fix(deploy): persist replica snapshot before stop for restore on start
Read live deployment replica counts from K8s before scaling to zero, store them on the application as suspendedReplicas, and use that snapshot when resuming so Start restores the pre-stop replica layout. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -189,6 +189,25 @@ export class ApplicationsService {
|
||||
return this.appsRepository.save(app);
|
||||
}
|
||||
|
||||
async saveSuspendedReplicas(
|
||||
id: string,
|
||||
snapshot: Record<string, number>,
|
||||
): Promise<Application> {
|
||||
const app = await this.findOne(id);
|
||||
app.suspendedReplicas = snapshot;
|
||||
app.suspendedAt = new Date();
|
||||
if (snapshot[app.name] !== undefined) {
|
||||
app.replicas = snapshot[app.name];
|
||||
}
|
||||
return this.appsRepository.save(app);
|
||||
}
|
||||
|
||||
async clearSuspendedReplicas(id: string): Promise<Application> {
|
||||
const app = await this.findOne(id);
|
||||
app.suspendedReplicas = undefined;
|
||||
return this.appsRepository.save(app);
|
||||
}
|
||||
|
||||
async uploadCode(id: string, userId: string, file: Express.Multer.File): Promise<Application> {
|
||||
if (!file) {
|
||||
throw new BadRequestException('No file uploaded');
|
||||
|
||||
@@ -159,6 +159,10 @@ export class Application {
|
||||
@Column({ type: 'timestamptz', nullable: true })
|
||||
suspendedAt: Date; // When the app was suspended (pods scaled to 0)
|
||||
|
||||
/** Per-deployment replica counts captured before suspend (deployment name → replicas). */
|
||||
@Column({ type: 'jsonb', nullable: true })
|
||||
suspendedReplicas?: Record<string, number>;
|
||||
|
||||
@Column({ type: 'timestamptz', nullable: true })
|
||||
scheduledDeletionAt: Date; // When the app will be permanently deleted
|
||||
|
||||
|
||||
@@ -240,7 +240,8 @@ export class DeploymentsService {
|
||||
|
||||
async stopDeployment(applicationId: string, userId: string): Promise<Deployment | null> {
|
||||
const app = await this.applicationsService.findOne(applicationId, userId);
|
||||
await this.kubernetesService.suspendApplication(app);
|
||||
const snapshot = await this.kubernetesService.suspendApplication(app);
|
||||
await this.applicationsService.saveSuspendedReplicas(app.id, snapshot);
|
||||
|
||||
const latest = await this.deploymentsRepository.findOne({
|
||||
where: { applicationId },
|
||||
@@ -257,6 +258,7 @@ export class DeploymentsService {
|
||||
async startDeployment(applicationId: string, userId: string): Promise<Deployment | null> {
|
||||
const app = await this.applicationsService.findOne(applicationId, userId);
|
||||
await this.kubernetesService.resumeApplication(app);
|
||||
await this.applicationsService.clearSuspendedReplicas(app.id);
|
||||
|
||||
const latest = await this.deploymentsRepository.findOne({
|
||||
where: { applicationId },
|
||||
|
||||
@@ -1353,7 +1353,7 @@ export class KubernetesService implements OnModuleInit {
|
||||
);
|
||||
}
|
||||
|
||||
/** All K8s Deployments that belong to an application stack. */
|
||||
/** All K8s Deployments that belong to an application stack (default replica targets). */
|
||||
private getApplicationWorkloadDeployments(app: Application): { name: string; runningReplicas: number }[] {
|
||||
const workloads: { name: string; runningReplicas: number }[] = [
|
||||
{ name: app.name, runningReplicas: app.replicas || 1 },
|
||||
@@ -1372,6 +1372,35 @@ export class KubernetesService implements OnModuleInit {
|
||||
return workloads;
|
||||
}
|
||||
|
||||
private resolveWorkloadReplicas(app: Application): { name: string; runningReplicas: number }[] {
|
||||
const workloads = this.getApplicationWorkloadDeployments(app);
|
||||
const saved = app.suspendedReplicas;
|
||||
if (!saved) return workloads;
|
||||
|
||||
return workloads.map((workload) => ({
|
||||
name: workload.name,
|
||||
runningReplicas: saved[workload.name] ?? workload.runningReplicas,
|
||||
}));
|
||||
}
|
||||
|
||||
async captureWorkloadReplicaSnapshot(app: Application): Promise<Record<string, number>> {
|
||||
const { appsApi } = await this.getK8sClient(app.clusterId);
|
||||
const namespace = `user-${app.userId.split('-')[0]}`;
|
||||
const snapshot: Record<string, number> = {};
|
||||
|
||||
for (const workload of this.getApplicationWorkloadDeployments(app)) {
|
||||
try {
|
||||
const deployment = await appsApi.readNamespacedDeployment(workload.name, namespace);
|
||||
snapshot[workload.name] = deployment.body.spec?.replicas ?? workload.runningReplicas;
|
||||
} catch (e: any) {
|
||||
if (e?.response?.statusCode === 404) continue;
|
||||
snapshot[workload.name] = workload.runningReplicas;
|
||||
}
|
||||
}
|
||||
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
private async patchDeploymentReplicas(
|
||||
appsApi: k8s.AppsV1Api,
|
||||
namespace: string,
|
||||
@@ -1393,15 +1422,18 @@ export class KubernetesService implements OnModuleInit {
|
||||
|
||||
/**
|
||||
* Suspend an application by scaling all stack deployments to 0 replicas.
|
||||
* Keeps PVCs, Services, and Ingress in place.
|
||||
* Returns the replica snapshot captured before scaling.
|
||||
*/
|
||||
async suspendApplication(app: Application): Promise<void> {
|
||||
async suspendApplication(app: Application): Promise<Record<string, number>> {
|
||||
const { appsApi } = await this.getK8sClient(app.clusterId);
|
||||
const namespace = `user-${app.userId.split('-')[0]}`;
|
||||
|
||||
this.logger.log(`Suspending application ${app.name} in namespace ${namespace}`);
|
||||
|
||||
for (const workload of this.getApplicationWorkloadDeployments(app)) {
|
||||
const snapshot = await this.captureWorkloadReplicaSnapshot(app);
|
||||
const workloads = this.getApplicationWorkloadDeployments(app);
|
||||
|
||||
for (const workload of workloads) {
|
||||
try {
|
||||
await this.patchDeploymentReplicas(appsApi, namespace, workload.name, 0);
|
||||
this.logger.log(`Scaled ${workload.name} to 0 replicas`);
|
||||
@@ -1411,10 +1443,12 @@ export class KubernetesService implements OnModuleInit {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resume a suspended application by scaling all stack deployments back up.
|
||||
* Resume a suspended application using saved replica counts when available.
|
||||
*/
|
||||
async resumeApplication(app: Application): Promise<void> {
|
||||
const { appsApi } = await this.getK8sClient(app.clusterId);
|
||||
@@ -1422,7 +1456,7 @@ export class KubernetesService implements OnModuleInit {
|
||||
|
||||
this.logger.log(`Resuming application ${app.name} in namespace ${namespace}`);
|
||||
|
||||
const workloads = this.getApplicationWorkloadDeployments(app);
|
||||
const workloads = this.resolveWorkloadReplicas(app);
|
||||
const dependencies = workloads.filter((w) => w.name !== app.name);
|
||||
const main = workloads.find((w) => w.name === app.name);
|
||||
|
||||
|
||||
@@ -74,6 +74,8 @@ export class AppLifecycleService implements OnModuleInit, OnModuleDestroy {
|
||||
if (app.latestImageTag) {
|
||||
try {
|
||||
await this.kubernetesService.resumeApplication(app);
|
||||
app.suspendedReplicas = undefined;
|
||||
await this.appRepo.save(app);
|
||||
this.logger.log(`Reactivated ${app.name} — resumed with ${app.replicas} replicas, expires ${expiresAt.toISOString()}`);
|
||||
} catch (e: any) {
|
||||
this.logger.warn(`Failed to resume ${app.name} on reactivation: ${e.message}`);
|
||||
@@ -138,7 +140,11 @@ export class AppLifecycleService implements OnModuleInit, OnModuleDestroy {
|
||||
|
||||
// Suspend: scale app and database to 0
|
||||
try {
|
||||
await this.kubernetesService.suspendApplication(app);
|
||||
const snapshot = await this.kubernetesService.suspendApplication(app);
|
||||
app.suspendedReplicas = snapshot;
|
||||
if (snapshot[app.name] !== undefined) {
|
||||
app.replicas = snapshot[app.name];
|
||||
}
|
||||
} catch (e: any) {
|
||||
this.logger.warn(`Failed to suspend ${app.name} in K8s: ${e.message}`);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user