fix(deploy): persist replica snapshot before stop for restore on start
Read live deployment replica counts from K8s before scaling to zero, store them on the application as suspendedReplicas, and use that snapshot when resuming so Start restores the pre-stop replica layout. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -189,6 +189,25 @@ export class ApplicationsService {
|
|||||||
return this.appsRepository.save(app);
|
return this.appsRepository.save(app);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async saveSuspendedReplicas(
|
||||||
|
id: string,
|
||||||
|
snapshot: Record<string, number>,
|
||||||
|
): Promise<Application> {
|
||||||
|
const app = await this.findOne(id);
|
||||||
|
app.suspendedReplicas = snapshot;
|
||||||
|
app.suspendedAt = new Date();
|
||||||
|
if (snapshot[app.name] !== undefined) {
|
||||||
|
app.replicas = snapshot[app.name];
|
||||||
|
}
|
||||||
|
return this.appsRepository.save(app);
|
||||||
|
}
|
||||||
|
|
||||||
|
async clearSuspendedReplicas(id: string): Promise<Application> {
|
||||||
|
const app = await this.findOne(id);
|
||||||
|
app.suspendedReplicas = undefined;
|
||||||
|
return this.appsRepository.save(app);
|
||||||
|
}
|
||||||
|
|
||||||
async uploadCode(id: string, userId: string, file: Express.Multer.File): Promise<Application> {
|
async uploadCode(id: string, userId: string, file: Express.Multer.File): Promise<Application> {
|
||||||
if (!file) {
|
if (!file) {
|
||||||
throw new BadRequestException('No file uploaded');
|
throw new BadRequestException('No file uploaded');
|
||||||
|
|||||||
@@ -159,6 +159,10 @@ export class Application {
|
|||||||
@Column({ type: 'timestamptz', nullable: true })
|
@Column({ type: 'timestamptz', nullable: true })
|
||||||
suspendedAt: Date; // When the app was suspended (pods scaled to 0)
|
suspendedAt: Date; // When the app was suspended (pods scaled to 0)
|
||||||
|
|
||||||
|
/** Per-deployment replica counts captured before suspend (deployment name → replicas). */
|
||||||
|
@Column({ type: 'jsonb', nullable: true })
|
||||||
|
suspendedReplicas?: Record<string, number>;
|
||||||
|
|
||||||
@Column({ type: 'timestamptz', nullable: true })
|
@Column({ type: 'timestamptz', nullable: true })
|
||||||
scheduledDeletionAt: Date; // When the app will be permanently deleted
|
scheduledDeletionAt: Date; // When the app will be permanently deleted
|
||||||
|
|
||||||
|
|||||||
@@ -240,7 +240,8 @@ export class DeploymentsService {
|
|||||||
|
|
||||||
async stopDeployment(applicationId: string, userId: string): Promise<Deployment | null> {
|
async stopDeployment(applicationId: string, userId: string): Promise<Deployment | null> {
|
||||||
const app = await this.applicationsService.findOne(applicationId, userId);
|
const app = await this.applicationsService.findOne(applicationId, userId);
|
||||||
await this.kubernetesService.suspendApplication(app);
|
const snapshot = await this.kubernetesService.suspendApplication(app);
|
||||||
|
await this.applicationsService.saveSuspendedReplicas(app.id, snapshot);
|
||||||
|
|
||||||
const latest = await this.deploymentsRepository.findOne({
|
const latest = await this.deploymentsRepository.findOne({
|
||||||
where: { applicationId },
|
where: { applicationId },
|
||||||
@@ -257,6 +258,7 @@ export class DeploymentsService {
|
|||||||
async startDeployment(applicationId: string, userId: string): Promise<Deployment | null> {
|
async startDeployment(applicationId: string, userId: string): Promise<Deployment | null> {
|
||||||
const app = await this.applicationsService.findOne(applicationId, userId);
|
const app = await this.applicationsService.findOne(applicationId, userId);
|
||||||
await this.kubernetesService.resumeApplication(app);
|
await this.kubernetesService.resumeApplication(app);
|
||||||
|
await this.applicationsService.clearSuspendedReplicas(app.id);
|
||||||
|
|
||||||
const latest = await this.deploymentsRepository.findOne({
|
const latest = await this.deploymentsRepository.findOne({
|
||||||
where: { applicationId },
|
where: { applicationId },
|
||||||
|
|||||||
@@ -1353,7 +1353,7 @@ export class KubernetesService implements OnModuleInit {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** All K8s Deployments that belong to an application stack. */
|
/** All K8s Deployments that belong to an application stack (default replica targets). */
|
||||||
private getApplicationWorkloadDeployments(app: Application): { name: string; runningReplicas: number }[] {
|
private getApplicationWorkloadDeployments(app: Application): { name: string; runningReplicas: number }[] {
|
||||||
const workloads: { name: string; runningReplicas: number }[] = [
|
const workloads: { name: string; runningReplicas: number }[] = [
|
||||||
{ name: app.name, runningReplicas: app.replicas || 1 },
|
{ name: app.name, runningReplicas: app.replicas || 1 },
|
||||||
@@ -1372,6 +1372,35 @@ export class KubernetesService implements OnModuleInit {
|
|||||||
return workloads;
|
return workloads;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private resolveWorkloadReplicas(app: Application): { name: string; runningReplicas: number }[] {
|
||||||
|
const workloads = this.getApplicationWorkloadDeployments(app);
|
||||||
|
const saved = app.suspendedReplicas;
|
||||||
|
if (!saved) return workloads;
|
||||||
|
|
||||||
|
return workloads.map((workload) => ({
|
||||||
|
name: workload.name,
|
||||||
|
runningReplicas: saved[workload.name] ?? workload.runningReplicas,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
async captureWorkloadReplicaSnapshot(app: Application): Promise<Record<string, number>> {
|
||||||
|
const { appsApi } = await this.getK8sClient(app.clusterId);
|
||||||
|
const namespace = `user-${app.userId.split('-')[0]}`;
|
||||||
|
const snapshot: Record<string, number> = {};
|
||||||
|
|
||||||
|
for (const workload of this.getApplicationWorkloadDeployments(app)) {
|
||||||
|
try {
|
||||||
|
const deployment = await appsApi.readNamespacedDeployment(workload.name, namespace);
|
||||||
|
snapshot[workload.name] = deployment.body.spec?.replicas ?? workload.runningReplicas;
|
||||||
|
} catch (e: any) {
|
||||||
|
if (e?.response?.statusCode === 404) continue;
|
||||||
|
snapshot[workload.name] = workload.runningReplicas;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return snapshot;
|
||||||
|
}
|
||||||
|
|
||||||
private async patchDeploymentReplicas(
|
private async patchDeploymentReplicas(
|
||||||
appsApi: k8s.AppsV1Api,
|
appsApi: k8s.AppsV1Api,
|
||||||
namespace: string,
|
namespace: string,
|
||||||
@@ -1393,15 +1422,18 @@ export class KubernetesService implements OnModuleInit {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Suspend an application by scaling all stack deployments to 0 replicas.
|
* Suspend an application by scaling all stack deployments to 0 replicas.
|
||||||
* Keeps PVCs, Services, and Ingress in place.
|
* Returns the replica snapshot captured before scaling.
|
||||||
*/
|
*/
|
||||||
async suspendApplication(app: Application): Promise<void> {
|
async suspendApplication(app: Application): Promise<Record<string, number>> {
|
||||||
const { appsApi } = await this.getK8sClient(app.clusterId);
|
const { appsApi } = await this.getK8sClient(app.clusterId);
|
||||||
const namespace = `user-${app.userId.split('-')[0]}`;
|
const namespace = `user-${app.userId.split('-')[0]}`;
|
||||||
|
|
||||||
this.logger.log(`Suspending application ${app.name} in namespace ${namespace}`);
|
this.logger.log(`Suspending application ${app.name} in namespace ${namespace}`);
|
||||||
|
|
||||||
for (const workload of this.getApplicationWorkloadDeployments(app)) {
|
const snapshot = await this.captureWorkloadReplicaSnapshot(app);
|
||||||
|
const workloads = this.getApplicationWorkloadDeployments(app);
|
||||||
|
|
||||||
|
for (const workload of workloads) {
|
||||||
try {
|
try {
|
||||||
await this.patchDeploymentReplicas(appsApi, namespace, workload.name, 0);
|
await this.patchDeploymentReplicas(appsApi, namespace, workload.name, 0);
|
||||||
this.logger.log(`Scaled ${workload.name} to 0 replicas`);
|
this.logger.log(`Scaled ${workload.name} to 0 replicas`);
|
||||||
@@ -1411,10 +1443,12 @@ export class KubernetesService implements OnModuleInit {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return snapshot;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Resume a suspended application by scaling all stack deployments back up.
|
* Resume a suspended application using saved replica counts when available.
|
||||||
*/
|
*/
|
||||||
async resumeApplication(app: Application): Promise<void> {
|
async resumeApplication(app: Application): Promise<void> {
|
||||||
const { appsApi } = await this.getK8sClient(app.clusterId);
|
const { appsApi } = await this.getK8sClient(app.clusterId);
|
||||||
@@ -1422,7 +1456,7 @@ export class KubernetesService implements OnModuleInit {
|
|||||||
|
|
||||||
this.logger.log(`Resuming application ${app.name} in namespace ${namespace}`);
|
this.logger.log(`Resuming application ${app.name} in namespace ${namespace}`);
|
||||||
|
|
||||||
const workloads = this.getApplicationWorkloadDeployments(app);
|
const workloads = this.resolveWorkloadReplicas(app);
|
||||||
const dependencies = workloads.filter((w) => w.name !== app.name);
|
const dependencies = workloads.filter((w) => w.name !== app.name);
|
||||||
const main = workloads.find((w) => w.name === app.name);
|
const main = workloads.find((w) => w.name === app.name);
|
||||||
|
|
||||||
|
|||||||
@@ -74,6 +74,8 @@ export class AppLifecycleService implements OnModuleInit, OnModuleDestroy {
|
|||||||
if (app.latestImageTag) {
|
if (app.latestImageTag) {
|
||||||
try {
|
try {
|
||||||
await this.kubernetesService.resumeApplication(app);
|
await this.kubernetesService.resumeApplication(app);
|
||||||
|
app.suspendedReplicas = undefined;
|
||||||
|
await this.appRepo.save(app);
|
||||||
this.logger.log(`Reactivated ${app.name} — resumed with ${app.replicas} replicas, expires ${expiresAt.toISOString()}`);
|
this.logger.log(`Reactivated ${app.name} — resumed with ${app.replicas} replicas, expires ${expiresAt.toISOString()}`);
|
||||||
} catch (e: any) {
|
} catch (e: any) {
|
||||||
this.logger.warn(`Failed to resume ${app.name} on reactivation: ${e.message}`);
|
this.logger.warn(`Failed to resume ${app.name} on reactivation: ${e.message}`);
|
||||||
@@ -138,7 +140,11 @@ export class AppLifecycleService implements OnModuleInit, OnModuleDestroy {
|
|||||||
|
|
||||||
// Suspend: scale app and database to 0
|
// Suspend: scale app and database to 0
|
||||||
try {
|
try {
|
||||||
await this.kubernetesService.suspendApplication(app);
|
const snapshot = await this.kubernetesService.suspendApplication(app);
|
||||||
|
app.suspendedReplicas = snapshot;
|
||||||
|
if (snapshot[app.name] !== undefined) {
|
||||||
|
app.replicas = snapshot[app.name];
|
||||||
|
}
|
||||||
} catch (e: any) {
|
} catch (e: any) {
|
||||||
this.logger.warn(`Failed to suspend ${app.name} in K8s: ${e.message}`);
|
this.logger.warn(`Failed to suspend ${app.name} in K8s: ${e.message}`);
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user