diff --git a/backend/src/deployments/deployments.service.ts b/backend/src/deployments/deployments.service.ts index c016a8a..c98c060 100644 --- a/backend/src/deployments/deployments.service.ts +++ b/backend/src/deployments/deployments.service.ts @@ -92,7 +92,17 @@ export class DeploymentsService { } } - // Step 4: Mark success + // Step 4: wait for every workload owned by this app before marking it running. + // Helm/API apply success only means resources were accepted; quota or scheduling + // pressure can still leave DB/addon/app pods Pending. + this.buildService.setProgress(deploymentId, { + phase: 'deploying', + percent: 96, + message: 'Waiting for all application pods to become ready...', + }); + await this.kubernetesService.waitForApplicationReady(app); + + // Step 5: Mark success this.buildService.setProgress(deploymentId, { phase: 'done', percent: 100, diff --git a/backend/src/kubernetes/kubernetes.service.ts b/backend/src/kubernetes/kubernetes.service.ts index 728125b..c1894de 100644 --- a/backend/src/kubernetes/kubernetes.service.ts +++ b/backend/src/kubernetes/kubernetes.service.ts @@ -218,6 +218,43 @@ export class KubernetesService implements OnModuleInit { } } + async waitForApplicationReady(app: Application, timeoutMs = 600_000): Promise { + const { coreApi, appsApi } = await this.getK8sClient(app.clusterId); + const namespace = `user-${app.userId.split('-')[0]}`; + const workloads = [ + { name: app.name, replicas: app.replicas || 1 }, + ...(app.databaseType !== DatabaseType.NONE ? [{ name: `${app.name}-db`, replicas: 1 }] : []), + ...(app.enableRedis ? [{ name: `${app.name}-redis`, replicas: 1 }] : []), + ...(app.enableRabbitmq ? [{ name: `${app.name}-rabbitmq`, replicas: 1 }] : []), + ]; + const start = Date.now(); + let lastSummary = ''; + + this.logger.log( + `Waiting for ${app.name} workloads to become Ready in ${namespace}: ${workloads.map((w) => w.name).join(', ')}`, + ); + + while (Date.now() - start < timeoutMs) { + const statuses = await Promise.all( + workloads.map((workload) => this.getDeploymentReadiness(appsApi, namespace, workload.name, workload.replicas)), + ); + lastSummary = statuses.map((s) => `${s.name} ${s.readyReplicas}/${s.desiredReplicas}`).join(', '); + + if (statuses.every((s) => s.ready)) { + this.logger.log(`All workloads for ${app.name} are Ready (${lastSummary})`); + return; + } + + await new Promise((resolve) => setTimeout(resolve, 5000)); + } + + const podSummary = await this.describeWorkloadPods(coreApi, namespace, workloads.map((w) => w.name)); + throw new Error( + `Application workloads did not become ready within ${Math.round(timeoutMs / 1000)}s. ` + + `Readiness: ${lastSummary || 'no deployment status available'}. ${podSummary}`, + ); + } + async updateIngress(app: Application): Promise { const domain = this.configService.get('platform.domain'); const subdomain = app.subdomain || app.name; @@ -2336,6 +2373,83 @@ export class KubernetesService implements OnModuleInit { this.logger.warn(`DB pod did not become Ready within ${timeoutMs / 1000}s — proceeding anyway`); } + private async getDeploymentReadiness( + appsApi: k8s.AppsV1Api, + namespace: string, + name: string, + expectedReplicas: number, + ): Promise<{ name: string; desiredReplicas: number; readyReplicas: number; availableReplicas: number; ready: boolean }> { + try { + const response = await appsApi.readNamespacedDeployment(name, namespace); + const deployment = response.body; + const desiredReplicas = deployment.spec?.replicas ?? expectedReplicas; + const readyReplicas = deployment.status?.readyReplicas ?? 0; + const availableReplicas = deployment.status?.availableReplicas ?? 0; + const observedGeneration = deployment.status?.observedGeneration ?? 0; + const generation = deployment.metadata?.generation ?? 0; + + return { + name, + desiredReplicas, + readyReplicas, + availableReplicas, + ready: + desiredReplicas === 0 || + (readyReplicas >= desiredReplicas && + availableReplicas >= desiredReplicas && + observedGeneration >= generation), + }; + } catch { + return { + name, + desiredReplicas: expectedReplicas, + readyReplicas: 0, + availableReplicas: 0, + ready: false, + }; + } + } + + private async describeWorkloadPods( + coreApi: k8s.CoreV1Api, + namespace: string, + workloadNames: string[], + ): Promise { + const podLines: string[] = []; + + for (const workloadName of workloadNames) { + try { + const pods = await coreApi.listNamespacedPod( + namespace, + undefined, + undefined, + undefined, + undefined, + `app=${workloadName}`, + ); + + for (const pod of pods.body.items) { + const ready = pod.status?.conditions?.find((c) => c.type === 'Ready')?.status === 'True'; + const scheduled = pod.status?.conditions?.find((c) => c.type === 'PodScheduled'); + const waitingReasons = (pod.status?.containerStatuses || []) + .map((status) => status.state?.waiting?.reason) + .filter(Boolean) + .join('/'); + const reason = waitingReasons || scheduled?.reason || pod.status?.reason || pod.status?.phase || 'Unknown'; + const message = scheduled?.message || pod.status?.message || ''; + + podLines.push( + `${pod.metadata?.name || workloadName}: phase=${pod.status?.phase || 'Unknown'}, ready=${ready}, reason=${reason}${message ? ` (${message})` : ''}`, + ); + } + } catch { + podLines.push(`${workloadName}: pods unavailable`); + } + } + + return podLines.length ? `Pods: ${podLines.join('; ')}` : 'No pods found for expected workloads.'; + } + /** * Restore a SQL dump file into the application's database. * Uses a PVC + helper pod + kubectl cp to transfer the dump (supports large files),