Files
cloud-host/backend/src/deployments/deployments.service.ts
T
keyhan 880521c576 Wait for all app workloads before marking deployments running.
Deployment success now reflects readiness across the application, database, and enabled add-on workloads so scheduling or resource failures do not appear as a running app.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-05-18 19:46:20 +03:30

323 lines
12 KiB
TypeScript

import { Injectable, NotFoundException, BadRequestException, Logger, Inject, forwardRef } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import * as fs from 'fs';
import { Deployment } from './entities/deployment.entity';
import { ApplicationsService } from '../applications/applications.service';
import { KubernetesService } from '../kubernetes/kubernetes.service';
import { BuildService, BuildProgress, BuildCancelledError } from '../build/build.service';
import { DeploymentStatus } from '../common/enums';
@Injectable()
export class DeploymentsService {
private readonly logger = new Logger(DeploymentsService.name);
constructor(
@InjectRepository(Deployment)
private deploymentsRepository: Repository<Deployment>,
@Inject(forwardRef(() => ApplicationsService))
private applicationsService: ApplicationsService,
private kubernetesService: KubernetesService,
private buildService: BuildService,
) {}
async triggerDeployment(applicationId: string, userId: string): Promise<Deployment> {
const app = await this.applicationsService.findOne(applicationId, userId);
// Create deployment record
const deployment = this.deploymentsRepository.create({
applicationId: app.id,
triggeredBy: userId,
imageTag: `${app.name}:${Date.now()}`,
status: DeploymentStatus.PENDING,
version: `v${Date.now()}`,
});
const saved = await this.deploymentsRepository.save(deployment);
// Trigger async build & deploy pipeline
this.executePipeline(saved.id, app).catch((error) => {
this.logger.error(`Pipeline failed for deployment ${saved.id}:`, error);
});
return saved;
}
private async executePipeline(deploymentId: string, app: any): Promise<void> {
try {
// Step 1: Build image
await this.updateStatus(deploymentId, DeploymentStatus.BUILDING);
const { imageUri, buildLog } = await this.buildService.buildImage(app, deploymentId);
// Save build log
await this.deploymentsRepository.update(deploymentId, { buildLog });
// Step 2: Update app with new image tag
await this.applicationsService.updateImageTag(app.id, imageUri);
// Step 3: Deploy to Kubernetes
await this.updateStatus(deploymentId, DeploymentStatus.DEPLOYING);
this.buildService.setProgress(deploymentId, {
phase: 'deploying',
percent: 92,
message: 'Deploying to Kubernetes...',
});
// If a DB dump will be restored, deploy with 0 replicas first so WordPress
// does not initialize empty tables before the dump is imported.
const hasDbDump = app.dbDumpPath && fs.existsSync(app.dbDumpPath);
const deployApp = hasDbDump ? { ...app, replicas: 0 } : app;
const k8sResources = await this.kubernetesService.deployApplication(deployApp, imageUri);
// Step 3.5: Restore DB dump if one was uploaded (must happen after deploy creates the namespace + DB)
if (hasDbDump) {
this.logger.log(`Restoring DB dump for ${app.name} from ${app.dbDumpPath}`);
try {
await this.kubernetesService.waitForDatabaseReady(app, 120_000);
const freshApp = await this.applicationsService.findOne(app.id);
const result = await this.kubernetesService.restoreDatabaseDump(freshApp, freshApp.dbDumpPath!);
if (result.success) {
this.logger.log(`DB dump restored successfully for ${app.name}`);
} else {
this.logger.warn(`DB dump restore failed for ${app.name}: ${result.logs}`);
}
} catch (e: any) {
this.logger.warn(`DB dump restore error for ${app.name}: ${e.message}`);
}
// Scale WordPress app up after restore (or even if restore failed)
try {
await this.kubernetesService.scaleDeployment(app, app.replicas || 1);
this.logger.log(`Scaled ${app.name} to ${app.replicas || 1} replica(s) after DB restore`);
} catch (e: any) {
this.logger.warn(`Failed to scale up ${app.name} after DB restore: ${e.message}`);
}
}
// Step 4: wait for every workload owned by this app before marking it running.
// Helm/API apply success only means resources were accepted; quota or scheduling
// pressure can still leave DB/addon/app pods Pending.
this.buildService.setProgress(deploymentId, {
phase: 'deploying',
percent: 96,
message: 'Waiting for all application pods to become ready...',
});
await this.kubernetesService.waitForApplicationReady(app);
// Step 5: Mark success
this.buildService.setProgress(deploymentId, {
phase: 'done',
percent: 100,
message: 'Deployment complete',
});
await this.deploymentsRepository.update(deploymentId, {
status: DeploymentStatus.RUNNING,
k8sResources,
finishedAt: new Date(),
});
} catch (error: any) {
if (error instanceof BuildCancelledError || error?.name === 'BuildCancelledError') {
this.logger.log(`Deployment ${deploymentId} cancelled by user`);
await this.deploymentsRepository.update(deploymentId, {
status: DeploymentStatus.FAILED,
errorMessage: 'Cancelled by user',
finishedAt: new Date(),
});
return;
}
this.logger.error(`Deployment ${deploymentId} failed:`, error);
this.buildService.setProgress(deploymentId, {
phase: 'failed',
percent: 0,
message: error.message || 'Deployment failed',
});
// Save build log if available (attached by build service on failure)
const buildLog = error.buildLog || null;
await this.deploymentsRepository.update(deploymentId, {
status: DeploymentStatus.FAILED,
errorMessage: error.message,
...(buildLog ? { buildLog } : {}),
finishedAt: new Date(),
});
}
}
async updateStatus(id: string, status: DeploymentStatus): Promise<void> {
await this.deploymentsRepository.update(id, { status });
}
async findByApplication(applicationId: string): Promise<Deployment[]> {
return this.deploymentsRepository.find({
where: { applicationId },
order: { createdAt: 'DESC' },
});
}
async findOne(id: string): Promise<Deployment> {
const deployment = await this.deploymentsRepository.findOne({
where: { id },
relations: ['application'],
});
if (!deployment) {
throw new NotFoundException('Deployment not found');
}
return deployment;
}
async getLogs(applicationId: string, userId: string): Promise<string> {
const app = await this.applicationsService.findOne(applicationId, userId);
return this.kubernetesService.getPodLogs(app);
}
async getBuildLogs(applicationId: string, userId: string): Promise<{ buildLog: string | null; status: string; version: string | null; createdAt: Date }> {
// Verify user access
await this.applicationsService.findOne(applicationId, userId);
const latest = await this.deploymentsRepository.findOne({
where: { applicationId },
order: { createdAt: 'DESC' },
});
if (!latest) {
return { buildLog: null, status: 'no_deployment', version: null, createdAt: new Date() };
}
return {
buildLog: latest.buildLog || null,
status: latest.status,
version: latest.version,
createdAt: latest.createdAt,
};
}
async getBuildProgress(applicationId: string, userId: string): Promise<BuildProgress | null> {
await this.applicationsService.findOne(applicationId, userId);
const latest = await this.deploymentsRepository.findOne({
where: { applicationId },
order: { createdAt: 'DESC' },
});
if (!latest) return null;
const progress = this.buildService.getProgress(latest.id);
if (progress) return progress;
// No in-memory progress — infer from deployment status
if (latest.status === DeploymentStatus.RUNNING) {
return { phase: 'done', percent: 100, message: 'Deployment complete' };
}
if (latest.status === DeploymentStatus.FAILED) {
return { phase: 'failed', percent: 0, message: latest.errorMessage || 'Deployment failed' };
}
if (latest.status === DeploymentStatus.BUILDING) {
return { phase: 'building', percent: 0, message: 'Building...' };
}
if (latest.status === DeploymentStatus.DEPLOYING) {
return { phase: 'deploying', percent: 90, message: 'Deploying...' };
}
return null;
}
async cancelDeployment(applicationId: string, userId: string): Promise<Deployment> {
const app = await this.applicationsService.findOne(applicationId, userId);
const latest = await this.deploymentsRepository.findOne({
where: { applicationId },
order: { createdAt: 'DESC' },
});
if (!latest) {
throw new NotFoundException('No deployment found');
}
const inProgress = [
DeploymentStatus.PENDING,
DeploymentStatus.BUILDING,
DeploymentStatus.DEPLOYING,
];
if (!inProgress.includes(latest.status as DeploymentStatus)) {
throw new BadRequestException('No deployment in progress to cancel');
}
await this.buildService.cancelBuild(latest.id);
await this.buildService.cleanupBuildResourcesForApp(app);
latest.status = DeploymentStatus.FAILED;
latest.errorMessage = 'Cancelled by user';
latest.finishedAt = new Date();
return this.deploymentsRepository.save(latest);
}
async stopDeployment(applicationId: string, userId: string): Promise<Deployment | null> {
const app = await this.applicationsService.findOne(applicationId, userId);
const snapshot = await this.kubernetesService.suspendApplication(app);
await this.applicationsService.saveSuspendedReplicas(app.id, snapshot);
const latest = await this.deploymentsRepository.findOne({
where: { applicationId },
order: { createdAt: 'DESC' },
});
if (latest) {
latest.status = DeploymentStatus.STOPPED;
latest.finishedAt = latest.finishedAt || new Date();
await this.deploymentsRepository.save(latest);
}
return latest;
}
async startDeployment(applicationId: string, userId: string): Promise<Deployment | null> {
const app = await this.applicationsService.findOne(applicationId, userId);
await this.kubernetesService.resumeApplication(app);
await this.applicationsService.clearSuspendedReplicas(app.id);
const latest = await this.deploymentsRepository.findOne({
where: { applicationId },
order: { createdAt: 'DESC' },
});
if (latest) {
latest.status = DeploymentStatus.RUNNING;
await this.deploymentsRepository.save(latest);
}
return latest;
}
async restartDeployment(applicationId: string, userId: string): Promise<void> {
const app = await this.applicationsService.findOne(applicationId, userId);
await this.kubernetesService.restartDeployment(app);
}
/**
* Redeploy: re-build from latest source (git pull or existing zip) and deploy new version.
* For git-based apps this pulls the latest code, for zip-based it rebuilds from last uploaded zip.
*/
async redeployApplication(applicationId: string, userId: string): Promise<Deployment> {
const app = await this.applicationsService.findOne(applicationId, userId);
if (!app.codePath && !app.gitUrl) {
throw new NotFoundException('No source code available. Upload code or set a git URL first.');
}
// Create new deployment record
const deployment = this.deploymentsRepository.create({
applicationId: app.id,
triggeredBy: userId,
imageTag: `${app.name}:${Date.now()}`,
status: DeploymentStatus.PENDING,
version: `v${Date.now()}`,
});
const saved = await this.deploymentsRepository.save(deployment);
// Trigger async build & deploy pipeline (same as initial deploy)
this.executePipeline(saved.id, app).catch((error) => {
this.logger.error(`Redeploy pipeline failed for deployment ${saved.id}:`, error);
});
this.logger.log(`Redeploy triggered for ${app.name} (${app.gitUrl ? 'git: ' + app.gitUrl : 'zip'})`);
return saved;
}
async deleteAllForApplication(applicationId: string): Promise<void> {
await this.deploymentsRepository.delete({ applicationId });
}
}