import { Injectable, NotFoundException, BadRequestException, Logger, Inject, forwardRef } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; import { Repository } from 'typeorm'; import * as fs from 'fs'; import { Deployment } from './entities/deployment.entity'; import { ApplicationsService } from '../applications/applications.service'; import { KubernetesService } from '../kubernetes/kubernetes.service'; import { BuildService, BuildProgress, BuildCancelledError } from '../build/build.service'; import * as crypto from 'crypto'; import { AppLifecycleStatus, DeploymentStatus, isManagedProductType, MANAGED_DEPLOY_MARKER, } from '../common/enums'; import { ClustersService } from '../clusters/clusters.service'; @Injectable() export class DeploymentsService { private readonly logger = new Logger(DeploymentsService.name); constructor( @InjectRepository(Deployment) private deploymentsRepository: Repository, @Inject(forwardRef(() => ApplicationsService)) private applicationsService: ApplicationsService, private kubernetesService: KubernetesService, private buildService: BuildService, private clustersService: ClustersService, ) {} /** * Random 7-digit suffix for the preview host: -<7-digit>.. * Generated once per application (see resolvePreviewNumber) and persisted. */ private generatePreviewNumber(): string { return String(crypto.randomInt(1_000_000, 10_000_000)); } /** * Returns a STABLE preview number for an application: reuse the one already * assigned to a previous deployment so the public preview URL never changes * across redeploys (otherwise old links 404). Only generates a new number the * first time the app is deployed. Apps with a custom domain get no preview host. */ private async resolvePreviewNumber(applicationId: string): Promise { const existing = await this.deploymentsRepository .createQueryBuilder('d') .select('d.previewSubdomain', 'previewSubdomain') .where('d.applicationId = :applicationId', { applicationId }) .andWhere('d.previewSubdomain IS NOT NULL') .orderBy('d.createdAt', 'DESC') .limit(1) .getRawOne<{ previewSubdomain: string }>(); return existing?.previewSubdomain || this.generatePreviewNumber(); } async triggerDeployment(applicationId: string, userId: string): Promise { const app = await this.applicationsService.findOne(applicationId, userId); // Create deployment record const deployment = this.deploymentsRepository.create({ applicationId: app.id, triggeredBy: userId, imageTag: `${app.name}:${Date.now()}`, status: DeploymentStatus.PENDING, version: `v${Date.now()}`, previewSubdomain: null, }); const saved = await this.deploymentsRepository.save(deployment); // Fill deterministic preview number after we have the deployment id. let previewSubdomain: string | null = null; if (!app.customDomain) { previewSubdomain = await this.resolvePreviewNumber(app.id); await this.deploymentsRepository.update(saved.id, { previewSubdomain }); saved.previewSubdomain = previewSubdomain; } // Trigger async pipeline (Helm-only for managed services, build+deploy for apps) const run = isManagedProductType(app.productType) ? this.executeManagedPipeline(saved.id, app) : this.executePipeline(saved.id, app, previewSubdomain); run.catch((error) => { this.logger.error(`Pipeline failed for deployment ${saved.id}:`, error); }); return saved; } /** Provision managed database/redis/rabbitmq via Helm only — no image build. */ private async executeManagedPipeline(deploymentId: string, app: any): Promise { try { await this.updateStatus(deploymentId, DeploymentStatus.DEPLOYING); this.buildService.setProgress(deploymentId, { phase: 'deploying', percent: 10, message: 'Provisioning service via Helm...', }); const hasDbDump = app.dbDumpPath && fs.existsSync(app.dbDumpPath); const { app: deployedApp, k8sResources } = await this.deployManagedWithClusterFallback( deploymentId, app, hasDbDump, ); app = deployedApp; if (hasDbDump) { this.logger.log(`Restoring DB dump for ${app.name} from ${app.dbDumpPath}`); try { await this.kubernetesService.waitForDatabaseReady(app, 120_000); const freshApp = await this.applicationsService.findOne(app.id); const result = await this.kubernetesService.restoreDatabaseDump(freshApp, freshApp.dbDumpPath!); if (result.success) { this.logger.log(`DB dump restored successfully for ${app.name}`); } else { this.logger.warn(`DB dump restore failed for ${app.name}: ${result.logs}`); } } catch (e: any) { this.logger.warn(`DB dump restore error for ${app.name}: ${e.message}`); } } this.buildService.setProgress(deploymentId, { phase: 'deploying', percent: 96, message: 'Waiting for service pods to become ready...', }); await this.kubernetesService.waitForApplicationReady( app, 600_000, () => this.isDeploymentCancelled(deploymentId), ); if (await this.isDeploymentCancelled(deploymentId)) { return; } await this.applicationsService.updateImageTag(app.id, MANAGED_DEPLOY_MARKER); this.buildService.setProgress(deploymentId, { phase: 'done', percent: 100, message: 'Service provisioned', }); await this.deploymentsRepository.update(deploymentId, { status: DeploymentStatus.RUNNING, k8sResources, finishedAt: new Date(), }); } catch (error: any) { if (this.isCancellationError(error) || (await this.isDeploymentCancelled(deploymentId))) { this.logger.log(`Deployment ${deploymentId} cancelled by user`); await this.deploymentsRepository.update(deploymentId, { status: DeploymentStatus.CANCELLED, errorMessage: 'Cancelled by user', finishedAt: new Date(), }); return; } this.logger.error(`Managed deployment ${deploymentId} failed:`, error); this.buildService.setProgress(deploymentId, { phase: 'failed', percent: 0, message: error.message || 'Provisioning failed', }); await this.deploymentsRepository.update(deploymentId, { status: DeploymentStatus.FAILED, errorMessage: error.message, finishedAt: new Date(), }); } } private async executePipeline( deploymentId: string, app: any, previewSubdomain: string | null, ): Promise { try { // Step 1: Build image await this.updateStatus(deploymentId, DeploymentStatus.BUILDING); const buildResult = await this.buildService.buildImage(app, deploymentId); const imageUri = buildResult.imageUri; // Save build log await this.deploymentsRepository.update(deploymentId, { buildLog: buildResult.buildLog }); // Step 2: Update app with new image tag await this.applicationsService.updateImageTag(app.id, imageUri); // Step 3: Deploy to Kubernetes await this.updateStatus(deploymentId, DeploymentStatus.DEPLOYING); this.buildService.setProgress(deploymentId, { phase: 'deploying', percent: 92, message: 'Deploying to Kubernetes...', }); const hasDbDump = app.dbDumpPath && fs.existsSync(app.dbDumpPath); const { app: deployedApp, k8sResources } = await this.deployWithClusterFallback( deploymentId, app, imageUri, hasDbDump, previewSubdomain, ); app = deployedApp; // Step 3.5: Restore DB dump if one was uploaded (must happen after deploy creates the namespace + DB) if (hasDbDump) { this.logger.log(`Restoring DB dump for ${app.name} from ${app.dbDumpPath}`); try { await this.kubernetesService.waitForDatabaseReady(app, 120_000); const freshApp = await this.applicationsService.findOne(app.id); const result = await this.kubernetesService.restoreDatabaseDump(freshApp, freshApp.dbDumpPath!); if (result.success) { this.logger.log(`DB dump restored successfully for ${app.name}`); } else { this.logger.warn(`DB dump restore failed for ${app.name}: ${result.logs}`); } } catch (e: any) { this.logger.warn(`DB dump restore error for ${app.name}: ${e.message}`); } // Scale WordPress app up after restore (or even if restore failed) try { await this.kubernetesService.scaleDeployment(app, app.replicas || 1); this.logger.log(`Scaled ${app.name} to ${app.replicas || 1} replica(s) after DB restore`); } catch (e: any) { this.logger.warn(`Failed to scale up ${app.name} after DB restore: ${e.message}`); } } // Step 4: wait for every workload owned by this app before marking it running. // Helm/API apply success only means resources were accepted; quota or scheduling // pressure can still leave DB/addon/app pods Pending. this.buildService.setProgress(deploymentId, { phase: 'deploying', percent: 96, message: 'Waiting for all application pods to become ready...', }); await this.kubernetesService.waitForApplicationReady( app, 600_000, () => this.isDeploymentCancelled(deploymentId), ); if (await this.isDeploymentCancelled(deploymentId)) { return; } // Step 5: Mark success this.buildService.setProgress(deploymentId, { phase: 'done', percent: 100, message: 'Deployment complete', }); await this.deploymentsRepository.update(deploymentId, { status: DeploymentStatus.RUNNING, k8sResources, finishedAt: new Date(), }); } catch (error: any) { if (this.isCancellationError(error) || await this.isDeploymentCancelled(deploymentId)) { this.logger.log(`Deployment ${deploymentId} cancelled by user`); await this.deploymentsRepository.update(deploymentId, { status: DeploymentStatus.CANCELLED, errorMessage: 'Cancelled by user', finishedAt: new Date(), }); return; } this.logger.error(`Deployment ${deploymentId} failed:`, error); this.buildService.setProgress(deploymentId, { phase: 'failed', percent: 0, message: error.message || 'Deployment failed', }); // Save build log if available (attached by build service on failure) const buildLog = error.buildLog || null; await this.deploymentsRepository.update(deploymentId, { status: DeploymentStatus.FAILED, errorMessage: error.message, ...(buildLog ? { buildLog } : {}), finishedAt: new Date(), }); } } private async deployManagedWithClusterFallback( deploymentId: string, app: any, hasDbDump: boolean, ): Promise<{ app: any; k8sResources: Record }> { const failedClusterIds: string[] = []; let currentApp = app; let lastError: any; const maxAttempts = Number(process.env.CLUSTER_DEPLOY_FALLBACK_ATTEMPTS || 3); for (let attempt = 1; attempt <= maxAttempts; attempt++) { if (await this.isDeploymentCancelled(deploymentId)) { throw new Error('Deployment cancelled by user'); } try { this.buildService.setProgress(deploymentId, { phase: 'deploying', percent: Math.min(20 + attempt * 5, 90), message: attempt === 1 ? 'Installing Helm release...' : `Retrying Helm install on fallback cluster (${attempt}/${maxAttempts})...`, }); const k8sResources = await this.kubernetesService.deployManagedService(currentApp); return { app: currentApp, k8sResources }; } catch (error: any) { lastError = error; failedClusterIds.push(currentApp.clusterId); const failureMessage = error?.message || 'Helm provisioning failed on selected cluster'; await this.clustersService.markAllocationFailure(currentApp.id, currentApp.clusterId, failureMessage); if (attempt >= maxAttempts) { break; } try { const fallback = await this.clustersService.chooseFallbackClusterForApplication( currentApp, failedClusterIds, failureMessage, ); const updatedApp = await this.applicationsService.updateClusterAssignment( currentApp.id, fallback.cluster.id, fallback.pool?.id, ); await this.clustersService.attachAllocationToApplication(fallback.allocationLogId, currentApp.id); this.logger.warn( `Managed deployment ${deploymentId} falling back from cluster ${currentApp.clusterId || 'none'} to ${fallback.cluster.id}`, ); currentApp = { ...currentApp, ...updatedApp, clusterId: fallback.cluster.id, poolId: fallback.pool?.id || currentApp.poolId, }; } catch (fallbackError: any) { lastError = fallbackError; break; } } } throw lastError || new Error('Managed service provisioning failed'); } private async deployWithClusterFallback( deploymentId: string, app: any, imageUri: string, hasDbDump: boolean, previewSubdomain: string | null, ): Promise<{ app: any; k8sResources: Record }> { const failedClusterIds: string[] = []; let currentApp = app; let lastError: any; const maxAttempts = Number(process.env.CLUSTER_DEPLOY_FALLBACK_ATTEMPTS || 3); for (let attempt = 1; attempt <= maxAttempts; attempt++) { if (await this.isDeploymentCancelled(deploymentId)) { throw new Error('Deployment cancelled by user'); } try { this.buildService.setProgress(deploymentId, { phase: 'deploying', percent: Math.min(92 + attempt, 95), message: attempt === 1 ? 'Deploying to selected cluster...' : `Retrying deployment on fallback cluster (${attempt}/${maxAttempts})...`, }); const deployApp = hasDbDump && !isManagedProductType(currentApp.productType) ? { ...currentApp, replicas: 0 } : currentApp; const k8sResources = await this.kubernetesService.deployApplication(deployApp, imageUri, { previewNumber: previewSubdomain, }); return { app: currentApp, k8sResources }; } catch (error: any) { lastError = error; failedClusterIds.push(currentApp.clusterId); const failureMessage = error?.message || 'Deployment failed on selected cluster'; await this.clustersService.markAllocationFailure(currentApp.id, currentApp.clusterId, failureMessage); if (attempt >= maxAttempts) { break; } try { const fallback = await this.clustersService.chooseFallbackClusterForApplication( currentApp, failedClusterIds, failureMessage, ); const updatedApp = await this.applicationsService.updateClusterAssignment( currentApp.id, fallback.cluster.id, fallback.pool?.id, ); await this.clustersService.attachAllocationToApplication(fallback.allocationLogId, currentApp.id); this.logger.warn( `Deployment ${deploymentId} falling back from cluster ${currentApp.clusterId || 'none'} to ${fallback.cluster.id}`, ); currentApp = { ...currentApp, ...updatedApp, clusterId: fallback.cluster.id, poolId: fallback.pool?.id || currentApp.poolId }; } catch (fallbackError: any) { this.logger.warn(`No fallback cluster available for deployment ${deploymentId}: ${fallbackError.message}`); break; } } } throw lastError; } async updateStatus(id: string, status: DeploymentStatus): Promise { if (await this.isDeploymentCancelled(id)) { return; } await this.deploymentsRepository.update(id, { status }); } private async isDeploymentCancelled(id: string): Promise { const deployment = await this.deploymentsRepository.findOne({ where: { id } }); return deployment?.status === DeploymentStatus.CANCELLED || deployment?.errorMessage === 'Cancelled by user'; } private isCancellationError(error: any): boolean { return error instanceof BuildCancelledError || error?.name === 'BuildCancelledError' || String(error?.message || '').toLowerCase().includes('cancelled'); } /** Managed DB/Redis/RabbitMQ or rows already provisioned via Helm without an app image build. */ private isManagedOrHelmOnlyApp(app: { productType?: string; latestImageTag?: string }): boolean { if (isManagedProductType(app.productType)) return true; return app.latestImageTag === MANAGED_DEPLOY_MARKER; } private ensureRedeployAllowed(app: any): void { if (!app.billingCycle) return; const isActive = app.lifecycleStatus === AppLifecycleStatus.ACTIVE; const expiresAt = app.planExpiresAt ? new Date(app.planExpiresAt) : null; const hasPaidTimeRemaining = !!expiresAt && expiresAt > new Date(); if (!isActive || !hasPaidTimeRemaining) { throw new BadRequestException('Payment must be completed successfully before redeploying this application.'); } } async findByApplication(applicationId: string, userId?: string): Promise { await this.applicationsService.findOne(applicationId, userId); return this.deploymentsRepository.find({ where: { applicationId }, order: { createdAt: 'DESC' }, }); } async findOne(id: string, userId?: string): Promise { const deployment = await this.deploymentsRepository.findOne({ where: { id }, relations: { application: true }, }); if (!deployment) { throw new NotFoundException('Deployment not found'); } await this.applicationsService.findOne(deployment.applicationId, userId); return deployment; } async getLogs(applicationId: string, userId: string): Promise { const app = await this.applicationsService.findOne(applicationId, userId); return this.kubernetesService.getPodLogs(app); } async getBuildLogs(applicationId: string, userId: string): Promise<{ buildLog: string | null; status: string; version: string | null; createdAt: Date }> { const app = await this.applicationsService.findOne(applicationId, userId); const latest = await this.deploymentsRepository.findOne({ where: { applicationId }, order: { createdAt: 'DESC' }, }); if (!latest) { return { buildLog: null, status: 'no_deployment', version: null, createdAt: new Date() }; } if (this.isManagedOrHelmOnlyApp(app)) { return { buildLog: null, status: latest.status, version: latest.version, createdAt: latest.createdAt, }; } // While the build is in progress the persisted buildLog isn't written yet — // stream the live build pod logs so the deploy modal can show them in real time. let buildLog = latest.buildLog || null; if ( latest.status === DeploymentStatus.BUILDING || latest.status === DeploymentStatus.DEPLOYING ) { const live = await this.buildService.getLiveBuildLog(latest.id); if (live) buildLog = live; } return { buildLog, status: latest.status, version: latest.version, createdAt: latest.createdAt, }; } async getBuildProgress(applicationId: string, userId: string): Promise { const app = await this.applicationsService.findOne(applicationId, userId); const managed = this.isManagedOrHelmOnlyApp(app); const latest = await this.deploymentsRepository.findOne({ where: { applicationId }, order: { createdAt: 'DESC' }, }); if (!latest) return null; const progress = await this.buildService.getProgress(latest.id); if (progress) return progress; // No in-memory progress — infer from deployment status if (latest.status === DeploymentStatus.RUNNING) { return { phase: 'done', percent: 100, message: 'Deployment complete' }; } if (latest.status === DeploymentStatus.FAILED) { return { phase: 'failed', percent: 0, message: latest.errorMessage || 'Deployment failed' }; } if (latest.status === DeploymentStatus.CANCELLED) { return { phase: 'cancelled', percent: 0, message: latest.errorMessage || 'Cancelled by user' }; } if (latest.status === DeploymentStatus.BUILDING) { return { phase: managed ? 'deploying' : 'building', percent: 0, message: managed ? 'Provisioning...' : 'Building...', }; } if (latest.status === DeploymentStatus.DEPLOYING) { return { phase: 'deploying', percent: 90, message: managed ? 'Provisioning via Helm...' : 'Deploying...', }; } return null; } async cancelDeployment(applicationId: string, userId: string): Promise { const app = await this.applicationsService.findOne(applicationId, userId); const latest = await this.deploymentsRepository.findOne({ where: { applicationId }, order: { createdAt: 'DESC' }, }); if (!latest) { throw new NotFoundException('No deployment found'); } const inProgress = [ DeploymentStatus.PENDING, DeploymentStatus.BUILDING, DeploymentStatus.DEPLOYING, ]; if (!inProgress.includes(latest.status as DeploymentStatus)) { throw new BadRequestException('No deployment in progress to cancel'); } latest.status = DeploymentStatus.CANCELLED; latest.errorMessage = 'Cancelled by user'; latest.finishedAt = new Date(); await this.deploymentsRepository.save(latest); this.buildService.setProgress(latest.id, { phase: 'cancelled', percent: 0, message: 'Cancelled by user' }); try { await this.buildService.cancelBuild(latest.id); } catch (error: any) { this.logger.warn(`Failed to cancel build resources for ${latest.id}: ${error.message}`); } try { await this.buildService.cleanupBuildResourcesForApp(app); } catch (error: any) { this.logger.warn(`Failed to cleanup build resources for ${app.name}: ${error.message}`); } try { await this.kubernetesService.deleteApplication(app); } catch (error: any) { this.logger.warn(`Failed to cleanup deployed resources for cancelled app ${app.name}: ${error.message}`); } return this.deploymentsRepository.findOneOrFail({ where: { id: latest.id } }); } async stopDeployment(applicationId: string, userId: string): Promise { const app = await this.applicationsService.findOne(applicationId, userId); const snapshot = await this.kubernetesService.suspendApplication(app); await this.applicationsService.saveSuspendedReplicas(app.id, snapshot); const latest = await this.deploymentsRepository.findOne({ where: { applicationId }, order: { createdAt: 'DESC' }, }); if (latest) { latest.status = DeploymentStatus.STOPPED; latest.finishedAt = latest.finishedAt || new Date(); await this.deploymentsRepository.save(latest); } return latest; } async startDeployment(applicationId: string, userId: string): Promise { const app = await this.applicationsService.findOne(applicationId, userId); await this.kubernetesService.resumeApplication(app); await this.applicationsService.clearSuspendedReplicas(app.id); const latest = await this.deploymentsRepository.findOne({ where: { applicationId }, order: { createdAt: 'DESC' }, }); if (latest) { latest.status = DeploymentStatus.RUNNING; await this.deploymentsRepository.save(latest); } return latest; } async restartDeployment(applicationId: string, userId: string): Promise { const app = await this.applicationsService.findOne(applicationId, userId); await this.kubernetesService.restartDeployment(app); } /** * Redeploy: re-build from latest source (git pull or existing zip) and deploy new version. * For git-based apps this pulls the latest code, for zip-based it rebuilds from last uploaded zip. */ async redeployApplication(applicationId: string, userId: string): Promise { const app = await this.applicationsService.findOne(applicationId, userId); if (this.isManagedOrHelmOnlyApp(app)) { this.logger.log(`Re-provisioning ${app.name} via Helm (no build)`); return this.triggerDeployment(applicationId, userId); } if (!app.codePath && !app.gitUrl) { throw new NotFoundException('No source code available. Upload code or set a git URL first.'); } this.ensureRedeployAllowed(app); // Create new deployment record const deployment = this.deploymentsRepository.create({ applicationId: app.id, triggeredBy: userId, imageTag: `${app.name}:${Date.now()}`, status: DeploymentStatus.PENDING, version: `v${Date.now()}`, previewSubdomain: null, }); const saved = await this.deploymentsRepository.save(deployment); let previewSubdomain: string | null = null; if (!app.customDomain) { previewSubdomain = await this.resolvePreviewNumber(app.id); await this.deploymentsRepository.update(saved.id, { previewSubdomain }); saved.previewSubdomain = previewSubdomain; } // Trigger async build & deploy pipeline (same as initial deploy) this.executePipeline(saved.id, app, previewSubdomain).catch((error) => { this.logger.error(`Redeploy pipeline failed for deployment ${saved.id}:`, error); }); this.logger.log(`Redeploy triggered for ${app.name} (${app.gitUrl ? 'git: ' + app.gitUrl : 'zip'})`); return saved; } async deleteAllForApplication(applicationId: string): Promise { await this.deploymentsRepository.delete({ applicationId }); } }