Files
cloud-host/backend/src/deployments/deployments.service.ts
T
keyhan 837f0fa63f Harden platform security, reliability, and CI after full audit.
Close deployment IDOR and gate stub payment endpoints, add production
secret validation, health probes, Redis-backed build progress, GitHub
Actions CI, expanded tests, billing/k8s refactors, and ops runbooks.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-06-29 20:59:49 +03:30

707 lines
26 KiB
TypeScript

import { Injectable, NotFoundException, BadRequestException, Logger, Inject, forwardRef } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import * as fs from 'fs';
import { Deployment } from './entities/deployment.entity';
import { ApplicationsService } from '../applications/applications.service';
import { KubernetesService } from '../kubernetes/kubernetes.service';
import { BuildService, BuildProgress, BuildCancelledError } from '../build/build.service';
import * as crypto from 'crypto';
import {
AppLifecycleStatus,
DeploymentStatus,
isManagedProductType,
MANAGED_DEPLOY_MARKER,
} from '../common/enums';
import { ClustersService } from '../clusters/clusters.service';
@Injectable()
export class DeploymentsService {
private readonly logger = new Logger(DeploymentsService.name);
constructor(
@InjectRepository(Deployment)
private deploymentsRepository: Repository<Deployment>,
@Inject(forwardRef(() => ApplicationsService))
private applicationsService: ApplicationsService,
private kubernetesService: KubernetesService,
private buildService: BuildService,
private clustersService: ClustersService,
) {}
/**
* Random 7-digit suffix for the preview host: <userId>-<7-digit>.<baseDomain>.
* Generated once per application (see resolvePreviewNumber) and persisted.
*/
private generatePreviewNumber(): string {
return String(crypto.randomInt(1_000_000, 10_000_000));
}
/**
* Returns a STABLE preview number for an application: reuse the one already
* assigned to a previous deployment so the public preview URL never changes
* across redeploys (otherwise old links 404). Only generates a new number the
* first time the app is deployed. Apps with a custom domain get no preview host.
*/
private async resolvePreviewNumber(applicationId: string): Promise<string> {
const existing = await this.deploymentsRepository
.createQueryBuilder('d')
.select('d.previewSubdomain', 'previewSubdomain')
.where('d.applicationId = :applicationId', { applicationId })
.andWhere('d.previewSubdomain IS NOT NULL')
.orderBy('d.createdAt', 'DESC')
.limit(1)
.getRawOne<{ previewSubdomain: string }>();
return existing?.previewSubdomain || this.generatePreviewNumber();
}
async triggerDeployment(applicationId: string, userId: string): Promise<Deployment> {
const app = await this.applicationsService.findOne(applicationId, userId);
// Create deployment record
const deployment = this.deploymentsRepository.create({
applicationId: app.id,
triggeredBy: userId,
imageTag: `${app.name}:${Date.now()}`,
status: DeploymentStatus.PENDING,
version: `v${Date.now()}`,
previewSubdomain: null,
});
const saved = await this.deploymentsRepository.save(deployment);
// Fill deterministic preview number after we have the deployment id.
let previewSubdomain: string | null = null;
if (!app.customDomain) {
previewSubdomain = await this.resolvePreviewNumber(app.id);
await this.deploymentsRepository.update(saved.id, { previewSubdomain });
saved.previewSubdomain = previewSubdomain;
}
// Trigger async pipeline (Helm-only for managed services, build+deploy for apps)
const run = isManagedProductType(app.productType)
? this.executeManagedPipeline(saved.id, app)
: this.executePipeline(saved.id, app, previewSubdomain);
run.catch((error) => {
this.logger.error(`Pipeline failed for deployment ${saved.id}:`, error);
});
return saved;
}
/** Provision managed database/redis/rabbitmq via Helm only — no image build. */
private async executeManagedPipeline(deploymentId: string, app: any): Promise<void> {
try {
await this.updateStatus(deploymentId, DeploymentStatus.DEPLOYING);
this.buildService.setProgress(deploymentId, {
phase: 'deploying',
percent: 10,
message: 'Provisioning service via Helm...',
});
const hasDbDump = app.dbDumpPath && fs.existsSync(app.dbDumpPath);
const { app: deployedApp, k8sResources } = await this.deployManagedWithClusterFallback(
deploymentId,
app,
hasDbDump,
);
app = deployedApp;
if (hasDbDump) {
this.logger.log(`Restoring DB dump for ${app.name} from ${app.dbDumpPath}`);
try {
await this.kubernetesService.waitForDatabaseReady(app, 120_000);
const freshApp = await this.applicationsService.findOne(app.id);
const result = await this.kubernetesService.restoreDatabaseDump(freshApp, freshApp.dbDumpPath!);
if (result.success) {
this.logger.log(`DB dump restored successfully for ${app.name}`);
} else {
this.logger.warn(`DB dump restore failed for ${app.name}: ${result.logs}`);
}
} catch (e: any) {
this.logger.warn(`DB dump restore error for ${app.name}: ${e.message}`);
}
}
this.buildService.setProgress(deploymentId, {
phase: 'deploying',
percent: 96,
message: 'Waiting for service pods to become ready...',
});
await this.kubernetesService.waitForApplicationReady(
app,
600_000,
() => this.isDeploymentCancelled(deploymentId),
);
if (await this.isDeploymentCancelled(deploymentId)) {
return;
}
await this.applicationsService.updateImageTag(app.id, MANAGED_DEPLOY_MARKER);
this.buildService.setProgress(deploymentId, {
phase: 'done',
percent: 100,
message: 'Service provisioned',
});
await this.deploymentsRepository.update(deploymentId, {
status: DeploymentStatus.RUNNING,
k8sResources,
finishedAt: new Date(),
});
} catch (error: any) {
if (this.isCancellationError(error) || (await this.isDeploymentCancelled(deploymentId))) {
this.logger.log(`Deployment ${deploymentId} cancelled by user`);
await this.deploymentsRepository.update(deploymentId, {
status: DeploymentStatus.CANCELLED,
errorMessage: 'Cancelled by user',
finishedAt: new Date(),
});
return;
}
this.logger.error(`Managed deployment ${deploymentId} failed:`, error);
this.buildService.setProgress(deploymentId, {
phase: 'failed',
percent: 0,
message: error.message || 'Provisioning failed',
});
await this.deploymentsRepository.update(deploymentId, {
status: DeploymentStatus.FAILED,
errorMessage: error.message,
finishedAt: new Date(),
});
}
}
private async executePipeline(
deploymentId: string,
app: any,
previewSubdomain: string | null,
): Promise<void> {
try {
// Step 1: Build image
await this.updateStatus(deploymentId, DeploymentStatus.BUILDING);
const buildResult = await this.buildService.buildImage(app, deploymentId);
const imageUri = buildResult.imageUri;
// Save build log
await this.deploymentsRepository.update(deploymentId, { buildLog: buildResult.buildLog });
// Step 2: Update app with new image tag
await this.applicationsService.updateImageTag(app.id, imageUri);
// Step 3: Deploy to Kubernetes
await this.updateStatus(deploymentId, DeploymentStatus.DEPLOYING);
this.buildService.setProgress(deploymentId, {
phase: 'deploying',
percent: 92,
message: 'Deploying to Kubernetes...',
});
const hasDbDump = app.dbDumpPath && fs.existsSync(app.dbDumpPath);
const { app: deployedApp, k8sResources } = await this.deployWithClusterFallback(
deploymentId,
app,
imageUri,
hasDbDump,
previewSubdomain,
);
app = deployedApp;
// Step 3.5: Restore DB dump if one was uploaded (must happen after deploy creates the namespace + DB)
if (hasDbDump) {
this.logger.log(`Restoring DB dump for ${app.name} from ${app.dbDumpPath}`);
try {
await this.kubernetesService.waitForDatabaseReady(app, 120_000);
const freshApp = await this.applicationsService.findOne(app.id);
const result = await this.kubernetesService.restoreDatabaseDump(freshApp, freshApp.dbDumpPath!);
if (result.success) {
this.logger.log(`DB dump restored successfully for ${app.name}`);
} else {
this.logger.warn(`DB dump restore failed for ${app.name}: ${result.logs}`);
}
} catch (e: any) {
this.logger.warn(`DB dump restore error for ${app.name}: ${e.message}`);
}
// Scale WordPress app up after restore (or even if restore failed)
try {
await this.kubernetesService.scaleDeployment(app, app.replicas || 1);
this.logger.log(`Scaled ${app.name} to ${app.replicas || 1} replica(s) after DB restore`);
} catch (e: any) {
this.logger.warn(`Failed to scale up ${app.name} after DB restore: ${e.message}`);
}
}
// Step 4: wait for every workload owned by this app before marking it running.
// Helm/API apply success only means resources were accepted; quota or scheduling
// pressure can still leave DB/addon/app pods Pending.
this.buildService.setProgress(deploymentId, {
phase: 'deploying',
percent: 96,
message: 'Waiting for all application pods to become ready...',
});
await this.kubernetesService.waitForApplicationReady(
app,
600_000,
() => this.isDeploymentCancelled(deploymentId),
);
if (await this.isDeploymentCancelled(deploymentId)) {
return;
}
// Step 5: Mark success
this.buildService.setProgress(deploymentId, {
phase: 'done',
percent: 100,
message: 'Deployment complete',
});
await this.deploymentsRepository.update(deploymentId, {
status: DeploymentStatus.RUNNING,
k8sResources,
finishedAt: new Date(),
});
} catch (error: any) {
if (this.isCancellationError(error) || await this.isDeploymentCancelled(deploymentId)) {
this.logger.log(`Deployment ${deploymentId} cancelled by user`);
await this.deploymentsRepository.update(deploymentId, {
status: DeploymentStatus.CANCELLED,
errorMessage: 'Cancelled by user',
finishedAt: new Date(),
});
return;
}
this.logger.error(`Deployment ${deploymentId} failed:`, error);
this.buildService.setProgress(deploymentId, {
phase: 'failed',
percent: 0,
message: error.message || 'Deployment failed',
});
// Save build log if available (attached by build service on failure)
const buildLog = error.buildLog || null;
await this.deploymentsRepository.update(deploymentId, {
status: DeploymentStatus.FAILED,
errorMessage: error.message,
...(buildLog ? { buildLog } : {}),
finishedAt: new Date(),
});
}
}
private async deployManagedWithClusterFallback(
deploymentId: string,
app: any,
hasDbDump: boolean,
): Promise<{ app: any; k8sResources: Record<string, any> }> {
const failedClusterIds: string[] = [];
let currentApp = app;
let lastError: any;
const maxAttempts = Number(process.env.CLUSTER_DEPLOY_FALLBACK_ATTEMPTS || 3);
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
if (await this.isDeploymentCancelled(deploymentId)) {
throw new Error('Deployment cancelled by user');
}
try {
this.buildService.setProgress(deploymentId, {
phase: 'deploying',
percent: Math.min(20 + attempt * 5, 90),
message:
attempt === 1
? 'Installing Helm release...'
: `Retrying Helm install on fallback cluster (${attempt}/${maxAttempts})...`,
});
const k8sResources = await this.kubernetesService.deployManagedService(currentApp);
return { app: currentApp, k8sResources };
} catch (error: any) {
lastError = error;
failedClusterIds.push(currentApp.clusterId);
const failureMessage = error?.message || 'Helm provisioning failed on selected cluster';
await this.clustersService.markAllocationFailure(currentApp.id, currentApp.clusterId, failureMessage);
if (attempt >= maxAttempts) {
break;
}
try {
const fallback = await this.clustersService.chooseFallbackClusterForApplication(
currentApp,
failedClusterIds,
failureMessage,
);
const updatedApp = await this.applicationsService.updateClusterAssignment(
currentApp.id,
fallback.cluster.id,
fallback.pool?.id,
);
await this.clustersService.attachAllocationToApplication(fallback.allocationLogId, currentApp.id);
this.logger.warn(
`Managed deployment ${deploymentId} falling back from cluster ${currentApp.clusterId || 'none'} to ${fallback.cluster.id}`,
);
currentApp = {
...currentApp,
...updatedApp,
clusterId: fallback.cluster.id,
poolId: fallback.pool?.id || currentApp.poolId,
};
} catch (fallbackError: any) {
lastError = fallbackError;
break;
}
}
}
throw lastError || new Error('Managed service provisioning failed');
}
private async deployWithClusterFallback(
deploymentId: string,
app: any,
imageUri: string,
hasDbDump: boolean,
previewSubdomain: string | null,
): Promise<{ app: any; k8sResources: Record<string, any> }> {
const failedClusterIds: string[] = [];
let currentApp = app;
let lastError: any;
const maxAttempts = Number(process.env.CLUSTER_DEPLOY_FALLBACK_ATTEMPTS || 3);
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
if (await this.isDeploymentCancelled(deploymentId)) {
throw new Error('Deployment cancelled by user');
}
try {
this.buildService.setProgress(deploymentId, {
phase: 'deploying',
percent: Math.min(92 + attempt, 95),
message: attempt === 1
? 'Deploying to selected cluster...'
: `Retrying deployment on fallback cluster (${attempt}/${maxAttempts})...`,
});
const deployApp =
hasDbDump && !isManagedProductType(currentApp.productType)
? { ...currentApp, replicas: 0 }
: currentApp;
const k8sResources = await this.kubernetesService.deployApplication(deployApp, imageUri, {
previewNumber: previewSubdomain,
});
return { app: currentApp, k8sResources };
} catch (error: any) {
lastError = error;
failedClusterIds.push(currentApp.clusterId);
const failureMessage = error?.message || 'Deployment failed on selected cluster';
await this.clustersService.markAllocationFailure(currentApp.id, currentApp.clusterId, failureMessage);
if (attempt >= maxAttempts) {
break;
}
try {
const fallback = await this.clustersService.chooseFallbackClusterForApplication(
currentApp,
failedClusterIds,
failureMessage,
);
const updatedApp = await this.applicationsService.updateClusterAssignment(
currentApp.id,
fallback.cluster.id,
fallback.pool?.id,
);
await this.clustersService.attachAllocationToApplication(fallback.allocationLogId, currentApp.id);
this.logger.warn(
`Deployment ${deploymentId} falling back from cluster ${currentApp.clusterId || 'none'} to ${fallback.cluster.id}`,
);
currentApp = { ...currentApp, ...updatedApp, clusterId: fallback.cluster.id, poolId: fallback.pool?.id || currentApp.poolId };
} catch (fallbackError: any) {
this.logger.warn(`No fallback cluster available for deployment ${deploymentId}: ${fallbackError.message}`);
break;
}
}
}
throw lastError;
}
async updateStatus(id: string, status: DeploymentStatus): Promise<void> {
if (await this.isDeploymentCancelled(id)) {
return;
}
await this.deploymentsRepository.update(id, { status });
}
private async isDeploymentCancelled(id: string): Promise<boolean> {
const deployment = await this.deploymentsRepository.findOne({ where: { id } });
return deployment?.status === DeploymentStatus.CANCELLED || deployment?.errorMessage === 'Cancelled by user';
}
private isCancellationError(error: any): boolean {
return error instanceof BuildCancelledError ||
error?.name === 'BuildCancelledError' ||
String(error?.message || '').toLowerCase().includes('cancelled');
}
/** Managed DB/Redis/RabbitMQ or rows already provisioned via Helm without an app image build. */
private isManagedOrHelmOnlyApp(app: { productType?: string; latestImageTag?: string }): boolean {
if (isManagedProductType(app.productType)) return true;
return app.latestImageTag === MANAGED_DEPLOY_MARKER;
}
private ensureRedeployAllowed(app: any): void {
if (!app.billingCycle) return;
const isActive = app.lifecycleStatus === AppLifecycleStatus.ACTIVE;
const expiresAt = app.planExpiresAt ? new Date(app.planExpiresAt) : null;
const hasPaidTimeRemaining = !!expiresAt && expiresAt > new Date();
if (!isActive || !hasPaidTimeRemaining) {
throw new BadRequestException('Payment must be completed successfully before redeploying this application.');
}
}
async findByApplication(applicationId: string, userId?: string): Promise<Deployment[]> {
await this.applicationsService.findOne(applicationId, userId);
return this.deploymentsRepository.find({
where: { applicationId },
order: { createdAt: 'DESC' },
});
}
async findOne(id: string, userId?: string): Promise<Deployment> {
const deployment = await this.deploymentsRepository.findOne({
where: { id },
relations: { application: true },
});
if (!deployment) {
throw new NotFoundException('Deployment not found');
}
await this.applicationsService.findOne(deployment.applicationId, userId);
return deployment;
}
async getLogs(applicationId: string, userId: string): Promise<string> {
const app = await this.applicationsService.findOne(applicationId, userId);
return this.kubernetesService.getPodLogs(app);
}
async getBuildLogs(applicationId: string, userId: string): Promise<{ buildLog: string | null; status: string; version: string | null; createdAt: Date }> {
const app = await this.applicationsService.findOne(applicationId, userId);
const latest = await this.deploymentsRepository.findOne({
where: { applicationId },
order: { createdAt: 'DESC' },
});
if (!latest) {
return { buildLog: null, status: 'no_deployment', version: null, createdAt: new Date() };
}
if (this.isManagedOrHelmOnlyApp(app)) {
return {
buildLog: null,
status: latest.status,
version: latest.version,
createdAt: latest.createdAt,
};
}
// While the build is in progress the persisted buildLog isn't written yet —
// stream the live build pod logs so the deploy modal can show them in real time.
let buildLog = latest.buildLog || null;
if (
latest.status === DeploymentStatus.BUILDING ||
latest.status === DeploymentStatus.DEPLOYING
) {
const live = await this.buildService.getLiveBuildLog(latest.id);
if (live) buildLog = live;
}
return {
buildLog,
status: latest.status,
version: latest.version,
createdAt: latest.createdAt,
};
}
async getBuildProgress(applicationId: string, userId: string): Promise<BuildProgress | null> {
const app = await this.applicationsService.findOne(applicationId, userId);
const managed = this.isManagedOrHelmOnlyApp(app);
const latest = await this.deploymentsRepository.findOne({
where: { applicationId },
order: { createdAt: 'DESC' },
});
if (!latest) return null;
const progress = await this.buildService.getProgress(latest.id);
if (progress) return progress;
// No in-memory progress — infer from deployment status
if (latest.status === DeploymentStatus.RUNNING) {
return { phase: 'done', percent: 100, message: 'Deployment complete' };
}
if (latest.status === DeploymentStatus.FAILED) {
return { phase: 'failed', percent: 0, message: latest.errorMessage || 'Deployment failed' };
}
if (latest.status === DeploymentStatus.CANCELLED) {
return { phase: 'cancelled', percent: 0, message: latest.errorMessage || 'Cancelled by user' };
}
if (latest.status === DeploymentStatus.BUILDING) {
return {
phase: managed ? 'deploying' : 'building',
percent: 0,
message: managed ? 'Provisioning...' : 'Building...',
};
}
if (latest.status === DeploymentStatus.DEPLOYING) {
return {
phase: 'deploying',
percent: 90,
message: managed ? 'Provisioning via Helm...' : 'Deploying...',
};
}
return null;
}
async cancelDeployment(applicationId: string, userId: string): Promise<Deployment> {
const app = await this.applicationsService.findOne(applicationId, userId);
const latest = await this.deploymentsRepository.findOne({
where: { applicationId },
order: { createdAt: 'DESC' },
});
if (!latest) {
throw new NotFoundException('No deployment found');
}
const inProgress = [
DeploymentStatus.PENDING,
DeploymentStatus.BUILDING,
DeploymentStatus.DEPLOYING,
];
if (!inProgress.includes(latest.status as DeploymentStatus)) {
throw new BadRequestException('No deployment in progress to cancel');
}
latest.status = DeploymentStatus.CANCELLED;
latest.errorMessage = 'Cancelled by user';
latest.finishedAt = new Date();
await this.deploymentsRepository.save(latest);
this.buildService.setProgress(latest.id, { phase: 'cancelled', percent: 0, message: 'Cancelled by user' });
try {
await this.buildService.cancelBuild(latest.id);
} catch (error: any) {
this.logger.warn(`Failed to cancel build resources for ${latest.id}: ${error.message}`);
}
try {
await this.buildService.cleanupBuildResourcesForApp(app);
} catch (error: any) {
this.logger.warn(`Failed to cleanup build resources for ${app.name}: ${error.message}`);
}
try {
await this.kubernetesService.deleteApplication(app);
} catch (error: any) {
this.logger.warn(`Failed to cleanup deployed resources for cancelled app ${app.name}: ${error.message}`);
}
return this.deploymentsRepository.findOneOrFail({ where: { id: latest.id } });
}
async stopDeployment(applicationId: string, userId: string): Promise<Deployment | null> {
const app = await this.applicationsService.findOne(applicationId, userId);
const snapshot = await this.kubernetesService.suspendApplication(app);
await this.applicationsService.saveSuspendedReplicas(app.id, snapshot);
const latest = await this.deploymentsRepository.findOne({
where: { applicationId },
order: { createdAt: 'DESC' },
});
if (latest) {
latest.status = DeploymentStatus.STOPPED;
latest.finishedAt = latest.finishedAt || new Date();
await this.deploymentsRepository.save(latest);
}
return latest;
}
async startDeployment(applicationId: string, userId: string): Promise<Deployment | null> {
const app = await this.applicationsService.findOne(applicationId, userId);
await this.kubernetesService.resumeApplication(app);
await this.applicationsService.clearSuspendedReplicas(app.id);
const latest = await this.deploymentsRepository.findOne({
where: { applicationId },
order: { createdAt: 'DESC' },
});
if (latest) {
latest.status = DeploymentStatus.RUNNING;
await this.deploymentsRepository.save(latest);
}
return latest;
}
async restartDeployment(applicationId: string, userId: string): Promise<void> {
const app = await this.applicationsService.findOne(applicationId, userId);
await this.kubernetesService.restartDeployment(app);
}
/**
* Redeploy: re-build from latest source (git pull or existing zip) and deploy new version.
* For git-based apps this pulls the latest code, for zip-based it rebuilds from last uploaded zip.
*/
async redeployApplication(applicationId: string, userId: string): Promise<Deployment> {
const app = await this.applicationsService.findOne(applicationId, userId);
if (this.isManagedOrHelmOnlyApp(app)) {
this.logger.log(`Re-provisioning ${app.name} via Helm (no build)`);
return this.triggerDeployment(applicationId, userId);
}
if (!app.codePath && !app.gitUrl) {
throw new NotFoundException('No source code available. Upload code or set a git URL first.');
}
this.ensureRedeployAllowed(app);
// Create new deployment record
const deployment = this.deploymentsRepository.create({
applicationId: app.id,
triggeredBy: userId,
imageTag: `${app.name}:${Date.now()}`,
status: DeploymentStatus.PENDING,
version: `v${Date.now()}`,
previewSubdomain: null,
});
const saved = await this.deploymentsRepository.save(deployment);
let previewSubdomain: string | null = null;
if (!app.customDomain) {
previewSubdomain = await this.resolvePreviewNumber(app.id);
await this.deploymentsRepository.update(saved.id, { previewSubdomain });
saved.previewSubdomain = previewSubdomain;
}
// Trigger async build & deploy pipeline (same as initial deploy)
this.executePipeline(saved.id, app, previewSubdomain).catch((error) => {
this.logger.error(`Redeploy pipeline failed for deployment ${saved.id}:`, error);
});
this.logger.log(`Redeploy triggered for ${app.name} (${app.gitUrl ? 'git: ' + app.gitUrl : 'zip'})`);
return saved;
}
async deleteAllForApplication(applicationId: string): Promise<void> {
await this.deploymentsRepository.delete({ applicationId });
}
}