revert(build): remove app build pipeline revamp (Nixpacks/MinIO/Trivy/registry GC)
Reverts commits3eff38fandc379a23and restores the previous Kaniko-only build pipeline (runtime detection + per-runtime Dockerfile generation, disk-based source upload). Removed: Nixpacks Dockerfile generation, MinIO source storage (common/storage), Bull build queue + Redis build state (common/redis, deployment.processor), Trivy image scan (scan.service, deployment.vulnerabilitySummary), and daily registry garbage collection (registry-gc). Nothing outside the build/deploy path depended on these. Backend tsc + 105/106 tests green (the pre-existing helm.service chartPath failure is unrelated); frontend tsc green. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -1,2 +0,0 @@
|
||||
/** Bull queue name for build+deploy pipelines. */
|
||||
export const DEPLOY_QUEUE = 'app-deploy';
|
||||
@@ -1,27 +0,0 @@
|
||||
import { Process, Processor } from '@nestjs/bull';
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { Job } from 'bull';
|
||||
import { DeploymentsService, DeploymentJobData } from './deployments.service';
|
||||
import { DEPLOY_QUEUE } from './deployment.constants';
|
||||
|
||||
/**
|
||||
* Processes build+deploy pipelines off the `app-deploy` queue with a bounded
|
||||
* concurrency (BUILD_CONCURRENCY, default 3) so simultaneous user deploys can't
|
||||
* flood the cluster with Kaniko build jobs (2 CPU / 4Gi each).
|
||||
*
|
||||
* Concurrency is read at module-load time from env because Bull's `@Process`
|
||||
* decorator option must be a constant.
|
||||
*/
|
||||
@Processor(DEPLOY_QUEUE)
|
||||
export class DeploymentProcessor {
|
||||
private readonly logger = new Logger(DeploymentProcessor.name);
|
||||
|
||||
constructor(private readonly deploymentsService: DeploymentsService) {}
|
||||
|
||||
@Process({ name: 'run', concurrency: parseInt(process.env.BUILD_CONCURRENCY || '3', 10) })
|
||||
async handleRun(job: Job<DeploymentJobData>): Promise<void> {
|
||||
const { deploymentId } = job.data;
|
||||
this.logger.log(`Processing deployment ${deploymentId} (job ${job.id})`);
|
||||
await this.deploymentsService.processDeploymentJob(job.data);
|
||||
}
|
||||
}
|
||||
@@ -1,10 +1,7 @@
|
||||
import { Module, forwardRef } from '@nestjs/common';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
import { BullModule } from '@nestjs/bull';
|
||||
import { DeploymentsService } from './deployments.service';
|
||||
import { DeploymentsController } from './deployments.controller';
|
||||
import { DeploymentProcessor } from './deployment.processor';
|
||||
import { DEPLOY_QUEUE } from './deployment.constants';
|
||||
import { Deployment } from './entities/deployment.entity';
|
||||
import { ApplicationsModule } from '../applications/applications.module';
|
||||
import { KubernetesModule } from '../kubernetes/kubernetes.module';
|
||||
@@ -14,14 +11,13 @@ import { ClustersModule } from '../clusters/clusters.module';
|
||||
@Module({
|
||||
imports: [
|
||||
TypeOrmModule.forFeature([Deployment]),
|
||||
BullModule.registerQueue({ name: DEPLOY_QUEUE }),
|
||||
forwardRef(() => ApplicationsModule),
|
||||
forwardRef(() => ClustersModule),
|
||||
KubernetesModule,
|
||||
BuildModule,
|
||||
],
|
||||
controllers: [DeploymentsController],
|
||||
providers: [DeploymentsService, DeploymentProcessor],
|
||||
providers: [DeploymentsService],
|
||||
exports: [DeploymentsService],
|
||||
})
|
||||
export class DeploymentsModule {}
|
||||
|
||||
@@ -1,15 +1,11 @@
|
||||
import { Injectable, NotFoundException, BadRequestException, Logger, Inject, forwardRef, OnModuleInit } from '@nestjs/common';
|
||||
import { Injectable, NotFoundException, BadRequestException, Logger, Inject, forwardRef } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import { InjectQueue } from '@nestjs/bull';
|
||||
import { Queue } from 'bull';
|
||||
import { Repository } from 'typeorm';
|
||||
import * as fs from 'fs';
|
||||
import { DEPLOY_QUEUE } from './deployment.constants';
|
||||
import { Deployment } from './entities/deployment.entity';
|
||||
import { ApplicationsService } from '../applications/applications.service';
|
||||
import { KubernetesService } from '../kubernetes/kubernetes.service';
|
||||
import { BuildService, BuildProgress, BuildCancelledError } from '../build/build.service';
|
||||
import { ScanService } from '../build/scan.service';
|
||||
import * as crypto from 'crypto';
|
||||
import {
|
||||
AppLifecycleStatus,
|
||||
@@ -19,15 +15,8 @@ import {
|
||||
} from '../common/enums';
|
||||
import { ClustersService } from '../clusters/clusters.service';
|
||||
|
||||
/** Payload enqueued on the `app-deploy` queue for the build+deploy pipeline. */
|
||||
export interface DeploymentJobData {
|
||||
deploymentId: string;
|
||||
applicationId: string;
|
||||
previewSubdomain: string | null;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class DeploymentsService implements OnModuleInit {
|
||||
export class DeploymentsService {
|
||||
private readonly logger = new Logger(DeploymentsService.name);
|
||||
|
||||
constructor(
|
||||
@@ -38,41 +27,8 @@ export class DeploymentsService implements OnModuleInit {
|
||||
private kubernetesService: KubernetesService,
|
||||
private buildService: BuildService,
|
||||
private clustersService: ClustersService,
|
||||
private scanService: ScanService,
|
||||
@InjectQueue(DEPLOY_QUEUE)
|
||||
private deployQueue: Queue<DeploymentJobData>,
|
||||
) {}
|
||||
|
||||
async onModuleInit(): Promise<void> {
|
||||
try {
|
||||
await this.deploymentsRepository.query(
|
||||
`ALTER TABLE deployments ADD COLUMN IF NOT EXISTS "vulnerabilitySummary" jsonb`,
|
||||
);
|
||||
} catch (e: any) {
|
||||
this.logger.warn(`Could not ensure deployments.vulnerabilitySummary column: ${e.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Queue worker entrypoint — runs one build+deploy pipeline. Bounded
|
||||
* concurrency lives on the Bull processor, so this just dispatches to the
|
||||
* managed (Helm-only) or app (build+deploy) pipeline. Both pipelines handle
|
||||
* their own errors, so a failure here never triggers a Bull retry.
|
||||
*/
|
||||
async processDeploymentJob(data: DeploymentJobData): Promise<void> {
|
||||
const { deploymentId, applicationId, previewSubdomain } = data;
|
||||
if (await this.isDeploymentCancelled(deploymentId)) {
|
||||
this.logger.log(`Deployment ${deploymentId} already cancelled before pickup — skipping`);
|
||||
return;
|
||||
}
|
||||
const app = await this.applicationsService.findOne(applicationId);
|
||||
if (isManagedProductType(app.productType)) {
|
||||
await this.executeManagedPipeline(deploymentId, app);
|
||||
} else {
|
||||
await this.executePipeline(deploymentId, app, previewSubdomain);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Random 7-digit suffix for the preview host: <userId>-<7-digit>.<baseDomain>.
|
||||
* Generated once per application (see resolvePreviewNumber) and persisted.
|
||||
@@ -121,13 +77,13 @@ export class DeploymentsService implements OnModuleInit {
|
||||
saved.previewSubdomain = previewSubdomain;
|
||||
}
|
||||
|
||||
// Enqueue the build+deploy pipeline. The Bull processor runs it with bounded
|
||||
// concurrency so concurrent user deploys can't flood the cluster.
|
||||
await this.deployQueue.add(
|
||||
'run',
|
||||
{ deploymentId: saved.id, applicationId: app.id, previewSubdomain },
|
||||
{ removeOnComplete: true, removeOnFail: true },
|
||||
);
|
||||
// Trigger async pipeline (Helm-only for managed services, build+deploy for apps)
|
||||
const run = isManagedProductType(app.productType)
|
||||
? this.executeManagedPipeline(saved.id, app)
|
||||
: this.executePipeline(saved.id, app, previewSubdomain);
|
||||
run.catch((error) => {
|
||||
this.logger.error(`Pipeline failed for deployment ${saved.id}:`, error);
|
||||
});
|
||||
|
||||
return saved;
|
||||
}
|
||||
@@ -231,19 +187,6 @@ export class DeploymentsService implements OnModuleInit {
|
||||
// Save build log
|
||||
await this.deploymentsRepository.update(deploymentId, { buildLog: buildResult.buildLog });
|
||||
|
||||
// Report-only vulnerability scan — runs alongside the deploy and is
|
||||
// persisted when it finishes. Never blocks or fails the deployment.
|
||||
void this.scanService
|
||||
.scanImage(app, imageUri)
|
||||
.then((summary) => {
|
||||
if (summary) {
|
||||
return this.deploymentsRepository.update(deploymentId, {
|
||||
vulnerabilitySummary: summary as Record<string, any>,
|
||||
});
|
||||
}
|
||||
})
|
||||
.catch((e) => this.logger.warn(`Scan persistence failed for ${deploymentId}: ${e.message}`));
|
||||
|
||||
// Step 2: Update app with new image tag
|
||||
await this.applicationsService.updateImageTag(app.id, imageUri);
|
||||
|
||||
@@ -543,10 +486,7 @@ export class DeploymentsService implements OnModuleInit {
|
||||
return this.kubernetesService.getPodLogs(app);
|
||||
}
|
||||
|
||||
async getBuildLogs(
|
||||
applicationId: string,
|
||||
userId: string,
|
||||
): Promise<{ buildLog: string | null; status: string; version: string | null; createdAt: Date; vulnerabilitySummary: Record<string, any> | null }> {
|
||||
async getBuildLogs(applicationId: string, userId: string): Promise<{ buildLog: string | null; status: string; version: string | null; createdAt: Date }> {
|
||||
const app = await this.applicationsService.findOne(applicationId, userId);
|
||||
|
||||
const latest = await this.deploymentsRepository.findOne({
|
||||
@@ -555,7 +495,7 @@ export class DeploymentsService implements OnModuleInit {
|
||||
});
|
||||
|
||||
if (!latest) {
|
||||
return { buildLog: null, status: 'no_deployment', version: null, createdAt: new Date(), vulnerabilitySummary: null };
|
||||
return { buildLog: null, status: 'no_deployment', version: null, createdAt: new Date() };
|
||||
}
|
||||
|
||||
if (this.isManagedOrHelmOnlyApp(app)) {
|
||||
@@ -564,7 +504,6 @@ export class DeploymentsService implements OnModuleInit {
|
||||
status: latest.status,
|
||||
version: latest.version,
|
||||
createdAt: latest.createdAt,
|
||||
vulnerabilitySummary: null,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -584,7 +523,6 @@ export class DeploymentsService implements OnModuleInit {
|
||||
status: latest.status,
|
||||
version: latest.version,
|
||||
createdAt: latest.createdAt,
|
||||
vulnerabilitySummary: latest.vulnerabilitySummary ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -599,7 +537,7 @@ export class DeploymentsService implements OnModuleInit {
|
||||
|
||||
if (!latest) return null;
|
||||
|
||||
const progress = await this.buildService.getProgress(latest.id);
|
||||
const progress = this.buildService.getProgress(latest.id);
|
||||
if (progress) return progress;
|
||||
|
||||
// No in-memory progress — infer from deployment status
|
||||
@@ -751,12 +689,10 @@ export class DeploymentsService implements OnModuleInit {
|
||||
saved.previewSubdomain = previewSubdomain;
|
||||
}
|
||||
|
||||
// Enqueue the build & deploy pipeline (same queue as initial deploy)
|
||||
await this.deployQueue.add(
|
||||
'run',
|
||||
{ deploymentId: saved.id, applicationId: app.id, previewSubdomain },
|
||||
{ removeOnComplete: true, removeOnFail: true },
|
||||
);
|
||||
// Trigger async build & deploy pipeline (same as initial deploy)
|
||||
this.executePipeline(saved.id, app, previewSubdomain).catch((error) => {
|
||||
this.logger.error(`Redeploy pipeline failed for deployment ${saved.id}:`, error);
|
||||
});
|
||||
|
||||
this.logger.log(`Redeploy triggered for ${app.name} (${app.gitUrl ? 'git: ' + app.gitUrl : 'zip'})`);
|
||||
return saved;
|
||||
|
||||
@@ -33,14 +33,6 @@ export class Deployment {
|
||||
@Column({ type: 'text', nullable: true })
|
||||
deployLog: string;
|
||||
|
||||
/**
|
||||
* Report-only Trivy vulnerability summary for the built image
|
||||
* (e.g. { critical, high, medium, low, unknown, total, scannedAt }).
|
||||
* Non-blocking — never gates a deployment.
|
||||
*/
|
||||
@Column({ type: 'jsonb', nullable: true })
|
||||
vulnerabilitySummary: Record<string, any> | null;
|
||||
|
||||
/**
|
||||
* Per-deployment preview number (derived deterministically from deployment.id).
|
||||
* Used to build preview ingress host under the main frontend domain.
|
||||
|
||||
Reference in New Issue
Block a user