revert(build): remove app build pipeline revamp (Nixpacks/MinIO/Trivy/registry GC)

Reverts commits 3eff38f and c379a23 and restores the previous Kaniko-only
build pipeline (runtime detection + per-runtime Dockerfile generation,
disk-based source upload).

Removed: Nixpacks Dockerfile generation, MinIO source storage (common/storage),
Bull build queue + Redis build state (common/redis, deployment.processor),
Trivy image scan (scan.service, deployment.vulnerabilitySummary), and daily
registry garbage collection (registry-gc). Nothing outside the build/deploy
path depended on these. Backend tsc + 105/106 tests green (the pre-existing
helm.service chartPath failure is unrelated); frontend tsc green.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
keyhan
2026-06-23 19:21:30 +03:30
parent bd14eb2daa
commit 9c16b462f4
27 changed files with 1297 additions and 1951 deletions
-8
View File
@@ -15,8 +15,6 @@ import { SnapshotsModule } from './snapshots/snapshots.module';
import { LifecycleModule } from './lifecycle/lifecycle.module';
import { ApplicationMigrationsModule } from './application-migrations/application-migrations.module';
import { AdminModule } from './admin/admin.module';
import { RedisModule } from './common/redis/redis.module';
import { StorageModule } from './common/storage/storage.module';
import configuration from './config/configuration';
@Module({
@@ -56,12 +54,6 @@ import configuration from './config/configuration';
inject: [ConfigService],
}),
// Shared Redis client (build state across replicas)
RedisModule,
// Shared MinIO storage (application source archives)
StorageModule,
// Feature modules
AuthModule,
UsersModule,
@@ -17,7 +17,6 @@ import {
} from '../common/enums';
import { ensureAppUrlEnv } from './app-url.util';
import { normalizeCreateApplicationDto } from './managed-service.util';
import { StorageService } from '../common/storage/storage.service';
@Injectable()
export class ApplicationsService {
@@ -28,7 +27,6 @@ export class ApplicationsService {
private appsRepository: Repository<Application>,
private clustersService: ClustersService,
private configService: ConfigService,
private storageService: StorageService,
) {}
private toDnsLabel(value: string): string {
@@ -206,20 +204,18 @@ export class ApplicationsService {
async delete(id: string, userId: string): Promise<Application> {
const app = await this.findOne(id, userId);
// Delete the uploaded source archive from object storage.
// Delete uploaded files
if (app.codePath) {
await this.storageService.removeSource(app.codePath);
}
// Remove any legacy on-disk dump/source dir (db dumps are still stored locally).
try {
const uploadDir = this.configService.get<string>('platform.uploadDir') || './uploads';
const appDir = path.join(uploadDir, app.userId, app.id);
if (fs.existsSync(appDir)) {
fs.rmSync(appDir, { recursive: true, force: true });
this.logger.log(`Deleted upload directory: ${appDir}`);
try {
const uploadDir = this.configService.get<string>('platform.uploadDir') || './uploads';
const appDir = path.join(uploadDir, app.userId, app.id);
if (fs.existsSync(appDir)) {
fs.rmSync(appDir, { recursive: true, force: true });
this.logger.log(`Deleted upload directory: ${appDir}`);
}
} catch (e: any) {
this.logger.warn(`Failed to delete upload dir for ${app.name}: ${e.message}`);
}
} catch (e: any) {
this.logger.warn(`Failed to delete upload dir for ${app.name}: ${e.message}`);
}
await this.appsRepository.remove(app);
@@ -265,14 +261,21 @@ export class ApplicationsService {
}
const app = await this.findOne(id, userId);
const uploadDir = this.configService.get<string>('platform.uploadDir') || './uploads';
const appDir = path.join(uploadDir, app.userId, app.id);
// Stream the archive to MinIO; codePath stores the object key (build pods
// pull it via a presigned URL — no local disk, no PVC, no kubectl cp).
const key = await this.storageService.putSource(app.userId, app.id, file.buffer);
app.codePath = key;
// Ensure directory exists
fs.mkdirSync(appDir, { recursive: true });
// Save the zip file
const zipPath = path.join(appDir, 'source.zip');
fs.writeFileSync(zipPath, file.buffer);
// Update app with code path
app.codePath = zipPath;
const saved = await this.appsRepository.save(app);
this.logger.log(`Uploaded code for ${app.name}${key} (${(file.size / 1024).toFixed(1)} KB)`);
this.logger.log(`Uploaded code for ${app.name}${zipPath} (${(file.size / 1024).toFixed(1)} KB)`);
return saved;
}
+2 -3
View File
@@ -1,6 +1,5 @@
import { Module, forwardRef } from '@nestjs/common';
import { BuildService } from './build.service';
import { ScanService } from './scan.service';
import { KubernetesModule } from '../kubernetes/kubernetes.module';
import { ClustersModule } from '../clusters/clusters.module';
@@ -9,7 +8,7 @@ import { ClustersModule } from '../clusters/clusters.module';
forwardRef(() => KubernetesModule),
ClustersModule,
],
providers: [BuildService, ScanService],
exports: [BuildService, ScanService],
providers: [BuildService],
exports: [BuildService],
})
export class BuildModule {}
+290 -80
View File
@@ -1,16 +1,298 @@
import { AppRuntime } from '../common/enums';
/**
* Tests for build service:
* • Nixpacks build preparation (BYO Dockerfile vs generated) for code runtimes
* • WordPress templated Dockerfile + helper-pod / entrypoint / zip-structure logic
*
* NOTE: like the rest of this file, the Nixpacks tests reproduce the pure logic
* locally instead of importing BuildService — the service pulls in the ESM
* `@kubernetes/client-node`, which this project's Jest config does not transform.
* Keep these copies in sync with nixpacksPrepareInitContainer in build.service.ts.
* Tests for build service — Dockerfile generation for all runtimes
*/
// ─────────────────────────────────────────────────────────────────────────────
// Go Dockerfile tests
// ─────────────────────────────────────────────────────────────────────────────
describe('Go Dockerfile generation', () => {
function goDockerfile(app: { runtimeVersion?: string; port?: number }): string {
const goVersion = app.runtimeVersion || '1.22';
const port = app.port || 8080;
return `FROM golang:${goVersion}-alpine AS builder
WORKDIR /app
RUN apk add --no-cache git
COPY go.mod go.sum* ./
RUN go mod download || true
COPY . .
RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags="-w -s" -o main .
FROM alpine:3.19
WORKDIR /app
RUN apk --no-cache add ca-certificates tzdata
RUN addgroup -g 1001 -S appgroup && adduser -S appuser -u 1001 -G appgroup
COPY --from=builder /app/main .
RUN mkdir -p /app/data && chown -R appuser:appgroup /app
USER appuser
ENV PORT=${port}
EXPOSE ${port}
HEALTHCHECK --interval=30s --timeout=3s CMD wget --no-verbose --tries=1 --spider http://localhost:${port}/health || exit 1
CMD ["./main"]
`;
}
it('should use correct Go version', () => {
const df = goDockerfile({ runtimeVersion: '1.21' });
expect(df).toContain('FROM golang:1.21-alpine');
});
it('should default to Go 1.22', () => {
const df = goDockerfile({});
expect(df).toContain('FROM golang:1.22-alpine');
});
it('should build static binary with CGO_ENABLED=0', () => {
const df = goDockerfile({});
expect(df).toContain('CGO_ENABLED=0');
});
it('should use multi-stage build for smaller image', () => {
const df = goDockerfile({});
expect(df).toContain('AS builder');
expect(df).toContain('FROM alpine:3.19');
});
it('should include health check', () => {
const df = goDockerfile({ port: 8080 });
expect(df).toContain('HEALTHCHECK');
expect(df).toContain('http://localhost:8080/health');
});
it('should create data directory for persistent storage', () => {
const df = goDockerfile({});
expect(df).toContain('mkdir -p /app/data');
});
it('should run as non-root user', () => {
const df = goDockerfile({});
expect(df).toContain('USER appuser');
});
});
// ─────────────────────────────────────────────────────────────────────────────
// Python Dockerfile tests
// ─────────────────────────────────────────────────────────────────────────────
describe('Python Dockerfile generation', () => {
function pythonDockerfile(app: { runtimeVersion?: string; port?: number }): string {
const pythonVersion = app.runtimeVersion || '3.12';
const port = app.port || 8000;
return `FROM python:${pythonVersion}-slim AS builder
WORKDIR /app
RUN apt-get update && apt-get install -y build-essential libpq-dev
COPY requirements.txt* ./
RUN pip install --no-cache-dir --user -r requirements.txt 2>/dev/null || pip install --no-cache-dir --user flask gunicorn
FROM python:${pythonVersion}-slim
WORKDIR /app
RUN groupadd -g 1001 appgroup && useradd -r -u 1001 -g appgroup appuser
COPY --from=builder /root/.local /home/appuser/.local
COPY . .
RUN mkdir -p /app/data && chown -R appuser:appgroup /app
USER appuser
ENV PATH=/home/appuser/.local/bin:$PATH
ENV PORT=${port}
EXPOSE ${port}
HEALTHCHECK --interval=30s --timeout=3s CMD curl -f http://localhost:${port}/health || exit 1
CMD ["gunicorn", "-w", "4", "-b", "0.0.0.0:${port}", "app:app"]
`;
}
it('should use correct Python version', () => {
const df = pythonDockerfile({ runtimeVersion: '3.11' });
expect(df).toContain('FROM python:3.11-slim');
});
it('should default to Python 3.12', () => {
const df = pythonDockerfile({});
expect(df).toContain('FROM python:3.12-slim');
});
it('should use multi-stage build', () => {
const df = pythonDockerfile({});
expect(df).toContain('AS builder');
});
it('should install from requirements.txt', () => {
const df = pythonDockerfile({});
expect(df).toContain('requirements.txt');
});
it('should include health check', () => {
const df = pythonDockerfile({ port: 8000 });
expect(df).toContain('HEALTHCHECK');
expect(df).toContain('http://localhost:8000/health');
});
it('should run as non-root user', () => {
const df = pythonDockerfile({});
expect(df).toContain('USER appuser');
});
});
// ─────────────────────────────────────────────────────────────────────────────
// Django Dockerfile tests
// ─────────────────────────────────────────────────────────────────────────────
describe('Django Dockerfile generation', () => {
function djangoDockerfile(app: { runtimeVersion?: string; port?: number }): string {
const pythonVersion = app.runtimeVersion || '3.12';
const port = app.port || 8000;
return `FROM python:${pythonVersion}-slim AS builder
WORKDIR /app
RUN apt-get update && apt-get install -y build-essential libpq-dev
COPY requirements.txt* ./
RUN pip install --no-cache-dir --user -r requirements.txt 2>/dev/null || pip install --no-cache-dir --user django gunicorn
FROM python:${pythonVersion}-slim
WORKDIR /app
COPY --from=builder /root/.local /home/appuser/.local
COPY . .
RUN mkdir -p /app/staticfiles /app/media /app/data
USER appuser
ENV PORT=${port}
ENV DJANGO_SETTINGS_MODULE=config.settings
EXPOSE ${port}
HEALTHCHECK --interval=30s --timeout=3s CMD curl -f http://localhost:${port}/health/ || exit 1
CMD ["sh", "-c", "python manage.py migrate --noinput && gunicorn config.wsgi:application --bind 0.0.0.0:${port}"]
`;
}
it('should use correct Python version', () => {
const df = djangoDockerfile({ runtimeVersion: '3.10' });
expect(df).toContain('FROM python:3.10-slim');
});
it('should set DJANGO_SETTINGS_MODULE', () => {
const df = djangoDockerfile({});
expect(df).toContain('DJANGO_SETTINGS_MODULE');
});
it('should create staticfiles and media directories', () => {
const df = djangoDockerfile({});
expect(df).toContain('/app/staticfiles');
expect(df).toContain('/app/media');
});
it('should run migrations on startup', () => {
const df = djangoDockerfile({});
expect(df).toContain('migrate');
});
it('should use gunicorn for production', () => {
const df = djangoDockerfile({});
expect(df).toContain('gunicorn');
});
});
// ─────────────────────────────────────────────────────────────────────────────
// .NET Dockerfile tests
// ─────────────────────────────────────────────────────────────────────────────
describe('.NET Dockerfile generation', () => {
function dotnetDockerfile(app: { runtimeVersion?: string; port?: number }): string {
const dotnetVersion = app.runtimeVersion || '8.0';
const port = app.port || 5000;
return `FROM mcr.microsoft.com/dotnet/sdk:${dotnetVersion} AS build
WORKDIR /src
COPY *.csproj ./
RUN dotnet restore || true
COPY . .
RUN dotnet publish -c Release -o /app/publish
FROM mcr.microsoft.com/dotnet/aspnet:${dotnetVersion}
WORKDIR /app
COPY --from=build /app/publish .
RUN mkdir -p /app/data
USER appuser
ENV ASPNETCORE_URLS=http://+:${port}
ENV ASPNETCORE_ENVIRONMENT=Production
EXPOSE ${port}
HEALTHCHECK --interval=30s --timeout=3s CMD curl -f http://localhost:${port}/health || exit 1
CMD ["dotnet", "app.dll"]
`;
}
it('should use correct .NET version', () => {
const df = dotnetDockerfile({ runtimeVersion: '7.0' });
expect(df).toContain('dotnet/sdk:7.0');
expect(df).toContain('dotnet/aspnet:7.0');
});
it('should default to .NET 8.0', () => {
const df = dotnetDockerfile({});
expect(df).toContain('dotnet/sdk:8.0');
});
it('should use multi-stage build', () => {
const df = dotnetDockerfile({});
expect(df).toContain('AS build');
expect(df).toContain('dotnet/aspnet');
});
it('should publish in Release mode', () => {
const df = dotnetDockerfile({});
expect(df).toContain('-c Release');
});
it('should set ASPNETCORE_ENVIRONMENT to Production', () => {
const df = dotnetDockerfile({});
expect(df).toContain('ASPNETCORE_ENVIRONMENT=Production');
});
it('should configure ASPNETCORE_URLS for correct port', () => {
const df = dotnetDockerfile({ port: 8080 });
expect(df).toContain('ASPNETCORE_URLS=http://+:8080');
});
});
// ─────────────────────────────────────────────────────────────────────────────
// PHP Dockerfile tests
// ─────────────────────────────────────────────────────────────────────────────
describe('PHP Dockerfile generation', () => {
function phpDockerfile(app: { phpVersion?: string; port?: number }): string {
const phpVersion = app.phpVersion || '8.3';
const port = app.port || 80;
return `FROM php:${phpVersion}-fpm-alpine
RUN apk add --no-cache nginx supervisor curl
RUN docker-php-ext-install pdo pdo_mysql opcache
WORKDIR /var/www/html
COPY . .
RUN mkdir -p /var/www/html/uploads /var/www/html/data
RUN chown -R www-data:www-data /var/www/html
EXPOSE ${port}
CMD ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf"]
`;
}
it('should use correct PHP version', () => {
const df = phpDockerfile({ phpVersion: '8.2' });
expect(df).toContain('FROM php:8.2-fpm-alpine');
});
it('should default to PHP 8.3', () => {
const df = phpDockerfile({});
expect(df).toContain('FROM php:8.3-fpm-alpine');
});
it('should use FPM with nginx via supervisord', () => {
const df = phpDockerfile({});
expect(df).toContain('supervisord');
expect(df).toContain('nginx');
});
it('should install common PHP extensions', () => {
const df = phpDockerfile({});
expect(df).toContain('pdo');
expect(df).toContain('opcache');
});
it('should create upload and data directories', () => {
const df = phpDockerfile({});
expect(df).toContain('/var/www/html/uploads');
expect(df).toContain('/var/www/html/data');
});
});
/**
* Tests for the WordPress build flow — specifically:
* 1. Helper pod PVC race condition (must wait for termination)
@@ -185,75 +467,3 @@ describe('WordPress zip structure handling', () => {
expect(copiedAsIs).toBe(true);
});
});
describe('Nixpacks build preparation', () => {
// Local copies of the pure logic in build.service.ts (see NOTE at top of file).
function shellQuote(value: string): string {
return `'${value.replace(/'/g, `'\\''`)}'`;
}
function nixpacksPlanEnv(app: { runtime: AppRuntime; runtimeVersion?: string }): { name: string; value: string }[] {
const env: { name: string; value: string }[] = [];
if (app.runtime === AppRuntime.NODEJS && app.runtimeVersion) {
env.push({ name: 'NIXPACKS_NODE_VERSION', value: String(app.runtimeVersion) });
}
if ((app.runtime === AppRuntime.PYTHON || app.runtime === AppRuntime.DJANGO) && app.runtimeVersion) {
env.push({ name: 'NIXPACKS_PYTHON_VERSION', value: String(app.runtimeVersion) });
}
return env;
}
function nixpacksPrepareInitContainer(
app: { runtime: AppRuntime; runtimeVersion?: string },
config: { nixpacksImage?: string; nixpacksBuildEnv?: string[] } = {},
): any {
const image = config.nixpacksImage || 'ghcr.io/railwayapp/nixpacks:latest';
const buildEnv = config.nixpacksBuildEnv || [];
const envFlags = buildEnv.map((kv) => `--env ${shellQuote(kv)}`).join(' ');
const planEnv = nixpacksPlanEnv(app);
return {
name: 'nixpacks-prepare',
image,
env: planEnv.length ? planEnv : undefined,
command: [
'sh',
'-c',
`if [ -f source/Dockerfile ]; then cp source/Dockerfile /workspace/Dockerfile; ` +
`else nixpacks build source --out source ${envFlags} && cp source/.nixpacks/Dockerfile /workspace/Dockerfile; fi`,
],
volumeMounts: [{ name: 'workspace', mountPath: '/workspace' }],
};
}
it('prefers a user-provided Dockerfile (BYO), falling back to Nixpacks', () => {
const script = nixpacksPrepareInitContainer({ runtime: AppRuntime.NODEJS }).command[2] as string;
expect(script).toContain('if [ -f source/Dockerfile ]');
expect(script).toContain('cp source/Dockerfile /workspace/Dockerfile');
expect(script).toContain('nixpacks build source --out source');
expect(script).toContain('cp source/.nixpacks/Dockerfile /workspace/Dockerfile');
});
it('uses the configured Nixpacks image (default when unset)', () => {
expect(nixpacksPrepareInitContainer({ runtime: AppRuntime.GO }).image).toBe('ghcr.io/railwayapp/nixpacks:latest');
expect(
nixpacksPrepareInitContainer({ runtime: AppRuntime.GO }, { nixpacksImage: 'registry.local/nixpacks:1.2.3' }).image,
).toBe('registry.local/nixpacks:1.2.3');
});
it('bakes build-time mirror env into the build via --env flags', () => {
const script = nixpacksPrepareInitContainer(
{ runtime: AppRuntime.NODEJS },
{ nixpacksBuildEnv: ['NPM_CONFIG_REGISTRY=https://registry.npmmirror.com'] },
).command[2] as string;
expect(script).toContain(`--env 'NPM_CONFIG_REGISTRY=https://registry.npmmirror.com'`);
});
it('maps the selected Node version to NIXPACKS_NODE_VERSION', () => {
const c = nixpacksPrepareInitContainer({ runtime: AppRuntime.NODEJS, runtimeVersion: '20' });
expect(c.env).toContainEqual({ name: 'NIXPACKS_NODE_VERSION', value: '20' });
});
it('shellQuote escapes embedded single quotes safely', () => {
expect(shellQuote("a'b")).toBe("'a'\\''b'");
});
});
File diff suppressed because it is too large Load Diff
-94
View File
@@ -1,94 +0,0 @@
/**
* Tests for the pure Trivy-report aggregation logic in ScanService.summarize.
*
* NOTE: like the other build specs, this reproduces the pure logic locally rather
* than importing ScanService — the service pulls in the ESM `@kubernetes/client-node`,
* which this project's Jest config does not transform. Keep in sync with scan.service.ts.
*/
interface VulnerabilitySummary {
critical: number;
high: number;
medium: number;
low: number;
unknown: number;
total: number;
scannedAt: string;
}
function parseTrivyJson(output: string): any | null {
if (!output) return null;
try {
return JSON.parse(output);
} catch {
const start = output.indexOf('{');
const end = output.lastIndexOf('}');
if (start >= 0 && end > start) {
try {
return JSON.parse(output.slice(start, end + 1));
} catch {
return null;
}
}
return null;
}
}
function summarize(trivyOutput: string): VulnerabilitySummary {
const counts = { critical: 0, high: 0, medium: 0, low: 0, unknown: 0 };
const parsed = parseTrivyJson(trivyOutput);
const results: any[] = Array.isArray(parsed?.Results) ? parsed.Results : [];
for (const result of results) {
const vulns: any[] = Array.isArray(result?.Vulnerabilities) ? result.Vulnerabilities : [];
for (const v of vulns) {
const sev = String(v?.Severity || 'UNKNOWN').toUpperCase();
if (sev === 'CRITICAL') counts.critical++;
else if (sev === 'HIGH') counts.high++;
else if (sev === 'MEDIUM') counts.medium++;
else if (sev === 'LOW') counts.low++;
else counts.unknown++;
}
}
return {
...counts,
total: counts.critical + counts.high + counts.medium + counts.low + counts.unknown,
scannedAt: new Date().toISOString(),
};
}
describe('ScanService.summarize', () => {
it('counts vulnerabilities per severity across results', () => {
const report = JSON.stringify({
Results: [
{ Vulnerabilities: [{ Severity: 'CRITICAL' }, { Severity: 'HIGH' }, { Severity: 'high' }] },
{ Vulnerabilities: [{ Severity: 'MEDIUM' }, { Severity: 'LOW' }, { Severity: 'WeIrD' }] },
{ Vulnerabilities: null },
{},
],
});
const s = summarize(report);
expect(s).toMatchObject({ critical: 1, high: 2, medium: 1, low: 1, unknown: 1, total: 6 });
expect(typeof s.scannedAt).toBe('string');
});
it('returns all-zero summary for a clean image', () => {
expect(summarize(JSON.stringify({ Results: [{ Target: 'x' }] }))).toMatchObject({
critical: 0,
high: 0,
medium: 0,
low: 0,
unknown: 0,
total: 0,
});
});
it('tolerates leading log noise before the JSON', () => {
const noisy = `2026-06-20 INFO Need to update DB\n{"Results":[{"Vulnerabilities":[{"Severity":"CRITICAL"}]}]}`;
expect(summarize(noisy).critical).toBe(1);
});
it('returns a zero summary on unparseable output', () => {
expect(summarize('not json at all').total).toBe(0);
expect(summarize('').total).toBe(0);
});
});
-169
View File
@@ -1,169 +0,0 @@
import { Injectable, Logger } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import * as k8s from '@kubernetes/client-node';
import { Application } from '../applications/entities/application.entity';
import { ClustersService } from '../clusters/clusters.service';
import { RegistryService } from '../kubernetes/registry.service';
export interface VulnerabilitySummary {
critical: number;
high: number;
medium: number;
low: number;
unknown: number;
total: number;
scannedAt: string;
}
/**
* Report-only image vulnerability scanning with Trivy. Runs a one-shot K8s Job
* that scans the freshly-pushed image in the in-cluster registry and stores a
* severity summary on the deployment. Never blocks a deployment — any failure is
* logged and ignored.
*/
@Injectable()
export class ScanService {
private readonly logger = new Logger(ScanService.name);
constructor(
private readonly configService: ConfigService,
private readonly clustersService: ClustersService,
private readonly registryService: RegistryService,
) {}
/** Aggregate a Trivy JSON report (raw stdout) into per-severity counts. Pure & testable. */
summarize(trivyOutput: string): VulnerabilitySummary {
const counts = { critical: 0, high: 0, medium: 0, low: 0, unknown: 0 };
const parsed = this.parseTrivyJson(trivyOutput);
const results: any[] = Array.isArray(parsed?.Results) ? parsed.Results : [];
for (const result of results) {
const vulns: any[] = Array.isArray(result?.Vulnerabilities) ? result.Vulnerabilities : [];
for (const v of vulns) {
const sev = String(v?.Severity || 'UNKNOWN').toUpperCase();
if (sev === 'CRITICAL') counts.critical++;
else if (sev === 'HIGH') counts.high++;
else if (sev === 'MEDIUM') counts.medium++;
else if (sev === 'LOW') counts.low++;
else counts.unknown++;
}
}
return {
...counts,
total: counts.critical + counts.high + counts.medium + counts.low + counts.unknown,
scannedAt: new Date().toISOString(),
};
}
/** Trivy prints clean JSON to stdout, but tolerate any leading noise from the log stream. */
private parseTrivyJson(output: string): any | null {
if (!output) return null;
try {
return JSON.parse(output);
} catch {
const start = output.indexOf('{');
const end = output.lastIndexOf('}');
if (start >= 0 && end > start) {
try {
return JSON.parse(output.slice(start, end + 1));
} catch {
return null;
}
}
return null;
}
}
/**
* Scan a pushed image and return a severity summary, or null on any failure.
* Report-only: callers must treat null as "no data", never as a deploy gate.
*/
async scanImage(app: Application, imageUri: string): Promise<VulnerabilitySummary | null> {
if (this.configService.get<boolean>('build.scanEnabled') === false) return null;
const buildNs = this.registryService.getBuildNamespace();
const image = this.configService.get<string>('build.trivyImage') || 'aquasec/trivy:latest';
const dbRepo = this.configService.get<string>('build.trivyDbRepository') || '';
const timeoutSeconds = this.configService.get<number>('build.scanTimeoutSeconds') || 300;
const { username, password } = this.registryService.getRegistryCredentials();
const jobName = `scan-${app.name}-${Date.now()}`.substring(0, 63).replace(/[^a-z0-9-]/g, '');
try {
const cluster = app.clusterId ? await this.clustersService.findOne(app.clusterId) : await this.clustersService.getDefault();
const kc = new k8s.KubeConfig();
kc.loadFromString(cluster.kubeconfig);
const coreApi = kc.makeApiClient(k8s.CoreV1Api);
const batchApi = kc.makeApiClient(k8s.BatchV1Api);
const env: { name: string; value: string }[] = [
{ name: 'TRIVY_INSECURE', value: 'true' }, // in-cluster registry is plain HTTP
{ name: 'TRIVY_NON_SSL', value: 'true' },
];
if (username) env.push({ name: 'TRIVY_USERNAME', value: username });
if (password) env.push({ name: 'TRIVY_PASSWORD', value: password });
if (dbRepo) env.push({ name: 'TRIVY_DB_REPOSITORY', value: dbRepo });
const job: k8s.V1Job = {
apiVersion: 'batch/v1',
kind: 'Job',
metadata: { name: jobName, namespace: buildNs },
spec: {
backoffLimit: 0,
ttlSecondsAfterFinished: 120,
template: {
spec: {
restartPolicy: 'Never',
containers: [
{
name: 'trivy',
image,
imagePullPolicy: 'IfNotPresent',
env,
args: ['image', '--quiet', '--no-progress', '--format', 'json', '--severity', 'CRITICAL,HIGH,MEDIUM,LOW', imageUri],
resources: {
requests: { cpu: '250m', memory: '512Mi' },
limits: { cpu: '1', memory: '1Gi' },
},
},
],
},
},
},
};
await batchApi.createNamespacedJob({ namespace: buildNs, body: job });
await this.waitForJob(batchApi, jobName, buildNs, timeoutSeconds);
const output = await this.getJobPodLogs(coreApi, jobName, buildNs);
const summary = this.summarize(output);
await batchApi
.deleteNamespacedJob({ name: jobName, namespace: buildNs, gracePeriodSeconds: 0, propagationPolicy: 'Foreground' })
.catch(() => undefined);
this.logger.log(`Scan complete for ${imageUri}: ${summary.critical}C/${summary.high}H/${summary.medium}M/${summary.low}L`);
return summary;
} catch (e: any) {
this.logger.warn(`Image scan failed for ${imageUri} (report-only, ignored): ${e.message}`);
return null;
}
}
private async waitForJob(batchApi: k8s.BatchV1Api, jobName: string, namespace: string, timeoutSeconds: number): Promise<void> {
const deadline = Date.now() + timeoutSeconds * 1000;
while (Date.now() < deadline) {
const job = await batchApi.readNamespacedJob({ name: jobName, namespace });
if (job.status?.succeeded) return;
if ((job.status?.failed ?? 0) > 0) return; // Trivy exits non-zero on findings with some flags; read logs anyway
await new Promise((r) => setTimeout(r, 4000));
}
throw new Error(`Scan job ${jobName} timed out after ${timeoutSeconds}s`);
}
private async getJobPodLogs(coreApi: k8s.CoreV1Api, jobName: string, namespace: string): Promise<string> {
const pods = await coreApi.listNamespacedPod({ namespace, labelSelector: `job-name=${jobName}` });
const podName = pods.items[0]?.metadata?.name;
if (!podName) throw new Error(`No pod found for scan job ${jobName}`);
return coreApi.readNamespacedPodLog({ name: podName, namespace, container: 'trivy' });
}
}
-124
View File
@@ -1102,133 +1102,9 @@ export class ClustersService implements OnModuleInit, OnModuleDestroy {
await this.ensureK3sRegistryMirrors(appsApi, registryUrl);
// ── 8. MinIO (S3-compatible) for application source archives ──
await this.ensureMinioInfrastructure(coreApi, appsApi, buildNs);
this.logger.log(`✅ Cluster bootstrap complete — registry: ${registryUrl}`);
}
/**
* Provision in-cluster MinIO (Deployment + PVC + Service + credentials Secret)
* in the build namespace. Application source archives are uploaded here by the
* API and pulled by build pods via presigned URLs.
*/
private async ensureMinioInfrastructure(coreApi: k8s.CoreV1Api, appsApi: k8s.AppsV1Api, buildNs: string): Promise<void> {
const accessKey = this.configService.get<string>('minio.accessKey') || 'cloudhost';
const secretKey = this.configService.get<string>('minio.secretKey') || '';
const pvcName = 'minio-data';
const deployName = 'minio';
const svcName = 'minio';
const secretName = 'minio-credentials';
// 1. Credentials Secret (shared by the MinIO server env and the API client)
const minioSecret: k8s.V1Secret = {
metadata: { name: secretName, namespace: buildNs },
type: 'Opaque',
data: {
accesskey: Buffer.from(accessKey).toString('base64'),
secretkey: Buffer.from(secretKey).toString('base64'),
},
};
try {
await coreApi.readNamespacedSecret({ name: secretName, namespace: buildNs });
await coreApi.replaceNamespacedSecret({ name: secretName, namespace: buildNs, body: minioSecret });
} catch (err: any) {
if (err.code === 404 || err.body?.code === 404) {
await coreApi.createNamespacedSecret({ namespace: buildNs, body: minioSecret });
this.logger.log(`Created ${secretName} Secret`);
} else {
throw err;
}
}
// 2. Data PVC
try {
await coreApi.readNamespacedPersistentVolumeClaim({ name: pvcName, namespace: buildNs });
} catch (err: any) {
if (err.code === 404 || err.body?.code === 404) {
await coreApi.createNamespacedPersistentVolumeClaim({
namespace: buildNs,
body: {
metadata: { name: pvcName, namespace: buildNs },
spec: { accessModes: ['ReadWriteOnce'], resources: { requests: { storage: '20Gi' } } },
},
});
this.logger.log(`Created PVC "${pvcName}" (20Gi)`);
} else {
throw err;
}
}
// 3. Deployment
try {
await appsApi.readNamespacedDeployment({ name: deployName, namespace: buildNs });
} catch (err: any) {
if (err.code === 404 || err.body?.code === 404) {
await appsApi.createNamespacedDeployment({
namespace: buildNs,
body: {
metadata: { name: deployName, namespace: buildNs, labels: { app: 'minio' } },
spec: {
replicas: 1,
selector: { matchLabels: { app: 'minio' } },
template: {
metadata: { labels: { app: 'minio' } },
spec: {
containers: [
{
name: 'minio',
image: process.env.MINIO_IMAGE || 'minio/minio:latest',
args: ['server', '/data', '--console-address', ':9001'],
env: [
{ name: 'MINIO_ROOT_USER', valueFrom: { secretKeyRef: { name: secretName, key: 'accesskey' } } },
{ name: 'MINIO_ROOT_PASSWORD', valueFrom: { secretKeyRef: { name: secretName, key: 'secretkey' } } },
],
ports: [{ containerPort: 9000 }, { containerPort: 9001 }],
volumeMounts: [{ name: 'data', mountPath: '/data' }],
resources: {
requests: { cpu: '100m', memory: '256Mi' },
limits: { cpu: '1', memory: '1Gi' },
},
},
],
volumes: [{ name: 'data', persistentVolumeClaim: { claimName: pvcName } }],
},
},
},
},
});
this.logger.log(`Created MinIO Deployment`);
} else {
throw err;
}
}
// 4. ClusterIP Service (API :9000, console :9001)
try {
await coreApi.readNamespacedService({ name: svcName, namespace: buildNs });
} catch (err: any) {
if (err.code === 404 || err.body?.code === 404) {
await coreApi.createNamespacedService({
namespace: buildNs,
body: {
metadata: { name: svcName, namespace: buildNs, labels: { app: 'minio' } },
spec: {
selector: { app: 'minio' },
ports: [
{ name: 'api', port: 9000, targetPort: 9000 as any, protocol: 'TCP' },
{ name: 'console', port: 9001, targetPort: 9001 as any, protocol: 'TCP' },
],
},
},
});
this.logger.log(`Created MinIO Service`);
} else {
throw err;
}
}
}
/** In-cluster registry mirror for k3s/containerd (HTTP). Removes legacy external-registry DaemonSet if present. */
private async ensureK3sRegistryMirrors(appsApi: k8s.AppsV1Api, registryUrl: string): Promise<void> {
const namespace = 'kube-system';
-51
View File
@@ -1,51 +0,0 @@
import { Global, Module, OnApplicationShutdown, Logger } from '@nestjs/common';
import { ModuleRef } from '@nestjs/core';
import { ConfigService } from '@nestjs/config';
import IORedis, { Redis } from 'ioredis';
/** Injection token for the shared ioredis client. */
export const REDIS_CLIENT = 'REDIS_CLIENT';
/**
* Shared Redis client used for build state (progress / cancel flags / build
* sessions) so the data survives across backend replicas — unlike the previous
* in-memory Maps that only worked with a single instance.
*
* Reuses the same connection details as the Bull queue (`redis.host/port`).
*/
@Global()
@Module({
providers: [
{
provide: REDIS_CLIENT,
inject: [ConfigService],
useFactory: (configService: ConfigService): Redis => {
const client = new IORedis({
host: configService.get<string>('redis.host'),
port: configService.get<number>('redis.port'),
// Build state writes are best-effort telemetry — never let a Redis
// hiccup take down the request that triggered them.
maxRetriesPerRequest: 2,
enableOfflineQueue: true,
});
client.on('error', (err) => {
new Logger('RedisClient').warn(`Redis connection error: ${err.message}`);
});
return client;
},
},
],
exports: [REDIS_CLIENT],
})
export class RedisModule implements OnApplicationShutdown {
constructor(private readonly moduleRef: ModuleRef) {}
async onApplicationShutdown(): Promise<void> {
try {
const client = this.moduleRef.get<Redis>(REDIS_CLIENT, { strict: false });
await client?.quit();
} catch {
/* ignore shutdown errors */
}
}
}
@@ -1,13 +0,0 @@
import { Global, Module } from '@nestjs/common';
import { StorageService } from './storage.service';
/**
* Global module exposing the MinIO-backed {@link StorageService} so both the API
* (source upload) and the build pipeline (presigned download) can inject it.
*/
@Global()
@Module({
providers: [StorageService],
exports: [StorageService],
})
export class StorageModule {}
@@ -1,64 +0,0 @@
import { Injectable, Logger } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import * as Minio from 'minio';
/**
* Object storage for application source archives, backed by the in-cluster MinIO
* (S3-compatible). Replaces the previous local-disk + PVC + kubectl-cp upload path:
* the API streams the uploaded zip straight to MinIO, and build pods pull it via a
* short-lived presigned URL (no credentials, no kubectl, no helper pod).
*/
@Injectable()
export class StorageService {
private readonly logger = new Logger(StorageService.name);
private readonly client: Minio.Client;
private readonly bucket: string;
private bucketReady = false;
constructor(private readonly configService: ConfigService) {
this.bucket = this.configService.get<string>('minio.bucket') || 'app-sources';
this.client = new Minio.Client({
endPoint: this.configService.get<string>('minio.endpoint') || 'minio.cloudhost-builds.svc.cluster.local',
port: this.configService.get<number>('minio.port') || 9000,
useSSL: this.configService.get<boolean>('minio.useSSL') || false,
accessKey: this.configService.get<string>('minio.accessKey') || 'cloudhost',
secretKey: this.configService.get<string>('minio.secretKey') || '',
});
}
/** Object key for an app's source archive. */
sourceKey(userId: string, appId: string): string {
return `${userId}/${appId}/source.zip`;
}
private async ensureBucket(): Promise<void> {
if (this.bucketReady) return;
const exists = await this.client.bucketExists(this.bucket).catch(() => false);
if (!exists) {
await this.client.makeBucket(this.bucket);
this.logger.log(`Created MinIO bucket "${this.bucket}"`);
}
this.bucketReady = true;
}
/** Upload an app's source archive; returns the stored object key (saved as app.codePath). */
async putSource(userId: string, appId: string, data: Buffer): Promise<string> {
await this.ensureBucket();
const key = this.sourceKey(userId, appId);
await this.client.putObject(this.bucket, key, data, data.length, { 'Content-Type': 'application/zip' });
this.logger.log(`Uploaded source ${(data.length / 1024 / 1024).toFixed(1)}MB → ${this.bucket}/${key}`);
return key;
}
/** Short-lived presigned GET URL the build pod uses to download the source. */
async presignSourceGet(key: string, expirySeconds = 3600): Promise<string> {
return this.client.presignedGetObject(this.bucket, key, expirySeconds);
}
/** Best-effort delete of an app's source object (e.g. on app deletion). */
async removeSource(key: string): Promise<void> {
await this.client.removeObject(this.bucket, key).catch((e) => {
this.logger.warn(`Failed to remove source ${key}: ${e.message}`);
});
}
}
-36
View File
@@ -125,45 +125,9 @@ export default () => ({
password: process.env.REGISTRY_PASSWORD || '',
},
// In-cluster MinIO (S3-compatible) for application source archives.
minio: {
endpoint: process.env.MINIO_ENDPOINT || 'minio.cloudhost-builds.svc.cluster.local',
port: parseInt(process.env.MINIO_PORT || '9000', 10),
useSSL: process.env.MINIO_USE_SSL === 'true',
accessKey: process.env.MINIO_ACCESS_KEY || 'cloudhost',
secretKey: process.env.MINIO_SECRET_KEY || 'CloudHost2024!Minio',
bucket: process.env.MINIO_BUCKET || 'app-sources',
},
build: {
namespace: process.env.BUILD_NAMESPACE || 'cloudhost-builds',
serviceAccount: process.env.BUILD_SERVICE_ACCOUNT || 'kaniko-builder',
/** Max number of build+deploy pipelines processed concurrently across the queue. */
concurrency: parseInt(process.env.BUILD_CONCURRENCY || '3', 10),
/** Per-image-build timeout (Kaniko job) in seconds. */
timeoutSeconds: parseInt(process.env.BUILD_TIMEOUT_SECONDS || '600', 10),
/** Nixpacks builder image used to generate a Dockerfile for code runtimes. */
nixpacksImage: process.env.NIXPACKS_IMAGE || 'ghcr.io/railwayapp/nixpacks:latest',
/**
* Build-time env baked into Nixpacks-generated images (mirrors/proxies for the
* Iran network, e.g. "NPM_CONFIG_REGISTRY=https://registry.npmmirror.com").
* Comma-separated KEY=VALUE pairs — set per the Phase 0 spike findings.
*/
nixpacksBuildEnv: (process.env.NIXPACKS_BUILD_ENV || '')
.split(',')
.map((s) => s.trim())
.filter(Boolean),
/** Report-only Trivy image scan after a successful build. */
scanEnabled: process.env.BUILD_SCAN_ENABLED !== 'false',
trivyImage: process.env.TRIVY_IMAGE || 'aquasec/trivy:latest',
/** Optional mirror for Trivy's vulnerability DB (Iran network); empty = default ghcr.io. */
trivyDbRepository: process.env.TRIVY_DB_REPOSITORY || '',
/** Max seconds to wait for the Trivy scan job. */
scanTimeoutSeconds: parseInt(process.env.BUILD_SCAN_TIMEOUT_SECONDS || '300', 10),
/** Registry garbage collection: keep the N most recent image tags per app repo. */
registryGcEnabled: process.env.REGISTRY_GC_ENABLED !== 'false',
registryKeepVersions: parseInt(process.env.REGISTRY_KEEP_VERSIONS || '3', 10),
registryGcIntervalMs: parseInt(process.env.REGISTRY_GC_INTERVAL_MS || '86400000', 10), // daily
},
elasticsearch: {
@@ -1,2 +0,0 @@
/** Bull queue name for build+deploy pipelines. */
export const DEPLOY_QUEUE = 'app-deploy';
@@ -1,27 +0,0 @@
import { Process, Processor } from '@nestjs/bull';
import { Logger } from '@nestjs/common';
import { Job } from 'bull';
import { DeploymentsService, DeploymentJobData } from './deployments.service';
import { DEPLOY_QUEUE } from './deployment.constants';
/**
* Processes build+deploy pipelines off the `app-deploy` queue with a bounded
* concurrency (BUILD_CONCURRENCY, default 3) so simultaneous user deploys can't
* flood the cluster with Kaniko build jobs (2 CPU / 4Gi each).
*
* Concurrency is read at module-load time from env because Bull's `@Process`
* decorator option must be a constant.
*/
@Processor(DEPLOY_QUEUE)
export class DeploymentProcessor {
private readonly logger = new Logger(DeploymentProcessor.name);
constructor(private readonly deploymentsService: DeploymentsService) {}
@Process({ name: 'run', concurrency: parseInt(process.env.BUILD_CONCURRENCY || '3', 10) })
async handleRun(job: Job<DeploymentJobData>): Promise<void> {
const { deploymentId } = job.data;
this.logger.log(`Processing deployment ${deploymentId} (job ${job.id})`);
await this.deploymentsService.processDeploymentJob(job.data);
}
}
@@ -1,10 +1,7 @@
import { Module, forwardRef } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { BullModule } from '@nestjs/bull';
import { DeploymentsService } from './deployments.service';
import { DeploymentsController } from './deployments.controller';
import { DeploymentProcessor } from './deployment.processor';
import { DEPLOY_QUEUE } from './deployment.constants';
import { Deployment } from './entities/deployment.entity';
import { ApplicationsModule } from '../applications/applications.module';
import { KubernetesModule } from '../kubernetes/kubernetes.module';
@@ -14,14 +11,13 @@ import { ClustersModule } from '../clusters/clusters.module';
@Module({
imports: [
TypeOrmModule.forFeature([Deployment]),
BullModule.registerQueue({ name: DEPLOY_QUEUE }),
forwardRef(() => ApplicationsModule),
forwardRef(() => ClustersModule),
KubernetesModule,
BuildModule,
],
controllers: [DeploymentsController],
providers: [DeploymentsService, DeploymentProcessor],
providers: [DeploymentsService],
exports: [DeploymentsService],
})
export class DeploymentsModule {}
+16 -80
View File
@@ -1,15 +1,11 @@
import { Injectable, NotFoundException, BadRequestException, Logger, Inject, forwardRef, OnModuleInit } from '@nestjs/common';
import { Injectable, NotFoundException, BadRequestException, Logger, Inject, forwardRef } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { InjectQueue } from '@nestjs/bull';
import { Queue } from 'bull';
import { Repository } from 'typeorm';
import * as fs from 'fs';
import { DEPLOY_QUEUE } from './deployment.constants';
import { Deployment } from './entities/deployment.entity';
import { ApplicationsService } from '../applications/applications.service';
import { KubernetesService } from '../kubernetes/kubernetes.service';
import { BuildService, BuildProgress, BuildCancelledError } from '../build/build.service';
import { ScanService } from '../build/scan.service';
import * as crypto from 'crypto';
import {
AppLifecycleStatus,
@@ -19,15 +15,8 @@ import {
} from '../common/enums';
import { ClustersService } from '../clusters/clusters.service';
/** Payload enqueued on the `app-deploy` queue for the build+deploy pipeline. */
export interface DeploymentJobData {
deploymentId: string;
applicationId: string;
previewSubdomain: string | null;
}
@Injectable()
export class DeploymentsService implements OnModuleInit {
export class DeploymentsService {
private readonly logger = new Logger(DeploymentsService.name);
constructor(
@@ -38,41 +27,8 @@ export class DeploymentsService implements OnModuleInit {
private kubernetesService: KubernetesService,
private buildService: BuildService,
private clustersService: ClustersService,
private scanService: ScanService,
@InjectQueue(DEPLOY_QUEUE)
private deployQueue: Queue<DeploymentJobData>,
) {}
async onModuleInit(): Promise<void> {
try {
await this.deploymentsRepository.query(
`ALTER TABLE deployments ADD COLUMN IF NOT EXISTS "vulnerabilitySummary" jsonb`,
);
} catch (e: any) {
this.logger.warn(`Could not ensure deployments.vulnerabilitySummary column: ${e.message}`);
}
}
/**
* Queue worker entrypoint — runs one build+deploy pipeline. Bounded
* concurrency lives on the Bull processor, so this just dispatches to the
* managed (Helm-only) or app (build+deploy) pipeline. Both pipelines handle
* their own errors, so a failure here never triggers a Bull retry.
*/
async processDeploymentJob(data: DeploymentJobData): Promise<void> {
const { deploymentId, applicationId, previewSubdomain } = data;
if (await this.isDeploymentCancelled(deploymentId)) {
this.logger.log(`Deployment ${deploymentId} already cancelled before pickup — skipping`);
return;
}
const app = await this.applicationsService.findOne(applicationId);
if (isManagedProductType(app.productType)) {
await this.executeManagedPipeline(deploymentId, app);
} else {
await this.executePipeline(deploymentId, app, previewSubdomain);
}
}
/**
* Random 7-digit suffix for the preview host: <userId>-<7-digit>.<baseDomain>.
* Generated once per application (see resolvePreviewNumber) and persisted.
@@ -121,13 +77,13 @@ export class DeploymentsService implements OnModuleInit {
saved.previewSubdomain = previewSubdomain;
}
// Enqueue the build+deploy pipeline. The Bull processor runs it with bounded
// concurrency so concurrent user deploys can't flood the cluster.
await this.deployQueue.add(
'run',
{ deploymentId: saved.id, applicationId: app.id, previewSubdomain },
{ removeOnComplete: true, removeOnFail: true },
);
// Trigger async pipeline (Helm-only for managed services, build+deploy for apps)
const run = isManagedProductType(app.productType)
? this.executeManagedPipeline(saved.id, app)
: this.executePipeline(saved.id, app, previewSubdomain);
run.catch((error) => {
this.logger.error(`Pipeline failed for deployment ${saved.id}:`, error);
});
return saved;
}
@@ -231,19 +187,6 @@ export class DeploymentsService implements OnModuleInit {
// Save build log
await this.deploymentsRepository.update(deploymentId, { buildLog: buildResult.buildLog });
// Report-only vulnerability scan — runs alongside the deploy and is
// persisted when it finishes. Never blocks or fails the deployment.
void this.scanService
.scanImage(app, imageUri)
.then((summary) => {
if (summary) {
return this.deploymentsRepository.update(deploymentId, {
vulnerabilitySummary: summary as Record<string, any>,
});
}
})
.catch((e) => this.logger.warn(`Scan persistence failed for ${deploymentId}: ${e.message}`));
// Step 2: Update app with new image tag
await this.applicationsService.updateImageTag(app.id, imageUri);
@@ -543,10 +486,7 @@ export class DeploymentsService implements OnModuleInit {
return this.kubernetesService.getPodLogs(app);
}
async getBuildLogs(
applicationId: string,
userId: string,
): Promise<{ buildLog: string | null; status: string; version: string | null; createdAt: Date; vulnerabilitySummary: Record<string, any> | null }> {
async getBuildLogs(applicationId: string, userId: string): Promise<{ buildLog: string | null; status: string; version: string | null; createdAt: Date }> {
const app = await this.applicationsService.findOne(applicationId, userId);
const latest = await this.deploymentsRepository.findOne({
@@ -555,7 +495,7 @@ export class DeploymentsService implements OnModuleInit {
});
if (!latest) {
return { buildLog: null, status: 'no_deployment', version: null, createdAt: new Date(), vulnerabilitySummary: null };
return { buildLog: null, status: 'no_deployment', version: null, createdAt: new Date() };
}
if (this.isManagedOrHelmOnlyApp(app)) {
@@ -564,7 +504,6 @@ export class DeploymentsService implements OnModuleInit {
status: latest.status,
version: latest.version,
createdAt: latest.createdAt,
vulnerabilitySummary: null,
};
}
@@ -584,7 +523,6 @@ export class DeploymentsService implements OnModuleInit {
status: latest.status,
version: latest.version,
createdAt: latest.createdAt,
vulnerabilitySummary: latest.vulnerabilitySummary ?? null,
};
}
@@ -599,7 +537,7 @@ export class DeploymentsService implements OnModuleInit {
if (!latest) return null;
const progress = await this.buildService.getProgress(latest.id);
const progress = this.buildService.getProgress(latest.id);
if (progress) return progress;
// No in-memory progress — infer from deployment status
@@ -751,12 +689,10 @@ export class DeploymentsService implements OnModuleInit {
saved.previewSubdomain = previewSubdomain;
}
// Enqueue the build & deploy pipeline (same queue as initial deploy)
await this.deployQueue.add(
'run',
{ deploymentId: saved.id, applicationId: app.id, previewSubdomain },
{ removeOnComplete: true, removeOnFail: true },
);
// Trigger async build & deploy pipeline (same as initial deploy)
this.executePipeline(saved.id, app, previewSubdomain).catch((error) => {
this.logger.error(`Redeploy pipeline failed for deployment ${saved.id}:`, error);
});
this.logger.log(`Redeploy triggered for ${app.name} (${app.gitUrl ? 'git: ' + app.gitUrl : 'zip'})`);
return saved;
@@ -33,14 +33,6 @@ export class Deployment {
@Column({ type: 'text', nullable: true })
deployLog: string;
/**
* Report-only Trivy vulnerability summary for the built image
* (e.g. { critical, high, medium, low, unknown, total, scannedAt }).
* Non-blocking — never gates a deployment.
*/
@Column({ type: 'jsonb', nullable: true })
vulnerabilitySummary: Record<string, any> | null;
/**
* Per-deployment preview number (derived deterministically from deployment.id).
* Used to build preview ingress host under the main frontend domain.
+1 -2
View File
@@ -3,7 +3,6 @@ import { TypeOrmModule } from '@nestjs/typeorm';
import { KubernetesService } from './kubernetes.service';
import { HelmService } from './helm.service';
import { RegistryService } from './registry.service';
import { RegistryGcService } from './registry-gc.service';
import { ElasticsearchService } from './elasticsearch.service';
import { ElasticsearchController } from './elasticsearch.controller';
import { LogsController } from './logs.controller';
@@ -14,7 +13,7 @@ import { Deployment } from '../deployments/entities/deployment.entity';
@Module({
imports: [forwardRef(() => ClustersModule), TypeOrmModule.forFeature([Application, Deployment])],
controllers: [ElasticsearchController, LogsController],
providers: [KubernetesService, HelmService, RegistryService, RegistryGcService, ElasticsearchService],
providers: [KubernetesService, HelmService, RegistryService, ElasticsearchService],
exports: [KubernetesService, HelmService, RegistryService, ElasticsearchService],
})
export class KubernetesModule {}
@@ -1,25 +0,0 @@
import { selectTagsToDelete } from './registry-gc.util';
describe('selectTagsToDelete', () => {
it('keeps the N newest numeric (Date.now) tags and deletes the rest', () => {
const tags = ['1000', '3000', '2000', '5000', '4000'];
const toDelete = selectTagsToDelete(tags, 3);
// newest 3 = 5000,4000,3000 → delete 2000,1000
expect(toDelete.sort()).toEqual(['1000', '2000']);
});
it('deletes nothing when tag count is within the keep limit', () => {
expect(selectTagsToDelete(['1000', '2000'], 3)).toEqual([]);
expect(selectTagsToDelete([], 3)).toEqual([]);
});
it('treats non-numeric tags as oldest (eligible for deletion first)', () => {
const tags = ['latest', '2000', '1000'];
// numeric newest kept first: 2000,1000 kept (keep=2) → delete latest
expect(selectTagsToDelete(tags, 2)).toEqual(['latest']);
});
it('keep=0 deletes every tag', () => {
expect(selectTagsToDelete(['1000', '2000'], 0).sort()).toEqual(['1000', '2000']);
});
});
@@ -1,164 +0,0 @@
import { Injectable, Logger, OnModuleInit, OnModuleDestroy, Inject } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { Writable } from 'stream';
import * as k8s from '@kubernetes/client-node';
import { Redis } from 'ioredis';
import { REDIS_CLIENT } from '../common/redis/redis.module';
import { ClustersService } from '../clusters/clusters.service';
import { RegistryService } from './registry.service';
import { selectTagsToDelete } from './registry-gc.util';
/**
* Periodic registry garbage collection: prunes each app image repo to the N most
* recent tags (deleting older manifests) and then reclaims disk by running
* `registry garbage-collect` in the registry pod. Runs on the local/default
* cluster's in-cluster registry (the one reachable via cluster DNS). A Redis lock
* keeps a single replica running it at a time (see [[multi-instance-interval-jobs]]).
*/
@Injectable()
export class RegistryGcService implements OnModuleInit, OnModuleDestroy {
private readonly logger = new Logger(RegistryGcService.name);
private timer?: NodeJS.Timeout;
private static readonly LOCK_KEY = 'registry:gc:lock';
constructor(
private readonly configService: ConfigService,
private readonly clustersService: ClustersService,
private readonly registryService: RegistryService,
@Inject(REDIS_CLIENT) private readonly redis: Redis,
) {}
onModuleInit(): void {
if (this.configService.get<boolean>('build.registryGcEnabled') === false) return;
const interval = this.configService.get<number>('build.registryGcIntervalMs') || 86_400_000;
this.timer = setInterval(() => void this.runGc(), interval);
// First pass shortly after boot.
setTimeout(() => void this.runGc(), 60_000);
this.logger.log(`Registry GC scheduled — interval: ${Math.round(interval / 3600000)}h`);
}
onModuleDestroy(): void {
if (this.timer) clearInterval(this.timer);
}
/** Run one GC pass guarded by a Redis lock so only one replica executes it. */
async runGc(): Promise<void> {
const keep = this.configService.get<number>('build.registryKeepVersions') || 3;
const locked = await this.redis.set(RegistryGcService.LOCK_KEY, '1', 'EX', 900, 'NX').catch(() => null);
if (locked !== 'OK') {
this.logger.debug('Registry GC already running on another replica — skipping');
return;
}
try {
await this.pruneDefaultClusterRegistry(keep);
} catch (e: any) {
this.logger.warn(`Registry GC failed: ${e.message}`);
} finally {
await this.redis.del(RegistryGcService.LOCK_KEY).catch(() => undefined);
}
}
private authHeader(): string {
const { username, password } = this.registryService.getRegistryCredentials();
return 'Basic ' + Buffer.from(`${username}:${password}`).toString('base64');
}
private async registryFetch(pathOrUrl: string, init: RequestInit = {}): Promise<Response> {
const base = `http://${this.registryService.getRegistryUrl()}`;
return fetch(`${base}${pathOrUrl}`, {
...init,
headers: { Authorization: this.authHeader(), ...(init.headers || {}) },
});
}
private async pruneDefaultClusterRegistry(keep: number): Promise<void> {
// 1. List repositories
const catalogRes = await this.registryFetch('/v2/_catalog?n=10000');
if (!catalogRes.ok) throw new Error(`catalog ${catalogRes.status}`);
const repositories: string[] = ((await catalogRes.json()) as any)?.repositories || [];
let deletedTotal = 0;
for (const repo of repositories) {
// Skip Kaniko cache repos — pruning them just slows the next build.
if (repo.endsWith('/cache')) continue;
deletedTotal += await this.pruneRepo(repo, keep);
}
if (deletedTotal > 0) {
this.logger.log(`Registry GC: deleted ${deletedTotal} old manifest(s); reclaiming disk…`);
await this.runGarbageCollect();
} else {
this.logger.debug('Registry GC: nothing to prune');
}
}
/** Delete all but the newest `keep` tags of one repo. Returns count deleted. */
private async pruneRepo(repo: string, keep: number): Promise<number> {
const tagsRes = await this.registryFetch(`/v2/${repo}/tags/list`);
if (!tagsRes.ok) return 0;
const tags: string[] = ((await tagsRes.json()) as any)?.tags || [];
const toDelete = selectTagsToDelete(tags, keep);
if (toDelete.length === 0) return 0;
const seenDigests = new Set<string>();
let deleted = 0;
for (const tag of toDelete) {
try {
const head = await this.registryFetch(`/v2/${repo}/manifests/${tag}`, {
method: 'GET',
headers: {
Accept: 'application/vnd.docker.distribution.manifest.v2+json, application/vnd.oci.image.manifest.v1+json',
},
});
const digest = head.headers.get('docker-content-digest');
if (!digest || seenDigests.has(digest)) continue; // multiple tags can share a digest
seenDigests.add(digest);
const del = await this.registryFetch(`/v2/${repo}/manifests/${digest}`, { method: 'DELETE' });
if (del.ok || del.status === 202) deleted++;
} catch (e: any) {
this.logger.warn(`Failed to delete ${repo}:${tag}: ${e.message}`);
}
}
if (deleted > 0) this.logger.log(`Registry GC: pruned ${deleted} tag(s) from ${repo} (kept ${keep})`);
return deleted;
}
/** Reclaim disk by running `registry garbage-collect` inside the registry pod. Best-effort. */
private async runGarbageCollect(): Promise<void> {
try {
const buildNs = this.registryService.getBuildNamespace();
const cluster = await this.clustersService.getDefault();
const kc = new k8s.KubeConfig();
kc.loadFromString(cluster.kubeconfig);
const coreApi = kc.makeApiClient(k8s.CoreV1Api);
const pods = await coreApi.listNamespacedPod({ namespace: buildNs, labelSelector: 'app=registry' });
const podName = pods.items[0]?.metadata?.name;
if (!podName) {
this.logger.warn('Registry GC: no registry pod found for garbage-collect');
return;
}
const exec = new k8s.Exec(kc);
const sink = new Writable({ write: (_c, _e, cb) => cb() });
await new Promise<void>((resolve, reject) => {
exec
.exec(
buildNs,
podName,
'registry',
['/bin/registry', 'garbage-collect', '/etc/docker/registry/config.yml'],
sink,
sink,
null,
false,
(status) => (status.status === 'Failure' ? reject(new Error(status.message)) : resolve()),
)
.catch(reject);
});
this.logger.log('Registry GC: garbage-collect completed');
} catch (e: any) {
this.logger.warn(`Registry garbage-collect failed (manifests already deleted): ${e.message}`);
}
}
}
@@ -1,19 +0,0 @@
/**
* Decide which image tags to delete, keeping the `keep` most recent. Tags are
* `Date.now()` strings, so newest = highest numeric value; non-numeric tags sort
* last (treated as oldest) and are eligible for deletion once `keep` is met.
*
* Kept in its own (dependency-free) module so it can be unit-tested without
* pulling in the ESM `@kubernetes/client-node` that the GC service imports.
*/
export function selectTagsToDelete(tags: string[], keep: number): string[] {
const sorted = [...tags].sort((a, b) => {
const na = Number(a);
const nb = Number(b);
if (Number.isNaN(na) && Number.isNaN(nb)) return a < b ? 1 : -1;
if (Number.isNaN(na)) return 1;
if (Number.isNaN(nb)) return -1;
return nb - na; // newest first
});
return sorted.slice(Math.max(0, keep));
}