feat(build): revamp app build pipeline (queue, Nixpacks, MinIO, Trivy, registry GC)

Rework the application build/deploy pipeline for scalability, reproducibility, and security: - Build queue: deploys run through a bounded-concurrency Bull queue (BUILD_CONCURRENCY, default 3) so concurrent user deploys can't flood the cluster with Kaniko jobs. Build state (progress / cancel / session) moves from in-memory Maps to Redis, so cancel + live logs work across backend replicas. - Nixpacks + BYO Dockerfile: code runtimes build via Nixpacks (or the user's own Dockerfile when present); the hand-written per-runtime Dockerfile generators and runtime auto-detection are removed. WordPress keeps its templated path. Build-time mirror env (NIXPACKS_BUILD_ENV) supports the Iran network. - Source upload to MinIO: archives stream to in-cluster MinIO; build pods pull via a presigned URL. Removes the PVC + helper pod + kubectl cp upload path. - Report-only Trivy scan after build; per-severity summary stored on the deployment and shown as a badge in the dashboard. Never gates a deploy. - Registry GC: a Redis-locked daily job keeps the newest N image tags per app (REGISTRY_KEEP_VERSIONS, default 3) and reclaims disk via garbage-collect. - Hardening: git tokens are delivered via a per-build Secret + git credential store instead of being embedded in the clone URL / Job manifest; build timeout is configurable. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-20 22:58:58 +03:30
parent 49726f1dfd
commit 3eff38f8d2
27 changed files with 1950 additions and 1295 deletions
@@ -15,6 +15,8 @@ import { SnapshotsModule } from './snapshots/snapshots.module';
 import { LifecycleModule } from './lifecycle/lifecycle.module';
 import { ApplicationMigrationsModule } from './application-migrations/application-migrations.module';
 import { AdminModule } from './admin/admin.module';
+import { RedisModule } from './common/redis/redis.module';
+import { StorageModule } from './common/storage/storage.module';
 import configuration from './config/configuration';

@Module({
@@ -54,6 +56,12 @@ import configuration from './config/configuration';
      inject: [ConfigService],
    }),

+    // Shared Redis client (build state across replicas)
+    RedisModule,
+
+    // Shared MinIO storage (application source archives)
+    StorageModule,
+
    // Feature modules
    AuthModule,
    UsersModule,
@@ -17,6 +17,7 @@ import {
 } from '../common/enums';
 import { ensureAppUrlEnv } from './app-url.util';
 import { normalizeCreateApplicationDto } from './managed-service.util';
+import { StorageService } from '../common/storage/storage.service';

@Injectable()
 export class ApplicationsService {
@@ -27,6 +28,7 @@ export class ApplicationsService {
    private appsRepository: Repository<Application>,
    private clustersService: ClustersService,
    private configService: ConfigService,
+    private storageService: StorageService,
  ) {}

  private toDnsLabel(value: string): string {
@@ -204,18 +206,20 @@ export class ApplicationsService {
  async delete(id: string, userId: string): Promise<Application> {
    const app = await this.findOne(id, userId);

-    // Delete uploaded files
+    // Delete the uploaded source archive from object storage.
    if (app.codePath) {
-      try {
-        const uploadDir = this.configService.get<string>('platform.uploadDir') || './uploads';
-        const appDir = path.join(uploadDir, app.userId, app.id);
-        if (fs.existsSync(appDir)) {
-          fs.rmSync(appDir, { recursive: true, force: true });
-          this.logger.log(`Deleted upload directory: ${appDir}`);
-        }
-      } catch (e: any) {
-        this.logger.warn(`Failed to delete upload dir for ${app.name}: ${e.message}`);
+      await this.storageService.removeSource(app.codePath);
+    }
+    // Remove any legacy on-disk dump/source dir (db dumps are still stored locally).
+    try {
+      const uploadDir = this.configService.get<string>('platform.uploadDir') || './uploads';
+      const appDir = path.join(uploadDir, app.userId, app.id);
+      if (fs.existsSync(appDir)) {
+        fs.rmSync(appDir, { recursive: true, force: true });
+        this.logger.log(`Deleted upload directory: ${appDir}`);
      }
+    } catch (e: any) {
+      this.logger.warn(`Failed to delete upload dir for ${app.name}: ${e.message}`);
    }

    await this.appsRepository.remove(app);
@@ -261,21 +265,14 @@ export class ApplicationsService {
    }

    const app = await this.findOne(id, userId);
-    const uploadDir = this.configService.get<string>('platform.uploadDir') || './uploads';
-    const appDir = path.join(uploadDir, app.userId, app.id);

-    // Ensure directory exists
-    fs.mkdirSync(appDir, { recursive: true });
-
-    // Save the zip file
-    const zipPath = path.join(appDir, 'source.zip');
-    fs.writeFileSync(zipPath, file.buffer);
-
-    // Update app with code path
-    app.codePath = zipPath;
+    // Stream the archive to MinIO; codePath stores the object key (build pods
+    // pull it via a presigned URL — no local disk, no PVC, no kubectl cp).
+    const key = await this.storageService.putSource(app.userId, app.id, file.buffer);
+    app.codePath = key;
    const saved = await this.appsRepository.save(app);

-    this.logger.log(`Uploaded code for ${app.name} → ${zipPath} (${(file.size / 1024).toFixed(1)} KB)`);
+    this.logger.log(`Uploaded code for ${app.name} → ${key} (${(file.size / 1024).toFixed(1)} KB)`);
    return saved;
  }

@@ -1,5 +1,6 @@
 import { Module, forwardRef } from '@nestjs/common';
 import { BuildService } from './build.service';
+import { ScanService } from './scan.service';
 import { KubernetesModule } from '../kubernetes/kubernetes.module';
 import { ClustersModule } from '../clusters/clusters.module';

@@ -8,7 +9,7 @@ import { ClustersModule } from '../clusters/clusters.module';
    forwardRef(() => KubernetesModule),
    ClustersModule,
  ],
-  providers: [BuildService],
-  exports: [BuildService],
+  providers: [BuildService, ScanService],
+  exports: [BuildService, ScanService],
 })
 export class BuildModule {}
@@ -1,298 +1,16 @@
 import { AppRuntime } from '../common/enums';

 /**
- * Tests for build service — Dockerfile generation for all runtimes
+ * Tests for build service:
+ *  • Nixpacks build preparation (BYO Dockerfile vs generated) for code runtimes
+ *  • WordPress templated Dockerfile + helper-pod / entrypoint / zip-structure logic
+ *
+ * NOTE: like the rest of this file, the Nixpacks tests reproduce the pure logic
+ * locally instead of importing BuildService — the service pulls in the ESM
+ * `@kubernetes/client-node`, which this project's Jest config does not transform.
+ * Keep these copies in sync with nixpacksPrepareInitContainer in build.service.ts.
 */

-// ─────────────────────────────────────────────────────────────────────────────
-// Go Dockerfile tests
-// ─────────────────────────────────────────────────────────────────────────────
-describe('Go Dockerfile generation', () => {
-  function goDockerfile(app: { runtimeVersion?: string; port?: number }): string {
-    const goVersion = app.runtimeVersion || '1.22';
-    const port = app.port || 8080;
-    return `FROM golang:${goVersion}-alpine AS builder
-WORKDIR /app
-RUN apk add --no-cache git
-COPY go.mod go.sum* ./
-RUN go mod download || true
-COPY . .
-RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags="-w -s" -o main .
-
-FROM alpine:3.19
-WORKDIR /app
-RUN apk --no-cache add ca-certificates tzdata
-RUN addgroup -g 1001 -S appgroup && adduser -S appuser -u 1001 -G appgroup
-COPY --from=builder /app/main .
-RUN mkdir -p /app/data && chown -R appuser:appgroup /app
-USER appuser
-ENV PORT=${port}
-EXPOSE ${port}
-HEALTHCHECK --interval=30s --timeout=3s CMD wget --no-verbose --tries=1 --spider http://localhost:${port}/health || exit 1
-CMD ["./main"]
-`;
-  }
-
-  it('should use correct Go version', () => {
-    const df = goDockerfile({ runtimeVersion: '1.21' });
-    expect(df).toContain('FROM golang:1.21-alpine');
-  });
-
-  it('should default to Go 1.22', () => {
-    const df = goDockerfile({});
-    expect(df).toContain('FROM golang:1.22-alpine');
-  });
-
-  it('should build static binary with CGO_ENABLED=0', () => {
-    const df = goDockerfile({});
-    expect(df).toContain('CGO_ENABLED=0');
-  });
-
-  it('should use multi-stage build for smaller image', () => {
-    const df = goDockerfile({});
-    expect(df).toContain('AS builder');
-    expect(df).toContain('FROM alpine:3.19');
-  });
-
-  it('should include health check', () => {
-    const df = goDockerfile({ port: 8080 });
-    expect(df).toContain('HEALTHCHECK');
-    expect(df).toContain('http://localhost:8080/health');
-  });
-
-  it('should create data directory for persistent storage', () => {
-    const df = goDockerfile({});
-    expect(df).toContain('mkdir -p /app/data');
-  });
-
-  it('should run as non-root user', () => {
-    const df = goDockerfile({});
-    expect(df).toContain('USER appuser');
-  });
-});
-
-// ─────────────────────────────────────────────────────────────────────────────
-// Python Dockerfile tests
-// ─────────────────────────────────────────────────────────────────────────────
-describe('Python Dockerfile generation', () => {
-  function pythonDockerfile(app: { runtimeVersion?: string; port?: number }): string {
-    const pythonVersion = app.runtimeVersion || '3.12';
-    const port = app.port || 8000;
-    return `FROM python:${pythonVersion}-slim AS builder
-WORKDIR /app
-RUN apt-get update && apt-get install -y build-essential libpq-dev
-COPY requirements.txt* ./
-RUN pip install --no-cache-dir --user -r requirements.txt 2>/dev/null || pip install --no-cache-dir --user flask gunicorn
-
-FROM python:${pythonVersion}-slim
-WORKDIR /app
-RUN groupadd -g 1001 appgroup && useradd -r -u 1001 -g appgroup appuser
-COPY --from=builder /root/.local /home/appuser/.local
-COPY . .
-RUN mkdir -p /app/data && chown -R appuser:appgroup /app
-USER appuser
-ENV PATH=/home/appuser/.local/bin:$PATH
-ENV PORT=${port}
-EXPOSE ${port}
-HEALTHCHECK --interval=30s --timeout=3s CMD curl -f http://localhost:${port}/health || exit 1
-CMD ["gunicorn", "-w", "4", "-b", "0.0.0.0:${port}", "app:app"]
-`;
-  }
-
-  it('should use correct Python version', () => {
-    const df = pythonDockerfile({ runtimeVersion: '3.11' });
-    expect(df).toContain('FROM python:3.11-slim');
-  });
-
-  it('should default to Python 3.12', () => {
-    const df = pythonDockerfile({});
-    expect(df).toContain('FROM python:3.12-slim');
-  });
-
-  it('should use multi-stage build', () => {
-    const df = pythonDockerfile({});
-    expect(df).toContain('AS builder');
-  });
-
-  it('should install from requirements.txt', () => {
-    const df = pythonDockerfile({});
-    expect(df).toContain('requirements.txt');
-  });
-
-  it('should include health check', () => {
-    const df = pythonDockerfile({ port: 8000 });
-    expect(df).toContain('HEALTHCHECK');
-    expect(df).toContain('http://localhost:8000/health');
-  });
-
-  it('should run as non-root user', () => {
-    const df = pythonDockerfile({});
-    expect(df).toContain('USER appuser');
-  });
-});
-
-// ─────────────────────────────────────────────────────────────────────────────
-// Django Dockerfile tests
-// ─────────────────────────────────────────────────────────────────────────────
-describe('Django Dockerfile generation', () => {
-  function djangoDockerfile(app: { runtimeVersion?: string; port?: number }): string {
-    const pythonVersion = app.runtimeVersion || '3.12';
-    const port = app.port || 8000;
-    return `FROM python:${pythonVersion}-slim AS builder
-WORKDIR /app
-RUN apt-get update && apt-get install -y build-essential libpq-dev
-COPY requirements.txt* ./
-RUN pip install --no-cache-dir --user -r requirements.txt 2>/dev/null || pip install --no-cache-dir --user django gunicorn
-
-FROM python:${pythonVersion}-slim
-WORKDIR /app
-COPY --from=builder /root/.local /home/appuser/.local
-COPY . .
-RUN mkdir -p /app/staticfiles /app/media /app/data
-USER appuser
-ENV PORT=${port}
-ENV DJANGO_SETTINGS_MODULE=config.settings
-EXPOSE ${port}
-HEALTHCHECK --interval=30s --timeout=3s CMD curl -f http://localhost:${port}/health/ || exit 1
-CMD ["sh", "-c", "python manage.py migrate --noinput && gunicorn config.wsgi:application --bind 0.0.0.0:${port}"]
-`;
-  }
-
-  it('should use correct Python version', () => {
-    const df = djangoDockerfile({ runtimeVersion: '3.10' });
-    expect(df).toContain('FROM python:3.10-slim');
-  });
-
-  it('should set DJANGO_SETTINGS_MODULE', () => {
-    const df = djangoDockerfile({});
-    expect(df).toContain('DJANGO_SETTINGS_MODULE');
-  });
-
-  it('should create staticfiles and media directories', () => {
-    const df = djangoDockerfile({});
-    expect(df).toContain('/app/staticfiles');
-    expect(df).toContain('/app/media');
-  });
-
-  it('should run migrations on startup', () => {
-    const df = djangoDockerfile({});
-    expect(df).toContain('migrate');
-  });
-
-  it('should use gunicorn for production', () => {
-    const df = djangoDockerfile({});
-    expect(df).toContain('gunicorn');
-  });
-});
-
-// ─────────────────────────────────────────────────────────────────────────────
-// .NET Dockerfile tests
-// ─────────────────────────────────────────────────────────────────────────────
-describe('.NET Dockerfile generation', () => {
-  function dotnetDockerfile(app: { runtimeVersion?: string; port?: number }): string {
-    const dotnetVersion = app.runtimeVersion || '8.0';
-    const port = app.port || 5000;
-    return `FROM mcr.microsoft.com/dotnet/sdk:${dotnetVersion} AS build
-WORKDIR /src
-COPY *.csproj ./
-RUN dotnet restore || true
-COPY . .
-RUN dotnet publish -c Release -o /app/publish
-
-FROM mcr.microsoft.com/dotnet/aspnet:${dotnetVersion}
-WORKDIR /app
-COPY --from=build /app/publish .
-RUN mkdir -p /app/data
-USER appuser
-ENV ASPNETCORE_URLS=http://+:${port}
-ENV ASPNETCORE_ENVIRONMENT=Production
-EXPOSE ${port}
-HEALTHCHECK --interval=30s --timeout=3s CMD curl -f http://localhost:${port}/health || exit 1
-CMD ["dotnet", "app.dll"]
-`;
-  }
-
-  it('should use correct .NET version', () => {
-    const df = dotnetDockerfile({ runtimeVersion: '7.0' });
-    expect(df).toContain('dotnet/sdk:7.0');
-    expect(df).toContain('dotnet/aspnet:7.0');
-  });
-
-  it('should default to .NET 8.0', () => {
-    const df = dotnetDockerfile({});
-    expect(df).toContain('dotnet/sdk:8.0');
-  });
-
-  it('should use multi-stage build', () => {
-    const df = dotnetDockerfile({});
-    expect(df).toContain('AS build');
-    expect(df).toContain('dotnet/aspnet');
-  });
-
-  it('should publish in Release mode', () => {
-    const df = dotnetDockerfile({});
-    expect(df).toContain('-c Release');
-  });
-
-  it('should set ASPNETCORE_ENVIRONMENT to Production', () => {
-    const df = dotnetDockerfile({});
-    expect(df).toContain('ASPNETCORE_ENVIRONMENT=Production');
-  });
-
-  it('should configure ASPNETCORE_URLS for correct port', () => {
-    const df = dotnetDockerfile({ port: 8080 });
-    expect(df).toContain('ASPNETCORE_URLS=http://+:8080');
-  });
-});
-
-// ─────────────────────────────────────────────────────────────────────────────
-// PHP Dockerfile tests
-// ─────────────────────────────────────────────────────────────────────────────
-describe('PHP Dockerfile generation', () => {
-  function phpDockerfile(app: { phpVersion?: string; port?: number }): string {
-    const phpVersion = app.phpVersion || '8.3';
-    const port = app.port || 80;
-    return `FROM php:${phpVersion}-fpm-alpine
-RUN apk add --no-cache nginx supervisor curl
-RUN docker-php-ext-install pdo pdo_mysql opcache
-WORKDIR /var/www/html
-COPY . .
-RUN mkdir -p /var/www/html/uploads /var/www/html/data
-RUN chown -R www-data:www-data /var/www/html
-EXPOSE ${port}
-CMD ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf"]
-`;
-  }
-
-  it('should use correct PHP version', () => {
-    const df = phpDockerfile({ phpVersion: '8.2' });
-    expect(df).toContain('FROM php:8.2-fpm-alpine');
-  });
-
-  it('should default to PHP 8.3', () => {
-    const df = phpDockerfile({});
-    expect(df).toContain('FROM php:8.3-fpm-alpine');
-  });
-
-  it('should use FPM with nginx via supervisord', () => {
-    const df = phpDockerfile({});
-    expect(df).toContain('supervisord');
-    expect(df).toContain('nginx');
-  });
-
-  it('should install common PHP extensions', () => {
-    const df = phpDockerfile({});
-    expect(df).toContain('pdo');
-    expect(df).toContain('opcache');
-  });
-
-  it('should create upload and data directories', () => {
-    const df = phpDockerfile({});
-    expect(df).toContain('/var/www/html/uploads');
-    expect(df).toContain('/var/www/html/data');
-  });
-});
-
 /**
 * Tests for the WordPress build flow — specifically:
 * 1. Helper pod PVC race condition (must wait for termination)
@@ -467,3 +185,75 @@ describe('WordPress zip structure handling', () => {
    expect(copiedAsIs).toBe(true);
  });
 });
+
+describe('Nixpacks build preparation', () => {
+  // Local copies of the pure logic in build.service.ts (see NOTE at top of file).
+  function shellQuote(value: string): string {
+    return `'${value.replace(/'/g, `'\\''`)}'`;
+  }
+
+  function nixpacksPlanEnv(app: { runtime: AppRuntime; runtimeVersion?: string }): { name: string; value: string }[] {
+    const env: { name: string; value: string }[] = [];
+    if (app.runtime === AppRuntime.NODEJS && app.runtimeVersion) {
+      env.push({ name: 'NIXPACKS_NODE_VERSION', value: String(app.runtimeVersion) });
+    }
+    if ((app.runtime === AppRuntime.PYTHON || app.runtime === AppRuntime.DJANGO) && app.runtimeVersion) {
+      env.push({ name: 'NIXPACKS_PYTHON_VERSION', value: String(app.runtimeVersion) });
+    }
+    return env;
+  }
+
+  function nixpacksPrepareInitContainer(
+    app: { runtime: AppRuntime; runtimeVersion?: string },
+    config: { nixpacksImage?: string; nixpacksBuildEnv?: string[] } = {},
+  ): any {
+    const image = config.nixpacksImage || 'ghcr.io/railwayapp/nixpacks:latest';
+    const buildEnv = config.nixpacksBuildEnv || [];
+    const envFlags = buildEnv.map((kv) => `--env ${shellQuote(kv)}`).join(' ');
+    const planEnv = nixpacksPlanEnv(app);
+    return {
+      name: 'nixpacks-prepare',
+      image,
+      env: planEnv.length ? planEnv : undefined,
+      command: [
+        'sh',
+        '-c',
+        `if [ -f source/Dockerfile ]; then cp source/Dockerfile /workspace/Dockerfile; ` +
+          `else nixpacks build source --out source ${envFlags} && cp source/.nixpacks/Dockerfile /workspace/Dockerfile; fi`,
+      ],
+      volumeMounts: [{ name: 'workspace', mountPath: '/workspace' }],
+    };
+  }
+
+  it('prefers a user-provided Dockerfile (BYO), falling back to Nixpacks', () => {
+    const script = nixpacksPrepareInitContainer({ runtime: AppRuntime.NODEJS }).command[2] as string;
+    expect(script).toContain('if [ -f source/Dockerfile ]');
+    expect(script).toContain('cp source/Dockerfile /workspace/Dockerfile');
+    expect(script).toContain('nixpacks build source --out source');
+    expect(script).toContain('cp source/.nixpacks/Dockerfile /workspace/Dockerfile');
+  });
+
+  it('uses the configured Nixpacks image (default when unset)', () => {
+    expect(nixpacksPrepareInitContainer({ runtime: AppRuntime.GO }).image).toBe('ghcr.io/railwayapp/nixpacks:latest');
+    expect(
+      nixpacksPrepareInitContainer({ runtime: AppRuntime.GO }, { nixpacksImage: 'registry.local/nixpacks:1.2.3' }).image,
+    ).toBe('registry.local/nixpacks:1.2.3');
+  });
+
+  it('bakes build-time mirror env into the build via --env flags', () => {
+    const script = nixpacksPrepareInitContainer(
+      { runtime: AppRuntime.NODEJS },
+      { nixpacksBuildEnv: ['NPM_CONFIG_REGISTRY=https://registry.npmmirror.com'] },
+    ).command[2] as string;
+    expect(script).toContain(`--env 'NPM_CONFIG_REGISTRY=https://registry.npmmirror.com'`);
+  });
+
+  it('maps the selected Node version to NIXPACKS_NODE_VERSION', () => {
+    const c = nixpacksPrepareInitContainer({ runtime: AppRuntime.NODEJS, runtimeVersion: '20' });
+    expect(c.env).toContainEqual({ name: 'NIXPACKS_NODE_VERSION', value: '20' });
+  });
+
+  it('shellQuote escapes embedded single quotes safely', () => {
+    expect(shellQuote("a'b")).toBe("'a'\\''b'");
+  });
+});
@@ -0,0 +1,94 @@
+/**
+ * Tests for the pure Trivy-report aggregation logic in ScanService.summarize.
+ *
+ * NOTE: like the other build specs, this reproduces the pure logic locally rather
+ * than importing ScanService — the service pulls in the ESM `@kubernetes/client-node`,
+ * which this project's Jest config does not transform. Keep in sync with scan.service.ts.
+ */
+
+interface VulnerabilitySummary {
+  critical: number;
+  high: number;
+  medium: number;
+  low: number;
+  unknown: number;
+  total: number;
+  scannedAt: string;
+}
+
+function parseTrivyJson(output: string): any | null {
+  if (!output) return null;
+  try {
+    return JSON.parse(output);
+  } catch {
+    const start = output.indexOf('{');
+    const end = output.lastIndexOf('}');
+    if (start >= 0 && end > start) {
+      try {
+        return JSON.parse(output.slice(start, end + 1));
+      } catch {
+        return null;
+      }
+    }
+    return null;
+  }
+}
+
+function summarize(trivyOutput: string): VulnerabilitySummary {
+  const counts = { critical: 0, high: 0, medium: 0, low: 0, unknown: 0 };
+  const parsed = parseTrivyJson(trivyOutput);
+  const results: any[] = Array.isArray(parsed?.Results) ? parsed.Results : [];
+  for (const result of results) {
+    const vulns: any[] = Array.isArray(result?.Vulnerabilities) ? result.Vulnerabilities : [];
+    for (const v of vulns) {
+      const sev = String(v?.Severity || 'UNKNOWN').toUpperCase();
+      if (sev === 'CRITICAL') counts.critical++;
+      else if (sev === 'HIGH') counts.high++;
+      else if (sev === 'MEDIUM') counts.medium++;
+      else if (sev === 'LOW') counts.low++;
+      else counts.unknown++;
+    }
+  }
+  return {
+    ...counts,
+    total: counts.critical + counts.high + counts.medium + counts.low + counts.unknown,
+    scannedAt: new Date().toISOString(),
+  };
+}
+
+describe('ScanService.summarize', () => {
+  it('counts vulnerabilities per severity across results', () => {
+    const report = JSON.stringify({
+      Results: [
+        { Vulnerabilities: [{ Severity: 'CRITICAL' }, { Severity: 'HIGH' }, { Severity: 'high' }] },
+        { Vulnerabilities: [{ Severity: 'MEDIUM' }, { Severity: 'LOW' }, { Severity: 'WeIrD' }] },
+        { Vulnerabilities: null },
+        {},
+      ],
+    });
+    const s = summarize(report);
+    expect(s).toMatchObject({ critical: 1, high: 2, medium: 1, low: 1, unknown: 1, total: 6 });
+    expect(typeof s.scannedAt).toBe('string');
+  });
+
+  it('returns all-zero summary for a clean image', () => {
+    expect(summarize(JSON.stringify({ Results: [{ Target: 'x' }] }))).toMatchObject({
+      critical: 0,
+      high: 0,
+      medium: 0,
+      low: 0,
+      unknown: 0,
+      total: 0,
+    });
+  });
+
+  it('tolerates leading log noise before the JSON', () => {
+    const noisy = `2026-06-20 INFO Need to update DB\n{"Results":[{"Vulnerabilities":[{"Severity":"CRITICAL"}]}]}`;
+    expect(summarize(noisy).critical).toBe(1);
+  });
+
+  it('returns a zero summary on unparseable output', () => {
+    expect(summarize('not json at all').total).toBe(0);
+    expect(summarize('').total).toBe(0);
+  });
+});
@@ -0,0 +1,169 @@
+import { Injectable, Logger } from '@nestjs/common';
+import { ConfigService } from '@nestjs/config';
+import * as k8s from '@kubernetes/client-node';
+import { Application } from '../applications/entities/application.entity';
+import { ClustersService } from '../clusters/clusters.service';
+import { RegistryService } from '../kubernetes/registry.service';
+
+export interface VulnerabilitySummary {
+  critical: number;
+  high: number;
+  medium: number;
+  low: number;
+  unknown: number;
+  total: number;
+  scannedAt: string;
+}
+
+/**
+ * Report-only image vulnerability scanning with Trivy. Runs a one-shot K8s Job
+ * that scans the freshly-pushed image in the in-cluster registry and stores a
+ * severity summary on the deployment. Never blocks a deployment — any failure is
+ * logged and ignored.
+ */
+@Injectable()
+export class ScanService {
+  private readonly logger = new Logger(ScanService.name);
+
+  constructor(
+    private readonly configService: ConfigService,
+    private readonly clustersService: ClustersService,
+    private readonly registryService: RegistryService,
+  ) {}
+
+  /** Aggregate a Trivy JSON report (raw stdout) into per-severity counts. Pure & testable. */
+  summarize(trivyOutput: string): VulnerabilitySummary {
+    const counts = { critical: 0, high: 0, medium: 0, low: 0, unknown: 0 };
+    const parsed = this.parseTrivyJson(trivyOutput);
+    const results: any[] = Array.isArray(parsed?.Results) ? parsed.Results : [];
+    for (const result of results) {
+      const vulns: any[] = Array.isArray(result?.Vulnerabilities) ? result.Vulnerabilities : [];
+      for (const v of vulns) {
+        const sev = String(v?.Severity || 'UNKNOWN').toUpperCase();
+        if (sev === 'CRITICAL') counts.critical++;
+        else if (sev === 'HIGH') counts.high++;
+        else if (sev === 'MEDIUM') counts.medium++;
+        else if (sev === 'LOW') counts.low++;
+        else counts.unknown++;
+      }
+    }
+    return {
+      ...counts,
+      total: counts.critical + counts.high + counts.medium + counts.low + counts.unknown,
+      scannedAt: new Date().toISOString(),
+    };
+  }
+
+  /** Trivy prints clean JSON to stdout, but tolerate any leading noise from the log stream. */
+  private parseTrivyJson(output: string): any | null {
+    if (!output) return null;
+    try {
+      return JSON.parse(output);
+    } catch {
+      const start = output.indexOf('{');
+      const end = output.lastIndexOf('}');
+      if (start >= 0 && end > start) {
+        try {
+          return JSON.parse(output.slice(start, end + 1));
+        } catch {
+          return null;
+        }
+      }
+      return null;
+    }
+  }
+
+  /**
+   * Scan a pushed image and return a severity summary, or null on any failure.
+   * Report-only: callers must treat null as "no data", never as a deploy gate.
+   */
+  async scanImage(app: Application, imageUri: string): Promise<VulnerabilitySummary | null> {
+    if (this.configService.get<boolean>('build.scanEnabled') === false) return null;
+
+    const buildNs = this.registryService.getBuildNamespace();
+    const image = this.configService.get<string>('build.trivyImage') || 'aquasec/trivy:latest';
+    const dbRepo = this.configService.get<string>('build.trivyDbRepository') || '';
+    const timeoutSeconds = this.configService.get<number>('build.scanTimeoutSeconds') || 300;
+    const { username, password } = this.registryService.getRegistryCredentials();
+
+    const jobName = `scan-${app.name}-${Date.now()}`.substring(0, 63).replace(/[^a-z0-9-]/g, '');
+
+    try {
+      const cluster = app.clusterId ? await this.clustersService.findOne(app.clusterId) : await this.clustersService.getDefault();
+      const kc = new k8s.KubeConfig();
+      kc.loadFromString(cluster.kubeconfig);
+      const coreApi = kc.makeApiClient(k8s.CoreV1Api);
+      const batchApi = kc.makeApiClient(k8s.BatchV1Api);
+
+      const env: { name: string; value: string }[] = [
+        { name: 'TRIVY_INSECURE', value: 'true' }, // in-cluster registry is plain HTTP
+        { name: 'TRIVY_NON_SSL', value: 'true' },
+      ];
+      if (username) env.push({ name: 'TRIVY_USERNAME', value: username });
+      if (password) env.push({ name: 'TRIVY_PASSWORD', value: password });
+      if (dbRepo) env.push({ name: 'TRIVY_DB_REPOSITORY', value: dbRepo });
+
+      const job: k8s.V1Job = {
+        apiVersion: 'batch/v1',
+        kind: 'Job',
+        metadata: { name: jobName, namespace: buildNs },
+        spec: {
+          backoffLimit: 0,
+          ttlSecondsAfterFinished: 120,
+          template: {
+            spec: {
+              restartPolicy: 'Never',
+              containers: [
+                {
+                  name: 'trivy',
+                  image,
+                  imagePullPolicy: 'IfNotPresent',
+                  env,
+                  args: ['image', '--quiet', '--no-progress', '--format', 'json', '--severity', 'CRITICAL,HIGH,MEDIUM,LOW', imageUri],
+                  resources: {
+                    requests: { cpu: '250m', memory: '512Mi' },
+                    limits: { cpu: '1', memory: '1Gi' },
+                  },
+                },
+              ],
+            },
+          },
+        },
+      };
+
+      await batchApi.createNamespacedJob({ namespace: buildNs, body: job });
+      await this.waitForJob(batchApi, jobName, buildNs, timeoutSeconds);
+
+      const output = await this.getJobPodLogs(coreApi, jobName, buildNs);
+      const summary = this.summarize(output);
+
+      await batchApi
+        .deleteNamespacedJob({ name: jobName, namespace: buildNs, gracePeriodSeconds: 0, propagationPolicy: 'Foreground' })
+        .catch(() => undefined);
+
+      this.logger.log(`Scan complete for ${imageUri}: ${summary.critical}C/${summary.high}H/${summary.medium}M/${summary.low}L`);
+      return summary;
+    } catch (e: any) {
+      this.logger.warn(`Image scan failed for ${imageUri} (report-only, ignored): ${e.message}`);
+      return null;
+    }
+  }
+
+  private async waitForJob(batchApi: k8s.BatchV1Api, jobName: string, namespace: string, timeoutSeconds: number): Promise<void> {
+    const deadline = Date.now() + timeoutSeconds * 1000;
+    while (Date.now() < deadline) {
+      const job = await batchApi.readNamespacedJob({ name: jobName, namespace });
+      if (job.status?.succeeded) return;
+      if ((job.status?.failed ?? 0) > 0) return; // Trivy exits non-zero on findings with some flags; read logs anyway
+      await new Promise((r) => setTimeout(r, 4000));
+    }
+    throw new Error(`Scan job ${jobName} timed out after ${timeoutSeconds}s`);
+  }
+
+  private async getJobPodLogs(coreApi: k8s.CoreV1Api, jobName: string, namespace: string): Promise<string> {
+    const pods = await coreApi.listNamespacedPod({ namespace, labelSelector: `job-name=${jobName}` });
+    const podName = pods.items[0]?.metadata?.name;
+    if (!podName) throw new Error(`No pod found for scan job ${jobName}`);
+    return coreApi.readNamespacedPodLog({ name: podName, namespace, container: 'trivy' });
+  }
+}
@@ -1102,9 +1102,133 @@ export class ClustersService implements OnModuleInit, OnModuleDestroy {

    await this.ensureK3sRegistryMirrors(appsApi, registryUrl);

+    // ── 8. MinIO (S3-compatible) for application source archives ──
+    await this.ensureMinioInfrastructure(coreApi, appsApi, buildNs);
+
    this.logger.log(`✅ Cluster bootstrap complete — registry: ${registryUrl}`);
  }

+  /**
+   * Provision in-cluster MinIO (Deployment + PVC + Service + credentials Secret)
+   * in the build namespace. Application source archives are uploaded here by the
+   * API and pulled by build pods via presigned URLs.
+   */
+  private async ensureMinioInfrastructure(coreApi: k8s.CoreV1Api, appsApi: k8s.AppsV1Api, buildNs: string): Promise<void> {
+    const accessKey = this.configService.get<string>('minio.accessKey') || 'cloudhost';
+    const secretKey = this.configService.get<string>('minio.secretKey') || '';
+    const pvcName = 'minio-data';
+    const deployName = 'minio';
+    const svcName = 'minio';
+    const secretName = 'minio-credentials';
+
+    // 1. Credentials Secret (shared by the MinIO server env and the API client)
+    const minioSecret: k8s.V1Secret = {
+      metadata: { name: secretName, namespace: buildNs },
+      type: 'Opaque',
+      data: {
+        accesskey: Buffer.from(accessKey).toString('base64'),
+        secretkey: Buffer.from(secretKey).toString('base64'),
+      },
+    };
+    try {
+      await coreApi.readNamespacedSecret({ name: secretName, namespace: buildNs });
+      await coreApi.replaceNamespacedSecret({ name: secretName, namespace: buildNs, body: minioSecret });
+    } catch (err: any) {
+      if (err.code === 404 || err.body?.code === 404) {
+        await coreApi.createNamespacedSecret({ namespace: buildNs, body: minioSecret });
+        this.logger.log(`Created ${secretName} Secret`);
+      } else {
+        throw err;
+      }
+    }
+
+    // 2. Data PVC
+    try {
+      await coreApi.readNamespacedPersistentVolumeClaim({ name: pvcName, namespace: buildNs });
+    } catch (err: any) {
+      if (err.code === 404 || err.body?.code === 404) {
+        await coreApi.createNamespacedPersistentVolumeClaim({
+          namespace: buildNs,
+          body: {
+            metadata: { name: pvcName, namespace: buildNs },
+            spec: { accessModes: ['ReadWriteOnce'], resources: { requests: { storage: '20Gi' } } },
+          },
+        });
+        this.logger.log(`Created PVC "${pvcName}" (20Gi)`);
+      } else {
+        throw err;
+      }
+    }
+
+    // 3. Deployment
+    try {
+      await appsApi.readNamespacedDeployment({ name: deployName, namespace: buildNs });
+    } catch (err: any) {
+      if (err.code === 404 || err.body?.code === 404) {
+        await appsApi.createNamespacedDeployment({
+          namespace: buildNs,
+          body: {
+            metadata: { name: deployName, namespace: buildNs, labels: { app: 'minio' } },
+            spec: {
+              replicas: 1,
+              selector: { matchLabels: { app: 'minio' } },
+              template: {
+                metadata: { labels: { app: 'minio' } },
+                spec: {
+                  containers: [
+                    {
+                      name: 'minio',
+                      image: process.env.MINIO_IMAGE || 'minio/minio:latest',
+                      args: ['server', '/data', '--console-address', ':9001'],
+                      env: [
+                        { name: 'MINIO_ROOT_USER', valueFrom: { secretKeyRef: { name: secretName, key: 'accesskey' } } },
+                        { name: 'MINIO_ROOT_PASSWORD', valueFrom: { secretKeyRef: { name: secretName, key: 'secretkey' } } },
+                      ],
+                      ports: [{ containerPort: 9000 }, { containerPort: 9001 }],
+                      volumeMounts: [{ name: 'data', mountPath: '/data' }],
+                      resources: {
+                        requests: { cpu: '100m', memory: '256Mi' },
+                        limits: { cpu: '1', memory: '1Gi' },
+                      },
+                    },
+                  ],
+                  volumes: [{ name: 'data', persistentVolumeClaim: { claimName: pvcName } }],
+                },
+              },
+            },
+          },
+        });
+        this.logger.log(`Created MinIO Deployment`);
+      } else {
+        throw err;
+      }
+    }
+
+    // 4. ClusterIP Service (API :9000, console :9001)
+    try {
+      await coreApi.readNamespacedService({ name: svcName, namespace: buildNs });
+    } catch (err: any) {
+      if (err.code === 404 || err.body?.code === 404) {
+        await coreApi.createNamespacedService({
+          namespace: buildNs,
+          body: {
+            metadata: { name: svcName, namespace: buildNs, labels: { app: 'minio' } },
+            spec: {
+              selector: { app: 'minio' },
+              ports: [
+                { name: 'api', port: 9000, targetPort: 9000 as any, protocol: 'TCP' },
+                { name: 'console', port: 9001, targetPort: 9001 as any, protocol: 'TCP' },
+              ],
+            },
+          },
+        });
+        this.logger.log(`Created MinIO Service`);
+      } else {
+        throw err;
+      }
+    }
+  }
+
  /** In-cluster registry mirror for k3s/containerd (HTTP). Removes legacy external-registry DaemonSet if present. */
  private async ensureK3sRegistryMirrors(appsApi: k8s.AppsV1Api, registryUrl: string): Promise<void> {
    const namespace = 'kube-system';
@@ -0,0 +1,51 @@
+import { Global, Module, OnApplicationShutdown, Logger } from '@nestjs/common';
+import { ModuleRef } from '@nestjs/core';
+import { ConfigService } from '@nestjs/config';
+import IORedis, { Redis } from 'ioredis';
+
+/** Injection token for the shared ioredis client. */
+export const REDIS_CLIENT = 'REDIS_CLIENT';
+
+/**
+ * Shared Redis client used for build state (progress / cancel flags / build
+ * sessions) so the data survives across backend replicas — unlike the previous
+ * in-memory Maps that only worked with a single instance.
+ *
+ * Reuses the same connection details as the Bull queue (`redis.host/port`).
+ */
+@Global()
+@Module({
+  providers: [
+    {
+      provide: REDIS_CLIENT,
+      inject: [ConfigService],
+      useFactory: (configService: ConfigService): Redis => {
+        const client = new IORedis({
+          host: configService.get<string>('redis.host'),
+          port: configService.get<number>('redis.port'),
+          // Build state writes are best-effort telemetry — never let a Redis
+          // hiccup take down the request that triggered them.
+          maxRetriesPerRequest: 2,
+          enableOfflineQueue: true,
+        });
+        client.on('error', (err) => {
+          new Logger('RedisClient').warn(`Redis connection error: ${err.message}`);
+        });
+        return client;
+      },
+    },
+  ],
+  exports: [REDIS_CLIENT],
+})
+export class RedisModule implements OnApplicationShutdown {
+  constructor(private readonly moduleRef: ModuleRef) {}
+
+  async onApplicationShutdown(): Promise<void> {
+    try {
+      const client = this.moduleRef.get<Redis>(REDIS_CLIENT, { strict: false });
+      await client?.quit();
+    } catch {
+      /* ignore shutdown errors */
+    }
+  }
+}
@@ -0,0 +1,13 @@
+import { Global, Module } from '@nestjs/common';
+import { StorageService } from './storage.service';
+
+/**
+ * Global module exposing the MinIO-backed {@link StorageService} so both the API
+ * (source upload) and the build pipeline (presigned download) can inject it.
+ */
+@Global()
+@Module({
+  providers: [StorageService],
+  exports: [StorageService],
+})
+export class StorageModule {}
@@ -0,0 +1,64 @@
+import { Injectable, Logger } from '@nestjs/common';
+import { ConfigService } from '@nestjs/config';
+import * as Minio from 'minio';
+
+/**
+ * Object storage for application source archives, backed by the in-cluster MinIO
+ * (S3-compatible). Replaces the previous local-disk + PVC + kubectl-cp upload path:
+ * the API streams the uploaded zip straight to MinIO, and build pods pull it via a
+ * short-lived presigned URL (no credentials, no kubectl, no helper pod).
+ */
+@Injectable()
+export class StorageService {
+  private readonly logger = new Logger(StorageService.name);
+  private readonly client: Minio.Client;
+  private readonly bucket: string;
+  private bucketReady = false;
+
+  constructor(private readonly configService: ConfigService) {
+    this.bucket = this.configService.get<string>('minio.bucket') || 'app-sources';
+    this.client = new Minio.Client({
+      endPoint: this.configService.get<string>('minio.endpoint') || 'minio.cloudhost-builds.svc.cluster.local',
+      port: this.configService.get<number>('minio.port') || 9000,
+      useSSL: this.configService.get<boolean>('minio.useSSL') || false,
+      accessKey: this.configService.get<string>('minio.accessKey') || 'cloudhost',
+      secretKey: this.configService.get<string>('minio.secretKey') || '',
+    });
+  }
+
+  /** Object key for an app's source archive. */
+  sourceKey(userId: string, appId: string): string {
+    return `${userId}/${appId}/source.zip`;
+  }
+
+  private async ensureBucket(): Promise<void> {
+    if (this.bucketReady) return;
+    const exists = await this.client.bucketExists(this.bucket).catch(() => false);
+    if (!exists) {
+      await this.client.makeBucket(this.bucket);
+      this.logger.log(`Created MinIO bucket "${this.bucket}"`);
+    }
+    this.bucketReady = true;
+  }
+
+  /** Upload an app's source archive; returns the stored object key (saved as app.codePath). */
+  async putSource(userId: string, appId: string, data: Buffer): Promise<string> {
+    await this.ensureBucket();
+    const key = this.sourceKey(userId, appId);
+    await this.client.putObject(this.bucket, key, data, data.length, { 'Content-Type': 'application/zip' });
+    this.logger.log(`Uploaded source ${(data.length / 1024 / 1024).toFixed(1)}MB → ${this.bucket}/${key}`);
+    return key;
+  }
+
+  /** Short-lived presigned GET URL the build pod uses to download the source. */
+  async presignSourceGet(key: string, expirySeconds = 3600): Promise<string> {
+    return this.client.presignedGetObject(this.bucket, key, expirySeconds);
+  }
+
+  /** Best-effort delete of an app's source object (e.g. on app deletion). */
+  async removeSource(key: string): Promise<void> {
+    await this.client.removeObject(this.bucket, key).catch((e) => {
+      this.logger.warn(`Failed to remove source ${key}: ${e.message}`);
+    });
+  }
+}
@@ -125,9 +125,45 @@ export default () => ({
    password: process.env.REGISTRY_PASSWORD || '',
  },

+  // In-cluster MinIO (S3-compatible) for application source archives.
+  minio: {
+    endpoint: process.env.MINIO_ENDPOINT || 'minio.cloudhost-builds.svc.cluster.local',
+    port: parseInt(process.env.MINIO_PORT || '9000', 10),
+    useSSL: process.env.MINIO_USE_SSL === 'true',
+    accessKey: process.env.MINIO_ACCESS_KEY || 'cloudhost',
+    secretKey: process.env.MINIO_SECRET_KEY || 'CloudHost2024!Minio',
+    bucket: process.env.MINIO_BUCKET || 'app-sources',
+  },
+
  build: {
    namespace: process.env.BUILD_NAMESPACE || 'cloudhost-builds',
    serviceAccount: process.env.BUILD_SERVICE_ACCOUNT || 'kaniko-builder',
+    /** Max number of build+deploy pipelines processed concurrently across the queue. */
+    concurrency: parseInt(process.env.BUILD_CONCURRENCY || '3', 10),
+    /** Per-image-build timeout (Kaniko job) in seconds. */
+    timeoutSeconds: parseInt(process.env.BUILD_TIMEOUT_SECONDS || '600', 10),
+    /** Nixpacks builder image used to generate a Dockerfile for code runtimes. */
+    nixpacksImage: process.env.NIXPACKS_IMAGE || 'ghcr.io/railwayapp/nixpacks:latest',
+    /**
+     * Build-time env baked into Nixpacks-generated images (mirrors/proxies for the
+     * Iran network, e.g. "NPM_CONFIG_REGISTRY=https://registry.npmmirror.com").
+     * Comma-separated KEY=VALUE pairs — set per the Phase 0 spike findings.
+     */
+    nixpacksBuildEnv: (process.env.NIXPACKS_BUILD_ENV || '')
+      .split(',')
+      .map((s) => s.trim())
+      .filter(Boolean),
+    /** Report-only Trivy image scan after a successful build. */
+    scanEnabled: process.env.BUILD_SCAN_ENABLED !== 'false',
+    trivyImage: process.env.TRIVY_IMAGE || 'aquasec/trivy:latest',
+    /** Optional mirror for Trivy's vulnerability DB (Iran network); empty = default ghcr.io. */
+    trivyDbRepository: process.env.TRIVY_DB_REPOSITORY || '',
+    /** Max seconds to wait for the Trivy scan job. */
+    scanTimeoutSeconds: parseInt(process.env.BUILD_SCAN_TIMEOUT_SECONDS || '300', 10),
+    /** Registry garbage collection: keep the N most recent image tags per app repo. */
+    registryGcEnabled: process.env.REGISTRY_GC_ENABLED !== 'false',
+    registryKeepVersions: parseInt(process.env.REGISTRY_KEEP_VERSIONS || '3', 10),
+    registryGcIntervalMs: parseInt(process.env.REGISTRY_GC_INTERVAL_MS || '86400000', 10), // daily
  },

  elasticsearch: {
@@ -0,0 +1,2 @@
+/** Bull queue name for build+deploy pipelines. */
+export const DEPLOY_QUEUE = 'app-deploy';
@@ -0,0 +1,27 @@
+import { Process, Processor } from '@nestjs/bull';
+import { Logger } from '@nestjs/common';
+import { Job } from 'bull';
+import { DeploymentsService, DeploymentJobData } from './deployments.service';
+import { DEPLOY_QUEUE } from './deployment.constants';
+
+/**
+ * Processes build+deploy pipelines off the `app-deploy` queue with a bounded
+ * concurrency (BUILD_CONCURRENCY, default 3) so simultaneous user deploys can't
+ * flood the cluster with Kaniko build jobs (2 CPU / 4Gi each).
+ *
+ * Concurrency is read at module-load time from env because Bull's `@Process`
+ * decorator option must be a constant.
+ */
+@Processor(DEPLOY_QUEUE)
+export class DeploymentProcessor {
+  private readonly logger = new Logger(DeploymentProcessor.name);
+
+  constructor(private readonly deploymentsService: DeploymentsService) {}
+
+  @Process({ name: 'run', concurrency: parseInt(process.env.BUILD_CONCURRENCY || '3', 10) })
+  async handleRun(job: Job<DeploymentJobData>): Promise<void> {
+    const { deploymentId } = job.data;
+    this.logger.log(`Processing deployment ${deploymentId} (job ${job.id})`);
+    await this.deploymentsService.processDeploymentJob(job.data);
+  }
+}
@@ -1,7 +1,10 @@
 import { Module, forwardRef } from '@nestjs/common';
 import { TypeOrmModule } from '@nestjs/typeorm';
+import { BullModule } from '@nestjs/bull';
 import { DeploymentsService } from './deployments.service';
 import { DeploymentsController } from './deployments.controller';
+import { DeploymentProcessor } from './deployment.processor';
+import { DEPLOY_QUEUE } from './deployment.constants';
 import { Deployment } from './entities/deployment.entity';
 import { ApplicationsModule } from '../applications/applications.module';
 import { KubernetesModule } from '../kubernetes/kubernetes.module';
@@ -11,13 +14,14 @@ import { ClustersModule } from '../clusters/clusters.module';
@Module({
  imports: [
    TypeOrmModule.forFeature([Deployment]),
+    BullModule.registerQueue({ name: DEPLOY_QUEUE }),
    forwardRef(() => ApplicationsModule),
    forwardRef(() => ClustersModule),
    KubernetesModule,
    BuildModule,
  ],
  controllers: [DeploymentsController],
-  providers: [DeploymentsService],
+  providers: [DeploymentsService, DeploymentProcessor],
  exports: [DeploymentsService],
 })
 export class DeploymentsModule {}
@@ -1,11 +1,15 @@
-import { Injectable, NotFoundException, BadRequestException, Logger, Inject, forwardRef } from '@nestjs/common';
+import { Injectable, NotFoundException, BadRequestException, Logger, Inject, forwardRef, OnModuleInit } from '@nestjs/common';
 import { InjectRepository } from '@nestjs/typeorm';
+import { InjectQueue } from '@nestjs/bull';
+import { Queue } from 'bull';
 import { Repository } from 'typeorm';
 import * as fs from 'fs';
+import { DEPLOY_QUEUE } from './deployment.constants';
 import { Deployment } from './entities/deployment.entity';
 import { ApplicationsService } from '../applications/applications.service';
 import { KubernetesService } from '../kubernetes/kubernetes.service';
 import { BuildService, BuildProgress, BuildCancelledError } from '../build/build.service';
+import { ScanService } from '../build/scan.service';
 import * as crypto from 'crypto';
 import {
  AppLifecycleStatus,
@@ -15,8 +19,15 @@ import {
 } from '../common/enums';
 import { ClustersService } from '../clusters/clusters.service';

+/** Payload enqueued on the `app-deploy` queue for the build+deploy pipeline. */
+export interface DeploymentJobData {
+  deploymentId: string;
+  applicationId: string;
+  previewSubdomain: string | null;
+}
+
@Injectable()
-export class DeploymentsService {
+export class DeploymentsService implements OnModuleInit {
  private readonly logger = new Logger(DeploymentsService.name);

  constructor(
@@ -27,8 +38,41 @@ export class DeploymentsService {
    private kubernetesService: KubernetesService,
    private buildService: BuildService,
    private clustersService: ClustersService,
+    private scanService: ScanService,
+    @InjectQueue(DEPLOY_QUEUE)
+    private deployQueue: Queue<DeploymentJobData>,
  ) {}

+  async onModuleInit(): Promise<void> {
+    try {
+      await this.deploymentsRepository.query(
+        `ALTER TABLE deployments ADD COLUMN IF NOT EXISTS "vulnerabilitySummary" jsonb`,
+      );
+    } catch (e: any) {
+      this.logger.warn(`Could not ensure deployments.vulnerabilitySummary column: ${e.message}`);
+    }
+  }
+
+  /**
+   * Queue worker entrypoint — runs one build+deploy pipeline. Bounded
+   * concurrency lives on the Bull processor, so this just dispatches to the
+   * managed (Helm-only) or app (build+deploy) pipeline. Both pipelines handle
+   * their own errors, so a failure here never triggers a Bull retry.
+   */
+  async processDeploymentJob(data: DeploymentJobData): Promise<void> {
+    const { deploymentId, applicationId, previewSubdomain } = data;
+    if (await this.isDeploymentCancelled(deploymentId)) {
+      this.logger.log(`Deployment ${deploymentId} already cancelled before pickup — skipping`);
+      return;
+    }
+    const app = await this.applicationsService.findOne(applicationId);
+    if (isManagedProductType(app.productType)) {
+      await this.executeManagedPipeline(deploymentId, app);
+    } else {
+      await this.executePipeline(deploymentId, app, previewSubdomain);
+    }
+  }
+
  /**
   * Random 7-digit suffix for the preview host: <userId>-<7-digit>.<baseDomain>.
   * Generated once per application (see resolvePreviewNumber) and persisted.
@@ -77,13 +121,13 @@ export class DeploymentsService {
      saved.previewSubdomain = previewSubdomain;
    }

-    // Trigger async pipeline (Helm-only for managed services, build+deploy for apps)
-    const run = isManagedProductType(app.productType)
-      ? this.executeManagedPipeline(saved.id, app)
-      : this.executePipeline(saved.id, app, previewSubdomain);
-    run.catch((error) => {
-      this.logger.error(`Pipeline failed for deployment ${saved.id}:`, error);
-    });
+    // Enqueue the build+deploy pipeline. The Bull processor runs it with bounded
+    // concurrency so concurrent user deploys can't flood the cluster.
+    await this.deployQueue.add(
+      'run',
+      { deploymentId: saved.id, applicationId: app.id, previewSubdomain },
+      { removeOnComplete: true, removeOnFail: true },
+    );

    return saved;
  }
@@ -187,6 +231,19 @@ export class DeploymentsService {
      // Save build log
      await this.deploymentsRepository.update(deploymentId, { buildLog: buildResult.buildLog });

+      // Report-only vulnerability scan — runs alongside the deploy and is
+      // persisted when it finishes. Never blocks or fails the deployment.
+      void this.scanService
+        .scanImage(app, imageUri)
+        .then((summary) => {
+          if (summary) {
+            return this.deploymentsRepository.update(deploymentId, {
+              vulnerabilitySummary: summary as Record<string, any>,
+            });
+          }
+        })
+        .catch((e) => this.logger.warn(`Scan persistence failed for ${deploymentId}: ${e.message}`));
+
      // Step 2: Update app with new image tag
      await this.applicationsService.updateImageTag(app.id, imageUri);

@@ -486,7 +543,10 @@ export class DeploymentsService {
    return this.kubernetesService.getPodLogs(app);
  }

-  async getBuildLogs(applicationId: string, userId: string): Promise<{ buildLog: string | null; status: string; version: string | null; createdAt: Date }> {
+  async getBuildLogs(
+    applicationId: string,
+    userId: string,
+  ): Promise<{ buildLog: string | null; status: string; version: string | null; createdAt: Date; vulnerabilitySummary: Record<string, any> | null }> {
    const app = await this.applicationsService.findOne(applicationId, userId);

    const latest = await this.deploymentsRepository.findOne({
@@ -495,7 +555,7 @@ export class DeploymentsService {
    });

    if (!latest) {
-      return { buildLog: null, status: 'no_deployment', version: null, createdAt: new Date() };
+      return { buildLog: null, status: 'no_deployment', version: null, createdAt: new Date(), vulnerabilitySummary: null };
    }

    if (this.isManagedOrHelmOnlyApp(app)) {
@@ -504,6 +564,7 @@ export class DeploymentsService {
        status: latest.status,
        version: latest.version,
        createdAt: latest.createdAt,
+        vulnerabilitySummary: null,
      };
    }

@@ -523,6 +584,7 @@ export class DeploymentsService {
      status: latest.status,
      version: latest.version,
      createdAt: latest.createdAt,
+      vulnerabilitySummary: latest.vulnerabilitySummary ?? null,
    };
  }

@@ -537,7 +599,7 @@ export class DeploymentsService {

    if (!latest) return null;

-    const progress = this.buildService.getProgress(latest.id);
+    const progress = await this.buildService.getProgress(latest.id);
    if (progress) return progress;

    // No in-memory progress — infer from deployment status
@@ -689,10 +751,12 @@ export class DeploymentsService {
      saved.previewSubdomain = previewSubdomain;
    }

-    // Trigger async build & deploy pipeline (same as initial deploy)
-    this.executePipeline(saved.id, app, previewSubdomain).catch((error) => {
-      this.logger.error(`Redeploy pipeline failed for deployment ${saved.id}:`, error);
-    });
+    // Enqueue the build & deploy pipeline (same queue as initial deploy)
+    await this.deployQueue.add(
+      'run',
+      { deploymentId: saved.id, applicationId: app.id, previewSubdomain },
+      { removeOnComplete: true, removeOnFail: true },
+    );

    this.logger.log(`Redeploy triggered for ${app.name} (${app.gitUrl ? 'git: ' + app.gitUrl : 'zip'})`);
    return saved;
@@ -33,6 +33,14 @@ export class Deployment {
  @Column({ type: 'text', nullable: true })
  deployLog: string;

+  /**
+   * Report-only Trivy vulnerability summary for the built image
+   * (e.g. { critical, high, medium, low, unknown, total, scannedAt }).
+   * Non-blocking — never gates a deployment.
+   */
+  @Column({ type: 'jsonb', nullable: true })
+  vulnerabilitySummary: Record<string, any> | null;
+
  /**
   * Per-deployment preview number (derived deterministically from deployment.id).
   * Used to build preview ingress host under the main frontend domain.
@@ -3,6 +3,7 @@ import { TypeOrmModule } from '@nestjs/typeorm';
 import { KubernetesService } from './kubernetes.service';
 import { HelmService } from './helm.service';
 import { RegistryService } from './registry.service';
+import { RegistryGcService } from './registry-gc.service';
 import { ElasticsearchService } from './elasticsearch.service';
 import { ElasticsearchController } from './elasticsearch.controller';
 import { LogsController } from './logs.controller';
@@ -13,7 +14,7 @@ import { Deployment } from '../deployments/entities/deployment.entity';
@Module({
  imports: [forwardRef(() => ClustersModule), TypeOrmModule.forFeature([Application, Deployment])],
  controllers: [ElasticsearchController, LogsController],
-  providers: [KubernetesService, HelmService, RegistryService, ElasticsearchService],
+  providers: [KubernetesService, HelmService, RegistryService, RegistryGcService, ElasticsearchService],
  exports: [KubernetesService, HelmService, RegistryService, ElasticsearchService],
 })
 export class KubernetesModule {}
@@ -0,0 +1,25 @@
+import { selectTagsToDelete } from './registry-gc.util';
+
+describe('selectTagsToDelete', () => {
+  it('keeps the N newest numeric (Date.now) tags and deletes the rest', () => {
+    const tags = ['1000', '3000', '2000', '5000', '4000'];
+    const toDelete = selectTagsToDelete(tags, 3);
+    // newest 3 = 5000,4000,3000 → delete 2000,1000
+    expect(toDelete.sort()).toEqual(['1000', '2000']);
+  });
+
+  it('deletes nothing when tag count is within the keep limit', () => {
+    expect(selectTagsToDelete(['1000', '2000'], 3)).toEqual([]);
+    expect(selectTagsToDelete([], 3)).toEqual([]);
+  });
+
+  it('treats non-numeric tags as oldest (eligible for deletion first)', () => {
+    const tags = ['latest', '2000', '1000'];
+    // numeric newest kept first: 2000,1000 kept (keep=2) → delete latest
+    expect(selectTagsToDelete(tags, 2)).toEqual(['latest']);
+  });
+
+  it('keep=0 deletes every tag', () => {
+    expect(selectTagsToDelete(['1000', '2000'], 0).sort()).toEqual(['1000', '2000']);
+  });
+});
@@ -0,0 +1,164 @@
+import { Injectable, Logger, OnModuleInit, OnModuleDestroy, Inject } from '@nestjs/common';
+import { ConfigService } from '@nestjs/config';
+import { Writable } from 'stream';
+import * as k8s from '@kubernetes/client-node';
+import { Redis } from 'ioredis';
+import { REDIS_CLIENT } from '../common/redis/redis.module';
+import { ClustersService } from '../clusters/clusters.service';
+import { RegistryService } from './registry.service';
+import { selectTagsToDelete } from './registry-gc.util';
+
+/**
+ * Periodic registry garbage collection: prunes each app image repo to the N most
+ * recent tags (deleting older manifests) and then reclaims disk by running
+ * `registry garbage-collect` in the registry pod. Runs on the local/default
+ * cluster's in-cluster registry (the one reachable via cluster DNS). A Redis lock
+ * keeps a single replica running it at a time (see [[multi-instance-interval-jobs]]).
+ */
+@Injectable()
+export class RegistryGcService implements OnModuleInit, OnModuleDestroy {
+  private readonly logger = new Logger(RegistryGcService.name);
+  private timer?: NodeJS.Timeout;
+  private static readonly LOCK_KEY = 'registry:gc:lock';
+
+  constructor(
+    private readonly configService: ConfigService,
+    private readonly clustersService: ClustersService,
+    private readonly registryService: RegistryService,
+    @Inject(REDIS_CLIENT) private readonly redis: Redis,
+  ) {}
+
+  onModuleInit(): void {
+    if (this.configService.get<boolean>('build.registryGcEnabled') === false) return;
+    const interval = this.configService.get<number>('build.registryGcIntervalMs') || 86_400_000;
+    this.timer = setInterval(() => void this.runGc(), interval);
+    // First pass shortly after boot.
+    setTimeout(() => void this.runGc(), 60_000);
+    this.logger.log(`Registry GC scheduled — interval: ${Math.round(interval / 3600000)}h`);
+  }
+
+  onModuleDestroy(): void {
+    if (this.timer) clearInterval(this.timer);
+  }
+
+  /** Run one GC pass guarded by a Redis lock so only one replica executes it. */
+  async runGc(): Promise<void> {
+    const keep = this.configService.get<number>('build.registryKeepVersions') || 3;
+    const locked = await this.redis.set(RegistryGcService.LOCK_KEY, '1', 'EX', 900, 'NX').catch(() => null);
+    if (locked !== 'OK') {
+      this.logger.debug('Registry GC already running on another replica — skipping');
+      return;
+    }
+    try {
+      await this.pruneDefaultClusterRegistry(keep);
+    } catch (e: any) {
+      this.logger.warn(`Registry GC failed: ${e.message}`);
+    } finally {
+      await this.redis.del(RegistryGcService.LOCK_KEY).catch(() => undefined);
+    }
+  }
+
+  private authHeader(): string {
+    const { username, password } = this.registryService.getRegistryCredentials();
+    return 'Basic ' + Buffer.from(`${username}:${password}`).toString('base64');
+  }
+
+  private async registryFetch(pathOrUrl: string, init: RequestInit = {}): Promise<Response> {
+    const base = `http://${this.registryService.getRegistryUrl()}`;
+    return fetch(`${base}${pathOrUrl}`, {
+      ...init,
+      headers: { Authorization: this.authHeader(), ...(init.headers || {}) },
+    });
+  }
+
+  private async pruneDefaultClusterRegistry(keep: number): Promise<void> {
+    // 1. List repositories
+    const catalogRes = await this.registryFetch('/v2/_catalog?n=10000');
+    if (!catalogRes.ok) throw new Error(`catalog ${catalogRes.status}`);
+    const repositories: string[] = ((await catalogRes.json()) as any)?.repositories || [];
+
+    let deletedTotal = 0;
+    for (const repo of repositories) {
+      // Skip Kaniko cache repos — pruning them just slows the next build.
+      if (repo.endsWith('/cache')) continue;
+      deletedTotal += await this.pruneRepo(repo, keep);
+    }
+
+    if (deletedTotal > 0) {
+      this.logger.log(`Registry GC: deleted ${deletedTotal} old manifest(s); reclaiming disk…`);
+      await this.runGarbageCollect();
+    } else {
+      this.logger.debug('Registry GC: nothing to prune');
+    }
+  }
+
+  /** Delete all but the newest `keep` tags of one repo. Returns count deleted. */
+  private async pruneRepo(repo: string, keep: number): Promise<number> {
+    const tagsRes = await this.registryFetch(`/v2/${repo}/tags/list`);
+    if (!tagsRes.ok) return 0;
+    const tags: string[] = ((await tagsRes.json()) as any)?.tags || [];
+    const toDelete = selectTagsToDelete(tags, keep);
+    if (toDelete.length === 0) return 0;
+
+    const seenDigests = new Set<string>();
+    let deleted = 0;
+    for (const tag of toDelete) {
+      try {
+        const head = await this.registryFetch(`/v2/${repo}/manifests/${tag}`, {
+          method: 'GET',
+          headers: {
+            Accept: 'application/vnd.docker.distribution.manifest.v2+json, application/vnd.oci.image.manifest.v1+json',
+          },
+        });
+        const digest = head.headers.get('docker-content-digest');
+        if (!digest || seenDigests.has(digest)) continue; // multiple tags can share a digest
+        seenDigests.add(digest);
+        const del = await this.registryFetch(`/v2/${repo}/manifests/${digest}`, { method: 'DELETE' });
+        if (del.ok || del.status === 202) deleted++;
+      } catch (e: any) {
+        this.logger.warn(`Failed to delete ${repo}:${tag}: ${e.message}`);
+      }
+    }
+    if (deleted > 0) this.logger.log(`Registry GC: pruned ${deleted} tag(s) from ${repo} (kept ${keep})`);
+    return deleted;
+  }
+
+  /** Reclaim disk by running `registry garbage-collect` inside the registry pod. Best-effort. */
+  private async runGarbageCollect(): Promise<void> {
+    try {
+      const buildNs = this.registryService.getBuildNamespace();
+      const cluster = await this.clustersService.getDefault();
+      const kc = new k8s.KubeConfig();
+      kc.loadFromString(cluster.kubeconfig);
+      const coreApi = kc.makeApiClient(k8s.CoreV1Api);
+
+      const pods = await coreApi.listNamespacedPod({ namespace: buildNs, labelSelector: 'app=registry' });
+      const podName = pods.items[0]?.metadata?.name;
+      if (!podName) {
+        this.logger.warn('Registry GC: no registry pod found for garbage-collect');
+        return;
+      }
+
+      const exec = new k8s.Exec(kc);
+      const sink = new Writable({ write: (_c, _e, cb) => cb() });
+      await new Promise<void>((resolve, reject) => {
+        exec
+          .exec(
+            buildNs,
+            podName,
+            'registry',
+            ['/bin/registry', 'garbage-collect', '/etc/docker/registry/config.yml'],
+            sink,
+            sink,
+            null,
+            false,
+            (status) => (status.status === 'Failure' ? reject(new Error(status.message)) : resolve()),
+          )
+          .catch(reject);
+      });
+      this.logger.log('Registry GC: garbage-collect completed');
+    } catch (e: any) {
+      this.logger.warn(`Registry garbage-collect failed (manifests already deleted): ${e.message}`);
+    }
+  }
+}
@@ -0,0 +1,19 @@
+/**
+ * Decide which image tags to delete, keeping the `keep` most recent. Tags are
+ * `Date.now()` strings, so newest = highest numeric value; non-numeric tags sort
+ * last (treated as oldest) and are eligible for deletion once `keep` is met.
+ *
+ * Kept in its own (dependency-free) module so it can be unit-tested without
+ * pulling in the ESM `@kubernetes/client-node` that the GC service imports.
+ */
+export function selectTagsToDelete(tags: string[], keep: number): string[] {
+  const sorted = [...tags].sort((a, b) => {
+    const na = Number(a);
+    const nb = Number(b);
+    if (Number.isNaN(na) && Number.isNaN(nb)) return a < b ? 1 : -1;
+    if (Number.isNaN(na)) return 1;
+    if (Number.isNaN(nb)) return -1;
+    return nb - na; // newest first
+  });
+  return sorted.slice(Math.max(0, keep));
+}