feat(build): revamp app build pipeline (queue, Nixpacks, MinIO, Trivy, registry GC)

Rework the application build/deploy pipeline for scalability, reproducibility,
and security:

- Build queue: deploys run through a bounded-concurrency Bull queue
  (BUILD_CONCURRENCY, default 3) so concurrent user deploys can't flood the
  cluster with Kaniko jobs. Build state (progress / cancel / session) moves from
  in-memory Maps to Redis, so cancel + live logs work across backend replicas.
- Nixpacks + BYO Dockerfile: code runtimes build via Nixpacks (or the user's own
  Dockerfile when present); the hand-written per-runtime Dockerfile generators
  and runtime auto-detection are removed. WordPress keeps its templated path.
  Build-time mirror env (NIXPACKS_BUILD_ENV) supports the Iran network.
- Source upload to MinIO: archives stream to in-cluster MinIO; build pods pull
  via a presigned URL. Removes the PVC + helper pod + kubectl cp upload path.
- Report-only Trivy scan after build; per-severity summary stored on the
  deployment and shown as a badge in the dashboard. Never gates a deploy.
- Registry GC: a Redis-locked daily job keeps the newest N image tags per app
  (REGISTRY_KEEP_VERSIONS, default 3) and reclaims disk via garbage-collect.
- Hardening: git tokens are delivered via a per-build Secret + git credential
  store instead of being embedded in the clone URL / Job manifest; build timeout
  is configurable.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

backend/src/kubernetes/kubernetes.module.ts

@@ -3,6 +3,7 @@ import { TypeOrmModule } from '@nestjs/typeorm';
 import { KubernetesService } from './kubernetes.service';
 import { HelmService } from './helm.service';
 import { RegistryService } from './registry.service';
+import { RegistryGcService } from './registry-gc.service';
 import { ElasticsearchService } from './elasticsearch.service';
 import { ElasticsearchController } from './elasticsearch.controller';
 import { LogsController } from './logs.controller';
@@ -13,7 +14,7 @@ import { Deployment } from '../deployments/entities/deployment.entity';
 @Module({
   imports: [forwardRef(() => ClustersModule), TypeOrmModule.forFeature([Application, Deployment])],
   controllers: [ElasticsearchController, LogsController],
-  providers: [KubernetesService, HelmService, RegistryService, ElasticsearchService],
+  providers: [KubernetesService, HelmService, RegistryService, RegistryGcService, ElasticsearchService],
   exports: [KubernetesService, HelmService, RegistryService, ElasticsearchService],
 })
 export class KubernetesModule {}

backend/src/kubernetes/registry-gc.service.spec.ts

@@ -0,0 +1,25 @@
+import { selectTagsToDelete } from './registry-gc.util';
+
+describe('selectTagsToDelete', () => {
+  it('keeps the N newest numeric (Date.now) tags and deletes the rest', () => {
+    const tags = ['1000', '3000', '2000', '5000', '4000'];
+    const toDelete = selectTagsToDelete(tags, 3);
+    // newest 3 = 5000,4000,3000 → delete 2000,1000
+    expect(toDelete.sort()).toEqual(['1000', '2000']);
+  });
+
+  it('deletes nothing when tag count is within the keep limit', () => {
+    expect(selectTagsToDelete(['1000', '2000'], 3)).toEqual([]);
+    expect(selectTagsToDelete([], 3)).toEqual([]);
+  });
+
+  it('treats non-numeric tags as oldest (eligible for deletion first)', () => {
+    const tags = ['latest', '2000', '1000'];
+    // numeric newest kept first: 2000,1000 kept (keep=2) → delete latest
+    expect(selectTagsToDelete(tags, 2)).toEqual(['latest']);
+  });
+
+  it('keep=0 deletes every tag', () => {
+    expect(selectTagsToDelete(['1000', '2000'], 0).sort()).toEqual(['1000', '2000']);
+  });
+});

backend/src/kubernetes/registry-gc.service.ts

@@ -0,0 +1,164 @@
+import { Injectable, Logger, OnModuleInit, OnModuleDestroy, Inject } from '@nestjs/common';
+import { ConfigService } from '@nestjs/config';
+import { Writable } from 'stream';
+import * as k8s from '@kubernetes/client-node';
+import { Redis } from 'ioredis';
+import { REDIS_CLIENT } from '../common/redis/redis.module';
+import { ClustersService } from '../clusters/clusters.service';
+import { RegistryService } from './registry.service';
+import { selectTagsToDelete } from './registry-gc.util';
+
+/**
+ * Periodic registry garbage collection: prunes each app image repo to the N most
+ * recent tags (deleting older manifests) and then reclaims disk by running
+ * `registry garbage-collect` in the registry pod. Runs on the local/default
+ * cluster's in-cluster registry (the one reachable via cluster DNS). A Redis lock
+ * keeps a single replica running it at a time (see [[multi-instance-interval-jobs]]).
+ */
+@Injectable()
+export class RegistryGcService implements OnModuleInit, OnModuleDestroy {
+  private readonly logger = new Logger(RegistryGcService.name);
+  private timer?: NodeJS.Timeout;
+  private static readonly LOCK_KEY = 'registry:gc:lock';
+
+  constructor(
+    private readonly configService: ConfigService,
+    private readonly clustersService: ClustersService,
+    private readonly registryService: RegistryService,
+    @Inject(REDIS_CLIENT) private readonly redis: Redis,
+  ) {}
+
+  onModuleInit(): void {
+    if (this.configService.get<boolean>('build.registryGcEnabled') === false) return;
+    const interval = this.configService.get<number>('build.registryGcIntervalMs') || 86_400_000;
+    this.timer = setInterval(() => void this.runGc(), interval);
+    // First pass shortly after boot.
+    setTimeout(() => void this.runGc(), 60_000);
+    this.logger.log(`Registry GC scheduled — interval: ${Math.round(interval / 3600000)}h`);
+  }
+
+  onModuleDestroy(): void {
+    if (this.timer) clearInterval(this.timer);
+  }
+
+  /** Run one GC pass guarded by a Redis lock so only one replica executes it. */
+  async runGc(): Promise<void> {
+    const keep = this.configService.get<number>('build.registryKeepVersions') || 3;
+    const locked = await this.redis.set(RegistryGcService.LOCK_KEY, '1', 'EX', 900, 'NX').catch(() => null);
+    if (locked !== 'OK') {
+      this.logger.debug('Registry GC already running on another replica — skipping');
+      return;
+    }
+    try {
+      await this.pruneDefaultClusterRegistry(keep);
+    } catch (e: any) {
+      this.logger.warn(`Registry GC failed: ${e.message}`);
+    } finally {
+      await this.redis.del(RegistryGcService.LOCK_KEY).catch(() => undefined);
+    }
+  }
+
+  private authHeader(): string {
+    const { username, password } = this.registryService.getRegistryCredentials();
+    return 'Basic ' + Buffer.from(`${username}:${password}`).toString('base64');
+  }
+
+  private async registryFetch(pathOrUrl: string, init: RequestInit = {}): Promise<Response> {
+    const base = `http://${this.registryService.getRegistryUrl()}`;
+    return fetch(`${base}${pathOrUrl}`, {
+      ...init,
+      headers: { Authorization: this.authHeader(), ...(init.headers || {}) },
+    });
+  }
+
+  private async pruneDefaultClusterRegistry(keep: number): Promise<void> {
+    // 1. List repositories
+    const catalogRes = await this.registryFetch('/v2/_catalog?n=10000');
+    if (!catalogRes.ok) throw new Error(`catalog ${catalogRes.status}`);
+    const repositories: string[] = ((await catalogRes.json()) as any)?.repositories || [];
+
+    let deletedTotal = 0;
+    for (const repo of repositories) {
+      // Skip Kaniko cache repos — pruning them just slows the next build.
+      if (repo.endsWith('/cache')) continue;
+      deletedTotal += await this.pruneRepo(repo, keep);
+    }
+
+    if (deletedTotal > 0) {
+      this.logger.log(`Registry GC: deleted ${deletedTotal} old manifest(s); reclaiming disk…`);
+      await this.runGarbageCollect();
+    } else {
+      this.logger.debug('Registry GC: nothing to prune');
+    }
+  }
+
+  /** Delete all but the newest `keep` tags of one repo. Returns count deleted. */
+  private async pruneRepo(repo: string, keep: number): Promise<number> {
+    const tagsRes = await this.registryFetch(`/v2/${repo}/tags/list`);
+    if (!tagsRes.ok) return 0;
+    const tags: string[] = ((await tagsRes.json()) as any)?.tags || [];
+    const toDelete = selectTagsToDelete(tags, keep);
+    if (toDelete.length === 0) return 0;
+
+    const seenDigests = new Set<string>();
+    let deleted = 0;
+    for (const tag of toDelete) {
+      try {
+        const head = await this.registryFetch(`/v2/${repo}/manifests/${tag}`, {
+          method: 'GET',
+          headers: {
+            Accept: 'application/vnd.docker.distribution.manifest.v2+json, application/vnd.oci.image.manifest.v1+json',
+          },
+        });
+        const digest = head.headers.get('docker-content-digest');
+        if (!digest || seenDigests.has(digest)) continue; // multiple tags can share a digest
+        seenDigests.add(digest);
+        const del = await this.registryFetch(`/v2/${repo}/manifests/${digest}`, { method: 'DELETE' });
+        if (del.ok || del.status === 202) deleted++;
+      } catch (e: any) {
+        this.logger.warn(`Failed to delete ${repo}:${tag}: ${e.message}`);
+      }
+    }
+    if (deleted > 0) this.logger.log(`Registry GC: pruned ${deleted} tag(s) from ${repo} (kept ${keep})`);
+    return deleted;
+  }
+
+  /** Reclaim disk by running `registry garbage-collect` inside the registry pod. Best-effort. */
+  private async runGarbageCollect(): Promise<void> {
+    try {
+      const buildNs = this.registryService.getBuildNamespace();
+      const cluster = await this.clustersService.getDefault();
+      const kc = new k8s.KubeConfig();
+      kc.loadFromString(cluster.kubeconfig);
+      const coreApi = kc.makeApiClient(k8s.CoreV1Api);
+
+      const pods = await coreApi.listNamespacedPod({ namespace: buildNs, labelSelector: 'app=registry' });
+      const podName = pods.items[0]?.metadata?.name;
+      if (!podName) {
+        this.logger.warn('Registry GC: no registry pod found for garbage-collect');
+        return;
+      }
+
+      const exec = new k8s.Exec(kc);
+      const sink = new Writable({ write: (_c, _e, cb) => cb() });
+      await new Promise<void>((resolve, reject) => {
+        exec
+          .exec(
+            buildNs,
+            podName,
+            'registry',
+            ['/bin/registry', 'garbage-collect', '/etc/docker/registry/config.yml'],
+            sink,
+            sink,
+            null,
+            false,
+            (status) => (status.status === 'Failure' ? reject(new Error(status.message)) : resolve()),
+          )
+          .catch(reject);
+      });
+      this.logger.log('Registry GC: garbage-collect completed');
+    } catch (e: any) {
+      this.logger.warn(`Registry garbage-collect failed (manifests already deleted): ${e.message}`);
+    }
+  }
+}

backend/src/kubernetes/registry-gc.util.ts

@@ -0,0 +1,19 @@
+/**
+ * Decide which image tags to delete, keeping the `keep` most recent. Tags are
+ * `Date.now()` strings, so newest = highest numeric value; non-numeric tags sort
+ * last (treated as oldest) and are eligible for deletion once `keep` is met.
+ *
+ * Kept in its own (dependency-free) module so it can be unit-tested without
+ * pulling in the ESM `@kubernetes/client-node` that the GC service imports.
+ */
+export function selectTagsToDelete(tags: string[], keep: number): string[] {
+  const sorted = [...tags].sort((a, b) => {
+    const na = Number(a);
+    const nb = Number(b);
+    if (Number.isNaN(na) && Number.isNaN(nb)) return a < b ? 1 : -1;
+    if (Number.isNaN(na)) return 1;
+    if (Number.isNaN(nb)) return -1;
+    return nb - na; // newest first
+  });
+  return sorted.slice(Math.max(0, keep));
+}