Serve preview URLs over Traefik+TLS, stabilize them, and speed up builds.

Ingress / preview URLs:
- Default the app Ingress class and ACME HTTP-01 solver to Traefik
  (k3s default) via a new INGRESS_CLASS env, instead of hardcoding nginx —
  fixes 404s on clusters without ingress-nginx.
- Only put public, real-TLD hosts (custom domain + preview) in the TLS
  block; the internal *.apps.cloudhost.local host no longer poisons the
  Let's Encrypt order, so certs actually issue.
- Make the per-app preview number stable across redeploys so URLs stop
  breaking, and let PREVIEW_BASE_DOMAIN configure the base domain.

Registry pulls:
- Point the k3s registries.yaml mirror endpoint at the registry NodePort on
  loopback so node containerd never depends on cluster DNS (image pulls
  survive node restarts).

Builds:
- Pin the Kaniko image, use IfNotPresent pull policy, drop the dead build
  queue/processor, and retry transient Kubernetes API errors while polling
  build jobs.

Logs & apps list:
- fluent-bit reads log files from head so startup output reaches
  Elasticsearch.
- Order joined deployments newest-first so the apps list shows the latest
  deployment status.

Allocation:
- Reserve in-flight (pending/building) capacity and stop globally degrading
  the cluster on a single allocation failure, so concurrent deploys don't
  starve or wrongly report "no healthy cluster".

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

backend/src/build/build.module.ts

@@ -1,17 +1,14 @@
 import { Module, forwardRef } from '@nestjs/common';
-import { BullModule } from '@nestjs/bull';
 import { BuildService } from './build.service';
-import { BuildProcessor } from './build.processor';
 import { KubernetesModule } from '../kubernetes/kubernetes.module';
 import { ClustersModule } from '../clusters/clusters.module';
 
 @Module({
   imports: [
-    BullModule.registerQueue({ name: 'build' }),
     forwardRef(() => KubernetesModule),
     ClustersModule,
   ],
-  providers: [BuildService, BuildProcessor],
+  providers: [BuildService],
   exports: [BuildService],
 })
 export class BuildModule {}

backend/src/build/build.processor.ts

@@ -1,42 +0,0 @@
-import { Process, Processor } from '@nestjs/bull';
-import { Logger } from '@nestjs/common';
-import { Job } from 'bull';
-import { BuildService } from './build.service';
-
-export interface BuildJobData {
-  applicationId: string;
-  deploymentId: string;
-  appName: string;
-  runtime: string;
-  gitUrl?: string;
-  codePath?: string;
-}
-
-@Processor('build')
-export class BuildProcessor {
-  private readonly logger = new Logger(BuildProcessor.name);
-
-  constructor(private buildService: BuildService) {}
-
-  @Process('build-image')
-  async handleBuild(job: Job<BuildJobData>) {
-    this.logger.log(`Processing build job ${job.id} for app: ${job.data.appName}`);
-
-    try {
-      await job.progress(10);
-
-      // The actual build logic is in BuildService
-      // This processor handles the queue job lifecycle
-      this.logger.log(`Build job ${job.id} started for ${job.data.appName}`);
-
-      await job.progress(50);
-      await job.progress(100);
-
-      this.logger.log(`Build job ${job.id} completed for ${job.data.appName}`);
-      return { status: 'completed', appName: job.data.appName };
-    } catch (error: any) {
-      this.logger.error(`Build job ${job.id} failed: ${error.message}`);
-      throw error;
-    }
-  }
-}

backend/src/build/build.service.ts

@@ -45,6 +45,12 @@ export class BuildService {
   private readonly logger = new Logger(BuildService.name);
   private readonly progressMap = new Map<string, BuildProgress>();
   private readonly activeBuilds = new Map<string, ActiveBuildSession>();
+  /**
+   * Kaniko executor image. Pinned (not `:latest`) so it can be cached on the node
+   * with imagePullPolicy=IfNotPresent — avoids re-pulling the ~250MB image on every build.
+   */
+  private readonly kanikoImage =
+    process.env.KANIKO_IMAGE || 'gcr.io/kaniko-project/executor:v1.23.2';
 
   constructor(
     private configService: ConfigService,
@@ -332,6 +338,7 @@ export class BuildService {
       initContainers.push({
         name: 'unzip-source',
         image: 'alpine:3.19',
+        imagePullPolicy: 'IfNotPresent',
         command: ['sh', '-c', `
           apk add --no-cache unzip tar gzip &&
           cp /workspace/Dockerfile /workspace-out/Dockerfile &&
@@ -390,6 +397,7 @@ export class BuildService {
       initContainers.push({
         name: 'git-clone',
         image: 'alpine/git:2.43.0',
+        imagePullPolicy: 'IfNotPresent',
         command: ['sh', '-c', `
           echo ">>> Cloning branch '${branch}' from ${app.gitUrl}" &&
           git clone --depth 1 --branch ${branch} ${cloneUrl} /workspace-out/source &&
@@ -417,6 +425,7 @@ export class BuildService {
       initContainers.push({
         name: 'prepare-workspace',
         image: 'alpine:3.19',
+        imagePullPolicy: 'IfNotPresent',
         command: ['sh', '-c', `
           mkdir -p /workspace-out/source &&
           cp /dockerfile/Dockerfile /workspace-out/Dockerfile &&
@@ -447,7 +456,8 @@ export class BuildService {
             containers: [
               {
                 name: 'kaniko',
-                image: 'gcr.io/kaniko-project/executor:latest',
+                image: this.kanikoImage,
+                imagePullPolicy: 'IfNotPresent',
                 args: kanikoArgs,
                 volumeMounts: kanikoVolumeMounts,
                 resources: {
@@ -519,17 +529,6 @@ export class BuildService {
     }
   }
 
-  /**
-   * Poll the helper pod until netcat finishes writing and reports the expected byte count.
-   */
-  private async waitForRemoteUploadDone(
-    kubeconfig: string,
-    namespace: string,
-    podName: string,
-    expectedSize: number,
-    deploymentId?: string,
-  ): Promise<void> { /* unused — kept for API compat */ }
-
   /**
    * Upload a local file to the helper pod using kubectl cp with progress tracking.
    * kubectl cp uses tar over the k8s exec API — reliable for any file size.
@@ -668,6 +667,7 @@ export class BuildService {
         containers: [{
           name: 'helper',
           image: 'alpine:3.19',
+          imagePullPolicy: 'IfNotPresent',
           command: ['sh', '-c', 'sleep 3600'],
           volumeMounts: [{ name: 'source', mountPath: '/data' }],
           resources: {
@@ -1467,6 +1467,30 @@ CMD ["sh", "-c", "DLL=$(find . -maxdepth 1 -name '*.dll' ! -name '*.deps.dll' !
 `;
   }
 
+  /**
+   * Whether a K8s API error is a transient connectivity/availability blip that
+   * should be retried rather than failing the operation. Covers socket-level
+   * errors, request timeouts (incl. the apiserver "request did not complete
+   * within the allotted timeout" message), DNS hiccups and 5xx/429 responses.
+   */
+  private isTransientK8sError(err: any): boolean {
+    const statusCode = err?.statusCode ?? err?.response?.statusCode ?? err?.body?.code;
+    if (typeof statusCode === 'number' && (statusCode >= 500 || statusCode === 429)) {
+      return true;
+    }
+    const haystack = [
+      err?.code,
+      err?.message,
+      err?.body?.message,
+      err?.cause?.code,
+    ]
+      .filter(Boolean)
+      .join(' ');
+    return /ECONNRESET|ECONNREFUSED|ETIMEDOUT|ESOCKETTIMEDOUT|EPIPE|EAI_AGAIN|ENOTFOUND|ENETUNREACH|socket hang up|timed? ?out|allotted timeout|did not complete|Client network socket disconnected/i.test(
+      haystack,
+    );
+  }
+
   private async waitForJobCompletion(
     batchApi: k8s.BatchV1Api,
     coreApi: k8s.CoreV1Api,
@@ -1493,9 +1517,12 @@ CMD ["sh", "-c", "DLL=$(find . -maxdepth 1 -name '*.dll' ! -name '*.deps.dll' !
       try {
         job = await batchApi.readNamespacedJob(jobName, namespace);
       } catch (pollErr: any) {
-        const code = pollErr?.code || pollErr?.message || '';
-        if (/ECONNRESET|ECONNREFUSED|ETIMEDOUT|socket hang up/i.test(String(code))) {
-          this.logger.warn(`Transient K8s API error polling job ${jobName}: ${code} — retrying in 5s`);
+        // The Kaniko job keeps running independently of these status polls.
+        // A single API blip (timeout, reset, 5xx, DNS) must NOT abort a build
+        // that is still progressing — just retry on the next poll tick.
+        if (this.isTransientK8sError(pollErr)) {
+          const detail = pollErr?.code || pollErr?.message || pollErr?.statusCode || 'unknown';
+          this.logger.warn(`Transient K8s API error polling job ${jobName}: ${detail} — retrying in 5s`);
           await new Promise(r => setTimeout(r, 5000));
           continue;
         }