Fix app image pulls and Harbor kubelet auth for user workloads.

Route k3s registry mirrors through harbor-core ClusterIP with hostname-only auth keys, use HTTP EXT_ENDPOINT so OAuth tokens work on port 80, extend deploy readiness timeout, and harden Kaniko build/dockerfile fallbacks.

Co-authored-by: Cursor <cursoragent@cursor.com>

backend/helm/cloudhost-harbor/values-registry.abrban.com.yaml

@@ -4,7 +4,7 @@
 ## Install:
 ##   helm upgrade --install harbor harbor/harbor -n cloudhost -f backend/helm/cloudhost-harbor/values-registry.abrban.com.yaml
 ##
-externalURL: https://registry.abrban.com
+externalURL: http://registry.abrban.com
 
 proxy:
   # Values are injected by install script from `cloudhost/registry-egress-proxy`.

backend/src/build/build.service.spec.ts

@@ -49,6 +49,41 @@ describe('BuildService', () => {
     service = module.get(BuildService);
   });
 
+  describe('baseImage', () => {
+    it('prefixes Docker Hub library images (tag colon must not block mirroring)', () => {
+      const config = (service as any).configService as { get: jest.Mock };
+      config.get.mockImplementation((key: string) => {
+        if (key === 'build.baseImageRegistry') return 'registry.abrban.com/abrban';
+        return undefined;
+      });
+
+      expect((service as any).baseImage('node:20-alpine')).toBe(
+        'registry.abrban.com/abrban/node:20-alpine',
+      );
+      expect((service as any).baseImage('alpine:3.19')).toBe(
+        'registry.abrban.com/abrban/alpine:3.19',
+      );
+    });
+
+    it('leaves images that already reference an external registry unchanged', () => {
+      const config = (service as any).configService as { get: jest.Mock };
+      config.get.mockImplementation((key: string) => {
+        if (key === 'build.baseImageRegistry') return 'registry.abrban.com/abrban';
+        return undefined;
+      });
+
+      expect((service as any).baseImage('mcr.microsoft.com/dotnet/sdk:8.0')).toBe(
+        'mcr.microsoft.com/dotnet/sdk:8.0',
+      );
+      expect((service as any).baseImage('registry.abrban.com/abrban/node:20-alpine')).toBe(
+        'registry.abrban.com/abrban/node:20-alpine',
+      );
+      expect((service as any).baseImage('localhost:5000/myapp:latest')).toBe(
+        'localhost:5000/myapp:latest',
+      );
+    });
+  });
+
   describe('generateDockerfile', () => {
     it('generates Go Dockerfile with requested runtime version', () => {
       const app = {
@@ -78,6 +113,53 @@ describe('BuildService', () => {
       expect(dockerfile).toContain('go build -a -installsuffix cgo -ldflags="-w -s" -o main ./cmd/server');
     });
 
+    it('generates Node.js Dockerfile with mirrored base images when registry prefix is set', async () => {
+      const module = await Test.createTestingModule({
+        providers: [
+          BuildService,
+          {
+            provide: ConfigService,
+            useValue: {
+              get: jest.fn((key: string) => {
+                const map: Record<string, string> = {
+                  'build.namespace': 'cloudhost-builds',
+                  'build.serviceAccount': 'kaniko-builder',
+                  'build.baseImageRegistry': 'registry.abrban.com/abrban',
+                  'registry.url': 'registry.local:5000',
+                };
+                return map[key];
+              }),
+            },
+          },
+          { provide: ClustersService, useValue: {} },
+          {
+            provide: BuildProgressStore,
+            useValue: { get: jest.fn(), set: jest.fn(), clear: jest.fn() },
+          },
+          { provide: RegistryService, useValue: {} },
+          {
+            provide: SourceStorageService,
+            useValue: {
+              isObjectStorage: () => false,
+              materializeToTempFile: jest.fn(),
+              getSize: jest.fn(),
+            },
+          },
+        ],
+      }).compile();
+
+      const mirrored = module.get(BuildService);
+      const app = {
+        runtime: AppRuntime.NODEJS,
+        runtimeVersion: '20',
+      } as Application;
+
+      const dockerfile = (mirrored as any).generateDockerfile(app) as string;
+
+      expect(dockerfile).toContain('FROM registry.abrban.com/abrban/node:20-alpine');
+      expect(dockerfile).toContain('EXPOSE 3000');
+    });
+
     it('generates Node.js Dockerfile with default port', () => {
       const app = {
         runtime: AppRuntime.NODEJS,

backend/src/build/build.service.ts

@@ -73,9 +73,19 @@ export class BuildService {
   private baseImage(image: string): string {
     const prefix = this.configService.get<string>('build.baseImageRegistry');
     if (!prefix) return image;
-    const firstSegment = image.split('/')[0];
-    const hasRegistry = firstSegment.includes('.') || firstSegment.includes(':');
-    if (hasRegistry) return image;
+
+    // Official Docker Hub library images have no slash (node:20-alpine, alpine:3.19).
+    // Do not treat the tag colon as a registry port — that was skipping the mirror.
+    if (!image.includes('/')) {
+      return `${prefix}/${image}`;
+    }
+
+    const registryHost = image.split('/')[0];
+    if (registryHost.includes('.') || registryHost.includes(':') || registryHost === 'localhost') {
+      return image;
+    }
+
+    // Docker Hub org/user image (e.g. bitnami/redis:7) — mirror through the prefix.
     return `${prefix}/${image}`;
   }
 
@@ -1195,8 +1205,8 @@ export class BuildService {
 FROM ${this.baseImage(`node:${nodeVersion}-alpine`)} AS builder
 WORKDIR /app
 COPY package*.json ./
-# Reproducible install from the lockfile when present
-RUN if [ -f package-lock.json ]; then npm ci --legacy-peer-deps; else npm install --legacy-peer-deps; fi \\
+# Prefer lockfile; fall back when package.json and lockfile are out of sync
+RUN if [ -f package-lock.json ]; then npm ci --legacy-peer-deps || npm install --legacy-peer-deps; else npm install --legacy-peer-deps; fi \\
     && npm cache clean --force
 COPY . .
 

backend/src/clusters/clusters.service.ts

@@ -870,7 +870,7 @@ export class ClustersService implements OnModuleInit, OnModuleDestroy {
 
     const buildNs = this.registryService.getBuildNamespace();
     const saName = this.configService.get<string>('build.serviceAccount') || 'kaniko-builder';
-    const registryUrl = this.registryService.getRegistryUrl();
+    const registryHost = this.registryService.getRegistryHost();
 
     this.logger.log(`Bootstrapping cluster — namespace: ${buildNs}`);
 
@@ -1100,13 +1100,17 @@ export class ClustersService implements OnModuleInit, OnModuleDestroy {
       }
     }
 
-    await this.ensureK3sRegistryMirrors(appsApi, registryUrl);
+    await this.ensureK3sRegistryMirrors(coreApi, appsApi, registryHost);
 
-    this.logger.log(`✅ Cluster bootstrap complete — registry: ${registryUrl}`);
+    this.logger.log(`✅ Cluster bootstrap complete — registry: ${registryHost}`);
   }
 
   /** In-cluster registry mirror for k3s/containerd (HTTP). Removes legacy external-registry DaemonSet if present. */
-  private async ensureK3sRegistryMirrors(appsApi: k8s.AppsV1Api, registryUrl: string): Promise<void> {
+  private async ensureK3sRegistryMirrors(
+    coreApi: k8s.CoreV1Api,
+    appsApi: k8s.AppsV1Api,
+    registryHost: string,
+  ): Promise<void> {
     const namespace = 'kube-system';
     const legacyDs = 'cloudhost-k3s-registry-config';
     try {
@@ -1120,28 +1124,25 @@ export class ClustersService implements OnModuleInit, OnModuleDestroy {
 
     const { username, password } = this.registryService.getRegistryCredentials();
     const dsName = 'cloudhost-k3s-registry-mirrors';
-    // The mirror endpoint must be reachable by the node's containerd, which does
-    // NOT use cluster DNS — so we point it at the registry NodePort on loopback
-    // (http://127.0.0.1:<nodePort>) instead of the in-cluster service DNS name.
-    // Otherwise image pulls break whenever node-level resolution of
-    // *.svc.cluster.local is unavailable (e.g. right after a node restart).
-    const registryNodePort = 30500;
-    const nodePortHost = `127.0.0.1:${registryNodePort}`;
+    // containerd on the node does not use cluster DNS — mirror via ClusterIP (Harbor)
+    // or loopback NodePort (legacy in-cluster registry).
+    const mirrorEndpoint = await this.resolveK3sRegistryMirrorEndpoint(coreApi);
+    const mirrorHost = mirrorEndpoint.replace(/^https?:\/\//, '');
     const configureScript = [
       'set -e',
       'REG=/host/etc/rancher/k3s/registries.yaml',
       'mkdir -p /host/etc/rancher/k3s',
       'cat > /tmp/cloudhost-registries.yaml <<EOFREG',
       'mirrors:',
-      `  "${registryUrl}":`,
+      `  "${registryHost}":`,
       '    endpoint:',
-      `      - "http://${nodePortHost}"`,
+      `      - "${mirrorEndpoint}"`,
       'configs:',
-      `  "${registryUrl}":`,
+      `  "${registryHost}":`,
       '    auth:',
       `      username: ${JSON.stringify(username)}`,
       `      password: ${JSON.stringify(password)}`,
-      `  "${nodePortHost}":`,
+      `  "${mirrorHost}":`,
       '    auth:',
       `      username: ${JSON.stringify(username)}`,
       `      password: ${JSON.stringify(password)}`,
@@ -1206,6 +1207,36 @@ export class ClustersService implements OnModuleInit, OnModuleDestroy {
     }
   }
 
+  /**
+   * containerd on nodes cannot resolve *.svc.cluster.local — use ClusterIP for
+   * Harbor (harbor-core HTTP) or legacy registry NodePort on loopback.
+   */
+  private async resolveK3sRegistryMirrorEndpoint(coreApi: k8s.CoreV1Api): Promise<string> {
+    const pushUrl = this.registryService.getRegistryPushUrl();
+    const platformNs = this.configService.get<string>('platform.namespace') || 'cloudhost';
+    const harborCoreService = this.configService.get<string>('registry.harborCoreService') || 'harbor-core';
+
+    if (pushUrl.includes('harbor-registry')) {
+      try {
+        const svc = await coreApi.readNamespacedService({
+          name: harborCoreService,
+          namespace: platformNs,
+        });
+        const clusterIp = svc.spec?.clusterIP;
+        if (clusterIp) {
+          return `http://${clusterIp}`;
+        }
+      } catch (err: any) {
+        this.logger.warn(
+          `Could not resolve ${harborCoreService} ClusterIP for k3s mirror: ${err.message}`,
+        );
+      }
+    }
+
+    const registryNodePort = 30500;
+    return `http://127.0.0.1:${registryNodePort}`;
+  }
+
   private parseCpuToMillicores(cpu: string): number {
     if (!cpu || cpu === '0') return 0;
     if (cpu.endsWith('n')) return parseFloat(cpu) / 1_000_000;

backend/src/deployments/deployments.service.ts

@@ -182,7 +182,7 @@ export class DeploymentsService implements OnModuleInit {
       });
       await this.kubernetesService.waitForApplicationReady(
         app,
-        600_000,
+        1_200_000,
         () => this.isDeploymentCancelled(deploymentId),
       );
 
@@ -295,7 +295,7 @@ export class DeploymentsService implements OnModuleInit {
       });
       await this.kubernetesService.waitForApplicationReady(
         app,
-        600_000,
+        1_200_000,
         () => this.isDeploymentCancelled(deploymentId),
       );