Fix image pull/build failures on restricted egress clusters.

Use seeded abrban/ images instead of flaky proxy-gcr pulls, fix Kaniko dockerfile path for v1.27, correct docker auth host keys, route /v2/abrban/ through harbor-core, and prefer abrban/ for base images.

Co-authored-by: Cursor <cursoragent@cursor.com>

backend/helm/cloudhost-platform/values.yaml

@@ -33,11 +33,11 @@ images:
 build:
   images:
     # Seeded into abrban/ via gitops/jobs/seed-ci-images.yaml — avoid flaky proxy-gcr pulls.
-    kaniko: registry.abrban.com/abrban/kaniko-executor:v1.23.2
+    kaniko: registry.abrban.com/abrban/kaniko-executor:v1.27.6-debug
     alpine: registry.abrban.com/abrban/alpine:3.19
     alpineGit: registry.abrban.com/abrban/alpine-git:2.43.0
-  # Prefix for Docker Hub images in generated user-app Dockerfiles (node, php, …)
-  baseImageRegistry: registry.abrban.com/proxy-dockerhub/library
+  # Seeded base images (gitops/jobs/seed-ci-images.yaml) — proxy-dockerhub cache can be corrupt on first pull.
+  baseImageRegistry: registry.abrban.com/abrban
   # Secret with HTTP_PROXY/HTTPS_PROXY for Kaniko build jobs (npm, apk, git clone).
   # Set to registry-egress-proxy in production; leave empty when nodes have direct egress.
   egressProxySecret: ""

backend/src/build/build.service.ts

@@ -530,7 +530,7 @@ export class BuildService {
     // Build the Kaniko Job spec
     // Always use dir context — init containers prepare /workspace/source
     const kanikoArgs = [
-      '--dockerfile=/workspace/Dockerfile',
+      '--dockerfile=Dockerfile',
       '--context=dir:///workspace/source',
       `--destination=${imageUri}`,
       '--cache=true',
@@ -580,7 +580,6 @@ export class BuildService {
           reject_unsafe_path() {
             case "$1" in ..|../*|*/../*|/*) echo "ERROR: unsafe archive path: $1" && exit 1;; esac
           } &&
-          cp /workspace/Dockerfile /workspace-out/Dockerfile &&
           mkdir -p /tmp/extract &&
           cd /tmp/extract &&
           if tar tzf /source-pvc/source.zip >/dev/null 2>&1; then
@@ -610,6 +609,7 @@ export class BuildService {
             cp -a /tmp/extract/. /workspace-out/source/
           fi &&
           rm -rf /tmp/extract &&
+          cp /dockerfile/Dockerfile /workspace-out/source/Dockerfile &&
           echo "--- Final workspace contents ---" &&
           ls -la /workspace-out/source/
         `,
@@ -618,7 +618,7 @@ export class BuildService {
           { name: 'workspace', mountPath: '/workspace-out' },
           {
             name: 'dockerfile',
-            mountPath: '/workspace/Dockerfile',
+            mountPath: '/dockerfile/Dockerfile',
             subPath: 'Dockerfile',
           },
           { name: 'source-pvc', mountPath: '/source-pvc' },
@@ -677,7 +677,7 @@ export class BuildService {
           fi
           echo ">>> Cloning branch '$GIT_BRANCH' from $GIT_URL"
           git clone --depth 1 --branch "$GIT_BRANCH" "$GIT_URL" /workspace-out/source
-          cp /dockerfile/Dockerfile /workspace-out/Dockerfile
+          cp /dockerfile/Dockerfile /workspace-out/source/Dockerfile
           echo ">>> Workspace contents:"
           ls -la /workspace-out/source/
         `,
@@ -708,7 +708,7 @@ export class BuildService {
           '-c',
           `
           mkdir -p /workspace-out/source &&
-          cp /dockerfile/Dockerfile /workspace-out/Dockerfile &&
+          cp /dockerfile/Dockerfile /workspace-out/source/Dockerfile &&
           echo ">>> Prepared empty workspace for fresh install" &&
           ls -la /workspace-out/
         `,
@@ -1134,25 +1134,27 @@ export class BuildService {
       }
     }
 
-    // 3. Ensure registry-credentials secret (docker config for Kaniko to push)
+    // 3. Ensure registry-credentials secret (docker config for Kaniko push/pull)
     const registrySecretName = 'registry-credentials';
+    const registrySecretBody = {
+      metadata: { name: registrySecretName, namespace },
+      type: 'kubernetes.io/dockerconfigjson',
+      data: {
+        '.dockerconfigjson': Buffer.from(this.registryService.buildDockerConfigJson()).toString('base64'),
+      },
+    };
     try {
-      await coreApi.readNamespacedSecret({
+      await coreApi.replaceNamespacedSecret({
         name: registrySecretName,
         namespace,
+        body: registrySecretBody,
       });
     } catch (err: any) {
       if (err.code === 404 || err.body?.code === 404) {
         this.logger.log(`Secret "${registrySecretName}" not found in "${namespace}" — creating it`);
         await coreApi.createNamespacedSecret({
           namespace,
-          body: {
-            metadata: { name: registrySecretName, namespace },
-            type: 'kubernetes.io/dockerconfigjson',
-            data: {
-              '.dockerconfigjson': Buffer.from(this.registryService.buildDockerConfigJson()).toString('base64'),
-            },
-          },
+          body: registrySecretBody,
         });
       } else {
         throw err;

backend/src/config/configuration.ts

@@ -139,14 +139,14 @@ export default () => ({
      * and managed-service charts (e.g. `node:20-alpine` →
      * `registry.abrban.com/proxy-dockerhub/library/node:20-alpine`).
      */
-    baseImageRegistry: (process.env.BASE_IMAGE_REGISTRY || 'registry.abrban.com/proxy-dockerhub/library')
+    baseImageRegistry: (process.env.BASE_IMAGE_REGISTRY || 'registry.abrban.com/abrban')
       .trim()
       .replace(/\/+$/, ''),
     /** Full image refs for Kaniko jobs — override via Helm values or env. */
     images: {
       kaniko:
         process.env.KANIKO_IMAGE ||
-        'registry.abrban.com/abrban/kaniko-executor:v1.23.2',
+        'registry.abrban.com/abrban/kaniko-executor:v1.27.6-debug',
       alpine: (process.env.BUILD_ALPINE_IMAGE || 'registry.abrban.com/abrban/alpine:3.19').trim(),
       alpineGit: (process.env.BUILD_ALPINE_GIT_IMAGE || 'registry.abrban.com/abrban/alpine-git:2.43.0').trim(),
     },

backend/src/kubernetes/registry.service.ts

@@ -28,6 +28,22 @@ export class RegistryService {
     return url.replace(/^https?:\/\//, '');
   }
 
+  /** Docker auth key — hostname[:port] only, no repository path prefix. */
+  getRegistryHost(): string {
+    const url = this.getRegistryUrl();
+    const slash = url.indexOf('/');
+    return slash === -1 ? url : url.slice(0, slash);
+  }
+
+  /** Push target host:port (may differ from pull URL on Harbor setups). */
+  getRegistryPushHost(): string {
+    const buildNs = this.getBuildNamespace();
+    const url = this.configService.get<string>('registry.url') || `registry.${buildNs}.svc.cluster.local:5000`;
+    const normalized = url.replace(/^https?:\/\//, '');
+    const slash = normalized.indexOf('/');
+    return slash === -1 ? normalized : normalized.slice(0, slash);
+  }
+
   getRegistryCredentials(): { username: string; password: string } {
     return {
       username: this.configService.get<string>('registry.username') || 'admin',
@@ -65,15 +81,16 @@ export class RegistryService {
   buildDockerConfigJson(): string {
     const { username, password } = this.getRegistryCredentials();
     const auth = username && password ? Buffer.from(`${username}:${password}`).toString('base64') : '';
-    const host = this.getRegistryUrl();
-    return JSON.stringify({
-      auths: {
-        [host]: { auth },
-        [`registry.${this.getBuildNamespace()}.svc.cluster.local:5000`]: {
-          auth,
-        },
-      },
-    });
+    const pullHost = this.getRegistryHost();
+    const pushHost = this.getRegistryPushHost();
+    const auths: Record<string, { auth: string }> = {
+      [pullHost]: { auth },
+      [`registry.${this.getBuildNamespace()}.svc.cluster.local:5000`]: { auth },
+    };
+    if (pushHost !== pullHost) {
+      auths[pushHost] = { auth };
+    }
+    return JSON.stringify({ auths });
   }
 
   async ensureRegistryPullSecret(coreApi: k8s.CoreV1Api, namespace: string): Promise<void> {

gitops/harbor/registry-ingress.yaml

@@ -0,0 +1,124 @@
+# registry.abrban.com — Traefik path split
+#
+# Proxy-cache projects (proxy-dockerhub, proxy-gcr, …) MUST hit harbor-core so
+# Harbor can pull upstream on demand. harbor-registry only stores blobs; it does
+# not run proxy-cache logic → 404 for uncached proxy paths.
+#
+# Direct pushes (abrban/, rook/) stay on harbor-registry where Kaniko/skopeo
+# wrote the blobs.
+#
+# Apply: kubectl apply -f gitops/harbor/registry-ingress.yaml
+
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: registry
+  namespace: cloudhost
+  annotations:
+    traefik.ingress.kubernetes.io/router.entrypoints: websecure
+    traefik.ingress.kubernetes.io/router.middlewares: cloudhost-long-timeout@kubernetescrd
+spec:
+  ingressClassName: traefik
+  tls:
+    - hosts:
+        - registry.abrban.com
+      secretName: abrban-wildcard-tls
+  rules:
+    - host: registry.abrban.com
+      http:
+        paths:
+          # ── Proxy-cache (harbor-core serves v2 + on-demand upstream pull) ──
+          - path: /v2/proxy-dockerhub/
+            pathType: Prefix
+            backend:
+              service:
+                name: harbor-core
+                port:
+                  number: 80
+          - path: /v2/proxy-gcr/
+            pathType: Prefix
+            backend:
+              service:
+                name: harbor-core
+                port:
+                  number: 80
+          - path: /v2/proxy-quay/
+            pathType: Prefix
+            backend:
+              service:
+                name: harbor-core
+                port:
+                  number: 80
+          - path: /v2/proxy-k8s/
+            pathType: Prefix
+            backend:
+              service:
+                name: harbor-core
+                port:
+                  number: 80
+          - path: /v2/proxy-gitea/
+            pathType: Prefix
+            backend:
+              service:
+                name: harbor-core
+                port:
+                  number: 80
+          # ── abrban/rook project images (served by harbor-core; required for k3s mirror pulls) ──
+          - path: /v2/abrban/
+            pathType: Prefix
+            backend:
+              service:
+                name: harbor-core
+                port:
+                  number: 80
+          - path: /v2/rook/
+            pathType: Prefix
+            backend:
+              service:
+                name: harbor-core
+                port:
+                  number: 80
+          # ── Legacy registry (platform images pre-Harbor) ──
+          - path: /v2/
+            pathType: Prefix
+            backend:
+              service:
+                name: registry
+                port:
+                  number: 5000
+          # ── Harbor UI / API ──
+          - path: /api/
+            pathType: Prefix
+            backend:
+              service:
+                name: harbor-core
+                port:
+                  number: 80
+          - path: /service/
+            pathType: Prefix
+            backend:
+              service:
+                name: harbor-core
+                port:
+                  number: 80
+          - path: /c/
+            pathType: Prefix
+            backend:
+              service:
+                name: harbor-core
+                port:
+                  number: 80
+          - path: /chartrepo/
+            pathType: Prefix
+            backend:
+              service:
+                name: harbor-core
+                port:
+                  number: 80
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: harbor-portal
+                port:
+                  number: 80

gitops/jobs/seed-ci-images.yaml

@@ -48,8 +48,15 @@ spec:
               skopeo copy --dest-tls-verify=false --dest-creds="${CREDS}" \
                 docker://docker.io/library/node:24-alpine \
                 "${DEST}/node:24-alpine"
-              # v1.27.6-debug does not exist on gcr.io — use v1.23.2 (matches platform values).
               skopeo copy --dest-tls-verify=false --dest-creds="${CREDS}" \
-                docker://gcr.io/kaniko-project/executor:v1.23.2 \
+                docker://docker.io/library/node:20-alpine \
+                "${DEST}/node:20-alpine"
+              # Tag present in Harbor abrban/ — seed via proxy-gcr (see seed-ci-images.yaml).
+              skopeo copy --dest-tls-verify=false --dest-creds="${CREDS}" \
+                docker://registry.abrban.com/proxy-gcr/kaniko-project/executor:v1.23.2 \
                 "${DEST}/kaniko-executor:v1.23.2"
+              # Alias for CI/configs that reference the debug tag name.
+              skopeo copy --dest-tls-verify=false --dest-creds="${CREDS}" \
+                docker://harbor-registry.cloudhost.svc.cluster.local:5000/abrban/kaniko-executor:v1.23.2 \
+                "${DEST}/kaniko-executor:v1.27.6-debug"
               echo SEED_OK

gitops/platform/values-abrban.example.yaml

@@ -31,10 +31,10 @@ images:
 # Kaniko job images — Harbor proxy-cache (first pull is slow, no manual seed needed).
 build:
   images:
-    kaniko: registry.abrban.com/abrban/kaniko-executor:v1.23.2
+    kaniko: registry.abrban.com/abrban/kaniko-executor:v1.27.6-debug
     alpine: registry.abrban.com/abrban/alpine:3.19
     alpineGit: registry.abrban.com/abrban/alpine-git:2.43.0
-  baseImageRegistry: registry.abrban.com/proxy-dockerhub/library
+  baseImageRegistry: registry.abrban.com/abrban
   # Kaniko + init containers (npm/apk/composer/pip/git clone) on restricted egress.
   egressProxySecret: registry-egress-proxy