Push Kaniko artifacts via harbor-core for Harbor UI visibility.

Add REGISTRY_PUSH_URL config, route CI Kaniko to harbor-core, and document dual-host kaniko auth for core push plus registry base-image pull.

Co-authored-by: Cursor <cursoragent@cursor.com>

.gitea/workflows/build-deploy.yaml

@@ -14,8 +14,8 @@ concurrency:
 env:
   # PULL_REGISTRY: kubelet pulls via k3s mirror → harbor-core (matches registry-pull-secret)
   PULL_REGISTRY: registry.abrban.com
-  # PUSH_REGISTRY: kaniko pushes directly to harbor-registry (internal, no TLS)
+  # PUSH_REGISTRY: kaniko pushes via harbor-core (Harbor UI metadata + blob storage)
-  PUSH_REGISTRY: harbor-registry.cloudhost.svc.cluster.local:5000
+  PUSH_REGISTRY: harbor-core.cloudhost.svc.cluster.local
   PROJECT: abrban
   BUILD_NS: cloudhost-builds
   GITEA_HOST: gitea-http.gitea.svc.cluster.local:3000
@@ -159,7 +159,7 @@ jobs:
                         mountPath: /workspace
                 containers:
                   - name: kaniko
-                    image: ${PULL_REGISTRY}/${PROJECT}/kaniko-executor:v1.27.6-debug
+                    image: ${PULL_REGISTRY}/${PROJECT}/kaniko-executor:v1.23.2
                     # Base image (node:24-alpine) is seeded in Harbor abrban/ — avoids
                     # flaky direct pulls from docker.io through the egress proxy.
                     envFrom:
@@ -224,7 +224,7 @@ jobs:
                         mountPath: /workspace
                 containers:
                   - name: kaniko
-                    image: ${PULL_REGISTRY}/${PROJECT}/kaniko-executor:v1.27.6-debug
+                    image: ${PULL_REGISTRY}/${PROJECT}/kaniko-executor:v1.23.2
                     envFrom:
                       - secretRef:
                           name: registry-egress-proxy

backend/helm/cloudhost-harbor/values-registry.abrban.com.yaml

@@ -4,7 +4,7 @@
 ## Install:
 ##   helm upgrade --install harbor harbor/harbor -n cloudhost -f backend/helm/cloudhost-harbor/values-registry.abrban.com.yaml
 ##
-externalURL: https://registry.abrban.com
+externalURL: http://registry.abrban.com
 
 proxy:
   # Values are injected by install script from `cloudhost/registry-egress-proxy`.

backend/helm/cloudhost-platform/templates/backend-deployment.yaml

@@ -125,6 +125,15 @@ spec:
                   name: {{ include "cloudhost-platform.secretName" . }}
                   key: mizbansms-password
             {{- end }}
+            {{- if .Values.registry.credentialsSecret }}
+            - name: REGISTRY_USERNAME
+              value: {{ .Values.registry.username | default "harbor_registry_user" | quote }}
+            - name: REGISTRY_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.registry.credentialsSecret | quote }}
+                  key: {{ .Values.registry.credentialsPasswordKey | default "REGISTRY_CREDENTIAL_PASSWORD" | quote }}
+            {{- end }}
             {{- include "cloudhost-platform.buildEnv" . | nindent 12 }}
             {{- range $key, $val := .Values.backend.env }}
             - name: {{ $key }}

backend/helm/cloudhost-platform/values.yaml

@@ -28,15 +28,22 @@ images:
     tag: "1.0.0"
     pullPolicy: IfNotPresent
 
+# Kaniko push credentials — harbor_registry_user for harbor-registry:5000 (Harbor production).
+registry:
+  credentialsSecret: ""
+  credentialsPasswordKey: REGISTRY_CREDENTIAL_PASSWORD
+  username: harbor_registry_user
+
 # Kaniko job images — defaults pull from Harbor proxy-cache.
 # Override any line for a different registry/tag.
 build:
   images:
-    kaniko: registry.abrban.com/proxy-gcr/kaniko-project/executor:v1.23.2
+    # Seeded into abrban/ via gitops/jobs/seed-ci-images.yaml — avoid flaky proxy-gcr pulls.
-    alpine: registry.abrban.com/proxy-dockerhub/library/alpine:3.19
+    kaniko: registry.abrban.com/abrban/kaniko-executor:v1.27.6-debug
-    alpineGit: registry.abrban.com/proxy-dockerhub/alpine/git:2.43.0
+    alpine: registry.abrban.com/abrban/alpine:3.19
-  # Prefix for Docker Hub images in generated user-app Dockerfiles (node, php, …)
+    alpineGit: registry.abrban.com/abrban/alpine-git:2.43.0
-  baseImageRegistry: registry.abrban.com/proxy-dockerhub/library
+  # Seeded base images (gitops/jobs/seed-ci-images.yaml) — proxy-dockerhub cache can be corrupt on first pull.
+  baseImageRegistry: registry.abrban.com/abrban
   # Secret with HTTP_PROXY/HTTPS_PROXY for Kaniko build jobs (npm, apk, git clone).
   # Set to registry-egress-proxy in production; leave empty when nodes have direct egress.
   egressProxySecret: ""

backend/src/build/build.service.spec.ts

@@ -49,6 +49,41 @@ describe('BuildService', () => {
     service = module.get(BuildService);
   });
 
+  describe('baseImage', () => {
+    it('prefixes Docker Hub library images (tag colon must not block mirroring)', () => {
+      const config = (service as any).configService as { get: jest.Mock };
+      config.get.mockImplementation((key: string) => {
+        if (key === 'build.baseImageRegistry') return 'registry.abrban.com/abrban';
+        return undefined;
+      });
+
+      expect((service as any).baseImage('node:20-alpine')).toBe(
+        'registry.abrban.com/abrban/node:20-alpine',
+      );
+      expect((service as any).baseImage('alpine:3.19')).toBe(
+        'registry.abrban.com/abrban/alpine:3.19',
+      );
+    });
+
+    it('leaves images that already reference an external registry unchanged', () => {
+      const config = (service as any).configService as { get: jest.Mock };
+      config.get.mockImplementation((key: string) => {
+        if (key === 'build.baseImageRegistry') return 'registry.abrban.com/abrban';
+        return undefined;
+      });
+
+      expect((service as any).baseImage('mcr.microsoft.com/dotnet/sdk:8.0')).toBe(
+        'mcr.microsoft.com/dotnet/sdk:8.0',
+      );
+      expect((service as any).baseImage('registry.abrban.com/abrban/node:20-alpine')).toBe(
+        'registry.abrban.com/abrban/node:20-alpine',
+      );
+      expect((service as any).baseImage('localhost:5000/myapp:latest')).toBe(
+        'localhost:5000/myapp:latest',
+      );
+    });
+  });
+
   describe('generateDockerfile', () => {
     it('generates Go Dockerfile with requested runtime version', () => {
       const app = {
@@ -78,6 +113,53 @@ describe('BuildService', () => {
       expect(dockerfile).toContain('go build -a -installsuffix cgo -ldflags="-w -s" -o main ./cmd/server');
     });
 
+    it('generates Node.js Dockerfile with mirrored base images when registry prefix is set', async () => {
+      const module = await Test.createTestingModule({
+        providers: [
+          BuildService,
+          {
+            provide: ConfigService,
+            useValue: {
+              get: jest.fn((key: string) => {
+                const map: Record<string, string> = {
+                  'build.namespace': 'cloudhost-builds',
+                  'build.serviceAccount': 'kaniko-builder',
+                  'build.baseImageRegistry': 'registry.abrban.com/abrban',
+                  'registry.url': 'registry.local:5000',
+                };
+                return map[key];
+              }),
+            },
+          },
+          { provide: ClustersService, useValue: {} },
+          {
+            provide: BuildProgressStore,
+            useValue: { get: jest.fn(), set: jest.fn(), clear: jest.fn() },
+          },
+          { provide: RegistryService, useValue: {} },
+          {
+            provide: SourceStorageService,
+            useValue: {
+              isObjectStorage: () => false,
+              materializeToTempFile: jest.fn(),
+              getSize: jest.fn(),
+            },
+          },
+        ],
+      }).compile();
+
+      const mirrored = module.get(BuildService);
+      const app = {
+        runtime: AppRuntime.NODEJS,
+        runtimeVersion: '20',
+      } as Application;
+
+      const dockerfile = (mirrored as any).generateDockerfile(app) as string;
+
+      expect(dockerfile).toContain('FROM registry.abrban.com/abrban/node:20-alpine');
+      expect(dockerfile).toContain('EXPOSE 3000');
+    });
+
     it('generates Node.js Dockerfile with default port', () => {
       const app = {
         runtime: AppRuntime.NODEJS,

backend/src/build/build.service.ts

@@ -73,9 +73,19 @@ export class BuildService {
   private baseImage(image: string): string {
     const prefix = this.configService.get<string>('build.baseImageRegistry');
     if (!prefix) return image;
-    const firstSegment = image.split('/')[0];
+
-    const hasRegistry = firstSegment.includes('.') || firstSegment.includes(':');
+    // Official Docker Hub library images have no slash (node:20-alpine, alpine:3.19).
-    if (hasRegistry) return image;
+    // Do not treat the tag colon as a registry port — that was skipping the mirror.
+    if (!image.includes('/')) {
+      return `${prefix}/${image}`;
+    }
+
+    const registryHost = image.split('/')[0];
+    if (registryHost.includes('.') || registryHost.includes(':') || registryHost === 'localhost') {
+      return image;
+    }
+
+    // Docker Hub org/user image (e.g. bitnami/redis:7) — mirror through the prefix.
     return `${prefix}/${image}`;
   }
 
@@ -434,10 +444,10 @@ export class BuildService {
    * Returns { imageUri, buildLog } — the full image URI and the build logs.
    */
   async buildImage(app: Application, deploymentId?: string): Promise<{ imageUri: string; buildLog: string }> {
-    const registryUrl = this.registryService.getRegistryUrl();
+    const registryPushUrl = this.registryService.getRegistryPushUrl();
     const buildNamespace = this.registryService.getBuildNamespace();
     const tag = `${Date.now()}`;
-    const imageUri = this.registryService.buildImageReference(app.userId, app.name, tag);
+    const imageUri = this.registryService.buildPushImageReference(app.userId, app.name, tag);
 
     this.logger.log(`Starting image build for ${app.name} → ${imageUri}`);
 
@@ -530,11 +540,11 @@ export class BuildService {
     // Build the Kaniko Job spec
     // Always use dir context — init containers prepare /workspace/source
     const kanikoArgs = [
-      '--dockerfile=/workspace/Dockerfile',
+      '--dockerfile=Dockerfile',
       '--context=dir:///workspace/source',
       `--destination=${imageUri}`,
       '--cache=true',
-      `--cache-repo=${registryUrl}/${app.userId}/cache`,
+      `--cache-repo=${registryPushUrl}/${app.userId}/cache`,
       '--insecure',
       '--skip-tls-verify',
       '--single-snapshot',
@@ -544,7 +554,10 @@ export class BuildService {
     const volumes: any[] = [
       {
         name: 'docker-config',
-        secret: { secretName: 'registry-credentials' },
+        secret: {
+          secretName: 'registry-credentials',
+          items: [{ key: '.dockerconfigjson', path: 'config.json' }],
+        },
       },
       {
         name: 'dockerfile',
@@ -580,7 +593,6 @@ export class BuildService {
           reject_unsafe_path() {
             case "$1" in ..|../*|*/../*|/*) echo "ERROR: unsafe archive path: $1" && exit 1;; esac
           } &&
-          cp /workspace/Dockerfile /workspace-out/Dockerfile &&
           mkdir -p /tmp/extract &&
           cd /tmp/extract &&
           if tar tzf /source-pvc/source.zip >/dev/null 2>&1; then
@@ -610,6 +622,7 @@ export class BuildService {
             cp -a /tmp/extract/. /workspace-out/source/
           fi &&
           rm -rf /tmp/extract &&
+          cp /dockerfile/Dockerfile /workspace-out/source/Dockerfile &&
           echo "--- Final workspace contents ---" &&
           ls -la /workspace-out/source/
         `,
@@ -618,7 +631,7 @@ export class BuildService {
           { name: 'workspace', mountPath: '/workspace-out' },
           {
             name: 'dockerfile',
-            mountPath: '/workspace/Dockerfile',
+            mountPath: '/dockerfile/Dockerfile',
             subPath: 'Dockerfile',
           },
           { name: 'source-pvc', mountPath: '/source-pvc' },
@@ -677,7 +690,7 @@ export class BuildService {
           fi
           echo ">>> Cloning branch '$GIT_BRANCH' from $GIT_URL"
           git clone --depth 1 --branch "$GIT_BRANCH" "$GIT_URL" /workspace-out/source
-          cp /dockerfile/Dockerfile /workspace-out/Dockerfile
+          cp /dockerfile/Dockerfile /workspace-out/source/Dockerfile
           echo ">>> Workspace contents:"
           ls -la /workspace-out/source/
         `,
@@ -708,7 +721,7 @@ export class BuildService {
           '-c',
           `
           mkdir -p /workspace-out/source &&
-          cp /dockerfile/Dockerfile /workspace-out/Dockerfile &&
+          cp /dockerfile/Dockerfile /workspace-out/source/Dockerfile &&
           echo ">>> Prepared empty workspace for fresh install" &&
           ls -la /workspace-out/
         `,
@@ -1134,25 +1147,27 @@ export class BuildService {
       }
     }
 
-    // 3. Ensure registry-credentials secret (docker config for Kaniko to push)
+    // 3. Ensure registry-credentials secret (docker config for Kaniko push/pull)
     const registrySecretName = 'registry-credentials';
+    const registrySecretBody = {
+      metadata: { name: registrySecretName, namespace },
+      type: 'kubernetes.io/dockerconfigjson',
+      data: {
+        '.dockerconfigjson': Buffer.from(this.registryService.buildDockerConfigJson()).toString('base64'),
+      },
+    };
     try {
-      await coreApi.readNamespacedSecret({
+      await coreApi.replaceNamespacedSecret({
         name: registrySecretName,
         namespace,
+        body: registrySecretBody,
       });
     } catch (err: any) {
       if (err.code === 404 || err.body?.code === 404) {
         this.logger.log(`Secret "${registrySecretName}" not found in "${namespace}" — creating it`);
         await coreApi.createNamespacedSecret({
           namespace,
-          body: {
+          body: registrySecretBody,
-            metadata: { name: registrySecretName, namespace },
-            type: 'kubernetes.io/dockerconfigjson',
-            data: {
-              '.dockerconfigjson': Buffer.from(this.registryService.buildDockerConfigJson()).toString('base64'),
-            },
-          },
         });
       } else {
         throw err;
@@ -1190,8 +1205,8 @@ export class BuildService {
 FROM ${this.baseImage(`node:${nodeVersion}-alpine`)} AS builder
 WORKDIR /app
 COPY package*.json ./
-# Reproducible install from the lockfile when present
+# Prefer lockfile; fall back when package.json and lockfile are out of sync
-RUN if [ -f package-lock.json ]; then npm ci --legacy-peer-deps; else npm install --legacy-peer-deps; fi \\
+RUN if [ -f package-lock.json ]; then npm ci --legacy-peer-deps || npm install --legacy-peer-deps; else npm install --legacy-peer-deps; fi \\
     && npm cache clean --force
 COPY . .
 

backend/src/clusters/clusters.service.ts

@@ -870,7 +870,7 @@ export class ClustersService implements OnModuleInit, OnModuleDestroy {
 
     const buildNs = this.registryService.getBuildNamespace();
     const saName = this.configService.get<string>('build.serviceAccount') || 'kaniko-builder';
-    const registryUrl = this.registryService.getRegistryUrl();
+    const registryHost = this.registryService.getRegistryHost();
 
     this.logger.log(`Bootstrapping cluster — namespace: ${buildNs}`);
 
@@ -1100,13 +1100,17 @@ export class ClustersService implements OnModuleInit, OnModuleDestroy {
       }
     }
 
-    await this.ensureK3sRegistryMirrors(appsApi, registryUrl);
+    await this.ensureK3sRegistryMirrors(coreApi, appsApi, registryHost);
 
-    this.logger.log(`✅ Cluster bootstrap complete — registry: ${registryUrl}`);
+    this.logger.log(`✅ Cluster bootstrap complete — registry: ${registryHost}`);
   }
 
   /** In-cluster registry mirror for k3s/containerd (HTTP). Removes legacy external-registry DaemonSet if present. */
-  private async ensureK3sRegistryMirrors(appsApi: k8s.AppsV1Api, registryUrl: string): Promise<void> {
+  private async ensureK3sRegistryMirrors(
+    coreApi: k8s.CoreV1Api,
+    appsApi: k8s.AppsV1Api,
+    registryHost: string,
+  ): Promise<void> {
     const namespace = 'kube-system';
     const legacyDs = 'cloudhost-k3s-registry-config';
     try {
@@ -1120,28 +1124,25 @@ export class ClustersService implements OnModuleInit, OnModuleDestroy {
 
     const { username, password } = this.registryService.getRegistryCredentials();
     const dsName = 'cloudhost-k3s-registry-mirrors';
-    // The mirror endpoint must be reachable by the node's containerd, which does
+    // containerd on the node does not use cluster DNS — mirror via ClusterIP (Harbor)
-    // NOT use cluster DNS — so we point it at the registry NodePort on loopback
+    // or loopback NodePort (legacy in-cluster registry).
-    // (http://127.0.0.1:<nodePort>) instead of the in-cluster service DNS name.
+    const mirrorEndpoint = await this.resolveK3sRegistryMirrorEndpoint(coreApi);
-    // Otherwise image pulls break whenever node-level resolution of
+    const mirrorHost = mirrorEndpoint.replace(/^https?:\/\//, '');
-    // *.svc.cluster.local is unavailable (e.g. right after a node restart).
-    const registryNodePort = 30500;
-    const nodePortHost = `127.0.0.1:${registryNodePort}`;
     const configureScript = [
       'set -e',
       'REG=/host/etc/rancher/k3s/registries.yaml',
       'mkdir -p /host/etc/rancher/k3s',
       'cat > /tmp/cloudhost-registries.yaml <<EOFREG',
       'mirrors:',
-      `  "${registryUrl}":`,
+      `  "${registryHost}":`,
       '    endpoint:',
-      `      - "http://${nodePortHost}"`,
+      `      - "${mirrorEndpoint}"`,
       'configs:',
-      `  "${registryUrl}":`,
+      `  "${registryHost}":`,
       '    auth:',
       `      username: ${JSON.stringify(username)}`,
       `      password: ${JSON.stringify(password)}`,
-      `  "${nodePortHost}":`,
+      `  "${mirrorHost}":`,
       '    auth:',
       `      username: ${JSON.stringify(username)}`,
       `      password: ${JSON.stringify(password)}`,
@@ -1206,6 +1207,36 @@ export class ClustersService implements OnModuleInit, OnModuleDestroy {
     }
   }
 
+  /**
+   * containerd on nodes cannot resolve *.svc.cluster.local — use ClusterIP for
+   * Harbor (harbor-core HTTP) or legacy registry NodePort on loopback.
+   */
+  private async resolveK3sRegistryMirrorEndpoint(coreApi: k8s.CoreV1Api): Promise<string> {
+    const pushUrl = this.registryService.getRegistryPushUrl();
+    const platformNs = this.configService.get<string>('platform.namespace') || 'cloudhost';
+    const harborCoreService = this.configService.get<string>('registry.harborCoreService') || 'harbor-core';
+
+    if (pushUrl.includes('harbor-registry')) {
+      try {
+        const svc = await coreApi.readNamespacedService({
+          name: harborCoreService,
+          namespace: platformNs,
+        });
+        const clusterIp = svc.spec?.clusterIP;
+        if (clusterIp) {
+          return `http://${clusterIp}`;
+        }
+      } catch (err: any) {
+        this.logger.warn(
+          `Could not resolve ${harborCoreService} ClusterIP for k3s mirror: ${err.message}`,
+        );
+      }
+    }
+
+    const registryNodePort = 30500;
+    return `http://127.0.0.1:${registryNodePort}`;
+  }
+
   private parseCpuToMillicores(cpu: string): number {
     if (!cpu || cpu === '0') return 0;
     if (cpu.endsWith('n')) return parseFloat(cpu) / 1_000_000;

backend/src/config/configuration.ts

@@ -124,11 +124,17 @@ export default () => ({
   },
 
   registry: {
-    /** In-cluster registry — Kaniko push and app image pull (same host). */
+    /** Kaniko push target — harbor-core when set (Harbor UI metadata); else in-cluster registry. */
     url: process.env.REGISTRY_URL || 'registry.cloudhost-builds.svc.cluster.local:5000',
+    pushUrl:
+      process.env.REGISTRY_PUSH_URL ||
+      process.env.REGISTRY_URL ||
+      'registry.cloudhost-builds.svc.cluster.local:5000',
+    /** Kubelet / workload pull (external hostname on Harbor setups). */
     pullUrl: process.env.REGISTRY_PULL_URL || process.env.REGISTRY_URL || 'registry.cloudhost-builds.svc.cluster.local:5000',
     username: process.env.REGISTRY_USERNAME || 'admin',
     password: process.env.REGISTRY_PASSWORD || '',
+    harborCoreService: process.env.HARBOR_CORE_SERVICE || 'harbor-core',
   },
 
   build: {
@@ -139,16 +145,16 @@ export default () => ({
      * and managed-service charts (e.g. `node:20-alpine` →
      * `registry.abrban.com/proxy-dockerhub/library/node:20-alpine`).
      */
-    baseImageRegistry: (process.env.BASE_IMAGE_REGISTRY || 'registry.abrban.com/proxy-dockerhub/library')
+    baseImageRegistry: (process.env.BASE_IMAGE_REGISTRY || 'registry.abrban.com/abrban')
       .trim()
       .replace(/\/+$/, ''),
     /** Full image refs for Kaniko jobs — override via Helm values or env. */
     images: {
       kaniko:
         process.env.KANIKO_IMAGE ||
-        'registry.abrban.com/proxy-gcr/kaniko-project/executor:v1.23.2',
+        'registry.abrban.com/abrban/kaniko-executor:v1.27.6-debug',
-      alpine: (process.env.BUILD_ALPINE_IMAGE || 'registry.abrban.com/proxy-dockerhub/library/alpine:3.19').trim(),
+      alpine: (process.env.BUILD_ALPINE_IMAGE || 'registry.abrban.com/abrban/alpine:3.19').trim(),
-      alpineGit: (process.env.BUILD_ALPINE_GIT_IMAGE || 'registry.abrban.com/proxy-dockerhub/alpine/git:2.43.0').trim(),
+      alpineGit: (process.env.BUILD_ALPINE_GIT_IMAGE || 'registry.abrban.com/abrban/alpine-git:2.43.0').trim(),
     },
     /**
      * Secret name with HTTP_PROXY / HTTPS_PROXY / NO_PROXY for build pods

backend/src/deployments/deployments.service.ts

@@ -182,7 +182,7 @@ export class DeploymentsService implements OnModuleInit {
       });
       await this.kubernetesService.waitForApplicationReady(
         app,
-        600_000,
+        1_200_000,
         () => this.isDeploymentCancelled(deploymentId),
       );
 
@@ -295,7 +295,7 @@ export class DeploymentsService implements OnModuleInit {
       });
       await this.kubernetesService.waitForApplicationReady(
         app,
-        600_000,
+        1_200_000,
         () => this.isDeploymentCancelled(deploymentId),
       );
 

backend/src/kubernetes/registry.service.ts

@@ -28,6 +28,30 @@ export class RegistryService {
     return url.replace(/^https?:\/\//, '');
   }
 
+  /** Docker auth key — hostname[:port] only, no repository path prefix. */
+  getRegistryHost(): string {
+    const url = this.getRegistryUrl();
+    const slash = url.indexOf('/');
+    return slash === -1 ? url : url.slice(0, slash);
+  }
+
+  /** Push target host[:port][/project] — Kaniko via harbor-core when configured (Harbor UI metadata). */
+  getRegistryPushUrl(): string {
+    const buildNs = this.getBuildNamespace();
+    const url =
+      this.configService.get<string>('registry.pushUrl') ||
+      this.configService.get<string>('registry.url') ||
+      `registry.${buildNs}.svc.cluster.local:5000`;
+    return url.replace(/^https?:\/\//, '');
+  }
+
+  /** Push target host:port only, no repository path prefix. */
+  getRegistryPushHost(): string {
+    const url = this.getRegistryPushUrl();
+    const slash = url.indexOf('/');
+    return slash === -1 ? url : url.slice(0, slash);
+  }
+
   getRegistryCredentials(): { username: string; password: string } {
     return {
       username: this.configService.get<string>('registry.username') || 'admin',
@@ -39,6 +63,11 @@ export class RegistryService {
     return `${this.getRegistryUrl()}/${userId}/${appName}:${tag}`;
   }
 
+  /** Kaniko push target — uses registry.url (in-cluster harbor-registry on Harbor setups). */
+  buildPushImageReference(userId: string, appName: string, tag: string): string {
+    return `${this.getRegistryPushUrl()}/${userId}/${appName}:${tag}`;
+  }
+
   parseImageReference(imageRef: string): ParsedImageReference {
     const normalized = imageRef.replace(/^https?:\/\//, '');
     const slashIdx = normalized.indexOf('/');
@@ -56,24 +85,41 @@ export class RegistryService {
     };
   }
 
-  /** Re-point any stored image (e.g. legacy external host) to the in-cluster registry. */
+  /** Re-point any stored image (e.g. push host) to the pull registry URL for kubelet. */
   normalizeImageReference(imageRef: string): string {
     const { repository, tag } = this.parseImageReference(imageRef);
-    return `${this.getRegistryUrl()}/${repository}:${tag}`;
+    const pullBase = this.getRegistryUrl().replace(/\/$/, '');
+    const slash = pullBase.indexOf('/');
+    const pullPath = slash === -1 ? '' : pullBase.slice(slash + 1);
+    let repo = repository;
+    if (pullPath && (repo === pullPath || repo.startsWith(`${pullPath}/`))) {
+      repo = repo === pullPath ? '' : repo.slice(pullPath.length + 1);
+    }
+    if (!repo) {
+      throw new Error(`Invalid image reference after normalization: ${imageRef}`);
+    }
+    return `${pullBase}/${repo}:${tag}`;
   }
 
   buildDockerConfigJson(): string {
     const { username, password } = this.getRegistryCredentials();
     const auth = username && password ? Buffer.from(`${username}:${password}`).toString('base64') : '';
-    const host = this.getRegistryUrl();
+    const pullHost = this.getRegistryHost();
-    return JSON.stringify({
+    const pushHost = this.getRegistryPushHost();
-      auths: {
+    const auths: Record<string, { auth: string }> = {
-        [host]: { auth },
+      [pullHost]: { auth },
-        [`registry.${this.getBuildNamespace()}.svc.cluster.local:5000`]: {
+      [`registry.${this.getBuildNamespace()}.svc.cluster.local:5000`]: { auth },
-          auth,
+    };
-        },
+    if (pushHost !== pullHost) {
-      },
+      auths[pushHost] = { auth };
-    });
+    }
+    // Legacy direct-registry push host (base image pulls during Kaniko build).
+    const directPushHost = this.configService.get<string>('registry.url')?.replace(/^https?:\/\//, '');
+    const directHostOnly = directPushHost?.includes('/') ? directPushHost.slice(0, directPushHost.indexOf('/')) : directPushHost;
+    if (directHostOnly && directHostOnly !== pushHost && directHostOnly !== pullHost) {
+      auths[directHostOnly] = { auth };
+    }
+    return JSON.stringify({ auths });
   }
 
   async ensureRegistryPullSecret(coreApi: k8s.CoreV1Api, namespace: string): Promise<void> {

gitops/harbor/registry-ingress.yaml

@@ -0,0 +1,124 @@
+# registry.abrban.com — Traefik path split
+#
+# Proxy-cache projects (proxy-dockerhub, proxy-gcr, …) MUST hit harbor-core so
+# Harbor can pull upstream on demand. harbor-registry only stores blobs; it does
+# not run proxy-cache logic → 404 for uncached proxy paths.
+#
+# Direct pushes (abrban/, rook/) stay on harbor-registry where Kaniko/skopeo
+# wrote the blobs.
+#
+# Apply: kubectl apply -f gitops/harbor/registry-ingress.yaml
+
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: registry
+  namespace: cloudhost
+  annotations:
+    traefik.ingress.kubernetes.io/router.entrypoints: websecure
+    traefik.ingress.kubernetes.io/router.middlewares: cloudhost-long-timeout@kubernetescrd
+spec:
+  ingressClassName: traefik
+  tls:
+    - hosts:
+        - registry.abrban.com
+      secretName: abrban-wildcard-tls
+  rules:
+    - host: registry.abrban.com
+      http:
+        paths:
+          # ── Proxy-cache (harbor-core serves v2 + on-demand upstream pull) ──
+          - path: /v2/proxy-dockerhub/
+            pathType: Prefix
+            backend:
+              service:
+                name: harbor-core
+                port:
+                  number: 80
+          - path: /v2/proxy-gcr/
+            pathType: Prefix
+            backend:
+              service:
+                name: harbor-core
+                port:
+                  number: 80
+          - path: /v2/proxy-quay/
+            pathType: Prefix
+            backend:
+              service:
+                name: harbor-core
+                port:
+                  number: 80
+          - path: /v2/proxy-k8s/
+            pathType: Prefix
+            backend:
+              service:
+                name: harbor-core
+                port:
+                  number: 80
+          - path: /v2/proxy-gitea/
+            pathType: Prefix
+            backend:
+              service:
+                name: harbor-core
+                port:
+                  number: 80
+          # ── abrban/rook project images (served by harbor-core; required for k3s mirror pulls) ──
+          - path: /v2/abrban/
+            pathType: Prefix
+            backend:
+              service:
+                name: harbor-core
+                port:
+                  number: 80
+          - path: /v2/rook/
+            pathType: Prefix
+            backend:
+              service:
+                name: harbor-core
+                port:
+                  number: 80
+          # ── Legacy registry (platform images pre-Harbor) ──
+          - path: /v2/
+            pathType: Prefix
+            backend:
+              service:
+                name: registry
+                port:
+                  number: 5000
+          # ── Harbor UI / API ──
+          - path: /api/
+            pathType: Prefix
+            backend:
+              service:
+                name: harbor-core
+                port:
+                  number: 80
+          - path: /service/
+            pathType: Prefix
+            backend:
+              service:
+                name: harbor-core
+                port:
+                  number: 80
+          - path: /c/
+            pathType: Prefix
+            backend:
+              service:
+                name: harbor-core
+                port:
+                  number: 80
+          - path: /chartrepo/
+            pathType: Prefix
+            backend:
+              service:
+                name: harbor-core
+                port:
+                  number: 80
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: harbor-portal
+                port:
+                  number: 80

gitops/jobs/build-platform-kaniko.yaml

@@ -31,7 +31,7 @@ spec:
               mountPath: /workspace
       containers:
         - name: kaniko
-          image: registry.abrban.com/proxy-gcr/kaniko-project/executor:v1.23.2
+          image: registry.abrban.com/abrban/kaniko-executor:v1.23.2
           env:
             - name: IMAGE_TAG
               value: bootstrap

gitops/jobs/kaniko-harbor-auth.example.yaml

@@ -2,21 +2,15 @@
 # The real secret is managed as a SealedSecret in the cloud-host-gitops repo
 # (sealed-secrets/kaniko-harbor-auth.yaml).
 #
-# Kaniko pushes directly to the internal registry endpoint
+# Kaniko pushes via harbor-core (Harbor UI metadata). Pull base images may still
-# (harbor-registry.cloudhost.svc.cluster.local:5000), which bypasses harbor-core.
+# use harbor-registry:5000 — include auth for both hosts in one dockerconfigjson.
-# That endpoint only accepts the internal registry credential — Harbor robot
-# accounts do NOT work there (their tokens are issued by harbor-core's token
-# service). Use the harbor_registry_user credential from the harbor-core secret:
 #
-#   REG_PASS="$(kubectl -n cloudhost get secret harbor-core \
+#   ADMIN="$(kubectl -n cloudhost get secret harbor-core -o jsonpath='{.data.HARBOR_ADMIN_PASSWORD}' | base64 -d)"
-#     -o jsonpath='{.data.REGISTRY_CREDENTIAL_PASSWORD}' | base64 -d)"
+#   REG_PASS="$(kubectl -n cloudhost get secret harbor-core -o jsonpath='{.data.REGISTRY_CREDENTIAL_PASSWORD}' | base64 -d)"
-#   kubectl -n cloudhost-builds create secret docker-registry kaniko-harbor-auth \
+#   kubectl -n cloudhost-builds create secret generic kaniko-harbor-auth \
-#     --docker-server=harbor-registry.cloudhost.svc.cluster.local:5000 \
+#     --from-literal=admin="${ADMIN}" --from-literal=reg_pass="${REG_PASS}" --dry-run=client -o yaml | ...
-#     --docker-username=harbor_registry_user \
-#     --docker-password="${REG_PASS}"
 #
-# The build-deploy workflow mounts this secret at /kaniko/.docker/config.json
+# See RUNBOOK-CICD.fa.md for the full procedure.
-# inside every Kaniko Job. See RUNBOOK-CICD.fa.md for the full procedure.
 apiVersion: v1
 kind: Secret
 metadata:
@@ -27,10 +21,20 @@ stringData:
   .dockerconfigjson: |
     {
       "auths": {
+        "harbor-core.cloudhost.svc.cluster.local": {
+          "username": "admin",
+          "password": "<HARBOR_ADMIN_PASSWORD>",
+          "auth": "<base64 of admin:password>"
+        },
         "harbor-registry.cloudhost.svc.cluster.local:5000": {
           "username": "harbor_registry_user",
           "password": "<REGISTRY_CREDENTIAL_PASSWORD>",
-          "auth": "<base64 of username:password>"
+          "auth": "<base64 of harbor_registry_user:password>"
+        },
+        "registry.abrban.com": {
+          "username": "harbor_registry_user",
+          "password": "<REGISTRY_CREDENTIAL_PASSWORD>",
+          "auth": "<base64>"
         }
       }
     }

gitops/jobs/seed-ci-images.yaml

@@ -7,7 +7,7 @@
 #   kubectl -n cloudhost wait --for=condition=complete job/seed-ci-images --timeout=15m
 #
 # Images copied (see RUNBOOK-CICD.fa.md):
-#   abrban/act-runner, abrban/alpine-git, abrban/node, abrban/kaniko-executor
+#   abrban/act-runner, abrban/alpine, abrban/alpine-git, abrban/node, abrban/kaniko-executor
 apiVersion: batch/v1
 kind: Job
 metadata:
@@ -39,6 +39,9 @@ spec:
               skopeo copy --dest-tls-verify=false --dest-creds="${CREDS}" \
                 docker://docker.io/gitea/act_runner:0.2.11 \
                 "${DEST}/act-runner:0.2.11"
+              skopeo copy --dest-tls-verify=false --dest-creds="${CREDS}" \
+                docker://docker.io/library/alpine:3.19 \
+                "${DEST}/alpine:3.19"
               skopeo copy --dest-tls-verify=false --dest-creds="${CREDS}" \
                 docker://docker.io/alpine/git:2.43.0 \
                 "${DEST}/alpine-git:2.43.0"
@@ -46,6 +49,14 @@ spec:
                 docker://docker.io/library/node:24-alpine \
                 "${DEST}/node:24-alpine"
               skopeo copy --dest-tls-verify=false --dest-creds="${CREDS}" \
-                docker://gcr.io/kaniko-project/executor:v1.27.6-debug \
+                docker://docker.io/library/node:20-alpine \
+                "${DEST}/node:20-alpine"
+              # Tag present in Harbor abrban/ — seed via proxy-gcr (see seed-ci-images.yaml).
+              skopeo copy --dest-tls-verify=false --dest-creds="${CREDS}" \
+                docker://registry.abrban.com/proxy-gcr/kaniko-project/executor:v1.23.2 \
+                "${DEST}/kaniko-executor:v1.23.2"
+              # Alias for CI/configs that reference the debug tag name.
+              skopeo copy --dest-tls-verify=false --dest-creds="${CREDS}" \
+                docker://harbor-registry.cloudhost.svc.cluster.local:5000/abrban/kaniko-executor:v1.23.2 \
                 "${DEST}/kaniko-executor:v1.27.6-debug"
               echo SEED_OK

gitops/k3s/registries.yaml

@@ -2,6 +2,8 @@
 # Proxy-cache only works through harbor-core (not harbor-registry or Traefik /v2/ alone).
 #
 # Apply: ./scripts/apply-k3s-registries.sh
+# Harbor EXT_ENDPOINT should be http://registry.abrban.com so OAuth realm uses HTTP
+# (kubelet mirror hits harbor-core on :80; https://<clusterIP>:443 times out).
 
 mirrors:
   registry.abrban.com:

gitops/platform/values-abrban.example.yaml

@@ -11,6 +11,11 @@
 namespace: cloudhost
 createNamespace: false
 
+registry:
+  credentialsSecret: harbor-core
+  credentialsPasswordKey: REGISTRY_CREDENTIAL_PASSWORD
+  username: harbor_registry_user
+
 global:
   storageClass: local-path
 
@@ -31,10 +36,10 @@ images:
 # Kaniko job images — Harbor proxy-cache (first pull is slow, no manual seed needed).
 build:
   images:
-    kaniko: registry.abrban.com/proxy-gcr/kaniko-project/executor:v1.23.2
+    kaniko: registry.abrban.com/abrban/kaniko-executor:v1.27.6-debug
-    alpine: registry.abrban.com/proxy-dockerhub/library/alpine:3.19
+    alpine: registry.abrban.com/abrban/alpine:3.19
-    alpineGit: registry.abrban.com/proxy-dockerhub/alpine/git:2.43.0
+    alpineGit: registry.abrban.com/abrban/alpine-git:2.43.0
-  baseImageRegistry: registry.abrban.com/proxy-dockerhub/library
+  baseImageRegistry: registry.abrban.com/abrban
   # Kaniko + init containers (npm/apk/composer/pip/git clone) on restricted egress.
   egressProxySecret: registry-egress-proxy
 
@@ -98,6 +103,8 @@ backend:
     PLATFORM_DOMAIN: apps.abrban.com
     PREVIEW_BASE_DOMAIN: apps.abrban.com
     FRONTEND_URL: https://panel.abrban.com,https://abrban.com
+    # Push via harbor-core so artifacts appear in Harbor UI; pull stays on registry.abrban.com.
+    REGISTRY_PUSH_URL: harbor-core.cloudhost.svc.cluster.local/abrban
     REGISTRY_URL: harbor-registry.cloudhost.svc.cluster.local:5000/abrban
     REGISTRY_PULL_URL: registry.abrban.com/abrban
     BUILD_NAMESPACE: cloudhost-builds

scripts/apply-k3s-registries.sh

@@ -8,6 +8,16 @@ HARBOR_CORE_IP="${HARBOR_CORE_IP:-$(kubectl -n cloudhost get svc harbor-core -o
 REG_USER="${REG_USER:-harbor_registry_user}"
 REG_PASS="${REG_PASS:-$(kubectl -n cloudhost get secret harbor-core -o jsonpath='{.data.REGISTRY_CREDENTIAL_PASSWORD}' | base64 -d)}"
 
+REG_PASS_B64="$(printf '%s' "$REG_PASS" | base64 | tr -d '\n')"
+
+kubectl -n "${NS}" delete pod k3s-registries-setup --ignore-not-found
+
+kubectl -n "${NS}" create secret generic k3s-registries-setup-env \
+  --from-literal=HARBOR_CORE_IP="${HARBOR_CORE_IP}" \
+  --from-literal=REG_USER="${REG_USER}" \
+  --from-literal=REG_PASS_B64="${REG_PASS_B64}" \
+  --dry-run=client -o yaml | kubectl apply -f -
+
 kubectl -n "${NS}" delete pod k3s-registries-setup --ignore-not-found
 
 kubectl -n "${NS}" run k3s-registries-setup \
@@ -23,7 +33,8 @@ kubectl -n "${NS}" run k3s-registries-setup \
       "name": "setup",
       "image": "rancher/mirrored-library-busybox:1.36.1",
       "securityContext": {"privileged": true},
-      "command": ["sh", "-ec", "mkdir -p /host/etc/rancher/k3s && cat > /host/etc/rancher/k3s/registries.yaml <<'REGEOF'\nmirrors:\n  registry.abrban.com:\n    endpoint:\n      - http://${HARBOR_CORE_IP}\n  \\\"registry.cloudhost-builds.svc.cluster.local:5000\\\":\n    endpoint:\n      - \\\"http://127.0.0.1:30500\\\"\nconfigs:\n  registry.abrban.com:\n    auth:\n      username: ${REG_USER}\n      password: ${REG_PASS}\n  \\\"${HARBOR_CORE_IP}\\\":\n    auth:\n      username: ${REG_USER}\n      password: ${REG_PASS}\n  \\\"registry.cloudhost-builds.svc.cluster.local:5000\\\":\n    auth:\n      username: admin\n      password: \\\"\\\"\n  \\\"127.0.0.1:30500\\\":\n    auth:\n      username: admin\n      password: \\\"\\\"\nREGEOF\nnsenter -t 1 -m -u -n -i -- systemctl restart k3s 2>/dev/null || true\necho k3s-restarted\nsleep 30"],
+      "envFrom": [{"secretRef": {"name": "k3s-registries-setup-env"}}],
+      "command": ["sh", "-ec", "REG_PASS=\$(echo \"\$REG_PASS_B64\" | base64 -d); mkdir -p /host/etc/rancher/k3s && cat > /host/etc/rancher/k3s/registries.yaml <<REGEOF\nmirrors:\n  registry.abrban.com:\n    endpoint:\n      - http://\${HARBOR_CORE_IP}\n  \\\"registry.cloudhost-builds.svc.cluster.local:5000\\\":\n    endpoint:\n      - \\\"http://127.0.0.1:30500\\\"\nconfigs:\n  registry.abrban.com:\n    auth:\n      username: \${REG_USER}\n      password: \${REG_PASS}\n  \\\"\${HARBOR_CORE_IP}\\\":\n    auth:\n      username: \${REG_USER}\n      password: \${REG_PASS}\n  \\\"registry.cloudhost-builds.svc.cluster.local:5000\\\":\n    auth:\n      username: admin\n      password: \\\"\\\"\n  \\\"127.0.0.1:30500\\\":\n    auth:\n      username: admin\n      password: \\\"\\\"\nREGEOF\nnsenter -t 1 -m -u -n -i -- systemctl restart k3s 2>/dev/null || true\necho k3s-restarted\nsleep 30"],
       "volumeMounts": [{"name": "host", "mountPath": "/host"}]
     }],
     "volumes": [{"name": "host", "hostPath": {"path": "/"}}]

scripts/trigger-platform-build.sh

@@ -40,7 +40,7 @@ spec:
               mountPath: /workspace
       containers:
         - name: kaniko
-          image: registry.abrban.com/proxy-gcr/kaniko-project/executor:v1.23.2
+          image: registry.abrban.com/abrban/kaniko-executor:v1.23.2
           command:
             - sh
             - -ec