From ec72ee4fca6f1172b719d200f38adcd4e9a373ce Mon Sep 17 00:00:00 2001 From: keyhan Date: Fri, 10 Jul 2026 18:38:05 +0330 Subject: [PATCH] Fix app image pulls and Harbor kubelet auth for user workloads. Route k3s registry mirrors through harbor-core ClusterIP with hostname-only auth keys, use HTTP EXT_ENDPOINT so OAuth tokens work on port 80, extend deploy readiness timeout, and harden Kaniko build/dockerfile fallbacks. Co-authored-by: Cursor --- .../values-registry.abrban.com.yaml | 2 +- backend/src/build/build.service.spec.ts | 82 +++++++++++++++++++ backend/src/build/build.service.ts | 20 +++-- backend/src/clusters/clusters.service.ts | 61 ++++++++++---- .../src/deployments/deployments.service.ts | 4 +- gitops/k3s/registries.yaml | 2 + scripts/apply-k3s-registries.sh | 13 ++- 7 files changed, 160 insertions(+), 24 deletions(-) diff --git a/backend/helm/cloudhost-harbor/values-registry.abrban.com.yaml b/backend/helm/cloudhost-harbor/values-registry.abrban.com.yaml index bbe6be0..22e3b54 100644 --- a/backend/helm/cloudhost-harbor/values-registry.abrban.com.yaml +++ b/backend/helm/cloudhost-harbor/values-registry.abrban.com.yaml @@ -4,7 +4,7 @@ ## Install: ## helm upgrade --install harbor harbor/harbor -n cloudhost -f backend/helm/cloudhost-harbor/values-registry.abrban.com.yaml ## -externalURL: https://registry.abrban.com +externalURL: http://registry.abrban.com proxy: # Values are injected by install script from `cloudhost/registry-egress-proxy`. diff --git a/backend/src/build/build.service.spec.ts b/backend/src/build/build.service.spec.ts index dc07854..04a2725 100644 --- a/backend/src/build/build.service.spec.ts +++ b/backend/src/build/build.service.spec.ts @@ -49,6 +49,41 @@ describe('BuildService', () => { service = module.get(BuildService); }); + describe('baseImage', () => { + it('prefixes Docker Hub library images (tag colon must not block mirroring)', () => { + const config = (service as any).configService as { get: jest.Mock }; + config.get.mockImplementation((key: string) => { + if (key === 'build.baseImageRegistry') return 'registry.abrban.com/abrban'; + return undefined; + }); + + expect((service as any).baseImage('node:20-alpine')).toBe( + 'registry.abrban.com/abrban/node:20-alpine', + ); + expect((service as any).baseImage('alpine:3.19')).toBe( + 'registry.abrban.com/abrban/alpine:3.19', + ); + }); + + it('leaves images that already reference an external registry unchanged', () => { + const config = (service as any).configService as { get: jest.Mock }; + config.get.mockImplementation((key: string) => { + if (key === 'build.baseImageRegistry') return 'registry.abrban.com/abrban'; + return undefined; + }); + + expect((service as any).baseImage('mcr.microsoft.com/dotnet/sdk:8.0')).toBe( + 'mcr.microsoft.com/dotnet/sdk:8.0', + ); + expect((service as any).baseImage('registry.abrban.com/abrban/node:20-alpine')).toBe( + 'registry.abrban.com/abrban/node:20-alpine', + ); + expect((service as any).baseImage('localhost:5000/myapp:latest')).toBe( + 'localhost:5000/myapp:latest', + ); + }); + }); + describe('generateDockerfile', () => { it('generates Go Dockerfile with requested runtime version', () => { const app = { @@ -78,6 +113,53 @@ describe('BuildService', () => { expect(dockerfile).toContain('go build -a -installsuffix cgo -ldflags="-w -s" -o main ./cmd/server'); }); + it('generates Node.js Dockerfile with mirrored base images when registry prefix is set', async () => { + const module = await Test.createTestingModule({ + providers: [ + BuildService, + { + provide: ConfigService, + useValue: { + get: jest.fn((key: string) => { + const map: Record = { + 'build.namespace': 'cloudhost-builds', + 'build.serviceAccount': 'kaniko-builder', + 'build.baseImageRegistry': 'registry.abrban.com/abrban', + 'registry.url': 'registry.local:5000', + }; + return map[key]; + }), + }, + }, + { provide: ClustersService, useValue: {} }, + { + provide: BuildProgressStore, + useValue: { get: jest.fn(), set: jest.fn(), clear: jest.fn() }, + }, + { provide: RegistryService, useValue: {} }, + { + provide: SourceStorageService, + useValue: { + isObjectStorage: () => false, + materializeToTempFile: jest.fn(), + getSize: jest.fn(), + }, + }, + ], + }).compile(); + + const mirrored = module.get(BuildService); + const app = { + runtime: AppRuntime.NODEJS, + runtimeVersion: '20', + } as Application; + + const dockerfile = (mirrored as any).generateDockerfile(app) as string; + + expect(dockerfile).toContain('FROM registry.abrban.com/abrban/node:20-alpine'); + expect(dockerfile).toContain('EXPOSE 3000'); + }); + it('generates Node.js Dockerfile with default port', () => { const app = { runtime: AppRuntime.NODEJS, diff --git a/backend/src/build/build.service.ts b/backend/src/build/build.service.ts index c5b49c7..0637fc3 100644 --- a/backend/src/build/build.service.ts +++ b/backend/src/build/build.service.ts @@ -73,9 +73,19 @@ export class BuildService { private baseImage(image: string): string { const prefix = this.configService.get('build.baseImageRegistry'); if (!prefix) return image; - const firstSegment = image.split('/')[0]; - const hasRegistry = firstSegment.includes('.') || firstSegment.includes(':'); - if (hasRegistry) return image; + + // Official Docker Hub library images have no slash (node:20-alpine, alpine:3.19). + // Do not treat the tag colon as a registry port — that was skipping the mirror. + if (!image.includes('/')) { + return `${prefix}/${image}`; + } + + const registryHost = image.split('/')[0]; + if (registryHost.includes('.') || registryHost.includes(':') || registryHost === 'localhost') { + return image; + } + + // Docker Hub org/user image (e.g. bitnami/redis:7) — mirror through the prefix. return `${prefix}/${image}`; } @@ -1195,8 +1205,8 @@ export class BuildService { FROM ${this.baseImage(`node:${nodeVersion}-alpine`)} AS builder WORKDIR /app COPY package*.json ./ -# Reproducible install from the lockfile when present -RUN if [ -f package-lock.json ]; then npm ci --legacy-peer-deps; else npm install --legacy-peer-deps; fi \\ +# Prefer lockfile; fall back when package.json and lockfile are out of sync +RUN if [ -f package-lock.json ]; then npm ci --legacy-peer-deps || npm install --legacy-peer-deps; else npm install --legacy-peer-deps; fi \\ && npm cache clean --force COPY . . diff --git a/backend/src/clusters/clusters.service.ts b/backend/src/clusters/clusters.service.ts index ffe88a1..9794734 100644 --- a/backend/src/clusters/clusters.service.ts +++ b/backend/src/clusters/clusters.service.ts @@ -870,7 +870,7 @@ export class ClustersService implements OnModuleInit, OnModuleDestroy { const buildNs = this.registryService.getBuildNamespace(); const saName = this.configService.get('build.serviceAccount') || 'kaniko-builder'; - const registryUrl = this.registryService.getRegistryUrl(); + const registryHost = this.registryService.getRegistryHost(); this.logger.log(`Bootstrapping cluster — namespace: ${buildNs}`); @@ -1100,13 +1100,17 @@ export class ClustersService implements OnModuleInit, OnModuleDestroy { } } - await this.ensureK3sRegistryMirrors(appsApi, registryUrl); + await this.ensureK3sRegistryMirrors(coreApi, appsApi, registryHost); - this.logger.log(`✅ Cluster bootstrap complete — registry: ${registryUrl}`); + this.logger.log(`✅ Cluster bootstrap complete — registry: ${registryHost}`); } /** In-cluster registry mirror for k3s/containerd (HTTP). Removes legacy external-registry DaemonSet if present. */ - private async ensureK3sRegistryMirrors(appsApi: k8s.AppsV1Api, registryUrl: string): Promise { + private async ensureK3sRegistryMirrors( + coreApi: k8s.CoreV1Api, + appsApi: k8s.AppsV1Api, + registryHost: string, + ): Promise { const namespace = 'kube-system'; const legacyDs = 'cloudhost-k3s-registry-config'; try { @@ -1120,28 +1124,25 @@ export class ClustersService implements OnModuleInit, OnModuleDestroy { const { username, password } = this.registryService.getRegistryCredentials(); const dsName = 'cloudhost-k3s-registry-mirrors'; - // The mirror endpoint must be reachable by the node's containerd, which does - // NOT use cluster DNS — so we point it at the registry NodePort on loopback - // (http://127.0.0.1:) instead of the in-cluster service DNS name. - // Otherwise image pulls break whenever node-level resolution of - // *.svc.cluster.local is unavailable (e.g. right after a node restart). - const registryNodePort = 30500; - const nodePortHost = `127.0.0.1:${registryNodePort}`; + // containerd on the node does not use cluster DNS — mirror via ClusterIP (Harbor) + // or loopback NodePort (legacy in-cluster registry). + const mirrorEndpoint = await this.resolveK3sRegistryMirrorEndpoint(coreApi); + const mirrorHost = mirrorEndpoint.replace(/^https?:\/\//, ''); const configureScript = [ 'set -e', 'REG=/host/etc/rancher/k3s/registries.yaml', 'mkdir -p /host/etc/rancher/k3s', 'cat > /tmp/cloudhost-registries.yaml < { + const pushUrl = this.registryService.getRegistryPushUrl(); + const platformNs = this.configService.get('platform.namespace') || 'cloudhost'; + const harborCoreService = this.configService.get('registry.harborCoreService') || 'harbor-core'; + + if (pushUrl.includes('harbor-registry')) { + try { + const svc = await coreApi.readNamespacedService({ + name: harborCoreService, + namespace: platformNs, + }); + const clusterIp = svc.spec?.clusterIP; + if (clusterIp) { + return `http://${clusterIp}`; + } + } catch (err: any) { + this.logger.warn( + `Could not resolve ${harborCoreService} ClusterIP for k3s mirror: ${err.message}`, + ); + } + } + + const registryNodePort = 30500; + return `http://127.0.0.1:${registryNodePort}`; + } + private parseCpuToMillicores(cpu: string): number { if (!cpu || cpu === '0') return 0; if (cpu.endsWith('n')) return parseFloat(cpu) / 1_000_000; diff --git a/backend/src/deployments/deployments.service.ts b/backend/src/deployments/deployments.service.ts index c96b37c..5e10906 100644 --- a/backend/src/deployments/deployments.service.ts +++ b/backend/src/deployments/deployments.service.ts @@ -182,7 +182,7 @@ export class DeploymentsService implements OnModuleInit { }); await this.kubernetesService.waitForApplicationReady( app, - 600_000, + 1_200_000, () => this.isDeploymentCancelled(deploymentId), ); @@ -295,7 +295,7 @@ export class DeploymentsService implements OnModuleInit { }); await this.kubernetesService.waitForApplicationReady( app, - 600_000, + 1_200_000, () => this.isDeploymentCancelled(deploymentId), ); diff --git a/gitops/k3s/registries.yaml b/gitops/k3s/registries.yaml index 46ba8d8..60f6e1d 100644 --- a/gitops/k3s/registries.yaml +++ b/gitops/k3s/registries.yaml @@ -2,6 +2,8 @@ # Proxy-cache only works through harbor-core (not harbor-registry or Traefik /v2/ alone). # # Apply: ./scripts/apply-k3s-registries.sh +# Harbor EXT_ENDPOINT should be http://registry.abrban.com so OAuth realm uses HTTP +# (kubelet mirror hits harbor-core on :80; https://:443 times out). mirrors: registry.abrban.com: diff --git a/scripts/apply-k3s-registries.sh b/scripts/apply-k3s-registries.sh index 4050ed8..590e385 100755 --- a/scripts/apply-k3s-registries.sh +++ b/scripts/apply-k3s-registries.sh @@ -8,6 +8,16 @@ HARBOR_CORE_IP="${HARBOR_CORE_IP:-$(kubectl -n cloudhost get svc harbor-core -o REG_USER="${REG_USER:-harbor_registry_user}" REG_PASS="${REG_PASS:-$(kubectl -n cloudhost get secret harbor-core -o jsonpath='{.data.REGISTRY_CREDENTIAL_PASSWORD}' | base64 -d)}" +REG_PASS_B64="$(printf '%s' "$REG_PASS" | base64 | tr -d '\n')" + +kubectl -n "${NS}" delete pod k3s-registries-setup --ignore-not-found + +kubectl -n "${NS}" create secret generic k3s-registries-setup-env \ + --from-literal=HARBOR_CORE_IP="${HARBOR_CORE_IP}" \ + --from-literal=REG_USER="${REG_USER}" \ + --from-literal=REG_PASS_B64="${REG_PASS_B64}" \ + --dry-run=client -o yaml | kubectl apply -f - + kubectl -n "${NS}" delete pod k3s-registries-setup --ignore-not-found kubectl -n "${NS}" run k3s-registries-setup \ @@ -23,7 +33,8 @@ kubectl -n "${NS}" run k3s-registries-setup \ "name": "setup", "image": "rancher/mirrored-library-busybox:1.36.1", "securityContext": {"privileged": true}, - "command": ["sh", "-ec", "mkdir -p /host/etc/rancher/k3s && cat > /host/etc/rancher/k3s/registries.yaml <<'REGEOF'\nmirrors:\n registry.abrban.com:\n endpoint:\n - http://${HARBOR_CORE_IP}\n \\\"registry.cloudhost-builds.svc.cluster.local:5000\\\":\n endpoint:\n - \\\"http://127.0.0.1:30500\\\"\nconfigs:\n registry.abrban.com:\n auth:\n username: ${REG_USER}\n password: ${REG_PASS}\n \\\"${HARBOR_CORE_IP}\\\":\n auth:\n username: ${REG_USER}\n password: ${REG_PASS}\n \\\"registry.cloudhost-builds.svc.cluster.local:5000\\\":\n auth:\n username: admin\n password: \\\"\\\"\n \\\"127.0.0.1:30500\\\":\n auth:\n username: admin\n password: \\\"\\\"\nREGEOF\nnsenter -t 1 -m -u -n -i -- systemctl restart k3s 2>/dev/null || true\necho k3s-restarted\nsleep 30"], + "envFrom": [{"secretRef": {"name": "k3s-registries-setup-env"}}], + "command": ["sh", "-ec", "REG_PASS=\$(echo \"\$REG_PASS_B64\" | base64 -d); mkdir -p /host/etc/rancher/k3s && cat > /host/etc/rancher/k3s/registries.yaml </dev/null || true\necho k3s-restarted\nsleep 30"], "volumeMounts": [{"name": "host", "mountPath": "/host"}] }], "volumes": [{"name": "host", "hostPath": {"path": "/"}}]