abrban/cloud-host
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix app image pulls and Harbor kubelet auth for user workloads.

Browse Source 
Route k3s registry mirrors through harbor-core ClusterIP with hostname-only auth keys, use HTTP EXT_ENDPOINT so OAuth tokens work on port 80, extend deploy readiness timeout, and harden Kaniko build/dockerfile fallbacks.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
keyhan
2026-07-10 18:38:05 +03:30
parent 54ab2f2f05
commit ec72ee4fca
7 changed files with 160 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/helm/cloudhost-harbor/values-registry.abrban.com.yaml
+1 -1
View File
@@ -4,7 +4,7 @@
## Install:
## helm upgrade --install harbor harbor/harbor -n cloudhost -f backend/helm/cloudhost-harbor/values-registry.abrban.com.yaml
##
externalURL: https://registry.abrban.com
externalURL: http://registry.abrban.com
proxy:
# Values are injected by install script from `cloudhost/registry-egress-proxy`.
backend/src/build/build.service.spec.ts
+82
View File
@@ -49,6 +49,41 @@ describe('BuildService', () => {
service = module.get(BuildService);
});
describe('baseImage', () => {
it('prefixes Docker Hub library images (tag colon must not block mirroring)', () => {
const config = (service as any).configService as { get: jest.Mock };
config.get.mockImplementation((key: string) => {
if (key === 'build.baseImageRegistry') return 'registry.abrban.com/abrban';
return undefined;
});
expect((service as any).baseImage('node:20-alpine')).toBe(
'registry.abrban.com/abrban/node:20-alpine',
);
expect((service as any).baseImage('alpine:3.19')).toBe(
'registry.abrban.com/abrban/alpine:3.19',
);
});
it('leaves images that already reference an external registry unchanged', () => {
const config = (service as any).configService as { get: jest.Mock };
config.get.mockImplementation((key: string) => {
if (key === 'build.baseImageRegistry') return 'registry.abrban.com/abrban';
return undefined;
});
expect((service as any).baseImage('mcr.microsoft.com/dotnet/sdk:8.0')).toBe(
'mcr.microsoft.com/dotnet/sdk:8.0',
);
expect((service as any).baseImage('registry.abrban.com/abrban/node:20-alpine')).toBe(
'registry.abrban.com/abrban/node:20-alpine',
);
expect((service as any).baseImage('localhost:5000/myapp:latest')).toBe(
'localhost:5000/myapp:latest',
);
});
});
describe('generateDockerfile', () => {
it('generates Go Dockerfile with requested runtime version', () => {
const app = {
@@ -78,6 +113,53 @@ describe('BuildService', () => {
expect(dockerfile).toContain('go build -a -installsuffix cgo -ldflags="-w -s" -o main ./cmd/server');
});
it('generates Node.js Dockerfile with mirrored base images when registry prefix is set', async () => {
const module = await Test.createTestingModule({
providers: [
BuildService,
{
provide: ConfigService,
useValue: {
get: jest.fn((key: string) => {
const map: Record<string, string> = {
'build.namespace': 'cloudhost-builds',
'build.serviceAccount': 'kaniko-builder',
'build.baseImageRegistry': 'registry.abrban.com/abrban',
'registry.url': 'registry.local:5000',
};
return map[key];
}),
},
},
{ provide: ClustersService, useValue: {} },
{
provide: BuildProgressStore,
useValue: { get: jest.fn(), set: jest.fn(), clear: jest.fn() },
},
{ provide: RegistryService, useValue: {} },
{
provide: SourceStorageService,
useValue: {
isObjectStorage: () => false,
materializeToTempFile: jest.fn(),
getSize: jest.fn(),
},
},
],
}).compile();
const mirrored = module.get(BuildService);
const app = {
runtime: AppRuntime.NODEJS,
runtimeVersion: '20',
} as Application;
const dockerfile = (mirrored as any).generateDockerfile(app) as string;
expect(dockerfile).toContain('FROM registry.abrban.com/abrban/node:20-alpine');
expect(dockerfile).toContain('EXPOSE 3000');
});
it('generates Node.js Dockerfile with default port', () => {
const app = {
runtime: AppRuntime.NODEJS,
backend/src/build/build.service.ts
+15 -5
View File
@@ -73,9 +73,19 @@ export class BuildService {
private baseImage(image: string): string {
const prefix = this.configService.get<string>('build.baseImageRegistry');
if (!prefix) return image;
const firstSegment = image.split('/')[0];
const hasRegistry = firstSegment.includes('.') || firstSegment.includes(':');
if (hasRegistry) return image;
// Official Docker Hub library images have no slash (node:20-alpine, alpine:3.19).
// Do not treat the tag colon as a registry port — that was skipping the mirror.
if (!image.includes('/')) {
return `${prefix}/${image}`;
}
const registryHost = image.split('/')[0];
if (registryHost.includes('.') || registryHost.includes(':') || registryHost === 'localhost') {
return image;
}
// Docker Hub org/user image (e.g. bitnami/redis:7) — mirror through the prefix.
return `${prefix}/${image}`;
}
@@ -1195,8 +1205,8 @@ export class BuildService {
FROM ${this.baseImage(`node:${nodeVersion}-alpine`)} AS builder
WORKDIR /app
COPY package*.json ./
# Reproducible install from the lockfile when present
RUN if [ -f package-lock.json ]; then npm ci --legacy-peer-deps; else npm install --legacy-peer-deps; fi \\
# Prefer lockfile; fall back when package.json and lockfile are out of sync
RUN if [ -f package-lock.json ]; then npm ci --legacy-peer-deps || npm install --legacy-peer-deps; else npm install --legacy-peer-deps; fi \\
&& npm cache clean --force
COPY . .
backend/src/clusters/clusters.service.ts
+46 -15
View File
@@ -870,7 +870,7 @@ export class ClustersService implements OnModuleInit, OnModuleDestroy {
const buildNs = this.registryService.getBuildNamespace();
const saName = this.configService.get<string>('build.serviceAccount') || 'kaniko-builder';
const registryUrl = this.registryService.getRegistryUrl();
const registryHost = this.registryService.getRegistryHost();
this.logger.log(`Bootstrapping cluster — namespace: ${buildNs}`);
@@ -1100,13 +1100,17 @@ export class ClustersService implements OnModuleInit, OnModuleDestroy {
}
}
await this.ensureK3sRegistryMirrors(appsApi, registryUrl);
await this.ensureK3sRegistryMirrors(coreApi, appsApi, registryHost);
this.logger.log(`✅ Cluster bootstrap complete — registry: ${registryUrl}`);
this.logger.log(`✅ Cluster bootstrap complete — registry: ${registryHost}`);
}
/** In-cluster registry mirror for k3s/containerd (HTTP). Removes legacy external-registry DaemonSet if present. */
private async ensureK3sRegistryMirrors(appsApi: k8s.AppsV1Api, registryUrl: string): Promise<void> {
private async ensureK3sRegistryMirrors(
coreApi: k8s.CoreV1Api,
appsApi: k8s.AppsV1Api,
registryHost: string,
): Promise<void> {
const namespace = 'kube-system';
const legacyDs = 'cloudhost-k3s-registry-config';
try {
@@ -1120,28 +1124,25 @@ export class ClustersService implements OnModuleInit, OnModuleDestroy {
const { username, password } = this.registryService.getRegistryCredentials();
const dsName = 'cloudhost-k3s-registry-mirrors';
// The mirror endpoint must be reachable by the node's containerd, which does
// NOT use cluster DNS — so we point it at the registry NodePort on loopback
// (http://127.0.0.1:<nodePort>) instead of the in-cluster service DNS name.
// Otherwise image pulls break whenever node-level resolution of
// *.svc.cluster.local is unavailable (e.g. right after a node restart).
const registryNodePort = 30500;
const nodePortHost = `127.0.0.1:${registryNodePort}`;
// containerd on the node does not use cluster DNS — mirror via ClusterIP (Harbor)
// or loopback NodePort (legacy in-cluster registry).
const mirrorEndpoint = await this.resolveK3sRegistryMirrorEndpoint(coreApi);
const mirrorHost = mirrorEndpoint.replace(/^https?:\/\//, '');
const configureScript = [
'set -e',
'REG=/host/etc/rancher/k3s/registries.yaml',
'mkdir -p /host/etc/rancher/k3s',
'cat > /tmp/cloudhost-registries.yaml <<EOFREG',
'mirrors:',
` "${registryUrl}":`,
` "${registryHost}":`,
' endpoint:',
` - "http://${nodePortHost}"`,
` - "${mirrorEndpoint}"`,
'configs:',
` "${registryUrl}":`,
` "${registryHost}":`,
' auth:',
` username: ${JSON.stringify(username)}`,
` password: ${JSON.stringify(password)}`,
` "${nodePortHost}":`,
` "${mirrorHost}":`,
' auth:',
` username: ${JSON.stringify(username)}`,
` password: ${JSON.stringify(password)}`,
@@ -1206,6 +1207,36 @@ export class ClustersService implements OnModuleInit, OnModuleDestroy {
}
}
/**
* containerd on nodes cannot resolve *.svc.cluster.local — use ClusterIP for
* Harbor (harbor-core HTTP) or legacy registry NodePort on loopback.
*/
private async resolveK3sRegistryMirrorEndpoint(coreApi: k8s.CoreV1Api): Promise<string> {
const pushUrl = this.registryService.getRegistryPushUrl();
const platformNs = this.configService.get<string>('platform.namespace') || 'cloudhost';
const harborCoreService = this.configService.get<string>('registry.harborCoreService') || 'harbor-core';
if (pushUrl.includes('harbor-registry')) {
try {
const svc = await coreApi.readNamespacedService({
name: harborCoreService,
namespace: platformNs,
});
const clusterIp = svc.spec?.clusterIP;
if (clusterIp) {
return `http://${clusterIp}`;
}
} catch (err: any) {
this.logger.warn(
`Could not resolve ${harborCoreService} ClusterIP for k3s mirror: ${err.message}`,
);
}
}
const registryNodePort = 30500;
return `http://127.0.0.1:${registryNodePort}`;
}
private parseCpuToMillicores(cpu: string): number {
if (!cpu || cpu === '0') return 0;
if (cpu.endsWith('n')) return parseFloat(cpu) / 1_000_000;
backend/src/deployments/deployments.service.ts
+2 -2
View File
@@ -182,7 +182,7 @@ export class DeploymentsService implements OnModuleInit {
});
await this.kubernetesService.waitForApplicationReady(
app,
600_000,
1_200_000,
() => this.isDeploymentCancelled(deploymentId),
);
@@ -295,7 +295,7 @@ export class DeploymentsService implements OnModuleInit {
});
await this.kubernetesService.waitForApplicationReady(
app,
600_000,
1_200_000,
() => this.isDeploymentCancelled(deploymentId),
);
gitops/k3s/registries.yaml
+2
View File
@@ -2,6 +2,8 @@
# Proxy-cache only works through harbor-core (not harbor-registry or Traefik /v2/ alone).
#
# Apply: ./scripts/apply-k3s-registries.sh
# Harbor EXT_ENDPOINT should be http://registry.abrban.com so OAuth realm uses HTTP
# (kubelet mirror hits harbor-core on :80; https://<clusterIP>:443 times out).
mirrors:
registry.abrban.com:
scripts/apply-k3s-registries.sh
+12 -1
View File
@@ -8,6 +8,16 @@ HARBOR_CORE_IP="${HARBOR_CORE_IP:-$(kubectl -n cloudhost get svc harbor-core -o
REG_USER="${REG_USER:-harbor_registry_user}"
REG_PASS="${REG_PASS:-$(kubectl -n cloudhost get secret harbor-core -o jsonpath='{.data.REGISTRY_CREDENTIAL_PASSWORD}' | base64 -d)}"
REG_PASS_B64="$(printf '%s' "$REG_PASS" | base64 | tr -d '\n')"
kubectl -n "${NS}" delete pod k3s-registries-setup --ignore-not-found
kubectl -n "${NS}" create secret generic k3s-registries-setup-env \
--from-literal=HARBOR_CORE_IP="${HARBOR_CORE_IP}" \
--from-literal=REG_USER="${REG_USER}" \
--from-literal=REG_PASS_B64="${REG_PASS_B64}" \
--dry-run=client -o yaml | kubectl apply -f -
kubectl -n "${NS}" delete pod k3s-registries-setup --ignore-not-found
kubectl -n "${NS}" run k3s-registries-setup \
@@ -23,7 +33,8 @@ kubectl -n "${NS}" run k3s-registries-setup \
"name": "setup",
"image": "rancher/mirrored-library-busybox:1.36.1",
"securityContext": {"privileged": true},
"command": ["sh", "-ec", "mkdir -p /host/etc/rancher/k3s && cat > /host/etc/rancher/k3s/registries.yaml <<'REGEOF'\nmirrors:\n registry.abrban.com:\n endpoint:\n - http://${HARBOR_CORE_IP}\n \\\"registry.cloudhost-builds.svc.cluster.local:5000\\\":\n endpoint:\n - \\\"http://127.0.0.1:30500\\\"\nconfigs:\n registry.abrban.com:\n auth:\n username: ${REG_USER}\n password: ${REG_PASS}\n \\\"${HARBOR_CORE_IP}\\\":\n auth:\n username: ${REG_USER}\n password: ${REG_PASS}\n \\\"registry.cloudhost-builds.svc.cluster.local:5000\\\":\n auth:\n username: admin\n password: \\\"\\\"\n \\\"127.0.0.1:30500\\\":\n auth:\n username: admin\n password: \\\"\\\"\nREGEOF\nnsenter -t 1 -m -u -n -i -- systemctl restart k3s 2>/dev/null || true\necho k3s-restarted\nsleep 30"],
"envFrom": [{"secretRef": {"name": "k3s-registries-setup-env"}}],
"command": ["sh", "-ec", "REG_PASS=\$(echo \"\$REG_PASS_B64\" | base64 -d); mkdir -p /host/etc/rancher/k3s && cat > /host/etc/rancher/k3s/registries.yaml <<REGEOF\nmirrors:\n registry.abrban.com:\n endpoint:\n - http://\${HARBOR_CORE_IP}\n \\\"registry.cloudhost-builds.svc.cluster.local:5000\\\":\n endpoint:\n - \\\"http://127.0.0.1:30500\\\"\nconfigs:\n registry.abrban.com:\n auth:\n username: \${REG_USER}\n password: \${REG_PASS}\n \\\"\${HARBOR_CORE_IP}\\\":\n auth:\n username: \${REG_USER}\n password: \${REG_PASS}\n \\\"registry.cloudhost-builds.svc.cluster.local:5000\\\":\n auth:\n username: admin\n password: \\\"\\\"\n \\\"127.0.0.1:30500\\\":\n auth:\n username: admin\n password: \\\"\\\"\nREGEOF\nnsenter -t 1 -m -u -n -i -- systemctl restart k3s 2>/dev/null || true\necho k3s-restarted\nsleep 30"],
"volumeMounts": [{"name": "host", "mountPath": "/host"}]
}],
"volumes": [{"name": "host", "hostPath": {"path": "/"}}]