From 3435eff25616b46777548af739051efc97bac500 Mon Sep 17 00:00:00 2001 From: keyhan Date: Wed, 27 May 2026 11:36:07 +0330 Subject: [PATCH] Fix kubelet registry pulls via node cluster DNS and in-cluster mirrors. Bootstrap configures systemd-resolved for *.cluster.local, installs k3s registries.yaml for the internal registry only, removes the legacy external-registry DaemonSet, and aligns Helm REGISTRY_PULL_URL with the in-cluster registry URL. Co-authored-by: Cursor --- backend/.env.example | 2 + backend/helm/cloudhost-platform/values.yaml | 2 +- backend/src/clusters/clusters.service.ts | 180 ++++++++++++++++++++ 3 files changed, 183 insertions(+), 1 deletion(-) diff --git a/backend/.env.example b/backend/.env.example index 875de11..69d6617 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -23,6 +23,8 @@ REDIS_PORT=6379 # In-cluster Docker Registry (Kaniko push + app image pull — same URL) REGISTRY_URL=registry.cloudhost-builds.svc.cluster.local:5000 # REGISTRY_PULL_URL=registry.cloudhost-builds.svc.cluster.local:5000 +# Cluster bootstrap installs cloudhost-node-cluster-dns so nodes resolve *.cluster.local +# (required for kubelet image pulls via registry..svc.cluster.local). REGISTRY_USERNAME=admin REGISTRY_PASSWORD=registry_secret diff --git a/backend/helm/cloudhost-platform/values.yaml b/backend/helm/cloudhost-platform/values.yaml index e989d4f..dd726e7 100644 --- a/backend/helm/cloudhost-platform/values.yaml +++ b/backend/helm/cloudhost-platform/values.yaml @@ -53,7 +53,7 @@ backend: JWT_REFRESH_EXPIRES_IN: 7d PLATFORM_DOMAIN: apps.cloudhost.local REGISTRY_URL: registry.cloudhost-builds.svc.cluster.local:5000 - REGISTRY_PULL_URL: localhost:30500 + REGISTRY_PULL_URL: registry.cloudhost-builds.svc.cluster.local:5000 BUILD_NAMESPACE: cloudhost-builds BUILD_SERVICE_ACCOUNT: kaniko-builder UPLOAD_DIR: /app/uploads diff --git a/backend/src/clusters/clusters.service.ts b/backend/src/clusters/clusters.service.ts index f713a78..9413402 100644 --- a/backend/src/clusters/clusters.service.ts +++ b/backend/src/clusters/clusters.service.ts @@ -1005,9 +1005,189 @@ export class ClustersService implements OnModuleInit, OnModuleDestroy { } } + await this.ensureNodeClusterDns(coreApi, appsApi); + await this.ensureK3sRegistryMirrors(appsApi, registryUrl); + this.logger.log(`✅ Cluster bootstrap complete — registry: ${registryUrl}`); } + /** + * Kubelet/containerd pull images on the host network stack, which uses the node's + * resolver (often systemd-resolved) — not pod DNS. Forward *.cluster.local to CoreDNS + * so registry.cloudhost-builds.svc.cluster.local resolves during image pulls. + */ + private async ensureNodeClusterDns( + coreApi: k8s.CoreV1Api, + appsApi: k8s.AppsV1Api, + ): Promise { + const dsName = 'cloudhost-node-cluster-dns'; + const namespace = 'kube-system'; + let clusterDnsIp = '10.43.0.10'; + try { + const dnsSvc = await coreApi.readNamespacedService('kube-dns', namespace); + clusterDnsIp = dnsSvc.body.spec?.clusterIP || clusterDnsIp; + } catch (err: any) { + this.logger.warn( + `Could not read kube-dns ClusterIP (${err.message}); using ${clusterDnsIp}`, + ); + } + + const configureScript = [ + 'set -e', + 'CONF=/host/etc/systemd/resolved.conf.d/k8s-cluster-dns.conf', + 'mkdir -p /host/etc/systemd/resolved.conf.d', + `cat > /tmp/k8s-cluster-dns.conf <<'EOF'`, + '[Resolve]', + `DNS=${clusterDnsIp}`, + 'Domains=~cluster.local', + 'EOF', + 'if [ ! -f "$CONF" ] || ! cmp -s /tmp/k8s-cluster-dns.conf "$CONF"; then', + ' cp /tmp/k8s-cluster-dns.conf "$CONF"', + ' echo "Updated k8s-cluster-dns.conf"', + ' if nsenter -t 1 -m -u -i -n -p -- systemctl is-active systemd-resolved >/dev/null 2>&1; then', + ' nsenter -t 1 -m -u -i -n -p -- systemctl restart systemd-resolved', + ' fi', + 'fi', + 'sleep infinity', + ].join('\n'); + + const daemonSet: k8s.V1DaemonSet = { + metadata: { + name: dsName, + namespace, + labels: { 'app.kubernetes.io/managed-by': 'cloudhost' }, + }, + spec: { + selector: { matchLabels: { app: dsName } }, + template: { + metadata: { labels: { app: dsName } }, + spec: { + hostPID: true, + tolerations: [{ operator: 'Exists' }], + containers: [ + { + name: 'configure', + image: 'rancher/mirrored-library-busybox:1.36.1', + command: ['/bin/sh', '-ec'], + args: [configureScript], + securityContext: { privileged: true }, + volumeMounts: [{ name: 'etc', mountPath: '/host/etc' }], + }, + ], + volumes: [ + { + name: 'etc', + hostPath: { path: '/etc', type: 'Directory' }, + }, + ], + }, + }, + }, + }; + + try { + await appsApi.readNamespacedDaemonSet(dsName, namespace); + await appsApi.replaceNamespacedDaemonSet(dsName, namespace, daemonSet); + this.logger.log(`Updated DaemonSet "${dsName}" (cluster DNS ${clusterDnsIp})`); + } catch (err: any) { + if (err.statusCode === 404 || err.body?.code === 404) { + await appsApi.createNamespacedDaemonSet(namespace, daemonSet); + this.logger.log(`Created DaemonSet "${dsName}" (cluster DNS ${clusterDnsIp})`); + } else { + throw err; + } + } + } + + /** In-cluster registry mirror for k3s/containerd (HTTP). Removes legacy external-registry DaemonSet if present. */ + private async ensureK3sRegistryMirrors( + appsApi: k8s.AppsV1Api, + registryUrl: string, + ): Promise { + const namespace = 'kube-system'; + const legacyDs = 'cloudhost-k3s-registry-config'; + try { + await appsApi.deleteNamespacedDaemonSet(legacyDs, namespace); + this.logger.log(`Removed legacy DaemonSet "${legacyDs}"`); + } catch (err: any) { + if (err.statusCode !== 404 && err.body?.code !== 404) { + this.logger.warn(`Could not delete legacy DaemonSet "${legacyDs}": ${err.message}`); + } + } + + const { username, password } = this.registryService.getRegistryCredentials(); + const dsName = 'cloudhost-k3s-registry-mirrors'; + const configureScript = [ + 'set -e', + 'REG=/host/etc/rancher/k3s/registries.yaml', + 'mkdir -p /host/etc/rancher/k3s', + 'cat > /tmp/cloudhost-registries.yaml </dev/null || true', + 'fi', + 'sleep infinity', + ].join('\n'); + + const daemonSet: k8s.V1DaemonSet = { + metadata: { + name: dsName, + namespace, + labels: { 'app.kubernetes.io/managed-by': 'cloudhost' }, + }, + spec: { + selector: { matchLabels: { app: dsName } }, + template: { + metadata: { labels: { app: dsName } }, + spec: { + hostPID: true, + tolerations: [{ operator: 'Exists' }], + containers: [ + { + name: 'configure', + image: 'rancher/mirrored-library-busybox:1.36.1', + command: ['/bin/sh', '-ec'], + args: [configureScript], + securityContext: { privileged: true }, + volumeMounts: [{ name: 'etc', mountPath: '/host/etc' }], + }, + ], + volumes: [ + { + name: 'etc', + hostPath: { path: '/etc', type: 'Directory' }, + }, + ], + }, + }, + }, + }; + + try { + await appsApi.readNamespacedDaemonSet(dsName, namespace); + await appsApi.replaceNamespacedDaemonSet(dsName, namespace, daemonSet); + this.logger.log(`Updated DaemonSet "${dsName}"`); + } catch (err: any) { + if (err.statusCode === 404 || err.body?.code === 404) { + await appsApi.createNamespacedDaemonSet(namespace, daemonSet); + this.logger.log(`Created DaemonSet "${dsName}"`); + } else { + throw err; + } + } + } + private parseCpuToMillicores(cpu: string): number { if (!cpu || cpu === '0') return 0; if (cpu.endsWith('n')) return parseFloat(cpu) / 1_000_000;