Fix kubelet registry pulls via node cluster DNS and in-cluster mirrors.

Bootstrap configures systemd-resolved for *.cluster.local, installs k3s registries.yaml for the internal registry only, removes the legacy external-registry DaemonSet, and aligns Helm REGISTRY_PULL_URL with the in-cluster registry URL.

Co-authored-by: Cursor <cursoragent@cursor.com>

backend/.env.example

@@ -23,6 +23,8 @@ REDIS_PORT=6379
 # In-cluster Docker Registry (Kaniko push + app image pull — same URL)
 REGISTRY_URL=registry.cloudhost-builds.svc.cluster.local:5000
 # REGISTRY_PULL_URL=registry.cloudhost-builds.svc.cluster.local:5000
+# Cluster bootstrap installs cloudhost-node-cluster-dns so nodes resolve *.cluster.local
+# (required for kubelet image pulls via registry.<ns>.svc.cluster.local).
 REGISTRY_USERNAME=admin
 REGISTRY_PASSWORD=registry_secret
 

backend/helm/cloudhost-platform/values.yaml

@@ -53,7 +53,7 @@ backend:
     JWT_REFRESH_EXPIRES_IN: 7d
     PLATFORM_DOMAIN: apps.cloudhost.local
     REGISTRY_URL: registry.cloudhost-builds.svc.cluster.local:5000
-    REGISTRY_PULL_URL: localhost:30500
+    REGISTRY_PULL_URL: registry.cloudhost-builds.svc.cluster.local:5000
     BUILD_NAMESPACE: cloudhost-builds
     BUILD_SERVICE_ACCOUNT: kaniko-builder
     UPLOAD_DIR: /app/uploads

backend/src/clusters/clusters.service.ts

@@ -1005,9 +1005,189 @@ export class ClustersService implements OnModuleInit, OnModuleDestroy {
       }
     }
 
+    await this.ensureNodeClusterDns(coreApi, appsApi);
+    await this.ensureK3sRegistryMirrors(appsApi, registryUrl);
+
     this.logger.log(`✅ Cluster bootstrap complete — registry: ${registryUrl}`);
   }
 
+  /**
+   * Kubelet/containerd pull images on the host network stack, which uses the node's
+   * resolver (often systemd-resolved) — not pod DNS. Forward *.cluster.local to CoreDNS
+   * so registry.cloudhost-builds.svc.cluster.local resolves during image pulls.
+   */
+  private async ensureNodeClusterDns(
+    coreApi: k8s.CoreV1Api,
+    appsApi: k8s.AppsV1Api,
+  ): Promise<void> {
+    const dsName = 'cloudhost-node-cluster-dns';
+    const namespace = 'kube-system';
+    let clusterDnsIp = '10.43.0.10';
+    try {
+      const dnsSvc = await coreApi.readNamespacedService('kube-dns', namespace);
+      clusterDnsIp = dnsSvc.body.spec?.clusterIP || clusterDnsIp;
+    } catch (err: any) {
+      this.logger.warn(
+        `Could not read kube-dns ClusterIP (${err.message}); using ${clusterDnsIp}`,
+      );
+    }
+
+    const configureScript = [
+      'set -e',
+      'CONF=/host/etc/systemd/resolved.conf.d/k8s-cluster-dns.conf',
+      'mkdir -p /host/etc/systemd/resolved.conf.d',
+      `cat > /tmp/k8s-cluster-dns.conf <<'EOF'`,
+      '[Resolve]',
+      `DNS=${clusterDnsIp}`,
+      'Domains=~cluster.local',
+      'EOF',
+      'if [ ! -f "$CONF" ] || ! cmp -s /tmp/k8s-cluster-dns.conf "$CONF"; then',
+      '  cp /tmp/k8s-cluster-dns.conf "$CONF"',
+      '  echo "Updated k8s-cluster-dns.conf"',
+      '  if nsenter -t 1 -m -u -i -n -p -- systemctl is-active systemd-resolved >/dev/null 2>&1; then',
+      '    nsenter -t 1 -m -u -i -n -p -- systemctl restart systemd-resolved',
+      '  fi',
+      'fi',
+      'sleep infinity',
+    ].join('\n');
+
+    const daemonSet: k8s.V1DaemonSet = {
+      metadata: {
+        name: dsName,
+        namespace,
+        labels: { 'app.kubernetes.io/managed-by': 'cloudhost' },
+      },
+      spec: {
+        selector: { matchLabels: { app: dsName } },
+        template: {
+          metadata: { labels: { app: dsName } },
+          spec: {
+            hostPID: true,
+            tolerations: [{ operator: 'Exists' }],
+            containers: [
+              {
+                name: 'configure',
+                image: 'rancher/mirrored-library-busybox:1.36.1',
+                command: ['/bin/sh', '-ec'],
+                args: [configureScript],
+                securityContext: { privileged: true },
+                volumeMounts: [{ name: 'etc', mountPath: '/host/etc' }],
+              },
+            ],
+            volumes: [
+              {
+                name: 'etc',
+                hostPath: { path: '/etc', type: 'Directory' },
+              },
+            ],
+          },
+        },
+      },
+    };
+
+    try {
+      await appsApi.readNamespacedDaemonSet(dsName, namespace);
+      await appsApi.replaceNamespacedDaemonSet(dsName, namespace, daemonSet);
+      this.logger.log(`Updated DaemonSet "${dsName}" (cluster DNS ${clusterDnsIp})`);
+    } catch (err: any) {
+      if (err.statusCode === 404 || err.body?.code === 404) {
+        await appsApi.createNamespacedDaemonSet(namespace, daemonSet);
+        this.logger.log(`Created DaemonSet "${dsName}" (cluster DNS ${clusterDnsIp})`);
+      } else {
+        throw err;
+      }
+    }
+  }
+
+  /** In-cluster registry mirror for k3s/containerd (HTTP). Removes legacy external-registry DaemonSet if present. */
+  private async ensureK3sRegistryMirrors(
+    appsApi: k8s.AppsV1Api,
+    registryUrl: string,
+  ): Promise<void> {
+    const namespace = 'kube-system';
+    const legacyDs = 'cloudhost-k3s-registry-config';
+    try {
+      await appsApi.deleteNamespacedDaemonSet(legacyDs, namespace);
+      this.logger.log(`Removed legacy DaemonSet "${legacyDs}"`);
+    } catch (err: any) {
+      if (err.statusCode !== 404 && err.body?.code !== 404) {
+        this.logger.warn(`Could not delete legacy DaemonSet "${legacyDs}": ${err.message}`);
+      }
+    }
+
+    const { username, password } = this.registryService.getRegistryCredentials();
+    const dsName = 'cloudhost-k3s-registry-mirrors';
+    const configureScript = [
+      'set -e',
+      'REG=/host/etc/rancher/k3s/registries.yaml',
+      'mkdir -p /host/etc/rancher/k3s',
+      'cat > /tmp/cloudhost-registries.yaml <<EOFREG',
+      'mirrors:',
+      `  "${registryUrl}":`,
+      '    endpoint:',
+      `      - "http://${registryUrl}"`,
+      'configs:',
+      `  "${registryUrl}":`,
+      '    auth:',
+      `      username: ${JSON.stringify(username)}`,
+      `      password: ${JSON.stringify(password)}`,
+      'EOFREG',
+      'if [ ! -f "$REG" ] || ! cmp -s /tmp/cloudhost-registries.yaml "$REG"; then',
+      '  cp /tmp/cloudhost-registries.yaml "$REG"',
+      '  echo "Updated registries.yaml"',
+      '  nsenter -t 1 -m -u -i -n -p -- systemctl restart k3s 2>/dev/null || true',
+      'fi',
+      'sleep infinity',
+    ].join('\n');
+
+    const daemonSet: k8s.V1DaemonSet = {
+      metadata: {
+        name: dsName,
+        namespace,
+        labels: { 'app.kubernetes.io/managed-by': 'cloudhost' },
+      },
+      spec: {
+        selector: { matchLabels: { app: dsName } },
+        template: {
+          metadata: { labels: { app: dsName } },
+          spec: {
+            hostPID: true,
+            tolerations: [{ operator: 'Exists' }],
+            containers: [
+              {
+                name: 'configure',
+                image: 'rancher/mirrored-library-busybox:1.36.1',
+                command: ['/bin/sh', '-ec'],
+                args: [configureScript],
+                securityContext: { privileged: true },
+                volumeMounts: [{ name: 'etc', mountPath: '/host/etc' }],
+              },
+            ],
+            volumes: [
+              {
+                name: 'etc',
+                hostPath: { path: '/etc', type: 'Directory' },
+              },
+            ],
+          },
+        },
+      },
+    };
+
+    try {
+      await appsApi.readNamespacedDaemonSet(dsName, namespace);
+      await appsApi.replaceNamespacedDaemonSet(dsName, namespace, daemonSet);
+      this.logger.log(`Updated DaemonSet "${dsName}"`);
+    } catch (err: any) {
+      if (err.statusCode === 404 || err.body?.code === 404) {
+        await appsApi.createNamespacedDaemonSet(namespace, daemonSet);
+        this.logger.log(`Created DaemonSet "${dsName}"`);
+      } else {
+        throw err;
+      }
+    }
+  }
+
   private parseCpuToMillicores(cpu: string): number {
     if (!cpu || cpu === '0') return 0;
     if (cpu.endsWith('n')) return parseFloat(cpu) / 1_000_000;