Fix kubelet registry pulls via node cluster DNS and in-cluster mirrors.
Bootstrap configures systemd-resolved for *.cluster.local, installs k3s registries.yaml for the internal registry only, removes the legacy external-registry DaemonSet, and aligns Helm REGISTRY_PULL_URL with the in-cluster registry URL. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
keyhan
@@ -1005,9 +1005,189 @@ export class ClustersService implements OnModuleInit, OnModuleDestroy {
|
||||
}
|
||||
}
|
||||
|
||||
await this.ensureNodeClusterDns(coreApi, appsApi);
|
||||
await this.ensureK3sRegistryMirrors(appsApi, registryUrl);
|
||||
|
||||
this.logger.log(`✅ Cluster bootstrap complete — registry: ${registryUrl}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Kubelet/containerd pull images on the host network stack, which uses the node's
|
||||
* resolver (often systemd-resolved) — not pod DNS. Forward *.cluster.local to CoreDNS
|
||||
* so registry.cloudhost-builds.svc.cluster.local resolves during image pulls.
|
||||
*/
|
||||
private async ensureNodeClusterDns(
|
||||
coreApi: k8s.CoreV1Api,
|
||||
appsApi: k8s.AppsV1Api,
|
||||
): Promise<void> {
|
||||
const dsName = 'cloudhost-node-cluster-dns';
|
||||
const namespace = 'kube-system';
|
||||
let clusterDnsIp = '10.43.0.10';
|
||||
try {
|
||||
const dnsSvc = await coreApi.readNamespacedService('kube-dns', namespace);
|
||||
clusterDnsIp = dnsSvc.body.spec?.clusterIP || clusterDnsIp;
|
||||
} catch (err: any) {
|
||||
this.logger.warn(
|
||||
`Could not read kube-dns ClusterIP (${err.message}); using ${clusterDnsIp}`,
|
||||
);
|
||||
}
|
||||
|
||||
const configureScript = [
|
||||
'set -e',
|
||||
'CONF=/host/etc/systemd/resolved.conf.d/k8s-cluster-dns.conf',
|
||||
'mkdir -p /host/etc/systemd/resolved.conf.d',
|
||||
`cat > /tmp/k8s-cluster-dns.conf <<'EOF'`,
|
||||
'[Resolve]',
|
||||
`DNS=${clusterDnsIp}`,
|
||||
'Domains=~cluster.local',
|
||||
'EOF',
|
||||
'if [ ! -f "$CONF" ] || ! cmp -s /tmp/k8s-cluster-dns.conf "$CONF"; then',
|
||||
' cp /tmp/k8s-cluster-dns.conf "$CONF"',
|
||||
' echo "Updated k8s-cluster-dns.conf"',
|
||||
' if nsenter -t 1 -m -u -i -n -p -- systemctl is-active systemd-resolved >/dev/null 2>&1; then',
|
||||
' nsenter -t 1 -m -u -i -n -p -- systemctl restart systemd-resolved',
|
||||
' fi',
|
||||
'fi',
|
||||
'sleep infinity',
|
||||
].join('\n');
|
||||
|
||||
const daemonSet: k8s.V1DaemonSet = {
|
||||
metadata: {
|
||||
name: dsName,
|
||||
namespace,
|
||||
labels: { 'app.kubernetes.io/managed-by': 'cloudhost' },
|
||||
},
|
||||
spec: {
|
||||
selector: { matchLabels: { app: dsName } },
|
||||
template: {
|
||||
metadata: { labels: { app: dsName } },
|
||||
spec: {
|
||||
hostPID: true,
|
||||
tolerations: [{ operator: 'Exists' }],
|
||||
containers: [
|
||||
{
|
||||
name: 'configure',
|
||||
image: 'rancher/mirrored-library-busybox:1.36.1',
|
||||
command: ['/bin/sh', '-ec'],
|
||||
args: [configureScript],
|
||||
securityContext: { privileged: true },
|
||||
volumeMounts: [{ name: 'etc', mountPath: '/host/etc' }],
|
||||
},
|
||||
],
|
||||
volumes: [
|
||||
{
|
||||
name: 'etc',
|
||||
hostPath: { path: '/etc', type: 'Directory' },
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
await appsApi.readNamespacedDaemonSet(dsName, namespace);
|
||||
await appsApi.replaceNamespacedDaemonSet(dsName, namespace, daemonSet);
|
||||
this.logger.log(`Updated DaemonSet "${dsName}" (cluster DNS ${clusterDnsIp})`);
|
||||
} catch (err: any) {
|
||||
if (err.statusCode === 404 || err.body?.code === 404) {
|
||||
await appsApi.createNamespacedDaemonSet(namespace, daemonSet);
|
||||
this.logger.log(`Created DaemonSet "${dsName}" (cluster DNS ${clusterDnsIp})`);
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** In-cluster registry mirror for k3s/containerd (HTTP). Removes legacy external-registry DaemonSet if present. */
|
||||
private async ensureK3sRegistryMirrors(
|
||||
appsApi: k8s.AppsV1Api,
|
||||
registryUrl: string,
|
||||
): Promise<void> {
|
||||
const namespace = 'kube-system';
|
||||
const legacyDs = 'cloudhost-k3s-registry-config';
|
||||
try {
|
||||
await appsApi.deleteNamespacedDaemonSet(legacyDs, namespace);
|
||||
this.logger.log(`Removed legacy DaemonSet "${legacyDs}"`);
|
||||
} catch (err: any) {
|
||||
if (err.statusCode !== 404 && err.body?.code !== 404) {
|
||||
this.logger.warn(`Could not delete legacy DaemonSet "${legacyDs}": ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
const { username, password } = this.registryService.getRegistryCredentials();
|
||||
const dsName = 'cloudhost-k3s-registry-mirrors';
|
||||
const configureScript = [
|
||||
'set -e',
|
||||
'REG=/host/etc/rancher/k3s/registries.yaml',
|
||||
'mkdir -p /host/etc/rancher/k3s',
|
||||
'cat > /tmp/cloudhost-registries.yaml <<EOFREG',
|
||||
'mirrors:',
|
||||
` "${registryUrl}":`,
|
||||
' endpoint:',
|
||||
` - "http://${registryUrl}"`,
|
||||
'configs:',
|
||||
` "${registryUrl}":`,
|
||||
' auth:',
|
||||
` username: ${JSON.stringify(username)}`,
|
||||
` password: ${JSON.stringify(password)}`,
|
||||
'EOFREG',
|
||||
'if [ ! -f "$REG" ] || ! cmp -s /tmp/cloudhost-registries.yaml "$REG"; then',
|
||||
' cp /tmp/cloudhost-registries.yaml "$REG"',
|
||||
' echo "Updated registries.yaml"',
|
||||
' nsenter -t 1 -m -u -i -n -p -- systemctl restart k3s 2>/dev/null || true',
|
||||
'fi',
|
||||
'sleep infinity',
|
||||
].join('\n');
|
||||
|
||||
const daemonSet: k8s.V1DaemonSet = {
|
||||
metadata: {
|
||||
name: dsName,
|
||||
namespace,
|
||||
labels: { 'app.kubernetes.io/managed-by': 'cloudhost' },
|
||||
},
|
||||
spec: {
|
||||
selector: { matchLabels: { app: dsName } },
|
||||
template: {
|
||||
metadata: { labels: { app: dsName } },
|
||||
spec: {
|
||||
hostPID: true,
|
||||
tolerations: [{ operator: 'Exists' }],
|
||||
containers: [
|
||||
{
|
||||
name: 'configure',
|
||||
image: 'rancher/mirrored-library-busybox:1.36.1',
|
||||
command: ['/bin/sh', '-ec'],
|
||||
args: [configureScript],
|
||||
securityContext: { privileged: true },
|
||||
volumeMounts: [{ name: 'etc', mountPath: '/host/etc' }],
|
||||
},
|
||||
],
|
||||
volumes: [
|
||||
{
|
||||
name: 'etc',
|
||||
hostPath: { path: '/etc', type: 'Directory' },
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
await appsApi.readNamespacedDaemonSet(dsName, namespace);
|
||||
await appsApi.replaceNamespacedDaemonSet(dsName, namespace, daemonSet);
|
||||
this.logger.log(`Updated DaemonSet "${dsName}"`);
|
||||
} catch (err: any) {
|
||||
if (err.statusCode === 404 || err.body?.code === 404) {
|
||||
await appsApi.createNamespacedDaemonSet(namespace, daemonSet);
|
||||
this.logger.log(`Created DaemonSet "${dsName}"`);
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private parseCpuToMillicores(cpu: string): number {
|
||||
if (!cpu || cpu === '0') return 0;
|
||||
if (cpu.endsWith('n')) return parseFloat(cpu) / 1_000_000;
|
||||
|
||||
Reference in New Issue
Block a user