import { Injectable, Logger } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; import * as k8s from '@kubernetes/client-node'; import * as fs from 'fs'; import * as path from 'path'; import { execFile, spawn, ChildProcess } from 'child_process'; import * as net from 'net'; import { promisify } from 'util'; import { Application } from '../applications/entities/application.entity'; import { AppRuntime } from '../common/enums'; import { ClustersService } from '../clusters/clusters.service'; import { RegistryService } from '../kubernetes/registry.service'; import { BuildProgressStore } from './build-progress.store'; import { detectDjangoSettingsModule, detectGoBuildTarget, detectShallowCsproj, listArchiveEntries, validateRuntimeFromArchive, } from './runtime-detector'; const execFileAsync = promisify(execFile); export class BuildCancelledError extends Error { constructor() { super('Build cancelled by user'); this.name = 'BuildCancelledError'; } } interface ActiveBuildSession { cancelled: boolean; coreApi?: k8s.CoreV1Api; batchApi?: k8s.BatchV1Api; namespace?: string; buildPodName?: string; sourcePvcName?: string; helperPodName?: string; processes: ChildProcess[]; socket?: net.Socket; } export interface BuildProgress { phase: 'uploading' | 'building' | 'deploying' | 'done' | 'failed' | 'cancelled'; percent: number; bytesUploaded?: number; totalBytes?: number; message?: string; } @Injectable() export class BuildService { private readonly logger = new Logger(BuildService.name); private readonly progressMap = new Map(); private readonly activeBuilds = new Map(); /** * Kaniko executor image. Pinned (not `:latest`) so it can be cached on the node * with imagePullPolicy=IfNotPresent — avoids re-pulling the ~250MB image on every build. */ private readonly kanikoImage = process.env.KANIKO_IMAGE || 'gcr.io/kaniko-project/executor:v1.23.2'; constructor( private configService: ConfigService, private clustersService: ClustersService, private registryService: RegistryService, private progressStore: BuildProgressStore, ) {} private beginBuildSession(deploymentId: string): void { this.activeBuilds.set(deploymentId, { cancelled: false, processes: [] }); } private getSession(deploymentId?: string): ActiveBuildSession | undefined { if (!deploymentId) return undefined; return this.activeBuilds.get(deploymentId); } private updateBuildSession(deploymentId: string, update: Partial): void { const session = this.activeBuilds.get(deploymentId); if (session) Object.assign(session, update); } private registerProcess(deploymentId: string | undefined, proc: ChildProcess): void { const session = this.getSession(deploymentId); if (!session) return; session.processes.push(proc); if (session.cancelled) { try { proc.kill('SIGKILL'); } catch { /* ignore */ } } } private registerSocket(deploymentId: string | undefined, socket: net.Socket): void { const session = this.getSession(deploymentId); if (!session) return; if (session.socket) { try { session.socket.destroy(); } catch { /* ignore */ } } session.socket = socket; if (session.cancelled) { try { socket.destroy(); } catch { /* ignore */ } } } private throwIfCancelled(deploymentId?: string): void { if (deploymentId && this.activeBuilds.get(deploymentId)?.cancelled) { throw new BuildCancelledError(); } } private endBuildSession(deploymentId?: string): void { if (deploymentId) this.activeBuilds.delete(deploymentId); } async cancelBuild(deploymentId: string): Promise { const session = this.activeBuilds.get(deploymentId); if (!session) { this.setProgress(deploymentId, { phase: 'cancelled', percent: 0, message: 'Cancelled by user', }); return; } session.cancelled = true; this.logger.log(`Cancelling build for deployment ${deploymentId}`); if (session.socket) { try { session.socket.destroy(); } catch { /* ignore */ } } for (const proc of session.processes) { try { proc.kill('SIGKILL'); } catch { /* ignore */ } } const { coreApi, batchApi, namespace, buildPodName, sourcePvcName, helperPodName } = session; if (coreApi && namespace) { const cleanup: Promise[] = []; if (helperPodName) { cleanup.push( coreApi .deleteNamespacedPod({ name: helperPodName, namespace, gracePeriodSeconds: 0, }) .catch(() => undefined), ); } if (buildPodName && batchApi) { cleanup.push( batchApi .deleteNamespacedJob({ name: buildPodName, namespace, gracePeriodSeconds: 0, propagationPolicy: 'Foreground', }) .catch(() => undefined), ); } if (sourcePvcName) { cleanup.push( coreApi .deleteNamespacedPersistentVolumeClaim({ name: sourcePvcName, namespace, }) .catch(() => undefined), ); } if (buildPodName) { cleanup.push( coreApi .deleteNamespacedConfigMap({ name: `${buildPodName}-dockerfile`, namespace, }) .catch(() => undefined), ); } await Promise.all(cleanup); this.logger.log(`Cleaned up K8s build resources for deployment ${deploymentId}`); } this.setProgress(deploymentId, { phase: 'cancelled', percent: 0, message: 'Cancelled by user', }); this.activeBuilds.delete(deploymentId); } /** Delete all in-flight build artifacts for an app (helper pods, jobs, PVCs, configmaps). */ async cleanupBuildResourcesForApp(app: Application): Promise { const buildNamespace = this.configService.get('build.namespace') || 'cloudhost-builds'; const prefix = `build-${app.name}-`; const cluster = app.clusterId ? await this.clustersService.findOne(app.clusterId) : await this.clustersService.getDefault(); const kc = new k8s.KubeConfig(); kc.loadFromString(cluster.kubeconfig); const coreApi = kc.makeApiClient(k8s.CoreV1Api); const batchApi = kc.makeApiClient(k8s.BatchV1Api); const cleanup: Promise[] = []; const [pods, pvcs, jobs, configMaps] = await Promise.all([ coreApi.listNamespacedPod({ namespace: buildNamespace }), coreApi.listNamespacedPersistentVolumeClaim({ namespace: buildNamespace, }), batchApi.listNamespacedJob({ namespace: buildNamespace }), coreApi.listNamespacedConfigMap({ namespace: buildNamespace }), ]); for (const pod of pods.items) { const name = pod.metadata?.name || ''; if (name.startsWith(prefix)) { cleanup.push( coreApi .deleteNamespacedPod({ name, namespace: buildNamespace, gracePeriodSeconds: 0, }) .catch(() => undefined), ); } } for (const pvc of pvcs.items) { const name = pvc.metadata?.name || ''; if (name.startsWith(prefix)) { cleanup.push( coreApi .deleteNamespacedPersistentVolumeClaim({ name, namespace: buildNamespace, }) .catch(() => undefined), ); } } for (const job of jobs.items) { const name = job.metadata?.name || ''; if (name.startsWith(prefix)) { cleanup.push( batchApi .deleteNamespacedJob({ name, namespace: buildNamespace, gracePeriodSeconds: 0, propagationPolicy: 'Foreground', }) .catch(() => undefined), ); } } for (const cm of configMaps.items) { const name = cm.metadata?.name || ''; if (name.startsWith(prefix)) { cleanup.push(coreApi.deleteNamespacedConfigMap({ name, namespace: buildNamespace }).catch(() => undefined)); } } await Promise.all(cleanup); this.logger.log(`Cleaned up all build resources matching "${prefix}*" in ${buildNamespace}`); } async getProgress(deploymentId: string): Promise { const local = this.progressMap.get(deploymentId); if (local) return local; const remote = await this.progressStore.get(deploymentId); if (remote) this.progressMap.set(deploymentId, remote); return remote; } setProgress(deploymentId: string | undefined, progress: BuildProgress): void { if (!deploymentId) return; this.progressMap.set(deploymentId, progress); void this.progressStore.set(deploymentId, progress); } clearProgress(deploymentId: string): void { this.progressMap.delete(deploymentId); void this.progressStore.clear(deploymentId); } /** * Builds a Docker image for the application using Kaniko inside K8s. * Returns { imageUri, buildLog } — the full image URI and the build logs. */ async buildImage(app: Application, deploymentId?: string): Promise<{ imageUri: string; buildLog: string }> { const registryUrl = this.registryService.getRegistryUrl(); const buildNamespace = this.registryService.getBuildNamespace(); const tag = `${Date.now()}`; const imageUri = this.registryService.buildImageReference(app.userId, app.name, tag); this.logger.log(`Starting image build for ${app.name} → ${imageUri}`); if (deploymentId) { this.beginBuildSession(deploymentId); } const codePath = app.codePath ? path.resolve(app.codePath) : null; const hasUploadedCode = codePath && fs.existsSync(codePath); if (hasUploadedCode) { await validateRuntimeFromArchive(app.runtime, codePath); } const archiveEntries = hasUploadedCode ? await listArchiveEntries(codePath!) : []; // Determine Dockerfile based on runtime const dockerfileContent = this.generateDockerfile(app, archiveEntries); // Create Kaniko build pod const buildPodName = `build-${app.name}-${tag}`.substring(0, 63).replace(/[^a-z0-9-]/g, ''); if (deploymentId) { this.updateBuildSession(deploymentId, { buildPodName }); } // Use the cluster's kubeconfig instead of default const cluster = app.clusterId ? await this.clustersService.findOne(app.clusterId) : await this.clustersService.getDefault(); const kc = new k8s.KubeConfig(); kc.loadFromString(cluster.kubeconfig); const coreApi = kc.makeApiClient(k8s.CoreV1Api); const batchApi = kc.makeApiClient(k8s.BatchV1Api); if (deploymentId) { this.updateBuildSession(deploymentId, { coreApi, batchApi, namespace: buildNamespace, }); } // Ensure the build namespace exists await this.ensureNamespace(coreApi, buildNamespace); this.throwIfCancelled(deploymentId); // Determine if we have uploaded code or git URL const hasGitUrl = !!app.gitUrl; // Create ConfigMap with Dockerfile const dockerfileConfigMap = { apiVersion: 'v1', kind: 'ConfigMap', metadata: { name: `${buildPodName}-dockerfile`, namespace: buildNamespace, }, data: { Dockerfile: dockerfileContent, }, }; // If we have uploaded code, create a PVC and upload via kubectl cp let sourcePvcName: string | undefined; if (hasUploadedCode) { sourcePvcName = `${buildPodName}-source`; if (deploymentId) { this.updateBuildSession(deploymentId, { sourcePvcName }); } const zipSize = fs.statSync(codePath!).size; // Allocate PVC size = zip size * 3 (zip + extracted), min 1Gi const pvcSizeGi = Math.max(1, Math.ceil((zipSize * 3) / (1024 * 1024 * 1024))); await this.uploadSourceViaPVC(kc, coreApi, buildNamespace!, sourcePvcName, codePath!, pvcSizeGi, deploymentId); } // Build the Kaniko Job spec // Always use dir context — init containers prepare /workspace/source const kanikoArgs = [ '--dockerfile=/workspace/Dockerfile', '--context=dir:///workspace/source', `--destination=${imageUri}`, '--cache=true', `--cache-repo=${registryUrl}/${app.userId}/cache`, '--insecure', '--skip-tls-verify', '--single-snapshot', '--snapshot-mode=redo', ]; const volumes: any[] = [ { name: 'docker-config', secret: { secretName: 'registry-credentials' }, }, { name: 'dockerfile', configMap: { name: `${buildPodName}-dockerfile`, }, }, { name: 'workspace', emptyDir: {}, }, ]; const initContainers: any[] = []; if (hasUploadedCode && sourcePvcName) { // Add the source PVC as a volume volumes.push({ name: 'source-pvc', persistentVolumeClaim: { claimName: sourcePvcName }, }); // Add init container that unzips the source code from PVC initContainers.push({ name: 'unzip-source', image: 'alpine:3.19', imagePullPolicy: 'IfNotPresent', command: [ 'sh', '-c', ` apk add --no-cache unzip tar gzip && cp /workspace/Dockerfile /workspace-out/Dockerfile && mkdir -p /tmp/extract && cd /tmp/extract && if tar tzf /source-pvc/source.zip >/dev/null 2>&1; then echo ">>> Detected gzip tarball" && tar xzf /source-pvc/source.zip elif unzip -t /source-pvc/source.zip >/dev/null 2>&1; then echo ">>> Detected zip archive" && unzip -q /source-pvc/source.zip else echo "ERROR: source archive is not a valid zip or tar.gz" && exit 1 fi && echo "--- Extracted contents ---" && ls -la /tmp/extract/ && mkdir -p /workspace-out/source && ITEMS=$(ls -1 /tmp/extract/ | head -5) && COUNT=$(ls -1 /tmp/extract/ | wc -l) && if [ "$COUNT" -eq 1 ] && [ -d "/tmp/extract/$ITEMS" ]; then echo ">>> Single subfolder detected: $ITEMS — flattening to root" && cp -a /tmp/extract/$ITEMS/. /workspace-out/source/ else echo ">>> Multiple items or files — copying as-is" && cp -a /tmp/extract/. /workspace-out/source/ fi && rm -rf /tmp/extract && echo "--- Final workspace contents ---" && ls -la /workspace-out/source/ `, ], volumeMounts: [ { name: 'workspace', mountPath: '/workspace-out' }, { name: 'dockerfile', mountPath: '/workspace/Dockerfile', subPath: 'Dockerfile', }, { name: 'source-pvc', mountPath: '/source-pvc' }, ], }); } else if (hasGitUrl) { // Build the git clone URL — inject token for private repos let cloneUrl = app.gitUrl!; if (app.gitToken) { // Convert https://github.com/user/repo.git → https://@github.com/user/repo.git // Also works for GitLab, Bitbucket, etc. try { const url = new URL(cloneUrl); url.username = app.gitToken; url.password = ''; // Some providers use token as username, others as password cloneUrl = url.toString(); } catch { // If URL parsing fails, try simple injection after protocol cloneUrl = cloneUrl.replace('https://', `https://${app.gitToken}@`); } } const branch = app.gitBranch || 'main'; // Clone git repo into /workspace/source, then copy our generated Dockerfile initContainers.push({ name: 'git-clone', image: 'alpine/git:2.43.0', imagePullPolicy: 'IfNotPresent', command: [ 'sh', '-c', ` echo ">>> Cloning branch '${branch}' from ${app.gitUrl}" && git clone --depth 1 --branch ${branch} ${cloneUrl} /workspace-out/source && cp /dockerfile/Dockerfile /workspace-out/Dockerfile && echo ">>> Workspace contents:" && ls -la /workspace-out/source/ `, ], volumeMounts: [ { name: 'workspace', mountPath: '/workspace-out' }, { name: 'dockerfile', mountPath: '/dockerfile' }, ], }); } // Kaniko container volume mounts const kanikoVolumeMounts: any[] = [ { name: 'docker-config', mountPath: '/kaniko/.docker' }, { name: 'workspace', mountPath: '/workspace' }, ]; // If no uploaded code and no git, we need to prepare the workspace if (!hasUploadedCode && !hasGitUrl) { // For runtimes that don't need source (e.g. fresh WordPress), // add an init container that creates empty source dir + copies Dockerfile initContainers.push({ name: 'prepare-workspace', image: 'alpine:3.19', imagePullPolicy: 'IfNotPresent', command: [ 'sh', '-c', ` mkdir -p /workspace-out/source && cp /dockerfile/Dockerfile /workspace-out/Dockerfile && echo ">>> Prepared empty workspace for fresh install" && ls -la /workspace-out/ `, ], volumeMounts: [ { name: 'workspace', mountPath: '/workspace-out' }, { name: 'dockerfile', mountPath: '/dockerfile' }, ], }); } const buildJob: k8s.V1Job = { apiVersion: 'batch/v1', kind: 'Job', metadata: { name: buildPodName, namespace: buildNamespace, }, spec: { backoffLimit: 1, ttlSecondsAfterFinished: 300, template: { spec: { serviceAccountName: this.configService.get('build.serviceAccount'), initContainers: initContainers.length > 0 ? initContainers : undefined, containers: [ { name: 'kaniko', image: this.kanikoImage, imagePullPolicy: 'IfNotPresent', args: kanikoArgs, volumeMounts: kanikoVolumeMounts, resources: { requests: { cpu: '500m', memory: '1Gi' }, limits: { cpu: '2', memory: '4Gi' }, }, }, ], restartPolicy: 'Never', volumes, }, }, }, }; try { const t0 = Date.now(); await coreApi.createNamespacedConfigMap({ namespace: buildNamespace!, body: dockerfileConfigMap, }); this.logger.log(`[timing] ConfigMap created in ${Date.now() - t0}ms`); const t1 = Date.now(); await batchApi.createNamespacedJob({ namespace: buildNamespace!, body: buildJob, }); this.logger.log(`[timing] Job created in ${Date.now() - t1}ms`); // Wait for build to complete this.setProgress(deploymentId, { phase: 'building', percent: 15, message: 'Building Docker image...', }); await this.waitForJobCompletion(batchApi, coreApi, buildPodName, buildNamespace!, 600, deploymentId); // Capture build logs on success let buildLog = ''; try { buildLog = await this.getBuildLogs(coreApi, buildPodName, buildNamespace!); } catch {} this.logger.log(`Build completed successfully: ${imageUri}`); return { imageUri, buildLog }; } catch (error: any) { if (error instanceof BuildCancelledError || error?.name === 'BuildCancelledError') { throw error; } // Try to get build logs for debugging let buildLog = ''; try { buildLog = await this.getBuildLogs(coreApi, buildPodName, buildNamespace!); this.logger.error(`Build logs for ${buildPodName}:\n${buildLog}`); } catch {} this.logger.error(`Build failed for ${app.name}:`, error.body || error.message); const err = new Error(`Image build failed: ${error.body?.message || error.message}`); (err as any).buildLog = buildLog; throw err; } finally { // Clean up build resources if (sourcePvcName) { try { await coreApi.deleteNamespacedPersistentVolumeClaim({ name: sourcePvcName, namespace: buildNamespace!, }); this.logger.log(`Cleaned up source PVC: ${sourcePvcName}`); } catch (e: any) { this.logger.warn(`Failed to clean up source PVC ${sourcePvcName}: ${e.message}`); } } // Clean up Dockerfile ConfigMap try { await coreApi.deleteNamespacedConfigMap({ name: `${buildPodName}-dockerfile`, namespace: buildNamespace!, }); } catch (e: any) { this.logger.warn(`Failed to clean up ConfigMap: ${e.message}`); } this.endBuildSession(deploymentId); } } /** * Upload a local file to the helper pod using kubectl cp with progress tracking. * kubectl cp uses tar over the k8s exec API — reliable for any file size. */ private streamFileToHelperPod(kubeconfig: string, namespace: string, podName: string, filePath: string, fileSize: number, deploymentId?: string): Promise { const maxAttempts = 3; const runOnce = () => new Promise((resolve, reject) => { this.throwIfCancelled(deploymentId); const kubectl = spawn('kubectl', ['--kubeconfig', kubeconfig, 'cp', filePath, `${namespace}/${podName}:/data/source.zip`, '-c', 'helper', '--retries', '3'], { stdio: ['ignore', 'pipe', 'pipe'], }); this.registerProcess(deploymentId, kubectl); let stderr = ''; kubectl.stderr.on('data', (chunk: Buffer) => { stderr += chunk.toString(); }); // Track progress by polling remote file size let progressTimer: NodeJS.Timeout | undefined; const pollProgress = () => { execFileAsync('kubectl', ['--kubeconfig', kubeconfig, 'exec', '-n', namespace, podName, '-c', 'helper', '--', 'sh', '-c', 'wc -c < /data/source.zip 2>/dev/null || echo 0'], { timeout: 10_000, }) .then(({ stdout }) => { const remoteSize = parseInt(stdout.trim(), 10) || 0; const percent = Math.min(99, Math.round((remoteSize / fileSize) * 100)); this.setProgress(deploymentId, { phase: 'uploading', percent, bytesUploaded: remoteSize, totalBytes: fileSize, message: `Uploading to cluster... ${percent}%`, }); }) .catch(() => { /* polling failure is non-fatal */ }); }; progressTimer = setInterval(pollProgress, 3000); pollProgress(); kubectl.on('error', (err) => { clearInterval(progressTimer); reject(new Error(`kubectl cp spawn error: ${err.message}`)); }); kubectl.on('close', (code) => { clearInterval(progressTimer); if (code === 0) resolve(); else reject(new Error(`kubectl cp failed (code ${code}): ${stderr.trim()}`)); }); }); return (async () => { for (let attempt = 1; attempt <= maxAttempts; attempt++) { try { this.throwIfCancelled(deploymentId); if (attempt > 1) { this.logger.warn(`Retrying source upload (attempt ${attempt}/${maxAttempts})...`); await execFileAsync('kubectl', ['--kubeconfig', kubeconfig, 'exec', '-n', namespace, podName, '-c', 'helper', '--', 'rm', '-f', '/data/source.zip'], { timeout: 15_000 }).catch( () => undefined, ); this.setProgress(deploymentId, { phase: 'uploading', percent: 0, bytesUploaded: 0, totalBytes: fileSize, message: `Retrying upload (attempt ${attempt})...`, }); } await runOnce(); return; } catch (err) { if (err instanceof BuildCancelledError || (err as Error)?.name === 'BuildCancelledError') throw err; if (attempt === maxAttempts) throw err; this.logger.warn(`Upload attempt ${attempt} failed: ${(err as Error).message}`); } } })(); } /** * Upload source zip to K8s via PVC + helper pod. * This handles files of any size (unlike Secret/ConfigMap which are limited to ~1MB). */ private async uploadSourceViaPVC(kc: k8s.KubeConfig, coreApi: k8s.CoreV1Api, namespace: string, pvcName: string, zipPath: string, sizeGi: number, deploymentId?: string): Promise { const t0 = Date.now(); const helperPodName = `${pvcName}-helper`; const zipSize = fs.statSync(zipPath).size; if (deploymentId) { this.updateBuildSession(deploymentId, { helperPodName }); } this.logger.log(`Uploading source via PVC (${(zipSize / 1024 / 1024).toFixed(1)} MB) → ${pvcName}`); // 1. Create PVC await coreApi.createNamespacedPersistentVolumeClaim({ namespace, body: { apiVersion: 'v1', kind: 'PersistentVolumeClaim', metadata: { name: pvcName, namespace }, spec: { accessModes: ['ReadWriteOnce'], resources: { requests: { storage: `${sizeGi}Gi` } }, }, }, }); this.logger.log(`[timing] PVC ${pvcName} created in ${Date.now() - t0}ms`); // 2. Create a helper pod that mounts the PVC and waits for data via a simple HTTP listener. // We use alpine + nc (netcat) to receive the file over a port — much more reliable // than kubectl cp or kubectl exec stdin pipe for large files. const helperPod: k8s.V1Pod = { apiVersion: 'v1', kind: 'Pod', metadata: { name: helperPodName, namespace }, spec: { containers: [ { name: 'helper', image: 'alpine:3.19', imagePullPolicy: 'IfNotPresent', command: ['sh', '-c', 'sleep 3600'], volumeMounts: [{ name: 'source', mountPath: '/data' }], resources: { requests: { cpu: '100m', memory: '128Mi' }, limits: { cpu: '500m', memory: '256Mi' }, }, }, ], volumes: [ { name: 'source', persistentVolumeClaim: { claimName: pvcName }, }, ], restartPolicy: 'Never', }, }; await coreApi.createNamespacedPod({ namespace, body: helperPod }); // 3. Wait for helper pod to be Running const podTimeout = 120_000; // 2 minutes const podStart = Date.now(); while (Date.now() - podStart < podTimeout) { this.throwIfCancelled(deploymentId); const pod = await coreApi.readNamespacedPod({ name: helperPodName, namespace, }); const phase = pod.status?.phase; if (phase === 'Running') break; if (phase === 'Failed' || phase === 'Unknown') { throw new Error(`Helper pod ${helperPodName} failed to start: phase=${phase}`); } await new Promise((r) => setTimeout(r, 2000)); } if (Date.now() - podStart >= podTimeout) { throw new Error(`Helper pod ${helperPodName} did not become Running within 2 minutes`); } this.logger.log(`[timing] Helper pod Running in ${Date.now() - t0}ms`); // 4. Write kubeconfig to temp file for kubectl const tmpKubeconfig = path.join('/tmp', `kubeconfig-${pvcName}.yaml`); const kcYaml = kc.exportConfig(); fs.writeFileSync(tmpKubeconfig, kcYaml); try { // 5. Upload the zip via kubectl cp (tar-based, reliable for any size). const t2 = Date.now(); this.setProgress(deploymentId, { phase: 'uploading', percent: 0, bytesUploaded: 0, totalBytes: zipSize, message: 'Uploading source to cluster...', }); await this.streamFileToHelperPod(tmpKubeconfig, namespace, helperPodName, zipPath, zipSize, deploymentId); this.logger.log(`[timing] Source stream upload completed in ${Date.now() - t2}ms (${(zipSize / 1024 / 1024).toFixed(1)} MB)`); this.setProgress(deploymentId, { phase: 'uploading', percent: 100, bytesUploaded: zipSize, totalBytes: zipSize, message: 'Upload complete, verifying...', }); // 5b. Verify the file was written correctly (exact size) const { stdout: sizeStr } = await execFileAsync( 'kubectl', ['--kubeconfig', tmpKubeconfig, 'exec', '-n', namespace, helperPodName, '-c', 'helper', '--', 'sh', '-c', 'wc -c < /data/source.zip'], { timeout: 30_000 }, ); const remoteSize = parseInt(sizeStr.trim(), 10); if (isNaN(remoteSize) || remoteSize !== zipSize) { throw new Error( `Source upload incomplete: expected ${zipSize} bytes but got ${remoteSize} bytes on remote. ` + `(${(zipSize / 1024 / 1024).toFixed(1)} MB expected, ${(remoteSize / 1024 / 1024).toFixed(1)} MB received)`, ); } this.logger.log(`[verify] Remote file size: ${remoteSize} bytes (expected ${zipSize}) ✓`); } finally { // Clean up temp kubeconfig try { fs.unlinkSync(tmpKubeconfig); } catch {} // 6. Delete the helper pod and WAIT for it to be fully terminated // (PVC is ReadWriteOnce — if the pod is still terminating when the // build Job starts, Kaniko can't mount the PVC → stuck in Pending) try { await coreApi.deleteNamespacedPod({ name: helperPodName, namespace, gracePeriodSeconds: 0, }); this.logger.log(`Helper pod ${helperPodName} delete requested — waiting for termination…`); const delTimeout = 60_000; const delStart = Date.now(); while (Date.now() - delStart < delTimeout) { try { await coreApi.readNamespacedPod({ name: helperPodName, namespace }); // Pod still exists — wait await new Promise((r) => setTimeout(r, 2000)); } catch (err: any) { if (err.code === 404 || err.body?.code === 404) { this.logger.log(`Helper pod ${helperPodName} fully terminated`); break; } // Other error — stop waiting break; } } } catch (e: any) { this.logger.warn(`Failed to delete helper pod: ${e.message}`); } } this.logger.log(`[timing] Source upload via PVC completed in ${Date.now() - t0}ms`); } /** * Ensure the build namespace exists with all required resources * (namespace, service account, registry-credentials secret). */ private async ensureNamespace(coreApi: k8s.CoreV1Api, namespace: string): Promise { // 1. Ensure namespace try { await coreApi.readNamespace({ name: namespace }); } catch (err: any) { if (err.code === 404 || err.body?.code === 404) { this.logger.log(`Namespace "${namespace}" not found — creating it`); await coreApi.createNamespace({ body: { metadata: { name: namespace } }, }); } else { throw err; } } // 2. Ensure service account for Kaniko const saName = this.configService.get('build.serviceAccount') || 'kaniko-builder'; try { await coreApi.readNamespacedServiceAccount({ name: saName, namespace }); } catch (err: any) { if (err.code === 404 || err.body?.code === 404) { this.logger.log(`ServiceAccount "${saName}" not found in "${namespace}" — creating it`); await coreApi.createNamespacedServiceAccount({ namespace, body: { metadata: { name: saName, namespace } }, }); } else { throw err; } } // 3. Ensure registry-credentials secret (docker config for Kaniko to push) const registrySecretName = 'registry-credentials'; try { await coreApi.readNamespacedSecret({ name: registrySecretName, namespace, }); } catch (err: any) { if (err.code === 404 || err.body?.code === 404) { this.logger.log(`Secret "${registrySecretName}" not found in "${namespace}" — creating it`); await coreApi.createNamespacedSecret({ namespace, body: { metadata: { name: registrySecretName, namespace }, type: 'kubernetes.io/dockerconfigjson', data: { '.dockerconfigjson': Buffer.from(this.registryService.buildDockerConfigJson()).toString('base64'), }, }, }); } else { throw err; } } } private generateDockerfile(app: Application, archiveEntries: string[] = []): string { switch (app.runtime) { case AppRuntime.NODEJS: return this.nodeDockerfile(app); case AppRuntime.LARAVEL: return this.laravelDockerfile(app); case AppRuntime.WORDPRESS: return this.wordpressDockerfile(app); case AppRuntime.GO: return this.goDockerfile(app, archiveEntries); case AppRuntime.PHP: return this.phpDockerfile(app); case AppRuntime.PYTHON: return this.pythonDockerfile(app); case AppRuntime.DJANGO: return this.djangoDockerfile(app, archiveEntries); case AppRuntime.DOTNET: return this.dotnetDockerfile(app, archiveEntries); default: throw new Error(`Unsupported runtime: ${app.runtime}`); } } private nodeDockerfile(app: Application): string { const port = app.port || 3000; const nodeVersion = app.runtimeVersion || '20'; return `# --- Build stage --- FROM node:${nodeVersion}-alpine AS builder WORKDIR /app COPY package*.json ./ RUN npm install --legacy-peer-deps && npm cache clean --force COPY . . # Auto-detect Next.js and enable standalone output RUN for cfg in next.config.js next.config.mjs next.config.ts; do \\ [ -f "$cfg" ] || continue; \\ if grep -q standalone "$cfg"; then \\ echo "$cfg already has standalone"; \\ else \\ echo ">>> Next.js detected, injecting standalone output"; \\ node -e 'var f=require("fs"),c=f.readFileSync(process.argv[1],"utf8");if(!c.includes("standalone")){f.writeFileSync(process.argv[1],c.replace("{","{ output: \\"standalone\\","))}' "$cfg"; \\ echo "Patched $cfg:"; head -5 "$cfg"; \\ fi; \\ break; \\ done RUN npm run build || echo ">>> Build script failed or not found — continuing" # Clean up dev dependencies and caches to reduce image size RUN rm -rf node_modules/.cache .next/cache /tmp/* /root/.npm 2>/dev/null; true # --- Production stage --- FROM node:${nodeVersion}-alpine AS runner WORKDIR /app RUN addgroup -g 1001 -S appgroup && adduser -S appuser -u 1001 # Copy all build output to temp COPY --from=builder /app /tmp/fullapp # Detect: Next.js standalone vs regular Node.js RUN if [ -d /tmp/fullapp/.next/standalone ]; then \\ echo ">>> Next.js standalone mode"; \\ cp -a /tmp/fullapp/.next/standalone/. .; \\ mkdir -p .next/static; \\ [ -d /tmp/fullapp/.next/static ] && cp -a /tmp/fullapp/.next/static/. .next/static/; \\ [ -d /tmp/fullapp/public ] && cp -a /tmp/fullapp/public ./public; \\ echo "standalone" > /app/.mode; \\ else \\ echo ">>> Regular Node.js app"; \\ cp -a /tmp/fullapp/. .; \\ echo "regular" > /app/.mode; \\ fi && rm -rf /tmp/fullapp USER appuser ENV PORT=${port} ENV HOSTNAME=0.0.0.0 EXPOSE ${port} CMD ["sh", "-c", "if [ \\"$(cat /app/.mode)\\" = \\"standalone\\" ] && [ -f server.js ]; then node server.js; else npm start; fi"] `; } private laravelDockerfile(app: Application): string { const phpVersion = app.phpVersion || '8.3'; const port = app.port || 80; return `# --- Build stage (match production PHP version for Composer) --- FROM php:${phpVersion}-cli-alpine AS composer RUN apk add --no-cache git unzip COPY --from=composer:2 /usr/bin/composer /usr/bin/composer WORKDIR /app COPY composer.json composer.lock* ./ RUN composer install --no-dev --no-scripts --no-autoloader --prefer-dist --ignore-platform-reqs COPY . . RUN composer dump-autoload --optimize --no-dev --no-scripts # --- Production stage --- FROM php:${phpVersion}-fpm-alpine RUN apk add --no-cache nginx supervisor curl openssl \\ && docker-php-ext-install pdo pdo_mysql opcache \\ && docker-php-ext-install pdo_pgsql 2>/dev/null || true WORKDIR /var/www/html COPY --from=composer /app . # Generate nginx config inline (no dependency on user files) RUN mkdir -p /etc/nginx/http.d && \\ echo 'server {' > /etc/nginx/http.d/default.conf && \\ echo ' listen ${port};' >> /etc/nginx/http.d/default.conf && \\ echo ' root /var/www/html/public;' >> /etc/nginx/http.d/default.conf && \\ echo ' index index.php index.html;' >> /etc/nginx/http.d/default.conf && \\ echo ' client_max_body_size 64M;' >> /etc/nginx/http.d/default.conf && \\ echo ' location / { try_files \\$uri \\$uri/ /index.php?\\$query_string; }' >> /etc/nginx/http.d/default.conf && \\ echo ' location ~ \\.php\\$ {' >> /etc/nginx/http.d/default.conf && \\ echo ' fastcgi_pass 127.0.0.1:9000;' >> /etc/nginx/http.d/default.conf && \\ echo ' fastcgi_param SCRIPT_FILENAME \\$document_root\\$fastcgi_script_name;' >> /etc/nginx/http.d/default.conf && \\ echo ' include fastcgi_params;' >> /etc/nginx/http.d/default.conf && \\ echo ' }' >> /etc/nginx/http.d/default.conf && \\ echo ' location ~ /\\.ht { deny all; }' >> /etc/nginx/http.d/default.conf && \\ echo '}' >> /etc/nginx/http.d/default.conf # Generate supervisord config inline RUN echo '[supervisord]' > /etc/supervisord.conf && \\ echo 'nodaemon=true' >> /etc/supervisord.conf && \\ echo 'logfile=/dev/stdout' >> /etc/supervisord.conf && \\ echo 'logfile_maxbytes=0' >> /etc/supervisord.conf && \\ echo '' >> /etc/supervisord.conf && \\ echo '[program:php-fpm]' >> /etc/supervisord.conf && \\ echo 'command=php-fpm -F' >> /etc/supervisord.conf && \\ echo 'autostart=true' >> /etc/supervisord.conf && \\ echo 'autorestart=true' >> /etc/supervisord.conf && \\ echo 'stdout_logfile=/dev/stdout' >> /etc/supervisord.conf && \\ echo 'stdout_logfile_maxbytes=0' >> /etc/supervisord.conf && \\ echo 'stderr_logfile=/dev/stderr' >> /etc/supervisord.conf && \\ echo 'stderr_logfile_maxbytes=0' >> /etc/supervisord.conf && \\ echo '' >> /etc/supervisord.conf && \\ echo '[program:nginx]' >> /etc/supervisord.conf && \\ echo 'command=nginx -g "daemon off;"' >> /etc/supervisord.conf && \\ echo 'autostart=true' >> /etc/supervisord.conf && \\ echo 'autorestart=true' >> /etc/supervisord.conf && \\ echo 'stdout_logfile=/dev/stdout' >> /etc/supervisord.conf && \\ echo 'stdout_logfile_maxbytes=0' >> /etc/supervisord.conf && \\ echo 'stderr_logfile=/dev/stderr' >> /etc/supervisord.conf && \\ echo 'stderr_logfile_maxbytes=0' >> /etc/supervisord.conf # If user provides their own nginx/supervisor configs, use those instead RUN [ -f docker/nginx.conf ] && cp docker/nginx.conf /etc/nginx/http.d/default.conf || true RUN [ -f docker/supervisord.conf ] && cp docker/supervisord.conf /etc/supervisord.conf || true # Ensure storage and cache directories exist and are writable RUN mkdir -p storage/logs storage/framework/cache storage/framework/sessions storage/framework/views bootstrap/cache \\ && chown -R www-data:www-data /var/www/html/storage /var/www/html/bootstrap/cache RUN echo '#!/bin/sh' > /usr/local/bin/cloudhost-laravel-entrypoint.sh && \\ echo 'set -e' >> /usr/local/bin/cloudhost-laravel-entrypoint.sh && \\ echo 'cd /var/www/html' >> /usr/local/bin/cloudhost-laravel-entrypoint.sh && \\ echo 'mkdir -p storage/logs storage/framework/cache storage/framework/sessions storage/framework/views storage/app/public bootstrap/cache' >> /usr/local/bin/cloudhost-laravel-entrypoint.sh && \\ echo 'chown -R www-data:www-data storage bootstrap/cache' >> /usr/local/bin/cloudhost-laravel-entrypoint.sh && \\ echo 'if [ -z "$APP_KEY" ] || [ "$APP_KEY" = "null" ]; then export APP_KEY="base64:$(openssl rand -base64 32)"; fi' >> /usr/local/bin/cloudhost-laravel-entrypoint.sh && \\ echo 'php artisan config:clear || true' >> /usr/local/bin/cloudhost-laravel-entrypoint.sh && \\ echo 'php artisan cache:clear || true' >> /usr/local/bin/cloudhost-laravel-entrypoint.sh && \\ echo 'php artisan migrate --force --no-interaction || true' >> /usr/local/bin/cloudhost-laravel-entrypoint.sh && \\ echo 'php artisan storage:link || true' >> /usr/local/bin/cloudhost-laravel-entrypoint.sh && \\ echo 'php artisan config:cache || true' >> /usr/local/bin/cloudhost-laravel-entrypoint.sh && \\ echo 'php artisan route:cache || true' >> /usr/local/bin/cloudhost-laravel-entrypoint.sh && \\ echo 'php artisan view:cache || true' >> /usr/local/bin/cloudhost-laravel-entrypoint.sh && \\ echo 'exec /usr/bin/supervisord -c /etc/supervisord.conf' >> /usr/local/bin/cloudhost-laravel-entrypoint.sh && \\ chmod +x /usr/local/bin/cloudhost-laravel-entrypoint.sh EXPOSE ${port} CMD ["/usr/local/bin/cloudhost-laravel-entrypoint.sh"] `; } private wordpressDockerfile(app: Application): string { const wpVersion = app.runtimeVersion || '6.7'; const phpVersion = app.phpVersion || '8.3'; const hasUploadedCode = !!app.codePath; return `FROM wordpress:${wpVersion}-php${phpVersion}-apache # Install additional PHP extensions commonly needed by WordPress RUN docker-php-ext-install opcache # Enable Apache mod_rewrite for pretty permalinks RUN a2enmod rewrite # Increase PHP upload limits for WordPress media RUN echo "upload_max_filesize = 64M\\npost_max_size = 64M\\nmax_execution_time = 300\\nmemory_limit = 256M" > /usr/local/etc/php/conf.d/uploads.ini ${ hasUploadedCode ? `# Copy user's custom WordPress files COPY . /tmp/user-content # Auto-detect: full public_html root (has wp-admin) vs wp-content only # ── public_html mode ── # wp-admin/ and wp-includes/ replace the base-image core so the user's # exact WordPress version & patches are preserved. # wp-content/ is staged in /usr/src/wordpress-user/ and merged into the # PVC on first boot (same as migrate mode). # wp-config.php is saved separately so docker-entrypoint.sh can still # inject WORDPRESS_DB_* env-vars when no config exists yet. # ── migrate mode (no wp-admin) ── # Only wp-content + config files are processed. RUN mkdir -p /usr/src/wordpress-user && \\ if [ -d /tmp/user-content/wp-admin ]; then \\ echo ">>> Full WordPress root (public_html) detected" && \\ echo ">>> Copying wp-admin/ to /var/www/html/" && \\ rm -rf /var/www/html/wp-admin && \\ cp -a /tmp/user-content/wp-admin /var/www/html/wp-admin && \\ echo ">>> Copying wp-includes/ to /var/www/html/" && \\ rm -rf /var/www/html/wp-includes && \\ cp -a /tmp/user-content/wp-includes /var/www/html/wp-includes && \\ echo ">>> Copying root PHP files (index.php, wp-login.php, ...)" && \\ find /tmp/user-content -maxdepth 1 -name "*.php" ! -name "wp-config.php" \\ -exec cp {} /var/www/html/ \\; 2>/dev/null || true && \\ echo ">>> Copying other root files/dirs (fonts, assets, etc.)" && \\ for item in /tmp/user-content/*; do \\ name=$(basename "$item"); \\ case "$name" in \\ wp-admin|wp-includes|wp-content|wp-config.php|.htaccess) ;; \\ *.php) ;; \\ *) \\ if [ -f "$item" ]; then \\ echo " root file: $name" && \\ cp "$item" /var/www/html/; \\ elif [ -d "$item" ]; then \\ echo " root dir: $name/" && \\ cp -a "$item" /var/www/html/; \\ fi ;; \\ esac; \\ done; \\ else \\ echo ">>> wp-content / config files only (migrate mode)"; \\ fi && \\ if [ -d /tmp/user-content/wp-content ]; then \\ echo ">>> Staging user wp-content (themes, plugins, uploads)..." && \\ cp -a /tmp/user-content/wp-content /usr/src/wordpress-user/wp-content; \\ elif [ -d /tmp/user-content/themes ] || [ -d /tmp/user-content/plugins ] || [ -d /tmp/user-content/uploads ]; then \\ echo ">>> Staging loose themes/plugins/uploads into wp-content..." && \\ mkdir -p /usr/src/wordpress-user/wp-content && \\ [ -d /tmp/user-content/themes ] && cp -a /tmp/user-content/themes /usr/src/wordpress-user/wp-content/ || true && \\ [ -d /tmp/user-content/plugins ] && cp -a /tmp/user-content/plugins /usr/src/wordpress-user/wp-content/ || true && \\ [ -d /tmp/user-content/uploads ] && cp -a /tmp/user-content/uploads /usr/src/wordpress-user/wp-content/ || true; \\ fi && \\ # wp-config.php is intentionally NOT copied — docker-entrypoint.sh generates it # from WORDPRESS_DB_* env vars so credentials always match the deployed database. if [ -f /tmp/user-content/.htaccess ]; then \\ echo ">>> Copying .htaccess" && \\ cp /tmp/user-content/.htaccess /var/www/html/.htaccess; \\ fi && \\ rm -rf /tmp/user-content && \\ echo ">>> WordPress user content staged" # Custom entrypoint: # 1. Merge staged wp-content into the PVC mount (every start — idempotent) # 2. Hand off to official docker-entrypoint.sh which creates wp-config.php # from WORDPRESS_DB_* env vars (never use uploaded wp-config with old credentials) RUN { \\ echo '#!/bin/bash'; \\ echo 'set -e'; \\ echo ''; \\ echo '# ── Merge user wp-content into PVC ──'; \\ echo 'if [ -d /usr/src/wordpress-user/wp-content ]; then'; \\ echo ' echo ">>> Merging user wp-content into PVC..."'; \\ echo ' mkdir -p /var/www/html/wp-content'; \\ echo ' cp -a /usr/src/wordpress-user/wp-content/. /var/www/html/wp-content/'; \\ echo ' chown -R www-data:www-data /var/www/html/wp-content'; \\ echo ' echo ">>> User wp-content merged successfully"'; \\ echo 'fi'; \\ echo ''; \\ echo 'exec docker-entrypoint.sh apache2-foreground'; \\ } > /usr/local/bin/cloudhost-entrypoint.sh && chmod +x /usr/local/bin/cloudhost-entrypoint.sh ` : `# Fresh install — no user content to merge ` } # Set proper ownership RUN chown -R www-data:www-data /var/www/html EXPOSE 80 ${ hasUploadedCode ? `ENTRYPOINT ["cloudhost-entrypoint.sh"] CMD []` : `CMD ["apache2-foreground"]` } `; } // ─── Go Dockerfile ───────────────────────────────────────────────── private goDockerfile(app: Application, archiveEntries: string[] = []): string { const goVersion = app.runtimeVersion || '1.22'; const port = app.port || 8080; const buildTarget = detectGoBuildTarget(archiveEntries); return `# --- Build stage --- FROM golang:${goVersion}-alpine AS builder WORKDIR /app # Install git for fetching dependencies RUN apk add --no-cache git # Copy go mod files first for better caching COPY go.mod go.sum* ./ RUN go mod download || true # Copy source code COPY . . # Build the application RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags="-w -s" -o main ${buildTarget} # --- Production stage --- FROM alpine:3.19 WORKDIR /app # Add CA certificates for HTTPS requests RUN apk --no-cache add ca-certificates tzdata # Create non-root user RUN addgroup -g 1001 -S appgroup && adduser -S appuser -u 1001 -G appgroup # Copy the binary from builder COPY --from=builder /app/main . COPY --from=builder /app/static ./static 2>/dev/null || true COPY --from=builder /app/templates ./templates 2>/dev/null || true COPY --from=builder /app/public ./public 2>/dev/null || true # Create data directory for persistent storage RUN mkdir -p /app/data && chown -R appuser:appgroup /app USER appuser ENV PORT=${port} EXPOSE ${port} HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \\ CMD wget --no-verbose --tries=1 --spider http://localhost:${port}/health || exit 1 CMD ["./main"] `; } // ─── PHP (Plain) Dockerfile ──────────────────────────────────────── private phpDockerfile(app: Application): string { const phpVersion = app.phpVersion || '8.3'; const port = app.port || 80; return `FROM php:${phpVersion}-fpm-alpine RUN apk add --no-cache nginx supervisor curl \\ && docker-php-ext-install pdo pdo_mysql opcache \\ && docker-php-ext-install pdo_pgsql 2>/dev/null || true # Install common PHP extensions RUN apk add --no-cache libpng-dev libjpeg-turbo-dev freetype-dev \\ && docker-php-ext-configure gd --with-freetype --with-jpeg \\ && docker-php-ext-install gd WORKDIR /var/www/html COPY . . # Generate nginx config RUN mkdir -p /etc/nginx/http.d && \\ echo 'server {' > /etc/nginx/http.d/default.conf && \\ echo ' listen ${port};' >> /etc/nginx/http.d/default.conf && \\ echo ' root /var/www/html;' >> /etc/nginx/http.d/default.conf && \\ echo ' index index.php index.html;' >> /etc/nginx/http.d/default.conf && \\ echo ' client_max_body_size 64M;' >> /etc/nginx/http.d/default.conf && \\ echo ' location / { try_files \\$uri \\$uri/ /index.php?\\$query_string; }' >> /etc/nginx/http.d/default.conf && \\ echo ' location ~ \\.php\\$ {' >> /etc/nginx/http.d/default.conf && \\ echo ' fastcgi_pass 127.0.0.1:9000;' >> /etc/nginx/http.d/default.conf && \\ echo ' fastcgi_param SCRIPT_FILENAME \\$document_root\\$fastcgi_script_name;' >> /etc/nginx/http.d/default.conf && \\ echo ' include fastcgi_params;' >> /etc/nginx/http.d/default.conf && \\ echo ' }' >> /etc/nginx/http.d/default.conf && \\ echo ' location ~ /\\.ht { deny all; }' >> /etc/nginx/http.d/default.conf && \\ echo '}' >> /etc/nginx/http.d/default.conf # Generate supervisord config RUN echo '[supervisord]' > /etc/supervisord.conf && \\ echo 'nodaemon=true' >> /etc/supervisord.conf && \\ echo 'logfile=/dev/stdout' >> /etc/supervisord.conf && \\ echo 'logfile_maxbytes=0' >> /etc/supervisord.conf && \\ echo '[program:php-fpm]' >> /etc/supervisord.conf && \\ echo 'command=php-fpm -F' >> /etc/supervisord.conf && \\ echo 'autostart=true' >> /etc/supervisord.conf && \\ echo 'autorestart=true' >> /etc/supervisord.conf && \\ echo 'stdout_logfile=/dev/stdout' >> /etc/supervisord.conf && \\ echo 'stdout_logfile_maxbytes=0' >> /etc/supervisord.conf && \\ echo 'stderr_logfile=/dev/stderr' >> /etc/supervisord.conf && \\ echo 'stderr_logfile_maxbytes=0' >> /etc/supervisord.conf && \\ echo '[program:nginx]' >> /etc/supervisord.conf && \\ echo 'command=nginx -g "daemon off;"' >> /etc/supervisord.conf && \\ echo 'autostart=true' >> /etc/supervisord.conf && \\ echo 'autorestart=true' >> /etc/supervisord.conf && \\ echo 'stdout_logfile=/dev/stdout' >> /etc/supervisord.conf && \\ echo 'stdout_logfile_maxbytes=0' >> /etc/supervisord.conf && \\ echo 'stderr_logfile=/dev/stderr' >> /etc/supervisord.conf && \\ echo 'stderr_logfile_maxbytes=0' >> /etc/supervisord.conf # Use custom configs if provided RUN [ -f docker/nginx.conf ] && cp docker/nginx.conf /etc/nginx/http.d/default.conf || true RUN [ -f docker/supervisord.conf ] && cp docker/supervisord.conf /etc/supervisord.conf || true # Create upload and data directories RUN mkdir -p /var/www/html/uploads /var/www/html/data \\ && chown -R www-data:www-data /var/www/html EXPOSE ${port} CMD ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf"] `; } // ─── Python Dockerfile ───────────────────────────────────────────── private pythonDockerfile(app: Application): string { const pythonVersion = app.runtimeVersion || '3.12'; const port = app.port || 8000; return `# --- Build stage --- FROM python:${pythonVersion}-slim AS builder WORKDIR /app # Install build dependencies RUN apt-get update && apt-get install -y --no-install-recommends \\ build-essential libpq-dev \\ && rm -rf /var/lib/apt/lists/* # Copy requirements and install dependencies COPY requirements.txt* ./ RUN pip install --no-cache-dir --user -r requirements.txt 2>/dev/null || \\ pip install --no-cache-dir --user flask gunicorn # --- Production stage --- FROM python:${pythonVersion}-slim WORKDIR /app # Install runtime dependencies RUN apt-get update && apt-get install -y --no-install-recommends \\ libpq5 curl \\ && rm -rf /var/lib/apt/lists/* # Create non-root user RUN groupadd -g 1001 appgroup && useradd -r -u 1001 -g appgroup appuser # Copy installed packages from builder COPY --from=builder /root/.local /home/appuser/.local # Copy application code COPY . . # Create data directory RUN mkdir -p /app/data && chown -R appuser:appgroup /app USER appuser ENV PATH=/home/appuser/.local/bin:$PATH ENV PORT=${port} ENV PYTHONUNBUFFERED=1 EXPOSE ${port} HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \\ CMD curl -f http://localhost:${port}/health || exit 1 # Auto-detect: Flask, FastAPI, or plain Python CMD sh -c "if [ -f main.py ]; then if grep -qi fastapi main.py; then exec uvicorn main:app --host 0.0.0.0 --port ${port}; elif grep -qi flask main.py; then exec gunicorn -w 4 -b 0.0.0.0:${port} main:app; else exec python main.py; fi; elif [ -f app.py ]; then if grep -qi fastapi app.py; then exec uvicorn app:app --host 0.0.0.0 --port ${port}; elif grep -qi flask app.py; then exec gunicorn -w 4 -b 0.0.0.0:${port} app:app; else exec python app.py; fi; else exec gunicorn -w 4 -b 0.0.0.0:${port} app:app; fi" `; } // ─── Django Dockerfile ───────────────────────────────────────────── private djangoDockerfile(app: Application, archiveEntries: string[] = []): string { const pythonVersion = app.runtimeVersion || '3.12'; const port = app.port || 8000; const settingsModule = detectDjangoSettingsModule(archiveEntries); return `# --- Build stage --- FROM python:${pythonVersion}-slim AS builder WORKDIR /app # Install build dependencies RUN apt-get update && apt-get install -y --no-install-recommends \\ build-essential libpq-dev \\ && rm -rf /var/lib/apt/lists/* # Copy requirements and install dependencies COPY requirements.txt* ./ RUN pip install --no-cache-dir --user -r requirements.txt 2>/dev/null || \\ pip install --no-cache-dir --user django gunicorn psycopg2-binary mysqlclient # --- Production stage --- FROM python:${pythonVersion}-slim WORKDIR /app # Install runtime dependencies RUN apt-get update && apt-get install -y --no-install-recommends \\ libpq5 default-libmysqlclient-dev curl \\ && rm -rf /var/lib/apt/lists/* # Create non-root user RUN groupadd -g 1001 appgroup && useradd -r -u 1001 -g appgroup appuser # Copy installed packages from builder COPY --from=builder /root/.local /home/appuser/.local # Copy application code COPY . . # Create directories for static files and media RUN mkdir -p /app/staticfiles /app/media /app/data \\ && chown -R appuser:appgroup /app USER appuser ENV PATH=/home/appuser/.local/bin:$PATH ENV PORT=${port} ENV PYTHONUNBUFFERED=1 ENV DJANGO_SETTINGS_MODULE=${settingsModule} EXPOSE ${port} HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \\ CMD curl -f http://localhost:${port}/health/ || curl -f http://localhost:${port}/ || exit 1 # Auto-detect project structure and run migrations + collectstatic CMD sh -c "\\ PROJECT_NAME=\\$(find . -maxdepth 2 -name 'wsgi.py' | head -1 | cut -d'/' -f2) && \\ if [ -z \\\"\\$PROJECT_NAME\\\" ]; then PROJECT_NAME='config'; fi && \\ echo \\\"Django project: \\$PROJECT_NAME\\\" && \\ python manage.py migrate --noinput 2>/dev/null || true && \\ python manage.py collectstatic --noinput 2>/dev/null || true && \\ exec gunicorn \\$PROJECT_NAME.wsgi:application --bind 0.0.0.0:${port} --workers 4 --threads 2 \\ " `; } // ─── .NET Dockerfile ─────────────────────────────────────────────── private dotnetDockerfile(app: Application, archiveEntries: string[] = []): string { const dotnetVersion = app.runtimeVersion || '8.0'; const port = app.port || 5000; const csprojHint = detectShallowCsproj(archiveEntries); const csprojFind = csprojHint ? `CSPROJ="${csprojHint}"` : `CSPROJ=$(find . -maxdepth 3 -name '*.csproj' | head -1)`; return `# --- Build stage --- FROM mcr.microsoft.com/dotnet/sdk:${dotnetVersion} AS build WORKDIR /src # Copy source and locate project file (supports nested csproj layouts) COPY . . RUN ${csprojFind} && \\ test -n "$CSPROJ" && \\ dotnet restore "$CSPROJ" && \\ dotnet publish "$CSPROJ" -c Release -o /app/publish # --- Production stage --- FROM mcr.microsoft.com/dotnet/aspnet:${dotnetVersion} WORKDIR /app # Create non-root user RUN groupadd -g 1001 appgroup && useradd -r -u 1001 -g appgroup appuser # Copy published app COPY --from=build /app/publish . # Create data directory RUN mkdir -p /app/data && chown -R appuser:appgroup /app USER appuser ENV ASPNETCORE_URLS=http://+:${port} ENV DOTNET_RUNNING_IN_CONTAINER=true ENV ASPNETCORE_ENVIRONMENT=Production EXPOSE ${port} HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \\ CMD curl -f http://localhost:${port}/health || curl -f http://localhost:${port}/ || exit 1 # Auto-detect entry point DLL CMD ["sh", "-c", "DLL=$(find . -maxdepth 1 -name '*.dll' ! -name '*.deps.dll' ! -name '*.runtimeconfig.dll' | head -1) && dotnet $DLL"] `; } /** * Whether a K8s API error is a transient connectivity/availability blip that * should be retried rather than failing the operation. Covers socket-level * errors, request timeouts (incl. the apiserver "request did not complete * within the allotted timeout" message), DNS hiccups and 5xx/429 responses. */ private isTransientK8sError(err: any): boolean { const statusCode = err?.statusCode ?? err?.response?.statusCode ?? err?.body?.code; if (typeof statusCode === 'number' && (statusCode >= 500 || statusCode === 429)) { return true; } const haystack = [err?.code, err?.message, err?.body?.message, err?.cause?.code].filter(Boolean).join(' '); return /ECONNRESET|ECONNREFUSED|ETIMEDOUT|ESOCKETTIMEDOUT|EPIPE|EAI_AGAIN|ENOTFOUND|ENETUNREACH|socket hang up|timed? ?out|allotted timeout|did not complete|Client network socket disconnected/i.test( haystack, ); } private async waitForJobCompletion(batchApi: k8s.BatchV1Api, coreApi: k8s.CoreV1Api, jobName: string, namespace: string, timeoutSeconds: number, deploymentId?: string): Promise { const startTime = Date.now(); const timeoutMs = timeoutSeconds * 1000; let lastLoggedStatus = ''; while (Date.now() - startTime < timeoutMs) { this.throwIfCancelled(deploymentId); const elapsed = Date.now() - startTime; const buildPercent = Math.min(90, 15 + Math.round((elapsed / timeoutMs) * 75)); this.setProgress(deploymentId, { phase: 'building', percent: buildPercent, message: 'Building Docker image...', }); // ── Check Job status (with retry for transient connection errors) ── let job: k8s.V1Job; try { job = await batchApi.readNamespacedJob({ name: jobName, namespace }); } catch (pollErr: any) { // The Kaniko job keeps running independently of these status polls. // A single API blip (timeout, reset, 5xx, DNS) must NOT abort a build // that is still progressing — just retry on the next poll tick. if (this.isTransientK8sError(pollErr)) { const detail = pollErr?.code || pollErr?.message || pollErr?.statusCode || 'unknown'; this.logger.warn(`Transient K8s API error polling job ${jobName}: ${detail} — retrying in 5s`); await new Promise((r) => setTimeout(r, 5000)); continue; } throw pollErr; } const status = job.status; if (status?.succeeded && status.succeeded > 0) { this.logger.log(`Build job ${jobName} succeeded`); return; } // Check if the Job has permanently failed (all retries exhausted) const failedCondition = (status?.conditions || []).find((c) => c.type === 'Failed' && c.status === 'True'); if (failedCondition) { const logs = await this.getBuildLogs(coreApi, jobName, namespace); throw new Error(`Build job ${jobName} failed.\nLogs:\n${logs}`); } // Safety net: if failures exceed backoffLimit and no pod is still running const backoffLimit = job.spec?.backoffLimit ?? 0; const failedCount = status?.failed ?? 0; if (failedCount > backoffLimit) { // Double-check: are there still active pods? const activePods = (status as any)?.active ?? 0; if (activePods === 0) { const logs = await this.getBuildLogs(coreApi, jobName, namespace); throw new Error(`Build job ${jobName} failed: ${failedCount} failures exceeded backoffLimit=${backoffLimit}.\nLogs:\n${logs}`); } } // Log intermediate pod failures (retries still available) if (failedCount > 0) { this.logger.warn(`Build job ${jobName}: ${failedCount} pod failure(s), backoffLimit=${backoffLimit} — retrying...`); } // ── Check Pod status for early failure detection ── try { const pods = await coreApi.listNamespacedPod({ namespace, labelSelector: `job-name=${jobName}`, }); for (const pod of pods.items) { const podName = pod.metadata?.name || 'unknown'; const phase = pod.status?.phase; // Check all container statuses (init + regular) for stuck states const allStatuses = [...(pod.status?.initContainerStatuses || []), ...(pod.status?.containerStatuses || [])]; for (const cs of allStatuses) { const waiting = cs.state?.waiting; if (waiting?.reason) { const reason = waiting.reason; const msg = waiting.message || ''; // These are unrecoverable — fail fast instead of waiting 10 minutes const fatalReasons = ['ErrImagePull', 'ImagePullBackOff', 'CreateContainerConfigError', 'InvalidImageName', 'CrashLoopBackOff']; if (fatalReasons.includes(reason)) { const logs = await this.getBuildLogs(coreApi, jobName, namespace); throw new Error(`Build pod ${podName} stuck: ${reason} — ${msg}\nLogs:\n${logs}`); } // Log non-fatal waiting states periodically const statusKey = `${podName}:${cs.name}:${reason}`; if (statusKey !== lastLoggedStatus) { this.logger.warn(`Pod ${podName} container "${cs.name}": ${reason} — ${msg}`); lastLoggedStatus = statusKey; } } } // Log phase changes const phaseKey = `${podName}:${phase}`; if (phaseKey !== lastLoggedStatus && phase !== 'Succeeded') { this.logger.log(`Build pod ${podName}: phase=${phase}`); lastLoggedStatus = phaseKey; } } } catch (podErr: any) { // Don't fail the whole build just because pod status check failed if (podErr.message?.includes('stuck:') || podErr.message?.includes('Build pod')) { throw podErr; // Re-throw our own fatal errors } this.logger.warn(`Could not check pod status: ${podErr.message}`); } // Wait 5 seconds before polling again await new Promise((resolve) => setTimeout(resolve, 5000)); } // Timeout — get logs for debugging let logs = ''; try { logs = await this.getBuildLogs(coreApi, jobName, namespace); } catch {} throw new Error(`Build job ${jobName} timed out after ${timeoutSeconds}s\nLogs:\n${logs}`); } /** * Live build logs for an in-progress build, read straight from the running * build pod (init + kaniko containers). Returns null when there is no active * build session for this deployment (e.g. build already finished/cleaned up), * so callers can fall back to the persisted build log. */ async getLiveBuildLog(deploymentId: string): Promise { const session = this.activeBuilds.get(deploymentId); if (!session?.coreApi || !session.namespace || !session.buildPodName) { return null; } try { return await this.getBuildLogs(session.coreApi, session.buildPodName, session.namespace); } catch { return null; } } private async getBuildLogs(coreApi: k8s.CoreV1Api, jobName: string, namespace: string): Promise { try { const pods = await coreApi.listNamespacedPod({ namespace, labelSelector: `job-name=${jobName}`, }); if (pods.items.length === 0) { return 'No pods found for build job.'; } const podName = pods.items[0].metadata?.name; if (!podName) return 'Pod name not found.'; // Get logs from all containers (init + kaniko) let allLogs = ''; const containers = [...(pods.items[0].spec?.initContainers || []), ...(pods.items[0].spec?.containers || [])]; for (const container of containers) { try { const logResponse = await coreApi.readNamespacedPodLog({ name: podName, namespace, container: container.name, tailLines: 500, }); allLogs += `\n--- ${container.name} ---\n${logResponse}`; } catch { allLogs += `\n--- ${container.name} --- (no logs available)`; } } return allLogs; } catch (e: any) { return `Failed to retrieve logs: ${e.message}`; } } }