import { Injectable, Logger } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; import * as k8s from '@kubernetes/client-node'; import * as fs from 'fs'; import * as path from 'path'; import { Application } from '../applications/entities/application.entity'; import { AppRuntime } from '../common/enums'; import { ClustersService } from '../clusters/clusters.service'; @Injectable() export class BuildService { private readonly logger = new Logger(BuildService.name); constructor( private configService: ConfigService, private clustersService: ClustersService, ) {} /** * Builds a Docker image for the application using Kaniko inside K8s. * Returns the full image URI (registry/repo:tag). */ async buildImage(app: Application): Promise { // Internal registry (used by Kaniko inside K8s for pushing) const internalRegistryUrl = this.configService.get('registry.url') || 'registry.cloudhost-builds.svc.cluster.local:5000'; // External registry URL (used by kubelet for pulling — NodePort or external) const pullRegistryUrl = this.configService.get('registry.pullUrl') || 'localhost:30500'; const buildNamespace = this.configService.get('build.namespace') || 'cloudhost-builds'; const tag = `${Date.now()}`; const pushImageUri = `${internalRegistryUrl}/${app.userId}/${app.name}:${tag}`; const pullImageUri = `${pullRegistryUrl}/${app.userId}/${app.name}:${tag}`; this.logger.log(`Starting image build for ${app.name} → push: ${pushImageUri}, pull: ${pullImageUri}`); // Determine Dockerfile based on runtime const dockerfileContent = this.generateDockerfile(app); // Create Kaniko build pod const buildPodName = `build-${app.name}-${tag}`.substring(0, 63).replace(/[^a-z0-9-]/g, ''); // Use the cluster's kubeconfig instead of default const cluster = app.clusterId ? await this.clustersService.findOne(app.clusterId) : await this.clustersService.getDefault(); const kc = new k8s.KubeConfig(); kc.loadFromString(cluster.kubeconfig); const coreApi = kc.makeApiClient(k8s.CoreV1Api); const batchApi = kc.makeApiClient(k8s.BatchV1Api); // Determine if we have uploaded code or git URL const codePath = app.codePath ? path.resolve(app.codePath) : null; const hasUploadedCode = codePath && fs.existsSync(codePath); const hasGitUrl = !!app.gitUrl; // Create ConfigMap with Dockerfile const dockerfileConfigMap = { apiVersion: 'v1', kind: 'ConfigMap', metadata: { name: `${buildPodName}-dockerfile`, namespace: buildNamespace, }, data: { Dockerfile: dockerfileContent, }, }; // If we have uploaded code, create a ConfigMap with the zip as base64 let sourceConfigMapName: string | undefined; if (hasUploadedCode) { const zipBuffer = fs.readFileSync(codePath!); const zipBase64 = zipBuffer.toString('base64'); sourceConfigMapName = `${buildPodName}-source`; // ConfigMap has 1MB limit, for larger files we'd need a PVC approach // For now, use a Secret (which can hold up to 1MB too, but binary-safe) const sourceSecret = { apiVersion: 'v1', kind: 'Secret', metadata: { name: sourceConfigMapName, namespace: buildNamespace, }, data: { 'source.zip': zipBase64, }, }; await coreApi.createNamespacedSecret(buildNamespace!, sourceSecret); this.logger.log(`Created source secret: ${sourceConfigMapName} (${(zipBuffer.length / 1024).toFixed(1)} KB)`); } // Build the Kaniko Job spec // Always use dir context — init containers prepare /workspace/source const kanikoArgs = [ '--dockerfile=/workspace/Dockerfile', '--context=dir:///workspace/source', `--destination=${pushImageUri}`, '--cache=true', `--cache-repo=${internalRegistryUrl}/${app.userId}/cache`, '--insecure', '--skip-tls-verify', ]; const volumes: any[] = [ { name: 'docker-config', secret: { secretName: 'registry-credentials' }, }, { name: 'dockerfile', configMap: { name: `${buildPodName}-dockerfile`, }, }, { name: 'workspace', emptyDir: {}, }, ]; const initContainers: any[] = []; if (hasUploadedCode && sourceConfigMapName) { // Add the source secret as a volume volumes.push({ name: 'source-zip', secret: { secretName: sourceConfigMapName }, }); // Add init container that unzips the source code initContainers.push({ name: 'unzip-source', image: 'alpine:3.19', command: ['sh', '-c', ` apk add --no-cache unzip && cp /workspace/Dockerfile /workspace-out/Dockerfile && mkdir -p /workspace-out/source && cd /workspace-out/source && unzip /source/source.zip && ls -la /workspace-out/source/ `], volumeMounts: [ { name: 'workspace', mountPath: '/workspace-out' }, { name: 'dockerfile', mountPath: '/workspace/Dockerfile', subPath: 'Dockerfile' }, { name: 'source-zip', mountPath: '/source' }, ], }); } else if (hasGitUrl) { // Clone git repo into /workspace/source, then copy our generated Dockerfile initContainers.push({ name: 'git-clone', image: 'alpine/git:2.43.0', command: ['sh', '-c', ` echo ">>> Cloning ${app.gitUrl}" && git clone --depth 1 ${app.gitUrl} /workspace-out/source && cp /dockerfile/Dockerfile /workspace-out/Dockerfile && echo ">>> Workspace contents:" && ls -la /workspace-out/source/ `], volumeMounts: [ { name: 'workspace', mountPath: '/workspace-out' }, { name: 'dockerfile', mountPath: '/dockerfile' }, ], }); } // Kaniko container volume mounts const kanikoVolumeMounts: any[] = [ { name: 'docker-config', mountPath: '/kaniko/.docker' }, { name: 'workspace', mountPath: '/workspace' }, ]; // If no uploaded code and no git, mount dockerfile directly if (!hasUploadedCode && !hasGitUrl) { kanikoVolumeMounts.push({ name: 'dockerfile', mountPath: '/workspace/Dockerfile', subPath: 'Dockerfile', }); } const buildJob: k8s.V1Job = { apiVersion: 'batch/v1', kind: 'Job', metadata: { name: buildPodName, namespace: buildNamespace, }, spec: { backoffLimit: 2, ttlSecondsAfterFinished: 300, template: { spec: { serviceAccountName: this.configService.get('build.serviceAccount'), initContainers: initContainers.length > 0 ? initContainers : undefined, containers: [ { name: 'kaniko', image: 'gcr.io/kaniko-project/executor:latest', args: kanikoArgs, volumeMounts: kanikoVolumeMounts, resources: { requests: { cpu: '500m', memory: '1Gi' }, limits: { cpu: '2', memory: '4Gi' }, }, }, ], restartPolicy: 'Never', volumes, }, }, }, }; try { await coreApi.createNamespacedConfigMap(buildNamespace!, dockerfileConfigMap); await batchApi.createNamespacedJob(buildNamespace!, buildJob); // Wait for build to complete await this.waitForJobCompletion(batchApi, coreApi, buildPodName, buildNamespace!, 600); this.logger.log(`Build completed successfully: ${pullImageUri}`); return pullImageUri; } catch (error: any) { // Try to get build logs for debugging try { const logs = await this.getBuildLogs(coreApi, buildPodName, buildNamespace!); this.logger.error(`Build logs for ${buildPodName}:\n${logs}`); } catch {} this.logger.error(`Build failed for ${app.name}:`, error.body || error.message); throw new Error(`Image build failed: ${error.body?.message || error.message}`); } } private generateDockerfile(app: Application): string { switch (app.runtime) { case AppRuntime.NODEJS: return this.nodeDockerfile(app); case AppRuntime.LARAVEL: return this.laravelDockerfile(app); default: throw new Error(`Unsupported runtime: ${app.runtime}`); } } private nodeDockerfile(app: Application): string { const port = app.port || 3000; return `# --- Build stage --- FROM node:20-alpine AS builder WORKDIR /app COPY package*.json ./ RUN if [ -f package-lock.json ]; then npm ci; else npm install; fi && npm cache clean --force COPY . . # Auto-detect Next.js and enable standalone output RUN if ([ -f next.config.js ] || [ -f next.config.mjs ] || [ -f next.config.ts ]); then \\ echo ">>> Next.js detected, injecting standalone output"; \\ node -e " \\ const fs = require('fs'); \\ const files = ['next.config.js','next.config.mjs','next.config.ts']; \\ for (const f of files) { \\ if (fs.existsSync(f)) { \\ let c = fs.readFileSync(f,'utf8'); \\ if (!c.includes('standalone')) { \\ c = c.replace(/output\\s*:\\s*['\\\"][^'\\\"]*['\\\"]\\s*,?/g, ''); \\ c = c.replace(/(\\{)/, '\\$1 output: \\\"standalone\\\",'); \\ fs.writeFileSync(f, c); \\ console.log('Patched ' + f + ' with standalone output'); \\ } else { \\ console.log(f + ' already has standalone'); \\ } \\ break; \\ } \\ } \\ "; \\ fi RUN npm run build 2>/dev/null || true # --- Production stage --- FROM node:20-alpine AS runner WORKDIR /app RUN addgroup -g 1001 -S appgroup && adduser -S appuser -u 1001 # Copy all build output to temp COPY --from=builder /app /tmp/fullapp # Detect: Next.js standalone vs regular Node.js RUN if [ -d /tmp/fullapp/.next/standalone ]; then \\ echo ">>> Next.js standalone mode"; \\ cp -a /tmp/fullapp/.next/standalone/. .; \\ mkdir -p .next/static; \\ [ -d /tmp/fullapp/.next/static ] && cp -a /tmp/fullapp/.next/static/. .next/static/; \\ [ -d /tmp/fullapp/public ] && cp -a /tmp/fullapp/public ./public; \\ echo "standalone" > /app/.mode; \\ else \\ echo ">>> Regular Node.js app"; \\ cp -a /tmp/fullapp/. .; \\ echo "regular" > /app/.mode; \\ fi && rm -rf /tmp/fullapp USER appuser ENV PORT=${port} ENV HOSTNAME=0.0.0.0 EXPOSE ${port} CMD ["sh", "-c", "if [ \\"$(cat /app/.mode)\\" = \\"standalone\\" ] && [ -f server.js ]; then node server.js; else npm start; fi"] `; } private laravelDockerfile(app: Application): string { return `# --- Build stage --- FROM composer:2 AS composer WORKDIR /app COPY composer.json composer.lock ./ RUN composer install --no-dev --no-scripts --no-autoloader --prefer-dist COPY . . RUN composer dump-autoload --optimize --no-dev # --- Production stage --- FROM php:8.3-fpm-alpine RUN apk add --no-cache nginx supervisor \\ && docker-php-ext-install pdo pdo_mysql pdo_pgsql opcache WORKDIR /var/www/html COPY --from=composer /app . COPY docker/nginx.conf /etc/nginx/http.d/default.conf COPY docker/supervisord.conf /etc/supervisord.conf RUN chown -R www-data:www-data /var/www/html/storage /var/www/html/bootstrap/cache RUN php artisan config:cache && php artisan route:cache && php artisan view:cache || true EXPOSE ${app.port || 8000} CMD ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf"] `; } private async waitForJobCompletion( batchApi: k8s.BatchV1Api, coreApi: k8s.CoreV1Api, jobName: string, namespace: string, timeoutSeconds: number, ): Promise { const startTime = Date.now(); const timeoutMs = timeoutSeconds * 1000; while (Date.now() - startTime < timeoutMs) { const job = await batchApi.readNamespacedJob(jobName, namespace); const status = job.body.status; if (status?.succeeded && status.succeeded > 0) { return; // Build completed } if (status?.failed && status.failed > 0) { // Try to get pod logs for more info const logs = await this.getBuildLogs(coreApi, jobName, namespace); throw new Error(`Build job ${jobName} failed.\nLogs:\n${logs}`); } // Wait 5 seconds before polling again await new Promise((resolve) => setTimeout(resolve, 5000)); } throw new Error(`Build job ${jobName} timed out after ${timeoutSeconds}s`); } private async getBuildLogs( coreApi: k8s.CoreV1Api, jobName: string, namespace: string, ): Promise { try { const pods = await coreApi.listNamespacedPod( namespace, undefined, undefined, undefined, undefined, `job-name=${jobName}`, ); if (pods.body.items.length === 0) { return 'No pods found for build job.'; } const podName = pods.body.items[0].metadata?.name; if (!podName) return 'Pod name not found.'; // Get logs from all containers (init + kaniko) let allLogs = ''; const containers = [ ...(pods.body.items[0].spec?.initContainers || []), ...(pods.body.items[0].spec?.containers || []), ]; for (const container of containers) { try { const logResponse = await coreApi.readNamespacedPodLog( podName, namespace, container.name, undefined, undefined, undefined, undefined, undefined, undefined, 500, ); allLogs += `\n--- ${container.name} ---\n${logResponse.body}`; } catch { allLogs += `\n--- ${container.name} --- (no logs available)`; } } return allLogs; } catch (e: any) { return `Failed to retrieve logs: ${e.message}`; } } }