keyhan
426 lines
14 KiB
TypeScript
426 lines
14 KiB
TypeScript
import { Injectable, Logger } from '@nestjs/common';
|
|
import { ConfigService } from '@nestjs/config';
|
|
import * as k8s from '@kubernetes/client-node';
|
|
import * as fs from 'fs';
|
|
import * as path from 'path';
|
|
import { Application } from '../applications/entities/application.entity';
|
|
import { AppRuntime } from '../common/enums';
|
|
import { ClustersService } from '../clusters/clusters.service';
|
|
|
|
@Injectable()
|
|
export class BuildService {
|
|
private readonly logger = new Logger(BuildService.name);
|
|
|
|
constructor(
|
|
private configService: ConfigService,
|
|
private clustersService: ClustersService,
|
|
) {}
|
|
|
|
/**
|
|
* Builds a Docker image for the application using Kaniko inside K8s.
|
|
* Returns the full image URI (registry/repo:tag).
|
|
*/
|
|
async buildImage(app: Application): Promise<string> {
|
|
// Internal registry (used by Kaniko inside K8s for pushing)
|
|
const internalRegistryUrl = this.configService.get<string>('registry.url') || 'registry.cloudhost-builds.svc.cluster.local:5000';
|
|
// External registry URL (used by kubelet for pulling — NodePort or external)
|
|
const pullRegistryUrl = this.configService.get<string>('registry.pullUrl') || 'localhost:30500';
|
|
const buildNamespace = this.configService.get<string>('build.namespace') || 'cloudhost-builds';
|
|
const tag = `${Date.now()}`;
|
|
const pushImageUri = `${internalRegistryUrl}/${app.userId}/${app.name}:${tag}`;
|
|
const pullImageUri = `${pullRegistryUrl}/${app.userId}/${app.name}:${tag}`;
|
|
|
|
this.logger.log(`Starting image build for ${app.name} → push: ${pushImageUri}, pull: ${pullImageUri}`);
|
|
|
|
// Determine Dockerfile based on runtime
|
|
const dockerfileContent = this.generateDockerfile(app);
|
|
|
|
// Create Kaniko build pod
|
|
const buildPodName = `build-${app.name}-${tag}`.substring(0, 63).replace(/[^a-z0-9-]/g, '');
|
|
|
|
// Use the cluster's kubeconfig instead of default
|
|
const cluster = app.clusterId
|
|
? await this.clustersService.findOne(app.clusterId)
|
|
: await this.clustersService.getDefault();
|
|
|
|
const kc = new k8s.KubeConfig();
|
|
kc.loadFromString(cluster.kubeconfig);
|
|
const coreApi = kc.makeApiClient(k8s.CoreV1Api);
|
|
const batchApi = kc.makeApiClient(k8s.BatchV1Api);
|
|
|
|
// Determine if we have uploaded code or git URL
|
|
const codePath = app.codePath ? path.resolve(app.codePath) : null;
|
|
const hasUploadedCode = codePath && fs.existsSync(codePath);
|
|
const hasGitUrl = !!app.gitUrl;
|
|
|
|
// Create ConfigMap with Dockerfile
|
|
const dockerfileConfigMap = {
|
|
apiVersion: 'v1',
|
|
kind: 'ConfigMap',
|
|
metadata: {
|
|
name: `${buildPodName}-dockerfile`,
|
|
namespace: buildNamespace,
|
|
},
|
|
data: {
|
|
Dockerfile: dockerfileContent,
|
|
},
|
|
};
|
|
|
|
// If we have uploaded code, create a ConfigMap with the zip as base64
|
|
let sourceConfigMapName: string | undefined;
|
|
if (hasUploadedCode) {
|
|
const zipBuffer = fs.readFileSync(codePath!);
|
|
const zipBase64 = zipBuffer.toString('base64');
|
|
|
|
sourceConfigMapName = `${buildPodName}-source`;
|
|
|
|
// ConfigMap has 1MB limit, for larger files we'd need a PVC approach
|
|
// For now, use a Secret (which can hold up to 1MB too, but binary-safe)
|
|
const sourceSecret = {
|
|
apiVersion: 'v1',
|
|
kind: 'Secret',
|
|
metadata: {
|
|
name: sourceConfigMapName,
|
|
namespace: buildNamespace,
|
|
},
|
|
data: {
|
|
'source.zip': zipBase64,
|
|
},
|
|
};
|
|
|
|
await coreApi.createNamespacedSecret(buildNamespace!, sourceSecret);
|
|
this.logger.log(`Created source secret: ${sourceConfigMapName} (${(zipBuffer.length / 1024).toFixed(1)} KB)`);
|
|
}
|
|
|
|
// Build the Kaniko Job spec
|
|
// Always use dir context — init containers prepare /workspace/source
|
|
const kanikoArgs = [
|
|
'--dockerfile=/workspace/Dockerfile',
|
|
'--context=dir:///workspace/source',
|
|
`--destination=${pushImageUri}`,
|
|
'--cache=true',
|
|
`--cache-repo=${internalRegistryUrl}/${app.userId}/cache`,
|
|
'--insecure',
|
|
'--skip-tls-verify',
|
|
];
|
|
|
|
const volumes: any[] = [
|
|
{
|
|
name: 'docker-config',
|
|
secret: { secretName: 'registry-credentials' },
|
|
},
|
|
{
|
|
name: 'dockerfile',
|
|
configMap: {
|
|
name: `${buildPodName}-dockerfile`,
|
|
},
|
|
},
|
|
{
|
|
name: 'workspace',
|
|
emptyDir: {},
|
|
},
|
|
];
|
|
|
|
const initContainers: any[] = [];
|
|
|
|
if (hasUploadedCode && sourceConfigMapName) {
|
|
// Add the source secret as a volume
|
|
volumes.push({
|
|
name: 'source-zip',
|
|
secret: { secretName: sourceConfigMapName },
|
|
});
|
|
|
|
// Add init container that unzips the source code
|
|
initContainers.push({
|
|
name: 'unzip-source',
|
|
image: 'alpine:3.19',
|
|
command: ['sh', '-c', `
|
|
apk add --no-cache unzip &&
|
|
cp /workspace/Dockerfile /workspace-out/Dockerfile &&
|
|
mkdir -p /workspace-out/source &&
|
|
cd /workspace-out/source &&
|
|
unzip /source/source.zip &&
|
|
ls -la /workspace-out/source/
|
|
`],
|
|
volumeMounts: [
|
|
{ name: 'workspace', mountPath: '/workspace-out' },
|
|
{ name: 'dockerfile', mountPath: '/workspace/Dockerfile', subPath: 'Dockerfile' },
|
|
{ name: 'source-zip', mountPath: '/source' },
|
|
],
|
|
});
|
|
} else if (hasGitUrl) {
|
|
// Clone git repo into /workspace/source, then copy our generated Dockerfile
|
|
initContainers.push({
|
|
name: 'git-clone',
|
|
image: 'alpine/git:2.43.0',
|
|
command: ['sh', '-c', `
|
|
echo ">>> Cloning ${app.gitUrl}" &&
|
|
git clone --depth 1 ${app.gitUrl} /workspace-out/source &&
|
|
cp /dockerfile/Dockerfile /workspace-out/Dockerfile &&
|
|
echo ">>> Workspace contents:" &&
|
|
ls -la /workspace-out/source/
|
|
`],
|
|
volumeMounts: [
|
|
{ name: 'workspace', mountPath: '/workspace-out' },
|
|
{ name: 'dockerfile', mountPath: '/dockerfile' },
|
|
],
|
|
});
|
|
}
|
|
|
|
// Kaniko container volume mounts
|
|
const kanikoVolumeMounts: any[] = [
|
|
{ name: 'docker-config', mountPath: '/kaniko/.docker' },
|
|
{ name: 'workspace', mountPath: '/workspace' },
|
|
];
|
|
|
|
// If no uploaded code and no git, mount dockerfile directly
|
|
if (!hasUploadedCode && !hasGitUrl) {
|
|
kanikoVolumeMounts.push({
|
|
name: 'dockerfile',
|
|
mountPath: '/workspace/Dockerfile',
|
|
subPath: 'Dockerfile',
|
|
});
|
|
}
|
|
|
|
const buildJob: k8s.V1Job = {
|
|
apiVersion: 'batch/v1',
|
|
kind: 'Job',
|
|
metadata: {
|
|
name: buildPodName,
|
|
namespace: buildNamespace,
|
|
},
|
|
spec: {
|
|
backoffLimit: 2,
|
|
ttlSecondsAfterFinished: 300,
|
|
template: {
|
|
spec: {
|
|
serviceAccountName: this.configService.get<string>('build.serviceAccount'),
|
|
initContainers: initContainers.length > 0 ? initContainers : undefined,
|
|
containers: [
|
|
{
|
|
name: 'kaniko',
|
|
image: 'gcr.io/kaniko-project/executor:latest',
|
|
args: kanikoArgs,
|
|
volumeMounts: kanikoVolumeMounts,
|
|
resources: {
|
|
requests: { cpu: '500m', memory: '1Gi' },
|
|
limits: { cpu: '2', memory: '4Gi' },
|
|
},
|
|
},
|
|
],
|
|
restartPolicy: 'Never',
|
|
volumes,
|
|
},
|
|
},
|
|
},
|
|
};
|
|
|
|
try {
|
|
await coreApi.createNamespacedConfigMap(buildNamespace!, dockerfileConfigMap);
|
|
await batchApi.createNamespacedJob(buildNamespace!, buildJob);
|
|
|
|
// Wait for build to complete
|
|
await this.waitForJobCompletion(batchApi, coreApi, buildPodName, buildNamespace!, 600);
|
|
|
|
this.logger.log(`Build completed successfully: ${pullImageUri}`);
|
|
return pullImageUri;
|
|
} catch (error: any) {
|
|
// Try to get build logs for debugging
|
|
try {
|
|
const logs = await this.getBuildLogs(coreApi, buildPodName, buildNamespace!);
|
|
this.logger.error(`Build logs for ${buildPodName}:\n${logs}`);
|
|
} catch {}
|
|
|
|
this.logger.error(`Build failed for ${app.name}:`, error.body || error.message);
|
|
throw new Error(`Image build failed: ${error.body?.message || error.message}`);
|
|
}
|
|
}
|
|
|
|
private generateDockerfile(app: Application): string {
|
|
switch (app.runtime) {
|
|
case AppRuntime.NODEJS:
|
|
return this.nodeDockerfile(app);
|
|
case AppRuntime.LARAVEL:
|
|
return this.laravelDockerfile(app);
|
|
default:
|
|
throw new Error(`Unsupported runtime: ${app.runtime}`);
|
|
}
|
|
}
|
|
|
|
private nodeDockerfile(app: Application): string {
|
|
const port = app.port || 3000;
|
|
return `# --- Build stage ---
|
|
FROM node:20-alpine AS builder
|
|
WORKDIR /app
|
|
COPY package*.json ./
|
|
RUN if [ -f package-lock.json ]; then npm ci; else npm install; fi && npm cache clean --force
|
|
COPY . .
|
|
|
|
# Auto-detect Next.js and enable standalone output
|
|
RUN if ([ -f next.config.js ] || [ -f next.config.mjs ] || [ -f next.config.ts ]); then \\
|
|
echo ">>> Next.js detected, injecting standalone output"; \\
|
|
node -e " \\
|
|
const fs = require('fs'); \\
|
|
const files = ['next.config.js','next.config.mjs','next.config.ts']; \\
|
|
for (const f of files) { \\
|
|
if (fs.existsSync(f)) { \\
|
|
let c = fs.readFileSync(f,'utf8'); \\
|
|
if (!c.includes('standalone')) { \\
|
|
c = c.replace(/output\\s*:\\s*['\\\"][^'\\\"]*['\\\"]\\s*,?/g, ''); \\
|
|
c = c.replace(/(\\{)/, '\\$1 output: \\\"standalone\\\",'); \\
|
|
fs.writeFileSync(f, c); \\
|
|
console.log('Patched ' + f + ' with standalone output'); \\
|
|
} else { \\
|
|
console.log(f + ' already has standalone'); \\
|
|
} \\
|
|
break; \\
|
|
} \\
|
|
} \\
|
|
"; \\
|
|
fi
|
|
|
|
RUN npm run build 2>/dev/null || true
|
|
|
|
# --- Production stage ---
|
|
FROM node:20-alpine AS runner
|
|
WORKDIR /app
|
|
RUN addgroup -g 1001 -S appgroup && adduser -S appuser -u 1001
|
|
|
|
# Copy all build output to temp
|
|
COPY --from=builder /app /tmp/fullapp
|
|
|
|
# Detect: Next.js standalone vs regular Node.js
|
|
RUN if [ -d /tmp/fullapp/.next/standalone ]; then \\
|
|
echo ">>> Next.js standalone mode"; \\
|
|
cp -a /tmp/fullapp/.next/standalone/. .; \\
|
|
mkdir -p .next/static; \\
|
|
[ -d /tmp/fullapp/.next/static ] && cp -a /tmp/fullapp/.next/static/. .next/static/; \\
|
|
[ -d /tmp/fullapp/public ] && cp -a /tmp/fullapp/public ./public; \\
|
|
echo "standalone" > /app/.mode; \\
|
|
else \\
|
|
echo ">>> Regular Node.js app"; \\
|
|
cp -a /tmp/fullapp/. .; \\
|
|
echo "regular" > /app/.mode; \\
|
|
fi && rm -rf /tmp/fullapp
|
|
|
|
USER appuser
|
|
ENV PORT=${port}
|
|
ENV HOSTNAME=0.0.0.0
|
|
EXPOSE ${port}
|
|
CMD ["sh", "-c", "if [ \\"$(cat /app/.mode)\\" = \\"standalone\\" ] && [ -f server.js ]; then node server.js; else npm start; fi"]
|
|
`;
|
|
}
|
|
|
|
private laravelDockerfile(app: Application): string {
|
|
return `# --- Build stage ---
|
|
FROM composer:2 AS composer
|
|
WORKDIR /app
|
|
COPY composer.json composer.lock ./
|
|
RUN composer install --no-dev --no-scripts --no-autoloader --prefer-dist
|
|
COPY . .
|
|
RUN composer dump-autoload --optimize --no-dev
|
|
|
|
# --- Production stage ---
|
|
FROM php:8.3-fpm-alpine
|
|
RUN apk add --no-cache nginx supervisor \\
|
|
&& docker-php-ext-install pdo pdo_mysql pdo_pgsql opcache
|
|
|
|
WORKDIR /var/www/html
|
|
COPY --from=composer /app .
|
|
COPY docker/nginx.conf /etc/nginx/http.d/default.conf
|
|
COPY docker/supervisord.conf /etc/supervisord.conf
|
|
|
|
RUN chown -R www-data:www-data /var/www/html/storage /var/www/html/bootstrap/cache
|
|
RUN php artisan config:cache && php artisan route:cache && php artisan view:cache || true
|
|
|
|
EXPOSE ${app.port || 8000}
|
|
CMD ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf"]
|
|
`;
|
|
}
|
|
|
|
private async waitForJobCompletion(
|
|
batchApi: k8s.BatchV1Api,
|
|
coreApi: k8s.CoreV1Api,
|
|
jobName: string,
|
|
namespace: string,
|
|
timeoutSeconds: number,
|
|
): Promise<void> {
|
|
const startTime = Date.now();
|
|
const timeoutMs = timeoutSeconds * 1000;
|
|
|
|
while (Date.now() - startTime < timeoutMs) {
|
|
const job = await batchApi.readNamespacedJob(jobName, namespace);
|
|
const status = job.body.status;
|
|
|
|
if (status?.succeeded && status.succeeded > 0) {
|
|
return; // Build completed
|
|
}
|
|
|
|
if (status?.failed && status.failed > 0) {
|
|
// Try to get pod logs for more info
|
|
const logs = await this.getBuildLogs(coreApi, jobName, namespace);
|
|
throw new Error(`Build job ${jobName} failed.\nLogs:\n${logs}`);
|
|
}
|
|
|
|
// Wait 5 seconds before polling again
|
|
await new Promise((resolve) => setTimeout(resolve, 5000));
|
|
}
|
|
|
|
throw new Error(`Build job ${jobName} timed out after ${timeoutSeconds}s`);
|
|
}
|
|
|
|
private async getBuildLogs(
|
|
coreApi: k8s.CoreV1Api,
|
|
jobName: string,
|
|
namespace: string,
|
|
): Promise<string> {
|
|
try {
|
|
const pods = await coreApi.listNamespacedPod(
|
|
namespace,
|
|
undefined,
|
|
undefined,
|
|
undefined,
|
|
undefined,
|
|
`job-name=${jobName}`,
|
|
);
|
|
|
|
if (pods.body.items.length === 0) {
|
|
return 'No pods found for build job.';
|
|
}
|
|
|
|
const podName = pods.body.items[0].metadata?.name;
|
|
if (!podName) return 'Pod name not found.';
|
|
|
|
// Get logs from all containers (init + kaniko)
|
|
let allLogs = '';
|
|
const containers = [
|
|
...(pods.body.items[0].spec?.initContainers || []),
|
|
...(pods.body.items[0].spec?.containers || []),
|
|
];
|
|
|
|
for (const container of containers) {
|
|
try {
|
|
const logResponse = await coreApi.readNamespacedPodLog(
|
|
podName,
|
|
namespace,
|
|
container.name,
|
|
undefined,
|
|
undefined,
|
|
undefined,
|
|
undefined,
|
|
undefined,
|
|
undefined,
|
|
500,
|
|
);
|
|
allLogs += `\n--- ${container.name} ---\n${logResponse.body}`;
|
|
} catch {
|
|
allLogs += `\n--- ${container.name} --- (no logs available)`;
|
|
}
|
|
}
|
|
return allLogs;
|
|
} catch (e: any) {
|
|
return `Failed to retrieve logs: ${e.message}`;
|
|
}
|
|
}
|
|
}
|