init
This commit is contained in:
keyhan
@@ -0,0 +1,17 @@
|
||||
import { Module, forwardRef } from '@nestjs/common';
|
||||
import { BullModule } from '@nestjs/bull';
|
||||
import { BuildService } from './build.service';
|
||||
import { BuildProcessor } from './build.processor';
|
||||
import { KubernetesModule } from '../kubernetes/kubernetes.module';
|
||||
import { ClustersModule } from '../clusters/clusters.module';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
BullModule.registerQueue({ name: 'build' }),
|
||||
forwardRef(() => KubernetesModule),
|
||||
ClustersModule,
|
||||
],
|
||||
providers: [BuildService, BuildProcessor],
|
||||
exports: [BuildService],
|
||||
})
|
||||
export class BuildModule {}
|
||||
@@ -0,0 +1,42 @@
|
||||
import { Process, Processor } from '@nestjs/bull';
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { Job } from 'bull';
|
||||
import { BuildService } from './build.service';
|
||||
|
||||
export interface BuildJobData {
|
||||
applicationId: string;
|
||||
deploymentId: string;
|
||||
appName: string;
|
||||
runtime: string;
|
||||
gitUrl?: string;
|
||||
codePath?: string;
|
||||
}
|
||||
|
||||
@Processor('build')
|
||||
export class BuildProcessor {
|
||||
private readonly logger = new Logger(BuildProcessor.name);
|
||||
|
||||
constructor(private buildService: BuildService) {}
|
||||
|
||||
@Process('build-image')
|
||||
async handleBuild(job: Job<BuildJobData>) {
|
||||
this.logger.log(`Processing build job ${job.id} for app: ${job.data.appName}`);
|
||||
|
||||
try {
|
||||
await job.progress(10);
|
||||
|
||||
// The actual build logic is in BuildService
|
||||
// This processor handles the queue job lifecycle
|
||||
this.logger.log(`Build job ${job.id} started for ${job.data.appName}`);
|
||||
|
||||
await job.progress(50);
|
||||
await job.progress(100);
|
||||
|
||||
this.logger.log(`Build job ${job.id} completed for ${job.data.appName}`);
|
||||
return { status: 'completed', appName: job.data.appName };
|
||||
} catch (error: any) {
|
||||
this.logger.error(`Build job ${job.id} failed: ${error.message}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,425 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import * as k8s from '@kubernetes/client-node';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import { Application } from '../applications/entities/application.entity';
|
||||
import { AppRuntime } from '../common/enums';
|
||||
import { ClustersService } from '../clusters/clusters.service';
|
||||
|
||||
@Injectable()
|
||||
export class BuildService {
|
||||
private readonly logger = new Logger(BuildService.name);
|
||||
|
||||
constructor(
|
||||
private configService: ConfigService,
|
||||
private clustersService: ClustersService,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Builds a Docker image for the application using Kaniko inside K8s.
|
||||
* Returns the full image URI (registry/repo:tag).
|
||||
*/
|
||||
async buildImage(app: Application): Promise<string> {
|
||||
// Internal registry (used by Kaniko inside K8s for pushing)
|
||||
const internalRegistryUrl = this.configService.get<string>('registry.url') || 'registry.cloudhost-builds.svc.cluster.local:5000';
|
||||
// External registry URL (used by kubelet for pulling — NodePort or external)
|
||||
const pullRegistryUrl = this.configService.get<string>('registry.pullUrl') || 'localhost:30500';
|
||||
const buildNamespace = this.configService.get<string>('build.namespace') || 'cloudhost-builds';
|
||||
const tag = `${Date.now()}`;
|
||||
const pushImageUri = `${internalRegistryUrl}/${app.userId}/${app.name}:${tag}`;
|
||||
const pullImageUri = `${pullRegistryUrl}/${app.userId}/${app.name}:${tag}`;
|
||||
|
||||
this.logger.log(`Starting image build for ${app.name} → push: ${pushImageUri}, pull: ${pullImageUri}`);
|
||||
|
||||
// Determine Dockerfile based on runtime
|
||||
const dockerfileContent = this.generateDockerfile(app);
|
||||
|
||||
// Create Kaniko build pod
|
||||
const buildPodName = `build-${app.name}-${tag}`.substring(0, 63).replace(/[^a-z0-9-]/g, '');
|
||||
|
||||
// Use the cluster's kubeconfig instead of default
|
||||
const cluster = app.clusterId
|
||||
? await this.clustersService.findOne(app.clusterId)
|
||||
: await this.clustersService.getDefault();
|
||||
|
||||
const kc = new k8s.KubeConfig();
|
||||
kc.loadFromString(cluster.kubeconfig);
|
||||
const coreApi = kc.makeApiClient(k8s.CoreV1Api);
|
||||
const batchApi = kc.makeApiClient(k8s.BatchV1Api);
|
||||
|
||||
// Determine if we have uploaded code or git URL
|
||||
const codePath = app.codePath ? path.resolve(app.codePath) : null;
|
||||
const hasUploadedCode = codePath && fs.existsSync(codePath);
|
||||
const hasGitUrl = !!app.gitUrl;
|
||||
|
||||
// Create ConfigMap with Dockerfile
|
||||
const dockerfileConfigMap = {
|
||||
apiVersion: 'v1',
|
||||
kind: 'ConfigMap',
|
||||
metadata: {
|
||||
name: `${buildPodName}-dockerfile`,
|
||||
namespace: buildNamespace,
|
||||
},
|
||||
data: {
|
||||
Dockerfile: dockerfileContent,
|
||||
},
|
||||
};
|
||||
|
||||
// If we have uploaded code, create a ConfigMap with the zip as base64
|
||||
let sourceConfigMapName: string | undefined;
|
||||
if (hasUploadedCode) {
|
||||
const zipBuffer = fs.readFileSync(codePath!);
|
||||
const zipBase64 = zipBuffer.toString('base64');
|
||||
|
||||
sourceConfigMapName = `${buildPodName}-source`;
|
||||
|
||||
// ConfigMap has 1MB limit, for larger files we'd need a PVC approach
|
||||
// For now, use a Secret (which can hold up to 1MB too, but binary-safe)
|
||||
const sourceSecret = {
|
||||
apiVersion: 'v1',
|
||||
kind: 'Secret',
|
||||
metadata: {
|
||||
name: sourceConfigMapName,
|
||||
namespace: buildNamespace,
|
||||
},
|
||||
data: {
|
||||
'source.zip': zipBase64,
|
||||
},
|
||||
};
|
||||
|
||||
await coreApi.createNamespacedSecret(buildNamespace!, sourceSecret);
|
||||
this.logger.log(`Created source secret: ${sourceConfigMapName} (${(zipBuffer.length / 1024).toFixed(1)} KB)`);
|
||||
}
|
||||
|
||||
// Build the Kaniko Job spec
|
||||
// Always use dir context — init containers prepare /workspace/source
|
||||
const kanikoArgs = [
|
||||
'--dockerfile=/workspace/Dockerfile',
|
||||
'--context=dir:///workspace/source',
|
||||
`--destination=${pushImageUri}`,
|
||||
'--cache=true',
|
||||
`--cache-repo=${internalRegistryUrl}/${app.userId}/cache`,
|
||||
'--insecure',
|
||||
'--skip-tls-verify',
|
||||
];
|
||||
|
||||
const volumes: any[] = [
|
||||
{
|
||||
name: 'docker-config',
|
||||
secret: { secretName: 'registry-credentials' },
|
||||
},
|
||||
{
|
||||
name: 'dockerfile',
|
||||
configMap: {
|
||||
name: `${buildPodName}-dockerfile`,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'workspace',
|
||||
emptyDir: {},
|
||||
},
|
||||
];
|
||||
|
||||
const initContainers: any[] = [];
|
||||
|
||||
if (hasUploadedCode && sourceConfigMapName) {
|
||||
// Add the source secret as a volume
|
||||
volumes.push({
|
||||
name: 'source-zip',
|
||||
secret: { secretName: sourceConfigMapName },
|
||||
});
|
||||
|
||||
// Add init container that unzips the source code
|
||||
initContainers.push({
|
||||
name: 'unzip-source',
|
||||
image: 'alpine:3.19',
|
||||
command: ['sh', '-c', `
|
||||
apk add --no-cache unzip &&
|
||||
cp /workspace/Dockerfile /workspace-out/Dockerfile &&
|
||||
mkdir -p /workspace-out/source &&
|
||||
cd /workspace-out/source &&
|
||||
unzip /source/source.zip &&
|
||||
ls -la /workspace-out/source/
|
||||
`],
|
||||
volumeMounts: [
|
||||
{ name: 'workspace', mountPath: '/workspace-out' },
|
||||
{ name: 'dockerfile', mountPath: '/workspace/Dockerfile', subPath: 'Dockerfile' },
|
||||
{ name: 'source-zip', mountPath: '/source' },
|
||||
],
|
||||
});
|
||||
} else if (hasGitUrl) {
|
||||
// Clone git repo into /workspace/source, then copy our generated Dockerfile
|
||||
initContainers.push({
|
||||
name: 'git-clone',
|
||||
image: 'alpine/git:2.43.0',
|
||||
command: ['sh', '-c', `
|
||||
echo ">>> Cloning ${app.gitUrl}" &&
|
||||
git clone --depth 1 ${app.gitUrl} /workspace-out/source &&
|
||||
cp /dockerfile/Dockerfile /workspace-out/Dockerfile &&
|
||||
echo ">>> Workspace contents:" &&
|
||||
ls -la /workspace-out/source/
|
||||
`],
|
||||
volumeMounts: [
|
||||
{ name: 'workspace', mountPath: '/workspace-out' },
|
||||
{ name: 'dockerfile', mountPath: '/dockerfile' },
|
||||
],
|
||||
});
|
||||
}
|
||||
|
||||
// Kaniko container volume mounts
|
||||
const kanikoVolumeMounts: any[] = [
|
||||
{ name: 'docker-config', mountPath: '/kaniko/.docker' },
|
||||
{ name: 'workspace', mountPath: '/workspace' },
|
||||
];
|
||||
|
||||
// If no uploaded code and no git, mount dockerfile directly
|
||||
if (!hasUploadedCode && !hasGitUrl) {
|
||||
kanikoVolumeMounts.push({
|
||||
name: 'dockerfile',
|
||||
mountPath: '/workspace/Dockerfile',
|
||||
subPath: 'Dockerfile',
|
||||
});
|
||||
}
|
||||
|
||||
const buildJob: k8s.V1Job = {
|
||||
apiVersion: 'batch/v1',
|
||||
kind: 'Job',
|
||||
metadata: {
|
||||
name: buildPodName,
|
||||
namespace: buildNamespace,
|
||||
},
|
||||
spec: {
|
||||
backoffLimit: 2,
|
||||
ttlSecondsAfterFinished: 300,
|
||||
template: {
|
||||
spec: {
|
||||
serviceAccountName: this.configService.get<string>('build.serviceAccount'),
|
||||
initContainers: initContainers.length > 0 ? initContainers : undefined,
|
||||
containers: [
|
||||
{
|
||||
name: 'kaniko',
|
||||
image: 'gcr.io/kaniko-project/executor:latest',
|
||||
args: kanikoArgs,
|
||||
volumeMounts: kanikoVolumeMounts,
|
||||
resources: {
|
||||
requests: { cpu: '500m', memory: '1Gi' },
|
||||
limits: { cpu: '2', memory: '4Gi' },
|
||||
},
|
||||
},
|
||||
],
|
||||
restartPolicy: 'Never',
|
||||
volumes,
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
await coreApi.createNamespacedConfigMap(buildNamespace!, dockerfileConfigMap);
|
||||
await batchApi.createNamespacedJob(buildNamespace!, buildJob);
|
||||
|
||||
// Wait for build to complete
|
||||
await this.waitForJobCompletion(batchApi, coreApi, buildPodName, buildNamespace!, 600);
|
||||
|
||||
this.logger.log(`Build completed successfully: ${pullImageUri}`);
|
||||
return pullImageUri;
|
||||
} catch (error: any) {
|
||||
// Try to get build logs for debugging
|
||||
try {
|
||||
const logs = await this.getBuildLogs(coreApi, buildPodName, buildNamespace!);
|
||||
this.logger.error(`Build logs for ${buildPodName}:\n${logs}`);
|
||||
} catch {}
|
||||
|
||||
this.logger.error(`Build failed for ${app.name}:`, error.body || error.message);
|
||||
throw new Error(`Image build failed: ${error.body?.message || error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
private generateDockerfile(app: Application): string {
|
||||
switch (app.runtime) {
|
||||
case AppRuntime.NODEJS:
|
||||
return this.nodeDockerfile(app);
|
||||
case AppRuntime.LARAVEL:
|
||||
return this.laravelDockerfile(app);
|
||||
default:
|
||||
throw new Error(`Unsupported runtime: ${app.runtime}`);
|
||||
}
|
||||
}
|
||||
|
||||
private nodeDockerfile(app: Application): string {
|
||||
const port = app.port || 3000;
|
||||
return `# --- Build stage ---
|
||||
FROM node:20-alpine AS builder
|
||||
WORKDIR /app
|
||||
COPY package*.json ./
|
||||
RUN if [ -f package-lock.json ]; then npm ci; else npm install; fi && npm cache clean --force
|
||||
COPY . .
|
||||
|
||||
# Auto-detect Next.js and enable standalone output
|
||||
RUN if ([ -f next.config.js ] || [ -f next.config.mjs ] || [ -f next.config.ts ]); then \\
|
||||
echo ">>> Next.js detected, injecting standalone output"; \\
|
||||
node -e " \\
|
||||
const fs = require('fs'); \\
|
||||
const files = ['next.config.js','next.config.mjs','next.config.ts']; \\
|
||||
for (const f of files) { \\
|
||||
if (fs.existsSync(f)) { \\
|
||||
let c = fs.readFileSync(f,'utf8'); \\
|
||||
if (!c.includes('standalone')) { \\
|
||||
c = c.replace(/output\\s*:\\s*['\\\"][^'\\\"]*['\\\"]\\s*,?/g, ''); \\
|
||||
c = c.replace(/(\\{)/, '\\$1 output: \\\"standalone\\\",'); \\
|
||||
fs.writeFileSync(f, c); \\
|
||||
console.log('Patched ' + f + ' with standalone output'); \\
|
||||
} else { \\
|
||||
console.log(f + ' already has standalone'); \\
|
||||
} \\
|
||||
break; \\
|
||||
} \\
|
||||
} \\
|
||||
"; \\
|
||||
fi
|
||||
|
||||
RUN npm run build 2>/dev/null || true
|
||||
|
||||
# --- Production stage ---
|
||||
FROM node:20-alpine AS runner
|
||||
WORKDIR /app
|
||||
RUN addgroup -g 1001 -S appgroup && adduser -S appuser -u 1001
|
||||
|
||||
# Copy all build output to temp
|
||||
COPY --from=builder /app /tmp/fullapp
|
||||
|
||||
# Detect: Next.js standalone vs regular Node.js
|
||||
RUN if [ -d /tmp/fullapp/.next/standalone ]; then \\
|
||||
echo ">>> Next.js standalone mode"; \\
|
||||
cp -a /tmp/fullapp/.next/standalone/. .; \\
|
||||
mkdir -p .next/static; \\
|
||||
[ -d /tmp/fullapp/.next/static ] && cp -a /tmp/fullapp/.next/static/. .next/static/; \\
|
||||
[ -d /tmp/fullapp/public ] && cp -a /tmp/fullapp/public ./public; \\
|
||||
echo "standalone" > /app/.mode; \\
|
||||
else \\
|
||||
echo ">>> Regular Node.js app"; \\
|
||||
cp -a /tmp/fullapp/. .; \\
|
||||
echo "regular" > /app/.mode; \\
|
||||
fi && rm -rf /tmp/fullapp
|
||||
|
||||
USER appuser
|
||||
ENV PORT=${port}
|
||||
ENV HOSTNAME=0.0.0.0
|
||||
EXPOSE ${port}
|
||||
CMD ["sh", "-c", "if [ \\"$(cat /app/.mode)\\" = \\"standalone\\" ] && [ -f server.js ]; then node server.js; else npm start; fi"]
|
||||
`;
|
||||
}
|
||||
|
||||
private laravelDockerfile(app: Application): string {
|
||||
return `# --- Build stage ---
|
||||
FROM composer:2 AS composer
|
||||
WORKDIR /app
|
||||
COPY composer.json composer.lock ./
|
||||
RUN composer install --no-dev --no-scripts --no-autoloader --prefer-dist
|
||||
COPY . .
|
||||
RUN composer dump-autoload --optimize --no-dev
|
||||
|
||||
# --- Production stage ---
|
||||
FROM php:8.3-fpm-alpine
|
||||
RUN apk add --no-cache nginx supervisor \\
|
||||
&& docker-php-ext-install pdo pdo_mysql pdo_pgsql opcache
|
||||
|
||||
WORKDIR /var/www/html
|
||||
COPY --from=composer /app .
|
||||
COPY docker/nginx.conf /etc/nginx/http.d/default.conf
|
||||
COPY docker/supervisord.conf /etc/supervisord.conf
|
||||
|
||||
RUN chown -R www-data:www-data /var/www/html/storage /var/www/html/bootstrap/cache
|
||||
RUN php artisan config:cache && php artisan route:cache && php artisan view:cache || true
|
||||
|
||||
EXPOSE ${app.port || 8000}
|
||||
CMD ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf"]
|
||||
`;
|
||||
}
|
||||
|
||||
private async waitForJobCompletion(
|
||||
batchApi: k8s.BatchV1Api,
|
||||
coreApi: k8s.CoreV1Api,
|
||||
jobName: string,
|
||||
namespace: string,
|
||||
timeoutSeconds: number,
|
||||
): Promise<void> {
|
||||
const startTime = Date.now();
|
||||
const timeoutMs = timeoutSeconds * 1000;
|
||||
|
||||
while (Date.now() - startTime < timeoutMs) {
|
||||
const job = await batchApi.readNamespacedJob(jobName, namespace);
|
||||
const status = job.body.status;
|
||||
|
||||
if (status?.succeeded && status.succeeded > 0) {
|
||||
return; // Build completed
|
||||
}
|
||||
|
||||
if (status?.failed && status.failed > 0) {
|
||||
// Try to get pod logs for more info
|
||||
const logs = await this.getBuildLogs(coreApi, jobName, namespace);
|
||||
throw new Error(`Build job ${jobName} failed.\nLogs:\n${logs}`);
|
||||
}
|
||||
|
||||
// Wait 5 seconds before polling again
|
||||
await new Promise((resolve) => setTimeout(resolve, 5000));
|
||||
}
|
||||
|
||||
throw new Error(`Build job ${jobName} timed out after ${timeoutSeconds}s`);
|
||||
}
|
||||
|
||||
private async getBuildLogs(
|
||||
coreApi: k8s.CoreV1Api,
|
||||
jobName: string,
|
||||
namespace: string,
|
||||
): Promise<string> {
|
||||
try {
|
||||
const pods = await coreApi.listNamespacedPod(
|
||||
namespace,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
`job-name=${jobName}`,
|
||||
);
|
||||
|
||||
if (pods.body.items.length === 0) {
|
||||
return 'No pods found for build job.';
|
||||
}
|
||||
|
||||
const podName = pods.body.items[0].metadata?.name;
|
||||
if (!podName) return 'Pod name not found.';
|
||||
|
||||
// Get logs from all containers (init + kaniko)
|
||||
let allLogs = '';
|
||||
const containers = [
|
||||
...(pods.body.items[0].spec?.initContainers || []),
|
||||
...(pods.body.items[0].spec?.containers || []),
|
||||
];
|
||||
|
||||
for (const container of containers) {
|
||||
try {
|
||||
const logResponse = await coreApi.readNamespacedPodLog(
|
||||
podName,
|
||||
namespace,
|
||||
container.name,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
500,
|
||||
);
|
||||
allLogs += `\n--- ${container.name} ---\n${logResponse.body}`;
|
||||
} catch {
|
||||
allLogs += `\n--- ${container.name} --- (no logs available)`;
|
||||
}
|
||||
}
|
||||
return allLogs;
|
||||
} catch (e: any) {
|
||||
return `Failed to retrieve logs: ${e.message}`;
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user