keyhan
8d1855b89c
Reject zip uploads when the selected runtime does not match archive contents, and re-validate before Kaniko builds to fail fast instead of producing the wrong Dockerfile. Co-authored-by: Cursor <cursoragent@cursor.com>
1744 lines
65 KiB
TypeScript
1744 lines
65 KiB
TypeScript
import { Injectable, Logger } from '@nestjs/common';
|
|
import { ConfigService } from '@nestjs/config';
|
|
import * as k8s from '@kubernetes/client-node';
|
|
import * as fs from 'fs';
|
|
import * as path from 'path';
|
|
import { execFile, spawn, ChildProcess } from 'child_process';
|
|
import * as net from 'net';
|
|
import { promisify } from 'util';
|
|
import { Application } from '../applications/entities/application.entity';
|
|
import { AppRuntime } from '../common/enums';
|
|
import { ClustersService } from '../clusters/clusters.service';
|
|
import { RegistryService } from '../kubernetes/registry.service';
|
|
import { BuildProgressStore } from './build-progress.store';
|
|
import {
|
|
detectDjangoSettingsModule,
|
|
detectGoBuildTarget,
|
|
detectShallowCsproj,
|
|
listArchiveEntries,
|
|
validateRuntimeFromArchive,
|
|
} from './runtime-detector';
|
|
|
|
const execFileAsync = promisify(execFile);
|
|
|
|
export class BuildCancelledError extends Error {
|
|
constructor() {
|
|
super('Build cancelled by user');
|
|
this.name = 'BuildCancelledError';
|
|
}
|
|
}
|
|
|
|
interface ActiveBuildSession {
|
|
cancelled: boolean;
|
|
coreApi?: k8s.CoreV1Api;
|
|
batchApi?: k8s.BatchV1Api;
|
|
namespace?: string;
|
|
buildPodName?: string;
|
|
sourcePvcName?: string;
|
|
helperPodName?: string;
|
|
processes: ChildProcess[];
|
|
socket?: net.Socket;
|
|
}
|
|
|
|
export interface BuildProgress {
|
|
phase: 'uploading' | 'building' | 'deploying' | 'done' | 'failed' | 'cancelled';
|
|
percent: number;
|
|
bytesUploaded?: number;
|
|
totalBytes?: number;
|
|
message?: string;
|
|
}
|
|
|
|
@Injectable()
|
|
export class BuildService {
|
|
private readonly logger = new Logger(BuildService.name);
|
|
private readonly progressMap = new Map<string, BuildProgress>();
|
|
private readonly activeBuilds = new Map<string, ActiveBuildSession>();
|
|
/**
|
|
* Kaniko executor image. Pinned (not `:latest`) so it can be cached on the node
|
|
* with imagePullPolicy=IfNotPresent — avoids re-pulling the ~250MB image on every build.
|
|
*/
|
|
private readonly kanikoImage = process.env.KANIKO_IMAGE || 'gcr.io/kaniko-project/executor:v1.23.2';
|
|
|
|
constructor(
|
|
private configService: ConfigService,
|
|
private clustersService: ClustersService,
|
|
private registryService: RegistryService,
|
|
private progressStore: BuildProgressStore,
|
|
) {}
|
|
|
|
private beginBuildSession(deploymentId: string): void {
|
|
this.activeBuilds.set(deploymentId, { cancelled: false, processes: [] });
|
|
}
|
|
|
|
private getSession(deploymentId?: string): ActiveBuildSession | undefined {
|
|
if (!deploymentId) return undefined;
|
|
return this.activeBuilds.get(deploymentId);
|
|
}
|
|
|
|
private updateBuildSession(deploymentId: string, update: Partial<ActiveBuildSession>): void {
|
|
const session = this.activeBuilds.get(deploymentId);
|
|
if (session) Object.assign(session, update);
|
|
}
|
|
|
|
private registerProcess(deploymentId: string | undefined, proc: ChildProcess): void {
|
|
const session = this.getSession(deploymentId);
|
|
if (!session) return;
|
|
session.processes.push(proc);
|
|
if (session.cancelled) {
|
|
try {
|
|
proc.kill('SIGKILL');
|
|
} catch {
|
|
/* ignore */
|
|
}
|
|
}
|
|
}
|
|
|
|
private registerSocket(deploymentId: string | undefined, socket: net.Socket): void {
|
|
const session = this.getSession(deploymentId);
|
|
if (!session) return;
|
|
if (session.socket) {
|
|
try {
|
|
session.socket.destroy();
|
|
} catch {
|
|
/* ignore */
|
|
}
|
|
}
|
|
session.socket = socket;
|
|
if (session.cancelled) {
|
|
try {
|
|
socket.destroy();
|
|
} catch {
|
|
/* ignore */
|
|
}
|
|
}
|
|
}
|
|
|
|
private throwIfCancelled(deploymentId?: string): void {
|
|
if (deploymentId && this.activeBuilds.get(deploymentId)?.cancelled) {
|
|
throw new BuildCancelledError();
|
|
}
|
|
}
|
|
|
|
private endBuildSession(deploymentId?: string): void {
|
|
if (deploymentId) this.activeBuilds.delete(deploymentId);
|
|
}
|
|
|
|
async cancelBuild(deploymentId: string): Promise<void> {
|
|
const session = this.activeBuilds.get(deploymentId);
|
|
if (!session) {
|
|
this.setProgress(deploymentId, {
|
|
phase: 'cancelled',
|
|
percent: 0,
|
|
message: 'Cancelled by user',
|
|
});
|
|
return;
|
|
}
|
|
|
|
session.cancelled = true;
|
|
this.logger.log(`Cancelling build for deployment ${deploymentId}`);
|
|
|
|
if (session.socket) {
|
|
try {
|
|
session.socket.destroy();
|
|
} catch {
|
|
/* ignore */
|
|
}
|
|
}
|
|
for (const proc of session.processes) {
|
|
try {
|
|
proc.kill('SIGKILL');
|
|
} catch {
|
|
/* ignore */
|
|
}
|
|
}
|
|
|
|
const { coreApi, batchApi, namespace, buildPodName, sourcePvcName, helperPodName } = session;
|
|
if (coreApi && namespace) {
|
|
const cleanup: Promise<unknown>[] = [];
|
|
if (helperPodName) {
|
|
cleanup.push(
|
|
coreApi
|
|
.deleteNamespacedPod({
|
|
name: helperPodName,
|
|
namespace,
|
|
gracePeriodSeconds: 0,
|
|
})
|
|
.catch(() => undefined),
|
|
);
|
|
}
|
|
if (buildPodName && batchApi) {
|
|
cleanup.push(
|
|
batchApi
|
|
.deleteNamespacedJob({
|
|
name: buildPodName,
|
|
namespace,
|
|
gracePeriodSeconds: 0,
|
|
propagationPolicy: 'Foreground',
|
|
})
|
|
.catch(() => undefined),
|
|
);
|
|
}
|
|
if (sourcePvcName) {
|
|
cleanup.push(
|
|
coreApi
|
|
.deleteNamespacedPersistentVolumeClaim({
|
|
name: sourcePvcName,
|
|
namespace,
|
|
})
|
|
.catch(() => undefined),
|
|
);
|
|
}
|
|
if (buildPodName) {
|
|
cleanup.push(
|
|
coreApi
|
|
.deleteNamespacedConfigMap({
|
|
name: `${buildPodName}-dockerfile`,
|
|
namespace,
|
|
})
|
|
.catch(() => undefined),
|
|
);
|
|
}
|
|
await Promise.all(cleanup);
|
|
this.logger.log(`Cleaned up K8s build resources for deployment ${deploymentId}`);
|
|
}
|
|
|
|
this.setProgress(deploymentId, {
|
|
phase: 'cancelled',
|
|
percent: 0,
|
|
message: 'Cancelled by user',
|
|
});
|
|
this.activeBuilds.delete(deploymentId);
|
|
}
|
|
|
|
/** Delete all in-flight build artifacts for an app (helper pods, jobs, PVCs, configmaps). */
|
|
async cleanupBuildResourcesForApp(app: Application): Promise<void> {
|
|
const buildNamespace = this.configService.get<string>('build.namespace') || 'cloudhost-builds';
|
|
const prefix = `build-${app.name}-`;
|
|
|
|
const cluster = app.clusterId ? await this.clustersService.findOne(app.clusterId) : await this.clustersService.getDefault();
|
|
|
|
const kc = new k8s.KubeConfig();
|
|
kc.loadFromString(cluster.kubeconfig);
|
|
const coreApi = kc.makeApiClient(k8s.CoreV1Api);
|
|
const batchApi = kc.makeApiClient(k8s.BatchV1Api);
|
|
|
|
const cleanup: Promise<unknown>[] = [];
|
|
|
|
const [pods, pvcs, jobs, configMaps] = await Promise.all([
|
|
coreApi.listNamespacedPod({ namespace: buildNamespace }),
|
|
coreApi.listNamespacedPersistentVolumeClaim({
|
|
namespace: buildNamespace,
|
|
}),
|
|
batchApi.listNamespacedJob({ namespace: buildNamespace }),
|
|
coreApi.listNamespacedConfigMap({ namespace: buildNamespace }),
|
|
]);
|
|
|
|
for (const pod of pods.items) {
|
|
const name = pod.metadata?.name || '';
|
|
if (name.startsWith(prefix)) {
|
|
cleanup.push(
|
|
coreApi
|
|
.deleteNamespacedPod({
|
|
name,
|
|
namespace: buildNamespace,
|
|
gracePeriodSeconds: 0,
|
|
})
|
|
.catch(() => undefined),
|
|
);
|
|
}
|
|
}
|
|
for (const pvc of pvcs.items) {
|
|
const name = pvc.metadata?.name || '';
|
|
if (name.startsWith(prefix)) {
|
|
cleanup.push(
|
|
coreApi
|
|
.deleteNamespacedPersistentVolumeClaim({
|
|
name,
|
|
namespace: buildNamespace,
|
|
})
|
|
.catch(() => undefined),
|
|
);
|
|
}
|
|
}
|
|
for (const job of jobs.items) {
|
|
const name = job.metadata?.name || '';
|
|
if (name.startsWith(prefix)) {
|
|
cleanup.push(
|
|
batchApi
|
|
.deleteNamespacedJob({
|
|
name,
|
|
namespace: buildNamespace,
|
|
gracePeriodSeconds: 0,
|
|
propagationPolicy: 'Foreground',
|
|
})
|
|
.catch(() => undefined),
|
|
);
|
|
}
|
|
}
|
|
for (const cm of configMaps.items) {
|
|
const name = cm.metadata?.name || '';
|
|
if (name.startsWith(prefix)) {
|
|
cleanup.push(coreApi.deleteNamespacedConfigMap({ name, namespace: buildNamespace }).catch(() => undefined));
|
|
}
|
|
}
|
|
|
|
await Promise.all(cleanup);
|
|
this.logger.log(`Cleaned up all build resources matching "${prefix}*" in ${buildNamespace}`);
|
|
}
|
|
|
|
async getProgress(deploymentId: string): Promise<BuildProgress | null> {
|
|
const local = this.progressMap.get(deploymentId);
|
|
if (local) return local;
|
|
const remote = await this.progressStore.get(deploymentId);
|
|
if (remote) this.progressMap.set(deploymentId, remote);
|
|
return remote;
|
|
}
|
|
|
|
setProgress(deploymentId: string | undefined, progress: BuildProgress): void {
|
|
if (!deploymentId) return;
|
|
this.progressMap.set(deploymentId, progress);
|
|
void this.progressStore.set(deploymentId, progress);
|
|
}
|
|
|
|
clearProgress(deploymentId: string): void {
|
|
this.progressMap.delete(deploymentId);
|
|
void this.progressStore.clear(deploymentId);
|
|
}
|
|
|
|
/**
|
|
* Builds a Docker image for the application using Kaniko inside K8s.
|
|
* Returns { imageUri, buildLog } — the full image URI and the build logs.
|
|
*/
|
|
async buildImage(app: Application, deploymentId?: string): Promise<{ imageUri: string; buildLog: string }> {
|
|
const registryUrl = this.registryService.getRegistryUrl();
|
|
const buildNamespace = this.registryService.getBuildNamespace();
|
|
const tag = `${Date.now()}`;
|
|
const imageUri = this.registryService.buildImageReference(app.userId, app.name, tag);
|
|
|
|
this.logger.log(`Starting image build for ${app.name} → ${imageUri}`);
|
|
|
|
if (deploymentId) {
|
|
this.beginBuildSession(deploymentId);
|
|
}
|
|
|
|
const codePath = app.codePath ? path.resolve(app.codePath) : null;
|
|
const hasUploadedCode = codePath && fs.existsSync(codePath);
|
|
if (hasUploadedCode) {
|
|
await validateRuntimeFromArchive(app.runtime, codePath);
|
|
}
|
|
|
|
const archiveEntries = hasUploadedCode ? await listArchiveEntries(codePath!) : [];
|
|
|
|
// Determine Dockerfile based on runtime
|
|
const dockerfileContent = this.generateDockerfile(app, archiveEntries);
|
|
|
|
// Create Kaniko build pod
|
|
const buildPodName = `build-${app.name}-${tag}`.substring(0, 63).replace(/[^a-z0-9-]/g, '');
|
|
|
|
if (deploymentId) {
|
|
this.updateBuildSession(deploymentId, { buildPodName });
|
|
}
|
|
|
|
// Use the cluster's kubeconfig instead of default
|
|
const cluster = app.clusterId ? await this.clustersService.findOne(app.clusterId) : await this.clustersService.getDefault();
|
|
|
|
const kc = new k8s.KubeConfig();
|
|
kc.loadFromString(cluster.kubeconfig);
|
|
const coreApi = kc.makeApiClient(k8s.CoreV1Api);
|
|
const batchApi = kc.makeApiClient(k8s.BatchV1Api);
|
|
|
|
if (deploymentId) {
|
|
this.updateBuildSession(deploymentId, {
|
|
coreApi,
|
|
batchApi,
|
|
namespace: buildNamespace,
|
|
});
|
|
}
|
|
|
|
// Ensure the build namespace exists
|
|
await this.ensureNamespace(coreApi, buildNamespace);
|
|
this.throwIfCancelled(deploymentId);
|
|
|
|
// Determine if we have uploaded code or git URL
|
|
const hasGitUrl = !!app.gitUrl;
|
|
|
|
// Create ConfigMap with Dockerfile
|
|
const dockerfileConfigMap = {
|
|
apiVersion: 'v1',
|
|
kind: 'ConfigMap',
|
|
metadata: {
|
|
name: `${buildPodName}-dockerfile`,
|
|
namespace: buildNamespace,
|
|
},
|
|
data: {
|
|
Dockerfile: dockerfileContent,
|
|
},
|
|
};
|
|
|
|
// If we have uploaded code, create a PVC and upload via kubectl cp
|
|
let sourcePvcName: string | undefined;
|
|
if (hasUploadedCode) {
|
|
sourcePvcName = `${buildPodName}-source`;
|
|
if (deploymentId) {
|
|
this.updateBuildSession(deploymentId, { sourcePvcName });
|
|
}
|
|
const zipSize = fs.statSync(codePath!).size;
|
|
// Allocate PVC size = zip size * 3 (zip + extracted), min 1Gi
|
|
const pvcSizeGi = Math.max(1, Math.ceil((zipSize * 3) / (1024 * 1024 * 1024)));
|
|
|
|
await this.uploadSourceViaPVC(kc, coreApi, buildNamespace!, sourcePvcName, codePath!, pvcSizeGi, deploymentId);
|
|
}
|
|
|
|
// Build the Kaniko Job spec
|
|
// Always use dir context — init containers prepare /workspace/source
|
|
const kanikoArgs = [
|
|
'--dockerfile=/workspace/Dockerfile',
|
|
'--context=dir:///workspace/source',
|
|
`--destination=${imageUri}`,
|
|
'--cache=true',
|
|
`--cache-repo=${registryUrl}/${app.userId}/cache`,
|
|
'--insecure',
|
|
'--skip-tls-verify',
|
|
'--single-snapshot',
|
|
'--snapshot-mode=redo',
|
|
];
|
|
|
|
const volumes: any[] = [
|
|
{
|
|
name: 'docker-config',
|
|
secret: { secretName: 'registry-credentials' },
|
|
},
|
|
{
|
|
name: 'dockerfile',
|
|
configMap: {
|
|
name: `${buildPodName}-dockerfile`,
|
|
},
|
|
},
|
|
{
|
|
name: 'workspace',
|
|
emptyDir: {},
|
|
},
|
|
];
|
|
|
|
const initContainers: any[] = [];
|
|
|
|
if (hasUploadedCode && sourcePvcName) {
|
|
// Add the source PVC as a volume
|
|
volumes.push({
|
|
name: 'source-pvc',
|
|
persistentVolumeClaim: { claimName: sourcePvcName },
|
|
});
|
|
|
|
// Add init container that unzips the source code from PVC
|
|
initContainers.push({
|
|
name: 'unzip-source',
|
|
image: 'alpine:3.19',
|
|
imagePullPolicy: 'IfNotPresent',
|
|
command: [
|
|
'sh',
|
|
'-c',
|
|
`
|
|
apk add --no-cache unzip tar gzip &&
|
|
cp /workspace/Dockerfile /workspace-out/Dockerfile &&
|
|
mkdir -p /tmp/extract &&
|
|
cd /tmp/extract &&
|
|
if tar tzf /source-pvc/source.zip >/dev/null 2>&1; then
|
|
echo ">>> Detected gzip tarball" &&
|
|
tar xzf /source-pvc/source.zip
|
|
elif unzip -t /source-pvc/source.zip >/dev/null 2>&1; then
|
|
echo ">>> Detected zip archive" &&
|
|
unzip -q /source-pvc/source.zip
|
|
else
|
|
echo "ERROR: source archive is not a valid zip or tar.gz" && exit 1
|
|
fi &&
|
|
echo "--- Extracted contents ---" &&
|
|
ls -la /tmp/extract/ &&
|
|
mkdir -p /workspace-out/source &&
|
|
ITEMS=$(ls -1 /tmp/extract/ | head -5) &&
|
|
COUNT=$(ls -1 /tmp/extract/ | wc -l) &&
|
|
if [ "$COUNT" -eq 1 ] && [ -d "/tmp/extract/$ITEMS" ]; then
|
|
echo ">>> Single subfolder detected: $ITEMS — flattening to root" &&
|
|
cp -a /tmp/extract/$ITEMS/. /workspace-out/source/
|
|
else
|
|
echo ">>> Multiple items or files — copying as-is" &&
|
|
cp -a /tmp/extract/. /workspace-out/source/
|
|
fi &&
|
|
rm -rf /tmp/extract &&
|
|
echo "--- Final workspace contents ---" &&
|
|
ls -la /workspace-out/source/
|
|
`,
|
|
],
|
|
volumeMounts: [
|
|
{ name: 'workspace', mountPath: '/workspace-out' },
|
|
{
|
|
name: 'dockerfile',
|
|
mountPath: '/workspace/Dockerfile',
|
|
subPath: 'Dockerfile',
|
|
},
|
|
{ name: 'source-pvc', mountPath: '/source-pvc' },
|
|
],
|
|
});
|
|
} else if (hasGitUrl) {
|
|
// Build the git clone URL — inject token for private repos
|
|
let cloneUrl = app.gitUrl!;
|
|
if (app.gitToken) {
|
|
// Convert https://github.com/user/repo.git → https://<token>@github.com/user/repo.git
|
|
// Also works for GitLab, Bitbucket, etc.
|
|
try {
|
|
const url = new URL(cloneUrl);
|
|
url.username = app.gitToken;
|
|
url.password = ''; // Some providers use token as username, others as password
|
|
cloneUrl = url.toString();
|
|
} catch {
|
|
// If URL parsing fails, try simple injection after protocol
|
|
cloneUrl = cloneUrl.replace('https://', `https://${app.gitToken}@`);
|
|
}
|
|
}
|
|
const branch = app.gitBranch || 'main';
|
|
|
|
// Clone git repo into /workspace/source, then copy our generated Dockerfile
|
|
initContainers.push({
|
|
name: 'git-clone',
|
|
image: 'alpine/git:2.43.0',
|
|
imagePullPolicy: 'IfNotPresent',
|
|
command: [
|
|
'sh',
|
|
'-c',
|
|
`
|
|
echo ">>> Cloning branch '${branch}' from ${app.gitUrl}" &&
|
|
git clone --depth 1 --branch ${branch} ${cloneUrl} /workspace-out/source &&
|
|
cp /dockerfile/Dockerfile /workspace-out/Dockerfile &&
|
|
echo ">>> Workspace contents:" &&
|
|
ls -la /workspace-out/source/
|
|
`,
|
|
],
|
|
volumeMounts: [
|
|
{ name: 'workspace', mountPath: '/workspace-out' },
|
|
{ name: 'dockerfile', mountPath: '/dockerfile' },
|
|
],
|
|
});
|
|
}
|
|
|
|
// Kaniko container volume mounts
|
|
const kanikoVolumeMounts: any[] = [
|
|
{ name: 'docker-config', mountPath: '/kaniko/.docker' },
|
|
{ name: 'workspace', mountPath: '/workspace' },
|
|
];
|
|
|
|
// If no uploaded code and no git, we need to prepare the workspace
|
|
if (!hasUploadedCode && !hasGitUrl) {
|
|
// For runtimes that don't need source (e.g. fresh WordPress),
|
|
// add an init container that creates empty source dir + copies Dockerfile
|
|
initContainers.push({
|
|
name: 'prepare-workspace',
|
|
image: 'alpine:3.19',
|
|
imagePullPolicy: 'IfNotPresent',
|
|
command: [
|
|
'sh',
|
|
'-c',
|
|
`
|
|
mkdir -p /workspace-out/source &&
|
|
cp /dockerfile/Dockerfile /workspace-out/Dockerfile &&
|
|
echo ">>> Prepared empty workspace for fresh install" &&
|
|
ls -la /workspace-out/
|
|
`,
|
|
],
|
|
volumeMounts: [
|
|
{ name: 'workspace', mountPath: '/workspace-out' },
|
|
{ name: 'dockerfile', mountPath: '/dockerfile' },
|
|
],
|
|
});
|
|
}
|
|
|
|
const buildJob: k8s.V1Job = {
|
|
apiVersion: 'batch/v1',
|
|
kind: 'Job',
|
|
metadata: {
|
|
name: buildPodName,
|
|
namespace: buildNamespace,
|
|
},
|
|
spec: {
|
|
backoffLimit: 1,
|
|
ttlSecondsAfterFinished: 300,
|
|
template: {
|
|
spec: {
|
|
serviceAccountName: this.configService.get<string>('build.serviceAccount'),
|
|
initContainers: initContainers.length > 0 ? initContainers : undefined,
|
|
containers: [
|
|
{
|
|
name: 'kaniko',
|
|
image: this.kanikoImage,
|
|
imagePullPolicy: 'IfNotPresent',
|
|
args: kanikoArgs,
|
|
volumeMounts: kanikoVolumeMounts,
|
|
resources: {
|
|
requests: { cpu: '500m', memory: '1Gi' },
|
|
limits: { cpu: '2', memory: '4Gi' },
|
|
},
|
|
},
|
|
],
|
|
restartPolicy: 'Never',
|
|
volumes,
|
|
},
|
|
},
|
|
},
|
|
};
|
|
|
|
try {
|
|
const t0 = Date.now();
|
|
await coreApi.createNamespacedConfigMap({
|
|
namespace: buildNamespace!,
|
|
body: dockerfileConfigMap,
|
|
});
|
|
this.logger.log(`[timing] ConfigMap created in ${Date.now() - t0}ms`);
|
|
|
|
const t1 = Date.now();
|
|
await batchApi.createNamespacedJob({
|
|
namespace: buildNamespace!,
|
|
body: buildJob,
|
|
});
|
|
this.logger.log(`[timing] Job created in ${Date.now() - t1}ms`);
|
|
|
|
// Wait for build to complete
|
|
this.setProgress(deploymentId, {
|
|
phase: 'building',
|
|
percent: 15,
|
|
message: 'Building Docker image...',
|
|
});
|
|
await this.waitForJobCompletion(batchApi, coreApi, buildPodName, buildNamespace!, 600, deploymentId);
|
|
|
|
// Capture build logs on success
|
|
let buildLog = '';
|
|
try {
|
|
buildLog = await this.getBuildLogs(coreApi, buildPodName, buildNamespace!);
|
|
} catch {}
|
|
|
|
this.logger.log(`Build completed successfully: ${imageUri}`);
|
|
return { imageUri, buildLog };
|
|
} catch (error: any) {
|
|
if (error instanceof BuildCancelledError || error?.name === 'BuildCancelledError') {
|
|
throw error;
|
|
}
|
|
// Try to get build logs for debugging
|
|
let buildLog = '';
|
|
try {
|
|
buildLog = await this.getBuildLogs(coreApi, buildPodName, buildNamespace!);
|
|
this.logger.error(`Build logs for ${buildPodName}:\n${buildLog}`);
|
|
} catch {}
|
|
|
|
this.logger.error(`Build failed for ${app.name}:`, error.body || error.message);
|
|
const err = new Error(`Image build failed: ${error.body?.message || error.message}`);
|
|
(err as any).buildLog = buildLog;
|
|
throw err;
|
|
} finally {
|
|
// Clean up build resources
|
|
if (sourcePvcName) {
|
|
try {
|
|
await coreApi.deleteNamespacedPersistentVolumeClaim({
|
|
name: sourcePvcName,
|
|
namespace: buildNamespace!,
|
|
});
|
|
this.logger.log(`Cleaned up source PVC: ${sourcePvcName}`);
|
|
} catch (e: any) {
|
|
this.logger.warn(`Failed to clean up source PVC ${sourcePvcName}: ${e.message}`);
|
|
}
|
|
}
|
|
// Clean up Dockerfile ConfigMap
|
|
try {
|
|
await coreApi.deleteNamespacedConfigMap({
|
|
name: `${buildPodName}-dockerfile`,
|
|
namespace: buildNamespace!,
|
|
});
|
|
} catch (e: any) {
|
|
this.logger.warn(`Failed to clean up ConfigMap: ${e.message}`);
|
|
}
|
|
this.endBuildSession(deploymentId);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Upload a local file to the helper pod using kubectl cp with progress tracking.
|
|
* kubectl cp uses tar over the k8s exec API — reliable for any file size.
|
|
*/
|
|
private streamFileToHelperPod(kubeconfig: string, namespace: string, podName: string, filePath: string, fileSize: number, deploymentId?: string): Promise<void> {
|
|
const maxAttempts = 3;
|
|
|
|
const runOnce = () =>
|
|
new Promise<void>((resolve, reject) => {
|
|
this.throwIfCancelled(deploymentId);
|
|
|
|
const kubectl = spawn('kubectl', ['--kubeconfig', kubeconfig, 'cp', filePath, `${namespace}/${podName}:/data/source.zip`, '-c', 'helper', '--retries', '3'], {
|
|
stdio: ['ignore', 'pipe', 'pipe'],
|
|
});
|
|
this.registerProcess(deploymentId, kubectl);
|
|
|
|
let stderr = '';
|
|
kubectl.stderr.on('data', (chunk: Buffer) => {
|
|
stderr += chunk.toString();
|
|
});
|
|
|
|
// Track progress by polling remote file size
|
|
let progressTimer: NodeJS.Timeout | undefined;
|
|
const pollProgress = () => {
|
|
execFileAsync('kubectl', ['--kubeconfig', kubeconfig, 'exec', '-n', namespace, podName, '-c', 'helper', '--', 'sh', '-c', 'wc -c < /data/source.zip 2>/dev/null || echo 0'], {
|
|
timeout: 10_000,
|
|
})
|
|
.then(({ stdout }) => {
|
|
const remoteSize = parseInt(stdout.trim(), 10) || 0;
|
|
const percent = Math.min(99, Math.round((remoteSize / fileSize) * 100));
|
|
this.setProgress(deploymentId, {
|
|
phase: 'uploading',
|
|
percent,
|
|
bytesUploaded: remoteSize,
|
|
totalBytes: fileSize,
|
|
message: `Uploading to cluster... ${percent}%`,
|
|
});
|
|
})
|
|
.catch(() => {
|
|
/* polling failure is non-fatal */
|
|
});
|
|
};
|
|
progressTimer = setInterval(pollProgress, 3000);
|
|
pollProgress();
|
|
|
|
kubectl.on('error', (err) => {
|
|
clearInterval(progressTimer);
|
|
reject(new Error(`kubectl cp spawn error: ${err.message}`));
|
|
});
|
|
|
|
kubectl.on('close', (code) => {
|
|
clearInterval(progressTimer);
|
|
if (code === 0) resolve();
|
|
else reject(new Error(`kubectl cp failed (code ${code}): ${stderr.trim()}`));
|
|
});
|
|
});
|
|
|
|
return (async () => {
|
|
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
try {
|
|
this.throwIfCancelled(deploymentId);
|
|
if (attempt > 1) {
|
|
this.logger.warn(`Retrying source upload (attempt ${attempt}/${maxAttempts})...`);
|
|
await execFileAsync('kubectl', ['--kubeconfig', kubeconfig, 'exec', '-n', namespace, podName, '-c', 'helper', '--', 'rm', '-f', '/data/source.zip'], { timeout: 15_000 }).catch(
|
|
() => undefined,
|
|
);
|
|
this.setProgress(deploymentId, {
|
|
phase: 'uploading',
|
|
percent: 0,
|
|
bytesUploaded: 0,
|
|
totalBytes: fileSize,
|
|
message: `Retrying upload (attempt ${attempt})...`,
|
|
});
|
|
}
|
|
await runOnce();
|
|
return;
|
|
} catch (err) {
|
|
if (err instanceof BuildCancelledError || (err as Error)?.name === 'BuildCancelledError') throw err;
|
|
if (attempt === maxAttempts) throw err;
|
|
this.logger.warn(`Upload attempt ${attempt} failed: ${(err as Error).message}`);
|
|
}
|
|
}
|
|
})();
|
|
}
|
|
|
|
/**
|
|
* Upload source zip to K8s via PVC + helper pod.
|
|
* This handles files of any size (unlike Secret/ConfigMap which are limited to ~1MB).
|
|
*/
|
|
private async uploadSourceViaPVC(kc: k8s.KubeConfig, coreApi: k8s.CoreV1Api, namespace: string, pvcName: string, zipPath: string, sizeGi: number, deploymentId?: string): Promise<void> {
|
|
const t0 = Date.now();
|
|
const helperPodName = `${pvcName}-helper`;
|
|
const zipSize = fs.statSync(zipPath).size;
|
|
|
|
if (deploymentId) {
|
|
this.updateBuildSession(deploymentId, { helperPodName });
|
|
}
|
|
|
|
this.logger.log(`Uploading source via PVC (${(zipSize / 1024 / 1024).toFixed(1)} MB) → ${pvcName}`);
|
|
|
|
// 1. Create PVC
|
|
await coreApi.createNamespacedPersistentVolumeClaim({
|
|
namespace,
|
|
body: {
|
|
apiVersion: 'v1',
|
|
kind: 'PersistentVolumeClaim',
|
|
metadata: { name: pvcName, namespace },
|
|
spec: {
|
|
accessModes: ['ReadWriteOnce'],
|
|
resources: { requests: { storage: `${sizeGi}Gi` } },
|
|
},
|
|
},
|
|
});
|
|
this.logger.log(`[timing] PVC ${pvcName} created in ${Date.now() - t0}ms`);
|
|
|
|
// 2. Create a helper pod that mounts the PVC and waits for data via a simple HTTP listener.
|
|
// We use alpine + nc (netcat) to receive the file over a port — much more reliable
|
|
// than kubectl cp or kubectl exec stdin pipe for large files.
|
|
const helperPod: k8s.V1Pod = {
|
|
apiVersion: 'v1',
|
|
kind: 'Pod',
|
|
metadata: { name: helperPodName, namespace },
|
|
spec: {
|
|
containers: [
|
|
{
|
|
name: 'helper',
|
|
image: 'alpine:3.19',
|
|
imagePullPolicy: 'IfNotPresent',
|
|
command: ['sh', '-c', 'sleep 3600'],
|
|
volumeMounts: [{ name: 'source', mountPath: '/data' }],
|
|
resources: {
|
|
requests: { cpu: '100m', memory: '128Mi' },
|
|
limits: { cpu: '500m', memory: '256Mi' },
|
|
},
|
|
},
|
|
],
|
|
volumes: [
|
|
{
|
|
name: 'source',
|
|
persistentVolumeClaim: { claimName: pvcName },
|
|
},
|
|
],
|
|
restartPolicy: 'Never',
|
|
},
|
|
};
|
|
|
|
await coreApi.createNamespacedPod({ namespace, body: helperPod });
|
|
|
|
// 3. Wait for helper pod to be Running
|
|
const podTimeout = 120_000; // 2 minutes
|
|
const podStart = Date.now();
|
|
while (Date.now() - podStart < podTimeout) {
|
|
this.throwIfCancelled(deploymentId);
|
|
const pod = await coreApi.readNamespacedPod({
|
|
name: helperPodName,
|
|
namespace,
|
|
});
|
|
const phase = pod.status?.phase;
|
|
if (phase === 'Running') break;
|
|
if (phase === 'Failed' || phase === 'Unknown') {
|
|
throw new Error(`Helper pod ${helperPodName} failed to start: phase=${phase}`);
|
|
}
|
|
await new Promise((r) => setTimeout(r, 2000));
|
|
}
|
|
if (Date.now() - podStart >= podTimeout) {
|
|
throw new Error(`Helper pod ${helperPodName} did not become Running within 2 minutes`);
|
|
}
|
|
this.logger.log(`[timing] Helper pod Running in ${Date.now() - t0}ms`);
|
|
|
|
// 4. Write kubeconfig to temp file for kubectl
|
|
const tmpKubeconfig = path.join('/tmp', `kubeconfig-${pvcName}.yaml`);
|
|
const kcYaml = kc.exportConfig();
|
|
fs.writeFileSync(tmpKubeconfig, kcYaml);
|
|
|
|
try {
|
|
// 5. Upload the zip via kubectl cp (tar-based, reliable for any size).
|
|
const t2 = Date.now();
|
|
this.setProgress(deploymentId, {
|
|
phase: 'uploading',
|
|
percent: 0,
|
|
bytesUploaded: 0,
|
|
totalBytes: zipSize,
|
|
message: 'Uploading source to cluster...',
|
|
});
|
|
|
|
await this.streamFileToHelperPod(tmpKubeconfig, namespace, helperPodName, zipPath, zipSize, deploymentId);
|
|
|
|
this.logger.log(`[timing] Source stream upload completed in ${Date.now() - t2}ms (${(zipSize / 1024 / 1024).toFixed(1)} MB)`);
|
|
this.setProgress(deploymentId, {
|
|
phase: 'uploading',
|
|
percent: 100,
|
|
bytesUploaded: zipSize,
|
|
totalBytes: zipSize,
|
|
message: 'Upload complete, verifying...',
|
|
});
|
|
|
|
// 5b. Verify the file was written correctly (exact size)
|
|
const { stdout: sizeStr } = await execFileAsync(
|
|
'kubectl',
|
|
['--kubeconfig', tmpKubeconfig, 'exec', '-n', namespace, helperPodName, '-c', 'helper', '--', 'sh', '-c', 'wc -c < /data/source.zip'],
|
|
{ timeout: 30_000 },
|
|
);
|
|
|
|
const remoteSize = parseInt(sizeStr.trim(), 10);
|
|
if (isNaN(remoteSize) || remoteSize !== zipSize) {
|
|
throw new Error(
|
|
`Source upload incomplete: expected ${zipSize} bytes but got ${remoteSize} bytes on remote. ` +
|
|
`(${(zipSize / 1024 / 1024).toFixed(1)} MB expected, ${(remoteSize / 1024 / 1024).toFixed(1)} MB received)`,
|
|
);
|
|
}
|
|
|
|
this.logger.log(`[verify] Remote file size: ${remoteSize} bytes (expected ${zipSize}) ✓`);
|
|
} finally {
|
|
// Clean up temp kubeconfig
|
|
try {
|
|
fs.unlinkSync(tmpKubeconfig);
|
|
} catch {}
|
|
|
|
// 6. Delete the helper pod and WAIT for it to be fully terminated
|
|
// (PVC is ReadWriteOnce — if the pod is still terminating when the
|
|
// build Job starts, Kaniko can't mount the PVC → stuck in Pending)
|
|
try {
|
|
await coreApi.deleteNamespacedPod({
|
|
name: helperPodName,
|
|
namespace,
|
|
gracePeriodSeconds: 0,
|
|
});
|
|
this.logger.log(`Helper pod ${helperPodName} delete requested — waiting for termination…`);
|
|
|
|
const delTimeout = 60_000;
|
|
const delStart = Date.now();
|
|
while (Date.now() - delStart < delTimeout) {
|
|
try {
|
|
await coreApi.readNamespacedPod({ name: helperPodName, namespace });
|
|
// Pod still exists — wait
|
|
await new Promise((r) => setTimeout(r, 2000));
|
|
} catch (err: any) {
|
|
if (err.code === 404 || err.body?.code === 404) {
|
|
this.logger.log(`Helper pod ${helperPodName} fully terminated`);
|
|
break;
|
|
}
|
|
// Other error — stop waiting
|
|
break;
|
|
}
|
|
}
|
|
} catch (e: any) {
|
|
this.logger.warn(`Failed to delete helper pod: ${e.message}`);
|
|
}
|
|
}
|
|
|
|
this.logger.log(`[timing] Source upload via PVC completed in ${Date.now() - t0}ms`);
|
|
}
|
|
|
|
/**
|
|
* Ensure the build namespace exists with all required resources
|
|
* (namespace, service account, registry-credentials secret).
|
|
*/
|
|
private async ensureNamespace(coreApi: k8s.CoreV1Api, namespace: string): Promise<void> {
|
|
// 1. Ensure namespace
|
|
try {
|
|
await coreApi.readNamespace({ name: namespace });
|
|
} catch (err: any) {
|
|
if (err.code === 404 || err.body?.code === 404) {
|
|
this.logger.log(`Namespace "${namespace}" not found — creating it`);
|
|
await coreApi.createNamespace({
|
|
body: { metadata: { name: namespace } },
|
|
});
|
|
} else {
|
|
throw err;
|
|
}
|
|
}
|
|
|
|
// 2. Ensure service account for Kaniko
|
|
const saName = this.configService.get<string>('build.serviceAccount') || 'kaniko-builder';
|
|
try {
|
|
await coreApi.readNamespacedServiceAccount({ name: saName, namespace });
|
|
} catch (err: any) {
|
|
if (err.code === 404 || err.body?.code === 404) {
|
|
this.logger.log(`ServiceAccount "${saName}" not found in "${namespace}" — creating it`);
|
|
await coreApi.createNamespacedServiceAccount({
|
|
namespace,
|
|
body: { metadata: { name: saName, namespace } },
|
|
});
|
|
} else {
|
|
throw err;
|
|
}
|
|
}
|
|
|
|
// 3. Ensure registry-credentials secret (docker config for Kaniko to push)
|
|
const registrySecretName = 'registry-credentials';
|
|
try {
|
|
await coreApi.readNamespacedSecret({
|
|
name: registrySecretName,
|
|
namespace,
|
|
});
|
|
} catch (err: any) {
|
|
if (err.code === 404 || err.body?.code === 404) {
|
|
this.logger.log(`Secret "${registrySecretName}" not found in "${namespace}" — creating it`);
|
|
await coreApi.createNamespacedSecret({
|
|
namespace,
|
|
body: {
|
|
metadata: { name: registrySecretName, namespace },
|
|
type: 'kubernetes.io/dockerconfigjson',
|
|
data: {
|
|
'.dockerconfigjson': Buffer.from(this.registryService.buildDockerConfigJson()).toString('base64'),
|
|
},
|
|
},
|
|
});
|
|
} else {
|
|
throw err;
|
|
}
|
|
}
|
|
}
|
|
|
|
private generateDockerfile(app: Application, archiveEntries: string[] = []): string {
|
|
switch (app.runtime) {
|
|
case AppRuntime.NODEJS:
|
|
return this.nodeDockerfile(app);
|
|
case AppRuntime.LARAVEL:
|
|
return this.laravelDockerfile(app);
|
|
case AppRuntime.WORDPRESS:
|
|
return this.wordpressDockerfile(app);
|
|
case AppRuntime.GO:
|
|
return this.goDockerfile(app, archiveEntries);
|
|
case AppRuntime.PHP:
|
|
return this.phpDockerfile(app);
|
|
case AppRuntime.PYTHON:
|
|
return this.pythonDockerfile(app);
|
|
case AppRuntime.DJANGO:
|
|
return this.djangoDockerfile(app, archiveEntries);
|
|
case AppRuntime.DOTNET:
|
|
return this.dotnetDockerfile(app, archiveEntries);
|
|
default:
|
|
throw new Error(`Unsupported runtime: ${app.runtime}`);
|
|
}
|
|
}
|
|
|
|
private nodeDockerfile(app: Application): string {
|
|
const port = app.port || 3000;
|
|
const nodeVersion = app.runtimeVersion || '20';
|
|
return `# --- Build stage ---
|
|
FROM node:${nodeVersion}-alpine AS builder
|
|
WORKDIR /app
|
|
COPY package*.json ./
|
|
RUN npm install --legacy-peer-deps && npm cache clean --force
|
|
COPY . .
|
|
|
|
# Auto-detect Next.js and enable standalone output
|
|
RUN for cfg in next.config.js next.config.mjs next.config.ts; do \\
|
|
[ -f "$cfg" ] || continue; \\
|
|
if grep -q standalone "$cfg"; then \\
|
|
echo "$cfg already has standalone"; \\
|
|
else \\
|
|
echo ">>> Next.js detected, injecting standalone output"; \\
|
|
node -e 'var f=require("fs"),c=f.readFileSync(process.argv[1],"utf8");if(!c.includes("standalone")){f.writeFileSync(process.argv[1],c.replace("{","{ output: \\"standalone\\","))}' "$cfg"; \\
|
|
echo "Patched $cfg:"; head -5 "$cfg"; \\
|
|
fi; \\
|
|
break; \\
|
|
done
|
|
|
|
RUN npm run build || echo ">>> Build script failed or not found — continuing"
|
|
|
|
# Clean up dev dependencies and caches to reduce image size
|
|
RUN rm -rf node_modules/.cache .next/cache /tmp/* /root/.npm 2>/dev/null; true
|
|
|
|
# --- Production stage ---
|
|
FROM node:${nodeVersion}-alpine AS runner
|
|
WORKDIR /app
|
|
RUN addgroup -g 1001 -S appgroup && adduser -S appuser -u 1001
|
|
|
|
# Copy all build output to temp
|
|
COPY --from=builder /app /tmp/fullapp
|
|
|
|
# Detect: Next.js standalone vs regular Node.js
|
|
RUN if [ -d /tmp/fullapp/.next/standalone ]; then \\
|
|
echo ">>> Next.js standalone mode"; \\
|
|
cp -a /tmp/fullapp/.next/standalone/. .; \\
|
|
mkdir -p .next/static; \\
|
|
[ -d /tmp/fullapp/.next/static ] && cp -a /tmp/fullapp/.next/static/. .next/static/; \\
|
|
[ -d /tmp/fullapp/public ] && cp -a /tmp/fullapp/public ./public; \\
|
|
echo "standalone" > /app/.mode; \\
|
|
else \\
|
|
echo ">>> Regular Node.js app"; \\
|
|
cp -a /tmp/fullapp/. .; \\
|
|
echo "regular" > /app/.mode; \\
|
|
fi && rm -rf /tmp/fullapp
|
|
|
|
USER appuser
|
|
ENV PORT=${port}
|
|
ENV HOSTNAME=0.0.0.0
|
|
EXPOSE ${port}
|
|
CMD ["sh", "-c", "if [ \\"$(cat /app/.mode)\\" = \\"standalone\\" ] && [ -f server.js ]; then node server.js; else npm start; fi"]
|
|
`;
|
|
}
|
|
|
|
private laravelDockerfile(app: Application): string {
|
|
const phpVersion = app.phpVersion || '8.3';
|
|
const port = app.port || 80;
|
|
return `# --- Build stage (match production PHP version for Composer) ---
|
|
FROM php:${phpVersion}-cli-alpine AS composer
|
|
RUN apk add --no-cache git unzip
|
|
COPY --from=composer:2 /usr/bin/composer /usr/bin/composer
|
|
WORKDIR /app
|
|
COPY composer.json composer.lock* ./
|
|
RUN composer install --no-dev --no-scripts --no-autoloader --prefer-dist --ignore-platform-reqs
|
|
COPY . .
|
|
RUN composer dump-autoload --optimize --no-dev --no-scripts
|
|
|
|
# --- Production stage ---
|
|
FROM php:${phpVersion}-fpm-alpine
|
|
|
|
RUN apk add --no-cache nginx supervisor curl openssl \\
|
|
&& docker-php-ext-install pdo pdo_mysql opcache \\
|
|
&& docker-php-ext-install pdo_pgsql 2>/dev/null || true
|
|
|
|
WORKDIR /var/www/html
|
|
COPY --from=composer /app .
|
|
|
|
# Generate nginx config inline (no dependency on user files)
|
|
RUN mkdir -p /etc/nginx/http.d && \\
|
|
echo 'server {' > /etc/nginx/http.d/default.conf && \\
|
|
echo ' listen ${port};' >> /etc/nginx/http.d/default.conf && \\
|
|
echo ' root /var/www/html/public;' >> /etc/nginx/http.d/default.conf && \\
|
|
echo ' index index.php index.html;' >> /etc/nginx/http.d/default.conf && \\
|
|
echo ' client_max_body_size 64M;' >> /etc/nginx/http.d/default.conf && \\
|
|
echo ' location / { try_files \\$uri \\$uri/ /index.php?\\$query_string; }' >> /etc/nginx/http.d/default.conf && \\
|
|
echo ' location ~ \\.php\\$ {' >> /etc/nginx/http.d/default.conf && \\
|
|
echo ' fastcgi_pass 127.0.0.1:9000;' >> /etc/nginx/http.d/default.conf && \\
|
|
echo ' fastcgi_param SCRIPT_FILENAME \\$document_root\\$fastcgi_script_name;' >> /etc/nginx/http.d/default.conf && \\
|
|
echo ' include fastcgi_params;' >> /etc/nginx/http.d/default.conf && \\
|
|
echo ' }' >> /etc/nginx/http.d/default.conf && \\
|
|
echo ' location ~ /\\.ht { deny all; }' >> /etc/nginx/http.d/default.conf && \\
|
|
echo '}' >> /etc/nginx/http.d/default.conf
|
|
|
|
# Generate supervisord config inline
|
|
RUN echo '[supervisord]' > /etc/supervisord.conf && \\
|
|
echo 'nodaemon=true' >> /etc/supervisord.conf && \\
|
|
echo 'logfile=/dev/stdout' >> /etc/supervisord.conf && \\
|
|
echo 'logfile_maxbytes=0' >> /etc/supervisord.conf && \\
|
|
echo '' >> /etc/supervisord.conf && \\
|
|
echo '[program:php-fpm]' >> /etc/supervisord.conf && \\
|
|
echo 'command=php-fpm -F' >> /etc/supervisord.conf && \\
|
|
echo 'autostart=true' >> /etc/supervisord.conf && \\
|
|
echo 'autorestart=true' >> /etc/supervisord.conf && \\
|
|
echo 'stdout_logfile=/dev/stdout' >> /etc/supervisord.conf && \\
|
|
echo 'stdout_logfile_maxbytes=0' >> /etc/supervisord.conf && \\
|
|
echo 'stderr_logfile=/dev/stderr' >> /etc/supervisord.conf && \\
|
|
echo 'stderr_logfile_maxbytes=0' >> /etc/supervisord.conf && \\
|
|
echo '' >> /etc/supervisord.conf && \\
|
|
echo '[program:nginx]' >> /etc/supervisord.conf && \\
|
|
echo 'command=nginx -g "daemon off;"' >> /etc/supervisord.conf && \\
|
|
echo 'autostart=true' >> /etc/supervisord.conf && \\
|
|
echo 'autorestart=true' >> /etc/supervisord.conf && \\
|
|
echo 'stdout_logfile=/dev/stdout' >> /etc/supervisord.conf && \\
|
|
echo 'stdout_logfile_maxbytes=0' >> /etc/supervisord.conf && \\
|
|
echo 'stderr_logfile=/dev/stderr' >> /etc/supervisord.conf && \\
|
|
echo 'stderr_logfile_maxbytes=0' >> /etc/supervisord.conf
|
|
|
|
# If user provides their own nginx/supervisor configs, use those instead
|
|
RUN [ -f docker/nginx.conf ] && cp docker/nginx.conf /etc/nginx/http.d/default.conf || true
|
|
RUN [ -f docker/supervisord.conf ] && cp docker/supervisord.conf /etc/supervisord.conf || true
|
|
|
|
# Ensure storage and cache directories exist and are writable
|
|
RUN mkdir -p storage/logs storage/framework/cache storage/framework/sessions storage/framework/views bootstrap/cache \\
|
|
&& chown -R www-data:www-data /var/www/html/storage /var/www/html/bootstrap/cache
|
|
|
|
RUN echo '#!/bin/sh' > /usr/local/bin/cloudhost-laravel-entrypoint.sh && \\
|
|
echo 'set -e' >> /usr/local/bin/cloudhost-laravel-entrypoint.sh && \\
|
|
echo 'cd /var/www/html' >> /usr/local/bin/cloudhost-laravel-entrypoint.sh && \\
|
|
echo 'mkdir -p storage/logs storage/framework/cache storage/framework/sessions storage/framework/views storage/app/public bootstrap/cache' >> /usr/local/bin/cloudhost-laravel-entrypoint.sh && \\
|
|
echo 'chown -R www-data:www-data storage bootstrap/cache' >> /usr/local/bin/cloudhost-laravel-entrypoint.sh && \\
|
|
echo 'if [ -z "$APP_KEY" ] || [ "$APP_KEY" = "null" ]; then export APP_KEY="base64:$(openssl rand -base64 32)"; fi' >> /usr/local/bin/cloudhost-laravel-entrypoint.sh && \\
|
|
echo 'php artisan config:clear || true' >> /usr/local/bin/cloudhost-laravel-entrypoint.sh && \\
|
|
echo 'php artisan cache:clear || true' >> /usr/local/bin/cloudhost-laravel-entrypoint.sh && \\
|
|
echo 'php artisan migrate --force --no-interaction || true' >> /usr/local/bin/cloudhost-laravel-entrypoint.sh && \\
|
|
echo 'php artisan storage:link || true' >> /usr/local/bin/cloudhost-laravel-entrypoint.sh && \\
|
|
echo 'php artisan config:cache || true' >> /usr/local/bin/cloudhost-laravel-entrypoint.sh && \\
|
|
echo 'php artisan route:cache || true' >> /usr/local/bin/cloudhost-laravel-entrypoint.sh && \\
|
|
echo 'php artisan view:cache || true' >> /usr/local/bin/cloudhost-laravel-entrypoint.sh && \\
|
|
echo 'exec /usr/bin/supervisord -c /etc/supervisord.conf' >> /usr/local/bin/cloudhost-laravel-entrypoint.sh && \\
|
|
chmod +x /usr/local/bin/cloudhost-laravel-entrypoint.sh
|
|
|
|
EXPOSE ${port}
|
|
CMD ["/usr/local/bin/cloudhost-laravel-entrypoint.sh"]
|
|
`;
|
|
}
|
|
|
|
private wordpressDockerfile(app: Application): string {
|
|
const wpVersion = app.runtimeVersion || '6.7';
|
|
const phpVersion = app.phpVersion || '8.3';
|
|
const hasUploadedCode = !!app.codePath;
|
|
|
|
return `FROM wordpress:${wpVersion}-php${phpVersion}-apache
|
|
|
|
# Install additional PHP extensions commonly needed by WordPress
|
|
RUN docker-php-ext-install opcache
|
|
|
|
# Enable Apache mod_rewrite for pretty permalinks
|
|
RUN a2enmod rewrite
|
|
|
|
# Increase PHP upload limits for WordPress media
|
|
RUN echo "upload_max_filesize = 64M\\npost_max_size = 64M\\nmax_execution_time = 300\\nmemory_limit = 256M" > /usr/local/etc/php/conf.d/uploads.ini
|
|
|
|
${
|
|
hasUploadedCode
|
|
? `# Copy user's custom WordPress files
|
|
COPY . /tmp/user-content
|
|
|
|
# Auto-detect: full public_html root (has wp-admin) vs wp-content only
|
|
# ── public_html mode ──
|
|
# wp-admin/ and wp-includes/ replace the base-image core so the user's
|
|
# exact WordPress version & patches are preserved.
|
|
# wp-content/ is staged in /usr/src/wordpress-user/ and merged into the
|
|
# PVC on first boot (same as migrate mode).
|
|
# wp-config.php is saved separately so docker-entrypoint.sh can still
|
|
# inject WORDPRESS_DB_* env-vars when no config exists yet.
|
|
# ── migrate mode (no wp-admin) ──
|
|
# Only wp-content + config files are processed.
|
|
RUN mkdir -p /usr/src/wordpress-user && \\
|
|
if [ -d /tmp/user-content/wp-admin ]; then \\
|
|
echo ">>> Full WordPress root (public_html) detected" && \\
|
|
echo ">>> Copying wp-admin/ to /var/www/html/" && \\
|
|
rm -rf /var/www/html/wp-admin && \\
|
|
cp -a /tmp/user-content/wp-admin /var/www/html/wp-admin && \\
|
|
echo ">>> Copying wp-includes/ to /var/www/html/" && \\
|
|
rm -rf /var/www/html/wp-includes && \\
|
|
cp -a /tmp/user-content/wp-includes /var/www/html/wp-includes && \\
|
|
echo ">>> Copying root PHP files (index.php, wp-login.php, ...)" && \\
|
|
find /tmp/user-content -maxdepth 1 -name "*.php" ! -name "wp-config.php" \\
|
|
-exec cp {} /var/www/html/ \\; 2>/dev/null || true && \\
|
|
echo ">>> Copying other root files/dirs (fonts, assets, etc.)" && \\
|
|
for item in /tmp/user-content/*; do \\
|
|
name=$(basename "$item"); \\
|
|
case "$name" in \\
|
|
wp-admin|wp-includes|wp-content|wp-config.php|.htaccess) ;; \\
|
|
*.php) ;; \\
|
|
*) \\
|
|
if [ -f "$item" ]; then \\
|
|
echo " root file: $name" && \\
|
|
cp "$item" /var/www/html/; \\
|
|
elif [ -d "$item" ]; then \\
|
|
echo " root dir: $name/" && \\
|
|
cp -a "$item" /var/www/html/; \\
|
|
fi ;; \\
|
|
esac; \\
|
|
done; \\
|
|
else \\
|
|
echo ">>> wp-content / config files only (migrate mode)"; \\
|
|
fi && \\
|
|
if [ -d /tmp/user-content/wp-content ]; then \\
|
|
echo ">>> Staging user wp-content (themes, plugins, uploads)..." && \\
|
|
cp -a /tmp/user-content/wp-content /usr/src/wordpress-user/wp-content; \\
|
|
elif [ -d /tmp/user-content/themes ] || [ -d /tmp/user-content/plugins ] || [ -d /tmp/user-content/uploads ]; then \\
|
|
echo ">>> Staging loose themes/plugins/uploads into wp-content..." && \\
|
|
mkdir -p /usr/src/wordpress-user/wp-content && \\
|
|
[ -d /tmp/user-content/themes ] && cp -a /tmp/user-content/themes /usr/src/wordpress-user/wp-content/ || true && \\
|
|
[ -d /tmp/user-content/plugins ] && cp -a /tmp/user-content/plugins /usr/src/wordpress-user/wp-content/ || true && \\
|
|
[ -d /tmp/user-content/uploads ] && cp -a /tmp/user-content/uploads /usr/src/wordpress-user/wp-content/ || true; \\
|
|
fi && \\
|
|
# wp-config.php is intentionally NOT copied — docker-entrypoint.sh generates it
|
|
# from WORDPRESS_DB_* env vars so credentials always match the deployed database.
|
|
if [ -f /tmp/user-content/.htaccess ]; then \\
|
|
echo ">>> Copying .htaccess" && \\
|
|
cp /tmp/user-content/.htaccess /var/www/html/.htaccess; \\
|
|
fi && \\
|
|
rm -rf /tmp/user-content && \\
|
|
echo ">>> WordPress user content staged"
|
|
|
|
# Custom entrypoint:
|
|
# 1. Merge staged wp-content into the PVC mount (every start — idempotent)
|
|
# 2. Hand off to official docker-entrypoint.sh which creates wp-config.php
|
|
# from WORDPRESS_DB_* env vars (never use uploaded wp-config with old credentials)
|
|
RUN { \\
|
|
echo '#!/bin/bash'; \\
|
|
echo 'set -e'; \\
|
|
echo ''; \\
|
|
echo '# ── Merge user wp-content into PVC ──'; \\
|
|
echo 'if [ -d /usr/src/wordpress-user/wp-content ]; then'; \\
|
|
echo ' echo ">>> Merging user wp-content into PVC..."'; \\
|
|
echo ' mkdir -p /var/www/html/wp-content'; \\
|
|
echo ' cp -a /usr/src/wordpress-user/wp-content/. /var/www/html/wp-content/'; \\
|
|
echo ' chown -R www-data:www-data /var/www/html/wp-content'; \\
|
|
echo ' echo ">>> User wp-content merged successfully"'; \\
|
|
echo 'fi'; \\
|
|
echo ''; \\
|
|
echo 'exec docker-entrypoint.sh apache2-foreground'; \\
|
|
} > /usr/local/bin/cloudhost-entrypoint.sh && chmod +x /usr/local/bin/cloudhost-entrypoint.sh
|
|
`
|
|
: `# Fresh install — no user content to merge
|
|
`
|
|
}
|
|
# Set proper ownership
|
|
RUN chown -R www-data:www-data /var/www/html
|
|
|
|
EXPOSE 80
|
|
${
|
|
hasUploadedCode
|
|
? `ENTRYPOINT ["cloudhost-entrypoint.sh"]
|
|
CMD []`
|
|
: `CMD ["apache2-foreground"]`
|
|
}
|
|
`;
|
|
}
|
|
|
|
// ─── Go Dockerfile ─────────────────────────────────────────────────
|
|
private goDockerfile(app: Application, archiveEntries: string[] = []): string {
|
|
const goVersion = app.runtimeVersion || '1.22';
|
|
const port = app.port || 8080;
|
|
const buildTarget = detectGoBuildTarget(archiveEntries);
|
|
return `# --- Build stage ---
|
|
FROM golang:${goVersion}-alpine AS builder
|
|
WORKDIR /app
|
|
|
|
# Install git for fetching dependencies
|
|
RUN apk add --no-cache git
|
|
|
|
# Copy go mod files first for better caching
|
|
COPY go.mod go.sum* ./
|
|
RUN go mod download || true
|
|
|
|
# Copy source code
|
|
COPY . .
|
|
|
|
# Build the application
|
|
RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags="-w -s" -o main ${buildTarget}
|
|
|
|
# --- Production stage ---
|
|
FROM alpine:3.19
|
|
WORKDIR /app
|
|
|
|
# Add CA certificates for HTTPS requests
|
|
RUN apk --no-cache add ca-certificates tzdata
|
|
|
|
# Create non-root user
|
|
RUN addgroup -g 1001 -S appgroup && adduser -S appuser -u 1001 -G appgroup
|
|
|
|
# Copy the binary from builder
|
|
COPY --from=builder /app/main .
|
|
COPY --from=builder /app/static ./static 2>/dev/null || true
|
|
COPY --from=builder /app/templates ./templates 2>/dev/null || true
|
|
COPY --from=builder /app/public ./public 2>/dev/null || true
|
|
|
|
# Create data directory for persistent storage
|
|
RUN mkdir -p /app/data && chown -R appuser:appgroup /app
|
|
|
|
USER appuser
|
|
|
|
ENV PORT=${port}
|
|
EXPOSE ${port}
|
|
HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \\
|
|
CMD wget --no-verbose --tries=1 --spider http://localhost:${port}/health || exit 1
|
|
|
|
CMD ["./main"]
|
|
`;
|
|
}
|
|
|
|
// ─── PHP (Plain) Dockerfile ────────────────────────────────────────
|
|
private phpDockerfile(app: Application): string {
|
|
const phpVersion = app.phpVersion || '8.3';
|
|
const port = app.port || 80;
|
|
return `FROM php:${phpVersion}-fpm-alpine
|
|
|
|
RUN apk add --no-cache nginx supervisor curl \\
|
|
&& docker-php-ext-install pdo pdo_mysql opcache \\
|
|
&& docker-php-ext-install pdo_pgsql 2>/dev/null || true
|
|
|
|
# Install common PHP extensions
|
|
RUN apk add --no-cache libpng-dev libjpeg-turbo-dev freetype-dev \\
|
|
&& docker-php-ext-configure gd --with-freetype --with-jpeg \\
|
|
&& docker-php-ext-install gd
|
|
|
|
WORKDIR /var/www/html
|
|
COPY . .
|
|
|
|
# Generate nginx config
|
|
RUN mkdir -p /etc/nginx/http.d && \\
|
|
echo 'server {' > /etc/nginx/http.d/default.conf && \\
|
|
echo ' listen ${port};' >> /etc/nginx/http.d/default.conf && \\
|
|
echo ' root /var/www/html;' >> /etc/nginx/http.d/default.conf && \\
|
|
echo ' index index.php index.html;' >> /etc/nginx/http.d/default.conf && \\
|
|
echo ' client_max_body_size 64M;' >> /etc/nginx/http.d/default.conf && \\
|
|
echo ' location / { try_files \\$uri \\$uri/ /index.php?\\$query_string; }' >> /etc/nginx/http.d/default.conf && \\
|
|
echo ' location ~ \\.php\\$ {' >> /etc/nginx/http.d/default.conf && \\
|
|
echo ' fastcgi_pass 127.0.0.1:9000;' >> /etc/nginx/http.d/default.conf && \\
|
|
echo ' fastcgi_param SCRIPT_FILENAME \\$document_root\\$fastcgi_script_name;' >> /etc/nginx/http.d/default.conf && \\
|
|
echo ' include fastcgi_params;' >> /etc/nginx/http.d/default.conf && \\
|
|
echo ' }' >> /etc/nginx/http.d/default.conf && \\
|
|
echo ' location ~ /\\.ht { deny all; }' >> /etc/nginx/http.d/default.conf && \\
|
|
echo '}' >> /etc/nginx/http.d/default.conf
|
|
|
|
# Generate supervisord config
|
|
RUN echo '[supervisord]' > /etc/supervisord.conf && \\
|
|
echo 'nodaemon=true' >> /etc/supervisord.conf && \\
|
|
echo 'logfile=/dev/stdout' >> /etc/supervisord.conf && \\
|
|
echo 'logfile_maxbytes=0' >> /etc/supervisord.conf && \\
|
|
echo '[program:php-fpm]' >> /etc/supervisord.conf && \\
|
|
echo 'command=php-fpm -F' >> /etc/supervisord.conf && \\
|
|
echo 'autostart=true' >> /etc/supervisord.conf && \\
|
|
echo 'autorestart=true' >> /etc/supervisord.conf && \\
|
|
echo 'stdout_logfile=/dev/stdout' >> /etc/supervisord.conf && \\
|
|
echo 'stdout_logfile_maxbytes=0' >> /etc/supervisord.conf && \\
|
|
echo 'stderr_logfile=/dev/stderr' >> /etc/supervisord.conf && \\
|
|
echo 'stderr_logfile_maxbytes=0' >> /etc/supervisord.conf && \\
|
|
echo '[program:nginx]' >> /etc/supervisord.conf && \\
|
|
echo 'command=nginx -g "daemon off;"' >> /etc/supervisord.conf && \\
|
|
echo 'autostart=true' >> /etc/supervisord.conf && \\
|
|
echo 'autorestart=true' >> /etc/supervisord.conf && \\
|
|
echo 'stdout_logfile=/dev/stdout' >> /etc/supervisord.conf && \\
|
|
echo 'stdout_logfile_maxbytes=0' >> /etc/supervisord.conf && \\
|
|
echo 'stderr_logfile=/dev/stderr' >> /etc/supervisord.conf && \\
|
|
echo 'stderr_logfile_maxbytes=0' >> /etc/supervisord.conf
|
|
|
|
# Use custom configs if provided
|
|
RUN [ -f docker/nginx.conf ] && cp docker/nginx.conf /etc/nginx/http.d/default.conf || true
|
|
RUN [ -f docker/supervisord.conf ] && cp docker/supervisord.conf /etc/supervisord.conf || true
|
|
|
|
# Create upload and data directories
|
|
RUN mkdir -p /var/www/html/uploads /var/www/html/data \\
|
|
&& chown -R www-data:www-data /var/www/html
|
|
|
|
EXPOSE ${port}
|
|
CMD ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf"]
|
|
`;
|
|
}
|
|
|
|
// ─── Python Dockerfile ─────────────────────────────────────────────
|
|
private pythonDockerfile(app: Application): string {
|
|
const pythonVersion = app.runtimeVersion || '3.12';
|
|
const port = app.port || 8000;
|
|
return `# --- Build stage ---
|
|
FROM python:${pythonVersion}-slim AS builder
|
|
WORKDIR /app
|
|
|
|
# Install build dependencies
|
|
RUN apt-get update && apt-get install -y --no-install-recommends \\
|
|
build-essential libpq-dev \\
|
|
&& rm -rf /var/lib/apt/lists/*
|
|
|
|
# Copy requirements and install dependencies
|
|
COPY requirements.txt* ./
|
|
RUN pip install --no-cache-dir --user -r requirements.txt 2>/dev/null || \\
|
|
pip install --no-cache-dir --user flask gunicorn
|
|
|
|
# --- Production stage ---
|
|
FROM python:${pythonVersion}-slim
|
|
WORKDIR /app
|
|
|
|
# Install runtime dependencies
|
|
RUN apt-get update && apt-get install -y --no-install-recommends \\
|
|
libpq5 curl \\
|
|
&& rm -rf /var/lib/apt/lists/*
|
|
|
|
# Create non-root user
|
|
RUN groupadd -g 1001 appgroup && useradd -r -u 1001 -g appgroup appuser
|
|
|
|
# Copy installed packages from builder
|
|
COPY --from=builder /root/.local /home/appuser/.local
|
|
|
|
# Copy application code
|
|
COPY . .
|
|
|
|
# Create data directory
|
|
RUN mkdir -p /app/data && chown -R appuser:appgroup /app
|
|
|
|
USER appuser
|
|
ENV PATH=/home/appuser/.local/bin:$PATH
|
|
ENV PORT=${port}
|
|
ENV PYTHONUNBUFFERED=1
|
|
|
|
EXPOSE ${port}
|
|
HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \\
|
|
CMD curl -f http://localhost:${port}/health || exit 1
|
|
|
|
# Auto-detect: Flask, FastAPI, or plain Python
|
|
CMD sh -c "if [ -f main.py ]; then if grep -qi fastapi main.py; then exec uvicorn main:app --host 0.0.0.0 --port ${port}; elif grep -qi flask main.py; then exec gunicorn -w 4 -b 0.0.0.0:${port} main:app; else exec python main.py; fi; elif [ -f app.py ]; then if grep -qi fastapi app.py; then exec uvicorn app:app --host 0.0.0.0 --port ${port}; elif grep -qi flask app.py; then exec gunicorn -w 4 -b 0.0.0.0:${port} app:app; else exec python app.py; fi; else exec gunicorn -w 4 -b 0.0.0.0:${port} app:app; fi"
|
|
`;
|
|
}
|
|
|
|
// ─── Django Dockerfile ─────────────────────────────────────────────
|
|
private djangoDockerfile(app: Application, archiveEntries: string[] = []): string {
|
|
const pythonVersion = app.runtimeVersion || '3.12';
|
|
const port = app.port || 8000;
|
|
const settingsModule = detectDjangoSettingsModule(archiveEntries);
|
|
return `# --- Build stage ---
|
|
FROM python:${pythonVersion}-slim AS builder
|
|
WORKDIR /app
|
|
|
|
# Install build dependencies
|
|
RUN apt-get update && apt-get install -y --no-install-recommends \\
|
|
build-essential libpq-dev \\
|
|
&& rm -rf /var/lib/apt/lists/*
|
|
|
|
# Copy requirements and install dependencies
|
|
COPY requirements.txt* ./
|
|
RUN pip install --no-cache-dir --user -r requirements.txt 2>/dev/null || \\
|
|
pip install --no-cache-dir --user django gunicorn psycopg2-binary mysqlclient
|
|
|
|
# --- Production stage ---
|
|
FROM python:${pythonVersion}-slim
|
|
WORKDIR /app
|
|
|
|
# Install runtime dependencies
|
|
RUN apt-get update && apt-get install -y --no-install-recommends \\
|
|
libpq5 default-libmysqlclient-dev curl \\
|
|
&& rm -rf /var/lib/apt/lists/*
|
|
|
|
# Create non-root user
|
|
RUN groupadd -g 1001 appgroup && useradd -r -u 1001 -g appgroup appuser
|
|
|
|
# Copy installed packages from builder
|
|
COPY --from=builder /root/.local /home/appuser/.local
|
|
|
|
# Copy application code
|
|
COPY . .
|
|
|
|
# Create directories for static files and media
|
|
RUN mkdir -p /app/staticfiles /app/media /app/data \\
|
|
&& chown -R appuser:appgroup /app
|
|
|
|
USER appuser
|
|
ENV PATH=/home/appuser/.local/bin:$PATH
|
|
ENV PORT=${port}
|
|
ENV PYTHONUNBUFFERED=1
|
|
ENV DJANGO_SETTINGS_MODULE=${settingsModule}
|
|
|
|
EXPOSE ${port}
|
|
HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \\
|
|
CMD curl -f http://localhost:${port}/health/ || curl -f http://localhost:${port}/ || exit 1
|
|
|
|
# Auto-detect project structure and run migrations + collectstatic
|
|
CMD sh -c "\\
|
|
PROJECT_NAME=\\$(find . -maxdepth 2 -name 'wsgi.py' | head -1 | cut -d'/' -f2) && \\
|
|
if [ -z \\\"\\$PROJECT_NAME\\\" ]; then PROJECT_NAME='config'; fi && \\
|
|
echo \\\"Django project: \\$PROJECT_NAME\\\" && \\
|
|
python manage.py migrate --noinput 2>/dev/null || true && \\
|
|
python manage.py collectstatic --noinput 2>/dev/null || true && \\
|
|
exec gunicorn \\$PROJECT_NAME.wsgi:application --bind 0.0.0.0:${port} --workers 4 --threads 2 \\
|
|
"
|
|
`;
|
|
}
|
|
|
|
// ─── .NET Dockerfile ───────────────────────────────────────────────
|
|
private dotnetDockerfile(app: Application, archiveEntries: string[] = []): string {
|
|
const dotnetVersion = app.runtimeVersion || '8.0';
|
|
const port = app.port || 5000;
|
|
const csprojHint = detectShallowCsproj(archiveEntries);
|
|
const csprojFind = csprojHint
|
|
? `CSPROJ="${csprojHint}"`
|
|
: `CSPROJ=$(find . -maxdepth 3 -name '*.csproj' | head -1)`;
|
|
return `# --- Build stage ---
|
|
FROM mcr.microsoft.com/dotnet/sdk:${dotnetVersion} AS build
|
|
WORKDIR /src
|
|
|
|
# Copy source and locate project file (supports nested csproj layouts)
|
|
COPY . .
|
|
RUN ${csprojFind} && \\
|
|
test -n "$CSPROJ" && \\
|
|
dotnet restore "$CSPROJ" && \\
|
|
dotnet publish "$CSPROJ" -c Release -o /app/publish
|
|
|
|
# --- Production stage ---
|
|
FROM mcr.microsoft.com/dotnet/aspnet:${dotnetVersion}
|
|
WORKDIR /app
|
|
|
|
# Create non-root user
|
|
RUN groupadd -g 1001 appgroup && useradd -r -u 1001 -g appgroup appuser
|
|
|
|
# Copy published app
|
|
COPY --from=build /app/publish .
|
|
|
|
# Create data directory
|
|
RUN mkdir -p /app/data && chown -R appuser:appgroup /app
|
|
|
|
USER appuser
|
|
|
|
ENV ASPNETCORE_URLS=http://+:${port}
|
|
ENV DOTNET_RUNNING_IN_CONTAINER=true
|
|
ENV ASPNETCORE_ENVIRONMENT=Production
|
|
|
|
EXPOSE ${port}
|
|
HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \\
|
|
CMD curl -f http://localhost:${port}/health || curl -f http://localhost:${port}/ || exit 1
|
|
|
|
# Auto-detect entry point DLL
|
|
CMD ["sh", "-c", "DLL=$(find . -maxdepth 1 -name '*.dll' ! -name '*.deps.dll' ! -name '*.runtimeconfig.dll' | head -1) && dotnet $DLL"]
|
|
`;
|
|
}
|
|
|
|
/**
|
|
* Whether a K8s API error is a transient connectivity/availability blip that
|
|
* should be retried rather than failing the operation. Covers socket-level
|
|
* errors, request timeouts (incl. the apiserver "request did not complete
|
|
* within the allotted timeout" message), DNS hiccups and 5xx/429 responses.
|
|
*/
|
|
private isTransientK8sError(err: any): boolean {
|
|
const statusCode = err?.statusCode ?? err?.response?.statusCode ?? err?.body?.code;
|
|
if (typeof statusCode === 'number' && (statusCode >= 500 || statusCode === 429)) {
|
|
return true;
|
|
}
|
|
const haystack = [err?.code, err?.message, err?.body?.message, err?.cause?.code].filter(Boolean).join(' ');
|
|
return /ECONNRESET|ECONNREFUSED|ETIMEDOUT|ESOCKETTIMEDOUT|EPIPE|EAI_AGAIN|ENOTFOUND|ENETUNREACH|socket hang up|timed? ?out|allotted timeout|did not complete|Client network socket disconnected/i.test(
|
|
haystack,
|
|
);
|
|
}
|
|
|
|
private async waitForJobCompletion(batchApi: k8s.BatchV1Api, coreApi: k8s.CoreV1Api, jobName: string, namespace: string, timeoutSeconds: number, deploymentId?: string): Promise<void> {
|
|
const startTime = Date.now();
|
|
const timeoutMs = timeoutSeconds * 1000;
|
|
let lastLoggedStatus = '';
|
|
|
|
while (Date.now() - startTime < timeoutMs) {
|
|
this.throwIfCancelled(deploymentId);
|
|
const elapsed = Date.now() - startTime;
|
|
const buildPercent = Math.min(90, 15 + Math.round((elapsed / timeoutMs) * 75));
|
|
this.setProgress(deploymentId, {
|
|
phase: 'building',
|
|
percent: buildPercent,
|
|
message: 'Building Docker image...',
|
|
});
|
|
// ── Check Job status (with retry for transient connection errors) ──
|
|
let job: k8s.V1Job;
|
|
try {
|
|
job = await batchApi.readNamespacedJob({ name: jobName, namespace });
|
|
} catch (pollErr: any) {
|
|
// The Kaniko job keeps running independently of these status polls.
|
|
// A single API blip (timeout, reset, 5xx, DNS) must NOT abort a build
|
|
// that is still progressing — just retry on the next poll tick.
|
|
if (this.isTransientK8sError(pollErr)) {
|
|
const detail = pollErr?.code || pollErr?.message || pollErr?.statusCode || 'unknown';
|
|
this.logger.warn(`Transient K8s API error polling job ${jobName}: ${detail} — retrying in 5s`);
|
|
await new Promise((r) => setTimeout(r, 5000));
|
|
continue;
|
|
}
|
|
throw pollErr;
|
|
}
|
|
const status = job.status;
|
|
|
|
if (status?.succeeded && status.succeeded > 0) {
|
|
this.logger.log(`Build job ${jobName} succeeded`);
|
|
return;
|
|
}
|
|
|
|
// Check if the Job has permanently failed (all retries exhausted)
|
|
const failedCondition = (status?.conditions || []).find((c) => c.type === 'Failed' && c.status === 'True');
|
|
if (failedCondition) {
|
|
const logs = await this.getBuildLogs(coreApi, jobName, namespace);
|
|
throw new Error(`Build job ${jobName} failed.\nLogs:\n${logs}`);
|
|
}
|
|
|
|
// Safety net: if failures exceed backoffLimit and no pod is still running
|
|
const backoffLimit = job.spec?.backoffLimit ?? 0;
|
|
const failedCount = status?.failed ?? 0;
|
|
if (failedCount > backoffLimit) {
|
|
// Double-check: are there still active pods?
|
|
const activePods = (status as any)?.active ?? 0;
|
|
if (activePods === 0) {
|
|
const logs = await this.getBuildLogs(coreApi, jobName, namespace);
|
|
throw new Error(`Build job ${jobName} failed: ${failedCount} failures exceeded backoffLimit=${backoffLimit}.\nLogs:\n${logs}`);
|
|
}
|
|
}
|
|
|
|
// Log intermediate pod failures (retries still available)
|
|
if (failedCount > 0) {
|
|
this.logger.warn(`Build job ${jobName}: ${failedCount} pod failure(s), backoffLimit=${backoffLimit} — retrying...`);
|
|
}
|
|
|
|
// ── Check Pod status for early failure detection ──
|
|
try {
|
|
const pods = await coreApi.listNamespacedPod({
|
|
namespace,
|
|
labelSelector: `job-name=${jobName}`,
|
|
});
|
|
|
|
for (const pod of pods.items) {
|
|
const podName = pod.metadata?.name || 'unknown';
|
|
const phase = pod.status?.phase;
|
|
|
|
// Check all container statuses (init + regular) for stuck states
|
|
const allStatuses = [...(pod.status?.initContainerStatuses || []), ...(pod.status?.containerStatuses || [])];
|
|
|
|
for (const cs of allStatuses) {
|
|
const waiting = cs.state?.waiting;
|
|
if (waiting?.reason) {
|
|
const reason = waiting.reason;
|
|
const msg = waiting.message || '';
|
|
|
|
// These are unrecoverable — fail fast instead of waiting 10 minutes
|
|
const fatalReasons = ['ErrImagePull', 'ImagePullBackOff', 'CreateContainerConfigError', 'InvalidImageName', 'CrashLoopBackOff'];
|
|
|
|
if (fatalReasons.includes(reason)) {
|
|
const logs = await this.getBuildLogs(coreApi, jobName, namespace);
|
|
throw new Error(`Build pod ${podName} stuck: ${reason} — ${msg}\nLogs:\n${logs}`);
|
|
}
|
|
|
|
// Log non-fatal waiting states periodically
|
|
const statusKey = `${podName}:${cs.name}:${reason}`;
|
|
if (statusKey !== lastLoggedStatus) {
|
|
this.logger.warn(`Pod ${podName} container "${cs.name}": ${reason} — ${msg}`);
|
|
lastLoggedStatus = statusKey;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Log phase changes
|
|
const phaseKey = `${podName}:${phase}`;
|
|
if (phaseKey !== lastLoggedStatus && phase !== 'Succeeded') {
|
|
this.logger.log(`Build pod ${podName}: phase=${phase}`);
|
|
lastLoggedStatus = phaseKey;
|
|
}
|
|
}
|
|
} catch (podErr: any) {
|
|
// Don't fail the whole build just because pod status check failed
|
|
if (podErr.message?.includes('stuck:') || podErr.message?.includes('Build pod')) {
|
|
throw podErr; // Re-throw our own fatal errors
|
|
}
|
|
this.logger.warn(`Could not check pod status: ${podErr.message}`);
|
|
}
|
|
|
|
// Wait 5 seconds before polling again
|
|
await new Promise((resolve) => setTimeout(resolve, 5000));
|
|
}
|
|
|
|
// Timeout — get logs for debugging
|
|
let logs = '';
|
|
try {
|
|
logs = await this.getBuildLogs(coreApi, jobName, namespace);
|
|
} catch {}
|
|
throw new Error(`Build job ${jobName} timed out after ${timeoutSeconds}s\nLogs:\n${logs}`);
|
|
}
|
|
|
|
/**
|
|
* Live build logs for an in-progress build, read straight from the running
|
|
* build pod (init + kaniko containers). Returns null when there is no active
|
|
* build session for this deployment (e.g. build already finished/cleaned up),
|
|
* so callers can fall back to the persisted build log.
|
|
*/
|
|
async getLiveBuildLog(deploymentId: string): Promise<string | null> {
|
|
const session = this.activeBuilds.get(deploymentId);
|
|
if (!session?.coreApi || !session.namespace || !session.buildPodName) {
|
|
return null;
|
|
}
|
|
try {
|
|
return await this.getBuildLogs(session.coreApi, session.buildPodName, session.namespace);
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
private async getBuildLogs(coreApi: k8s.CoreV1Api, jobName: string, namespace: string): Promise<string> {
|
|
try {
|
|
const pods = await coreApi.listNamespacedPod({
|
|
namespace,
|
|
labelSelector: `job-name=${jobName}`,
|
|
});
|
|
|
|
if (pods.items.length === 0) {
|
|
return 'No pods found for build job.';
|
|
}
|
|
|
|
const podName = pods.items[0].metadata?.name;
|
|
if (!podName) return 'Pod name not found.';
|
|
|
|
// Get logs from all containers (init + kaniko)
|
|
let allLogs = '';
|
|
const containers = [...(pods.items[0].spec?.initContainers || []), ...(pods.items[0].spec?.containers || [])];
|
|
|
|
for (const container of containers) {
|
|
try {
|
|
const logResponse = await coreApi.readNamespacedPodLog({
|
|
name: podName,
|
|
namespace,
|
|
container: container.name,
|
|
tailLines: 500,
|
|
});
|
|
allLogs += `\n--- ${container.name} ---\n${logResponse}`;
|
|
} catch {
|
|
allLogs += `\n--- ${container.name} --- (no logs available)`;
|
|
}
|
|
}
|
|
return allLogs;
|
|
} catch (e: any) {
|
|
return `Failed to retrieve logs: ${e.message}`;
|
|
}
|
|
}
|
|
}
|