keyhan
3eff38f8d2
Rework the application build/deploy pipeline for scalability, reproducibility, and security: - Build queue: deploys run through a bounded-concurrency Bull queue (BUILD_CONCURRENCY, default 3) so concurrent user deploys can't flood the cluster with Kaniko jobs. Build state (progress / cancel / session) moves from in-memory Maps to Redis, so cancel + live logs work across backend replicas. - Nixpacks + BYO Dockerfile: code runtimes build via Nixpacks (or the user's own Dockerfile when present); the hand-written per-runtime Dockerfile generators and runtime auto-detection are removed. WordPress keeps its templated path. Build-time mirror env (NIXPACKS_BUILD_ENV) supports the Iran network. - Source upload to MinIO: archives stream to in-cluster MinIO; build pods pull via a presigned URL. Removes the PVC + helper pod + kubectl cp upload path. - Report-only Trivy scan after build; per-severity summary stored on the deployment and shown as a badge in the dashboard. Never gates a deploy. - Registry GC: a Redis-locked daily job keeps the newest N image tags per app (REGISTRY_KEEP_VERSIONS, default 3) and reclaims disk via garbage-collect. - Hardening: git tokens are delivered via a per-build Secret + git credential store instead of being embedded in the clone URL / Job manifest; build timeout is configurable. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
1206 lines
46 KiB
TypeScript
1206 lines
46 KiB
TypeScript
import { Injectable, Logger, Inject } from '@nestjs/common';
|
|
import { ConfigService } from '@nestjs/config';
|
|
import { Redis } from 'ioredis';
|
|
import { REDIS_CLIENT } from '../common/redis/redis.module';
|
|
import * as k8s from '@kubernetes/client-node';
|
|
import { ChildProcess } from 'child_process';
|
|
import * as net from 'net';
|
|
import { Application } from '../applications/entities/application.entity';
|
|
import { AppRuntime } from '../common/enums';
|
|
import { ClustersService } from '../clusters/clusters.service';
|
|
import { RegistryService } from '../kubernetes/registry.service';
|
|
import { StorageService } from '../common/storage/storage.service';
|
|
|
|
export class BuildCancelledError extends Error {
|
|
constructor() {
|
|
super('Build cancelled by user');
|
|
this.name = 'BuildCancelledError';
|
|
}
|
|
}
|
|
|
|
interface ActiveBuildSession {
|
|
cancelled: boolean;
|
|
coreApi?: k8s.CoreV1Api;
|
|
batchApi?: k8s.BatchV1Api;
|
|
namespace?: string;
|
|
buildPodName?: string;
|
|
sourcePvcName?: string;
|
|
helperPodName?: string;
|
|
clusterId?: string;
|
|
processes: ChildProcess[];
|
|
socket?: net.Socket;
|
|
}
|
|
|
|
/**
|
|
* Serializable subset of a build session persisted to Redis so a build can be
|
|
* cancelled (or have its live logs read) from a backend replica other than the
|
|
* one running the build. `clusterId` lets that replica rebuild a K8s client.
|
|
*/
|
|
interface PersistedBuildSession {
|
|
namespace?: string;
|
|
buildPodName?: string;
|
|
sourcePvcName?: string;
|
|
helperPodName?: string;
|
|
clusterId?: string;
|
|
}
|
|
|
|
export interface BuildProgress {
|
|
phase: 'uploading' | 'building' | 'deploying' | 'done' | 'failed' | 'cancelled';
|
|
percent: number;
|
|
bytesUploaded?: number;
|
|
totalBytes?: number;
|
|
message?: string;
|
|
}
|
|
|
|
@Injectable()
|
|
export class BuildService {
|
|
private readonly logger = new Logger(BuildService.name);
|
|
/** Local, non-serializable session state (child processes, sockets, API clients). */
|
|
private readonly activeBuilds = new Map<string, ActiveBuildSession>();
|
|
/** Build state TTL in Redis (1h) — long enough for the slowest build + final read. */
|
|
private static readonly STATE_TTL_SECONDS = 3600;
|
|
/**
|
|
* Kaniko executor image. Pinned (not `:latest`) so it can be cached on the node
|
|
* with imagePullPolicy=IfNotPresent — avoids re-pulling the ~250MB image on every build.
|
|
*/
|
|
private readonly kanikoImage = process.env.KANIKO_IMAGE || 'gcr.io/kaniko-project/executor:v1.23.2';
|
|
|
|
constructor(
|
|
private configService: ConfigService,
|
|
private clustersService: ClustersService,
|
|
private registryService: RegistryService,
|
|
private storageService: StorageService,
|
|
@Inject(REDIS_CLIENT) private readonly redis: Redis,
|
|
) {}
|
|
|
|
// ─── Redis keys for cross-replica build state ──────────────────────
|
|
private progressKey(id: string): string {
|
|
return `build:progress:${id}`;
|
|
}
|
|
private sessionKey(id: string): string {
|
|
return `build:session:${id}`;
|
|
}
|
|
private cancelKey(id: string): string {
|
|
return `build:cancelled:${id}`;
|
|
}
|
|
|
|
private beginBuildSession(deploymentId: string): void {
|
|
this.activeBuilds.set(deploymentId, { cancelled: false, processes: [] });
|
|
}
|
|
|
|
private getSession(deploymentId?: string): ActiveBuildSession | undefined {
|
|
if (!deploymentId) return undefined;
|
|
return this.activeBuilds.get(deploymentId);
|
|
}
|
|
|
|
private updateBuildSession(deploymentId: string, update: Partial<ActiveBuildSession>): void {
|
|
const session = this.activeBuilds.get(deploymentId);
|
|
if (session) Object.assign(session, update);
|
|
// Mirror the serializable subset to Redis so another replica can cancel /
|
|
// read logs for this build. Fire-and-forget — never block the build on it.
|
|
const persisted: PersistedBuildSession = {
|
|
namespace: session?.namespace,
|
|
buildPodName: session?.buildPodName,
|
|
sourcePvcName: session?.sourcePvcName,
|
|
helperPodName: session?.helperPodName,
|
|
clusterId: session?.clusterId,
|
|
};
|
|
void this.redis
|
|
.set(this.sessionKey(deploymentId), JSON.stringify(persisted), 'EX', BuildService.STATE_TTL_SECONDS)
|
|
.catch(() => undefined);
|
|
}
|
|
|
|
/** Read the persisted (cross-replica) session metadata for a build. */
|
|
private async readPersistedSession(deploymentId: string): Promise<PersistedBuildSession | null> {
|
|
try {
|
|
const raw = await this.redis.get(this.sessionKey(deploymentId));
|
|
return raw ? (JSON.parse(raw) as PersistedBuildSession) : null;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
/** Build a CoreV1Api/BatchV1Api pair for a cluster (used for cross-replica cleanup). */
|
|
private async makeClusterApis(clusterId?: string): Promise<{ coreApi: k8s.CoreV1Api; batchApi: k8s.BatchV1Api } | null> {
|
|
try {
|
|
const cluster = clusterId ? await this.clustersService.findOne(clusterId) : await this.clustersService.getDefault();
|
|
const kc = new k8s.KubeConfig();
|
|
kc.loadFromString(cluster.kubeconfig);
|
|
return { coreApi: kc.makeApiClient(k8s.CoreV1Api), batchApi: kc.makeApiClient(k8s.BatchV1Api) };
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Whether this build has been cancelled — checks both the local session flag
|
|
* and the shared Redis flag, so a cancel issued on any replica is observed by
|
|
* the replica actually running the build.
|
|
*/
|
|
private async isCancelledShared(deploymentId?: string): Promise<boolean> {
|
|
if (!deploymentId) return false;
|
|
if (this.activeBuilds.get(deploymentId)?.cancelled) return true;
|
|
try {
|
|
return (await this.redis.exists(this.cancelKey(deploymentId))) === 1;
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
private async throwIfCancelledShared(deploymentId?: string): Promise<void> {
|
|
if (await this.isCancelledShared(deploymentId)) {
|
|
throw new BuildCancelledError();
|
|
}
|
|
}
|
|
|
|
private registerProcess(deploymentId: string | undefined, proc: ChildProcess): void {
|
|
const session = this.getSession(deploymentId);
|
|
if (!session) return;
|
|
session.processes.push(proc);
|
|
if (session.cancelled) {
|
|
try {
|
|
proc.kill('SIGKILL');
|
|
} catch {
|
|
/* ignore */
|
|
}
|
|
}
|
|
}
|
|
|
|
private registerSocket(deploymentId: string | undefined, socket: net.Socket): void {
|
|
const session = this.getSession(deploymentId);
|
|
if (!session) return;
|
|
if (session.socket) {
|
|
try {
|
|
session.socket.destroy();
|
|
} catch {
|
|
/* ignore */
|
|
}
|
|
}
|
|
session.socket = socket;
|
|
if (session.cancelled) {
|
|
try {
|
|
socket.destroy();
|
|
} catch {
|
|
/* ignore */
|
|
}
|
|
}
|
|
}
|
|
|
|
private throwIfCancelled(deploymentId?: string): void {
|
|
if (deploymentId && this.activeBuilds.get(deploymentId)?.cancelled) {
|
|
throw new BuildCancelledError();
|
|
}
|
|
}
|
|
|
|
private endBuildSession(deploymentId?: string): void {
|
|
if (!deploymentId) return;
|
|
this.activeBuilds.delete(deploymentId);
|
|
void this.redis.del(this.sessionKey(deploymentId), this.cancelKey(deploymentId)).catch(() => undefined);
|
|
}
|
|
|
|
/** Delete the K8s artifacts (helper pod, build job, source PVC, dockerfile configmap) of one build. */
|
|
private async deleteBuildArtifacts(
|
|
coreApi: k8s.CoreV1Api,
|
|
batchApi: k8s.BatchV1Api,
|
|
namespace: string,
|
|
names: { buildPodName?: string; sourcePvcName?: string; helperPodName?: string },
|
|
): Promise<void> {
|
|
const { buildPodName, sourcePvcName, helperPodName } = names;
|
|
const cleanup: Promise<unknown>[] = [];
|
|
if (helperPodName) {
|
|
cleanup.push(coreApi.deleteNamespacedPod({ name: helperPodName, namespace, gracePeriodSeconds: 0 }).catch(() => undefined));
|
|
}
|
|
if (buildPodName) {
|
|
cleanup.push(
|
|
batchApi
|
|
.deleteNamespacedJob({ name: buildPodName, namespace, gracePeriodSeconds: 0, propagationPolicy: 'Foreground' })
|
|
.catch(() => undefined),
|
|
);
|
|
}
|
|
if (sourcePvcName) {
|
|
cleanup.push(coreApi.deleteNamespacedPersistentVolumeClaim({ name: sourcePvcName, namespace }).catch(() => undefined));
|
|
}
|
|
if (buildPodName) {
|
|
cleanup.push(coreApi.deleteNamespacedConfigMap({ name: `${buildPodName}-dockerfile`, namespace }).catch(() => undefined));
|
|
}
|
|
await Promise.all(cleanup);
|
|
}
|
|
|
|
async cancelBuild(deploymentId: string): Promise<void> {
|
|
this.logger.log(`Cancelling build for deployment ${deploymentId}`);
|
|
// Shared flag so the (possibly different) replica running the build observes
|
|
// the cancellation via isCancelledShared and aborts its wait loop.
|
|
void this.redis.set(this.cancelKey(deploymentId), '1', 'EX', BuildService.STATE_TTL_SECONDS).catch(() => undefined);
|
|
|
|
const session = this.activeBuilds.get(deploymentId);
|
|
if (session) {
|
|
// Local build — stop in-process work and clean up via the live API clients.
|
|
session.cancelled = true;
|
|
if (session.socket) {
|
|
try {
|
|
session.socket.destroy();
|
|
} catch {
|
|
/* ignore */
|
|
}
|
|
}
|
|
for (const proc of session.processes) {
|
|
try {
|
|
proc.kill('SIGKILL');
|
|
} catch {
|
|
/* ignore */
|
|
}
|
|
}
|
|
if (session.coreApi && session.batchApi && session.namespace) {
|
|
await this.deleteBuildArtifacts(session.coreApi, session.batchApi, session.namespace, session);
|
|
this.logger.log(`Cleaned up K8s build resources for deployment ${deploymentId}`);
|
|
}
|
|
} else {
|
|
// Build is running on another replica (or already finished) — reconstruct
|
|
// a client from the persisted session metadata and clean up its artifacts.
|
|
const persisted = await this.readPersistedSession(deploymentId);
|
|
if (persisted?.namespace) {
|
|
const apis = await this.makeClusterApis(persisted.clusterId);
|
|
if (apis) {
|
|
await this.deleteBuildArtifacts(apis.coreApi, apis.batchApi, persisted.namespace, persisted);
|
|
this.logger.log(`Cleaned up cross-replica K8s build resources for deployment ${deploymentId}`);
|
|
}
|
|
}
|
|
}
|
|
|
|
this.setProgress(deploymentId, { phase: 'cancelled', percent: 0, message: 'Cancelled by user' });
|
|
this.endBuildSession(deploymentId);
|
|
}
|
|
|
|
/** Delete all in-flight build artifacts for an app (helper pods, jobs, PVCs, configmaps). */
|
|
async cleanupBuildResourcesForApp(app: Application): Promise<void> {
|
|
const buildNamespace = this.configService.get<string>('build.namespace') || 'cloudhost-builds';
|
|
const prefix = `build-${app.name}-`;
|
|
|
|
const cluster = app.clusterId ? await this.clustersService.findOne(app.clusterId) : await this.clustersService.getDefault();
|
|
|
|
const kc = new k8s.KubeConfig();
|
|
kc.loadFromString(cluster.kubeconfig);
|
|
const coreApi = kc.makeApiClient(k8s.CoreV1Api);
|
|
const batchApi = kc.makeApiClient(k8s.BatchV1Api);
|
|
|
|
const cleanup: Promise<unknown>[] = [];
|
|
|
|
const [pods, pvcs, jobs, configMaps] = await Promise.all([
|
|
coreApi.listNamespacedPod({ namespace: buildNamespace }),
|
|
coreApi.listNamespacedPersistentVolumeClaim({
|
|
namespace: buildNamespace,
|
|
}),
|
|
batchApi.listNamespacedJob({ namespace: buildNamespace }),
|
|
coreApi.listNamespacedConfigMap({ namespace: buildNamespace }),
|
|
]);
|
|
|
|
for (const pod of pods.items) {
|
|
const name = pod.metadata?.name || '';
|
|
if (name.startsWith(prefix)) {
|
|
cleanup.push(
|
|
coreApi
|
|
.deleteNamespacedPod({
|
|
name,
|
|
namespace: buildNamespace,
|
|
gracePeriodSeconds: 0,
|
|
})
|
|
.catch(() => undefined),
|
|
);
|
|
}
|
|
}
|
|
for (const pvc of pvcs.items) {
|
|
const name = pvc.metadata?.name || '';
|
|
if (name.startsWith(prefix)) {
|
|
cleanup.push(
|
|
coreApi
|
|
.deleteNamespacedPersistentVolumeClaim({
|
|
name,
|
|
namespace: buildNamespace,
|
|
})
|
|
.catch(() => undefined),
|
|
);
|
|
}
|
|
}
|
|
for (const job of jobs.items) {
|
|
const name = job.metadata?.name || '';
|
|
if (name.startsWith(prefix)) {
|
|
cleanup.push(
|
|
batchApi
|
|
.deleteNamespacedJob({
|
|
name,
|
|
namespace: buildNamespace,
|
|
gracePeriodSeconds: 0,
|
|
propagationPolicy: 'Foreground',
|
|
})
|
|
.catch(() => undefined),
|
|
);
|
|
}
|
|
}
|
|
for (const cm of configMaps.items) {
|
|
const name = cm.metadata?.name || '';
|
|
if (name.startsWith(prefix)) {
|
|
cleanup.push(coreApi.deleteNamespacedConfigMap({ name, namespace: buildNamespace }).catch(() => undefined));
|
|
}
|
|
}
|
|
|
|
await Promise.all(cleanup);
|
|
this.logger.log(`Cleaned up all build resources matching "${prefix}*" in ${buildNamespace}`);
|
|
}
|
|
|
|
async getProgress(deploymentId: string): Promise<BuildProgress | null> {
|
|
try {
|
|
const raw = await this.redis.get(this.progressKey(deploymentId));
|
|
return raw ? (JSON.parse(raw) as BuildProgress) : null;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Best-effort progress write. Kept synchronous (fire-and-forget) so the many
|
|
* call sites in the deploy pipeline don't need to await telemetry; a Redis
|
|
* blip must never fail a build.
|
|
*/
|
|
setProgress(deploymentId: string | undefined, progress: BuildProgress): void {
|
|
if (!deploymentId) return;
|
|
void this.redis
|
|
.set(this.progressKey(deploymentId), JSON.stringify(progress), 'EX', BuildService.STATE_TTL_SECONDS)
|
|
.catch(() => undefined);
|
|
}
|
|
|
|
async clearProgress(deploymentId: string): Promise<void> {
|
|
await this.redis.del(this.progressKey(deploymentId)).catch(() => undefined);
|
|
}
|
|
|
|
/**
|
|
* Builds a Docker image for the application using Kaniko inside K8s.
|
|
* Returns { imageUri, buildLog } — the full image URI and the build logs.
|
|
*/
|
|
async buildImage(app: Application, deploymentId?: string): Promise<{ imageUri: string; buildLog: string }> {
|
|
const registryUrl = this.registryService.getRegistryUrl();
|
|
const buildNamespace = this.registryService.getBuildNamespace();
|
|
const tag = `${Date.now()}`;
|
|
const imageUri = this.registryService.buildImageReference(app.userId, app.name, tag);
|
|
|
|
this.logger.log(`Starting image build for ${app.name} → ${imageUri}`);
|
|
|
|
if (deploymentId) {
|
|
this.beginBuildSession(deploymentId);
|
|
}
|
|
|
|
// Build mode:
|
|
// • templated → WordPress (and fresh installs): a generated Dockerfile is
|
|
// injected via ConfigMap (Nixpacks can't build a WordPress upload).
|
|
// • nixpacks → every code runtime: an init container picks the user's own
|
|
// Dockerfile if present (BYO), otherwise generates one with Nixpacks.
|
|
const useTemplated = app.runtime === AppRuntime.WORDPRESS;
|
|
const dockerfileContent = useTemplated ? this.wordpressDockerfile(app) : null;
|
|
|
|
// Create Kaniko build pod
|
|
const buildPodName = `build-${app.name}-${tag}`.substring(0, 63).replace(/[^a-z0-9-]/g, '');
|
|
|
|
if (deploymentId) {
|
|
this.updateBuildSession(deploymentId, { buildPodName });
|
|
}
|
|
|
|
// Use the cluster's kubeconfig instead of default
|
|
const cluster = app.clusterId ? await this.clustersService.findOne(app.clusterId) : await this.clustersService.getDefault();
|
|
|
|
const kc = new k8s.KubeConfig();
|
|
kc.loadFromString(cluster.kubeconfig);
|
|
const coreApi = kc.makeApiClient(k8s.CoreV1Api);
|
|
const batchApi = kc.makeApiClient(k8s.BatchV1Api);
|
|
|
|
if (deploymentId) {
|
|
this.updateBuildSession(deploymentId, {
|
|
coreApi,
|
|
batchApi,
|
|
namespace: buildNamespace,
|
|
clusterId: cluster.id,
|
|
});
|
|
}
|
|
|
|
// Ensure the build namespace exists
|
|
await this.ensureNamespace(coreApi, buildNamespace);
|
|
await this.throwIfCancelledShared(deploymentId);
|
|
|
|
// Determine source: uploaded code (MinIO object key in app.codePath) or git URL.
|
|
const hasUploadedCode = !!app.codePath;
|
|
const hasGitUrl = !!app.gitUrl;
|
|
|
|
// For private git repos, the token is delivered via a per-build Secret (env)
|
|
// and used through git's credential store inside the pod — never embedded in
|
|
// the clone URL/args or the Job manifest (which would leak it into etcd/logs).
|
|
let gitHost = '';
|
|
if (hasGitUrl) {
|
|
try {
|
|
gitHost = new URL(app.gitUrl!).host;
|
|
} catch {
|
|
/* malformed URL — fall back to inline injection below */
|
|
}
|
|
}
|
|
const useGitTokenSecret = hasGitUrl && !!app.gitToken && !!gitHost;
|
|
const gitSecretName = `${buildPodName}-git`;
|
|
|
|
// Create ConfigMap with Dockerfile
|
|
const dockerfileConfigMap = {
|
|
apiVersion: 'v1',
|
|
kind: 'ConfigMap',
|
|
metadata: {
|
|
name: `${buildPodName}-dockerfile`,
|
|
namespace: buildNamespace,
|
|
},
|
|
data: {
|
|
Dockerfile: dockerfileContent ?? '',
|
|
},
|
|
};
|
|
|
|
// For uploaded code, mint a short-lived presigned URL the build pod downloads
|
|
// from MinIO (replaces the PVC + helper pod + kubectl cp upload path).
|
|
let sourceDownloadUrl: string | undefined;
|
|
if (hasUploadedCode) {
|
|
sourceDownloadUrl = await this.storageService.presignSourceGet(app.codePath!);
|
|
}
|
|
|
|
// Build the Kaniko Job spec
|
|
// Always use dir context — init containers prepare /workspace/source
|
|
const kanikoArgs = [
|
|
'--dockerfile=/workspace/Dockerfile',
|
|
'--context=dir:///workspace/source',
|
|
`--destination=${imageUri}`,
|
|
'--cache=true',
|
|
`--cache-repo=${registryUrl}/${app.userId}/cache`,
|
|
'--insecure',
|
|
'--skip-tls-verify',
|
|
'--single-snapshot',
|
|
'--snapshot-mode=redo',
|
|
];
|
|
|
|
const volumes: any[] = [
|
|
{
|
|
name: 'docker-config',
|
|
secret: { secretName: 'registry-credentials' },
|
|
},
|
|
{
|
|
name: 'workspace',
|
|
emptyDir: {},
|
|
},
|
|
];
|
|
|
|
// The generated Dockerfile is only mounted (via ConfigMap) in templated mode.
|
|
if (useTemplated) {
|
|
volumes.push({ name: 'dockerfile', configMap: { name: `${buildPodName}-dockerfile` } });
|
|
}
|
|
|
|
// In templated mode each staging container copies the ConfigMap Dockerfile to
|
|
// /workspace/Dockerfile; in nixpacks mode the nixpacks-prepare container writes
|
|
// it instead, so staging just lays down the source.
|
|
const copyTemplatedDockerfile = useTemplated ? 'cp /workspace/Dockerfile /workspace-out/Dockerfile &&' : '';
|
|
const stagingDockerfileMounts = useTemplated
|
|
? [{ name: 'dockerfile', mountPath: '/workspace/Dockerfile', subPath: 'Dockerfile' }]
|
|
: [];
|
|
|
|
const initContainers: any[] = [];
|
|
|
|
if (hasUploadedCode && sourceDownloadUrl) {
|
|
// Download the source archive from MinIO via the presigned URL, then unzip
|
|
// it into /workspace/source (no PVC, no helper pod, no credentials in-pod).
|
|
initContainers.push({
|
|
name: 'fetch-source',
|
|
image: 'alpine:3.19',
|
|
imagePullPolicy: 'IfNotPresent',
|
|
env: [{ name: 'SOURCE_URL', value: sourceDownloadUrl }],
|
|
command: [
|
|
'sh',
|
|
'-c',
|
|
`
|
|
apk add --no-cache unzip tar gzip wget &&
|
|
${copyTemplatedDockerfile}
|
|
echo ">>> Downloading source archive from object storage..." &&
|
|
wget -q -O /tmp/source.zip "$SOURCE_URL" &&
|
|
mkdir -p /tmp/extract &&
|
|
cd /tmp/extract &&
|
|
if tar tzf /tmp/source.zip >/dev/null 2>&1; then
|
|
echo ">>> Detected gzip tarball" &&
|
|
tar xzf /tmp/source.zip
|
|
elif unzip -t /tmp/source.zip >/dev/null 2>&1; then
|
|
echo ">>> Detected zip archive" &&
|
|
unzip -q /tmp/source.zip
|
|
else
|
|
echo "ERROR: source archive is not a valid zip or tar.gz" && exit 1
|
|
fi &&
|
|
echo "--- Extracted contents ---" &&
|
|
ls -la /tmp/extract/ &&
|
|
mkdir -p /workspace-out/source &&
|
|
ITEMS=$(ls -1 /tmp/extract/ | head -5) &&
|
|
COUNT=$(ls -1 /tmp/extract/ | wc -l) &&
|
|
if [ "$COUNT" -eq 1 ] && [ -d "/tmp/extract/$ITEMS" ]; then
|
|
echo ">>> Single subfolder detected: $ITEMS — flattening to root" &&
|
|
cp -a /tmp/extract/$ITEMS/. /workspace-out/source/
|
|
else
|
|
echo ">>> Multiple items or files — copying as-is" &&
|
|
cp -a /tmp/extract/. /workspace-out/source/
|
|
fi &&
|
|
rm -rf /tmp/extract /tmp/source.zip &&
|
|
echo "--- Final workspace contents ---" &&
|
|
ls -la /workspace-out/source/
|
|
`,
|
|
],
|
|
volumeMounts: [
|
|
{ name: 'workspace', mountPath: '/workspace-out' },
|
|
...stagingDockerfileMounts,
|
|
],
|
|
});
|
|
} else if (hasGitUrl) {
|
|
const branch = app.gitBranch || 'main';
|
|
const gitCopyDockerfile = useTemplated ? 'cp /dockerfile/Dockerfile /workspace-out/Dockerfile &&' : '';
|
|
|
|
// Clone command. Three cases:
|
|
// • token + parseable host → token comes from $GIT_TOKEN (Secret env) via
|
|
// git's credential store; the clone URL stays token-free.
|
|
// • token + unparseable host (rare) → fall back to inline token injection.
|
|
// • no token (public repo) → plain clone.
|
|
let cloneCmd: string;
|
|
let gitEnv: any[] | undefined;
|
|
if (useGitTokenSecret) {
|
|
gitEnv = [{ name: 'GIT_TOKEN', valueFrom: { secretKeyRef: { name: gitSecretName, key: 'token' } } }];
|
|
cloneCmd =
|
|
`git config --global credential.helper store && ` +
|
|
`printf 'https://%s@%s\\n' "$GIT_TOKEN" '${gitHost}' > "$HOME/.git-credentials" && ` +
|
|
`chmod 600 "$HOME/.git-credentials" && ` +
|
|
`git clone --depth 1 --branch ${branch} '${app.gitUrl}' /workspace-out/source && ` +
|
|
`rm -f "$HOME/.git-credentials" &&`;
|
|
} else if (app.gitToken) {
|
|
const cloneUrl = app.gitUrl!.replace('https://', `https://${app.gitToken}@`);
|
|
cloneCmd = `git clone --depth 1 --branch ${branch} ${cloneUrl} /workspace-out/source &&`;
|
|
} else {
|
|
cloneCmd = `git clone --depth 1 --branch ${branch} '${app.gitUrl}' /workspace-out/source &&`;
|
|
}
|
|
|
|
// Clone git repo into /workspace/source
|
|
initContainers.push({
|
|
name: 'git-clone',
|
|
image: 'alpine/git:2.43.0',
|
|
imagePullPolicy: 'IfNotPresent',
|
|
env: gitEnv,
|
|
command: [
|
|
'sh',
|
|
'-c',
|
|
`
|
|
echo ">>> Cloning branch '${branch}' from ${app.gitUrl}" &&
|
|
${cloneCmd}
|
|
${gitCopyDockerfile}
|
|
echo ">>> Workspace contents:" &&
|
|
ls -la /workspace-out/source/
|
|
`,
|
|
],
|
|
volumeMounts: [
|
|
{ name: 'workspace', mountPath: '/workspace-out' },
|
|
...(useTemplated ? [{ name: 'dockerfile', mountPath: '/dockerfile' }] : []),
|
|
],
|
|
});
|
|
} else if (useTemplated) {
|
|
// No uploaded code and no git — only valid for templated fresh installs
|
|
// (e.g. fresh WordPress). Create empty source dir + copy Dockerfile.
|
|
initContainers.push({
|
|
name: 'prepare-workspace',
|
|
image: 'alpine:3.19',
|
|
imagePullPolicy: 'IfNotPresent',
|
|
command: [
|
|
'sh',
|
|
'-c',
|
|
`
|
|
mkdir -p /workspace-out/source &&
|
|
cp /dockerfile/Dockerfile /workspace-out/Dockerfile &&
|
|
echo ">>> Prepared empty workspace for fresh install" &&
|
|
ls -la /workspace-out/
|
|
`,
|
|
],
|
|
volumeMounts: [
|
|
{ name: 'workspace', mountPath: '/workspace-out' },
|
|
{ name: 'dockerfile', mountPath: '/dockerfile' },
|
|
],
|
|
});
|
|
}
|
|
|
|
// Nixpacks mode: after the source is staged, pick the user's Dockerfile (BYO)
|
|
// or generate one with Nixpacks, writing the result to /workspace/Dockerfile.
|
|
if (!useTemplated) {
|
|
initContainers.push(this.nixpacksPrepareInitContainer(app));
|
|
}
|
|
|
|
// Kaniko container volume mounts
|
|
const kanikoVolumeMounts: any[] = [
|
|
{ name: 'docker-config', mountPath: '/kaniko/.docker' },
|
|
{ name: 'workspace', mountPath: '/workspace' },
|
|
];
|
|
|
|
const buildJob: k8s.V1Job = {
|
|
apiVersion: 'batch/v1',
|
|
kind: 'Job',
|
|
metadata: {
|
|
name: buildPodName,
|
|
namespace: buildNamespace,
|
|
},
|
|
spec: {
|
|
backoffLimit: 1,
|
|
ttlSecondsAfterFinished: 300,
|
|
template: {
|
|
spec: {
|
|
serviceAccountName: this.configService.get<string>('build.serviceAccount'),
|
|
initContainers: initContainers.length > 0 ? initContainers : undefined,
|
|
containers: [
|
|
{
|
|
name: 'kaniko',
|
|
image: this.kanikoImage,
|
|
imagePullPolicy: 'IfNotPresent',
|
|
args: kanikoArgs,
|
|
volumeMounts: kanikoVolumeMounts,
|
|
resources: {
|
|
requests: { cpu: '500m', memory: '1Gi' },
|
|
limits: { cpu: '2', memory: '4Gi' },
|
|
},
|
|
},
|
|
],
|
|
restartPolicy: 'Never',
|
|
volumes,
|
|
},
|
|
},
|
|
},
|
|
};
|
|
|
|
try {
|
|
if (useTemplated) {
|
|
const t0 = Date.now();
|
|
await coreApi.createNamespacedConfigMap({
|
|
namespace: buildNamespace!,
|
|
body: dockerfileConfigMap,
|
|
});
|
|
this.logger.log(`[timing] ConfigMap created in ${Date.now() - t0}ms`);
|
|
}
|
|
|
|
// Per-build Secret holding the git token (mounted as $GIT_TOKEN env).
|
|
if (useGitTokenSecret) {
|
|
await coreApi.createNamespacedSecret({
|
|
namespace: buildNamespace!,
|
|
body: {
|
|
metadata: { name: gitSecretName, namespace: buildNamespace },
|
|
type: 'Opaque',
|
|
data: { token: Buffer.from(app.gitToken!).toString('base64') },
|
|
},
|
|
});
|
|
}
|
|
|
|
const t1 = Date.now();
|
|
await batchApi.createNamespacedJob({
|
|
namespace: buildNamespace!,
|
|
body: buildJob,
|
|
});
|
|
this.logger.log(`[timing] Job created in ${Date.now() - t1}ms`);
|
|
|
|
// Wait for build to complete
|
|
this.setProgress(deploymentId, {
|
|
phase: 'building',
|
|
percent: 15,
|
|
message: 'Building Docker image...',
|
|
});
|
|
const buildTimeout = this.configService.get<number>('build.timeoutSeconds') || 600;
|
|
await this.waitForJobCompletion(batchApi, coreApi, buildPodName, buildNamespace!, buildTimeout, deploymentId);
|
|
|
|
// Capture build logs on success
|
|
let buildLog = '';
|
|
try {
|
|
buildLog = await this.getBuildLogs(coreApi, buildPodName, buildNamespace!);
|
|
} catch {}
|
|
|
|
this.logger.log(`Build completed successfully: ${imageUri}`);
|
|
return { imageUri, buildLog };
|
|
} catch (error: any) {
|
|
if (error instanceof BuildCancelledError || error?.name === 'BuildCancelledError') {
|
|
throw error;
|
|
}
|
|
// Try to get build logs for debugging
|
|
let buildLog = '';
|
|
try {
|
|
buildLog = await this.getBuildLogs(coreApi, buildPodName, buildNamespace!);
|
|
this.logger.error(`Build logs for ${buildPodName}:\n${buildLog}`);
|
|
} catch {}
|
|
|
|
this.logger.error(`Build failed for ${app.name}:`, error.body || error.message);
|
|
const err = new Error(`Image build failed: ${error.body?.message || error.message}`);
|
|
(err as any).buildLog = buildLog;
|
|
throw err;
|
|
} finally {
|
|
// Clean up Dockerfile ConfigMap (templated mode only)
|
|
if (useTemplated) {
|
|
try {
|
|
await coreApi.deleteNamespacedConfigMap({
|
|
name: `${buildPodName}-dockerfile`,
|
|
namespace: buildNamespace!,
|
|
});
|
|
} catch (e: any) {
|
|
this.logger.warn(`Failed to clean up ConfigMap: ${e.message}`);
|
|
}
|
|
}
|
|
// Clean up the per-build git-token Secret.
|
|
if (useGitTokenSecret) {
|
|
try {
|
|
await coreApi.deleteNamespacedSecret({ name: gitSecretName, namespace: buildNamespace! });
|
|
} catch (e: any) {
|
|
this.logger.warn(`Failed to clean up git-token Secret: ${e.message}`);
|
|
}
|
|
}
|
|
this.endBuildSession(deploymentId);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Ensure the build namespace exists with all required resources
|
|
* (namespace, service account, registry-credentials secret).
|
|
*/
|
|
private async ensureNamespace(coreApi: k8s.CoreV1Api, namespace: string): Promise<void> {
|
|
// 1. Ensure namespace
|
|
try {
|
|
await coreApi.readNamespace({ name: namespace });
|
|
} catch (err: any) {
|
|
if (err.code === 404 || err.body?.code === 404) {
|
|
this.logger.log(`Namespace "${namespace}" not found — creating it`);
|
|
await coreApi.createNamespace({
|
|
body: { metadata: { name: namespace } },
|
|
});
|
|
} else {
|
|
throw err;
|
|
}
|
|
}
|
|
|
|
// 2. Ensure service account for Kaniko
|
|
const saName = this.configService.get<string>('build.serviceAccount') || 'kaniko-builder';
|
|
try {
|
|
await coreApi.readNamespacedServiceAccount({ name: saName, namespace });
|
|
} catch (err: any) {
|
|
if (err.code === 404 || err.body?.code === 404) {
|
|
this.logger.log(`ServiceAccount "${saName}" not found in "${namespace}" — creating it`);
|
|
await coreApi.createNamespacedServiceAccount({
|
|
namespace,
|
|
body: { metadata: { name: saName, namespace } },
|
|
});
|
|
} else {
|
|
throw err;
|
|
}
|
|
}
|
|
|
|
// 3. Ensure registry-credentials secret (docker config for Kaniko to push)
|
|
const registrySecretName = 'registry-credentials';
|
|
try {
|
|
await coreApi.readNamespacedSecret({
|
|
name: registrySecretName,
|
|
namespace,
|
|
});
|
|
} catch (err: any) {
|
|
if (err.code === 404 || err.body?.code === 404) {
|
|
this.logger.log(`Secret "${registrySecretName}" not found in "${namespace}" — creating it`);
|
|
await coreApi.createNamespacedSecret({
|
|
namespace,
|
|
body: {
|
|
metadata: { name: registrySecretName, namespace },
|
|
type: 'kubernetes.io/dockerconfigjson',
|
|
data: {
|
|
'.dockerconfigjson': Buffer.from(this.registryService.buildDockerConfigJson()).toString('base64'),
|
|
},
|
|
},
|
|
});
|
|
} else {
|
|
throw err;
|
|
}
|
|
}
|
|
}
|
|
|
|
/** Single-quote a string for safe inclusion in a `sh -c` command. */
|
|
private shellQuote(value: string): string {
|
|
return `'${value.replace(/'/g, `'\\''`)}'`;
|
|
}
|
|
|
|
/**
|
|
* NIXPACKS_* planning env (read by the nixpacks process itself) derived from the
|
|
* app's selected runtime version. Other versions are inferred by Nixpacks from
|
|
* project files (go.mod, .python-version, …), the Nixpacks-idiomatic way.
|
|
*/
|
|
private nixpacksPlanEnv(app: Application): { name: string; value: string }[] {
|
|
const env: { name: string; value: string }[] = [];
|
|
if (app.runtime === AppRuntime.NODEJS && app.runtimeVersion) {
|
|
env.push({ name: 'NIXPACKS_NODE_VERSION', value: String(app.runtimeVersion) });
|
|
}
|
|
if ((app.runtime === AppRuntime.PYTHON || app.runtime === AppRuntime.DJANGO) && app.runtimeVersion) {
|
|
env.push({ name: 'NIXPACKS_PYTHON_VERSION', value: String(app.runtimeVersion) });
|
|
}
|
|
return env;
|
|
}
|
|
|
|
/**
|
|
* Init container for nixpacks (non-WordPress) builds. Turns staged source at
|
|
* /workspace/source into a Dockerfile at /workspace/Dockerfile:
|
|
* • if the source ships its own Dockerfile → use it (BYO, full user control),
|
|
* • otherwise generate one with Nixpacks (`nixpacks build --out`).
|
|
* Mirror/proxy env from `build.nixpacksBuildEnv` is baked into the generated
|
|
* image so package installs in the Kaniko stage work behind the Iran network.
|
|
*/
|
|
private nixpacksPrepareInitContainer(app: Application): any {
|
|
const image = this.configService.get<string>('build.nixpacksImage') || 'ghcr.io/railwayapp/nixpacks:latest';
|
|
const buildEnv = this.configService.get<string[]>('build.nixpacksBuildEnv') || [];
|
|
const envFlags = buildEnv.map((kv) => `--env ${this.shellQuote(kv)}`).join(' ');
|
|
const planEnv = this.nixpacksPlanEnv(app);
|
|
|
|
return {
|
|
name: 'nixpacks-prepare',
|
|
image,
|
|
imagePullPolicy: 'IfNotPresent',
|
|
env: planEnv.length ? planEnv : undefined,
|
|
command: [
|
|
'sh',
|
|
'-c',
|
|
`
|
|
set -e
|
|
cd /workspace
|
|
if [ -f source/Dockerfile ]; then
|
|
echo ">>> Using user-provided Dockerfile (BYO)" &&
|
|
cp source/Dockerfile /workspace/Dockerfile
|
|
else
|
|
echo ">>> No Dockerfile found — generating with Nixpacks" &&
|
|
nixpacks build source --out source ${envFlags} &&
|
|
cp source/.nixpacks/Dockerfile /workspace/Dockerfile &&
|
|
echo "--- Generated Dockerfile ---" &&
|
|
cat /workspace/Dockerfile
|
|
fi
|
|
`,
|
|
],
|
|
volumeMounts: [{ name: 'workspace', mountPath: '/workspace' }],
|
|
};
|
|
}
|
|
|
|
private wordpressDockerfile(app: Application): string {
|
|
const wpVersion = app.runtimeVersion || '6.7';
|
|
const phpVersion = app.phpVersion || '8.3';
|
|
const hasUploadedCode = !!app.codePath;
|
|
|
|
return `FROM wordpress:${wpVersion}-php${phpVersion}-apache
|
|
|
|
# Install additional PHP extensions commonly needed by WordPress
|
|
RUN docker-php-ext-install opcache
|
|
|
|
# Enable Apache mod_rewrite for pretty permalinks
|
|
RUN a2enmod rewrite
|
|
|
|
# Increase PHP upload limits for WordPress media
|
|
RUN echo "upload_max_filesize = 64M\\npost_max_size = 64M\\nmax_execution_time = 300\\nmemory_limit = 256M" > /usr/local/etc/php/conf.d/uploads.ini
|
|
|
|
${
|
|
hasUploadedCode
|
|
? `# Copy user's custom WordPress files
|
|
COPY . /tmp/user-content
|
|
|
|
# Auto-detect: full public_html root (has wp-admin) vs wp-content only
|
|
# ── public_html mode ──
|
|
# wp-admin/ and wp-includes/ replace the base-image core so the user's
|
|
# exact WordPress version & patches are preserved.
|
|
# wp-content/ is staged in /usr/src/wordpress-user/ and merged into the
|
|
# PVC on first boot (same as migrate mode).
|
|
# wp-config.php is saved separately so docker-entrypoint.sh can still
|
|
# inject WORDPRESS_DB_* env-vars when no config exists yet.
|
|
# ── migrate mode (no wp-admin) ──
|
|
# Only wp-content + config files are processed.
|
|
RUN mkdir -p /usr/src/wordpress-user && \\
|
|
if [ -d /tmp/user-content/wp-admin ]; then \\
|
|
echo ">>> Full WordPress root (public_html) detected" && \\
|
|
echo ">>> Copying wp-admin/ to /var/www/html/" && \\
|
|
rm -rf /var/www/html/wp-admin && \\
|
|
cp -a /tmp/user-content/wp-admin /var/www/html/wp-admin && \\
|
|
echo ">>> Copying wp-includes/ to /var/www/html/" && \\
|
|
rm -rf /var/www/html/wp-includes && \\
|
|
cp -a /tmp/user-content/wp-includes /var/www/html/wp-includes && \\
|
|
echo ">>> Copying root PHP files (index.php, wp-login.php, ...)" && \\
|
|
find /tmp/user-content -maxdepth 1 -name "*.php" ! -name "wp-config.php" \\
|
|
-exec cp {} /var/www/html/ \\; 2>/dev/null || true && \\
|
|
echo ">>> Copying other root files/dirs (fonts, assets, etc.)" && \\
|
|
for item in /tmp/user-content/*; do \\
|
|
name=$(basename "$item"); \\
|
|
case "$name" in \\
|
|
wp-admin|wp-includes|wp-content|wp-config.php|.htaccess) ;; \\
|
|
*.php) ;; \\
|
|
*) \\
|
|
if [ -f "$item" ]; then \\
|
|
echo " root file: $name" && \\
|
|
cp "$item" /var/www/html/; \\
|
|
elif [ -d "$item" ]; then \\
|
|
echo " root dir: $name/" && \\
|
|
cp -a "$item" /var/www/html/; \\
|
|
fi ;; \\
|
|
esac; \\
|
|
done; \\
|
|
else \\
|
|
echo ">>> wp-content / config files only (migrate mode)"; \\
|
|
fi && \\
|
|
if [ -d /tmp/user-content/wp-content ]; then \\
|
|
echo ">>> Staging user wp-content (themes, plugins, uploads)..." && \\
|
|
cp -a /tmp/user-content/wp-content /usr/src/wordpress-user/wp-content; \\
|
|
elif [ -d /tmp/user-content/themes ] || [ -d /tmp/user-content/plugins ] || [ -d /tmp/user-content/uploads ]; then \\
|
|
echo ">>> Staging loose themes/plugins/uploads into wp-content..." && \\
|
|
mkdir -p /usr/src/wordpress-user/wp-content && \\
|
|
[ -d /tmp/user-content/themes ] && cp -a /tmp/user-content/themes /usr/src/wordpress-user/wp-content/ || true && \\
|
|
[ -d /tmp/user-content/plugins ] && cp -a /tmp/user-content/plugins /usr/src/wordpress-user/wp-content/ || true && \\
|
|
[ -d /tmp/user-content/uploads ] && cp -a /tmp/user-content/uploads /usr/src/wordpress-user/wp-content/ || true; \\
|
|
fi && \\
|
|
# wp-config.php is intentionally NOT copied — docker-entrypoint.sh generates it
|
|
# from WORDPRESS_DB_* env vars so credentials always match the deployed database.
|
|
if [ -f /tmp/user-content/.htaccess ]; then \\
|
|
echo ">>> Copying .htaccess" && \\
|
|
cp /tmp/user-content/.htaccess /var/www/html/.htaccess; \\
|
|
fi && \\
|
|
rm -rf /tmp/user-content && \\
|
|
echo ">>> WordPress user content staged"
|
|
|
|
# Custom entrypoint:
|
|
# 1. Merge staged wp-content into the PVC mount (every start — idempotent)
|
|
# 2. Hand off to official docker-entrypoint.sh which creates wp-config.php
|
|
# from WORDPRESS_DB_* env vars (never use uploaded wp-config with old credentials)
|
|
RUN { \\
|
|
echo '#!/bin/bash'; \\
|
|
echo 'set -e'; \\
|
|
echo ''; \\
|
|
echo '# ── Merge user wp-content into PVC ──'; \\
|
|
echo 'if [ -d /usr/src/wordpress-user/wp-content ]; then'; \\
|
|
echo ' echo ">>> Merging user wp-content into PVC..."'; \\
|
|
echo ' mkdir -p /var/www/html/wp-content'; \\
|
|
echo ' cp -a /usr/src/wordpress-user/wp-content/. /var/www/html/wp-content/'; \\
|
|
echo ' chown -R www-data:www-data /var/www/html/wp-content'; \\
|
|
echo ' echo ">>> User wp-content merged successfully"'; \\
|
|
echo 'fi'; \\
|
|
echo ''; \\
|
|
echo 'exec docker-entrypoint.sh apache2-foreground'; \\
|
|
} > /usr/local/bin/cloudhost-entrypoint.sh && chmod +x /usr/local/bin/cloudhost-entrypoint.sh
|
|
`
|
|
: `# Fresh install — no user content to merge
|
|
`
|
|
}
|
|
# Set proper ownership
|
|
RUN chown -R www-data:www-data /var/www/html
|
|
|
|
EXPOSE 80
|
|
${
|
|
hasUploadedCode
|
|
? `ENTRYPOINT ["cloudhost-entrypoint.sh"]
|
|
CMD []`
|
|
: `CMD ["apache2-foreground"]`
|
|
}
|
|
`;
|
|
}
|
|
|
|
/**
|
|
* Whether a K8s API error is a transient connectivity/availability blip that
|
|
* should be retried rather than failing the operation. Covers socket-level
|
|
* errors, request timeouts (incl. the apiserver "request did not complete
|
|
* within the allotted timeout" message), DNS hiccups and 5xx/429 responses.
|
|
*/
|
|
private isTransientK8sError(err: any): boolean {
|
|
const statusCode = err?.statusCode ?? err?.response?.statusCode ?? err?.body?.code;
|
|
if (typeof statusCode === 'number' && (statusCode >= 500 || statusCode === 429)) {
|
|
return true;
|
|
}
|
|
const haystack = [err?.code, err?.message, err?.body?.message, err?.cause?.code].filter(Boolean).join(' ');
|
|
return /ECONNRESET|ECONNREFUSED|ETIMEDOUT|ESOCKETTIMEDOUT|EPIPE|EAI_AGAIN|ENOTFOUND|ENETUNREACH|socket hang up|timed? ?out|allotted timeout|did not complete|Client network socket disconnected/i.test(
|
|
haystack,
|
|
);
|
|
}
|
|
|
|
private async waitForJobCompletion(batchApi: k8s.BatchV1Api, coreApi: k8s.CoreV1Api, jobName: string, namespace: string, timeoutSeconds: number, deploymentId?: string): Promise<void> {
|
|
const startTime = Date.now();
|
|
const timeoutMs = timeoutSeconds * 1000;
|
|
let lastLoggedStatus = '';
|
|
|
|
while (Date.now() - startTime < timeoutMs) {
|
|
await this.throwIfCancelledShared(deploymentId);
|
|
const elapsed = Date.now() - startTime;
|
|
const buildPercent = Math.min(90, 15 + Math.round((elapsed / timeoutMs) * 75));
|
|
this.setProgress(deploymentId, {
|
|
phase: 'building',
|
|
percent: buildPercent,
|
|
message: 'Building Docker image...',
|
|
});
|
|
// ── Check Job status (with retry for transient connection errors) ──
|
|
let job: k8s.V1Job;
|
|
try {
|
|
job = await batchApi.readNamespacedJob({ name: jobName, namespace });
|
|
} catch (pollErr: any) {
|
|
// The Kaniko job keeps running independently of these status polls.
|
|
// A single API blip (timeout, reset, 5xx, DNS) must NOT abort a build
|
|
// that is still progressing — just retry on the next poll tick.
|
|
if (this.isTransientK8sError(pollErr)) {
|
|
const detail = pollErr?.code || pollErr?.message || pollErr?.statusCode || 'unknown';
|
|
this.logger.warn(`Transient K8s API error polling job ${jobName}: ${detail} — retrying in 5s`);
|
|
await new Promise((r) => setTimeout(r, 5000));
|
|
continue;
|
|
}
|
|
throw pollErr;
|
|
}
|
|
const status = job.status;
|
|
|
|
if (status?.succeeded && status.succeeded > 0) {
|
|
this.logger.log(`Build job ${jobName} succeeded`);
|
|
return;
|
|
}
|
|
|
|
// Check if the Job has permanently failed (all retries exhausted)
|
|
const failedCondition = (status?.conditions || []).find((c) => c.type === 'Failed' && c.status === 'True');
|
|
if (failedCondition) {
|
|
const logs = await this.getBuildLogs(coreApi, jobName, namespace);
|
|
throw new Error(`Build job ${jobName} failed.\nLogs:\n${logs}`);
|
|
}
|
|
|
|
// Safety net: if failures exceed backoffLimit and no pod is still running
|
|
const backoffLimit = job.spec?.backoffLimit ?? 0;
|
|
const failedCount = status?.failed ?? 0;
|
|
if (failedCount > backoffLimit) {
|
|
// Double-check: are there still active pods?
|
|
const activePods = (status as any)?.active ?? 0;
|
|
if (activePods === 0) {
|
|
const logs = await this.getBuildLogs(coreApi, jobName, namespace);
|
|
throw new Error(`Build job ${jobName} failed: ${failedCount} failures exceeded backoffLimit=${backoffLimit}.\nLogs:\n${logs}`);
|
|
}
|
|
}
|
|
|
|
// Log intermediate pod failures (retries still available)
|
|
if (failedCount > 0) {
|
|
this.logger.warn(`Build job ${jobName}: ${failedCount} pod failure(s), backoffLimit=${backoffLimit} — retrying...`);
|
|
}
|
|
|
|
// ── Check Pod status for early failure detection ──
|
|
try {
|
|
const pods = await coreApi.listNamespacedPod({
|
|
namespace,
|
|
labelSelector: `job-name=${jobName}`,
|
|
});
|
|
|
|
for (const pod of pods.items) {
|
|
const podName = pod.metadata?.name || 'unknown';
|
|
const phase = pod.status?.phase;
|
|
|
|
// Check all container statuses (init + regular) for stuck states
|
|
const allStatuses = [...(pod.status?.initContainerStatuses || []), ...(pod.status?.containerStatuses || [])];
|
|
|
|
for (const cs of allStatuses) {
|
|
const waiting = cs.state?.waiting;
|
|
if (waiting?.reason) {
|
|
const reason = waiting.reason;
|
|
const msg = waiting.message || '';
|
|
|
|
// These are unrecoverable — fail fast instead of waiting 10 minutes
|
|
const fatalReasons = ['ErrImagePull', 'ImagePullBackOff', 'CreateContainerConfigError', 'InvalidImageName', 'CrashLoopBackOff'];
|
|
|
|
if (fatalReasons.includes(reason)) {
|
|
const logs = await this.getBuildLogs(coreApi, jobName, namespace);
|
|
throw new Error(`Build pod ${podName} stuck: ${reason} — ${msg}\nLogs:\n${logs}`);
|
|
}
|
|
|
|
// Log non-fatal waiting states periodically
|
|
const statusKey = `${podName}:${cs.name}:${reason}`;
|
|
if (statusKey !== lastLoggedStatus) {
|
|
this.logger.warn(`Pod ${podName} container "${cs.name}": ${reason} — ${msg}`);
|
|
lastLoggedStatus = statusKey;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Log phase changes
|
|
const phaseKey = `${podName}:${phase}`;
|
|
if (phaseKey !== lastLoggedStatus && phase !== 'Succeeded') {
|
|
this.logger.log(`Build pod ${podName}: phase=${phase}`);
|
|
lastLoggedStatus = phaseKey;
|
|
}
|
|
}
|
|
} catch (podErr: any) {
|
|
// Don't fail the whole build just because pod status check failed
|
|
if (podErr.message?.includes('stuck:') || podErr.message?.includes('Build pod')) {
|
|
throw podErr; // Re-throw our own fatal errors
|
|
}
|
|
this.logger.warn(`Could not check pod status: ${podErr.message}`);
|
|
}
|
|
|
|
// Wait 5 seconds before polling again
|
|
await new Promise((resolve) => setTimeout(resolve, 5000));
|
|
}
|
|
|
|
// Timeout — get logs for debugging
|
|
let logs = '';
|
|
try {
|
|
logs = await this.getBuildLogs(coreApi, jobName, namespace);
|
|
} catch {}
|
|
throw new Error(`Build job ${jobName} timed out after ${timeoutSeconds}s\nLogs:\n${logs}`);
|
|
}
|
|
|
|
/**
|
|
* Live build logs for an in-progress build, read straight from the running
|
|
* build pod (init + kaniko containers). Returns null when there is no active
|
|
* build session for this deployment (e.g. build already finished/cleaned up),
|
|
* so callers can fall back to the persisted build log.
|
|
*/
|
|
async getLiveBuildLog(deploymentId: string): Promise<string | null> {
|
|
const session = this.activeBuilds.get(deploymentId);
|
|
if (session?.coreApi && session.namespace && session.buildPodName) {
|
|
try {
|
|
return await this.getBuildLogs(session.coreApi, session.buildPodName, session.namespace);
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
// Build is running on another replica — read logs via the persisted session.
|
|
const persisted = await this.readPersistedSession(deploymentId);
|
|
if (persisted?.namespace && persisted.buildPodName) {
|
|
const apis = await this.makeClusterApis(persisted.clusterId);
|
|
if (apis) {
|
|
try {
|
|
return await this.getBuildLogs(apis.coreApi, persisted.buildPodName, persisted.namespace);
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
private async getBuildLogs(coreApi: k8s.CoreV1Api, jobName: string, namespace: string): Promise<string> {
|
|
try {
|
|
const pods = await coreApi.listNamespacedPod({
|
|
namespace,
|
|
labelSelector: `job-name=${jobName}`,
|
|
});
|
|
|
|
if (pods.items.length === 0) {
|
|
return 'No pods found for build job.';
|
|
}
|
|
|
|
const podName = pods.items[0].metadata?.name;
|
|
if (!podName) return 'Pod name not found.';
|
|
|
|
// Get logs from all containers (init + kaniko)
|
|
let allLogs = '';
|
|
const containers = [...(pods.items[0].spec?.initContainers || []), ...(pods.items[0].spec?.containers || [])];
|
|
|
|
for (const container of containers) {
|
|
try {
|
|
const logResponse = await coreApi.readNamespacedPodLog({
|
|
name: podName,
|
|
namespace,
|
|
container: container.name,
|
|
tailLines: 500,
|
|
});
|
|
allLogs += `\n--- ${container.name} ---\n${logResponse}`;
|
|
} catch {
|
|
allLogs += `\n--- ${container.name} --- (no logs available)`;
|
|
}
|
|
}
|
|
return allLogs;
|
|
} catch (e: any) {
|
|
return `Failed to retrieve logs: ${e.message}`;
|
|
}
|
|
}
|
|
}
|