fix: reliable source upload, build cancel, WordPress port 80 default

- Replace port-forward/netcat PVC upload with kubectl cp for integrity
- Add build cancellation API and session cleanup; deploy catches cancel
- Default port 80 for WordPress, PHP, and Laravel on create
- Build progress modal with cancel; Helm/K8s adjustments for deployments
- Update build and kubernetes specs

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
keyhan
2026-05-14 16:03:45 +03:30
parent 3d56a2cc5d
commit 0c0a6cd5be
12 changed files with 637 additions and 93 deletions
+380 -40
View File
@@ -3,7 +3,8 @@ import { ConfigService } from '@nestjs/config';
import * as k8s from '@kubernetes/client-node';
import * as fs from 'fs';
import * as path from 'path';
import { execFile } from 'child_process';
import { execFile, spawn, ChildProcess } from 'child_process';
import * as net from 'net';
import { promisify } from 'util';
import { Application } from '../applications/entities/application.entity';
import { AppRuntime } from '../common/enums';
@@ -11,20 +12,207 @@ import { ClustersService } from '../clusters/clusters.service';
const execFileAsync = promisify(execFile);
export class BuildCancelledError extends Error {
constructor() {
super('Build cancelled by user');
this.name = 'BuildCancelledError';
}
}
interface ActiveBuildSession {
cancelled: boolean;
coreApi?: k8s.CoreV1Api;
batchApi?: k8s.BatchV1Api;
namespace?: string;
buildPodName?: string;
sourcePvcName?: string;
helperPodName?: string;
processes: ChildProcess[];
socket?: net.Socket;
}
export interface BuildProgress {
phase: 'uploading' | 'building' | 'deploying' | 'done' | 'failed';
percent: number;
bytesUploaded?: number;
totalBytes?: number;
message?: string;
}
@Injectable()
export class BuildService {
private readonly logger = new Logger(BuildService.name);
private readonly progressMap = new Map<string, BuildProgress>();
private readonly activeBuilds = new Map<string, ActiveBuildSession>();
constructor(
private configService: ConfigService,
private clustersService: ClustersService,
) {}
private beginBuildSession(deploymentId: string): void {
this.activeBuilds.set(deploymentId, { cancelled: false, processes: [] });
}
private getSession(deploymentId?: string): ActiveBuildSession | undefined {
if (!deploymentId) return undefined;
return this.activeBuilds.get(deploymentId);
}
private updateBuildSession(deploymentId: string, update: Partial<ActiveBuildSession>): void {
const session = this.activeBuilds.get(deploymentId);
if (session) Object.assign(session, update);
}
private registerProcess(deploymentId: string | undefined, proc: ChildProcess): void {
const session = this.getSession(deploymentId);
if (!session) return;
session.processes.push(proc);
if (session.cancelled) {
try { proc.kill('SIGKILL'); } catch { /* ignore */ }
}
}
private registerSocket(deploymentId: string | undefined, socket: net.Socket): void {
const session = this.getSession(deploymentId);
if (!session) return;
if (session.socket) {
try { session.socket.destroy(); } catch { /* ignore */ }
}
session.socket = socket;
if (session.cancelled) {
try { socket.destroy(); } catch { /* ignore */ }
}
}
private throwIfCancelled(deploymentId?: string): void {
if (deploymentId && this.activeBuilds.get(deploymentId)?.cancelled) {
throw new BuildCancelledError();
}
}
private endBuildSession(deploymentId?: string): void {
if (deploymentId) this.activeBuilds.delete(deploymentId);
}
async cancelBuild(deploymentId: string): Promise<void> {
const session = this.activeBuilds.get(deploymentId);
if (!session) {
this.setProgress(deploymentId, { phase: 'failed', percent: 0, message: 'Cancelled by user' });
return;
}
session.cancelled = true;
this.logger.log(`Cancelling build for deployment ${deploymentId}`);
if (session.socket) {
try { session.socket.destroy(); } catch { /* ignore */ }
}
for (const proc of session.processes) {
try { proc.kill('SIGKILL'); } catch { /* ignore */ }
}
const { coreApi, batchApi, namespace, buildPodName, sourcePvcName, helperPodName } = session;
if (coreApi && namespace) {
const cleanup: Promise<unknown>[] = [];
if (helperPodName) {
cleanup.push(
coreApi.deleteNamespacedPod(helperPodName, namespace, undefined, undefined, 0).catch(() => undefined),
);
}
if (buildPodName && batchApi) {
cleanup.push(
batchApi.deleteNamespacedJob(buildPodName, namespace, undefined, undefined, 0, undefined, 'Foreground').catch(() => undefined),
);
}
if (sourcePvcName) {
cleanup.push(
coreApi.deleteNamespacedPersistentVolumeClaim(sourcePvcName, namespace).catch(() => undefined),
);
}
if (buildPodName) {
cleanup.push(
coreApi.deleteNamespacedConfigMap(`${buildPodName}-dockerfile`, namespace).catch(() => undefined),
);
}
await Promise.all(cleanup);
this.logger.log(`Cleaned up K8s build resources for deployment ${deploymentId}`);
}
this.setProgress(deploymentId, { phase: 'failed', percent: 0, message: 'Cancelled by user' });
this.activeBuilds.delete(deploymentId);
}
/** Delete all in-flight build artifacts for an app (helper pods, jobs, PVCs, configmaps). */
async cleanupBuildResourcesForApp(app: Application): Promise<void> {
const buildNamespace = this.configService.get<string>('build.namespace') || 'cloudhost-builds';
const prefix = `build-${app.name}-`;
const cluster = app.clusterId
? await this.clustersService.findOne(app.clusterId)
: await this.clustersService.getDefault();
const kc = new k8s.KubeConfig();
kc.loadFromString(cluster.kubeconfig);
const coreApi = kc.makeApiClient(k8s.CoreV1Api);
const batchApi = kc.makeApiClient(k8s.BatchV1Api);
const cleanup: Promise<unknown>[] = [];
const [pods, pvcs, jobs, configMaps] = await Promise.all([
coreApi.listNamespacedPod(buildNamespace),
coreApi.listNamespacedPersistentVolumeClaim(buildNamespace),
batchApi.listNamespacedJob(buildNamespace),
coreApi.listNamespacedConfigMap(buildNamespace),
]);
for (const pod of pods.body.items) {
const name = pod.metadata?.name || '';
if (name.startsWith(prefix)) {
cleanup.push(coreApi.deleteNamespacedPod(name, buildNamespace, undefined, undefined, 0).catch(() => undefined));
}
}
for (const pvc of pvcs.body.items) {
const name = pvc.metadata?.name || '';
if (name.startsWith(prefix)) {
cleanup.push(coreApi.deleteNamespacedPersistentVolumeClaim(name, buildNamespace).catch(() => undefined));
}
}
for (const job of jobs.body.items) {
const name = job.metadata?.name || '';
if (name.startsWith(prefix)) {
cleanup.push(batchApi.deleteNamespacedJob(name, buildNamespace, undefined, undefined, 0, undefined, 'Foreground').catch(() => undefined));
}
}
for (const cm of configMaps.body.items) {
const name = cm.metadata?.name || '';
if (name.startsWith(prefix)) {
cleanup.push(coreApi.deleteNamespacedConfigMap(name, buildNamespace).catch(() => undefined));
}
}
await Promise.all(cleanup);
this.logger.log(`Cleaned up all build resources matching "${prefix}*" in ${buildNamespace}`);
}
getProgress(deploymentId: string): BuildProgress | null {
return this.progressMap.get(deploymentId) ?? null;
}
setProgress(deploymentId: string | undefined, progress: BuildProgress): void {
if (!deploymentId) return;
this.progressMap.set(deploymentId, progress);
}
clearProgress(deploymentId: string): void {
this.progressMap.delete(deploymentId);
}
/**
* Builds a Docker image for the application using Kaniko inside K8s.
* Returns { imageUri, buildLog } — the full image URI and the build logs.
*/
async buildImage(app: Application): Promise<{ imageUri: string; buildLog: string }> {
async buildImage(app: Application, deploymentId?: string): Promise<{ imageUri: string; buildLog: string }> {
// Internal registry (used by Kaniko inside K8s for pushing)
const internalRegistryUrl = this.configService.get<string>('registry.url') || 'registry.cloudhost-builds.svc.cluster.local:5000';
// External registry URL (used by kubelet for pulling — NodePort or external)
@@ -36,12 +224,20 @@ export class BuildService {
this.logger.log(`Starting image build for ${app.name} → push: ${pushImageUri}, pull: ${pullImageUri}`);
if (deploymentId) {
this.beginBuildSession(deploymentId);
}
// Determine Dockerfile based on runtime
const dockerfileContent = this.generateDockerfile(app);
// Create Kaniko build pod
const buildPodName = `build-${app.name}-${tag}`.substring(0, 63).replace(/[^a-z0-9-]/g, '');
if (deploymentId) {
this.updateBuildSession(deploymentId, { buildPodName });
}
// Use the cluster's kubeconfig instead of default
const cluster = app.clusterId
? await this.clustersService.findOne(app.clusterId)
@@ -52,8 +248,13 @@ export class BuildService {
const coreApi = kc.makeApiClient(k8s.CoreV1Api);
const batchApi = kc.makeApiClient(k8s.BatchV1Api);
if (deploymentId) {
this.updateBuildSession(deploymentId, { coreApi, batchApi, namespace: buildNamespace });
}
// Ensure the build namespace exists
await this.ensureNamespace(coreApi, buildNamespace);
this.throwIfCancelled(deploymentId);
// Determine if we have uploaded code or git URL
const codePath = app.codePath ? path.resolve(app.codePath) : null;
@@ -77,12 +278,15 @@ export class BuildService {
let sourcePvcName: string | undefined;
if (hasUploadedCode) {
sourcePvcName = `${buildPodName}-source`;
if (deploymentId) {
this.updateBuildSession(deploymentId, { sourcePvcName });
}
const zipSize = fs.statSync(codePath!).size;
// Allocate PVC size = zip size * 3 (zip + extracted), min 1Gi
const pvcSizeGi = Math.max(1, Math.ceil((zipSize * 3) / (1024 * 1024 * 1024)));
await this.uploadSourceViaPVC(
kc, coreApi, buildNamespace!, sourcePvcName, codePath!, pvcSizeGi,
kc, coreApi, buildNamespace!, sourcePvcName, codePath!, pvcSizeGi, deploymentId,
);
}
@@ -131,11 +335,19 @@ export class BuildService {
name: 'unzip-source',
image: 'alpine:3.19',
command: ['sh', '-c', `
apk add --no-cache unzip &&
apk add --no-cache unzip tar gzip &&
cp /workspace/Dockerfile /workspace-out/Dockerfile &&
mkdir -p /tmp/extract &&
cd /tmp/extract &&
unzip /source-pvc/source.zip &&
if tar tzf /source-pvc/source.zip >/dev/null 2>&1; then
echo ">>> Detected gzip tarball" &&
tar xzf /source-pvc/source.zip
elif unzip -t /source-pvc/source.zip >/dev/null 2>&1; then
echo ">>> Detected zip archive" &&
unzip -q /source-pvc/source.zip
else
echo "ERROR: source archive is not a valid zip or tar.gz" && exit 1
fi &&
echo "--- Extracted contents ---" &&
ls -la /tmp/extract/ &&
mkdir -p /workspace-out/source &&
@@ -263,7 +475,8 @@ export class BuildService {
this.logger.log(`[timing] Job created in ${Date.now() - t1}ms`);
// Wait for build to complete
await this.waitForJobCompletion(batchApi, coreApi, buildPodName, buildNamespace!, 600);
this.setProgress(deploymentId, { phase: 'building', percent: 15, message: 'Building Docker image...' });
await this.waitForJobCompletion(batchApi, coreApi, buildPodName, buildNamespace!, 600, deploymentId);
// Capture build logs on success
let buildLog = '';
@@ -274,6 +487,9 @@ export class BuildService {
this.logger.log(`Build completed successfully: ${pullImageUri}`);
return { imageUri: pullImageUri, buildLog };
} catch (error: any) {
if (error instanceof BuildCancelledError || error?.name === 'BuildCancelledError') {
throw error;
}
// Try to get build logs for debugging
let buildLog = '';
try {
@@ -301,11 +517,115 @@ export class BuildService {
} catch (e: any) {
this.logger.warn(`Failed to clean up ConfigMap: ${e.message}`);
}
this.endBuildSession(deploymentId);
}
}
/**
* Upload source zip to K8s via PVC + helper pod + kubectl cp.
* Poll the helper pod until netcat finishes writing and reports the expected byte count.
*/
private async waitForRemoteUploadDone(
kubeconfig: string,
namespace: string,
podName: string,
expectedSize: number,
deploymentId?: string,
): Promise<void> { /* unused — kept for API compat */ }
/**
* Upload a local file to the helper pod using kubectl cp with progress tracking.
* kubectl cp uses tar over the k8s exec API — reliable for any file size.
*/
private streamFileToHelperPod(
kubeconfig: string,
namespace: string,
podName: string,
filePath: string,
fileSize: number,
deploymentId?: string,
): Promise<void> {
const maxAttempts = 3;
const runOnce = () => new Promise<void>((resolve, reject) => {
this.throwIfCancelled(deploymentId);
const kubectl = spawn('kubectl', [
'--kubeconfig', kubeconfig,
'cp', filePath, `${namespace}/${podName}:/data/source.zip`,
'-c', 'helper',
'--retries', '3',
], { stdio: ['ignore', 'pipe', 'pipe'] });
this.registerProcess(deploymentId, kubectl);
let stderr = '';
kubectl.stderr.on('data', (chunk: Buffer) => { stderr += chunk.toString(); });
// Track progress by polling remote file size
let progressTimer: NodeJS.Timeout | undefined;
const pollProgress = () => {
execFileAsync('kubectl', [
'--kubeconfig', kubeconfig,
'exec', '-n', namespace, podName, '-c', 'helper', '--',
'sh', '-c', 'wc -c < /data/source.zip 2>/dev/null || echo 0',
], { timeout: 10_000 }).then(({ stdout }) => {
const remoteSize = parseInt(stdout.trim(), 10) || 0;
const percent = Math.min(99, Math.round((remoteSize / fileSize) * 100));
this.setProgress(deploymentId, {
phase: 'uploading',
percent,
bytesUploaded: remoteSize,
totalBytes: fileSize,
message: `Uploading to cluster... ${percent}%`,
});
}).catch(() => { /* polling failure is non-fatal */ });
};
progressTimer = setInterval(pollProgress, 3000);
pollProgress();
kubectl.on('error', (err) => {
clearInterval(progressTimer);
reject(new Error(`kubectl cp spawn error: ${err.message}`));
});
kubectl.on('close', (code) => {
clearInterval(progressTimer);
if (code === 0) resolve();
else reject(new Error(`kubectl cp failed (code ${code}): ${stderr.trim()}`));
});
});
return (async () => {
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
try {
this.throwIfCancelled(deploymentId);
if (attempt > 1) {
this.logger.warn(`Retrying source upload (attempt ${attempt}/${maxAttempts})...`);
await execFileAsync('kubectl', [
'--kubeconfig', kubeconfig,
'exec', '-n', namespace, podName, '-c', 'helper', '--',
'rm', '-f', '/data/source.zip',
], { timeout: 15_000 }).catch(() => undefined);
this.setProgress(deploymentId, {
phase: 'uploading',
percent: 0,
bytesUploaded: 0,
totalBytes: fileSize,
message: `Retrying upload (attempt ${attempt})...`,
});
}
await runOnce();
return;
} catch (err) {
if (err instanceof BuildCancelledError || (err as Error)?.name === 'BuildCancelledError') throw err;
if (attempt === maxAttempts) throw err;
this.logger.warn(`Upload attempt ${attempt} failed: ${(err as Error).message}`);
}
}
})();
}
/**
* Upload source zip to K8s via PVC + helper pod.
* This handles files of any size (unlike Secret/ConfigMap which are limited to ~1MB).
*/
private async uploadSourceViaPVC(
@@ -315,11 +635,16 @@ export class BuildService {
pvcName: string,
zipPath: string,
sizeGi: number,
deploymentId?: string,
): Promise<void> {
const t0 = Date.now();
const helperPodName = `${pvcName}-helper`;
const zipSize = fs.statSync(zipPath).size;
if (deploymentId) {
this.updateBuildSession(deploymentId, { helperPodName });
}
this.logger.log(`Uploading source via PVC (${(zipSize / 1024 / 1024).toFixed(1)} MB) → ${pvcName}`);
// 1. Create PVC
@@ -366,6 +691,7 @@ export class BuildService {
const podTimeout = 120_000; // 2 minutes
const podStart = Date.now();
while (Date.now() - podStart < podTimeout) {
this.throwIfCancelled(deploymentId);
const pod = await coreApi.readNamespacedPod(helperPodName, namespace);
const phase = pod.body.status?.phase;
if (phase === 'Running') break;
@@ -385,35 +711,44 @@ export class BuildService {
fs.writeFileSync(tmpKubeconfig, kcYaml);
try {
// 5. Copy the zip into the helper pod via kubectl cp.
// Both k8s.Exec (WebSocket) and kubectl exec -i stdin piping are unreliable
// for binary transfers — data can be lost before the connection is ready.
// kubectl cp uses tar internally and handles connection timing correctly.
// 5. Upload the zip via kubectl cp (tar-based, reliable for any size).
const t2 = Date.now();
this.setProgress(deploymentId, {
phase: 'uploading',
percent: 0,
bytesUploaded: 0,
totalBytes: zipSize,
message: 'Uploading source to cluster...',
});
await execFileAsync('kubectl', [
'--kubeconfig', tmpKubeconfig,
'cp', zipPath,
`${namespace}/${helperPodName}:/data/source.zip`,
'-c', 'helper',
], { timeout: 1200_000 });
await this.streamFileToHelperPod(
tmpKubeconfig, namespace, helperPodName, zipPath, zipSize, deploymentId,
);
this.logger.log(`[timing] kubectl cp upload completed in ${Date.now() - t2}ms (${(zipSize / 1024 / 1024).toFixed(1)} MB)`);
this.logger.log(`[timing] Source stream upload completed in ${Date.now() - t2}ms (${(zipSize / 1024 / 1024).toFixed(1)} MB)`);
this.setProgress(deploymentId, {
phase: 'uploading',
percent: 100,
bytesUploaded: zipSize,
totalBytes: zipSize,
message: 'Upload complete, verifying...',
});
// 5b. Verify the file was written correctly
// 5b. Verify the file was written correctly (exact size)
const { stdout: sizeStr } = await execFileAsync('kubectl', [
'--kubeconfig', tmpKubeconfig,
'exec', '-n', namespace, helperPodName,
'exec', '-n', namespace, helperPodName, '-c', 'helper',
'--', 'sh', '-c', 'wc -c < /data/source.zip',
], { timeout: 30_000 });
const remoteSize = parseInt(sizeStr.trim(), 10);
if (isNaN(remoteSize) || remoteSize < zipSize * 0.95) {
if (isNaN(remoteSize) || remoteSize !== zipSize) {
throw new Error(
`Source upload incomplete: expected ${zipSize} bytes but got ${remoteSize} bytes on remote. ` +
`(${(zipSize / 1024 / 1024).toFixed(1)} MB expected, ${(remoteSize / 1024 / 1024).toFixed(1)} MB received)`,
);
}
this.logger.log(`[verify] Remote file size: ${remoteSize} bytes (expected ${zipSize}) ✓`);
} finally {
// Clean up temp kubeconfig
@@ -799,11 +1134,15 @@ RUN mkdir -p /usr/src/wordpress-user && \\
if [ -d /tmp/user-content/wp-content ]; then \\
echo ">>> Staging user wp-content (themes, plugins, uploads)..." && \\
cp -a /tmp/user-content/wp-content /usr/src/wordpress-user/wp-content; \\
elif [ -d /tmp/user-content/themes ] || [ -d /tmp/user-content/plugins ] || [ -d /tmp/user-content/uploads ]; then \\
echo ">>> Staging loose themes/plugins/uploads into wp-content..." && \\
mkdir -p /usr/src/wordpress-user/wp-content && \\
[ -d /tmp/user-content/themes ] && cp -a /tmp/user-content/themes /usr/src/wordpress-user/wp-content/ || true && \\
[ -d /tmp/user-content/plugins ] && cp -a /tmp/user-content/plugins /usr/src/wordpress-user/wp-content/ || true && \\
[ -d /tmp/user-content/uploads ] && cp -a /tmp/user-content/uploads /usr/src/wordpress-user/wp-content/ || true; \\
fi && \\
if [ -f /tmp/user-content/wp-config.php ]; then \\
echo ">>> Saving custom wp-config.php" && \\
cp /tmp/user-content/wp-config.php /usr/src/wordpress-user/wp-config.php; \\
fi && \\
# wp-config.php is intentionally NOT copied — docker-entrypoint.sh generates it
# from WORDPRESS_DB_* env vars so credentials always match the deployed database.
if [ -f /tmp/user-content/.htaccess ]; then \\
echo ">>> Copying .htaccess" && \\
cp /tmp/user-content/.htaccess /var/www/html/.htaccess; \\
@@ -812,30 +1151,22 @@ RUN mkdir -p /usr/src/wordpress-user && \\
echo ">>> WordPress user content staged"
# Custom entrypoint:
# 1. Merge staged wp-content into the PVC mount on first run
# 2. Optionally apply user's wp-config.php (saved during build)
# 3. Hand off to the official docker-entrypoint.sh which handles
# WORDPRESS_DB_* env-var injection and first-run setup
# 1. Merge staged wp-content into the PVC mount (every start — idempotent)
# 2. Hand off to official docker-entrypoint.sh which creates wp-config.php
# from WORDPRESS_DB_* env vars (never use uploaded wp-config with old credentials)
RUN { \\
echo '#!/bin/bash'; \\
echo 'set -e'; \\
echo ''; \\
echo '# ── Merge user wp-content into PVC (first run only) ──'; \\
echo 'if [ -d /usr/src/wordpress-user/wp-content ] && [ ! -f /var/www/html/wp-content/.user-content-merged ]; then'; \\
echo ' echo ">>> First run: merging user wp-content into PVC..."'; \\
echo '# ── Merge user wp-content into PVC ──'; \\
echo 'if [ -d /usr/src/wordpress-user/wp-content ]; then'; \\
echo ' echo ">>> Merging user wp-content into PVC..."'; \\
echo ' mkdir -p /var/www/html/wp-content'; \\
echo ' cp -a /usr/src/wordpress-user/wp-content/. /var/www/html/wp-content/'; \\
echo ' touch /var/www/html/wp-content/.user-content-merged'; \\
echo ' chown -R www-data:www-data /var/www/html/wp-content'; \\
echo ' echo ">>> User wp-content merged successfully"'; \\
echo 'fi'; \\
echo ''; \\
echo '# ── Apply user wp-config.php if docker-entrypoint has not created one yet ──'; \\
echo 'if [ -f /usr/src/wordpress-user/wp-config.php ] && [ ! -f /var/www/html/wp-config.php ]; then'; \\
echo ' echo ">>> Applying user wp-config.php"'; \\
echo ' cp /usr/src/wordpress-user/wp-config.php /var/www/html/wp-config.php'; \\
echo ' chown www-data:www-data /var/www/html/wp-config.php'; \\
echo 'fi'; \\
echo ''; \\
echo 'exec docker-entrypoint.sh apache2-foreground'; \\
} > /usr/local/bin/cloudhost-entrypoint.sh && chmod +x /usr/local/bin/cloudhost-entrypoint.sh
` : `# Fresh install — no user content to merge
@@ -1135,12 +1466,21 @@ CMD ["sh", "-c", "DLL=$(find . -maxdepth 1 -name '*.dll' ! -name '*.deps.dll' !
jobName: string,
namespace: string,
timeoutSeconds: number,
deploymentId?: string,
): Promise<void> {
const startTime = Date.now();
const timeoutMs = timeoutSeconds * 1000;
let lastLoggedStatus = '';
while (Date.now() - startTime < timeoutMs) {
this.throwIfCancelled(deploymentId);
const elapsed = Date.now() - startTime;
const buildPercent = Math.min(90, 15 + Math.round((elapsed / timeoutMs) * 75));
this.setProgress(deploymentId, {
phase: 'building',
percent: buildPercent,
message: 'Building Docker image...',
});
// ── Check Job status (with retry for transient connection errors) ──
let job: { body: k8s.V1Job };
try {