diff --git a/backend/src/kubernetes/kubernetes.service.ts b/backend/src/kubernetes/kubernetes.service.ts index cd9c2f8..5ac4071 100644 --- a/backend/src/kubernetes/kubernetes.service.ts +++ b/backend/src/kubernetes/kubernetes.service.ts @@ -1285,6 +1285,8 @@ export class KubernetesService implements OnModuleInit { /** * Export (dump) the application database to a local file via a K8s Job. * Returns the dump as a Buffer, or null on failure. + * + * Strategy: Run dump command, then sleep for 60s to allow exec retrieval. */ async exportDatabaseDump(app: Application): Promise<{ data: Buffer | null; logs: string }> { const { coreApi, kc } = await this.getK8sClient(app.clusterId); @@ -1299,17 +1301,18 @@ export class KubernetesService implements OnModuleInit { const dbVer = app.dbVersion || defaultDbVer; const image = isPostgres ? `postgres:${dbVer}-alpine` : `mysql:${dbVer}`; - // Dump command writes to /dump/output.sql inside an emptyDir volume + // Dump command writes to /dump/output.sql, then sleeps to allow exec retrieval const command = isPostgres - ? ['sh', '-c', `PGPASSWORD="$DB_PASSWORD" pg_dump -h ${dbName} -U "$DB_USER" -d ${dbDatabase} --no-owner --no-acl > /dump/output.sql 2>/tmp/err; cat /tmp/err >&2; echo "DUMP_DONE"`] - : ['sh', '-c', `mysqldump -h ${dbName} -u "$DB_USER" -p"$DB_PASSWORD" ${dbDatabase} > /dump/output.sql 2>/tmp/err; cat /tmp/err >&2; echo "DUMP_DONE"`]; + ? ['sh', '-c', `PGPASSWORD="$DB_PASSWORD" pg_dump -h ${dbName} -U "$DB_USER" -d ${dbDatabase} --no-owner --no-acl > /dump/output.sql 2>/tmp/err; cat /tmp/err >&2; echo "DUMP_DONE"; sleep 120`] + : ['sh', '-c', `mysqldump -h ${dbName} -u "$DB_USER" -p"$DB_PASSWORD" ${dbDatabase} > /dump/output.sql 2>/tmp/err; cat /tmp/err >&2; echo "DUMP_DONE"; sleep 120`]; const job: k8s.V1Job = { apiVersion: 'batch/v1', kind: 'Job', metadata: { name: jobName, namespace }, spec: { - ttlSecondsAfterFinished: 120, + ttlSecondsAfterFinished: 180, + activeDeadlineSeconds: 300, backoffLimit: 0, template: { spec: { @@ -1338,83 +1341,92 @@ export class KubernetesService implements OnModuleInit { return { data: null, logs: `Failed to create dump job: ${e.message}` }; } - // Wait for completion (max 5 min) + // Wait for dump to complete (check logs for DUMP_DONE marker) const timeout = 300_000; const start = Date.now(); - let succeeded = false; + let dumpDone = false; + let podName: string | undefined; while (Date.now() - start < timeout) { await new Promise((r) => setTimeout(r, 3000)); try { - const st = await batchApi.readNamespacedJob(jobName, namespace); - if (st.body.status?.succeeded && st.body.status.succeeded > 0) { succeeded = true; break; } - if (st.body.status?.failed && st.body.status.failed > 0) break; + const pods = await coreApi.listNamespacedPod(namespace, undefined, undefined, undefined, undefined, `job-name=${jobName}`); + if (pods.body.items.length > 0) { + podName = pods.body.items[0].metadata?.name; + const phase = pods.body.items[0].status?.phase; + + // Check if pod is Running (container is in sleep phase after dump) + if (podName && phase === 'Running') { + try { + const logRes = await coreApi.readNamespacedPodLog(podName, namespace, 'dump', false, undefined, undefined, undefined, undefined, undefined, 50); + if (logRes.body?.includes('DUMP_DONE')) { + dumpDone = true; + break; + } + } catch {} + } + + // If Failed, exit early + if (phase === 'Failed') break; + } } catch {} } - // Get dump by exec-ing into the pod and cat-ing the file let dumpBuffer: Buffer | null = null; let logs = ''; - try { - const pods = await coreApi.listNamespacedPod(namespace, undefined, undefined, undefined, undefined, `job-name=${jobName}`); - if (pods.body.items.length > 0) { - const podName = pods.body.items[0].metadata?.name; - if (podName && succeeded) { - // Use exec to cat the dump file from the pod - const exec = new k8s.Exec(kc); - const chunks: Buffer[] = []; - - await new Promise((resolve, reject) => { - exec.exec( - namespace, podName, 'dump', - ['cat', '/dump/output.sql'], - { - write: (data: string) => { chunks.push(Buffer.from(data)); }, - } as any, - null, - { - write: (data: string) => { logs += data; }, - } as any, - false, - (status: k8s.V1Status) => { - if (status.status === 'Success') resolve(); - else reject(new Error(status.message || 'exec failed')); - }, - ); - }).catch(() => { - this.logger.warn(`Exec cat failed for ${podName}, trying readNamespacedPodLog`); - }); + if (dumpDone && podName) { + try { + // Use kubectl cp equivalent via Exec + const exec = new k8s.Exec(kc); + const chunks: Buffer[] = []; + + await new Promise((resolve, reject) => { + exec.exec( + namespace, podName!, 'dump', + ['cat', '/dump/output.sql'], + { + write: (data: string) => { chunks.push(Buffer.from(data, 'binary')); }, + } as any, + null, + { + write: (data: string) => { logs += data; }, + } as any, + false, + (status: k8s.V1Status) => { + if (status.status === 'Success') resolve(); + else reject(new Error(status.message || 'exec failed')); + }, + ); + }); - if (chunks.length > 0) { - dumpBuffer = Buffer.concat(chunks); - } - } - - // Fallback: get logs - if (!dumpBuffer && pods.body.items[0].metadata?.name) { - try { - const logRes = await coreApi.readNamespacedPodLog(pods.body.items[0].metadata.name, namespace); - logs = logRes.body || ''; - } catch {} + if (chunks.length > 0) { + dumpBuffer = Buffer.concat(chunks); + this.logger.log(`DB dump retrieved: ${dumpBuffer.length} bytes`); } + } catch (e: any) { + this.logger.warn(`Exec failed for DB dump: ${e.message}`); + logs = e.message; } - } catch (e: any) { - this.logger.warn(`Could not retrieve dump: ${e.message}`); - logs = e.message; } - if (!succeeded) { - return { data: null, logs: logs || 'Dump job failed or timed out' }; + // Cleanup: delete the job early to free resources + try { + await batchApi.deleteNamespacedJob(jobName, namespace, undefined, undefined, undefined, undefined, 'Background'); + } catch {} + + if (!dumpBuffer) { + return { data: null, logs: logs || 'Dump failed or could not be retrieved' }; } - return { data: dumpBuffer, logs }; + return { data: dumpBuffer, logs: 'OK' }; } /** * Archive the wp-content directory from a WordPress app's PVC via a K8s Job. * The job creates a tar.gz of /var/www/html/wp-content and we retrieve it via exec. - * Returns the archive as a Buffer, or null on failure. + * + * Strategy: Create archive, then sleep to allow exec retrieval. */ async archiveWpContent(app: Application): Promise<{ data: Buffer | null; logs: string }> { const { coreApi, kc } = await this.getK8sClient(app.clusterId); @@ -1428,7 +1440,8 @@ export class KubernetesService implements OnModuleInit { kind: 'Job', metadata: { name: jobName, namespace }, spec: { - ttlSecondsAfterFinished: 120, + ttlSecondsAfterFinished: 180, + activeDeadlineSeconds: 300, backoffLimit: 0, template: { spec: { @@ -1436,12 +1449,12 @@ export class KubernetesService implements OnModuleInit { containers: [{ name: 'archiver', image: 'alpine:3.19', - command: ['sh', '-c', 'apk add --no-cache tar gzip > /dev/null 2>&1 && cd /wp-content && tar czf /output/wp-content.tar.gz . && echo "ARCHIVE_DONE"'], + command: ['sh', '-c', 'apk add --no-cache tar gzip > /dev/null 2>&1 && cd /wp-content && tar czf /output/wp-content.tar.gz . && echo "ARCHIVE_DONE" && sleep 120'], volumeMounts: [ { name: 'wp-content', mountPath: '/wp-content', readOnly: true }, { name: 'output', mountPath: '/output' }, ], - resources: { requests: { cpu: '100m', memory: '64Mi' }, limits: { cpu: '500m', memory: '256Mi' } }, + resources: { requests: { cpu: '100m', memory: '64Mi' }, limits: { cpu: '500m', memory: '512Mi' } }, }], volumes: [ { name: 'wp-content', persistentVolumeClaim: { claimName: pvcName } }, @@ -1459,65 +1472,82 @@ export class KubernetesService implements OnModuleInit { return { data: null, logs: `Failed to create archive job: ${e.message}` }; } - // Wait for completion + // Wait for archive to complete (check logs for ARCHIVE_DONE marker) const timeout = 300_000; const start = Date.now(); - let succeeded = false; + let archiveDone = false; + let podName: string | undefined; + while (Date.now() - start < timeout) { await new Promise((r) => setTimeout(r, 3000)); try { - const st = await batchApi.readNamespacedJob(jobName, namespace); - if (st.body.status?.succeeded && st.body.status.succeeded > 0) { succeeded = true; break; } - if (st.body.status?.failed && st.body.status.failed > 0) break; + const pods = await coreApi.listNamespacedPod(namespace, undefined, undefined, undefined, undefined, `job-name=${jobName}`); + if (pods.body.items.length > 0) { + podName = pods.body.items[0].metadata?.name; + const phase = pods.body.items[0].status?.phase; + + if (podName && phase === 'Running') { + try { + const logRes = await coreApi.readNamespacedPodLog(podName, namespace, 'archiver', false, undefined, undefined, undefined, undefined, undefined, 50); + if (logRes.body?.includes('ARCHIVE_DONE')) { + archiveDone = true; + break; + } + } catch {} + } + + if (phase === 'Failed') break; + } } catch {} } let archiveBuffer: Buffer | null = null; let logs = ''; - if (succeeded) { + if (archiveDone && podName) { try { - const pods = await coreApi.listNamespacedPod(namespace, undefined, undefined, undefined, undefined, `job-name=${jobName}`); - if (pods.body.items.length > 0) { - const podName = pods.body.items[0].metadata?.name; - if (podName) { - const exec = new k8s.Exec(kc); - const chunks: Buffer[] = []; + const exec = new k8s.Exec(kc); + const chunks: Buffer[] = []; - await new Promise((resolve, reject) => { - exec.exec( - namespace, podName, 'archiver', - ['cat', '/output/wp-content.tar.gz'], - { - write: (data: string) => { chunks.push(Buffer.from(data)); }, - } as any, - null, - { - write: (data: string) => { logs += data; }, - } as any, - false, - (status: k8s.V1Status) => { - if (status.status === 'Success') resolve(); - else reject(new Error(status.message || 'exec failed')); - }, - ); - }).catch((err) => { - this.logger.warn(`Exec failed for wp-content archive: ${err.message}`); - }); + await new Promise((resolve, reject) => { + exec.exec( + namespace, podName!, 'archiver', + ['cat', '/output/wp-content.tar.gz'], + { + write: (data: string) => { chunks.push(Buffer.from(data, 'binary')); }, + } as any, + null, + { + write: (data: string) => { logs += data; }, + } as any, + false, + (status: k8s.V1Status) => { + if (status.status === 'Success') resolve(); + else reject(new Error(status.message || 'exec failed')); + }, + ); + }); - if (chunks.length > 0) { - archiveBuffer = Buffer.concat(chunks); - } - } + if (chunks.length > 0) { + archiveBuffer = Buffer.concat(chunks); + this.logger.log(`wp-content archive retrieved: ${archiveBuffer.length} bytes`); } } catch (e: any) { + this.logger.warn(`Exec failed for wp-content archive: ${e.message}`); logs = e.message; } - } else { - logs = 'Archive job failed or timed out'; } - return { data: archiveBuffer, logs }; + // Cleanup + try { + await batchApi.deleteNamespacedJob(jobName, namespace, undefined, undefined, undefined, undefined, 'Background'); + } catch {} + + if (!archiveBuffer) { + return { data: null, logs: logs || 'Archive failed or could not be retrieved' }; + } + + return { data: archiveBuffer, logs: 'OK' }; } /**