abrban/cloud-host
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(snapshots): add sleep after dump/archive for exec retrieval

Browse Source 
- Job containers now sleep 120s after completing dump/archive
- This allows exec to retrieve files before container exits
- Wait for DUMP_DONE/ARCHIVE_DONE marker before attempting exec
- Cleanup job immediately after retrieval
- Fix Helm registry-pull-secret ownership conflict with lookup
This commit is contained in:
keyhan
2026-04-22 18:49:03 +03:30
parent b8355f1fc1
commit f481a57d8f
1 changed files with 129 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/kubernetes/kubernetes.service.ts
+129 -99
View File
@@ -1285,6 +1285,8 @@ export class KubernetesService implements OnModuleInit {
/** /**
* Export (dump) the application database to a local file via a K8s Job. * Export (dump) the application database to a local file via a K8s Job.
* Returns the dump as a Buffer, or null on failure. * Returns the dump as a Buffer, or null on failure.
*
* Strategy: Run dump command, then sleep for 60s to allow exec retrieval.
*/ */
async exportDatabaseDump(app: Application): Promise<{ data: Buffer | null; logs: string }> { async exportDatabaseDump(app: Application): Promise<{ data: Buffer | null; logs: string }> {
const { coreApi, kc } = await this.getK8sClient(app.clusterId); const { coreApi, kc } = await this.getK8sClient(app.clusterId);
@@ -1299,17 +1301,18 @@ export class KubernetesService implements OnModuleInit {
const dbVer = app.dbVersion || defaultDbVer; const dbVer = app.dbVersion || defaultDbVer;
const image = isPostgres ? `postgres:${dbVer}-alpine` : `mysql:${dbVer}`; const image = isPostgres ? `postgres:${dbVer}-alpine` : `mysql:${dbVer}`;
// Dump command writes to /dump/output.sql inside an emptyDir volume // Dump command writes to /dump/output.sql, then sleeps to allow exec retrieval
const command = isPostgres const command = isPostgres
? ['sh', '-c', `PGPASSWORD="$DB_PASSWORD" pg_dump -h ${dbName} -U "$DB_USER" -d ${dbDatabase} --no-owner --no-acl > /dump/output.sql 2>/tmp/err; cat /tmp/err >&2; echo "DUMP_DONE"`] ? ['sh', '-c', `PGPASSWORD="$DB_PASSWORD" pg_dump -h ${dbName} -U "$DB_USER" -d ${dbDatabase} --no-owner --no-acl > /dump/output.sql 2>/tmp/err; cat /tmp/err >&2; echo "DUMP_DONE"; sleep 120`]
: ['sh', '-c', `mysqldump -h ${dbName} -u "$DB_USER" -p"$DB_PASSWORD" ${dbDatabase} > /dump/output.sql 2>/tmp/err; cat /tmp/err >&2; echo "DUMP_DONE"`]; : ['sh', '-c', `mysqldump -h ${dbName} -u "$DB_USER" -p"$DB_PASSWORD" ${dbDatabase} > /dump/output.sql 2>/tmp/err; cat /tmp/err >&2; echo "DUMP_DONE"; sleep 120`];
const job: k8s.V1Job = { const job: k8s.V1Job = {
apiVersion: 'batch/v1', apiVersion: 'batch/v1',
kind: 'Job', kind: 'Job',
metadata: { name: jobName, namespace }, metadata: { name: jobName, namespace },
spec: { spec: {
ttlSecondsAfterFinished: 120, ttlSecondsAfterFinished: 180,
activeDeadlineSeconds: 300,
backoffLimit: 0, backoffLimit: 0,
template: { template: {
spec: { spec: {
@@ -1338,83 +1341,92 @@ export class KubernetesService implements OnModuleInit {
return { data: null, logs: `Failed to create dump job: ${e.message}` }; return { data: null, logs: `Failed to create dump job: ${e.message}` };
} }
// Wait for completion (max 5 min) // Wait for dump to complete (check logs for DUMP_DONE marker)
const timeout = 300_000; const timeout = 300_000;
const start = Date.now(); const start = Date.now();
let succeeded = false; let dumpDone = false;
let podName: string | undefined;
while (Date.now() - start < timeout) { while (Date.now() - start < timeout) {
await new Promise((r) => setTimeout(r, 3000)); await new Promise((r) => setTimeout(r, 3000));
try { try {
const st = await batchApi.readNamespacedJob(jobName, namespace); const pods = await coreApi.listNamespacedPod(namespace, undefined, undefined, undefined, undefined, `job-name=${jobName}`);
if (st.body.status?.succeeded && st.body.status.succeeded > 0) { succeeded = true; break; } if (pods.body.items.length > 0) {
if (st.body.status?.failed && st.body.status.failed > 0) break; podName = pods.body.items[0].metadata?.name;
const phase = pods.body.items[0].status?.phase;
// Check if pod is Running (container is in sleep phase after dump)
if (podName && phase === 'Running') {
try {
const logRes = await coreApi.readNamespacedPodLog(podName, namespace, 'dump', false, undefined, undefined, undefined, undefined, undefined, 50);
if (logRes.body?.includes('DUMP_DONE')) {
dumpDone = true;
break;
}
} catch {}
}
// If Failed, exit early
if (phase === 'Failed') break;
}
} catch {} } catch {}
} }
// Get dump by exec-ing into the pod and cat-ing the file
let dumpBuffer: Buffer | null = null; let dumpBuffer: Buffer | null = null;
let logs = ''; let logs = '';
try { if (dumpDone && podName) {
const pods = await coreApi.listNamespacedPod(namespace, undefined, undefined, undefined, undefined, `job-name=${jobName}`); try {
if (pods.body.items.length > 0) { // Use kubectl cp equivalent via Exec
const podName = pods.body.items[0].metadata?.name; const exec = new k8s.Exec(kc);
if (podName && succeeded) { const chunks: Buffer[] = [];
// Use exec to cat the dump file from the pod
const exec = new k8s.Exec(kc); await new Promise<void>((resolve, reject) => {
const chunks: Buffer[] = []; exec.exec(
namespace, podName!, 'dump',
await new Promise<void>((resolve, reject) => { ['cat', '/dump/output.sql'],
exec.exec( {
namespace, podName, 'dump', write: (data: string) => { chunks.push(Buffer.from(data, 'binary')); },
['cat', '/dump/output.sql'], } as any,
{ null,
write: (data: string) => { chunks.push(Buffer.from(data)); }, {
} as any, write: (data: string) => { logs += data; },
null, } as any,
{ false,
write: (data: string) => { logs += data; }, (status: k8s.V1Status) => {
} as any, if (status.status === 'Success') resolve();
false, else reject(new Error(status.message || 'exec failed'));
(status: k8s.V1Status) => { },
if (status.status === 'Success') resolve(); );
else reject(new Error(status.message || 'exec failed')); });
},
);
}).catch(() => {
this.logger.warn(`Exec cat failed for ${podName}, trying readNamespacedPodLog`);
});
if (chunks.length > 0) { if (chunks.length > 0) {
dumpBuffer = Buffer.concat(chunks); dumpBuffer = Buffer.concat(chunks);
} this.logger.log(`DB dump retrieved: ${dumpBuffer.length} bytes`);
}
// Fallback: get logs
if (!dumpBuffer && pods.body.items[0].metadata?.name) {
try {
const logRes = await coreApi.readNamespacedPodLog(pods.body.items[0].metadata.name, namespace);
logs = logRes.body || '';
} catch {}
} }
} catch (e: any) {
this.logger.warn(`Exec failed for DB dump: ${e.message}`);
logs = e.message;
} }
} catch (e: any) {
this.logger.warn(`Could not retrieve dump: ${e.message}`);
logs = e.message;
} }
if (!succeeded) { // Cleanup: delete the job early to free resources
return { data: null, logs: logs || 'Dump job failed or timed out' }; try {
await batchApi.deleteNamespacedJob(jobName, namespace, undefined, undefined, undefined, undefined, 'Background');
} catch {}
if (!dumpBuffer) {
return { data: null, logs: logs || 'Dump failed or could not be retrieved' };
} }
return { data: dumpBuffer, logs }; return { data: dumpBuffer, logs: 'OK' };
} }
/** /**
* Archive the wp-content directory from a WordPress app's PVC via a K8s Job. * Archive the wp-content directory from a WordPress app's PVC via a K8s Job.
* The job creates a tar.gz of /var/www/html/wp-content and we retrieve it via exec. * The job creates a tar.gz of /var/www/html/wp-content and we retrieve it via exec.
* Returns the archive as a Buffer, or null on failure. *
* Strategy: Create archive, then sleep to allow exec retrieval.
*/ */
async archiveWpContent(app: Application): Promise<{ data: Buffer | null; logs: string }> { async archiveWpContent(app: Application): Promise<{ data: Buffer | null; logs: string }> {
const { coreApi, kc } = await this.getK8sClient(app.clusterId); const { coreApi, kc } = await this.getK8sClient(app.clusterId);
@@ -1428,7 +1440,8 @@ export class KubernetesService implements OnModuleInit {
kind: 'Job', kind: 'Job',
metadata: { name: jobName, namespace }, metadata: { name: jobName, namespace },
spec: { spec: {
ttlSecondsAfterFinished: 120, ttlSecondsAfterFinished: 180,
activeDeadlineSeconds: 300,
backoffLimit: 0, backoffLimit: 0,
template: { template: {
spec: { spec: {
@@ -1436,12 +1449,12 @@ export class KubernetesService implements OnModuleInit {
containers: [{ containers: [{
name: 'archiver', name: 'archiver',
image: 'alpine:3.19', image: 'alpine:3.19',
command: ['sh', '-c', 'apk add --no-cache tar gzip > /dev/null 2>&1 && cd /wp-content && tar czf /output/wp-content.tar.gz . && echo "ARCHIVE_DONE"'], command: ['sh', '-c', 'apk add --no-cache tar gzip > /dev/null 2>&1 && cd /wp-content && tar czf /output/wp-content.tar.gz . && echo "ARCHIVE_DONE" && sleep 120'],
volumeMounts: [ volumeMounts: [
{ name: 'wp-content', mountPath: '/wp-content', readOnly: true }, { name: 'wp-content', mountPath: '/wp-content', readOnly: true },
{ name: 'output', mountPath: '/output' }, { name: 'output', mountPath: '/output' },
], ],
resources: { requests: { cpu: '100m', memory: '64Mi' }, limits: { cpu: '500m', memory: '256Mi' } }, resources: { requests: { cpu: '100m', memory: '64Mi' }, limits: { cpu: '500m', memory: '512Mi' } },
}], }],
volumes: [ volumes: [
{ name: 'wp-content', persistentVolumeClaim: { claimName: pvcName } }, { name: 'wp-content', persistentVolumeClaim: { claimName: pvcName } },
@@ -1459,65 +1472,82 @@ export class KubernetesService implements OnModuleInit {
return { data: null, logs: `Failed to create archive job: ${e.message}` }; return { data: null, logs: `Failed to create archive job: ${e.message}` };
} }
// Wait for completion // Wait for archive to complete (check logs for ARCHIVE_DONE marker)
const timeout = 300_000; const timeout = 300_000;
const start = Date.now(); const start = Date.now();
let succeeded = false; let archiveDone = false;
let podName: string | undefined;
while (Date.now() - start < timeout) { while (Date.now() - start < timeout) {
await new Promise((r) => setTimeout(r, 3000)); await new Promise((r) => setTimeout(r, 3000));
try { try {
const st = await batchApi.readNamespacedJob(jobName, namespace); const pods = await coreApi.listNamespacedPod(namespace, undefined, undefined, undefined, undefined, `job-name=${jobName}`);
if (st.body.status?.succeeded && st.body.status.succeeded > 0) { succeeded = true; break; } if (pods.body.items.length > 0) {
if (st.body.status?.failed && st.body.status.failed > 0) break; podName = pods.body.items[0].metadata?.name;
const phase = pods.body.items[0].status?.phase;
if (podName && phase === 'Running') {
try {
const logRes = await coreApi.readNamespacedPodLog(podName, namespace, 'archiver', false, undefined, undefined, undefined, undefined, undefined, 50);
if (logRes.body?.includes('ARCHIVE_DONE')) {
archiveDone = true;
break;
}
} catch {}
}
if (phase === 'Failed') break;
}
} catch {} } catch {}
} }
let archiveBuffer: Buffer | null = null; let archiveBuffer: Buffer | null = null;
let logs = ''; let logs = '';
if (succeeded) { if (archiveDone && podName) {
try { try {
const pods = await coreApi.listNamespacedPod(namespace, undefined, undefined, undefined, undefined, `job-name=${jobName}`); const exec = new k8s.Exec(kc);
if (pods.body.items.length > 0) { const chunks: Buffer[] = [];
const podName = pods.body.items[0].metadata?.name;
if (podName) {
const exec = new k8s.Exec(kc);
const chunks: Buffer[] = [];
await new Promise<void>((resolve, reject) => { await new Promise<void>((resolve, reject) => {
exec.exec( exec.exec(
namespace, podName, 'archiver', namespace, podName!, 'archiver',
['cat', '/output/wp-content.tar.gz'], ['cat', '/output/wp-content.tar.gz'],
{ {
write: (data: string) => { chunks.push(Buffer.from(data)); }, write: (data: string) => { chunks.push(Buffer.from(data, 'binary')); },
} as any, } as any,
null, null,
{ {
write: (data: string) => { logs += data; }, write: (data: string) => { logs += data; },
} as any, } as any,
false, false,
(status: k8s.V1Status) => { (status: k8s.V1Status) => {
if (status.status === 'Success') resolve(); if (status.status === 'Success') resolve();
else reject(new Error(status.message || 'exec failed')); else reject(new Error(status.message || 'exec failed'));
}, },
); );
}).catch((err) => { });
this.logger.warn(`Exec failed for wp-content archive: ${err.message}`);
});
if (chunks.length > 0) { if (chunks.length > 0) {
archiveBuffer = Buffer.concat(chunks); archiveBuffer = Buffer.concat(chunks);
} this.logger.log(`wp-content archive retrieved: ${archiveBuffer.length} bytes`);
}
} }
} catch (e: any) { } catch (e: any) {
this.logger.warn(`Exec failed for wp-content archive: ${e.message}`);
logs = e.message; logs = e.message;
} }
} else {
logs = 'Archive job failed or timed out';
} }
return { data: archiveBuffer, logs }; // Cleanup
try {
await batchApi.deleteNamespacedJob(jobName, namespace, undefined, undefined, undefined, undefined, 'Background');
} catch {}
if (!archiveBuffer) {
return { data: null, logs: logs || 'Archive failed or could not be retrieved' };
}
return { data: archiveBuffer, logs: 'OK' };
} }
/** /**