abrban/cloud-host
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(snapshots): add sleep after dump/archive for exec retrieval

Browse Source 
- Job containers now sleep 120s after completing dump/archive
- This allows exec to retrieve files before container exits
- Wait for DUMP_DONE/ARCHIVE_DONE marker before attempting exec
- Cleanup job immediately after retrieval
- Fix Helm registry-pull-secret ownership conflict with lookup
This commit is contained in:
keyhan
2026-04-22 18:49:03 +03:30
parent b8355f1fc1
commit f481a57d8f
1 changed files with 129 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/kubernetes/kubernetes.service.ts
+87 -57
View File
@@ -1285,6 +1285,8 @@ export class KubernetesService implements OnModuleInit {
/**
* Export (dump) the application database to a local file via a K8s Job.
* Returns the dump as a Buffer, or null on failure.
*
* Strategy: Run dump command, then sleep for 60s to allow exec retrieval.
*/
async exportDatabaseDump(app: Application): Promise<{ data: Buffer | null; logs: string }> {
const { coreApi, kc } = await this.getK8sClient(app.clusterId);
@@ -1299,17 +1301,18 @@ export class KubernetesService implements OnModuleInit {
const dbVer = app.dbVersion || defaultDbVer;
const image = isPostgres ? `postgres:${dbVer}-alpine` : `mysql:${dbVer}`;
// Dump command writes to /dump/output.sql inside an emptyDir volume
// Dump command writes to /dump/output.sql, then sleeps to allow exec retrieval
const command = isPostgres
? ['sh', '-c', `PGPASSWORD="$DB_PASSWORD" pg_dump -h ${dbName} -U "$DB_USER" -d ${dbDatabase} --no-owner --no-acl > /dump/output.sql 2>/tmp/err; cat /tmp/err >&2; echo "DUMP_DONE"`]
: ['sh', '-c', `mysqldump -h ${dbName} -u "$DB_USER" -p"$DB_PASSWORD" ${dbDatabase} > /dump/output.sql 2>/tmp/err; cat /tmp/err >&2; echo "DUMP_DONE"`];
? ['sh', '-c', `PGPASSWORD="$DB_PASSWORD" pg_dump -h ${dbName} -U "$DB_USER" -d ${dbDatabase} --no-owner --no-acl > /dump/output.sql 2>/tmp/err; cat /tmp/err >&2; echo "DUMP_DONE"; sleep 120`]
: ['sh', '-c', `mysqldump -h ${dbName} -u "$DB_USER" -p"$DB_PASSWORD" ${dbDatabase} > /dump/output.sql 2>/tmp/err; cat /tmp/err >&2; echo "DUMP_DONE"; sleep 120`];
const job: k8s.V1Job = {
apiVersion: 'batch/v1',
kind: 'Job',
metadata: { name: jobName, namespace },
spec: {
ttlSecondsAfterFinished: 120,
ttlSecondsAfterFinished: 180,
activeDeadlineSeconds: 300,
backoffLimit: 0,
template: {
spec: {
@@ -1338,39 +1341,52 @@ export class KubernetesService implements OnModuleInit {
return { data: null, logs: `Failed to create dump job: ${e.message}` };
}
// Wait for completion (max 5 min)
// Wait for dump to complete (check logs for DUMP_DONE marker)
const timeout = 300_000;
const start = Date.now();
let succeeded = false;
let dumpDone = false;
let podName: string | undefined;
while (Date.now() - start < timeout) {
await new Promise((r) => setTimeout(r, 3000));
try {
const st = await batchApi.readNamespacedJob(jobName, namespace);
if (st.body.status?.succeeded && st.body.status.succeeded > 0) { succeeded = true; break; }
if (st.body.status?.failed && st.body.status.failed > 0) break;
const pods = await coreApi.listNamespacedPod(namespace, undefined, undefined, undefined, undefined, `job-name=${jobName}`);
if (pods.body.items.length > 0) {
podName = pods.body.items[0].metadata?.name;
const phase = pods.body.items[0].status?.phase;
// Check if pod is Running (container is in sleep phase after dump)
if (podName && phase === 'Running') {
try {
const logRes = await coreApi.readNamespacedPodLog(podName, namespace, 'dump', false, undefined, undefined, undefined, undefined, undefined, 50);
if (logRes.body?.includes('DUMP_DONE')) {
dumpDone = true;
break;
}
} catch {}
}
// If Failed, exit early
if (phase === 'Failed') break;
}
} catch {}
}
// Get dump by exec-ing into the pod and cat-ing the file
let dumpBuffer: Buffer | null = null;
let logs = '';
if (dumpDone && podName) {
try {
const pods = await coreApi.listNamespacedPod(namespace, undefined, undefined, undefined, undefined, `job-name=${jobName}`);
if (pods.body.items.length > 0) {
const podName = pods.body.items[0].metadata?.name;
if (podName && succeeded) {
// Use exec to cat the dump file from the pod
// Use kubectl cp equivalent via Exec
const exec = new k8s.Exec(kc);
const chunks: Buffer[] = [];
await new Promise<void>((resolve, reject) => {
exec.exec(
namespace, podName, 'dump',
namespace, podName!, 'dump',
['cat', '/dump/output.sql'],
{
write: (data: string) => { chunks.push(Buffer.from(data)); },
write: (data: string) => { chunks.push(Buffer.from(data, 'binary')); },
} as any,
null,
{
@@ -1382,39 +1398,35 @@ export class KubernetesService implements OnModuleInit {
else reject(new Error(status.message || 'exec failed'));
},
);
}).catch(() => {
this.logger.warn(`Exec cat failed for ${podName}, trying readNamespacedPodLog`);
});
if (chunks.length > 0) {
dumpBuffer = Buffer.concat(chunks);
}
}
// Fallback: get logs
if (!dumpBuffer && pods.body.items[0].metadata?.name) {
try {
const logRes = await coreApi.readNamespacedPodLog(pods.body.items[0].metadata.name, namespace);
logs = logRes.body || '';
} catch {}
}
this.logger.log(`DB dump retrieved: ${dumpBuffer.length} bytes`);
}
} catch (e: any) {
this.logger.warn(`Could not retrieve dump: ${e.message}`);
this.logger.warn(`Exec failed for DB dump: ${e.message}`);
logs = e.message;
}
if (!succeeded) {
return { data: null, logs: logs || 'Dump job failed or timed out' };
}
return { data: dumpBuffer, logs };
// Cleanup: delete the job early to free resources
try {
await batchApi.deleteNamespacedJob(jobName, namespace, undefined, undefined, undefined, undefined, 'Background');
} catch {}
if (!dumpBuffer) {
return { data: null, logs: logs || 'Dump failed or could not be retrieved' };
}
return { data: dumpBuffer, logs: 'OK' };
}
/**
* Archive the wp-content directory from a WordPress app's PVC via a K8s Job.
* The job creates a tar.gz of /var/www/html/wp-content and we retrieve it via exec.
* Returns the archive as a Buffer, or null on failure.
*
* Strategy: Create archive, then sleep to allow exec retrieval.
*/
async archiveWpContent(app: Application): Promise<{ data: Buffer | null; logs: string }> {
const { coreApi, kc } = await this.getK8sClient(app.clusterId);
@@ -1428,7 +1440,8 @@ export class KubernetesService implements OnModuleInit {
kind: 'Job',
metadata: { name: jobName, namespace },
spec: {
ttlSecondsAfterFinished: 120,
ttlSecondsAfterFinished: 180,
activeDeadlineSeconds: 300,
backoffLimit: 0,
template: {
spec: {
@@ -1436,12 +1449,12 @@ export class KubernetesService implements OnModuleInit {
containers: [{
name: 'archiver',
image: 'alpine:3.19',
command: ['sh', '-c', 'apk add --no-cache tar gzip > /dev/null 2>&1 && cd /wp-content && tar czf /output/wp-content.tar.gz . && echo "ARCHIVE_DONE"'],
command: ['sh', '-c', 'apk add --no-cache tar gzip > /dev/null 2>&1 && cd /wp-content && tar czf /output/wp-content.tar.gz . && echo "ARCHIVE_DONE" && sleep 120'],
volumeMounts: [
{ name: 'wp-content', mountPath: '/wp-content', readOnly: true },
{ name: 'output', mountPath: '/output' },
],
resources: { requests: { cpu: '100m', memory: '64Mi' }, limits: { cpu: '500m', memory: '256Mi' } },
resources: { requests: { cpu: '100m', memory: '64Mi' }, limits: { cpu: '500m', memory: '512Mi' } },
}],
volumes: [
{ name: 'wp-content', persistentVolumeClaim: { claimName: pvcName } },
@@ -1459,37 +1472,49 @@ export class KubernetesService implements OnModuleInit {
return { data: null, logs: `Failed to create archive job: ${e.message}` };
}
// Wait for completion
// Wait for archive to complete (check logs for ARCHIVE_DONE marker)
const timeout = 300_000;
const start = Date.now();
let succeeded = false;
let archiveDone = false;
let podName: string | undefined;
while (Date.now() - start < timeout) {
await new Promise((r) => setTimeout(r, 3000));
try {
const st = await batchApi.readNamespacedJob(jobName, namespace);
if (st.body.status?.succeeded && st.body.status.succeeded > 0) { succeeded = true; break; }
if (st.body.status?.failed && st.body.status.failed > 0) break;
const pods = await coreApi.listNamespacedPod(namespace, undefined, undefined, undefined, undefined, `job-name=${jobName}`);
if (pods.body.items.length > 0) {
podName = pods.body.items[0].metadata?.name;
const phase = pods.body.items[0].status?.phase;
if (podName && phase === 'Running') {
try {
const logRes = await coreApi.readNamespacedPodLog(podName, namespace, 'archiver', false, undefined, undefined, undefined, undefined, undefined, 50);
if (logRes.body?.includes('ARCHIVE_DONE')) {
archiveDone = true;
break;
}
} catch {}
}
if (phase === 'Failed') break;
}
} catch {}
}
let archiveBuffer: Buffer | null = null;
let logs = '';
if (succeeded) {
if (archiveDone && podName) {
try {
const pods = await coreApi.listNamespacedPod(namespace, undefined, undefined, undefined, undefined, `job-name=${jobName}`);
if (pods.body.items.length > 0) {
const podName = pods.body.items[0].metadata?.name;
if (podName) {
const exec = new k8s.Exec(kc);
const chunks: Buffer[] = [];
await new Promise<void>((resolve, reject) => {
exec.exec(
namespace, podName, 'archiver',
namespace, podName!, 'archiver',
['cat', '/output/wp-content.tar.gz'],
{
write: (data: string) => { chunks.push(Buffer.from(data)); },
write: (data: string) => { chunks.push(Buffer.from(data, 'binary')); },
} as any,
null,
{
@@ -1501,23 +1526,28 @@ export class KubernetesService implements OnModuleInit {
else reject(new Error(status.message || 'exec failed'));
},
);
}).catch((err) => {
this.logger.warn(`Exec failed for wp-content archive: ${err.message}`);
});
if (chunks.length > 0) {
archiveBuffer = Buffer.concat(chunks);
}
}
this.logger.log(`wp-content archive retrieved: ${archiveBuffer.length} bytes`);
}
} catch (e: any) {
this.logger.warn(`Exec failed for wp-content archive: ${e.message}`);
logs = e.message;
}
} else {
logs = 'Archive job failed or timed out';
}
return { data: archiveBuffer, logs };
// Cleanup
try {
await batchApi.deleteNamespacedJob(jobName, namespace, undefined, undefined, undefined, undefined, 'Background');
} catch {}
if (!archiveBuffer) {
return { data: null, logs: logs || 'Archive failed or could not be retrieved' };
}
return { data: archiveBuffer, logs: 'OK' };
}
/**