feat: add snapshot/rollback system with download capability
- Add AppSnapshot entity with type (pre_deploy/manual), status tracking, and file paths - Add SnapshotsService with create, capture, rollback, prune (max 10), and download logic - Add SnapshotsController with REST endpoints for CRUD, rollback, and file downloads - Add K8s methods: exportDatabaseDump, archiveWpContent, restoreWpContent - Auto-create pre-deploy snapshots before each deployment for rollback safety - Support downloading current live state (source, wp-content, database) without snapshots - Add snapshot management UI in app detail page with create, rollback, download, delete - Wire circular dependencies with forwardRef between Deployments and Snapshots modules
This commit is contained in:
keyhan
@@ -1093,6 +1093,335 @@ export class KubernetesService implements OnModuleInit {
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Snapshot helpers ───────────────────────────────
|
||||
|
||||
/**
|
||||
* Export (dump) the application database to a local file via a K8s Job.
|
||||
* Returns the dump as a Buffer, or null on failure.
|
||||
*/
|
||||
async exportDatabaseDump(app: Application): Promise<{ data: Buffer | null; logs: string }> {
|
||||
const { coreApi, kc } = await this.getK8sClient(app.clusterId);
|
||||
const batchApi = kc.makeApiClient(k8s.BatchV1Api);
|
||||
const namespace = `user-${app.userId.split('-')[0]}`;
|
||||
const dbName = `${app.name}-db`;
|
||||
const jobName = `${app.name}-db-dump-${Date.now()}`;
|
||||
const isPostgres = app.databaseType === DatabaseType.POSTGRESQL;
|
||||
const dbDatabase = app.name.replace(/-/g, '_');
|
||||
|
||||
const defaultDbVer = isPostgres ? '16' : '8.0';
|
||||
const dbVer = app.dbVersion || defaultDbVer;
|
||||
const image = isPostgres ? `postgres:${dbVer}-alpine` : `mysql:${dbVer}`;
|
||||
|
||||
// Dump command writes to /dump/output.sql inside an emptyDir volume
|
||||
const command = isPostgres
|
||||
? ['sh', '-c', `PGPASSWORD="$DB_PASSWORD" pg_dump -h ${dbName} -U "$DB_USER" -d ${dbDatabase} --no-owner --no-acl > /dump/output.sql 2>/tmp/err; cat /tmp/err >&2; echo "DUMP_DONE"`]
|
||||
: ['sh', '-c', `mysqldump -h ${dbName} -u "$DB_USER" -p"$DB_PASSWORD" ${dbDatabase} > /dump/output.sql 2>/tmp/err; cat /tmp/err >&2; echo "DUMP_DONE"`];
|
||||
|
||||
const job: k8s.V1Job = {
|
||||
apiVersion: 'batch/v1',
|
||||
kind: 'Job',
|
||||
metadata: { name: jobName, namespace },
|
||||
spec: {
|
||||
ttlSecondsAfterFinished: 120,
|
||||
backoffLimit: 0,
|
||||
template: {
|
||||
spec: {
|
||||
restartPolicy: 'Never',
|
||||
containers: [{
|
||||
name: 'dump',
|
||||
image,
|
||||
command,
|
||||
env: [
|
||||
{ name: 'DB_USER', valueFrom: { secretKeyRef: { name: `${app.name}-db-secret`, key: 'username' } } },
|
||||
{ name: 'DB_PASSWORD', valueFrom: { secretKeyRef: { name: `${app.name}-db-secret`, key: 'password' } } },
|
||||
],
|
||||
volumeMounts: [{ name: 'dump-vol', mountPath: '/dump' }],
|
||||
resources: { requests: { cpu: '100m', memory: '128Mi' }, limits: { cpu: '500m', memory: '512Mi' } },
|
||||
}],
|
||||
volumes: [{ name: 'dump-vol', emptyDir: {} }],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
await batchApi.createNamespacedJob(namespace, job);
|
||||
} catch (e: any) {
|
||||
this.logger.error(`Failed to create DB dump job: ${e.message}`);
|
||||
return { data: null, logs: `Failed to create dump job: ${e.message}` };
|
||||
}
|
||||
|
||||
// Wait for completion (max 5 min)
|
||||
const timeout = 300_000;
|
||||
const start = Date.now();
|
||||
let succeeded = false;
|
||||
|
||||
while (Date.now() - start < timeout) {
|
||||
await new Promise((r) => setTimeout(r, 3000));
|
||||
try {
|
||||
const st = await batchApi.readNamespacedJob(jobName, namespace);
|
||||
if (st.body.status?.succeeded && st.body.status.succeeded > 0) { succeeded = true; break; }
|
||||
if (st.body.status?.failed && st.body.status.failed > 0) break;
|
||||
} catch {}
|
||||
}
|
||||
|
||||
// Get dump by exec-ing into the pod and cat-ing the file
|
||||
let dumpBuffer: Buffer | null = null;
|
||||
let logs = '';
|
||||
|
||||
try {
|
||||
const pods = await coreApi.listNamespacedPod(namespace, undefined, undefined, undefined, undefined, `job-name=${jobName}`);
|
||||
if (pods.body.items.length > 0) {
|
||||
const podName = pods.body.items[0].metadata?.name;
|
||||
if (podName && succeeded) {
|
||||
// Use exec to cat the dump file from the pod
|
||||
const exec = new k8s.Exec(kc);
|
||||
const chunks: Buffer[] = [];
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
exec.exec(
|
||||
namespace, podName, 'dump',
|
||||
['cat', '/dump/output.sql'],
|
||||
{
|
||||
write: (data: string) => { chunks.push(Buffer.from(data)); },
|
||||
} as any,
|
||||
null,
|
||||
{
|
||||
write: (data: string) => { logs += data; },
|
||||
} as any,
|
||||
false,
|
||||
(status: k8s.V1Status) => {
|
||||
if (status.status === 'Success') resolve();
|
||||
else reject(new Error(status.message || 'exec failed'));
|
||||
},
|
||||
);
|
||||
}).catch(() => {
|
||||
this.logger.warn(`Exec cat failed for ${podName}, trying readNamespacedPodLog`);
|
||||
});
|
||||
|
||||
if (chunks.length > 0) {
|
||||
dumpBuffer = Buffer.concat(chunks);
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: get logs
|
||||
if (!dumpBuffer && pods.body.items[0].metadata?.name) {
|
||||
try {
|
||||
const logRes = await coreApi.readNamespacedPodLog(pods.body.items[0].metadata.name, namespace);
|
||||
logs = logRes.body || '';
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
} catch (e: any) {
|
||||
this.logger.warn(`Could not retrieve dump: ${e.message}`);
|
||||
logs = e.message;
|
||||
}
|
||||
|
||||
if (!succeeded) {
|
||||
return { data: null, logs: logs || 'Dump job failed or timed out' };
|
||||
}
|
||||
|
||||
return { data: dumpBuffer, logs };
|
||||
}
|
||||
|
||||
/**
|
||||
* Archive the wp-content directory from a WordPress app's PVC via a K8s Job.
|
||||
* The job creates a tar.gz of /var/www/html/wp-content and we retrieve it via exec.
|
||||
* Returns the archive as a Buffer, or null on failure.
|
||||
*/
|
||||
async archiveWpContent(app: Application): Promise<{ data: Buffer | null; logs: string }> {
|
||||
const { coreApi, kc } = await this.getK8sClient(app.clusterId);
|
||||
const batchApi = kc.makeApiClient(k8s.BatchV1Api);
|
||||
const namespace = `user-${app.userId.split('-')[0]}`;
|
||||
const pvcName = `${app.name}-wp-content`;
|
||||
const jobName = `${app.name}-wp-archive-${Date.now()}`;
|
||||
|
||||
const job: k8s.V1Job = {
|
||||
apiVersion: 'batch/v1',
|
||||
kind: 'Job',
|
||||
metadata: { name: jobName, namespace },
|
||||
spec: {
|
||||
ttlSecondsAfterFinished: 120,
|
||||
backoffLimit: 0,
|
||||
template: {
|
||||
spec: {
|
||||
restartPolicy: 'Never',
|
||||
containers: [{
|
||||
name: 'archiver',
|
||||
image: 'alpine:3.19',
|
||||
command: ['sh', '-c', 'apk add --no-cache tar gzip > /dev/null 2>&1 && cd /wp-content && tar czf /output/wp-content.tar.gz . && echo "ARCHIVE_DONE"'],
|
||||
volumeMounts: [
|
||||
{ name: 'wp-content', mountPath: '/wp-content', readOnly: true },
|
||||
{ name: 'output', mountPath: '/output' },
|
||||
],
|
||||
resources: { requests: { cpu: '100m', memory: '64Mi' }, limits: { cpu: '500m', memory: '256Mi' } },
|
||||
}],
|
||||
volumes: [
|
||||
{ name: 'wp-content', persistentVolumeClaim: { claimName: pvcName } },
|
||||
{ name: 'output', emptyDir: {} },
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
await batchApi.createNamespacedJob(namespace, job);
|
||||
} catch (e: any) {
|
||||
this.logger.error(`Failed to create wp-content archive job: ${e.message}`);
|
||||
return { data: null, logs: `Failed to create archive job: ${e.message}` };
|
||||
}
|
||||
|
||||
// Wait for completion
|
||||
const timeout = 300_000;
|
||||
const start = Date.now();
|
||||
let succeeded = false;
|
||||
while (Date.now() - start < timeout) {
|
||||
await new Promise((r) => setTimeout(r, 3000));
|
||||
try {
|
||||
const st = await batchApi.readNamespacedJob(jobName, namespace);
|
||||
if (st.body.status?.succeeded && st.body.status.succeeded > 0) { succeeded = true; break; }
|
||||
if (st.body.status?.failed && st.body.status.failed > 0) break;
|
||||
} catch {}
|
||||
}
|
||||
|
||||
let archiveBuffer: Buffer | null = null;
|
||||
let logs = '';
|
||||
|
||||
if (succeeded) {
|
||||
try {
|
||||
const pods = await coreApi.listNamespacedPod(namespace, undefined, undefined, undefined, undefined, `job-name=${jobName}`);
|
||||
if (pods.body.items.length > 0) {
|
||||
const podName = pods.body.items[0].metadata?.name;
|
||||
if (podName) {
|
||||
const exec = new k8s.Exec(kc);
|
||||
const chunks: Buffer[] = [];
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
exec.exec(
|
||||
namespace, podName, 'archiver',
|
||||
['cat', '/output/wp-content.tar.gz'],
|
||||
{
|
||||
write: (data: string) => { chunks.push(Buffer.from(data)); },
|
||||
} as any,
|
||||
null,
|
||||
{
|
||||
write: (data: string) => { logs += data; },
|
||||
} as any,
|
||||
false,
|
||||
(status: k8s.V1Status) => {
|
||||
if (status.status === 'Success') resolve();
|
||||
else reject(new Error(status.message || 'exec failed'));
|
||||
},
|
||||
);
|
||||
}).catch((err) => {
|
||||
this.logger.warn(`Exec failed for wp-content archive: ${err.message}`);
|
||||
});
|
||||
|
||||
if (chunks.length > 0) {
|
||||
archiveBuffer = Buffer.concat(chunks);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (e: any) {
|
||||
logs = e.message;
|
||||
}
|
||||
} else {
|
||||
logs = 'Archive job failed or timed out';
|
||||
}
|
||||
|
||||
return { data: archiveBuffer, logs };
|
||||
}
|
||||
|
||||
/**
|
||||
* Restore wp-content from a tar.gz archive into the WordPress PVC.
|
||||
*/
|
||||
async restoreWpContent(app: Application, archiveBuffer: Buffer): Promise<{ success: boolean; logs: string }> {
|
||||
const { coreApi, kc } = await this.getK8sClient(app.clusterId);
|
||||
const batchApi = kc.makeApiClient(k8s.BatchV1Api);
|
||||
const namespace = `user-${app.userId.split('-')[0]}`;
|
||||
const pvcName = `${app.name}-wp-content`;
|
||||
const jobName = `${app.name}-wp-restore-${Date.now()}`;
|
||||
const secretName = `${jobName}-archive`;
|
||||
|
||||
// Store archive in a secret
|
||||
const archiveSecret = {
|
||||
apiVersion: 'v1',
|
||||
kind: 'Secret',
|
||||
metadata: { name: secretName, namespace },
|
||||
data: { 'wp-content.tar.gz': archiveBuffer.toString('base64') },
|
||||
};
|
||||
|
||||
try {
|
||||
await coreApi.createNamespacedSecret(namespace, archiveSecret);
|
||||
} catch (e: any) {
|
||||
return { success: false, logs: `Failed to create archive secret: ${e.message}` };
|
||||
}
|
||||
|
||||
const job: k8s.V1Job = {
|
||||
apiVersion: 'batch/v1',
|
||||
kind: 'Job',
|
||||
metadata: { name: jobName, namespace },
|
||||
spec: {
|
||||
ttlSecondsAfterFinished: 120,
|
||||
backoffLimit: 0,
|
||||
template: {
|
||||
spec: {
|
||||
restartPolicy: 'Never',
|
||||
containers: [{
|
||||
name: 'restore',
|
||||
image: 'alpine:3.19',
|
||||
command: ['sh', '-c', 'apk add --no-cache tar gzip > /dev/null 2>&1 && rm -rf /wp-content/* && cd /wp-content && tar xzf /archive/wp-content.tar.gz && echo "RESTORE_DONE"'],
|
||||
volumeMounts: [
|
||||
{ name: 'wp-content', mountPath: '/wp-content' },
|
||||
{ name: 'archive', mountPath: '/archive', readOnly: true },
|
||||
],
|
||||
resources: { requests: { cpu: '100m', memory: '64Mi' }, limits: { cpu: '500m', memory: '256Mi' } },
|
||||
}],
|
||||
volumes: [
|
||||
{ name: 'wp-content', persistentVolumeClaim: { claimName: pvcName } },
|
||||
{ name: 'archive', secret: { secretName } },
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
await batchApi.createNamespacedJob(namespace, job);
|
||||
} catch (e: any) {
|
||||
try { await coreApi.deleteNamespacedSecret(secretName, namespace); } catch {}
|
||||
return { success: false, logs: `Failed to create restore job: ${e.message}` };
|
||||
}
|
||||
|
||||
// Wait
|
||||
const timeout = 300_000;
|
||||
const start = Date.now();
|
||||
let succeeded = false;
|
||||
let failed = false;
|
||||
while (Date.now() - start < timeout) {
|
||||
await new Promise((r) => setTimeout(r, 3000));
|
||||
try {
|
||||
const st = await batchApi.readNamespacedJob(jobName, namespace);
|
||||
if (st.body.status?.succeeded && st.body.status.succeeded > 0) { succeeded = true; break; }
|
||||
if (st.body.status?.failed && st.body.status.failed > 0) { failed = true; break; }
|
||||
} catch {}
|
||||
}
|
||||
|
||||
let logs = '';
|
||||
try {
|
||||
const pods = await coreApi.listNamespacedPod(namespace, undefined, undefined, undefined, undefined, `job-name=${jobName}`);
|
||||
if (pods.body.items.length > 0 && pods.body.items[0].metadata?.name) {
|
||||
const logRes = await coreApi.readNamespacedPodLog(pods.body.items[0].metadata.name, namespace);
|
||||
logs = logRes.body || '';
|
||||
}
|
||||
} catch {}
|
||||
|
||||
try { await coreApi.deleteNamespacedSecret(secretName, namespace); } catch {}
|
||||
|
||||
return { success: succeeded && !failed, logs: logs || (succeeded ? 'Restore completed' : 'Restore failed or timed out') };
|
||||
}
|
||||
|
||||
private generatePassword(length = 24): string {
|
||||
const chars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!@#$%';
|
||||
let password = '';
|
||||
|
||||
Reference in New Issue
Block a user