feat: add snapshot/rollback system with download capability
- Add AppSnapshot entity with type (pre_deploy/manual), status tracking, and file paths - Add SnapshotsService with create, capture, rollback, prune (max 10), and download logic - Add SnapshotsController with REST endpoints for CRUD, rollback, and file downloads - Add K8s methods: exportDatabaseDump, archiveWpContent, restoreWpContent - Auto-create pre-deploy snapshots before each deployment for rollback safety - Support downloading current live state (source, wp-content, database) without snapshots - Add snapshot management UI in app detail page with create, rollback, download, delete - Wire circular dependencies with forwardRef between Deployments and Snapshots modules
This commit is contained in:
keyhan
@@ -11,6 +11,7 @@ import { KubernetesModule } from './kubernetes/kubernetes.module';
|
||||
import { BuildModule } from './build/build.module';
|
||||
import { TicketsModule } from './tickets/tickets.module';
|
||||
import { BillingModule } from './billing/billing.module';
|
||||
import { SnapshotsModule } from './snapshots/snapshots.module';
|
||||
import configuration from './config/configuration';
|
||||
|
||||
@Module({
|
||||
@@ -60,6 +61,7 @@ import configuration from './config/configuration';
|
||||
BuildModule,
|
||||
TicketsModule,
|
||||
BillingModule,
|
||||
SnapshotsModule,
|
||||
],
|
||||
})
|
||||
export class AppModule {}
|
||||
|
||||
@@ -6,6 +6,7 @@ import { Deployment } from './entities/deployment.entity';
|
||||
import { ApplicationsModule } from '../applications/applications.module';
|
||||
import { KubernetesModule } from '../kubernetes/kubernetes.module';
|
||||
import { BuildModule } from '../build/build.module';
|
||||
import { SnapshotsModule } from '../snapshots/snapshots.module';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
@@ -13,6 +14,7 @@ import { BuildModule } from '../build/build.module';
|
||||
forwardRef(() => ApplicationsModule),
|
||||
KubernetesModule,
|
||||
BuildModule,
|
||||
forwardRef(() => SnapshotsModule),
|
||||
],
|
||||
controllers: [DeploymentsController],
|
||||
providers: [DeploymentsService],
|
||||
|
||||
@@ -6,6 +6,7 @@ import { ApplicationsService } from '../applications/applications.service';
|
||||
import { KubernetesService } from '../kubernetes/kubernetes.service';
|
||||
import { BuildService } from '../build/build.service';
|
||||
import { DeploymentStatus } from '../common/enums';
|
||||
import { SnapshotsService } from '../snapshots/snapshots.service';
|
||||
|
||||
@Injectable()
|
||||
export class DeploymentsService {
|
||||
@@ -18,11 +19,22 @@ export class DeploymentsService {
|
||||
private applicationsService: ApplicationsService,
|
||||
private kubernetesService: KubernetesService,
|
||||
private buildService: BuildService,
|
||||
@Inject(forwardRef(() => SnapshotsService))
|
||||
private snapshotsService: SnapshotsService,
|
||||
) {}
|
||||
|
||||
async triggerDeployment(applicationId: string, userId: string): Promise<Deployment> {
|
||||
const app = await this.applicationsService.findOne(applicationId, userId);
|
||||
|
||||
// Auto-snapshot before deploying (capture current state for rollback)
|
||||
try {
|
||||
const version = `v${Date.now()}`;
|
||||
await this.snapshotsService.createPreDeploySnapshot(app.id, userId, version);
|
||||
this.logger.log(`Pre-deploy snapshot created for ${app.name}`);
|
||||
} catch (e: any) {
|
||||
this.logger.warn(`Pre-deploy snapshot failed for ${app.name}: ${e.message} — continuing deploy`);
|
||||
}
|
||||
|
||||
// Create deployment record
|
||||
const deployment = this.deploymentsRepository.create({
|
||||
applicationId: app.id,
|
||||
@@ -173,6 +185,15 @@ export class DeploymentsService {
|
||||
throw new NotFoundException('No source code available. Upload code or set a git URL first.');
|
||||
}
|
||||
|
||||
// Auto-snapshot before redeploy
|
||||
try {
|
||||
const version = `v${Date.now()}`;
|
||||
await this.snapshotsService.createPreDeploySnapshot(app.id, userId, version);
|
||||
this.logger.log(`Pre-redeploy snapshot created for ${app.name}`);
|
||||
} catch (e: any) {
|
||||
this.logger.warn(`Pre-redeploy snapshot failed for ${app.name}: ${e.message} — continuing`);
|
||||
}
|
||||
|
||||
// Create new deployment record
|
||||
const deployment = this.deploymentsRepository.create({
|
||||
applicationId: app.id,
|
||||
|
||||
@@ -1093,6 +1093,335 @@ export class KubernetesService implements OnModuleInit {
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Snapshot helpers ───────────────────────────────
|
||||
|
||||
/**
|
||||
* Export (dump) the application database to a local file via a K8s Job.
|
||||
* Returns the dump as a Buffer, or null on failure.
|
||||
*/
|
||||
async exportDatabaseDump(app: Application): Promise<{ data: Buffer | null; logs: string }> {
|
||||
const { coreApi, kc } = await this.getK8sClient(app.clusterId);
|
||||
const batchApi = kc.makeApiClient(k8s.BatchV1Api);
|
||||
const namespace = `user-${app.userId.split('-')[0]}`;
|
||||
const dbName = `${app.name}-db`;
|
||||
const jobName = `${app.name}-db-dump-${Date.now()}`;
|
||||
const isPostgres = app.databaseType === DatabaseType.POSTGRESQL;
|
||||
const dbDatabase = app.name.replace(/-/g, '_');
|
||||
|
||||
const defaultDbVer = isPostgres ? '16' : '8.0';
|
||||
const dbVer = app.dbVersion || defaultDbVer;
|
||||
const image = isPostgres ? `postgres:${dbVer}-alpine` : `mysql:${dbVer}`;
|
||||
|
||||
// Dump command writes to /dump/output.sql inside an emptyDir volume
|
||||
const command = isPostgres
|
||||
? ['sh', '-c', `PGPASSWORD="$DB_PASSWORD" pg_dump -h ${dbName} -U "$DB_USER" -d ${dbDatabase} --no-owner --no-acl > /dump/output.sql 2>/tmp/err; cat /tmp/err >&2; echo "DUMP_DONE"`]
|
||||
: ['sh', '-c', `mysqldump -h ${dbName} -u "$DB_USER" -p"$DB_PASSWORD" ${dbDatabase} > /dump/output.sql 2>/tmp/err; cat /tmp/err >&2; echo "DUMP_DONE"`];
|
||||
|
||||
const job: k8s.V1Job = {
|
||||
apiVersion: 'batch/v1',
|
||||
kind: 'Job',
|
||||
metadata: { name: jobName, namespace },
|
||||
spec: {
|
||||
ttlSecondsAfterFinished: 120,
|
||||
backoffLimit: 0,
|
||||
template: {
|
||||
spec: {
|
||||
restartPolicy: 'Never',
|
||||
containers: [{
|
||||
name: 'dump',
|
||||
image,
|
||||
command,
|
||||
env: [
|
||||
{ name: 'DB_USER', valueFrom: { secretKeyRef: { name: `${app.name}-db-secret`, key: 'username' } } },
|
||||
{ name: 'DB_PASSWORD', valueFrom: { secretKeyRef: { name: `${app.name}-db-secret`, key: 'password' } } },
|
||||
],
|
||||
volumeMounts: [{ name: 'dump-vol', mountPath: '/dump' }],
|
||||
resources: { requests: { cpu: '100m', memory: '128Mi' }, limits: { cpu: '500m', memory: '512Mi' } },
|
||||
}],
|
||||
volumes: [{ name: 'dump-vol', emptyDir: {} }],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
await batchApi.createNamespacedJob(namespace, job);
|
||||
} catch (e: any) {
|
||||
this.logger.error(`Failed to create DB dump job: ${e.message}`);
|
||||
return { data: null, logs: `Failed to create dump job: ${e.message}` };
|
||||
}
|
||||
|
||||
// Wait for completion (max 5 min)
|
||||
const timeout = 300_000;
|
||||
const start = Date.now();
|
||||
let succeeded = false;
|
||||
|
||||
while (Date.now() - start < timeout) {
|
||||
await new Promise((r) => setTimeout(r, 3000));
|
||||
try {
|
||||
const st = await batchApi.readNamespacedJob(jobName, namespace);
|
||||
if (st.body.status?.succeeded && st.body.status.succeeded > 0) { succeeded = true; break; }
|
||||
if (st.body.status?.failed && st.body.status.failed > 0) break;
|
||||
} catch {}
|
||||
}
|
||||
|
||||
// Get dump by exec-ing into the pod and cat-ing the file
|
||||
let dumpBuffer: Buffer | null = null;
|
||||
let logs = '';
|
||||
|
||||
try {
|
||||
const pods = await coreApi.listNamespacedPod(namespace, undefined, undefined, undefined, undefined, `job-name=${jobName}`);
|
||||
if (pods.body.items.length > 0) {
|
||||
const podName = pods.body.items[0].metadata?.name;
|
||||
if (podName && succeeded) {
|
||||
// Use exec to cat the dump file from the pod
|
||||
const exec = new k8s.Exec(kc);
|
||||
const chunks: Buffer[] = [];
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
exec.exec(
|
||||
namespace, podName, 'dump',
|
||||
['cat', '/dump/output.sql'],
|
||||
{
|
||||
write: (data: string) => { chunks.push(Buffer.from(data)); },
|
||||
} as any,
|
||||
null,
|
||||
{
|
||||
write: (data: string) => { logs += data; },
|
||||
} as any,
|
||||
false,
|
||||
(status: k8s.V1Status) => {
|
||||
if (status.status === 'Success') resolve();
|
||||
else reject(new Error(status.message || 'exec failed'));
|
||||
},
|
||||
);
|
||||
}).catch(() => {
|
||||
this.logger.warn(`Exec cat failed for ${podName}, trying readNamespacedPodLog`);
|
||||
});
|
||||
|
||||
if (chunks.length > 0) {
|
||||
dumpBuffer = Buffer.concat(chunks);
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: get logs
|
||||
if (!dumpBuffer && pods.body.items[0].metadata?.name) {
|
||||
try {
|
||||
const logRes = await coreApi.readNamespacedPodLog(pods.body.items[0].metadata.name, namespace);
|
||||
logs = logRes.body || '';
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
} catch (e: any) {
|
||||
this.logger.warn(`Could not retrieve dump: ${e.message}`);
|
||||
logs = e.message;
|
||||
}
|
||||
|
||||
if (!succeeded) {
|
||||
return { data: null, logs: logs || 'Dump job failed or timed out' };
|
||||
}
|
||||
|
||||
return { data: dumpBuffer, logs };
|
||||
}
|
||||
|
||||
/**
|
||||
* Archive the wp-content directory from a WordPress app's PVC via a K8s Job.
|
||||
* The job creates a tar.gz of /var/www/html/wp-content and we retrieve it via exec.
|
||||
* Returns the archive as a Buffer, or null on failure.
|
||||
*/
|
||||
async archiveWpContent(app: Application): Promise<{ data: Buffer | null; logs: string }> {
|
||||
const { coreApi, kc } = await this.getK8sClient(app.clusterId);
|
||||
const batchApi = kc.makeApiClient(k8s.BatchV1Api);
|
||||
const namespace = `user-${app.userId.split('-')[0]}`;
|
||||
const pvcName = `${app.name}-wp-content`;
|
||||
const jobName = `${app.name}-wp-archive-${Date.now()}`;
|
||||
|
||||
const job: k8s.V1Job = {
|
||||
apiVersion: 'batch/v1',
|
||||
kind: 'Job',
|
||||
metadata: { name: jobName, namespace },
|
||||
spec: {
|
||||
ttlSecondsAfterFinished: 120,
|
||||
backoffLimit: 0,
|
||||
template: {
|
||||
spec: {
|
||||
restartPolicy: 'Never',
|
||||
containers: [{
|
||||
name: 'archiver',
|
||||
image: 'alpine:3.19',
|
||||
command: ['sh', '-c', 'apk add --no-cache tar gzip > /dev/null 2>&1 && cd /wp-content && tar czf /output/wp-content.tar.gz . && echo "ARCHIVE_DONE"'],
|
||||
volumeMounts: [
|
||||
{ name: 'wp-content', mountPath: '/wp-content', readOnly: true },
|
||||
{ name: 'output', mountPath: '/output' },
|
||||
],
|
||||
resources: { requests: { cpu: '100m', memory: '64Mi' }, limits: { cpu: '500m', memory: '256Mi' } },
|
||||
}],
|
||||
volumes: [
|
||||
{ name: 'wp-content', persistentVolumeClaim: { claimName: pvcName } },
|
||||
{ name: 'output', emptyDir: {} },
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
await batchApi.createNamespacedJob(namespace, job);
|
||||
} catch (e: any) {
|
||||
this.logger.error(`Failed to create wp-content archive job: ${e.message}`);
|
||||
return { data: null, logs: `Failed to create archive job: ${e.message}` };
|
||||
}
|
||||
|
||||
// Wait for completion
|
||||
const timeout = 300_000;
|
||||
const start = Date.now();
|
||||
let succeeded = false;
|
||||
while (Date.now() - start < timeout) {
|
||||
await new Promise((r) => setTimeout(r, 3000));
|
||||
try {
|
||||
const st = await batchApi.readNamespacedJob(jobName, namespace);
|
||||
if (st.body.status?.succeeded && st.body.status.succeeded > 0) { succeeded = true; break; }
|
||||
if (st.body.status?.failed && st.body.status.failed > 0) break;
|
||||
} catch {}
|
||||
}
|
||||
|
||||
let archiveBuffer: Buffer | null = null;
|
||||
let logs = '';
|
||||
|
||||
if (succeeded) {
|
||||
try {
|
||||
const pods = await coreApi.listNamespacedPod(namespace, undefined, undefined, undefined, undefined, `job-name=${jobName}`);
|
||||
if (pods.body.items.length > 0) {
|
||||
const podName = pods.body.items[0].metadata?.name;
|
||||
if (podName) {
|
||||
const exec = new k8s.Exec(kc);
|
||||
const chunks: Buffer[] = [];
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
exec.exec(
|
||||
namespace, podName, 'archiver',
|
||||
['cat', '/output/wp-content.tar.gz'],
|
||||
{
|
||||
write: (data: string) => { chunks.push(Buffer.from(data)); },
|
||||
} as any,
|
||||
null,
|
||||
{
|
||||
write: (data: string) => { logs += data; },
|
||||
} as any,
|
||||
false,
|
||||
(status: k8s.V1Status) => {
|
||||
if (status.status === 'Success') resolve();
|
||||
else reject(new Error(status.message || 'exec failed'));
|
||||
},
|
||||
);
|
||||
}).catch((err) => {
|
||||
this.logger.warn(`Exec failed for wp-content archive: ${err.message}`);
|
||||
});
|
||||
|
||||
if (chunks.length > 0) {
|
||||
archiveBuffer = Buffer.concat(chunks);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (e: any) {
|
||||
logs = e.message;
|
||||
}
|
||||
} else {
|
||||
logs = 'Archive job failed or timed out';
|
||||
}
|
||||
|
||||
return { data: archiveBuffer, logs };
|
||||
}
|
||||
|
||||
/**
|
||||
* Restore wp-content from a tar.gz archive into the WordPress PVC.
|
||||
*/
|
||||
async restoreWpContent(app: Application, archiveBuffer: Buffer): Promise<{ success: boolean; logs: string }> {
|
||||
const { coreApi, kc } = await this.getK8sClient(app.clusterId);
|
||||
const batchApi = kc.makeApiClient(k8s.BatchV1Api);
|
||||
const namespace = `user-${app.userId.split('-')[0]}`;
|
||||
const pvcName = `${app.name}-wp-content`;
|
||||
const jobName = `${app.name}-wp-restore-${Date.now()}`;
|
||||
const secretName = `${jobName}-archive`;
|
||||
|
||||
// Store archive in a secret
|
||||
const archiveSecret = {
|
||||
apiVersion: 'v1',
|
||||
kind: 'Secret',
|
||||
metadata: { name: secretName, namespace },
|
||||
data: { 'wp-content.tar.gz': archiveBuffer.toString('base64') },
|
||||
};
|
||||
|
||||
try {
|
||||
await coreApi.createNamespacedSecret(namespace, archiveSecret);
|
||||
} catch (e: any) {
|
||||
return { success: false, logs: `Failed to create archive secret: ${e.message}` };
|
||||
}
|
||||
|
||||
const job: k8s.V1Job = {
|
||||
apiVersion: 'batch/v1',
|
||||
kind: 'Job',
|
||||
metadata: { name: jobName, namespace },
|
||||
spec: {
|
||||
ttlSecondsAfterFinished: 120,
|
||||
backoffLimit: 0,
|
||||
template: {
|
||||
spec: {
|
||||
restartPolicy: 'Never',
|
||||
containers: [{
|
||||
name: 'restore',
|
||||
image: 'alpine:3.19',
|
||||
command: ['sh', '-c', 'apk add --no-cache tar gzip > /dev/null 2>&1 && rm -rf /wp-content/* && cd /wp-content && tar xzf /archive/wp-content.tar.gz && echo "RESTORE_DONE"'],
|
||||
volumeMounts: [
|
||||
{ name: 'wp-content', mountPath: '/wp-content' },
|
||||
{ name: 'archive', mountPath: '/archive', readOnly: true },
|
||||
],
|
||||
resources: { requests: { cpu: '100m', memory: '64Mi' }, limits: { cpu: '500m', memory: '256Mi' } },
|
||||
}],
|
||||
volumes: [
|
||||
{ name: 'wp-content', persistentVolumeClaim: { claimName: pvcName } },
|
||||
{ name: 'archive', secret: { secretName } },
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
await batchApi.createNamespacedJob(namespace, job);
|
||||
} catch (e: any) {
|
||||
try { await coreApi.deleteNamespacedSecret(secretName, namespace); } catch {}
|
||||
return { success: false, logs: `Failed to create restore job: ${e.message}` };
|
||||
}
|
||||
|
||||
// Wait
|
||||
const timeout = 300_000;
|
||||
const start = Date.now();
|
||||
let succeeded = false;
|
||||
let failed = false;
|
||||
while (Date.now() - start < timeout) {
|
||||
await new Promise((r) => setTimeout(r, 3000));
|
||||
try {
|
||||
const st = await batchApi.readNamespacedJob(jobName, namespace);
|
||||
if (st.body.status?.succeeded && st.body.status.succeeded > 0) { succeeded = true; break; }
|
||||
if (st.body.status?.failed && st.body.status.failed > 0) { failed = true; break; }
|
||||
} catch {}
|
||||
}
|
||||
|
||||
let logs = '';
|
||||
try {
|
||||
const pods = await coreApi.listNamespacedPod(namespace, undefined, undefined, undefined, undefined, `job-name=${jobName}`);
|
||||
if (pods.body.items.length > 0 && pods.body.items[0].metadata?.name) {
|
||||
const logRes = await coreApi.readNamespacedPodLog(pods.body.items[0].metadata.name, namespace);
|
||||
logs = logRes.body || '';
|
||||
}
|
||||
} catch {}
|
||||
|
||||
try { await coreApi.deleteNamespacedSecret(secretName, namespace); } catch {}
|
||||
|
||||
return { success: succeeded && !failed, logs: logs || (succeeded ? 'Restore completed' : 'Restore failed or timed out') };
|
||||
}
|
||||
|
||||
private generatePassword(length = 24): string {
|
||||
const chars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!@#$%';
|
||||
let password = '';
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
import {
|
||||
Entity,
|
||||
PrimaryGeneratedColumn,
|
||||
Column,
|
||||
CreateDateColumn,
|
||||
ManyToOne,
|
||||
JoinColumn,
|
||||
} from 'typeorm';
|
||||
import { Application } from '../../applications/entities/application.entity';
|
||||
|
||||
export enum SnapshotType {
|
||||
PRE_DEPLOY = 'pre_deploy', // Automatic snapshot before each deploy
|
||||
MANUAL = 'manual', // User-triggered manual snapshot
|
||||
}
|
||||
|
||||
export enum SnapshotStatus {
|
||||
IN_PROGRESS = 'in_progress',
|
||||
COMPLETED = 'completed',
|
||||
FAILED = 'failed',
|
||||
}
|
||||
|
||||
@Entity('snapshots')
|
||||
export class AppSnapshot {
|
||||
@PrimaryGeneratedColumn('uuid')
|
||||
id: string;
|
||||
|
||||
@Column({ type: 'enum', enum: SnapshotType, default: SnapshotType.MANUAL })
|
||||
type: SnapshotType;
|
||||
|
||||
@Column({ type: 'enum', enum: SnapshotStatus, default: SnapshotStatus.IN_PROGRESS })
|
||||
status: SnapshotStatus;
|
||||
|
||||
@Column({ nullable: true })
|
||||
label: string; // Human-readable label e.g. "Before deploy v1712345678"
|
||||
|
||||
// ─── App snapshot data ────────────────────────────
|
||||
@Column({ nullable: true })
|
||||
appArchivePath: string; // Local path to archived app source (zip)
|
||||
|
||||
@Column({ nullable: true })
|
||||
wpContentArchivePath: string; // Local path to archived wp-content (tar.gz from PVC)
|
||||
|
||||
@Column({ nullable: true })
|
||||
imageTag: string; // Docker image tag at time of snapshot
|
||||
|
||||
// ─── DB snapshot data ─────────────────────────────
|
||||
@Column({ nullable: true })
|
||||
dbDumpPath: string; // Local path to DB dump file
|
||||
|
||||
@Column({ default: false })
|
||||
hasDatabase: boolean;
|
||||
|
||||
// ─── Metadata ─────────────────────────────────────
|
||||
@Column({ type: 'bigint', nullable: true })
|
||||
appArchiveSize: number; // bytes
|
||||
|
||||
@Column({ type: 'bigint', nullable: true })
|
||||
dbDumpSize: number; // bytes
|
||||
|
||||
@Column({ type: 'bigint', nullable: true })
|
||||
wpContentSize: number; // bytes
|
||||
|
||||
@Column({ nullable: true })
|
||||
errorMessage: string;
|
||||
|
||||
// ─── Relations ────────────────────────────────────
|
||||
@ManyToOne(() => Application, { onDelete: 'CASCADE' })
|
||||
@JoinColumn({ name: 'applicationId' })
|
||||
application: Application;
|
||||
|
||||
@Column()
|
||||
applicationId: string;
|
||||
|
||||
@Column()
|
||||
createdBy: string; // userId
|
||||
|
||||
@CreateDateColumn()
|
||||
createdAt: Date;
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
import {
|
||||
Controller,
|
||||
Get,
|
||||
Post,
|
||||
Delete,
|
||||
Param,
|
||||
Query,
|
||||
Res,
|
||||
UseGuards,
|
||||
Request,
|
||||
Logger,
|
||||
} from '@nestjs/common';
|
||||
import { AuthGuard } from '@nestjs/passport';
|
||||
import { ApiTags, ApiOperation, ApiBearerAuth } from '@nestjs/swagger';
|
||||
import { Response } from 'express';
|
||||
import * as fs from 'fs';
|
||||
import { SnapshotsService } from './snapshots.service';
|
||||
import { RolesGuard } from '../common/guards/roles.guard';
|
||||
import { SnapshotType } from './entities/snapshot.entity';
|
||||
|
||||
@ApiTags('Snapshots')
|
||||
@ApiBearerAuth()
|
||||
@Controller('snapshots')
|
||||
@UseGuards(AuthGuard('jwt'), RolesGuard)
|
||||
export class SnapshotsController {
|
||||
private readonly logger = new Logger(SnapshotsController.name);
|
||||
|
||||
constructor(private readonly snapshotsService: SnapshotsService) {}
|
||||
|
||||
// ─── Snapshot CRUD ──────────────────────────────────
|
||||
|
||||
@Post('applications/:appId')
|
||||
@ApiOperation({ summary: 'Create a manual snapshot of the current app state' })
|
||||
async createSnapshot(
|
||||
@Param('appId') appId: string,
|
||||
@Request() req: any,
|
||||
@Query('label') label?: string,
|
||||
) {
|
||||
const snapshot = await this.snapshotsService.createSnapshot(
|
||||
appId,
|
||||
req.user.id,
|
||||
SnapshotType.MANUAL,
|
||||
label,
|
||||
);
|
||||
return { message: 'Snapshot creation started', snapshot };
|
||||
}
|
||||
|
||||
@Get('applications/:appId')
|
||||
@ApiOperation({ summary: 'List all snapshots for an application (max 10)' })
|
||||
async listSnapshots(@Param('appId') appId: string, @Request() req: any) {
|
||||
return this.snapshotsService.listSnapshots(appId, req.user.id);
|
||||
}
|
||||
|
||||
@Get(':id')
|
||||
@ApiOperation({ summary: 'Get snapshot details' })
|
||||
async getSnapshot(@Param('id') id: string, @Request() req: any) {
|
||||
return this.snapshotsService.findOne(id, req.user.id);
|
||||
}
|
||||
|
||||
@Delete(':id')
|
||||
@ApiOperation({ summary: 'Delete a specific snapshot' })
|
||||
async deleteSnapshot(@Param('id') id: string, @Request() req: any) {
|
||||
await this.snapshotsService.deleteSnapshot(id, req.user.id);
|
||||
return { message: 'Snapshot deleted' };
|
||||
}
|
||||
|
||||
// ─── Rollback ───────────────────────────────────────
|
||||
|
||||
@Post(':id/rollback')
|
||||
@ApiOperation({ summary: 'Rollback application to a specific snapshot' })
|
||||
async rollback(@Param('id') id: string, @Request() req: any) {
|
||||
const result = await this.snapshotsService.rollbackToSnapshot(id, req.user.id);
|
||||
return result;
|
||||
}
|
||||
|
||||
// ─── Download snapshot artifacts ────────────────────
|
||||
|
||||
@Get(':id/download/source')
|
||||
@ApiOperation({ summary: 'Download source code from a snapshot' })
|
||||
async downloadSource(@Param('id') id: string, @Request() req: any, @Res() res: Response) {
|
||||
const { filePath, fileName } = await this.snapshotsService.getDownloadPath(id, req.user.id, 'source');
|
||||
res.setHeader('Content-Disposition', `attachment; filename="${fileName}"`);
|
||||
res.setHeader('Content-Type', 'application/zip');
|
||||
const stream = fs.createReadStream(filePath);
|
||||
stream.pipe(res);
|
||||
}
|
||||
|
||||
@Get(':id/download/wp-content')
|
||||
@ApiOperation({ summary: 'Download wp-content archive from a snapshot' })
|
||||
async downloadWpContent(@Param('id') id: string, @Request() req: any, @Res() res: Response) {
|
||||
const { filePath, fileName } = await this.snapshotsService.getDownloadPath(id, req.user.id, 'wp-content');
|
||||
res.setHeader('Content-Disposition', `attachment; filename="${fileName}"`);
|
||||
res.setHeader('Content-Type', 'application/gzip');
|
||||
const stream = fs.createReadStream(filePath);
|
||||
stream.pipe(res);
|
||||
}
|
||||
|
||||
@Get(':id/download/database')
|
||||
@ApiOperation({ summary: 'Download database dump from a snapshot' })
|
||||
async downloadDatabase(@Param('id') id: string, @Request() req: any, @Res() res: Response) {
|
||||
const { filePath, fileName } = await this.snapshotsService.getDownloadPath(id, req.user.id, 'database');
|
||||
res.setHeader('Content-Disposition', `attachment; filename="${fileName}"`);
|
||||
res.setHeader('Content-Type', 'application/sql');
|
||||
const stream = fs.createReadStream(filePath);
|
||||
stream.pipe(res);
|
||||
}
|
||||
|
||||
// ─── Download CURRENT live state ────────────────────
|
||||
|
||||
@Get('applications/:appId/current/source')
|
||||
@ApiOperation({ summary: 'Download the current source code of an application' })
|
||||
async downloadCurrentSource(@Param('appId') appId: string, @Request() req: any, @Res() res: Response) {
|
||||
const result = await this.snapshotsService.downloadCurrentSource(appId, req.user.id);
|
||||
if (!result) {
|
||||
res.status(404).json({ message: 'No source code available' });
|
||||
return;
|
||||
}
|
||||
res.setHeader('Content-Disposition', `attachment; filename="${result.fileName}"`);
|
||||
res.setHeader('Content-Type', 'application/zip');
|
||||
const stream = fs.createReadStream(result.filePath);
|
||||
stream.pipe(res);
|
||||
}
|
||||
|
||||
@Get('applications/:appId/current/wp-content')
|
||||
@ApiOperation({ summary: 'Download the current live wp-content from a WordPress app' })
|
||||
async downloadCurrentWpContent(@Param('appId') appId: string, @Request() req: any, @Res() res: Response) {
|
||||
const data = await this.snapshotsService.downloadCurrentWpContent(appId, req.user.id);
|
||||
if (!data) {
|
||||
res.status(404).json({ message: 'Could not retrieve wp-content' });
|
||||
return;
|
||||
}
|
||||
res.setHeader('Content-Disposition', `attachment; filename="wp-content-live.tar.gz"`);
|
||||
res.setHeader('Content-Type', 'application/gzip');
|
||||
res.send(data);
|
||||
}
|
||||
|
||||
@Get('applications/:appId/current/database')
|
||||
@ApiOperation({ summary: 'Download the current live database dump' })
|
||||
async downloadCurrentDatabase(@Param('appId') appId: string, @Request() req: any, @Res() res: Response) {
|
||||
const data = await this.snapshotsService.downloadCurrentDatabase(appId, req.user.id);
|
||||
if (!data) {
|
||||
res.status(404).json({ message: 'Could not dump database' });
|
||||
return;
|
||||
}
|
||||
res.setHeader('Content-Disposition', `attachment; filename="database-live.sql"`);
|
||||
res.setHeader('Content-Type', 'application/sql');
|
||||
res.send(data);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
import { Module, forwardRef } from '@nestjs/common';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
import { SnapshotsService } from './snapshots.service';
|
||||
import { SnapshotsController } from './snapshots.controller';
|
||||
import { AppSnapshot } from './entities/snapshot.entity';
|
||||
import { ApplicationsModule } from '../applications/applications.module';
|
||||
import { KubernetesModule } from '../kubernetes/kubernetes.module';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
TypeOrmModule.forFeature([AppSnapshot]),
|
||||
forwardRef(() => ApplicationsModule),
|
||||
KubernetesModule,
|
||||
],
|
||||
controllers: [SnapshotsController],
|
||||
providers: [SnapshotsService],
|
||||
exports: [SnapshotsService],
|
||||
})
|
||||
export class SnapshotsModule {}
|
||||
@@ -0,0 +1,369 @@
|
||||
import { Injectable, Logger, NotFoundException, BadRequestException, Inject, forwardRef } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import { Repository } from 'typeorm';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import { AppSnapshot, SnapshotType, SnapshotStatus } from './entities/snapshot.entity';
|
||||
import { ApplicationsService } from '../applications/applications.service';
|
||||
import { KubernetesService } from '../kubernetes/kubernetes.service';
|
||||
import { AppRuntime, DatabaseType } from '../common/enums';
|
||||
|
||||
const MAX_SNAPSHOTS = 10;
|
||||
|
||||
@Injectable()
|
||||
export class SnapshotsService {
|
||||
private readonly logger = new Logger(SnapshotsService.name);
|
||||
|
||||
constructor(
|
||||
@InjectRepository(AppSnapshot)
|
||||
private snapshotsRepo: Repository<AppSnapshot>,
|
||||
@Inject(forwardRef(() => ApplicationsService))
|
||||
private applicationsService: ApplicationsService,
|
||||
private kubernetesService: KubernetesService,
|
||||
private configService: ConfigService,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Create a snapshot of the current state of an application.
|
||||
* Captures: source code zip, wp-content (for WordPress), and DB dump.
|
||||
*/
|
||||
async createSnapshot(
|
||||
applicationId: string,
|
||||
userId: string,
|
||||
type: SnapshotType = SnapshotType.MANUAL,
|
||||
label?: string,
|
||||
): Promise<AppSnapshot> {
|
||||
const app = await this.applicationsService.findOne(applicationId, userId);
|
||||
|
||||
const snapshot = this.snapshotsRepo.create({
|
||||
applicationId: app.id,
|
||||
createdBy: userId,
|
||||
type,
|
||||
status: SnapshotStatus.IN_PROGRESS,
|
||||
label: label || `Snapshot ${new Date().toLocaleString()}`,
|
||||
imageTag: app.latestImageTag || undefined,
|
||||
hasDatabase: app.databaseType !== DatabaseType.NONE,
|
||||
} as Partial<AppSnapshot>);
|
||||
const saved = await this.snapshotsRepo.save(snapshot);
|
||||
|
||||
// Run snapshot capture async
|
||||
this.captureSnapshot(saved.id, app).catch((err) => {
|
||||
this.logger.error(`Snapshot ${saved.id} capture failed: ${err.message}`);
|
||||
});
|
||||
|
||||
return saved;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a pre-deploy snapshot (called automatically before each deploy).
|
||||
* Does NOT require userId ownership check — called internally.
|
||||
*/
|
||||
async createPreDeploySnapshot(applicationId: string, userId: string, version: string): Promise<AppSnapshot> {
|
||||
// Use findOne without userId to bypass ownership (internal call)
|
||||
const app = await this.applicationsService.findOne(applicationId);
|
||||
|
||||
const snapshot = this.snapshotsRepo.create({
|
||||
applicationId: app.id,
|
||||
createdBy: userId,
|
||||
type: SnapshotType.PRE_DEPLOY,
|
||||
status: SnapshotStatus.IN_PROGRESS,
|
||||
label: `Before deploy ${version}`,
|
||||
imageTag: app.latestImageTag || undefined,
|
||||
hasDatabase: app.databaseType !== DatabaseType.NONE,
|
||||
} as Partial<AppSnapshot>);
|
||||
const saved = await this.snapshotsRepo.save(snapshot);
|
||||
|
||||
// Run snapshot capture (await it for pre-deploy to ensure it completes before deploy)
|
||||
await this.captureSnapshot(saved.id, app);
|
||||
|
||||
return saved;
|
||||
}
|
||||
|
||||
private async captureSnapshot(snapshotId: string, app: any): Promise<void> {
|
||||
const uploadDir = this.configService.get<string>('platform.uploadDir') || './uploads';
|
||||
const snapshotDir = path.join(uploadDir, app.userId, app.id, 'snapshots', snapshotId);
|
||||
fs.mkdirSync(snapshotDir, { recursive: true });
|
||||
|
||||
const updates: Partial<AppSnapshot> = {};
|
||||
|
||||
try {
|
||||
// 1. Copy current source code zip
|
||||
if (app.codePath && fs.existsSync(app.codePath)) {
|
||||
const destPath = path.join(snapshotDir, 'source.zip');
|
||||
fs.copyFileSync(app.codePath, destPath);
|
||||
updates.appArchivePath = destPath;
|
||||
updates.appArchiveSize = fs.statSync(destPath).size;
|
||||
this.logger.log(`Snapshot ${snapshotId}: copied source code (${(updates.appArchiveSize / 1024).toFixed(1)} KB)`);
|
||||
}
|
||||
|
||||
// 2. Archive wp-content for WordPress apps
|
||||
if (app.runtime === AppRuntime.WORDPRESS) {
|
||||
try {
|
||||
const { data, logs } = await this.kubernetesService.archiveWpContent(app);
|
||||
if (data && data.length > 0) {
|
||||
const wpPath = path.join(snapshotDir, 'wp-content.tar.gz');
|
||||
fs.writeFileSync(wpPath, data);
|
||||
updates.wpContentArchivePath = wpPath;
|
||||
updates.wpContentSize = data.length;
|
||||
this.logger.log(`Snapshot ${snapshotId}: archived wp-content (${(data.length / 1024).toFixed(1)} KB)`);
|
||||
} else {
|
||||
this.logger.warn(`Snapshot ${snapshotId}: wp-content archive empty — ${logs}`);
|
||||
}
|
||||
} catch (e: any) {
|
||||
this.logger.warn(`Snapshot ${snapshotId}: wp-content archive failed — ${e.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Dump database
|
||||
if (app.databaseType !== DatabaseType.NONE) {
|
||||
try {
|
||||
const { data, logs } = await this.kubernetesService.exportDatabaseDump(app);
|
||||
if (data && data.length > 0) {
|
||||
const dbPath = path.join(snapshotDir, 'database.sql');
|
||||
fs.writeFileSync(dbPath, data);
|
||||
updates.dbDumpPath = dbPath;
|
||||
updates.dbDumpSize = data.length;
|
||||
this.logger.log(`Snapshot ${snapshotId}: dumped database (${(data.length / 1024).toFixed(1)} KB)`);
|
||||
} else {
|
||||
this.logger.warn(`Snapshot ${snapshotId}: DB dump empty — ${logs}`);
|
||||
}
|
||||
} catch (e: any) {
|
||||
this.logger.warn(`Snapshot ${snapshotId}: DB dump failed — ${e.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
updates.status = SnapshotStatus.COMPLETED;
|
||||
} catch (error: any) {
|
||||
updates.status = SnapshotStatus.FAILED;
|
||||
updates.errorMessage = error.message;
|
||||
this.logger.error(`Snapshot ${snapshotId} failed: ${error.message}`);
|
||||
}
|
||||
|
||||
await this.snapshotsRepo.update(snapshotId, updates);
|
||||
|
||||
// Prune old snapshots (keep max 10)
|
||||
await this.pruneSnapshots(app.id);
|
||||
}
|
||||
|
||||
/**
|
||||
* Keep only the latest MAX_SNAPSHOTS per application. Delete older ones + their files.
|
||||
*/
|
||||
private async pruneSnapshots(applicationId: string): Promise<void> {
|
||||
const all = await this.snapshotsRepo.find({
|
||||
where: { applicationId },
|
||||
order: { createdAt: 'DESC' },
|
||||
});
|
||||
|
||||
if (all.length <= MAX_SNAPSHOTS) return;
|
||||
|
||||
const toDelete = all.slice(MAX_SNAPSHOTS);
|
||||
for (const snap of toDelete) {
|
||||
this.deleteSnapshotFiles(snap);
|
||||
await this.snapshotsRepo.remove(snap);
|
||||
}
|
||||
this.logger.log(`Pruned ${toDelete.length} old snapshot(s) for app ${applicationId}`);
|
||||
}
|
||||
|
||||
private deleteSnapshotFiles(snap: AppSnapshot): void {
|
||||
for (const filePath of [snap.appArchivePath, snap.wpContentArchivePath, snap.dbDumpPath]) {
|
||||
if (filePath && fs.existsSync(filePath)) {
|
||||
try {
|
||||
fs.unlinkSync(filePath);
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
// Try to remove the snapshot directory
|
||||
if (snap.appArchivePath) {
|
||||
const dir = path.dirname(snap.appArchivePath);
|
||||
try {
|
||||
if (fs.existsSync(dir)) fs.rmdirSync(dir);
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* List snapshots for an application (newest first).
|
||||
*/
|
||||
async listSnapshots(applicationId: string, userId: string): Promise<AppSnapshot[]> {
|
||||
// Verify user has access
|
||||
await this.applicationsService.findOne(applicationId, userId);
|
||||
|
||||
return this.snapshotsRepo.find({
|
||||
where: { applicationId },
|
||||
order: { createdAt: 'DESC' },
|
||||
take: MAX_SNAPSHOTS,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a single snapshot (with ownership check).
|
||||
*/
|
||||
async findOne(snapshotId: string, userId: string): Promise<AppSnapshot> {
|
||||
const snapshot = await this.snapshotsRepo.findOne({
|
||||
where: { id: snapshotId },
|
||||
relations: ['application'],
|
||||
});
|
||||
if (!snapshot) throw new NotFoundException('Snapshot not found');
|
||||
|
||||
// Verify ownership
|
||||
await this.applicationsService.findOne(snapshot.applicationId, userId);
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
/**
|
||||
* Download a snapshot artifact (source, wp-content, or database).
|
||||
*/
|
||||
async getDownloadPath(
|
||||
snapshotId: string,
|
||||
userId: string,
|
||||
artifact: 'source' | 'wp-content' | 'database',
|
||||
): Promise<{ filePath: string; fileName: string }> {
|
||||
const snapshot = await this.findOne(snapshotId, userId);
|
||||
|
||||
let filePath: string | null = null;
|
||||
let fileName = '';
|
||||
|
||||
switch (artifact) {
|
||||
case 'source':
|
||||
filePath = snapshot.appArchivePath;
|
||||
fileName = `${snapshot.applicationId}-source-${snapshot.id.slice(0, 8)}.zip`;
|
||||
break;
|
||||
case 'wp-content':
|
||||
filePath = snapshot.wpContentArchivePath;
|
||||
fileName = `${snapshot.applicationId}-wp-content-${snapshot.id.slice(0, 8)}.tar.gz`;
|
||||
break;
|
||||
case 'database':
|
||||
filePath = snapshot.dbDumpPath;
|
||||
fileName = `${snapshot.applicationId}-database-${snapshot.id.slice(0, 8)}.sql`;
|
||||
break;
|
||||
}
|
||||
|
||||
if (!filePath || !fs.existsSync(filePath)) {
|
||||
throw new NotFoundException(`Snapshot artifact "${artifact}" not found or has been deleted`);
|
||||
}
|
||||
|
||||
return { filePath, fileName };
|
||||
}
|
||||
|
||||
/**
|
||||
* Download the CURRENT live state of the app (not from a snapshot).
|
||||
* Creates a temporary archive of the current source code.
|
||||
*/
|
||||
async downloadCurrentSource(applicationId: string, userId: string): Promise<{ filePath: string; fileName: string } | null> {
|
||||
const app = await this.applicationsService.findOne(applicationId, userId);
|
||||
|
||||
if (app.codePath && fs.existsSync(app.codePath)) {
|
||||
return { filePath: app.codePath, fileName: `${app.name}-current-source.zip` };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Download the current live wp-content from the running WordPress app.
|
||||
*/
|
||||
async downloadCurrentWpContent(applicationId: string, userId: string): Promise<Buffer | null> {
|
||||
const app = await this.applicationsService.findOne(applicationId, userId);
|
||||
if (app.runtime !== AppRuntime.WORDPRESS) {
|
||||
throw new BadRequestException('Only WordPress applications have wp-content');
|
||||
}
|
||||
|
||||
const { data } = await this.kubernetesService.archiveWpContent(app);
|
||||
return data;
|
||||
}
|
||||
|
||||
/**
|
||||
* Download the current live database dump.
|
||||
*/
|
||||
async downloadCurrentDatabase(applicationId: string, userId: string): Promise<Buffer | null> {
|
||||
const app = await this.applicationsService.findOne(applicationId, userId);
|
||||
if (app.databaseType === DatabaseType.NONE) {
|
||||
throw new BadRequestException('This application does not have a database');
|
||||
}
|
||||
|
||||
const { data } = await this.kubernetesService.exportDatabaseDump(app);
|
||||
return data;
|
||||
}
|
||||
|
||||
/**
|
||||
* Rollback an application to a specific snapshot.
|
||||
* Restores: source code, wp-content (WordPress), and database.
|
||||
*/
|
||||
async rollbackToSnapshot(snapshotId: string, userId: string): Promise<{ success: boolean; details: string[] }> {
|
||||
const snapshot = await this.findOne(snapshotId, userId);
|
||||
|
||||
if (snapshot.status !== SnapshotStatus.COMPLETED) {
|
||||
throw new BadRequestException('Cannot rollback to an incomplete or failed snapshot');
|
||||
}
|
||||
|
||||
const app = await this.applicationsService.findOne(snapshot.applicationId, userId);
|
||||
const details: string[] = [];
|
||||
|
||||
// 1. Restore source code
|
||||
if (snapshot.appArchivePath && fs.existsSync(snapshot.appArchivePath)) {
|
||||
const uploadDir = this.configService.get<string>('platform.uploadDir') || './uploads';
|
||||
const appDir = path.join(uploadDir, app.userId, app.id);
|
||||
const destPath = path.join(appDir, 'source.zip');
|
||||
fs.mkdirSync(appDir, { recursive: true });
|
||||
fs.copyFileSync(snapshot.appArchivePath, destPath);
|
||||
// Update app codePath in DB
|
||||
await this.applicationsService.update(app.id, app.userId, { codePath: destPath } as any);
|
||||
details.push('✅ Source code restored');
|
||||
this.logger.log(`Rollback ${snapshotId}: restored source code`);
|
||||
}
|
||||
|
||||
// 2. Restore wp-content (WordPress)
|
||||
if (snapshot.wpContentArchivePath && fs.existsSync(snapshot.wpContentArchivePath)) {
|
||||
const archiveBuffer = fs.readFileSync(snapshot.wpContentArchivePath);
|
||||
const result = await this.kubernetesService.restoreWpContent(app, archiveBuffer);
|
||||
if (result.success) {
|
||||
details.push('✅ wp-content restored');
|
||||
this.logger.log(`Rollback ${snapshotId}: restored wp-content`);
|
||||
} else {
|
||||
details.push(`⚠️ wp-content restore failed: ${result.logs}`);
|
||||
this.logger.warn(`Rollback ${snapshotId}: wp-content restore failed`);
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Restore database
|
||||
if (snapshot.dbDumpPath && fs.existsSync(snapshot.dbDumpPath)) {
|
||||
const dumpBuffer = fs.readFileSync(snapshot.dbDumpPath);
|
||||
const result = await this.kubernetesService.restoreDatabaseDump(app, dumpBuffer);
|
||||
if (result.success) {
|
||||
details.push('✅ Database restored');
|
||||
this.logger.log(`Rollback ${snapshotId}: restored database`);
|
||||
} else {
|
||||
details.push(`⚠️ Database restore failed: ${result.logs}`);
|
||||
this.logger.warn(`Rollback ${snapshotId}: database restore failed`);
|
||||
}
|
||||
}
|
||||
|
||||
if (details.length === 0) {
|
||||
details.push('⚠️ No artifacts found in this snapshot to restore');
|
||||
}
|
||||
|
||||
return { success: true, details };
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a specific snapshot.
|
||||
*/
|
||||
async deleteSnapshot(snapshotId: string, userId: string): Promise<void> {
|
||||
const snapshot = await this.findOne(snapshotId, userId);
|
||||
this.deleteSnapshotFiles(snapshot);
|
||||
await this.snapshotsRepo.remove(snapshot);
|
||||
this.logger.log(`Deleted snapshot ${snapshotId}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete all snapshots for an application (called when app is deleted).
|
||||
*/
|
||||
async deleteAllForApplication(applicationId: string): Promise<void> {
|
||||
const all = await this.snapshotsRepo.find({ where: { applicationId } });
|
||||
for (const snap of all) {
|
||||
this.deleteSnapshotFiles(snap);
|
||||
}
|
||||
await this.snapshotsRepo.delete({ applicationId });
|
||||
this.logger.log(`Deleted all snapshots for app ${applicationId}`);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user