feat: database management — custom credentials, dump upload/restore
Backend: - Add dbUsername/dbPassword columns to Application entity - Add optional DB credential fields to CreateApplicationDto - Auto-generate dbPassword (crypto.randomBytes) and default dbUsername='appuser' when databaseType != 'none' on app creation - Store both username and password in K8s DB secret (was password-only) - Read DB_USER/POSTGRES_USER/MYSQL_USER from secretKeyRef instead of hardcoded - New restoreDatabaseDump() in KubernetesService: creates K8s Job with psql/mysql client to restore uploaded SQL dump, waits for completion, returns logs - New POST /applications/:id/db-upload endpoint with 500MB file limit Frontend: - Add dbUsername/dbPassword to Application and CreateApplicationDto types - Deploy page: show username/password fields when database is selected, with generate-random-password button and show/hide toggle - App detail page: new Database section with connection info (host, port, db name, username, password with copy-to-clipboard), SQL dump upload area with drag-and-drop, and restore output logs display Security: - Database remains ClusterIP only (no external exposure) - Credentials stored in K8s Secrets (base64-encoded) - Dump file uploaded as temporary K8s Secret, auto-cleaned after restore
This commit is contained in:
@@ -14,6 +14,7 @@ import {
|
||||
Logger,
|
||||
Inject,
|
||||
forwardRef,
|
||||
BadRequestException,
|
||||
} from '@nestjs/common';
|
||||
import { AuthGuard } from '@nestjs/passport';
|
||||
import { FileInterceptor } from '@nestjs/platform-express';
|
||||
@@ -22,7 +23,7 @@ import { ApplicationsService } from './applications.service';
|
||||
import { CreateApplicationDto, UpdateApplicationDto, ScaleResourcesDto } from './dto/application.dto';
|
||||
import { RolesGuard } from '../common/guards/roles.guard';
|
||||
import { Roles } from '../common/decorators/roles.decorator';
|
||||
import { UserRole } from '../common/enums';
|
||||
import { UserRole, DatabaseType } from '../common/enums';
|
||||
import { KubernetesService } from '../kubernetes/kubernetes.service';
|
||||
import { DeploymentsService } from '../deployments/deployments.service';
|
||||
|
||||
@@ -60,6 +61,38 @@ export class ApplicationsController {
|
||||
return this.applicationsService.uploadCode(id, req.user.id, file);
|
||||
}
|
||||
|
||||
@Post(':id/db-upload')
|
||||
@ApiOperation({ summary: 'Upload and restore a SQL dump into the application database' })
|
||||
@ApiConsumes('multipart/form-data')
|
||||
@UseInterceptors(FileInterceptor('file', {
|
||||
limits: { fileSize: 500 * 1024 * 1024 }, // 500MB for DB dumps
|
||||
}))
|
||||
async uploadDbDump(
|
||||
@Param('id') id: string,
|
||||
@Request() req: any,
|
||||
@UploadedFile() file: Express.Multer.File,
|
||||
) {
|
||||
if (!file) {
|
||||
throw new BadRequestException('No file uploaded');
|
||||
}
|
||||
|
||||
const isStaff = req.user.role === UserRole.ADMIN || req.user.role === UserRole.TECHNICAL;
|
||||
const app = await this.applicationsService.findOne(id, isStaff ? undefined : req.user.id);
|
||||
|
||||
if (app.databaseType === DatabaseType.NONE) {
|
||||
throw new BadRequestException('This application does not have a database configured');
|
||||
}
|
||||
|
||||
this.logger.log(`DB dump upload for ${app.name} — ${(file.size / 1024).toFixed(1)} KB`);
|
||||
const result = await this.kubernetesService.restoreDatabaseDump(app, file.buffer);
|
||||
|
||||
return {
|
||||
success: result.success,
|
||||
message: result.success ? 'Database restored successfully' : 'Database restore failed',
|
||||
logs: result.logs,
|
||||
};
|
||||
}
|
||||
|
||||
@Get()
|
||||
@ApiOperation({ summary: 'List my applications' })
|
||||
async findAll(@Request() req: any) {
|
||||
|
||||
@@ -4,10 +4,11 @@ import { Repository } from 'typeorm';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as crypto from 'crypto';
|
||||
import { Application } from './entities/application.entity';
|
||||
import { CreateApplicationDto, UpdateApplicationDto } from './dto/application.dto';
|
||||
import { ClustersService } from '../clusters/clusters.service';
|
||||
import { UserRole } from '../common/enums';
|
||||
import { UserRole, DatabaseType } from '../common/enums';
|
||||
|
||||
@Injectable()
|
||||
export class ApplicationsService {
|
||||
@@ -66,11 +67,22 @@ export class ApplicationsService {
|
||||
this.logger.log(`Manual cluster assignment for app "${dto.name}" → cluster ${clusterId}`);
|
||||
}
|
||||
|
||||
// Generate database credentials if a database is requested
|
||||
let dbUsername: string | undefined;
|
||||
let dbPassword: string | undefined;
|
||||
if (dto.databaseType && dto.databaseType !== DatabaseType.NONE) {
|
||||
dbUsername = dto.dbUsername?.trim() || 'appuser';
|
||||
dbPassword = dto.dbPassword?.trim() || crypto.randomBytes(16).toString('hex');
|
||||
this.logger.log(`Generated DB credentials for app "${dto.name}" — user: ${dbUsername}`);
|
||||
}
|
||||
|
||||
const app = this.appsRepository.create({
|
||||
...dto,
|
||||
userId,
|
||||
clusterId,
|
||||
poolId,
|
||||
dbUsername,
|
||||
dbPassword,
|
||||
subdomain: `${dto.name}-${userId.split('-')[0]}`,
|
||||
});
|
||||
return this.appsRepository.save(app);
|
||||
|
||||
@@ -32,6 +32,16 @@ export class CreateApplicationDto {
|
||||
@IsEnum(DatabaseType)
|
||||
databaseType: DatabaseType;
|
||||
|
||||
@ApiPropertyOptional({ example: 'appuser', description: 'Database username (default: appuser)' })
|
||||
@IsOptional()
|
||||
@IsString()
|
||||
dbUsername?: string;
|
||||
|
||||
@ApiPropertyOptional({ example: 'mySecurePass123', description: 'Database password (auto-generated if empty)' })
|
||||
@IsOptional()
|
||||
@IsString()
|
||||
dbPassword?: string;
|
||||
|
||||
@ApiPropertyOptional({ example: 'https://github.com/user/repo.git' })
|
||||
@IsOptional()
|
||||
@IsString()
|
||||
|
||||
@@ -29,6 +29,12 @@ export class Application {
|
||||
@Column({ type: 'enum', enum: DatabaseType, default: DatabaseType.NONE })
|
||||
databaseType: DatabaseType;
|
||||
|
||||
@Column({ nullable: true })
|
||||
dbUsername: string;
|
||||
|
||||
@Column({ nullable: true })
|
||||
dbPassword: string;
|
||||
|
||||
@Column({ nullable: true })
|
||||
gitUrl: string;
|
||||
|
||||
|
||||
@@ -23,6 +23,8 @@ interface ManifestContext {
|
||||
databaseType: DatabaseType;
|
||||
domain: string;
|
||||
subdomain: string;
|
||||
dbUsername: string;
|
||||
dbPassword: string;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
@@ -93,6 +95,8 @@ export class KubernetesService implements OnModuleInit {
|
||||
databaseType: app.databaseType,
|
||||
domain: domain,
|
||||
subdomain: app.subdomain || app.name,
|
||||
dbUsername: app.dbUsername || 'appuser',
|
||||
dbPassword: app.dbPassword || this.generatePassword(),
|
||||
};
|
||||
|
||||
const manifests: Record<string, any> = {};
|
||||
@@ -177,7 +181,7 @@ export class KubernetesService implements OnModuleInit {
|
||||
{ name: 'DB_HOST', value: `${ctx.appName}-db` },
|
||||
{ name: 'DB_PORT', value: '5432' },
|
||||
{ name: 'DB_NAME', value: ctx.appName.replace(/-/g, '_') },
|
||||
{ name: 'DB_USER', value: 'appuser' },
|
||||
{ name: 'DB_USER', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'username' } } },
|
||||
{ name: 'DB_PASSWORD', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'password' } } },
|
||||
);
|
||||
} else if (ctx.databaseType === DatabaseType.MYSQL) {
|
||||
@@ -185,7 +189,7 @@ export class KubernetesService implements OnModuleInit {
|
||||
{ name: 'DB_HOST', value: `${ctx.appName}-db` },
|
||||
{ name: 'DB_PORT', value: '3306' },
|
||||
{ name: 'DB_NAME', value: ctx.appName.replace(/-/g, '_') },
|
||||
{ name: 'DB_USER', value: 'appuser' },
|
||||
{ name: 'DB_USER', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'username' } } },
|
||||
{ name: 'DB_PASSWORD', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'password' } } },
|
||||
);
|
||||
}
|
||||
@@ -318,11 +322,10 @@ export class KubernetesService implements OnModuleInit {
|
||||
appsApi: k8s.AppsV1Api,
|
||||
ctx: ManifestContext,
|
||||
): Promise<any> {
|
||||
const dbPassword = this.generatePassword();
|
||||
const dbName = `${ctx.appName}-db`;
|
||||
|
||||
// Create DB secret
|
||||
await this.createDbSecret(coreApi, ctx.namespace, ctx.appName, dbPassword);
|
||||
// Create DB secret with username + password from app entity
|
||||
await this.createDbSecret(coreApi, ctx.namespace, ctx.appName, ctx.dbPassword, ctx.dbUsername);
|
||||
|
||||
// Create PVC for DB
|
||||
await this.createPVC(coreApi, ctx.namespace, dbName, '5Gi');
|
||||
@@ -334,12 +337,12 @@ export class KubernetesService implements OnModuleInit {
|
||||
const envVars = isPostgres
|
||||
? [
|
||||
{ name: 'POSTGRES_DB', value: ctx.appName.replace(/-/g, '_') },
|
||||
{ name: 'POSTGRES_USER', value: 'appuser' },
|
||||
{ name: 'POSTGRES_USER', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'username' } } },
|
||||
{ name: 'POSTGRES_PASSWORD', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'password' } } },
|
||||
]
|
||||
: [
|
||||
{ name: 'MYSQL_DATABASE', value: ctx.appName.replace(/-/g, '_') },
|
||||
{ name: 'MYSQL_USER', value: 'appuser' },
|
||||
{ name: 'MYSQL_USER', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'username' } } },
|
||||
{ name: 'MYSQL_PASSWORD', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'password' } } },
|
||||
{ name: 'MYSQL_ROOT_PASSWORD', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'password' } } },
|
||||
];
|
||||
@@ -407,12 +410,16 @@ export class KubernetesService implements OnModuleInit {
|
||||
namespace: string,
|
||||
appName: string,
|
||||
password: string,
|
||||
username: string = 'appuser',
|
||||
): Promise<void> {
|
||||
const secret = {
|
||||
apiVersion: 'v1',
|
||||
kind: 'Secret',
|
||||
metadata: { name: `${appName}-db-secret`, namespace },
|
||||
data: { password: Buffer.from(password).toString('base64') },
|
||||
data: {
|
||||
username: Buffer.from(username).toString('base64'),
|
||||
password: Buffer.from(password).toString('base64'),
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
@@ -787,6 +794,165 @@ export class KubernetesService implements OnModuleInit {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Restore a SQL dump file into the application's database.
|
||||
* Creates a K8s Job that runs psql/mysql to import the dump.
|
||||
*/
|
||||
async restoreDatabaseDump(app: Application, fileBuffer: Buffer): Promise<{ success: boolean; logs: string }> {
|
||||
const { coreApi, kc } = await this.getK8sClient(app.clusterId);
|
||||
const batchApi = kc.makeApiClient(k8s.BatchV1Api);
|
||||
const namespace = `user-${app.userId.split('-')[0]}`;
|
||||
const dbName = `${app.name}-db`;
|
||||
const jobName = `${app.name}-db-restore-${Date.now()}`;
|
||||
const secretName = `${jobName}-dump`;
|
||||
const isPostgres = app.databaseType === DatabaseType.POSTGRESQL;
|
||||
const dbDatabase = app.name.replace(/-/g, '_');
|
||||
|
||||
// 1. Create a temporary secret holding the dump file
|
||||
const dumpSecret = {
|
||||
apiVersion: 'v1',
|
||||
kind: 'Secret',
|
||||
metadata: { name: secretName, namespace },
|
||||
data: {
|
||||
'dump.sql': fileBuffer.toString('base64'),
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
await coreApi.createNamespacedSecret(namespace, dumpSecret);
|
||||
} catch (e: any) {
|
||||
this.logger.error(`Failed to create dump secret: ${e.message}`);
|
||||
throw new Error('Failed to prepare database dump for restore');
|
||||
}
|
||||
|
||||
// 2. Build the restore command
|
||||
const command = isPostgres
|
||||
? [
|
||||
'sh', '-c',
|
||||
`PGPASSWORD="$DB_PASSWORD" psql -h ${dbName} -U "$DB_USER" -d ${dbDatabase} -f /dump/dump.sql 2>&1`,
|
||||
]
|
||||
: [
|
||||
'sh', '-c',
|
||||
`mysql -h ${dbName} -u "$DB_USER" -p"$DB_PASSWORD" ${dbDatabase} < /dump/dump.sql 2>&1`,
|
||||
];
|
||||
|
||||
const image = isPostgres ? 'postgres:16-alpine' : 'mysql:8.0';
|
||||
|
||||
// 3. Create the restore Job
|
||||
const job: k8s.V1Job = {
|
||||
apiVersion: 'batch/v1',
|
||||
kind: 'Job',
|
||||
metadata: { name: jobName, namespace },
|
||||
spec: {
|
||||
ttlSecondsAfterFinished: 300,
|
||||
backoffLimit: 0,
|
||||
template: {
|
||||
spec: {
|
||||
restartPolicy: 'Never',
|
||||
containers: [
|
||||
{
|
||||
name: 'restore',
|
||||
image,
|
||||
command,
|
||||
env: [
|
||||
{ name: 'DB_USER', valueFrom: { secretKeyRef: { name: `${app.name}-db-secret`, key: 'username' } } },
|
||||
{ name: 'DB_PASSWORD', valueFrom: { secretKeyRef: { name: `${app.name}-db-secret`, key: 'password' } } },
|
||||
],
|
||||
volumeMounts: [
|
||||
{ name: 'dump-volume', mountPath: '/dump', readOnly: true },
|
||||
],
|
||||
resources: {
|
||||
requests: { cpu: '100m', memory: '128Mi' },
|
||||
limits: { cpu: '500m', memory: '512Mi' },
|
||||
},
|
||||
},
|
||||
],
|
||||
volumes: [
|
||||
{
|
||||
name: 'dump-volume',
|
||||
secret: { secretName },
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
await batchApi.createNamespacedJob(namespace, job);
|
||||
this.logger.log(`Created DB restore job ${jobName} for ${app.name}`);
|
||||
} catch (e: any) {
|
||||
// Clean up the dump secret on failure
|
||||
try { await coreApi.deleteNamespacedSecret(secretName, namespace); } catch {}
|
||||
this.logger.error(`Failed to create restore job: ${e.message}`);
|
||||
throw new Error('Failed to create database restore job');
|
||||
}
|
||||
|
||||
// 4. Wait for the job to complete (max 5 minutes)
|
||||
const timeout = 300_000;
|
||||
const start = Date.now();
|
||||
let completed = false;
|
||||
let failed = false;
|
||||
|
||||
while (Date.now() - start < timeout) {
|
||||
await new Promise((r) => setTimeout(r, 3000));
|
||||
try {
|
||||
const jobStatus = await batchApi.readNamespacedJob(jobName, namespace);
|
||||
const status = jobStatus.body.status;
|
||||
if (status?.succeeded && status.succeeded > 0) {
|
||||
completed = true;
|
||||
break;
|
||||
}
|
||||
if (status?.failed && status.failed > 0) {
|
||||
failed = true;
|
||||
break;
|
||||
}
|
||||
} catch {
|
||||
// Job may not be ready yet
|
||||
}
|
||||
}
|
||||
|
||||
// 5. Get logs from the job pod
|
||||
let logs = '';
|
||||
try {
|
||||
const pods = await coreApi.listNamespacedPod(
|
||||
namespace,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
`job-name=${jobName}`,
|
||||
);
|
||||
if (pods.body.items.length > 0) {
|
||||
const podName = pods.body.items[0].metadata?.name;
|
||||
if (podName) {
|
||||
const logResponse = await coreApi.readNamespacedPodLog(podName, namespace);
|
||||
logs = logResponse.body || '';
|
||||
}
|
||||
}
|
||||
} catch (e: any) {
|
||||
this.logger.warn(`Could not get restore job logs: ${e.message}`);
|
||||
}
|
||||
|
||||
// 6. Clean up the dump secret
|
||||
try {
|
||||
await coreApi.deleteNamespacedSecret(secretName, namespace);
|
||||
} catch {}
|
||||
|
||||
if (!completed && !failed) {
|
||||
this.logger.warn(`DB restore job ${jobName} timed out`);
|
||||
return { success: false, logs: logs || 'Restore job timed out after 5 minutes' };
|
||||
}
|
||||
|
||||
if (failed) {
|
||||
this.logger.warn(`DB restore job ${jobName} failed`);
|
||||
return { success: false, logs: logs || 'Restore job failed' };
|
||||
}
|
||||
|
||||
this.logger.log(`DB restore for ${app.name} completed successfully`);
|
||||
return { success: true, logs };
|
||||
}
|
||||
|
||||
private generatePassword(length = 24): string {
|
||||
const chars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!@#$%';
|
||||
let password = '';
|
||||
|
||||
Reference in New Issue
Block a user