feat: database management — custom credentials, dump upload/restore

Backend:
- Add dbUsername/dbPassword columns to Application entity
- Add optional DB credential fields to CreateApplicationDto
- Auto-generate dbPassword (crypto.randomBytes) and default dbUsername='appuser'
  when databaseType != 'none' on app creation
- Store both username and password in K8s DB secret (was password-only)
- Read DB_USER/POSTGRES_USER/MYSQL_USER from secretKeyRef instead of hardcoded
- New restoreDatabaseDump() in KubernetesService: creates K8s Job with
  psql/mysql client to restore uploaded SQL dump, waits for completion,
  returns logs
- New POST /applications/:id/db-upload endpoint with 500MB file limit

Frontend:
- Add dbUsername/dbPassword to Application and CreateApplicationDto types
- Deploy page: show username/password fields when database is selected,
  with generate-random-password button and show/hide toggle
- App detail page: new Database section with connection info (host, port,
  db name, username, password with copy-to-clipboard), SQL dump upload
  area with drag-and-drop, and restore output logs display

Security:
- Database remains ClusterIP only (no external exposure)
- Credentials stored in K8s Secrets (base64-encoded)
- Dump file uploaded as temporary K8s Secret, auto-cleaned after restore
This commit is contained in:
keyhan
2026-04-06 22:47:41 +03:30
parent 3c3e0e48fa
commit 9e3347cb71
9 changed files with 496 additions and 13 deletions
@@ -14,6 +14,7 @@ import {
Logger,
Inject,
forwardRef,
BadRequestException,
} from '@nestjs/common';
import { AuthGuard } from '@nestjs/passport';
import { FileInterceptor } from '@nestjs/platform-express';
@@ -22,7 +23,7 @@ import { ApplicationsService } from './applications.service';
import { CreateApplicationDto, UpdateApplicationDto, ScaleResourcesDto } from './dto/application.dto';
import { RolesGuard } from '../common/guards/roles.guard';
import { Roles } from '../common/decorators/roles.decorator';
import { UserRole } from '../common/enums';
import { UserRole, DatabaseType } from '../common/enums';
import { KubernetesService } from '../kubernetes/kubernetes.service';
import { DeploymentsService } from '../deployments/deployments.service';
@@ -60,6 +61,38 @@ export class ApplicationsController {
return this.applicationsService.uploadCode(id, req.user.id, file);
}
@Post(':id/db-upload')
@ApiOperation({ summary: 'Upload and restore a SQL dump into the application database' })
@ApiConsumes('multipart/form-data')
@UseInterceptors(FileInterceptor('file', {
limits: { fileSize: 500 * 1024 * 1024 }, // 500MB for DB dumps
}))
async uploadDbDump(
@Param('id') id: string,
@Request() req: any,
@UploadedFile() file: Express.Multer.File,
) {
if (!file) {
throw new BadRequestException('No file uploaded');
}
const isStaff = req.user.role === UserRole.ADMIN || req.user.role === UserRole.TECHNICAL;
const app = await this.applicationsService.findOne(id, isStaff ? undefined : req.user.id);
if (app.databaseType === DatabaseType.NONE) {
throw new BadRequestException('This application does not have a database configured');
}
this.logger.log(`DB dump upload for ${app.name}${(file.size / 1024).toFixed(1)} KB`);
const result = await this.kubernetesService.restoreDatabaseDump(app, file.buffer);
return {
success: result.success,
message: result.success ? 'Database restored successfully' : 'Database restore failed',
logs: result.logs,
};
}
@Get()
@ApiOperation({ summary: 'List my applications' })
async findAll(@Request() req: any) {
@@ -4,10 +4,11 @@ import { Repository } from 'typeorm';
import { ConfigService } from '@nestjs/config';
import * as fs from 'fs';
import * as path from 'path';
import * as crypto from 'crypto';
import { Application } from './entities/application.entity';
import { CreateApplicationDto, UpdateApplicationDto } from './dto/application.dto';
import { ClustersService } from '../clusters/clusters.service';
import { UserRole } from '../common/enums';
import { UserRole, DatabaseType } from '../common/enums';
@Injectable()
export class ApplicationsService {
@@ -66,11 +67,22 @@ export class ApplicationsService {
this.logger.log(`Manual cluster assignment for app "${dto.name}" → cluster ${clusterId}`);
}
// Generate database credentials if a database is requested
let dbUsername: string | undefined;
let dbPassword: string | undefined;
if (dto.databaseType && dto.databaseType !== DatabaseType.NONE) {
dbUsername = dto.dbUsername?.trim() || 'appuser';
dbPassword = dto.dbPassword?.trim() || crypto.randomBytes(16).toString('hex');
this.logger.log(`Generated DB credentials for app "${dto.name}" — user: ${dbUsername}`);
}
const app = this.appsRepository.create({
...dto,
userId,
clusterId,
poolId,
dbUsername,
dbPassword,
subdomain: `${dto.name}-${userId.split('-')[0]}`,
});
return this.appsRepository.save(app);
@@ -32,6 +32,16 @@ export class CreateApplicationDto {
@IsEnum(DatabaseType)
databaseType: DatabaseType;
@ApiPropertyOptional({ example: 'appuser', description: 'Database username (default: appuser)' })
@IsOptional()
@IsString()
dbUsername?: string;
@ApiPropertyOptional({ example: 'mySecurePass123', description: 'Database password (auto-generated if empty)' })
@IsOptional()
@IsString()
dbPassword?: string;
@ApiPropertyOptional({ example: 'https://github.com/user/repo.git' })
@IsOptional()
@IsString()
@@ -29,6 +29,12 @@ export class Application {
@Column({ type: 'enum', enum: DatabaseType, default: DatabaseType.NONE })
databaseType: DatabaseType;
@Column({ nullable: true })
dbUsername: string;
@Column({ nullable: true })
dbPassword: string;
@Column({ nullable: true })
gitUrl: string;
+174 -8
View File
@@ -23,6 +23,8 @@ interface ManifestContext {
databaseType: DatabaseType;
domain: string;
subdomain: string;
dbUsername: string;
dbPassword: string;
}
@Injectable()
@@ -93,6 +95,8 @@ export class KubernetesService implements OnModuleInit {
databaseType: app.databaseType,
domain: domain,
subdomain: app.subdomain || app.name,
dbUsername: app.dbUsername || 'appuser',
dbPassword: app.dbPassword || this.generatePassword(),
};
const manifests: Record<string, any> = {};
@@ -177,7 +181,7 @@ export class KubernetesService implements OnModuleInit {
{ name: 'DB_HOST', value: `${ctx.appName}-db` },
{ name: 'DB_PORT', value: '5432' },
{ name: 'DB_NAME', value: ctx.appName.replace(/-/g, '_') },
{ name: 'DB_USER', value: 'appuser' },
{ name: 'DB_USER', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'username' } } },
{ name: 'DB_PASSWORD', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'password' } } },
);
} else if (ctx.databaseType === DatabaseType.MYSQL) {
@@ -185,7 +189,7 @@ export class KubernetesService implements OnModuleInit {
{ name: 'DB_HOST', value: `${ctx.appName}-db` },
{ name: 'DB_PORT', value: '3306' },
{ name: 'DB_NAME', value: ctx.appName.replace(/-/g, '_') },
{ name: 'DB_USER', value: 'appuser' },
{ name: 'DB_USER', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'username' } } },
{ name: 'DB_PASSWORD', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'password' } } },
);
}
@@ -318,11 +322,10 @@ export class KubernetesService implements OnModuleInit {
appsApi: k8s.AppsV1Api,
ctx: ManifestContext,
): Promise<any> {
const dbPassword = this.generatePassword();
const dbName = `${ctx.appName}-db`;
// Create DB secret
await this.createDbSecret(coreApi, ctx.namespace, ctx.appName, dbPassword);
// Create DB secret with username + password from app entity
await this.createDbSecret(coreApi, ctx.namespace, ctx.appName, ctx.dbPassword, ctx.dbUsername);
// Create PVC for DB
await this.createPVC(coreApi, ctx.namespace, dbName, '5Gi');
@@ -334,12 +337,12 @@ export class KubernetesService implements OnModuleInit {
const envVars = isPostgres
? [
{ name: 'POSTGRES_DB', value: ctx.appName.replace(/-/g, '_') },
{ name: 'POSTGRES_USER', value: 'appuser' },
{ name: 'POSTGRES_USER', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'username' } } },
{ name: 'POSTGRES_PASSWORD', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'password' } } },
]
: [
{ name: 'MYSQL_DATABASE', value: ctx.appName.replace(/-/g, '_') },
{ name: 'MYSQL_USER', value: 'appuser' },
{ name: 'MYSQL_USER', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'username' } } },
{ name: 'MYSQL_PASSWORD', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'password' } } },
{ name: 'MYSQL_ROOT_PASSWORD', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'password' } } },
];
@@ -407,12 +410,16 @@ export class KubernetesService implements OnModuleInit {
namespace: string,
appName: string,
password: string,
username: string = 'appuser',
): Promise<void> {
const secret = {
apiVersion: 'v1',
kind: 'Secret',
metadata: { name: `${appName}-db-secret`, namespace },
data: { password: Buffer.from(password).toString('base64') },
data: {
username: Buffer.from(username).toString('base64'),
password: Buffer.from(password).toString('base64'),
},
};
try {
@@ -787,6 +794,165 @@ export class KubernetesService implements OnModuleInit {
}
}
/**
* Restore a SQL dump file into the application's database.
* Creates a K8s Job that runs psql/mysql to import the dump.
*/
async restoreDatabaseDump(app: Application, fileBuffer: Buffer): Promise<{ success: boolean; logs: string }> {
const { coreApi, kc } = await this.getK8sClient(app.clusterId);
const batchApi = kc.makeApiClient(k8s.BatchV1Api);
const namespace = `user-${app.userId.split('-')[0]}`;
const dbName = `${app.name}-db`;
const jobName = `${app.name}-db-restore-${Date.now()}`;
const secretName = `${jobName}-dump`;
const isPostgres = app.databaseType === DatabaseType.POSTGRESQL;
const dbDatabase = app.name.replace(/-/g, '_');
// 1. Create a temporary secret holding the dump file
const dumpSecret = {
apiVersion: 'v1',
kind: 'Secret',
metadata: { name: secretName, namespace },
data: {
'dump.sql': fileBuffer.toString('base64'),
},
};
try {
await coreApi.createNamespacedSecret(namespace, dumpSecret);
} catch (e: any) {
this.logger.error(`Failed to create dump secret: ${e.message}`);
throw new Error('Failed to prepare database dump for restore');
}
// 2. Build the restore command
const command = isPostgres
? [
'sh', '-c',
`PGPASSWORD="$DB_PASSWORD" psql -h ${dbName} -U "$DB_USER" -d ${dbDatabase} -f /dump/dump.sql 2>&1`,
]
: [
'sh', '-c',
`mysql -h ${dbName} -u "$DB_USER" -p"$DB_PASSWORD" ${dbDatabase} < /dump/dump.sql 2>&1`,
];
const image = isPostgres ? 'postgres:16-alpine' : 'mysql:8.0';
// 3. Create the restore Job
const job: k8s.V1Job = {
apiVersion: 'batch/v1',
kind: 'Job',
metadata: { name: jobName, namespace },
spec: {
ttlSecondsAfterFinished: 300,
backoffLimit: 0,
template: {
spec: {
restartPolicy: 'Never',
containers: [
{
name: 'restore',
image,
command,
env: [
{ name: 'DB_USER', valueFrom: { secretKeyRef: { name: `${app.name}-db-secret`, key: 'username' } } },
{ name: 'DB_PASSWORD', valueFrom: { secretKeyRef: { name: `${app.name}-db-secret`, key: 'password' } } },
],
volumeMounts: [
{ name: 'dump-volume', mountPath: '/dump', readOnly: true },
],
resources: {
requests: { cpu: '100m', memory: '128Mi' },
limits: { cpu: '500m', memory: '512Mi' },
},
},
],
volumes: [
{
name: 'dump-volume',
secret: { secretName },
},
],
},
},
},
};
try {
await batchApi.createNamespacedJob(namespace, job);
this.logger.log(`Created DB restore job ${jobName} for ${app.name}`);
} catch (e: any) {
// Clean up the dump secret on failure
try { await coreApi.deleteNamespacedSecret(secretName, namespace); } catch {}
this.logger.error(`Failed to create restore job: ${e.message}`);
throw new Error('Failed to create database restore job');
}
// 4. Wait for the job to complete (max 5 minutes)
const timeout = 300_000;
const start = Date.now();
let completed = false;
let failed = false;
while (Date.now() - start < timeout) {
await new Promise((r) => setTimeout(r, 3000));
try {
const jobStatus = await batchApi.readNamespacedJob(jobName, namespace);
const status = jobStatus.body.status;
if (status?.succeeded && status.succeeded > 0) {
completed = true;
break;
}
if (status?.failed && status.failed > 0) {
failed = true;
break;
}
} catch {
// Job may not be ready yet
}
}
// 5. Get logs from the job pod
let logs = '';
try {
const pods = await coreApi.listNamespacedPod(
namespace,
undefined,
undefined,
undefined,
undefined,
`job-name=${jobName}`,
);
if (pods.body.items.length > 0) {
const podName = pods.body.items[0].metadata?.name;
if (podName) {
const logResponse = await coreApi.readNamespacedPodLog(podName, namespace);
logs = logResponse.body || '';
}
}
} catch (e: any) {
this.logger.warn(`Could not get restore job logs: ${e.message}`);
}
// 6. Clean up the dump secret
try {
await coreApi.deleteNamespacedSecret(secretName, namespace);
} catch {}
if (!completed && !failed) {
this.logger.warn(`DB restore job ${jobName} timed out`);
return { success: false, logs: logs || 'Restore job timed out after 5 minutes' };
}
if (failed) {
this.logger.warn(`DB restore job ${jobName} failed`);
return { success: false, logs: logs || 'Restore job failed' };
}
this.logger.log(`DB restore for ${app.name} completed successfully`);
return { success: true, logs };
}
private generatePassword(length = 24): string {
const chars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!@#$%';
let password = '';