fix(apps): always attempt K8s cleanup on delete, cluster service fixes

- Delete endpoint no longer requires clusterId && latestImageTag
- Clusters: getDefault fallback logic, reassign apps on delete
- Unit tests for cluster default/delete logic
This commit is contained in:
keyhan
2026-04-22 16:44:43 +03:30
parent 8ca787d46e
commit 5f6eccc483
7 changed files with 423 additions and 38 deletions
@@ -62,7 +62,7 @@ export class ApplicationsController {
}
@Post(':id/db-upload')
@ApiOperation({ summary: 'Upload and restore a SQL dump into the application database' })
@ApiOperation({ summary: 'Upload a SQL dump file for later restore during deployment' })
@ApiConsumes('multipart/form-data')
@UseInterceptors(FileInterceptor('file', {
limits: { fileSize: 500 * 1024 * 1024 }, // 500MB for DB dumps
@@ -84,12 +84,13 @@ export class ApplicationsController {
}
this.logger.log(`DB dump upload for ${app.name}${(file.size / 1024).toFixed(1)} KB`);
const result = await this.kubernetesService.restoreDatabaseDump(app, file.buffer);
// Save to disk (restore happens after deploy when namespace exists)
await this.applicationsService.uploadDbDump(id, isStaff ? app.userId : req.user.id, file);
return {
success: result.success,
message: result.success ? 'Database restored successfully' : 'Database restore failed',
logs: result.logs,
success: true,
message: 'Database dump uploaded. It will be restored automatically after deployment.',
};
}
@@ -225,12 +226,10 @@ export class ApplicationsController {
// 1. Get the app first
const app = await this.applicationsService.findOne(id, isStaff ? undefined : req.user.id);
// 2. Delete K8s resources (deployment, service, ingress, db, secrets)
// 2. Delete K8s resources (deployment, service, ingress, db, secrets, PVCs)
try {
if (app.clusterId && app.latestImageTag) {
await this.kubernetesService.deleteApplication(app);
this.logger.log(`Deleted K8s resources for ${app.name}`);
}
await this.kubernetesService.deleteApplication(app);
this.logger.log(`Deleted K8s resources for ${app.name}`);
} catch (e: any) {
this.logger.warn(`K8s cleanup failed for ${app.name}: ${e.message}`);
}
@@ -188,4 +188,28 @@ export class ApplicationsService {
this.logger.log(`Uploaded code for ${app.name}${zipPath} (${(file.size / 1024).toFixed(1)} KB)`);
return saved;
}
async uploadDbDump(id: string, userId: string, file: Express.Multer.File): Promise<Application> {
if (!file) {
throw new BadRequestException('No file uploaded');
}
const app = await this.findOne(id, userId);
const uploadDir = this.configService.get<string>('platform.uploadDir') || './uploads';
const appDir = path.join(uploadDir, app.userId, app.id);
// Ensure directory exists
fs.mkdirSync(appDir, { recursive: true });
// Save the SQL dump file
const dumpPath = path.join(appDir, 'dump.sql');
fs.writeFileSync(dumpPath, file.buffer);
// Update app with dump path
app.dbDumpPath = dumpPath;
const saved = await this.appsRepository.save(app);
this.logger.log(`Uploaded DB dump for ${app.name}${dumpPath} (${(file.size / 1024).toFixed(1)} KB)`);
return saved;
}
}
@@ -0,0 +1,105 @@
import { ClusterStatus } from '../common/enums';
/**
* Tests for ClustersService — getDefault and delete logic.
*/
describe('ClustersService getDefault logic', () => {
// Simulate the fixed getDefault behavior
function getDefault(clusters: { id: string; isDefault: boolean; status: string }[]): { id: string } | null {
// Step 1: active + default
let result = clusters.find(c => c.isDefault && c.status === ClusterStatus.ACTIVE);
if (result) return { id: result.id };
// Step 2: any active (fallback)
result = clusters.find(c => c.status === ClusterStatus.ACTIVE);
if (result) return { id: result.id };
return null;
}
it('should return active default cluster', () => {
const clusters = [
{ id: '1', isDefault: true, status: ClusterStatus.ACTIVE },
{ id: '2', isDefault: false, status: ClusterStatus.ACTIVE },
];
expect(getDefault(clusters)?.id).toBe('1');
});
it('should skip inactive default and return active cluster', () => {
const clusters = [
{ id: '1', isDefault: true, status: ClusterStatus.INACTIVE },
{ id: '2', isDefault: false, status: ClusterStatus.ACTIVE },
];
expect(getDefault(clusters)?.id).toBe('2');
});
it('should return null when no active clusters exist', () => {
const clusters = [
{ id: '1', isDefault: true, status: ClusterStatus.INACTIVE },
];
expect(getDefault(clusters)).toBeNull();
});
it('should handle both clusters being default (picks active one)', () => {
const clusters = [
{ id: 'inactive', isDefault: true, status: ClusterStatus.INACTIVE },
{ id: 'active', isDefault: true, status: ClusterStatus.ACTIVE },
];
expect(getDefault(clusters)?.id).toBe('active');
});
});
describe('ClustersService delete logic', () => {
it('should reassign apps to replacement cluster on delete', () => {
// Simulate: cluster A (being deleted) has 3 apps, cluster B is active
const apps = [
{ id: 'app1', clusterId: 'A' },
{ id: 'app2', clusterId: 'A' },
{ id: 'app3', clusterId: 'B' },
];
const deletedClusterId = 'A';
const replacementId = 'B';
// Reassign
for (const app of apps) {
if (app.clusterId === deletedClusterId) {
app.clusterId = replacementId;
}
}
expect(apps.filter(a => a.clusterId === 'A')).toHaveLength(0);
expect(apps.filter(a => a.clusterId === 'B')).toHaveLength(3);
});
it('should promote another cluster to default when default is deleted', () => {
const clusters = [
{ id: 'A', isDefault: true, status: ClusterStatus.ACTIVE },
{ id: 'B', isDefault: false, status: ClusterStatus.ACTIVE },
];
// Delete A
const deleted = clusters.splice(0, 1)[0];
expect(deleted.isDefault).toBe(true);
// Promote
const newDefault = clusters.find(c => c.status === ClusterStatus.ACTIVE);
if (newDefault) newDefault.isDefault = true;
expect(clusters[0].isDefault).toBe(true);
expect(clusters[0].id).toBe('B');
});
it('should nullify clusterId when no replacement cluster exists', () => {
const apps = [{ id: 'app1', clusterId: 'A' as string | null }];
const hasReplacement = false;
if (!hasReplacement) {
for (const app of apps) {
app.clusterId = null;
}
}
expect(apps[0].clusterId).toBeNull();
});
});
+262 -2
View File
@@ -1,5 +1,6 @@
import { Injectable, NotFoundException, Logger, BadRequestException } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { ConfigService } from '@nestjs/config';
import { Repository, DataSource, In } from 'typeorm';
import * as k8s from '@kubernetes/client-node';
import { Cluster } from './entities/cluster.entity';
@@ -20,6 +21,7 @@ export class ClustersService {
@InjectRepository(ClusterPool)
private poolsRepository: Repository<ClusterPool>,
private dataSource: DataSource,
private configService: ConfigService,
) {}
/**
@@ -73,6 +75,12 @@ export class ClustersService {
});
const saved = await this.clustersRepository.save(cluster);
this.logger.log(`Cluster "${saved.name}" registered (active) — K8s ${connectionTest.version}`);
// Bootstrap the cluster with build infrastructure (namespace, registry, SA, etc.)
this.bootstrapCluster(saved.kubeconfig).catch((err) => {
this.logger.error(`Failed to bootstrap cluster "${saved.name}": ${err.message}`);
});
return saved;
}
@@ -92,9 +100,23 @@ export class ClustersService {
}
async getDefault(): Promise<Cluster> {
const cluster = await this.clustersRepository.findOne({ where: { isDefault: true } });
// Prefer active default cluster; fall back to any active cluster
let cluster = await this.clustersRepository.findOne({
where: { isDefault: true, status: ClusterStatus.ACTIVE },
});
if (!cluster) {
throw new NotFoundException('No default cluster configured');
// Fallback: pick any active cluster and promote it to default
cluster = await this.clustersRepository.findOne({
where: { status: ClusterStatus.ACTIVE },
});
if (cluster) {
cluster.isDefault = true;
await this.clustersRepository.save(cluster);
this.logger.warn(`No active default cluster — promoted "${cluster.name}" to default`);
}
}
if (!cluster) {
throw new NotFoundException('No active cluster available');
}
return cluster;
}
@@ -112,6 +134,11 @@ export class ClustersService {
}
dto.status = ClusterStatus.ACTIVE;
this.logger.log(`Cluster "${cluster.name}" kubeconfig updated — connection verified (K8s ${connectionTest.version})`);
// Re-bootstrap build infrastructure on the new/updated cluster
this.bootstrapCluster(dto.kubeconfig).catch((err: any) => {
this.logger.error(`Failed to bootstrap cluster "${cluster.name}": ${err.message}`);
});
}
if (dto.isDefault === true) {
@@ -206,7 +233,55 @@ export class ClustersService {
async delete(id: string): Promise<void> {
const cluster = await this.findOne(id);
const wasDefault = cluster.isDefault;
// Reassign applications that were on this cluster to another active cluster
try {
const replacement = await this.clustersRepository.findOne({
where: { status: ClusterStatus.ACTIVE, id: undefined as any },
});
// Use raw query to exclude the deleted cluster
const activeReplacement = await this.clustersRepository
.createQueryBuilder('c')
.where('c.id != :id', { id })
.andWhere('c.status = :status', { status: ClusterStatus.ACTIVE })
.getOne();
if (activeReplacement) {
const result = await this.dataSource.query(
`UPDATE applications SET "clusterId" = $1 WHERE "clusterId" = $2`,
[activeReplacement.id, id],
);
const count = result?.[1] || 0;
if (count > 0) {
this.logger.log(`Reassigned ${count} application(s) from cluster "${cluster.name}" to "${activeReplacement.name}"`);
}
} else {
// No replacement — nullify clusterId so apps aren't orphaned with a dangling FK
await this.dataSource.query(
`UPDATE applications SET "clusterId" = NULL WHERE "clusterId" = $1`,
[id],
);
this.logger.warn(`No active replacement cluster — cleared clusterId for apps on "${cluster.name}"`);
}
} catch (e: any) {
this.logger.warn(`Failed to reassign apps from cluster "${cluster.name}": ${e.message}`);
}
await this.clustersRepository.remove(cluster);
this.logger.log(`Cluster "${cluster.name}" deleted`);
// If deleted cluster was default, promote another active cluster
if (wasDefault) {
const newDefault = await this.clustersRepository.findOne({
where: { status: ClusterStatus.ACTIVE },
});
if (newDefault) {
newDefault.isDefault = true;
await this.clustersRepository.save(newDefault);
this.logger.log(`Promoted cluster "${newDefault.name}" to default after deleting "${cluster.name}"`);
}
}
}
// ─── Cluster Pool Methods ─────────────────────────────────────────
@@ -429,6 +504,191 @@ export class ClustersService {
}
}
/**
* Bootstrap a newly-added cluster with the build infrastructure:
* 1. cloudhost-builds namespace
* 2. kaniko-builder ServiceAccount
* 3. Docker Registry Deployment + PVC + Service (ClusterIP) + NodePort Service
* 4. registry-credentials Secret (for Kaniko docker auth)
*/
async bootstrapCluster(kubeconfig: string): Promise<void> {
const kc = new k8s.KubeConfig();
kc.loadFromString(kubeconfig);
const coreApi = kc.makeApiClient(k8s.CoreV1Api);
const appsApi = kc.makeApiClient(k8s.AppsV1Api);
const buildNs = this.configService.get<string>('build.namespace') || 'cloudhost-builds';
const saName = this.configService.get<string>('build.serviceAccount') || 'kaniko-builder';
const registryUrl = this.configService.get<string>('registry.url') || `registry.${buildNs}.svc.cluster.local:5000`;
this.logger.log(`Bootstrapping cluster — namespace: ${buildNs}`);
// ── 1. Namespace ──────────────────────────────────────────────
try {
await coreApi.readNamespace(buildNs);
this.logger.log(`Namespace "${buildNs}" already exists`);
} catch (err: any) {
if (err.statusCode === 404 || err.body?.code === 404) {
await coreApi.createNamespace({ metadata: { name: buildNs } });
this.logger.log(`Created namespace "${buildNs}"`);
} else {
throw err;
}
}
// ── 2. ServiceAccount for Kaniko ──────────────────────────────
try {
await coreApi.readNamespacedServiceAccount(saName, buildNs);
this.logger.log(`ServiceAccount "${saName}" already exists`);
} catch (err: any) {
if (err.statusCode === 404 || err.body?.code === 404) {
await coreApi.createNamespacedServiceAccount(buildNs, {
metadata: { name: saName, namespace: buildNs },
});
this.logger.log(`Created ServiceAccount "${saName}"`);
} else {
throw err;
}
}
// ── 3. Docker Registry PVC ────────────────────────────────────
const registryPvcName = 'registry-data';
try {
await coreApi.readNamespacedPersistentVolumeClaim(registryPvcName, buildNs);
this.logger.log(`PVC "${registryPvcName}" already exists`);
} catch (err: any) {
if (err.statusCode === 404 || err.body?.code === 404) {
await coreApi.createNamespacedPersistentVolumeClaim(buildNs, {
metadata: { name: registryPvcName, namespace: buildNs },
spec: {
accessModes: ['ReadWriteOnce'],
resources: { requests: { storage: '10Gi' } },
},
});
this.logger.log(`Created PVC "${registryPvcName}" (10Gi)`);
} else {
throw err;
}
}
// ── 4. Docker Registry Deployment ─────────────────────────────
const registryDeployName = 'registry';
try {
await appsApi.readNamespacedDeployment(registryDeployName, buildNs);
this.logger.log(`Deployment "${registryDeployName}" already exists`);
} catch (err: any) {
if (err.statusCode === 404 || err.body?.code === 404) {
await appsApi.createNamespacedDeployment(buildNs, {
metadata: { name: registryDeployName, namespace: buildNs, labels: { app: 'registry' } },
spec: {
replicas: 1,
selector: { matchLabels: { app: 'registry' } },
template: {
metadata: { labels: { app: 'registry' } },
spec: {
containers: [{
name: 'registry',
image: 'registry:2',
ports: [{ containerPort: 5000 }],
env: [
{ name: 'REGISTRY_STORAGE_DELETE_ENABLED', value: 'true' },
],
volumeMounts: [{
name: 'registry-data',
mountPath: '/var/lib/registry',
}],
resources: {
requests: { cpu: '100m', memory: '128Mi' },
limits: { cpu: '500m', memory: '512Mi' },
},
}],
volumes: [{
name: 'registry-data',
persistentVolumeClaim: { claimName: registryPvcName },
}],
},
},
},
});
this.logger.log(`Created Docker Registry Deployment`);
} else {
throw err;
}
}
// ── 5. Registry ClusterIP Service (for Kaniko to push) ────────
const registrySvcName = 'registry';
try {
await coreApi.readNamespacedService(registrySvcName, buildNs);
this.logger.log(`Service "${registrySvcName}" already exists`);
} catch (err: any) {
if (err.statusCode === 404 || err.body?.code === 404) {
await coreApi.createNamespacedService(buildNs, {
metadata: { name: registrySvcName, namespace: buildNs, labels: { app: 'registry' } },
spec: {
type: 'ClusterIP',
selector: { app: 'registry' },
ports: [{ port: 5000, targetPort: 5000 as any, protocol: 'TCP' }],
},
});
this.logger.log(`Created Registry ClusterIP Service (port 5000)`);
} else {
throw err;
}
}
// ── 6. Registry NodePort Service (for kubelet to pull) ────────
const registryNodePortName = 'registry-nodeport';
try {
await coreApi.readNamespacedService(registryNodePortName, buildNs);
this.logger.log(`Service "${registryNodePortName}" already exists`);
} catch (err: any) {
if (err.statusCode === 404 || err.body?.code === 404) {
await coreApi.createNamespacedService(buildNs, {
metadata: { name: registryNodePortName, namespace: buildNs, labels: { app: 'registry' } },
spec: {
type: 'NodePort',
selector: { app: 'registry' },
ports: [{ port: 5000, targetPort: 5000 as any, nodePort: 30500, protocol: 'TCP' }],
},
});
this.logger.log(`Created Registry NodePort Service (30500 → 5000)`);
} else {
throw err;
}
}
// ── 7. registry-credentials Secret (docker config for Kaniko) ─
const registrySecretName = 'registry-credentials';
try {
await coreApi.readNamespacedSecret(registrySecretName, buildNs);
this.logger.log(`Secret "${registrySecretName}" already exists`);
} catch (err: any) {
if (err.statusCode === 404 || err.body?.code === 404) {
// Parse registry host (without port path) for the docker config
const dockerConfig = JSON.stringify({
auths: {
[registryUrl]: { auth: '' },
[`registry.${buildNs}.svc.cluster.local:5000`]: { auth: '' },
'localhost:30500': { auth: '' },
},
});
await coreApi.createNamespacedSecret(buildNs, {
metadata: { name: registrySecretName, namespace: buildNs },
type: 'kubernetes.io/dockerconfigjson',
data: {
'.dockerconfigjson': Buffer.from(dockerConfig).toString('base64'),
},
});
this.logger.log(`Created registry-credentials Secret`);
} else {
throw err;
}
}
this.logger.log(`✅ Cluster bootstrap complete — build infrastructure ready`);
}
private parseCpuToMillicores(cpu: string): number {
if (!cpu || cpu === '0') return 0;
if (cpu.endsWith('n')) return parseFloat(cpu) / 1_000_000;
@@ -6,7 +6,6 @@ import { Deployment } from './entities/deployment.entity';
import { ApplicationsModule } from '../applications/applications.module';
import { KubernetesModule } from '../kubernetes/kubernetes.module';
import { BuildModule } from '../build/build.module';
import { SnapshotsModule } from '../snapshots/snapshots.module';
@Module({
imports: [
@@ -14,7 +13,6 @@ import { SnapshotsModule } from '../snapshots/snapshots.module';
forwardRef(() => ApplicationsModule),
KubernetesModule,
BuildModule,
forwardRef(() => SnapshotsModule),
],
controllers: [DeploymentsController],
providers: [DeploymentsService],
+21 -21
View File
@@ -1,12 +1,12 @@
import { Injectable, NotFoundException, Logger, Inject, forwardRef } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import * as fs from 'fs';
import { Deployment } from './entities/deployment.entity';
import { ApplicationsService } from '../applications/applications.service';
import { KubernetesService } from '../kubernetes/kubernetes.service';
import { BuildService } from '../build/build.service';
import { DeploymentStatus } from '../common/enums';
import { SnapshotsService } from '../snapshots/snapshots.service';
@Injectable()
export class DeploymentsService {
@@ -19,22 +19,11 @@ export class DeploymentsService {
private applicationsService: ApplicationsService,
private kubernetesService: KubernetesService,
private buildService: BuildService,
@Inject(forwardRef(() => SnapshotsService))
private snapshotsService: SnapshotsService,
) {}
async triggerDeployment(applicationId: string, userId: string): Promise<Deployment> {
const app = await this.applicationsService.findOne(applicationId, userId);
// Auto-snapshot before deploying (capture current state for rollback)
try {
const version = `v${Date.now()}`;
await this.snapshotsService.createPreDeploySnapshot(app.id, userId, version);
this.logger.log(`Pre-deploy snapshot created for ${app.name}`);
} catch (e: any) {
this.logger.warn(`Pre-deploy snapshot failed for ${app.name}: ${e.message} — continuing deploy`);
}
// Create deployment record
const deployment = this.deploymentsRepository.create({
applicationId: app.id,
@@ -69,6 +58,26 @@ export class DeploymentsService {
await this.updateStatus(deploymentId, DeploymentStatus.DEPLOYING);
const k8sResources = await this.kubernetesService.deployApplication(app, imageUri);
// Step 3.5: Restore DB dump if one was uploaded (must happen after deploy creates the namespace + DB)
if (app.dbDumpPath && fs.existsSync(app.dbDumpPath)) {
this.logger.log(`Restoring DB dump for ${app.name} from ${app.dbDumpPath}`);
try {
// Wait for the database pod to be Ready before restoring
await this.kubernetesService.waitForDatabaseReady(app, 120_000);
// Re-fetch app to ensure we have latest data
const freshApp = await this.applicationsService.findOne(app.id);
const result = await this.kubernetesService.restoreDatabaseDump(freshApp, freshApp.dbDumpPath!);
if (result.success) {
this.logger.log(`DB dump restored successfully for ${app.name}`);
} else {
this.logger.warn(`DB dump restore failed for ${app.name}: ${result.logs}`);
}
} catch (e: any) {
this.logger.warn(`DB dump restore error for ${app.name}: ${e.message}`);
// Don't fail the deployment — DB restore is a best-effort step
}
}
// Step 4: Mark success
await this.deploymentsRepository.update(deploymentId, {
status: DeploymentStatus.RUNNING,
@@ -185,15 +194,6 @@ export class DeploymentsService {
throw new NotFoundException('No source code available. Upload code or set a git URL first.');
}
// Auto-snapshot before redeploy
try {
const version = `v${Date.now()}`;
await this.snapshotsService.createPreDeploySnapshot(app.id, userId, version);
this.logger.log(`Pre-redeploy snapshot created for ${app.name}`);
} catch (e: any) {
this.logger.warn(`Pre-redeploy snapshot failed for ${app.name}: ${e.message} — continuing`);
}
// Create new deployment record
const deployment = this.deploymentsRepository.create({
applicationId: app.id,
+2 -3
View File
@@ -351,10 +351,9 @@ export class SnapshotsService {
}
}
// 3. Restore database — file-based
// 3. Restore database — file-based (pass file path for PVC-based transfer)
if (snapshot.dbDumpPath && fs.existsSync(snapshot.dbDumpPath)) {
const dumpBuffer = fs.readFileSync(snapshot.dbDumpPath);
const result = await this.kubernetesService.restoreDatabaseDump(app, dumpBuffer);
const result = await this.kubernetesService.restoreDatabaseDump(app, snapshot.dbDumpPath);
if (result.success) {
details.push('✅ Database restored');
this.logger.log(`Rollback ${snapshotId}: restored database`);