diff --git a/backend/src/applications/applications.controller.ts b/backend/src/applications/applications.controller.ts index 1b7e145..c52283a 100644 --- a/backend/src/applications/applications.controller.ts +++ b/backend/src/applications/applications.controller.ts @@ -62,7 +62,7 @@ export class ApplicationsController { } @Post(':id/db-upload') - @ApiOperation({ summary: 'Upload and restore a SQL dump into the application database' }) + @ApiOperation({ summary: 'Upload a SQL dump file for later restore during deployment' }) @ApiConsumes('multipart/form-data') @UseInterceptors(FileInterceptor('file', { limits: { fileSize: 500 * 1024 * 1024 }, // 500MB for DB dumps @@ -84,12 +84,13 @@ export class ApplicationsController { } this.logger.log(`DB dump upload for ${app.name} — ${(file.size / 1024).toFixed(1)} KB`); - const result = await this.kubernetesService.restoreDatabaseDump(app, file.buffer); + + // Save to disk (restore happens after deploy when namespace exists) + await this.applicationsService.uploadDbDump(id, isStaff ? app.userId : req.user.id, file); return { - success: result.success, - message: result.success ? 'Database restored successfully' : 'Database restore failed', - logs: result.logs, + success: true, + message: 'Database dump uploaded. It will be restored automatically after deployment.', }; } @@ -225,12 +226,10 @@ export class ApplicationsController { // 1. Get the app first const app = await this.applicationsService.findOne(id, isStaff ? undefined : req.user.id); - // 2. Delete K8s resources (deployment, service, ingress, db, secrets) + // 2. Delete K8s resources (deployment, service, ingress, db, secrets, PVCs) try { - if (app.clusterId && app.latestImageTag) { - await this.kubernetesService.deleteApplication(app); - this.logger.log(`Deleted K8s resources for ${app.name}`); - } + await this.kubernetesService.deleteApplication(app); + this.logger.log(`Deleted K8s resources for ${app.name}`); } catch (e: any) { this.logger.warn(`K8s cleanup failed for ${app.name}: ${e.message}`); } diff --git a/backend/src/applications/applications.service.ts b/backend/src/applications/applications.service.ts index 35a025a..44fad5f 100644 --- a/backend/src/applications/applications.service.ts +++ b/backend/src/applications/applications.service.ts @@ -188,4 +188,28 @@ export class ApplicationsService { this.logger.log(`Uploaded code for ${app.name} → ${zipPath} (${(file.size / 1024).toFixed(1)} KB)`); return saved; } + + async uploadDbDump(id: string, userId: string, file: Express.Multer.File): Promise { + if (!file) { + throw new BadRequestException('No file uploaded'); + } + + const app = await this.findOne(id, userId); + const uploadDir = this.configService.get('platform.uploadDir') || './uploads'; + const appDir = path.join(uploadDir, app.userId, app.id); + + // Ensure directory exists + fs.mkdirSync(appDir, { recursive: true }); + + // Save the SQL dump file + const dumpPath = path.join(appDir, 'dump.sql'); + fs.writeFileSync(dumpPath, file.buffer); + + // Update app with dump path + app.dbDumpPath = dumpPath; + const saved = await this.appsRepository.save(app); + + this.logger.log(`Uploaded DB dump for ${app.name} → ${dumpPath} (${(file.size / 1024).toFixed(1)} KB)`); + return saved; + } } diff --git a/backend/src/clusters/clusters.service.spec.ts b/backend/src/clusters/clusters.service.spec.ts new file mode 100644 index 0000000..58c47ab --- /dev/null +++ b/backend/src/clusters/clusters.service.spec.ts @@ -0,0 +1,105 @@ +import { ClusterStatus } from '../common/enums'; + +/** + * Tests for ClustersService — getDefault and delete logic. + */ + +describe('ClustersService getDefault logic', () => { + // Simulate the fixed getDefault behavior + function getDefault(clusters: { id: string; isDefault: boolean; status: string }[]): { id: string } | null { + // Step 1: active + default + let result = clusters.find(c => c.isDefault && c.status === ClusterStatus.ACTIVE); + if (result) return { id: result.id }; + + // Step 2: any active (fallback) + result = clusters.find(c => c.status === ClusterStatus.ACTIVE); + if (result) return { id: result.id }; + + return null; + } + + it('should return active default cluster', () => { + const clusters = [ + { id: '1', isDefault: true, status: ClusterStatus.ACTIVE }, + { id: '2', isDefault: false, status: ClusterStatus.ACTIVE }, + ]; + expect(getDefault(clusters)?.id).toBe('1'); + }); + + it('should skip inactive default and return active cluster', () => { + const clusters = [ + { id: '1', isDefault: true, status: ClusterStatus.INACTIVE }, + { id: '2', isDefault: false, status: ClusterStatus.ACTIVE }, + ]; + expect(getDefault(clusters)?.id).toBe('2'); + }); + + it('should return null when no active clusters exist', () => { + const clusters = [ + { id: '1', isDefault: true, status: ClusterStatus.INACTIVE }, + ]; + expect(getDefault(clusters)).toBeNull(); + }); + + it('should handle both clusters being default (picks active one)', () => { + const clusters = [ + { id: 'inactive', isDefault: true, status: ClusterStatus.INACTIVE }, + { id: 'active', isDefault: true, status: ClusterStatus.ACTIVE }, + ]; + expect(getDefault(clusters)?.id).toBe('active'); + }); +}); + +describe('ClustersService delete logic', () => { + it('should reassign apps to replacement cluster on delete', () => { + // Simulate: cluster A (being deleted) has 3 apps, cluster B is active + const apps = [ + { id: 'app1', clusterId: 'A' }, + { id: 'app2', clusterId: 'A' }, + { id: 'app3', clusterId: 'B' }, + ]; + const deletedClusterId = 'A'; + const replacementId = 'B'; + + // Reassign + for (const app of apps) { + if (app.clusterId === deletedClusterId) { + app.clusterId = replacementId; + } + } + + expect(apps.filter(a => a.clusterId === 'A')).toHaveLength(0); + expect(apps.filter(a => a.clusterId === 'B')).toHaveLength(3); + }); + + it('should promote another cluster to default when default is deleted', () => { + const clusters = [ + { id: 'A', isDefault: true, status: ClusterStatus.ACTIVE }, + { id: 'B', isDefault: false, status: ClusterStatus.ACTIVE }, + ]; + + // Delete A + const deleted = clusters.splice(0, 1)[0]; + expect(deleted.isDefault).toBe(true); + + // Promote + const newDefault = clusters.find(c => c.status === ClusterStatus.ACTIVE); + if (newDefault) newDefault.isDefault = true; + + expect(clusters[0].isDefault).toBe(true); + expect(clusters[0].id).toBe('B'); + }); + + it('should nullify clusterId when no replacement cluster exists', () => { + const apps = [{ id: 'app1', clusterId: 'A' as string | null }]; + const hasReplacement = false; + + if (!hasReplacement) { + for (const app of apps) { + app.clusterId = null; + } + } + + expect(apps[0].clusterId).toBeNull(); + }); +}); diff --git a/backend/src/clusters/clusters.service.ts b/backend/src/clusters/clusters.service.ts index c5a02da..fde93d8 100644 --- a/backend/src/clusters/clusters.service.ts +++ b/backend/src/clusters/clusters.service.ts @@ -1,5 +1,6 @@ import { Injectable, NotFoundException, Logger, BadRequestException } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; +import { ConfigService } from '@nestjs/config'; import { Repository, DataSource, In } from 'typeorm'; import * as k8s from '@kubernetes/client-node'; import { Cluster } from './entities/cluster.entity'; @@ -20,6 +21,7 @@ export class ClustersService { @InjectRepository(ClusterPool) private poolsRepository: Repository, private dataSource: DataSource, + private configService: ConfigService, ) {} /** @@ -73,6 +75,12 @@ export class ClustersService { }); const saved = await this.clustersRepository.save(cluster); this.logger.log(`Cluster "${saved.name}" registered (active) — K8s ${connectionTest.version}`); + + // Bootstrap the cluster with build infrastructure (namespace, registry, SA, etc.) + this.bootstrapCluster(saved.kubeconfig).catch((err) => { + this.logger.error(`Failed to bootstrap cluster "${saved.name}": ${err.message}`); + }); + return saved; } @@ -92,9 +100,23 @@ export class ClustersService { } async getDefault(): Promise { - const cluster = await this.clustersRepository.findOne({ where: { isDefault: true } }); + // Prefer active default cluster; fall back to any active cluster + let cluster = await this.clustersRepository.findOne({ + where: { isDefault: true, status: ClusterStatus.ACTIVE }, + }); if (!cluster) { - throw new NotFoundException('No default cluster configured'); + // Fallback: pick any active cluster and promote it to default + cluster = await this.clustersRepository.findOne({ + where: { status: ClusterStatus.ACTIVE }, + }); + if (cluster) { + cluster.isDefault = true; + await this.clustersRepository.save(cluster); + this.logger.warn(`No active default cluster — promoted "${cluster.name}" to default`); + } + } + if (!cluster) { + throw new NotFoundException('No active cluster available'); } return cluster; } @@ -112,6 +134,11 @@ export class ClustersService { } dto.status = ClusterStatus.ACTIVE; this.logger.log(`Cluster "${cluster.name}" kubeconfig updated — connection verified (K8s ${connectionTest.version})`); + + // Re-bootstrap build infrastructure on the new/updated cluster + this.bootstrapCluster(dto.kubeconfig).catch((err: any) => { + this.logger.error(`Failed to bootstrap cluster "${cluster.name}": ${err.message}`); + }); } if (dto.isDefault === true) { @@ -206,7 +233,55 @@ export class ClustersService { async delete(id: string): Promise { const cluster = await this.findOne(id); + const wasDefault = cluster.isDefault; + + // Reassign applications that were on this cluster to another active cluster + try { + const replacement = await this.clustersRepository.findOne({ + where: { status: ClusterStatus.ACTIVE, id: undefined as any }, + }); + // Use raw query to exclude the deleted cluster + const activeReplacement = await this.clustersRepository + .createQueryBuilder('c') + .where('c.id != :id', { id }) + .andWhere('c.status = :status', { status: ClusterStatus.ACTIVE }) + .getOne(); + + if (activeReplacement) { + const result = await this.dataSource.query( + `UPDATE applications SET "clusterId" = $1 WHERE "clusterId" = $2`, + [activeReplacement.id, id], + ); + const count = result?.[1] || 0; + if (count > 0) { + this.logger.log(`Reassigned ${count} application(s) from cluster "${cluster.name}" to "${activeReplacement.name}"`); + } + } else { + // No replacement — nullify clusterId so apps aren't orphaned with a dangling FK + await this.dataSource.query( + `UPDATE applications SET "clusterId" = NULL WHERE "clusterId" = $1`, + [id], + ); + this.logger.warn(`No active replacement cluster — cleared clusterId for apps on "${cluster.name}"`); + } + } catch (e: any) { + this.logger.warn(`Failed to reassign apps from cluster "${cluster.name}": ${e.message}`); + } + await this.clustersRepository.remove(cluster); + this.logger.log(`Cluster "${cluster.name}" deleted`); + + // If deleted cluster was default, promote another active cluster + if (wasDefault) { + const newDefault = await this.clustersRepository.findOne({ + where: { status: ClusterStatus.ACTIVE }, + }); + if (newDefault) { + newDefault.isDefault = true; + await this.clustersRepository.save(newDefault); + this.logger.log(`Promoted cluster "${newDefault.name}" to default after deleting "${cluster.name}"`); + } + } } // ─── Cluster Pool Methods ───────────────────────────────────────── @@ -429,6 +504,191 @@ export class ClustersService { } } + /** + * Bootstrap a newly-added cluster with the build infrastructure: + * 1. cloudhost-builds namespace + * 2. kaniko-builder ServiceAccount + * 3. Docker Registry Deployment + PVC + Service (ClusterIP) + NodePort Service + * 4. registry-credentials Secret (for Kaniko docker auth) + */ + async bootstrapCluster(kubeconfig: string): Promise { + const kc = new k8s.KubeConfig(); + kc.loadFromString(kubeconfig); + const coreApi = kc.makeApiClient(k8s.CoreV1Api); + const appsApi = kc.makeApiClient(k8s.AppsV1Api); + + const buildNs = this.configService.get('build.namespace') || 'cloudhost-builds'; + const saName = this.configService.get('build.serviceAccount') || 'kaniko-builder'; + const registryUrl = this.configService.get('registry.url') || `registry.${buildNs}.svc.cluster.local:5000`; + + this.logger.log(`Bootstrapping cluster — namespace: ${buildNs}`); + + // ── 1. Namespace ────────────────────────────────────────────── + try { + await coreApi.readNamespace(buildNs); + this.logger.log(`Namespace "${buildNs}" already exists`); + } catch (err: any) { + if (err.statusCode === 404 || err.body?.code === 404) { + await coreApi.createNamespace({ metadata: { name: buildNs } }); + this.logger.log(`Created namespace "${buildNs}"`); + } else { + throw err; + } + } + + // ── 2. ServiceAccount for Kaniko ────────────────────────────── + try { + await coreApi.readNamespacedServiceAccount(saName, buildNs); + this.logger.log(`ServiceAccount "${saName}" already exists`); + } catch (err: any) { + if (err.statusCode === 404 || err.body?.code === 404) { + await coreApi.createNamespacedServiceAccount(buildNs, { + metadata: { name: saName, namespace: buildNs }, + }); + this.logger.log(`Created ServiceAccount "${saName}"`); + } else { + throw err; + } + } + + // ── 3. Docker Registry PVC ──────────────────────────────────── + const registryPvcName = 'registry-data'; + try { + await coreApi.readNamespacedPersistentVolumeClaim(registryPvcName, buildNs); + this.logger.log(`PVC "${registryPvcName}" already exists`); + } catch (err: any) { + if (err.statusCode === 404 || err.body?.code === 404) { + await coreApi.createNamespacedPersistentVolumeClaim(buildNs, { + metadata: { name: registryPvcName, namespace: buildNs }, + spec: { + accessModes: ['ReadWriteOnce'], + resources: { requests: { storage: '10Gi' } }, + }, + }); + this.logger.log(`Created PVC "${registryPvcName}" (10Gi)`); + } else { + throw err; + } + } + + // ── 4. Docker Registry Deployment ───────────────────────────── + const registryDeployName = 'registry'; + try { + await appsApi.readNamespacedDeployment(registryDeployName, buildNs); + this.logger.log(`Deployment "${registryDeployName}" already exists`); + } catch (err: any) { + if (err.statusCode === 404 || err.body?.code === 404) { + await appsApi.createNamespacedDeployment(buildNs, { + metadata: { name: registryDeployName, namespace: buildNs, labels: { app: 'registry' } }, + spec: { + replicas: 1, + selector: { matchLabels: { app: 'registry' } }, + template: { + metadata: { labels: { app: 'registry' } }, + spec: { + containers: [{ + name: 'registry', + image: 'registry:2', + ports: [{ containerPort: 5000 }], + env: [ + { name: 'REGISTRY_STORAGE_DELETE_ENABLED', value: 'true' }, + ], + volumeMounts: [{ + name: 'registry-data', + mountPath: '/var/lib/registry', + }], + resources: { + requests: { cpu: '100m', memory: '128Mi' }, + limits: { cpu: '500m', memory: '512Mi' }, + }, + }], + volumes: [{ + name: 'registry-data', + persistentVolumeClaim: { claimName: registryPvcName }, + }], + }, + }, + }, + }); + this.logger.log(`Created Docker Registry Deployment`); + } else { + throw err; + } + } + + // ── 5. Registry ClusterIP Service (for Kaniko to push) ──────── + const registrySvcName = 'registry'; + try { + await coreApi.readNamespacedService(registrySvcName, buildNs); + this.logger.log(`Service "${registrySvcName}" already exists`); + } catch (err: any) { + if (err.statusCode === 404 || err.body?.code === 404) { + await coreApi.createNamespacedService(buildNs, { + metadata: { name: registrySvcName, namespace: buildNs, labels: { app: 'registry' } }, + spec: { + type: 'ClusterIP', + selector: { app: 'registry' }, + ports: [{ port: 5000, targetPort: 5000 as any, protocol: 'TCP' }], + }, + }); + this.logger.log(`Created Registry ClusterIP Service (port 5000)`); + } else { + throw err; + } + } + + // ── 6. Registry NodePort Service (for kubelet to pull) ──────── + const registryNodePortName = 'registry-nodeport'; + try { + await coreApi.readNamespacedService(registryNodePortName, buildNs); + this.logger.log(`Service "${registryNodePortName}" already exists`); + } catch (err: any) { + if (err.statusCode === 404 || err.body?.code === 404) { + await coreApi.createNamespacedService(buildNs, { + metadata: { name: registryNodePortName, namespace: buildNs, labels: { app: 'registry' } }, + spec: { + type: 'NodePort', + selector: { app: 'registry' }, + ports: [{ port: 5000, targetPort: 5000 as any, nodePort: 30500, protocol: 'TCP' }], + }, + }); + this.logger.log(`Created Registry NodePort Service (30500 → 5000)`); + } else { + throw err; + } + } + + // ── 7. registry-credentials Secret (docker config for Kaniko) ─ + const registrySecretName = 'registry-credentials'; + try { + await coreApi.readNamespacedSecret(registrySecretName, buildNs); + this.logger.log(`Secret "${registrySecretName}" already exists`); + } catch (err: any) { + if (err.statusCode === 404 || err.body?.code === 404) { + // Parse registry host (without port path) for the docker config + const dockerConfig = JSON.stringify({ + auths: { + [registryUrl]: { auth: '' }, + [`registry.${buildNs}.svc.cluster.local:5000`]: { auth: '' }, + 'localhost:30500': { auth: '' }, + }, + }); + await coreApi.createNamespacedSecret(buildNs, { + metadata: { name: registrySecretName, namespace: buildNs }, + type: 'kubernetes.io/dockerconfigjson', + data: { + '.dockerconfigjson': Buffer.from(dockerConfig).toString('base64'), + }, + }); + this.logger.log(`Created registry-credentials Secret`); + } else { + throw err; + } + } + + this.logger.log(`✅ Cluster bootstrap complete — build infrastructure ready`); + } + private parseCpuToMillicores(cpu: string): number { if (!cpu || cpu === '0') return 0; if (cpu.endsWith('n')) return parseFloat(cpu) / 1_000_000; diff --git a/backend/src/deployments/deployments.module.ts b/backend/src/deployments/deployments.module.ts index 2b9e6f6..7545194 100644 --- a/backend/src/deployments/deployments.module.ts +++ b/backend/src/deployments/deployments.module.ts @@ -6,7 +6,6 @@ import { Deployment } from './entities/deployment.entity'; import { ApplicationsModule } from '../applications/applications.module'; import { KubernetesModule } from '../kubernetes/kubernetes.module'; import { BuildModule } from '../build/build.module'; -import { SnapshotsModule } from '../snapshots/snapshots.module'; @Module({ imports: [ @@ -14,7 +13,6 @@ import { SnapshotsModule } from '../snapshots/snapshots.module'; forwardRef(() => ApplicationsModule), KubernetesModule, BuildModule, - forwardRef(() => SnapshotsModule), ], controllers: [DeploymentsController], providers: [DeploymentsService], diff --git a/backend/src/deployments/deployments.service.ts b/backend/src/deployments/deployments.service.ts index 98296ff..c0a5b11 100644 --- a/backend/src/deployments/deployments.service.ts +++ b/backend/src/deployments/deployments.service.ts @@ -1,12 +1,12 @@ import { Injectable, NotFoundException, Logger, Inject, forwardRef } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; import { Repository } from 'typeorm'; +import * as fs from 'fs'; import { Deployment } from './entities/deployment.entity'; import { ApplicationsService } from '../applications/applications.service'; import { KubernetesService } from '../kubernetes/kubernetes.service'; import { BuildService } from '../build/build.service'; import { DeploymentStatus } from '../common/enums'; -import { SnapshotsService } from '../snapshots/snapshots.service'; @Injectable() export class DeploymentsService { @@ -19,22 +19,11 @@ export class DeploymentsService { private applicationsService: ApplicationsService, private kubernetesService: KubernetesService, private buildService: BuildService, - @Inject(forwardRef(() => SnapshotsService)) - private snapshotsService: SnapshotsService, ) {} async triggerDeployment(applicationId: string, userId: string): Promise { const app = await this.applicationsService.findOne(applicationId, userId); - // Auto-snapshot before deploying (capture current state for rollback) - try { - const version = `v${Date.now()}`; - await this.snapshotsService.createPreDeploySnapshot(app.id, userId, version); - this.logger.log(`Pre-deploy snapshot created for ${app.name}`); - } catch (e: any) { - this.logger.warn(`Pre-deploy snapshot failed for ${app.name}: ${e.message} — continuing deploy`); - } - // Create deployment record const deployment = this.deploymentsRepository.create({ applicationId: app.id, @@ -69,6 +58,26 @@ export class DeploymentsService { await this.updateStatus(deploymentId, DeploymentStatus.DEPLOYING); const k8sResources = await this.kubernetesService.deployApplication(app, imageUri); + // Step 3.5: Restore DB dump if one was uploaded (must happen after deploy creates the namespace + DB) + if (app.dbDumpPath && fs.existsSync(app.dbDumpPath)) { + this.logger.log(`Restoring DB dump for ${app.name} from ${app.dbDumpPath}`); + try { + // Wait for the database pod to be Ready before restoring + await this.kubernetesService.waitForDatabaseReady(app, 120_000); + // Re-fetch app to ensure we have latest data + const freshApp = await this.applicationsService.findOne(app.id); + const result = await this.kubernetesService.restoreDatabaseDump(freshApp, freshApp.dbDumpPath!); + if (result.success) { + this.logger.log(`DB dump restored successfully for ${app.name}`); + } else { + this.logger.warn(`DB dump restore failed for ${app.name}: ${result.logs}`); + } + } catch (e: any) { + this.logger.warn(`DB dump restore error for ${app.name}: ${e.message}`); + // Don't fail the deployment — DB restore is a best-effort step + } + } + // Step 4: Mark success await this.deploymentsRepository.update(deploymentId, { status: DeploymentStatus.RUNNING, @@ -185,15 +194,6 @@ export class DeploymentsService { throw new NotFoundException('No source code available. Upload code or set a git URL first.'); } - // Auto-snapshot before redeploy - try { - const version = `v${Date.now()}`; - await this.snapshotsService.createPreDeploySnapshot(app.id, userId, version); - this.logger.log(`Pre-redeploy snapshot created for ${app.name}`); - } catch (e: any) { - this.logger.warn(`Pre-redeploy snapshot failed for ${app.name}: ${e.message} — continuing`); - } - // Create new deployment record const deployment = this.deploymentsRepository.create({ applicationId: app.id, diff --git a/backend/src/snapshots/snapshots.service.ts b/backend/src/snapshots/snapshots.service.ts index 5afda1c..479d772 100644 --- a/backend/src/snapshots/snapshots.service.ts +++ b/backend/src/snapshots/snapshots.service.ts @@ -351,10 +351,9 @@ export class SnapshotsService { } } - // 3. Restore database — file-based + // 3. Restore database — file-based (pass file path for PVC-based transfer) if (snapshot.dbDumpPath && fs.existsSync(snapshot.dbDumpPath)) { - const dumpBuffer = fs.readFileSync(snapshot.dbDumpPath); - const result = await this.kubernetesService.restoreDatabaseDump(app, dumpBuffer); + const result = await this.kubernetesService.restoreDatabaseDump(app, snapshot.dbDumpPath); if (result.success) { details.push('✅ Database restored'); this.logger.log(`Rollback ${snapshotId}: restored database`);