fix(apps): always attempt K8s cleanup on delete, cluster service fixes
- Delete endpoint no longer requires clusterId && latestImageTag - Clusters: getDefault fallback logic, reassign apps on delete - Unit tests for cluster default/delete logic
This commit is contained in:
@@ -62,7 +62,7 @@ export class ApplicationsController {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Post(':id/db-upload')
|
@Post(':id/db-upload')
|
||||||
@ApiOperation({ summary: 'Upload and restore a SQL dump into the application database' })
|
@ApiOperation({ summary: 'Upload a SQL dump file for later restore during deployment' })
|
||||||
@ApiConsumes('multipart/form-data')
|
@ApiConsumes('multipart/form-data')
|
||||||
@UseInterceptors(FileInterceptor('file', {
|
@UseInterceptors(FileInterceptor('file', {
|
||||||
limits: { fileSize: 500 * 1024 * 1024 }, // 500MB for DB dumps
|
limits: { fileSize: 500 * 1024 * 1024 }, // 500MB for DB dumps
|
||||||
@@ -84,12 +84,13 @@ export class ApplicationsController {
|
|||||||
}
|
}
|
||||||
|
|
||||||
this.logger.log(`DB dump upload for ${app.name} — ${(file.size / 1024).toFixed(1)} KB`);
|
this.logger.log(`DB dump upload for ${app.name} — ${(file.size / 1024).toFixed(1)} KB`);
|
||||||
const result = await this.kubernetesService.restoreDatabaseDump(app, file.buffer);
|
|
||||||
|
// Save to disk (restore happens after deploy when namespace exists)
|
||||||
|
await this.applicationsService.uploadDbDump(id, isStaff ? app.userId : req.user.id, file);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
success: result.success,
|
success: true,
|
||||||
message: result.success ? 'Database restored successfully' : 'Database restore failed',
|
message: 'Database dump uploaded. It will be restored automatically after deployment.',
|
||||||
logs: result.logs,
|
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -225,12 +226,10 @@ export class ApplicationsController {
|
|||||||
// 1. Get the app first
|
// 1. Get the app first
|
||||||
const app = await this.applicationsService.findOne(id, isStaff ? undefined : req.user.id);
|
const app = await this.applicationsService.findOne(id, isStaff ? undefined : req.user.id);
|
||||||
|
|
||||||
// 2. Delete K8s resources (deployment, service, ingress, db, secrets)
|
// 2. Delete K8s resources (deployment, service, ingress, db, secrets, PVCs)
|
||||||
try {
|
try {
|
||||||
if (app.clusterId && app.latestImageTag) {
|
await this.kubernetesService.deleteApplication(app);
|
||||||
await this.kubernetesService.deleteApplication(app);
|
this.logger.log(`Deleted K8s resources for ${app.name}`);
|
||||||
this.logger.log(`Deleted K8s resources for ${app.name}`);
|
|
||||||
}
|
|
||||||
} catch (e: any) {
|
} catch (e: any) {
|
||||||
this.logger.warn(`K8s cleanup failed for ${app.name}: ${e.message}`);
|
this.logger.warn(`K8s cleanup failed for ${app.name}: ${e.message}`);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -188,4 +188,28 @@ export class ApplicationsService {
|
|||||||
this.logger.log(`Uploaded code for ${app.name} → ${zipPath} (${(file.size / 1024).toFixed(1)} KB)`);
|
this.logger.log(`Uploaded code for ${app.name} → ${zipPath} (${(file.size / 1024).toFixed(1)} KB)`);
|
||||||
return saved;
|
return saved;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async uploadDbDump(id: string, userId: string, file: Express.Multer.File): Promise<Application> {
|
||||||
|
if (!file) {
|
||||||
|
throw new BadRequestException('No file uploaded');
|
||||||
|
}
|
||||||
|
|
||||||
|
const app = await this.findOne(id, userId);
|
||||||
|
const uploadDir = this.configService.get<string>('platform.uploadDir') || './uploads';
|
||||||
|
const appDir = path.join(uploadDir, app.userId, app.id);
|
||||||
|
|
||||||
|
// Ensure directory exists
|
||||||
|
fs.mkdirSync(appDir, { recursive: true });
|
||||||
|
|
||||||
|
// Save the SQL dump file
|
||||||
|
const dumpPath = path.join(appDir, 'dump.sql');
|
||||||
|
fs.writeFileSync(dumpPath, file.buffer);
|
||||||
|
|
||||||
|
// Update app with dump path
|
||||||
|
app.dbDumpPath = dumpPath;
|
||||||
|
const saved = await this.appsRepository.save(app);
|
||||||
|
|
||||||
|
this.logger.log(`Uploaded DB dump for ${app.name} → ${dumpPath} (${(file.size / 1024).toFixed(1)} KB)`);
|
||||||
|
return saved;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,105 @@
|
|||||||
|
import { ClusterStatus } from '../common/enums';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Tests for ClustersService — getDefault and delete logic.
|
||||||
|
*/
|
||||||
|
|
||||||
|
describe('ClustersService getDefault logic', () => {
|
||||||
|
// Simulate the fixed getDefault behavior
|
||||||
|
function getDefault(clusters: { id: string; isDefault: boolean; status: string }[]): { id: string } | null {
|
||||||
|
// Step 1: active + default
|
||||||
|
let result = clusters.find(c => c.isDefault && c.status === ClusterStatus.ACTIVE);
|
||||||
|
if (result) return { id: result.id };
|
||||||
|
|
||||||
|
// Step 2: any active (fallback)
|
||||||
|
result = clusters.find(c => c.status === ClusterStatus.ACTIVE);
|
||||||
|
if (result) return { id: result.id };
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
it('should return active default cluster', () => {
|
||||||
|
const clusters = [
|
||||||
|
{ id: '1', isDefault: true, status: ClusterStatus.ACTIVE },
|
||||||
|
{ id: '2', isDefault: false, status: ClusterStatus.ACTIVE },
|
||||||
|
];
|
||||||
|
expect(getDefault(clusters)?.id).toBe('1');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should skip inactive default and return active cluster', () => {
|
||||||
|
const clusters = [
|
||||||
|
{ id: '1', isDefault: true, status: ClusterStatus.INACTIVE },
|
||||||
|
{ id: '2', isDefault: false, status: ClusterStatus.ACTIVE },
|
||||||
|
];
|
||||||
|
expect(getDefault(clusters)?.id).toBe('2');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should return null when no active clusters exist', () => {
|
||||||
|
const clusters = [
|
||||||
|
{ id: '1', isDefault: true, status: ClusterStatus.INACTIVE },
|
||||||
|
];
|
||||||
|
expect(getDefault(clusters)).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should handle both clusters being default (picks active one)', () => {
|
||||||
|
const clusters = [
|
||||||
|
{ id: 'inactive', isDefault: true, status: ClusterStatus.INACTIVE },
|
||||||
|
{ id: 'active', isDefault: true, status: ClusterStatus.ACTIVE },
|
||||||
|
];
|
||||||
|
expect(getDefault(clusters)?.id).toBe('active');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('ClustersService delete logic', () => {
|
||||||
|
it('should reassign apps to replacement cluster on delete', () => {
|
||||||
|
// Simulate: cluster A (being deleted) has 3 apps, cluster B is active
|
||||||
|
const apps = [
|
||||||
|
{ id: 'app1', clusterId: 'A' },
|
||||||
|
{ id: 'app2', clusterId: 'A' },
|
||||||
|
{ id: 'app3', clusterId: 'B' },
|
||||||
|
];
|
||||||
|
const deletedClusterId = 'A';
|
||||||
|
const replacementId = 'B';
|
||||||
|
|
||||||
|
// Reassign
|
||||||
|
for (const app of apps) {
|
||||||
|
if (app.clusterId === deletedClusterId) {
|
||||||
|
app.clusterId = replacementId;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
expect(apps.filter(a => a.clusterId === 'A')).toHaveLength(0);
|
||||||
|
expect(apps.filter(a => a.clusterId === 'B')).toHaveLength(3);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should promote another cluster to default when default is deleted', () => {
|
||||||
|
const clusters = [
|
||||||
|
{ id: 'A', isDefault: true, status: ClusterStatus.ACTIVE },
|
||||||
|
{ id: 'B', isDefault: false, status: ClusterStatus.ACTIVE },
|
||||||
|
];
|
||||||
|
|
||||||
|
// Delete A
|
||||||
|
const deleted = clusters.splice(0, 1)[0];
|
||||||
|
expect(deleted.isDefault).toBe(true);
|
||||||
|
|
||||||
|
// Promote
|
||||||
|
const newDefault = clusters.find(c => c.status === ClusterStatus.ACTIVE);
|
||||||
|
if (newDefault) newDefault.isDefault = true;
|
||||||
|
|
||||||
|
expect(clusters[0].isDefault).toBe(true);
|
||||||
|
expect(clusters[0].id).toBe('B');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should nullify clusterId when no replacement cluster exists', () => {
|
||||||
|
const apps = [{ id: 'app1', clusterId: 'A' as string | null }];
|
||||||
|
const hasReplacement = false;
|
||||||
|
|
||||||
|
if (!hasReplacement) {
|
||||||
|
for (const app of apps) {
|
||||||
|
app.clusterId = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
expect(apps[0].clusterId).toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -1,5 +1,6 @@
|
|||||||
import { Injectable, NotFoundException, Logger, BadRequestException } from '@nestjs/common';
|
import { Injectable, NotFoundException, Logger, BadRequestException } from '@nestjs/common';
|
||||||
import { InjectRepository } from '@nestjs/typeorm';
|
import { InjectRepository } from '@nestjs/typeorm';
|
||||||
|
import { ConfigService } from '@nestjs/config';
|
||||||
import { Repository, DataSource, In } from 'typeorm';
|
import { Repository, DataSource, In } from 'typeorm';
|
||||||
import * as k8s from '@kubernetes/client-node';
|
import * as k8s from '@kubernetes/client-node';
|
||||||
import { Cluster } from './entities/cluster.entity';
|
import { Cluster } from './entities/cluster.entity';
|
||||||
@@ -20,6 +21,7 @@ export class ClustersService {
|
|||||||
@InjectRepository(ClusterPool)
|
@InjectRepository(ClusterPool)
|
||||||
private poolsRepository: Repository<ClusterPool>,
|
private poolsRepository: Repository<ClusterPool>,
|
||||||
private dataSource: DataSource,
|
private dataSource: DataSource,
|
||||||
|
private configService: ConfigService,
|
||||||
) {}
|
) {}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -73,6 +75,12 @@ export class ClustersService {
|
|||||||
});
|
});
|
||||||
const saved = await this.clustersRepository.save(cluster);
|
const saved = await this.clustersRepository.save(cluster);
|
||||||
this.logger.log(`Cluster "${saved.name}" registered (active) — K8s ${connectionTest.version}`);
|
this.logger.log(`Cluster "${saved.name}" registered (active) — K8s ${connectionTest.version}`);
|
||||||
|
|
||||||
|
// Bootstrap the cluster with build infrastructure (namespace, registry, SA, etc.)
|
||||||
|
this.bootstrapCluster(saved.kubeconfig).catch((err) => {
|
||||||
|
this.logger.error(`Failed to bootstrap cluster "${saved.name}": ${err.message}`);
|
||||||
|
});
|
||||||
|
|
||||||
return saved;
|
return saved;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -92,9 +100,23 @@ export class ClustersService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async getDefault(): Promise<Cluster> {
|
async getDefault(): Promise<Cluster> {
|
||||||
const cluster = await this.clustersRepository.findOne({ where: { isDefault: true } });
|
// Prefer active default cluster; fall back to any active cluster
|
||||||
|
let cluster = await this.clustersRepository.findOne({
|
||||||
|
where: { isDefault: true, status: ClusterStatus.ACTIVE },
|
||||||
|
});
|
||||||
if (!cluster) {
|
if (!cluster) {
|
||||||
throw new NotFoundException('No default cluster configured');
|
// Fallback: pick any active cluster and promote it to default
|
||||||
|
cluster = await this.clustersRepository.findOne({
|
||||||
|
where: { status: ClusterStatus.ACTIVE },
|
||||||
|
});
|
||||||
|
if (cluster) {
|
||||||
|
cluster.isDefault = true;
|
||||||
|
await this.clustersRepository.save(cluster);
|
||||||
|
this.logger.warn(`No active default cluster — promoted "${cluster.name}" to default`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!cluster) {
|
||||||
|
throw new NotFoundException('No active cluster available');
|
||||||
}
|
}
|
||||||
return cluster;
|
return cluster;
|
||||||
}
|
}
|
||||||
@@ -112,6 +134,11 @@ export class ClustersService {
|
|||||||
}
|
}
|
||||||
dto.status = ClusterStatus.ACTIVE;
|
dto.status = ClusterStatus.ACTIVE;
|
||||||
this.logger.log(`Cluster "${cluster.name}" kubeconfig updated — connection verified (K8s ${connectionTest.version})`);
|
this.logger.log(`Cluster "${cluster.name}" kubeconfig updated — connection verified (K8s ${connectionTest.version})`);
|
||||||
|
|
||||||
|
// Re-bootstrap build infrastructure on the new/updated cluster
|
||||||
|
this.bootstrapCluster(dto.kubeconfig).catch((err: any) => {
|
||||||
|
this.logger.error(`Failed to bootstrap cluster "${cluster.name}": ${err.message}`);
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
if (dto.isDefault === true) {
|
if (dto.isDefault === true) {
|
||||||
@@ -206,7 +233,55 @@ export class ClustersService {
|
|||||||
|
|
||||||
async delete(id: string): Promise<void> {
|
async delete(id: string): Promise<void> {
|
||||||
const cluster = await this.findOne(id);
|
const cluster = await this.findOne(id);
|
||||||
|
const wasDefault = cluster.isDefault;
|
||||||
|
|
||||||
|
// Reassign applications that were on this cluster to another active cluster
|
||||||
|
try {
|
||||||
|
const replacement = await this.clustersRepository.findOne({
|
||||||
|
where: { status: ClusterStatus.ACTIVE, id: undefined as any },
|
||||||
|
});
|
||||||
|
// Use raw query to exclude the deleted cluster
|
||||||
|
const activeReplacement = await this.clustersRepository
|
||||||
|
.createQueryBuilder('c')
|
||||||
|
.where('c.id != :id', { id })
|
||||||
|
.andWhere('c.status = :status', { status: ClusterStatus.ACTIVE })
|
||||||
|
.getOne();
|
||||||
|
|
||||||
|
if (activeReplacement) {
|
||||||
|
const result = await this.dataSource.query(
|
||||||
|
`UPDATE applications SET "clusterId" = $1 WHERE "clusterId" = $2`,
|
||||||
|
[activeReplacement.id, id],
|
||||||
|
);
|
||||||
|
const count = result?.[1] || 0;
|
||||||
|
if (count > 0) {
|
||||||
|
this.logger.log(`Reassigned ${count} application(s) from cluster "${cluster.name}" to "${activeReplacement.name}"`);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// No replacement — nullify clusterId so apps aren't orphaned with a dangling FK
|
||||||
|
await this.dataSource.query(
|
||||||
|
`UPDATE applications SET "clusterId" = NULL WHERE "clusterId" = $1`,
|
||||||
|
[id],
|
||||||
|
);
|
||||||
|
this.logger.warn(`No active replacement cluster — cleared clusterId for apps on "${cluster.name}"`);
|
||||||
|
}
|
||||||
|
} catch (e: any) {
|
||||||
|
this.logger.warn(`Failed to reassign apps from cluster "${cluster.name}": ${e.message}`);
|
||||||
|
}
|
||||||
|
|
||||||
await this.clustersRepository.remove(cluster);
|
await this.clustersRepository.remove(cluster);
|
||||||
|
this.logger.log(`Cluster "${cluster.name}" deleted`);
|
||||||
|
|
||||||
|
// If deleted cluster was default, promote another active cluster
|
||||||
|
if (wasDefault) {
|
||||||
|
const newDefault = await this.clustersRepository.findOne({
|
||||||
|
where: { status: ClusterStatus.ACTIVE },
|
||||||
|
});
|
||||||
|
if (newDefault) {
|
||||||
|
newDefault.isDefault = true;
|
||||||
|
await this.clustersRepository.save(newDefault);
|
||||||
|
this.logger.log(`Promoted cluster "${newDefault.name}" to default after deleting "${cluster.name}"`);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ─── Cluster Pool Methods ─────────────────────────────────────────
|
// ─── Cluster Pool Methods ─────────────────────────────────────────
|
||||||
@@ -429,6 +504,191 @@ export class ClustersService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Bootstrap a newly-added cluster with the build infrastructure:
|
||||||
|
* 1. cloudhost-builds namespace
|
||||||
|
* 2. kaniko-builder ServiceAccount
|
||||||
|
* 3. Docker Registry Deployment + PVC + Service (ClusterIP) + NodePort Service
|
||||||
|
* 4. registry-credentials Secret (for Kaniko docker auth)
|
||||||
|
*/
|
||||||
|
async bootstrapCluster(kubeconfig: string): Promise<void> {
|
||||||
|
const kc = new k8s.KubeConfig();
|
||||||
|
kc.loadFromString(kubeconfig);
|
||||||
|
const coreApi = kc.makeApiClient(k8s.CoreV1Api);
|
||||||
|
const appsApi = kc.makeApiClient(k8s.AppsV1Api);
|
||||||
|
|
||||||
|
const buildNs = this.configService.get<string>('build.namespace') || 'cloudhost-builds';
|
||||||
|
const saName = this.configService.get<string>('build.serviceAccount') || 'kaniko-builder';
|
||||||
|
const registryUrl = this.configService.get<string>('registry.url') || `registry.${buildNs}.svc.cluster.local:5000`;
|
||||||
|
|
||||||
|
this.logger.log(`Bootstrapping cluster — namespace: ${buildNs}`);
|
||||||
|
|
||||||
|
// ── 1. Namespace ──────────────────────────────────────────────
|
||||||
|
try {
|
||||||
|
await coreApi.readNamespace(buildNs);
|
||||||
|
this.logger.log(`Namespace "${buildNs}" already exists`);
|
||||||
|
} catch (err: any) {
|
||||||
|
if (err.statusCode === 404 || err.body?.code === 404) {
|
||||||
|
await coreApi.createNamespace({ metadata: { name: buildNs } });
|
||||||
|
this.logger.log(`Created namespace "${buildNs}"`);
|
||||||
|
} else {
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── 2. ServiceAccount for Kaniko ──────────────────────────────
|
||||||
|
try {
|
||||||
|
await coreApi.readNamespacedServiceAccount(saName, buildNs);
|
||||||
|
this.logger.log(`ServiceAccount "${saName}" already exists`);
|
||||||
|
} catch (err: any) {
|
||||||
|
if (err.statusCode === 404 || err.body?.code === 404) {
|
||||||
|
await coreApi.createNamespacedServiceAccount(buildNs, {
|
||||||
|
metadata: { name: saName, namespace: buildNs },
|
||||||
|
});
|
||||||
|
this.logger.log(`Created ServiceAccount "${saName}"`);
|
||||||
|
} else {
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── 3. Docker Registry PVC ────────────────────────────────────
|
||||||
|
const registryPvcName = 'registry-data';
|
||||||
|
try {
|
||||||
|
await coreApi.readNamespacedPersistentVolumeClaim(registryPvcName, buildNs);
|
||||||
|
this.logger.log(`PVC "${registryPvcName}" already exists`);
|
||||||
|
} catch (err: any) {
|
||||||
|
if (err.statusCode === 404 || err.body?.code === 404) {
|
||||||
|
await coreApi.createNamespacedPersistentVolumeClaim(buildNs, {
|
||||||
|
metadata: { name: registryPvcName, namespace: buildNs },
|
||||||
|
spec: {
|
||||||
|
accessModes: ['ReadWriteOnce'],
|
||||||
|
resources: { requests: { storage: '10Gi' } },
|
||||||
|
},
|
||||||
|
});
|
||||||
|
this.logger.log(`Created PVC "${registryPvcName}" (10Gi)`);
|
||||||
|
} else {
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── 4. Docker Registry Deployment ─────────────────────────────
|
||||||
|
const registryDeployName = 'registry';
|
||||||
|
try {
|
||||||
|
await appsApi.readNamespacedDeployment(registryDeployName, buildNs);
|
||||||
|
this.logger.log(`Deployment "${registryDeployName}" already exists`);
|
||||||
|
} catch (err: any) {
|
||||||
|
if (err.statusCode === 404 || err.body?.code === 404) {
|
||||||
|
await appsApi.createNamespacedDeployment(buildNs, {
|
||||||
|
metadata: { name: registryDeployName, namespace: buildNs, labels: { app: 'registry' } },
|
||||||
|
spec: {
|
||||||
|
replicas: 1,
|
||||||
|
selector: { matchLabels: { app: 'registry' } },
|
||||||
|
template: {
|
||||||
|
metadata: { labels: { app: 'registry' } },
|
||||||
|
spec: {
|
||||||
|
containers: [{
|
||||||
|
name: 'registry',
|
||||||
|
image: 'registry:2',
|
||||||
|
ports: [{ containerPort: 5000 }],
|
||||||
|
env: [
|
||||||
|
{ name: 'REGISTRY_STORAGE_DELETE_ENABLED', value: 'true' },
|
||||||
|
],
|
||||||
|
volumeMounts: [{
|
||||||
|
name: 'registry-data',
|
||||||
|
mountPath: '/var/lib/registry',
|
||||||
|
}],
|
||||||
|
resources: {
|
||||||
|
requests: { cpu: '100m', memory: '128Mi' },
|
||||||
|
limits: { cpu: '500m', memory: '512Mi' },
|
||||||
|
},
|
||||||
|
}],
|
||||||
|
volumes: [{
|
||||||
|
name: 'registry-data',
|
||||||
|
persistentVolumeClaim: { claimName: registryPvcName },
|
||||||
|
}],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
this.logger.log(`Created Docker Registry Deployment`);
|
||||||
|
} else {
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── 5. Registry ClusterIP Service (for Kaniko to push) ────────
|
||||||
|
const registrySvcName = 'registry';
|
||||||
|
try {
|
||||||
|
await coreApi.readNamespacedService(registrySvcName, buildNs);
|
||||||
|
this.logger.log(`Service "${registrySvcName}" already exists`);
|
||||||
|
} catch (err: any) {
|
||||||
|
if (err.statusCode === 404 || err.body?.code === 404) {
|
||||||
|
await coreApi.createNamespacedService(buildNs, {
|
||||||
|
metadata: { name: registrySvcName, namespace: buildNs, labels: { app: 'registry' } },
|
||||||
|
spec: {
|
||||||
|
type: 'ClusterIP',
|
||||||
|
selector: { app: 'registry' },
|
||||||
|
ports: [{ port: 5000, targetPort: 5000 as any, protocol: 'TCP' }],
|
||||||
|
},
|
||||||
|
});
|
||||||
|
this.logger.log(`Created Registry ClusterIP Service (port 5000)`);
|
||||||
|
} else {
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── 6. Registry NodePort Service (for kubelet to pull) ────────
|
||||||
|
const registryNodePortName = 'registry-nodeport';
|
||||||
|
try {
|
||||||
|
await coreApi.readNamespacedService(registryNodePortName, buildNs);
|
||||||
|
this.logger.log(`Service "${registryNodePortName}" already exists`);
|
||||||
|
} catch (err: any) {
|
||||||
|
if (err.statusCode === 404 || err.body?.code === 404) {
|
||||||
|
await coreApi.createNamespacedService(buildNs, {
|
||||||
|
metadata: { name: registryNodePortName, namespace: buildNs, labels: { app: 'registry' } },
|
||||||
|
spec: {
|
||||||
|
type: 'NodePort',
|
||||||
|
selector: { app: 'registry' },
|
||||||
|
ports: [{ port: 5000, targetPort: 5000 as any, nodePort: 30500, protocol: 'TCP' }],
|
||||||
|
},
|
||||||
|
});
|
||||||
|
this.logger.log(`Created Registry NodePort Service (30500 → 5000)`);
|
||||||
|
} else {
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── 7. registry-credentials Secret (docker config for Kaniko) ─
|
||||||
|
const registrySecretName = 'registry-credentials';
|
||||||
|
try {
|
||||||
|
await coreApi.readNamespacedSecret(registrySecretName, buildNs);
|
||||||
|
this.logger.log(`Secret "${registrySecretName}" already exists`);
|
||||||
|
} catch (err: any) {
|
||||||
|
if (err.statusCode === 404 || err.body?.code === 404) {
|
||||||
|
// Parse registry host (without port path) for the docker config
|
||||||
|
const dockerConfig = JSON.stringify({
|
||||||
|
auths: {
|
||||||
|
[registryUrl]: { auth: '' },
|
||||||
|
[`registry.${buildNs}.svc.cluster.local:5000`]: { auth: '' },
|
||||||
|
'localhost:30500': { auth: '' },
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await coreApi.createNamespacedSecret(buildNs, {
|
||||||
|
metadata: { name: registrySecretName, namespace: buildNs },
|
||||||
|
type: 'kubernetes.io/dockerconfigjson',
|
||||||
|
data: {
|
||||||
|
'.dockerconfigjson': Buffer.from(dockerConfig).toString('base64'),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
this.logger.log(`Created registry-credentials Secret`);
|
||||||
|
} else {
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
this.logger.log(`✅ Cluster bootstrap complete — build infrastructure ready`);
|
||||||
|
}
|
||||||
|
|
||||||
private parseCpuToMillicores(cpu: string): number {
|
private parseCpuToMillicores(cpu: string): number {
|
||||||
if (!cpu || cpu === '0') return 0;
|
if (!cpu || cpu === '0') return 0;
|
||||||
if (cpu.endsWith('n')) return parseFloat(cpu) / 1_000_000;
|
if (cpu.endsWith('n')) return parseFloat(cpu) / 1_000_000;
|
||||||
|
|||||||
@@ -6,7 +6,6 @@ import { Deployment } from './entities/deployment.entity';
|
|||||||
import { ApplicationsModule } from '../applications/applications.module';
|
import { ApplicationsModule } from '../applications/applications.module';
|
||||||
import { KubernetesModule } from '../kubernetes/kubernetes.module';
|
import { KubernetesModule } from '../kubernetes/kubernetes.module';
|
||||||
import { BuildModule } from '../build/build.module';
|
import { BuildModule } from '../build/build.module';
|
||||||
import { SnapshotsModule } from '../snapshots/snapshots.module';
|
|
||||||
|
|
||||||
@Module({
|
@Module({
|
||||||
imports: [
|
imports: [
|
||||||
@@ -14,7 +13,6 @@ import { SnapshotsModule } from '../snapshots/snapshots.module';
|
|||||||
forwardRef(() => ApplicationsModule),
|
forwardRef(() => ApplicationsModule),
|
||||||
KubernetesModule,
|
KubernetesModule,
|
||||||
BuildModule,
|
BuildModule,
|
||||||
forwardRef(() => SnapshotsModule),
|
|
||||||
],
|
],
|
||||||
controllers: [DeploymentsController],
|
controllers: [DeploymentsController],
|
||||||
providers: [DeploymentsService],
|
providers: [DeploymentsService],
|
||||||
|
|||||||
@@ -1,12 +1,12 @@
|
|||||||
import { Injectable, NotFoundException, Logger, Inject, forwardRef } from '@nestjs/common';
|
import { Injectable, NotFoundException, Logger, Inject, forwardRef } from '@nestjs/common';
|
||||||
import { InjectRepository } from '@nestjs/typeorm';
|
import { InjectRepository } from '@nestjs/typeorm';
|
||||||
import { Repository } from 'typeorm';
|
import { Repository } from 'typeorm';
|
||||||
|
import * as fs from 'fs';
|
||||||
import { Deployment } from './entities/deployment.entity';
|
import { Deployment } from './entities/deployment.entity';
|
||||||
import { ApplicationsService } from '../applications/applications.service';
|
import { ApplicationsService } from '../applications/applications.service';
|
||||||
import { KubernetesService } from '../kubernetes/kubernetes.service';
|
import { KubernetesService } from '../kubernetes/kubernetes.service';
|
||||||
import { BuildService } from '../build/build.service';
|
import { BuildService } from '../build/build.service';
|
||||||
import { DeploymentStatus } from '../common/enums';
|
import { DeploymentStatus } from '../common/enums';
|
||||||
import { SnapshotsService } from '../snapshots/snapshots.service';
|
|
||||||
|
|
||||||
@Injectable()
|
@Injectable()
|
||||||
export class DeploymentsService {
|
export class DeploymentsService {
|
||||||
@@ -19,22 +19,11 @@ export class DeploymentsService {
|
|||||||
private applicationsService: ApplicationsService,
|
private applicationsService: ApplicationsService,
|
||||||
private kubernetesService: KubernetesService,
|
private kubernetesService: KubernetesService,
|
||||||
private buildService: BuildService,
|
private buildService: BuildService,
|
||||||
@Inject(forwardRef(() => SnapshotsService))
|
|
||||||
private snapshotsService: SnapshotsService,
|
|
||||||
) {}
|
) {}
|
||||||
|
|
||||||
async triggerDeployment(applicationId: string, userId: string): Promise<Deployment> {
|
async triggerDeployment(applicationId: string, userId: string): Promise<Deployment> {
|
||||||
const app = await this.applicationsService.findOne(applicationId, userId);
|
const app = await this.applicationsService.findOne(applicationId, userId);
|
||||||
|
|
||||||
// Auto-snapshot before deploying (capture current state for rollback)
|
|
||||||
try {
|
|
||||||
const version = `v${Date.now()}`;
|
|
||||||
await this.snapshotsService.createPreDeploySnapshot(app.id, userId, version);
|
|
||||||
this.logger.log(`Pre-deploy snapshot created for ${app.name}`);
|
|
||||||
} catch (e: any) {
|
|
||||||
this.logger.warn(`Pre-deploy snapshot failed for ${app.name}: ${e.message} — continuing deploy`);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create deployment record
|
// Create deployment record
|
||||||
const deployment = this.deploymentsRepository.create({
|
const deployment = this.deploymentsRepository.create({
|
||||||
applicationId: app.id,
|
applicationId: app.id,
|
||||||
@@ -69,6 +58,26 @@ export class DeploymentsService {
|
|||||||
await this.updateStatus(deploymentId, DeploymentStatus.DEPLOYING);
|
await this.updateStatus(deploymentId, DeploymentStatus.DEPLOYING);
|
||||||
const k8sResources = await this.kubernetesService.deployApplication(app, imageUri);
|
const k8sResources = await this.kubernetesService.deployApplication(app, imageUri);
|
||||||
|
|
||||||
|
// Step 3.5: Restore DB dump if one was uploaded (must happen after deploy creates the namespace + DB)
|
||||||
|
if (app.dbDumpPath && fs.existsSync(app.dbDumpPath)) {
|
||||||
|
this.logger.log(`Restoring DB dump for ${app.name} from ${app.dbDumpPath}`);
|
||||||
|
try {
|
||||||
|
// Wait for the database pod to be Ready before restoring
|
||||||
|
await this.kubernetesService.waitForDatabaseReady(app, 120_000);
|
||||||
|
// Re-fetch app to ensure we have latest data
|
||||||
|
const freshApp = await this.applicationsService.findOne(app.id);
|
||||||
|
const result = await this.kubernetesService.restoreDatabaseDump(freshApp, freshApp.dbDumpPath!);
|
||||||
|
if (result.success) {
|
||||||
|
this.logger.log(`DB dump restored successfully for ${app.name}`);
|
||||||
|
} else {
|
||||||
|
this.logger.warn(`DB dump restore failed for ${app.name}: ${result.logs}`);
|
||||||
|
}
|
||||||
|
} catch (e: any) {
|
||||||
|
this.logger.warn(`DB dump restore error for ${app.name}: ${e.message}`);
|
||||||
|
// Don't fail the deployment — DB restore is a best-effort step
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Step 4: Mark success
|
// Step 4: Mark success
|
||||||
await this.deploymentsRepository.update(deploymentId, {
|
await this.deploymentsRepository.update(deploymentId, {
|
||||||
status: DeploymentStatus.RUNNING,
|
status: DeploymentStatus.RUNNING,
|
||||||
@@ -185,15 +194,6 @@ export class DeploymentsService {
|
|||||||
throw new NotFoundException('No source code available. Upload code or set a git URL first.');
|
throw new NotFoundException('No source code available. Upload code or set a git URL first.');
|
||||||
}
|
}
|
||||||
|
|
||||||
// Auto-snapshot before redeploy
|
|
||||||
try {
|
|
||||||
const version = `v${Date.now()}`;
|
|
||||||
await this.snapshotsService.createPreDeploySnapshot(app.id, userId, version);
|
|
||||||
this.logger.log(`Pre-redeploy snapshot created for ${app.name}`);
|
|
||||||
} catch (e: any) {
|
|
||||||
this.logger.warn(`Pre-redeploy snapshot failed for ${app.name}: ${e.message} — continuing`);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create new deployment record
|
// Create new deployment record
|
||||||
const deployment = this.deploymentsRepository.create({
|
const deployment = this.deploymentsRepository.create({
|
||||||
applicationId: app.id,
|
applicationId: app.id,
|
||||||
|
|||||||
@@ -351,10 +351,9 @@ export class SnapshotsService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 3. Restore database — file-based
|
// 3. Restore database — file-based (pass file path for PVC-based transfer)
|
||||||
if (snapshot.dbDumpPath && fs.existsSync(snapshot.dbDumpPath)) {
|
if (snapshot.dbDumpPath && fs.existsSync(snapshot.dbDumpPath)) {
|
||||||
const dumpBuffer = fs.readFileSync(snapshot.dbDumpPath);
|
const result = await this.kubernetesService.restoreDatabaseDump(app, snapshot.dbDumpPath);
|
||||||
const result = await this.kubernetesService.restoreDatabaseDump(app, dumpBuffer);
|
|
||||||
if (result.success) {
|
if (result.success) {
|
||||||
details.push('✅ Database restored');
|
details.push('✅ Database restored');
|
||||||
this.logger.log(`Rollback ${snapshotId}: restored database`);
|
this.logger.log(`Rollback ${snapshotId}: restored database`);
|
||||||
|
|||||||
Reference in New Issue
Block a user