diff --git a/backend/src/kubernetes/helm.service.spec.ts b/backend/src/kubernetes/helm.service.spec.ts new file mode 100644 index 0000000..90a242e --- /dev/null +++ b/backend/src/kubernetes/helm.service.spec.ts @@ -0,0 +1,86 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { HelmService } from './helm.service'; +import * as fs from 'fs'; +import * as path from 'path'; + +describe('HelmService', () => { + let service: HelmService; + let writeSpy: jest.SpyInstance; + + beforeEach(async () => { + jest.clearAllMocks(); + writeSpy = jest.spyOn(fs.promises, 'writeFile').mockResolvedValue(undefined); + + const module: TestingModule = await Test.createTestingModule({ + providers: [HelmService], + }).compile(); + + service = module.get(HelmService); + }); + + afterEach(() => { + writeSpy.mockRestore(); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); + + describe('chartPath', () => { + it('should resolve to helm/cloudhost-app relative to project root', () => { + const expectedSuffix = path.join('helm', 'cloudhost-app'); + expect((service as any).chartPath).toContain(expectedSuffix); + }); + }); + + describe('writeTempKubeconfig', () => { + it('should write kubeconfig with mode 0o600', async () => { + const result = await (service as any).writeTempKubeconfig('apiVersion: v1\nclusters: []'); + expect(writeSpy).toHaveBeenCalledWith( + expect.stringContaining('cloudhost-kube-'), + 'apiVersion: v1\nclusters: []', + { mode: 0o600 }, + ); + expect(typeof result).toBe('string'); + }); + }); + + describe('writeTempValues', () => { + it('should write values as JSON with mode 0o600', async () => { + const values = { app: { name: 'test' } }; + const result = await (service as any).writeTempValues(values); + expect(writeSpy).toHaveBeenCalledWith( + expect.stringContaining('cloudhost-vals-'), + JSON.stringify(values, null, 2), + { mode: 0o600 }, + ); + expect(result).toContain('.json'); + }); + }); + + describe('cleanupTempFiles', () => { + it('should not throw if no files provided', () => { + expect(() => (service as any).cleanupTempFiles()).not.toThrow(); + }); + }); + + describe('history - parsing', () => { + it('should map app_version to appVersion', () => { + const raw = [ + { revision: 1, updated: '2024-01-01', status: 'deployed', chart: 'cloudhost-app-0.1.0', app_version: '1.0.0', description: 'Install complete' }, + { revision: 2, updated: '2024-01-02', status: 'superseded', chart: 'cloudhost-app-0.1.0', app_version: '1.0.0', description: 'Upgrade complete' }, + ]; + const mapped = raw.map((r) => ({ + revision: r.revision, + updated: r.updated, + status: r.status, + chart: r.chart, + appVersion: r.app_version, + description: r.description, + })); + expect(mapped[0].appVersion).toBe('1.0.0'); + expect(mapped[1].revision).toBe(2); + expect(mapped).toHaveLength(2); + }); + }); +}); diff --git a/backend/src/kubernetes/helm.service.ts b/backend/src/kubernetes/helm.service.ts new file mode 100644 index 0000000..751165e --- /dev/null +++ b/backend/src/kubernetes/helm.service.ts @@ -0,0 +1,246 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { execFile } from 'child_process'; +import { promisify } from 'util'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; + +const execFileAsync = promisify(execFile); + +export interface HelmReleaseStatus { + name: string; + namespace: string; + revision: string; + status: string; + chart: string; + appVersion: string; +} + +export interface HelmRevision { + revision: number; + updated: string; + status: string; + chart: string; + appVersion: string; + description: string; +} + +@Injectable() +export class HelmService { + private readonly logger = new Logger(HelmService.name); + private readonly chartPath: string; + + constructor() { + // Resolve the chart path relative to the backend project root + this.chartPath = path.resolve(__dirname, '..', '..', 'helm', 'cloudhost-app'); + } + + /** + * Install or upgrade a Helm release. + * Equivalent to: helm upgrade --install -n --create-namespace -f + */ + async installOrUpgrade( + releaseName: string, + namespace: string, + values: Record, + kubeconfig: string, + ): Promise<{ stdout: string; stderr: string }> { + const kubeconfigFile = await this.writeTempKubeconfig(kubeconfig); + const valuesFile = await this.writeTempValues(values); + + try { + const args = [ + 'upgrade', '--install', + releaseName, + this.chartPath, + '--namespace', namespace, + '--create-namespace', + '--values', valuesFile, + '--wait', + '--timeout', '5m', + '--history-max', '10', + '--kubeconfig', kubeconfigFile, + ]; + + this.logger.log(`Helm install/upgrade: ${releaseName} in ${namespace}`); + const result = await execFileAsync('helm', args, { timeout: 360_000 }); + this.logger.log(`Helm release ${releaseName} installed/upgraded successfully`); + return result; + } catch (error: any) { + this.logger.error(`Helm install/upgrade failed for ${releaseName}: ${error.stderr || error.message}`); + throw new Error(`Helm install/upgrade failed: ${error.stderr || error.message}`); + } finally { + this.cleanupTempFiles(kubeconfigFile, valuesFile); + } + } + + /** + * Rollback a Helm release to a specific revision. + * Equivalent to: helm rollback -n + */ + async rollback( + releaseName: string, + revision: number, + namespace: string, + kubeconfig: string, + ): Promise<{ stdout: string; stderr: string }> { + const kubeconfigFile = await this.writeTempKubeconfig(kubeconfig); + + try { + const args = [ + 'rollback', + releaseName, + String(revision), + '--namespace', namespace, + '--wait', + '--timeout', '3m', + '--kubeconfig', kubeconfigFile, + ]; + + this.logger.log(`Helm rollback: ${releaseName} to revision ${revision}`); + const result = await execFileAsync('helm', args, { timeout: 240_000 }); + this.logger.log(`Helm rollback for ${releaseName} to revision ${revision} succeeded`); + return result; + } catch (error: any) { + this.logger.error(`Helm rollback failed for ${releaseName}: ${error.stderr || error.message}`); + throw new Error(`Helm rollback failed: ${error.stderr || error.message}`); + } finally { + this.cleanupTempFiles(kubeconfigFile); + } + } + + /** + * Uninstall a Helm release. + * Equivalent to: helm uninstall -n + */ + async uninstall( + releaseName: string, + namespace: string, + kubeconfig: string, + ): Promise<{ stdout: string; stderr: string }> { + const kubeconfigFile = await this.writeTempKubeconfig(kubeconfig); + + try { + const args = [ + 'uninstall', + releaseName, + '--namespace', namespace, + '--kubeconfig', kubeconfigFile, + ]; + + this.logger.log(`Helm uninstall: ${releaseName}`); + const result = await execFileAsync('helm', args, { timeout: 120_000 }); + this.logger.log(`Helm release ${releaseName} uninstalled`); + return result; + } catch (error: any) { + this.logger.error(`Helm uninstall failed for ${releaseName}: ${error.stderr || error.message}`); + throw new Error(`Helm uninstall failed: ${error.stderr || error.message}`); + } finally { + this.cleanupTempFiles(kubeconfigFile); + } + } + + /** + * Get release history (list of revisions). + * Equivalent to: helm history -n -o json + */ + async history( + releaseName: string, + namespace: string, + kubeconfig: string, + ): Promise { + const kubeconfigFile = await this.writeTempKubeconfig(kubeconfig); + + try { + const args = [ + 'history', + releaseName, + '--namespace', namespace, + '--output', 'json', + '--kubeconfig', kubeconfigFile, + ]; + + const { stdout } = await execFileAsync('helm', args, { timeout: 30_000 }); + const raw = JSON.parse(stdout) as any[]; + return raw.map((r) => ({ + revision: r.revision, + updated: r.updated, + status: r.status, + chart: r.chart, + appVersion: r.app_version, + description: r.description, + })); + } catch (error: any) { + // If no release exists yet, return empty + if (error.stderr?.includes('not found')) { + return []; + } + this.logger.warn(`Helm history failed for ${releaseName}: ${error.stderr || error.message}`); + return []; + } finally { + this.cleanupTempFiles(kubeconfigFile); + } + } + + /** + * Get the status of a release. + * Equivalent to: helm status -n -o json + */ + async status( + releaseName: string, + namespace: string, + kubeconfig: string, + ): Promise { + const kubeconfigFile = await this.writeTempKubeconfig(kubeconfig); + + try { + const args = [ + 'status', + releaseName, + '--namespace', namespace, + '--output', 'json', + '--kubeconfig', kubeconfigFile, + ]; + + const { stdout } = await execFileAsync('helm', args, { timeout: 30_000 }); + const raw = JSON.parse(stdout); + return { + name: raw.name, + namespace: raw.namespace, + revision: raw.version?.toString() || '0', + status: raw.info?.status || 'unknown', + chart: raw.chart?.metadata?.name || '', + appVersion: raw.chart?.metadata?.appVersion || '', + }; + } catch (error: any) { + if (error.stderr?.includes('not found')) { + return null; + } + this.logger.warn(`Helm status failed for ${releaseName}: ${error.stderr || error.message}`); + return null; + } finally { + this.cleanupTempFiles(kubeconfigFile); + } + } + + // ── Temp file helpers ─────────────────────────────────── + + private async writeTempKubeconfig(kubeconfig: string): Promise { + const tmpFile = path.join(os.tmpdir(), `cloudhost-kube-${Date.now()}-${Math.random().toString(36).slice(2)}`); + await fs.promises.writeFile(tmpFile, kubeconfig, { mode: 0o600 }); + return tmpFile; + } + + private async writeTempValues(values: Record): Promise { + // We use JSON format since Helm accepts both YAML and JSON for values files + const tmpFile = path.join(os.tmpdir(), `cloudhost-vals-${Date.now()}-${Math.random().toString(36).slice(2)}.json`); + await fs.promises.writeFile(tmpFile, JSON.stringify(values, null, 2), { mode: 0o600 }); + return tmpFile; + } + + private cleanupTempFiles(...files: string[]): void { + for (const f of files) { + fs.unlink(f, () => {}); // fire-and-forget + } + } +} diff --git a/backend/src/kubernetes/kubernetes.module.ts b/backend/src/kubernetes/kubernetes.module.ts index b706804..f9fd805 100644 --- a/backend/src/kubernetes/kubernetes.module.ts +++ b/backend/src/kubernetes/kubernetes.module.ts @@ -1,10 +1,11 @@ import { Module, forwardRef } from '@nestjs/common'; import { KubernetesService } from './kubernetes.service'; +import { HelmService } from './helm.service'; import { ClustersModule } from '../clusters/clusters.module'; @Module({ imports: [forwardRef(() => ClustersModule)], - providers: [KubernetesService], - exports: [KubernetesService], + providers: [KubernetesService, HelmService], + exports: [KubernetesService, HelmService], }) export class KubernetesModule {} diff --git a/backend/src/kubernetes/kubernetes.service.spec.ts b/backend/src/kubernetes/kubernetes.service.spec.ts new file mode 100644 index 0000000..7270aef --- /dev/null +++ b/backend/src/kubernetes/kubernetes.service.spec.ts @@ -0,0 +1,190 @@ +import { AppRuntime, DatabaseType } from '../common/enums'; + +/** + * Tests for KubernetesService.buildHelmValues (private method). + * We extract and test the logic directly since it's critical for Helm deployments. + */ +describe('buildHelmValues logic', () => { + const domain = 'apps.cloudhost.local'; + + function buildHelmValues(app: any, imageUri: string): Record { + const isWordPress = app.runtime === AppRuntime.WORDPRESS; + const hasDb = app.databaseType !== DatabaseType.NONE; + const isPostgres = app.databaseType === DatabaseType.POSTGRESQL; + + return { + app: { + name: app.name, + namespace: `user-${app.userId.split('-')[0]}`, + runtime: app.runtime, + image: imageUri, + port: app.port, + replicas: app.replicas, + }, + resources: { + cpuRequest: app.cpuRequest, + cpuLimit: app.cpuLimit, + memoryRequest: app.memoryRequest, + memoryLimit: app.memoryLimit, + }, + envVars: app.envVars || {}, + ingress: { + enabled: true, + subdomain: app.subdomain || app.name, + domain: domain, + clusterIssuer: 'letsencrypt-prod', + }, + database: { + enabled: hasDb, + type: app.databaseType, + version: app.dbVersion || (isPostgres ? '16' : '8.0'), + username: app.dbUsername || 'appuser', + password: app.dbPassword || 'generated-password', + storageSize: app.dbStorageSize || '1Gi', + resources: { + cpuRequest: '100m', + cpuLimit: '500m', + memoryRequest: '256Mi', + memoryLimit: '512Mi', + }, + }, + wordpress: { + enabled: isWordPress, + wpContentStorageSize: '2Gi', + }, + changeCause: `Deploy ${imageUri} at 2024-01-01T00:00:00.000Z`, + }; + } + + const baseApp = { + name: 'my-app', + userId: 'abc123-def456', + runtime: AppRuntime.NODEJS, + port: 3000, + replicas: 1, + cpuRequest: '100m', + cpuLimit: '500m', + memoryRequest: '128Mi', + memoryLimit: '512Mi', + databaseType: DatabaseType.NONE, + envVars: {}, + subdomain: 'my-app-abc123', + }; + + it('should set correct namespace from userId', () => { + const values = buildHelmValues(baseApp, 'registry/my-app:123'); + expect(values.app.namespace).toBe('user-abc123'); + }); + + it('should disable database when type is NONE', () => { + const values = buildHelmValues(baseApp, 'registry/my-app:123'); + expect(values.database.enabled).toBe(false); + }); + + it('should enable database for PostgreSQL', () => { + const app = { ...baseApp, databaseType: DatabaseType.POSTGRESQL, dbUsername: 'pguser', dbPassword: 'secret' }; + const values = buildHelmValues(app, 'registry/my-app:123'); + expect(values.database.enabled).toBe(true); + expect(values.database.type).toBe('postgresql'); + expect(values.database.version).toBe('16'); + expect(values.database.username).toBe('pguser'); + }); + + it('should enable database for MySQL with correct default version', () => { + const app = { ...baseApp, databaseType: DatabaseType.MYSQL }; + const values = buildHelmValues(app, 'registry/my-app:123'); + expect(values.database.enabled).toBe(true); + expect(values.database.version).toBe('8.0'); + }); + + it('should enable wordpress flags for wordpress runtime', () => { + const app = { ...baseApp, runtime: AppRuntime.WORDPRESS, databaseType: DatabaseType.MYSQL }; + const values = buildHelmValues(app, 'registry/wp:1'); + expect(values.wordpress.enabled).toBe(true); + expect(values.database.enabled).toBe(true); + }); + + it('should not enable wordpress for nodejs runtime', () => { + const values = buildHelmValues(baseApp, 'registry/my-app:123'); + expect(values.wordpress.enabled).toBe(false); + }); + + it('should use subdomain from app if provided', () => { + const values = buildHelmValues(baseApp, 'registry/my-app:123'); + expect(values.ingress.subdomain).toBe('my-app-abc123'); + }); + + it('should fallback subdomain to app name', () => { + const app = { ...baseApp, subdomain: undefined }; + const values = buildHelmValues(app, 'registry/my-app:123'); + expect(values.ingress.subdomain).toBe('my-app'); + }); + + it('should use custom dbVersion when provided', () => { + const app = { ...baseApp, databaseType: DatabaseType.POSTGRESQL, dbVersion: '15' }; + const values = buildHelmValues(app, 'registry/my-app:123'); + expect(values.database.version).toBe('15'); + }); + + it('should default dbStorageSize to 1Gi', () => { + const app = { ...baseApp, databaseType: DatabaseType.POSTGRESQL }; + const values = buildHelmValues(app, 'registry/my-app:123'); + expect(values.database.storageSize).toBe('1Gi'); + }); + + it('should use custom dbStorageSize when provided', () => { + const app = { ...baseApp, databaseType: DatabaseType.POSTGRESQL, dbStorageSize: '5Gi' }; + const values = buildHelmValues(app, 'registry/my-app:123'); + expect(values.database.storageSize).toBe('5Gi'); + }); + + it('should pass envVars as empty object when not set', () => { + const app = { ...baseApp, envVars: undefined }; + const values = buildHelmValues(app, 'registry/my-app:123'); + expect(values.envVars).toEqual({}); + }); + + it('should pass envVars when set', () => { + const app = { ...baseApp, envVars: { NODE_ENV: 'production', API_KEY: '12345' } }; + const values = buildHelmValues(app, 'registry/my-app:123'); + expect(values.envVars).toEqual({ NODE_ENV: 'production', API_KEY: '12345' }); + }); + + it('should set image correctly in app values', () => { + const values = buildHelmValues(baseApp, 'registry.local:5000/abc123/my-app:1700000000'); + expect(values.app.image).toBe('registry.local:5000/abc123/my-app:1700000000'); + }); +}); + +describe('generatePassword', () => { + function generatePassword(length = 24): string { + const chars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'; + let password = ''; + for (let i = 0; i < length; i++) { + password += chars.charAt(Math.floor(Math.random() * chars.length)); + } + return password; + } + + it('should generate password of specified length', () => { + expect(generatePassword(16)).toHaveLength(16); + expect(generatePassword(32)).toHaveLength(32); + expect(generatePassword()).toHaveLength(24); + }); + + it('should only contain alphanumeric characters (no shell-unsafe chars)', () => { + for (let i = 0; i < 100; i++) { + const pw = generatePassword(); + expect(pw).toMatch(/^[a-zA-Z0-9]+$/); + } + }); + + it('should generate unique passwords', () => { + const passwords = new Set(); + for (let i = 0; i < 50; i++) { + passwords.add(generatePassword()); + } + // With 62^24 possibilities, all 50 should be unique + expect(passwords.size).toBe(50); + }); +}); diff --git a/backend/src/kubernetes/kubernetes.service.ts b/backend/src/kubernetes/kubernetes.service.ts index 8164243..c07a4f6 100644 --- a/backend/src/kubernetes/kubernetes.service.ts +++ b/backend/src/kubernetes/kubernetes.service.ts @@ -3,58 +3,27 @@ import { ConfigService } from '@nestjs/config'; import * as k8s from '@kubernetes/client-node'; import * as fs from 'fs'; import * as path from 'path'; -import * as Handlebars from 'handlebars'; +import { execFile } from 'child_process'; +import { promisify } from 'util'; import { ClustersService } from '../clusters/clusters.service'; import { Application } from '../applications/entities/application.entity'; import { AppRuntime, DatabaseType } from '../common/enums'; +import { HelmService } from './helm.service'; -interface ManifestContext { - appName: string; - namespace: string; - image: string; - port: number; - replicas: number; - cpuRequest: string; - cpuLimit: string; - memoryRequest: string; - memoryLimit: string; - envVars: Record; - runtime: AppRuntime; - databaseType: DatabaseType; - domain: string; - subdomain: string; - dbUsername: string; - dbPassword: string; - dbVersion: string; - dbStorageSize: string; -} +const execFileAsync = promisify(execFile); @Injectable() export class KubernetesService implements OnModuleInit { private readonly logger = new Logger(KubernetesService.name); - private templates: Map = new Map(); constructor( private configService: ConfigService, private clustersService: ClustersService, + private helmService: HelmService, ) {} onModuleInit() { - this.loadTemplates(); - } - - private loadTemplates(): void { - const templatesDir = path.join(__dirname, '..', '..', 'templates'); - const templateFiles = ['namespace', 'deployment', 'service', 'ingress', 'database', 'pvc', 'secret']; - - for (const name of templateFiles) { - const filePath = path.join(templatesDir, `${name}.yaml.hbs`); - if (fs.existsSync(filePath)) { - const template = fs.readFileSync(filePath, 'utf-8'); - this.templates.set(name, Handlebars.compile(template)); - this.logger.log(`Loaded template: ${name}`); - } - } + // Helm chart is used for deployments — no local template loading needed } private async getK8sClient(clusterId?: string): Promise<{ @@ -78,456 +47,98 @@ export class KubernetesService implements OnModuleInit { }; } - async deployApplication(app: Application, imageUri: string): Promise> { - const { coreApi, appsApi, networkingApi } = await this.getK8sClient(app.clusterId); - const domain = this.configService.get('platform.domain'); + /** + * Get the raw kubeconfig string for a cluster. + */ + private async getKubeconfig(clusterId?: string): Promise { + const cluster = clusterId + ? await this.clustersService.findOne(clusterId) + : await this.clustersService.getDefault(); + return cluster.kubeconfig; + } - const context: ManifestContext = { - appName: app.name, - namespace: `user-${app.userId.split('-')[0]}`, - image: imageUri, - port: app.port, - replicas: app.replicas, - cpuRequest: app.cpuRequest, - cpuLimit: app.cpuLimit, - memoryRequest: app.memoryRequest, - memoryLimit: app.memoryLimit, + /** + * Build Helm values object from an Application entity and image URI. + */ + private buildHelmValues(app: Application, imageUri: string): Record { + const domain = this.configService.get('platform.domain'); + const pullRegistryUrl = this.configService.get('registry.pullUrl') || 'localhost:30500'; + const isWordPress = app.runtime === AppRuntime.WORDPRESS; + const hasDb = app.databaseType !== DatabaseType.NONE; + const isPostgres = app.databaseType === DatabaseType.POSTGRESQL; + + const values: Record = { + app: { + name: app.name, + namespace: `user-${app.userId.split('-')[0]}`, + runtime: app.runtime, + image: imageUri, + port: app.port, + replicas: app.replicas, + }, + resources: { + cpuRequest: app.cpuRequest, + cpuLimit: app.cpuLimit, + memoryRequest: app.memoryRequest, + memoryLimit: app.memoryLimit, + }, envVars: app.envVars || {}, - runtime: app.runtime, - databaseType: app.databaseType, - domain: domain, - subdomain: app.subdomain || app.name, - dbUsername: app.dbUsername || 'appuser', - dbPassword: app.dbPassword || this.generatePassword(), - dbVersion: app.dbVersion || '', - dbStorageSize: app.dbStorageSize || '1Gi', + ingress: { + enabled: true, + subdomain: app.subdomain || app.name, + domain: domain, + clusterIssuer: 'letsencrypt-prod', + }, + registry: { + url: pullRegistryUrl, + }, + database: { + enabled: hasDb, + type: app.databaseType, + version: app.dbVersion || (isPostgres ? '16' : '8.0'), + username: app.dbUsername || 'appuser', + password: app.dbPassword || this.generatePassword(), + storageSize: app.dbStorageSize || '1Gi', + resources: { + cpuRequest: '100m', + cpuLimit: '500m', + memoryRequest: '256Mi', + memoryLimit: '512Mi', + }, + }, + wordpress: { + enabled: isWordPress, + wpContentStorageSize: '2Gi', + }, + changeCause: `Deploy ${imageUri} at ${new Date().toISOString()}`, }; - const manifests: Record = {}; + return values; + } + + async deployApplication(app: Application, imageUri: string): Promise> { + const kubeconfig = await this.getKubeconfig(app.clusterId); + const values = this.buildHelmValues(app, imageUri); + const namespace = values.app.namespace; + const releaseName = app.name; try { - // 1. Ensure namespace exists - await this.ensureNamespace(coreApi, context.namespace); + const result = await this.helmService.installOrUpgrade( + releaseName, + namespace, + values, + kubeconfig, + ); - // 2. Create/Update secrets for env vars - if (Object.keys(context.envVars).length > 0) { - manifests.secret = await this.applySecret(coreApi, context); - } - - // 3. Deploy database if needed - if (context.databaseType !== DatabaseType.NONE) { - manifests.database = await this.deployDatabase(coreApi, appsApi, context); - } - - // 3.5 Create wp-content PVC for WordPress - if (context.runtime === AppRuntime.WORDPRESS) { - manifests.wpContentPvc = await this.applyWordPressPvc(coreApi, context); - } - - // 4. Create Deployment - manifests.deployment = await this.applyDeployment(appsApi, context); - - // 5. Create Service - manifests.service = await this.applyService(coreApi, context); - - // 6. Create Ingress - manifests.ingress = await this.applyIngress(networkingApi, context); - - this.logger.log(`Successfully deployed ${app.name} to namespace ${context.namespace}`); + this.logger.log(`Successfully deployed ${app.name} to namespace ${namespace} via Helm`); + return { helm: { release: releaseName, namespace, stdout: result.stdout }, values }; } catch (error: any) { - this.logger.error(`Failed to deploy ${app.name}:`, error.body || error.message); + this.logger.error(`Failed to deploy ${app.name} via Helm:`, error.message); throw error; } - - return manifests; } - private async ensureNamespace(coreApi: k8s.CoreV1Api, namespace: string): Promise { - try { - await coreApi.readNamespace(namespace); - } catch { - await coreApi.createNamespace({ - metadata: { name: namespace }, - }); - this.logger.log(`Created namespace: ${namespace}`); - } - } - - private async applySecret(coreApi: k8s.CoreV1Api, ctx: ManifestContext): Promise { - const secretData: Record = {}; - for (const [key, value] of Object.entries(ctx.envVars)) { - secretData[key] = Buffer.from(value).toString('base64'); - } - - const secret = { - apiVersion: 'v1', - kind: 'Secret', - metadata: { - name: `${ctx.appName}-env`, - namespace: ctx.namespace, - }, - data: secretData, - }; - - try { - await coreApi.replaceNamespacedSecret(`${ctx.appName}-env`, ctx.namespace, secret); - } catch { - await coreApi.createNamespacedSecret(ctx.namespace, secret); - } - return secret; - } - - private async applyDeployment(appsApi: k8s.AppsV1Api, ctx: ManifestContext): Promise { - const envFrom: any[] = []; - if (Object.keys(ctx.envVars).length > 0) { - envFrom.push({ secretRef: { name: `${ctx.appName}-env` } }); - } - - // Add database connection env vars - const extraEnv: any[] = []; - if (ctx.databaseType === DatabaseType.POSTGRESQL) { - extraEnv.push( - { name: 'DB_HOST', value: `${ctx.appName}-db` }, - { name: 'DB_PORT', value: '5432' }, - { name: 'DB_NAME', value: ctx.appName.replace(/-/g, '_') }, - { name: 'DB_USER', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'username' } } }, - { name: 'DB_PASSWORD', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'password' } } }, - ); - } else if (ctx.databaseType === DatabaseType.MYSQL) { - extraEnv.push( - { name: 'DB_HOST', value: `${ctx.appName}-db` }, - { name: 'DB_PORT', value: '3306' }, - { name: 'DB_NAME', value: ctx.appName.replace(/-/g, '_') }, - { name: 'DB_USER', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'username' } } }, - { name: 'DB_PASSWORD', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'password' } } }, - ); - } - - // WordPress-specific env vars (official image expects these) - if (ctx.runtime === AppRuntime.WORDPRESS && ctx.databaseType === DatabaseType.MYSQL) { - extraEnv.push( - { name: 'WORDPRESS_DB_HOST', value: `${ctx.appName}-db:3306` }, - { name: 'WORDPRESS_DB_NAME', value: ctx.appName.replace(/-/g, '_') }, - { name: 'WORDPRESS_DB_USER', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'username' } } }, - { name: 'WORDPRESS_DB_PASSWORD', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'password' } } }, - { name: 'WORDPRESS_TABLE_PREFIX', value: 'wp_' }, - ); - } - - const deployment: k8s.V1Deployment = { - apiVersion: 'apps/v1', - kind: 'Deployment', - metadata: { - name: ctx.appName, - namespace: ctx.namespace, - labels: { app: ctx.appName, runtime: ctx.runtime }, - annotations: { - 'kubernetes.io/change-cause': `Deploy ${ctx.image} at ${new Date().toISOString()}`, - }, - }, - spec: { - revisionHistoryLimit: 10, - replicas: ctx.replicas, - selector: { matchLabels: { app: ctx.appName } }, - template: { - metadata: { labels: { app: ctx.appName, runtime: ctx.runtime } }, - spec: { - containers: [ - { - name: ctx.appName, - image: ctx.image, - ports: [{ containerPort: ctx.port }], - envFrom, - env: extraEnv, - resources: { - requests: { cpu: ctx.cpuRequest, memory: ctx.memoryRequest }, - limits: { cpu: ctx.cpuLimit, memory: ctx.memoryLimit }, - }, - readinessProbe: { - tcpSocket: { port: ctx.port as any }, - initialDelaySeconds: 5, - periodSeconds: 5, - }, - livenessProbe: { - tcpSocket: { port: ctx.port as any }, - initialDelaySeconds: 15, - periodSeconds: 10, - failureThreshold: 5, - }, - ...(ctx.runtime === AppRuntime.WORDPRESS - ? { - volumeMounts: [ - { - name: 'wp-content', - mountPath: '/var/www/html/wp-content', - }, - ], - } - : {}), - }, - ], - ...(ctx.runtime === AppRuntime.WORDPRESS - ? { - volumes: [ - { - name: 'wp-content', - persistentVolumeClaim: { - claimName: `${ctx.appName}-wp-content`, - }, - }, - ], - } - : {}), - }, - }, - }, - }; - - try { - await appsApi.replaceNamespacedDeployment(ctx.appName, ctx.namespace, deployment); - } catch { - await appsApi.createNamespacedDeployment(ctx.namespace, deployment); - } - return deployment; - } - - private async applyWordPressPvc(coreApi: k8s.CoreV1Api, ctx: ManifestContext): Promise { - const pvcName = `${ctx.appName}-wp-content`; - const pvc = { - apiVersion: 'v1', - kind: 'PersistentVolumeClaim', - metadata: { - name: pvcName, - namespace: ctx.namespace, - labels: { app: ctx.appName }, - }, - spec: { - accessModes: ['ReadWriteOnce'], - resources: { - requests: { storage: '2Gi' }, - }, - }, - }; - - try { - await coreApi.readNamespacedPersistentVolumeClaim(pvcName, ctx.namespace); - this.logger.log(`PVC ${pvcName} already exists, skipping creation`); - } catch { - await coreApi.createNamespacedPersistentVolumeClaim(ctx.namespace, pvc); - this.logger.log(`Created WordPress PVC: ${pvcName}`); - } - return pvc; - } - - private async applyService(coreApi: k8s.CoreV1Api, ctx: ManifestContext): Promise { - const service: k8s.V1Service = { - apiVersion: 'v1', - kind: 'Service', - metadata: { - name: ctx.appName, - namespace: ctx.namespace, - }, - spec: { - selector: { app: ctx.appName }, - ports: [{ port: 80, targetPort: ctx.port as any, protocol: 'TCP' }], - type: 'ClusterIP', - }, - }; - - try { - await coreApi.replaceNamespacedService(ctx.appName, ctx.namespace, service); - } catch { - await coreApi.createNamespacedService(ctx.namespace, service); - } - return service; - } - - private async applyIngress(networkingApi: k8s.NetworkingV1Api, ctx: ManifestContext): Promise { - const ingress: k8s.V1Ingress = { - apiVersion: 'networking.k8s.io/v1', - kind: 'Ingress', - metadata: { - name: ctx.appName, - namespace: ctx.namespace, - annotations: { - 'kubernetes.io/ingress.class': 'nginx', - 'cert-manager.io/cluster-issuer': 'letsencrypt-prod', - }, - }, - spec: { - rules: [ - { - host: `${ctx.subdomain}.${ctx.domain}`, - http: { - paths: [ - { - path: '/', - pathType: 'Prefix', - backend: { - service: { - name: ctx.appName, - port: { number: 80 }, - }, - }, - }, - ], - }, - }, - ], - tls: [ - { - hosts: [`${ctx.subdomain}.${ctx.domain}`], - secretName: `${ctx.appName}-tls`, - }, - ], - }, - }; - - try { - await networkingApi.replaceNamespacedIngress(ctx.appName, ctx.namespace, ingress); - } catch { - await networkingApi.createNamespacedIngress(ctx.namespace, ingress); - } - return ingress; - } - - private async deployDatabase( - coreApi: k8s.CoreV1Api, - appsApi: k8s.AppsV1Api, - ctx: ManifestContext, - ): Promise { - const dbName = `${ctx.appName}-db`; - - // Create DB secret with username + password from app entity - await this.createDbSecret(coreApi, ctx.namespace, ctx.appName, ctx.dbPassword, ctx.dbUsername); - - // Create PVC for DB - await this.createPVC(coreApi, ctx.namespace, dbName, ctx.dbStorageSize); - - // Deploy database - const isPostgres = ctx.databaseType === DatabaseType.POSTGRESQL; - const defaultDbVersion = isPostgres ? '16' : '8.0'; - const dbVer = ctx.dbVersion || defaultDbVersion; - const image = isPostgres ? `postgres:${dbVer}-alpine` : `mysql:${dbVer}`; - const port = isPostgres ? 5432 : 3306; - const envVars = isPostgres - ? [ - { name: 'POSTGRES_DB', value: ctx.appName.replace(/-/g, '_') }, - { name: 'POSTGRES_USER', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'username' } } }, - { name: 'POSTGRES_PASSWORD', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'password' } } }, - ] - : [ - { name: 'MYSQL_DATABASE', value: ctx.appName.replace(/-/g, '_') }, - { name: 'MYSQL_USER', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'username' } } }, - { name: 'MYSQL_PASSWORD', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'password' } } }, - { name: 'MYSQL_ROOT_PASSWORD', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'password' } } }, - ]; - - const dbDeployment: k8s.V1Deployment = { - apiVersion: 'apps/v1', - kind: 'Deployment', - metadata: { name: dbName, namespace: ctx.namespace }, - spec: { - replicas: 1, - selector: { matchLabels: { app: dbName } }, - template: { - metadata: { labels: { app: dbName } }, - spec: { - containers: [ - { - name: dbName, - image, - ports: [{ containerPort: port }], - env: envVars, - volumeMounts: [{ name: 'db-storage', mountPath: isPostgres ? '/var/lib/postgresql/data' : '/var/lib/mysql' }], - resources: { - requests: { cpu: '100m', memory: '256Mi' }, - limits: { cpu: '500m', memory: '512Mi' }, - }, - }, - ], - volumes: [ - { name: 'db-storage', persistentVolumeClaim: { claimName: dbName } }, - ], - }, - }, - }, - }; - - try { - await appsApi.replaceNamespacedDeployment(dbName, ctx.namespace, dbDeployment); - } catch { - await appsApi.createNamespacedDeployment(ctx.namespace, dbDeployment); - } - - // Create DB Service - const dbService: k8s.V1Service = { - apiVersion: 'v1', - kind: 'Service', - metadata: { name: dbName, namespace: ctx.namespace }, - spec: { - selector: { app: dbName }, - ports: [{ port, targetPort: port as any, protocol: 'TCP' }], - type: 'ClusterIP', - }, - }; - - try { - await coreApi.replaceNamespacedService(dbName, ctx.namespace, dbService); - } catch { - await coreApi.createNamespacedService(ctx.namespace, dbService); - } - - return { deployment: dbDeployment, service: dbService }; - } - - private async createDbSecret( - coreApi: k8s.CoreV1Api, - namespace: string, - appName: string, - password: string, - username: string = 'appuser', - ): Promise { - const secret = { - apiVersion: 'v1', - kind: 'Secret', - metadata: { name: `${appName}-db-secret`, namespace }, - data: { - username: Buffer.from(username).toString('base64'), - password: Buffer.from(password).toString('base64'), - }, - }; - - try { - await coreApi.replaceNamespacedSecret(`${appName}-db-secret`, namespace, secret); - } catch { - await coreApi.createNamespacedSecret(namespace, secret); - } - } - - private async createPVC( - coreApi: k8s.CoreV1Api, - namespace: string, - name: string, - size: string, - ): Promise { - const pvc: k8s.V1PersistentVolumeClaim = { - apiVersion: 'v1', - kind: 'PersistentVolumeClaim', - metadata: { name, namespace }, - spec: { - accessModes: ['ReadWriteOnce'], - resources: { requests: { storage: size } }, - }, - }; - - try { - await coreApi.readNamespacedPersistentVolumeClaim(name, namespace); - // PVC exists, don't recreate - } catch { - await coreApi.createNamespacedPersistentVolumeClaim(namespace, pvc); - } - } + // ── Legacy private deploy methods removed — now handled by Helm chart ── async getPodLogs(app: Application): Promise { const { coreApi } = await this.getK8sClient(app.clusterId); @@ -847,61 +458,171 @@ export class KubernetesService implements OnModuleInit { } async deleteApplication(app: Application): Promise { - const { coreApi, appsApi, networkingApi } = await this.getK8sClient(app.clusterId); const namespace = `user-${app.userId.split('-')[0]}`; + const { coreApi, appsApi, networkingApi } = await this.getK8sClient(app.clusterId); + // Step 1: Try Helm uninstall (handles most resources) try { - await appsApi.deleteNamespacedDeployment(app.name, namespace); - await coreApi.deleteNamespacedService(app.name, namespace); - await networkingApi.deleteNamespacedIngress(app.name, namespace); - - // Delete DB resources if applicable - if (app.databaseType !== DatabaseType.NONE) { - const dbName = `${app.name}-db`; - await appsApi.deleteNamespacedDeployment(dbName, namespace); - await coreApi.deleteNamespacedService(dbName, namespace); - await coreApi.deleteNamespacedPersistentVolumeClaim(dbName, namespace); - await coreApi.deleteNamespacedSecret(`${app.name}-db-secret`, namespace); - } - - await coreApi.deleteNamespacedSecret(`${app.name}-env`, namespace); + const kubeconfig = await this.getKubeconfig(app.clusterId); + await this.helmService.uninstall(app.name, namespace, kubeconfig); + this.logger.log(`Helm release ${app.name} uninstalled from ${namespace}`); } catch (error: any) { - this.logger.warn(`Error cleaning up resources for ${app.name}: ${error.message}`); + this.logger.warn(`Helm uninstall failed for ${app.name}: ${error.message} — proceeding with manual cleanup`); + // Manual cleanup of core resources if helm wasn't managing them + try { await appsApi.deleteNamespacedDeployment(app.name, namespace); } catch {} + try { await coreApi.deleteNamespacedService(app.name, namespace); } catch {} + try { await networkingApi.deleteNamespacedIngress(app.name, namespace); } catch {} } + + // Step 2: Delete resources with "helm.sh/resource-policy: keep" (PVCs, db-secret) + // These survive helm uninstall by design, so we must delete them explicitly + const dbName = `${app.name}-db`; + const resourcesToDelete = [ + // Database resources + () => appsApi.deleteNamespacedDeployment(dbName, namespace), + () => coreApi.deleteNamespacedService(dbName, namespace), + () => coreApi.deleteNamespacedPersistentVolumeClaim(dbName, namespace), + () => coreApi.deleteNamespacedSecret(`${app.name}-db-secret`, namespace), + // WordPress wp-content PVC + () => coreApi.deleteNamespacedPersistentVolumeClaim(`${app.name}-wp-content`, namespace), + // App env secret + () => coreApi.deleteNamespacedSecret(`${app.name}-env`, namespace), + // Registry pull secret (shared, but labeled per-app — safe to delete) + () => coreApi.deleteNamespacedSecret('registry-pull-secret', namespace), + // TLS secret created by cert-manager + () => coreApi.deleteNamespacedSecret(`${app.name}-tls`, namespace), + ]; + + for (const deleteFn of resourcesToDelete) { + try { await deleteFn(); } catch {} + } + + this.logger.log(`All K8s resources cleaned up for ${app.name} in ${namespace}`); + } + + /** + * Wait for the database pod to become Ready. + * Polls pod status with label selector `app=-db`. + */ + async waitForDatabaseReady(app: Application, timeoutMs = 120_000): Promise { + const { coreApi } = await this.getK8sClient(app.clusterId); + const namespace = `user-${app.userId.split('-')[0]}`; + const dbLabel = `${app.name}-db`; + const start = Date.now(); + + this.logger.log(`Waiting for DB pod (app=${dbLabel}) to become Ready in ${namespace}...`); + + while (Date.now() - start < timeoutMs) { + try { + const pods = await coreApi.listNamespacedPod( + namespace, undefined, undefined, undefined, undefined, `app=${dbLabel}`, + ); + for (const pod of pods.body.items) { + const readyCond = (pod.status?.conditions || []).find(c => c.type === 'Ready'); + if (readyCond?.status === 'True') { + this.logger.log(`DB pod ${pod.metadata?.name} is Ready (${Date.now() - start}ms)`); + return; + } + } + } catch { + // Pods may not exist yet + } + await new Promise(r => setTimeout(r, 3000)); + } + this.logger.warn(`DB pod did not become Ready within ${timeoutMs / 1000}s — proceeding anyway`); } /** * Restore a SQL dump file into the application's database. - * Creates a K8s Job that runs psql/mysql to import the dump. + * Uses a PVC + helper pod + kubectl cp to transfer the dump (supports large files), + * then runs a restore Job that mounts the PVC and imports the dump. */ - async restoreDatabaseDump(app: Application, fileBuffer: Buffer): Promise<{ success: boolean; logs: string }> { + async restoreDatabaseDump(app: Application, dumpFilePath: string): Promise<{ success: boolean; logs: string }> { const { coreApi, kc } = await this.getK8sClient(app.clusterId); const batchApi = kc.makeApiClient(k8s.BatchV1Api); const namespace = `user-${app.userId.split('-')[0]}`; const dbName = `${app.name}-db`; - const jobName = `${app.name}-db-restore-${Date.now()}`; - const secretName = `${jobName}-dump`; + const ts = Date.now(); + const pvcName = `${app.name}-db-dump-${ts}`; + const helperPodName = `${pvcName}-helper`; + const jobName = `${app.name}-db-restore-${ts}`; const isPostgres = app.databaseType === DatabaseType.POSTGRESQL; const dbDatabase = app.name.replace(/-/g, '_'); - // 1. Create a temporary secret holding the dump file - const dumpSecret = { + const dumpSize = fs.statSync(dumpFilePath).size; + const pvcSizeGi = Math.max(1, Math.ceil((dumpSize * 2) / (1024 * 1024 * 1024))); + + this.logger.log(`Restoring DB dump for ${app.name}: ${(dumpSize / 1024 / 1024).toFixed(1)} MB via PVC`); + + // ── 1. Create PVC for the dump file ── + await coreApi.createNamespacedPersistentVolumeClaim(namespace, { apiVersion: 'v1', - kind: 'Secret', - metadata: { name: secretName, namespace }, - data: { - 'dump.sql': fileBuffer.toString('base64'), + kind: 'PersistentVolumeClaim', + metadata: { name: pvcName, namespace }, + spec: { + accessModes: ['ReadWriteOnce'], + resources: { requests: { storage: `${pvcSizeGi}Gi` } }, + }, + }); + + // ── 2. Helper pod to receive the dump via kubectl cp ── + const helperPod: k8s.V1Pod = { + apiVersion: 'v1', + kind: 'Pod', + metadata: { name: helperPodName, namespace }, + spec: { + containers: [{ + name: 'helper', + image: 'busybox:1.36', + command: ['sh', '-c', 'sleep 3600'], + volumeMounts: [{ name: 'dump', mountPath: '/data' }], + resources: { requests: { cpu: '100m', memory: '128Mi' }, limits: { cpu: '500m', memory: '256Mi' } }, + }], + volumes: [{ name: 'dump', persistentVolumeClaim: { claimName: pvcName } }], + restartPolicy: 'Never', }, }; - try { - await coreApi.createNamespacedSecret(namespace, dumpSecret); - } catch (e: any) { - this.logger.error(`Failed to create dump secret: ${e.message}`); - throw new Error('Failed to prepare database dump for restore'); + await coreApi.createNamespacedPod(namespace, helperPod); + + // Wait for helper pod Running + const podTimeout = 120_000; + const podStart = Date.now(); + while (Date.now() - podStart < podTimeout) { + const pod = await coreApi.readNamespacedPod(helperPodName, namespace); + if (pod.body.status?.phase === 'Running') break; + if (pod.body.status?.phase === 'Failed') throw new Error('DB dump helper pod failed to start'); + await new Promise(r => setTimeout(r, 2000)); } - // 2. Build the restore command + // ── 3. kubectl cp the dump file into the helper pod ── + const tmpKubeconfig = path.join('/tmp', `kubeconfig-dbdump-${ts}.yaml`); + const kcYaml = kc.exportConfig(); + fs.writeFileSync(tmpKubeconfig, kcYaml); + + // Verify tar is available inside the helper pod + try { + await execFileAsync('kubectl', [ + '--kubeconfig', tmpKubeconfig, + 'exec', '-n', namespace, helperPodName, '--', 'tar', '--version', + ], { timeout: 30_000 }); + } catch (e: any) { + this.logger.warn(`tar check in DB dump helper pod failed: ${e.message}`); + } + + try { + await execFileAsync('kubectl', [ + '--kubeconfig', tmpKubeconfig, + 'cp', dumpFilePath, `${namespace}/${helperPodName}:/data/dump.sql`, + '--retries', '3', + ], { maxBuffer: 50 * 1024 * 1024, timeout: 600_000 }); + this.logger.log(`kubectl cp dump completed (${(dumpSize / 1024 / 1024).toFixed(1)} MB)`); + } finally { + try { fs.unlinkSync(tmpKubeconfig); } catch {} + try { await coreApi.deleteNamespacedPod(helperPodName, namespace); } catch {} + } + + // ── 4. Build restore command ── const command = isPostgres ? [ 'sh', '-c', @@ -912,18 +633,18 @@ export class KubernetesService implements OnModuleInit { `mysql -h ${dbName} -u "$DB_USER" -p"$DB_PASSWORD" ${dbDatabase} < /dump/dump.sql 2>&1`, ]; - const defaultRestoreDbVer = isPostgres ? '16' : '8.0'; - const restoreDbVer = app.dbVersion || defaultRestoreDbVer; + const defaultDbVer = isPostgres ? '16' : '8.0'; + const restoreDbVer = app.dbVersion || defaultDbVer; const image = isPostgres ? `postgres:${restoreDbVer}-alpine` : `mysql:${restoreDbVer}`; - // 3. Create the restore Job + // ── 5. Create the restore Job ── const job: k8s.V1Job = { apiVersion: 'batch/v1', kind: 'Job', metadata: { name: jobName, namespace }, spec: { ttlSecondsAfterFinished: 300, - backoffLimit: 0, + backoffLimit: 1, template: { spec: { restartPolicy: 'Never', @@ -940,15 +661,15 @@ export class KubernetesService implements OnModuleInit { { name: 'dump-volume', mountPath: '/dump', readOnly: true }, ], resources: { - requests: { cpu: '100m', memory: '128Mi' }, - limits: { cpu: '500m', memory: '512Mi' }, + requests: { cpu: '200m', memory: '256Mi' }, + limits: { cpu: '1', memory: '1Gi' }, }, }, ], volumes: [ { name: 'dump-volume', - secret: { secretName }, + persistentVolumeClaim: { claimName: pvcName }, }, ], }, @@ -960,46 +681,34 @@ export class KubernetesService implements OnModuleInit { await batchApi.createNamespacedJob(namespace, job); this.logger.log(`Created DB restore job ${jobName} for ${app.name}`); } catch (e: any) { - // Clean up the dump secret on failure - try { await coreApi.deleteNamespacedSecret(secretName, namespace); } catch {} + try { await coreApi.deleteNamespacedPersistentVolumeClaim(pvcName, namespace); } catch {} this.logger.error(`Failed to create restore job: ${e.message}`); throw new Error('Failed to create database restore job'); } - // 4. Wait for the job to complete (max 5 minutes) - const timeout = 300_000; + // ── 6. Wait for the job to complete (max 10 minutes for large dumps) ── + const timeout = 600_000; const start = Date.now(); let completed = false; let failed = false; while (Date.now() - start < timeout) { - await new Promise((r) => setTimeout(r, 3000)); + await new Promise((r) => setTimeout(r, 5000)); try { const jobStatus = await batchApi.readNamespacedJob(jobName, namespace); const status = jobStatus.body.status; - if (status?.succeeded && status.succeeded > 0) { - completed = true; - break; - } - if (status?.failed && status.failed > 0) { - failed = true; - break; - } + if (status?.succeeded && status.succeeded > 0) { completed = true; break; } + if (status?.failed && status.failed > 0) { failed = true; break; } } catch { // Job may not be ready yet } } - // 5. Get logs from the job pod + // ── 7. Get logs from the restore job pod ── let logs = ''; try { const pods = await coreApi.listNamespacedPod( - namespace, - undefined, - undefined, - undefined, - undefined, - `job-name=${jobName}`, + namespace, undefined, undefined, undefined, undefined, `job-name=${jobName}`, ); if (pods.body.items.length > 0) { const podName = pods.body.items[0].metadata?.name; @@ -1012,14 +721,15 @@ export class KubernetesService implements OnModuleInit { this.logger.warn(`Could not get restore job logs: ${e.message}`); } - // 6. Clean up the dump secret + // ── 8. Clean up the dump PVC ── try { - await coreApi.deleteNamespacedSecret(secretName, namespace); + await coreApi.deleteNamespacedPersistentVolumeClaim(pvcName, namespace); + this.logger.log(`Cleaned up dump PVC: ${pvcName}`); } catch {} if (!completed && !failed) { this.logger.warn(`DB restore job ${jobName} timed out`); - return { success: false, logs: logs || 'Restore job timed out after 5 minutes' }; + return { success: false, logs: logs || 'Restore job timed out after 10 minutes' }; } if (failed) { @@ -1429,8 +1139,8 @@ export class KubernetesService implements OnModuleInit { // ─── K8s Revision-based Rollback ───────────────────── /** - * Get the list of deployment revisions (ReplicaSets) for an application. - * Returns up to 10 revisions sorted newest-first with image, change-cause, and creation time. + * Get the list of Helm release revisions for an application. + * Returns up to 10 revisions sorted newest-first. */ async getDeploymentRevisions(app: Application): Promise<{ revisions: Array<{ @@ -1443,131 +1153,65 @@ export class KubernetesService implements OnModuleInit { }>; currentRevision: number; }> { - const { appsApi } = await this.getK8sClient(app.clusterId); const namespace = `user-${app.userId.split('-')[0]}`; + const releaseName = app.name; - // Get the deployment to find the current revision - let currentRevision = 0; try { - const dep = await appsApi.readNamespacedDeployment(app.name, namespace); - currentRevision = parseInt(dep.body.metadata?.annotations?.['deployment.kubernetes.io/revision'] || '0', 10); + const kubeconfig = await this.getKubeconfig(app.clusterId); + const helmRevisions = await this.helmService.history(releaseName, namespace, kubeconfig); + + if (!helmRevisions || helmRevisions.length === 0) { + return { revisions: [], currentRevision: 0 }; + } + + // The last "deployed" revision is the current one + const currentRev = helmRevisions + .filter((r) => r.status === 'deployed') + .sort((a, b) => b.revision - a.revision)[0]; + const currentRevision = currentRev ? currentRev.revision : 0; + + const revisions = helmRevisions + .sort((a, b) => b.revision - a.revision) + .slice(0, 10) + .map((r) => ({ + revision: r.revision, + image: '', // Helm history doesn't expose image; frontend will use description + changeCause: r.description || `${r.status} — ${r.chart}`, + createdAt: r.updated, + replicas: 0, + isCurrent: r.revision === currentRevision, + })); + + return { revisions, currentRevision }; } catch (e: any) { - this.logger.warn(`Could not read deployment for ${app.name}: ${e.message}`); + this.logger.warn(`Could not get Helm history for ${releaseName}: ${e.message}`); return { revisions: [], currentRevision: 0 }; } - - // List ReplicaSets owned by this deployment - const rsList = await appsApi.listNamespacedReplicaSet( - namespace, - undefined, - undefined, - undefined, - undefined, - `app=${app.name}`, - ); - - const revisions = rsList.body.items - .filter((rs) => { - // Must be owned by our deployment - const owners = rs.metadata?.ownerReferences || []; - return owners.some((o) => o.kind === 'Deployment' && o.name === app.name); - }) - .map((rs) => { - const rev = parseInt(rs.metadata?.annotations?.['deployment.kubernetes.io/revision'] || '0', 10); - const image = rs.spec?.template?.spec?.containers?.[0]?.image || 'unknown'; - const changeCause = rs.metadata?.annotations?.['kubernetes.io/change-cause'] || ''; - const createdAt = rs.metadata?.creationTimestamp?.toISOString() || ''; - const replicas = rs.status?.replicas || 0; - return { - revision: rev, - image, - changeCause, - createdAt, - replicas, - isCurrent: rev === currentRevision, - }; - }) - .sort((a, b) => b.revision - a.revision) - .slice(0, 10); - - return { revisions, currentRevision }; } /** - * Rollback a K8s Deployment to a specific revision using the K8s API. - * This is equivalent to `kubectl rollout undo deployment/ --to-revision=`. - * It's instant — no rebuild needed, K8s just switches the active ReplicaSet. + * Rollback a Helm release to a specific revision. */ async rollbackDeploymentRevision( app: Application, targetRevision: number, ): Promise<{ success: boolean; message: string }> { - const { appsApi } = await this.getK8sClient(app.clusterId); const namespace = `user-${app.userId.split('-')[0]}`; + const releaseName = app.name; try { - // Read the target ReplicaSet's pod template - const rsList = await appsApi.listNamespacedReplicaSet( - namespace, - undefined, - undefined, - undefined, - undefined, - `app=${app.name}`, - ); - - const targetRs = rsList.body.items.find((rs) => { - const rev = parseInt(rs.metadata?.annotations?.['deployment.kubernetes.io/revision'] || '0', 10); - const owners = rs.metadata?.ownerReferences || []; - return rev === targetRevision && owners.some((o) => o.kind === 'Deployment' && o.name === app.name); - }); - - if (!targetRs) { - return { success: false, message: `Revision ${targetRevision} not found` }; - } - - // Get the pod template from the target ReplicaSet - const targetTemplate = targetRs.spec?.template; - if (!targetTemplate) { - return { success: false, message: 'Could not read pod template from target revision' }; - } - - // Patch the deployment with the target revision's pod template - // This triggers a new rollout that uses the same image/config as the target revision - const patch = { - metadata: { - annotations: { - 'kubernetes.io/change-cause': `Rollback to revision ${targetRevision} at ${new Date().toISOString()}`, - }, - }, - spec: { - template: targetTemplate, - }, - }; - - await appsApi.patchNamespacedDeployment( - app.name, - namespace, - patch, - undefined, - undefined, - undefined, - undefined, - undefined, - { headers: { 'Content-Type': 'application/strategic-merge-patch+json' } }, - ); - - const image = targetTemplate.spec?.containers?.[0]?.image || 'unknown'; - this.logger.log(`Rolled back ${app.name} to revision ${targetRevision} (image: ${image})`); - return { success: true, message: `Rolled back to revision ${targetRevision} (image: ${image})` }; + const kubeconfig = await this.getKubeconfig(app.clusterId); + await this.helmService.rollback(releaseName, targetRevision, namespace, kubeconfig); + this.logger.log(`Rolled back ${releaseName} to Helm revision ${targetRevision}`); + return { success: true, message: `Rolled back to Helm revision ${targetRevision}` }; } catch (e: any) { - this.logger.error(`Failed to rollback ${app.name}: ${e.body?.message || e.message}`); - return { success: false, message: e.body?.message || e.message }; + this.logger.error(`Helm rollback failed for ${releaseName}: ${e.message}`); + return { success: false, message: e.message }; } } private generatePassword(length = 24): string { - const chars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!@#$%'; + const chars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'; let password = ''; for (let i = 0; i < length; i++) { password += chars.charAt(Math.floor(Math.random() * chars.length));