keyhan
f481a57d8f
- Job containers now sleep 120s after completing dump/archive - This allows exec to retrieve files before container exits - Wait for DUMP_DONE/ARCHIVE_DONE marker before attempting exec - Cleanup job immediately after retrieval - Fix Helm registry-pull-secret ownership conflict with lookup
1725 lines
60 KiB
TypeScript
1725 lines
60 KiB
TypeScript
import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
|
|
import { ConfigService } from '@nestjs/config';
|
|
import * as k8s from '@kubernetes/client-node';
|
|
import * as fs from 'fs';
|
|
import * as path from 'path';
|
|
import { execFile } from 'child_process';
|
|
import { promisify } from 'util';
|
|
import { ClustersService } from '../clusters/clusters.service';
|
|
import { Application } from '../applications/entities/application.entity';
|
|
import { AppRuntime, DatabaseType } from '../common/enums';
|
|
import { HelmService } from './helm.service';
|
|
|
|
const execFileAsync = promisify(execFile);
|
|
|
|
interface ManifestContext {
|
|
appName: string;
|
|
namespace: string;
|
|
image: string;
|
|
port: number;
|
|
replicas: number;
|
|
cpuRequest: string;
|
|
cpuLimit: string;
|
|
memoryRequest: string;
|
|
memoryLimit: string;
|
|
envVars: Record<string, string>;
|
|
runtime: string;
|
|
databaseType: string;
|
|
domain: string;
|
|
subdomain: string;
|
|
dbUsername: string;
|
|
dbPassword: string;
|
|
dbVersion: string;
|
|
dbStorageSize: string;
|
|
}
|
|
|
|
@Injectable()
|
|
export class KubernetesService implements OnModuleInit {
|
|
private readonly logger = new Logger(KubernetesService.name);
|
|
|
|
constructor(
|
|
private configService: ConfigService,
|
|
private clustersService: ClustersService,
|
|
private helmService: HelmService,
|
|
) {}
|
|
|
|
onModuleInit() {
|
|
// Helm chart is used for deployments — no local template loading needed
|
|
}
|
|
|
|
private async getK8sClient(clusterId?: string): Promise<{
|
|
coreApi: k8s.CoreV1Api;
|
|
appsApi: k8s.AppsV1Api;
|
|
networkingApi: k8s.NetworkingV1Api;
|
|
kc: k8s.KubeConfig;
|
|
}> {
|
|
const cluster = clusterId
|
|
? await this.clustersService.findOne(clusterId)
|
|
: await this.clustersService.getDefault();
|
|
|
|
const kc = new k8s.KubeConfig();
|
|
kc.loadFromString(cluster.kubeconfig);
|
|
|
|
return {
|
|
coreApi: kc.makeApiClient(k8s.CoreV1Api),
|
|
appsApi: kc.makeApiClient(k8s.AppsV1Api),
|
|
networkingApi: kc.makeApiClient(k8s.NetworkingV1Api),
|
|
kc,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Get the raw kubeconfig string for a cluster.
|
|
*/
|
|
private async getKubeconfig(clusterId?: string): Promise<string> {
|
|
const cluster = clusterId
|
|
? await this.clustersService.findOne(clusterId)
|
|
: await this.clustersService.getDefault();
|
|
return cluster.kubeconfig;
|
|
}
|
|
|
|
/**
|
|
* Build Helm values object from an Application entity and image URI.
|
|
*/
|
|
private buildHelmValues(app: Application, imageUri: string): Record<string, any> {
|
|
const domain = this.configService.get('platform.domain');
|
|
const pullRegistryUrl = this.configService.get<string>('registry.pullUrl') || 'localhost:30500';
|
|
const isWordPress = app.runtime === AppRuntime.WORDPRESS;
|
|
const hasDb = app.databaseType !== DatabaseType.NONE;
|
|
const isPostgres = app.databaseType === DatabaseType.POSTGRESQL;
|
|
|
|
const values: Record<string, any> = {
|
|
app: {
|
|
name: app.name,
|
|
namespace: `user-${app.userId.split('-')[0]}`,
|
|
runtime: app.runtime,
|
|
image: imageUri,
|
|
port: app.port,
|
|
replicas: app.replicas,
|
|
},
|
|
resources: {
|
|
cpuRequest: app.cpuRequest,
|
|
cpuLimit: app.cpuLimit,
|
|
memoryRequest: app.memoryRequest,
|
|
memoryLimit: app.memoryLimit,
|
|
},
|
|
envVars: app.envVars || {},
|
|
ingress: {
|
|
enabled: true,
|
|
subdomain: app.subdomain || app.name,
|
|
domain: domain,
|
|
clusterIssuer: 'letsencrypt-prod',
|
|
},
|
|
registry: {
|
|
url: pullRegistryUrl,
|
|
},
|
|
database: {
|
|
enabled: hasDb,
|
|
type: app.databaseType,
|
|
version: app.dbVersion || (isPostgres ? '16' : '8.0'),
|
|
username: app.dbUsername || 'appuser',
|
|
password: app.dbPassword || this.generatePassword(),
|
|
storageSize: app.dbStorageSize || '1Gi',
|
|
resources: {
|
|
cpuRequest: '100m',
|
|
cpuLimit: '500m',
|
|
memoryRequest: '256Mi',
|
|
memoryLimit: '512Mi',
|
|
},
|
|
},
|
|
wordpress: {
|
|
enabled: isWordPress,
|
|
wpContentStorageSize: '2Gi',
|
|
},
|
|
changeCause: `Deploy ${imageUri} at ${new Date().toISOString()}`,
|
|
};
|
|
|
|
return values;
|
|
}
|
|
|
|
async deployApplication(app: Application, imageUri: string): Promise<Record<string, any>> {
|
|
// Try Helm first, fall back to direct K8s API if Helm is unavailable
|
|
try {
|
|
return await this.deployViaHelm(app, imageUri);
|
|
} catch (helmError: any) {
|
|
this.logger.warn(
|
|
`Helm deploy failed for ${app.name}, falling back to direct K8s API: ${helmError.message}`,
|
|
);
|
|
return await this.deployViaK8sApi(app, imageUri);
|
|
}
|
|
}
|
|
|
|
// ── Helm-based deployment ─────────────────────────────────────────
|
|
|
|
private async deployViaHelm(app: Application, imageUri: string): Promise<Record<string, any>> {
|
|
const kubeconfig = await this.getKubeconfig(app.clusterId);
|
|
const values = this.buildHelmValues(app, imageUri);
|
|
const namespace = values.app.namespace;
|
|
const releaseName = app.name;
|
|
|
|
const result = await this.helmService.installOrUpgrade(
|
|
releaseName,
|
|
namespace,
|
|
values,
|
|
kubeconfig,
|
|
);
|
|
|
|
this.logger.log(`Successfully deployed ${app.name} to namespace ${namespace} via Helm`);
|
|
return { helm: { release: releaseName, namespace, stdout: result.stdout }, values };
|
|
}
|
|
|
|
// ── Direct K8s API deployment (fallback) ──────────────────────────
|
|
|
|
private async deployViaK8sApi(app: Application, imageUri: string): Promise<Record<string, any>> {
|
|
const { coreApi, appsApi, networkingApi } = await this.getK8sClient(app.clusterId);
|
|
const domain = this.configService.get('platform.domain');
|
|
|
|
const context: ManifestContext = {
|
|
appName: app.name,
|
|
namespace: `user-${app.userId.split('-')[0]}`,
|
|
image: imageUri,
|
|
port: app.port,
|
|
replicas: app.replicas,
|
|
cpuRequest: app.cpuRequest,
|
|
cpuLimit: app.cpuLimit,
|
|
memoryRequest: app.memoryRequest,
|
|
memoryLimit: app.memoryLimit,
|
|
envVars: app.envVars || {},
|
|
runtime: app.runtime,
|
|
databaseType: app.databaseType,
|
|
domain: domain,
|
|
subdomain: app.subdomain || app.name,
|
|
dbUsername: app.dbUsername || 'appuser',
|
|
dbPassword: app.dbPassword || this.generatePassword(),
|
|
dbVersion: app.dbVersion || '',
|
|
dbStorageSize: app.dbStorageSize || '1Gi',
|
|
};
|
|
|
|
const manifests: Record<string, any> = {};
|
|
|
|
try {
|
|
// 1. Ensure namespace exists
|
|
await this.ensureNamespace(coreApi, context.namespace);
|
|
|
|
// 2. Create/Update secrets for env vars
|
|
if (Object.keys(context.envVars).length > 0) {
|
|
manifests.secret = await this.applySecret(coreApi, context);
|
|
}
|
|
|
|
// 3. Deploy database if needed
|
|
if (context.databaseType !== DatabaseType.NONE) {
|
|
manifests.database = await this.deployDatabase(coreApi, appsApi, context);
|
|
}
|
|
|
|
// 3.5 Create wp-content PVC for WordPress
|
|
if (context.runtime === AppRuntime.WORDPRESS) {
|
|
manifests.wpContentPvc = await this.applyWordPressPvc(coreApi, context);
|
|
}
|
|
|
|
// 4. Create Deployment
|
|
manifests.deployment = await this.applyDeployment(appsApi, context);
|
|
|
|
// 5. Create Service
|
|
manifests.service = await this.applyService(coreApi, context);
|
|
|
|
// 6. Create Ingress
|
|
manifests.ingress = await this.applyIngress(networkingApi, context);
|
|
|
|
this.logger.log(`Successfully deployed ${app.name} to namespace ${context.namespace} via K8s API`);
|
|
} catch (error: any) {
|
|
this.logger.error(`Failed to deploy ${app.name} via K8s API:`, error.body || error.message);
|
|
throw error;
|
|
}
|
|
|
|
return manifests;
|
|
}
|
|
|
|
// ── Private K8s resource methods ──────────────────────────────────
|
|
|
|
private async ensureNamespace(coreApi: k8s.CoreV1Api, namespace: string): Promise<void> {
|
|
try {
|
|
await coreApi.readNamespace(namespace);
|
|
} catch {
|
|
await coreApi.createNamespace({
|
|
metadata: { name: namespace },
|
|
});
|
|
this.logger.log(`Created namespace: ${namespace}`);
|
|
}
|
|
}
|
|
|
|
private async applySecret(coreApi: k8s.CoreV1Api, ctx: ManifestContext): Promise<any> {
|
|
const secretData: Record<string, string> = {};
|
|
for (const [key, value] of Object.entries(ctx.envVars)) {
|
|
secretData[key] = Buffer.from(String(value)).toString('base64');
|
|
}
|
|
|
|
const secret = {
|
|
apiVersion: 'v1',
|
|
kind: 'Secret',
|
|
metadata: {
|
|
name: `${ctx.appName}-env`,
|
|
namespace: ctx.namespace,
|
|
},
|
|
data: secretData,
|
|
};
|
|
|
|
try {
|
|
await coreApi.replaceNamespacedSecret(`${ctx.appName}-env`, ctx.namespace, secret);
|
|
} catch {
|
|
await coreApi.createNamespacedSecret(ctx.namespace, secret);
|
|
}
|
|
return secret;
|
|
}
|
|
|
|
private async applyDeployment(appsApi: k8s.AppsV1Api, ctx: ManifestContext): Promise<any> {
|
|
const envFrom: any[] = [];
|
|
if (Object.keys(ctx.envVars).length > 0) {
|
|
envFrom.push({ secretRef: { name: `${ctx.appName}-env` } });
|
|
}
|
|
|
|
// Database connection env vars
|
|
const extraEnv: any[] = [];
|
|
if (ctx.databaseType === DatabaseType.POSTGRESQL) {
|
|
extraEnv.push(
|
|
{ name: 'DB_HOST', value: `${ctx.appName}-db` },
|
|
{ name: 'DB_PORT', value: '5432' },
|
|
{ name: 'DB_NAME', value: ctx.appName.replace(/-/g, '_') },
|
|
{ name: 'DB_USER', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'username' } } },
|
|
{ name: 'DB_PASSWORD', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'password' } } },
|
|
{ name: 'DATABASE_URL', value: `postgresql://$(DB_USER):$(DB_PASSWORD)@${ctx.appName}-db:5432/${ctx.appName.replace(/-/g, '_')}` },
|
|
);
|
|
} else if (ctx.databaseType === DatabaseType.MYSQL) {
|
|
extraEnv.push(
|
|
{ name: 'DB_HOST', value: `${ctx.appName}-db` },
|
|
{ name: 'DB_PORT', value: '3306' },
|
|
{ name: 'DB_NAME', value: ctx.appName.replace(/-/g, '_') },
|
|
{ name: 'DB_USER', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'username' } } },
|
|
{ name: 'DB_PASSWORD', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'password' } } },
|
|
{ name: 'DATABASE_URL', value: `mysql://$(DB_USER):$(DB_PASSWORD)@${ctx.appName}-db:3306/${ctx.appName.replace(/-/g, '_')}` },
|
|
);
|
|
}
|
|
|
|
// WordPress-specific env vars
|
|
if (ctx.runtime === AppRuntime.WORDPRESS && ctx.databaseType !== DatabaseType.NONE) {
|
|
const dbPort = ctx.databaseType === DatabaseType.POSTGRESQL ? '5432' : '3306';
|
|
extraEnv.push(
|
|
{ name: 'WORDPRESS_DB_HOST', value: `${ctx.appName}-db:${dbPort}` },
|
|
{ name: 'WORDPRESS_DB_NAME', value: ctx.appName.replace(/-/g, '_') },
|
|
{ name: 'WORDPRESS_DB_USER', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'username' } } },
|
|
{ name: 'WORDPRESS_DB_PASSWORD', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'password' } } },
|
|
{ name: 'WORDPRESS_TABLE_PREFIX', value: 'wp_' },
|
|
);
|
|
}
|
|
|
|
const deployment: k8s.V1Deployment = {
|
|
apiVersion: 'apps/v1',
|
|
kind: 'Deployment',
|
|
metadata: {
|
|
name: ctx.appName,
|
|
namespace: ctx.namespace,
|
|
labels: { app: ctx.appName, runtime: ctx.runtime },
|
|
annotations: {
|
|
'kubernetes.io/change-cause': `Deploy ${ctx.image} at ${new Date().toISOString()}`,
|
|
},
|
|
},
|
|
spec: {
|
|
revisionHistoryLimit: 10,
|
|
replicas: ctx.replicas,
|
|
selector: { matchLabels: { app: ctx.appName } },
|
|
template: {
|
|
metadata: { labels: { app: ctx.appName, runtime: ctx.runtime } },
|
|
spec: {
|
|
containers: [
|
|
{
|
|
name: ctx.appName,
|
|
image: ctx.image,
|
|
imagePullPolicy: 'Always',
|
|
ports: [{ containerPort: ctx.port }],
|
|
envFrom,
|
|
env: extraEnv,
|
|
resources: {
|
|
requests: { cpu: ctx.cpuRequest, memory: ctx.memoryRequest },
|
|
limits: { cpu: ctx.cpuLimit, memory: ctx.memoryLimit },
|
|
},
|
|
readinessProbe: {
|
|
tcpSocket: { port: ctx.port as any },
|
|
initialDelaySeconds: 5,
|
|
periodSeconds: 5,
|
|
},
|
|
livenessProbe: {
|
|
tcpSocket: { port: ctx.port as any },
|
|
initialDelaySeconds: 15,
|
|
periodSeconds: 10,
|
|
failureThreshold: 5,
|
|
},
|
|
...(ctx.runtime === AppRuntime.WORDPRESS
|
|
? {
|
|
volumeMounts: [
|
|
{ name: 'wp-content', mountPath: '/var/www/html/wp-content' },
|
|
],
|
|
}
|
|
: {}),
|
|
},
|
|
],
|
|
...(ctx.runtime === AppRuntime.WORDPRESS
|
|
? {
|
|
volumes: [
|
|
{
|
|
name: 'wp-content',
|
|
persistentVolumeClaim: { claimName: `${ctx.appName}-wp-content` },
|
|
},
|
|
],
|
|
}
|
|
: {}),
|
|
},
|
|
},
|
|
},
|
|
};
|
|
|
|
try {
|
|
await appsApi.replaceNamespacedDeployment(ctx.appName, ctx.namespace, deployment);
|
|
} catch {
|
|
await appsApi.createNamespacedDeployment(ctx.namespace, deployment);
|
|
}
|
|
return deployment;
|
|
}
|
|
|
|
private async applyWordPressPvc(coreApi: k8s.CoreV1Api, ctx: ManifestContext): Promise<any> {
|
|
const pvcName = `${ctx.appName}-wp-content`;
|
|
const pvc = {
|
|
apiVersion: 'v1',
|
|
kind: 'PersistentVolumeClaim',
|
|
metadata: { name: pvcName, namespace: ctx.namespace, labels: { app: ctx.appName } },
|
|
spec: {
|
|
accessModes: ['ReadWriteOnce'],
|
|
resources: { requests: { storage: '2Gi' } },
|
|
},
|
|
};
|
|
|
|
try {
|
|
await coreApi.readNamespacedPersistentVolumeClaim(pvcName, ctx.namespace);
|
|
this.logger.log(`PVC ${pvcName} already exists, skipping`);
|
|
} catch {
|
|
await coreApi.createNamespacedPersistentVolumeClaim(ctx.namespace, pvc);
|
|
this.logger.log(`Created WordPress PVC: ${pvcName}`);
|
|
}
|
|
return pvc;
|
|
}
|
|
|
|
private async applyService(coreApi: k8s.CoreV1Api, ctx: ManifestContext): Promise<any> {
|
|
const service: k8s.V1Service = {
|
|
apiVersion: 'v1',
|
|
kind: 'Service',
|
|
metadata: { name: ctx.appName, namespace: ctx.namespace },
|
|
spec: {
|
|
selector: { app: ctx.appName },
|
|
ports: [{ port: 80, targetPort: ctx.port as any, protocol: 'TCP' }],
|
|
type: 'ClusterIP',
|
|
},
|
|
};
|
|
|
|
try {
|
|
await coreApi.replaceNamespacedService(ctx.appName, ctx.namespace, service);
|
|
} catch {
|
|
await coreApi.createNamespacedService(ctx.namespace, service);
|
|
}
|
|
return service;
|
|
}
|
|
|
|
private async applyIngress(networkingApi: k8s.NetworkingV1Api, ctx: ManifestContext): Promise<any> {
|
|
const host = `${ctx.subdomain}.${ctx.domain}`;
|
|
const ingress: k8s.V1Ingress = {
|
|
apiVersion: 'networking.k8s.io/v1',
|
|
kind: 'Ingress',
|
|
metadata: {
|
|
name: ctx.appName,
|
|
namespace: ctx.namespace,
|
|
annotations: {
|
|
'cert-manager.io/cluster-issuer': 'letsencrypt-prod',
|
|
},
|
|
},
|
|
spec: {
|
|
ingressClassName: 'nginx',
|
|
rules: [
|
|
{
|
|
host,
|
|
http: {
|
|
paths: [
|
|
{
|
|
path: '/',
|
|
pathType: 'Prefix',
|
|
backend: { service: { name: ctx.appName, port: { number: 80 } } },
|
|
},
|
|
],
|
|
},
|
|
},
|
|
],
|
|
tls: [{ hosts: [host], secretName: `${ctx.appName}-tls` }],
|
|
},
|
|
};
|
|
|
|
try {
|
|
await networkingApi.replaceNamespacedIngress(ctx.appName, ctx.namespace, ingress);
|
|
} catch {
|
|
await networkingApi.createNamespacedIngress(ctx.namespace, ingress);
|
|
}
|
|
return ingress;
|
|
}
|
|
|
|
private async deployDatabase(
|
|
coreApi: k8s.CoreV1Api,
|
|
appsApi: k8s.AppsV1Api,
|
|
ctx: ManifestContext,
|
|
): Promise<any> {
|
|
const dbName = `${ctx.appName}-db`;
|
|
|
|
// Create DB secret
|
|
await this.createDbSecret(coreApi, ctx.namespace, ctx.appName, ctx.dbPassword, ctx.dbUsername);
|
|
|
|
// Create PVC for DB
|
|
await this.createPVC(coreApi, ctx.namespace, dbName, ctx.dbStorageSize);
|
|
|
|
// Deploy database
|
|
const isPostgres = ctx.databaseType === DatabaseType.POSTGRESQL;
|
|
const defaultDbVersion = isPostgres ? '16' : '8.0';
|
|
const dbVer = ctx.dbVersion || defaultDbVersion;
|
|
const image = isPostgres ? `postgres:${dbVer}-alpine` : `mysql:${dbVer}`;
|
|
const port = isPostgres ? 5432 : 3306;
|
|
const dataPath = isPostgres ? '/var/lib/postgresql/data' : '/var/lib/mysql';
|
|
|
|
const envVars = isPostgres
|
|
? [
|
|
{ name: 'PGDATA', value: '/var/lib/postgresql/data/pgdata' },
|
|
{ name: 'POSTGRES_DB', value: ctx.appName.replace(/-/g, '_') },
|
|
{ name: 'POSTGRES_USER', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'username' } } },
|
|
{ name: 'POSTGRES_PASSWORD', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'password' } } },
|
|
]
|
|
: [
|
|
{ name: 'MYSQL_DATABASE', value: ctx.appName.replace(/-/g, '_') },
|
|
{ name: 'MYSQL_USER', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'username' } } },
|
|
{ name: 'MYSQL_PASSWORD', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'password' } } },
|
|
{ name: 'MYSQL_ROOT_PASSWORD', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'password' } } },
|
|
];
|
|
|
|
const dbDeployment: k8s.V1Deployment = {
|
|
apiVersion: 'apps/v1',
|
|
kind: 'Deployment',
|
|
metadata: { name: dbName, namespace: ctx.namespace, labels: { app: dbName } },
|
|
spec: {
|
|
replicas: 1,
|
|
selector: { matchLabels: { app: dbName } },
|
|
template: {
|
|
metadata: { labels: { app: dbName } },
|
|
spec: {
|
|
containers: [
|
|
{
|
|
name: dbName,
|
|
image,
|
|
ports: [{ containerPort: port }],
|
|
env: envVars,
|
|
volumeMounts: [{ name: 'db-storage', mountPath: dataPath }],
|
|
resources: {
|
|
requests: { cpu: '100m', memory: '256Mi' },
|
|
limits: { cpu: '500m', memory: '512Mi' },
|
|
},
|
|
readinessProbe: isPostgres
|
|
? { exec: { command: ['pg_isready', '-U', ctx.dbUsername] }, initialDelaySeconds: 10, periodSeconds: 5, failureThreshold: 6 }
|
|
: { exec: { command: ['mysqladmin', 'ping', '-h', '127.0.0.1'] }, initialDelaySeconds: 10, periodSeconds: 5, failureThreshold: 6 },
|
|
livenessProbe: isPostgres
|
|
? { exec: { command: ['pg_isready', '-U', ctx.dbUsername] }, initialDelaySeconds: 30, periodSeconds: 10, failureThreshold: 5 }
|
|
: { exec: { command: ['mysqladmin', 'ping', '-h', '127.0.0.1'] }, initialDelaySeconds: 30, periodSeconds: 10, failureThreshold: 5 },
|
|
},
|
|
],
|
|
volumes: [
|
|
{ name: 'db-storage', persistentVolumeClaim: { claimName: dbName } },
|
|
],
|
|
},
|
|
},
|
|
},
|
|
};
|
|
|
|
try {
|
|
await appsApi.replaceNamespacedDeployment(dbName, ctx.namespace, dbDeployment);
|
|
} catch {
|
|
await appsApi.createNamespacedDeployment(ctx.namespace, dbDeployment);
|
|
}
|
|
|
|
// Create DB Service
|
|
const dbService: k8s.V1Service = {
|
|
apiVersion: 'v1',
|
|
kind: 'Service',
|
|
metadata: { name: dbName, namespace: ctx.namespace },
|
|
spec: {
|
|
selector: { app: dbName },
|
|
ports: [{ port, targetPort: port as any, protocol: 'TCP' }],
|
|
type: 'ClusterIP',
|
|
},
|
|
};
|
|
|
|
try {
|
|
await coreApi.replaceNamespacedService(dbName, ctx.namespace, dbService);
|
|
} catch {
|
|
await coreApi.createNamespacedService(ctx.namespace, dbService);
|
|
}
|
|
|
|
return { deployment: dbDeployment, service: dbService };
|
|
}
|
|
|
|
private async createDbSecret(
|
|
coreApi: k8s.CoreV1Api,
|
|
namespace: string,
|
|
appName: string,
|
|
password: string,
|
|
username: string = 'appuser',
|
|
): Promise<void> {
|
|
const secret = {
|
|
apiVersion: 'v1',
|
|
kind: 'Secret',
|
|
metadata: { name: `${appName}-db-secret`, namespace },
|
|
data: {
|
|
username: Buffer.from(username).toString('base64'),
|
|
password: Buffer.from(password).toString('base64'),
|
|
},
|
|
};
|
|
|
|
try {
|
|
await coreApi.replaceNamespacedSecret(`${appName}-db-secret`, namespace, secret);
|
|
} catch {
|
|
await coreApi.createNamespacedSecret(namespace, secret);
|
|
}
|
|
}
|
|
|
|
private async createPVC(
|
|
coreApi: k8s.CoreV1Api,
|
|
namespace: string,
|
|
name: string,
|
|
size: string,
|
|
): Promise<void> {
|
|
const pvc: k8s.V1PersistentVolumeClaim = {
|
|
apiVersion: 'v1',
|
|
kind: 'PersistentVolumeClaim',
|
|
metadata: { name, namespace },
|
|
spec: {
|
|
accessModes: ['ReadWriteOnce'],
|
|
resources: { requests: { storage: size } },
|
|
},
|
|
};
|
|
|
|
try {
|
|
await coreApi.readNamespacedPersistentVolumeClaim(name, namespace);
|
|
// PVC exists, don't recreate
|
|
} catch {
|
|
await coreApi.createNamespacedPersistentVolumeClaim(namespace, pvc);
|
|
}
|
|
}
|
|
|
|
async getPodLogs(app: Application): Promise<string> {
|
|
const { coreApi } = await this.getK8sClient(app.clusterId);
|
|
const namespace = `user-${app.userId.split('-')[0]}`;
|
|
|
|
const pods = await coreApi.listNamespacedPod(
|
|
namespace,
|
|
undefined,
|
|
undefined,
|
|
undefined,
|
|
undefined,
|
|
`app=${app.name}`,
|
|
);
|
|
|
|
if (pods.body.items.length === 0) {
|
|
return 'No pods found for this application.';
|
|
}
|
|
|
|
const podName = pods.body.items[0].metadata?.name;
|
|
if (!podName) return 'Pod name not found.';
|
|
|
|
const logResponse = await coreApi.readNamespacedPodLog(
|
|
podName,
|
|
namespace,
|
|
undefined,
|
|
undefined,
|
|
undefined,
|
|
undefined,
|
|
undefined,
|
|
undefined,
|
|
undefined,
|
|
200,
|
|
);
|
|
|
|
return logResponse.body;
|
|
}
|
|
|
|
async scaleDeployment(app: Application, replicas: number): Promise<void> {
|
|
const { appsApi } = await this.getK8sClient(app.clusterId);
|
|
const namespace = `user-${app.userId.split('-')[0]}`;
|
|
|
|
await appsApi.patchNamespacedDeployment(
|
|
app.name,
|
|
namespace,
|
|
{ spec: { replicas } },
|
|
undefined,
|
|
undefined,
|
|
undefined,
|
|
undefined,
|
|
undefined,
|
|
{ headers: { 'Content-Type': 'application/merge-patch+json' } },
|
|
);
|
|
}
|
|
|
|
async restartDeployment(app: Application): Promise<void> {
|
|
const { appsApi } = await this.getK8sClient(app.clusterId);
|
|
const namespace = `user-${app.userId.split('-')[0]}`;
|
|
|
|
await appsApi.patchNamespacedDeployment(
|
|
app.name,
|
|
namespace,
|
|
{
|
|
spec: {
|
|
template: {
|
|
metadata: {
|
|
annotations: {
|
|
'kubectl.kubernetes.io/restartedAt': new Date().toISOString(),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
undefined,
|
|
undefined,
|
|
undefined,
|
|
undefined,
|
|
undefined,
|
|
{ headers: { 'Content-Type': 'application/merge-patch+json' } },
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Get real-time resource usage (CPU/Memory) for an app's pods via metrics-server.
|
|
* Also returns the configured requests/limits and pod status.
|
|
*/
|
|
async getResourceUsage(app: Application): Promise<any> {
|
|
const { coreApi, appsApi, kc } = await this.getK8sClient(app.clusterId);
|
|
const namespace = `user-${app.userId.split('-')[0]}`;
|
|
|
|
// Get deployment info for configured resources
|
|
let deployment: k8s.V1Deployment | null = null;
|
|
try {
|
|
const depResponse = await appsApi.readNamespacedDeployment(app.name, namespace);
|
|
deployment = depResponse.body;
|
|
} catch {
|
|
// Deployment may not exist yet
|
|
}
|
|
|
|
// Get pods
|
|
const podsResponse = await coreApi.listNamespacedPod(
|
|
namespace,
|
|
undefined,
|
|
undefined,
|
|
undefined,
|
|
undefined,
|
|
`app=${app.name}`,
|
|
);
|
|
|
|
const pods = podsResponse.body.items.map((pod) => ({
|
|
name: pod.metadata?.name,
|
|
status: pod.status?.phase,
|
|
ready: pod.status?.conditions?.find((c) => c.type === 'Ready')?.status === 'True',
|
|
restarts: pod.status?.containerStatuses?.[0]?.restartCount || 0,
|
|
startedAt: pod.status?.startTime,
|
|
}));
|
|
|
|
// Try to get metrics from metrics-server via custom API
|
|
let podMetrics: any[] = [];
|
|
try {
|
|
const metricsClient = new k8s.CustomObjectsApi(kc.getCurrentCluster()?.server);
|
|
// Use the kc to make a raw request to metrics API
|
|
const opts: any = {};
|
|
await kc.applyToRequest(opts);
|
|
|
|
const metricsUrl = `${kc.getCurrentCluster()?.server}/apis/metrics.k8s.io/v1beta1/namespaces/${namespace}/pods`;
|
|
|
|
const https = require('https');
|
|
const http = require('http');
|
|
const url = new URL(metricsUrl);
|
|
|
|
podMetrics = await new Promise((resolve) => {
|
|
const client = url.protocol === 'https:' ? https : http;
|
|
const reqOpts: any = {
|
|
hostname: url.hostname,
|
|
port: url.port,
|
|
path: url.pathname + `?labelSelector=app%3D${app.name}`,
|
|
method: 'GET',
|
|
headers: opts.headers || {},
|
|
rejectUnauthorized: false,
|
|
};
|
|
|
|
// Apply TLS from kubeconfig
|
|
if (opts.ca) reqOpts.ca = opts.ca;
|
|
if (opts.cert) reqOpts.cert = opts.cert;
|
|
if (opts.key) reqOpts.key = opts.key;
|
|
|
|
const req = client.request(reqOpts, (res: any) => {
|
|
let data = '';
|
|
res.on('data', (chunk: string) => (data += chunk));
|
|
res.on('end', () => {
|
|
try {
|
|
const parsed = JSON.parse(data);
|
|
const items = (parsed.items || []).map((item: any) => ({
|
|
name: item.metadata?.name,
|
|
cpu: item.containers?.[0]?.usage?.cpu || '0',
|
|
memory: item.containers?.[0]?.usage?.memory || '0',
|
|
}));
|
|
resolve(items);
|
|
} catch {
|
|
resolve([]);
|
|
}
|
|
});
|
|
});
|
|
req.on('error', () => resolve([]));
|
|
req.end();
|
|
});
|
|
} catch {
|
|
this.logger.warn(`Metrics not available for ${app.name}`);
|
|
}
|
|
|
|
// Get configured resources from deployment
|
|
const container = deployment?.spec?.template?.spec?.containers?.[0];
|
|
const configured = {
|
|
cpuRequest: container?.resources?.requests?.cpu || app.cpuRequest,
|
|
cpuLimit: container?.resources?.limits?.cpu || app.cpuLimit,
|
|
memoryRequest: container?.resources?.requests?.memory || app.memoryRequest,
|
|
memoryLimit: container?.resources?.limits?.memory || app.memoryLimit,
|
|
replicas: deployment?.spec?.replicas ?? app.replicas,
|
|
readyReplicas: deployment?.status?.readyReplicas || 0,
|
|
availableReplicas: deployment?.status?.availableReplicas || 0,
|
|
};
|
|
|
|
return {
|
|
configured,
|
|
pods,
|
|
metrics: podMetrics,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Update resource limits/requests and replicas on a live K8s deployment.
|
|
*/
|
|
async updateResources(
|
|
app: Application,
|
|
resources: { cpuRequest?: string; cpuLimit?: string; memoryRequest?: string; memoryLimit?: string; replicas?: number },
|
|
): Promise<void> {
|
|
const { appsApi } = await this.getK8sClient(app.clusterId);
|
|
const namespace = `user-${app.userId.split('-')[0]}`;
|
|
|
|
const patch: any = { spec: {} };
|
|
|
|
if (resources.replicas !== undefined) {
|
|
patch.spec.replicas = resources.replicas;
|
|
}
|
|
|
|
if (resources.cpuRequest || resources.cpuLimit || resources.memoryRequest || resources.memoryLimit) {
|
|
patch.spec.template = {
|
|
spec: {
|
|
containers: [
|
|
{
|
|
name: app.name,
|
|
resources: {
|
|
requests: {
|
|
...(resources.cpuRequest && { cpu: resources.cpuRequest }),
|
|
...(resources.memoryRequest && { memory: resources.memoryRequest }),
|
|
},
|
|
limits: {
|
|
...(resources.cpuLimit && { cpu: resources.cpuLimit }),
|
|
...(resources.memoryLimit && { memory: resources.memoryLimit }),
|
|
},
|
|
},
|
|
},
|
|
],
|
|
},
|
|
};
|
|
}
|
|
|
|
await appsApi.patchNamespacedDeployment(
|
|
app.name,
|
|
namespace,
|
|
patch,
|
|
undefined,
|
|
undefined,
|
|
undefined,
|
|
undefined,
|
|
undefined,
|
|
{ headers: { 'Content-Type': 'application/strategic-merge-patch+json' } },
|
|
);
|
|
|
|
this.logger.log(`Updated resources for ${app.name}: ${JSON.stringify(resources)}`);
|
|
}
|
|
|
|
/**
|
|
* Get preview info for a deployed application.
|
|
* Patches the service to NodePort if needed, and returns the access URL.
|
|
*/
|
|
async getPreviewInfo(app: Application): Promise<{
|
|
url: string;
|
|
nodePort: number;
|
|
host: string;
|
|
ingressUrl?: string;
|
|
}> {
|
|
const { coreApi, networkingApi, kc } = await this.getK8sClient(app.clusterId);
|
|
const namespace = `user-${app.userId.split('-')[0]}`;
|
|
const domain = this.configService.get('platform.domain');
|
|
const clusterServer = kc.getCurrentCluster()?.server || '';
|
|
// Extract host IP from cluster API server URL (e.g., https://217.197.107.252:6443 → 217.197.107.252)
|
|
let hostIp = '127.0.0.1';
|
|
try {
|
|
const serverUrl = new URL(clusterServer);
|
|
hostIp = serverUrl.hostname;
|
|
} catch {}
|
|
|
|
// Read current service
|
|
let nodePort = 0;
|
|
try {
|
|
const svcResponse = await coreApi.readNamespacedService(app.name, namespace);
|
|
const svc = svcResponse.body;
|
|
|
|
if (svc.spec?.type === 'NodePort') {
|
|
// Already NodePort, read the assigned port
|
|
nodePort = svc.spec.ports?.[0]?.nodePort || 0;
|
|
} else {
|
|
// Patch ClusterIP → NodePort so we can access from outside
|
|
const patchBody = {
|
|
spec: {
|
|
type: 'NodePort',
|
|
ports: [
|
|
{
|
|
port: 80,
|
|
targetPort: app.port,
|
|
protocol: 'TCP',
|
|
},
|
|
],
|
|
},
|
|
};
|
|
|
|
const patchedResponse = await coreApi.patchNamespacedService(
|
|
app.name,
|
|
namespace,
|
|
patchBody,
|
|
undefined,
|
|
undefined,
|
|
undefined,
|
|
undefined,
|
|
undefined,
|
|
{ headers: { 'Content-Type': 'application/strategic-merge-patch+json' } },
|
|
);
|
|
nodePort = patchedResponse.body.spec?.ports?.[0]?.nodePort || 0;
|
|
this.logger.log(`Patched service ${app.name} to NodePort: ${nodePort}`);
|
|
}
|
|
} catch (e: any) {
|
|
this.logger.warn(`Failed to get/patch service for ${app.name}: ${e.message}`);
|
|
throw new Error(`Service not found for "${app.name}". Make sure the app is deployed.`);
|
|
}
|
|
|
|
// Build ingress URL
|
|
const subdomain = app.subdomain || app.name;
|
|
const ingressUrl = `https://${subdomain}.${domain}`;
|
|
|
|
return {
|
|
url: `http://${hostIp}:${nodePort}`,
|
|
nodePort,
|
|
host: hostIp,
|
|
ingressUrl,
|
|
};
|
|
}
|
|
|
|
async deleteApplication(app: Application): Promise<void> {
|
|
const namespace = `user-${app.userId.split('-')[0]}`;
|
|
const { coreApi, appsApi, networkingApi } = await this.getK8sClient(app.clusterId);
|
|
|
|
// Step 1: Try Helm uninstall (handles most resources)
|
|
try {
|
|
const kubeconfig = await this.getKubeconfig(app.clusterId);
|
|
await this.helmService.uninstall(app.name, namespace, kubeconfig);
|
|
this.logger.log(`Helm release ${app.name} uninstalled from ${namespace}`);
|
|
} catch (error: any) {
|
|
this.logger.warn(`Helm uninstall failed for ${app.name}: ${error.message} — proceeding with manual cleanup`);
|
|
// Manual cleanup of core resources if helm wasn't managing them
|
|
try { await appsApi.deleteNamespacedDeployment(app.name, namespace); } catch {}
|
|
try { await coreApi.deleteNamespacedService(app.name, namespace); } catch {}
|
|
try { await networkingApi.deleteNamespacedIngress(app.name, namespace); } catch {}
|
|
}
|
|
|
|
// Step 2: Delete resources with "helm.sh/resource-policy: keep" (PVCs, db-secret)
|
|
// These survive helm uninstall by design, so we must delete them explicitly
|
|
const dbName = `${app.name}-db`;
|
|
const resourcesToDelete = [
|
|
// Database resources
|
|
() => appsApi.deleteNamespacedDeployment(dbName, namespace),
|
|
() => coreApi.deleteNamespacedService(dbName, namespace),
|
|
() => coreApi.deleteNamespacedPersistentVolumeClaim(dbName, namespace),
|
|
() => coreApi.deleteNamespacedSecret(`${app.name}-db-secret`, namespace),
|
|
// WordPress wp-content PVC
|
|
() => coreApi.deleteNamespacedPersistentVolumeClaim(`${app.name}-wp-content`, namespace),
|
|
// App env secret
|
|
() => coreApi.deleteNamespacedSecret(`${app.name}-env`, namespace),
|
|
// Registry pull secret (shared, but labeled per-app — safe to delete)
|
|
() => coreApi.deleteNamespacedSecret('registry-pull-secret', namespace),
|
|
// TLS secret created by cert-manager
|
|
() => coreApi.deleteNamespacedSecret(`${app.name}-tls`, namespace),
|
|
];
|
|
|
|
for (const deleteFn of resourcesToDelete) {
|
|
try { await deleteFn(); } catch {}
|
|
}
|
|
|
|
this.logger.log(`All K8s resources cleaned up for ${app.name} in ${namespace}`);
|
|
}
|
|
|
|
/**
|
|
* Wait for the database pod to become Ready.
|
|
* Polls pod status with label selector `app=<appName>-db`.
|
|
*/
|
|
async waitForDatabaseReady(app: Application, timeoutMs = 120_000): Promise<void> {
|
|
const { coreApi } = await this.getK8sClient(app.clusterId);
|
|
const namespace = `user-${app.userId.split('-')[0]}`;
|
|
const dbLabel = `${app.name}-db`;
|
|
const start = Date.now();
|
|
|
|
this.logger.log(`Waiting for DB pod (app=${dbLabel}) to become Ready in ${namespace}...`);
|
|
|
|
while (Date.now() - start < timeoutMs) {
|
|
try {
|
|
const pods = await coreApi.listNamespacedPod(
|
|
namespace, undefined, undefined, undefined, undefined, `app=${dbLabel}`,
|
|
);
|
|
for (const pod of pods.body.items) {
|
|
const readyCond = (pod.status?.conditions || []).find(c => c.type === 'Ready');
|
|
if (readyCond?.status === 'True') {
|
|
this.logger.log(`DB pod ${pod.metadata?.name} is Ready (${Date.now() - start}ms)`);
|
|
return;
|
|
}
|
|
}
|
|
} catch {
|
|
// Pods may not exist yet
|
|
}
|
|
await new Promise(r => setTimeout(r, 3000));
|
|
}
|
|
this.logger.warn(`DB pod did not become Ready within ${timeoutMs / 1000}s — proceeding anyway`);
|
|
}
|
|
|
|
/**
|
|
* Restore a SQL dump file into the application's database.
|
|
* Uses a PVC + helper pod + kubectl cp to transfer the dump (supports large files),
|
|
* then runs a restore Job that mounts the PVC and imports the dump.
|
|
*/
|
|
async restoreDatabaseDump(app: Application, dumpFilePath: string): Promise<{ success: boolean; logs: string }> {
|
|
const { coreApi, kc } = await this.getK8sClient(app.clusterId);
|
|
const batchApi = kc.makeApiClient(k8s.BatchV1Api);
|
|
const namespace = `user-${app.userId.split('-')[0]}`;
|
|
const dbName = `${app.name}-db`;
|
|
const ts = Date.now();
|
|
const pvcName = `${app.name}-db-dump-${ts}`;
|
|
const helperPodName = `${pvcName}-helper`;
|
|
const jobName = `${app.name}-db-restore-${ts}`;
|
|
const isPostgres = app.databaseType === DatabaseType.POSTGRESQL;
|
|
const dbDatabase = app.name.replace(/-/g, '_');
|
|
|
|
const dumpSize = fs.statSync(dumpFilePath).size;
|
|
const pvcSizeGi = Math.max(1, Math.ceil((dumpSize * 2) / (1024 * 1024 * 1024)));
|
|
|
|
this.logger.log(`Restoring DB dump for ${app.name}: ${(dumpSize / 1024 / 1024).toFixed(1)} MB via PVC`);
|
|
|
|
// ── 1. Create PVC for the dump file ──
|
|
await coreApi.createNamespacedPersistentVolumeClaim(namespace, {
|
|
apiVersion: 'v1',
|
|
kind: 'PersistentVolumeClaim',
|
|
metadata: { name: pvcName, namespace },
|
|
spec: {
|
|
accessModes: ['ReadWriteOnce'],
|
|
resources: { requests: { storage: `${pvcSizeGi}Gi` } },
|
|
},
|
|
});
|
|
|
|
// ── 2. Helper pod to receive the dump via kubectl cp ──
|
|
const helperPod: k8s.V1Pod = {
|
|
apiVersion: 'v1',
|
|
kind: 'Pod',
|
|
metadata: { name: helperPodName, namespace },
|
|
spec: {
|
|
containers: [{
|
|
name: 'helper',
|
|
image: 'busybox:1.36',
|
|
command: ['sh', '-c', 'sleep 3600'],
|
|
volumeMounts: [{ name: 'dump', mountPath: '/data' }],
|
|
resources: { requests: { cpu: '100m', memory: '128Mi' }, limits: { cpu: '500m', memory: '256Mi' } },
|
|
}],
|
|
volumes: [{ name: 'dump', persistentVolumeClaim: { claimName: pvcName } }],
|
|
restartPolicy: 'Never',
|
|
},
|
|
};
|
|
|
|
await coreApi.createNamespacedPod(namespace, helperPod);
|
|
|
|
// Wait for helper pod Running
|
|
const podTimeout = 120_000;
|
|
const podStart = Date.now();
|
|
while (Date.now() - podStart < podTimeout) {
|
|
const pod = await coreApi.readNamespacedPod(helperPodName, namespace);
|
|
if (pod.body.status?.phase === 'Running') break;
|
|
if (pod.body.status?.phase === 'Failed') throw new Error('DB dump helper pod failed to start');
|
|
await new Promise(r => setTimeout(r, 2000));
|
|
}
|
|
|
|
// ── 3. kubectl cp the dump file into the helper pod ──
|
|
const tmpKubeconfig = path.join('/tmp', `kubeconfig-dbdump-${ts}.yaml`);
|
|
const kcYaml = kc.exportConfig();
|
|
fs.writeFileSync(tmpKubeconfig, kcYaml);
|
|
|
|
// Verify tar is available inside the helper pod
|
|
try {
|
|
await execFileAsync('kubectl', [
|
|
'--kubeconfig', tmpKubeconfig,
|
|
'exec', '-n', namespace, helperPodName, '--', 'tar', '--version',
|
|
], { timeout: 30_000 });
|
|
} catch (e: any) {
|
|
this.logger.warn(`tar check in DB dump helper pod failed: ${e.message}`);
|
|
}
|
|
|
|
try {
|
|
await execFileAsync('kubectl', [
|
|
'--kubeconfig', tmpKubeconfig,
|
|
'cp', dumpFilePath, `${namespace}/${helperPodName}:/data/dump.sql`,
|
|
'--retries', '3',
|
|
], { maxBuffer: 50 * 1024 * 1024, timeout: 600_000 });
|
|
this.logger.log(`kubectl cp dump completed (${(dumpSize / 1024 / 1024).toFixed(1)} MB)`);
|
|
} finally {
|
|
try { fs.unlinkSync(tmpKubeconfig); } catch {}
|
|
try { await coreApi.deleteNamespacedPod(helperPodName, namespace); } catch {}
|
|
}
|
|
|
|
// ── 4. Build restore command ──
|
|
const command = isPostgres
|
|
? [
|
|
'sh', '-c',
|
|
`PGPASSWORD="$DB_PASSWORD" psql -h ${dbName} -U "$DB_USER" -d ${dbDatabase} -f /dump/dump.sql 2>&1`,
|
|
]
|
|
: [
|
|
'sh', '-c',
|
|
`mysql -h ${dbName} -u "$DB_USER" -p"$DB_PASSWORD" ${dbDatabase} < /dump/dump.sql 2>&1`,
|
|
];
|
|
|
|
const defaultDbVer = isPostgres ? '16' : '8.0';
|
|
const restoreDbVer = app.dbVersion || defaultDbVer;
|
|
const image = isPostgres ? `postgres:${restoreDbVer}-alpine` : `mysql:${restoreDbVer}`;
|
|
|
|
// ── 5. Create the restore Job ──
|
|
const job: k8s.V1Job = {
|
|
apiVersion: 'batch/v1',
|
|
kind: 'Job',
|
|
metadata: { name: jobName, namespace },
|
|
spec: {
|
|
ttlSecondsAfterFinished: 300,
|
|
backoffLimit: 1,
|
|
template: {
|
|
spec: {
|
|
restartPolicy: 'Never',
|
|
containers: [
|
|
{
|
|
name: 'restore',
|
|
image,
|
|
command,
|
|
env: [
|
|
{ name: 'DB_USER', valueFrom: { secretKeyRef: { name: `${app.name}-db-secret`, key: 'username' } } },
|
|
{ name: 'DB_PASSWORD', valueFrom: { secretKeyRef: { name: `${app.name}-db-secret`, key: 'password' } } },
|
|
],
|
|
volumeMounts: [
|
|
{ name: 'dump-volume', mountPath: '/dump', readOnly: true },
|
|
],
|
|
resources: {
|
|
requests: { cpu: '200m', memory: '256Mi' },
|
|
limits: { cpu: '1', memory: '1Gi' },
|
|
},
|
|
},
|
|
],
|
|
volumes: [
|
|
{
|
|
name: 'dump-volume',
|
|
persistentVolumeClaim: { claimName: pvcName },
|
|
},
|
|
],
|
|
},
|
|
},
|
|
},
|
|
};
|
|
|
|
try {
|
|
await batchApi.createNamespacedJob(namespace, job);
|
|
this.logger.log(`Created DB restore job ${jobName} for ${app.name}`);
|
|
} catch (e: any) {
|
|
try { await coreApi.deleteNamespacedPersistentVolumeClaim(pvcName, namespace); } catch {}
|
|
this.logger.error(`Failed to create restore job: ${e.message}`);
|
|
throw new Error('Failed to create database restore job');
|
|
}
|
|
|
|
// ── 6. Wait for the job to complete (max 10 minutes for large dumps) ──
|
|
const timeout = 600_000;
|
|
const start = Date.now();
|
|
let completed = false;
|
|
let failed = false;
|
|
|
|
while (Date.now() - start < timeout) {
|
|
await new Promise((r) => setTimeout(r, 5000));
|
|
try {
|
|
const jobStatus = await batchApi.readNamespacedJob(jobName, namespace);
|
|
const status = jobStatus.body.status;
|
|
if (status?.succeeded && status.succeeded > 0) { completed = true; break; }
|
|
if (status?.failed && status.failed > 0) { failed = true; break; }
|
|
} catch {
|
|
// Job may not be ready yet
|
|
}
|
|
}
|
|
|
|
// ── 7. Get logs from the restore job pod ──
|
|
let logs = '';
|
|
try {
|
|
const pods = await coreApi.listNamespacedPod(
|
|
namespace, undefined, undefined, undefined, undefined, `job-name=${jobName}`,
|
|
);
|
|
if (pods.body.items.length > 0) {
|
|
const podName = pods.body.items[0].metadata?.name;
|
|
if (podName) {
|
|
const logResponse = await coreApi.readNamespacedPodLog(podName, namespace);
|
|
logs = logResponse.body || '';
|
|
}
|
|
}
|
|
} catch (e: any) {
|
|
this.logger.warn(`Could not get restore job logs: ${e.message}`);
|
|
}
|
|
|
|
// ── 8. Clean up the dump PVC ──
|
|
try {
|
|
await coreApi.deleteNamespacedPersistentVolumeClaim(pvcName, namespace);
|
|
this.logger.log(`Cleaned up dump PVC: ${pvcName}`);
|
|
} catch {}
|
|
|
|
if (!completed && !failed) {
|
|
this.logger.warn(`DB restore job ${jobName} timed out`);
|
|
return { success: false, logs: logs || 'Restore job timed out after 10 minutes' };
|
|
}
|
|
|
|
if (failed) {
|
|
this.logger.warn(`DB restore job ${jobName} failed`);
|
|
return { success: false, logs: logs || 'Restore job failed' };
|
|
}
|
|
|
|
this.logger.log(`DB restore for ${app.name} completed successfully`);
|
|
return { success: true, logs };
|
|
}
|
|
|
|
/**
|
|
* Resize (expand) the database PVC for an application.
|
|
* K8s only supports PVC expansion, not shrinking.
|
|
*/
|
|
async resizeDatabasePvc(app: Application, newSize: string): Promise<{ success: boolean; message: string }> {
|
|
const { coreApi } = await this.getK8sClient(app.clusterId);
|
|
const namespace = `user-${app.userId.split('-')[0]}`;
|
|
const pvcName = `${app.name}-db`;
|
|
|
|
try {
|
|
// Read current PVC to check current size
|
|
const currentPvc = await coreApi.readNamespacedPersistentVolumeClaim(pvcName, namespace);
|
|
const currentSize = currentPvc.body.spec?.resources?.requests?.storage || '1Gi';
|
|
|
|
const currentGi = parseInt(currentSize.replace('Gi', ''), 10) || 1;
|
|
const newGi = parseInt(newSize.replace('Gi', ''), 10) || 1;
|
|
|
|
if (newGi <= currentGi) {
|
|
return { success: false, message: `New size (${newSize}) must be larger than current size (${currentSize})` };
|
|
}
|
|
|
|
// Patch PVC to expand
|
|
const patch = [
|
|
{
|
|
op: 'replace',
|
|
path: '/spec/resources/requests/storage',
|
|
value: newSize,
|
|
},
|
|
];
|
|
|
|
await coreApi.patchNamespacedPersistentVolumeClaim(
|
|
pvcName,
|
|
namespace,
|
|
patch,
|
|
undefined,
|
|
undefined,
|
|
undefined,
|
|
undefined,
|
|
undefined,
|
|
{ headers: { 'Content-Type': 'application/json-patch+json' } },
|
|
);
|
|
|
|
this.logger.log(`Resized PVC ${pvcName} from ${currentSize} to ${newSize}`);
|
|
return { success: true, message: `Database storage expanded from ${currentSize} to ${newSize}` };
|
|
} catch (e: any) {
|
|
this.logger.error(`Failed to resize PVC ${pvcName}: ${e.message}`);
|
|
return { success: false, message: e.body?.message || e.message || 'Failed to resize database storage' };
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Get current PVC size for an application's database.
|
|
*/
|
|
async getDatabasePvcSize(app: Application): Promise<string> {
|
|
try {
|
|
const { coreApi } = await this.getK8sClient(app.clusterId);
|
|
const namespace = `user-${app.userId.split('-')[0]}`;
|
|
const pvcName = `${app.name}-db`;
|
|
|
|
const pvc = await coreApi.readNamespacedPersistentVolumeClaim(pvcName, namespace);
|
|
return pvc.body.spec?.resources?.requests?.storage || app.dbStorageSize || '1Gi';
|
|
} catch {
|
|
return app.dbStorageSize || '1Gi';
|
|
}
|
|
}
|
|
|
|
// ─── Snapshot helpers ───────────────────────────────
|
|
|
|
/**
|
|
* Export (dump) the application database to a local file via a K8s Job.
|
|
* Returns the dump as a Buffer, or null on failure.
|
|
*
|
|
* Strategy: Run dump command, then sleep for 60s to allow exec retrieval.
|
|
*/
|
|
async exportDatabaseDump(app: Application): Promise<{ data: Buffer | null; logs: string }> {
|
|
const { coreApi, kc } = await this.getK8sClient(app.clusterId);
|
|
const batchApi = kc.makeApiClient(k8s.BatchV1Api);
|
|
const namespace = `user-${app.userId.split('-')[0]}`;
|
|
const dbName = `${app.name}-db`;
|
|
const jobName = `${app.name}-db-dump-${Date.now()}`;
|
|
const isPostgres = app.databaseType === DatabaseType.POSTGRESQL;
|
|
const dbDatabase = app.name.replace(/-/g, '_');
|
|
|
|
const defaultDbVer = isPostgres ? '16' : '8.0';
|
|
const dbVer = app.dbVersion || defaultDbVer;
|
|
const image = isPostgres ? `postgres:${dbVer}-alpine` : `mysql:${dbVer}`;
|
|
|
|
// Dump command writes to /dump/output.sql, then sleeps to allow exec retrieval
|
|
const command = isPostgres
|
|
? ['sh', '-c', `PGPASSWORD="$DB_PASSWORD" pg_dump -h ${dbName} -U "$DB_USER" -d ${dbDatabase} --no-owner --no-acl > /dump/output.sql 2>/tmp/err; cat /tmp/err >&2; echo "DUMP_DONE"; sleep 120`]
|
|
: ['sh', '-c', `mysqldump -h ${dbName} -u "$DB_USER" -p"$DB_PASSWORD" ${dbDatabase} > /dump/output.sql 2>/tmp/err; cat /tmp/err >&2; echo "DUMP_DONE"; sleep 120`];
|
|
|
|
const job: k8s.V1Job = {
|
|
apiVersion: 'batch/v1',
|
|
kind: 'Job',
|
|
metadata: { name: jobName, namespace },
|
|
spec: {
|
|
ttlSecondsAfterFinished: 180,
|
|
activeDeadlineSeconds: 300,
|
|
backoffLimit: 0,
|
|
template: {
|
|
spec: {
|
|
restartPolicy: 'Never',
|
|
containers: [{
|
|
name: 'dump',
|
|
image,
|
|
command,
|
|
env: [
|
|
{ name: 'DB_USER', valueFrom: { secretKeyRef: { name: `${app.name}-db-secret`, key: 'username' } } },
|
|
{ name: 'DB_PASSWORD', valueFrom: { secretKeyRef: { name: `${app.name}-db-secret`, key: 'password' } } },
|
|
],
|
|
volumeMounts: [{ name: 'dump-vol', mountPath: '/dump' }],
|
|
resources: { requests: { cpu: '100m', memory: '128Mi' }, limits: { cpu: '500m', memory: '512Mi' } },
|
|
}],
|
|
volumes: [{ name: 'dump-vol', emptyDir: {} }],
|
|
},
|
|
},
|
|
},
|
|
};
|
|
|
|
try {
|
|
await batchApi.createNamespacedJob(namespace, job);
|
|
} catch (e: any) {
|
|
this.logger.error(`Failed to create DB dump job: ${e.message}`);
|
|
return { data: null, logs: `Failed to create dump job: ${e.message}` };
|
|
}
|
|
|
|
// Wait for dump to complete (check logs for DUMP_DONE marker)
|
|
const timeout = 300_000;
|
|
const start = Date.now();
|
|
let dumpDone = false;
|
|
let podName: string | undefined;
|
|
|
|
while (Date.now() - start < timeout) {
|
|
await new Promise((r) => setTimeout(r, 3000));
|
|
try {
|
|
const pods = await coreApi.listNamespacedPod(namespace, undefined, undefined, undefined, undefined, `job-name=${jobName}`);
|
|
if (pods.body.items.length > 0) {
|
|
podName = pods.body.items[0].metadata?.name;
|
|
const phase = pods.body.items[0].status?.phase;
|
|
|
|
// Check if pod is Running (container is in sleep phase after dump)
|
|
if (podName && phase === 'Running') {
|
|
try {
|
|
const logRes = await coreApi.readNamespacedPodLog(podName, namespace, 'dump', false, undefined, undefined, undefined, undefined, undefined, 50);
|
|
if (logRes.body?.includes('DUMP_DONE')) {
|
|
dumpDone = true;
|
|
break;
|
|
}
|
|
} catch {}
|
|
}
|
|
|
|
// If Failed, exit early
|
|
if (phase === 'Failed') break;
|
|
}
|
|
} catch {}
|
|
}
|
|
|
|
let dumpBuffer: Buffer | null = null;
|
|
let logs = '';
|
|
|
|
if (dumpDone && podName) {
|
|
try {
|
|
// Use kubectl cp equivalent via Exec
|
|
const exec = new k8s.Exec(kc);
|
|
const chunks: Buffer[] = [];
|
|
|
|
await new Promise<void>((resolve, reject) => {
|
|
exec.exec(
|
|
namespace, podName!, 'dump',
|
|
['cat', '/dump/output.sql'],
|
|
{
|
|
write: (data: string) => { chunks.push(Buffer.from(data, 'binary')); },
|
|
} as any,
|
|
null,
|
|
{
|
|
write: (data: string) => { logs += data; },
|
|
} as any,
|
|
false,
|
|
(status: k8s.V1Status) => {
|
|
if (status.status === 'Success') resolve();
|
|
else reject(new Error(status.message || 'exec failed'));
|
|
},
|
|
);
|
|
});
|
|
|
|
if (chunks.length > 0) {
|
|
dumpBuffer = Buffer.concat(chunks);
|
|
this.logger.log(`DB dump retrieved: ${dumpBuffer.length} bytes`);
|
|
}
|
|
} catch (e: any) {
|
|
this.logger.warn(`Exec failed for DB dump: ${e.message}`);
|
|
logs = e.message;
|
|
}
|
|
}
|
|
|
|
// Cleanup: delete the job early to free resources
|
|
try {
|
|
await batchApi.deleteNamespacedJob(jobName, namespace, undefined, undefined, undefined, undefined, 'Background');
|
|
} catch {}
|
|
|
|
if (!dumpBuffer) {
|
|
return { data: null, logs: logs || 'Dump failed or could not be retrieved' };
|
|
}
|
|
|
|
return { data: dumpBuffer, logs: 'OK' };
|
|
}
|
|
|
|
/**
|
|
* Archive the wp-content directory from a WordPress app's PVC via a K8s Job.
|
|
* The job creates a tar.gz of /var/www/html/wp-content and we retrieve it via exec.
|
|
*
|
|
* Strategy: Create archive, then sleep to allow exec retrieval.
|
|
*/
|
|
async archiveWpContent(app: Application): Promise<{ data: Buffer | null; logs: string }> {
|
|
const { coreApi, kc } = await this.getK8sClient(app.clusterId);
|
|
const batchApi = kc.makeApiClient(k8s.BatchV1Api);
|
|
const namespace = `user-${app.userId.split('-')[0]}`;
|
|
const pvcName = `${app.name}-wp-content`;
|
|
const jobName = `${app.name}-wp-archive-${Date.now()}`;
|
|
|
|
const job: k8s.V1Job = {
|
|
apiVersion: 'batch/v1',
|
|
kind: 'Job',
|
|
metadata: { name: jobName, namespace },
|
|
spec: {
|
|
ttlSecondsAfterFinished: 180,
|
|
activeDeadlineSeconds: 300,
|
|
backoffLimit: 0,
|
|
template: {
|
|
spec: {
|
|
restartPolicy: 'Never',
|
|
containers: [{
|
|
name: 'archiver',
|
|
image: 'alpine:3.19',
|
|
command: ['sh', '-c', 'apk add --no-cache tar gzip > /dev/null 2>&1 && cd /wp-content && tar czf /output/wp-content.tar.gz . && echo "ARCHIVE_DONE" && sleep 120'],
|
|
volumeMounts: [
|
|
{ name: 'wp-content', mountPath: '/wp-content', readOnly: true },
|
|
{ name: 'output', mountPath: '/output' },
|
|
],
|
|
resources: { requests: { cpu: '100m', memory: '64Mi' }, limits: { cpu: '500m', memory: '512Mi' } },
|
|
}],
|
|
volumes: [
|
|
{ name: 'wp-content', persistentVolumeClaim: { claimName: pvcName } },
|
|
{ name: 'output', emptyDir: {} },
|
|
],
|
|
},
|
|
},
|
|
},
|
|
};
|
|
|
|
try {
|
|
await batchApi.createNamespacedJob(namespace, job);
|
|
} catch (e: any) {
|
|
this.logger.error(`Failed to create wp-content archive job: ${e.message}`);
|
|
return { data: null, logs: `Failed to create archive job: ${e.message}` };
|
|
}
|
|
|
|
// Wait for archive to complete (check logs for ARCHIVE_DONE marker)
|
|
const timeout = 300_000;
|
|
const start = Date.now();
|
|
let archiveDone = false;
|
|
let podName: string | undefined;
|
|
|
|
while (Date.now() - start < timeout) {
|
|
await new Promise((r) => setTimeout(r, 3000));
|
|
try {
|
|
const pods = await coreApi.listNamespacedPod(namespace, undefined, undefined, undefined, undefined, `job-name=${jobName}`);
|
|
if (pods.body.items.length > 0) {
|
|
podName = pods.body.items[0].metadata?.name;
|
|
const phase = pods.body.items[0].status?.phase;
|
|
|
|
if (podName && phase === 'Running') {
|
|
try {
|
|
const logRes = await coreApi.readNamespacedPodLog(podName, namespace, 'archiver', false, undefined, undefined, undefined, undefined, undefined, 50);
|
|
if (logRes.body?.includes('ARCHIVE_DONE')) {
|
|
archiveDone = true;
|
|
break;
|
|
}
|
|
} catch {}
|
|
}
|
|
|
|
if (phase === 'Failed') break;
|
|
}
|
|
} catch {}
|
|
}
|
|
|
|
let archiveBuffer: Buffer | null = null;
|
|
let logs = '';
|
|
|
|
if (archiveDone && podName) {
|
|
try {
|
|
const exec = new k8s.Exec(kc);
|
|
const chunks: Buffer[] = [];
|
|
|
|
await new Promise<void>((resolve, reject) => {
|
|
exec.exec(
|
|
namespace, podName!, 'archiver',
|
|
['cat', '/output/wp-content.tar.gz'],
|
|
{
|
|
write: (data: string) => { chunks.push(Buffer.from(data, 'binary')); },
|
|
} as any,
|
|
null,
|
|
{
|
|
write: (data: string) => { logs += data; },
|
|
} as any,
|
|
false,
|
|
(status: k8s.V1Status) => {
|
|
if (status.status === 'Success') resolve();
|
|
else reject(new Error(status.message || 'exec failed'));
|
|
},
|
|
);
|
|
});
|
|
|
|
if (chunks.length > 0) {
|
|
archiveBuffer = Buffer.concat(chunks);
|
|
this.logger.log(`wp-content archive retrieved: ${archiveBuffer.length} bytes`);
|
|
}
|
|
} catch (e: any) {
|
|
this.logger.warn(`Exec failed for wp-content archive: ${e.message}`);
|
|
logs = e.message;
|
|
}
|
|
}
|
|
|
|
// Cleanup
|
|
try {
|
|
await batchApi.deleteNamespacedJob(jobName, namespace, undefined, undefined, undefined, undefined, 'Background');
|
|
} catch {}
|
|
|
|
if (!archiveBuffer) {
|
|
return { data: null, logs: logs || 'Archive failed or could not be retrieved' };
|
|
}
|
|
|
|
return { data: archiveBuffer, logs: 'OK' };
|
|
}
|
|
|
|
/**
|
|
* Restore wp-content from a tar.gz archive into the WordPress PVC.
|
|
*/
|
|
async restoreWpContent(app: Application, archiveBuffer: Buffer): Promise<{ success: boolean; logs: string }> {
|
|
const { coreApi, kc } = await this.getK8sClient(app.clusterId);
|
|
const batchApi = kc.makeApiClient(k8s.BatchV1Api);
|
|
const namespace = `user-${app.userId.split('-')[0]}`;
|
|
const pvcName = `${app.name}-wp-content`;
|
|
const jobName = `${app.name}-wp-restore-${Date.now()}`;
|
|
const secretName = `${jobName}-archive`;
|
|
|
|
// Store archive in a secret
|
|
const archiveSecret = {
|
|
apiVersion: 'v1',
|
|
kind: 'Secret',
|
|
metadata: { name: secretName, namespace },
|
|
data: { 'wp-content.tar.gz': archiveBuffer.toString('base64') },
|
|
};
|
|
|
|
try {
|
|
await coreApi.createNamespacedSecret(namespace, archiveSecret);
|
|
} catch (e: any) {
|
|
return { success: false, logs: `Failed to create archive secret: ${e.message}` };
|
|
}
|
|
|
|
const job: k8s.V1Job = {
|
|
apiVersion: 'batch/v1',
|
|
kind: 'Job',
|
|
metadata: { name: jobName, namespace },
|
|
spec: {
|
|
ttlSecondsAfterFinished: 120,
|
|
backoffLimit: 0,
|
|
template: {
|
|
spec: {
|
|
restartPolicy: 'Never',
|
|
containers: [{
|
|
name: 'restore',
|
|
image: 'alpine:3.19',
|
|
command: ['sh', '-c', 'apk add --no-cache tar gzip > /dev/null 2>&1 && rm -rf /wp-content/* && cd /wp-content && tar xzf /archive/wp-content.tar.gz && echo "RESTORE_DONE"'],
|
|
volumeMounts: [
|
|
{ name: 'wp-content', mountPath: '/wp-content' },
|
|
{ name: 'archive', mountPath: '/archive', readOnly: true },
|
|
],
|
|
resources: { requests: { cpu: '100m', memory: '64Mi' }, limits: { cpu: '500m', memory: '256Mi' } },
|
|
}],
|
|
volumes: [
|
|
{ name: 'wp-content', persistentVolumeClaim: { claimName: pvcName } },
|
|
{ name: 'archive', secret: { secretName } },
|
|
],
|
|
},
|
|
},
|
|
},
|
|
};
|
|
|
|
try {
|
|
await batchApi.createNamespacedJob(namespace, job);
|
|
} catch (e: any) {
|
|
try { await coreApi.deleteNamespacedSecret(secretName, namespace); } catch {}
|
|
return { success: false, logs: `Failed to create restore job: ${e.message}` };
|
|
}
|
|
|
|
// Wait
|
|
const timeout = 300_000;
|
|
const start = Date.now();
|
|
let succeeded = false;
|
|
let failed = false;
|
|
while (Date.now() - start < timeout) {
|
|
await new Promise((r) => setTimeout(r, 3000));
|
|
try {
|
|
const st = await batchApi.readNamespacedJob(jobName, namespace);
|
|
if (st.body.status?.succeeded && st.body.status.succeeded > 0) { succeeded = true; break; }
|
|
if (st.body.status?.failed && st.body.status.failed > 0) { failed = true; break; }
|
|
} catch {}
|
|
}
|
|
|
|
let logs = '';
|
|
try {
|
|
const pods = await coreApi.listNamespacedPod(namespace, undefined, undefined, undefined, undefined, `job-name=${jobName}`);
|
|
if (pods.body.items.length > 0 && pods.body.items[0].metadata?.name) {
|
|
const logRes = await coreApi.readNamespacedPodLog(pods.body.items[0].metadata.name, namespace);
|
|
logs = logRes.body || '';
|
|
}
|
|
} catch {}
|
|
|
|
try { await coreApi.deleteNamespacedSecret(secretName, namespace); } catch {}
|
|
|
|
return { success: succeeded && !failed, logs: logs || (succeeded ? 'Restore completed' : 'Restore failed or timed out') };
|
|
}
|
|
|
|
// ─── K8s Revision-based Rollback ─────────────────────
|
|
|
|
/**
|
|
* Get the list of Helm release revisions for an application.
|
|
* Returns up to 10 revisions sorted newest-first.
|
|
*/
|
|
async getDeploymentRevisions(app: Application): Promise<{
|
|
revisions: Array<{
|
|
revision: number;
|
|
image: string;
|
|
changeCause: string;
|
|
createdAt: string;
|
|
replicas: number;
|
|
isCurrent: boolean;
|
|
}>;
|
|
currentRevision: number;
|
|
}> {
|
|
const namespace = `user-${app.userId.split('-')[0]}`;
|
|
const releaseName = app.name;
|
|
|
|
try {
|
|
const kubeconfig = await this.getKubeconfig(app.clusterId);
|
|
const helmRevisions = await this.helmService.history(releaseName, namespace, kubeconfig);
|
|
|
|
if (!helmRevisions || helmRevisions.length === 0) {
|
|
return { revisions: [], currentRevision: 0 };
|
|
}
|
|
|
|
// The last "deployed" revision is the current one
|
|
const currentRev = helmRevisions
|
|
.filter((r) => r.status === 'deployed')
|
|
.sort((a, b) => b.revision - a.revision)[0];
|
|
const currentRevision = currentRev ? currentRev.revision : 0;
|
|
|
|
const revisions = helmRevisions
|
|
.sort((a, b) => b.revision - a.revision)
|
|
.slice(0, 10)
|
|
.map((r) => ({
|
|
revision: r.revision,
|
|
image: '', // Helm history doesn't expose image; frontend will use description
|
|
changeCause: r.description || `${r.status} — ${r.chart}`,
|
|
createdAt: r.updated,
|
|
replicas: 0,
|
|
isCurrent: r.revision === currentRevision,
|
|
}));
|
|
|
|
return { revisions, currentRevision };
|
|
} catch (e: any) {
|
|
this.logger.warn(`Could not get Helm history for ${releaseName}: ${e.message}`);
|
|
return { revisions: [], currentRevision: 0 };
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Rollback a Helm release to a specific revision.
|
|
*/
|
|
async rollbackDeploymentRevision(
|
|
app: Application,
|
|
targetRevision: number,
|
|
): Promise<{ success: boolean; message: string }> {
|
|
const namespace = `user-${app.userId.split('-')[0]}`;
|
|
const releaseName = app.name;
|
|
|
|
try {
|
|
const kubeconfig = await this.getKubeconfig(app.clusterId);
|
|
await this.helmService.rollback(releaseName, targetRevision, namespace, kubeconfig);
|
|
this.logger.log(`Rolled back ${releaseName} to Helm revision ${targetRevision}`);
|
|
return { success: true, message: `Rolled back to Helm revision ${targetRevision}` };
|
|
} catch (e: any) {
|
|
this.logger.error(`Helm rollback failed for ${releaseName}: ${e.message}`);
|
|
return { success: false, message: e.message };
|
|
}
|
|
}
|
|
|
|
private generatePassword(length = 24): string {
|
|
const chars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789';
|
|
let password = '';
|
|
for (let i = 0; i < length; i++) {
|
|
password += chars.charAt(Math.floor(Math.random() * chars.length));
|
|
}
|
|
return password;
|
|
}
|
|
}
|