abrban/cloud-host
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
58ab81469bba5479f499b9033647073f354ef945
cloud-host/backend/src/kubernetes/kubernetes.service.ts
T
keyhan 8163665c86 fix(platform): close remaining audit findings from security review 
Harden preview/deploy flows, OTP generation, zip extraction, and multi-replica billing races; document full remediation status in AUDIT-STATUS.fa.md.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-07-03 12:30:22 +03:30

4941 lines
168 KiB
TypeScript
Raw Blame History

import { Injectable, Logger, OnModuleInit, BadRequestException } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import * as k8s from '@kubernetes/client-node';
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import { execFile } from 'child_process';
import { promisify } from 'util';
import { PassThrough } from 'stream';
import { ClustersService } from '../clusters/clusters.service';
import { Application } from '../applications/entities/application.entity';
import { Deployment } from '../deployments/entities/deployment.entity';
import { ensureAppUrlEnv } from '../applications/app-url.util';
import { AppRuntime, DatabaseType, CustomDomainStatus, ServiceAccessTarget, ProductType, isManagedProductType } from '../common/enums';
import { HelmService } from './helm.service';
import { RegistryService } from './registry.service';
import { K8sClientService } from './k8s-client.service';
import { K8sLifecycleService } from './k8s-lifecycle.service';
import { userNamespace, userIdSlug } from './k8s-workload.util';
import { registerKubeconfigNoProxy } from '../common/kubernetes-proxy.util';
const execFileAsync = promisify(execFile);
interface ManifestContext {
appName: string;
namespace: string;
image: string;
port: number;
replicas: number;
cpuRequest: string;
cpuLimit: string;
memoryRequest: string;
memoryLimit: string;
envVars: Record<string, string>;
runtime: string;
databaseType: string;
domain: string;
subdomain: string;
dbUsername: string;
dbPassword: string;
dbVersion: string;
dbStorageSize: string;
dbCpuRequest: string;
dbCpuLimit: string;
dbMemoryRequest: string;
dbMemoryLimit: string;
appStorageSize: string;
enableRedis: boolean;
redisVersion: string;
enableRabbitmq: boolean;
rabbitmqVersion: string;
enableElasticsearch: boolean;
elasticsearchVersion: string;
logPaths: string[];
ownerId: string;
applicationId: string;
}
type StorageUsageSlice = {
allocatedRaw: string;
allocatedGi: number;
usedGi: number;
availableGi: number;
usedPercent: number;
allocated: string;
used: string;
available: string;
};
@Injectable()
export class KubernetesService implements OnModuleInit {
private readonly logger = new Logger(KubernetesService.name);
constructor(
private configService: ConfigService,
private clustersService: ClustersService,
private helmService: HelmService,
private registryService: RegistryService,
private k8sClientService: K8sClientService,
private k8sLifecycleService: K8sLifecycleService,
@InjectRepository(Deployment)
private deploymentsRepository: Repository<Deployment>,
) {}
/**
* Stable per-app preview number, reused from the latest deployment that has
* one. Used to restore the preview host on the Ingress (e.g. after a custom
* domain is removed) without triggering a fresh deploy.
*/
private async resolvePreviewNumber(applicationId: string): Promise<string | null> {
const existing = await this.deploymentsRepository
.createQueryBuilder('d')
.select('d.previewSubdomain', 'previewSubdomain')
.where('d.applicationId = :applicationId', { applicationId })
.andWhere('d.previewSubdomain IS NOT NULL')
.orderBy('d.createdAt', 'DESC')
.limit(1)
.getRawOne<{ previewSubdomain: string }>();
return existing?.previewSubdomain || null;
}
onModuleInit() {
// Helm chart is used for deployments — no local template loading needed
}
/**
* Build Helm values object from an Application entity and image URI.
*/
/**
* Resolve the database workload's CPU/RAM. Prefers the user-selected
* per-database resources (optionalServiceResources.database); falls back to
* the app's main resources for managed databases / legacy apps.
*/
private resolveDatabaseResources(app: Application) {
const res = app.optionalServiceResources?.database;
return {
cpuRequest: res?.cpuRequest || app.cpuRequest || '100m',
cpuLimit: res?.cpuLimit || app.cpuLimit || '500m',
memoryRequest: res?.memoryRequest || app.memoryRequest || '256Mi',
memoryLimit: res?.memoryLimit || app.memoryLimit || '512Mi',
};
}
private buildRedisHelmBlock(app: Application) {
const res = app.optionalServiceResources?.redis;
const storageGi = res?.storageGi ?? 1;
return {
enabled: app.enableRedis || false,
version: app.redisVersion || '7.2',
storageSize: `${storageGi}Gi`,
resources: {
cpuRequest: res?.cpuRequest || '50m',
cpuLimit: res?.cpuLimit || '200m',
memoryRequest: res?.memoryRequest || '64Mi',
memoryLimit: res?.memoryLimit || '256Mi',
},
};
}
private buildRabbitmqHelmBlock(app: Application) {
const res = app.optionalServiceResources?.rabbitmq;
const storageGi = res?.storageGi ?? 2;
return {
enabled: app.enableRabbitmq || false,
version: app.rabbitmqVersion || '3.13',
storageSize: `${storageGi}Gi`,
resources: {
cpuRequest: res?.cpuRequest || '100m',
cpuLimit: res?.cpuLimit || '500m',
memoryRequest: res?.memoryRequest || '256Mi',
memoryLimit: res?.memoryLimit || '512Mi',
},
};
}
private resolveEnvVars(app: Application): Record<string, string> {
const platformDomain = this.configService.get('platform.domain') || 'apps.cloudhost.ir';
return ensureAppUrlEnv(app, platformDomain);
}
private helmGlobalStorageValues(): Record<string, unknown> {
const storageClass = this.configService.get<string>('platform.storageClass') || '';
const createStorageClass = this.configService.get<boolean>('platform.createStorageClass') === true;
const storageProvisioner = this.configService.get<string>('platform.storageProvisioner') || 'rancher.io/local-path';
if (!storageClass) {
return {
storageClass: '',
createStorageClass: false,
storageProvisioner,
};
}
return { storageClass, createStorageClass, storageProvisioner };
}
/** Create platform StorageClass on the target cluster when configured (Helm chart + K8s API fallback). */
private async ensurePlatformStorageClass(kubeconfig: string): Promise<void> {
const storageClass = this.configService.get<string>('platform.storageClass')?.trim();
const createStorageClass = this.configService.get<boolean>('platform.createStorageClass') === true;
if (!storageClass || !createStorageClass) {
return;
}
const kc = new k8s.KubeConfig();
kc.loadFromString(kubeconfig);
const storageApi = kc.makeApiClient(k8s.StorageV1Api);
const provisioner = this.configService.get<string>('platform.storageProvisioner') || 'rancher.io/local-path';
try {
await storageApi.readStorageClass({ name: storageClass });
return;
} catch (err: any) {
if (err.code !== 404 && err.body?.code !== 404) {
throw err;
}
}
await storageApi.createStorageClass({
body: {
apiVersion: 'storage.k8s.io/v1',
kind: 'StorageClass',
metadata: { name: storageClass },
provisioner,
allowVolumeExpansion: true,
reclaimPolicy: 'Delete',
volumeBindingMode: 'WaitForFirstConsumer',
},
});
this.logger.log(`Created StorageClass "${storageClass}" (provisioner: ${provisioner})`);
}
/** Helm values for managed_database / managed_redis / managed_rabbitmq (no app workload). */
/**
* Return the app's database password, generating and PERSISTING one if it is
* missing. Without persistence a fresh password would be generated on every
* helm upgrade, breaking auth against the database's persisted volume.
*/
private ensureDbPassword(app: Application): string {
if (!app.dbPassword) {
app.dbPassword = this.generatePassword();
this.deploymentsRepository.manager
.getRepository(Application)
.update(app.id, { dbPassword: app.dbPassword })
.catch((e: any) =>
this.logger.warn(`Failed to persist generated dbPassword for ${app.name}: ${e.message}`),
);
this.logger.warn(`App ${app.name} had no dbPassword — generated and persisted one`);
}
return app.dbPassword;
}
private buildManagedHelmValues(app: Application): Record<string, any> {
const namespace = this.getUserNamespace(app.userId);
const pullRegistryUrl = this.registryService.getRegistryUrl();
const isPostgres = app.databaseType === DatabaseType.POSTGRESQL;
const productType = app.productType;
const values: Record<string, any> = {
global: this.helmGlobalStorageValues(),
app: {
enabled: false,
name: app.name,
namespace,
runtime: app.runtime,
image: '',
port: app.port || 3000,
replicas: 0,
storageSize: app.appStorageSize || '2Gi',
},
resources: {
cpuRequest: app.cpuRequest,
cpuLimit: app.cpuLimit,
memoryRequest: app.memoryRequest,
memoryLimit: app.memoryLimit,
},
envVars: {},
ingress: {
enabled: false,
subdomain: app.subdomain || app.name,
domain: this.configService.get('platform.domain'),
clusterIssuer: 'letsencrypt-prod',
customDomain: '',
},
registry: { url: pullRegistryUrl },
database: {
enabled: false,
type: app.databaseType,
version: app.dbVersion || (isPostgres ? '16' : '8.0'),
username: app.dbUsername || 'appuser',
password:
app.databaseType && app.databaseType !== DatabaseType.NONE
? this.ensureDbPassword(app)
: '',
storageSize: app.dbStorageSize || '1Gi',
resources: this.resolveDatabaseResources(app),
},
redis: { enabled: false, storageSize: '1Gi', resources: {} },
rabbitmq: { enabled: false, storageSize: '2Gi', resources: {} },
wordpress: { enabled: false },
elasticsearch: {
enabled: false,
logPaths: [],
ownerId: app.userId,
applicationId: app.id,
},
images: { baseRegistry: this.configService.get<string>('build.baseImageRegistry') || '' },
changeCause: `Helm provision ${app.name} (${productType}) at ${new Date().toISOString()}`,
};
switch (productType) {
case ProductType.MANAGED_DATABASE:
values.database.enabled = true;
values.redis.enabled = false;
values.rabbitmq.enabled = false;
break;
case ProductType.MANAGED_REDIS:
values.redis = this.buildRedisHelmBlock(app);
values.redis.enabled = true;
break;
case ProductType.MANAGED_RABBITMQ:
values.rabbitmq = this.buildRabbitmqHelmBlock(app);
values.rabbitmq.enabled = true;
break;
default:
break;
}
return values;
}
private buildHelmValues(app: Application, imageUri: string, previewNumber?: string | null): Record<string, any> {
const domain = this.configService.get('platform.domain');
const previewRootDomain = this.configService.get<string>('platform.previewRootDomain') || domain;
const namespacePrefix = userIdSlug(app.userId);
const previewHost = previewNumber && !this.hasVerifiedCustomDomain(app)
? `${namespacePrefix}-${previewNumber}.${previewRootDomain}`
: '';
const pullRegistryUrl = this.registryService.getRegistryUrl();
const isWordPress = app.runtime === AppRuntime.WORDPRESS;
const hasDb = app.databaseType !== DatabaseType.NONE;
const isPostgres = app.databaseType === DatabaseType.POSTGRESQL;
const values: Record<string, any> = {
global: this.helmGlobalStorageValues(),
app: {
enabled: true,
name: app.name,
namespace: this.getUserNamespace(app.userId),
runtime: app.runtime,
image: imageUri,
port: app.port,
replicas: app.replicas || 1,
storageSize: app.appStorageSize || '2Gi',
},
resources: {
cpuRequest: app.cpuRequest,
cpuLimit: app.cpuLimit,
memoryRequest: app.memoryRequest,
memoryLimit: app.memoryLimit,
},
envVars: this.resolveEnvVars(app),
ingress: {
enabled: true,
className: this.configService.get<string>('platform.ingressClass') || 'traefik',
subdomain: app.subdomain || app.name,
domain: domain,
clusterIssuer: 'letsencrypt-prod',
customDomain: app.customDomain && app.customDomainStatus === CustomDomainStatus.VERIFIED ? app.customDomain : '',
previewHost,
},
registry: {
url: pullRegistryUrl,
},
database: {
enabled: hasDb,
type: app.databaseType,
version: app.dbVersion || (isPostgres ? '16' : '8.0'),
username: app.dbUsername || 'appuser',
password: hasDb ? this.ensureDbPassword(app) : '',
storageSize: app.dbStorageSize || '1Gi',
resources: this.resolveDatabaseResources(app),
},
wordpress: {
enabled: isWordPress,
},
redis: this.buildRedisHelmBlock(app),
rabbitmq: this.buildRabbitmqHelmBlock(app),
elasticsearch: {
enabled: app.enableElasticsearch || false,
logPaths: app.logPaths || [],
ownerId: app.userId,
applicationId: app.id,
elasticPassword: this.configService.get<string>('elasticsearch.password'),
fluentbitPassword: this.configService.get<string>('elasticsearch.fluentbitPassword'),
kibanaPassword: this.configService.get<string>('elasticsearch.kibanaPassword'),
},
images: { baseRegistry: this.configService.get<string>('build.baseImageRegistry') || '' },
changeCause: `Deploy ${imageUri} at ${new Date().toISOString()}`,
};
return values;
}
/** Install or upgrade only the workload for a managed service (database, Redis, or RabbitMQ). */
async deployManagedService(app: Application): Promise<Record<string, any>> {
if (!isManagedProductType(app.productType)) {
throw new BadRequestException('deployManagedService requires a managed product type');
}
try {
return await this.deployManagedViaHelm(app);
} catch (helmError: any) {
this.logger.warn(`Helm provision failed for managed ${app.name}, falling back to direct K8s API: ${helmError.message}`);
return await this.deployManagedViaK8sApi(app);
}
}
async deployApplication(app: Application, imageUri: string, opts?: { previewNumber?: string | null }): Promise<Record<string, any>> {
if (isManagedProductType(app.productType)) {
return this.deployManagedService(app);
}
const previewNumber = opts?.previewNumber ?? null;
const workloadImage = this.registryService.normalizeImageReference(imageUri);
if (workloadImage !== imageUri) {
this.logger.log(`Using in-cluster registry image for ${app.name}: ${workloadImage}`);
}
// Try Helm first, fall back to direct K8s API if Helm is unavailable
try {
return await this.deployViaHelm(app, workloadImage, previewNumber);
} catch (helmError: any) {
this.logger.warn(`Helm deploy failed for ${app.name}, falling back to direct K8s API: ${helmError.message}`);
return await this.deployViaK8sApi(app, workloadImage, previewNumber);
}
}
async waitForApplicationReady(app: Application, timeoutMs = 600_000, shouldAbort?: () => Promise<boolean>): Promise<void> {
const { coreApi, appsApi } = await this.k8sClientService.getK8sClient(app.clusterId);
const namespace = this.getUserNamespace(app.userId);
const managed = isManagedProductType(app.productType);
const workloads = [
...(!managed ? [{ name: app.name, replicas: app.replicas || 1 }] : []),
...(app.databaseType !== DatabaseType.NONE ? [{ name: `${app.name}-db`, replicas: 1 }] : []),
...(app.enableRedis ? [{ name: `${app.name}-redis`, replicas: 1 }] : []),
...(app.enableRabbitmq ? [{ name: `${app.name}-rabbitmq`, replicas: 1 }] : []),
];
const start = Date.now();
let lastSummary = '';
this.logger.log(`Waiting for ${app.name} workloads to become Ready in ${namespace}: ${workloads.map((w) => w.name).join(', ')}`);
while (Date.now() - start < timeoutMs) {
if (await shouldAbort?.()) {
throw new Error('Deployment cancelled by user');
}
const statuses = await Promise.all(workloads.map((workload) => this.getDeploymentReadiness(appsApi, namespace, workload.name, workload.replicas)));
lastSummary = statuses.map((s) => `${s.name} ${s.readyReplicas}/${s.desiredReplicas}`).join(', ');
if (statuses.every((s) => s.ready)) {
this.logger.log(`All workloads for ${app.name} are Ready (${lastSummary})`);
return;
}
await new Promise((resolve) => setTimeout(resolve, 5000));
}
const podSummary = await this.describeWorkloadPods(
coreApi,
namespace,
workloads.map((w) => w.name),
);
throw new Error(`Application workloads did not become ready within ${Math.round(timeoutMs / 1000)}s. ` + `Readiness: ${lastSummary || 'no deployment status available'}. ${podSummary}`);
}
async updateIngress(app: Application): Promise<void> {
const domain = this.configService.get('platform.domain');
const subdomain = app.subdomain || app.name;
const namespace = this.getUserNamespace(app.userId);
const customDomain = app.customDomain && app.customDomainStatus === CustomDomainStatus.VERIFIED ? app.customDomain : undefined;
// When there's no verified custom domain, restore the stable preview host so
// the app stays reachable (and keeps its TLS) — e.g. after a custom domain is
// removed. buildHelmValues only emits a preview host when there's no
// app.customDomain at all, so we only resolve it when none is set.
const previewNumber = app.customDomain ? null : await this.resolvePreviewNumber(app.id);
try {
const kubeconfig = await this.k8sClientService.getKubeconfig(app.clusterId);
const imageUri = app.latestImageTag ? this.registryService.normalizeImageReference(app.latestImageTag) : '';
const values = this.buildHelmValues(app, imageUri, previewNumber);
await this.helmService.installOrUpgrade(app.name, namespace, values, kubeconfig);
this.logger.log(`Updated ingress for ${app.name} via Helm (customDomain: ${customDomain || 'none'})`);
} catch (helmError: any) {
this.logger.warn(`Helm ingress update failed for ${app.name}, using direct K8s API: ${helmError.message}`);
const { networkingApi } = await this.k8sClientService.getK8sClient(app.clusterId);
const ctx: ManifestContext = {
appName: app.name,
namespace,
image: '',
port: app.port,
replicas: app.replicas,
cpuRequest: app.cpuRequest,
cpuLimit: app.cpuLimit,
memoryRequest: app.memoryRequest,
memoryLimit: app.memoryLimit,
envVars: this.resolveEnvVars(app),
runtime: app.runtime,
databaseType: app.databaseType,
domain,
subdomain,
dbUsername: app.dbUsername || '',
dbPassword: app.dbPassword || '',
dbVersion: app.dbVersion || '',
dbStorageSize: app.dbStorageSize || '1Gi',
dbCpuRequest: this.resolveDatabaseResources(app).cpuRequest,
dbCpuLimit: this.resolveDatabaseResources(app).cpuLimit,
dbMemoryRequest: this.resolveDatabaseResources(app).memoryRequest,
dbMemoryLimit: this.resolveDatabaseResources(app).memoryLimit,
appStorageSize: app.appStorageSize || '2Gi',
enableRedis: app.enableRedis || false,
redisVersion: app.redisVersion || '7.2',
enableRabbitmq: app.enableRabbitmq || false,
rabbitmqVersion: app.rabbitmqVersion || '3.13',
enableElasticsearch: app.enableElasticsearch || false,
elasticsearchVersion: app.elasticsearchVersion || '8.12',
logPaths: app.logPaths || [],
ownerId: app.userId,
applicationId: app.id,
};
await this.applyIngress(networkingApi, ctx, customDomain, previewNumber);
this.logger.log(`Updated ingress for ${app.name} via K8s API (customDomain: ${customDomain || 'none'})`);
}
}
// ── Helm-based deployment ─────────────────────────────────────────
private async deployViaHelm(app: Application, imageUri: string, previewNumber?: string | null): Promise<Record<string, any>> {
const kubeconfig = await this.k8sClientService.getKubeconfig(app.clusterId);
await this.ensurePlatformStorageClass(kubeconfig);
const { coreApi } = await this.k8sClientService.getK8sClient(app.clusterId);
const values = this.buildHelmValues(app, imageUri, previewNumber);
const namespace = values.app.namespace as string;
await this.registryService.ensureRegistryPullSecret(coreApi, namespace);
const releaseName = app.name;
const result = await this.helmService.installOrUpgrade(releaseName, namespace, values, kubeconfig);
this.logger.log(`Successfully deployed ${app.name} to namespace ${namespace} via Helm`);
return {
helm: { release: releaseName, namespace, stdout: result.stdout },
values,
};
}
private async deployManagedViaHelm(app: Application): Promise<Record<string, any>> {
const kubeconfig = await this.k8sClientService.getKubeconfig(app.clusterId);
await this.ensurePlatformStorageClass(kubeconfig);
const values = this.buildManagedHelmValues(app);
const namespace = values.app.namespace;
const releaseName = app.name;
const result = await this.helmService.installOrUpgrade(releaseName, namespace, values, kubeconfig);
this.logger.log(`Successfully provisioned managed service ${app.name} in ${namespace} via Helm`);
return {
helm: { release: releaseName, namespace, stdout: result.stdout },
values,
};
}
// ── Direct K8s API deployment (fallback) ──────────────────────────
private async deployManagedViaK8sApi(app: Application): Promise<Record<string, any>> {
const { coreApi, appsApi } = await this.k8sClientService.getK8sClient(app.clusterId);
const kubeconfig = await this.k8sClientService.getKubeconfig(app.clusterId);
await this.ensurePlatformStorageClass(kubeconfig);
const namespace = this.getUserNamespace(app.userId);
const context: ManifestContext = {
appName: app.name,
namespace,
image: '',
port: app.port || 3000,
replicas: 0,
cpuRequest: app.cpuRequest,
cpuLimit: app.cpuLimit,
memoryRequest: app.memoryRequest,
memoryLimit: app.memoryLimit,
envVars: {},
runtime: app.runtime,
databaseType: app.databaseType,
domain: this.configService.get('platform.domain') || 'apps.cloudhost.ir',
subdomain: app.subdomain || app.name,
dbUsername: app.dbUsername || 'appuser',
dbPassword:
app.databaseType && app.databaseType !== DatabaseType.NONE
? this.ensureDbPassword(app)
: '',
dbVersion: app.dbVersion || '',
dbStorageSize: app.dbStorageSize || '1Gi',
dbCpuRequest: this.resolveDatabaseResources(app).cpuRequest,
dbCpuLimit: this.resolveDatabaseResources(app).cpuLimit,
dbMemoryRequest: this.resolveDatabaseResources(app).memoryRequest,
dbMemoryLimit: this.resolveDatabaseResources(app).memoryLimit,
appStorageSize: app.appStorageSize || '2Gi',
enableRedis: false,
redisVersion: app.redisVersion || '7.2',
enableRabbitmq: false,
rabbitmqVersion: app.rabbitmqVersion || '3.13',
enableElasticsearch: false,
elasticsearchVersion: app.elasticsearchVersion || '8.12',
logPaths: [],
ownerId: app.userId,
applicationId: app.id,
};
const manifests: Record<string, any> = {};
await this.ensureNamespace(coreApi, namespace);
switch (app.productType) {
case ProductType.MANAGED_DATABASE:
context.databaseType = app.databaseType;
manifests.database = await this.deployDatabase(coreApi, appsApi, context);
break;
case ProductType.MANAGED_REDIS:
context.enableRedis = true;
await this.deployRedis(coreApi, appsApi, context);
manifests.redis = true;
break;
case ProductType.MANAGED_RABBITMQ:
context.enableRabbitmq = true;
await this.deployRabbitmq(coreApi, appsApi, context);
manifests.rabbitmq = true;
break;
default:
throw new BadRequestException(`Unsupported managed product type: ${app.productType}`);
}
this.logger.log(`Provisioned managed service ${app.name} in ${namespace} via K8s API`);
return manifests;
}
private async deployViaK8sApi(app: Application, imageUri: string, previewNumber?: string | null): Promise<Record<string, any>> {
if (isManagedProductType(app.productType)) {
return this.deployManagedViaK8sApi(app);
}
const { coreApi, appsApi, networkingApi } = await this.k8sClientService.getK8sClient(app.clusterId);
const kubeconfig = await this.k8sClientService.getKubeconfig(app.clusterId);
await this.ensurePlatformStorageClass(kubeconfig);
const domain = this.configService.get('platform.domain');
const context: ManifestContext = {
appName: app.name,
namespace: this.getUserNamespace(app.userId),
image: imageUri,
port: app.port,
replicas: app.replicas,
cpuRequest: app.cpuRequest,
cpuLimit: app.cpuLimit,
memoryRequest: app.memoryRequest,
memoryLimit: app.memoryLimit,
envVars: this.resolveEnvVars(app),
runtime: app.runtime,
databaseType: app.databaseType,
domain: domain,
subdomain: app.subdomain || app.name,
dbUsername: app.dbUsername || 'appuser',
dbPassword:
app.databaseType && app.databaseType !== DatabaseType.NONE
? this.ensureDbPassword(app)
: '',
dbVersion: app.dbVersion || '',
dbStorageSize: app.dbStorageSize || '1Gi',
dbCpuRequest: this.resolveDatabaseResources(app).cpuRequest,
dbCpuLimit: this.resolveDatabaseResources(app).cpuLimit,
dbMemoryRequest: this.resolveDatabaseResources(app).memoryRequest,
dbMemoryLimit: this.resolveDatabaseResources(app).memoryLimit,
appStorageSize: app.appStorageSize || '2Gi',
enableRedis: app.enableRedis || false,
redisVersion: app.redisVersion || '7.2',
enableRabbitmq: app.enableRabbitmq || false,
rabbitmqVersion: app.rabbitmqVersion || '3.13',
enableElasticsearch: app.enableElasticsearch || false,
elasticsearchVersion: app.elasticsearchVersion || '8.12',
logPaths: app.logPaths || [],
ownerId: app.userId,
applicationId: app.id,
};
const manifests: Record<string, any> = {};
try {
// 1. Ensure namespace exists
await this.ensureNamespace(coreApi, context.namespace);
// 1b. Image pull secret for in-cluster registry
await this.registryService.ensureRegistryPullSecret(coreApi, context.namespace);
// 2. Create/Update secrets for env vars
if (Object.keys(context.envVars).length > 0) {
manifests.secret = await this.applySecret(coreApi, context);
}
// 3. Deploy database if needed
if (context.databaseType !== DatabaseType.NONE) {
manifests.database = await this.deployDatabase(coreApi, appsApi, context);
}
// 3.5 Deploy optional services
if (context.enableRedis) {
await this.deployRedis(coreApi, appsApi, context);
manifests.redis = true;
}
if (context.enableRabbitmq) {
await this.deployRabbitmq(coreApi, appsApi, context);
manifests.rabbitmq = true;
}
// 3.7 Logging: credentials secret + Fluent Bit config
if (context.enableElasticsearch) {
await this.ensureElasticsearchCredentialsSecret(coreApi, context.namespace);
await this.createFluentBitConfigMap(coreApi, context);
manifests.fluentBitConfig = true;
}
// 3.8 Create app storage PVC (for all app types)
manifests.appStoragePvc = await this.applyAppStoragePvc(coreApi, context);
// 4. Create Deployment
manifests.deployment = await this.applyDeployment(appsApi, context);
// 5. Create Service
manifests.service = await this.applyService(coreApi, context);
// 6. Create Ingress
const customDomain = app.customDomain && app.customDomainStatus === CustomDomainStatus.VERIFIED ? app.customDomain : undefined;
manifests.ingress = await this.applyIngress(networkingApi, context, customDomain, previewNumber);
this.logger.log(`Successfully deployed ${app.name} to namespace ${context.namespace} via K8s API`);
} catch (error: any) {
this.logger.error(`Failed to deploy ${app.name} via K8s API:`, error.body || error.message);
throw error;
}
return manifests;
}
// ── Private K8s resource methods ──────────────────────────────────
private async ensureNamespace(coreApi: k8s.CoreV1Api, namespace: string): Promise<void> {
try {
await coreApi.readNamespace({ name: namespace });
} catch {
await coreApi.createNamespace({
body: { metadata: { name: namespace } },
});
this.logger.log(`Created namespace: ${namespace}`);
}
}
private async applySecret(coreApi: k8s.CoreV1Api, ctx: ManifestContext): Promise<any> {
const secretData: Record<string, string> = {};
for (const [key, value] of Object.entries(ctx.envVars)) {
secretData[key] = Buffer.from(String(value)).toString('base64');
}
const secret = {
apiVersion: 'v1',
kind: 'Secret',
metadata: {
name: `${ctx.appName}-env`,
namespace: ctx.namespace,
},
data: secretData,
};
try {
await coreApi.replaceNamespacedSecret({
name: `${ctx.appName}-env`,
namespace: ctx.namespace,
body: secret,
});
} catch {
await coreApi.createNamespacedSecret({
namespace: ctx.namespace,
body: secret,
});
}
return secret;
}
private async applyDeployment(appsApi: k8s.AppsV1Api, ctx: ManifestContext): Promise<any> {
const envFrom: any[] = [];
if (Object.keys(ctx.envVars).length > 0) {
envFrom.push({ secretRef: { name: `${ctx.appName}-env` } });
}
// Database connection env vars
const extraEnv: any[] = [];
if (ctx.databaseType === DatabaseType.POSTGRESQL) {
extraEnv.push(
{ name: 'DB_HOST', value: `${ctx.appName}-db` },
{ name: 'DB_PORT', value: '5432' },
{ name: 'DB_NAME', value: ctx.appName.replace(/-/g, '_') },
{
name: 'DB_USER',
valueFrom: {
secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'username' },
},
},
{
name: 'DB_PASSWORD',
valueFrom: {
secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'password' },
},
},
{
name: 'DATABASE_URL',
value: `postgresql://$(DB_USER):$(DB_PASSWORD)@${ctx.appName}-db:5432/${ctx.appName.replace(/-/g, '_')}`,
},
);
} else if (ctx.databaseType === DatabaseType.MYSQL || ctx.databaseType === DatabaseType.MARIADB) {
extraEnv.push(
{ name: 'DB_HOST', value: `${ctx.appName}-db` },
{ name: 'DB_PORT', value: '3306' },
{ name: 'DB_NAME', value: ctx.appName.replace(/-/g, '_') },
{
name: 'DB_USER',
valueFrom: {
secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'username' },
},
},
{
name: 'DB_PASSWORD',
valueFrom: {
secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'password' },
},
},
{
name: 'DATABASE_URL',
value: `mysql://$(DB_USER):$(DB_PASSWORD)@${ctx.appName}-db:3306/${ctx.appName.replace(/-/g, '_')}`,
},
);
} else if (ctx.databaseType === DatabaseType.MONGODB) {
extraEnv.push(
{ name: 'DB_HOST', value: `${ctx.appName}-db` },
{ name: 'DB_PORT', value: '27017' },
{ name: 'DB_NAME', value: ctx.appName.replace(/-/g, '_') },
{
name: 'DB_USER',
valueFrom: {
secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'username' },
},
},
{
name: 'DB_PASSWORD',
valueFrom: {
secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'password' },
},
},
{
name: 'MONGODB_URI',
value: `mongodb://$(DB_USER):$(DB_PASSWORD)@${ctx.appName}-db:27017/${ctx.appName.replace(/-/g, '_')}?authSource=admin`,
},
{
name: 'DATABASE_URL',
value: `mongodb://$(DB_USER):$(DB_PASSWORD)@${ctx.appName}-db:27017/${ctx.appName.replace(/-/g, '_')}?authSource=admin`,
},
);
}
if ((ctx.runtime === AppRuntime.LARAVEL || ctx.runtime === AppRuntime.PHP) && ctx.databaseType !== DatabaseType.NONE) {
const dbConnection = ctx.databaseType === DatabaseType.POSTGRESQL ? 'pgsql' : ctx.databaseType === DatabaseType.MONGODB ? 'mongodb' : 'mysql';
extraEnv.push(
{ name: 'DB_CONNECTION', value: dbConnection },
{ name: 'DB_DATABASE', value: ctx.appName.replace(/-/g, '_') },
{
name: 'DB_USERNAME',
valueFrom: {
secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'username' },
},
},
);
}
// Redis connection env vars
if (ctx.enableRedis) {
const redisName = `${ctx.appName}-redis`;
extraEnv.push(
{ name: 'REDIS_HOST', value: redisName },
{ name: 'REDIS_PORT', value: '6379' },
{
name: 'REDIS_PASSWORD',
valueFrom: {
secretKeyRef: { name: `${redisName}-secret`, key: 'password' },
},
},
{
name: 'REDIS_URL',
value: `redis://:$(REDIS_PASSWORD)@${redisName}:6379`,
},
);
}
// RabbitMQ connection env vars
if (ctx.enableRabbitmq) {
const rabbitName = `${ctx.appName}-rabbitmq`;
extraEnv.push(
{ name: 'RABBITMQ_HOST', value: rabbitName },
{ name: 'RABBITMQ_PORT', value: '5672' },
{ name: 'RABBITMQ_MANAGEMENT_PORT', value: '15672' },
{
name: 'RABBITMQ_USER',
valueFrom: {
secretKeyRef: { name: `${rabbitName}-secret`, key: 'username' },
},
},
{
name: 'RABBITMQ_PASSWORD',
valueFrom: {
secretKeyRef: { name: `${rabbitName}-secret`, key: 'password' },
},
},
{
name: 'AMQP_URL',
value: `amqp://$(RABBITMQ_USER):$(RABBITMQ_PASSWORD)@${rabbitName}:5672`,
},
);
}
// WordPress-specific env vars
if (ctx.runtime === AppRuntime.WORDPRESS && ctx.databaseType !== DatabaseType.NONE) {
const dbPort = ctx.databaseType === DatabaseType.POSTGRESQL ? '5432' : '3306';
extraEnv.push(
{ name: 'WORDPRESS_DB_HOST', value: `${ctx.appName}-db:${dbPort}` },
{ name: 'WORDPRESS_DB_NAME', value: ctx.appName.replace(/-/g, '_') },
{
name: 'WORDPRESS_DB_USER',
valueFrom: {
secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'username' },
},
},
{
name: 'WORDPRESS_DB_PASSWORD',
valueFrom: {
secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'password' },
},
},
{ name: 'WORDPRESS_TABLE_PREFIX', value: 'wp_' },
);
}
const deployment: k8s.V1Deployment = {
apiVersion: 'apps/v1',
kind: 'Deployment',
metadata: {
name: ctx.appName,
namespace: ctx.namespace,
labels: { app: ctx.appName, runtime: ctx.runtime },
annotations: {
'kubernetes.io/change-cause': `Deploy ${ctx.image} at ${new Date().toISOString()}`,
},
},
spec: {
revisionHistoryLimit: 10,
replicas: ctx.replicas,
selector: { matchLabels: { app: ctx.appName } },
template: {
metadata: { labels: { app: ctx.appName, runtime: ctx.runtime } },
spec: {
imagePullSecrets: [{ name: 'registry-pull-secret' }],
containers: this.buildContainersSpec(ctx, envFrom, extraEnv),
volumes: this.buildVolumesSpec(ctx),
},
},
},
};
try {
await appsApi.replaceNamespacedDeployment({
name: ctx.appName,
namespace: ctx.namespace,
body: deployment,
});
} catch {
await appsApi.createNamespacedDeployment({
namespace: ctx.namespace,
body: deployment,
});
}
return deployment;
}
private async applyAppStoragePvc(coreApi: k8s.CoreV1Api, ctx: ManifestContext): Promise<any> {
const pvcName = `${ctx.appName}-storage`;
const pvc = {
apiVersion: 'v1',
kind: 'PersistentVolumeClaim',
metadata: {
name: pvcName,
namespace: ctx.namespace,
labels: { app: ctx.appName, runtime: ctx.runtime },
},
spec: {
accessModes: ['ReadWriteOnce'],
resources: { requests: { storage: ctx.appStorageSize || '2Gi' } },
...(this.configService.get<string>('platform.storageClass')
? {
storageClassName: this.configService.get<string>('platform.storageClass'),
}
: {}),
},
};
try {
await coreApi.readNamespacedPersistentVolumeClaim({
name: pvcName,
namespace: ctx.namespace,
});
this.logger.log(`PVC ${pvcName} already exists, skipping`);
} catch {
await coreApi.createNamespacedPersistentVolumeClaim({
namespace: ctx.namespace,
body: pvc,
});
this.logger.log(`Created app storage PVC: ${pvcName}`);
}
return pvc;
}
/**
* Get the storage mount path based on runtime type
*/
private getStorageMountPath(runtime: string): string {
switch (runtime) {
case AppRuntime.WORDPRESS:
return '/var/www/html/wp-content';
case AppRuntime.LARAVEL:
case AppRuntime.PHP:
return '/var/www/html/storage';
case AppRuntime.DJANGO:
return '/app/media';
case AppRuntime.PYTHON:
case AppRuntime.GO:
case AppRuntime.DOTNET:
case AppRuntime.NODEJS:
default:
return '/app/data';
}
}
/**
* Build containers spec for the app deployment, including optional Fluent Bit sidecar
*/
private buildContainersSpec(ctx: ManifestContext, envFrom: any[], extraEnv: any[]): any[] {
const containers: any[] = [];
// Main application container
const appContainer: any = {
name: ctx.appName,
// Image tags are unique per build (name:timestamp) and immutable, so
// IfNotPresent is correct and avoids re-pulling on every restart/scale-up.
image: ctx.image,
imagePullPolicy: 'IfNotPresent',
ports: [{ containerPort: ctx.port }],
envFrom,
env: extraEnv,
resources: {
requests: { cpu: ctx.cpuRequest, memory: ctx.memoryRequest },
limits: { cpu: ctx.cpuLimit, memory: ctx.memoryLimit },
},
readinessProbe: {
tcpSocket: { port: ctx.port as any },
initialDelaySeconds: 5,
periodSeconds: 5,
},
livenessProbe: {
tcpSocket: { port: ctx.port as any },
initialDelaySeconds: 15,
periodSeconds: 10,
failureThreshold: 5,
},
volumeMounts: [
{
name: 'app-storage',
mountPath: this.getStorageMountPath(ctx.runtime),
},
],
};
// Add log volume mount if Elasticsearch is enabled
if (ctx.enableElasticsearch) {
appContainer.volumeMounts.push({
name: 'app-logs',
mountPath: '/var/log/app',
});
this.applyLoggingCommandWrapper(appContainer, ctx.runtime);
}
containers.push(appContainer);
// Add Fluent Bit sidecar for log collection if Elasticsearch is enabled
if (ctx.enableElasticsearch) {
containers.push({
name: 'fluent-bit',
image: 'fluent/fluent-bit:2.2',
resources: {
requests: { cpu: '10m', memory: '32Mi' },
limits: { cpu: '50m', memory: '64Mi' },
},
volumeMounts: [
{ name: 'app-logs', mountPath: '/var/log/app', readOnly: true },
{ name: 'fluent-bit-config', mountPath: '/fluent-bit/etc' },
],
env: [
{ name: 'APP_NAME', value: ctx.appName },
{ name: 'APP_NAMESPACE', value: ctx.namespace },
{ name: 'ES_HOST', value: 'elasticsearch.logging.svc.cluster.local' },
{ name: 'ES_PORT', value: '9200' },
{
name: 'ES_PASSWORD',
valueFrom: {
secretKeyRef: {
name: 'elasticsearch-credentials',
key: 'ELASTIC_PASSWORD',
optional: true,
},
},
},
],
});
}
return containers;
}
/**
* Build volumes spec for the app deployment
*/
private buildVolumesSpec(ctx: ManifestContext): any[] {
const volumes: any[] = [
{
name: 'app-storage',
persistentVolumeClaim: { claimName: `${ctx.appName}-storage` },
},
];
if (ctx.enableElasticsearch) {
// Shared log volume between app and fluent-bit
volumes.push({
name: 'app-logs',
emptyDir: {},
});
// Fluent Bit config as ConfigMap
volumes.push({
name: 'fluent-bit-config',
configMap: { name: `${ctx.appName}-fluent-bit-config` },
});
}
return volumes;
}
/**
* Get default log paths based on runtime type
*/
private getDefaultLogPaths(runtime: string): string[] {
switch (runtime) {
case AppRuntime.WORDPRESS:
return ['/var/www/html/wp-content/debug.log', '/var/log/app/*.log'];
case AppRuntime.LARAVEL:
return ['/var/www/html/storage/logs/*.log', '/var/log/app/*.log'];
case AppRuntime.PHP:
return ['/var/www/html/storage/logs/*.log', '/var/log/php/*.log', '/var/log/app/*.log'];
default:
// Node/Go/Python/.NET log to stdout — captured into /var/log/app/app.log at runtime
return ['/var/log/app/*.log'];
}
}
/**
* Redirect stdout/stderr into the shared log volume so Fluent Bit can tail them.
*/
private applyLoggingCommandWrapper(container: any, runtime: string): void {
const startCmd = this.getRuntimeStartCommand(runtime);
if (!startCmd) return;
container.command = ['sh', '-c'];
container.args = [`mkdir -p /var/log/app && (${startCmd}) >> /var/log/app/app.log 2>&1`];
}
/** Shell command that mirrors CloudHost-generated image ENTRYPOINT/CMD per runtime. */
private getRuntimeStartCommand(runtime: string): string | null {
switch (runtime) {
case AppRuntime.NODEJS:
return 'if [ -f /app/.mode ] && [ "$(cat /app/.mode)" = "standalone" ] && [ -f server.js ]; ' + 'then node server.js; else npm start; fi';
case AppRuntime.GO:
return './main';
case AppRuntime.PYTHON:
return (
'if [ -f main.py ]; then ' +
'if grep -qi fastapi main.py; then exec uvicorn main:app --host 0.0.0.0 --port ${PORT:-3000}; ' +
'elif grep -qi flask main.py; then exec gunicorn -w 4 -b 0.0.0.0:${PORT:-3000} main:app; ' +
'else exec python main.py; fi; ' +
'elif [ -f app.py ]; then ' +
'if grep -qi fastapi app.py; then exec uvicorn app:app --host 0.0.0.0 --port ${PORT:-3000}; ' +
'elif grep -qi flask app.py; then exec gunicorn -w 4 -b 0.0.0.0:${PORT:-3000} app:app; ' +
'else exec python app.py; fi; ' +
'else exec gunicorn -w 4 -b 0.0.0.0:${PORT:-3000} app:app; fi'
);
case AppRuntime.DJANGO:
return 'python manage.py runserver 0.0.0.0:${PORT:-8000}';
case AppRuntime.DOTNET:
return 'DLL=$(find . -maxdepth 1 -name "*.dll" ! -name "*.deps.dll" ! -name "*.runtimeconfig.dll" | head -1) ' + '&& dotnet "$DLL"';
case AppRuntime.WORDPRESS:
case AppRuntime.LARAVEL:
case AppRuntime.PHP:
return null;
default:
return null;
}
}
/** Replicate logging credentials into the app namespace for Fluent Bit sidecars. */
private async ensureElasticsearchCredentialsSecret(coreApi: k8s.CoreV1Api, namespace: string): Promise<void> {
const name = 'elasticsearch-credentials';
const stringData: { [key: string]: string } = {
ELASTIC_PASSWORD: this.configService.get<string>('elasticsearch.password') || '',
FLUENTBIT_PASSWORD: this.configService.get<string>('elasticsearch.fluentbitPassword') || '',
KIBANA_SYSTEM_PASSWORD: this.configService.get<string>('elasticsearch.kibanaPassword') || '',
};
try {
await coreApi.readNamespacedSecret({ name, namespace });
await coreApi.replaceNamespacedSecret({
name,
namespace,
body: {
apiVersion: 'v1',
kind: 'Secret',
metadata: { name, namespace },
type: 'Opaque',
stringData,
},
});
} catch (err: any) {
if (err.code === 404 || err.body?.code === 404) {
await coreApi.createNamespacedSecret({
namespace,
body: {
apiVersion: 'v1',
kind: 'Secret',
metadata: { name, namespace },
type: 'Opaque',
stringData,
},
});
this.logger.log(`Created ${name} secret in ${namespace}`);
} else {
throw err;
}
}
}
/**
* Build Fluent Bit configuration for log collection
*/
private buildFluentBitConfig(appName: string, namespace: string, runtime: string, ownerId: string, applicationId: string, workload: string, customLogPaths?: string[]): string {
const logPaths = customLogPaths && customLogPaths.length > 0 ? customLogPaths : this.getDefaultLogPaths(runtime);
const pathsStr = logPaths.join(',');
return `
[SERVICE]
Flush 5
Daemon Off
Log_Level info
Parsers_File /fluent-bit/etc/parsers.conf
[INPUT]
Name tail
Path ${pathsStr}
Tag app.${appName}
Refresh_Interval 5
Mem_Buf_Limit 5MB
Skip_Long_Lines On
[FILTER]
Name record_modifier
Match *
Record app ${appName}
Record applicationName ${appName}
Record namespace ${namespace}
Record runtime ${runtime}
Record ownerId ${ownerId}
Record applicationId ${applicationId}
Record workload ${workload}
[FILTER]
Name parser
Match *
Key_Name log
Parser json
Reserve_Data On
Preserve_Key On
[OUTPUT]
Name es
Match *
Host \${ES_HOST}
Port \${ES_PORT}
HTTP_User elastic
HTTP_Passwd \${ES_PASSWORD}
Index logs-${namespace}-${appName}
Logstash_Format On
Logstash_Prefix logs-${namespace}
Suppress_Type_Name On
tls Off
Retry_Limit 3
`;
}
/**
* Create Fluent Bit ConfigMap for an app
*/
private async createFluentBitConfigMap(coreApi: k8s.CoreV1Api, ctx: ManifestContext): Promise<void> {
if (!ctx.enableElasticsearch) return;
const customLogPaths = ctx.logPaths && ctx.logPaths.length > 0 ? ctx.logPaths : undefined;
const configMap = {
apiVersion: 'v1',
kind: 'ConfigMap',
metadata: {
name: `${ctx.appName}-fluent-bit-config`,
namespace: ctx.namespace,
labels: { app: ctx.appName },
},
data: {
'fluent-bit.conf': this.buildFluentBitConfig(ctx.appName, ctx.namespace, ctx.runtime, ctx.ownerId, ctx.applicationId, 'app', customLogPaths),
'parsers.conf': `
[PARSER]
Name json
Format json
Time_Key time
Time_Format %Y-%m-%dT%H:%M:%S.%L
[PARSER]
Name docker
Format json
Time_Key time
Time_Format %Y-%m-%dT%H:%M:%S.%L
`,
},
};
try {
await coreApi.replaceNamespacedConfigMap({
name: `${ctx.appName}-fluent-bit-config`,
namespace: ctx.namespace,
body: configMap,
});
} catch {
await coreApi.createNamespacedConfigMap({
namespace: ctx.namespace,
body: configMap,
});
}
this.logger.log(`Created Fluent Bit ConfigMap for ${ctx.appName}`);
}
private buildWorkloadFluentBitConfig(ctx: ManifestContext, workload: 'redis' | 'rabbitmq' | 'database', resourceName: string): string {
const logGlob = `/var/log/pods/*${resourceName}*/*/*.log`;
return `
[SERVICE]
Flush 5
Daemon Off
Log_Level info
Parsers_File /fluent-bit/etc/parsers.conf
[INPUT]
Name tail
Path ${logGlob}
Tag ${workload}.${resourceName}
Refresh_Interval 5
Mem_Buf_Limit 5MB
Skip_Long_Lines On
Parser docker
[FILTER]
Name record_modifier
Match *
Record app ${ctx.appName}
Record applicationName ${ctx.appName}
Record namespace ${ctx.namespace}
Record ownerId ${ctx.ownerId}
Record applicationId ${ctx.applicationId}
Record workload ${workload}
[OUTPUT]
Name es
Match *
Host \${ES_HOST}
Port \${ES_PORT}
HTTP_User elastic
HTTP_Passwd \${ES_PASSWORD}
Index logs-${ctx.namespace}-${ctx.appName}
Logstash_Format On
Logstash_Prefix logs-${ctx.namespace}
Suppress_Type_Name On
tls Off
Retry_Limit 3
`;
}
private async attachWorkloadLogShipper(
coreApi: k8s.CoreV1Api,
ctx: ManifestContext,
workload: 'redis' | 'rabbitmq' | 'database',
resourceName: string,
): Promise<{ containers: k8s.V1Container[]; volumes: k8s.V1Volume[] }> {
if (!ctx.enableElasticsearch) {
return { containers: [], volumes: [] };
}
const configMapName = `${resourceName}-log-shipper-config`;
const configMap = {
apiVersion: 'v1',
kind: 'ConfigMap',
metadata: {
name: configMapName,
namespace: ctx.namespace,
labels: { app: resourceName, 'cloudhost.io/log-shipper': 'true' },
},
data: {
'fluent-bit.conf': this.buildWorkloadFluentBitConfig(ctx, workload, resourceName),
'parsers.conf': `
[PARSER]
Name docker
Format json
Time_Key time
Time_Format %Y-%m-%dT%H:%M:%S.%L
`,
},
};
try {
await coreApi.replaceNamespacedConfigMap({
name: configMapName,
namespace: ctx.namespace,
body: configMap,
});
} catch {
await coreApi.createNamespacedConfigMap({
namespace: ctx.namespace,
body: configMap,
});
}
return {
containers: [
{
name: 'log-shipper',
image: 'fluent/fluent-bit:2.2',
resources: {
requests: { cpu: '10m', memory: '32Mi' },
limits: { cpu: '50m', memory: '64Mi' },
},
volumeMounts: [
{ name: 'varlogpods', mountPath: '/var/log/pods', readOnly: true },
{ name: 'log-shipper-config', mountPath: '/fluent-bit/etc' },
],
env: [
{
name: 'ES_HOST',
value: 'elasticsearch.logging.svc.cluster.local',
},
{ name: 'ES_PORT', value: '9200' },
{
name: 'ES_PASSWORD',
valueFrom: {
secretKeyRef: {
name: 'elasticsearch-credentials',
key: 'ELASTIC_PASSWORD',
optional: true,
},
},
},
],
},
],
volumes: [
{
name: 'varlogpods',
hostPath: { path: '/var/log/pods', type: 'Directory' },
},
{ name: 'log-shipper-config', configMap: { name: configMapName } },
],
};
}
private async applyService(coreApi: k8s.CoreV1Api, ctx: ManifestContext): Promise<any> {
const service: k8s.V1Service = {
apiVersion: 'v1',
kind: 'Service',
metadata: { name: ctx.appName, namespace: ctx.namespace },
spec: {
selector: { app: ctx.appName },
ports: [{ port: 80, targetPort: ctx.port as any, protocol: 'TCP' }],
type: 'ClusterIP',
},
};
try {
await coreApi.replaceNamespacedService({
name: ctx.appName,
namespace: ctx.namespace,
body: service,
});
} catch {
await coreApi.createNamespacedService({
namespace: ctx.namespace,
body: service,
});
}
return service;
}
private async applyIngress(networkingApi: k8s.NetworkingV1Api, ctx: ManifestContext, customDomain?: string, previewNumber?: string | null): Promise<any> {
const host = `${ctx.subdomain}.${ctx.domain}`;
const rules: k8s.V1IngressRule[] = [
{
host,
http: {
paths: [
{
path: '/',
pathType: 'Prefix',
backend: { service: { name: ctx.appName, port: { number: 80 } } },
},
],
},
},
];
// Only PUBLIC, real-TLD hosts may go into the TLS block. The internal
// `${subdomain}.${domain}` host (e.g. *.apps.cloudhost.local) is not a valid
// public suffix — including it makes Let's Encrypt reject the whole order,
// which would also block the cert for the legitimate preview/custom domains.
const tlsHosts: string[] = [];
if (customDomain) {
rules.push({
host: customDomain,
http: {
paths: [
{
path: '/',
pathType: 'Prefix',
backend: { service: { name: ctx.appName, port: { number: 80 } } },
},
],
},
});
tlsHosts.push(customDomain);
}
const previewRootDomain = this.configService.get<string>('platform.previewRootDomain') || ctx.domain;
const namespacePrefix = userIdSlug(ctx.ownerId);
const previewHost = previewNumber && !customDomain ? `${namespacePrefix}-${previewNumber}.${previewRootDomain}` : '';
if (previewHost) {
rules.push({
host: previewHost,
http: {
paths: [
{
path: '/',
pathType: 'Prefix',
backend: { service: { name: ctx.appName, port: { number: 80 } } },
},
],
},
});
tlsHosts.push(previewHost);
}
const ingressClass = this.configService.get<string>('platform.ingressClass') || 'traefik';
// Request a managed cert only when we actually have a public host to issue for.
const annotations: Record<string, string> = {};
const tls: k8s.V1IngressTLS[] = [];
if (tlsHosts.length > 0) {
annotations['cert-manager.io/cluster-issuer'] = 'letsencrypt-prod';
tls.push({ hosts: tlsHosts, secretName: `${ctx.appName}-tls` });
}
const ingress: k8s.V1Ingress = {
apiVersion: 'networking.k8s.io/v1',
kind: 'Ingress',
metadata: {
name: ctx.appName,
namespace: ctx.namespace,
annotations,
},
spec: {
ingressClassName: ingressClass,
rules,
...(tls.length > 0 ? { tls } : {}),
},
};
try {
await networkingApi.replaceNamespacedIngress({
name: ctx.appName,
namespace: ctx.namespace,
body: ingress,
});
} catch {
await networkingApi.createNamespacedIngress({
namespace: ctx.namespace,
body: ingress,
});
}
return ingress;
}
private async deployDatabase(coreApi: k8s.CoreV1Api, appsApi: k8s.AppsV1Api, ctx: ManifestContext): Promise<any> {
const dbName = `${ctx.appName}-db`;
// Create DB secret
await this.createDbSecret(coreApi, ctx.namespace, ctx.appName, ctx.dbPassword, ctx.dbUsername);
// Create PVC for DB
await this.createPVC(coreApi, ctx.namespace, dbName, ctx.dbStorageSize);
// Deploy database based on type
const dbType = ctx.databaseType;
let image: string;
let port: number;
let dataPath: string;
let envVars: any[];
let readinessProbe: any;
let livenessProbe: any;
switch (dbType) {
case DatabaseType.POSTGRESQL: {
const pgVer = ctx.dbVersion || '16';
const pgDatabase = ctx.appName.replace(/-/g, '_');
image = `postgres:${pgVer}-alpine`;
port = 5432;
dataPath = '/var/lib/postgresql/data';
envVars = [
{ name: 'PGDATA', value: '/var/lib/postgresql/data/pgdata' },
{ name: 'POSTGRES_DB', value: pgDatabase },
{
name: 'POSTGRES_USER',
valueFrom: {
secretKeyRef: {
name: `${ctx.appName}-db-secret`,
key: 'username',
},
},
},
{
name: 'POSTGRES_PASSWORD',
valueFrom: {
secretKeyRef: {
name: `${ctx.appName}-db-secret`,
key: 'password',
},
},
},
];
const pgReady = ['pg_isready', '-U', ctx.dbUsername, '-d', pgDatabase];
readinessProbe = {
exec: { command: pgReady },
initialDelaySeconds: 10,
periodSeconds: 5,
failureThreshold: 6,
};
livenessProbe = {
exec: { command: pgReady },
initialDelaySeconds: 30,
periodSeconds: 10,
failureThreshold: 5,
};
break;
}
case DatabaseType.MYSQL:
const mysqlVer = ctx.dbVersion || '8.0';
image = `mysql:${mysqlVer}`;
port = 3306;
dataPath = '/var/lib/mysql';
envVars = [
{ name: 'MYSQL_DATABASE', value: ctx.appName.replace(/-/g, '_') },
{
name: 'MYSQL_USER',
valueFrom: {
secretKeyRef: {
name: `${ctx.appName}-db-secret`,
key: 'username',
},
},
},
{
name: 'MYSQL_PASSWORD',
valueFrom: {
secretKeyRef: {
name: `${ctx.appName}-db-secret`,
key: 'password',
},
},
},
{
name: 'MYSQL_ROOT_PASSWORD',
valueFrom: {
secretKeyRef: {
name: `${ctx.appName}-db-secret`,
key: 'password',
},
},
},
];
readinessProbe = {
exec: { command: ['mysqladmin', 'ping', '-h', '127.0.0.1'] },
initialDelaySeconds: 10,
periodSeconds: 5,
failureThreshold: 6,
};
livenessProbe = {
exec: { command: ['mysqladmin', 'ping', '-h', '127.0.0.1'] },
initialDelaySeconds: 30,
periodSeconds: 10,
failureThreshold: 5,
};
break;
case DatabaseType.MARIADB:
const mariaVer = ctx.dbVersion || '11.4';
image = `mariadb:${mariaVer}`;
port = 3306;
dataPath = '/var/lib/mysql';
envVars = [
{ name: 'MARIADB_DATABASE', value: ctx.appName.replace(/-/g, '_') },
{
name: 'MARIADB_USER',
valueFrom: {
secretKeyRef: {
name: `${ctx.appName}-db-secret`,
key: 'username',
},
},
},
{
name: 'MARIADB_PASSWORD',
valueFrom: {
secretKeyRef: {
name: `${ctx.appName}-db-secret`,
key: 'password',
},
},
},
{
name: 'MARIADB_ROOT_PASSWORD',
valueFrom: {
secretKeyRef: {
name: `${ctx.appName}-db-secret`,
key: 'password',
},
},
},
];
readinessProbe = {
exec: {
command: ['healthcheck.sh', '--connect', '--innodb_initialized'],
},
initialDelaySeconds: 10,
periodSeconds: 5,
failureThreshold: 6,
};
livenessProbe = {
exec: {
command: ['healthcheck.sh', '--connect', '--innodb_initialized'],
},
initialDelaySeconds: 30,
periodSeconds: 10,
failureThreshold: 5,
};
break;
case DatabaseType.MONGODB:
const mongoVer = ctx.dbVersion || '7.0';
image = `mongo:${mongoVer}`;
port = 27017;
dataPath = '/data/db';
envVars = [
{
name: 'MONGO_INITDB_DATABASE',
value: ctx.appName.replace(/-/g, '_'),
},
{
name: 'MONGO_INITDB_ROOT_USERNAME',
valueFrom: {
secretKeyRef: {
name: `${ctx.appName}-db-secret`,
key: 'username',
},
},
},
{
name: 'MONGO_INITDB_ROOT_PASSWORD',
valueFrom: {
secretKeyRef: {
name: `${ctx.appName}-db-secret`,
key: 'password',
},
},
},
];
readinessProbe = {
exec: { command: ['mongosh', '--eval', 'db.adminCommand("ping")'] },
initialDelaySeconds: 10,
periodSeconds: 5,
failureThreshold: 6,
};
livenessProbe = {
exec: { command: ['mongosh', '--eval', 'db.adminCommand("ping")'] },
initialDelaySeconds: 30,
periodSeconds: 10,
failureThreshold: 5,
};
break;
default:
throw new Error(`Unsupported database type: ${dbType}`);
}
const dbLogShipper = await this.attachWorkloadLogShipper(coreApi, ctx, 'database', dbName);
const dbDeployment: k8s.V1Deployment = {
apiVersion: 'apps/v1',
kind: 'Deployment',
metadata: {
name: dbName,
namespace: ctx.namespace,
labels: { app: dbName },
},
spec: {
replicas: 1,
selector: { matchLabels: { app: dbName } },
template: {
metadata: { labels: { app: dbName } },
spec: {
containers: [
{
name: dbName,
image,
ports: [{ containerPort: port }],
env: envVars,
volumeMounts: [{ name: 'db-storage', mountPath: dataPath }],
resources: {
requests: { cpu: ctx.dbCpuRequest, memory: ctx.dbMemoryRequest },
limits: { cpu: ctx.dbCpuLimit, memory: ctx.dbMemoryLimit },
},
readinessProbe,
livenessProbe,
},
...dbLogShipper.containers,
],
volumes: [
{
name: 'db-storage',
persistentVolumeClaim: { claimName: dbName },
},
...dbLogShipper.volumes,
],
},
},
},
};
try {
await appsApi.replaceNamespacedDeployment({
name: dbName,
namespace: ctx.namespace,
body: dbDeployment,
});
} catch {
await appsApi.createNamespacedDeployment({
namespace: ctx.namespace,
body: dbDeployment,
});
}
// Create DB Service
const dbService: k8s.V1Service = {
apiVersion: 'v1',
kind: 'Service',
metadata: { name: dbName, namespace: ctx.namespace },
spec: {
selector: { app: dbName },
ports: [{ port, targetPort: port as any, protocol: 'TCP' }],
type: 'ClusterIP',
},
};
try {
await coreApi.replaceNamespacedService({
name: dbName,
namespace: ctx.namespace,
body: dbService,
});
} catch {
await coreApi.createNamespacedService({
namespace: ctx.namespace,
body: dbService,
});
}
return { deployment: dbDeployment, service: dbService };
}
private async createDbSecret(coreApi: k8s.CoreV1Api, namespace: string, appName: string, password: string, username: string = 'appuser'): Promise<void> {
const secret = {
apiVersion: 'v1',
kind: 'Secret',
metadata: { name: `${appName}-db-secret`, namespace },
data: {
username: Buffer.from(username).toString('base64'),
password: Buffer.from(password).toString('base64'),
},
};
try {
await coreApi.replaceNamespacedSecret({
name: `${appName}-db-secret`,
namespace,
body: secret,
});
} catch {
await coreApi.createNamespacedSecret({ namespace, body: secret });
}
}
private async createPVC(coreApi: k8s.CoreV1Api, namespace: string, name: string, size: string): Promise<void> {
const storageClass = this.configService.get<string>('platform.storageClass');
const pvc: k8s.V1PersistentVolumeClaim = {
apiVersion: 'v1',
kind: 'PersistentVolumeClaim',
metadata: { name, namespace },
spec: {
accessModes: ['ReadWriteOnce'],
resources: { requests: { storage: size } },
...(storageClass ? { storageClassName: storageClass } : {}),
},
};
try {
await coreApi.readNamespacedPersistentVolumeClaim({ name, namespace });
// PVC exists, don't recreate
} catch {
await coreApi.createNamespacedPersistentVolumeClaim({
namespace,
body: pvc,
});
}
}
/**
* Deploy Redis for an application
*/
private async deployRedis(coreApi: k8s.CoreV1Api, appsApi: k8s.AppsV1Api, ctx: ManifestContext): Promise<void> {
const redisName = `${ctx.appName}-redis`;
// Create PVC for Redis persistence
await this.createPVC(coreApi, ctx.namespace, `${redisName}-data`, '1Gi');
// Only create Redis password secret if it doesn't already exist
try {
await coreApi.readNamespacedSecret({
name: `${redisName}-secret`,
namespace: ctx.namespace,
});
this.logger.log(`Redis secret ${redisName}-secret already exists, skipping`);
} catch {
const redisPassword = this.generatePassword(16);
const redisSecret = {
apiVersion: 'v1',
kind: 'Secret',
metadata: { name: `${redisName}-secret`, namespace: ctx.namespace },
data: {
password: Buffer.from(redisPassword).toString('base64'),
},
};
await coreApi.createNamespacedSecret({
namespace: ctx.namespace,
body: redisSecret,
});
}
const logShipper = await this.attachWorkloadLogShipper(coreApi, ctx, 'redis', redisName);
// Create Redis Deployment
const redisDeployment: k8s.V1Deployment = {
apiVersion: 'apps/v1',
kind: 'Deployment',
metadata: { name: redisName, namespace: ctx.namespace },
spec: {
replicas: 1,
selector: { matchLabels: { app: redisName } },
template: {
metadata: { labels: { app: redisName } },
spec: {
containers: [
{
name: 'redis',
image: `redis:${ctx.redisVersion}-alpine`,
args: ['--requirepass', '$(REDIS_PASSWORD)'],
ports: [{ containerPort: 6379 }],
env: [
{
name: 'REDIS_PASSWORD',
valueFrom: {
secretKeyRef: {
name: `${redisName}-secret`,
key: 'password',
},
},
},
],
volumeMounts: [{ name: 'redis-data', mountPath: '/data' }],
resources: {
requests: { cpu: '50m', memory: '64Mi' },
limits: { cpu: '200m', memory: '256Mi' },
},
readinessProbe: {
exec: { command: ['redis-cli', 'ping'] },
initialDelaySeconds: 5,
periodSeconds: 10,
},
livenessProbe: {
exec: { command: ['redis-cli', 'ping'] },
initialDelaySeconds: 15,
periodSeconds: 20,
},
},
...logShipper.containers,
],
volumes: [
{
name: 'redis-data',
persistentVolumeClaim: { claimName: `${redisName}-data` },
},
...logShipper.volumes,
],
},
},
},
};
try {
await appsApi.replaceNamespacedDeployment({
name: redisName,
namespace: ctx.namespace,
body: redisDeployment,
});
} catch {
await appsApi.createNamespacedDeployment({
namespace: ctx.namespace,
body: redisDeployment,
});
}
// Create Redis Service
const redisService = {
apiVersion: 'v1',
kind: 'Service',
metadata: { name: redisName, namespace: ctx.namespace },
spec: {
selector: { app: redisName },
ports: [{ port: 6379, targetPort: 6379 as any, protocol: 'TCP' }],
type: 'ClusterIP',
},
};
try {
await coreApi.replaceNamespacedService({
name: redisName,
namespace: ctx.namespace,
body: redisService,
});
} catch {
await coreApi.createNamespacedService({
namespace: ctx.namespace,
body: redisService,
});
}
this.logger.log(`Redis deployed for ${ctx.appName}`);
}
/**
* Deploy RabbitMQ for an application
*/
private async deployRabbitmq(coreApi: k8s.CoreV1Api, appsApi: k8s.AppsV1Api, ctx: ManifestContext): Promise<void> {
const rabbitName = `${ctx.appName}-rabbitmq`;
// Create PVC for RabbitMQ persistence
await this.createPVC(coreApi, ctx.namespace, `${rabbitName}-data`, '2Gi');
// Only create RabbitMQ credentials secret if it doesn't already exist
try {
await coreApi.readNamespacedSecret({
name: `${rabbitName}-secret`,
namespace: ctx.namespace,
});
this.logger.log(`RabbitMQ secret ${rabbitName}-secret already exists, skipping`);
} catch {
const rabbitUser = 'appuser';
const rabbitPassword = this.generatePassword(16);
const rabbitSecret = {
apiVersion: 'v1',
kind: 'Secret',
metadata: { name: `${rabbitName}-secret`, namespace: ctx.namespace },
data: {
username: Buffer.from(rabbitUser).toString('base64'),
password: Buffer.from(rabbitPassword).toString('base64'),
},
};
await coreApi.createNamespacedSecret({
namespace: ctx.namespace,
body: rabbitSecret,
});
}
const rabbitLogShipper = await this.attachWorkloadLogShipper(coreApi, ctx, 'rabbitmq', rabbitName);
// Create RabbitMQ Deployment
const rabbitDeployment: k8s.V1Deployment = {
apiVersion: 'apps/v1',
kind: 'Deployment',
metadata: { name: rabbitName, namespace: ctx.namespace },
spec: {
replicas: 1,
selector: { matchLabels: { app: rabbitName } },
template: {
metadata: { labels: { app: rabbitName } },
spec: {
containers: [
{
name: 'rabbitmq',
image: `rabbitmq:${ctx.rabbitmqVersion}-management-alpine`,
ports: [
{ containerPort: 5672, name: 'amqp' },
{ containerPort: 15672, name: 'management' },
],
env: [
{
name: 'RABBITMQ_DEFAULT_USER',
valueFrom: {
secretKeyRef: {
name: `${rabbitName}-secret`,
key: 'username',
},
},
},
{
name: 'RABBITMQ_DEFAULT_PASS',
valueFrom: {
secretKeyRef: {
name: `${rabbitName}-secret`,
key: 'password',
},
},
},
],
volumeMounts: [{ name: 'rabbitmq-data', mountPath: '/var/lib/rabbitmq' }],
resources: {
requests: { cpu: '100m', memory: '256Mi' },
limits: { cpu: '500m', memory: '512Mi' },
},
readinessProbe: {
exec: { command: ['rabbitmq-diagnostics', '-q', 'ping'] },
initialDelaySeconds: 20,
periodSeconds: 10,
timeoutSeconds: 5,
},
livenessProbe: {
exec: { command: ['rabbitmq-diagnostics', '-q', 'status'] },
initialDelaySeconds: 60,
periodSeconds: 30,
timeoutSeconds: 10,
},
},
...rabbitLogShipper.containers,
],
volumes: [
{
name: 'rabbitmq-data',
persistentVolumeClaim: { claimName: `${rabbitName}-data` },
},
...rabbitLogShipper.volumes,
],
},
},
},
};
try {
await appsApi.replaceNamespacedDeployment({
name: rabbitName,
namespace: ctx.namespace,
body: rabbitDeployment,
});
} catch {
await appsApi.createNamespacedDeployment({
namespace: ctx.namespace,
body: rabbitDeployment,
});
}
// Create RabbitMQ Services (AMQP and Management)
const rabbitService = {
apiVersion: 'v1',
kind: 'Service',
metadata: { name: rabbitName, namespace: ctx.namespace },
spec: {
selector: { app: rabbitName },
ports: [
{
port: 5672,
targetPort: 5672 as any,
protocol: 'TCP',
name: 'amqp',
},
{
port: 15672,
targetPort: 15672 as any,
protocol: 'TCP',
name: 'management',
},
],
type: 'ClusterIP',
},
};
try {
await coreApi.replaceNamespacedService({
name: rabbitName,
namespace: ctx.namespace,
body: rabbitService,
});
} catch {
await coreApi.createNamespacedService({
namespace: ctx.namespace,
body: rabbitService,
});
}
this.logger.log(`RabbitMQ deployed for ${ctx.appName}`);
}
private primaryWorkloadLabel(app: Application): string {
if (isManagedProductType(app.productType)) {
switch (app.productType) {
case ProductType.MANAGED_DATABASE:
return `${app.name}-db`;
case ProductType.MANAGED_REDIS:
return `${app.name}-redis`;
case ProductType.MANAGED_RABBITMQ:
return `${app.name}-rabbitmq`;
default:
break;
}
}
return app.name;
}
async getPodLogs(app: Application): Promise<string> {
return this.k8sLifecycleService.getPodLogs(app);
}
async scaleDeployment(app: Application, replicas: number): Promise<void> {
const { appsApi } = await this.k8sClientService.getK8sClient(app.clusterId);
const namespace = this.getUserNamespace(app.userId);
await appsApi.patchNamespacedDeployment({ name: app.name, namespace, body: { spec: { replicas } } }, k8s.setHeaderOptions('Content-Type', 'application/merge-patch+json'));
}
/** All K8s Deployments that belong to an application stack (default replica targets). */
private getApplicationWorkloadDeployments(app: Application): { name: string; runningReplicas: number }[] {
const managed = isManagedProductType(app.productType);
const workloads: { name: string; runningReplicas: number }[] = [];
if (!managed) {
workloads.push({ name: app.name, runningReplicas: app.replicas || 1 });
}
if (app.databaseType && app.databaseType !== DatabaseType.NONE) {
workloads.push({ name: `${app.name}-db`, runningReplicas: 1 });
}
if (app.enableRedis) {
workloads.push({ name: `${app.name}-redis`, runningReplicas: 1 });
}
if (app.enableRabbitmq) {
workloads.push({ name: `${app.name}-rabbitmq`, runningReplicas: 1 });
}
return workloads;
}
private resolveWorkloadReplicas(app: Application): { name: string; runningReplicas: number }[] {
const workloads = this.getApplicationWorkloadDeployments(app);
const saved = app.suspendedReplicas;
if (!saved) return workloads;
return workloads.map((workload) => ({
name: workload.name,
runningReplicas: saved[workload.name] ?? workload.runningReplicas,
}));
}
async captureWorkloadReplicaSnapshot(app: Application): Promise<Record<string, number>> {
const { appsApi } = await this.k8sClientService.getK8sClient(app.clusterId);
const namespace = this.getUserNamespace(app.userId);
const snapshot: Record<string, number> = {};
for (const workload of this.getApplicationWorkloadDeployments(app)) {
try {
const deployment = await appsApi.readNamespacedDeployment({
name: workload.name,
namespace,
});
snapshot[workload.name] = deployment.spec?.replicas ?? workload.runningReplicas;
} catch (e: any) {
if (e?.code === 404 || e?.response?.statusCode === 404) continue;
snapshot[workload.name] = workload.runningReplicas;
}
}
return snapshot;
}
private async patchDeploymentReplicas(appsApi: k8s.AppsV1Api, namespace: string, deploymentName: string, replicas: number): Promise<void> {
await appsApi.patchNamespacedDeployment({ name: deploymentName, namespace, body: { spec: { replicas } } }, k8s.setHeaderOptions('Content-Type', 'application/merge-patch+json'));
}
/**
* Suspend an application by scaling all stack deployments to 0 replicas.
* Returns the replica snapshot captured before scaling.
*/
async suspendApplication(app: Application): Promise<Record<string, number>> {
const { appsApi } = await this.k8sClientService.getK8sClient(app.clusterId);
const namespace = this.getUserNamespace(app.userId);
this.logger.log(`Suspending application ${app.name} in namespace ${namespace}`);
const snapshot = await this.captureWorkloadReplicaSnapshot(app);
const workloads = this.getApplicationWorkloadDeployments(app);
for (const workload of workloads) {
try {
await this.patchDeploymentReplicas(appsApi, namespace, workload.name, 0);
this.logger.log(`Scaled ${workload.name} to 0 replicas`);
} catch (e: any) {
if (e?.response?.statusCode !== 404) {
this.logger.warn(`Failed to scale ${workload.name}: ${e.message}`);
}
}
}
await this.deleteTemporaryAccessServicesForApp(app);
return snapshot;
}
/**
* Resume a suspended application using saved replica counts when available.
*/
async resumeApplication(app: Application): Promise<void> {
const { appsApi } = await this.k8sClientService.getK8sClient(app.clusterId);
const namespace = this.getUserNamespace(app.userId);
this.logger.log(`Resuming application ${app.name} in namespace ${namespace}`);
const workloads = this.resolveWorkloadReplicas(app);
const dependencies = workloads.filter((w) => w.name !== app.name);
const main = workloads.find((w) => w.name === app.name);
for (const workload of dependencies) {
try {
await this.patchDeploymentReplicas(appsApi, namespace, workload.name, workload.runningReplicas);
this.logger.log(`Scaled ${workload.name} to ${workload.runningReplicas} replica(s)`);
} catch (e: any) {
if (e?.response?.statusCode !== 404) {
this.logger.warn(`Failed to scale ${workload.name}: ${e.message}`);
}
}
}
if (!main) return;
try {
await this.patchDeploymentReplicas(appsApi, namespace, main.name, main.runningReplicas);
this.logger.log(`Scaled ${main.name} to ${main.runningReplicas} replica(s)`);
} catch (e: any) {
this.logger.warn(`Failed to scale ${main.name}: ${e.message}`);
throw e;
}
}
async restartDeployment(app: Application): Promise<void> {
const { appsApi } = await this.k8sClientService.getK8sClient(app.clusterId);
const namespace = this.getUserNamespace(app.userId);
const deploymentName = isManagedProductType(app.productType) ? this.primaryWorkloadLabel(app) : app.name;
await appsApi.patchNamespacedDeployment(
{
name: deploymentName,
namespace,
body: {
spec: {
template: {
metadata: {
annotations: {
'kubectl.kubernetes.io/restartedAt': new Date().toISOString(),
},
},
},
},
},
},
k8s.setHeaderOptions('Content-Type', 'application/merge-patch+json'),
);
}
/** Map optional workload keys to Deployment + primary container name */
workloadDeploymentTarget(app: Application, workload: 'app' | 'database' | 'redis' | 'rabbitmq'): { deploymentName: string; containerName: string } | null {
switch (workload) {
case 'app':
return { deploymentName: app.name, containerName: app.name };
case 'database':
if (!app.databaseType || app.databaseType === DatabaseType.NONE) return null;
return {
deploymentName: `${app.name}-db`,
containerName: `${app.name}-db`,
};
case 'redis':
if (!app.enableRedis) return null;
return { deploymentName: `${app.name}-redis`, containerName: 'redis' };
case 'rabbitmq':
if (!app.enableRabbitmq) return null;
return {
deploymentName: `${app.name}-rabbitmq`,
containerName: 'rabbitmq',
};
default:
return null;
}
}
private async fetchNamespacePodMetrics(kc: k8s.KubeConfig, namespace: string, appLabelValue: string): Promise<any[]> {
try {
const opts: any = {};
await kc.applyToHTTPSOptions(opts);
const metricsUrl = `${kc.getCurrentCluster()?.server}/apis/metrics.k8s.io/v1beta1/namespaces/${namespace}/pods`;
const https = require('https');
const http = require('http');
const url = new URL(metricsUrl);
const labelQ = `labelSelector=${encodeURIComponent(`app=${appLabelValue}`)}`;
return await new Promise((resolve) => {
const client = url.protocol === 'https:' ? https : http;
const reqOpts: any = {
hostname: url.hostname,
port: url.port,
path: `${url.pathname}?${labelQ}`,
method: 'GET',
headers: opts.headers || {},
rejectUnauthorized: false,
};
if (opts.ca) reqOpts.ca = opts.ca;
if (opts.cert) reqOpts.cert = opts.cert;
if (opts.key) reqOpts.key = opts.key;
const req = client.request(reqOpts, (res: any) => {
let data = '';
res.on('data', (chunk: string) => (data += chunk));
res.on('end', () => {
try {
const parsed = JSON.parse(data);
const items = (parsed.items || []).map((item: any) => ({
name: item.metadata?.name,
containers: (item.containers || []).map((c: any) => ({
name: c.name,
cpu: c.usage?.cpu || '0',
memory: c.usage?.memory || '0',
})),
cpu: item.containers?.[0]?.usage?.cpu || '0',
memory: item.containers?.[0]?.usage?.memory || '0',
}));
resolve(items);
} catch {
resolve([]);
}
});
});
req.on('error', () => resolve([]));
req.end();
});
} catch {
return [];
}
}
private async getSingleWorkloadUsage(
app: Application,
namespace: string,
appsApi: k8s.AppsV1Api,
coreApi: k8s.CoreV1Api,
kc: k8s.KubeConfig,
opts: {
key: string;
title: string;
deploymentName: string;
primaryContainerName: string;
fallbackCpuRequest?: string;
fallbackCpuLimit?: string;
fallbackMemoryRequest?: string;
fallbackMemoryLimit?: string;
fallbackReplicas?: number;
},
): Promise<any | null> {
let deployment: k8s.V1Deployment | null = null;
try {
const depResponse = await appsApi.readNamespacedDeployment({
name: opts.deploymentName,
namespace,
});
deployment = depResponse;
} catch {
return null;
}
const podsResponse = await coreApi.listNamespacedPod({
namespace,
labelSelector: `app=${opts.deploymentName}`,
});
const pods = podsResponse.items.map((pod) => ({
name: pod.metadata?.name,
status: pod.status?.phase,
ready: pod.status?.conditions?.find((c) => c.type === 'Ready')?.status === 'True',
restarts: pod.status?.containerStatuses?.reduce((s, c) => s + (c.restartCount || 0), 0) || 0,
startedAt: pod.status?.startTime,
}));
const podMetrics = await this.fetchNamespacePodMetrics(kc, namespace, opts.deploymentName);
const container = deployment.spec?.template?.spec?.containers?.find((c) => c.name === opts.primaryContainerName) || deployment.spec?.template?.spec?.containers?.[0];
const sidecars = (deployment.spec?.template?.spec?.containers || [])
.filter((c) => c.name !== opts.primaryContainerName)
.map((c) => ({
name: c.name,
cpuRequest: c.resources?.requests?.cpu,
cpuLimit: c.resources?.limits?.cpu,
memoryRequest: c.resources?.requests?.memory,
memoryLimit: c.resources?.limits?.memory,
}));
const configured = {
cpuRequest: container?.resources?.requests?.cpu || opts.fallbackCpuRequest,
cpuLimit: container?.resources?.limits?.cpu || opts.fallbackCpuLimit,
memoryRequest: container?.resources?.requests?.memory || opts.fallbackMemoryRequest,
memoryLimit: container?.resources?.limits?.memory || opts.fallbackMemoryLimit,
replicas: deployment.spec?.replicas ?? opts.fallbackReplicas ?? 1,
readyReplicas: deployment.status?.readyReplicas || 0,
availableReplicas: deployment.status?.availableReplicas || 0,
};
const metrics = podMetrics.map((m: any) => {
const matchC = m.containers?.find((c: any) => c.name === opts.primaryContainerName) || m.containers?.[0];
return {
name: m.name,
cpu: matchC?.cpu || m.cpu || '0',
memory: matchC?.memory || m.memory || '0',
containers: m.containers,
};
});
return {
key: opts.key,
title: opts.title,
deploymentName: opts.deploymentName,
configured,
pods,
metrics,
sidecars: sidecars.length ? sidecars : undefined,
};
}
/**
* Get real-time resource usage (CPU/Memory) for an app's pods via metrics-server.
* Includes application workload, database, and optional Redis / RabbitMQ when enabled.
*/
async getResourceUsage(app: Application): Promise<any> {
const { coreApi, appsApi, kc } = await this.k8sClientService.getK8sClient(app.clusterId);
const namespace = this.getUserNamespace(app.userId);
const workloads: any[] = [];
const appW = await this.getSingleWorkloadUsage(app, namespace, appsApi, coreApi, kc, {
key: 'app',
title: 'Application',
deploymentName: app.name,
primaryContainerName: app.name,
fallbackCpuRequest: app.cpuRequest,
fallbackCpuLimit: app.cpuLimit,
fallbackMemoryRequest: app.memoryRequest,
fallbackMemoryLimit: app.memoryLimit,
fallbackReplicas: app.replicas,
});
if (appW) workloads.push(appW);
if (app.databaseType && app.databaseType !== DatabaseType.NONE) {
const dbW = await this.getSingleWorkloadUsage(app, namespace, appsApi, coreApi, kc, {
key: 'database',
title: 'Database',
deploymentName: `${app.name}-db`,
primaryContainerName: `${app.name}-db`,
fallbackCpuRequest: '100m',
fallbackCpuLimit: '500m',
fallbackMemoryRequest: '256Mi',
fallbackMemoryLimit: '512Mi',
fallbackReplicas: 1,
});
if (dbW) workloads.push(dbW);
}
if (app.enableRedis) {
const r = await this.getSingleWorkloadUsage(app, namespace, appsApi, coreApi, kc, {
key: 'redis',
title: 'Redis',
deploymentName: `${app.name}-redis`,
primaryContainerName: 'redis',
fallbackCpuRequest: '50m',
fallbackCpuLimit: '200m',
fallbackMemoryRequest: '64Mi',
fallbackMemoryLimit: '256Mi',
fallbackReplicas: 1,
});
if (r) workloads.push(r);
}
if (app.enableRabbitmq) {
const mq = await this.getSingleWorkloadUsage(app, namespace, appsApi, coreApi, kc, {
key: 'rabbitmq',
title: 'RabbitMQ',
deploymentName: `${app.name}-rabbitmq`,
primaryContainerName: 'rabbitmq',
fallbackCpuRequest: '100m',
fallbackCpuLimit: '500m',
fallbackMemoryRequest: '256Mi',
fallbackMemoryLimit: '512Mi',
fallbackReplicas: 1,
});
if (mq) workloads.push(mq);
}
const primary = workloads.find((w) => w.key === 'app') || workloads[0];
const loggingNote = app.enableElasticsearch ? 'Elasticsearch log forwarding is enabled via Fluent Bit sidecar on application pods.' : undefined;
return {
workloads,
loggingNote,
configured: primary?.configured,
pods: primary?.pods || [],
metrics: primary?.metrics || [],
};
}
/**
* Update resource limits/requests and replicas on a live K8s deployment.
* @param workload Which deployment to patch (default: main application). Replicas only apply to `app`.
*/
async updateResources(
app: Application,
resources: {
cpuRequest?: string;
cpuLimit?: string;
memoryRequest?: string;
memoryLimit?: string;
replicas?: number;
},
workload: 'app' | 'database' | 'redis' | 'rabbitmq' = 'app',
): Promise<void> {
const { appsApi } = await this.k8sClientService.getK8sClient(app.clusterId);
const namespace = this.getUserNamespace(app.userId);
const target = this.workloadDeploymentTarget(app, workload);
if (!target) {
throw new BadRequestException(workload === 'database' ? 'This application does not have a database deployment.' : `Workload "${workload}" is not enabled for this application.`);
}
const patch: any = { spec: {} };
if (resources.replicas !== undefined && workload === 'app') {
patch.spec.replicas = resources.replicas;
}
if (resources.cpuRequest || resources.cpuLimit || resources.memoryRequest || resources.memoryLimit) {
patch.spec.template = {
spec: {
containers: [
{
name: target.containerName,
resources: {
requests: {
...(resources.cpuRequest && { cpu: resources.cpuRequest }),
...(resources.memoryRequest && {
memory: resources.memoryRequest,
}),
},
limits: {
...(resources.cpuLimit && { cpu: resources.cpuLimit }),
...(resources.memoryLimit && {
memory: resources.memoryLimit,
}),
},
},
},
],
},
};
}
await appsApi.patchNamespacedDeployment({ name: target.deploymentName, namespace, body: patch }, k8s.setHeaderOptions('Content-Type', 'application/strategic-merge-patch+json'));
this.logger.log(`Updated resources for ${target.deploymentName} (${workload}): ${JSON.stringify(resources)}`);
}
getUserNamespace(userId: string): string {
return userNamespace(userId);
}
/** Preview URL stays available until the custom domain is verified (not merely requested). */
private hasVerifiedCustomDomain(app: Application): boolean {
return !!(app.customDomain && app.customDomainStatus === CustomDomainStatus.VERIFIED);
}
private getClusterHostIp(kc: k8s.KubeConfig): string {
const clusterServer = kc.getCurrentCluster()?.server || '';
try {
return new URL(clusterServer).hostname;
} catch {
return '127.0.0.1';
}
}
private toDnsLabel(value: string): string {
return value
.toLowerCase()
.replace(/[^a-z0-9-]/g, '-')
.replace(/-+/g, '-')
.replace(/^-|-$/g, '')
.slice(0, 63)
.replace(/-$/g, '');
}
resolveAccessTarget(
app: Application,
target: ServiceAccessTarget,
): {
selector: Record<string, string>;
targetPort: number;
portName?: string;
} {
switch (target) {
case ServiceAccessTarget.DATABASE: {
if (!app.databaseType || app.databaseType === DatabaseType.NONE) {
throw new BadRequestException('Application has no database');
}
let targetPort = 3306;
if (app.databaseType === DatabaseType.POSTGRESQL) targetPort = 5432;
else if (app.databaseType === DatabaseType.MONGODB) targetPort = 27017;
return { selector: { app: `${app.name}-db` }, targetPort };
}
case ServiceAccessTarget.REDIS:
if (!app.enableRedis) throw new BadRequestException('Redis is not enabled for this application');
return { selector: { app: `${app.name}-redis` }, targetPort: 6379 };
case ServiceAccessTarget.RABBITMQ_AMQP:
if (!app.enableRabbitmq) throw new BadRequestException('RabbitMQ is not enabled for this application');
return {
selector: { app: `${app.name}-rabbitmq` },
targetPort: 5672,
portName: 'amqp',
};
case ServiceAccessTarget.RABBITMQ_MANAGEMENT:
if (!app.enableRabbitmq) throw new BadRequestException('RabbitMQ is not enabled for this application');
return {
selector: { app: `${app.name}-rabbitmq` },
targetPort: 15672,
portName: 'management',
};
default:
throw new BadRequestException(`Unknown access target: ${target}`);
}
}
async createTemporaryAccess(
app: Application,
target: ServiceAccessTarget,
grantId: string,
): Promise<{
host: string;
nodePort: number;
k8sServiceName: string;
targetPort: number;
}> {
if (!app.clusterId) {
throw new BadRequestException('Application is not assigned to a cluster');
}
const { coreApi, kc } = await this.k8sClientService.getK8sClient(app.clusterId);
const namespace = this.getUserNamespace(app.userId);
const { selector, targetPort, portName } = this.resolveAccessTarget(app, target);
const shortId = grantId.split('-')[0];
const k8sServiceName = this.toDnsLabel(`${app.name}-${target}-access-${shortId}`);
const portSpec: k8s.V1ServicePort = {
port: targetPort,
targetPort: targetPort,
protocol: 'TCP',
};
if (portName) portSpec.name = portName;
const service: k8s.V1Service = {
apiVersion: 'v1',
kind: 'Service',
metadata: {
name: k8sServiceName,
namespace,
labels: {
app: selector.app,
'cloudhost.io/access-grant': 'true',
'cloudhost.io/grant-id': grantId,
'cloudhost.io/application-id': app.id,
},
},
spec: {
type: 'NodePort',
selector,
ports: [portSpec],
},
};
let created: k8s.V1Service;
try {
created = await coreApi.createNamespacedService({
namespace,
body: service,
});
} catch (error: any) {
const message = error.body?.message || error.response?.body?.message || error.message || 'HTTP request failed';
this.logger.warn(`K8s client failed to create temporary access service ${k8sServiceName}; trying kubectl fallback: ${message}`);
created = await this.applyServiceWithKubectl(app.clusterId, namespace, service);
}
const nodePort = created.spec?.ports?.[0]?.nodePort;
if (!nodePort) {
try {
await coreApi.deleteNamespacedService({
name: k8sServiceName,
namespace,
});
} catch {}
throw new Error('Failed to allocate NodePort for temporary access');
}
const host = this.getClusterHostIp(kc);
this.logger.log(`Temporary access for ${app.name} target=${target}: ${host}:${nodePort} (service ${k8sServiceName})`);
return { host, nodePort, k8sServiceName, targetPort };
}
private async applyServiceWithKubectl(clusterId: string | undefined, namespace: string, service: k8s.V1Service): Promise<k8s.V1Service> {
const tmpDir = fs.mkdtempSync(path.join('/tmp', 'cloudhost-access-'));
const kubeconfigPath = path.join(tmpDir, 'kubeconfig.yaml');
const manifestPath = path.join(tmpDir, 'service.json');
try {
fs.writeFileSync(kubeconfigPath, await this.k8sClientService.getKubeconfig(clusterId), {
mode: 0o600,
});
fs.writeFileSync(manifestPath, JSON.stringify(service), { mode: 0o600 });
await execFileAsync('kubectl', ['--kubeconfig', kubeconfigPath, 'apply', '-n', namespace, '-f', manifestPath]);
const { stdout } = await execFileAsync('kubectl', ['--kubeconfig', kubeconfigPath, 'get', 'service', service.metadata!.name!, '-n', namespace, '-o', 'json']);
return JSON.parse(stdout) as k8s.V1Service;
} catch (error: any) {
const message = error.stderr || error.message || 'kubectl failed';
throw new Error(`Failed to create temporary access service: ${message}`);
} finally {
try {
fs.rmSync(tmpDir, { recursive: true, force: true });
} catch {}
}
}
async revokeTemporaryAccess(clusterId: string, namespace: string, k8sServiceName: string): Promise<void> {
const { coreApi } = await this.k8sClientService.getK8sClient(clusterId);
try {
await coreApi.deleteNamespacedService({
name: k8sServiceName,
namespace,
});
this.logger.log(`Revoked temporary access service ${k8sServiceName} in ${namespace}`);
} catch (e: any) {
if (e.code !== 404 && e.response?.statusCode !== 404) {
this.logger.warn(`Failed to delete access service ${k8sServiceName}: ${e.message}`);
}
}
}
async deleteTemporaryAccessServicesForApp(app: Application): Promise<void> {
if (!app.clusterId) return;
const namespace = this.getUserNamespace(app.userId);
const { coreApi } = await this.k8sClientService.getK8sClient(app.clusterId);
try {
const services = await coreApi.listNamespacedService({
namespace,
labelSelector: 'cloudhost.io/access-grant=true',
});
for (const svc of services.items) {
const appId = svc.metadata?.labels?.['cloudhost.io/application-id'];
if (appId === app.id && svc.metadata?.name) {
await this.revokeTemporaryAccess(app.clusterId, namespace, svc.metadata.name);
}
}
} catch (e: any) {
this.logger.warn(`Failed to list temporary access services for ${app.name}: ${e.message}`);
}
}
async readAccessCredentials(app: Application, target: ServiceAccessTarget): Promise<Record<string, string | number | undefined>> {
const namespace = this.getUserNamespace(app.userId);
switch (target) {
case ServiceAccessTarget.DATABASE:
return {
username: app.dbUsername || 'appuser',
password: app.dbPassword || undefined,
database: app.name.replace(/-/g, '_'),
};
case ServiceAccessTarget.REDIS: {
if (!app.clusterId) return {};
const { coreApi } = await this.k8sClientService.getK8sClient(app.clusterId);
const secret = await coreApi.readNamespacedSecret({
name: `${app.name}-redis-secret`,
namespace,
});
const password = secret.data?.password ? Buffer.from(secret.data.password, 'base64').toString('utf8') : undefined;
return { password };
}
case ServiceAccessTarget.RABBITMQ_AMQP:
case ServiceAccessTarget.RABBITMQ_MANAGEMENT: {
if (!app.clusterId) return {};
const { coreApi } = await this.k8sClientService.getK8sClient(app.clusterId);
const secret = await coreApi.readNamespacedSecret({
name: `${app.name}-rabbitmq-secret`,
namespace,
});
const username = secret.data?.username ? Buffer.from(secret.data.username, 'base64').toString('utf8') : 'appuser';
const password = secret.data?.password ? Buffer.from(secret.data.password, 'base64').toString('utf8') : undefined;
return { username, password };
}
default:
return {};
}
}
/**
* Get preview info for a deployed application.
* Returns ingress URL when available; only reads an existing NodePort (never patches ClusterIP).
*/
async getPreviewInfo(
app: Application,
previewNumber?: string | null,
): Promise<{
url: string;
nodePort: number;
host: string;
ingressUrl?: string;
}> {
const { coreApi, kc } = await this.k8sClientService.getK8sClient(app.clusterId);
const namespace = this.getUserNamespace(app.userId);
const domain = this.configService.get('platform.domain');
const hostIp = this.getClusterHostIp(kc);
const subdomain = app.subdomain || app.name;
const verifiedCustomDomain = app.customDomain && app.customDomainStatus === CustomDomainStatus.VERIFIED ? app.customDomain : null;
const previewRootDomain = this.configService.get<string>('platform.previewRootDomain') || domain;
const namespacePrefix = userIdSlug(app.userId);
let ingressUrl = `https://${subdomain}.${domain}`;
if (verifiedCustomDomain) {
ingressUrl = `https://${verifiedCustomDomain}`;
} else if (previewNumber) {
ingressUrl = `https://${namespacePrefix}-${previewNumber}.${previewRootDomain}`;
}
let nodePort = 0;
try {
const svcResponse = await coreApi.readNamespacedService({
name: app.name,
namespace,
});
if (svcResponse.spec?.type === 'NodePort') {
nodePort = svcResponse.spec.ports?.[0]?.nodePort || 0;
}
} catch (e: any) {
this.logger.warn(`Failed to read service for ${app.name}: ${e.message}`);
}
const url = ingressUrl || (nodePort > 0 ? `http://${hostIp}:${nodePort}` : '');
return {
url,
nodePort,
host: hostIp,
ingressUrl,
};
}
async deleteApplication(app: Application): Promise<void> {
const namespace = this.getUserNamespace(app.userId);
const { coreApi, appsApi, networkingApi } = await this.k8sClientService.getK8sClient(app.clusterId);
await this.deleteTemporaryAccessServicesForApp(app);
// Step 1: Try Helm uninstall (handles most resources)
try {
const kubeconfig = await this.k8sClientService.getKubeconfig(app.clusterId);
await this.helmService.uninstall(app.name, namespace, kubeconfig);
this.logger.log(`Helm release ${app.name} uninstalled from ${namespace}`);
} catch (error: any) {
this.logger.warn(`Helm uninstall failed for ${app.name}: ${error.message} — proceeding with manual cleanup`);
// Manual cleanup of core resources if helm wasn't managing them
try {
await appsApi.deleteNamespacedDeployment({ name: app.name, namespace });
} catch {}
try {
await coreApi.deleteNamespacedService({ name: app.name, namespace });
} catch {}
try {
await networkingApi.deleteNamespacedIngress({
name: app.name,
namespace,
});
} catch {}
}
// Step 2: Delete resources with "helm.sh/resource-policy: keep" (PVCs, db-secret)
// These survive helm uninstall by design, so we must delete them explicitly
const dbName = `${app.name}-db`;
const resourcesToDelete = [
// Database resources
() => appsApi.deleteNamespacedDeployment({ name: dbName, namespace }),
() => coreApi.deleteNamespacedService({ name: dbName, namespace }),
() =>
coreApi.deleteNamespacedPersistentVolumeClaim({
name: dbName,
namespace,
}),
() =>
coreApi.deleteNamespacedSecret({
name: `${app.name}-db-secret`,
namespace,
}),
// App storage PVC (all app types)
() =>
coreApi.deleteNamespacedPersistentVolumeClaim({
name: `${app.name}-storage`,
namespace,
}),
// Legacy: also try deleting old wp-content PVC name for backward compatibility
() =>
coreApi.deleteNamespacedPersistentVolumeClaim({
name: `${app.name}-wp-content`,
namespace,
}),
// App env secret
() => coreApi.deleteNamespacedSecret({ name: `${app.name}-env`, namespace }),
// Fluent Bit config (if elasticsearch was enabled)
() =>
coreApi.deleteNamespacedConfigMap({
name: `${app.name}-fluent-bit-config`,
namespace,
}),
// Redis resources
() =>
appsApi.deleteNamespacedDeployment({
name: `${app.name}-redis`,
namespace,
}),
() =>
coreApi.deleteNamespacedService({
name: `${app.name}-redis`,
namespace,
}),
() =>
coreApi.deleteNamespacedPersistentVolumeClaim({
name: `${app.name}-redis-data`,
namespace,
}),
() =>
coreApi.deleteNamespacedSecret({
name: `${app.name}-redis-secret`,
namespace,
}),
// RabbitMQ resources
() =>
appsApi.deleteNamespacedDeployment({
name: `${app.name}-rabbitmq`,
namespace,
}),
() =>
coreApi.deleteNamespacedService({
name: `${app.name}-rabbitmq`,
namespace,
}),
() =>
coreApi.deleteNamespacedPersistentVolumeClaim({
name: `${app.name}-rabbitmq-data`,
namespace,
}),
() =>
coreApi.deleteNamespacedSecret({
name: `${app.name}-rabbitmq-secret`,
namespace,
}),
// TLS secret created by cert-manager
() => coreApi.deleteNamespacedSecret({ name: `${app.name}-tls`, namespace }),
];
for (const deleteFn of resourcesToDelete) {
try {
await deleteFn();
} catch {}
}
this.logger.log(`All K8s resources cleaned up for ${app.name} in ${namespace}`);
}
async prepareApplicationMigration(
app: Application,
targetClusterId: string,
options: {
migrateStorage?: boolean;
log?: (step: string, message: string, metadata?: Record<string, any>) => Promise<void>;
} = {},
): Promise<void> {
const namespace = this.getUserNamespace(app.userId);
const source = await this.k8sClientService.getK8sClient(app.clusterId);
const target = await this.k8sClientService.getK8sClient(targetClusterId);
await this.ensureNamespaceOnCluster(target.coreApi, namespace);
await options.log?.('transfer-secrets-configs', 'Target namespace ensured', { namespace });
const secretNames = [`${app.name}-env`, `${app.name}-db-secret`, `${app.name}-redis-secret`, `${app.name}-rabbitmq-secret`, `${app.name}-tls`];
for (const secretName of secretNames) {
try {
const secret = await source.coreApi.readNamespacedSecret({
name: secretName,
namespace,
});
await this.upsertSecret(target.coreApi, namespace, this.cleanK8sObject(secret));
await options.log?.('transfer-secrets-configs', `Secret ${secretName} copied`);
} catch (error: any) {
if (!this.isK8sNotFound(error)) {
throw error;
}
}
}
const configMapNames = [`${app.name}-fluent-bit-config`];
for (const configMapName of configMapNames) {
try {
const configMap = await source.coreApi.readNamespacedConfigMap({
name: configMapName,
namespace,
});
await this.upsertConfigMap(target.coreApi, namespace, this.cleanK8sObject(configMap));
await options.log?.('transfer-secrets-configs', `ConfigMap ${configMapName} copied`);
} catch (error: any) {
if (!this.isK8sNotFound(error)) {
throw error;
}
}
}
if (options.migrateStorage !== false) {
const pvcNames = [`${app.name}-storage`, `${app.name}-wp-content`, `${app.name}-db`, `${app.name}-redis-data`, `${app.name}-rabbitmq-data`];
for (const pvcName of pvcNames) {
try {
const pvc = await source.coreApi.readNamespacedPersistentVolumeClaim({
name: pvcName,
namespace,
});
await this.upsertPvcDefinition(target.coreApi, namespace, this.cleanPvcForMigration(pvc));
await options.log?.('transfer-volumes', `PVC definition ${pvcName} prepared on target cluster`, {
note: 'Starting best-effort PVC data copy through migration helper pods.',
});
await this.copyPvcDataBetweenClusters(app.clusterId!, targetClusterId, namespace, pvcName, options.log);
} catch (error: any) {
if (!this.isK8sNotFound(error)) {
throw error;
}
}
}
}
}
private async copyPvcDataBetweenClusters(
sourceClusterId: string,
targetClusterId: string,
namespace: string,
pvcName: string,
log?: (step: string, message: string, metadata?: Record<string, any>) => Promise<void>,
): Promise<void> {
const safeName = pvcName
.replace(/[^a-z0-9-]/gi, '-')
.toLowerCase()
.slice(0, 32);
const suffix = `${Date.now()}`.slice(-6);
const sourcePod = `migrate-src-${safeName}-${suffix}`;
const targetPod = `migrate-dst-${safeName}-${suffix}`;
const tempDir = path.join('/tmp', `app-migration-${safeName}-${suffix}`);
const sourceKubeconfig = path.join(tempDir, 'source.kubeconfig');
const targetKubeconfig = path.join(tempDir, 'target.kubeconfig');
fs.mkdirSync(tempDir, { recursive: true });
fs.writeFileSync(sourceKubeconfig, await this.k8sClientService.getKubeconfig(sourceClusterId), { mode: 0o600 });
fs.writeFileSync(targetKubeconfig, await this.k8sClientService.getKubeconfig(targetClusterId), { mode: 0o600 });
try {
await this.createPvcCopyPod(sourceKubeconfig, namespace, sourcePod, pvcName);
await this.createPvcCopyPod(targetKubeconfig, namespace, targetPod, pvcName);
await log?.('transfer-volumes', `Copy helper pods ready for PVC ${pvcName}`);
const localDataPath = path.join(tempDir, 'data');
await execFileAsync('kubectl', ['--kubeconfig', sourceKubeconfig, '-n', namespace, 'cp', `${sourcePod}:/data`, localDataPath], {
timeout: 30 * 60 * 1000,
});
await execFileAsync('kubectl', ['--kubeconfig', targetKubeconfig, '-n', namespace, 'cp', `${localDataPath}/.`, `${targetPod}:/data`], {
timeout: 30 * 60 * 1000,
});
await log?.('transfer-volumes', `PVC data copied for ${pvcName}`);
} finally {
await execFileAsync('kubectl', ['--kubeconfig', sourceKubeconfig, '-n', namespace, 'delete', 'pod', sourcePod, '--ignore-not-found=true']).catch(() => undefined);
await execFileAsync('kubectl', ['--kubeconfig', targetKubeconfig, '-n', namespace, 'delete', 'pod', targetPod, '--ignore-not-found=true']).catch(() => undefined);
fs.rmSync(tempDir, { recursive: true, force: true });
}
}
private async createPvcCopyPod(kubeconfigPath: string, namespace: string, podName: string, pvcName: string): Promise<void> {
const manifestPath = path.join('/tmp', `${podName}.json`);
const manifest = {
apiVersion: 'v1',
kind: 'Pod',
metadata: { name: podName, namespace },
spec: {
restartPolicy: 'Never',
containers: [
{
name: 'copy',
image: 'busybox:1.36',
command: ['sh', '-c', 'mkdir -p /data && sleep 3600'],
volumeMounts: [{ name: 'data', mountPath: '/data' }],
},
],
volumes: [{ name: 'data', persistentVolumeClaim: { claimName: pvcName } }],
},
};
fs.writeFileSync(manifestPath, JSON.stringify(manifest));
try {
await execFileAsync('kubectl', ['--kubeconfig', kubeconfigPath, 'apply', '-f', manifestPath], { timeout: 120_000 });
await execFileAsync('kubectl', ['--kubeconfig', kubeconfigPath, '-n', namespace, 'wait', '--for=condition=Ready', `pod/${podName}`, '--timeout=180s'], { timeout: 210_000 });
} finally {
fs.rmSync(manifestPath, { force: true });
}
}
private async ensureNamespaceOnCluster(coreApi: k8s.CoreV1Api, namespace: string): Promise<void> {
try {
await coreApi.readNamespace({ name: namespace });
} catch (error: any) {
if (this.isK8sNotFound(error)) {
await coreApi.createNamespace({
body: { metadata: { name: namespace } },
});
return;
}
throw error;
}
}
private async upsertSecret(coreApi: k8s.CoreV1Api, namespace: string, secret: k8s.V1Secret): Promise<void> {
secret.metadata = { ...(secret.metadata || {}), namespace };
try {
await coreApi.replaceNamespacedSecret({
name: secret.metadata.name!,
namespace,
body: secret,
});
} catch (error: any) {
if (this.isK8sNotFound(error)) {
await coreApi.createNamespacedSecret({ namespace, body: secret });
return;
}
throw error;
}
}
private async upsertConfigMap(coreApi: k8s.CoreV1Api, namespace: string, configMap: k8s.V1ConfigMap): Promise<void> {
configMap.metadata = { ...(configMap.metadata || {}), namespace };
try {
await coreApi.replaceNamespacedConfigMap({
name: configMap.metadata.name!,
namespace,
body: configMap,
});
} catch (error: any) {
if (this.isK8sNotFound(error)) {
await coreApi.createNamespacedConfigMap({ namespace, body: configMap });
return;
}
throw error;
}
}
private async upsertPvcDefinition(coreApi: k8s.CoreV1Api, namespace: string, pvc: k8s.V1PersistentVolumeClaim): Promise<void> {
pvc.metadata = { ...(pvc.metadata || {}), namespace };
try {
await coreApi.readNamespacedPersistentVolumeClaim({
name: pvc.metadata.name!,
namespace,
});
} catch (error: any) {
if (this.isK8sNotFound(error)) {
await coreApi.createNamespacedPersistentVolumeClaim({
namespace,
body: pvc,
});
return;
}
throw error;
}
}
private cleanK8sObject<T extends { metadata?: k8s.V1ObjectMeta }>(obj: T): T {
const metadata = { ...(obj.metadata || {}) };
delete metadata.uid;
delete metadata.resourceVersion;
delete metadata.generation;
delete metadata.creationTimestamp;
delete metadata.managedFields;
delete metadata.selfLink;
return {
...obj,
metadata,
};
}
private cleanPvcForMigration(pvc: k8s.V1PersistentVolumeClaim): k8s.V1PersistentVolumeClaim {
const cleaned = this.cleanK8sObject(pvc);
return {
apiVersion: cleaned.apiVersion,
kind: cleaned.kind,
metadata: cleaned.metadata,
spec: {
accessModes: cleaned.spec?.accessModes,
resources: cleaned.spec?.resources,
storageClassName: cleaned.spec?.storageClassName,
volumeMode: cleaned.spec?.volumeMode,
},
};
}
private isK8sNotFound(error: any): boolean {
return error?.statusCode === 404 || error?.body?.code === 404;
}
/**
* Wait for the database pod to become Ready.
* Polls pod status with label selector `app=<appName>-db`.
*/
async waitForDatabaseReady(app: Application, timeoutMs = 120_000): Promise<void> {
const { coreApi } = await this.k8sClientService.getK8sClient(app.clusterId);
const namespace = this.getUserNamespace(app.userId);
const dbLabel = `${app.name}-db`;
const start = Date.now();
this.logger.log(`Waiting for DB pod (app=${dbLabel}) to become Ready in ${namespace}...`);
while (Date.now() - start < timeoutMs) {
try {
const pods = await coreApi.listNamespacedPod({
namespace,
labelSelector: `app=${dbLabel}`,
});
for (const pod of pods.items) {
const readyCond = (pod.status?.conditions || []).find((c) => c.type === 'Ready');
if (readyCond?.status === 'True') {
this.logger.log(`DB pod ${pod.metadata?.name} is Ready (${Date.now() - start}ms)`);
return;
}
}
} catch {
// Pods may not exist yet
}
await new Promise((r) => setTimeout(r, 3000));
}
this.logger.warn(`DB pod did not become Ready within ${timeoutMs / 1000}s — proceeding anyway`);
}
private async getDeploymentReadiness(
appsApi: k8s.AppsV1Api,
namespace: string,
name: string,
expectedReplicas: number,
): Promise<{
name: string;
desiredReplicas: number;
readyReplicas: number;
availableReplicas: number;
ready: boolean;
}> {
try {
const response = await appsApi.readNamespacedDeployment({
name,
namespace,
});
const deployment = response;
const desiredReplicas = deployment.spec?.replicas ?? expectedReplicas;
const readyReplicas = deployment.status?.readyReplicas ?? 0;
const availableReplicas = deployment.status?.availableReplicas ?? 0;
const observedGeneration = deployment.status?.observedGeneration ?? 0;
const generation = deployment.metadata?.generation ?? 0;
return {
name,
desiredReplicas,
readyReplicas,
availableReplicas,
ready: desiredReplicas === 0 || (readyReplicas >= desiredReplicas && availableReplicas >= desiredReplicas && observedGeneration >= generation),
};
} catch {
return {
name,
desiredReplicas: expectedReplicas,
readyReplicas: 0,
availableReplicas: 0,
ready: false,
};
}
}
private async describeWorkloadPods(coreApi: k8s.CoreV1Api, namespace: string, workloadNames: string[]): Promise<string> {
const podLines: string[] = [];
for (const workloadName of workloadNames) {
try {
const pods = await coreApi.listNamespacedPod({
namespace,
labelSelector: `app=${workloadName}`,
});
for (const pod of pods.items) {
const ready = pod.status?.conditions?.find((c) => c.type === 'Ready')?.status === 'True';
const scheduled = pod.status?.conditions?.find((c) => c.type === 'PodScheduled');
const waitingReasons = (pod.status?.containerStatuses || [])
.map((status) => status.state?.waiting?.reason)
.filter(Boolean)
.join('/');
const reason = waitingReasons || scheduled?.reason || pod.status?.reason || pod.status?.phase || 'Unknown';
const message = scheduled?.message || pod.status?.message || '';
podLines.push(`${pod.metadata?.name || workloadName}: phase=${pod.status?.phase || 'Unknown'}, ready=${ready}, reason=${reason}${message ? ` (${message})` : ''}`);
}
} catch {
podLines.push(`${workloadName}: pods unavailable`);
}
}
return podLines.length ? `Pods: ${podLines.join('; ')}` : 'No pods found for expected workloads.';
}
/**
* Restore a SQL dump file into the application's database.
* Uses a PVC + helper pod + kubectl cp to transfer the dump (supports large files),
* then runs a restore Job that mounts the PVC and imports the dump.
*/
async restoreDatabaseDump(app: Application, dumpFilePath: string): Promise<{ success: boolean; logs: string }> {
const { coreApi, kc } = await this.k8sClientService.getK8sClient(app.clusterId);
const batchApi = kc.makeApiClient(k8s.BatchV1Api);
const namespace = this.getUserNamespace(app.userId);
const dbName = `${app.name}-db`;
const ts = Date.now();
const pvcName = `${app.name}-db-dump-${ts}`;
const helperPodName = `${pvcName}-helper`;
const jobName = `${app.name}-db-restore-${ts}`;
const dumpSize = fs.statSync(dumpFilePath).size;
const pvcSizeGi = Math.max(1, Math.ceil((dumpSize * 2) / (1024 * 1024 * 1024)));
this.logger.log(`Restoring DB dump for ${app.name}: ${(dumpSize / 1024 / 1024).toFixed(1)} MB via PVC`);
// ── 1. Create PVC for the dump file ──
await coreApi.createNamespacedPersistentVolumeClaim({
namespace,
body: {
apiVersion: 'v1',
kind: 'PersistentVolumeClaim',
metadata: { name: pvcName, namespace },
spec: {
accessModes: ['ReadWriteOnce'],
resources: { requests: { storage: `${pvcSizeGi}Gi` } },
},
},
});
// ── 2. Helper pod to receive the dump via kubectl cp ──
const helperPod: k8s.V1Pod = {
apiVersion: 'v1',
kind: 'Pod',
metadata: { name: helperPodName, namespace },
spec: {
containers: [
{
name: 'helper',
image: 'busybox:1.36',
command: ['sh', '-c', 'sleep 3600'],
volumeMounts: [{ name: 'dump', mountPath: '/data' }],
resources: {
requests: { cpu: '100m', memory: '128Mi' },
limits: { cpu: '500m', memory: '256Mi' },
},
},
],
volumes: [{ name: 'dump', persistentVolumeClaim: { claimName: pvcName } }],
restartPolicy: 'Never',
},
};
await coreApi.createNamespacedPod({ namespace, body: helperPod });
// Wait for helper pod Running
const podTimeout = 120_000;
const podStart = Date.now();
while (Date.now() - podStart < podTimeout) {
const pod = await coreApi.readNamespacedPod({
name: helperPodName,
namespace,
});
if (pod.status?.phase === 'Running') break;
if (pod.status?.phase === 'Failed') throw new Error('DB dump helper pod failed to start');
await new Promise((r) => setTimeout(r, 2000));
}
// ── 3. kubectl cp the dump file into the helper pod ──
const tmpKubeconfig = path.join('/tmp', `kubeconfig-dbdump-${ts}.yaml`);
const kcYaml = kc.exportConfig();
fs.writeFileSync(tmpKubeconfig, kcYaml);
// Verify tar is available inside the helper pod
try {
await execFileAsync('kubectl', ['--kubeconfig', tmpKubeconfig, 'exec', '-n', namespace, helperPodName, '--', 'tar', '--version'], { timeout: 30_000 });
} catch (e: any) {
this.logger.warn(`tar check in DB dump helper pod failed: ${e.message}`);
}
try {
await execFileAsync('kubectl', ['--kubeconfig', tmpKubeconfig, 'cp', dumpFilePath, `${namespace}/${helperPodName}:/data/dump.sql`, '--retries', '3'], {
maxBuffer: 50 * 1024 * 1024,
timeout: 600_000,
});
this.logger.log(`kubectl cp dump completed (${(dumpSize / 1024 / 1024).toFixed(1)} MB)`);
} finally {
try {
fs.unlinkSync(tmpKubeconfig);
} catch {}
try {
await coreApi.deleteNamespacedPod({ name: helperPodName, namespace });
} catch {}
}
// ── 4. Build restore command (per database type) ──
const { image, restoreCommand: command } = this.databaseDumpSpec(app, dbName);
// ── 5. Create the restore Job ──
const job: k8s.V1Job = {
apiVersion: 'batch/v1',
kind: 'Job',
metadata: { name: jobName, namespace },
spec: {
ttlSecondsAfterFinished: 300,
backoffLimit: 1,
template: {
spec: {
restartPolicy: 'Never',
containers: [
{
name: 'restore',
image,
command,
env: [
{
name: 'DB_USER',
valueFrom: {
secretKeyRef: {
name: `${app.name}-db-secret`,
key: 'username',
},
},
},
{
name: 'DB_PASSWORD',
valueFrom: {
secretKeyRef: {
name: `${app.name}-db-secret`,
key: 'password',
},
},
},
],
volumeMounts: [{ name: 'dump-volume', mountPath: '/dump', readOnly: true }],
resources: {
requests: { cpu: '200m', memory: '256Mi' },
limits: { cpu: '1', memory: '1Gi' },
},
},
],
volumes: [
{
name: 'dump-volume',
persistentVolumeClaim: { claimName: pvcName },
},
],
},
},
},
};
try {
await batchApi.createNamespacedJob({ namespace, body: job });
this.logger.log(`Created DB restore job ${jobName} for ${app.name}`);
} catch (e: any) {
try {
await coreApi.deleteNamespacedPersistentVolumeClaim({
name: pvcName,
namespace,
});
} catch {}
this.logger.error(`Failed to create restore job: ${e.message}`);
throw new Error('Failed to create database restore job');
}
// ── 6. Wait for the job to complete (max 10 minutes for large dumps) ──
const timeout = 600_000;
const start = Date.now();
let completed = false;
let failed = false;
while (Date.now() - start < timeout) {
await new Promise((r) => setTimeout(r, 5000));
try {
const jobStatus = await batchApi.readNamespacedJob({
name: jobName,
namespace,
});
const status = jobStatus.status;
if (status?.succeeded && status.succeeded > 0) {
completed = true;
break;
}
if (status?.failed && status.failed > 0) {
failed = true;
break;
}
} catch {
// Job may not be ready yet
}
}
// ── 7. Get logs from the restore job pod ──
let logs = '';
try {
const pods = await coreApi.listNamespacedPod({
namespace,
labelSelector: `job-name=${jobName}`,
});
if (pods.items.length > 0) {
const podName = pods.items[0].metadata?.name;
if (podName) {
const logResponse = await coreApi.readNamespacedPodLog({
name: podName,
namespace,
});
logs = logResponse || '';
}
}
} catch (e: any) {
this.logger.warn(`Could not get restore job logs: ${e.message}`);
}
// ── 8. Clean up the dump PVC ──
try {
await coreApi.deleteNamespacedPersistentVolumeClaim({
name: pvcName,
namespace,
});
this.logger.log(`Cleaned up dump PVC: ${pvcName}`);
} catch {}
if (!completed && !failed) {
this.logger.warn(`DB restore job ${jobName} timed out`);
return {
success: false,
logs: logs || 'Restore job timed out after 10 minutes',
};
}
if (failed) {
this.logger.warn(`DB restore job ${jobName} failed`);
return { success: false, logs: logs || 'Restore job failed' };
}
this.logger.log(`DB restore for ${app.name} completed successfully`);
return { success: true, logs };
}
private isPvcResizeForbiddenError(err: unknown): boolean {
const msg = (err as { body?: { message?: string }; message?: string })?.body?.message || (err as Error)?.message || '';
return /forbidden|resize|storageclass|dynamically provisioned/i.test(msg);
}
private async resolvePvcStorageClassName(coreApi: k8s.CoreV1Api, pvc: k8s.V1PersistentVolumeClaim): Promise<string | undefined> {
let scName = pvc.spec?.storageClassName;
if (scName) return scName;
const volumeName = pvc.spec?.volumeName;
if (!volumeName) return undefined;
try {
const pv = await coreApi.readPersistentVolume({ name: volumeName });
scName = pv.spec?.storageClassName;
if (pv.spec?.hostPath || pv.spec?.nfs || pv.spec?.local) {
return undefined;
}
return scName;
} catch {
return undefined;
}
}
private async ensureStorageClassAllowsExpansion(kc: k8s.KubeConfig, storageClassName: string): Promise<{ ok: boolean; message?: string }> {
const storageApi = kc.makeApiClient(k8s.StorageV1Api);
try {
const sc = await storageApi.readStorageClass({ name: storageClassName });
if (sc.allowVolumeExpansion) {
return { ok: true };
}
await storageApi.patchStorageClass({ name: storageClassName, body: { allowVolumeExpansion: true } }, k8s.setHeaderOptions('Content-Type', 'application/strategic-merge-patch+json'));
this.logger.log(`Enabled allowVolumeExpansion on StorageClass ${storageClassName}`);
return { ok: true };
} catch (e: any) {
return {
ok: false,
message: `StorageClass "${storageClassName}" does not support expansion: ${e.message}`,
};
}
}
private async patchPvcStorageSize(coreApi: k8s.CoreV1Api, pvcName: string, namespace: string, newSize: string): Promise<void> {
await coreApi.patchNamespacedPersistentVolumeClaim(
{
name: pvcName,
namespace,
body: [
{
op: 'replace',
path: '/spec/resources/requests/storage',
value: newSize,
},
],
},
k8s.setHeaderOptions('Content-Type', 'application/json-patch+json'),
);
}
/**
* Migrate DB PVC to a resize-capable StorageClass (one-time copy).
* Used when legacy PVCs were created without storageClassName.
*/
private async migrateDatabasePvcToResizableStorage(app: Application, newSize: string, storageClassName: string): Promise<{ success: boolean; message: string }> {
const { coreApi, appsApi, kc } = await this.k8sClientService.getK8sClient(app.clusterId);
const batchApi = kc.makeApiClient(k8s.BatchV1Api);
const namespace = this.getUserNamespace(app.userId);
const oldPvcName = `${app.name}-db`;
const newPvcName = `${app.name}-db-resizable`;
const deploymentName = `${app.name}-db`;
try {
try {
await coreApi.readNamespacedPersistentVolumeClaim({
name: newPvcName,
namespace,
});
} catch {
await coreApi.createNamespacedPersistentVolumeClaim({
namespace,
body: {
apiVersion: 'v1',
kind: 'PersistentVolumeClaim',
metadata: { name: newPvcName, namespace },
spec: {
accessModes: ['ReadWriteOnce'],
storageClassName,
resources: { requests: { storage: newSize } },
},
},
});
}
await appsApi.patchNamespacedDeployment({ name: deploymentName, namespace, body: { spec: { replicas: 0 } } }, k8s.setHeaderOptions('Content-Type', 'application/strategic-merge-patch+json'));
await this.waitForDeploymentReplicas(appsApi, namespace, deploymentName, 0, 120_000);
const jobName = `${app.name}-pvc-migrate-${Date.now()}`;
await batchApi.createNamespacedJob({
namespace,
body: {
apiVersion: 'batch/v1',
kind: 'Job',
metadata: { name: jobName, namespace },
spec: {
ttlSecondsAfterFinished: 300,
backoffLimit: 1,
template: {
spec: {
restartPolicy: 'Never',
containers: [
{
name: 'copy',
image: 'busybox:1.36',
command: ['sh', '-c', 'set -e; mkdir -p /dest; if [ -d /src ] && [ "$(ls -A /src 2>/dev/null)" ]; then cp -a /src/. /dest/; fi; touch /dest/.cloudhost-migrated'],
volumeMounts: [
{ name: 'src', mountPath: '/src', readOnly: true },
{ name: 'dest', mountPath: '/dest' },
],
},
],
volumes: [
{
name: 'src',
persistentVolumeClaim: { claimName: oldPvcName },
},
{
name: 'dest',
persistentVolumeClaim: { claimName: newPvcName },
},
],
},
},
},
},
});
const jobOk = await this.waitForJobComplete(batchApi, coreApi, namespace, jobName, 600_000);
if (!jobOk) {
return {
success: false,
message: 'Storage migration job failed or timed out. Database was scaled down; check cluster jobs.',
};
}
const depRes = await appsApi.readNamespacedDeployment({
name: deploymentName,
namespace,
});
const existingVolumes = depRes.spec?.template?.spec?.volumes || [];
const updatedVolumes = existingVolumes.map((vol) => {
if (vol.name === 'db-storage' && vol.persistentVolumeClaim) {
return { ...vol, persistentVolumeClaim: { claimName: newPvcName } };
}
return vol;
});
if (!updatedVolumes.some((v) => v.name === 'db-storage')) {
updatedVolumes.push({
name: 'db-storage',
persistentVolumeClaim: { claimName: newPvcName },
});
}
await appsApi.patchNamespacedDeployment(
{
name: deploymentName,
namespace,
body: {
spec: {
replicas: 1,
template: { spec: { volumes: updatedVolumes } },
},
},
},
k8s.setHeaderOptions('Content-Type', 'application/strategic-merge-patch+json'),
);
try {
await coreApi.deleteNamespacedPersistentVolumeClaim({
name: oldPvcName,
namespace,
});
} catch {
this.logger.warn(`Could not delete old PVC ${oldPvcName} after migration`);
}
this.logger.log(`Migrated database PVC ${oldPvcName} → ${newPvcName} (${newSize})`);
return {
success: true,
message: `Database storage migrated to expandable disk and set to ${newSize}. A brief restart was required.`,
};
} catch (e: any) {
this.logger.error(`PVC migration failed for ${app.name}: ${e.message}`);
try {
await appsApi.patchNamespacedDeployment({ name: deploymentName, namespace, body: { spec: { replicas: 1 } } }, k8s.setHeaderOptions('Content-Type', 'application/strategic-merge-patch+json'));
} catch {}
return {
success: false,
message: e.body?.message || e.message || 'Failed to migrate database storage',
};
}
}
private async waitForDeploymentReplicas(appsApi: k8s.AppsV1Api, namespace: string, name: string, target: number, timeoutMs: number): Promise<boolean> {
const start = Date.now();
while (Date.now() - start < timeoutMs) {
try {
const dep = await appsApi.readNamespacedDeployment({ name, namespace });
const ready = dep.status?.readyReplicas ?? 0;
const replicas = dep.spec?.replicas ?? 0;
if (target === 0 && replicas === 0) return true;
if (target > 0 && ready >= target && replicas >= target) return true;
} catch {}
await new Promise((r) => setTimeout(r, 3000));
}
return false;
}
private async waitForJobComplete(batchApi: k8s.BatchV1Api, coreApi: k8s.CoreV1Api, namespace: string, jobName: string, timeoutMs: number): Promise<boolean> {
const start = Date.now();
while (Date.now() - start < timeoutMs) {
try {
const job = await batchApi.readNamespacedJob({
name: jobName,
namespace,
});
const succeeded = job.status?.succeeded ?? 0;
const failed = job.status?.failed ?? 0;
if (succeeded > 0) return true;
if (failed > 0) return false;
} catch {}
await new Promise((r) => setTimeout(r, 3000));
}
return false;
}
/**
* Resize (expand) the database PVC for an application.
* K8s only supports PVC expansion, not shrinking.
*/
async resizeDatabasePvc(app: Application, newSize: string): Promise<{ success: boolean; message: string }> {
const { coreApi, kc } = await this.k8sClientService.getK8sClient(app.clusterId);
const namespace = this.getUserNamespace(app.userId);
const pvcName = `${app.name}-db`;
try {
const currentPvc = await coreApi.readNamespacedPersistentVolumeClaim({
name: pvcName,
namespace,
});
const currentSize = currentPvc.spec?.resources?.requests?.storage || '1Gi';
const currentGi = parseInt(String(currentSize).replace(/Gi/i, ''), 10) || 1;
const newGi = parseInt(String(newSize).replace(/Gi/i, ''), 10) || 1;
if (newGi <= currentGi) {
return {
success: false,
message: `New size (${newSize}) must be larger than current size (${currentSize})`,
};
}
const scName = await this.resolvePvcStorageClassName(coreApi, currentPvc);
if (scName) {
const scCheck = await this.ensureStorageClassAllowsExpansion(kc, scName);
if (!scCheck.ok) {
return {
success: false,
message: scCheck.message || 'StorageClass does not allow expansion',
};
}
}
try {
await this.patchPvcStorageSize(coreApi, pvcName, namespace, newSize);
this.logger.log(`Resized PVC ${pvcName} from ${currentSize} to ${newSize}`);
return {
success: true,
message: `Database storage expanded from ${currentSize} to ${newSize}`,
};
} catch (patchErr: any) {
if (!this.isPvcResizeForbiddenError(patchErr)) {
throw patchErr;
}
const targetSc = this.configService.get<string>('platform.storageClass');
if (!targetSc) {
return {
success: false,
message: 'This disk cannot be expanded in place. Set PLATFORM_STORAGE_CLASS (e.g. cloudhost-expandable) and redeploy, or contact support.',
};
}
if (pvcName.endsWith('-resizable')) {
return {
success: false,
message: patchErr.body?.message || patchErr.message || 'Failed to resize database storage',
};
}
this.logger.warn(`In-place resize failed for ${pvcName}, migrating to StorageClass ${targetSc}`);
return this.migrateDatabasePvcToResizableStorage(app, newSize, targetSc);
}
} catch (e: any) {
this.logger.error(`Failed to resize PVC ${pvcName}: ${e.message}`);
return {
success: false,
message: e.body?.message || e.message || 'Failed to resize database storage',
};
}
}
/**
* Get current PVC size for an application's database.
*/
async getDatabasePvcSize(app: Application): Promise<string> {
try {
const { coreApi } = await this.k8sClientService.getK8sClient(app.clusterId);
const namespace = this.getUserNamespace(app.userId);
const pvcName = `${app.name}-db`;
const pvc = await coreApi.readNamespacedPersistentVolumeClaim({
name: pvcName,
namespace,
});
return pvc.spec?.resources?.requests?.storage || app.dbStorageSize || '1Gi';
} catch {
return app.dbStorageSize || '1Gi';
}
}
/**
* Get comprehensive storage usage for an application.
* Includes app + DB PVCs and optional Redis/RabbitMQ data volumes when enabled.
*/
async getStorageUsage(app: Application): Promise<{
database: StorageUsageSlice | null;
appStorage: StorageUsageSlice | null;
redisStorage: StorageUsageSlice | null;
rabbitmqStorage: StorageUsageSlice | null;
totalAllocatedGb: number;
totalUsedGb: number;
}> {
const { coreApi } = await this.k8sClientService.getK8sClient(app.clusterId);
const namespace = this.getUserNamespace(app.userId);
const result = {
database: null as StorageUsageSlice | null,
appStorage: null as StorageUsageSlice | null,
redisStorage: null as StorageUsageSlice | null,
rabbitmqStorage: null as StorageUsageSlice | null,
totalAllocatedGb: 0,
totalUsedGb: 0,
};
const parseToGb = (size: string): number => {
if (!size) return 0;
const match = size.match(/^(\d+(?:\.\d+)?)(Ki|Mi|Gi|Ti)?$/i);
if (!match) return 0;
const value = parseFloat(match[1]);
const unit = (match[2] || 'Gi').toLowerCase();
switch (unit) {
case 'ki':
return value / (1024 * 1024);
case 'mi':
return value / 1024;
case 'gi':
return value;
case 'ti':
return value * 1024;
default:
return value;
}
};
const formatSize = (gb: number): string => {
if (gb >= 1) return `${gb.toFixed(1)}Gi`;
const mb = gb * 1024;
if (mb >= 1) return `${mb.toFixed(0)}Mi`;
return `${(mb * 1024).toFixed(0)}Ki`;
};
const makeSlice = (allocatedStr: string, allocatedGb: number, usedGb: number): StorageUsageSlice => {
const availableGb = Math.max(0, allocatedGb - usedGb);
const usedPercent = allocatedGb > 0 ? Math.round((usedGb / allocatedGb) * 100) : 0;
return {
allocatedRaw: allocatedStr,
allocatedGi: allocatedGb,
usedGi: usedGb,
availableGi: availableGb,
usedPercent,
allocated: allocatedStr,
used: formatSize(usedGb),
available: formatSize(availableGb),
};
};
if (app.databaseType && app.databaseType !== DatabaseType.NONE) {
try {
const pvcName = `${app.name}-db`;
const pvc = await coreApi.readNamespacedPersistentVolumeClaim({
name: pvcName,
namespace,
});
const allocatedStr = pvc.spec?.resources?.requests?.storage || app.dbStorageSize || '1Gi';
const allocatedGb = parseToGb(allocatedStr);
let usedGb = 0;
try {
const dbDep = `${app.name}-db`;
const dbContainer = `${app.name}-db`;
if (app.databaseType === DatabaseType.POSTGRESQL) {
usedGb = await this.getPvcUsageFromPod(app, dbDep, '/var/lib/postgresql/data', namespace, dbContainer);
} else if (app.databaseType === DatabaseType.MYSQL || app.databaseType === DatabaseType.MARIADB) {
usedGb = await this.getPvcUsageFromPod(app, dbDep, '/var/lib/mysql', namespace, dbContainer);
} else if (app.databaseType === DatabaseType.MONGODB) {
usedGb = await this.getPvcUsageFromPod(app, dbDep, '/data/db', namespace, dbContainer);
}
} catch {
usedGb = allocatedGb * 0.1;
}
result.database = makeSlice(allocatedStr, allocatedGb, usedGb);
result.totalAllocatedGb += allocatedGb;
result.totalUsedGb += usedGb;
} catch (e: any) {
this.logger.warn(`Failed to get DB storage usage for ${app.name}: ${e.message}`);
}
}
try {
let pvc;
let pvcName = `${app.name}-storage`;
try {
pvc = await coreApi.readNamespacedPersistentVolumeClaim({
name: pvcName,
namespace,
});
} catch {
pvcName = `${app.name}-wp-content`;
pvc = await coreApi.readNamespacedPersistentVolumeClaim({
name: pvcName,
namespace,
});
}
const allocatedStr = pvc.spec?.resources?.requests?.storage || app.appStorageSize || '2Gi';
const allocatedGb = parseToGb(allocatedStr);
const mountPath = this.getStorageMountPath(app.runtime);
let usedGb = 0;
try {
usedGb = await this.getPvcUsageFromPod(app, app.name, mountPath, namespace, app.name);
} catch {
usedGb = allocatedGb * 0.1;
}
result.appStorage = makeSlice(allocatedStr, allocatedGb, usedGb);
result.totalAllocatedGb += allocatedGb;
result.totalUsedGb += usedGb;
} catch (e: any) {
this.logger.warn(`Failed to get app storage usage for ${app.name}: ${e.message}`);
}
if (app.enableRedis) {
try {
const redisPvc = `${app.name}-redis-data`;
const pvc = await coreApi.readNamespacedPersistentVolumeClaim({
name: redisPvc,
namespace,
});
const allocatedStr = pvc.spec?.resources?.requests?.storage || '1Gi';
const allocatedGb = parseToGb(allocatedStr);
let usedGb = 0;
try {
usedGb = await this.getPvcUsageFromPod(app, `${app.name}-redis`, '/data', namespace, 'redis');
} catch {
usedGb = allocatedGb * 0.05;
}
result.redisStorage = makeSlice(allocatedStr, allocatedGb, usedGb);
result.totalAllocatedGb += allocatedGb;
result.totalUsedGb += usedGb;
} catch (e: any) {
this.logger.warn(`Failed to get Redis storage usage for ${app.name}: ${e.message}`);
}
}
if (app.enableRabbitmq) {
try {
const mqPvc = `${app.name}-rabbitmq-data`;
const pvc = await coreApi.readNamespacedPersistentVolumeClaim({
name: mqPvc,
namespace,
});
const allocatedStr = pvc.spec?.resources?.requests?.storage || '1Gi';
const allocatedGb = parseToGb(allocatedStr);
let usedGb = 0;
try {
usedGb = await this.getPvcUsageFromPod(app, `${app.name}-rabbitmq`, '/var/lib/rabbitmq', namespace, 'rabbitmq');
} catch {
usedGb = allocatedGb * 0.05;
}
result.rabbitmqStorage = makeSlice(allocatedStr, allocatedGb, usedGb);
result.totalAllocatedGb += allocatedGb;
result.totalUsedGb += usedGb;
} catch (e: any) {
this.logger.warn(`Failed to get RabbitMQ storage usage for ${app.name}: ${e.message}`);
}
}
return result;
}
/**
* Get PVC usage by executing du command in a pod.
* Returns usage in GB.
*/
private async getPvcUsageFromPod(app: Application, deploymentName: string, mountPath: string, namespace: string, containerName: string): Promise<number> {
const { coreApi, kc } = await this.k8sClientService.getK8sClient(app.clusterId);
const pods = await coreApi.listNamespacedPod({
namespace,
labelSelector: `app=${deploymentName}`,
});
const runningPod = pods.items.find((p) => p.status?.phase === 'Running');
if (!runningPod?.metadata?.name) {
throw new Error('No running pod found');
}
const exec = new k8s.Exec(kc);
const chunks: Buffer[] = [];
const stdout = new PassThrough();
stdout.on('data', (chunk: Buffer) => chunks.push(chunk));
await new Promise<void>((resolve, reject) => {
exec.exec(namespace, runningPod.metadata!.name!, containerName, ['du', '-sb', mountPath], stdout, null, null, false, (status: k8s.V1Status) => {
if (status.status === 'Success') resolve();
else reject(new Error(status.message || 'du command failed'));
});
});
const output = Buffer.concat(chunks).toString().trim();
const bytes = parseInt(output.split(/\s+/)[0], 10) || 0;
return bytes / (1024 * 1024 * 1024);
}
/**
* Expand a named PVC (Redis, RabbitMQ, or other optional service volumes).
*/
async resizeNamedPvc(app: Application, pvcName: string, newSize: string, label: string): Promise<{ success: boolean; message: string }> {
const { coreApi } = await this.k8sClientService.getK8sClient(app.clusterId);
const namespace = this.getUserNamespace(app.userId);
try {
const pvc = await coreApi.readNamespacedPersistentVolumeClaim({
name: pvcName,
namespace,
});
const currentSize = pvc.spec?.resources?.requests?.storage || '1Gi';
const parseGi = (s: string) => parseInt(String(s).replace(/Gi/i, ''), 10) || 0;
if (parseGi(newSize) <= parseGi(currentSize)) {
return {
success: false,
message: `New size (${newSize}) must be larger than current size (${currentSize})`,
};
}
await this.patchPvcStorageSize(coreApi, pvcName, namespace, newSize);
this.logger.log(`Expanded ${pvcName} from ${currentSize} to ${newSize}`);
return {
success: true,
message: `${label} storage expanded from ${currentSize} to ${newSize}`,
};
} catch (e: any) {
this.logger.error(`Failed to resize ${pvcName}: ${e.message}`);
return {
success: false,
message: e.body?.message || e.message || `Failed to resize ${label} storage`,
};
}
}
async resizeRedisStoragePvc(app: Application, newSize: string): Promise<{ success: boolean; message: string }> {
if (!app.enableRedis) {
return {
success: false,
message: 'Redis is not enabled for this application',
};
}
return this.resizeNamedPvc(app, `${app.name}-redis-data`, newSize, 'Redis');
}
async resizeRabbitmqStoragePvc(app: Application, newSize: string): Promise<{ success: boolean; message: string }> {
if (!app.enableRabbitmq) {
return {
success: false,
message: 'RabbitMQ is not enabled for this application',
};
}
return this.resizeNamedPvc(app, `${app.name}-rabbitmq-data`, newSize, 'RabbitMQ');
}
/**
* Resize app storage PVC (all app types).
*/
async resizeAppStoragePvc(app: Application, newSize: string): Promise<{ success: boolean; message: string }> {
const { coreApi } = await this.k8sClientService.getK8sClient(app.clusterId);
const namespace = this.getUserNamespace(app.userId);
// Try new unified name first, then legacy wp-content name
let pvcName = `${app.name}-storage`;
let pvcResponse;
try {
pvcResponse = await coreApi.readNamespacedPersistentVolumeClaim({
name: pvcName,
namespace,
});
} catch {
// Fallback to legacy WordPress PVC name
pvcName = `${app.name}-wp-content`;
pvcResponse = await coreApi.readNamespacedPersistentVolumeClaim({
name: pvcName,
namespace,
});
}
try {
const currentSize = pvcResponse.spec?.resources?.requests?.storage || '2Gi';
// Parse sizes for comparison
const parseGi = (s: string) => parseInt(s.replace(/Gi$/i, ''), 10) || 0;
if (parseGi(newSize) <= parseGi(currentSize)) {
return {
success: false,
message: `Cannot shrink storage. Current: ${currentSize}, Requested: ${newSize}`,
};
}
// Patch PVC to expand
await coreApi.patchNamespacedPersistentVolumeClaim(
{
name: pvcName,
namespace,
body: { spec: { resources: { requests: { storage: newSize } } } },
},
k8s.setHeaderOptions('Content-Type', 'application/merge-patch+json'),
);
this.logger.log(`Expanded ${pvcName} from ${currentSize} to ${newSize}`);
return {
success: true,
message: `App storage expanded from ${currentSize} to ${newSize}`,
};
} catch (e: any) {
this.logger.error(`Failed to resize ${pvcName}: ${e.message}`);
return {
success: false,
message: e.body?.message || e.message || 'Failed to resize app storage',
};
}
}
// ─── Snapshot helpers ───────────────────────────────
/**
* Per-database tooling for dump/restore jobs. `dumpCommand` writes to
* `outputPath`; `restoreCommand` reads from `/dump/dump.sql` (the copied
* dump file keeps that name regardless of format — mongodump archives are
* binary but mongorestore does not care about the extension).
*/
private databaseDumpSpec(app: Application, dbHost: string): {
image: string;
outputPath: string;
dumpCommand: string[];
restoreCommand: string[];
} {
const dbDatabase = app.name.replace(/-/g, '_');
switch (app.databaseType) {
case DatabaseType.POSTGRESQL: {
const image = `postgres:${app.dbVersion || '16'}-alpine`;
return {
image,
outputPath: '/dump/output.sql',
dumpCommand: ['sh', '-c', `PGPASSWORD="$DB_PASSWORD" pg_dump -h ${dbHost} -U "$DB_USER" -d ${dbDatabase} --no-owner --no-acl > /dump/output.sql 2>/tmp/err; cat /tmp/err >&2; echo "DUMP_DONE"; sleep 900`],
restoreCommand: ['sh', '-c', `PGPASSWORD="$DB_PASSWORD" psql -h ${dbHost} -U "$DB_USER" -d ${dbDatabase} -f /dump/dump.sql 2>&1`],
};
}
case DatabaseType.MONGODB: {
const image = `mongo:${app.dbVersion || '7.0'}`;
const auth = `-u "$DB_USER" -p "$DB_PASSWORD" --authenticationDatabase admin`;
return {
image,
outputPath: '/dump/output.archive',
dumpCommand: ['sh', '-c', `mongodump --host ${dbHost} ${auth} --db ${dbDatabase} --archive=/dump/output.archive --gzip 2>/tmp/err; cat /tmp/err >&2; echo "DUMP_DONE"; sleep 900`],
restoreCommand: ['sh', '-c', `mongorestore --host ${dbHost} ${auth} --nsInclude '${dbDatabase}.*' --archive=/dump/dump.sql --gzip --drop 2>&1`],
};
}
case DatabaseType.MARIADB: {
const image = `mariadb:${app.dbVersion || '11'}`;
return {
image,
outputPath: '/dump/output.sql',
dumpCommand: ['sh', '-c', `mariadb-dump -h ${dbHost} -u "$DB_USER" -p"$DB_PASSWORD" ${dbDatabase} > /dump/output.sql 2>/tmp/err; cat /tmp/err >&2; echo "DUMP_DONE"; sleep 900`],
restoreCommand: ['sh', '-c', `mariadb -h ${dbHost} -u "$DB_USER" -p"$DB_PASSWORD" ${dbDatabase} < /dump/dump.sql 2>&1`],
};
}
default: {
const image = `mysql:${app.dbVersion || '8.0'}`;
return {
image,
outputPath: '/dump/output.sql',
dumpCommand: ['sh', '-c', `mysqldump -h ${dbHost} -u "$DB_USER" -p"$DB_PASSWORD" ${dbDatabase} > /dump/output.sql 2>/tmp/err; cat /tmp/err >&2; echo "DUMP_DONE"; sleep 900`],
restoreCommand: ['sh', '-c', `mysql -h ${dbHost} -u "$DB_USER" -p"$DB_PASSWORD" ${dbDatabase} < /dump/dump.sql 2>&1`],
};
}
}
}
/**
* Export (dump) the application database to a local file via a K8s Job.
* Returns the dump as a Buffer, or null on failure.
*
* Strategy: Run dump command, then sleep for 60s to allow exec retrieval.
*/
async exportDatabaseDump(app: Application, onProgress?: (percent: number) => void): Promise<{ data: Buffer | null; logs: string }> {
const { coreApi, kc } = await this.k8sClientService.getK8sClient(app.clusterId);
const batchApi = kc.makeApiClient(k8s.BatchV1Api);
const namespace = this.getUserNamespace(app.userId);
const dbName = `${app.name}-db`;
const jobName = `${app.name}-db-dump-${Date.now()}`;
// Dump command writes to spec.outputPath, then sleeps to allow exec retrieval
const { image, outputPath, dumpCommand: command } = this.databaseDumpSpec(app, dbName);
const job: k8s.V1Job = {
apiVersion: 'batch/v1',
kind: 'Job',
metadata: { name: jobName, namespace },
spec: {
ttlSecondsAfterFinished: 180,
activeDeadlineSeconds: 1200,
backoffLimit: 0,
template: {
spec: {
restartPolicy: 'Never',
containers: [
{
name: 'dump',
image,
command,
env: [
{
name: 'DB_USER',
valueFrom: {
secretKeyRef: {
name: `${app.name}-db-secret`,
key: 'username',
},
},
},
{
name: 'DB_PASSWORD',
valueFrom: {
secretKeyRef: {
name: `${app.name}-db-secret`,
key: 'password',
},
},
},
],
volumeMounts: [{ name: 'dump-vol', mountPath: '/dump' }],
resources: {
requests: { cpu: '100m', memory: '128Mi' },
limits: { cpu: '500m', memory: '512Mi' },
},
},
],
volumes: [{ name: 'dump-vol', emptyDir: {} }],
},
},
},
};
try {
await batchApi.createNamespacedJob({ namespace, body: job });
} catch (e: any) {
this.logger.error(`Failed to create DB dump job: ${e.message}`);
return { data: null, logs: `Failed to create dump job: ${e.message}` };
}
// Wait for dump to complete (check logs for DUMP_DONE marker)
const timeout = 900_000;
const start = Date.now();
let dumpDone = false;
let podName: string | undefined;
while (Date.now() - start < timeout) {
await new Promise((r) => setTimeout(r, 3000));
const elapsed = Date.now() - start;
const waitPct = Math.min(75, Math.round((elapsed / timeout) * 75));
onProgress?.(10 + waitPct);
try {
const pods = await coreApi.listNamespacedPod({
namespace,
labelSelector: `job-name=${jobName}`,
});
if (pods.items.length > 0) {
podName = pods.items[0].metadata?.name;
const phase = pods.items[0].status?.phase;
// Check if pod is Running (container is in sleep phase after dump)
if (podName && phase === 'Running') {
try {
const logRes = await coreApi.readNamespacedPodLog({
name: podName,
namespace,
container: 'dump',
follow: false,
tailLines: 50,
});
if (logRes?.includes('DUMP_DONE')) {
dumpDone = true;
onProgress?.(88);
break;
}
} catch {}
}
// If Failed, exit early
if (phase === 'Failed') break;
}
} catch {}
}
let dumpBuffer: Buffer | null = null;
let logs = '';
if (dumpDone && podName) {
try {
// Use kubectl cp equivalent via Exec with proper streams
const exec = new k8s.Exec(kc);
const chunks: Buffer[] = [];
const stdoutStream = new PassThrough();
const stderrStream = new PassThrough();
stdoutStream.on('data', (chunk: Buffer) => {
chunks.push(chunk);
});
stderrStream.on('data', (chunk: Buffer) => {
logs += chunk.toString();
});
await new Promise<void>((resolve, reject) => {
exec.exec(namespace, podName!, 'dump', ['cat', outputPath], stdoutStream, stderrStream, null, false, (status: k8s.V1Status) => {
if (status.status === 'Success') resolve();
else reject(new Error(status.message || 'exec failed'));
});
});
if (chunks.length > 0) {
dumpBuffer = Buffer.concat(chunks);
onProgress?.(95);
this.logger.log(`DB dump retrieved: ${dumpBuffer.length} bytes`);
}
} catch (e: any) {
this.logger.warn(`Exec failed for DB dump: ${e.message}`);
logs = e.message;
}
}
// Cleanup: delete the job early to free resources
try {
await batchApi.deleteNamespacedJob({
name: jobName,
namespace,
propagationPolicy: 'Background',
});
} catch {}
if (!dumpBuffer) {
return {
data: null,
logs: logs || 'Dump failed or could not be retrieved',
};
}
return { data: dumpBuffer, logs: 'OK' };
}
/**
* Archive the wp-content directory from a WordPress app's PVC via a K8s Job.
* The job creates a tar.gz of /var/www/html/wp-content and we retrieve it via exec.
*
* Strategy: Create archive, then sleep to allow exec retrieval.
*/
async archiveWpContent(app: Application): Promise<{ data: Buffer | null; logs: string }> {
const { coreApi, kc } = await this.k8sClientService.getK8sClient(app.clusterId);
const batchApi = kc.makeApiClient(k8s.BatchV1Api);
const namespace = this.getUserNamespace(app.userId);
const pvcName = `${app.name}-storage`;
const jobName = `${app.name}-wp-archive-${Date.now()}`;
const job: k8s.V1Job = {
apiVersion: 'batch/v1',
kind: 'Job',
metadata: { name: jobName, namespace },
spec: {
ttlSecondsAfterFinished: 180,
activeDeadlineSeconds: 1200,
backoffLimit: 0,
template: {
spec: {
restartPolicy: 'Never',
containers: [
{
name: 'archiver',
image: 'alpine:3.19',
command: ['sh', '-c', 'apk add --no-cache tar gzip > /dev/null 2>&1 && cd /wp-content && tar czf /output/wp-content.tar.gz . && echo "ARCHIVE_DONE" && sleep 900'],
volumeMounts: [
{
name: 'wp-content',
mountPath: '/wp-content',
readOnly: true,
},
{ name: 'output', mountPath: '/output' },
],
resources: {
requests: { cpu: '100m', memory: '64Mi' },
limits: { cpu: '500m', memory: '512Mi' },
},
},
],
volumes: [
{
name: 'wp-content',
persistentVolumeClaim: { claimName: pvcName },
},
{ name: 'output', emptyDir: {} },
],
},
},
},
};
try {
await batchApi.createNamespacedJob({ namespace, body: job });
} catch (e: any) {
this.logger.error(`Failed to create wp-content archive job: ${e.message}`);
return { data: null, logs: `Failed to create archive job: ${e.message}` };
}
// Wait for archive to complete (check logs for ARCHIVE_DONE marker)
const timeout = 900_000;
const start = Date.now();
let archiveDone = false;
let podName: string | undefined;
while (Date.now() - start < timeout) {
await new Promise((r) => setTimeout(r, 3000));
try {
const pods = await coreApi.listNamespacedPod({
namespace,
labelSelector: `job-name=${jobName}`,
});
if (pods.items.length > 0) {
podName = pods.items[0].metadata?.name;
const phase = pods.items[0].status?.phase;
if (podName && phase === 'Running') {
try {
const logRes = await coreApi.readNamespacedPodLog({
name: podName,
namespace,
container: 'archiver',
follow: false,
tailLines: 50,
});
if (logRes?.includes('ARCHIVE_DONE')) {
archiveDone = true;
break;
}
} catch {}
}
if (phase === 'Failed') break;
}
} catch {}
}
let archiveBuffer: Buffer | null = null;
let logs = '';
if (archiveDone && podName) {
try {
const exec = new k8s.Exec(kc);
const chunks: Buffer[] = [];
const stdoutStream = new PassThrough();
const stderrStream = new PassThrough();
stdoutStream.on('data', (chunk: Buffer) => {
chunks.push(chunk);
});
stderrStream.on('data', (chunk: Buffer) => {
logs += chunk.toString();
});
await new Promise<void>((resolve, reject) => {
exec.exec(namespace, podName!, 'archiver', ['cat', '/output/wp-content.tar.gz'], stdoutStream, stderrStream, null, false, (status: k8s.V1Status) => {
if (status.status === 'Success') resolve();
else reject(new Error(status.message || 'exec failed'));
});
});
if (chunks.length > 0) {
archiveBuffer = Buffer.concat(chunks);
this.logger.log(`wp-content archive retrieved: ${archiveBuffer.length} bytes`);
}
} catch (e: any) {
this.logger.warn(`Exec failed for wp-content archive: ${e.message}`);
logs = e.message;
}
}
// Cleanup
try {
await batchApi.deleteNamespacedJob({
name: jobName,
namespace,
propagationPolicy: 'Background',
});
} catch {}
if (!archiveBuffer) {
return {
data: null,
logs: logs || 'Archive failed or could not be retrieved',
};
}
return { data: archiveBuffer, logs: 'OK' };
}
/**
* Restore wp-content from a tar.gz archive into the WordPress PVC.
*
* The archive is streamed into a helper pod with `kubectl cp` (a Secret
* would be capped at ~1MiB — far too small for real wp-content) and
* extracted in place onto the mounted PVC.
*/
async restoreWpContent(app: Application, archiveBuffer: Buffer): Promise<{ success: boolean; logs: string }> {
const { coreApi, kc } = await this.k8sClientService.getK8sClient(app.clusterId);
const namespace = this.getUserNamespace(app.userId);
const pvcName = `${app.name}-storage`;
const ts = Date.now();
const helperPodName = `${app.name}-wp-restore-${ts}`;
const helperPod: k8s.V1Pod = {
apiVersion: 'v1',
kind: 'Pod',
metadata: { name: helperPodName, namespace },
spec: {
containers: [
{
name: 'restore',
image: 'alpine:3.19',
command: ['sh', '-c', 'sleep 3600'],
volumeMounts: [{ name: 'wp-content', mountPath: '/wp-content' }],
resources: {
requests: { cpu: '100m', memory: '128Mi' },
limits: { cpu: '500m', memory: '512Mi' },
},
},
],
volumes: [{ name: 'wp-content', persistentVolumeClaim: { claimName: pvcName } }],
restartPolicy: 'Never',
},
};
const tmpArchive = path.join(os.tmpdir(), `wp-content-restore-${ts}.tar.gz`);
const tmpKubeconfig = path.join(os.tmpdir(), `kubeconfig-wprestore-${ts}.yaml`);
try {
fs.writeFileSync(tmpArchive, archiveBuffer);
fs.writeFileSync(tmpKubeconfig, kc.exportConfig());
await coreApi.createNamespacedPod({ namespace, body: helperPod });
// Wait for helper pod Running
const podTimeout = 120_000;
const podStart = Date.now();
while (Date.now() - podStart < podTimeout) {
const pod = await coreApi.readNamespacedPod({ name: helperPodName, namespace });
if (pod.status?.phase === 'Running') break;
if (pod.status?.phase === 'Failed') throw new Error('wp-content restore helper pod failed to start');
await new Promise((r) => setTimeout(r, 2000));
}
await execFileAsync(
'kubectl',
['--kubeconfig', tmpKubeconfig, 'cp', tmpArchive, `${namespace}/${helperPodName}:/tmp/wp-content.tar.gz`, '--retries', '3'],
{ maxBuffer: 50 * 1024 * 1024, timeout: 600_000 },
);
const { stdout, stderr } = await execFileAsync(
'kubectl',
[
'--kubeconfig', tmpKubeconfig, 'exec', '-n', namespace, helperPodName, '--',
'sh', '-c',
'rm -rf /wp-content/* /wp-content/.[!.]* 2>/dev/null; tar xzf /tmp/wp-content.tar.gz -C /wp-content && echo RESTORE_DONE',
],
{ maxBuffer: 10 * 1024 * 1024, timeout: 600_000 },
);
const logs = `${stdout || ''}${stderr || ''}`;
const success = logs.includes('RESTORE_DONE');
return { success, logs: logs || (success ? 'Restore completed' : 'Restore failed') };
} catch (e: any) {
this.logger.error(`wp-content restore failed for ${app.name}: ${e.message}`);
return { success: false, logs: e.message || 'wp-content restore failed' };
} finally {
try {
fs.unlinkSync(tmpArchive);
} catch {}
try {
fs.unlinkSync(tmpKubeconfig);
} catch {}
try {
await coreApi.deleteNamespacedPod({ name: helperPodName, namespace });
} catch {}
}
}
// ─── K8s Revision-based Rollback ─────────────────────
/**
* Get the list of Helm release revisions for an application.
* Returns up to 10 revisions sorted newest-first.
*/
async getDeploymentRevisions(app: Application): Promise<{
revisions: Array<{
revision: number;
image: string;
changeCause: string;
createdAt: string;
replicas: number;
isCurrent: boolean;
}>;
currentRevision: number;
}> {
const namespace = this.getUserNamespace(app.userId);
const releaseName = app.name;
try {
const kubeconfig = await this.k8sClientService.getKubeconfig(app.clusterId);
const helmRevisions = await this.helmService.history(releaseName, namespace, kubeconfig);
if (!helmRevisions || helmRevisions.length === 0) {
return { revisions: [], currentRevision: 0 };
}
// The last "deployed" revision is the current one
const currentRev = helmRevisions.filter((r) => r.status === 'deployed').sort((a, b) => b.revision - a.revision)[0];
const currentRevision = currentRev ? currentRev.revision : 0;
const revisions = helmRevisions
.sort((a, b) => b.revision - a.revision)
.slice(0, 10)
.map((r) => ({
revision: r.revision,
image: '', // Helm history doesn't expose image; frontend will use description
changeCause: r.description || `${r.status} — ${r.chart}`,
createdAt: r.updated,
replicas: 0,
isCurrent: r.revision === currentRevision,
}));
return { revisions, currentRevision };
} catch (e: any) {
this.logger.warn(`Could not get Helm history for ${releaseName}: ${e.message}`);
return { revisions: [], currentRevision: 0 };
}
}
/**
* Rollback a Helm release to a specific revision.
*/
async rollbackDeploymentRevision(app: Application, targetRevision: number): Promise<{ success: boolean; message: string }> {
const namespace = this.getUserNamespace(app.userId);
const releaseName = app.name;
try {
const kubeconfig = await this.k8sClientService.getKubeconfig(app.clusterId);
await this.helmService.rollback(releaseName, targetRevision, namespace, kubeconfig);
this.logger.log(`Rolled back ${releaseName} to Helm revision ${targetRevision}`);
return {
success: true,
message: `Rolled back to Helm revision ${targetRevision}`,
};
} catch (e: any) {
this.logger.error(`Helm rollback failed for ${releaseName}: ${e.message}`);
return { success: false, message: e.message };
}
}
private generatePassword(length = 24): string {
const chars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789';
let password = '';
for (let i = 0; i < length; i++) {
password += chars.charAt(Math.floor(Math.random() * chars.length));
}
return password;
}
}
Reference in New Issue View Git Blame Copy Permalink