fix(k8s): restore direct K8s API deploy as fallback when Helm unavailable
- deployApplication now tries Helm first, falls back to direct K8s API - Restored all private methods: ensureNamespace, applySecret, applyDeployment, applyService, applyIngress, applyWordPressPvc, deployDatabase, createDbSecret, createPVC - Added ManifestContext interface - Added DATABASE_URL env var, PGDATA, health probes for DB, ingressClassName
This commit is contained in:
keyhan
@@ -12,6 +12,27 @@ import { HelmService } from './helm.service';
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
interface ManifestContext {
|
||||
appName: string;
|
||||
namespace: string;
|
||||
image: string;
|
||||
port: number;
|
||||
replicas: number;
|
||||
cpuRequest: string;
|
||||
cpuLimit: string;
|
||||
memoryRequest: string;
|
||||
memoryLimit: string;
|
||||
envVars: Record<string, string>;
|
||||
runtime: string;
|
||||
databaseType: string;
|
||||
domain: string;
|
||||
subdomain: string;
|
||||
dbUsername: string;
|
||||
dbPassword: string;
|
||||
dbVersion: string;
|
||||
dbStorageSize: string;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class KubernetesService implements OnModuleInit {
|
||||
private readonly logger = new Logger(KubernetesService.name);
|
||||
@@ -117,12 +138,25 @@ export class KubernetesService implements OnModuleInit {
|
||||
}
|
||||
|
||||
async deployApplication(app: Application, imageUri: string): Promise<Record<string, any>> {
|
||||
// Try Helm first, fall back to direct K8s API if Helm is unavailable
|
||||
try {
|
||||
return await this.deployViaHelm(app, imageUri);
|
||||
} catch (helmError: any) {
|
||||
this.logger.warn(
|
||||
`Helm deploy failed for ${app.name}, falling back to direct K8s API: ${helmError.message}`,
|
||||
);
|
||||
return await this.deployViaK8sApi(app, imageUri);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Helm-based deployment ─────────────────────────────────────────
|
||||
|
||||
private async deployViaHelm(app: Application, imageUri: string): Promise<Record<string, any>> {
|
||||
const kubeconfig = await this.getKubeconfig(app.clusterId);
|
||||
const values = this.buildHelmValues(app, imageUri);
|
||||
const namespace = values.app.namespace;
|
||||
const releaseName = app.name;
|
||||
|
||||
try {
|
||||
const result = await this.helmService.installOrUpgrade(
|
||||
releaseName,
|
||||
namespace,
|
||||
@@ -132,13 +166,452 @@ export class KubernetesService implements OnModuleInit {
|
||||
|
||||
this.logger.log(`Successfully deployed ${app.name} to namespace ${namespace} via Helm`);
|
||||
return { helm: { release: releaseName, namespace, stdout: result.stdout }, values };
|
||||
}
|
||||
|
||||
// ── Direct K8s API deployment (fallback) ──────────────────────────
|
||||
|
||||
private async deployViaK8sApi(app: Application, imageUri: string): Promise<Record<string, any>> {
|
||||
const { coreApi, appsApi, networkingApi } = await this.getK8sClient(app.clusterId);
|
||||
const domain = this.configService.get('platform.domain');
|
||||
|
||||
const context: ManifestContext = {
|
||||
appName: app.name,
|
||||
namespace: `user-${app.userId.split('-')[0]}`,
|
||||
image: imageUri,
|
||||
port: app.port,
|
||||
replicas: app.replicas,
|
||||
cpuRequest: app.cpuRequest,
|
||||
cpuLimit: app.cpuLimit,
|
||||
memoryRequest: app.memoryRequest,
|
||||
memoryLimit: app.memoryLimit,
|
||||
envVars: app.envVars || {},
|
||||
runtime: app.runtime,
|
||||
databaseType: app.databaseType,
|
||||
domain: domain,
|
||||
subdomain: app.subdomain || app.name,
|
||||
dbUsername: app.dbUsername || 'appuser',
|
||||
dbPassword: app.dbPassword || this.generatePassword(),
|
||||
dbVersion: app.dbVersion || '',
|
||||
dbStorageSize: app.dbStorageSize || '1Gi',
|
||||
};
|
||||
|
||||
const manifests: Record<string, any> = {};
|
||||
|
||||
try {
|
||||
// 1. Ensure namespace exists
|
||||
await this.ensureNamespace(coreApi, context.namespace);
|
||||
|
||||
// 2. Create/Update secrets for env vars
|
||||
if (Object.keys(context.envVars).length > 0) {
|
||||
manifests.secret = await this.applySecret(coreApi, context);
|
||||
}
|
||||
|
||||
// 3. Deploy database if needed
|
||||
if (context.databaseType !== DatabaseType.NONE) {
|
||||
manifests.database = await this.deployDatabase(coreApi, appsApi, context);
|
||||
}
|
||||
|
||||
// 3.5 Create wp-content PVC for WordPress
|
||||
if (context.runtime === AppRuntime.WORDPRESS) {
|
||||
manifests.wpContentPvc = await this.applyWordPressPvc(coreApi, context);
|
||||
}
|
||||
|
||||
// 4. Create Deployment
|
||||
manifests.deployment = await this.applyDeployment(appsApi, context);
|
||||
|
||||
// 5. Create Service
|
||||
manifests.service = await this.applyService(coreApi, context);
|
||||
|
||||
// 6. Create Ingress
|
||||
manifests.ingress = await this.applyIngress(networkingApi, context);
|
||||
|
||||
this.logger.log(`Successfully deployed ${app.name} to namespace ${context.namespace} via K8s API`);
|
||||
} catch (error: any) {
|
||||
this.logger.error(`Failed to deploy ${app.name} via Helm:`, error.message);
|
||||
this.logger.error(`Failed to deploy ${app.name} via K8s API:`, error.body || error.message);
|
||||
throw error;
|
||||
}
|
||||
|
||||
return manifests;
|
||||
}
|
||||
|
||||
// ── Private K8s resource methods ──────────────────────────────────
|
||||
|
||||
private async ensureNamespace(coreApi: k8s.CoreV1Api, namespace: string): Promise<void> {
|
||||
try {
|
||||
await coreApi.readNamespace(namespace);
|
||||
} catch {
|
||||
await coreApi.createNamespace({
|
||||
metadata: { name: namespace },
|
||||
});
|
||||
this.logger.log(`Created namespace: ${namespace}`);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Legacy private deploy methods removed — now handled by Helm chart ──
|
||||
private async applySecret(coreApi: k8s.CoreV1Api, ctx: ManifestContext): Promise<any> {
|
||||
const secretData: Record<string, string> = {};
|
||||
for (const [key, value] of Object.entries(ctx.envVars)) {
|
||||
secretData[key] = Buffer.from(String(value)).toString('base64');
|
||||
}
|
||||
|
||||
const secret = {
|
||||
apiVersion: 'v1',
|
||||
kind: 'Secret',
|
||||
metadata: {
|
||||
name: `${ctx.appName}-env`,
|
||||
namespace: ctx.namespace,
|
||||
},
|
||||
data: secretData,
|
||||
};
|
||||
|
||||
try {
|
||||
await coreApi.replaceNamespacedSecret(`${ctx.appName}-env`, ctx.namespace, secret);
|
||||
} catch {
|
||||
await coreApi.createNamespacedSecret(ctx.namespace, secret);
|
||||
}
|
||||
return secret;
|
||||
}
|
||||
|
||||
private async applyDeployment(appsApi: k8s.AppsV1Api, ctx: ManifestContext): Promise<any> {
|
||||
const envFrom: any[] = [];
|
||||
if (Object.keys(ctx.envVars).length > 0) {
|
||||
envFrom.push({ secretRef: { name: `${ctx.appName}-env` } });
|
||||
}
|
||||
|
||||
// Database connection env vars
|
||||
const extraEnv: any[] = [];
|
||||
if (ctx.databaseType === DatabaseType.POSTGRESQL) {
|
||||
extraEnv.push(
|
||||
{ name: 'DB_HOST', value: `${ctx.appName}-db` },
|
||||
{ name: 'DB_PORT', value: '5432' },
|
||||
{ name: 'DB_NAME', value: ctx.appName.replace(/-/g, '_') },
|
||||
{ name: 'DB_USER', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'username' } } },
|
||||
{ name: 'DB_PASSWORD', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'password' } } },
|
||||
{ name: 'DATABASE_URL', value: `postgresql://$(DB_USER):$(DB_PASSWORD)@${ctx.appName}-db:5432/${ctx.appName.replace(/-/g, '_')}` },
|
||||
);
|
||||
} else if (ctx.databaseType === DatabaseType.MYSQL) {
|
||||
extraEnv.push(
|
||||
{ name: 'DB_HOST', value: `${ctx.appName}-db` },
|
||||
{ name: 'DB_PORT', value: '3306' },
|
||||
{ name: 'DB_NAME', value: ctx.appName.replace(/-/g, '_') },
|
||||
{ name: 'DB_USER', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'username' } } },
|
||||
{ name: 'DB_PASSWORD', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'password' } } },
|
||||
{ name: 'DATABASE_URL', value: `mysql://$(DB_USER):$(DB_PASSWORD)@${ctx.appName}-db:3306/${ctx.appName.replace(/-/g, '_')}` },
|
||||
);
|
||||
}
|
||||
|
||||
// WordPress-specific env vars
|
||||
if (ctx.runtime === AppRuntime.WORDPRESS && ctx.databaseType !== DatabaseType.NONE) {
|
||||
const dbPort = ctx.databaseType === DatabaseType.POSTGRESQL ? '5432' : '3306';
|
||||
extraEnv.push(
|
||||
{ name: 'WORDPRESS_DB_HOST', value: `${ctx.appName}-db:${dbPort}` },
|
||||
{ name: 'WORDPRESS_DB_NAME', value: ctx.appName.replace(/-/g, '_') },
|
||||
{ name: 'WORDPRESS_DB_USER', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'username' } } },
|
||||
{ name: 'WORDPRESS_DB_PASSWORD', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'password' } } },
|
||||
{ name: 'WORDPRESS_TABLE_PREFIX', value: 'wp_' },
|
||||
);
|
||||
}
|
||||
|
||||
const deployment: k8s.V1Deployment = {
|
||||
apiVersion: 'apps/v1',
|
||||
kind: 'Deployment',
|
||||
metadata: {
|
||||
name: ctx.appName,
|
||||
namespace: ctx.namespace,
|
||||
labels: { app: ctx.appName, runtime: ctx.runtime },
|
||||
annotations: {
|
||||
'kubernetes.io/change-cause': `Deploy ${ctx.image} at ${new Date().toISOString()}`,
|
||||
},
|
||||
},
|
||||
spec: {
|
||||
revisionHistoryLimit: 10,
|
||||
replicas: ctx.replicas,
|
||||
selector: { matchLabels: { app: ctx.appName } },
|
||||
template: {
|
||||
metadata: { labels: { app: ctx.appName, runtime: ctx.runtime } },
|
||||
spec: {
|
||||
containers: [
|
||||
{
|
||||
name: ctx.appName,
|
||||
image: ctx.image,
|
||||
imagePullPolicy: 'Always',
|
||||
ports: [{ containerPort: ctx.port }],
|
||||
envFrom,
|
||||
env: extraEnv,
|
||||
resources: {
|
||||
requests: { cpu: ctx.cpuRequest, memory: ctx.memoryRequest },
|
||||
limits: { cpu: ctx.cpuLimit, memory: ctx.memoryLimit },
|
||||
},
|
||||
readinessProbe: {
|
||||
tcpSocket: { port: ctx.port as any },
|
||||
initialDelaySeconds: 5,
|
||||
periodSeconds: 5,
|
||||
},
|
||||
livenessProbe: {
|
||||
tcpSocket: { port: ctx.port as any },
|
||||
initialDelaySeconds: 15,
|
||||
periodSeconds: 10,
|
||||
failureThreshold: 5,
|
||||
},
|
||||
...(ctx.runtime === AppRuntime.WORDPRESS
|
||||
? {
|
||||
volumeMounts: [
|
||||
{ name: 'wp-content', mountPath: '/var/www/html/wp-content' },
|
||||
],
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
],
|
||||
...(ctx.runtime === AppRuntime.WORDPRESS
|
||||
? {
|
||||
volumes: [
|
||||
{
|
||||
name: 'wp-content',
|
||||
persistentVolumeClaim: { claimName: `${ctx.appName}-wp-content` },
|
||||
},
|
||||
],
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
await appsApi.replaceNamespacedDeployment(ctx.appName, ctx.namespace, deployment);
|
||||
} catch {
|
||||
await appsApi.createNamespacedDeployment(ctx.namespace, deployment);
|
||||
}
|
||||
return deployment;
|
||||
}
|
||||
|
||||
private async applyWordPressPvc(coreApi: k8s.CoreV1Api, ctx: ManifestContext): Promise<any> {
|
||||
const pvcName = `${ctx.appName}-wp-content`;
|
||||
const pvc = {
|
||||
apiVersion: 'v1',
|
||||
kind: 'PersistentVolumeClaim',
|
||||
metadata: { name: pvcName, namespace: ctx.namespace, labels: { app: ctx.appName } },
|
||||
spec: {
|
||||
accessModes: ['ReadWriteOnce'],
|
||||
resources: { requests: { storage: '2Gi' } },
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
await coreApi.readNamespacedPersistentVolumeClaim(pvcName, ctx.namespace);
|
||||
this.logger.log(`PVC ${pvcName} already exists, skipping`);
|
||||
} catch {
|
||||
await coreApi.createNamespacedPersistentVolumeClaim(ctx.namespace, pvc);
|
||||
this.logger.log(`Created WordPress PVC: ${pvcName}`);
|
||||
}
|
||||
return pvc;
|
||||
}
|
||||
|
||||
private async applyService(coreApi: k8s.CoreV1Api, ctx: ManifestContext): Promise<any> {
|
||||
const service: k8s.V1Service = {
|
||||
apiVersion: 'v1',
|
||||
kind: 'Service',
|
||||
metadata: { name: ctx.appName, namespace: ctx.namespace },
|
||||
spec: {
|
||||
selector: { app: ctx.appName },
|
||||
ports: [{ port: 80, targetPort: ctx.port as any, protocol: 'TCP' }],
|
||||
type: 'ClusterIP',
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
await coreApi.replaceNamespacedService(ctx.appName, ctx.namespace, service);
|
||||
} catch {
|
||||
await coreApi.createNamespacedService(ctx.namespace, service);
|
||||
}
|
||||
return service;
|
||||
}
|
||||
|
||||
private async applyIngress(networkingApi: k8s.NetworkingV1Api, ctx: ManifestContext): Promise<any> {
|
||||
const host = `${ctx.subdomain}.${ctx.domain}`;
|
||||
const ingress: k8s.V1Ingress = {
|
||||
apiVersion: 'networking.k8s.io/v1',
|
||||
kind: 'Ingress',
|
||||
metadata: {
|
||||
name: ctx.appName,
|
||||
namespace: ctx.namespace,
|
||||
annotations: {
|
||||
'cert-manager.io/cluster-issuer': 'letsencrypt-prod',
|
||||
},
|
||||
},
|
||||
spec: {
|
||||
ingressClassName: 'nginx',
|
||||
rules: [
|
||||
{
|
||||
host,
|
||||
http: {
|
||||
paths: [
|
||||
{
|
||||
path: '/',
|
||||
pathType: 'Prefix',
|
||||
backend: { service: { name: ctx.appName, port: { number: 80 } } },
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
tls: [{ hosts: [host], secretName: `${ctx.appName}-tls` }],
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
await networkingApi.replaceNamespacedIngress(ctx.appName, ctx.namespace, ingress);
|
||||
} catch {
|
||||
await networkingApi.createNamespacedIngress(ctx.namespace, ingress);
|
||||
}
|
||||
return ingress;
|
||||
}
|
||||
|
||||
private async deployDatabase(
|
||||
coreApi: k8s.CoreV1Api,
|
||||
appsApi: k8s.AppsV1Api,
|
||||
ctx: ManifestContext,
|
||||
): Promise<any> {
|
||||
const dbName = `${ctx.appName}-db`;
|
||||
|
||||
// Create DB secret
|
||||
await this.createDbSecret(coreApi, ctx.namespace, ctx.appName, ctx.dbPassword, ctx.dbUsername);
|
||||
|
||||
// Create PVC for DB
|
||||
await this.createPVC(coreApi, ctx.namespace, dbName, ctx.dbStorageSize);
|
||||
|
||||
// Deploy database
|
||||
const isPostgres = ctx.databaseType === DatabaseType.POSTGRESQL;
|
||||
const defaultDbVersion = isPostgres ? '16' : '8.0';
|
||||
const dbVer = ctx.dbVersion || defaultDbVersion;
|
||||
const image = isPostgres ? `postgres:${dbVer}-alpine` : `mysql:${dbVer}`;
|
||||
const port = isPostgres ? 5432 : 3306;
|
||||
const dataPath = isPostgres ? '/var/lib/postgresql/data' : '/var/lib/mysql';
|
||||
|
||||
const envVars = isPostgres
|
||||
? [
|
||||
{ name: 'PGDATA', value: '/var/lib/postgresql/data/pgdata' },
|
||||
{ name: 'POSTGRES_DB', value: ctx.appName.replace(/-/g, '_') },
|
||||
{ name: 'POSTGRES_USER', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'username' } } },
|
||||
{ name: 'POSTGRES_PASSWORD', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'password' } } },
|
||||
]
|
||||
: [
|
||||
{ name: 'MYSQL_DATABASE', value: ctx.appName.replace(/-/g, '_') },
|
||||
{ name: 'MYSQL_USER', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'username' } } },
|
||||
{ name: 'MYSQL_PASSWORD', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'password' } } },
|
||||
{ name: 'MYSQL_ROOT_PASSWORD', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'password' } } },
|
||||
];
|
||||
|
||||
const dbDeployment: k8s.V1Deployment = {
|
||||
apiVersion: 'apps/v1',
|
||||
kind: 'Deployment',
|
||||
metadata: { name: dbName, namespace: ctx.namespace, labels: { app: dbName } },
|
||||
spec: {
|
||||
replicas: 1,
|
||||
selector: { matchLabels: { app: dbName } },
|
||||
template: {
|
||||
metadata: { labels: { app: dbName } },
|
||||
spec: {
|
||||
containers: [
|
||||
{
|
||||
name: dbName,
|
||||
image,
|
||||
ports: [{ containerPort: port }],
|
||||
env: envVars,
|
||||
volumeMounts: [{ name: 'db-storage', mountPath: dataPath }],
|
||||
resources: {
|
||||
requests: { cpu: '100m', memory: '256Mi' },
|
||||
limits: { cpu: '500m', memory: '512Mi' },
|
||||
},
|
||||
readinessProbe: isPostgres
|
||||
? { exec: { command: ['pg_isready', '-U', ctx.dbUsername] }, initialDelaySeconds: 10, periodSeconds: 5, failureThreshold: 6 }
|
||||
: { exec: { command: ['mysqladmin', 'ping', '-h', '127.0.0.1'] }, initialDelaySeconds: 10, periodSeconds: 5, failureThreshold: 6 },
|
||||
livenessProbe: isPostgres
|
||||
? { exec: { command: ['pg_isready', '-U', ctx.dbUsername] }, initialDelaySeconds: 30, periodSeconds: 10, failureThreshold: 5 }
|
||||
: { exec: { command: ['mysqladmin', 'ping', '-h', '127.0.0.1'] }, initialDelaySeconds: 30, periodSeconds: 10, failureThreshold: 5 },
|
||||
},
|
||||
],
|
||||
volumes: [
|
||||
{ name: 'db-storage', persistentVolumeClaim: { claimName: dbName } },
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
await appsApi.replaceNamespacedDeployment(dbName, ctx.namespace, dbDeployment);
|
||||
} catch {
|
||||
await appsApi.createNamespacedDeployment(ctx.namespace, dbDeployment);
|
||||
}
|
||||
|
||||
// Create DB Service
|
||||
const dbService: k8s.V1Service = {
|
||||
apiVersion: 'v1',
|
||||
kind: 'Service',
|
||||
metadata: { name: dbName, namespace: ctx.namespace },
|
||||
spec: {
|
||||
selector: { app: dbName },
|
||||
ports: [{ port, targetPort: port as any, protocol: 'TCP' }],
|
||||
type: 'ClusterIP',
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
await coreApi.replaceNamespacedService(dbName, ctx.namespace, dbService);
|
||||
} catch {
|
||||
await coreApi.createNamespacedService(ctx.namespace, dbService);
|
||||
}
|
||||
|
||||
return { deployment: dbDeployment, service: dbService };
|
||||
}
|
||||
|
||||
private async createDbSecret(
|
||||
coreApi: k8s.CoreV1Api,
|
||||
namespace: string,
|
||||
appName: string,
|
||||
password: string,
|
||||
username: string = 'appuser',
|
||||
): Promise<void> {
|
||||
const secret = {
|
||||
apiVersion: 'v1',
|
||||
kind: 'Secret',
|
||||
metadata: { name: `${appName}-db-secret`, namespace },
|
||||
data: {
|
||||
username: Buffer.from(username).toString('base64'),
|
||||
password: Buffer.from(password).toString('base64'),
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
await coreApi.replaceNamespacedSecret(`${appName}-db-secret`, namespace, secret);
|
||||
} catch {
|
||||
await coreApi.createNamespacedSecret(namespace, secret);
|
||||
}
|
||||
}
|
||||
|
||||
private async createPVC(
|
||||
coreApi: k8s.CoreV1Api,
|
||||
namespace: string,
|
||||
name: string,
|
||||
size: string,
|
||||
): Promise<void> {
|
||||
const pvc: k8s.V1PersistentVolumeClaim = {
|
||||
apiVersion: 'v1',
|
||||
kind: 'PersistentVolumeClaim',
|
||||
metadata: { name, namespace },
|
||||
spec: {
|
||||
accessModes: ['ReadWriteOnce'],
|
||||
resources: { requests: { storage: size } },
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
await coreApi.readNamespacedPersistentVolumeClaim(name, namespace);
|
||||
// PVC exists, don't recreate
|
||||
} catch {
|
||||
await coreApi.createNamespacedPersistentVolumeClaim(namespace, pvc);
|
||||
}
|
||||
}
|
||||
|
||||
async getPodLogs(app: Application): Promise<string> {
|
||||
const { coreApi } = await this.getK8sClient(app.clusterId);
|
||||
|
||||
Reference in New Issue
Block a user