init
This commit is contained in:
keyhan
@@ -0,0 +1,558 @@
|
||||
import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import * as k8s from '@kubernetes/client-node';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as Handlebars from 'handlebars';
|
||||
import { ClustersService } from '../clusters/clusters.service';
|
||||
import { Application } from '../applications/entities/application.entity';
|
||||
import { AppRuntime, DatabaseType } from '../common/enums';
|
||||
|
||||
interface ManifestContext {
|
||||
appName: string;
|
||||
namespace: string;
|
||||
image: string;
|
||||
port: number;
|
||||
replicas: number;
|
||||
cpuRequest: string;
|
||||
cpuLimit: string;
|
||||
memoryRequest: string;
|
||||
memoryLimit: string;
|
||||
envVars: Record<string, string>;
|
||||
runtime: AppRuntime;
|
||||
databaseType: DatabaseType;
|
||||
domain: string;
|
||||
subdomain: string;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class KubernetesService implements OnModuleInit {
|
||||
private readonly logger = new Logger(KubernetesService.name);
|
||||
private templates: Map<string, Handlebars.TemplateDelegate> = new Map();
|
||||
|
||||
constructor(
|
||||
private configService: ConfigService,
|
||||
private clustersService: ClustersService,
|
||||
) {}
|
||||
|
||||
onModuleInit() {
|
||||
this.loadTemplates();
|
||||
}
|
||||
|
||||
private loadTemplates(): void {
|
||||
const templatesDir = path.join(__dirname, '..', '..', 'templates');
|
||||
const templateFiles = ['namespace', 'deployment', 'service', 'ingress', 'database', 'pvc', 'secret'];
|
||||
|
||||
for (const name of templateFiles) {
|
||||
const filePath = path.join(templatesDir, `${name}.yaml.hbs`);
|
||||
if (fs.existsSync(filePath)) {
|
||||
const template = fs.readFileSync(filePath, 'utf-8');
|
||||
this.templates.set(name, Handlebars.compile(template));
|
||||
this.logger.log(`Loaded template: ${name}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async getK8sClient(clusterId?: string): Promise<{
|
||||
coreApi: k8s.CoreV1Api;
|
||||
appsApi: k8s.AppsV1Api;
|
||||
networkingApi: k8s.NetworkingV1Api;
|
||||
}> {
|
||||
const cluster = clusterId
|
||||
? await this.clustersService.findOne(clusterId)
|
||||
: await this.clustersService.getDefault();
|
||||
|
||||
const kc = new k8s.KubeConfig();
|
||||
kc.loadFromString(cluster.kubeconfig);
|
||||
|
||||
return {
|
||||
coreApi: kc.makeApiClient(k8s.CoreV1Api),
|
||||
appsApi: kc.makeApiClient(k8s.AppsV1Api),
|
||||
networkingApi: kc.makeApiClient(k8s.NetworkingV1Api),
|
||||
};
|
||||
}
|
||||
|
||||
async deployApplication(app: Application, imageUri: string): Promise<Record<string, any>> {
|
||||
const { coreApi, appsApi, networkingApi } = await this.getK8sClient(app.clusterId);
|
||||
const domain = this.configService.get('platform.domain');
|
||||
|
||||
const context: ManifestContext = {
|
||||
appName: app.name,
|
||||
namespace: `user-${app.userId.split('-')[0]}`,
|
||||
image: imageUri,
|
||||
port: app.port,
|
||||
replicas: app.replicas,
|
||||
cpuRequest: app.cpuRequest,
|
||||
cpuLimit: app.cpuLimit,
|
||||
memoryRequest: app.memoryRequest,
|
||||
memoryLimit: app.memoryLimit,
|
||||
envVars: app.envVars || {},
|
||||
runtime: app.runtime,
|
||||
databaseType: app.databaseType,
|
||||
domain: domain,
|
||||
subdomain: app.subdomain || app.name,
|
||||
};
|
||||
|
||||
const manifests: Record<string, any> = {};
|
||||
|
||||
try {
|
||||
// 1. Ensure namespace exists
|
||||
await this.ensureNamespace(coreApi, context.namespace);
|
||||
|
||||
// 2. Create/Update secrets for env vars
|
||||
if (Object.keys(context.envVars).length > 0) {
|
||||
manifests.secret = await this.applySecret(coreApi, context);
|
||||
}
|
||||
|
||||
// 3. Deploy database if needed
|
||||
if (context.databaseType !== DatabaseType.NONE) {
|
||||
manifests.database = await this.deployDatabase(coreApi, appsApi, context);
|
||||
}
|
||||
|
||||
// 4. Create Deployment
|
||||
manifests.deployment = await this.applyDeployment(appsApi, context);
|
||||
|
||||
// 5. Create Service
|
||||
manifests.service = await this.applyService(coreApi, context);
|
||||
|
||||
// 6. Create Ingress
|
||||
manifests.ingress = await this.applyIngress(networkingApi, context);
|
||||
|
||||
this.logger.log(`Successfully deployed ${app.name} to namespace ${context.namespace}`);
|
||||
} catch (error: any) {
|
||||
this.logger.error(`Failed to deploy ${app.name}:`, error.body || error.message);
|
||||
throw error;
|
||||
}
|
||||
|
||||
return manifests;
|
||||
}
|
||||
|
||||
private async ensureNamespace(coreApi: k8s.CoreV1Api, namespace: string): Promise<void> {
|
||||
try {
|
||||
await coreApi.readNamespace(namespace);
|
||||
} catch {
|
||||
await coreApi.createNamespace({
|
||||
metadata: { name: namespace },
|
||||
});
|
||||
this.logger.log(`Created namespace: ${namespace}`);
|
||||
}
|
||||
}
|
||||
|
||||
private async applySecret(coreApi: k8s.CoreV1Api, ctx: ManifestContext): Promise<any> {
|
||||
const secretData: Record<string, string> = {};
|
||||
for (const [key, value] of Object.entries(ctx.envVars)) {
|
||||
secretData[key] = Buffer.from(value).toString('base64');
|
||||
}
|
||||
|
||||
const secret = {
|
||||
apiVersion: 'v1',
|
||||
kind: 'Secret',
|
||||
metadata: {
|
||||
name: `${ctx.appName}-env`,
|
||||
namespace: ctx.namespace,
|
||||
},
|
||||
data: secretData,
|
||||
};
|
||||
|
||||
try {
|
||||
await coreApi.replaceNamespacedSecret(`${ctx.appName}-env`, ctx.namespace, secret);
|
||||
} catch {
|
||||
await coreApi.createNamespacedSecret(ctx.namespace, secret);
|
||||
}
|
||||
return secret;
|
||||
}
|
||||
|
||||
private async applyDeployment(appsApi: k8s.AppsV1Api, ctx: ManifestContext): Promise<any> {
|
||||
const envFrom: any[] = [];
|
||||
if (Object.keys(ctx.envVars).length > 0) {
|
||||
envFrom.push({ secretRef: { name: `${ctx.appName}-env` } });
|
||||
}
|
||||
|
||||
// Add database connection env vars
|
||||
const extraEnv: any[] = [];
|
||||
if (ctx.databaseType === DatabaseType.POSTGRESQL) {
|
||||
extraEnv.push(
|
||||
{ name: 'DB_HOST', value: `${ctx.appName}-db` },
|
||||
{ name: 'DB_PORT', value: '5432' },
|
||||
{ name: 'DB_NAME', value: ctx.appName.replace(/-/g, '_') },
|
||||
{ name: 'DB_USER', value: 'appuser' },
|
||||
{ name: 'DB_PASSWORD', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'password' } } },
|
||||
);
|
||||
} else if (ctx.databaseType === DatabaseType.MYSQL) {
|
||||
extraEnv.push(
|
||||
{ name: 'DB_HOST', value: `${ctx.appName}-db` },
|
||||
{ name: 'DB_PORT', value: '3306' },
|
||||
{ name: 'DB_NAME', value: ctx.appName.replace(/-/g, '_') },
|
||||
{ name: 'DB_USER', value: 'appuser' },
|
||||
{ name: 'DB_PASSWORD', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'password' } } },
|
||||
);
|
||||
}
|
||||
|
||||
const deployment: k8s.V1Deployment = {
|
||||
apiVersion: 'apps/v1',
|
||||
kind: 'Deployment',
|
||||
metadata: {
|
||||
name: ctx.appName,
|
||||
namespace: ctx.namespace,
|
||||
labels: { app: ctx.appName, runtime: ctx.runtime },
|
||||
},
|
||||
spec: {
|
||||
replicas: ctx.replicas,
|
||||
selector: { matchLabels: { app: ctx.appName } },
|
||||
template: {
|
||||
metadata: { labels: { app: ctx.appName, runtime: ctx.runtime } },
|
||||
spec: {
|
||||
containers: [
|
||||
{
|
||||
name: ctx.appName,
|
||||
image: ctx.image,
|
||||
ports: [{ containerPort: ctx.port }],
|
||||
envFrom,
|
||||
env: extraEnv,
|
||||
resources: {
|
||||
requests: { cpu: ctx.cpuRequest, memory: ctx.memoryRequest },
|
||||
limits: { cpu: ctx.cpuLimit, memory: ctx.memoryLimit },
|
||||
},
|
||||
readinessProbe: {
|
||||
httpGet: { path: '/health', port: ctx.port as any },
|
||||
initialDelaySeconds: 10,
|
||||
periodSeconds: 5,
|
||||
},
|
||||
livenessProbe: {
|
||||
httpGet: { path: '/health', port: ctx.port as any },
|
||||
initialDelaySeconds: 30,
|
||||
periodSeconds: 10,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
await appsApi.replaceNamespacedDeployment(ctx.appName, ctx.namespace, deployment);
|
||||
} catch {
|
||||
await appsApi.createNamespacedDeployment(ctx.namespace, deployment);
|
||||
}
|
||||
return deployment;
|
||||
}
|
||||
|
||||
private async applyService(coreApi: k8s.CoreV1Api, ctx: ManifestContext): Promise<any> {
|
||||
const service: k8s.V1Service = {
|
||||
apiVersion: 'v1',
|
||||
kind: 'Service',
|
||||
metadata: {
|
||||
name: ctx.appName,
|
||||
namespace: ctx.namespace,
|
||||
},
|
||||
spec: {
|
||||
selector: { app: ctx.appName },
|
||||
ports: [{ port: 80, targetPort: ctx.port as any, protocol: 'TCP' }],
|
||||
type: 'ClusterIP',
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
await coreApi.replaceNamespacedService(ctx.appName, ctx.namespace, service);
|
||||
} catch {
|
||||
await coreApi.createNamespacedService(ctx.namespace, service);
|
||||
}
|
||||
return service;
|
||||
}
|
||||
|
||||
private async applyIngress(networkingApi: k8s.NetworkingV1Api, ctx: ManifestContext): Promise<any> {
|
||||
const ingress: k8s.V1Ingress = {
|
||||
apiVersion: 'networking.k8s.io/v1',
|
||||
kind: 'Ingress',
|
||||
metadata: {
|
||||
name: ctx.appName,
|
||||
namespace: ctx.namespace,
|
||||
annotations: {
|
||||
'kubernetes.io/ingress.class': 'nginx',
|
||||
'cert-manager.io/cluster-issuer': 'letsencrypt-prod',
|
||||
},
|
||||
},
|
||||
spec: {
|
||||
rules: [
|
||||
{
|
||||
host: `${ctx.subdomain}.${ctx.domain}`,
|
||||
http: {
|
||||
paths: [
|
||||
{
|
||||
path: '/',
|
||||
pathType: 'Prefix',
|
||||
backend: {
|
||||
service: {
|
||||
name: ctx.appName,
|
||||
port: { number: 80 },
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
tls: [
|
||||
{
|
||||
hosts: [`${ctx.subdomain}.${ctx.domain}`],
|
||||
secretName: `${ctx.appName}-tls`,
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
await networkingApi.replaceNamespacedIngress(ctx.appName, ctx.namespace, ingress);
|
||||
} catch {
|
||||
await networkingApi.createNamespacedIngress(ctx.namespace, ingress);
|
||||
}
|
||||
return ingress;
|
||||
}
|
||||
|
||||
private async deployDatabase(
|
||||
coreApi: k8s.CoreV1Api,
|
||||
appsApi: k8s.AppsV1Api,
|
||||
ctx: ManifestContext,
|
||||
): Promise<any> {
|
||||
const dbPassword = this.generatePassword();
|
||||
const dbName = `${ctx.appName}-db`;
|
||||
|
||||
// Create DB secret
|
||||
await this.createDbSecret(coreApi, ctx.namespace, ctx.appName, dbPassword);
|
||||
|
||||
// Create PVC for DB
|
||||
await this.createPVC(coreApi, ctx.namespace, dbName, '5Gi');
|
||||
|
||||
// Deploy database
|
||||
const isPostgres = ctx.databaseType === DatabaseType.POSTGRESQL;
|
||||
const image = isPostgres ? 'postgres:16-alpine' : 'mysql:8.0';
|
||||
const port = isPostgres ? 5432 : 3306;
|
||||
const envVars = isPostgres
|
||||
? [
|
||||
{ name: 'POSTGRES_DB', value: ctx.appName.replace(/-/g, '_') },
|
||||
{ name: 'POSTGRES_USER', value: 'appuser' },
|
||||
{ name: 'POSTGRES_PASSWORD', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'password' } } },
|
||||
]
|
||||
: [
|
||||
{ name: 'MYSQL_DATABASE', value: ctx.appName.replace(/-/g, '_') },
|
||||
{ name: 'MYSQL_USER', value: 'appuser' },
|
||||
{ name: 'MYSQL_PASSWORD', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'password' } } },
|
||||
{ name: 'MYSQL_ROOT_PASSWORD', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'password' } } },
|
||||
];
|
||||
|
||||
const dbDeployment: k8s.V1Deployment = {
|
||||
apiVersion: 'apps/v1',
|
||||
kind: 'Deployment',
|
||||
metadata: { name: dbName, namespace: ctx.namespace },
|
||||
spec: {
|
||||
replicas: 1,
|
||||
selector: { matchLabels: { app: dbName } },
|
||||
template: {
|
||||
metadata: { labels: { app: dbName } },
|
||||
spec: {
|
||||
containers: [
|
||||
{
|
||||
name: dbName,
|
||||
image,
|
||||
ports: [{ containerPort: port }],
|
||||
env: envVars,
|
||||
volumeMounts: [{ name: 'db-storage', mountPath: isPostgres ? '/var/lib/postgresql/data' : '/var/lib/mysql' }],
|
||||
resources: {
|
||||
requests: { cpu: '100m', memory: '256Mi' },
|
||||
limits: { cpu: '500m', memory: '512Mi' },
|
||||
},
|
||||
},
|
||||
],
|
||||
volumes: [
|
||||
{ name: 'db-storage', persistentVolumeClaim: { claimName: dbName } },
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
await appsApi.replaceNamespacedDeployment(dbName, ctx.namespace, dbDeployment);
|
||||
} catch {
|
||||
await appsApi.createNamespacedDeployment(ctx.namespace, dbDeployment);
|
||||
}
|
||||
|
||||
// Create DB Service
|
||||
const dbService: k8s.V1Service = {
|
||||
apiVersion: 'v1',
|
||||
kind: 'Service',
|
||||
metadata: { name: dbName, namespace: ctx.namespace },
|
||||
spec: {
|
||||
selector: { app: dbName },
|
||||
ports: [{ port, targetPort: port as any, protocol: 'TCP' }],
|
||||
type: 'ClusterIP',
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
await coreApi.replaceNamespacedService(dbName, ctx.namespace, dbService);
|
||||
} catch {
|
||||
await coreApi.createNamespacedService(ctx.namespace, dbService);
|
||||
}
|
||||
|
||||
return { deployment: dbDeployment, service: dbService };
|
||||
}
|
||||
|
||||
private async createDbSecret(
|
||||
coreApi: k8s.CoreV1Api,
|
||||
namespace: string,
|
||||
appName: string,
|
||||
password: string,
|
||||
): Promise<void> {
|
||||
const secret = {
|
||||
apiVersion: 'v1',
|
||||
kind: 'Secret',
|
||||
metadata: { name: `${appName}-db-secret`, namespace },
|
||||
data: { password: Buffer.from(password).toString('base64') },
|
||||
};
|
||||
|
||||
try {
|
||||
await coreApi.replaceNamespacedSecret(`${appName}-db-secret`, namespace, secret);
|
||||
} catch {
|
||||
await coreApi.createNamespacedSecret(namespace, secret);
|
||||
}
|
||||
}
|
||||
|
||||
private async createPVC(
|
||||
coreApi: k8s.CoreV1Api,
|
||||
namespace: string,
|
||||
name: string,
|
||||
size: string,
|
||||
): Promise<void> {
|
||||
const pvc: k8s.V1PersistentVolumeClaim = {
|
||||
apiVersion: 'v1',
|
||||
kind: 'PersistentVolumeClaim',
|
||||
metadata: { name, namespace },
|
||||
spec: {
|
||||
accessModes: ['ReadWriteOnce'],
|
||||
resources: { requests: { storage: size } },
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
await coreApi.readNamespacedPersistentVolumeClaim(name, namespace);
|
||||
// PVC exists, don't recreate
|
||||
} catch {
|
||||
await coreApi.createNamespacedPersistentVolumeClaim(namespace, pvc);
|
||||
}
|
||||
}
|
||||
|
||||
async getPodLogs(app: Application): Promise<string> {
|
||||
const { coreApi } = await this.getK8sClient(app.clusterId);
|
||||
const namespace = `user-${app.userId.split('-')[0]}`;
|
||||
|
||||
const pods = await coreApi.listNamespacedPod(
|
||||
namespace,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
`app=${app.name}`,
|
||||
);
|
||||
|
||||
if (pods.body.items.length === 0) {
|
||||
return 'No pods found for this application.';
|
||||
}
|
||||
|
||||
const podName = pods.body.items[0].metadata?.name;
|
||||
if (!podName) return 'Pod name not found.';
|
||||
|
||||
const logResponse = await coreApi.readNamespacedPodLog(
|
||||
podName,
|
||||
namespace,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
200,
|
||||
);
|
||||
|
||||
return logResponse.body;
|
||||
}
|
||||
|
||||
async scaleDeployment(app: Application, replicas: number): Promise<void> {
|
||||
const { appsApi } = await this.getK8sClient(app.clusterId);
|
||||
const namespace = `user-${app.userId.split('-')[0]}`;
|
||||
|
||||
await appsApi.patchNamespacedDeployment(
|
||||
app.name,
|
||||
namespace,
|
||||
{ spec: { replicas } },
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
{ headers: { 'Content-Type': 'application/merge-patch+json' } },
|
||||
);
|
||||
}
|
||||
|
||||
async restartDeployment(app: Application): Promise<void> {
|
||||
const { appsApi } = await this.getK8sClient(app.clusterId);
|
||||
const namespace = `user-${app.userId.split('-')[0]}`;
|
||||
|
||||
await appsApi.patchNamespacedDeployment(
|
||||
app.name,
|
||||
namespace,
|
||||
{
|
||||
spec: {
|
||||
template: {
|
||||
metadata: {
|
||||
annotations: {
|
||||
'kubectl.kubernetes.io/restartedAt': new Date().toISOString(),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
{ headers: { 'Content-Type': 'application/merge-patch+json' } },
|
||||
);
|
||||
}
|
||||
|
||||
async deleteApplication(app: Application): Promise<void> {
|
||||
const { coreApi, appsApi, networkingApi } = await this.getK8sClient(app.clusterId);
|
||||
const namespace = `user-${app.userId.split('-')[0]}`;
|
||||
|
||||
try {
|
||||
await appsApi.deleteNamespacedDeployment(app.name, namespace);
|
||||
await coreApi.deleteNamespacedService(app.name, namespace);
|
||||
await networkingApi.deleteNamespacedIngress(app.name, namespace);
|
||||
|
||||
// Delete DB resources if applicable
|
||||
if (app.databaseType !== DatabaseType.NONE) {
|
||||
const dbName = `${app.name}-db`;
|
||||
await appsApi.deleteNamespacedDeployment(dbName, namespace);
|
||||
await coreApi.deleteNamespacedService(dbName, namespace);
|
||||
await coreApi.deleteNamespacedPersistentVolumeClaim(dbName, namespace);
|
||||
await coreApi.deleteNamespacedSecret(`${app.name}-db-secret`, namespace);
|
||||
}
|
||||
|
||||
await coreApi.deleteNamespacedSecret(`${app.name}-env`, namespace);
|
||||
} catch (error: any) {
|
||||
this.logger.warn(`Error cleaning up resources for ${app.name}: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
private generatePassword(length = 24): string {
|
||||
const chars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!@#$%';
|
||||
let password = '';
|
||||
for (let i = 0; i < length; i++) {
|
||||
password += chars.charAt(Math.floor(Math.random() * chars.length));
|
||||
}
|
||||
return password;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user