import { Injectable, NotFoundException, Logger, BadRequestException, Inject, forwardRef } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; import { ConfigService } from '@nestjs/config'; import { Repository, DataSource, In } from 'typeorm'; import * as k8s from '@kubernetes/client-node'; import { Cluster } from './entities/cluster.entity'; import { ClusterPool } from './entities/cluster-pool.entity'; import { CreateClusterDto, UpdateClusterDto } from './dto/cluster.dto'; import { CreateClusterPoolDto, UpdateClusterPoolDto } from './dto/cluster-pool.dto'; import { ClusterStatus } from '../common/enums'; import { ElasticsearchService } from '../kubernetes/elasticsearch.service'; @Injectable() export class ClustersService { private readonly logger = new Logger(ClustersService.name); private roundRobinIndex = 0; private poolRoundRobinIndices = new Map(); constructor( @InjectRepository(Cluster) private clustersRepository: Repository, @InjectRepository(ClusterPool) private poolsRepository: Repository, private dataSource: DataSource, private configService: ConfigService, @Inject(forwardRef(() => ElasticsearchService)) private elasticsearchService: ElasticsearchService, ) {} /** * Test connection to a Kubernetes cluster using its kubeconfig. * Calls the /version endpoint to verify the cluster is reachable. */ async testConnection(kubeconfig: string): Promise<{ connected: boolean; version?: string; error?: string }> { try { const kc = new k8s.KubeConfig(); kc.loadFromString(kubeconfig); const versionApi = kc.makeApiClient(k8s.VersionApi); const result = await versionApi.getCode(); const info = result.body; this.logger.log(`Cluster connection OK: Kubernetes ${info.gitVersion}`); return { connected: true, version: info.gitVersion, }; } catch (err: any) { const message = err?.body?.message || err?.message || 'Unknown connection error'; this.logger.warn(`Cluster connection failed: ${message}`); return { connected: false, error: message, }; } } async create(dto: CreateClusterDto): Promise { // Validate kubeconfig by testing actual connection const connectionTest = await this.testConnection(dto.kubeconfig); if (!connectionTest.connected) { throw new BadRequestException( `Cannot connect to Kubernetes cluster: ${connectionTest.error}`, ); } if (dto.isDefault === true) { const existingDefaults = await this.clustersRepository.find({ where: { isDefault: true } }); for (const c of existingDefaults) { c.isDefault = false; await this.clustersRepository.save(c); } } const cluster = this.clustersRepository.create({ ...dto, status: ClusterStatus.ACTIVE, // Connection verified — mark active }); const saved = await this.clustersRepository.save(cluster); this.logger.log(`Cluster "${saved.name}" registered (active) — K8s ${connectionTest.version}`); // Bootstrap the cluster with build infrastructure (namespace, registry, SA, etc.) this.bootstrapCluster(saved.kubeconfig).catch((err) => { this.logger.error(`Failed to bootstrap cluster "${saved.name}": ${err.message}`); }); // Deploy central logging (Elasticsearch + Kibana) via Helm this.elasticsearchService.deploy(saved.id).catch((err) => { this.logger.error(`Failed to deploy central logging on "${saved.name}": ${err.message}`); }); return saved; } async findAll(): Promise { return this.clustersRepository.find({ select: ['id', 'name', 'description', 'status', 'apiServer', 'region', 'provider', 'isDefault', 'createdAt'], order: { createdAt: 'DESC' }, }); } async findOne(id: string): Promise { const cluster = await this.clustersRepository.findOne({ where: { id } }); if (!cluster) { throw new NotFoundException('Cluster not found'); } return cluster; } async getDefault(): Promise { // Prefer active default cluster; fall back to any active cluster let cluster = await this.clustersRepository.findOne({ where: { isDefault: true, status: ClusterStatus.ACTIVE }, }); if (!cluster) { // Fallback: pick any active cluster and promote it to default cluster = await this.clustersRepository.findOne({ where: { status: ClusterStatus.ACTIVE }, }); if (cluster) { cluster.isDefault = true; await this.clustersRepository.save(cluster); this.logger.warn(`No active default cluster — promoted "${cluster.name}" to default`); } } if (!cluster) { throw new NotFoundException('No active cluster available'); } return cluster; } async update(id: string, dto: UpdateClusterDto): Promise { const cluster = await this.findOne(id); // If kubeconfig is being updated, re-test connection if (dto.kubeconfig) { const connectionTest = await this.testConnection(dto.kubeconfig); if (!connectionTest.connected) { throw new BadRequestException( `Cannot connect to Kubernetes cluster: ${connectionTest.error}`, ); } dto.status = ClusterStatus.ACTIVE; this.logger.log(`Cluster "${cluster.name}" kubeconfig updated — connection verified (K8s ${connectionTest.version})`); // Re-bootstrap build infrastructure on the new/updated cluster this.bootstrapCluster(dto.kubeconfig).catch((err: any) => { this.logger.error(`Failed to bootstrap cluster "${cluster.name}": ${err.message}`); }); } if (dto.isDefault === true) { const existingDefaults = await this.clustersRepository.find({ where: { isDefault: true } }); for (const c of existingDefaults) { if (c.id !== id) { c.isDefault = false; await this.clustersRepository.save(c); } } } Object.assign(cluster, dto); return this.clustersRepository.save(cluster); } /** * Manually test connectivity to an existing cluster. * Updates status to active/inactive based on result. */ async testClusterById(id: string): Promise<{ connected: boolean; version?: string; error?: string }> { const cluster = await this.findOne(id); const result = await this.testConnection(cluster.kubeconfig); cluster.status = result.connected ? ClusterStatus.ACTIVE : ClusterStatus.INACTIVE; await this.clustersRepository.save(cluster); this.logger.log(`Cluster "${cluster.name}" test: ${result.connected ? 'ACTIVE' : 'INACTIVE'}`); return result; } /** * Public cluster list (no sensitive data) — for users to select a cluster. */ async findAllPublic(): Promise[]> { return this.clustersRepository.find({ select: ['id', 'name', 'region', 'provider', 'isDefault', 'status'], where: { status: ClusterStatus.ACTIVE }, order: { isDefault: 'DESC', name: 'ASC' }, }); } /** * Get the optimal cluster using load-balancing strategy. * Strategy: 'least-apps' — picks the active cluster with fewest deployed applications. * Falls back to default cluster if only one active cluster exists. */ async getOptimalCluster(strategy: 'least-apps' | 'round-robin' = 'least-apps'): Promise { const activeClusters = await this.clustersRepository.find({ where: { status: ClusterStatus.ACTIVE }, }); if (activeClusters.length === 0) { throw new NotFoundException('No active clusters available'); } if (activeClusters.length === 1) { return activeClusters[0]; } if (strategy === 'round-robin') { const cluster = activeClusters[this.roundRobinIndex % activeClusters.length]; this.roundRobinIndex++; this.logger.log(`Round-robin selected cluster "${cluster.name}" (index: ${this.roundRobinIndex - 1})`); return cluster; } // least-apps: count applications per cluster const appCounts: { clusterId: string; count: string }[] = await this.dataSource.query(` SELECT "clusterId", COUNT(*) as count FROM applications WHERE "clusterId" IS NOT NULL GROUP BY "clusterId" `); const countMap = new Map(); for (const row of appCounts) { countMap.set(row.clusterId, parseInt(row.count, 10)); } // Sort by app count ascending (least apps first) activeClusters.sort((a, b) => { const countA = countMap.get(a.id) || 0; const countB = countMap.get(b.id) || 0; return countA - countB; }); const selected = activeClusters[0]; const selectedCount = countMap.get(selected.id) || 0; this.logger.log(`Least-apps selected cluster "${selected.name}" (${selectedCount} apps)`); return selected; } async delete(id: string): Promise { const cluster = await this.findOne(id); const wasDefault = cluster.isDefault; // Reassign applications that were on this cluster to another active cluster try { const replacement = await this.clustersRepository.findOne({ where: { status: ClusterStatus.ACTIVE, id: undefined as any }, }); // Use raw query to exclude the deleted cluster const activeReplacement = await this.clustersRepository .createQueryBuilder('c') .where('c.id != :id', { id }) .andWhere('c.status = :status', { status: ClusterStatus.ACTIVE }) .getOne(); if (activeReplacement) { const result = await this.dataSource.query( `UPDATE applications SET "clusterId" = $1 WHERE "clusterId" = $2`, [activeReplacement.id, id], ); const count = result?.[1] || 0; if (count > 0) { this.logger.log(`Reassigned ${count} application(s) from cluster "${cluster.name}" to "${activeReplacement.name}"`); } } else { // No replacement — nullify clusterId so apps aren't orphaned with a dangling FK await this.dataSource.query( `UPDATE applications SET "clusterId" = NULL WHERE "clusterId" = $1`, [id], ); this.logger.warn(`No active replacement cluster — cleared clusterId for apps on "${cluster.name}"`); } } catch (e: any) { this.logger.warn(`Failed to reassign apps from cluster "${cluster.name}": ${e.message}`); } await this.clustersRepository.remove(cluster); this.logger.log(`Cluster "${cluster.name}" deleted`); // If deleted cluster was default, promote another active cluster if (wasDefault) { const newDefault = await this.clustersRepository.findOne({ where: { status: ClusterStatus.ACTIVE }, }); if (newDefault) { newDefault.isDefault = true; await this.clustersRepository.save(newDefault); this.logger.log(`Promoted cluster "${newDefault.name}" to default after deleting "${cluster.name}"`); } } } // ─── Cluster Pool Methods ───────────────────────────────────────── async createPool(dto: CreateClusterPoolDto): Promise { // Validate that all cluster IDs exist if (dto.clusterIds.length === 0) { throw new BadRequestException('Pool must contain at least one cluster'); } const clusters = await this.clustersRepository.find({ where: { id: In(dto.clusterIds) }, }); if (clusters.length !== dto.clusterIds.length) { const foundIds = clusters.map((c) => c.id); const missingIds = dto.clusterIds.filter((id) => !foundIds.includes(id)); throw new BadRequestException(`Clusters not found: ${missingIds.join(', ')}`); } const pool = this.poolsRepository.create(dto); const saved = await this.poolsRepository.save(pool); this.logger.log(`Cluster pool "${saved.name}" created with ${dto.clusterIds.length} clusters (strategy: ${dto.strategy})`); return saved; } async findAllPools(): Promise { return this.poolsRepository.find({ order: { createdAt: 'DESC' } }); } /** * Public pool list — returns pools with resolved cluster names for UI. */ async findAllPoolsPublic(): Promise<(ClusterPool & { clusters: Pick[] })[]> { const pools = await this.poolsRepository.find({ where: { isActive: true }, order: { createdAt: 'DESC' }, }); const allClusterIds = [...new Set(pools.flatMap((p) => p.clusterIds))]; const clusters = allClusterIds.length > 0 ? await this.clustersRepository.find({ where: { id: In(allClusterIds) }, select: ['id', 'name', 'region', 'provider', 'status'], }) : []; const clusterMap = new Map(clusters.map((c) => [c.id, c])); return pools.map((pool) => ({ ...pool, clusters: pool.clusterIds .map((id) => clusterMap.get(id)) .filter(Boolean) as Pick[], })); } async findOnePool(id: string): Promise { const pool = await this.poolsRepository.findOne({ where: { id } }); if (!pool) { throw new NotFoundException('Cluster pool not found'); } return pool; } async updatePool(id: string, dto: UpdateClusterPoolDto): Promise { const pool = await this.findOnePool(id); if (dto.clusterIds && dto.clusterIds.length > 0) { const clusters = await this.clustersRepository.find({ where: { id: In(dto.clusterIds) }, }); if (clusters.length !== dto.clusterIds.length) { const foundIds = clusters.map((c) => c.id); const missingIds = dto.clusterIds.filter((cid) => !foundIds.includes(cid)); throw new BadRequestException(`Clusters not found: ${missingIds.join(', ')}`); } } Object.assign(pool, dto); return this.poolsRepository.save(pool); } async deletePool(id: string): Promise { const pool = await this.findOnePool(id); await this.poolsRepository.remove(pool); this.poolRoundRobinIndices.delete(id); } /** * Get the optimal cluster from a specific pool using the pool's strategy. * Only considers ACTIVE clusters within the pool. */ async getOptimalClusterFromPool(poolId: string): Promise { const pool = await this.findOnePool(poolId); if (!pool.isActive) { throw new BadRequestException(`Pool "${pool.name}" is not active`); } const activeClusters = await this.clustersRepository.find({ where: { id: In(pool.clusterIds), status: ClusterStatus.ACTIVE, }, }); if (activeClusters.length === 0) { throw new NotFoundException(`No active clusters in pool "${pool.name}"`); } if (activeClusters.length === 1) { return activeClusters[0]; } if (pool.strategy === 'round-robin') { const idx = this.poolRoundRobinIndices.get(pool.id) || 0; const cluster = activeClusters[idx % activeClusters.length]; this.poolRoundRobinIndices.set(pool.id, idx + 1); this.logger.log(`Pool "${pool.name}" round-robin → cluster "${cluster.name}"`); return cluster; } // least-apps strategy const appCounts: { clusterId: string; count: string }[] = await this.dataSource.query(` SELECT "clusterId", COUNT(*) as count FROM applications WHERE "clusterId" = ANY($1) GROUP BY "clusterId" `, [pool.clusterIds]); const countMap = new Map(); for (const row of appCounts) { countMap.set(row.clusterId, parseInt(row.count, 10)); } activeClusters.sort((a, b) => { return (countMap.get(a.id) || 0) - (countMap.get(b.id) || 0); }); const selected = activeClusters[0]; this.logger.log(`Pool "${pool.name}" least-apps → cluster "${selected.name}" (${countMap.get(selected.id) || 0} apps)`); return selected; } /** * Get resource usage for a specific cluster — nodes, total CPU/memory, pod counts. */ async getClusterResources(id: string): Promise<{ nodes: { name: string; status: string; roles: string; cpuCapacity: string; memoryCapacity: string; cpuAllocatable: string; memoryAllocatable: string; }[]; totalCpuCapacity: string; totalMemoryCapacity: string; totalCpuAllocatable: string; totalMemoryAllocatable: string; podCount: number; nodeCount: number; appCount: number; }> { const cluster = await this.findOne(id); const kc = new k8s.KubeConfig(); kc.loadFromString(cluster.kubeconfig); const coreApi = kc.makeApiClient(k8s.CoreV1Api); try { // Get nodes const nodesRes = await coreApi.listNode(); const nodes = nodesRes.body.items.map((node) => { const conditions = node.status?.conditions || []; const readyCondition = conditions.find((c) => c.type === 'Ready'); const roles = Object.keys(node.metadata?.labels || {}) .filter((l) => l.startsWith('node-role.kubernetes.io/')) .map((l) => l.replace('node-role.kubernetes.io/', '')) .join(', ') || 'worker'; return { name: node.metadata?.name || 'unknown', status: readyCondition?.status === 'True' ? 'Ready' : 'NotReady', roles, cpuCapacity: node.status?.capacity?.cpu || '0', memoryCapacity: node.status?.capacity?.memory || '0', cpuAllocatable: node.status?.allocatable?.cpu || '0', memoryAllocatable: node.status?.allocatable?.memory || '0', }; }); // Total capacity let totalCpuCap = 0; let totalMemCap = 0; let totalCpuAlloc = 0; let totalMemAlloc = 0; for (const node of nodes) { totalCpuCap += this.parseCpuToMillicores(node.cpuCapacity); totalMemCap += this.parseMemoryToMi(node.memoryCapacity); totalCpuAlloc += this.parseCpuToMillicores(node.cpuAllocatable); totalMemAlloc += this.parseMemoryToMi(node.memoryAllocatable); } // Get all pods count const podsRes = await coreApi.listPodForAllNamespaces(); const podCount = podsRes.body.items.length; // Get app count for this cluster const appCountResult = await this.dataSource.query( `SELECT COUNT(*) as count FROM applications WHERE "clusterId" = $1`, [id], ); const appCount = parseInt(appCountResult[0]?.count || '0', 10); return { nodes, totalCpuCapacity: `${totalCpuCap}m`, totalMemoryCapacity: `${totalMemCap.toFixed(0)}Mi`, totalCpuAllocatable: `${totalCpuAlloc}m`, totalMemoryAllocatable: `${totalMemAlloc.toFixed(0)}Mi`, podCount, nodeCount: nodes.length, appCount, }; } catch (err: any) { this.logger.error(`Failed to get cluster resources for "${cluster.name}": ${err.message}`); throw new BadRequestException(`Cannot fetch resources: ${err.message}`); } } /** * Bootstrap a newly-added cluster with the build infrastructure: * 1. cloudhost-builds namespace * 2. kaniko-builder ServiceAccount * 3. Docker Registry Deployment + PVC + Service (ClusterIP) + NodePort Service * 4. registry-credentials Secret (for Kaniko docker auth) */ async bootstrapCluster(kubeconfig: string): Promise { const kc = new k8s.KubeConfig(); kc.loadFromString(kubeconfig); const coreApi = kc.makeApiClient(k8s.CoreV1Api); const appsApi = kc.makeApiClient(k8s.AppsV1Api); const buildNs = this.configService.get('build.namespace') || 'cloudhost-builds'; const saName = this.configService.get('build.serviceAccount') || 'kaniko-builder'; const registryUrl = this.configService.get('registry.url') || `registry.${buildNs}.svc.cluster.local:5000`; this.logger.log(`Bootstrapping cluster — namespace: ${buildNs}`); // ── 1. Namespace ────────────────────────────────────────────── try { await coreApi.readNamespace(buildNs); this.logger.log(`Namespace "${buildNs}" already exists`); } catch (err: any) { if (err.statusCode === 404 || err.body?.code === 404) { await coreApi.createNamespace({ metadata: { name: buildNs } }); this.logger.log(`Created namespace "${buildNs}"`); } else { throw err; } } // ── 2. ServiceAccount for Kaniko ────────────────────────────── try { await coreApi.readNamespacedServiceAccount(saName, buildNs); this.logger.log(`ServiceAccount "${saName}" already exists`); } catch (err: any) { if (err.statusCode === 404 || err.body?.code === 404) { await coreApi.createNamespacedServiceAccount(buildNs, { metadata: { name: saName, namespace: buildNs }, }); this.logger.log(`Created ServiceAccount "${saName}"`); } else { throw err; } } // ── 3. Docker Registry PVC ──────────────────────────────────── const registryPvcName = 'registry-data'; try { await coreApi.readNamespacedPersistentVolumeClaim(registryPvcName, buildNs); this.logger.log(`PVC "${registryPvcName}" already exists`); } catch (err: any) { if (err.statusCode === 404 || err.body?.code === 404) { await coreApi.createNamespacedPersistentVolumeClaim(buildNs, { metadata: { name: registryPvcName, namespace: buildNs }, spec: { accessModes: ['ReadWriteOnce'], resources: { requests: { storage: '10Gi' } }, }, }); this.logger.log(`Created PVC "${registryPvcName}" (10Gi)`); } else { throw err; } } // ── 4. Docker Registry Deployment ───────────────────────────── const registryDeployName = 'registry'; try { await appsApi.readNamespacedDeployment(registryDeployName, buildNs); this.logger.log(`Deployment "${registryDeployName}" already exists`); } catch (err: any) { if (err.statusCode === 404 || err.body?.code === 404) { await appsApi.createNamespacedDeployment(buildNs, { metadata: { name: registryDeployName, namespace: buildNs, labels: { app: 'registry' } }, spec: { replicas: 1, selector: { matchLabels: { app: 'registry' } }, template: { metadata: { labels: { app: 'registry' } }, spec: { containers: [{ name: 'registry', image: 'registry:2', ports: [{ containerPort: 5000 }], env: [ { name: 'REGISTRY_STORAGE_DELETE_ENABLED', value: 'true' }, ], volumeMounts: [{ name: 'registry-data', mountPath: '/var/lib/registry', }], resources: { requests: { cpu: '100m', memory: '128Mi' }, limits: { cpu: '500m', memory: '512Mi' }, }, }], volumes: [{ name: 'registry-data', persistentVolumeClaim: { claimName: registryPvcName }, }], }, }, }, }); this.logger.log(`Created Docker Registry Deployment`); } else { throw err; } } // ── 5. Registry ClusterIP Service (for Kaniko to push) ──────── const registrySvcName = 'registry'; try { await coreApi.readNamespacedService(registrySvcName, buildNs); this.logger.log(`Service "${registrySvcName}" already exists`); } catch (err: any) { if (err.statusCode === 404 || err.body?.code === 404) { await coreApi.createNamespacedService(buildNs, { metadata: { name: registrySvcName, namespace: buildNs, labels: { app: 'registry' } }, spec: { type: 'ClusterIP', selector: { app: 'registry' }, ports: [{ port: 5000, targetPort: 5000 as any, protocol: 'TCP' }], }, }); this.logger.log(`Created Registry ClusterIP Service (port 5000)`); } else { throw err; } } // ── 6. Registry NodePort Service (for kubelet to pull) ──────── const registryNodePortName = 'registry-nodeport'; try { await coreApi.readNamespacedService(registryNodePortName, buildNs); this.logger.log(`Service "${registryNodePortName}" already exists`); } catch (err: any) { if (err.statusCode === 404 || err.body?.code === 404) { await coreApi.createNamespacedService(buildNs, { metadata: { name: registryNodePortName, namespace: buildNs, labels: { app: 'registry' } }, spec: { type: 'NodePort', selector: { app: 'registry' }, ports: [{ port: 5000, targetPort: 5000 as any, nodePort: 30500, protocol: 'TCP' }], }, }); this.logger.log(`Created Registry NodePort Service (30500 → 5000)`); } else { throw err; } } // ── 7. registry-credentials Secret (docker config for Kaniko) ─ const registrySecretName = 'registry-credentials'; try { await coreApi.readNamespacedSecret(registrySecretName, buildNs); this.logger.log(`Secret "${registrySecretName}" already exists`); } catch (err: any) { if (err.statusCode === 404 || err.body?.code === 404) { // Parse registry host (without port path) for the docker config const dockerConfig = JSON.stringify({ auths: { [registryUrl]: { auth: '' }, [`registry.${buildNs}.svc.cluster.local:5000`]: { auth: '' }, 'localhost:30500': { auth: '' }, }, }); await coreApi.createNamespacedSecret(buildNs, { metadata: { name: registrySecretName, namespace: buildNs }, type: 'kubernetes.io/dockerconfigjson', data: { '.dockerconfigjson': Buffer.from(dockerConfig).toString('base64'), }, }); this.logger.log(`Created registry-credentials Secret`); } else { throw err; } } this.logger.log(`✅ Cluster bootstrap complete — build infrastructure ready`); } private parseCpuToMillicores(cpu: string): number { if (!cpu || cpu === '0') return 0; if (cpu.endsWith('n')) return parseFloat(cpu) / 1_000_000; if (cpu.endsWith('u')) return parseFloat(cpu) / 1_000; if (cpu.endsWith('m')) return parseFloat(cpu); return parseFloat(cpu) * 1000; } private parseMemoryToMi(memory: string): number { if (!memory || memory === '0') return 0; if (memory.endsWith('Ki')) return parseFloat(memory) / 1024; if (memory.endsWith('Mi')) return parseFloat(memory); if (memory.endsWith('Gi')) return parseFloat(memory) * 1024; if (memory.endsWith('Ti')) return parseFloat(memory) * 1024 * 1024; return parseFloat(memory) / (1024 * 1024); // bytes } }