Add automatic cluster pool allocation.

Co-authored-by: Cursor <cursoragent@cursor.com>

backend/src/clusters/clusters.service.ts

@@ -3,12 +3,16 @@ import { InjectRepository } from '@nestjs/typeorm';
 import { ConfigService } from '@nestjs/config';
 import { Repository, DataSource, In } from 'typeorm';
 import * as k8s from '@kubernetes/client-node';
-import { Cluster } from './entities/cluster.entity';
+import * as crypto from 'crypto';
+import { Cluster, ClusterHealthStatus } from './entities/cluster.entity';
 import { ClusterPool } from './entities/cluster-pool.entity';
+import { ClusterHealth } from './entities/cluster-health.entity';
+import { ClusterAllocationLog } from './entities/cluster-allocation-log.entity';
 import { CreateClusterDto, UpdateClusterDto } from './dto/cluster.dto';
 import { CreateClusterPoolDto, UpdateClusterPoolDto } from './dto/cluster-pool.dto';
 import { ClusterStatus } from '../common/enums';
 import { ElasticsearchService } from '../kubernetes/elasticsearch.service';
+import { CreateApplicationDto } from '../applications/dto/application.dto';
 
 @Injectable()
 export class ClustersService {
@@ -21,6 +25,10 @@ export class ClustersService {
     private clustersRepository: Repository<Cluster>,
     @InjectRepository(ClusterPool)
     private poolsRepository: Repository<ClusterPool>,
+    @InjectRepository(ClusterHealth)
+    private healthRepository: Repository<ClusterHealth>,
+    @InjectRepository(ClusterAllocationLog)
+    private allocationLogsRepository: Repository<ClusterAllocationLog>,
     private dataSource: DataSource,
     private configService: ConfigService,
     @Inject(forwardRef(() => ElasticsearchService))
@@ -34,7 +42,7 @@ export class ClustersService {
   async testConnection(kubeconfig: string): Promise<{ connected: boolean; version?: string; error?: string }> {
     try {
       const kc = new k8s.KubeConfig();
-      kc.loadFromString(kubeconfig);
+      kc.loadFromString(this.decryptKubeconfig(kubeconfig));
 
       const versionApi = kc.makeApiClient(k8s.VersionApi);
       const result = await versionApi.getCode();
@@ -74,13 +82,18 @@ export class ClustersService {
 
     const cluster = this.clustersRepository.create({
       ...dto,
+      kubeconfig: this.encryptKubeconfig(dto.kubeconfig),
       status: ClusterStatus.ACTIVE, // Connection verified — mark active
+      healthStatus: 'healthy',
+      lastHealthCheckedAt: new Date(),
+      healthMessage: connectionTest.version ? `Kubernetes ${connectionTest.version}` : 'Connection verified',
     });
     const saved = await this.clustersRepository.save(cluster);
+    const usableCluster = this.withDecryptedKubeconfig(saved);
     this.logger.log(`Cluster "${saved.name}" registered (active) — K8s ${connectionTest.version}`);
 
     // Bootstrap the cluster with build infrastructure (namespace, registry, SA, etc.)
-    this.bootstrapCluster(saved.kubeconfig).catch((err) => {
+    this.bootstrapCluster(usableCluster.kubeconfig).catch((err) => {
       this.logger.error(`Failed to bootstrap cluster "${saved.name}": ${err.message}`);
     });
 
@@ -89,12 +102,33 @@ export class ClustersService {
       this.logger.error(`Failed to deploy central logging on "${saved.name}": ${err.message}`);
     });
 
-    return saved;
+    this.recordHealthSnapshot(saved, {
+      status: 'healthy',
+      message: connectionTest.version ? `Kubernetes ${connectionTest.version}` : 'Connection verified',
+    }).catch((err) => this.logger.warn(`Failed to record cluster health: ${err.message}`));
+
+    return usableCluster;
   }
 
   async findAll(): Promise<Cluster[]> {
     return this.clustersRepository.find({
-      select: ['id', 'name', 'description', 'status', 'apiServer', 'region', 'provider', 'isDefault', 'createdAt'],
+      select: [
+        'id',
+        'name',
+        'description',
+        'status',
+        'apiServer',
+        'region',
+        'provider',
+        'isDefault',
+        'weight',
+        'tags',
+        'healthStatus',
+        'lastHealthCheckedAt',
+        'healthMessage',
+        'availableResources',
+        'createdAt',
+      ],
       order: { createdAt: 'DESC' },
     });
   }
@@ -104,7 +138,7 @@ export class ClustersService {
     if (!cluster) {
       throw new NotFoundException('Cluster not found');
     }
-    return cluster;
+    return this.withDecryptedKubeconfig(cluster);
   }
 
   async getDefault(): Promise<Cluster> {
@@ -126,7 +160,7 @@ export class ClustersService {
     if (!cluster) {
       throw new NotFoundException('No active cluster available');
     }
-    return cluster;
+    return this.withDecryptedKubeconfig(cluster);
   }
 
   async update(id: string, dto: UpdateClusterDto): Promise<Cluster> {
@@ -141,10 +175,11 @@ export class ClustersService {
         );
       }
       dto.status = ClusterStatus.ACTIVE;
+      dto.kubeconfig = this.encryptKubeconfig(dto.kubeconfig);
       this.logger.log(`Cluster "${cluster.name}" kubeconfig updated — connection verified (K8s ${connectionTest.version})`);
 
       // Re-bootstrap build infrastructure on the new/updated cluster
-      this.bootstrapCluster(dto.kubeconfig).catch((err: any) => {
+      this.bootstrapCluster(this.decryptKubeconfig(dto.kubeconfig)).catch((err: any) => {
         this.logger.error(`Failed to bootstrap cluster "${cluster.name}": ${err.message}`);
       });
     }
@@ -159,7 +194,9 @@ export class ClustersService {
       }
     }
     Object.assign(cluster, dto);
-    return this.clustersRepository.save(cluster);
+    cluster.kubeconfig = this.encryptKubeconfig(cluster.kubeconfig);
+    const saved = await this.clustersRepository.save(cluster);
+    return this.withDecryptedKubeconfig(saved);
   }
 
   /**
@@ -171,7 +208,17 @@ export class ClustersService {
     const result = await this.testConnection(cluster.kubeconfig);
 
     cluster.status = result.connected ? ClusterStatus.ACTIVE : ClusterStatus.INACTIVE;
+    cluster.healthStatus = result.connected ? 'healthy' : 'unhealthy';
+    cluster.lastHealthCheckedAt = new Date();
+    cluster.healthMessage = result.connected
+      ? (result.version ? `Kubernetes ${result.version}` : 'Connection verified')
+      : result.error || 'Connection failed';
+    cluster.kubeconfig = this.encryptKubeconfig(cluster.kubeconfig);
     await this.clustersRepository.save(cluster);
+    await this.recordHealthSnapshot(cluster, {
+      status: cluster.healthStatus,
+      message: cluster.healthMessage,
+    });
     this.logger.log(`Cluster "${cluster.name}" test: ${result.connected ? 'ACTIVE' : 'INACTIVE'}`);
 
     return result;
@@ -188,6 +235,106 @@ export class ClustersService {
     });
   }
 
+  async selectClusterForApplication(
+    dto: CreateApplicationDto,
+    userId: string,
+  ): Promise<{ cluster: Cluster; pool?: ClusterPool; allocationLogId: string }> {
+    const estimatedRequest = this.estimateApplicationRequest(dto);
+    let pool = dto.poolId
+      ? await this.poolsRepository.findOne({ where: { id: dto.poolId, isActive: true } })
+      : null;
+    if (dto.poolId && !pool) {
+      throw new BadRequestException('Selected cluster pool is not active or does not exist');
+    }
+    if (!pool) {
+      pool = await this.poolsRepository.findOne({
+        where: { isActive: true, isDefault: true },
+        order: { priority: 'ASC', createdAt: 'ASC' },
+      }) || await this.poolsRepository.findOne({
+        where: { isActive: true },
+        order: { priority: 'ASC', createdAt: 'ASC' },
+      });
+    }
+
+    const candidateQuery = this.clustersRepository
+      .createQueryBuilder('cluster')
+      .where('cluster.status = :status', { status: ClusterStatus.ACTIVE });
+
+    if (pool?.clusterIds?.length) {
+      candidateQuery.andWhere('cluster.id IN (:...clusterIds)', { clusterIds: pool.clusterIds });
+    }
+
+    const candidates = await candidateQuery.getMany();
+    const appCounts = await this.getAppCounts(candidates.map((cluster) => cluster.id));
+    const candidateScores: Record<string, any>[] = [];
+    const rejectionReasons: Record<string, any>[] = [];
+
+    for (const cluster of candidates) {
+      const rejection = this.getClusterRejectionReason(cluster, estimatedRequest);
+      if (rejection) {
+        rejectionReasons.push({ clusterId: cluster.id, clusterName: cluster.name, reason: rejection });
+        continue;
+      }
+
+      const score = this.scoreCluster(cluster, estimatedRequest, appCounts.get(cluster.id) || 0, pool?.strategy || 'weighted-resource');
+      candidateScores.push({
+        clusterId: cluster.id,
+        clusterName: cluster.name,
+        score,
+        weight: cluster.weight || 1,
+        appCount: appCounts.get(cluster.id) || 0,
+        healthStatus: cluster.healthStatus,
+        availableResources: cluster.availableResources || null,
+      });
+    }
+
+    if (candidateScores.length === 0) {
+      const log = await this.allocationLogsRepository.save(this.allocationLogsRepository.create({
+        userId,
+        poolId: pool?.id,
+        selectedClusterId: null,
+        strategy: pool?.strategy || 'weighted-resource',
+        estimatedRequest,
+        candidateScores,
+        rejectionReasons,
+        status: 'failed',
+        message: 'No active healthy cluster has enough estimated capacity',
+      }));
+      throw new BadRequestException(
+        `No active healthy cluster has enough capacity for this application (allocation log: ${log.id})`,
+      );
+    }
+
+    candidateScores.sort((a, b) => b.score - a.score);
+    const selected = await this.findOne(candidateScores[0].clusterId);
+    const log = await this.allocationLogsRepository.save(this.allocationLogsRepository.create({
+      userId,
+      poolId: pool?.id,
+      selectedClusterId: selected.id,
+      strategy: pool?.strategy || 'weighted-resource',
+      estimatedRequest,
+      candidateScores,
+      rejectionReasons,
+      status: 'success',
+      message: `Selected ${selected.name}`,
+    }));
+
+    this.logger.log(`Allocator selected cluster "${selected.name}" for user ${userId} (score ${candidateScores[0].score.toFixed(2)})`);
+    return { cluster: selected, pool: pool || undefined, allocationLogId: log.id };
+  }
+
+  async attachAllocationToApplication(allocationLogId: string, applicationId: string): Promise<void> {
+    await this.allocationLogsRepository.update({ id: allocationLogId }, { applicationId });
+  }
+
+  async listAllocationLogs(limit = 100): Promise<ClusterAllocationLog[]> {
+    return this.allocationLogsRepository.find({
+      relations: ['selectedCluster', 'pool'],
+      order: { createdAt: 'DESC' },
+      take: limit,
+    });
+  }
+
   /**
    * Get the optimal cluster using load-balancing strategy.
    * Strategy: 'least-apps' — picks the active cluster with fewest deployed applications.
@@ -308,14 +455,21 @@ export class ClustersService {
       throw new BadRequestException(`Clusters not found: ${missingIds.join(', ')}`);
     }
 
-    const pool = this.poolsRepository.create(dto);
+    if (dto.isDefault === true) {
+      await this.poolsRepository.update({ isDefault: true }, { isDefault: false });
+    }
+
+    const pool = this.poolsRepository.create({
+      ...dto,
+      strategy: dto.strategy || 'weighted-resource',
+    });
     const saved = await this.poolsRepository.save(pool);
     this.logger.log(`Cluster pool "${saved.name}" created with ${dto.clusterIds.length} clusters (strategy: ${dto.strategy})`);
     return saved;
   }
 
   async findAllPools(): Promise<ClusterPool[]> {
-    return this.poolsRepository.find({ order: { createdAt: 'DESC' } });
+    return this.poolsRepository.find({ order: { isDefault: 'DESC', priority: 'ASC', createdAt: 'DESC' } });
   }
 
   /**
@@ -324,7 +478,7 @@ export class ClustersService {
   async findAllPoolsPublic(): Promise<(ClusterPool & { clusters: Pick<Cluster, 'id' | 'name' | 'region' | 'provider' | 'status'>[] })[]> {
     const pools = await this.poolsRepository.find({
       where: { isActive: true },
-      order: { createdAt: 'DESC' },
+      order: { isDefault: 'DESC', priority: 'ASC', createdAt: 'DESC' },
     });
 
     const allClusterIds = [...new Set(pools.flatMap((p) => p.clusterIds))];
@@ -367,6 +521,10 @@ export class ClustersService {
       }
     }
 
+    if (dto.isDefault === true) {
+      await this.poolsRepository.update({ isDefault: true }, { isDefault: false });
+    }
+
     Object.assign(pool, dto);
     return this.poolsRepository.save(pool);
   }
@@ -443,6 +601,7 @@ export class ClustersService {
     totalMemoryAllocatable: string;
     podCount: number;
     nodeCount: number;
+    readyNodeCount?: number;
     appCount: number;
   }> {
     const cluster = await this.findOne(id);
@@ -496,7 +655,8 @@ export class ClustersService {
       );
       const appCount = parseInt(appCountResult[0]?.count || '0', 10);
 
-      return {
+      const readyNodeCount = nodes.filter((node) => node.status === 'Ready').length;
+      const resources = {
         nodes,
         totalCpuCapacity: `${totalCpuCap}m`,
         totalMemoryCapacity: `${totalMemCap.toFixed(0)}Mi`,
@@ -504,10 +664,33 @@ export class ClustersService {
         totalMemoryAllocatable: `${totalMemAlloc.toFixed(0)}Mi`,
         podCount,
         nodeCount: nodes.length,
+        readyNodeCount,
         appCount,
       };
+      const healthStatus: ClusterHealthStatus = readyNodeCount === nodes.length && nodes.length > 0
+        ? 'healthy'
+        : readyNodeCount > 0
+          ? 'degraded'
+          : 'unhealthy';
+      const healthMessage = `${readyNodeCount}/${nodes.length} nodes ready`;
+
+      await this.clustersRepository.update(id, {
+        healthStatus,
+        healthMessage,
+        lastHealthCheckedAt: new Date(),
+        availableResources: resources as any,
+      });
+      await this.recordHealthSnapshot(cluster, { status: healthStatus, message: healthMessage, resources });
+
+      return resources;
     } catch (err: any) {
       this.logger.error(`Failed to get cluster resources for "${cluster.name}": ${err.message}`);
+      await this.clustersRepository.update(id, {
+        healthStatus: 'unhealthy',
+        healthMessage: err.message,
+        lastHealthCheckedAt: new Date(),
+      });
+      await this.recordHealthSnapshot(cluster, { status: 'unhealthy', message: err.message });
       throw new BadRequestException(`Cannot fetch resources: ${err.message}`);
     }
   }
@@ -713,4 +896,198 @@ export class ClustersService {
     if (memory.endsWith('Ti')) return parseFloat(memory) * 1024 * 1024;
     return parseFloat(memory) / (1024 * 1024); // bytes
   }
+
+  private estimateApplicationRequest(dto: CreateApplicationDto): Record<string, any> {
+    const replicas = Math.max(dto.replicas || 1, 1);
+    let cpuMillicores = this.parseCpuToMillicores(dto.cpuRequest || '100m') * replicas;
+    let memoryMi = this.parseMemoryToMi(dto.memoryRequest || '128Mi') * replicas;
+    let storageMi = this.parseStorageToMi(dto.dbStorageSize || '1Gi') + this.parseStorageToMi(dto.appStorageSize || '2Gi');
+
+    if (dto.databaseType && dto.databaseType !== 'none') {
+      cpuMillicores += 100;
+      memoryMi += 256;
+    }
+
+    for (const service of ['redis', 'rabbitmq'] as const) {
+      const enabled = service === 'redis' ? dto.enableRedis : dto.enableRabbitmq;
+      const custom = dto.optionalServiceResources?.[service];
+      if (!enabled && !custom) continue;
+
+      cpuMillicores += this.parseCpuToMillicores(custom?.cpuRequest || custom?.cpuLimit || '100m');
+      memoryMi += this.parseMemoryToMi(custom?.memoryRequest || custom?.memoryLimit || '128Mi');
+      storageMi += (custom?.storageGi || 1) * 1024;
+    }
+
+    return {
+      cpuMillicores,
+      memoryMi,
+      storageMi,
+      replicas,
+      podEstimate: replicas + (dto.databaseType && dto.databaseType !== 'none' ? 1 : 0) + (dto.enableRedis ? 1 : 0) + (dto.enableRabbitmq ? 1 : 0),
+    };
+  }
+
+  private getClusterRejectionReason(cluster: Cluster, estimatedRequest: Record<string, any>): string | null {
+    if (cluster.status !== ClusterStatus.ACTIVE) {
+      return `status=${cluster.status}`;
+    }
+
+    if (cluster.healthStatus && !['healthy', 'unknown'].includes(cluster.healthStatus)) {
+      return `health=${cluster.healthStatus}`;
+    }
+
+    const resources = cluster.availableResources || {};
+    const cpuFree = this.parseCpuToMillicores(resources.totalCpuAllocatable || resources.cpuAllocatable || '0');
+    const memoryFree = this.parseMemoryToMi(resources.totalMemoryAllocatable || resources.memoryAllocatable || '0');
+    const podCount = Number(resources.podCount || 0);
+    const nodeCount = Number(resources.nodeCount || 0);
+    const podCapacity = Number(resources.podCapacity || nodeCount * 110 || 0);
+
+    if (cpuFree > 0 && cpuFree < estimatedRequest.cpuMillicores) {
+      return `insufficient cpu (${cpuFree}m < ${estimatedRequest.cpuMillicores}m)`;
+    }
+    if (memoryFree > 0 && memoryFree < estimatedRequest.memoryMi) {
+      return `insufficient memory (${memoryFree}Mi < ${estimatedRequest.memoryMi}Mi)`;
+    }
+    if (podCapacity > 0 && podCount + estimatedRequest.podEstimate > podCapacity) {
+      return `pod pressure (${podCount}/${podCapacity})`;
+    }
+
+    return null;
+  }
+
+  private scoreCluster(
+    cluster: Cluster,
+    estimatedRequest: Record<string, any>,
+    appCount: number,
+    strategy: string,
+  ): number {
+    if (strategy === 'round-robin') {
+      const idx = this.poolRoundRobinIndices.get('allocator') || 0;
+      this.poolRoundRobinIndices.set('allocator', idx + 1);
+      return 1000 - idx;
+    }
+
+    const resources = cluster.availableResources || {};
+    const cpuFree = this.parseCpuToMillicores(resources.totalCpuAllocatable || resources.cpuAllocatable || '0');
+    const memoryFree = this.parseMemoryToMi(resources.totalMemoryAllocatable || resources.memoryAllocatable || '0');
+    const storageFree = this.parseStorageToMi(resources.storageAllocatable || resources.storageFree || '0');
+    const podCount = Number(resources.podCount || 0);
+    const nodeCount = Number(resources.nodeCount || 0);
+    const podCapacity = Number(resources.podCapacity || nodeCount * 110 || 0);
+
+    const capacityScore =
+      this.ratioScore(cpuFree, estimatedRequest.cpuMillicores) * 0.35 +
+      this.ratioScore(memoryFree, estimatedRequest.memoryMi) * 0.35 +
+      this.ratioScore(storageFree, estimatedRequest.storageMi) * 0.15 +
+      (podCapacity > 0 ? Math.max(0, 1 - podCount / podCapacity) : 0.7) * 0.15;
+    const appPenalty = Math.min(appCount, 100) * 0.75;
+
+    if (strategy === 'least-apps') {
+      return 1000 - appPenalty + capacityScore * 100;
+    }
+
+    return (cluster.weight || 1) * 100 + capacityScore * 100 - appPenalty;
+  }
+
+  private ratioScore(available: number, required: number): number {
+    if (!available) return 0.7;
+    if (!required) return 1;
+    return Math.min(available / required, 10) / 10;
+  }
+
+  private async getAppCounts(clusterIds: string[]): Promise<Map<string, number>> {
+    if (clusterIds.length === 0) {
+      return new Map();
+    }
+    const rows: { clusterId: string; count: string }[] = await this.dataSource.query(`
+      SELECT "clusterId", COUNT(*) as count
+      FROM applications
+      WHERE "clusterId" = ANY($1)
+      GROUP BY "clusterId"
+    `, [clusterIds]);
+    return new Map(rows.map((row) => [row.clusterId, parseInt(row.count, 10)]));
+  }
+
+  private parseStorageToMi(storage: string): number {
+    if (!storage) return 0;
+    if (storage.endsWith('Ki')) return parseFloat(storage) / 1024;
+    if (storage.endsWith('Mi')) return parseFloat(storage);
+    if (storage.endsWith('Gi')) return parseFloat(storage) * 1024;
+    if (storage.endsWith('Ti')) return parseFloat(storage) * 1024 * 1024;
+    return parseFloat(storage) || 0;
+  }
+
+  private async recordHealthSnapshot(
+    cluster: Cluster,
+    snapshot: {
+      status: ClusterHealthStatus;
+      message?: string;
+      resources?: Record<string, any>;
+    },
+  ): Promise<void> {
+    const resources = snapshot.resources || cluster.availableResources || {};
+    await this.healthRepository.save(this.healthRepository.create({
+      clusterId: cluster.id,
+      status: snapshot.status,
+      readyNodes: resources.readyNodeCount || resources.nodeCount || 0,
+      nodeCount: resources.nodeCount || 0,
+      cpuAllocatable: resources.totalCpuAllocatable || resources.cpuAllocatable || null,
+      memoryAllocatable: resources.totalMemoryAllocatable || resources.memoryAllocatable || null,
+      podCount: resources.podCount || 0,
+      appCount: resources.appCount || 0,
+      message: snapshot.message,
+      resources,
+    }));
+  }
+
+  private encryptKubeconfig(kubeconfig: string): string {
+    if (!kubeconfig || kubeconfig.startsWith('enc:v1:')) {
+      return kubeconfig;
+    }
+    const key = this.getKubeconfigEncryptionKey();
+    if (!key) {
+      return kubeconfig;
+    }
+    const iv = crypto.randomBytes(12);
+    const cipher = crypto.createCipheriv('aes-256-gcm', key, iv);
+    const encrypted = Buffer.concat([cipher.update(kubeconfig, 'utf8'), cipher.final()]);
+    const tag = cipher.getAuthTag();
+    return `enc:v1:${iv.toString('base64')}:${tag.toString('base64')}:${encrypted.toString('base64')}`;
+  }
+
+  private decryptKubeconfig(kubeconfig: string): string {
+    if (!kubeconfig?.startsWith('enc:v1:')) {
+      return kubeconfig;
+    }
+    const key = this.getKubeconfigEncryptionKey();
+    if (!key) {
+      throw new BadRequestException('Kubeconfig is encrypted but CLUSTER_KUBECONFIG_KEY is not configured');
+    }
+    const [, , ivRaw, tagRaw, encryptedRaw] = kubeconfig.split(':');
+    const decipher = crypto.createDecipheriv('aes-256-gcm', key, Buffer.from(ivRaw, 'base64'));
+    decipher.setAuthTag(Buffer.from(tagRaw, 'base64'));
+    return Buffer.concat([
+      decipher.update(Buffer.from(encryptedRaw, 'base64')),
+      decipher.final(),
+    ]).toString('utf8');
+  }
+
+  private withDecryptedKubeconfig(cluster: Cluster): Cluster {
+    return {
+      ...cluster,
+      kubeconfig: this.decryptKubeconfig(cluster.kubeconfig),
+    };
+  }
+
+  private getKubeconfigEncryptionKey(): Buffer | null {
+    const secret = this.configService.get<string>('CLUSTER_KUBECONFIG_KEY') || this.configService.get<string>('cluster.kubeconfigKey');
+    if (!secret) {
+      return null;
+    }
+    if (secret.length === 64 && /^[0-9a-f]+$/i.test(secret)) {
+      return Buffer.from(secret, 'hex');
+    }
+    return crypto.createHash('sha256').update(secret).digest();
+  }
 }