Improve cluster allocation strategy.

Co-authored-by: Cursor <cursoragent@cursor.com>

backend/migrations/012_cluster_pool_allocation.sql

@@ -12,6 +12,9 @@ ALTER TABLE cluster_pools
   ADD COLUMN IF NOT EXISTS "isDefault" BOOLEAN NOT NULL DEFAULT FALSE,
   ADD COLUMN IF NOT EXISTS priority INTEGER NOT NULL DEFAULT 100;
 
+ALTER TABLE cluster_pools
+  ALTER COLUMN strategy SET DEFAULT 'weighted-resource';
+
 CREATE INDEX IF NOT EXISTS idx_clusters_status_health
   ON clusters(status, "healthStatus");
 

backend/src/applications/applications.service.ts

@@ -179,6 +179,13 @@ export class ApplicationsService {
     return this.appsRepository.save(app);
   }
 
+  async updateClusterAssignment(id: string, clusterId: string, poolId?: string): Promise<Application> {
+    const app = await this.findOne(id);
+    app.clusterId = clusterId;
+    app.poolId = poolId || app.poolId;
+    return this.appsRepository.save(app);
+  }
+
   async saveSuspendedReplicas(
     id: string,
     snapshot: Record<string, number>,

backend/src/clusters/clusters.service.ts

@@ -1,11 +1,20 @@
-import { Injectable, NotFoundException, Logger, BadRequestException, Inject, forwardRef } from '@nestjs/common';
+import {
+  Injectable,
+  NotFoundException,
+  Logger,
+  BadRequestException,
+  Inject,
+  forwardRef,
+  OnModuleDestroy,
+  OnModuleInit,
+} from '@nestjs/common';
 import { InjectRepository } from '@nestjs/typeorm';
 import { ConfigService } from '@nestjs/config';
 import { Repository, DataSource, In } from 'typeorm';
 import * as k8s from '@kubernetes/client-node';
 import * as crypto from 'crypto';
 import { Cluster, ClusterHealthStatus } from './entities/cluster.entity';
-import { ClusterPool } from './entities/cluster-pool.entity';
+import { ClusterPool, PoolStrategy } from './entities/cluster-pool.entity';
 import { ClusterHealth } from './entities/cluster-health.entity';
 import { ClusterAllocationLog } from './entities/cluster-allocation-log.entity';
 import { CreateClusterDto, UpdateClusterDto } from './dto/cluster.dto';
@@ -15,10 +24,21 @@ import { ElasticsearchService } from '../kubernetes/elasticsearch.service';
 import { CreateApplicationDto } from '../applications/dto/application.dto';
 
 @Injectable()
-export class ClustersService {
+export class ClustersService implements OnModuleInit, OnModuleDestroy {
   private readonly logger = new Logger(ClustersService.name);
   private roundRobinIndex = 0;
   private poolRoundRobinIndices = new Map<string, number>();
+  private weightedRoundRobinState = new Map<string, number>();
+  private clusterHealthCache = new Map<string, { cluster: Cluster; cachedAt: number }>();
+  private healthCheckTimer?: NodeJS.Timeout;
+  private readonly cacheTtlMs =
+    Number(process.env.CLUSTER_HEALTH_CACHE_TTL_MS || 60_000);
+  private readonly overloadedCpuThreshold =
+    Number(process.env.CLUSTER_OVERLOADED_CPU_THRESHOLD || 0.85);
+  private readonly overloadedMemoryThreshold =
+    Number(process.env.CLUSTER_OVERLOADED_MEMORY_THRESHOLD || 0.85);
+  private readonly overloadedPodThreshold =
+    Number(process.env.CLUSTER_OVERLOADED_POD_THRESHOLD || 0.85);
 
   constructor(
     @InjectRepository(Cluster)
@@ -35,6 +55,22 @@ export class ClustersService {
     private elasticsearchService: ElasticsearchService,
   ) {}
 
+  onModuleInit(): void {
+    const intervalMs = Number(process.env.CLUSTER_HEALTH_INTERVAL_MS || 120_000);
+    this.healthCheckTimer = setInterval(() => {
+      this.refreshAllClusterHealth().catch((err) => {
+        this.logger.warn(`Periodic cluster health refresh failed: ${err.message}`);
+      });
+    }, intervalMs);
+    this.healthCheckTimer.unref?.();
+  }
+
+  onModuleDestroy(): void {
+    if (this.healthCheckTimer) {
+      clearInterval(this.healthCheckTimer);
+    }
+  }
+
   /**
    * Test connection to a Kubernetes cluster using its kubeconfig.
    * Calls the /version endpoint to verify the cluster is reachable.
@@ -238,6 +274,11 @@ export class ClustersService {
   async selectClusterForApplication(
     dto: CreateApplicationDto,
     userId: string,
+    options: {
+      excludeClusterIds?: string[];
+      applicationId?: string;
+      reason?: string;
+    } = {},
   ): Promise<{ cluster: Cluster; pool?: ClusterPool; allocationLogId: string }> {
     const estimatedRequest = this.estimateApplicationRequest(dto);
     let pool = dto.poolId
@@ -256,15 +297,7 @@ export class ClustersService {
       });
     }
 
-    const candidateQuery = this.clustersRepository
+    const candidates = await this.getCachedHealthyClusters(pool || undefined, options.excludeClusterIds || []);
-      .createQueryBuilder('cluster')
-      .where('cluster.status = :status', { status: ClusterStatus.ACTIVE });
-
-    if (pool?.clusterIds?.length) {
-      candidateQuery.andWhere('cluster.id IN (:...clusterIds)', { clusterIds: pool.clusterIds });
-    }
-
-    const candidates = await candidateQuery.getMany();
     const appCounts = await this.getAppCounts(candidates.map((cluster) => cluster.id));
     const candidateScores: Record<string, any>[] = [];
     const rejectionReasons: Record<string, any>[] = [];
@@ -276,14 +309,18 @@ export class ClustersService {
         continue;
       }
 
-      const score = this.scoreCluster(cluster, estimatedRequest, appCounts.get(cluster.id) || 0, pool?.strategy || 'weighted-resource');
+      const strategy = this.resolveStrategy(pool?.strategy);
+      const appCount = appCounts.get(cluster.id) || 0;
+      const score = this.scoreCluster(cluster, estimatedRequest, appCount, strategy, (dto as any).region);
+      const resourceMetrics = this.getResourceMetrics(cluster, estimatedRequest, appCount);
       candidateScores.push({
         clusterId: cluster.id,
         clusterName: cluster.name,
         score,
         weight: cluster.weight || 1,
-        appCount: appCounts.get(cluster.id) || 0,
+        appCount,
         healthStatus: cluster.healthStatus,
+        utilization: resourceMetrics.utilization,
         availableResources: cluster.availableResources || null,
       });
     }
@@ -292,13 +329,14 @@ export class ClustersService {
       const log = await this.allocationLogsRepository.save(this.allocationLogsRepository.create({
         userId,
         poolId: pool?.id,
+        applicationId: options.applicationId,
         selectedClusterId: null,
-        strategy: pool?.strategy || 'weighted-resource',
+        strategy: this.resolveStrategy(pool?.strategy),
         estimatedRequest,
         candidateScores,
         rejectionReasons,
         status: 'failed',
-        message: 'No active healthy cluster has enough estimated capacity',
+        message: options.reason || 'No active healthy cluster has enough estimated capacity',
       }));
       throw new BadRequestException(
         `No active healthy cluster has enough capacity for this application (allocation log: ${log.id})`,
@@ -310,13 +348,14 @@ export class ClustersService {
     const log = await this.allocationLogsRepository.save(this.allocationLogsRepository.create({
       userId,
       poolId: pool?.id,
+      applicationId: options.applicationId,
       selectedClusterId: selected.id,
-      strategy: pool?.strategy || 'weighted-resource',
+      strategy: this.resolveStrategy(pool?.strategy),
       estimatedRequest,
       candidateScores,
       rejectionReasons,
       status: 'success',
-      message: `Selected ${selected.name}`,
+      message: options.reason ? `${options.reason}: selected ${selected.name}` : `Selected ${selected.name}`,
     }));
 
     this.logger.log(`Allocator selected cluster "${selected.name}" for user ${userId} (score ${candidateScores[0].score.toFixed(2)})`);
@@ -335,6 +374,48 @@ export class ClustersService {
     });
   }
 
+  async markAllocationFailure(
+    applicationId: string,
+    clusterId: string | undefined,
+    message: string,
+  ): Promise<void> {
+    if (clusterId) {
+      await this.clustersRepository.update(clusterId, {
+        healthStatus: 'degraded',
+        healthMessage: message,
+        lastHealthCheckedAt: new Date(),
+      });
+      this.clusterHealthCache.delete(clusterId);
+    }
+
+    const latest = await this.allocationLogsRepository.findOne({
+      where: {
+        applicationId,
+        ...(clusterId ? { selectedClusterId: clusterId } : {}),
+      },
+      order: { createdAt: 'DESC' },
+    });
+
+    if (latest) {
+      await this.allocationLogsRepository.update(latest.id, {
+        status: 'failed',
+        message,
+      });
+    }
+  }
+
+  async chooseFallbackClusterForApplication(
+    app: CreateApplicationDto & { id: string; userId: string; clusterId?: string; poolId?: string },
+    failedClusterIds: string[],
+    reason: string,
+  ): Promise<{ cluster: Cluster; pool?: ClusterPool; allocationLogId: string }> {
+    return this.selectClusterForApplication(app, app.userId, {
+      excludeClusterIds: failedClusterIds,
+      applicationId: app.id,
+      reason,
+    });
+  }
+
   /**
    * Get the optimal cluster using load-balancing strategy.
    * Strategy: 'least-apps' — picks the active cluster with fewest deployed applications.
@@ -593,18 +674,32 @@ export class ClustersService {
   /**
    * Get resource usage for a specific cluster — nodes, total CPU/memory, pod counts.
    */
-  async getClusterResources(id: string): Promise<{
+  async getClusterResources(id: string, options: { allowCached?: boolean } = {}): Promise<{
     nodes: { name: string; status: string; roles: string; cpuCapacity: string; memoryCapacity: string; cpuAllocatable: string; memoryAllocatable: string; }[];
     totalCpuCapacity: string;
     totalMemoryCapacity: string;
     totalCpuAllocatable: string;
     totalMemoryAllocatable: string;
+    totalCpuRequested?: string;
+    totalMemoryRequested?: string;
+    cpuUtilization?: number;
+    memoryUtilization?: number;
+    podUtilization?: number;
+    podCapacity?: number;
     podCount: number;
     nodeCount: number;
     readyNodeCount?: number;
     appCount: number;
   }> {
     const cluster = await this.findOne(id);
+    if (
+      options.allowCached !== false &&
+      cluster.availableResources &&
+      cluster.lastHealthCheckedAt &&
+      Date.now() - new Date(cluster.lastHealthCheckedAt).getTime() < this.cacheTtlMs
+    ) {
+      return cluster.availableResources as any;
+    }
 
     const kc = new k8s.KubeConfig();
     kc.loadFromString(cluster.kubeconfig);
@@ -647,6 +742,14 @@ export class ClustersService {
       // Get all pods count
       const podsRes = await coreApi.listPodForAllNamespaces();
       const podCount = podsRes.body.items.length;
+      let totalCpuRequested = 0;
+      let totalMemoryRequested = 0;
+      for (const pod of podsRes.body.items) {
+        for (const container of pod.spec?.containers || []) {
+          totalCpuRequested += this.parseCpuToMillicores(container.resources?.requests?.cpu || '0');
+          totalMemoryRequested += this.parseMemoryToMi(container.resources?.requests?.memory || '0');
+        }
+      }
 
       // Get app count for this cluster
       const appCountResult = await this.dataSource.query(
@@ -656,12 +759,19 @@ export class ClustersService {
       const appCount = parseInt(appCountResult[0]?.count || '0', 10);
 
       const readyNodeCount = nodes.filter((node) => node.status === 'Ready').length;
+      const podCapacity = nodes.length * 110;
       const resources = {
         nodes,
         totalCpuCapacity: `${totalCpuCap}m`,
         totalMemoryCapacity: `${totalMemCap.toFixed(0)}Mi`,
         totalCpuAllocatable: `${totalCpuAlloc}m`,
         totalMemoryAllocatable: `${totalMemAlloc.toFixed(0)}Mi`,
+        totalCpuRequested: `${totalCpuRequested}m`,
+        totalMemoryRequested: `${totalMemoryRequested.toFixed(0)}Mi`,
+        cpuUtilization: this.utilizationRatio(totalCpuRequested, totalCpuAlloc),
+        memoryUtilization: this.utilizationRatio(totalMemoryRequested, totalMemAlloc),
+        podUtilization: this.utilizationRatio(podCount, podCapacity),
+        podCapacity,
         podCount,
         nodeCount: nodes.length,
         readyNodeCount,
@@ -680,6 +790,16 @@ export class ClustersService {
         lastHealthCheckedAt: new Date(),
         availableResources: resources as any,
       });
+      this.clusterHealthCache.set(id, {
+        cluster: {
+          ...cluster,
+          healthStatus,
+          healthMessage,
+          lastHealthCheckedAt: new Date(),
+          availableResources: resources,
+        },
+        cachedAt: Date.now(),
+      });
       await this.recordHealthSnapshot(cluster, { status: healthStatus, message: healthMessage, resources });
 
       return resources;
@@ -690,6 +810,7 @@ export class ClustersService {
         healthMessage: err.message,
         lastHealthCheckedAt: new Date(),
       });
+      this.clusterHealthCache.delete(id);
       await this.recordHealthSnapshot(cluster, { status: 'unhealthy', message: err.message });
       throw new BadRequestException(`Cannot fetch resources: ${err.message}`);
     }
@@ -927,30 +1048,106 @@ export class ClustersService {
     };
   }
 
+  private async getCachedHealthyClusters(
+    pool?: ClusterPool,
+    excludeClusterIds: string[] = [],
+  ): Promise<Cluster[]> {
+    const query = this.clustersRepository
+      .createQueryBuilder('cluster')
+      .where('cluster.status = :status', { status: ClusterStatus.ACTIVE });
+
+    if (pool?.clusterIds?.length) {
+      query.andWhere('cluster.id IN (:...clusterIds)', { clusterIds: pool.clusterIds });
+    }
+
+    if (excludeClusterIds.length > 0) {
+      query.andWhere('cluster.id NOT IN (:...excludeClusterIds)', { excludeClusterIds });
+    }
+
+    const clusters = await query.getMany();
+    const healthy: Cluster[] = [];
+    for (const cluster of clusters) {
+      const cached = await this.getClusterSnapshot(cluster);
+      if (cached.status === ClusterStatus.ACTIVE && cached.healthStatus === 'healthy') {
+        healthy.push(cached);
+      }
+    }
+    return healthy;
+  }
+
+  private async getClusterSnapshot(cluster: Cluster): Promise<Cluster> {
+    const cached = this.clusterHealthCache.get(cluster.id);
+    if (cached && Date.now() - cached.cachedAt < this.cacheTtlMs) {
+      return cached.cluster;
+    }
+
+    if (
+      cluster.availableResources &&
+      cluster.lastHealthCheckedAt &&
+      Date.now() - new Date(cluster.lastHealthCheckedAt).getTime() < this.cacheTtlMs
+    ) {
+      this.clusterHealthCache.set(cluster.id, { cluster, cachedAt: Date.now() });
+      return cluster;
+    }
+
+    try {
+      await this.getClusterResources(cluster.id, { allowCached: false });
+    } catch (err: any) {
+      this.logger.warn(`Health refresh failed for cluster "${cluster.name}": ${err.message}`);
+    }
+
+    const fresh = await this.clustersRepository.findOne({ where: { id: cluster.id } });
+    const snapshot = fresh || cluster;
+    this.clusterHealthCache.set(cluster.id, { cluster: snapshot, cachedAt: Date.now() });
+    return snapshot;
+  }
+
+  async refreshAllClusterHealth(): Promise<void> {
+    const clusters = await this.clustersRepository.find({
+      where: [
+        { status: ClusterStatus.ACTIVE },
+        { status: ClusterStatus.MAINTENANCE },
+      ],
+    });
+
+    for (const cluster of clusters) {
+      try {
+        await this.getClusterResources(cluster.id, { allowCached: false });
+      } catch (err: any) {
+        this.logger.warn(`Cluster health refresh failed for "${cluster.name}": ${err.message}`);
+      }
+    }
+  }
+
   private getClusterRejectionReason(cluster: Cluster, estimatedRequest: Record<string, any>): string | null {
     if (cluster.status !== ClusterStatus.ACTIVE) {
       return `status=${cluster.status}`;
     }
 
-    if (cluster.healthStatus && !['healthy', 'unknown'].includes(cluster.healthStatus)) {
+    if (cluster.healthStatus !== 'healthy') {
       return `health=${cluster.healthStatus}`;
     }
 
-    const resources = cluster.availableResources || {};
+    const metrics = this.getResourceMetrics(cluster, estimatedRequest, 0);
-    const cpuFree = this.parseCpuToMillicores(resources.totalCpuAllocatable || resources.cpuAllocatable || '0');
+    const { available, utilization } = metrics;
-    const memoryFree = this.parseMemoryToMi(resources.totalMemoryAllocatable || resources.memoryAllocatable || '0');
-    const podCount = Number(resources.podCount || 0);
-    const nodeCount = Number(resources.nodeCount || 0);
-    const podCapacity = Number(resources.podCapacity || nodeCount * 110 || 0);
 
-    if (cpuFree > 0 && cpuFree < estimatedRequest.cpuMillicores) {
+    if (available.cpuMillicores > 0 && available.cpuMillicores < estimatedRequest.cpuMillicores) {
-      return `insufficient cpu (${cpuFree}m < ${estimatedRequest.cpuMillicores}m)`;
+      return `insufficient cpu (${available.cpuMillicores}m < ${estimatedRequest.cpuMillicores}m)`;
     }
-    if (memoryFree > 0 && memoryFree < estimatedRequest.memoryMi) {
+    if (available.memoryMi > 0 && available.memoryMi < estimatedRequest.memoryMi) {
-      return `insufficient memory (${memoryFree}Mi < ${estimatedRequest.memoryMi}Mi)`;
+      return `insufficient memory (${available.memoryMi}Mi < ${estimatedRequest.memoryMi}Mi)`;
     }
-    if (podCapacity > 0 && podCount + estimatedRequest.podEstimate > podCapacity) {
+    if (available.pods > 0 && available.pods < estimatedRequest.podEstimate) {
-      return `pod pressure (${podCount}/${podCapacity})`;
+      return `pod pressure (${available.pods} pods available < ${estimatedRequest.podEstimate})`;
+    }
+    if (utilization.cpu >= this.overloadedCpuThreshold) {
+      return `cpu overloaded (${Math.round(utilization.cpu * 100)}%)`;
+    }
+    if (utilization.memory >= this.overloadedMemoryThreshold) {
+      return `memory overloaded (${Math.round(utilization.memory * 100)}%)`;
+    }
+    if (utilization.pods >= this.overloadedPodThreshold) {
+      return `pod overloaded (${Math.round(utilization.pods * 100)}%)`;
     }
 
     return null;
@@ -960,34 +1157,111 @@ export class ClustersService {
     cluster: Cluster,
     estimatedRequest: Record<string, any>,
     appCount: number,
-    strategy: string,
+    strategy: PoolStrategy,
+    desiredRegion?: string,
   ): number {
+    const metrics = this.getResourceMetrics(cluster, estimatedRequest, appCount);
+    const capacityScore = metrics.capacityScore;
+    const appPenalty = Math.min(appCount, 100) * 0.75;
+
     if (strategy === 'round-robin') {
-      const idx = this.poolRoundRobinIndices.get('allocator') || 0;
+      return this.nextRoundRobinScore('allocator');
-      this.poolRoundRobinIndices.set('allocator', idx + 1);
+    }
-      return 1000 - idx;
+    if (strategy === 'weighted-round-robin') {
+      return this.nextWeightedRoundRobinScore(cluster);
+    }
+    if (strategy === 'least-apps') {
+      return 1000 - appPenalty + capacityScore * 100;
+    }
+    if (strategy === 'least-loaded') {
+      return 1000 + capacityScore * 200 - metrics.utilization.average * 100 - appPenalty;
+    }
+    if (strategy === 'region-based') {
+      const regionBoost = desiredRegion && cluster.region === desiredRegion ? 250 : 0;
+      return regionBoost + (cluster.weight || 1) * 50 + capacityScore * 150 - appPenalty;
     }
 
+    return (cluster.weight || 1) * 100 + capacityScore * 100 - appPenalty;
+  }
+
+  private getResourceMetrics(
+    cluster: Cluster,
+    estimatedRequest: Record<string, any>,
+    appCount: number,
+  ): {
+    available: { cpuMillicores: number; memoryMi: number; storageMi: number; pods: number };
+    utilization: { cpu: number; memory: number; storage: number; pods: number; appPressure: number; average: number };
+    capacityScore: number;
+  } {
     const resources = cluster.availableResources || {};
-    const cpuFree = this.parseCpuToMillicores(resources.totalCpuAllocatable || resources.cpuAllocatable || '0');
+    const cpuCapacity = this.parseCpuToMillicores(resources.totalCpuAllocatable || resources.cpuAllocatable || resources.totalCpuCapacity || '0');
-    const memoryFree = this.parseMemoryToMi(resources.totalMemoryAllocatable || resources.memoryAllocatable || '0');
+    const memoryCapacity = this.parseMemoryToMi(resources.totalMemoryAllocatable || resources.memoryAllocatable || resources.totalMemoryCapacity || '0');
-    const storageFree = this.parseStorageToMi(resources.storageAllocatable || resources.storageFree || '0');
+    const storageCapacity = this.parseStorageToMi(resources.storageAllocatable || resources.storageCapacity || '0');
+    const cpuRequested = this.parseCpuToMillicores(resources.totalCpuRequested || resources.cpuRequested || '0');
+    const memoryRequested = this.parseMemoryToMi(resources.totalMemoryRequested || resources.memoryRequested || '0');
+    const storageUsed = this.parseStorageToMi(resources.storageUsed || '0');
     const podCount = Number(resources.podCount || 0);
     const nodeCount = Number(resources.nodeCount || 0);
     const podCapacity = Number(resources.podCapacity || nodeCount * 110 || 0);
 
-    const capacityScore =
+    const available = {
-      this.ratioScore(cpuFree, estimatedRequest.cpuMillicores) * 0.35 +
+      cpuMillicores: Math.max(cpuCapacity - cpuRequested, 0),
-      this.ratioScore(memoryFree, estimatedRequest.memoryMi) * 0.35 +
+      memoryMi: Math.max(memoryCapacity - memoryRequested, 0),
-      this.ratioScore(storageFree, estimatedRequest.storageMi) * 0.15 +
+      storageMi: storageCapacity > 0 ? Math.max(storageCapacity - storageUsed, 0) : 0,
-      (podCapacity > 0 ? Math.max(0, 1 - podCount / podCapacity) : 0.7) * 0.15;
+      pods: podCapacity > 0 ? Math.max(podCapacity - podCount, 0) : 0,
-    const appPenalty = Math.min(appCount, 100) * 0.75;
+    };
+    const utilization = {
+      cpu: this.utilizationRatio(cpuRequested, cpuCapacity),
+      memory: this.utilizationRatio(memoryRequested, memoryCapacity),
+      storage: this.utilizationRatio(storageUsed, storageCapacity),
+      pods: this.utilizationRatio(podCount, podCapacity),
+      appPressure: Math.min(appCount / 100, 1),
+      average: 0,
+    };
+    utilization.average = (
+      utilization.cpu * 0.35 +
+      utilization.memory * 0.35 +
+      utilization.pods * 0.2 +
+      utilization.appPressure * 0.1
+    );
 
-    if (strategy === 'least-apps') {
+    const capacityScore =
-      return 1000 - appPenalty + capacityScore * 100;
+      this.ratioScore(available.cpuMillicores, estimatedRequest.cpuMillicores) * 0.35 +
+      this.ratioScore(available.memoryMi, estimatedRequest.memoryMi) * 0.35 +
+      this.ratioScore(available.storageMi, estimatedRequest.storageMi) * 0.1 +
+      this.ratioScore(available.pods, estimatedRequest.podEstimate) * 0.2;
+
+    return { available, utilization, capacityScore };
   }
 
-    return (cluster.weight || 1) * 100 + capacityScore * 100 - appPenalty;
+  private resolveStrategy(strategy?: string): PoolStrategy {
+    const strategies: PoolStrategy[] = [
+      'round-robin',
+      'weighted-round-robin',
+      'least-apps',
+      'least-loaded',
+      'weighted-resource',
+      'region-based',
+    ];
+    return strategies.includes(strategy as PoolStrategy) ? strategy as PoolStrategy : 'weighted-resource';
+  }
+
+  private nextRoundRobinScore(scope: string): number {
+    const idx = this.poolRoundRobinIndices.get(scope) || 0;
+    this.poolRoundRobinIndices.set(scope, idx + 1);
+    return 1000 - idx;
+  }
+
+  private nextWeightedRoundRobinScore(cluster: Cluster): number {
+    const current = this.weightedRoundRobinState.get(cluster.id) || 0;
+    const next = current + Math.max(cluster.weight || 1, 1);
+    this.weightedRoundRobinState.set(cluster.id, next);
+    return next;
+  }
+
+  private utilizationRatio(used: number, capacity: number): number {
+    if (!capacity || capacity <= 0) return 0;
+    return Math.min(Math.max(used / capacity, 0), 1);
   }
 
   private ratioScore(available: number, required: number): number {
@@ -1004,6 +1278,7 @@ export class ClustersService {
       SELECT "clusterId", COUNT(*) as count
       FROM applications
       WHERE "clusterId" = ANY($1)
+        AND "lifecycleStatus" = 'active'
       GROUP BY "clusterId"
     `, [clusterIds]);
     return new Map(rows.map((row) => [row.clusterId, parseInt(row.count, 10)]));

backend/src/clusters/dto/cluster-pool.dto.ts

@@ -1,6 +1,9 @@
 import { IsString, IsOptional, IsBoolean, IsArray, IsIn, IsNumber, Min } from 'class-validator';
 import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
 
+const poolStrategies = ['least-apps', 'round-robin', 'weighted-round-robin', 'least-loaded', 'weighted-resource', 'region-based'] as const;
+type PoolStrategyDto = typeof poolStrategies[number];
+
 export class CreateClusterPoolDto {
   @ApiProperty({ example: 'production-pool' })
   @IsString()
@@ -11,9 +14,9 @@ export class CreateClusterPoolDto {
   @IsString()
   description?: string;
 
-  @ApiProperty({ example: 'weighted-resource', enum: ['least-apps', 'round-robin', 'weighted-resource'] })
+  @ApiProperty({ example: 'weighted-resource', enum: poolStrategies })
-  @IsIn(['least-apps', 'round-robin', 'weighted-resource'])
+  @IsIn(poolStrategies)
-  strategy: 'least-apps' | 'round-robin' | 'weighted-resource';
+  strategy: PoolStrategyDto;
 
   @ApiProperty({ example: ['uuid-1', 'uuid-2'], description: 'Array of cluster IDs in this pool' })
   @IsArray()
@@ -43,10 +46,10 @@ export class UpdateClusterPoolDto {
   @IsString()
   description?: string;
 
-  @ApiPropertyOptional({ enum: ['least-apps', 'round-robin', 'weighted-resource'] })
+  @ApiPropertyOptional({ enum: poolStrategies })
   @IsOptional()
-  @IsIn(['least-apps', 'round-robin', 'weighted-resource'])
+  @IsIn(poolStrategies)
-  strategy?: 'least-apps' | 'round-robin' | 'weighted-resource';
+  strategy?: PoolStrategyDto;
 
   @ApiPropertyOptional({ description: 'Array of cluster IDs in this pool' })
   @IsOptional()

backend/src/clusters/entities/cluster-pool.entity.ts

@@ -6,7 +6,13 @@ import {
   UpdateDateColumn,
 } from 'typeorm';
 
-export type PoolStrategy = 'least-apps' | 'round-robin' | 'weighted-resource';
+export type PoolStrategy =
+  | 'round-robin'
+  | 'weighted-round-robin'
+  | 'least-apps'
+  | 'least-loaded'
+  | 'weighted-resource'
+  | 'region-based';
 
 @Entity('cluster_pools')
 export class ClusterPool {
@@ -19,7 +25,7 @@ export class ClusterPool {
   @Column({ nullable: true })
   description: string;
 
-  @Column({ default: 'least-apps' })
+  @Column({ default: 'weighted-resource' })
   strategy: PoolStrategy;
 
   /**

backend/src/deployments/deployments.module.ts

@@ -6,11 +6,13 @@ import { Deployment } from './entities/deployment.entity';
 import { ApplicationsModule } from '../applications/applications.module';
 import { KubernetesModule } from '../kubernetes/kubernetes.module';
 import { BuildModule } from '../build/build.module';
+import { ClustersModule } from '../clusters/clusters.module';
 
 @Module({
   imports: [
     TypeOrmModule.forFeature([Deployment]),
     forwardRef(() => ApplicationsModule),
+    forwardRef(() => ClustersModule),
     KubernetesModule,
     BuildModule,
   ],

backend/src/deployments/deployments.service.ts

@@ -7,6 +7,7 @@ import { ApplicationsService } from '../applications/applications.service';
 import { KubernetesService } from '../kubernetes/kubernetes.service';
 import { BuildService, BuildProgress, BuildCancelledError } from '../build/build.service';
 import { AppLifecycleStatus, DeploymentStatus } from '../common/enums';
+import { ClustersService } from '../clusters/clusters.service';
 
 @Injectable()
 export class DeploymentsService {
@@ -19,6 +20,7 @@ export class DeploymentsService {
     private applicationsService: ApplicationsService,
     private kubernetesService: KubernetesService,
     private buildService: BuildService,
+    private clustersService: ClustersService,
   ) {}
 
   async triggerDeployment(applicationId: string, userId: string): Promise<Deployment> {
@@ -62,11 +64,9 @@ export class DeploymentsService {
         message: 'Deploying to Kubernetes...',
       });
 
-      // If a DB dump will be restored, deploy with 0 replicas first so WordPress
-      // does not initialize empty tables before the dump is imported.
       const hasDbDump = app.dbDumpPath && fs.existsSync(app.dbDumpPath);
-      const deployApp = hasDbDump ? { ...app, replicas: 0 } : app;
+      const { app: deployedApp, k8sResources } = await this.deployWithClusterFallback(deploymentId, app, imageUri, hasDbDump);
-      const k8sResources = await this.kubernetesService.deployApplication(deployApp, imageUri);
+      app = deployedApp;
 
       // Step 3.5: Restore DB dump if one was uploaded (must happen after deploy creates the namespace + DB)
       if (hasDbDump) {
@@ -149,6 +149,70 @@ export class DeploymentsService {
     }
   }
 
+  private async deployWithClusterFallback(
+    deploymentId: string,
+    app: any,
+    imageUri: string,
+    hasDbDump: boolean,
+  ): Promise<{ app: any; k8sResources: Record<string, any> }> {
+    const failedClusterIds: string[] = [];
+    let currentApp = app;
+    let lastError: any;
+    const maxAttempts = Number(process.env.CLUSTER_DEPLOY_FALLBACK_ATTEMPTS || 3);
+
+    for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+      if (await this.isDeploymentCancelled(deploymentId)) {
+        throw new Error('Deployment cancelled by user');
+      }
+
+      try {
+        this.buildService.setProgress(deploymentId, {
+          phase: 'deploying',
+          percent: Math.min(92 + attempt, 95),
+          message: attempt === 1
+            ? 'Deploying to selected cluster...'
+            : `Retrying deployment on fallback cluster (${attempt}/${maxAttempts})...`,
+        });
+
+        const deployApp = hasDbDump ? { ...currentApp, replicas: 0 } : currentApp;
+        const k8sResources = await this.kubernetesService.deployApplication(deployApp, imageUri);
+        return { app: currentApp, k8sResources };
+      } catch (error: any) {
+        lastError = error;
+        failedClusterIds.push(currentApp.clusterId);
+        const failureMessage = error?.message || 'Deployment failed on selected cluster';
+        await this.clustersService.markAllocationFailure(currentApp.id, currentApp.clusterId, failureMessage);
+
+        if (attempt >= maxAttempts) {
+          break;
+        }
+
+        try {
+          const fallback = await this.clustersService.chooseFallbackClusterForApplication(
+            currentApp,
+            failedClusterIds,
+            failureMessage,
+          );
+          const updatedApp = await this.applicationsService.updateClusterAssignment(
+            currentApp.id,
+            fallback.cluster.id,
+            fallback.pool?.id,
+          );
+          await this.clustersService.attachAllocationToApplication(fallback.allocationLogId, currentApp.id);
+          this.logger.warn(
+            `Deployment ${deploymentId} falling back from cluster ${currentApp.clusterId || 'none'} to ${fallback.cluster.id}`,
+          );
+          currentApp = { ...currentApp, ...updatedApp, clusterId: fallback.cluster.id, poolId: fallback.pool?.id || currentApp.poolId };
+        } catch (fallbackError: any) {
+          this.logger.warn(`No fallback cluster available for deployment ${deploymentId}: ${fallbackError.message}`);
+          break;
+        }
+      }
+    }
+
+    throw lastError;
+  }
+
   async updateStatus(id: string, status: DeploymentStatus): Promise<void> {
     if (await this.isDeploymentCancelled(id)) {
       return;

frontend/src/app/dashboard/admin/pools/page.tsx

@@ -16,7 +16,7 @@ export default function AdminPoolsPage() {
   const [form, setForm] = useState({
     name: '',
     description: '',
-    strategy: 'weighted-resource' as 'least-apps' | 'round-robin' | 'weighted-resource',
+    strategy: 'weighted-resource' as 'least-apps' | 'round-robin' | 'weighted-round-robin' | 'least-loaded' | 'weighted-resource' | 'region-based',
     clusterIds: [] as string[],
     isDefault: false,
     priority: 100,
@@ -143,6 +143,9 @@ export default function AdminPoolsPage() {
                 onChange={(e) => setForm({ ...form, strategy: e.target.value as any })}
               >
                 <option value="weighted-resource">Weighted Resource — prefer healthy capacity and higher weights</option>
+                <option value="least-loaded">Least Loaded — prefer lowest CPU, memory, and pod pressure</option>
+                <option value="weighted-round-robin">Weighted Round Robin — rotate proportionally by weight</option>
+                <option value="region-based">Region Based — prefer matching region, then weight and load</option>
                 <option value="least-apps">Least Apps — deploy to cluster with fewest apps</option>
                 <option value="round-robin">Round Robin — rotate across clusters evenly</option>
               </select>
@@ -315,6 +318,12 @@ export default function AdminPoolsPage() {
                       <span className="badge badge-purple flex items-center gap-1">
                         {pool.strategy === 'weighted-resource'
                           ? <><BarChart3 className="w-3 h-3" /> Weighted Resource</>
+                          : pool.strategy === 'least-loaded'
+                          ? <><BarChart3 className="w-3 h-3" /> Least Loaded</>
+                          : pool.strategy === 'weighted-round-robin'
+                          ? <><RotateCw className="w-3 h-3" /> Weighted RR</>
+                          : pool.strategy === 'region-based'
+                          ? <><BarChart3 className="w-3 h-3" /> Region Based</>
                           : pool.strategy === 'least-apps'
                           ? <><BarChart3 className="w-3 h-3" /> Least Apps</>
                           : <><RotateCw className="w-3 h-3" /> Round Robin</>}

frontend/src/app/dashboard/deploy/page.tsx

@@ -2281,6 +2281,12 @@ export default function DeployPage() {
                                 <span className="text-xs bg-purple-100 text-purple-700 px-1.5 py-0.5 rounded flex items-center gap-1">
                                   {pool.strategy === 'weighted-resource'
                                     ? <><BarChart3 className="w-3 h-3" /> Weighted Resource</>
+                                    : pool.strategy === 'least-loaded'
+                                    ? <><BarChart3 className="w-3 h-3" /> Least Loaded</>
+                                    : pool.strategy === 'weighted-round-robin'
+                                    ? <><RotateCw className="w-3 h-3" /> Weighted RR</>
+                                    : pool.strategy === 'region-based'
+                                    ? <><BarChart3 className="w-3 h-3" /> Region Based</>
                                     : pool.strategy === 'least-apps'
                                     ? <><BarChart3 className="w-3 h-3" /> Least Apps</>
                                     : <><RotateCw className="w-3 h-3" /> Round Robin</>}

frontend/src/types/index.ts

@@ -254,7 +254,7 @@ export interface ClusterPoolPublic {
   id: string;
   name: string;
   description?: string;
-  strategy: 'least-apps' | 'round-robin' | 'weighted-resource';
+  strategy: 'least-apps' | 'round-robin' | 'weighted-round-robin' | 'least-loaded' | 'weighted-resource' | 'region-based';
   clusterIds: string[];
   isActive: boolean;
   isDefault: boolean;
@@ -266,7 +266,7 @@ export interface ClusterPool {
   id: string;
   name: string;
   description?: string;
-  strategy: 'least-apps' | 'round-robin' | 'weighted-resource';
+  strategy: 'least-apps' | 'round-robin' | 'weighted-round-robin' | 'least-loaded' | 'weighted-resource' | 'region-based';
   clusterIds: string[];
   isActive: boolean;
   isDefault: boolean;
@@ -295,7 +295,7 @@ export interface ClusterAllocationLog {
   userId: string;
   poolId?: string;
   selectedClusterId?: string;
-  strategy: 'least-apps' | 'round-robin' | 'weighted-resource';
+  strategy: 'least-apps' | 'round-robin' | 'weighted-round-robin' | 'least-loaded' | 'weighted-resource' | 'region-based';
   estimatedRequest?: Record<string, any>;
   candidateScores?: Record<string, any>[];
   rejectionReasons?: Record<string, any>[];