diff --git a/backend/migrations/012_cluster_pool_allocation.sql b/backend/migrations/012_cluster_pool_allocation.sql index dc33ab2..5781037 100644 --- a/backend/migrations/012_cluster_pool_allocation.sql +++ b/backend/migrations/012_cluster_pool_allocation.sql @@ -12,6 +12,9 @@ ALTER TABLE cluster_pools ADD COLUMN IF NOT EXISTS "isDefault" BOOLEAN NOT NULL DEFAULT FALSE, ADD COLUMN IF NOT EXISTS priority INTEGER NOT NULL DEFAULT 100; +ALTER TABLE cluster_pools + ALTER COLUMN strategy SET DEFAULT 'weighted-resource'; + CREATE INDEX IF NOT EXISTS idx_clusters_status_health ON clusters(status, "healthStatus"); diff --git a/backend/src/applications/applications.service.ts b/backend/src/applications/applications.service.ts index 3d5489d..b1d16cf 100644 --- a/backend/src/applications/applications.service.ts +++ b/backend/src/applications/applications.service.ts @@ -179,6 +179,13 @@ export class ApplicationsService { return this.appsRepository.save(app); } + async updateClusterAssignment(id: string, clusterId: string, poolId?: string): Promise { + const app = await this.findOne(id); + app.clusterId = clusterId; + app.poolId = poolId || app.poolId; + return this.appsRepository.save(app); + } + async saveSuspendedReplicas( id: string, snapshot: Record, diff --git a/backend/src/clusters/clusters.service.ts b/backend/src/clusters/clusters.service.ts index 4012dcc..f5905bd 100644 --- a/backend/src/clusters/clusters.service.ts +++ b/backend/src/clusters/clusters.service.ts @@ -1,11 +1,20 @@ -import { Injectable, NotFoundException, Logger, BadRequestException, Inject, forwardRef } from '@nestjs/common'; +import { + Injectable, + NotFoundException, + Logger, + BadRequestException, + Inject, + forwardRef, + OnModuleDestroy, + OnModuleInit, +} from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; import { ConfigService } from '@nestjs/config'; import { Repository, DataSource, In } from 'typeorm'; import * as k8s from '@kubernetes/client-node'; import * as crypto from 'crypto'; import { Cluster, ClusterHealthStatus } from './entities/cluster.entity'; -import { ClusterPool } from './entities/cluster-pool.entity'; +import { ClusterPool, PoolStrategy } from './entities/cluster-pool.entity'; import { ClusterHealth } from './entities/cluster-health.entity'; import { ClusterAllocationLog } from './entities/cluster-allocation-log.entity'; import { CreateClusterDto, UpdateClusterDto } from './dto/cluster.dto'; @@ -15,10 +24,21 @@ import { ElasticsearchService } from '../kubernetes/elasticsearch.service'; import { CreateApplicationDto } from '../applications/dto/application.dto'; @Injectable() -export class ClustersService { +export class ClustersService implements OnModuleInit, OnModuleDestroy { private readonly logger = new Logger(ClustersService.name); private roundRobinIndex = 0; private poolRoundRobinIndices = new Map(); + private weightedRoundRobinState = new Map(); + private clusterHealthCache = new Map(); + private healthCheckTimer?: NodeJS.Timeout; + private readonly cacheTtlMs = + Number(process.env.CLUSTER_HEALTH_CACHE_TTL_MS || 60_000); + private readonly overloadedCpuThreshold = + Number(process.env.CLUSTER_OVERLOADED_CPU_THRESHOLD || 0.85); + private readonly overloadedMemoryThreshold = + Number(process.env.CLUSTER_OVERLOADED_MEMORY_THRESHOLD || 0.85); + private readonly overloadedPodThreshold = + Number(process.env.CLUSTER_OVERLOADED_POD_THRESHOLD || 0.85); constructor( @InjectRepository(Cluster) @@ -35,6 +55,22 @@ export class ClustersService { private elasticsearchService: ElasticsearchService, ) {} + onModuleInit(): void { + const intervalMs = Number(process.env.CLUSTER_HEALTH_INTERVAL_MS || 120_000); + this.healthCheckTimer = setInterval(() => { + this.refreshAllClusterHealth().catch((err) => { + this.logger.warn(`Periodic cluster health refresh failed: ${err.message}`); + }); + }, intervalMs); + this.healthCheckTimer.unref?.(); + } + + onModuleDestroy(): void { + if (this.healthCheckTimer) { + clearInterval(this.healthCheckTimer); + } + } + /** * Test connection to a Kubernetes cluster using its kubeconfig. * Calls the /version endpoint to verify the cluster is reachable. @@ -238,6 +274,11 @@ export class ClustersService { async selectClusterForApplication( dto: CreateApplicationDto, userId: string, + options: { + excludeClusterIds?: string[]; + applicationId?: string; + reason?: string; + } = {}, ): Promise<{ cluster: Cluster; pool?: ClusterPool; allocationLogId: string }> { const estimatedRequest = this.estimateApplicationRequest(dto); let pool = dto.poolId @@ -256,15 +297,7 @@ export class ClustersService { }); } - const candidateQuery = this.clustersRepository - .createQueryBuilder('cluster') - .where('cluster.status = :status', { status: ClusterStatus.ACTIVE }); - - if (pool?.clusterIds?.length) { - candidateQuery.andWhere('cluster.id IN (:...clusterIds)', { clusterIds: pool.clusterIds }); - } - - const candidates = await candidateQuery.getMany(); + const candidates = await this.getCachedHealthyClusters(pool || undefined, options.excludeClusterIds || []); const appCounts = await this.getAppCounts(candidates.map((cluster) => cluster.id)); const candidateScores: Record[] = []; const rejectionReasons: Record[] = []; @@ -276,14 +309,18 @@ export class ClustersService { continue; } - const score = this.scoreCluster(cluster, estimatedRequest, appCounts.get(cluster.id) || 0, pool?.strategy || 'weighted-resource'); + const strategy = this.resolveStrategy(pool?.strategy); + const appCount = appCounts.get(cluster.id) || 0; + const score = this.scoreCluster(cluster, estimatedRequest, appCount, strategy, (dto as any).region); + const resourceMetrics = this.getResourceMetrics(cluster, estimatedRequest, appCount); candidateScores.push({ clusterId: cluster.id, clusterName: cluster.name, score, weight: cluster.weight || 1, - appCount: appCounts.get(cluster.id) || 0, + appCount, healthStatus: cluster.healthStatus, + utilization: resourceMetrics.utilization, availableResources: cluster.availableResources || null, }); } @@ -292,13 +329,14 @@ export class ClustersService { const log = await this.allocationLogsRepository.save(this.allocationLogsRepository.create({ userId, poolId: pool?.id, + applicationId: options.applicationId, selectedClusterId: null, - strategy: pool?.strategy || 'weighted-resource', + strategy: this.resolveStrategy(pool?.strategy), estimatedRequest, candidateScores, rejectionReasons, status: 'failed', - message: 'No active healthy cluster has enough estimated capacity', + message: options.reason || 'No active healthy cluster has enough estimated capacity', })); throw new BadRequestException( `No active healthy cluster has enough capacity for this application (allocation log: ${log.id})`, @@ -310,13 +348,14 @@ export class ClustersService { const log = await this.allocationLogsRepository.save(this.allocationLogsRepository.create({ userId, poolId: pool?.id, + applicationId: options.applicationId, selectedClusterId: selected.id, - strategy: pool?.strategy || 'weighted-resource', + strategy: this.resolveStrategy(pool?.strategy), estimatedRequest, candidateScores, rejectionReasons, status: 'success', - message: `Selected ${selected.name}`, + message: options.reason ? `${options.reason}: selected ${selected.name}` : `Selected ${selected.name}`, })); this.logger.log(`Allocator selected cluster "${selected.name}" for user ${userId} (score ${candidateScores[0].score.toFixed(2)})`); @@ -335,6 +374,48 @@ export class ClustersService { }); } + async markAllocationFailure( + applicationId: string, + clusterId: string | undefined, + message: string, + ): Promise { + if (clusterId) { + await this.clustersRepository.update(clusterId, { + healthStatus: 'degraded', + healthMessage: message, + lastHealthCheckedAt: new Date(), + }); + this.clusterHealthCache.delete(clusterId); + } + + const latest = await this.allocationLogsRepository.findOne({ + where: { + applicationId, + ...(clusterId ? { selectedClusterId: clusterId } : {}), + }, + order: { createdAt: 'DESC' }, + }); + + if (latest) { + await this.allocationLogsRepository.update(latest.id, { + status: 'failed', + message, + }); + } + } + + async chooseFallbackClusterForApplication( + app: CreateApplicationDto & { id: string; userId: string; clusterId?: string; poolId?: string }, + failedClusterIds: string[], + reason: string, + ): Promise<{ cluster: Cluster; pool?: ClusterPool; allocationLogId: string }> { + return this.selectClusterForApplication(app, app.userId, { + excludeClusterIds: failedClusterIds, + applicationId: app.id, + reason, + }); + } + /** * Get the optimal cluster using load-balancing strategy. * Strategy: 'least-apps' — picks the active cluster with fewest deployed applications. @@ -593,18 +674,32 @@ export class ClustersService { /** * Get resource usage for a specific cluster — nodes, total CPU/memory, pod counts. */ - async getClusterResources(id: string): Promise<{ + async getClusterResources(id: string, options: { allowCached?: boolean } = {}): Promise<{ nodes: { name: string; status: string; roles: string; cpuCapacity: string; memoryCapacity: string; cpuAllocatable: string; memoryAllocatable: string; }[]; totalCpuCapacity: string; totalMemoryCapacity: string; totalCpuAllocatable: string; totalMemoryAllocatable: string; + totalCpuRequested?: string; + totalMemoryRequested?: string; + cpuUtilization?: number; + memoryUtilization?: number; + podUtilization?: number; + podCapacity?: number; podCount: number; nodeCount: number; readyNodeCount?: number; appCount: number; }> { const cluster = await this.findOne(id); + if ( + options.allowCached !== false && + cluster.availableResources && + cluster.lastHealthCheckedAt && + Date.now() - new Date(cluster.lastHealthCheckedAt).getTime() < this.cacheTtlMs + ) { + return cluster.availableResources as any; + } const kc = new k8s.KubeConfig(); kc.loadFromString(cluster.kubeconfig); @@ -647,6 +742,14 @@ export class ClustersService { // Get all pods count const podsRes = await coreApi.listPodForAllNamespaces(); const podCount = podsRes.body.items.length; + let totalCpuRequested = 0; + let totalMemoryRequested = 0; + for (const pod of podsRes.body.items) { + for (const container of pod.spec?.containers || []) { + totalCpuRequested += this.parseCpuToMillicores(container.resources?.requests?.cpu || '0'); + totalMemoryRequested += this.parseMemoryToMi(container.resources?.requests?.memory || '0'); + } + } // Get app count for this cluster const appCountResult = await this.dataSource.query( @@ -656,12 +759,19 @@ export class ClustersService { const appCount = parseInt(appCountResult[0]?.count || '0', 10); const readyNodeCount = nodes.filter((node) => node.status === 'Ready').length; + const podCapacity = nodes.length * 110; const resources = { nodes, totalCpuCapacity: `${totalCpuCap}m`, totalMemoryCapacity: `${totalMemCap.toFixed(0)}Mi`, totalCpuAllocatable: `${totalCpuAlloc}m`, totalMemoryAllocatable: `${totalMemAlloc.toFixed(0)}Mi`, + totalCpuRequested: `${totalCpuRequested}m`, + totalMemoryRequested: `${totalMemoryRequested.toFixed(0)}Mi`, + cpuUtilization: this.utilizationRatio(totalCpuRequested, totalCpuAlloc), + memoryUtilization: this.utilizationRatio(totalMemoryRequested, totalMemAlloc), + podUtilization: this.utilizationRatio(podCount, podCapacity), + podCapacity, podCount, nodeCount: nodes.length, readyNodeCount, @@ -680,6 +790,16 @@ export class ClustersService { lastHealthCheckedAt: new Date(), availableResources: resources as any, }); + this.clusterHealthCache.set(id, { + cluster: { + ...cluster, + healthStatus, + healthMessage, + lastHealthCheckedAt: new Date(), + availableResources: resources, + }, + cachedAt: Date.now(), + }); await this.recordHealthSnapshot(cluster, { status: healthStatus, message: healthMessage, resources }); return resources; @@ -690,6 +810,7 @@ export class ClustersService { healthMessage: err.message, lastHealthCheckedAt: new Date(), }); + this.clusterHealthCache.delete(id); await this.recordHealthSnapshot(cluster, { status: 'unhealthy', message: err.message }); throw new BadRequestException(`Cannot fetch resources: ${err.message}`); } @@ -927,30 +1048,106 @@ export class ClustersService { }; } + private async getCachedHealthyClusters( + pool?: ClusterPool, + excludeClusterIds: string[] = [], + ): Promise { + const query = this.clustersRepository + .createQueryBuilder('cluster') + .where('cluster.status = :status', { status: ClusterStatus.ACTIVE }); + + if (pool?.clusterIds?.length) { + query.andWhere('cluster.id IN (:...clusterIds)', { clusterIds: pool.clusterIds }); + } + + if (excludeClusterIds.length > 0) { + query.andWhere('cluster.id NOT IN (:...excludeClusterIds)', { excludeClusterIds }); + } + + const clusters = await query.getMany(); + const healthy: Cluster[] = []; + for (const cluster of clusters) { + const cached = await this.getClusterSnapshot(cluster); + if (cached.status === ClusterStatus.ACTIVE && cached.healthStatus === 'healthy') { + healthy.push(cached); + } + } + return healthy; + } + + private async getClusterSnapshot(cluster: Cluster): Promise { + const cached = this.clusterHealthCache.get(cluster.id); + if (cached && Date.now() - cached.cachedAt < this.cacheTtlMs) { + return cached.cluster; + } + + if ( + cluster.availableResources && + cluster.lastHealthCheckedAt && + Date.now() - new Date(cluster.lastHealthCheckedAt).getTime() < this.cacheTtlMs + ) { + this.clusterHealthCache.set(cluster.id, { cluster, cachedAt: Date.now() }); + return cluster; + } + + try { + await this.getClusterResources(cluster.id, { allowCached: false }); + } catch (err: any) { + this.logger.warn(`Health refresh failed for cluster "${cluster.name}": ${err.message}`); + } + + const fresh = await this.clustersRepository.findOne({ where: { id: cluster.id } }); + const snapshot = fresh || cluster; + this.clusterHealthCache.set(cluster.id, { cluster: snapshot, cachedAt: Date.now() }); + return snapshot; + } + + async refreshAllClusterHealth(): Promise { + const clusters = await this.clustersRepository.find({ + where: [ + { status: ClusterStatus.ACTIVE }, + { status: ClusterStatus.MAINTENANCE }, + ], + }); + + for (const cluster of clusters) { + try { + await this.getClusterResources(cluster.id, { allowCached: false }); + } catch (err: any) { + this.logger.warn(`Cluster health refresh failed for "${cluster.name}": ${err.message}`); + } + } + } + private getClusterRejectionReason(cluster: Cluster, estimatedRequest: Record): string | null { if (cluster.status !== ClusterStatus.ACTIVE) { return `status=${cluster.status}`; } - if (cluster.healthStatus && !['healthy', 'unknown'].includes(cluster.healthStatus)) { + if (cluster.healthStatus !== 'healthy') { return `health=${cluster.healthStatus}`; } - const resources = cluster.availableResources || {}; - const cpuFree = this.parseCpuToMillicores(resources.totalCpuAllocatable || resources.cpuAllocatable || '0'); - const memoryFree = this.parseMemoryToMi(resources.totalMemoryAllocatable || resources.memoryAllocatable || '0'); - const podCount = Number(resources.podCount || 0); - const nodeCount = Number(resources.nodeCount || 0); - const podCapacity = Number(resources.podCapacity || nodeCount * 110 || 0); + const metrics = this.getResourceMetrics(cluster, estimatedRequest, 0); + const { available, utilization } = metrics; - if (cpuFree > 0 && cpuFree < estimatedRequest.cpuMillicores) { - return `insufficient cpu (${cpuFree}m < ${estimatedRequest.cpuMillicores}m)`; + if (available.cpuMillicores > 0 && available.cpuMillicores < estimatedRequest.cpuMillicores) { + return `insufficient cpu (${available.cpuMillicores}m < ${estimatedRequest.cpuMillicores}m)`; } - if (memoryFree > 0 && memoryFree < estimatedRequest.memoryMi) { - return `insufficient memory (${memoryFree}Mi < ${estimatedRequest.memoryMi}Mi)`; + if (available.memoryMi > 0 && available.memoryMi < estimatedRequest.memoryMi) { + return `insufficient memory (${available.memoryMi}Mi < ${estimatedRequest.memoryMi}Mi)`; } - if (podCapacity > 0 && podCount + estimatedRequest.podEstimate > podCapacity) { - return `pod pressure (${podCount}/${podCapacity})`; + if (available.pods > 0 && available.pods < estimatedRequest.podEstimate) { + return `pod pressure (${available.pods} pods available < ${estimatedRequest.podEstimate})`; + } + if (utilization.cpu >= this.overloadedCpuThreshold) { + return `cpu overloaded (${Math.round(utilization.cpu * 100)}%)`; + } + if (utilization.memory >= this.overloadedMemoryThreshold) { + return `memory overloaded (${Math.round(utilization.memory * 100)}%)`; + } + if (utilization.pods >= this.overloadedPodThreshold) { + return `pod overloaded (${Math.round(utilization.pods * 100)}%)`; } return null; @@ -960,34 +1157,111 @@ export class ClustersService { cluster: Cluster, estimatedRequest: Record, appCount: number, - strategy: string, + strategy: PoolStrategy, + desiredRegion?: string, ): number { + const metrics = this.getResourceMetrics(cluster, estimatedRequest, appCount); + const capacityScore = metrics.capacityScore; + const appPenalty = Math.min(appCount, 100) * 0.75; + if (strategy === 'round-robin') { - const idx = this.poolRoundRobinIndices.get('allocator') || 0; - this.poolRoundRobinIndices.set('allocator', idx + 1); - return 1000 - idx; + return this.nextRoundRobinScore('allocator'); + } + if (strategy === 'weighted-round-robin') { + return this.nextWeightedRoundRobinScore(cluster); + } + if (strategy === 'least-apps') { + return 1000 - appPenalty + capacityScore * 100; + } + if (strategy === 'least-loaded') { + return 1000 + capacityScore * 200 - metrics.utilization.average * 100 - appPenalty; + } + if (strategy === 'region-based') { + const regionBoost = desiredRegion && cluster.region === desiredRegion ? 250 : 0; + return regionBoost + (cluster.weight || 1) * 50 + capacityScore * 150 - appPenalty; } + return (cluster.weight || 1) * 100 + capacityScore * 100 - appPenalty; + } + + private getResourceMetrics( + cluster: Cluster, + estimatedRequest: Record, + appCount: number, + ): { + available: { cpuMillicores: number; memoryMi: number; storageMi: number; pods: number }; + utilization: { cpu: number; memory: number; storage: number; pods: number; appPressure: number; average: number }; + capacityScore: number; + } { const resources = cluster.availableResources || {}; - const cpuFree = this.parseCpuToMillicores(resources.totalCpuAllocatable || resources.cpuAllocatable || '0'); - const memoryFree = this.parseMemoryToMi(resources.totalMemoryAllocatable || resources.memoryAllocatable || '0'); - const storageFree = this.parseStorageToMi(resources.storageAllocatable || resources.storageFree || '0'); + const cpuCapacity = this.parseCpuToMillicores(resources.totalCpuAllocatable || resources.cpuAllocatable || resources.totalCpuCapacity || '0'); + const memoryCapacity = this.parseMemoryToMi(resources.totalMemoryAllocatable || resources.memoryAllocatable || resources.totalMemoryCapacity || '0'); + const storageCapacity = this.parseStorageToMi(resources.storageAllocatable || resources.storageCapacity || '0'); + const cpuRequested = this.parseCpuToMillicores(resources.totalCpuRequested || resources.cpuRequested || '0'); + const memoryRequested = this.parseMemoryToMi(resources.totalMemoryRequested || resources.memoryRequested || '0'); + const storageUsed = this.parseStorageToMi(resources.storageUsed || '0'); const podCount = Number(resources.podCount || 0); const nodeCount = Number(resources.nodeCount || 0); const podCapacity = Number(resources.podCapacity || nodeCount * 110 || 0); + const available = { + cpuMillicores: Math.max(cpuCapacity - cpuRequested, 0), + memoryMi: Math.max(memoryCapacity - memoryRequested, 0), + storageMi: storageCapacity > 0 ? Math.max(storageCapacity - storageUsed, 0) : 0, + pods: podCapacity > 0 ? Math.max(podCapacity - podCount, 0) : 0, + }; + const utilization = { + cpu: this.utilizationRatio(cpuRequested, cpuCapacity), + memory: this.utilizationRatio(memoryRequested, memoryCapacity), + storage: this.utilizationRatio(storageUsed, storageCapacity), + pods: this.utilizationRatio(podCount, podCapacity), + appPressure: Math.min(appCount / 100, 1), + average: 0, + }; + utilization.average = ( + utilization.cpu * 0.35 + + utilization.memory * 0.35 + + utilization.pods * 0.2 + + utilization.appPressure * 0.1 + ); + const capacityScore = - this.ratioScore(cpuFree, estimatedRequest.cpuMillicores) * 0.35 + - this.ratioScore(memoryFree, estimatedRequest.memoryMi) * 0.35 + - this.ratioScore(storageFree, estimatedRequest.storageMi) * 0.15 + - (podCapacity > 0 ? Math.max(0, 1 - podCount / podCapacity) : 0.7) * 0.15; - const appPenalty = Math.min(appCount, 100) * 0.75; + this.ratioScore(available.cpuMillicores, estimatedRequest.cpuMillicores) * 0.35 + + this.ratioScore(available.memoryMi, estimatedRequest.memoryMi) * 0.35 + + this.ratioScore(available.storageMi, estimatedRequest.storageMi) * 0.1 + + this.ratioScore(available.pods, estimatedRequest.podEstimate) * 0.2; - if (strategy === 'least-apps') { - return 1000 - appPenalty + capacityScore * 100; - } + return { available, utilization, capacityScore }; + } - return (cluster.weight || 1) * 100 + capacityScore * 100 - appPenalty; + private resolveStrategy(strategy?: string): PoolStrategy { + const strategies: PoolStrategy[] = [ + 'round-robin', + 'weighted-round-robin', + 'least-apps', + 'least-loaded', + 'weighted-resource', + 'region-based', + ]; + return strategies.includes(strategy as PoolStrategy) ? strategy as PoolStrategy : 'weighted-resource'; + } + + private nextRoundRobinScore(scope: string): number { + const idx = this.poolRoundRobinIndices.get(scope) || 0; + this.poolRoundRobinIndices.set(scope, idx + 1); + return 1000 - idx; + } + + private nextWeightedRoundRobinScore(cluster: Cluster): number { + const current = this.weightedRoundRobinState.get(cluster.id) || 0; + const next = current + Math.max(cluster.weight || 1, 1); + this.weightedRoundRobinState.set(cluster.id, next); + return next; + } + + private utilizationRatio(used: number, capacity: number): number { + if (!capacity || capacity <= 0) return 0; + return Math.min(Math.max(used / capacity, 0), 1); } private ratioScore(available: number, required: number): number { @@ -1004,6 +1278,7 @@ export class ClustersService { SELECT "clusterId", COUNT(*) as count FROM applications WHERE "clusterId" = ANY($1) + AND "lifecycleStatus" = 'active' GROUP BY "clusterId" `, [clusterIds]); return new Map(rows.map((row) => [row.clusterId, parseInt(row.count, 10)])); diff --git a/backend/src/clusters/dto/cluster-pool.dto.ts b/backend/src/clusters/dto/cluster-pool.dto.ts index f9766c2..cb9e509 100644 --- a/backend/src/clusters/dto/cluster-pool.dto.ts +++ b/backend/src/clusters/dto/cluster-pool.dto.ts @@ -1,6 +1,9 @@ import { IsString, IsOptional, IsBoolean, IsArray, IsIn, IsNumber, Min } from 'class-validator'; import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger'; +const poolStrategies = ['least-apps', 'round-robin', 'weighted-round-robin', 'least-loaded', 'weighted-resource', 'region-based'] as const; +type PoolStrategyDto = typeof poolStrategies[number]; + export class CreateClusterPoolDto { @ApiProperty({ example: 'production-pool' }) @IsString() @@ -11,9 +14,9 @@ export class CreateClusterPoolDto { @IsString() description?: string; - @ApiProperty({ example: 'weighted-resource', enum: ['least-apps', 'round-robin', 'weighted-resource'] }) - @IsIn(['least-apps', 'round-robin', 'weighted-resource']) - strategy: 'least-apps' | 'round-robin' | 'weighted-resource'; + @ApiProperty({ example: 'weighted-resource', enum: poolStrategies }) + @IsIn(poolStrategies) + strategy: PoolStrategyDto; @ApiProperty({ example: ['uuid-1', 'uuid-2'], description: 'Array of cluster IDs in this pool' }) @IsArray() @@ -43,10 +46,10 @@ export class UpdateClusterPoolDto { @IsString() description?: string; - @ApiPropertyOptional({ enum: ['least-apps', 'round-robin', 'weighted-resource'] }) + @ApiPropertyOptional({ enum: poolStrategies }) @IsOptional() - @IsIn(['least-apps', 'round-robin', 'weighted-resource']) - strategy?: 'least-apps' | 'round-robin' | 'weighted-resource'; + @IsIn(poolStrategies) + strategy?: PoolStrategyDto; @ApiPropertyOptional({ description: 'Array of cluster IDs in this pool' }) @IsOptional() diff --git a/backend/src/clusters/entities/cluster-pool.entity.ts b/backend/src/clusters/entities/cluster-pool.entity.ts index 0a3d76d..69d3c0b 100644 --- a/backend/src/clusters/entities/cluster-pool.entity.ts +++ b/backend/src/clusters/entities/cluster-pool.entity.ts @@ -6,7 +6,13 @@ import { UpdateDateColumn, } from 'typeorm'; -export type PoolStrategy = 'least-apps' | 'round-robin' | 'weighted-resource'; +export type PoolStrategy = + | 'round-robin' + | 'weighted-round-robin' + | 'least-apps' + | 'least-loaded' + | 'weighted-resource' + | 'region-based'; @Entity('cluster_pools') export class ClusterPool { @@ -19,7 +25,7 @@ export class ClusterPool { @Column({ nullable: true }) description: string; - @Column({ default: 'least-apps' }) + @Column({ default: 'weighted-resource' }) strategy: PoolStrategy; /** diff --git a/backend/src/deployments/deployments.module.ts b/backend/src/deployments/deployments.module.ts index 7545194..0d7e0e3 100644 --- a/backend/src/deployments/deployments.module.ts +++ b/backend/src/deployments/deployments.module.ts @@ -6,11 +6,13 @@ import { Deployment } from './entities/deployment.entity'; import { ApplicationsModule } from '../applications/applications.module'; import { KubernetesModule } from '../kubernetes/kubernetes.module'; import { BuildModule } from '../build/build.module'; +import { ClustersModule } from '../clusters/clusters.module'; @Module({ imports: [ TypeOrmModule.forFeature([Deployment]), forwardRef(() => ApplicationsModule), + forwardRef(() => ClustersModule), KubernetesModule, BuildModule, ], diff --git a/backend/src/deployments/deployments.service.ts b/backend/src/deployments/deployments.service.ts index 014f521..ad9f0f1 100644 --- a/backend/src/deployments/deployments.service.ts +++ b/backend/src/deployments/deployments.service.ts @@ -7,6 +7,7 @@ import { ApplicationsService } from '../applications/applications.service'; import { KubernetesService } from '../kubernetes/kubernetes.service'; import { BuildService, BuildProgress, BuildCancelledError } from '../build/build.service'; import { AppLifecycleStatus, DeploymentStatus } from '../common/enums'; +import { ClustersService } from '../clusters/clusters.service'; @Injectable() export class DeploymentsService { @@ -19,6 +20,7 @@ export class DeploymentsService { private applicationsService: ApplicationsService, private kubernetesService: KubernetesService, private buildService: BuildService, + private clustersService: ClustersService, ) {} async triggerDeployment(applicationId: string, userId: string): Promise { @@ -62,11 +64,9 @@ export class DeploymentsService { message: 'Deploying to Kubernetes...', }); - // If a DB dump will be restored, deploy with 0 replicas first so WordPress - // does not initialize empty tables before the dump is imported. const hasDbDump = app.dbDumpPath && fs.existsSync(app.dbDumpPath); - const deployApp = hasDbDump ? { ...app, replicas: 0 } : app; - const k8sResources = await this.kubernetesService.deployApplication(deployApp, imageUri); + const { app: deployedApp, k8sResources } = await this.deployWithClusterFallback(deploymentId, app, imageUri, hasDbDump); + app = deployedApp; // Step 3.5: Restore DB dump if one was uploaded (must happen after deploy creates the namespace + DB) if (hasDbDump) { @@ -149,6 +149,70 @@ export class DeploymentsService { } } + private async deployWithClusterFallback( + deploymentId: string, + app: any, + imageUri: string, + hasDbDump: boolean, + ): Promise<{ app: any; k8sResources: Record }> { + const failedClusterIds: string[] = []; + let currentApp = app; + let lastError: any; + const maxAttempts = Number(process.env.CLUSTER_DEPLOY_FALLBACK_ATTEMPTS || 3); + + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + if (await this.isDeploymentCancelled(deploymentId)) { + throw new Error('Deployment cancelled by user'); + } + + try { + this.buildService.setProgress(deploymentId, { + phase: 'deploying', + percent: Math.min(92 + attempt, 95), + message: attempt === 1 + ? 'Deploying to selected cluster...' + : `Retrying deployment on fallback cluster (${attempt}/${maxAttempts})...`, + }); + + const deployApp = hasDbDump ? { ...currentApp, replicas: 0 } : currentApp; + const k8sResources = await this.kubernetesService.deployApplication(deployApp, imageUri); + return { app: currentApp, k8sResources }; + } catch (error: any) { + lastError = error; + failedClusterIds.push(currentApp.clusterId); + const failureMessage = error?.message || 'Deployment failed on selected cluster'; + await this.clustersService.markAllocationFailure(currentApp.id, currentApp.clusterId, failureMessage); + + if (attempt >= maxAttempts) { + break; + } + + try { + const fallback = await this.clustersService.chooseFallbackClusterForApplication( + currentApp, + failedClusterIds, + failureMessage, + ); + const updatedApp = await this.applicationsService.updateClusterAssignment( + currentApp.id, + fallback.cluster.id, + fallback.pool?.id, + ); + await this.clustersService.attachAllocationToApplication(fallback.allocationLogId, currentApp.id); + this.logger.warn( + `Deployment ${deploymentId} falling back from cluster ${currentApp.clusterId || 'none'} to ${fallback.cluster.id}`, + ); + currentApp = { ...currentApp, ...updatedApp, clusterId: fallback.cluster.id, poolId: fallback.pool?.id || currentApp.poolId }; + } catch (fallbackError: any) { + this.logger.warn(`No fallback cluster available for deployment ${deploymentId}: ${fallbackError.message}`); + break; + } + } + } + + throw lastError; + } + async updateStatus(id: string, status: DeploymentStatus): Promise { if (await this.isDeploymentCancelled(id)) { return; diff --git a/frontend/src/app/dashboard/admin/pools/page.tsx b/frontend/src/app/dashboard/admin/pools/page.tsx index 61f15a0..db9881c 100644 --- a/frontend/src/app/dashboard/admin/pools/page.tsx +++ b/frontend/src/app/dashboard/admin/pools/page.tsx @@ -16,7 +16,7 @@ export default function AdminPoolsPage() { const [form, setForm] = useState({ name: '', description: '', - strategy: 'weighted-resource' as 'least-apps' | 'round-robin' | 'weighted-resource', + strategy: 'weighted-resource' as 'least-apps' | 'round-robin' | 'weighted-round-robin' | 'least-loaded' | 'weighted-resource' | 'region-based', clusterIds: [] as string[], isDefault: false, priority: 100, @@ -143,6 +143,9 @@ export default function AdminPoolsPage() { onChange={(e) => setForm({ ...form, strategy: e.target.value as any })} > + + + @@ -315,6 +318,12 @@ export default function AdminPoolsPage() { {pool.strategy === 'weighted-resource' ? <> Weighted Resource + : pool.strategy === 'least-loaded' + ? <> Least Loaded + : pool.strategy === 'weighted-round-robin' + ? <> Weighted RR + : pool.strategy === 'region-based' + ? <> Region Based : pool.strategy === 'least-apps' ? <> Least Apps : <> Round Robin} diff --git a/frontend/src/app/dashboard/deploy/page.tsx b/frontend/src/app/dashboard/deploy/page.tsx index bffd105..5a2040d 100644 --- a/frontend/src/app/dashboard/deploy/page.tsx +++ b/frontend/src/app/dashboard/deploy/page.tsx @@ -2281,6 +2281,12 @@ export default function DeployPage() { {pool.strategy === 'weighted-resource' ? <> Weighted Resource + : pool.strategy === 'least-loaded' + ? <> Least Loaded + : pool.strategy === 'weighted-round-robin' + ? <> Weighted RR + : pool.strategy === 'region-based' + ? <> Region Based : pool.strategy === 'least-apps' ? <> Least Apps : <> Round Robin} diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.ts index ab7384a..35d5c0a 100644 --- a/frontend/src/types/index.ts +++ b/frontend/src/types/index.ts @@ -254,7 +254,7 @@ export interface ClusterPoolPublic { id: string; name: string; description?: string; - strategy: 'least-apps' | 'round-robin' | 'weighted-resource'; + strategy: 'least-apps' | 'round-robin' | 'weighted-round-robin' | 'least-loaded' | 'weighted-resource' | 'region-based'; clusterIds: string[]; isActive: boolean; isDefault: boolean; @@ -266,7 +266,7 @@ export interface ClusterPool { id: string; name: string; description?: string; - strategy: 'least-apps' | 'round-robin' | 'weighted-resource'; + strategy: 'least-apps' | 'round-robin' | 'weighted-round-robin' | 'least-loaded' | 'weighted-resource' | 'region-based'; clusterIds: string[]; isActive: boolean; isDefault: boolean; @@ -295,7 +295,7 @@ export interface ClusterAllocationLog { userId: string; poolId?: string; selectedClusterId?: string; - strategy: 'least-apps' | 'round-robin' | 'weighted-resource'; + strategy: 'least-apps' | 'round-robin' | 'weighted-round-robin' | 'least-loaded' | 'weighted-resource' | 'region-based'; estimatedRequest?: Record; candidateScores?: Record[]; rejectionReasons?: Record[];