abrban/cloud-host
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add automatic cluster pool allocation.

Browse Source 
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
keyhan
2026-05-18 23:08:06 +03:30
parent 97e4c865b6
commit 72a1519ea0
19 changed files with 983 additions and 151 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/applications/applications.service.ts
+24 -34
View File
@@ -23,49 +23,35 @@ export class ApplicationsService {
) {}
async create(userId: string, dto: CreateApplicationDto, userRole?: string): Promise<Application> {
// Only admin/technical users can manually select cluster or pool
// Regular users always get the default cluster assignment
if (userRole !== UserRole.ADMIN && userRole !== UserRole.TECHNICAL) {
// End users and technical staff cannot influence placement; only admins may manually assign.
const isAdmin = userRole === UserRole.ADMIN;
if (!isAdmin) {
if (dto.clusterId || dto.poolId) {
this.logger.warn(`Non-admin user ${userId} attempted manual cluster/pool selection — ignoring`);
this.logger.warn(`Non-admin user ${userId} attempted manual cluster/pool selection - ignoring`);
}
dto.clusterId = undefined;
dto.poolId = undefined;
}
// Cluster assignment: 3 modes
// 1. Manual: specific clusterId provided
// 2. Pool-based LB: poolId provided → pick from pool using pool's strategy
// 3. Default fallback: no clusterId/poolId → use default cluster
let clusterId = dto.clusterId;
let poolId = dto.poolId;
let allocationLogId: string | undefined;
if (!clusterId) {
if (poolId) {
// Mode 2: Pool-based load balancing
try {
const optimal = await this.clustersService.getOptimalClusterFromPool(poolId);
clusterId = optimal.id;
this.logger.log(`Pool-assigned cluster "${optimal.name}" to app "${dto.name}" (pool LB)`);
} catch (err: any) {
this.logger.warn(`Pool assignment failed for app "${dto.name}": ${err.message}`);
poolId = undefined; // Clear invalid pool
}
}
// Mode 3: Default fallback (no manual cluster, no pool, or pool failed)
if (!clusterId) {
try {
const defaultCluster = await this.clustersService.getDefault();
clusterId = defaultCluster.id;
this.logger.log(`Default-assigned cluster "${defaultCluster.name}" to app "${dto.name}"`);
} catch {
this.logger.warn('No cluster available — app will be created without cluster assignment');
}
}
if (isAdmin && clusterId) {
await this.clustersService.findOne(clusterId);
this.logger.log(`Manual cluster assignment for app "${dto.name}" -> cluster ${clusterId}`);
} else {
this.logger.log(`Manual cluster assignment for app "${dto.name}" → cluster ${clusterId}`);
const allocationDto = { ...dto };
if (poolId) {
allocationDto.poolId = poolId;
}
const allocation = await this.clustersService.selectClusterForApplication(allocationDto, userId);
clusterId = allocation.cluster.id;
poolId = allocation.pool?.id || (isAdmin ? poolId : undefined);
allocationLogId = allocation.allocationLogId;
if (!clusterId) {
throw new BadRequestException('No eligible cluster available for this application');
}
}
// Generate database credentials if a database is requested
@@ -109,7 +95,11 @@ export class ApplicationsService {
platformDomain,
),
});
return this.appsRepository.save(app);
const saved = await this.appsRepository.save(app);
if (allocationLogId) {
await this.clustersService.attachAllocationToApplication(allocationLogId, saved.id);
}
return saved;
}
async findAllByUser(userId: string): Promise<Application[]> {
backend/src/applications/dto/application.dto.ts
+2 -2
View File
@@ -182,12 +182,12 @@ export class CreateApplicationDto {
@IsNumber()
port?: number;
@ApiPropertyOptional({ description: 'Cluster ID to deploy to (auto-assigns default if empty)' })
@ApiPropertyOptional({ description: 'Admin-only manual cluster override. Ignored for non-admin users.' })
@IsOptional()
@IsString()
clusterId?: string;
@ApiPropertyOptional({ description: 'Cluster pool ID for load-balanced deployment' })
@ApiPropertyOptional({ description: 'Admin-only pool override. Ignored for non-admin users.' })
@IsOptional()
@IsString()
poolId?: string;
backend/src/applications/entities/application.entity.ts
+2 -2
View File
@@ -136,10 +136,10 @@ export class Application {
@Column()
userId: string;
@Column({ nullable: true })
@Column({ type: 'uuid', nullable: true })
clusterId: string;
@Column({ nullable: true })
@Column({ type: 'uuid', nullable: true })
poolId: string; // Cluster pool used for load-balanced assignment
@OneToMany(() => Deployment, (deployment: Deployment) => deployment.application)
backend/src/clusters/clusters.controller.ts
+61 -51
View File
@@ -24,15 +24,17 @@ import { UserRole } from '../common/enums';
export class ClustersController {
constructor(private readonly clustersService: ClustersService) {}
// ─── Public endpoints (any authenticated user) ────────────────────
// ─── Admin-safe lookup endpoints (no end-user exposure) ───────────
@Get('public')
@Roles(UserRole.ADMIN)
@ApiOperation({ summary: 'List available clusters (public info, no kubeconfig)' })
async findAllPublic() {
return this.clustersService.findAllPublic();
}
@Get('pools/public')
@Roles(UserRole.ADMIN)
@ApiOperation({ summary: 'List active cluster pools with resolved cluster names' })
async findAllPoolsPublic() {
return this.clustersService.findAllPoolsPublic();
@@ -41,90 +43,98 @@ export class ClustersController {
// ─── Cluster admin endpoints ──────────────────────────────────────
@Post()
@Roles(UserRole.ADMIN, UserRole.TECHNICAL)
@ApiOperation({ summary: 'Register a new Kubernetes cluster (Admin/Technical)' })
@Roles(UserRole.ADMIN)
@ApiOperation({ summary: 'Register a new Kubernetes cluster (Admin)' })
async create(@Body() dto: CreateClusterDto) {
return this.clustersService.create(dto);
}
@Get()
@Roles(UserRole.ADMIN, UserRole.TECHNICAL)
@ApiOperation({ summary: 'List all clusters (Admin/Technical)' })
@Roles(UserRole.ADMIN)
@ApiOperation({ summary: 'List all clusters (Admin)' })
async findAll() {
return this.clustersService.findAll();
}
@Get(':id/resources')
@Roles(UserRole.ADMIN, UserRole.TECHNICAL)
@ApiOperation({ summary: 'Get cluster resource usage — nodes, CPU, memory, pods (Admin/Technical)' })
async getClusterResources(@Param('id') id: string) {
return this.clustersService.getClusterResources(id);
}
@Get(':id')
@Roles(UserRole.ADMIN, UserRole.TECHNICAL)
@ApiOperation({ summary: 'Get cluster details (Admin/Technical)' })
async findOne(@Param('id') id: string) {
return this.clustersService.findOne(id);
}
@Patch(':id')
@Roles(UserRole.ADMIN, UserRole.TECHNICAL)
@ApiOperation({ summary: 'Update cluster configuration (Admin/Technical)' })
async update(@Param('id') id: string, @Body() dto: UpdateClusterDto) {
return this.clustersService.update(id, dto);
}
@Post(':id/test')
@Roles(UserRole.ADMIN, UserRole.TECHNICAL)
@ApiOperation({ summary: 'Test connectivity to a registered cluster (Admin/Technical)' })
async testConnection(@Param('id') id: string) {
return this.clustersService.testClusterById(id);
}
@Delete(':id')
@Roles(UserRole.ADMIN, UserRole.TECHNICAL)
@ApiOperation({ summary: 'Remove a cluster (Admin/Technical)' })
async delete(@Param('id') id: string) {
await this.clustersService.delete(id);
return { message: 'Cluster deleted' };
}
// ─── Cluster Pool admin endpoints ─────────────────────────────────
@Post('pools')
@Roles(UserRole.ADMIN, UserRole.TECHNICAL)
@ApiOperation({ summary: 'Create a cluster pool for load balancing (Admin/Technical)' })
@Roles(UserRole.ADMIN)
@ApiOperation({ summary: 'Create a cluster pool for load balancing (Admin)' })
async createPool(@Body() dto: CreateClusterPoolDto) {
return this.clustersService.createPool(dto);
}
@Get('pools')
@Roles(UserRole.ADMIN, UserRole.TECHNICAL)
@ApiOperation({ summary: 'List all cluster pools (Admin/Technical)' })
@Roles(UserRole.ADMIN)
@ApiOperation({ summary: 'List all cluster pools (Admin)' })
async findAllPools() {
return this.clustersService.findAllPools();
}
@Get('pools/:id')
@Roles(UserRole.ADMIN, UserRole.TECHNICAL)
@ApiOperation({ summary: 'Get cluster pool details (Admin/Technical)' })
@Roles(UserRole.ADMIN)
@ApiOperation({ summary: 'Get cluster pool details (Admin)' })
async findOnePool(@Param('id') id: string) {
return this.clustersService.findOnePool(id);
}
@Patch('pools/:id')
@Roles(UserRole.ADMIN, UserRole.TECHNICAL)
@ApiOperation({ summary: 'Update cluster pool (Admin/Technical)' })
@Roles(UserRole.ADMIN)
@ApiOperation({ summary: 'Update cluster pool (Admin)' })
async updatePool(@Param('id') id: string, @Body() dto: UpdateClusterPoolDto) {
return this.clustersService.updatePool(id, dto);
}
@Delete('pools/:id')
@Roles(UserRole.ADMIN, UserRole.TECHNICAL)
@ApiOperation({ summary: 'Delete a cluster pool (Admin/Technical)' })
@Roles(UserRole.ADMIN)
@ApiOperation({ summary: 'Delete a cluster pool (Admin)' })
async deletePool(@Param('id') id: string) {
await this.clustersService.deletePool(id);
return { message: 'Cluster pool deleted' };
}
@Get('allocation-logs')
@Roles(UserRole.ADMIN)
@ApiOperation({ summary: 'List recent cluster allocation decisions (Admin)' })
async listAllocationLogs() {
return this.clustersService.listAllocationLogs();
}
@Get(':id/resources')
@Roles(UserRole.ADMIN)
@ApiOperation({ summary: 'Get cluster resource usage - nodes, CPU, memory, pods (Admin)' })
async getClusterResources(@Param('id') id: string) {
return this.clustersService.getClusterResources(id);
}
@Get(':id')
@Roles(UserRole.ADMIN)
@ApiOperation({ summary: 'Get cluster details (Admin)' })
async findOne(@Param('id') id: string) {
const { kubeconfig, ...cluster } = await this.clustersService.findOne(id);
return cluster;
}
@Patch(':id')
@Roles(UserRole.ADMIN)
@ApiOperation({ summary: 'Update cluster configuration (Admin)' })
async update(@Param('id') id: string, @Body() dto: UpdateClusterDto) {
return this.clustersService.update(id, dto);
}
@Post(':id/test')
@Roles(UserRole.ADMIN)
@ApiOperation({ summary: 'Test connectivity to a registered cluster (Admin)' })
async testConnection(@Param('id') id: string) {
return this.clustersService.testClusterById(id);
}
@Delete(':id')
@Roles(UserRole.ADMIN)
@ApiOperation({ summary: 'Remove a cluster (Admin)' })
async delete(@Param('id') id: string) {
await this.clustersService.delete(id);
return { message: 'Cluster deleted' };
}
}
backend/src/clusters/clusters.module.ts
+3 -1
View File
@@ -4,11 +4,13 @@ import { ClustersService } from './clusters.service';
import { ClustersController } from './clusters.controller';
import { Cluster } from './entities/cluster.entity';
import { ClusterPool } from './entities/cluster-pool.entity';
import { ClusterHealth } from './entities/cluster-health.entity';
import { ClusterAllocationLog } from './entities/cluster-allocation-log.entity';
import { KubernetesModule } from '../kubernetes/kubernetes.module';
@Module({
imports: [
TypeOrmModule.forFeature([Cluster, ClusterPool]),
TypeOrmModule.forFeature([Cluster, ClusterPool, ClusterHealth, ClusterAllocationLog]),
forwardRef(() => KubernetesModule),
],
controllers: [ClustersController],
backend/src/clusters/clusters.service.ts
+390 -13
View File
@@ -3,12 +3,16 @@ import { InjectRepository } from '@nestjs/typeorm';
import { ConfigService } from '@nestjs/config';
import { Repository, DataSource, In } from 'typeorm';
import * as k8s from '@kubernetes/client-node';
import { Cluster } from './entities/cluster.entity';
import * as crypto from 'crypto';
import { Cluster, ClusterHealthStatus } from './entities/cluster.entity';
import { ClusterPool } from './entities/cluster-pool.entity';
import { ClusterHealth } from './entities/cluster-health.entity';
import { ClusterAllocationLog } from './entities/cluster-allocation-log.entity';
import { CreateClusterDto, UpdateClusterDto } from './dto/cluster.dto';
import { CreateClusterPoolDto, UpdateClusterPoolDto } from './dto/cluster-pool.dto';
import { ClusterStatus } from '../common/enums';
import { ElasticsearchService } from '../kubernetes/elasticsearch.service';
import { CreateApplicationDto } from '../applications/dto/application.dto';
@Injectable()
export class ClustersService {
@@ -21,6 +25,10 @@ export class ClustersService {
private clustersRepository: Repository<Cluster>,
@InjectRepository(ClusterPool)
private poolsRepository: Repository<ClusterPool>,
@InjectRepository(ClusterHealth)
private healthRepository: Repository<ClusterHealth>,
@InjectRepository(ClusterAllocationLog)
private allocationLogsRepository: Repository<ClusterAllocationLog>,
private dataSource: DataSource,
private configService: ConfigService,
@Inject(forwardRef(() => ElasticsearchService))
@@ -34,7 +42,7 @@ export class ClustersService {
async testConnection(kubeconfig: string): Promise<{ connected: boolean; version?: string; error?: string }> {
try {
const kc = new k8s.KubeConfig();
kc.loadFromString(kubeconfig);
kc.loadFromString(this.decryptKubeconfig(kubeconfig));
const versionApi = kc.makeApiClient(k8s.VersionApi);
const result = await versionApi.getCode();
@@ -74,13 +82,18 @@ export class ClustersService {
const cluster = this.clustersRepository.create({
...dto,
kubeconfig: this.encryptKubeconfig(dto.kubeconfig),
status: ClusterStatus.ACTIVE, // Connection verified — mark active
healthStatus: 'healthy',
lastHealthCheckedAt: new Date(),
healthMessage: connectionTest.version ? `Kubernetes ${connectionTest.version}` : 'Connection verified',
});
const saved = await this.clustersRepository.save(cluster);
const usableCluster = this.withDecryptedKubeconfig(saved);
this.logger.log(`Cluster "${saved.name}" registered (active) — K8s ${connectionTest.version}`);
// Bootstrap the cluster with build infrastructure (namespace, registry, SA, etc.)
this.bootstrapCluster(saved.kubeconfig).catch((err) => {
this.bootstrapCluster(usableCluster.kubeconfig).catch((err) => {
this.logger.error(`Failed to bootstrap cluster "${saved.name}": ${err.message}`);
});
@@ -89,12 +102,33 @@ export class ClustersService {
this.logger.error(`Failed to deploy central logging on "${saved.name}": ${err.message}`);
});
return saved;
this.recordHealthSnapshot(saved, {
status: 'healthy',
message: connectionTest.version ? `Kubernetes ${connectionTest.version}` : 'Connection verified',
}).catch((err) => this.logger.warn(`Failed to record cluster health: ${err.message}`));
return usableCluster;
}
async findAll(): Promise<Cluster[]> {
return this.clustersRepository.find({
select: ['id', 'name', 'description', 'status', 'apiServer', 'region', 'provider', 'isDefault', 'createdAt'],
select: [
'id',
'name',
'description',
'status',
'apiServer',
'region',
'provider',
'isDefault',
'weight',
'tags',
'healthStatus',
'lastHealthCheckedAt',
'healthMessage',
'availableResources',
'createdAt',
],
order: { createdAt: 'DESC' },
});
}
@@ -104,7 +138,7 @@ export class ClustersService {
if (!cluster) {
throw new NotFoundException('Cluster not found');
}
return cluster;
return this.withDecryptedKubeconfig(cluster);
}
async getDefault(): Promise<Cluster> {
@@ -126,7 +160,7 @@ export class ClustersService {
if (!cluster) {
throw new NotFoundException('No active cluster available');
}
return cluster;
return this.withDecryptedKubeconfig(cluster);
}
async update(id: string, dto: UpdateClusterDto): Promise<Cluster> {
@@ -141,10 +175,11 @@ export class ClustersService {
);
}
dto.status = ClusterStatus.ACTIVE;
dto.kubeconfig = this.encryptKubeconfig(dto.kubeconfig);
this.logger.log(`Cluster "${cluster.name}" kubeconfig updated — connection verified (K8s ${connectionTest.version})`);
// Re-bootstrap build infrastructure on the new/updated cluster
this.bootstrapCluster(dto.kubeconfig).catch((err: any) => {
this.bootstrapCluster(this.decryptKubeconfig(dto.kubeconfig)).catch((err: any) => {
this.logger.error(`Failed to bootstrap cluster "${cluster.name}": ${err.message}`);
});
}
@@ -159,7 +194,9 @@ export class ClustersService {
}
}
Object.assign(cluster, dto);
return this.clustersRepository.save(cluster);
cluster.kubeconfig = this.encryptKubeconfig(cluster.kubeconfig);
const saved = await this.clustersRepository.save(cluster);
return this.withDecryptedKubeconfig(saved);
}
/**
@@ -171,7 +208,17 @@ export class ClustersService {
const result = await this.testConnection(cluster.kubeconfig);
cluster.status = result.connected ? ClusterStatus.ACTIVE : ClusterStatus.INACTIVE;
cluster.healthStatus = result.connected ? 'healthy' : 'unhealthy';
cluster.lastHealthCheckedAt = new Date();
cluster.healthMessage = result.connected
? (result.version ? `Kubernetes ${result.version}` : 'Connection verified')
: result.error || 'Connection failed';
cluster.kubeconfig = this.encryptKubeconfig(cluster.kubeconfig);
await this.clustersRepository.save(cluster);
await this.recordHealthSnapshot(cluster, {
status: cluster.healthStatus,
message: cluster.healthMessage,
});
this.logger.log(`Cluster "${cluster.name}" test: ${result.connected ? 'ACTIVE' : 'INACTIVE'}`);
return result;
@@ -188,6 +235,106 @@ export class ClustersService {
});
}
async selectClusterForApplication(
dto: CreateApplicationDto,
userId: string,
): Promise<{ cluster: Cluster; pool?: ClusterPool; allocationLogId: string }> {
const estimatedRequest = this.estimateApplicationRequest(dto);
let pool = dto.poolId
? await this.poolsRepository.findOne({ where: { id: dto.poolId, isActive: true } })
: null;
if (dto.poolId && !pool) {
throw new BadRequestException('Selected cluster pool is not active or does not exist');
}
if (!pool) {
pool = await this.poolsRepository.findOne({
where: { isActive: true, isDefault: true },
order: { priority: 'ASC', createdAt: 'ASC' },
}) || await this.poolsRepository.findOne({
where: { isActive: true },
order: { priority: 'ASC', createdAt: 'ASC' },
});
}
const candidateQuery = this.clustersRepository
.createQueryBuilder('cluster')
.where('cluster.status = :status', { status: ClusterStatus.ACTIVE });
if (pool?.clusterIds?.length) {
candidateQuery.andWhere('cluster.id IN (:...clusterIds)', { clusterIds: pool.clusterIds });
}
const candidates = await candidateQuery.getMany();
const appCounts = await this.getAppCounts(candidates.map((cluster) => cluster.id));
const candidateScores: Record<string, any>[] = [];
const rejectionReasons: Record<string, any>[] = [];
for (const cluster of candidates) {
const rejection = this.getClusterRejectionReason(cluster, estimatedRequest);
if (rejection) {
rejectionReasons.push({ clusterId: cluster.id, clusterName: cluster.name, reason: rejection });
continue;
}
const score = this.scoreCluster(cluster, estimatedRequest, appCounts.get(cluster.id) || 0, pool?.strategy || 'weighted-resource');
candidateScores.push({
clusterId: cluster.id,
clusterName: cluster.name,
score,
weight: cluster.weight || 1,
appCount: appCounts.get(cluster.id) || 0,
healthStatus: cluster.healthStatus,
availableResources: cluster.availableResources || null,
});
}
if (candidateScores.length === 0) {
const log = await this.allocationLogsRepository.save(this.allocationLogsRepository.create({
userId,
poolId: pool?.id,
selectedClusterId: null,
strategy: pool?.strategy || 'weighted-resource',
estimatedRequest,
candidateScores,
rejectionReasons,
status: 'failed',
message: 'No active healthy cluster has enough estimated capacity',
}));
throw new BadRequestException(
`No active healthy cluster has enough capacity for this application (allocation log: ${log.id})`,
);
}
candidateScores.sort((a, b) => b.score - a.score);
const selected = await this.findOne(candidateScores[0].clusterId);
const log = await this.allocationLogsRepository.save(this.allocationLogsRepository.create({
userId,
poolId: pool?.id,
selectedClusterId: selected.id,
strategy: pool?.strategy || 'weighted-resource',
estimatedRequest,
candidateScores,
rejectionReasons,
status: 'success',
message: `Selected ${selected.name}`,
}));
this.logger.log(`Allocator selected cluster "${selected.name}" for user ${userId} (score ${candidateScores[0].score.toFixed(2)})`);
return { cluster: selected, pool: pool || undefined, allocationLogId: log.id };
}
async attachAllocationToApplication(allocationLogId: string, applicationId: string): Promise<void> {
await this.allocationLogsRepository.update({ id: allocationLogId }, { applicationId });
}
async listAllocationLogs(limit = 100): Promise<ClusterAllocationLog[]> {
return this.allocationLogsRepository.find({
relations: ['selectedCluster', 'pool'],
order: { createdAt: 'DESC' },
take: limit,
});
}
/**
* Get the optimal cluster using load-balancing strategy.
* Strategy: 'least-apps' — picks the active cluster with fewest deployed applications.
@@ -308,14 +455,21 @@ export class ClustersService {
throw new BadRequestException(`Clusters not found: ${missingIds.join(', ')}`);
}
const pool = this.poolsRepository.create(dto);
if (dto.isDefault === true) {
await this.poolsRepository.update({ isDefault: true }, { isDefault: false });
}
const pool = this.poolsRepository.create({
...dto,
strategy: dto.strategy || 'weighted-resource',
});
const saved = await this.poolsRepository.save(pool);
this.logger.log(`Cluster pool "${saved.name}" created with ${dto.clusterIds.length} clusters (strategy: ${dto.strategy})`);
return saved;
}
async findAllPools(): Promise<ClusterPool[]> {
return this.poolsRepository.find({ order: { createdAt: 'DESC' } });
return this.poolsRepository.find({ order: { isDefault: 'DESC', priority: 'ASC', createdAt: 'DESC' } });
}
/**
@@ -324,7 +478,7 @@ export class ClustersService {
async findAllPoolsPublic(): Promise<(ClusterPool & { clusters: Pick<Cluster, 'id' | 'name' | 'region' | 'provider' | 'status'>[] })[]> {
const pools = await this.poolsRepository.find({
where: { isActive: true },
order: { createdAt: 'DESC' },
order: { isDefault: 'DESC', priority: 'ASC', createdAt: 'DESC' },
});
const allClusterIds = [...new Set(pools.flatMap((p) => p.clusterIds))];
@@ -367,6 +521,10 @@ export class ClustersService {
}
}
if (dto.isDefault === true) {
await this.poolsRepository.update({ isDefault: true }, { isDefault: false });
}
Object.assign(pool, dto);
return this.poolsRepository.save(pool);
}
@@ -443,6 +601,7 @@ export class ClustersService {
totalMemoryAllocatable: string;
podCount: number;
nodeCount: number;
readyNodeCount?: number;
appCount: number;
}> {
const cluster = await this.findOne(id);
@@ -496,7 +655,8 @@ export class ClustersService {
);
const appCount = parseInt(appCountResult[0]?.count || '0', 10);
return {
const readyNodeCount = nodes.filter((node) => node.status === 'Ready').length;
const resources = {
nodes,
totalCpuCapacity: `${totalCpuCap}m`,
totalMemoryCapacity: `${totalMemCap.toFixed(0)}Mi`,
@@ -504,10 +664,33 @@ export class ClustersService {
totalMemoryAllocatable: `${totalMemAlloc.toFixed(0)}Mi`,
podCount,
nodeCount: nodes.length,
readyNodeCount,
appCount,
};
const healthStatus: ClusterHealthStatus = readyNodeCount === nodes.length && nodes.length > 0
? 'healthy'
: readyNodeCount > 0
? 'degraded'
: 'unhealthy';
const healthMessage = `${readyNodeCount}/${nodes.length} nodes ready`;
await this.clustersRepository.update(id, {
healthStatus,
healthMessage,
lastHealthCheckedAt: new Date(),
availableResources: resources as any,
});
await this.recordHealthSnapshot(cluster, { status: healthStatus, message: healthMessage, resources });
return resources;
} catch (err: any) {
this.logger.error(`Failed to get cluster resources for "${cluster.name}": ${err.message}`);
await this.clustersRepository.update(id, {
healthStatus: 'unhealthy',
healthMessage: err.message,
lastHealthCheckedAt: new Date(),
});
await this.recordHealthSnapshot(cluster, { status: 'unhealthy', message: err.message });
throw new BadRequestException(`Cannot fetch resources: ${err.message}`);
}
}
@@ -713,4 +896,198 @@ export class ClustersService {
if (memory.endsWith('Ti')) return parseFloat(memory) * 1024 * 1024;
return parseFloat(memory) / (1024 * 1024); // bytes
}
private estimateApplicationRequest(dto: CreateApplicationDto): Record<string, any> {
const replicas = Math.max(dto.replicas || 1, 1);
let cpuMillicores = this.parseCpuToMillicores(dto.cpuRequest || '100m') * replicas;
let memoryMi = this.parseMemoryToMi(dto.memoryRequest || '128Mi') * replicas;
let storageMi = this.parseStorageToMi(dto.dbStorageSize || '1Gi') + this.parseStorageToMi(dto.appStorageSize || '2Gi');
if (dto.databaseType && dto.databaseType !== 'none') {
cpuMillicores += 100;
memoryMi += 256;
}
for (const service of ['redis', 'rabbitmq'] as const) {
const enabled = service === 'redis' ? dto.enableRedis : dto.enableRabbitmq;
const custom = dto.optionalServiceResources?.[service];
if (!enabled && !custom) continue;
cpuMillicores += this.parseCpuToMillicores(custom?.cpuRequest || custom?.cpuLimit || '100m');
memoryMi += this.parseMemoryToMi(custom?.memoryRequest || custom?.memoryLimit || '128Mi');
storageMi += (custom?.storageGi || 1) * 1024;
}
return {
cpuMillicores,
memoryMi,
storageMi,
replicas,
podEstimate: replicas + (dto.databaseType && dto.databaseType !== 'none' ? 1 : 0) + (dto.enableRedis ? 1 : 0) + (dto.enableRabbitmq ? 1 : 0),
};
}
private getClusterRejectionReason(cluster: Cluster, estimatedRequest: Record<string, any>): string | null {
if (cluster.status !== ClusterStatus.ACTIVE) {
return `status=${cluster.status}`;
}
if (cluster.healthStatus && !['healthy', 'unknown'].includes(cluster.healthStatus)) {
return `health=${cluster.healthStatus}`;
}
const resources = cluster.availableResources || {};
const cpuFree = this.parseCpuToMillicores(resources.totalCpuAllocatable || resources.cpuAllocatable || '0');
const memoryFree = this.parseMemoryToMi(resources.totalMemoryAllocatable || resources.memoryAllocatable || '0');
const podCount = Number(resources.podCount || 0);
const nodeCount = Number(resources.nodeCount || 0);
const podCapacity = Number(resources.podCapacity || nodeCount * 110 || 0);
if (cpuFree > 0 && cpuFree < estimatedRequest.cpuMillicores) {
return `insufficient cpu (${cpuFree}m < ${estimatedRequest.cpuMillicores}m)`;
}
if (memoryFree > 0 && memoryFree < estimatedRequest.memoryMi) {
return `insufficient memory (${memoryFree}Mi < ${estimatedRequest.memoryMi}Mi)`;
}
if (podCapacity > 0 && podCount + estimatedRequest.podEstimate > podCapacity) {
return `pod pressure (${podCount}/${podCapacity})`;
}
return null;
}
private scoreCluster(
cluster: Cluster,
estimatedRequest: Record<string, any>,
appCount: number,
strategy: string,
): number {
if (strategy === 'round-robin') {
const idx = this.poolRoundRobinIndices.get('allocator') || 0;
this.poolRoundRobinIndices.set('allocator', idx + 1);
return 1000 - idx;
}
const resources = cluster.availableResources || {};
const cpuFree = this.parseCpuToMillicores(resources.totalCpuAllocatable || resources.cpuAllocatable || '0');
const memoryFree = this.parseMemoryToMi(resources.totalMemoryAllocatable || resources.memoryAllocatable || '0');
const storageFree = this.parseStorageToMi(resources.storageAllocatable || resources.storageFree || '0');
const podCount = Number(resources.podCount || 0);
const nodeCount = Number(resources.nodeCount || 0);
const podCapacity = Number(resources.podCapacity || nodeCount * 110 || 0);
const capacityScore =
this.ratioScore(cpuFree, estimatedRequest.cpuMillicores) * 0.35 +
this.ratioScore(memoryFree, estimatedRequest.memoryMi) * 0.35 +
this.ratioScore(storageFree, estimatedRequest.storageMi) * 0.15 +
(podCapacity > 0 ? Math.max(0, 1 - podCount / podCapacity) : 0.7) * 0.15;
const appPenalty = Math.min(appCount, 100) * 0.75;
if (strategy === 'least-apps') {
return 1000 - appPenalty + capacityScore * 100;
}
return (cluster.weight || 1) * 100 + capacityScore * 100 - appPenalty;
}
private ratioScore(available: number, required: number): number {
if (!available) return 0.7;
if (!required) return 1;
return Math.min(available / required, 10) / 10;
}
private async getAppCounts(clusterIds: string[]): Promise<Map<string, number>> {
if (clusterIds.length === 0) {
return new Map();
}
const rows: { clusterId: string; count: string }[] = await this.dataSource.query(`
SELECT "clusterId", COUNT(*) as count
FROM applications
WHERE "clusterId" = ANY($1)
GROUP BY "clusterId"
`, [clusterIds]);
return new Map(rows.map((row) => [row.clusterId, parseInt(row.count, 10)]));
}
private parseStorageToMi(storage: string): number {
if (!storage) return 0;
if (storage.endsWith('Ki')) return parseFloat(storage) / 1024;
if (storage.endsWith('Mi')) return parseFloat(storage);
if (storage.endsWith('Gi')) return parseFloat(storage) * 1024;
if (storage.endsWith('Ti')) return parseFloat(storage) * 1024 * 1024;
return parseFloat(storage) || 0;
}
private async recordHealthSnapshot(
cluster: Cluster,
snapshot: {
status: ClusterHealthStatus;
message?: string;
resources?: Record<string, any>;
},
): Promise<void> {
const resources = snapshot.resources || cluster.availableResources || {};
await this.healthRepository.save(this.healthRepository.create({
clusterId: cluster.id,
status: snapshot.status,
readyNodes: resources.readyNodeCount || resources.nodeCount || 0,
nodeCount: resources.nodeCount || 0,
cpuAllocatable: resources.totalCpuAllocatable || resources.cpuAllocatable || null,
memoryAllocatable: resources.totalMemoryAllocatable || resources.memoryAllocatable || null,
podCount: resources.podCount || 0,
appCount: resources.appCount || 0,
message: snapshot.message,
resources,
}));
}
private encryptKubeconfig(kubeconfig: string): string {
if (!kubeconfig || kubeconfig.startsWith('enc:v1:')) {
return kubeconfig;
}
const key = this.getKubeconfigEncryptionKey();
if (!key) {
return kubeconfig;
}
const iv = crypto.randomBytes(12);
const cipher = crypto.createCipheriv('aes-256-gcm', key, iv);
const encrypted = Buffer.concat([cipher.update(kubeconfig, 'utf8'), cipher.final()]);
const tag = cipher.getAuthTag();
return `enc:v1:${iv.toString('base64')}:${tag.toString('base64')}:${encrypted.toString('base64')}`;
}
private decryptKubeconfig(kubeconfig: string): string {
if (!kubeconfig?.startsWith('enc:v1:')) {
return kubeconfig;
}
const key = this.getKubeconfigEncryptionKey();
if (!key) {
throw new BadRequestException('Kubeconfig is encrypted but CLUSTER_KUBECONFIG_KEY is not configured');
}
const [, , ivRaw, tagRaw, encryptedRaw] = kubeconfig.split(':');
const decipher = crypto.createDecipheriv('aes-256-gcm', key, Buffer.from(ivRaw, 'base64'));
decipher.setAuthTag(Buffer.from(tagRaw, 'base64'));
return Buffer.concat([
decipher.update(Buffer.from(encryptedRaw, 'base64')),
decipher.final(),
]).toString('utf8');
}
private withDecryptedKubeconfig(cluster: Cluster): Cluster {
return {
...cluster,
kubeconfig: this.decryptKubeconfig(cluster.kubeconfig),
};
}
private getKubeconfigEncryptionKey(): Buffer | null {
const secret = this.configService.get<string>('CLUSTER_KUBECONFIG_KEY') || this.configService.get<string>('cluster.kubeconfigKey');
if (!secret) {
return null;
}
if (secret.length === 64 && /^[0-9a-f]+$/i.test(secret)) {
return Buffer.from(secret, 'hex');
}
return crypto.createHash('sha256').update(secret).digest();
}
}
backend/src/clusters/dto/cluster-pool.dto.ts
+29 -7
View File
@@ -1,4 +1,4 @@
import { IsString, IsOptional, IsBoolean, IsArray, IsIn } from 'class-validator';
import { IsString, IsOptional, IsBoolean, IsArray, IsIn, IsNumber, Min } from 'class-validator';
import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
export class CreateClusterPoolDto {
@@ -11,14 +11,25 @@ export class CreateClusterPoolDto {
@IsString()
description?: string;
@ApiProperty({ example: 'least-apps', enum: ['least-apps', 'round-robin'] })
@IsIn(['least-apps', 'round-robin'])
strategy: 'least-apps' | 'round-robin';
@ApiProperty({ example: 'weighted-resource', enum: ['least-apps', 'round-robin', 'weighted-resource'] })
@IsIn(['least-apps', 'round-robin', 'weighted-resource'])
strategy: 'least-apps' | 'round-robin' | 'weighted-resource';
@ApiProperty({ example: ['uuid-1', 'uuid-2'], description: 'Array of cluster IDs in this pool' })
@IsArray()
@IsString({ each: true })
clusterIds: string[];
@ApiPropertyOptional({ example: true })
@IsOptional()
@IsBoolean()
isDefault?: boolean;
@ApiPropertyOptional({ example: 100 })
@IsOptional()
@IsNumber()
@Min(1)
priority?: number;
}
export class UpdateClusterPoolDto {
@@ -32,10 +43,10 @@ export class UpdateClusterPoolDto {
@IsString()
description?: string;
@ApiPropertyOptional({ enum: ['least-apps', 'round-robin'] })
@ApiPropertyOptional({ enum: ['least-apps', 'round-robin', 'weighted-resource'] })
@IsOptional()
@IsIn(['least-apps', 'round-robin'])
strategy?: 'least-apps' | 'round-robin';
@IsIn(['least-apps', 'round-robin', 'weighted-resource'])
strategy?: 'least-apps' | 'round-robin' | 'weighted-resource';
@ApiPropertyOptional({ description: 'Array of cluster IDs in this pool' })
@IsOptional()
@@ -47,4 +58,15 @@ export class UpdateClusterPoolDto {
@IsOptional()
@IsBoolean()
isActive?: boolean;
@ApiPropertyOptional()
@IsOptional()
@IsBoolean()
isDefault?: boolean;
@ApiPropertyOptional()
@IsOptional()
@IsNumber()
@Min(1)
priority?: number;
}
backend/src/clusters/dto/cluster.dto.ts
+35 -1
View File
@@ -1,4 +1,4 @@
import { IsString, IsOptional, IsBoolean, IsNumber, IsEnum } from 'class-validator';
import { IsString, IsOptional, IsBoolean, IsNumber, IsEnum, IsArray, Min } from 'class-validator';
import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
import { ClusterStatus } from '../../common/enums';
@@ -25,6 +25,18 @@ export class CreateClusterDto {
@IsString()
region?: string;
@ApiPropertyOptional({ example: 10, description: 'Relative placement weight; higher means more preferred' })
@IsOptional()
@IsNumber()
@Min(1)
weight?: number;
@ApiPropertyOptional({ example: ['ssd', 'iran', 'production'] })
@IsOptional()
@IsArray()
@IsString({ each: true })
tags?: string[];
@ApiPropertyOptional({ example: 'aws' })
@IsOptional()
@IsString()
@@ -62,6 +74,16 @@ export class UpdateClusterDto {
@IsString()
description?: string;
@ApiPropertyOptional()
@IsOptional()
@IsString()
region?: string;
@ApiPropertyOptional()
@IsOptional()
@IsString()
provider?: string;
@ApiPropertyOptional()
@IsOptional()
@IsEnum(ClusterStatus)
@@ -77,6 +99,18 @@ export class UpdateClusterDto {
@IsBoolean()
isDefault?: boolean;
@ApiPropertyOptional()
@IsOptional()
@IsNumber()
@Min(1)
weight?: number;
@ApiPropertyOptional()
@IsOptional()
@IsArray()
@IsString({ each: true })
tags?: string[];
@ApiPropertyOptional()
@IsOptional()
@IsString()
backend/src/clusters/entities/cluster-allocation-log.entity.ts
+64
View File
@@ -0,0 +1,64 @@
import {
Entity,
PrimaryGeneratedColumn,
Column,
ManyToOne,
JoinColumn,
CreateDateColumn,
} from 'typeorm';
import { Cluster } from './cluster.entity';
import { ClusterPool, PoolStrategy } from './cluster-pool.entity';
import { Application } from '../../applications/entities/application.entity';
export type ClusterAllocationStatus = 'success' | 'failed';
@Entity('cluster_allocation_logs')
export class ClusterAllocationLog {
@PrimaryGeneratedColumn('uuid')
id: string;
@Column({ nullable: true })
applicationId?: string | null;
@ManyToOne(() => Application, { nullable: true, onDelete: 'SET NULL' })
@JoinColumn({ name: 'applicationId' })
application: Application;
@Column()
userId: string;
@Column({ nullable: true })
poolId?: string | null;
@ManyToOne(() => ClusterPool, { nullable: true, onDelete: 'SET NULL' })
@JoinColumn({ name: 'poolId' })
pool: ClusterPool;
@Column({ nullable: true })
selectedClusterId?: string | null;
@ManyToOne(() => Cluster, { nullable: true, onDelete: 'SET NULL' })
@JoinColumn({ name: 'selectedClusterId' })
selectedCluster: Cluster;
@Column({ default: 'weighted-resource' })
strategy: PoolStrategy;
@Column({ type: 'jsonb', nullable: true })
estimatedRequest: Record<string, any>;
@Column({ type: 'jsonb', nullable: true })
candidateScores: Record<string, any>[];
@Column({ type: 'jsonb', nullable: true })
rejectionReasons: Record<string, any>[];
@Column({ default: 'success' })
status: ClusterAllocationStatus;
@Column({ nullable: true })
message: string;
@CreateDateColumn()
createdAt: Date;
}
backend/src/clusters/entities/cluster-health.entity.ts
+52
View File
@@ -0,0 +1,52 @@
import {
Entity,
PrimaryGeneratedColumn,
Column,
ManyToOne,
JoinColumn,
CreateDateColumn,
} from 'typeorm';
import { Cluster, ClusterHealthStatus } from './cluster.entity';
@Entity('cluster_health')
export class ClusterHealth {
@PrimaryGeneratedColumn('uuid')
id: string;
@Column()
clusterId: string;
@ManyToOne(() => Cluster, { onDelete: 'CASCADE' })
@JoinColumn({ name: 'clusterId' })
cluster: Cluster;
@Column({ default: 'unknown' })
status: ClusterHealthStatus;
@Column({ default: 0 })
readyNodes: number;
@Column({ default: 0 })
nodeCount: number;
@Column({ nullable: true })
cpuAllocatable: string;
@Column({ nullable: true })
memoryAllocatable: string;
@Column({ default: 0 })
podCount: number;
@Column({ default: 0 })
appCount: number;
@Column({ nullable: true })
message: string;
@Column({ type: 'jsonb', nullable: true })
resources: Record<string, any>;
@CreateDateColumn()
checkedAt: Date;
}
backend/src/clusters/entities/cluster-pool.entity.ts
+7 -1
View File
@@ -6,7 +6,7 @@ import {
UpdateDateColumn,
} from 'typeorm';
export type PoolStrategy = 'least-apps' | 'round-robin';
export type PoolStrategy = 'least-apps' | 'round-robin' | 'weighted-resource';
@Entity('cluster_pools')
export class ClusterPool {
@@ -32,6 +32,12 @@ export class ClusterPool {
@Column({ default: true })
isActive: boolean;
@Column({ default: false })
isDefault: boolean;
@Column({ default: 100 })
priority: number;
@CreateDateColumn()
createdAt: Date;
backend/src/clusters/entities/cluster.entity.ts
+20
View File
@@ -7,6 +7,8 @@ import {
} from 'typeorm';
import { ClusterStatus } from '../../common/enums';
export type ClusterHealthStatus = 'unknown' | 'healthy' | 'degraded' | 'unhealthy';
@Entity('clusters')
export class Cluster {
@PrimaryGeneratedColumn('uuid')
@@ -30,6 +32,24 @@ export class Cluster {
@Column({ nullable: true })
region: string;
@Column({ default: 1 })
weight: number;
@Column({ type: 'jsonb', default: [] })
tags: string[];
@Column({ default: 'unknown' })
healthStatus: ClusterHealthStatus;
@Column({ type: 'timestamptz', nullable: true })
lastHealthCheckedAt: Date;
@Column({ nullable: true })
healthMessage: string;
@Column({ type: 'jsonb', nullable: true })
availableResources: Record<string, any>;
@Column({ nullable: true })
provider: string; // e.g., 'aws', 'gcp', 'azure', 'bare-metal'