keyhan
35dd771f63
Deploy cloudhost-logging on cluster registration, ship app and optional service logs to ES with owner isolation, and fix Kibana 8.12 auth via kibana_system. Co-authored-by: Cursor <cursoragent@cursor.com>
717 lines
27 KiB
TypeScript
717 lines
27 KiB
TypeScript
import { Injectable, NotFoundException, Logger, BadRequestException, Inject, forwardRef } from '@nestjs/common';
|
|
import { InjectRepository } from '@nestjs/typeorm';
|
|
import { ConfigService } from '@nestjs/config';
|
|
import { Repository, DataSource, In } from 'typeorm';
|
|
import * as k8s from '@kubernetes/client-node';
|
|
import { Cluster } from './entities/cluster.entity';
|
|
import { ClusterPool } from './entities/cluster-pool.entity';
|
|
import { CreateClusterDto, UpdateClusterDto } from './dto/cluster.dto';
|
|
import { CreateClusterPoolDto, UpdateClusterPoolDto } from './dto/cluster-pool.dto';
|
|
import { ClusterStatus } from '../common/enums';
|
|
import { ElasticsearchService } from '../kubernetes/elasticsearch.service';
|
|
|
|
@Injectable()
|
|
export class ClustersService {
|
|
private readonly logger = new Logger(ClustersService.name);
|
|
private roundRobinIndex = 0;
|
|
private poolRoundRobinIndices = new Map<string, number>();
|
|
|
|
constructor(
|
|
@InjectRepository(Cluster)
|
|
private clustersRepository: Repository<Cluster>,
|
|
@InjectRepository(ClusterPool)
|
|
private poolsRepository: Repository<ClusterPool>,
|
|
private dataSource: DataSource,
|
|
private configService: ConfigService,
|
|
@Inject(forwardRef(() => ElasticsearchService))
|
|
private elasticsearchService: ElasticsearchService,
|
|
) {}
|
|
|
|
/**
|
|
* Test connection to a Kubernetes cluster using its kubeconfig.
|
|
* Calls the /version endpoint to verify the cluster is reachable.
|
|
*/
|
|
async testConnection(kubeconfig: string): Promise<{ connected: boolean; version?: string; error?: string }> {
|
|
try {
|
|
const kc = new k8s.KubeConfig();
|
|
kc.loadFromString(kubeconfig);
|
|
|
|
const versionApi = kc.makeApiClient(k8s.VersionApi);
|
|
const result = await versionApi.getCode();
|
|
const info = result.body;
|
|
|
|
this.logger.log(`Cluster connection OK: Kubernetes ${info.gitVersion}`);
|
|
return {
|
|
connected: true,
|
|
version: info.gitVersion,
|
|
};
|
|
} catch (err: any) {
|
|
const message = err?.body?.message || err?.message || 'Unknown connection error';
|
|
this.logger.warn(`Cluster connection failed: ${message}`);
|
|
return {
|
|
connected: false,
|
|
error: message,
|
|
};
|
|
}
|
|
}
|
|
|
|
async create(dto: CreateClusterDto): Promise<Cluster> {
|
|
// Validate kubeconfig by testing actual connection
|
|
const connectionTest = await this.testConnection(dto.kubeconfig);
|
|
if (!connectionTest.connected) {
|
|
throw new BadRequestException(
|
|
`Cannot connect to Kubernetes cluster: ${connectionTest.error}`,
|
|
);
|
|
}
|
|
|
|
if (dto.isDefault === true) {
|
|
const existingDefaults = await this.clustersRepository.find({ where: { isDefault: true } });
|
|
for (const c of existingDefaults) {
|
|
c.isDefault = false;
|
|
await this.clustersRepository.save(c);
|
|
}
|
|
}
|
|
|
|
const cluster = this.clustersRepository.create({
|
|
...dto,
|
|
status: ClusterStatus.ACTIVE, // Connection verified — mark active
|
|
});
|
|
const saved = await this.clustersRepository.save(cluster);
|
|
this.logger.log(`Cluster "${saved.name}" registered (active) — K8s ${connectionTest.version}`);
|
|
|
|
// Bootstrap the cluster with build infrastructure (namespace, registry, SA, etc.)
|
|
this.bootstrapCluster(saved.kubeconfig).catch((err) => {
|
|
this.logger.error(`Failed to bootstrap cluster "${saved.name}": ${err.message}`);
|
|
});
|
|
|
|
// Deploy central logging (Elasticsearch + Kibana) via Helm
|
|
this.elasticsearchService.deploy(saved.id).catch((err) => {
|
|
this.logger.error(`Failed to deploy central logging on "${saved.name}": ${err.message}`);
|
|
});
|
|
|
|
return saved;
|
|
}
|
|
|
|
async findAll(): Promise<Cluster[]> {
|
|
return this.clustersRepository.find({
|
|
select: ['id', 'name', 'description', 'status', 'apiServer', 'region', 'provider', 'isDefault', 'createdAt'],
|
|
order: { createdAt: 'DESC' },
|
|
});
|
|
}
|
|
|
|
async findOne(id: string): Promise<Cluster> {
|
|
const cluster = await this.clustersRepository.findOne({ where: { id } });
|
|
if (!cluster) {
|
|
throw new NotFoundException('Cluster not found');
|
|
}
|
|
return cluster;
|
|
}
|
|
|
|
async getDefault(): Promise<Cluster> {
|
|
// Prefer active default cluster; fall back to any active cluster
|
|
let cluster = await this.clustersRepository.findOne({
|
|
where: { isDefault: true, status: ClusterStatus.ACTIVE },
|
|
});
|
|
if (!cluster) {
|
|
// Fallback: pick any active cluster and promote it to default
|
|
cluster = await this.clustersRepository.findOne({
|
|
where: { status: ClusterStatus.ACTIVE },
|
|
});
|
|
if (cluster) {
|
|
cluster.isDefault = true;
|
|
await this.clustersRepository.save(cluster);
|
|
this.logger.warn(`No active default cluster — promoted "${cluster.name}" to default`);
|
|
}
|
|
}
|
|
if (!cluster) {
|
|
throw new NotFoundException('No active cluster available');
|
|
}
|
|
return cluster;
|
|
}
|
|
|
|
async update(id: string, dto: UpdateClusterDto): Promise<Cluster> {
|
|
const cluster = await this.findOne(id);
|
|
|
|
// If kubeconfig is being updated, re-test connection
|
|
if (dto.kubeconfig) {
|
|
const connectionTest = await this.testConnection(dto.kubeconfig);
|
|
if (!connectionTest.connected) {
|
|
throw new BadRequestException(
|
|
`Cannot connect to Kubernetes cluster: ${connectionTest.error}`,
|
|
);
|
|
}
|
|
dto.status = ClusterStatus.ACTIVE;
|
|
this.logger.log(`Cluster "${cluster.name}" kubeconfig updated — connection verified (K8s ${connectionTest.version})`);
|
|
|
|
// Re-bootstrap build infrastructure on the new/updated cluster
|
|
this.bootstrapCluster(dto.kubeconfig).catch((err: any) => {
|
|
this.logger.error(`Failed to bootstrap cluster "${cluster.name}": ${err.message}`);
|
|
});
|
|
}
|
|
|
|
if (dto.isDefault === true) {
|
|
const existingDefaults = await this.clustersRepository.find({ where: { isDefault: true } });
|
|
for (const c of existingDefaults) {
|
|
if (c.id !== id) {
|
|
c.isDefault = false;
|
|
await this.clustersRepository.save(c);
|
|
}
|
|
}
|
|
}
|
|
Object.assign(cluster, dto);
|
|
return this.clustersRepository.save(cluster);
|
|
}
|
|
|
|
/**
|
|
* Manually test connectivity to an existing cluster.
|
|
* Updates status to active/inactive based on result.
|
|
*/
|
|
async testClusterById(id: string): Promise<{ connected: boolean; version?: string; error?: string }> {
|
|
const cluster = await this.findOne(id);
|
|
const result = await this.testConnection(cluster.kubeconfig);
|
|
|
|
cluster.status = result.connected ? ClusterStatus.ACTIVE : ClusterStatus.INACTIVE;
|
|
await this.clustersRepository.save(cluster);
|
|
this.logger.log(`Cluster "${cluster.name}" test: ${result.connected ? 'ACTIVE' : 'INACTIVE'}`);
|
|
|
|
return result;
|
|
}
|
|
|
|
/**
|
|
* Public cluster list (no sensitive data) — for users to select a cluster.
|
|
*/
|
|
async findAllPublic(): Promise<Pick<Cluster, 'id' | 'name' | 'region' | 'provider' | 'isDefault' | 'status'>[]> {
|
|
return this.clustersRepository.find({
|
|
select: ['id', 'name', 'region', 'provider', 'isDefault', 'status'],
|
|
where: { status: ClusterStatus.ACTIVE },
|
|
order: { isDefault: 'DESC', name: 'ASC' },
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Get the optimal cluster using load-balancing strategy.
|
|
* Strategy: 'least-apps' — picks the active cluster with fewest deployed applications.
|
|
* Falls back to default cluster if only one active cluster exists.
|
|
*/
|
|
async getOptimalCluster(strategy: 'least-apps' | 'round-robin' = 'least-apps'): Promise<Cluster> {
|
|
const activeClusters = await this.clustersRepository.find({
|
|
where: { status: ClusterStatus.ACTIVE },
|
|
});
|
|
|
|
if (activeClusters.length === 0) {
|
|
throw new NotFoundException('No active clusters available');
|
|
}
|
|
|
|
if (activeClusters.length === 1) {
|
|
return activeClusters[0];
|
|
}
|
|
|
|
if (strategy === 'round-robin') {
|
|
const cluster = activeClusters[this.roundRobinIndex % activeClusters.length];
|
|
this.roundRobinIndex++;
|
|
this.logger.log(`Round-robin selected cluster "${cluster.name}" (index: ${this.roundRobinIndex - 1})`);
|
|
return cluster;
|
|
}
|
|
|
|
// least-apps: count applications per cluster
|
|
const appCounts: { clusterId: string; count: string }[] = await this.dataSource.query(`
|
|
SELECT "clusterId", COUNT(*) as count
|
|
FROM applications
|
|
WHERE "clusterId" IS NOT NULL
|
|
GROUP BY "clusterId"
|
|
`);
|
|
|
|
const countMap = new Map<string, number>();
|
|
for (const row of appCounts) {
|
|
countMap.set(row.clusterId, parseInt(row.count, 10));
|
|
}
|
|
|
|
// Sort by app count ascending (least apps first)
|
|
activeClusters.sort((a, b) => {
|
|
const countA = countMap.get(a.id) || 0;
|
|
const countB = countMap.get(b.id) || 0;
|
|
return countA - countB;
|
|
});
|
|
|
|
const selected = activeClusters[0];
|
|
const selectedCount = countMap.get(selected.id) || 0;
|
|
this.logger.log(`Least-apps selected cluster "${selected.name}" (${selectedCount} apps)`);
|
|
return selected;
|
|
}
|
|
|
|
async delete(id: string): Promise<void> {
|
|
const cluster = await this.findOne(id);
|
|
const wasDefault = cluster.isDefault;
|
|
|
|
// Reassign applications that were on this cluster to another active cluster
|
|
try {
|
|
const replacement = await this.clustersRepository.findOne({
|
|
where: { status: ClusterStatus.ACTIVE, id: undefined as any },
|
|
});
|
|
// Use raw query to exclude the deleted cluster
|
|
const activeReplacement = await this.clustersRepository
|
|
.createQueryBuilder('c')
|
|
.where('c.id != :id', { id })
|
|
.andWhere('c.status = :status', { status: ClusterStatus.ACTIVE })
|
|
.getOne();
|
|
|
|
if (activeReplacement) {
|
|
const result = await this.dataSource.query(
|
|
`UPDATE applications SET "clusterId" = $1 WHERE "clusterId" = $2`,
|
|
[activeReplacement.id, id],
|
|
);
|
|
const count = result?.[1] || 0;
|
|
if (count > 0) {
|
|
this.logger.log(`Reassigned ${count} application(s) from cluster "${cluster.name}" to "${activeReplacement.name}"`);
|
|
}
|
|
} else {
|
|
// No replacement — nullify clusterId so apps aren't orphaned with a dangling FK
|
|
await this.dataSource.query(
|
|
`UPDATE applications SET "clusterId" = NULL WHERE "clusterId" = $1`,
|
|
[id],
|
|
);
|
|
this.logger.warn(`No active replacement cluster — cleared clusterId for apps on "${cluster.name}"`);
|
|
}
|
|
} catch (e: any) {
|
|
this.logger.warn(`Failed to reassign apps from cluster "${cluster.name}": ${e.message}`);
|
|
}
|
|
|
|
await this.clustersRepository.remove(cluster);
|
|
this.logger.log(`Cluster "${cluster.name}" deleted`);
|
|
|
|
// If deleted cluster was default, promote another active cluster
|
|
if (wasDefault) {
|
|
const newDefault = await this.clustersRepository.findOne({
|
|
where: { status: ClusterStatus.ACTIVE },
|
|
});
|
|
if (newDefault) {
|
|
newDefault.isDefault = true;
|
|
await this.clustersRepository.save(newDefault);
|
|
this.logger.log(`Promoted cluster "${newDefault.name}" to default after deleting "${cluster.name}"`);
|
|
}
|
|
}
|
|
}
|
|
|
|
// ─── Cluster Pool Methods ─────────────────────────────────────────
|
|
|
|
async createPool(dto: CreateClusterPoolDto): Promise<ClusterPool> {
|
|
// Validate that all cluster IDs exist
|
|
if (dto.clusterIds.length === 0) {
|
|
throw new BadRequestException('Pool must contain at least one cluster');
|
|
}
|
|
const clusters = await this.clustersRepository.find({
|
|
where: { id: In(dto.clusterIds) },
|
|
});
|
|
if (clusters.length !== dto.clusterIds.length) {
|
|
const foundIds = clusters.map((c) => c.id);
|
|
const missingIds = dto.clusterIds.filter((id) => !foundIds.includes(id));
|
|
throw new BadRequestException(`Clusters not found: ${missingIds.join(', ')}`);
|
|
}
|
|
|
|
const pool = this.poolsRepository.create(dto);
|
|
const saved = await this.poolsRepository.save(pool);
|
|
this.logger.log(`Cluster pool "${saved.name}" created with ${dto.clusterIds.length} clusters (strategy: ${dto.strategy})`);
|
|
return saved;
|
|
}
|
|
|
|
async findAllPools(): Promise<ClusterPool[]> {
|
|
return this.poolsRepository.find({ order: { createdAt: 'DESC' } });
|
|
}
|
|
|
|
/**
|
|
* Public pool list — returns pools with resolved cluster names for UI.
|
|
*/
|
|
async findAllPoolsPublic(): Promise<(ClusterPool & { clusters: Pick<Cluster, 'id' | 'name' | 'region' | 'provider' | 'status'>[] })[]> {
|
|
const pools = await this.poolsRepository.find({
|
|
where: { isActive: true },
|
|
order: { createdAt: 'DESC' },
|
|
});
|
|
|
|
const allClusterIds = [...new Set(pools.flatMap((p) => p.clusterIds))];
|
|
const clusters = allClusterIds.length > 0
|
|
? await this.clustersRepository.find({
|
|
where: { id: In(allClusterIds) },
|
|
select: ['id', 'name', 'region', 'provider', 'status'],
|
|
})
|
|
: [];
|
|
|
|
const clusterMap = new Map(clusters.map((c) => [c.id, c]));
|
|
|
|
return pools.map((pool) => ({
|
|
...pool,
|
|
clusters: pool.clusterIds
|
|
.map((id) => clusterMap.get(id))
|
|
.filter(Boolean) as Pick<Cluster, 'id' | 'name' | 'region' | 'provider' | 'status'>[],
|
|
}));
|
|
}
|
|
|
|
async findOnePool(id: string): Promise<ClusterPool> {
|
|
const pool = await this.poolsRepository.findOne({ where: { id } });
|
|
if (!pool) {
|
|
throw new NotFoundException('Cluster pool not found');
|
|
}
|
|
return pool;
|
|
}
|
|
|
|
async updatePool(id: string, dto: UpdateClusterPoolDto): Promise<ClusterPool> {
|
|
const pool = await this.findOnePool(id);
|
|
|
|
if (dto.clusterIds && dto.clusterIds.length > 0) {
|
|
const clusters = await this.clustersRepository.find({
|
|
where: { id: In(dto.clusterIds) },
|
|
});
|
|
if (clusters.length !== dto.clusterIds.length) {
|
|
const foundIds = clusters.map((c) => c.id);
|
|
const missingIds = dto.clusterIds.filter((cid) => !foundIds.includes(cid));
|
|
throw new BadRequestException(`Clusters not found: ${missingIds.join(', ')}`);
|
|
}
|
|
}
|
|
|
|
Object.assign(pool, dto);
|
|
return this.poolsRepository.save(pool);
|
|
}
|
|
|
|
async deletePool(id: string): Promise<void> {
|
|
const pool = await this.findOnePool(id);
|
|
await this.poolsRepository.remove(pool);
|
|
this.poolRoundRobinIndices.delete(id);
|
|
}
|
|
|
|
/**
|
|
* Get the optimal cluster from a specific pool using the pool's strategy.
|
|
* Only considers ACTIVE clusters within the pool.
|
|
*/
|
|
async getOptimalClusterFromPool(poolId: string): Promise<Cluster> {
|
|
const pool = await this.findOnePool(poolId);
|
|
if (!pool.isActive) {
|
|
throw new BadRequestException(`Pool "${pool.name}" is not active`);
|
|
}
|
|
|
|
const activeClusters = await this.clustersRepository.find({
|
|
where: {
|
|
id: In(pool.clusterIds),
|
|
status: ClusterStatus.ACTIVE,
|
|
},
|
|
});
|
|
|
|
if (activeClusters.length === 0) {
|
|
throw new NotFoundException(`No active clusters in pool "${pool.name}"`);
|
|
}
|
|
|
|
if (activeClusters.length === 1) {
|
|
return activeClusters[0];
|
|
}
|
|
|
|
if (pool.strategy === 'round-robin') {
|
|
const idx = this.poolRoundRobinIndices.get(pool.id) || 0;
|
|
const cluster = activeClusters[idx % activeClusters.length];
|
|
this.poolRoundRobinIndices.set(pool.id, idx + 1);
|
|
this.logger.log(`Pool "${pool.name}" round-robin → cluster "${cluster.name}"`);
|
|
return cluster;
|
|
}
|
|
|
|
// least-apps strategy
|
|
const appCounts: { clusterId: string; count: string }[] = await this.dataSource.query(`
|
|
SELECT "clusterId", COUNT(*) as count
|
|
FROM applications
|
|
WHERE "clusterId" = ANY($1)
|
|
GROUP BY "clusterId"
|
|
`, [pool.clusterIds]);
|
|
|
|
const countMap = new Map<string, number>();
|
|
for (const row of appCounts) {
|
|
countMap.set(row.clusterId, parseInt(row.count, 10));
|
|
}
|
|
|
|
activeClusters.sort((a, b) => {
|
|
return (countMap.get(a.id) || 0) - (countMap.get(b.id) || 0);
|
|
});
|
|
|
|
const selected = activeClusters[0];
|
|
this.logger.log(`Pool "${pool.name}" least-apps → cluster "${selected.name}" (${countMap.get(selected.id) || 0} apps)`);
|
|
return selected;
|
|
}
|
|
|
|
/**
|
|
* Get resource usage for a specific cluster — nodes, total CPU/memory, pod counts.
|
|
*/
|
|
async getClusterResources(id: string): Promise<{
|
|
nodes: { name: string; status: string; roles: string; cpuCapacity: string; memoryCapacity: string; cpuAllocatable: string; memoryAllocatable: string; }[];
|
|
totalCpuCapacity: string;
|
|
totalMemoryCapacity: string;
|
|
totalCpuAllocatable: string;
|
|
totalMemoryAllocatable: string;
|
|
podCount: number;
|
|
nodeCount: number;
|
|
appCount: number;
|
|
}> {
|
|
const cluster = await this.findOne(id);
|
|
|
|
const kc = new k8s.KubeConfig();
|
|
kc.loadFromString(cluster.kubeconfig);
|
|
const coreApi = kc.makeApiClient(k8s.CoreV1Api);
|
|
|
|
try {
|
|
// Get nodes
|
|
const nodesRes = await coreApi.listNode();
|
|
const nodes = nodesRes.body.items.map((node) => {
|
|
const conditions = node.status?.conditions || [];
|
|
const readyCondition = conditions.find((c) => c.type === 'Ready');
|
|
const roles = Object.keys(node.metadata?.labels || {})
|
|
.filter((l) => l.startsWith('node-role.kubernetes.io/'))
|
|
.map((l) => l.replace('node-role.kubernetes.io/', ''))
|
|
.join(', ') || 'worker';
|
|
|
|
return {
|
|
name: node.metadata?.name || 'unknown',
|
|
status: readyCondition?.status === 'True' ? 'Ready' : 'NotReady',
|
|
roles,
|
|
cpuCapacity: node.status?.capacity?.cpu || '0',
|
|
memoryCapacity: node.status?.capacity?.memory || '0',
|
|
cpuAllocatable: node.status?.allocatable?.cpu || '0',
|
|
memoryAllocatable: node.status?.allocatable?.memory || '0',
|
|
};
|
|
});
|
|
|
|
// Total capacity
|
|
let totalCpuCap = 0;
|
|
let totalMemCap = 0;
|
|
let totalCpuAlloc = 0;
|
|
let totalMemAlloc = 0;
|
|
for (const node of nodes) {
|
|
totalCpuCap += this.parseCpuToMillicores(node.cpuCapacity);
|
|
totalMemCap += this.parseMemoryToMi(node.memoryCapacity);
|
|
totalCpuAlloc += this.parseCpuToMillicores(node.cpuAllocatable);
|
|
totalMemAlloc += this.parseMemoryToMi(node.memoryAllocatable);
|
|
}
|
|
|
|
// Get all pods count
|
|
const podsRes = await coreApi.listPodForAllNamespaces();
|
|
const podCount = podsRes.body.items.length;
|
|
|
|
// Get app count for this cluster
|
|
const appCountResult = await this.dataSource.query(
|
|
`SELECT COUNT(*) as count FROM applications WHERE "clusterId" = $1`,
|
|
[id],
|
|
);
|
|
const appCount = parseInt(appCountResult[0]?.count || '0', 10);
|
|
|
|
return {
|
|
nodes,
|
|
totalCpuCapacity: `${totalCpuCap}m`,
|
|
totalMemoryCapacity: `${totalMemCap.toFixed(0)}Mi`,
|
|
totalCpuAllocatable: `${totalCpuAlloc}m`,
|
|
totalMemoryAllocatable: `${totalMemAlloc.toFixed(0)}Mi`,
|
|
podCount,
|
|
nodeCount: nodes.length,
|
|
appCount,
|
|
};
|
|
} catch (err: any) {
|
|
this.logger.error(`Failed to get cluster resources for "${cluster.name}": ${err.message}`);
|
|
throw new BadRequestException(`Cannot fetch resources: ${err.message}`);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Bootstrap a newly-added cluster with the build infrastructure:
|
|
* 1. cloudhost-builds namespace
|
|
* 2. kaniko-builder ServiceAccount
|
|
* 3. Docker Registry Deployment + PVC + Service (ClusterIP) + NodePort Service
|
|
* 4. registry-credentials Secret (for Kaniko docker auth)
|
|
*/
|
|
async bootstrapCluster(kubeconfig: string): Promise<void> {
|
|
const kc = new k8s.KubeConfig();
|
|
kc.loadFromString(kubeconfig);
|
|
const coreApi = kc.makeApiClient(k8s.CoreV1Api);
|
|
const appsApi = kc.makeApiClient(k8s.AppsV1Api);
|
|
|
|
const buildNs = this.configService.get<string>('build.namespace') || 'cloudhost-builds';
|
|
const saName = this.configService.get<string>('build.serviceAccount') || 'kaniko-builder';
|
|
const registryUrl = this.configService.get<string>('registry.url') || `registry.${buildNs}.svc.cluster.local:5000`;
|
|
|
|
this.logger.log(`Bootstrapping cluster — namespace: ${buildNs}`);
|
|
|
|
// ── 1. Namespace ──────────────────────────────────────────────
|
|
try {
|
|
await coreApi.readNamespace(buildNs);
|
|
this.logger.log(`Namespace "${buildNs}" already exists`);
|
|
} catch (err: any) {
|
|
if (err.statusCode === 404 || err.body?.code === 404) {
|
|
await coreApi.createNamespace({ metadata: { name: buildNs } });
|
|
this.logger.log(`Created namespace "${buildNs}"`);
|
|
} else {
|
|
throw err;
|
|
}
|
|
}
|
|
|
|
// ── 2. ServiceAccount for Kaniko ──────────────────────────────
|
|
try {
|
|
await coreApi.readNamespacedServiceAccount(saName, buildNs);
|
|
this.logger.log(`ServiceAccount "${saName}" already exists`);
|
|
} catch (err: any) {
|
|
if (err.statusCode === 404 || err.body?.code === 404) {
|
|
await coreApi.createNamespacedServiceAccount(buildNs, {
|
|
metadata: { name: saName, namespace: buildNs },
|
|
});
|
|
this.logger.log(`Created ServiceAccount "${saName}"`);
|
|
} else {
|
|
throw err;
|
|
}
|
|
}
|
|
|
|
// ── 3. Docker Registry PVC ────────────────────────────────────
|
|
const registryPvcName = 'registry-data';
|
|
try {
|
|
await coreApi.readNamespacedPersistentVolumeClaim(registryPvcName, buildNs);
|
|
this.logger.log(`PVC "${registryPvcName}" already exists`);
|
|
} catch (err: any) {
|
|
if (err.statusCode === 404 || err.body?.code === 404) {
|
|
await coreApi.createNamespacedPersistentVolumeClaim(buildNs, {
|
|
metadata: { name: registryPvcName, namespace: buildNs },
|
|
spec: {
|
|
accessModes: ['ReadWriteOnce'],
|
|
resources: { requests: { storage: '10Gi' } },
|
|
},
|
|
});
|
|
this.logger.log(`Created PVC "${registryPvcName}" (10Gi)`);
|
|
} else {
|
|
throw err;
|
|
}
|
|
}
|
|
|
|
// ── 4. Docker Registry Deployment ─────────────────────────────
|
|
const registryDeployName = 'registry';
|
|
try {
|
|
await appsApi.readNamespacedDeployment(registryDeployName, buildNs);
|
|
this.logger.log(`Deployment "${registryDeployName}" already exists`);
|
|
} catch (err: any) {
|
|
if (err.statusCode === 404 || err.body?.code === 404) {
|
|
await appsApi.createNamespacedDeployment(buildNs, {
|
|
metadata: { name: registryDeployName, namespace: buildNs, labels: { app: 'registry' } },
|
|
spec: {
|
|
replicas: 1,
|
|
selector: { matchLabels: { app: 'registry' } },
|
|
template: {
|
|
metadata: { labels: { app: 'registry' } },
|
|
spec: {
|
|
containers: [{
|
|
name: 'registry',
|
|
image: 'registry:2',
|
|
ports: [{ containerPort: 5000 }],
|
|
env: [
|
|
{ name: 'REGISTRY_STORAGE_DELETE_ENABLED', value: 'true' },
|
|
],
|
|
volumeMounts: [{
|
|
name: 'registry-data',
|
|
mountPath: '/var/lib/registry',
|
|
}],
|
|
resources: {
|
|
requests: { cpu: '100m', memory: '128Mi' },
|
|
limits: { cpu: '500m', memory: '512Mi' },
|
|
},
|
|
}],
|
|
volumes: [{
|
|
name: 'registry-data',
|
|
persistentVolumeClaim: { claimName: registryPvcName },
|
|
}],
|
|
},
|
|
},
|
|
},
|
|
});
|
|
this.logger.log(`Created Docker Registry Deployment`);
|
|
} else {
|
|
throw err;
|
|
}
|
|
}
|
|
|
|
// ── 5. Registry ClusterIP Service (for Kaniko to push) ────────
|
|
const registrySvcName = 'registry';
|
|
try {
|
|
await coreApi.readNamespacedService(registrySvcName, buildNs);
|
|
this.logger.log(`Service "${registrySvcName}" already exists`);
|
|
} catch (err: any) {
|
|
if (err.statusCode === 404 || err.body?.code === 404) {
|
|
await coreApi.createNamespacedService(buildNs, {
|
|
metadata: { name: registrySvcName, namespace: buildNs, labels: { app: 'registry' } },
|
|
spec: {
|
|
type: 'ClusterIP',
|
|
selector: { app: 'registry' },
|
|
ports: [{ port: 5000, targetPort: 5000 as any, protocol: 'TCP' }],
|
|
},
|
|
});
|
|
this.logger.log(`Created Registry ClusterIP Service (port 5000)`);
|
|
} else {
|
|
throw err;
|
|
}
|
|
}
|
|
|
|
// ── 6. Registry NodePort Service (for kubelet to pull) ────────
|
|
const registryNodePortName = 'registry-nodeport';
|
|
try {
|
|
await coreApi.readNamespacedService(registryNodePortName, buildNs);
|
|
this.logger.log(`Service "${registryNodePortName}" already exists`);
|
|
} catch (err: any) {
|
|
if (err.statusCode === 404 || err.body?.code === 404) {
|
|
await coreApi.createNamespacedService(buildNs, {
|
|
metadata: { name: registryNodePortName, namespace: buildNs, labels: { app: 'registry' } },
|
|
spec: {
|
|
type: 'NodePort',
|
|
selector: { app: 'registry' },
|
|
ports: [{ port: 5000, targetPort: 5000 as any, nodePort: 30500, protocol: 'TCP' }],
|
|
},
|
|
});
|
|
this.logger.log(`Created Registry NodePort Service (30500 → 5000)`);
|
|
} else {
|
|
throw err;
|
|
}
|
|
}
|
|
|
|
// ── 7. registry-credentials Secret (docker config for Kaniko) ─
|
|
const registrySecretName = 'registry-credentials';
|
|
try {
|
|
await coreApi.readNamespacedSecret(registrySecretName, buildNs);
|
|
this.logger.log(`Secret "${registrySecretName}" already exists`);
|
|
} catch (err: any) {
|
|
if (err.statusCode === 404 || err.body?.code === 404) {
|
|
// Parse registry host (without port path) for the docker config
|
|
const dockerConfig = JSON.stringify({
|
|
auths: {
|
|
[registryUrl]: { auth: '' },
|
|
[`registry.${buildNs}.svc.cluster.local:5000`]: { auth: '' },
|
|
'localhost:30500': { auth: '' },
|
|
},
|
|
});
|
|
await coreApi.createNamespacedSecret(buildNs, {
|
|
metadata: { name: registrySecretName, namespace: buildNs },
|
|
type: 'kubernetes.io/dockerconfigjson',
|
|
data: {
|
|
'.dockerconfigjson': Buffer.from(dockerConfig).toString('base64'),
|
|
},
|
|
});
|
|
this.logger.log(`Created registry-credentials Secret`);
|
|
} else {
|
|
throw err;
|
|
}
|
|
}
|
|
|
|
this.logger.log(`✅ Cluster bootstrap complete — build infrastructure ready`);
|
|
}
|
|
|
|
private parseCpuToMillicores(cpu: string): number {
|
|
if (!cpu || cpu === '0') return 0;
|
|
if (cpu.endsWith('n')) return parseFloat(cpu) / 1_000_000;
|
|
if (cpu.endsWith('u')) return parseFloat(cpu) / 1_000;
|
|
if (cpu.endsWith('m')) return parseFloat(cpu);
|
|
return parseFloat(cpu) * 1000;
|
|
}
|
|
|
|
private parseMemoryToMi(memory: string): number {
|
|
if (!memory || memory === '0') return 0;
|
|
if (memory.endsWith('Ki')) return parseFloat(memory) / 1024;
|
|
if (memory.endsWith('Mi')) return parseFloat(memory);
|
|
if (memory.endsWith('Gi')) return parseFloat(memory) * 1024;
|
|
if (memory.endsWith('Ti')) return parseFloat(memory) * 1024 * 1024;
|
|
return parseFloat(memory) / (1024 * 1024); // bytes
|
|
}
|
|
}
|