Use in-cluster registry for builds and deploys; improve logging and cluster ops.
Remove external registry Ingress (repo.3fase.ir) and route Kaniko push and app pulls through the internal ClusterIP registry. Add RegistryService, ensure StorageClass and pull secrets on deploy, make Elasticsearch install/repair more resilient, and add per-cluster Deploy Elastic controls in admin UI. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -19,13 +19,14 @@ export class ElasticsearchController {
|
||||
@ApiQuery({ name: 'clusterId', required: false, description: 'Target cluster ID' })
|
||||
@ApiResponse({ status: 200, description: 'Elasticsearch status' })
|
||||
async getStatus(@Query('clusterId') clusterId?: string) {
|
||||
const isDeployed = await this.esService.isDeployed(clusterId);
|
||||
const health = isDeployed ? await this.esService.getHealth(clusterId) : null;
|
||||
const state = await this.esService.getDeployState(clusterId);
|
||||
const connectionInfo = this.esService.getConnectionInfo();
|
||||
|
||||
|
||||
return {
|
||||
deployed: isDeployed,
|
||||
health,
|
||||
deployed: state.status === 'ready',
|
||||
deployStatus: state.status,
|
||||
helmReleaseStatus: state.helmReleaseStatus,
|
||||
health: state.health,
|
||||
namespace: 'logging',
|
||||
elasticsearch: {
|
||||
host: connectionInfo.host,
|
||||
@@ -41,28 +42,38 @@ export class ElasticsearchController {
|
||||
}
|
||||
|
||||
@Post('deploy')
|
||||
@ApiOperation({ summary: 'Deploy central Elasticsearch + Kibana stack' })
|
||||
@ApiOperation({ summary: 'Deploy or repair central Elasticsearch + Kibana stack' })
|
||||
@ApiQuery({ name: 'clusterId', required: false, description: 'Target cluster ID' })
|
||||
@ApiResponse({ status: 201, description: 'Elasticsearch deployed successfully' })
|
||||
async deploy(@Query('clusterId') clusterId?: string) {
|
||||
const alreadyDeployed = await this.esService.isDeployed(clusterId);
|
||||
|
||||
if (alreadyDeployed) {
|
||||
const health = await this.esService.getHealth(clusterId);
|
||||
const before = await this.esService.getDeployState(clusterId);
|
||||
|
||||
if (before.status === 'ready') {
|
||||
return {
|
||||
success: true,
|
||||
message: 'Elasticsearch is already deployed',
|
||||
message: 'Elasticsearch is already deployed and healthy',
|
||||
status: 'existing',
|
||||
health,
|
||||
deployStatus: before.status,
|
||||
health: before.health,
|
||||
};
|
||||
}
|
||||
|
||||
const result = await this.esService.deploy(clusterId);
|
||||
|
||||
const after = await this.esService.getDeployState(clusterId);
|
||||
|
||||
const status =
|
||||
before.status === 'not_installed'
|
||||
? 'created'
|
||||
: 'repaired';
|
||||
|
||||
return {
|
||||
success: true,
|
||||
message: 'Elasticsearch and Kibana deployed successfully. Please wait 2-3 minutes for pods to be ready.',
|
||||
status: 'created',
|
||||
message: result.deploying
|
||||
? 'Logging stack installed. Pods are starting — allow 2–5 minutes for Elasticsearch and Kibana to become ready.'
|
||||
: 'Elasticsearch and Kibana are ready.',
|
||||
status,
|
||||
deployStatus: after.status,
|
||||
health: after.health,
|
||||
credentials: {
|
||||
username: 'elastic',
|
||||
password: result.esPassword,
|
||||
@@ -87,7 +98,7 @@ export class ElasticsearchController {
|
||||
@ApiResponse({ status: 200, description: 'Elasticsearch undeployed' })
|
||||
async undeploy(@Query('clusterId') clusterId?: string) {
|
||||
await this.esService.undeploy(clusterId);
|
||||
|
||||
|
||||
return {
|
||||
success: true,
|
||||
message: 'Elasticsearch stack removed. PVC with data is preserved.',
|
||||
@@ -101,7 +112,7 @@ export class ElasticsearchController {
|
||||
async getCredentials() {
|
||||
const creds = this.esService.getFluentBitCredentials();
|
||||
const connInfo = this.esService.getConnectionInfo();
|
||||
|
||||
|
||||
return {
|
||||
elasticsearch: {
|
||||
host: connInfo.host,
|
||||
|
||||
@@ -59,6 +59,21 @@ export interface LogStatsResult {
|
||||
period: string;
|
||||
}
|
||||
|
||||
export type LoggingDeployStatus = 'not_installed' | 'installing' | 'failed' | 'ready';
|
||||
|
||||
export interface LoggingDeployState {
|
||||
status: LoggingDeployStatus;
|
||||
helmReleaseStatus?: string | null;
|
||||
health: {
|
||||
status: string;
|
||||
clusterName: string;
|
||||
numberOfNodes: number;
|
||||
activePrimaryShards: number;
|
||||
activeShards: number;
|
||||
kibanaReady: boolean;
|
||||
} | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Central Elasticsearch management service.
|
||||
* Deploys a shared Elasticsearch + Kibana stack in a dedicated namespace
|
||||
@@ -358,17 +373,124 @@ export class ElasticsearchService implements OnModuleInit, OnModuleDestroy {
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if central Elasticsearch is deployed in a cluster
|
||||
* Check if central Elasticsearch is deployed and healthy in a cluster.
|
||||
*/
|
||||
async isDeployed(clusterId?: string): Promise<boolean> {
|
||||
const { appsApi } = await this.getK8sClients(clusterId);
|
||||
const state = await this.getDeployState(clusterId);
|
||||
return state.status === 'ready';
|
||||
}
|
||||
|
||||
/**
|
||||
* Detailed logging stack state (Helm release + pod readiness).
|
||||
*/
|
||||
async getDeployState(clusterId?: string): Promise<LoggingDeployState> {
|
||||
const { cluster } = await this.getK8sClients(clusterId);
|
||||
const helmStatus = await this.helmService.status(
|
||||
LOGGING_HELM_RELEASE,
|
||||
LOGGING_HELM_NAMESPACE,
|
||||
cluster.kubeconfig,
|
||||
);
|
||||
|
||||
let hasEsWorkload = false;
|
||||
try {
|
||||
const kc = new k8s.KubeConfig();
|
||||
kc.loadFromString(cluster.kubeconfig);
|
||||
const appsApi = kc.makeApiClient(k8s.AppsV1Api);
|
||||
await appsApi.readNamespacedStatefulSet(this.ES_NAME, this.ES_NAMESPACE);
|
||||
hasEsWorkload = true;
|
||||
} catch {
|
||||
hasEsWorkload = false;
|
||||
}
|
||||
|
||||
if (!helmStatus && !hasEsWorkload) {
|
||||
return { status: 'not_installed', helmReleaseStatus: null, health: null };
|
||||
}
|
||||
|
||||
const health = await this.getHealth(clusterId);
|
||||
const helmReleaseStatus = helmStatus?.status || null;
|
||||
|
||||
if (health?.status === 'green') {
|
||||
return { status: 'ready', helmReleaseStatus, health };
|
||||
}
|
||||
|
||||
if (helmReleaseStatus === 'failed' || helmReleaseStatus === 'pending-install') {
|
||||
return { status: 'failed', helmReleaseStatus, health };
|
||||
}
|
||||
|
||||
if (hasEsWorkload || helmReleaseStatus === 'deployed') {
|
||||
return { status: 'installing', helmReleaseStatus, health };
|
||||
}
|
||||
|
||||
return { status: 'installing', helmReleaseStatus, health };
|
||||
}
|
||||
|
||||
private resolveLoggingStorageClass(): string | undefined {
|
||||
const storageClass = this.configService.get<string>('platform.storageClass');
|
||||
return storageClass?.trim() || undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure platform StorageClass exists before Helm creates PVCs (same as app deploy chart).
|
||||
*/
|
||||
private async ensureClusterStorageClass(kubeconfig: string): Promise<void> {
|
||||
const storageClass = this.resolveLoggingStorageClass();
|
||||
const createStorageClass = this.configService.get<boolean>('platform.createStorageClass') === true;
|
||||
if (!storageClass || !createStorageClass) {
|
||||
return;
|
||||
}
|
||||
|
||||
const kc = new k8s.KubeConfig();
|
||||
kc.loadFromString(kubeconfig);
|
||||
const storageApi = kc.makeApiClient(k8s.StorageV1Api);
|
||||
const provisioner =
|
||||
this.configService.get<string>('platform.storageProvisioner') || 'rancher.io/local-path';
|
||||
|
||||
try {
|
||||
await appsApi.readNamespacedStatefulSet(this.ES_NAME, this.ES_NAMESPACE);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
await storageApi.readStorageClass(storageClass);
|
||||
return;
|
||||
} catch (err: any) {
|
||||
if (err.statusCode !== 404 && err.body?.code !== 404) {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
await storageApi.createStorageClass({
|
||||
apiVersion: 'storage.k8s.io/v1',
|
||||
kind: 'StorageClass',
|
||||
metadata: { name: storageClass },
|
||||
provisioner,
|
||||
allowVolumeExpansion: true,
|
||||
reclaimPolicy: 'Delete',
|
||||
volumeBindingMode: 'WaitForFirstConsumer',
|
||||
});
|
||||
this.logger.log(`Created StorageClass "${storageClass}" (provisioner: ${provisioner})`);
|
||||
}
|
||||
|
||||
private buildLoggingHelmValues(): {
|
||||
elasticPassword: string;
|
||||
fluentbitPassword: string;
|
||||
kibanaSystemPassword: string;
|
||||
storageClass?: string;
|
||||
images: {
|
||||
elasticsearch?: string;
|
||||
kibana?: string;
|
||||
busybox?: string;
|
||||
curl?: string;
|
||||
};
|
||||
} {
|
||||
const storageClass = this.resolveLoggingStorageClass();
|
||||
return {
|
||||
elasticPassword: this.ELASTIC_PASSWORD,
|
||||
fluentbitPassword: this.FLUENTBIT_PASSWORD,
|
||||
kibanaSystemPassword: this.KIBANA_SYSTEM_PASSWORD,
|
||||
...(storageClass ? { storageClass } : {}),
|
||||
images: {
|
||||
elasticsearch: this.configService.get<string>('elasticsearch.images.elasticsearch'),
|
||||
kibana: this.configService.get<string>('elasticsearch.images.kibana'),
|
||||
busybox: this.configService.get<string>('elasticsearch.images.busybox'),
|
||||
curl: this.configService.get<string>('elasticsearch.images.curl'),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -432,28 +554,45 @@ export class ElasticsearchService implements OnModuleInit, OnModuleDestroy {
|
||||
}
|
||||
|
||||
/**
|
||||
* Deploy central Elasticsearch + Kibana stack via Helm.
|
||||
* Deploy or repair central Elasticsearch + Kibana stack via Helm.
|
||||
* Always runs upgrade --install; does not wait for pods (avoids timeout errors on slow image pulls).
|
||||
*/
|
||||
async deploy(clusterId?: string): Promise<{ esPassword: string; kibanaUrl: string }> {
|
||||
async deploy(clusterId?: string): Promise<{ esPassword: string; kibanaUrl: string; deploying: boolean }> {
|
||||
const { cluster } = await this.getK8sClients(clusterId);
|
||||
|
||||
await this.helmService.installLoggingStack(cluster.kubeconfig, {
|
||||
elasticPassword: this.ELASTIC_PASSWORD,
|
||||
fluentbitPassword: this.FLUENTBIT_PASSWORD,
|
||||
kibanaSystemPassword: this.KIBANA_SYSTEM_PASSWORD,
|
||||
images: {
|
||||
elasticsearch: this.configService.get<string>('elasticsearch.images.elasticsearch'),
|
||||
kibana: this.configService.get<string>('elasticsearch.images.kibana'),
|
||||
},
|
||||
});
|
||||
await this.ensureClusterStorageClass(cluster.kubeconfig);
|
||||
|
||||
const helmValues = this.buildLoggingHelmValues();
|
||||
|
||||
try {
|
||||
await this.helmService.installLoggingStack(cluster.kubeconfig, helmValues, {
|
||||
wait: false,
|
||||
timeout: '10m',
|
||||
});
|
||||
} catch (error: any) {
|
||||
const release = await this.helmService.status(
|
||||
LOGGING_HELM_RELEASE,
|
||||
LOGGING_HELM_NAMESPACE,
|
||||
cluster.kubeconfig,
|
||||
);
|
||||
if (!release) {
|
||||
throw error;
|
||||
}
|
||||
this.logger.warn(
|
||||
`Helm logging install reported an error but release exists (${release.status}); continuing: ${error.message}`,
|
||||
);
|
||||
}
|
||||
|
||||
this.logger.log(
|
||||
`Central logging stack deployed via Helm (${LOGGING_HELM_RELEASE} in ${LOGGING_HELM_NAMESPACE})`,
|
||||
`Central logging stack applied via Helm (${LOGGING_HELM_RELEASE} in ${LOGGING_HELM_NAMESPACE})`,
|
||||
);
|
||||
|
||||
const state = await this.getDeployState(clusterId);
|
||||
|
||||
return {
|
||||
esPassword: this.ELASTIC_PASSWORD,
|
||||
kibanaUrl: `http://kibana.${this.ES_NAMESPACE}.svc.cluster.local:5601`,
|
||||
deploying: state.status !== 'ready',
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -104,30 +104,39 @@ export class HelmService {
|
||||
elasticPassword: string;
|
||||
fluentbitPassword: string;
|
||||
kibanaSystemPassword: string;
|
||||
images?: { elasticsearch?: string; kibana?: string };
|
||||
storageClass?: string;
|
||||
images?: {
|
||||
elasticsearch?: string;
|
||||
kibana?: string;
|
||||
busybox?: string;
|
||||
curl?: string;
|
||||
};
|
||||
},
|
||||
options: HelmInstallOptions = { wait: false, timeout: '10m' },
|
||||
): Promise<{ stdout: string; stderr: string }> {
|
||||
const chartValues: Record<string, unknown> = {
|
||||
elasticPassword: values.elasticPassword,
|
||||
fluentbitPassword: values.fluentbitPassword,
|
||||
kibanaSystemPassword: values.kibanaSystemPassword,
|
||||
};
|
||||
if (values.storageClass) {
|
||||
chartValues.storageClass = values.storageClass;
|
||||
}
|
||||
if (values.images) {
|
||||
chartValues.images = {
|
||||
...(values.images.elasticsearch ? { elasticsearch: values.images.elasticsearch } : {}),
|
||||
...(values.images.kibana ? { kibana: values.images.kibana } : {}),
|
||||
...(values.images.busybox ? { busybox: values.images.busybox } : {}),
|
||||
...(values.images.curl ? { curl: values.images.curl } : {}),
|
||||
};
|
||||
}
|
||||
return this.installOrUpgradeFromChart(
|
||||
'cloudhost-logging',
|
||||
LOGGING_HELM_RELEASE,
|
||||
LOGGING_HELM_NAMESPACE,
|
||||
{
|
||||
elasticPassword: values.elasticPassword,
|
||||
fluentbitPassword: values.fluentbitPassword,
|
||||
kibanaSystemPassword: values.kibanaSystemPassword,
|
||||
...(values.images?.elasticsearch || values.images?.kibana
|
||||
? {
|
||||
images: {
|
||||
...(values.images.elasticsearch
|
||||
? { elasticsearch: values.images.elasticsearch }
|
||||
: {}),
|
||||
...(values.images.kibana ? { kibana: values.images.kibana } : {}),
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
chartValues,
|
||||
kubeconfig,
|
||||
{ wait: true, timeout: '10m' },
|
||||
options,
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ import { Module, forwardRef } from '@nestjs/common';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
import { KubernetesService } from './kubernetes.service';
|
||||
import { HelmService } from './helm.service';
|
||||
import { RegistryService } from './registry.service';
|
||||
import { ElasticsearchService } from './elasticsearch.service';
|
||||
import { ElasticsearchController } from './elasticsearch.controller';
|
||||
import { LogsController } from './logs.controller';
|
||||
@@ -11,7 +12,7 @@ import { Application } from '../applications/entities/application.entity';
|
||||
@Module({
|
||||
imports: [forwardRef(() => ClustersModule), TypeOrmModule.forFeature([Application])],
|
||||
controllers: [ElasticsearchController, LogsController],
|
||||
providers: [KubernetesService, HelmService, ElasticsearchService],
|
||||
exports: [KubernetesService, HelmService, ElasticsearchService],
|
||||
providers: [KubernetesService, HelmService, RegistryService, ElasticsearchService],
|
||||
exports: [KubernetesService, HelmService, RegistryService, ElasticsearchService],
|
||||
})
|
||||
export class KubernetesModule {}
|
||||
|
||||
@@ -18,6 +18,7 @@ import {
|
||||
isManagedProductType,
|
||||
} from '../common/enums';
|
||||
import { HelmService } from './helm.service';
|
||||
import { RegistryService } from './registry.service';
|
||||
import { registerKubeconfigNoProxy } from '../common/kubernetes-proxy.util';
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
@@ -72,6 +73,7 @@ export class KubernetesService implements OnModuleInit {
|
||||
private configService: ConfigService,
|
||||
private clustersService: ClustersService,
|
||||
private helmService: HelmService,
|
||||
private registryService: RegistryService,
|
||||
) {}
|
||||
|
||||
onModuleInit() {
|
||||
@@ -161,10 +163,45 @@ export class KubernetesService implements OnModuleInit {
|
||||
return { storageClass, createStorageClass, storageProvisioner };
|
||||
}
|
||||
|
||||
/** Create platform StorageClass on the target cluster when configured (Helm chart + K8s API fallback). */
|
||||
private async ensurePlatformStorageClass(kubeconfig: string): Promise<void> {
|
||||
const storageClass = this.configService.get<string>('platform.storageClass')?.trim();
|
||||
const createStorageClass = this.configService.get<boolean>('platform.createStorageClass') === true;
|
||||
if (!storageClass || !createStorageClass) {
|
||||
return;
|
||||
}
|
||||
|
||||
const kc = new k8s.KubeConfig();
|
||||
kc.loadFromString(kubeconfig);
|
||||
const storageApi = kc.makeApiClient(k8s.StorageV1Api);
|
||||
const provisioner =
|
||||
this.configService.get<string>('platform.storageProvisioner') || 'rancher.io/local-path';
|
||||
|
||||
try {
|
||||
await storageApi.readStorageClass(storageClass);
|
||||
return;
|
||||
} catch (err: any) {
|
||||
if (err.statusCode !== 404 && err.body?.code !== 404) {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
await storageApi.createStorageClass({
|
||||
apiVersion: 'storage.k8s.io/v1',
|
||||
kind: 'StorageClass',
|
||||
metadata: { name: storageClass },
|
||||
provisioner,
|
||||
allowVolumeExpansion: true,
|
||||
reclaimPolicy: 'Delete',
|
||||
volumeBindingMode: 'WaitForFirstConsumer',
|
||||
});
|
||||
this.logger.log(`Created StorageClass "${storageClass}" (provisioner: ${provisioner})`);
|
||||
}
|
||||
|
||||
/** Helm values for managed_database / managed_redis / managed_rabbitmq (no app workload). */
|
||||
private buildManagedHelmValues(app: Application): Record<string, any> {
|
||||
const namespace = `user-${app.userId.split('-')[0]}`;
|
||||
const pullRegistryUrl = this.configService.get<string>('registry.pullUrl') || 'localhost:30500';
|
||||
const pullRegistryUrl = this.registryService.getRegistryUrl();
|
||||
const isPostgres = app.databaseType === DatabaseType.POSTGRESQL;
|
||||
const productType = app.productType;
|
||||
|
||||
@@ -244,7 +281,7 @@ export class KubernetesService implements OnModuleInit {
|
||||
|
||||
private buildHelmValues(app: Application, imageUri: string): Record<string, any> {
|
||||
const domain = this.configService.get('platform.domain');
|
||||
const pullRegistryUrl = this.configService.get<string>('registry.pullUrl') || 'localhost:30500';
|
||||
const pullRegistryUrl = this.registryService.getRegistryUrl();
|
||||
const isWordPress = app.runtime === AppRuntime.WORDPRESS;
|
||||
const hasDb = app.databaseType !== DatabaseType.NONE;
|
||||
const isPostgres = app.databaseType === DatabaseType.POSTGRESQL;
|
||||
@@ -332,14 +369,20 @@ export class KubernetesService implements OnModuleInit {
|
||||
if (isManagedProductType(app.productType)) {
|
||||
return this.deployManagedService(app);
|
||||
}
|
||||
|
||||
const workloadImage = this.registryService.normalizeImageReference(imageUri);
|
||||
if (workloadImage !== imageUri) {
|
||||
this.logger.log(`Using in-cluster registry image for ${app.name}: ${workloadImage}`);
|
||||
}
|
||||
|
||||
// Try Helm first, fall back to direct K8s API if Helm is unavailable
|
||||
try {
|
||||
return await this.deployViaHelm(app, imageUri);
|
||||
return await this.deployViaHelm(app, workloadImage);
|
||||
} catch (helmError: any) {
|
||||
this.logger.warn(
|
||||
`Helm deploy failed for ${app.name}, falling back to direct K8s API: ${helmError.message}`,
|
||||
);
|
||||
return await this.deployViaK8sApi(app, imageUri);
|
||||
return await this.deployViaK8sApi(app, workloadImage);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -399,7 +442,7 @@ export class KubernetesService implements OnModuleInit {
|
||||
try {
|
||||
const kubeconfig = await this.getKubeconfig(app.clusterId);
|
||||
const imageUri = app.latestImageTag
|
||||
? `${this.configService.get<string>('registry.pullUrl') || 'localhost:30500'}/${app.name}:${app.latestImageTag}`
|
||||
? this.registryService.normalizeImageReference(app.latestImageTag)
|
||||
: '';
|
||||
const values = this.buildHelmValues(app, imageUri);
|
||||
await this.helmService.installOrUpgrade(app.name, namespace, values, kubeconfig);
|
||||
@@ -446,8 +489,11 @@ export class KubernetesService implements OnModuleInit {
|
||||
|
||||
private async deployViaHelm(app: Application, imageUri: string): Promise<Record<string, any>> {
|
||||
const kubeconfig = await this.getKubeconfig(app.clusterId);
|
||||
await this.ensurePlatformStorageClass(kubeconfig);
|
||||
const { coreApi } = await this.getK8sClient(app.clusterId);
|
||||
const values = this.buildHelmValues(app, imageUri);
|
||||
const namespace = values.app.namespace;
|
||||
const namespace = values.app.namespace as string;
|
||||
await this.registryService.ensureRegistryPullSecret(coreApi, namespace);
|
||||
const releaseName = app.name;
|
||||
|
||||
const result = await this.helmService.installOrUpgrade(
|
||||
@@ -463,6 +509,7 @@ export class KubernetesService implements OnModuleInit {
|
||||
|
||||
private async deployManagedViaHelm(app: Application): Promise<Record<string, any>> {
|
||||
const kubeconfig = await this.getKubeconfig(app.clusterId);
|
||||
await this.ensurePlatformStorageClass(kubeconfig);
|
||||
const values = this.buildManagedHelmValues(app);
|
||||
const namespace = values.app.namespace;
|
||||
const releaseName = app.name;
|
||||
@@ -482,6 +529,8 @@ export class KubernetesService implements OnModuleInit {
|
||||
|
||||
private async deployManagedViaK8sApi(app: Application): Promise<Record<string, any>> {
|
||||
const { coreApi, appsApi } = await this.getK8sClient(app.clusterId);
|
||||
const kubeconfig = await this.getKubeconfig(app.clusterId);
|
||||
await this.ensurePlatformStorageClass(kubeconfig);
|
||||
const namespace = `user-${app.userId.split('-')[0]}`;
|
||||
const context: ManifestContext = {
|
||||
appName: app.name,
|
||||
@@ -546,6 +595,8 @@ export class KubernetesService implements OnModuleInit {
|
||||
return this.deployManagedViaK8sApi(app);
|
||||
}
|
||||
const { coreApi, appsApi, networkingApi } = await this.getK8sClient(app.clusterId);
|
||||
const kubeconfig = await this.getKubeconfig(app.clusterId);
|
||||
await this.ensurePlatformStorageClass(kubeconfig);
|
||||
const domain = this.configService.get('platform.domain');
|
||||
|
||||
const context: ManifestContext = {
|
||||
@@ -585,6 +636,9 @@ export class KubernetesService implements OnModuleInit {
|
||||
// 1. Ensure namespace exists
|
||||
await this.ensureNamespace(coreApi, context.namespace);
|
||||
|
||||
// 1b. Image pull secret for in-cluster registry
|
||||
await this.registryService.ensureRegistryPullSecret(coreApi, context.namespace);
|
||||
|
||||
// 2. Create/Update secrets for env vars
|
||||
if (Object.keys(context.envVars).length > 0) {
|
||||
manifests.secret = await this.applySecret(coreApi, context);
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import * as k8s from '@kubernetes/client-node';
|
||||
|
||||
export interface ParsedImageReference {
|
||||
repository: string;
|
||||
tag: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* In-cluster Docker Registry v2 (ClusterIP).
|
||||
* Kaniko pushes and application workloads pull from the same registry URL.
|
||||
*/
|
||||
@Injectable()
|
||||
export class RegistryService {
|
||||
private readonly logger = new Logger(RegistryService.name);
|
||||
|
||||
constructor(private readonly configService: ConfigService) {}
|
||||
|
||||
getBuildNamespace(): string {
|
||||
return this.configService.get<string>('build.namespace') || 'cloudhost-builds';
|
||||
}
|
||||
|
||||
/** Registry host:port used for build push and app image pull. */
|
||||
getRegistryUrl(): string {
|
||||
const buildNs = this.getBuildNamespace();
|
||||
const url =
|
||||
this.configService.get<string>('registry.pullUrl') ||
|
||||
this.configService.get<string>('registry.url') ||
|
||||
`registry.${buildNs}.svc.cluster.local:5000`;
|
||||
return url.replace(/^https?:\/\//, '');
|
||||
}
|
||||
|
||||
getRegistryCredentials(): { username: string; password: string } {
|
||||
return {
|
||||
username: this.configService.get<string>('registry.username') || 'admin',
|
||||
password: this.configService.get<string>('registry.password') || '',
|
||||
};
|
||||
}
|
||||
|
||||
buildImageReference(userId: string, appName: string, tag: string): string {
|
||||
return `${this.getRegistryUrl()}/${userId}/${appName}:${tag}`;
|
||||
}
|
||||
|
||||
parseImageReference(imageRef: string): ParsedImageReference {
|
||||
const normalized = imageRef.replace(/^https?:\/\//, '');
|
||||
const slashIdx = normalized.indexOf('/');
|
||||
if (slashIdx === -1) {
|
||||
throw new Error(`Invalid image reference (missing repository path): ${imageRef}`);
|
||||
}
|
||||
const rest = normalized.slice(slashIdx + 1);
|
||||
const tagIdx = rest.lastIndexOf(':');
|
||||
if (tagIdx === -1) {
|
||||
throw new Error(`Invalid image reference (missing tag): ${imageRef}`);
|
||||
}
|
||||
return {
|
||||
repository: rest.slice(0, tagIdx),
|
||||
tag: rest.slice(tagIdx + 1),
|
||||
};
|
||||
}
|
||||
|
||||
/** Re-point any stored image (e.g. legacy external host) to the in-cluster registry. */
|
||||
normalizeImageReference(imageRef: string): string {
|
||||
const { repository, tag } = this.parseImageReference(imageRef);
|
||||
return `${this.getRegistryUrl()}/${repository}:${tag}`;
|
||||
}
|
||||
|
||||
buildDockerConfigJson(): string {
|
||||
const { username, password } = this.getRegistryCredentials();
|
||||
const auth =
|
||||
username && password
|
||||
? Buffer.from(`${username}:${password}`).toString('base64')
|
||||
: '';
|
||||
const host = this.getRegistryUrl();
|
||||
return JSON.stringify({
|
||||
auths: {
|
||||
[host]: { auth },
|
||||
[`registry.${this.getBuildNamespace()}.svc.cluster.local:5000`]: { auth },
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async ensureRegistryPullSecret(coreApi: k8s.CoreV1Api, namespace: string): Promise<void> {
|
||||
const secretName = 'registry-pull-secret';
|
||||
const secret: k8s.V1Secret = {
|
||||
apiVersion: 'v1',
|
||||
kind: 'Secret',
|
||||
metadata: {
|
||||
name: secretName,
|
||||
namespace,
|
||||
labels: { 'app.kubernetes.io/managed-by': 'cloudhost' },
|
||||
},
|
||||
type: 'kubernetes.io/dockerconfigjson',
|
||||
data: {
|
||||
'.dockerconfigjson': Buffer.from(this.buildDockerConfigJson()).toString('base64'),
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
await coreApi.replaceNamespacedSecret(secretName, namespace, secret);
|
||||
} catch (err: any) {
|
||||
if (err.statusCode === 404 || err.body?.code === 404) {
|
||||
await coreApi.createNamespacedSecret(namespace, secret);
|
||||
this.logger.log(`Created ${secretName} in ${namespace}`);
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user