diff --git a/backend/.env.example b/backend/.env.example index 18c9050..875de11 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -2,6 +2,7 @@ NODE_ENV=development PORT=4000 + # Database DB_HOST=localhost DB_PORT=5432 @@ -19,12 +20,9 @@ JWT_REFRESH_EXPIRES_IN=7d REDIS_HOST=localhost REDIS_PORT=6379 -# Container Registry (same Docker Registry v2, two hostnames) -# Internal — Kaniko/build pods push here (ClusterIP, HTTP, fast) +# In-cluster Docker Registry (Kaniko push + app image pull — same URL) REGISTRY_URL=registry.cloudhost-builds.svc.cluster.local:5000 -# External — kubelet pulls app images; also use for manual "docker push" (Ingress or NodePort) -REGISTRY_PULL_URL=repo.3fase.ir -# REGISTRY_PULL_URL=10.0.0.50:30500 +# REGISTRY_PULL_URL=registry.cloudhost-builds.svc.cluster.local:5000 REGISTRY_USERNAME=admin REGISTRY_PASSWORD=registry_secret @@ -47,6 +45,9 @@ BUILD_SERVICE_ACCOUNT=kaniko-builder PLATFORM_DOMAIN=apps.cloudhost.local UPLOAD_DIR=./uploads # PVC resize: use a dynamic StorageClass with allowVolumeExpansion (k3s: rancher.io/local-path) +# k3s: use local-path and skip creating a custom class (set CREATE=false) +# PLATFORM_STORAGE_CLASS=local-path +# PLATFORM_CREATE_STORAGE_CLASS=false PLATFORM_STORAGE_CLASS=cloudhost-expandable PLATFORM_CREATE_STORAGE_CLASS=true PLATFORM_STORAGE_PROVISIONER=rancher.io/local-path diff --git a/backend/helm/cloudhost-logging/templates/elasticsearch-pvc.yaml b/backend/helm/cloudhost-logging/templates/elasticsearch-pvc.yaml index 5320963..b86c9f5 100644 --- a/backend/helm/cloudhost-logging/templates/elasticsearch-pvc.yaml +++ b/backend/helm/cloudhost-logging/templates/elasticsearch-pvc.yaml @@ -12,6 +12,9 @@ metadata: spec: accessModes: - ReadWriteOnce + {{- if .Values.storageClass }} + storageClassName: {{ .Values.storageClass | quote }} + {{- end }} resources: requests: storage: {{ .Values.storage }} diff --git a/backend/helm/cloudhost-logging/templates/namespace.yaml b/backend/helm/cloudhost-logging/templates/namespace.yaml deleted file mode 100644 index 28477f8..0000000 --- a/backend/helm/cloudhost-logging/templates/namespace.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: {{ include "cloudhost-logging.namespace" . }} - labels: - app.kubernetes.io/managed-by: cloudhost - app.kubernetes.io/instance: {{ .Release.Name }} diff --git a/backend/helm/cloudhost-logging/values.yaml b/backend/helm/cloudhost-logging/values.yaml index 9b93d3c..e7dadf1 100644 --- a/backend/helm/cloudhost-logging/values.yaml +++ b/backend/helm/cloudhost-logging/values.yaml @@ -7,6 +7,8 @@ kibanaSystemPassword: "" clusterName: cloudhost-logs storage: 50Gi +# Leave empty to use the cluster default StorageClass (e.g. local-path on k3s). +storageClass: "" # Official Elastic images; require docker.elastic.co DNS + outbound HTTPS from nodes. # If pull fails with "lookup docker.elastic.co: Try again", mirror to your registry and override here. diff --git a/backend/helm/cloudhost-platform/values-production.example.yaml b/backend/helm/cloudhost-platform/values-production.example.yaml index 8d33436..ff96e1e 100644 --- a/backend/helm/cloudhost-platform/values-production.example.yaml +++ b/backend/helm/cloudhost-platform/values-production.example.yaml @@ -38,7 +38,8 @@ ingress: backend: env: PLATFORM_DOMAIN: apps.example.com - REGISTRY_PULL_URL: "10.0.0.50:30500" + REGISTRY_URL: registry.cloudhost-builds.svc.cluster.local:5000 + REGISTRY_PULL_URL: registry.cloudhost-builds.svc.cluster.local:5000 migrations: enabled: true diff --git a/backend/src/build/build.service.ts b/backend/src/build/build.service.ts index 5c61858..176ea4d 100644 --- a/backend/src/build/build.service.ts +++ b/backend/src/build/build.service.ts @@ -9,6 +9,7 @@ import { promisify } from 'util'; import { Application } from '../applications/entities/application.entity'; import { AppRuntime } from '../common/enums'; import { ClustersService } from '../clusters/clusters.service'; +import { RegistryService } from '../kubernetes/registry.service'; const execFileAsync = promisify(execFile); @@ -48,6 +49,7 @@ export class BuildService { constructor( private configService: ConfigService, private clustersService: ClustersService, + private registryService: RegistryService, ) {} private beginBuildSession(deploymentId: string): void { @@ -213,16 +215,12 @@ export class BuildService { * Returns { imageUri, buildLog } — the full image URI and the build logs. */ async buildImage(app: Application, deploymentId?: string): Promise<{ imageUri: string; buildLog: string }> { - // Internal registry (used by Kaniko inside K8s for pushing) - const internalRegistryUrl = this.configService.get('registry.url') || 'registry.cloudhost-builds.svc.cluster.local:5000'; - // External registry URL (used by kubelet for pulling — NodePort or external) - const pullRegistryUrl = this.configService.get('registry.pullUrl') || 'localhost:30500'; - const buildNamespace = this.configService.get('build.namespace') || 'cloudhost-builds'; + const registryUrl = this.registryService.getRegistryUrl(); + const buildNamespace = this.registryService.getBuildNamespace(); const tag = `${Date.now()}`; - const pushImageUri = `${internalRegistryUrl}/${app.userId}/${app.name}:${tag}`; - const pullImageUri = `${pullRegistryUrl}/${app.userId}/${app.name}:${tag}`; + const imageUri = this.registryService.buildImageReference(app.userId, app.name, tag); - this.logger.log(`Starting image build for ${app.name} → push: ${pushImageUri}, pull: ${pullImageUri}`); + this.logger.log(`Starting image build for ${app.name} → ${imageUri}`); if (deploymentId) { this.beginBuildSession(deploymentId); @@ -295,9 +293,9 @@ export class BuildService { const kanikoArgs = [ '--dockerfile=/workspace/Dockerfile', '--context=dir:///workspace/source', - `--destination=${pushImageUri}`, + `--destination=${imageUri}`, '--cache=true', - `--cache-repo=${internalRegistryUrl}/${app.userId}/cache`, + `--cache-repo=${registryUrl}/${app.userId}/cache`, '--insecure', '--skip-tls-verify', '--single-snapshot', @@ -484,8 +482,8 @@ export class BuildService { buildLog = await this.getBuildLogs(coreApi, buildPodName, buildNamespace!); } catch {} - this.logger.log(`Build completed successfully: ${pullImageUri}`); - return { imageUri: pullImageUri, buildLog }; + this.logger.log(`Build completed successfully: ${imageUri}`); + return { imageUri, buildLog }; } catch (error: any) { if (error instanceof BuildCancelledError || error?.name === 'BuildCancelledError') { throw error; @@ -826,18 +824,11 @@ export class BuildService { } catch (err: any) { if (err.statusCode === 404 || err.body?.code === 404) { this.logger.log(`Secret "${registrySecretName}" not found in "${namespace}" — creating it`); - const registryUrl = this.configService.get('registry.url') || 'registry.cloudhost-builds.svc.cluster.local:5000'; - // Create a docker config that allows insecure push (for internal registry) - const dockerConfig = JSON.stringify({ - auths: { - [registryUrl]: { auth: '' }, - }, - }); await coreApi.createNamespacedSecret(namespace, { metadata: { name: registrySecretName, namespace }, type: 'kubernetes.io/dockerconfigjson', data: { - '.dockerconfigjson': Buffer.from(dockerConfig).toString('base64'), + '.dockerconfigjson': Buffer.from(this.registryService.buildDockerConfigJson()).toString('base64'), }, }); } else { diff --git a/backend/src/clusters/clusters.service.ts b/backend/src/clusters/clusters.service.ts index 1253056..f713a78 100644 --- a/backend/src/clusters/clusters.service.ts +++ b/backend/src/clusters/clusters.service.ts @@ -21,6 +21,7 @@ import { CreateClusterDto, UpdateClusterDto } from './dto/cluster.dto'; import { CreateClusterPoolDto, UpdateClusterPoolDto } from './dto/cluster-pool.dto'; import { ClusterStatus } from '../common/enums'; import { ElasticsearchService } from '../kubernetes/elasticsearch.service'; +import { RegistryService } from '../kubernetes/registry.service'; import { CreateApplicationDto } from '../applications/dto/application.dto'; import { registerKubeconfigNoProxy } from '../common/kubernetes-proxy.util'; @@ -54,6 +55,8 @@ export class ClustersService implements OnModuleInit, OnModuleDestroy { private configService: ConfigService, @Inject(forwardRef(() => ElasticsearchService)) private elasticsearchService: ElasticsearchService, + @Inject(forwardRef(() => RegistryService)) + private registryService: RegistryService, ) {} onModuleInit(): void { @@ -835,9 +838,9 @@ export class ClustersService implements OnModuleInit, OnModuleDestroy { const coreApi = kc.makeApiClient(k8s.CoreV1Api); const appsApi = kc.makeApiClient(k8s.AppsV1Api); - const buildNs = this.configService.get('build.namespace') || 'cloudhost-builds'; + const buildNs = this.registryService.getBuildNamespace(); const saName = this.configService.get('build.serviceAccount') || 'kaniko-builder'; - const registryUrl = this.configService.get('registry.url') || `registry.${buildNs}.svc.cluster.local:5000`; + const registryUrl = this.registryService.getRegistryUrl(); this.logger.log(`Bootstrapping cluster — namespace: ${buildNs}`); @@ -908,9 +911,7 @@ export class ClustersService implements OnModuleInit, OnModuleDestroy { name: 'registry', image: 'registry:2', ports: [{ containerPort: 5000 }], - env: [ - { name: 'REGISTRY_STORAGE_DELETE_ENABLED', value: 'true' }, - ], + env: [{ name: 'REGISTRY_STORAGE_DELETE_ENABLED', value: 'true' }], volumeMounts: [{ name: 'registry-data', mountPath: '/var/lib/registry', @@ -934,7 +935,7 @@ export class ClustersService implements OnModuleInit, OnModuleDestroy { } } - // ── 5. Registry ClusterIP Service (for Kaniko to push) ──────── + // ── 5. Registry ClusterIP Service (Kaniko push + app pull) ─── const registrySvcName = 'registry'; try { await coreApi.readNamespacedService(registrySvcName, buildNs); @@ -955,7 +956,8 @@ export class ClustersService implements OnModuleInit, OnModuleDestroy { } } - // ── 6. Registry NodePort Service (for kubelet to pull) ──────── + // ── 6. Registry NodePort Service (optional host access :30500) ─ + const registryNodePort = 30500; const registryNodePortName = 'registry-nodeport'; try { await coreApi.readNamespacedService(registryNodePortName, buildNs); @@ -967,10 +969,15 @@ export class ClustersService implements OnModuleInit, OnModuleDestroy { spec: { type: 'NodePort', selector: { app: 'registry' }, - ports: [{ port: 5000, targetPort: 5000 as any, nodePort: 30500, protocol: 'TCP' }], + ports: [{ + port: 5000, + targetPort: 5000 as any, + nodePort: registryNodePort, + protocol: 'TCP', + }], }, }); - this.logger.log(`Created Registry NodePort Service (30500 → 5000)`); + this.logger.log(`Created Registry NodePort Service (${registryNodePort} → 5000)`); } else { throw err; } @@ -978,33 +985,27 @@ export class ClustersService implements OnModuleInit, OnModuleDestroy { // ── 7. registry-credentials Secret (docker config for Kaniko) ─ const registrySecretName = 'registry-credentials'; + const dockerConfig = this.registryService.buildDockerConfigJson(); + const kanikoRegistrySecret: k8s.V1Secret = { + metadata: { name: registrySecretName, namespace: buildNs }, + type: 'kubernetes.io/dockerconfigjson', + data: { + '.dockerconfigjson': Buffer.from(dockerConfig).toString('base64'), + }, + }; try { await coreApi.readNamespacedSecret(registrySecretName, buildNs); - this.logger.log(`Secret "${registrySecretName}" already exists`); + await coreApi.replaceNamespacedSecret(registrySecretName, buildNs, kanikoRegistrySecret); } catch (err: any) { if (err.statusCode === 404 || err.body?.code === 404) { - // Parse registry host (without port path) for the docker config - const dockerConfig = JSON.stringify({ - auths: { - [registryUrl]: { auth: '' }, - [`registry.${buildNs}.svc.cluster.local:5000`]: { auth: '' }, - 'localhost:30500': { auth: '' }, - }, - }); - await coreApi.createNamespacedSecret(buildNs, { - metadata: { name: registrySecretName, namespace: buildNs }, - type: 'kubernetes.io/dockerconfigjson', - data: { - '.dockerconfigjson': Buffer.from(dockerConfig).toString('base64'), - }, - }); + await coreApi.createNamespacedSecret(buildNs, kanikoRegistrySecret); this.logger.log(`Created registry-credentials Secret`); } else { throw err; } } - this.logger.log(`✅ Cluster bootstrap complete — build infrastructure ready`); + this.logger.log(`✅ Cluster bootstrap complete — registry: ${registryUrl}`); } private parseCpuToMillicores(cpu: string): number { diff --git a/backend/src/config/configuration.ts b/backend/src/config/configuration.ts index 9c0c904..9b7c825 100644 --- a/backend/src/config/configuration.ts +++ b/backend/src/config/configuration.ts @@ -23,8 +23,9 @@ export default () => ({ }, registry: { + /** In-cluster registry — Kaniko push and app image pull (same host). */ url: process.env.REGISTRY_URL || 'registry.cloudhost-builds.svc.cluster.local:5000', - pullUrl: process.env.REGISTRY_PULL_URL || process.env.REGISTRY_URL || 'localhost:30500', + pullUrl: process.env.REGISTRY_PULL_URL || process.env.REGISTRY_URL || 'registry.cloudhost-builds.svc.cluster.local:5000', username: process.env.REGISTRY_USERNAME || 'admin', password: process.env.REGISTRY_PASSWORD || '', }, @@ -49,6 +50,8 @@ export default () => ({ process.env.LOGGING_ELASTICSEARCH_IMAGE || 'docker.elastic.co/elasticsearch/elasticsearch:8.12.0', kibana: process.env.LOGGING_KIBANA_IMAGE || 'docker.elastic.co/kibana/kibana:8.12.0', + busybox: process.env.LOGGING_BUSYBOX_IMAGE || 'busybox:1.36', + curl: process.env.LOGGING_CURL_IMAGE || 'curlimages/curl:8.5.0', }, }, diff --git a/backend/src/kubernetes/elasticsearch.controller.ts b/backend/src/kubernetes/elasticsearch.controller.ts index 02e0817..f57d931 100644 --- a/backend/src/kubernetes/elasticsearch.controller.ts +++ b/backend/src/kubernetes/elasticsearch.controller.ts @@ -19,13 +19,14 @@ export class ElasticsearchController { @ApiQuery({ name: 'clusterId', required: false, description: 'Target cluster ID' }) @ApiResponse({ status: 200, description: 'Elasticsearch status' }) async getStatus(@Query('clusterId') clusterId?: string) { - const isDeployed = await this.esService.isDeployed(clusterId); - const health = isDeployed ? await this.esService.getHealth(clusterId) : null; + const state = await this.esService.getDeployState(clusterId); const connectionInfo = this.esService.getConnectionInfo(); - + return { - deployed: isDeployed, - health, + deployed: state.status === 'ready', + deployStatus: state.status, + helmReleaseStatus: state.helmReleaseStatus, + health: state.health, namespace: 'logging', elasticsearch: { host: connectionInfo.host, @@ -41,28 +42,38 @@ export class ElasticsearchController { } @Post('deploy') - @ApiOperation({ summary: 'Deploy central Elasticsearch + Kibana stack' }) + @ApiOperation({ summary: 'Deploy or repair central Elasticsearch + Kibana stack' }) @ApiQuery({ name: 'clusterId', required: false, description: 'Target cluster ID' }) @ApiResponse({ status: 201, description: 'Elasticsearch deployed successfully' }) async deploy(@Query('clusterId') clusterId?: string) { - const alreadyDeployed = await this.esService.isDeployed(clusterId); - - if (alreadyDeployed) { - const health = await this.esService.getHealth(clusterId); + const before = await this.esService.getDeployState(clusterId); + + if (before.status === 'ready') { return { success: true, - message: 'Elasticsearch is already deployed', + message: 'Elasticsearch is already deployed and healthy', status: 'existing', - health, + deployStatus: before.status, + health: before.health, }; } const result = await this.esService.deploy(clusterId); - + const after = await this.esService.getDeployState(clusterId); + + const status = + before.status === 'not_installed' + ? 'created' + : 'repaired'; + return { success: true, - message: 'Elasticsearch and Kibana deployed successfully. Please wait 2-3 minutes for pods to be ready.', - status: 'created', + message: result.deploying + ? 'Logging stack installed. Pods are starting — allow 2–5 minutes for Elasticsearch and Kibana to become ready.' + : 'Elasticsearch and Kibana are ready.', + status, + deployStatus: after.status, + health: after.health, credentials: { username: 'elastic', password: result.esPassword, @@ -87,7 +98,7 @@ export class ElasticsearchController { @ApiResponse({ status: 200, description: 'Elasticsearch undeployed' }) async undeploy(@Query('clusterId') clusterId?: string) { await this.esService.undeploy(clusterId); - + return { success: true, message: 'Elasticsearch stack removed. PVC with data is preserved.', @@ -101,7 +112,7 @@ export class ElasticsearchController { async getCredentials() { const creds = this.esService.getFluentBitCredentials(); const connInfo = this.esService.getConnectionInfo(); - + return { elasticsearch: { host: connInfo.host, diff --git a/backend/src/kubernetes/elasticsearch.service.ts b/backend/src/kubernetes/elasticsearch.service.ts index 8c01519..c4265ff 100644 --- a/backend/src/kubernetes/elasticsearch.service.ts +++ b/backend/src/kubernetes/elasticsearch.service.ts @@ -59,6 +59,21 @@ export interface LogStatsResult { period: string; } +export type LoggingDeployStatus = 'not_installed' | 'installing' | 'failed' | 'ready'; + +export interface LoggingDeployState { + status: LoggingDeployStatus; + helmReleaseStatus?: string | null; + health: { + status: string; + clusterName: string; + numberOfNodes: number; + activePrimaryShards: number; + activeShards: number; + kibanaReady: boolean; + } | null; +} + /** * Central Elasticsearch management service. * Deploys a shared Elasticsearch + Kibana stack in a dedicated namespace @@ -358,17 +373,124 @@ export class ElasticsearchService implements OnModuleInit, OnModuleDestroy { } /** - * Check if central Elasticsearch is deployed in a cluster + * Check if central Elasticsearch is deployed and healthy in a cluster. */ async isDeployed(clusterId?: string): Promise { - const { appsApi } = await this.getK8sClients(clusterId); + const state = await this.getDeployState(clusterId); + return state.status === 'ready'; + } + + /** + * Detailed logging stack state (Helm release + pod readiness). + */ + async getDeployState(clusterId?: string): Promise { + const { cluster } = await this.getK8sClients(clusterId); + const helmStatus = await this.helmService.status( + LOGGING_HELM_RELEASE, + LOGGING_HELM_NAMESPACE, + cluster.kubeconfig, + ); + + let hasEsWorkload = false; + try { + const kc = new k8s.KubeConfig(); + kc.loadFromString(cluster.kubeconfig); + const appsApi = kc.makeApiClient(k8s.AppsV1Api); + await appsApi.readNamespacedStatefulSet(this.ES_NAME, this.ES_NAMESPACE); + hasEsWorkload = true; + } catch { + hasEsWorkload = false; + } + + if (!helmStatus && !hasEsWorkload) { + return { status: 'not_installed', helmReleaseStatus: null, health: null }; + } + + const health = await this.getHealth(clusterId); + const helmReleaseStatus = helmStatus?.status || null; + + if (health?.status === 'green') { + return { status: 'ready', helmReleaseStatus, health }; + } + + if (helmReleaseStatus === 'failed' || helmReleaseStatus === 'pending-install') { + return { status: 'failed', helmReleaseStatus, health }; + } + + if (hasEsWorkload || helmReleaseStatus === 'deployed') { + return { status: 'installing', helmReleaseStatus, health }; + } + + return { status: 'installing', helmReleaseStatus, health }; + } + + private resolveLoggingStorageClass(): string | undefined { + const storageClass = this.configService.get('platform.storageClass'); + return storageClass?.trim() || undefined; + } + + /** + * Ensure platform StorageClass exists before Helm creates PVCs (same as app deploy chart). + */ + private async ensureClusterStorageClass(kubeconfig: string): Promise { + const storageClass = this.resolveLoggingStorageClass(); + const createStorageClass = this.configService.get('platform.createStorageClass') === true; + if (!storageClass || !createStorageClass) { + return; + } + + const kc = new k8s.KubeConfig(); + kc.loadFromString(kubeconfig); + const storageApi = kc.makeApiClient(k8s.StorageV1Api); + const provisioner = + this.configService.get('platform.storageProvisioner') || 'rancher.io/local-path'; try { - await appsApi.readNamespacedStatefulSet(this.ES_NAME, this.ES_NAMESPACE); - return true; - } catch { - return false; + await storageApi.readStorageClass(storageClass); + return; + } catch (err: any) { + if (err.statusCode !== 404 && err.body?.code !== 404) { + throw err; + } } + + await storageApi.createStorageClass({ + apiVersion: 'storage.k8s.io/v1', + kind: 'StorageClass', + metadata: { name: storageClass }, + provisioner, + allowVolumeExpansion: true, + reclaimPolicy: 'Delete', + volumeBindingMode: 'WaitForFirstConsumer', + }); + this.logger.log(`Created StorageClass "${storageClass}" (provisioner: ${provisioner})`); + } + + private buildLoggingHelmValues(): { + elasticPassword: string; + fluentbitPassword: string; + kibanaSystemPassword: string; + storageClass?: string; + images: { + elasticsearch?: string; + kibana?: string; + busybox?: string; + curl?: string; + }; + } { + const storageClass = this.resolveLoggingStorageClass(); + return { + elasticPassword: this.ELASTIC_PASSWORD, + fluentbitPassword: this.FLUENTBIT_PASSWORD, + kibanaSystemPassword: this.KIBANA_SYSTEM_PASSWORD, + ...(storageClass ? { storageClass } : {}), + images: { + elasticsearch: this.configService.get('elasticsearch.images.elasticsearch'), + kibana: this.configService.get('elasticsearch.images.kibana'), + busybox: this.configService.get('elasticsearch.images.busybox'), + curl: this.configService.get('elasticsearch.images.curl'), + }, + }; } /** @@ -432,28 +554,45 @@ export class ElasticsearchService implements OnModuleInit, OnModuleDestroy { } /** - * Deploy central Elasticsearch + Kibana stack via Helm. + * Deploy or repair central Elasticsearch + Kibana stack via Helm. + * Always runs upgrade --install; does not wait for pods (avoids timeout errors on slow image pulls). */ - async deploy(clusterId?: string): Promise<{ esPassword: string; kibanaUrl: string }> { + async deploy(clusterId?: string): Promise<{ esPassword: string; kibanaUrl: string; deploying: boolean }> { const { cluster } = await this.getK8sClients(clusterId); - await this.helmService.installLoggingStack(cluster.kubeconfig, { - elasticPassword: this.ELASTIC_PASSWORD, - fluentbitPassword: this.FLUENTBIT_PASSWORD, - kibanaSystemPassword: this.KIBANA_SYSTEM_PASSWORD, - images: { - elasticsearch: this.configService.get('elasticsearch.images.elasticsearch'), - kibana: this.configService.get('elasticsearch.images.kibana'), - }, - }); + await this.ensureClusterStorageClass(cluster.kubeconfig); + + const helmValues = this.buildLoggingHelmValues(); + + try { + await this.helmService.installLoggingStack(cluster.kubeconfig, helmValues, { + wait: false, + timeout: '10m', + }); + } catch (error: any) { + const release = await this.helmService.status( + LOGGING_HELM_RELEASE, + LOGGING_HELM_NAMESPACE, + cluster.kubeconfig, + ); + if (!release) { + throw error; + } + this.logger.warn( + `Helm logging install reported an error but release exists (${release.status}); continuing: ${error.message}`, + ); + } this.logger.log( - `Central logging stack deployed via Helm (${LOGGING_HELM_RELEASE} in ${LOGGING_HELM_NAMESPACE})`, + `Central logging stack applied via Helm (${LOGGING_HELM_RELEASE} in ${LOGGING_HELM_NAMESPACE})`, ); + const state = await this.getDeployState(clusterId); + return { esPassword: this.ELASTIC_PASSWORD, kibanaUrl: `http://kibana.${this.ES_NAMESPACE}.svc.cluster.local:5601`, + deploying: state.status !== 'ready', }; } diff --git a/backend/src/kubernetes/helm.service.ts b/backend/src/kubernetes/helm.service.ts index f87eaf7..23cddf2 100644 --- a/backend/src/kubernetes/helm.service.ts +++ b/backend/src/kubernetes/helm.service.ts @@ -104,30 +104,39 @@ export class HelmService { elasticPassword: string; fluentbitPassword: string; kibanaSystemPassword: string; - images?: { elasticsearch?: string; kibana?: string }; + storageClass?: string; + images?: { + elasticsearch?: string; + kibana?: string; + busybox?: string; + curl?: string; + }; }, + options: HelmInstallOptions = { wait: false, timeout: '10m' }, ): Promise<{ stdout: string; stderr: string }> { + const chartValues: Record = { + elasticPassword: values.elasticPassword, + fluentbitPassword: values.fluentbitPassword, + kibanaSystemPassword: values.kibanaSystemPassword, + }; + if (values.storageClass) { + chartValues.storageClass = values.storageClass; + } + if (values.images) { + chartValues.images = { + ...(values.images.elasticsearch ? { elasticsearch: values.images.elasticsearch } : {}), + ...(values.images.kibana ? { kibana: values.images.kibana } : {}), + ...(values.images.busybox ? { busybox: values.images.busybox } : {}), + ...(values.images.curl ? { curl: values.images.curl } : {}), + }; + } return this.installOrUpgradeFromChart( 'cloudhost-logging', LOGGING_HELM_RELEASE, LOGGING_HELM_NAMESPACE, - { - elasticPassword: values.elasticPassword, - fluentbitPassword: values.fluentbitPassword, - kibanaSystemPassword: values.kibanaSystemPassword, - ...(values.images?.elasticsearch || values.images?.kibana - ? { - images: { - ...(values.images.elasticsearch - ? { elasticsearch: values.images.elasticsearch } - : {}), - ...(values.images.kibana ? { kibana: values.images.kibana } : {}), - }, - } - : {}), - }, + chartValues, kubeconfig, - { wait: true, timeout: '10m' }, + options, ); } diff --git a/backend/src/kubernetes/kubernetes.module.ts b/backend/src/kubernetes/kubernetes.module.ts index 2f67e1b..8f04e60 100644 --- a/backend/src/kubernetes/kubernetes.module.ts +++ b/backend/src/kubernetes/kubernetes.module.ts @@ -2,6 +2,7 @@ import { Module, forwardRef } from '@nestjs/common'; import { TypeOrmModule } from '@nestjs/typeorm'; import { KubernetesService } from './kubernetes.service'; import { HelmService } from './helm.service'; +import { RegistryService } from './registry.service'; import { ElasticsearchService } from './elasticsearch.service'; import { ElasticsearchController } from './elasticsearch.controller'; import { LogsController } from './logs.controller'; @@ -11,7 +12,7 @@ import { Application } from '../applications/entities/application.entity'; @Module({ imports: [forwardRef(() => ClustersModule), TypeOrmModule.forFeature([Application])], controllers: [ElasticsearchController, LogsController], - providers: [KubernetesService, HelmService, ElasticsearchService], - exports: [KubernetesService, HelmService, ElasticsearchService], + providers: [KubernetesService, HelmService, RegistryService, ElasticsearchService], + exports: [KubernetesService, HelmService, RegistryService, ElasticsearchService], }) export class KubernetesModule {} diff --git a/backend/src/kubernetes/kubernetes.service.ts b/backend/src/kubernetes/kubernetes.service.ts index 97177d7..c4f9a99 100644 --- a/backend/src/kubernetes/kubernetes.service.ts +++ b/backend/src/kubernetes/kubernetes.service.ts @@ -18,6 +18,7 @@ import { isManagedProductType, } from '../common/enums'; import { HelmService } from './helm.service'; +import { RegistryService } from './registry.service'; import { registerKubeconfigNoProxy } from '../common/kubernetes-proxy.util'; const execFileAsync = promisify(execFile); @@ -72,6 +73,7 @@ export class KubernetesService implements OnModuleInit { private configService: ConfigService, private clustersService: ClustersService, private helmService: HelmService, + private registryService: RegistryService, ) {} onModuleInit() { @@ -161,10 +163,45 @@ export class KubernetesService implements OnModuleInit { return { storageClass, createStorageClass, storageProvisioner }; } + /** Create platform StorageClass on the target cluster when configured (Helm chart + K8s API fallback). */ + private async ensurePlatformStorageClass(kubeconfig: string): Promise { + const storageClass = this.configService.get('platform.storageClass')?.trim(); + const createStorageClass = this.configService.get('platform.createStorageClass') === true; + if (!storageClass || !createStorageClass) { + return; + } + + const kc = new k8s.KubeConfig(); + kc.loadFromString(kubeconfig); + const storageApi = kc.makeApiClient(k8s.StorageV1Api); + const provisioner = + this.configService.get('platform.storageProvisioner') || 'rancher.io/local-path'; + + try { + await storageApi.readStorageClass(storageClass); + return; + } catch (err: any) { + if (err.statusCode !== 404 && err.body?.code !== 404) { + throw err; + } + } + + await storageApi.createStorageClass({ + apiVersion: 'storage.k8s.io/v1', + kind: 'StorageClass', + metadata: { name: storageClass }, + provisioner, + allowVolumeExpansion: true, + reclaimPolicy: 'Delete', + volumeBindingMode: 'WaitForFirstConsumer', + }); + this.logger.log(`Created StorageClass "${storageClass}" (provisioner: ${provisioner})`); + } + /** Helm values for managed_database / managed_redis / managed_rabbitmq (no app workload). */ private buildManagedHelmValues(app: Application): Record { const namespace = `user-${app.userId.split('-')[0]}`; - const pullRegistryUrl = this.configService.get('registry.pullUrl') || 'localhost:30500'; + const pullRegistryUrl = this.registryService.getRegistryUrl(); const isPostgres = app.databaseType === DatabaseType.POSTGRESQL; const productType = app.productType; @@ -244,7 +281,7 @@ export class KubernetesService implements OnModuleInit { private buildHelmValues(app: Application, imageUri: string): Record { const domain = this.configService.get('platform.domain'); - const pullRegistryUrl = this.configService.get('registry.pullUrl') || 'localhost:30500'; + const pullRegistryUrl = this.registryService.getRegistryUrl(); const isWordPress = app.runtime === AppRuntime.WORDPRESS; const hasDb = app.databaseType !== DatabaseType.NONE; const isPostgres = app.databaseType === DatabaseType.POSTGRESQL; @@ -332,14 +369,20 @@ export class KubernetesService implements OnModuleInit { if (isManagedProductType(app.productType)) { return this.deployManagedService(app); } + + const workloadImage = this.registryService.normalizeImageReference(imageUri); + if (workloadImage !== imageUri) { + this.logger.log(`Using in-cluster registry image for ${app.name}: ${workloadImage}`); + } + // Try Helm first, fall back to direct K8s API if Helm is unavailable try { - return await this.deployViaHelm(app, imageUri); + return await this.deployViaHelm(app, workloadImage); } catch (helmError: any) { this.logger.warn( `Helm deploy failed for ${app.name}, falling back to direct K8s API: ${helmError.message}`, ); - return await this.deployViaK8sApi(app, imageUri); + return await this.deployViaK8sApi(app, workloadImage); } } @@ -399,7 +442,7 @@ export class KubernetesService implements OnModuleInit { try { const kubeconfig = await this.getKubeconfig(app.clusterId); const imageUri = app.latestImageTag - ? `${this.configService.get('registry.pullUrl') || 'localhost:30500'}/${app.name}:${app.latestImageTag}` + ? this.registryService.normalizeImageReference(app.latestImageTag) : ''; const values = this.buildHelmValues(app, imageUri); await this.helmService.installOrUpgrade(app.name, namespace, values, kubeconfig); @@ -446,8 +489,11 @@ export class KubernetesService implements OnModuleInit { private async deployViaHelm(app: Application, imageUri: string): Promise> { const kubeconfig = await this.getKubeconfig(app.clusterId); + await this.ensurePlatformStorageClass(kubeconfig); + const { coreApi } = await this.getK8sClient(app.clusterId); const values = this.buildHelmValues(app, imageUri); - const namespace = values.app.namespace; + const namespace = values.app.namespace as string; + await this.registryService.ensureRegistryPullSecret(coreApi, namespace); const releaseName = app.name; const result = await this.helmService.installOrUpgrade( @@ -463,6 +509,7 @@ export class KubernetesService implements OnModuleInit { private async deployManagedViaHelm(app: Application): Promise> { const kubeconfig = await this.getKubeconfig(app.clusterId); + await this.ensurePlatformStorageClass(kubeconfig); const values = this.buildManagedHelmValues(app); const namespace = values.app.namespace; const releaseName = app.name; @@ -482,6 +529,8 @@ export class KubernetesService implements OnModuleInit { private async deployManagedViaK8sApi(app: Application): Promise> { const { coreApi, appsApi } = await this.getK8sClient(app.clusterId); + const kubeconfig = await this.getKubeconfig(app.clusterId); + await this.ensurePlatformStorageClass(kubeconfig); const namespace = `user-${app.userId.split('-')[0]}`; const context: ManifestContext = { appName: app.name, @@ -546,6 +595,8 @@ export class KubernetesService implements OnModuleInit { return this.deployManagedViaK8sApi(app); } const { coreApi, appsApi, networkingApi } = await this.getK8sClient(app.clusterId); + const kubeconfig = await this.getKubeconfig(app.clusterId); + await this.ensurePlatformStorageClass(kubeconfig); const domain = this.configService.get('platform.domain'); const context: ManifestContext = { @@ -585,6 +636,9 @@ export class KubernetesService implements OnModuleInit { // 1. Ensure namespace exists await this.ensureNamespace(coreApi, context.namespace); + // 1b. Image pull secret for in-cluster registry + await this.registryService.ensureRegistryPullSecret(coreApi, context.namespace); + // 2. Create/Update secrets for env vars if (Object.keys(context.envVars).length > 0) { manifests.secret = await this.applySecret(coreApi, context); diff --git a/backend/src/kubernetes/registry.service.ts b/backend/src/kubernetes/registry.service.ts new file mode 100644 index 0000000..bd8f998 --- /dev/null +++ b/backend/src/kubernetes/registry.service.ts @@ -0,0 +1,110 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import * as k8s from '@kubernetes/client-node'; + +export interface ParsedImageReference { + repository: string; + tag: string; +} + +/** + * In-cluster Docker Registry v2 (ClusterIP). + * Kaniko pushes and application workloads pull from the same registry URL. + */ +@Injectable() +export class RegistryService { + private readonly logger = new Logger(RegistryService.name); + + constructor(private readonly configService: ConfigService) {} + + getBuildNamespace(): string { + return this.configService.get('build.namespace') || 'cloudhost-builds'; + } + + /** Registry host:port used for build push and app image pull. */ + getRegistryUrl(): string { + const buildNs = this.getBuildNamespace(); + const url = + this.configService.get('registry.pullUrl') || + this.configService.get('registry.url') || + `registry.${buildNs}.svc.cluster.local:5000`; + return url.replace(/^https?:\/\//, ''); + } + + getRegistryCredentials(): { username: string; password: string } { + return { + username: this.configService.get('registry.username') || 'admin', + password: this.configService.get('registry.password') || '', + }; + } + + buildImageReference(userId: string, appName: string, tag: string): string { + return `${this.getRegistryUrl()}/${userId}/${appName}:${tag}`; + } + + parseImageReference(imageRef: string): ParsedImageReference { + const normalized = imageRef.replace(/^https?:\/\//, ''); + const slashIdx = normalized.indexOf('/'); + if (slashIdx === -1) { + throw new Error(`Invalid image reference (missing repository path): ${imageRef}`); + } + const rest = normalized.slice(slashIdx + 1); + const tagIdx = rest.lastIndexOf(':'); + if (tagIdx === -1) { + throw new Error(`Invalid image reference (missing tag): ${imageRef}`); + } + return { + repository: rest.slice(0, tagIdx), + tag: rest.slice(tagIdx + 1), + }; + } + + /** Re-point any stored image (e.g. legacy external host) to the in-cluster registry. */ + normalizeImageReference(imageRef: string): string { + const { repository, tag } = this.parseImageReference(imageRef); + return `${this.getRegistryUrl()}/${repository}:${tag}`; + } + + buildDockerConfigJson(): string { + const { username, password } = this.getRegistryCredentials(); + const auth = + username && password + ? Buffer.from(`${username}:${password}`).toString('base64') + : ''; + const host = this.getRegistryUrl(); + return JSON.stringify({ + auths: { + [host]: { auth }, + [`registry.${this.getBuildNamespace()}.svc.cluster.local:5000`]: { auth }, + }, + }); + } + + async ensureRegistryPullSecret(coreApi: k8s.CoreV1Api, namespace: string): Promise { + const secretName = 'registry-pull-secret'; + const secret: k8s.V1Secret = { + apiVersion: 'v1', + kind: 'Secret', + metadata: { + name: secretName, + namespace, + labels: { 'app.kubernetes.io/managed-by': 'cloudhost' }, + }, + type: 'kubernetes.io/dockerconfigjson', + data: { + '.dockerconfigjson': Buffer.from(this.buildDockerConfigJson()).toString('base64'), + }, + }; + + try { + await coreApi.replaceNamespacedSecret(secretName, namespace, secret); + } catch (err: any) { + if (err.statusCode === 404 || err.body?.code === 404) { + await coreApi.createNamespacedSecret(namespace, secret); + this.logger.log(`Created ${secretName} in ${namespace}`); + } else { + throw err; + } + } + } +} diff --git a/frontend/src/app/dashboard/admin/clusters/page.tsx b/frontend/src/app/dashboard/admin/clusters/page.tsx index ce48b0f..494a2dc 100644 --- a/frontend/src/app/dashboard/admin/clusters/page.tsx +++ b/frontend/src/app/dashboard/admin/clusters/page.tsx @@ -120,28 +120,55 @@ function ResourcePanel({ clusterId }: { clusterId: string }) { ); } -function CentralLoggingPanel() { +function apiErrorMessage(err: unknown, fallback: string): string { + const e = err as { response?: { data?: { message?: string | string[] } } }; + const msg = e?.response?.data?.message; + if (Array.isArray(msg)) return msg.join(', '); + if (typeof msg === 'string' && msg.trim()) return msg; + return fallback; +} + +function CentralLoggingPanel({ + clusters, + selectedClusterId, + onSelectCluster, +}: { + clusters: Cluster[]; + selectedClusterId: string | null; + onSelectCluster: (id: string) => void; +}) { const queryClient = useQueryClient(); + const clusterId = selectedClusterId || clusters.find((c) => c.isDefault)?.id || clusters[0]?.id; + const { data: status, isLoading } = useQuery({ - queryKey: ['admin-elasticsearch-status'], - queryFn: () => api.get('/admin/elasticsearch/status').then((r) => r.data), + queryKey: ['admin-elasticsearch-status', clusterId], + queryFn: () => + api.get('/admin/elasticsearch/status', { params: clusterId ? { clusterId } : {} }).then((r) => r.data), + enabled: !!clusterId, }); const deployMutation = useMutation({ - mutationFn: () => api.post('/admin/elasticsearch/deploy'), - onSuccess: () => { + mutationFn: (targetClusterId?: string) => + api.post('/admin/elasticsearch/deploy', null, { + params: targetClusterId ? { clusterId: targetClusterId } : {}, + }), + onSuccess: (res) => { queryClient.invalidateQueries({ queryKey: ['admin-elasticsearch-status'] }); - toast.success('Logging stack deployment started'); + toast.success(res.data?.message || 'Logging stack deployment started'); }, - onError: () => toast.error('Failed to deploy logging stack'), + onError: (err) => toast.error(apiErrorMessage(err, 'Failed to deploy logging stack')), }); const undeployMutation = useMutation({ - mutationFn: () => api.delete('/admin/elasticsearch/undeploy'), + mutationFn: (targetClusterId?: string) => + api.delete('/admin/elasticsearch/undeploy', { + params: targetClusterId ? { clusterId: targetClusterId } : {}, + }), onSuccess: () => { queryClient.invalidateQueries({ queryKey: ['admin-elasticsearch-status'] }); toast.success('Logging stack removed'); }, + onError: (err) => toast.error(apiErrorMessage(err, 'Failed to remove logging stack')), }); const kibanaCmd = 'kubectl port-forward svc/kibana 5601:5601 -n logging'; @@ -158,21 +185,34 @@ function CentralLoggingPanel() { End users never get Kibana access — staff use port-forward.

-
+
+ {clusters.length > 1 && ( + + )} {!status?.deployed ? ( ) : (
) : ( -

- Not deployed on the default cluster. New clusters install this automatically; use Deploy for existing clusters. -

+
+

+ Not ready on this cluster + {status?.deployStatus ? ` (${status.deployStatus}${status.helmReleaseStatus ? ` · helm: ${status.helmReleaseStatus}` : ''})` : ''}. +

+

+ New clusters install automatically; if that failed (e.g. missing StorageClass), click Deploy Elastic to install or repair. +

+
)} ); } +function ClusterElasticButton({ clusterId, clusterName }: { clusterId: string; clusterName: string }) { + const queryClient = useQueryClient(); + const { data: status } = useQuery({ + queryKey: ['admin-elasticsearch-status', clusterId], + queryFn: () => api.get('/admin/elasticsearch/status', { params: { clusterId } }).then((r) => r.data), + }); + + const deployMutation = useMutation({ + mutationFn: () => api.post('/admin/elasticsearch/deploy', null, { params: { clusterId } }), + onSuccess: (res) => { + queryClient.invalidateQueries({ queryKey: ['admin-elasticsearch-status'] }); + toast.success(res.data?.message || `Elasticsearch deploy started on ${clusterName}`); + }, + onError: (err) => toast.error(apiErrorMessage(err, `Failed to deploy Elasticsearch on ${clusterName}`)), + }); + + if (status?.deployed) { + return ( + + Elastic + + ); + } + + return ( + + ); +} + export default function AdminClustersPage() { const queryClient = useQueryClient(); const confirm = useConfirm(); const [showForm, setShowForm] = useState(false); const [testingId, setTestingId] = useState(null); const [expandedResources, setExpandedResources] = useState>(new Set()); + const [loggingClusterId, setLoggingClusterId] = useState(null); const [form, setForm] = useState({ name: '', description: '', @@ -308,7 +393,11 @@ export default function AdminClustersPage() { - + {showForm && (
@@ -477,6 +566,7 @@ export default function AdminClustersPage() { > Resources +