diff --git a/backend/.env.example b/backend/.env.example index 69d6617..0bb706d 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -30,7 +30,7 @@ REGISTRY_PASSWORD=registry_secret # Central logging (Elasticsearch + Kibana) # In-cluster backend: leave ELASTICSEARCH_HOST unset (uses elasticsearch.logging.svc.cluster.local). -# Local backend (npm run dev): API auto-runs kubectl port-forward when host is loopback +# Local backend (npm run dev): defaults to 127.0.0.1 and auto-runs kubectl port-forward to the default cluster. # ELASTICSEARCH_HOST=127.0.0.1 # ELASTICSEARCH_PORT=9200 # ELASTICSEARCH_AUTO_PORT_FORWARD=false diff --git a/backend/helm/cloudhost-platform/values.yaml b/backend/helm/cloudhost-platform/values.yaml index dd726e7..920afff 100644 --- a/backend/helm/cloudhost-platform/values.yaml +++ b/backend/helm/cloudhost-platform/values.yaml @@ -60,6 +60,8 @@ backend: PLATFORM_CREATE_STORAGE_CLASS: "true" PLATFORM_STORAGE_CLASS: cloudhost-expandable PLATFORM_STORAGE_PROVISIONER: rancher.io/local-path + ELASTICSEARCH_HOST: elasticsearch.logging.svc.cluster.local + ELASTICSEARCH_AUTO_PORT_FORWARD: "false" frontend: enabled: true diff --git a/backend/src/config/configuration.ts b/backend/src/config/configuration.ts index 9b7c825..e5124f8 100644 --- a/backend/src/config/configuration.ts +++ b/backend/src/config/configuration.ts @@ -36,8 +36,12 @@ export default () => ({ }, elasticsearch: { - /** API host for log search. Use cluster DNS in-cluster; 127.0.0.1 + port-forward when backend runs locally. */ - host: process.env.ELASTICSEARCH_HOST || 'elasticsearch.logging.svc.cluster.local', + /** API host for log search. In-cluster: cluster DNS. Local dev: loopback + auto port-forward. */ + host: + process.env.ELASTICSEARCH_HOST || + (process.env.KUBERNETES_SERVICE_HOST + ? 'elasticsearch.logging.svc.cluster.local' + : '127.0.0.1'), port: parseInt(process.env.ELASTICSEARCH_PORT || '9200', 10), /** In development with loopback host, start kubectl port-forward on API boot (set false to manage manually). */ autoPortForward: process.env.ELASTICSEARCH_AUTO_PORT_FORWARD ?? 'true', diff --git a/backend/src/kubernetes/elasticsearch.service.ts b/backend/src/kubernetes/elasticsearch.service.ts index c4265ff..3cb658a 100644 --- a/backend/src/kubernetes/elasticsearch.service.ts +++ b/backend/src/kubernetes/elasticsearch.service.ts @@ -87,6 +87,8 @@ export class ElasticsearchService implements OnModuleInit, OnModuleDestroy { private readonly KIBANA_NAME = 'kibana'; private portForwardChild: ChildProcess | null = null; private portForwardStartedByUs = false; + private portForwardKubeconfigFile: string | null = null; + private portForwardClusterId: string | null = null; private ensureInFlight: Promise | null = null; private reconnectTimer: ReturnType | null = null; private healthCheckTimer: ReturnType | null = null; @@ -133,6 +135,37 @@ export class ElasticsearchService implements OnModuleInit, OnModuleDestroy { return host === '127.0.0.1' || host === 'localhost' || host === '::1'; } + private isRunningInKubernetes(): boolean { + return Boolean(process.env.KUBERNETES_SERVICE_HOST); + } + + /** Hostnames only resolvable from inside the workload cluster (not from a laptop). */ + private isClusterInternalHost(host: string): boolean { + const h = host.toLowerCase(); + return ( + h.includes('svc.cluster.local') || + h.includes('.cluster.') || + h === 'elasticsearch' || + h === 'kibana' + ); + } + + private configuredElasticsearchHost(): string { + return ( + this.configService.get('elasticsearch.host') || + `${this.ES_NAME}.${this.ES_NAMESPACE}.svc.cluster.local` + ); + } + + /** HTTP target: loopback when we tunnel; cluster DNS when the API pod runs in-cluster. */ + private effectiveElasticsearchHost(): string { + const configured = this.configuredElasticsearchHost(); + if (this.shouldAutoPortForward() && this.isClusterInternalHost(configured)) { + return '127.0.0.1'; + } + return configured; + } + private shouldAutoPortForward(): boolean { if (this.configService.get('elasticsearch.autoPortForward') === 'false') { return false; @@ -140,12 +173,15 @@ export class ElasticsearchService implements OnModuleInit, OnModuleDestroy { if (process.env.ELASTICSEARCH_AUTO_PORT_FORWARD === 'false') { return false; } + if (this.isRunningInKubernetes()) { + return false; + } const nodeEnv = process.env.NODE_ENV || 'development'; if (nodeEnv === 'production') { return false; } - const host = this.configService.get('elasticsearch.host') || ''; - return this.isLoopbackHost(host); + const host = this.configuredElasticsearchHost(); + return this.isLoopbackHost(host) || this.isClusterInternalHost(host); } private stopDevPortForward(): void { @@ -154,9 +190,15 @@ export class ElasticsearchService implements OnModuleInit, OnModuleDestroy { } const startedByUs = this.portForwardStartedByUs; const child = this.portForwardChild; + const kubeconfigFile = this.portForwardKubeconfigFile; this.portForwardChild = null; this.portForwardStartedByUs = false; + this.portForwardKubeconfigFile = null; + this.portForwardClusterId = null; child.kill('SIGTERM'); + if (kubeconfigFile) { + this.helmService.removeTempFile(kubeconfigFile); + } if (startedByUs) { this.logger.log('Stopped Elasticsearch kubectl port-forward'); } @@ -236,18 +278,36 @@ export class ElasticsearchService implements OnModuleInit, OnModuleDestroy { return false; } - private startDevPortForward(localPort: number): void { - if (this.portForwardChild) { + private async startDevPortForward(localPort: number, clusterId?: string): Promise { + const targetClusterId = clusterId || null; + if ( + this.portForwardChild && + this.portForwardClusterId === targetClusterId + ) { return; } + if (this.portForwardChild) { + this.stopDevPortForward(); + await new Promise((r) => setTimeout(r, 300)); + } + + const { cluster } = await this.getK8sClients(clusterId); + const kubeconfigFile = await this.helmService.createKubeconfigFile(cluster.kubeconfig); + this.portForwardKubeconfigFile = kubeconfigFile; + this.portForwardClusterId = targetClusterId; + const args = [ + '--kubeconfig', + kubeconfigFile, 'port-forward', '-n', this.ES_NAMESPACE, `svc/${this.ES_NAME}`, `${localPort}:9200`, ]; - this.logger.log(`Starting kubectl ${args.join(' ')} (local log search)`); + this.logger.log( + `Starting kubectl port-forward to Elasticsearch on cluster "${cluster.name}" (local log search)`, + ); const child = spawn('kubectl', args, { stdio: ['ignore', 'pipe', 'pipe'] }); this.portForwardChild = child; this.portForwardStartedByUs = true; @@ -281,6 +341,7 @@ export class ElasticsearchService implements OnModuleInit, OnModuleDestroy { */ private async ensureLocalElasticsearchAccess(options?: { waitForCluster?: boolean; + clusterId?: string; }): Promise { if (this.ensureInFlight) { await this.ensureInFlight; @@ -298,6 +359,7 @@ export class ElasticsearchService implements OnModuleInit, OnModuleDestroy { private async ensureLocalElasticsearchAccessImpl(options?: { waitForCluster?: boolean; + clusterId?: string; }): Promise { if (!this.shouldAutoPortForward()) { return; @@ -308,7 +370,8 @@ export class ElasticsearchService implements OnModuleInit, OnModuleDestroy { return; } - let deployed = await this.isDeployed(); + const clusterId = options?.clusterId; + let deployed = await this.isDeployed(clusterId); if (!deployed && options?.waitForCluster) { this.logger.log('Waiting for logging stack after cluster reconnect…'); deployed = await this.waitForLoggingStack(); @@ -325,7 +388,7 @@ export class ElasticsearchService implements OnModuleInit, OnModuleDestroy { await new Promise((r) => setTimeout(r, 300)); } - this.startDevPortForward(port); + await this.startDevPortForward(port, clusterId); const ready = await this.waitForElasticsearch(90_000); if (ready) { this.reconnectAttempt = 0; @@ -341,16 +404,23 @@ export class ElasticsearchService implements OnModuleInit, OnModuleDestroy { private localElasticsearchHint(): string { const conn = this.getConnectionInfo(); + const configured = this.configuredElasticsearchHost(); if (this.isLoopbackHost(conn.host)) { return ( `Ensure port ${conn.port} is forwarded to the cluster (the API auto-starts kubectl port-forward in development). ` + `Manual: kubectl port-forward -n ${this.ES_NAMESPACE} svc/${this.ES_NAME} ${conn.port}:9200` ); } - if (conn.host.includes('svc.cluster.local') || conn.host.includes('.cluster.')) { + if (this.isClusterInternalHost(configured)) { + if (this.isRunningInKubernetes()) { + return ( + 'Ensure the logging stack is deployed on the same cluster as this API pod ' + + `(Helm release ${LOGGING_HELM_RELEASE} in namespace ${LOGGING_HELM_NAMESPACE}).` + ); + } return ( - 'Run the API inside the cluster, or set ELASTICSEARCH_HOST=127.0.0.1 and keep port-forward running: ' + - `kubectl port-forward -n ${this.ES_NAMESPACE} svc/${this.ES_NAME} ${conn.port}:9200` + 'Run the API inside the cluster, or restart the API locally so it can auto port-forward Elasticsearch ' + + `(kubectl port-forward -n ${this.ES_NAMESPACE} svc/${this.ES_NAME} ${conn.port}:9200 using the cluster kubeconfig).` ); } return `Ensure Elasticsearch is listening on ${conn.host}:${conn.port}.`; @@ -620,9 +690,7 @@ export class ElasticsearchService implements OnModuleInit, OnModuleDestroy { */ getConnectionInfo(): { host: string; port: number; username: string; password: string } { return { - host: - this.configService.get('elasticsearch.host') || - `${this.ES_NAME}.${this.ES_NAMESPACE}.svc.cluster.local`, + host: this.effectiveElasticsearchHost(), port: this.configService.get('elasticsearch.port') || 9200, username: 'elastic', password: this.ELASTIC_PASSWORD, @@ -791,6 +859,10 @@ export class ElasticsearchService implements OnModuleInit, OnModuleDestroy { ); } + if (this.shouldAutoPortForward()) { + await this.ensureLocalElasticsearchAccess({ clusterId }); + } + const conn = this.getConnectionInfo(); const url = `http://${conn.host}:${conn.port}${path}`; const auth = Buffer.from(`${conn.username}:${conn.password}`).toString('base64'); @@ -800,7 +872,7 @@ export class ElasticsearchService implements OnModuleInit, OnModuleDestroy { response = await this.elasticsearchFetch(url, auth, body); } catch (err: any) { if (this.shouldAutoPortForward()) { - await this.ensureLocalElasticsearchAccess(); + await this.ensureLocalElasticsearchAccess({ clusterId }); try { response = await this.elasticsearchFetch(url, auth, body); } catch { @@ -963,7 +1035,7 @@ export class ElasticsearchService implements OnModuleInit, OnModuleDestroy { } if (this.shouldAutoPortForward() && !(await this.probeElasticsearch())) { - void this.ensureLocalElasticsearchAccess(); + void this.ensureLocalElasticsearchAccess({ clusterId }); } if (await this.probeElasticsearch()) { diff --git a/backend/src/kubernetes/helm.service.ts b/backend/src/kubernetes/helm.service.ts index 23cddf2..b750a13 100644 --- a/backend/src/kubernetes/helm.service.ts +++ b/backend/src/kubernetes/helm.service.ts @@ -329,6 +329,15 @@ export class HelmService { // ── Temp file helpers ─────────────────────────────────── + /** Write kubeconfig to a temp file for kubectl/helm CLIs (caller may delete when done). */ + async createKubeconfigFile(kubeconfig: string): Promise { + return this.writeTempKubeconfig(kubeconfig); + } + + removeTempFile(filePath: string): void { + this.cleanupTempFiles(filePath); + } + private async writeTempKubeconfig(kubeconfig: string): Promise { const tmpFile = path.join(os.tmpdir(), `cloudhost-kube-${Date.now()}-${Math.random().toString(36).slice(2)}`); await fs.promises.writeFile(tmpFile, kubeconfig, { mode: 0o600 }); diff --git a/backend/src/kubernetes/logs.controller.ts b/backend/src/kubernetes/logs.controller.ts index c6fa342..2b0a3ea 100644 --- a/backend/src/kubernetes/logs.controller.ts +++ b/backend/src/kubernetes/logs.controller.ts @@ -46,12 +46,16 @@ export class LogsController { userId: string, appId?: string, allowStaff = false, - ): Promise<{ applicationId?: string; applicationName?: string }> { + ): Promise<{ applicationId?: string; applicationName?: string; clusterId?: string }> { if (!appId) return {}; const where = allowStaff ? { id: appId } : { id: appId, userId }; const app = await this.appsRepo.findOne({ where }); if (!app) throw new NotFoundException('Application not found'); - return { applicationId: app.id, applicationName: app.name }; + return { + applicationId: app.id, + applicationName: app.name, + clusterId: app.clusterId || undefined, + }; } private isStaff(role: string): boolean { @@ -60,8 +64,15 @@ export class LogsController { @Get('status') @ApiOperation({ summary: 'Check if central logging is available' }) - async getStatus() { - return this.esService.getLoggingStatus(); + @ApiQuery({ name: 'appId', required: false, description: 'Resolve logging cluster from this app' }) + async getStatus( + @Request() req: AuthenticatedRequest, + @Query('appId') appId?: string, + ) { + const appFilters = appId + ? await this.resolveAppFilters(req.user.id, appId, this.isStaff(req.user.role)) + : {}; + return this.esService.getLoggingStatus(appFilters.clusterId); } @Get() @@ -102,16 +113,20 @@ export class LogsController { const appFilters = await this.resolveAppFilters(userId, appId, this.isStaff(req.user.role)); - return this.esService.searchLogs(userId, { - ...appFilters, - workload, - level: level?.toLowerCase(), - from, - to, - search, - page: parseInt(page || '1', 10), - limit: Math.min(parseInt(limit || '100', 10), 1000), - }); + return this.esService.searchLogs( + userId, + { + ...appFilters, + workload, + level: level?.toLowerCase(), + from, + to, + search, + page: parseInt(page || '1', 10), + limit: Math.min(parseInt(limit || '100', 10), 1000), + }, + appFilters.clusterId, + ); } @Get('stream') @@ -127,13 +142,17 @@ export class LogsController { const appFilters = await this.resolveAppFilters(userId, appId, this.isStaff(req.user.role)); const fiveMinAgo = new Date(Date.now() - 5 * 60 * 1000).toISOString(); - return this.esService.searchLogs(userId, { - ...appFilters, - workload, - from: fiveMinAgo, - limit: 100, - page: 1, - }); + return this.esService.searchLogs( + userId, + { + ...appFilters, + workload, + from: fiveMinAgo, + limit: 100, + page: 1, + }, + appFilters.clusterId, + ); } @Get('stats') @@ -149,11 +168,15 @@ export class LogsController { ) { const userId = req.user.id; const appFilters = await this.resolveAppFilters(userId, appId, this.isStaff(req.user.role)); - return this.esService.searchLogStats(userId, { - ...appFilters, - workload, - period: period || '24h', - }); + return this.esService.searchLogStats( + userId, + { + ...appFilters, + workload, + period: period || '24h', + }, + appFilters.clusterId, + ); } @Get('errors') @@ -171,12 +194,16 @@ export class LogsController { ) { const userId = req.user.id; const appFilters = await this.resolveAppFilters(userId, appId, this.isStaff(req.user.role)); - const hits = await this.esService.searchRecentErrors(userId, { - ...appFilters, - workload, - hours: parseInt(hours || '24', 10), - limit: Math.min(parseInt(limit || '50', 10), 500), - }); + const hits = await this.esService.searchRecentErrors( + userId, + { + ...appFilters, + workload, + hours: parseInt(hours || '24', 10), + limit: Math.min(parseInt(limit || '50', 10), 500), + }, + appFilters.clusterId, + ); return { hits, total: hits.length }; } diff --git a/frontend/src/app/dashboard/logs/page.tsx b/frontend/src/app/dashboard/logs/page.tsx index 6e1c363..7fa1458 100644 --- a/frontend/src/app/dashboard/logs/page.tsx +++ b/frontend/src/app/dashboard/logs/page.tsx @@ -68,10 +68,10 @@ function LogsPageContent() { }, [initialAppId]); const { data: loggingStatus, isFetching: statusFetching } = useQuery({ - queryKey: ['logs-status'], + queryKey: ['logs-status', appId], queryFn: () => api - .get('/logs/status') + .get('/logs/status', { params: appId ? { appId } : undefined }) .then((r) => r.data as { available: boolean; @@ -185,7 +185,7 @@ function LogsPageContent() { const totalPages = logsResult ? Math.max(1, Math.ceil(logsResult.total / logsResult.limit)) : 1; if (loggingStatus && !loggingStatus.available) { - const isRecovering = loggingStatus.recovering || loggingStatus.deployed === true; + const isRecovering = loggingStatus.recovering === true; return (
{isRecovering ? (