Improve logging recovery, resource scaling, and app deploy logging.

Auto-reconnect Elasticsearch port-forward after cluster or API restarts, poll log status in the UI, and apply storage changes through billing upgrade for all workloads. Add Redis/RabbitMQ PVC resize, Helm ES credentials for Fluent Bit, and fix deploy progress overlay behavior.

Co-authored-by: Cursor <cursoragent@cursor.com>

backend/.env.example

@@ -19,11 +19,26 @@ JWT_REFRESH_EXPIRES_IN=7d
 REDIS_HOST=localhost
 REDIS_PORT=6379
 
-# Container Registry
-REGISTRY_URL=registry.example.com
+# Container Registry (same Docker Registry v2, two hostnames)
+# Internal — Kaniko/build pods push here (ClusterIP, HTTP, fast)
+REGISTRY_URL=registry.cloudhost-builds.svc.cluster.local:5000
+# External — kubelet pulls app images; also use for manual "docker push" (Ingress or NodePort)
+REGISTRY_PULL_URL=repo.3fase.ir
+# REGISTRY_PULL_URL=10.0.0.50:30500
 REGISTRY_USERNAME=admin
 REGISTRY_PASSWORD=registry_secret
 
+# Central logging (Elasticsearch + Kibana)
+# In-cluster backend: leave ELASTICSEARCH_HOST unset (uses elasticsearch.logging.svc.cluster.local).
+# Local backend (npm run dev): API auto-runs kubectl port-forward when host is loopback
+# ELASTICSEARCH_HOST=127.0.0.1
+# ELASTICSEARCH_PORT=9200
+# ELASTICSEARCH_AUTO_PORT_FORWARD=false
+# ELASTIC_PASSWORD=CloudHost2024!Secure
+# KIBANA_SYSTEM_PASSWORD=Kibana2024!System
+# LOGGING_ELASTICSEARCH_IMAGE=localhost:30500/elasticsearch:8.12.0
+# LOGGING_KIBANA_IMAGE=localhost:30500/kibana:8.12.0
+
 # Build
 BUILD_NAMESPACE=cloudhost-builds
 BUILD_SERVICE_ACCOUNT=kaniko-builder

backend/helm/cloudhost-app/templates/_helpers.tpl

@@ -130,3 +130,21 @@ Default log paths based on runtime
 {{- else }}/var/log/app/*.log
 {{- end }}
 {{- end }}
+
+{{/*
+Shell start command for stdout capture (must match CloudHost-generated images).
+*/}}
+{{- define "cloudhost-app.runtimeStartCommand" -}}
+{{- if eq .Values.app.runtime "nodejs" -}}
+if [ -f /app/.mode ] && [ "$(cat /app/.mode)" = "standalone" ] && [ -f server.js ]; then node server.js; else npm start; fi
+{{- else if eq .Values.app.runtime "go" -}}
+./main
+{{- else if eq .Values.app.runtime "dotnet" -}}
+DLL=$(find . -maxdepth 1 -name '*.dll' ! -name '*.deps.dll' ! -name '*.runtimeconfig.dll' | head -1) && dotnet "$DLL"
+{{- else -}}
+{{- end }}
+{{- end }}
+
+{{- define "cloudhost-app.loggingWrapEnabled" -}}
+{{- and .Values.elasticsearch.enabled (include "cloudhost-app.runtimeStartCommand" .) -}}
+{{- end }}

backend/helm/cloudhost-app/templates/deployment.yaml

@@ -27,6 +27,11 @@ spec:
         - name: {{ $name }}
           image: {{ .Values.app.image | quote }}
           imagePullPolicy: Always
+          {{- if include "cloudhost-app.loggingWrapEnabled" . }}
+          command: ["sh", "-c"]
+          args:
+            - mkdir -p /var/log/app && ({{ include "cloudhost-app.runtimeStartCommand" . | trim }}) >> /var/log/app/app.log 2>&1
+          {{- end }}
           ports:
             - containerPort: {{ .Values.app.port }}
           {{- if and .Values.envVars (gt (len .Values.envVars) 0) }}

backend/helm/cloudhost-app/templates/elasticsearch-credentials-secret.yaml

@@ -0,0 +1,14 @@
+{{- if .Values.elasticsearch.enabled }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: elasticsearch-credentials
+  namespace: {{ include "cloudhost-app.namespace" . }}
+  labels:
+    {{- include "cloudhost-app.labels" . | nindent 4 }}
+type: Opaque
+stringData:
+  ELASTIC_PASSWORD: {{ .Values.elasticsearch.elasticPassword | default "CloudHost2024!Secure" | quote }}
+  FLUENTBIT_PASSWORD: {{ .Values.elasticsearch.fluentbitPassword | default "FluentBit2024!Writer" | quote }}
+  KIBANA_SYSTEM_PASSWORD: {{ .Values.elasticsearch.kibanaPassword | default "Kibana2024!System" | quote }}
+{{- end }}

backend/helm/cloudhost-app/values.yaml

@@ -87,6 +87,9 @@ elasticsearch:
   logPaths: []
   ownerId: ""
   applicationId: ""
+  elasticPassword: ""
+  fluentbitPassword: ""
+  kibanaPassword: ""
 
 # ── Change metadata ─────────────────────────────────────
 changeCause: ""

backend/helm/cloudhost-logging/values.yaml

@@ -8,6 +8,8 @@ clusterName: cloudhost-logs
 
 storage: 50Gi
 
+# Official Elastic images; require docker.elastic.co DNS + outbound HTTPS from nodes.
+# If pull fails with "lookup docker.elastic.co: Try again", mirror to your registry and override here.
 images:
   elasticsearch: docker.elastic.co/elasticsearch/elasticsearch:8.12.0
   kibana: docker.elastic.co/kibana/kibana:8.12.0

backend/src/applications/applications.controller.ts

@@ -199,6 +199,80 @@ export class ApplicationsController {
     return result;
   }
 
+  @Patch(':id/redis-storage')
+  @ApiOperation({ summary: 'Resize (expand) Redis PVC storage' })
+  async resizeRedisStorage(
+    @Param('id') id: string,
+    @Request() req: any,
+    @Body() body: { size: string },
+  ) {
+    const isStaff = req.user.role === UserRole.ADMIN || req.user.role === UserRole.TECHNICAL;
+    const app = await this.applicationsService.findOne(id, isStaff ? undefined : req.user.id);
+
+    if (!app.enableRedis) {
+      throw new BadRequestException('Redis is not enabled for this application');
+    }
+    if (!body.size || !/^\d+Gi$/.test(body.size)) {
+      throw new BadRequestException('Size must be in format like "1Gi", "5Gi", "10Gi"');
+    }
+
+    const result = await this.kubernetesService.resizeRedisStoragePvc(app, body.size);
+    if (result.success) {
+      const prev = app.optionalServiceResources?.redis;
+      const storageGi = parseInt(body.size.replace('Gi', ''), 10) || 1;
+      await this.applicationsService.update(id, app.userId, {
+        optionalServiceResources: {
+          ...app.optionalServiceResources,
+          redis: {
+            cpuRequest: prev?.cpuRequest,
+            cpuLimit: prev?.cpuLimit ?? '200m',
+            memoryRequest: prev?.memoryRequest,
+            memoryLimit: prev?.memoryLimit ?? '256Mi',
+            storageGi,
+          },
+        },
+      } as any);
+    }
+    return result;
+  }
+
+  @Patch(':id/rabbitmq-storage')
+  @ApiOperation({ summary: 'Resize (expand) RabbitMQ PVC storage' })
+  async resizeRabbitmqStorage(
+    @Param('id') id: string,
+    @Request() req: any,
+    @Body() body: { size: string },
+  ) {
+    const isStaff = req.user.role === UserRole.ADMIN || req.user.role === UserRole.TECHNICAL;
+    const app = await this.applicationsService.findOne(id, isStaff ? undefined : req.user.id);
+
+    if (!app.enableRabbitmq) {
+      throw new BadRequestException('RabbitMQ is not enabled for this application');
+    }
+    if (!body.size || !/^\d+Gi$/.test(body.size)) {
+      throw new BadRequestException('Size must be in format like "1Gi", "5Gi", "10Gi"');
+    }
+
+    const result = await this.kubernetesService.resizeRabbitmqStoragePvc(app, body.size);
+    if (result.success) {
+      const prev = app.optionalServiceResources?.rabbitmq;
+      const storageGi = parseInt(body.size.replace('Gi', ''), 10) || 2;
+      await this.applicationsService.update(id, app.userId, {
+        optionalServiceResources: {
+          ...app.optionalServiceResources,
+          rabbitmq: {
+            cpuRequest: prev?.cpuRequest,
+            cpuLimit: prev?.cpuLimit ?? '500m',
+            memoryRequest: prev?.memoryRequest,
+            memoryLimit: prev?.memoryLimit ?? '512Mi',
+            storageGi,
+          },
+        },
+      } as any);
+    }
+    return result;
+  }
+
   @Get()
   @ApiOperation({ summary: 'List my applications or managed services' })
   async findAll(

backend/src/billing/billing.controller.ts

@@ -830,7 +830,7 @@ export class BillingController {
   private buildUpgradeEntityPatch(app: Application, dto: UpgradeResourcesDto): Partial<Application> {
     const pt = app.productType ?? ProductType.APPLICATION;
 
-    if (pt === ProductType.MANAGED_REDIS && dto.redisResources) {
+    if (dto.redisResources) {
       return {
         optionalServiceResources: {
           ...app.optionalServiceResources,
@@ -844,7 +844,7 @@ export class BillingController {
       };
     }
 
-    if (pt === ProductType.MANAGED_RABBITMQ && dto.rabbitmqResources) {
+    if (dto.rabbitmqResources) {
       return {
         optionalServiceResources: {
           ...app.optionalServiceResources,
@@ -860,6 +860,10 @@ export class BillingController {
       };
     }
 
+    if (pt === ProductType.MANAGED_REDIS || pt === ProductType.MANAGED_RABBITMQ) {
+      return {};
+    }
+
     return {
       cpuRequest: dto.cpuRequest || app.cpuRequest,
       cpuLimit: dto.cpuLimit || app.cpuLimit,
@@ -899,46 +903,39 @@ export class BillingController {
     }
 
     if (pt === ProductType.MANAGED_REDIS) {
-      const res = app.optionalServiceResources?.redis;
-      if (res) {
-        await this.kubernetesService.updateResources(
-          app,
-          {
-            cpuRequest: res.cpuRequest,
-            cpuLimit: res.cpuLimit,
-            memoryRequest: res.memoryRequest,
-            memoryLimit: res.memoryLimit,
-          },
-          'redis',
-        );
-      }
+      await this.applyOptionalServiceUpgrade(app, dto, previous, 'redis');
       return;
     }
 
     if (pt === ProductType.MANAGED_RABBITMQ) {
-      const res = app.optionalServiceResources?.rabbitmq;
-      if (res) {
-        await this.kubernetesService.updateResources(
-          app,
-          {
-            cpuRequest: res.cpuRequest,
-            cpuLimit: res.cpuLimit,
-            memoryRequest: res.memoryRequest,
-            memoryLimit: res.memoryLimit,
-          },
-          'rabbitmq',
-        );
-      }
+      await this.applyOptionalServiceUpgrade(app, dto, previous, 'rabbitmq');
       return;
     }
 
-    await this.kubernetesService.updateResources(app, {
-      cpuRequest: dto.cpuRequest,
-      cpuLimit: dto.cpuLimit,
-      memoryRequest: dto.memoryRequest,
-      memoryLimit: dto.memoryLimit,
-      replicas: dto.replicas,
-    });
+    if (dto.redisResources && app.enableRedis) {
+      await this.applyOptionalServiceUpgrade(app, dto, previous, 'redis');
+    }
+
+    if (dto.rabbitmqResources && app.enableRabbitmq) {
+      await this.applyOptionalServiceUpgrade(app, dto, previous, 'rabbitmq');
+    }
+
+    const touchesAppWorkload =
+      dto.cpuRequest !== undefined ||
+      dto.cpuLimit !== undefined ||
+      dto.memoryRequest !== undefined ||
+      dto.memoryLimit !== undefined ||
+      dto.replicas !== undefined;
+
+    if (touchesAppWorkload) {
+      await this.kubernetesService.updateResources(app, {
+        cpuRequest: dto.cpuRequest,
+        cpuLimit: dto.cpuLimit,
+        memoryRequest: dto.memoryRequest,
+        memoryLimit: dto.memoryLimit,
+        replicas: dto.replicas,
+      });
+    }
 
     if (dto.appStorageSize && dto.appStorageSize !== previous.appStorageSize) {
       await this.kubernetesService.resizeAppStoragePvc(app, dto.appStorageSize);
@@ -957,6 +954,40 @@ export class BillingController {
     }
   }
 
+  private async applyOptionalServiceUpgrade(
+    app: Application,
+    dto: UpgradeResourcesDto,
+    previous: Application,
+    service: 'redis' | 'rabbitmq',
+  ): Promise<void> {
+    const res = app.optionalServiceResources?.[service];
+    const dtoRes = service === 'redis' ? dto.redisResources : dto.rabbitmqResources;
+    if (res) {
+      await this.kubernetesService.updateResources(
+        app,
+        {
+          cpuRequest: res.cpuRequest,
+          cpuLimit: res.cpuLimit,
+          memoryRequest: res.memoryRequest,
+          memoryLimit: res.memoryLimit,
+        },
+        service,
+      );
+    }
+    const prevGi =
+      previous.optionalServiceResources?.[service]?.storageGi ?? (service === 'redis' ? 1 : 2);
+    const nextGi = dtoRes?.storageGi;
+    if (nextGi != null && nextGi > prevGi) {
+      const resize =
+        service === 'redis'
+          ? await this.kubernetesService.resizeRedisStoragePvc(app, `${nextGi}Gi`)
+          : await this.kubernetesService.resizeRabbitmqStoragePvc(app, `${nextGi}Gi`);
+      if (!resize.success) {
+        throw new BadRequestException(resize.message);
+      }
+    }
+  }
+
   private async getAppWithAccess(user: any, applicationId: string) {
     const isAdminOrSales = user.role === UserRole.ADMIN || user.role === UserRole.SALES;
     

backend/src/config/configuration.ts

@@ -35,9 +35,21 @@ export default () => ({
   },
 
   elasticsearch: {
+    /** API host for log search. Use cluster DNS in-cluster; 127.0.0.1 + port-forward when backend runs locally. */
+    host: process.env.ELASTICSEARCH_HOST || 'elasticsearch.logging.svc.cluster.local',
+    port: parseInt(process.env.ELASTICSEARCH_PORT || '9200', 10),
+    /** In development with loopback host, start kubectl port-forward on API boot (set false to manage manually). */
+    autoPortForward: process.env.ELASTICSEARCH_AUTO_PORT_FORWARD ?? 'true',
     password: process.env.ELASTIC_PASSWORD || 'CloudHost2024!Secure',
     fluentbitPassword: process.env.FLUENTBIT_PASSWORD || 'FluentBit2024!Writer',
     kibanaPassword: process.env.KIBANA_SYSTEM_PASSWORD || 'Kibana2024!System',
+    /** Override when cluster nodes cannot reach docker.elastic.co (mirror to local registry). */
+    images: {
+      elasticsearch:
+        process.env.LOGGING_ELASTICSEARCH_IMAGE ||
+        'docker.elastic.co/elasticsearch/elasticsearch:8.12.0',
+      kibana: process.env.LOGGING_KIBANA_IMAGE || 'docker.elastic.co/kibana/kibana:8.12.0',
+    },
   },
 
   platform: {

backend/src/kubernetes/elasticsearch.service.ts

@@ -1,7 +1,16 @@
-import { Injectable, Logger, ServiceUnavailableException, Inject, forwardRef } from '@nestjs/common';
+import {
+  Injectable,
+  Logger,
+  ServiceUnavailableException,
+  Inject,
+  forwardRef,
+  OnModuleInit,
+  OnModuleDestroy,
+} from '@nestjs/common';
 import { ConfigService } from '@nestjs/config';
 import * as k8s from '@kubernetes/client-node';
 import * as crypto from 'crypto';
+import { ChildProcess, spawn } from 'child_process';
 import { ClustersService } from '../clusters/clusters.service';
 import { HelmService, LOGGING_HELM_NAMESPACE, LOGGING_HELM_RELEASE } from './helm.service';
 
@@ -56,12 +65,18 @@ export interface LogStatsResult {
  * that all user apps can send logs to via Fluent Bit sidecars.
  */
 @Injectable()
-export class ElasticsearchService {
+export class ElasticsearchService implements OnModuleInit, OnModuleDestroy {
   private readonly logger = new Logger(ElasticsearchService.name);
   private readonly ES_NAMESPACE = 'logging';
   private readonly ES_NAME = 'elasticsearch';
   private readonly KIBANA_NAME = 'kibana';
-  
+  private portForwardChild: ChildProcess | null = null;
+  private portForwardStartedByUs = false;
+  private ensureInFlight: Promise<void> | null = null;
+  private reconnectTimer: ReturnType<typeof setTimeout> | null = null;
+  private healthCheckTimer: ReturnType<typeof setInterval> | null = null;
+  private reconnectAttempt = 0;
+
   // Default credentials - should be overridden via env in production
   private readonly ELASTIC_PASSWORD: string;
   private readonly FLUENTBIT_PASSWORD: string;
@@ -78,6 +93,254 @@ export class ElasticsearchService {
     this.KIBANA_SYSTEM_PASSWORD = this.configService.get('elasticsearch.kibanaPassword') || 'Kibana2024!System';
   }
 
+  async onModuleInit(): Promise<void> {
+    await this.ensureLocalElasticsearchAccess({ waitForCluster: true });
+    if (this.shouldAutoPortForward()) {
+      this.healthCheckTimer = setInterval(() => {
+        void this.periodicElasticsearchHealthCheck();
+      }, 30_000);
+    }
+  }
+
+  onModuleDestroy(): void {
+    if (this.healthCheckTimer) {
+      clearInterval(this.healthCheckTimer);
+      this.healthCheckTimer = null;
+    }
+    if (this.reconnectTimer) {
+      clearTimeout(this.reconnectTimer);
+      this.reconnectTimer = null;
+    }
+    this.stopDevPortForward();
+  }
+
+  private isLoopbackHost(host: string): boolean {
+    return host === '127.0.0.1' || host === 'localhost' || host === '::1';
+  }
+
+  private shouldAutoPortForward(): boolean {
+    if (this.configService.get<string>('elasticsearch.autoPortForward') === 'false') {
+      return false;
+    }
+    if (process.env.ELASTICSEARCH_AUTO_PORT_FORWARD === 'false') {
+      return false;
+    }
+    const nodeEnv = process.env.NODE_ENV || 'development';
+    if (nodeEnv === 'production') {
+      return false;
+    }
+    const host = this.configService.get<string>('elasticsearch.host') || '';
+    return this.isLoopbackHost(host);
+  }
+
+  private stopDevPortForward(): void {
+    if (!this.portForwardChild) {
+      return;
+    }
+    const startedByUs = this.portForwardStartedByUs;
+    const child = this.portForwardChild;
+    this.portForwardChild = null;
+    this.portForwardStartedByUs = false;
+    child.kill('SIGTERM');
+    if (startedByUs) {
+      this.logger.log('Stopped Elasticsearch kubectl port-forward');
+    }
+  }
+
+  private schedulePortForwardReconnect(reason: string): void {
+    if (!this.shouldAutoPortForward()) {
+      return;
+    }
+    if (this.reconnectTimer) {
+      return;
+    }
+    const delay = Math.min(60_000, 2_000 * Math.pow(2, this.reconnectAttempt));
+    this.reconnectAttempt += 1;
+    this.logger.warn(
+      `Elasticsearch port-forward lost (${reason}). Reconnecting in ${Math.round(delay / 1000)}s…`,
+    );
+    this.reconnectTimer = setTimeout(() => {
+      this.reconnectTimer = null;
+      void this.ensureLocalElasticsearchAccess().then((ok) => {
+        if (ok) {
+          this.reconnectAttempt = 0;
+        }
+      });
+    }, delay);
+  }
+
+  private async periodicElasticsearchHealthCheck(): Promise<void> {
+    if (!this.shouldAutoPortForward()) {
+      return;
+    }
+    const deployed = await this.isDeployed();
+    if (!deployed) {
+      return;
+    }
+    if (await this.probeElasticsearch()) {
+      this.reconnectAttempt = 0;
+      return;
+    }
+    this.logger.debug('Elasticsearch health check failed; restoring tunnel…');
+    await this.ensureLocalElasticsearchAccess();
+  }
+
+  private async waitForLoggingStack(maxWaitMs = 120_000): Promise<boolean> {
+    const started = Date.now();
+    while (Date.now() - started < maxWaitMs) {
+      if (await this.isDeployed()) {
+        return true;
+      }
+      await new Promise((r) => setTimeout(r, 5_000));
+    }
+    return false;
+  }
+
+  private async probeElasticsearch(timeoutMs = 3000): Promise<boolean> {
+    try {
+      const conn = this.getConnectionInfo();
+      const auth = Buffer.from(`${conn.username}:${conn.password}`).toString('base64');
+      const response = await fetch(`http://${conn.host}:${conn.port}/_cluster/health`, {
+        headers: { Authorization: `Basic ${auth}` },
+        signal: AbortSignal.timeout(timeoutMs),
+      });
+      return response.ok;
+    } catch {
+      return false;
+    }
+  }
+
+  private async waitForElasticsearch(maxWaitMs = 15_000): Promise<boolean> {
+    const started = Date.now();
+    while (Date.now() - started < maxWaitMs) {
+      if (await this.probeElasticsearch(2000)) {
+        return true;
+      }
+      await new Promise((r) => setTimeout(r, 400));
+    }
+    return false;
+  }
+
+  private startDevPortForward(localPort: number): void {
+    if (this.portForwardChild) {
+      return;
+    }
+    const args = [
+      'port-forward',
+      '-n',
+      this.ES_NAMESPACE,
+      `svc/${this.ES_NAME}`,
+      `${localPort}:9200`,
+    ];
+    this.logger.log(`Starting kubectl ${args.join(' ')} (local log search)`);
+    const child = spawn('kubectl', args, { stdio: ['ignore', 'pipe', 'pipe'] });
+    this.portForwardChild = child;
+    this.portForwardStartedByUs = true;
+    child.on('exit', (code, signal) => {
+      const wasOurs = this.portForwardChild === child;
+      if (wasOurs) {
+        this.portForwardChild = null;
+        this.portForwardStartedByUs = false;
+      }
+      if (wasOurs) {
+        const reason =
+          code !== 0 && code !== null
+            ? `exit code ${code}`
+            : signal
+              ? `signal ${signal}`
+              : 'connection closed';
+        this.schedulePortForwardReconnect(reason);
+      }
+    });
+    child.stderr?.on('data', (chunk: Buffer) => {
+      const line = chunk.toString().trim();
+      if (line && !line.includes('Handling connection')) {
+        this.logger.debug(`kubectl port-forward: ${line}`);
+      }
+    });
+  }
+
+  /**
+   * When the API runs on the host with ELASTICSEARCH_HOST=127.0.0.1, open a tunnel to the cluster.
+   * Safe to call repeatedly (e.g. after cluster/API restart or port-forward drop).
+   */
+  private async ensureLocalElasticsearchAccess(options?: {
+    waitForCluster?: boolean;
+  }): Promise<boolean> {
+    if (this.ensureInFlight) {
+      await this.ensureInFlight;
+      return this.probeElasticsearch();
+    }
+
+    this.ensureInFlight = this.ensureLocalElasticsearchAccessImpl(options);
+    try {
+      await this.ensureInFlight;
+      return this.probeElasticsearch();
+    } finally {
+      this.ensureInFlight = null;
+    }
+  }
+
+  private async ensureLocalElasticsearchAccessImpl(options?: {
+    waitForCluster?: boolean;
+  }): Promise<void> {
+    if (!this.shouldAutoPortForward()) {
+      return;
+    }
+
+    if (await this.probeElasticsearch()) {
+      this.reconnectAttempt = 0;
+      return;
+    }
+
+    let deployed = await this.isDeployed();
+    if (!deployed && options?.waitForCluster) {
+      this.logger.log('Waiting for logging stack after cluster reconnect…');
+      deployed = await this.waitForLoggingStack();
+    }
+    if (!deployed) {
+      return;
+    }
+
+    const port = this.configService.get<number>('elasticsearch.port') || 9200;
+
+    // Stale tunnel after sleep/reboot: port may be bound but ES unreachable
+    if (this.portForwardChild) {
+      this.stopDevPortForward();
+      await new Promise((r) => setTimeout(r, 300));
+    }
+
+    this.startDevPortForward(port);
+    const ready = await this.waitForElasticsearch(90_000);
+    if (ready) {
+      this.reconnectAttempt = 0;
+      this.logger.log(`Elasticsearch reachable at 127.0.0.1:${port}`);
+    } else {
+      this.stopDevPortForward();
+      this.logger.warn(
+        `Could not reach Elasticsearch on 127.0.0.1:${port}. Will retry. Manual: kubectl port-forward -n ${this.ES_NAMESPACE} svc/${this.ES_NAME} ${port}:9200`,
+      );
+      this.schedulePortForwardReconnect('probe timeout');
+    }
+  }
+
+  private localElasticsearchHint(): string {
+    const conn = this.getConnectionInfo();
+    if (this.isLoopbackHost(conn.host)) {
+      return (
+        `Ensure port ${conn.port} is forwarded to the cluster (the API auto-starts kubectl port-forward in development). ` +
+        `Manual: kubectl port-forward -n ${this.ES_NAMESPACE} svc/${this.ES_NAME} ${conn.port}:9200`
+      );
+    }
+    if (conn.host.includes('svc.cluster.local') || conn.host.includes('.cluster.')) {
+      return (
+        'Run the API inside the cluster, or set ELASTICSEARCH_HOST=127.0.0.1 and keep port-forward running: ' +
+        `kubectl port-forward -n ${this.ES_NAMESPACE} svc/${this.ES_NAME} ${conn.port}:9200`
+      );
+    }
+    return `Ensure Elasticsearch is listening on ${conn.host}:${conn.port}.`;
+  }
+
   private async getK8sClients(clusterId?: string) {
     const cluster = clusterId
       ? await this.clustersService.findOne(clusterId)
@@ -178,6 +441,10 @@ export class ElasticsearchService {
       elasticPassword: this.ELASTIC_PASSWORD,
       fluentbitPassword: this.FLUENTBIT_PASSWORD,
       kibanaSystemPassword: this.KIBANA_SYSTEM_PASSWORD,
+      images: {
+        elasticsearch: this.configService.get<string>('elasticsearch.images.elasticsearch'),
+        kibana: this.configService.get<string>('elasticsearch.images.kibana'),
+      },
     });
 
     this.logger.log(
@@ -214,8 +481,10 @@ export class ElasticsearchService {
    */
   getConnectionInfo(): { host: string; port: number; username: string; password: string } {
     return {
-      host: `${this.ES_NAME}.${this.ES_NAMESPACE}.svc.cluster.local`,
-      port: 9200,
+      host:
+        this.configService.get<string>('elasticsearch.host') ||
+        `${this.ES_NAME}.${this.ES_NAMESPACE}.svc.cluster.local`,
+      port: this.configService.get<number>('elasticsearch.port') || 9200,
       username: 'elastic',
       password: this.ELASTIC_PASSWORD,
     };
@@ -363,6 +632,18 @@ export class ElasticsearchService {
     return `logs-user-${userId.split('-')[0]}-*`;
   }
 
+  private elasticsearchFetch(url: string, auth: string, body: unknown): Promise<Response> {
+    return fetch(url, {
+      method: body === undefined ? 'GET' : 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        Authorization: `Basic ${auth}`,
+      },
+      body: body === undefined ? undefined : JSON.stringify(body),
+      signal: AbortSignal.timeout(15_000),
+    });
+  }
+
   private async esRequest(path: string, body: unknown, clusterId?: string): Promise<any> {
     const deployed = await this.isDeployed(clusterId);
     if (!deployed) {
@@ -375,14 +656,23 @@ export class ElasticsearchService {
     const url = `http://${conn.host}:${conn.port}${path}`;
     const auth = Buffer.from(`${conn.username}:${conn.password}`).toString('base64');
 
-    const response = await fetch(url, {
-      method: body === undefined ? 'GET' : 'POST',
-      headers: {
-        'Content-Type': 'application/json',
-        Authorization: `Basic ${auth}`,
-      },
-      body: body === undefined ? undefined : JSON.stringify(body),
-    });
+    let response: Response | undefined;
+    try {
+      response = await this.elasticsearchFetch(url, auth, body);
+    } catch (err: any) {
+      if (this.shouldAutoPortForward()) {
+        await this.ensureLocalElasticsearchAccess();
+        try {
+          response = await this.elasticsearchFetch(url, auth, body);
+        } catch {
+          // retry failed
+        }
+      }
+      if (!response) {
+        this.logger.warn(`Elasticsearch unreachable at ${conn.host}:${conn.port}: ${err?.message || err}`);
+        throw new ServiceUnavailableException(`Cannot reach Elasticsearch. ${this.localElasticsearchHint()}`);
+      }
+    }
 
     if (!response.ok) {
       const text = await response.text();
@@ -518,8 +808,36 @@ export class ElasticsearchService {
     return (result.hits?.hits || []).map((h: any) => this.normalizeHit(h));
   }
 
-  async getLoggingStatus(clusterId?: string): Promise<{ available: boolean; deployed: boolean }> {
-    const deployed = await this.isDeployed(clusterId);
-    return { available: deployed, deployed };
+  async getLoggingStatus(
+    clusterId?: string,
+  ): Promise<{ available: boolean; deployed: boolean; recovering?: boolean; message?: string }> {
+    let deployed = await this.isDeployed(clusterId);
+    if (!deployed && this.shouldAutoPortForward()) {
+      deployed = await this.waitForLoggingStack(8_000);
+    }
+    if (!deployed) {
+      return {
+        available: false,
+        deployed: false,
+        message: 'Central logging is not deployed. Ask an administrator to deploy Elasticsearch.',
+      };
+    }
+
+    if (this.shouldAutoPortForward() && !(await this.probeElasticsearch())) {
+      void this.ensureLocalElasticsearchAccess();
+    }
+
+    if (await this.probeElasticsearch()) {
+      return { available: true, deployed: true };
+    }
+
+    return {
+      available: false,
+      deployed: true,
+      recovering: this.shouldAutoPortForward(),
+      message: this.shouldAutoPortForward()
+        ? 'Reconnecting to Elasticsearch after cluster or API restart. This usually takes under a minute.'
+        : `Elasticsearch is running in the cluster, but this backend cannot reach it. ${this.localElasticsearchHint()}`,
+    };
   }
 }

backend/src/kubernetes/helm.service.ts

@@ -100,7 +100,12 @@ export class HelmService {
    */
   async installLoggingStack(
     kubeconfig: string,
-    values: { elasticPassword: string; fluentbitPassword: string; kibanaSystemPassword: string },
+    values: {
+      elasticPassword: string;
+      fluentbitPassword: string;
+      kibanaSystemPassword: string;
+      images?: { elasticsearch?: string; kibana?: string };
+    },
   ): Promise<{ stdout: string; stderr: string }> {
     return this.installOrUpgradeFromChart(
       'cloudhost-logging',
@@ -110,6 +115,16 @@ export class HelmService {
         elasticPassword: values.elasticPassword,
         fluentbitPassword: values.fluentbitPassword,
         kibanaSystemPassword: values.kibanaSystemPassword,
+        ...(values.images?.elasticsearch || values.images?.kibana
+          ? {
+              images: {
+                ...(values.images.elasticsearch
+                  ? { elasticsearch: values.images.elasticsearch }
+                  : {}),
+                ...(values.images.kibana ? { kibana: values.images.kibana } : {}),
+              },
+            }
+          : {}),
       },
       kubeconfig,
       { wait: true, timeout: '10m' },

backend/src/kubernetes/kubernetes.service.ts

@@ -303,6 +303,9 @@ export class KubernetesService implements OnModuleInit {
         logPaths: app.logPaths || [],
         ownerId: app.userId,
         applicationId: app.id,
+        elasticPassword: this.configService.get<string>('elasticsearch.password') || 'CloudHost2024!Secure',
+        fluentbitPassword: this.configService.get<string>('elasticsearch.fluentbitPassword') || 'FluentBit2024!Writer',
+        kibanaPassword: this.configService.get<string>('elasticsearch.kibanaPassword') || 'Kibana2024!System',
       },
       changeCause: `Deploy ${imageUri} at ${new Date().toISOString()}`,
     };
@@ -602,8 +605,9 @@ export class KubernetesService implements OnModuleInit {
         manifests.rabbitmq = true;
       }
 
-      // 3.7 Create Fluent Bit ConfigMap if Elasticsearch is enabled
+      // 3.7 Logging: credentials secret + Fluent Bit config
       if (context.enableElasticsearch) {
+        await this.ensureElasticsearchCredentialsSecret(coreApi, context.namespace);
         await this.createFluentBitConfigMap(coreApi, context);
         manifests.fluentBitConfig = true;
       }
@@ -874,6 +878,7 @@ export class KubernetesService implements OnModuleInit {
     // Add log volume mount if Elasticsearch is enabled
     if (ctx.enableElasticsearch) {
       appContainer.volumeMounts.push({ name: 'app-logs', mountPath: '/var/log/app' });
+      this.applyLoggingCommandWrapper(appContainer, ctx.runtime);
     }
 
     containers.push(appContainer);
@@ -943,21 +948,97 @@ export class KubernetesService implements OnModuleInit {
         return ['/var/www/html/storage/logs/*.log', '/var/log/app/*.log'];
       case AppRuntime.PHP:
         return ['/var/www/html/storage/logs/*.log', '/var/log/php/*.log', '/var/log/app/*.log'];
-      case AppRuntime.DJANGO:
-        return ['/app/logs/*.log', '/var/log/app/*.log'];
-      case AppRuntime.PYTHON:
-        return ['/app/logs/*.log', '/var/log/app/*.log'];
-      case AppRuntime.NODEJS:
-        return ['/app/logs/*.log', '/var/log/app/*.log'];
-      case AppRuntime.GO:
-        return ['/app/logs/*.log', '/var/log/app/*.log'];
-      case AppRuntime.DOTNET:
-        return ['/app/logs/*.log', '/var/log/app/*.log'];
       default:
+        // Node/Go/Python/.NET log to stdout — captured into /var/log/app/app.log at runtime
         return ['/var/log/app/*.log'];
     }
   }
 
+  /**
+   * Redirect stdout/stderr into the shared log volume so Fluent Bit can tail them.
+   */
+  private applyLoggingCommandWrapper(container: any, runtime: string): void {
+    const startCmd = this.getRuntimeStartCommand(runtime);
+    if (!startCmd) return;
+    container.command = ['sh', '-c'];
+    container.args = [`mkdir -p /var/log/app && (${startCmd}) >> /var/log/app/app.log 2>&1`];
+  }
+
+  /** Shell command that mirrors CloudHost-generated image ENTRYPOINT/CMD per runtime. */
+  private getRuntimeStartCommand(runtime: string): string | null {
+    switch (runtime) {
+      case AppRuntime.NODEJS:
+        return (
+          'if [ -f /app/.mode ] && [ "$(cat /app/.mode)" = "standalone" ] && [ -f server.js ]; ' +
+          'then node server.js; else npm start; fi'
+        );
+      case AppRuntime.GO:
+        return './main';
+      case AppRuntime.PYTHON:
+        return (
+          'if [ -f main.py ]; then ' +
+          'if grep -qi fastapi main.py; then exec uvicorn main:app --host 0.0.0.0 --port ${PORT:-3000}; ' +
+          'elif grep -qi flask main.py; then exec gunicorn -w 4 -b 0.0.0.0:${PORT:-3000} main:app; ' +
+          'else exec python main.py; fi; ' +
+          'elif [ -f app.py ]; then ' +
+          'if grep -qi fastapi app.py; then exec uvicorn app:app --host 0.0.0.0 --port ${PORT:-3000}; ' +
+          'elif grep -qi flask app.py; then exec gunicorn -w 4 -b 0.0.0.0:${PORT:-3000} app:app; ' +
+          'else exec python app.py; fi; ' +
+          'else exec gunicorn -w 4 -b 0.0.0.0:${PORT:-3000} app:app; fi'
+        );
+      case AppRuntime.DJANGO:
+        return 'python manage.py runserver 0.0.0.0:${PORT:-8000}';
+      case AppRuntime.DOTNET:
+        return (
+          'DLL=$(find . -maxdepth 1 -name "*.dll" ! -name "*.deps.dll" ! -name "*.runtimeconfig.dll" | head -1) ' +
+          '&& dotnet "$DLL"'
+        );
+      case AppRuntime.WORDPRESS:
+      case AppRuntime.LARAVEL:
+      case AppRuntime.PHP:
+        return null;
+      default:
+        return null;
+    }
+  }
+
+  /** Replicate logging credentials into the app namespace for Fluent Bit sidecars. */
+  private async ensureElasticsearchCredentialsSecret(
+    coreApi: k8s.CoreV1Api,
+    namespace: string,
+  ): Promise<void> {
+    const name = 'elasticsearch-credentials';
+    const stringData = {
+      ELASTIC_PASSWORD: this.configService.get<string>('elasticsearch.password') || 'CloudHost2024!Secure',
+      FLUENTBIT_PASSWORD: this.configService.get<string>('elasticsearch.fluentbitPassword') || 'FluentBit2024!Writer',
+      KIBANA_SYSTEM_PASSWORD: this.configService.get<string>('elasticsearch.kibanaPassword') || 'Kibana2024!System',
+    };
+
+    try {
+      await coreApi.readNamespacedSecret(name, namespace);
+      await coreApi.replaceNamespacedSecret(name, namespace, {
+        apiVersion: 'v1',
+        kind: 'Secret',
+        metadata: { name, namespace },
+        type: 'Opaque',
+        stringData,
+      });
+    } catch (err: any) {
+      if (err.statusCode === 404 || err.body?.code === 404) {
+        await coreApi.createNamespacedSecret(namespace, {
+          apiVersion: 'v1',
+          kind: 'Secret',
+          metadata: { name, namespace },
+          type: 'Opaque',
+          stringData,
+        });
+        this.logger.log(`Created ${name} secret in ${namespace}`);
+      } else {
+        throw err;
+      }
+    }
+  }
+
   /**
    * Build Fluent Bit configuration for log collection
    */
@@ -3755,6 +3836,56 @@ export class KubernetesService implements OnModuleInit {
     return bytes / (1024 * 1024 * 1024);
   }
 
+  /**
+   * Expand a named PVC (Redis, RabbitMQ, or other optional service volumes).
+   */
+  async resizeNamedPvc(
+    app: Application,
+    pvcName: string,
+    newSize: string,
+    label: string,
+  ): Promise<{ success: boolean; message: string }> {
+    const { coreApi } = await this.getK8sClient(app.clusterId);
+    const namespace = `user-${app.userId.split('-')[0]}`;
+
+    try {
+      const pvc = await coreApi.readNamespacedPersistentVolumeClaim(pvcName, namespace);
+      const currentSize = pvc.body.spec?.resources?.requests?.storage || '1Gi';
+      const parseGi = (s: string) => parseInt(String(s).replace(/Gi/i, ''), 10) || 0;
+
+      if (parseGi(newSize) <= parseGi(currentSize)) {
+        return {
+          success: false,
+          message: `New size (${newSize}) must be larger than current size (${currentSize})`,
+        };
+      }
+
+      await this.patchPvcStorageSize(coreApi, pvcName, namespace, newSize);
+      this.logger.log(`Expanded ${pvcName} from ${currentSize} to ${newSize}`);
+      return { success: true, message: `${label} storage expanded from ${currentSize} to ${newSize}` };
+    } catch (e: any) {
+      this.logger.error(`Failed to resize ${pvcName}: ${e.message}`);
+      return {
+        success: false,
+        message: e.body?.message || e.message || `Failed to resize ${label} storage`,
+      };
+    }
+  }
+
+  async resizeRedisStoragePvc(app: Application, newSize: string): Promise<{ success: boolean; message: string }> {
+    if (!app.enableRedis) {
+      return { success: false, message: 'Redis is not enabled for this application' };
+    }
+    return this.resizeNamedPvc(app, `${app.name}-redis-data`, newSize, 'Redis');
+  }
+
+  async resizeRabbitmqStoragePvc(app: Application, newSize: string): Promise<{ success: boolean; message: string }> {
+    if (!app.enableRabbitmq) {
+      return { success: false, message: 'RabbitMQ is not enabled for this application' };
+    }
+    return this.resizeNamedPvc(app, `${app.name}-rabbitmq-data`, newSize, 'RabbitMQ');
+  }
+
   /**
    * Resize app storage PVC (all app types).
    */