Improve logging recovery, resource scaling, and app deploy logging.

Auto-reconnect Elasticsearch port-forward after cluster or API restarts, poll log status in the UI, and apply storage changes through billing upgrade for all workloads. Add Redis/RabbitMQ PVC resize, Helm ES credentials for Fluent Bit, and fix deploy progress overlay behavior.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
keyhan
2026-05-25 21:41:00 +03:30
parent bf9e827f85
commit dc9830383b
18 changed files with 1068 additions and 414 deletions
+334 -16
View File
@@ -1,7 +1,16 @@
import { Injectable, Logger, ServiceUnavailableException, Inject, forwardRef } from '@nestjs/common';
import {
Injectable,
Logger,
ServiceUnavailableException,
Inject,
forwardRef,
OnModuleInit,
OnModuleDestroy,
} from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import * as k8s from '@kubernetes/client-node';
import * as crypto from 'crypto';
import { ChildProcess, spawn } from 'child_process';
import { ClustersService } from '../clusters/clusters.service';
import { HelmService, LOGGING_HELM_NAMESPACE, LOGGING_HELM_RELEASE } from './helm.service';
@@ -56,12 +65,18 @@ export interface LogStatsResult {
* that all user apps can send logs to via Fluent Bit sidecars.
*/
@Injectable()
export class ElasticsearchService {
export class ElasticsearchService implements OnModuleInit, OnModuleDestroy {
private readonly logger = new Logger(ElasticsearchService.name);
private readonly ES_NAMESPACE = 'logging';
private readonly ES_NAME = 'elasticsearch';
private readonly KIBANA_NAME = 'kibana';
private portForwardChild: ChildProcess | null = null;
private portForwardStartedByUs = false;
private ensureInFlight: Promise<void> | null = null;
private reconnectTimer: ReturnType<typeof setTimeout> | null = null;
private healthCheckTimer: ReturnType<typeof setInterval> | null = null;
private reconnectAttempt = 0;
// Default credentials - should be overridden via env in production
private readonly ELASTIC_PASSWORD: string;
private readonly FLUENTBIT_PASSWORD: string;
@@ -78,6 +93,254 @@ export class ElasticsearchService {
this.KIBANA_SYSTEM_PASSWORD = this.configService.get('elasticsearch.kibanaPassword') || 'Kibana2024!System';
}
async onModuleInit(): Promise<void> {
await this.ensureLocalElasticsearchAccess({ waitForCluster: true });
if (this.shouldAutoPortForward()) {
this.healthCheckTimer = setInterval(() => {
void this.periodicElasticsearchHealthCheck();
}, 30_000);
}
}
onModuleDestroy(): void {
if (this.healthCheckTimer) {
clearInterval(this.healthCheckTimer);
this.healthCheckTimer = null;
}
if (this.reconnectTimer) {
clearTimeout(this.reconnectTimer);
this.reconnectTimer = null;
}
this.stopDevPortForward();
}
private isLoopbackHost(host: string): boolean {
return host === '127.0.0.1' || host === 'localhost' || host === '::1';
}
private shouldAutoPortForward(): boolean {
if (this.configService.get<string>('elasticsearch.autoPortForward') === 'false') {
return false;
}
if (process.env.ELASTICSEARCH_AUTO_PORT_FORWARD === 'false') {
return false;
}
const nodeEnv = process.env.NODE_ENV || 'development';
if (nodeEnv === 'production') {
return false;
}
const host = this.configService.get<string>('elasticsearch.host') || '';
return this.isLoopbackHost(host);
}
private stopDevPortForward(): void {
if (!this.portForwardChild) {
return;
}
const startedByUs = this.portForwardStartedByUs;
const child = this.portForwardChild;
this.portForwardChild = null;
this.portForwardStartedByUs = false;
child.kill('SIGTERM');
if (startedByUs) {
this.logger.log('Stopped Elasticsearch kubectl port-forward');
}
}
private schedulePortForwardReconnect(reason: string): void {
if (!this.shouldAutoPortForward()) {
return;
}
if (this.reconnectTimer) {
return;
}
const delay = Math.min(60_000, 2_000 * Math.pow(2, this.reconnectAttempt));
this.reconnectAttempt += 1;
this.logger.warn(
`Elasticsearch port-forward lost (${reason}). Reconnecting in ${Math.round(delay / 1000)}s…`,
);
this.reconnectTimer = setTimeout(() => {
this.reconnectTimer = null;
void this.ensureLocalElasticsearchAccess().then((ok) => {
if (ok) {
this.reconnectAttempt = 0;
}
});
}, delay);
}
private async periodicElasticsearchHealthCheck(): Promise<void> {
if (!this.shouldAutoPortForward()) {
return;
}
const deployed = await this.isDeployed();
if (!deployed) {
return;
}
if (await this.probeElasticsearch()) {
this.reconnectAttempt = 0;
return;
}
this.logger.debug('Elasticsearch health check failed; restoring tunnel…');
await this.ensureLocalElasticsearchAccess();
}
private async waitForLoggingStack(maxWaitMs = 120_000): Promise<boolean> {
const started = Date.now();
while (Date.now() - started < maxWaitMs) {
if (await this.isDeployed()) {
return true;
}
await new Promise((r) => setTimeout(r, 5_000));
}
return false;
}
private async probeElasticsearch(timeoutMs = 3000): Promise<boolean> {
try {
const conn = this.getConnectionInfo();
const auth = Buffer.from(`${conn.username}:${conn.password}`).toString('base64');
const response = await fetch(`http://${conn.host}:${conn.port}/_cluster/health`, {
headers: { Authorization: `Basic ${auth}` },
signal: AbortSignal.timeout(timeoutMs),
});
return response.ok;
} catch {
return false;
}
}
private async waitForElasticsearch(maxWaitMs = 15_000): Promise<boolean> {
const started = Date.now();
while (Date.now() - started < maxWaitMs) {
if (await this.probeElasticsearch(2000)) {
return true;
}
await new Promise((r) => setTimeout(r, 400));
}
return false;
}
private startDevPortForward(localPort: number): void {
if (this.portForwardChild) {
return;
}
const args = [
'port-forward',
'-n',
this.ES_NAMESPACE,
`svc/${this.ES_NAME}`,
`${localPort}:9200`,
];
this.logger.log(`Starting kubectl ${args.join(' ')} (local log search)`);
const child = spawn('kubectl', args, { stdio: ['ignore', 'pipe', 'pipe'] });
this.portForwardChild = child;
this.portForwardStartedByUs = true;
child.on('exit', (code, signal) => {
const wasOurs = this.portForwardChild === child;
if (wasOurs) {
this.portForwardChild = null;
this.portForwardStartedByUs = false;
}
if (wasOurs) {
const reason =
code !== 0 && code !== null
? `exit code ${code}`
: signal
? `signal ${signal}`
: 'connection closed';
this.schedulePortForwardReconnect(reason);
}
});
child.stderr?.on('data', (chunk: Buffer) => {
const line = chunk.toString().trim();
if (line && !line.includes('Handling connection')) {
this.logger.debug(`kubectl port-forward: ${line}`);
}
});
}
/**
* When the API runs on the host with ELASTICSEARCH_HOST=127.0.0.1, open a tunnel to the cluster.
* Safe to call repeatedly (e.g. after cluster/API restart or port-forward drop).
*/
private async ensureLocalElasticsearchAccess(options?: {
waitForCluster?: boolean;
}): Promise<boolean> {
if (this.ensureInFlight) {
await this.ensureInFlight;
return this.probeElasticsearch();
}
this.ensureInFlight = this.ensureLocalElasticsearchAccessImpl(options);
try {
await this.ensureInFlight;
return this.probeElasticsearch();
} finally {
this.ensureInFlight = null;
}
}
private async ensureLocalElasticsearchAccessImpl(options?: {
waitForCluster?: boolean;
}): Promise<void> {
if (!this.shouldAutoPortForward()) {
return;
}
if (await this.probeElasticsearch()) {
this.reconnectAttempt = 0;
return;
}
let deployed = await this.isDeployed();
if (!deployed && options?.waitForCluster) {
this.logger.log('Waiting for logging stack after cluster reconnect…');
deployed = await this.waitForLoggingStack();
}
if (!deployed) {
return;
}
const port = this.configService.get<number>('elasticsearch.port') || 9200;
// Stale tunnel after sleep/reboot: port may be bound but ES unreachable
if (this.portForwardChild) {
this.stopDevPortForward();
await new Promise((r) => setTimeout(r, 300));
}
this.startDevPortForward(port);
const ready = await this.waitForElasticsearch(90_000);
if (ready) {
this.reconnectAttempt = 0;
this.logger.log(`Elasticsearch reachable at 127.0.0.1:${port}`);
} else {
this.stopDevPortForward();
this.logger.warn(
`Could not reach Elasticsearch on 127.0.0.1:${port}. Will retry. Manual: kubectl port-forward -n ${this.ES_NAMESPACE} svc/${this.ES_NAME} ${port}:9200`,
);
this.schedulePortForwardReconnect('probe timeout');
}
}
private localElasticsearchHint(): string {
const conn = this.getConnectionInfo();
if (this.isLoopbackHost(conn.host)) {
return (
`Ensure port ${conn.port} is forwarded to the cluster (the API auto-starts kubectl port-forward in development). ` +
`Manual: kubectl port-forward -n ${this.ES_NAMESPACE} svc/${this.ES_NAME} ${conn.port}:9200`
);
}
if (conn.host.includes('svc.cluster.local') || conn.host.includes('.cluster.')) {
return (
'Run the API inside the cluster, or set ELASTICSEARCH_HOST=127.0.0.1 and keep port-forward running: ' +
`kubectl port-forward -n ${this.ES_NAMESPACE} svc/${this.ES_NAME} ${conn.port}:9200`
);
}
return `Ensure Elasticsearch is listening on ${conn.host}:${conn.port}.`;
}
private async getK8sClients(clusterId?: string) {
const cluster = clusterId
? await this.clustersService.findOne(clusterId)
@@ -178,6 +441,10 @@ export class ElasticsearchService {
elasticPassword: this.ELASTIC_PASSWORD,
fluentbitPassword: this.FLUENTBIT_PASSWORD,
kibanaSystemPassword: this.KIBANA_SYSTEM_PASSWORD,
images: {
elasticsearch: this.configService.get<string>('elasticsearch.images.elasticsearch'),
kibana: this.configService.get<string>('elasticsearch.images.kibana'),
},
});
this.logger.log(
@@ -214,8 +481,10 @@ export class ElasticsearchService {
*/
getConnectionInfo(): { host: string; port: number; username: string; password: string } {
return {
host: `${this.ES_NAME}.${this.ES_NAMESPACE}.svc.cluster.local`,
port: 9200,
host:
this.configService.get<string>('elasticsearch.host') ||
`${this.ES_NAME}.${this.ES_NAMESPACE}.svc.cluster.local`,
port: this.configService.get<number>('elasticsearch.port') || 9200,
username: 'elastic',
password: this.ELASTIC_PASSWORD,
};
@@ -363,6 +632,18 @@ export class ElasticsearchService {
return `logs-user-${userId.split('-')[0]}-*`;
}
private elasticsearchFetch(url: string, auth: string, body: unknown): Promise<Response> {
return fetch(url, {
method: body === undefined ? 'GET' : 'POST',
headers: {
'Content-Type': 'application/json',
Authorization: `Basic ${auth}`,
},
body: body === undefined ? undefined : JSON.stringify(body),
signal: AbortSignal.timeout(15_000),
});
}
private async esRequest(path: string, body: unknown, clusterId?: string): Promise<any> {
const deployed = await this.isDeployed(clusterId);
if (!deployed) {
@@ -375,14 +656,23 @@ export class ElasticsearchService {
const url = `http://${conn.host}:${conn.port}${path}`;
const auth = Buffer.from(`${conn.username}:${conn.password}`).toString('base64');
const response = await fetch(url, {
method: body === undefined ? 'GET' : 'POST',
headers: {
'Content-Type': 'application/json',
Authorization: `Basic ${auth}`,
},
body: body === undefined ? undefined : JSON.stringify(body),
});
let response: Response | undefined;
try {
response = await this.elasticsearchFetch(url, auth, body);
} catch (err: any) {
if (this.shouldAutoPortForward()) {
await this.ensureLocalElasticsearchAccess();
try {
response = await this.elasticsearchFetch(url, auth, body);
} catch {
// retry failed
}
}
if (!response) {
this.logger.warn(`Elasticsearch unreachable at ${conn.host}:${conn.port}: ${err?.message || err}`);
throw new ServiceUnavailableException(`Cannot reach Elasticsearch. ${this.localElasticsearchHint()}`);
}
}
if (!response.ok) {
const text = await response.text();
@@ -518,8 +808,36 @@ export class ElasticsearchService {
return (result.hits?.hits || []).map((h: any) => this.normalizeHit(h));
}
async getLoggingStatus(clusterId?: string): Promise<{ available: boolean; deployed: boolean }> {
const deployed = await this.isDeployed(clusterId);
return { available: deployed, deployed };
async getLoggingStatus(
clusterId?: string,
): Promise<{ available: boolean; deployed: boolean; recovering?: boolean; message?: string }> {
let deployed = await this.isDeployed(clusterId);
if (!deployed && this.shouldAutoPortForward()) {
deployed = await this.waitForLoggingStack(8_000);
}
if (!deployed) {
return {
available: false,
deployed: false,
message: 'Central logging is not deployed. Ask an administrator to deploy Elasticsearch.',
};
}
if (this.shouldAutoPortForward() && !(await this.probeElasticsearch())) {
void this.ensureLocalElasticsearchAccess();
}
if (await this.probeElasticsearch()) {
return { available: true, deployed: true };
}
return {
available: false,
deployed: true,
recovering: this.shouldAutoPortForward(),
message: this.shouldAutoPortForward()
? 'Reconnecting to Elasticsearch after cluster or API restart. This usually takes under a minute.'
: `Elasticsearch is running in the cluster, but this backend cannot reach it. ${this.localElasticsearchHint()}`,
};
}
}
+16 -1
View File
@@ -100,7 +100,12 @@ export class HelmService {
*/
async installLoggingStack(
kubeconfig: string,
values: { elasticPassword: string; fluentbitPassword: string; kibanaSystemPassword: string },
values: {
elasticPassword: string;
fluentbitPassword: string;
kibanaSystemPassword: string;
images?: { elasticsearch?: string; kibana?: string };
},
): Promise<{ stdout: string; stderr: string }> {
return this.installOrUpgradeFromChart(
'cloudhost-logging',
@@ -110,6 +115,16 @@ export class HelmService {
elasticPassword: values.elasticPassword,
fluentbitPassword: values.fluentbitPassword,
kibanaSystemPassword: values.kibanaSystemPassword,
...(values.images?.elasticsearch || values.images?.kibana
? {
images: {
...(values.images.elasticsearch
? { elasticsearch: values.images.elasticsearch }
: {}),
...(values.images.kibana ? { kibana: values.images.kibana } : {}),
},
}
: {}),
},
kubeconfig,
{ wait: true, timeout: '10m' },
+142 -11
View File
@@ -303,6 +303,9 @@ export class KubernetesService implements OnModuleInit {
logPaths: app.logPaths || [],
ownerId: app.userId,
applicationId: app.id,
elasticPassword: this.configService.get<string>('elasticsearch.password') || 'CloudHost2024!Secure',
fluentbitPassword: this.configService.get<string>('elasticsearch.fluentbitPassword') || 'FluentBit2024!Writer',
kibanaPassword: this.configService.get<string>('elasticsearch.kibanaPassword') || 'Kibana2024!System',
},
changeCause: `Deploy ${imageUri} at ${new Date().toISOString()}`,
};
@@ -602,8 +605,9 @@ export class KubernetesService implements OnModuleInit {
manifests.rabbitmq = true;
}
// 3.7 Create Fluent Bit ConfigMap if Elasticsearch is enabled
// 3.7 Logging: credentials secret + Fluent Bit config
if (context.enableElasticsearch) {
await this.ensureElasticsearchCredentialsSecret(coreApi, context.namespace);
await this.createFluentBitConfigMap(coreApi, context);
manifests.fluentBitConfig = true;
}
@@ -874,6 +878,7 @@ export class KubernetesService implements OnModuleInit {
// Add log volume mount if Elasticsearch is enabled
if (ctx.enableElasticsearch) {
appContainer.volumeMounts.push({ name: 'app-logs', mountPath: '/var/log/app' });
this.applyLoggingCommandWrapper(appContainer, ctx.runtime);
}
containers.push(appContainer);
@@ -943,21 +948,97 @@ export class KubernetesService implements OnModuleInit {
return ['/var/www/html/storage/logs/*.log', '/var/log/app/*.log'];
case AppRuntime.PHP:
return ['/var/www/html/storage/logs/*.log', '/var/log/php/*.log', '/var/log/app/*.log'];
case AppRuntime.DJANGO:
return ['/app/logs/*.log', '/var/log/app/*.log'];
case AppRuntime.PYTHON:
return ['/app/logs/*.log', '/var/log/app/*.log'];
case AppRuntime.NODEJS:
return ['/app/logs/*.log', '/var/log/app/*.log'];
case AppRuntime.GO:
return ['/app/logs/*.log', '/var/log/app/*.log'];
case AppRuntime.DOTNET:
return ['/app/logs/*.log', '/var/log/app/*.log'];
default:
// Node/Go/Python/.NET log to stdout — captured into /var/log/app/app.log at runtime
return ['/var/log/app/*.log'];
}
}
/**
* Redirect stdout/stderr into the shared log volume so Fluent Bit can tail them.
*/
private applyLoggingCommandWrapper(container: any, runtime: string): void {
const startCmd = this.getRuntimeStartCommand(runtime);
if (!startCmd) return;
container.command = ['sh', '-c'];
container.args = [`mkdir -p /var/log/app && (${startCmd}) >> /var/log/app/app.log 2>&1`];
}
/** Shell command that mirrors CloudHost-generated image ENTRYPOINT/CMD per runtime. */
private getRuntimeStartCommand(runtime: string): string | null {
switch (runtime) {
case AppRuntime.NODEJS:
return (
'if [ -f /app/.mode ] && [ "$(cat /app/.mode)" = "standalone" ] && [ -f server.js ]; ' +
'then node server.js; else npm start; fi'
);
case AppRuntime.GO:
return './main';
case AppRuntime.PYTHON:
return (
'if [ -f main.py ]; then ' +
'if grep -qi fastapi main.py; then exec uvicorn main:app --host 0.0.0.0 --port ${PORT:-3000}; ' +
'elif grep -qi flask main.py; then exec gunicorn -w 4 -b 0.0.0.0:${PORT:-3000} main:app; ' +
'else exec python main.py; fi; ' +
'elif [ -f app.py ]; then ' +
'if grep -qi fastapi app.py; then exec uvicorn app:app --host 0.0.0.0 --port ${PORT:-3000}; ' +
'elif grep -qi flask app.py; then exec gunicorn -w 4 -b 0.0.0.0:${PORT:-3000} app:app; ' +
'else exec python app.py; fi; ' +
'else exec gunicorn -w 4 -b 0.0.0.0:${PORT:-3000} app:app; fi'
);
case AppRuntime.DJANGO:
return 'python manage.py runserver 0.0.0.0:${PORT:-8000}';
case AppRuntime.DOTNET:
return (
'DLL=$(find . -maxdepth 1 -name "*.dll" ! -name "*.deps.dll" ! -name "*.runtimeconfig.dll" | head -1) ' +
'&& dotnet "$DLL"'
);
case AppRuntime.WORDPRESS:
case AppRuntime.LARAVEL:
case AppRuntime.PHP:
return null;
default:
return null;
}
}
/** Replicate logging credentials into the app namespace for Fluent Bit sidecars. */
private async ensureElasticsearchCredentialsSecret(
coreApi: k8s.CoreV1Api,
namespace: string,
): Promise<void> {
const name = 'elasticsearch-credentials';
const stringData = {
ELASTIC_PASSWORD: this.configService.get<string>('elasticsearch.password') || 'CloudHost2024!Secure',
FLUENTBIT_PASSWORD: this.configService.get<string>('elasticsearch.fluentbitPassword') || 'FluentBit2024!Writer',
KIBANA_SYSTEM_PASSWORD: this.configService.get<string>('elasticsearch.kibanaPassword') || 'Kibana2024!System',
};
try {
await coreApi.readNamespacedSecret(name, namespace);
await coreApi.replaceNamespacedSecret(name, namespace, {
apiVersion: 'v1',
kind: 'Secret',
metadata: { name, namespace },
type: 'Opaque',
stringData,
});
} catch (err: any) {
if (err.statusCode === 404 || err.body?.code === 404) {
await coreApi.createNamespacedSecret(namespace, {
apiVersion: 'v1',
kind: 'Secret',
metadata: { name, namespace },
type: 'Opaque',
stringData,
});
this.logger.log(`Created ${name} secret in ${namespace}`);
} else {
throw err;
}
}
}
/**
* Build Fluent Bit configuration for log collection
*/
@@ -3755,6 +3836,56 @@ export class KubernetesService implements OnModuleInit {
return bytes / (1024 * 1024 * 1024);
}
/**
* Expand a named PVC (Redis, RabbitMQ, or other optional service volumes).
*/
async resizeNamedPvc(
app: Application,
pvcName: string,
newSize: string,
label: string,
): Promise<{ success: boolean; message: string }> {
const { coreApi } = await this.getK8sClient(app.clusterId);
const namespace = `user-${app.userId.split('-')[0]}`;
try {
const pvc = await coreApi.readNamespacedPersistentVolumeClaim(pvcName, namespace);
const currentSize = pvc.body.spec?.resources?.requests?.storage || '1Gi';
const parseGi = (s: string) => parseInt(String(s).replace(/Gi/i, ''), 10) || 0;
if (parseGi(newSize) <= parseGi(currentSize)) {
return {
success: false,
message: `New size (${newSize}) must be larger than current size (${currentSize})`,
};
}
await this.patchPvcStorageSize(coreApi, pvcName, namespace, newSize);
this.logger.log(`Expanded ${pvcName} from ${currentSize} to ${newSize}`);
return { success: true, message: `${label} storage expanded from ${currentSize} to ${newSize}` };
} catch (e: any) {
this.logger.error(`Failed to resize ${pvcName}: ${e.message}`);
return {
success: false,
message: e.body?.message || e.message || `Failed to resize ${label} storage`,
};
}
}
async resizeRedisStoragePvc(app: Application, newSize: string): Promise<{ success: boolean; message: string }> {
if (!app.enableRedis) {
return { success: false, message: 'Redis is not enabled for this application' };
}
return this.resizeNamedPvc(app, `${app.name}-redis-data`, newSize, 'Redis');
}
async resizeRabbitmqStoragePvc(app: Application, newSize: string): Promise<{ success: boolean; message: string }> {
if (!app.enableRabbitmq) {
return { success: false, message: 'RabbitMQ is not enabled for this application' };
}
return this.resizeNamedPvc(app, `${app.name}-rabbitmq-data`, newSize, 'RabbitMQ');
}
/**
* Resize app storage PVC (all app types).
*/