Improve logging recovery, resource scaling, and app deploy logging.
Auto-reconnect Elasticsearch port-forward after cluster or API restarts, poll log status in the UI, and apply storage changes through billing upgrade for all workloads. Add Redis/RabbitMQ PVC resize, Helm ES credentials for Fluent Bit, and fix deploy progress overlay behavior. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -1,7 +1,16 @@
|
||||
import { Injectable, Logger, ServiceUnavailableException, Inject, forwardRef } from '@nestjs/common';
|
||||
import {
|
||||
Injectable,
|
||||
Logger,
|
||||
ServiceUnavailableException,
|
||||
Inject,
|
||||
forwardRef,
|
||||
OnModuleInit,
|
||||
OnModuleDestroy,
|
||||
} from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import * as k8s from '@kubernetes/client-node';
|
||||
import * as crypto from 'crypto';
|
||||
import { ChildProcess, spawn } from 'child_process';
|
||||
import { ClustersService } from '../clusters/clusters.service';
|
||||
import { HelmService, LOGGING_HELM_NAMESPACE, LOGGING_HELM_RELEASE } from './helm.service';
|
||||
|
||||
@@ -56,12 +65,18 @@ export interface LogStatsResult {
|
||||
* that all user apps can send logs to via Fluent Bit sidecars.
|
||||
*/
|
||||
@Injectable()
|
||||
export class ElasticsearchService {
|
||||
export class ElasticsearchService implements OnModuleInit, OnModuleDestroy {
|
||||
private readonly logger = new Logger(ElasticsearchService.name);
|
||||
private readonly ES_NAMESPACE = 'logging';
|
||||
private readonly ES_NAME = 'elasticsearch';
|
||||
private readonly KIBANA_NAME = 'kibana';
|
||||
|
||||
private portForwardChild: ChildProcess | null = null;
|
||||
private portForwardStartedByUs = false;
|
||||
private ensureInFlight: Promise<void> | null = null;
|
||||
private reconnectTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
private healthCheckTimer: ReturnType<typeof setInterval> | null = null;
|
||||
private reconnectAttempt = 0;
|
||||
|
||||
// Default credentials - should be overridden via env in production
|
||||
private readonly ELASTIC_PASSWORD: string;
|
||||
private readonly FLUENTBIT_PASSWORD: string;
|
||||
@@ -78,6 +93,254 @@ export class ElasticsearchService {
|
||||
this.KIBANA_SYSTEM_PASSWORD = this.configService.get('elasticsearch.kibanaPassword') || 'Kibana2024!System';
|
||||
}
|
||||
|
||||
async onModuleInit(): Promise<void> {
|
||||
await this.ensureLocalElasticsearchAccess({ waitForCluster: true });
|
||||
if (this.shouldAutoPortForward()) {
|
||||
this.healthCheckTimer = setInterval(() => {
|
||||
void this.periodicElasticsearchHealthCheck();
|
||||
}, 30_000);
|
||||
}
|
||||
}
|
||||
|
||||
onModuleDestroy(): void {
|
||||
if (this.healthCheckTimer) {
|
||||
clearInterval(this.healthCheckTimer);
|
||||
this.healthCheckTimer = null;
|
||||
}
|
||||
if (this.reconnectTimer) {
|
||||
clearTimeout(this.reconnectTimer);
|
||||
this.reconnectTimer = null;
|
||||
}
|
||||
this.stopDevPortForward();
|
||||
}
|
||||
|
||||
private isLoopbackHost(host: string): boolean {
|
||||
return host === '127.0.0.1' || host === 'localhost' || host === '::1';
|
||||
}
|
||||
|
||||
private shouldAutoPortForward(): boolean {
|
||||
if (this.configService.get<string>('elasticsearch.autoPortForward') === 'false') {
|
||||
return false;
|
||||
}
|
||||
if (process.env.ELASTICSEARCH_AUTO_PORT_FORWARD === 'false') {
|
||||
return false;
|
||||
}
|
||||
const nodeEnv = process.env.NODE_ENV || 'development';
|
||||
if (nodeEnv === 'production') {
|
||||
return false;
|
||||
}
|
||||
const host = this.configService.get<string>('elasticsearch.host') || '';
|
||||
return this.isLoopbackHost(host);
|
||||
}
|
||||
|
||||
private stopDevPortForward(): void {
|
||||
if (!this.portForwardChild) {
|
||||
return;
|
||||
}
|
||||
const startedByUs = this.portForwardStartedByUs;
|
||||
const child = this.portForwardChild;
|
||||
this.portForwardChild = null;
|
||||
this.portForwardStartedByUs = false;
|
||||
child.kill('SIGTERM');
|
||||
if (startedByUs) {
|
||||
this.logger.log('Stopped Elasticsearch kubectl port-forward');
|
||||
}
|
||||
}
|
||||
|
||||
private schedulePortForwardReconnect(reason: string): void {
|
||||
if (!this.shouldAutoPortForward()) {
|
||||
return;
|
||||
}
|
||||
if (this.reconnectTimer) {
|
||||
return;
|
||||
}
|
||||
const delay = Math.min(60_000, 2_000 * Math.pow(2, this.reconnectAttempt));
|
||||
this.reconnectAttempt += 1;
|
||||
this.logger.warn(
|
||||
`Elasticsearch port-forward lost (${reason}). Reconnecting in ${Math.round(delay / 1000)}s…`,
|
||||
);
|
||||
this.reconnectTimer = setTimeout(() => {
|
||||
this.reconnectTimer = null;
|
||||
void this.ensureLocalElasticsearchAccess().then((ok) => {
|
||||
if (ok) {
|
||||
this.reconnectAttempt = 0;
|
||||
}
|
||||
});
|
||||
}, delay);
|
||||
}
|
||||
|
||||
private async periodicElasticsearchHealthCheck(): Promise<void> {
|
||||
if (!this.shouldAutoPortForward()) {
|
||||
return;
|
||||
}
|
||||
const deployed = await this.isDeployed();
|
||||
if (!deployed) {
|
||||
return;
|
||||
}
|
||||
if (await this.probeElasticsearch()) {
|
||||
this.reconnectAttempt = 0;
|
||||
return;
|
||||
}
|
||||
this.logger.debug('Elasticsearch health check failed; restoring tunnel…');
|
||||
await this.ensureLocalElasticsearchAccess();
|
||||
}
|
||||
|
||||
private async waitForLoggingStack(maxWaitMs = 120_000): Promise<boolean> {
|
||||
const started = Date.now();
|
||||
while (Date.now() - started < maxWaitMs) {
|
||||
if (await this.isDeployed()) {
|
||||
return true;
|
||||
}
|
||||
await new Promise((r) => setTimeout(r, 5_000));
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private async probeElasticsearch(timeoutMs = 3000): Promise<boolean> {
|
||||
try {
|
||||
const conn = this.getConnectionInfo();
|
||||
const auth = Buffer.from(`${conn.username}:${conn.password}`).toString('base64');
|
||||
const response = await fetch(`http://${conn.host}:${conn.port}/_cluster/health`, {
|
||||
headers: { Authorization: `Basic ${auth}` },
|
||||
signal: AbortSignal.timeout(timeoutMs),
|
||||
});
|
||||
return response.ok;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private async waitForElasticsearch(maxWaitMs = 15_000): Promise<boolean> {
|
||||
const started = Date.now();
|
||||
while (Date.now() - started < maxWaitMs) {
|
||||
if (await this.probeElasticsearch(2000)) {
|
||||
return true;
|
||||
}
|
||||
await new Promise((r) => setTimeout(r, 400));
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private startDevPortForward(localPort: number): void {
|
||||
if (this.portForwardChild) {
|
||||
return;
|
||||
}
|
||||
const args = [
|
||||
'port-forward',
|
||||
'-n',
|
||||
this.ES_NAMESPACE,
|
||||
`svc/${this.ES_NAME}`,
|
||||
`${localPort}:9200`,
|
||||
];
|
||||
this.logger.log(`Starting kubectl ${args.join(' ')} (local log search)`);
|
||||
const child = spawn('kubectl', args, { stdio: ['ignore', 'pipe', 'pipe'] });
|
||||
this.portForwardChild = child;
|
||||
this.portForwardStartedByUs = true;
|
||||
child.on('exit', (code, signal) => {
|
||||
const wasOurs = this.portForwardChild === child;
|
||||
if (wasOurs) {
|
||||
this.portForwardChild = null;
|
||||
this.portForwardStartedByUs = false;
|
||||
}
|
||||
if (wasOurs) {
|
||||
const reason =
|
||||
code !== 0 && code !== null
|
||||
? `exit code ${code}`
|
||||
: signal
|
||||
? `signal ${signal}`
|
||||
: 'connection closed';
|
||||
this.schedulePortForwardReconnect(reason);
|
||||
}
|
||||
});
|
||||
child.stderr?.on('data', (chunk: Buffer) => {
|
||||
const line = chunk.toString().trim();
|
||||
if (line && !line.includes('Handling connection')) {
|
||||
this.logger.debug(`kubectl port-forward: ${line}`);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* When the API runs on the host with ELASTICSEARCH_HOST=127.0.0.1, open a tunnel to the cluster.
|
||||
* Safe to call repeatedly (e.g. after cluster/API restart or port-forward drop).
|
||||
*/
|
||||
private async ensureLocalElasticsearchAccess(options?: {
|
||||
waitForCluster?: boolean;
|
||||
}): Promise<boolean> {
|
||||
if (this.ensureInFlight) {
|
||||
await this.ensureInFlight;
|
||||
return this.probeElasticsearch();
|
||||
}
|
||||
|
||||
this.ensureInFlight = this.ensureLocalElasticsearchAccessImpl(options);
|
||||
try {
|
||||
await this.ensureInFlight;
|
||||
return this.probeElasticsearch();
|
||||
} finally {
|
||||
this.ensureInFlight = null;
|
||||
}
|
||||
}
|
||||
|
||||
private async ensureLocalElasticsearchAccessImpl(options?: {
|
||||
waitForCluster?: boolean;
|
||||
}): Promise<void> {
|
||||
if (!this.shouldAutoPortForward()) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (await this.probeElasticsearch()) {
|
||||
this.reconnectAttempt = 0;
|
||||
return;
|
||||
}
|
||||
|
||||
let deployed = await this.isDeployed();
|
||||
if (!deployed && options?.waitForCluster) {
|
||||
this.logger.log('Waiting for logging stack after cluster reconnect…');
|
||||
deployed = await this.waitForLoggingStack();
|
||||
}
|
||||
if (!deployed) {
|
||||
return;
|
||||
}
|
||||
|
||||
const port = this.configService.get<number>('elasticsearch.port') || 9200;
|
||||
|
||||
// Stale tunnel after sleep/reboot: port may be bound but ES unreachable
|
||||
if (this.portForwardChild) {
|
||||
this.stopDevPortForward();
|
||||
await new Promise((r) => setTimeout(r, 300));
|
||||
}
|
||||
|
||||
this.startDevPortForward(port);
|
||||
const ready = await this.waitForElasticsearch(90_000);
|
||||
if (ready) {
|
||||
this.reconnectAttempt = 0;
|
||||
this.logger.log(`Elasticsearch reachable at 127.0.0.1:${port}`);
|
||||
} else {
|
||||
this.stopDevPortForward();
|
||||
this.logger.warn(
|
||||
`Could not reach Elasticsearch on 127.0.0.1:${port}. Will retry. Manual: kubectl port-forward -n ${this.ES_NAMESPACE} svc/${this.ES_NAME} ${port}:9200`,
|
||||
);
|
||||
this.schedulePortForwardReconnect('probe timeout');
|
||||
}
|
||||
}
|
||||
|
||||
private localElasticsearchHint(): string {
|
||||
const conn = this.getConnectionInfo();
|
||||
if (this.isLoopbackHost(conn.host)) {
|
||||
return (
|
||||
`Ensure port ${conn.port} is forwarded to the cluster (the API auto-starts kubectl port-forward in development). ` +
|
||||
`Manual: kubectl port-forward -n ${this.ES_NAMESPACE} svc/${this.ES_NAME} ${conn.port}:9200`
|
||||
);
|
||||
}
|
||||
if (conn.host.includes('svc.cluster.local') || conn.host.includes('.cluster.')) {
|
||||
return (
|
||||
'Run the API inside the cluster, or set ELASTICSEARCH_HOST=127.0.0.1 and keep port-forward running: ' +
|
||||
`kubectl port-forward -n ${this.ES_NAMESPACE} svc/${this.ES_NAME} ${conn.port}:9200`
|
||||
);
|
||||
}
|
||||
return `Ensure Elasticsearch is listening on ${conn.host}:${conn.port}.`;
|
||||
}
|
||||
|
||||
private async getK8sClients(clusterId?: string) {
|
||||
const cluster = clusterId
|
||||
? await this.clustersService.findOne(clusterId)
|
||||
@@ -178,6 +441,10 @@ export class ElasticsearchService {
|
||||
elasticPassword: this.ELASTIC_PASSWORD,
|
||||
fluentbitPassword: this.FLUENTBIT_PASSWORD,
|
||||
kibanaSystemPassword: this.KIBANA_SYSTEM_PASSWORD,
|
||||
images: {
|
||||
elasticsearch: this.configService.get<string>('elasticsearch.images.elasticsearch'),
|
||||
kibana: this.configService.get<string>('elasticsearch.images.kibana'),
|
||||
},
|
||||
});
|
||||
|
||||
this.logger.log(
|
||||
@@ -214,8 +481,10 @@ export class ElasticsearchService {
|
||||
*/
|
||||
getConnectionInfo(): { host: string; port: number; username: string; password: string } {
|
||||
return {
|
||||
host: `${this.ES_NAME}.${this.ES_NAMESPACE}.svc.cluster.local`,
|
||||
port: 9200,
|
||||
host:
|
||||
this.configService.get<string>('elasticsearch.host') ||
|
||||
`${this.ES_NAME}.${this.ES_NAMESPACE}.svc.cluster.local`,
|
||||
port: this.configService.get<number>('elasticsearch.port') || 9200,
|
||||
username: 'elastic',
|
||||
password: this.ELASTIC_PASSWORD,
|
||||
};
|
||||
@@ -363,6 +632,18 @@ export class ElasticsearchService {
|
||||
return `logs-user-${userId.split('-')[0]}-*`;
|
||||
}
|
||||
|
||||
private elasticsearchFetch(url: string, auth: string, body: unknown): Promise<Response> {
|
||||
return fetch(url, {
|
||||
method: body === undefined ? 'GET' : 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Basic ${auth}`,
|
||||
},
|
||||
body: body === undefined ? undefined : JSON.stringify(body),
|
||||
signal: AbortSignal.timeout(15_000),
|
||||
});
|
||||
}
|
||||
|
||||
private async esRequest(path: string, body: unknown, clusterId?: string): Promise<any> {
|
||||
const deployed = await this.isDeployed(clusterId);
|
||||
if (!deployed) {
|
||||
@@ -375,14 +656,23 @@ export class ElasticsearchService {
|
||||
const url = `http://${conn.host}:${conn.port}${path}`;
|
||||
const auth = Buffer.from(`${conn.username}:${conn.password}`).toString('base64');
|
||||
|
||||
const response = await fetch(url, {
|
||||
method: body === undefined ? 'GET' : 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Basic ${auth}`,
|
||||
},
|
||||
body: body === undefined ? undefined : JSON.stringify(body),
|
||||
});
|
||||
let response: Response | undefined;
|
||||
try {
|
||||
response = await this.elasticsearchFetch(url, auth, body);
|
||||
} catch (err: any) {
|
||||
if (this.shouldAutoPortForward()) {
|
||||
await this.ensureLocalElasticsearchAccess();
|
||||
try {
|
||||
response = await this.elasticsearchFetch(url, auth, body);
|
||||
} catch {
|
||||
// retry failed
|
||||
}
|
||||
}
|
||||
if (!response) {
|
||||
this.logger.warn(`Elasticsearch unreachable at ${conn.host}:${conn.port}: ${err?.message || err}`);
|
||||
throw new ServiceUnavailableException(`Cannot reach Elasticsearch. ${this.localElasticsearchHint()}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
const text = await response.text();
|
||||
@@ -518,8 +808,36 @@ export class ElasticsearchService {
|
||||
return (result.hits?.hits || []).map((h: any) => this.normalizeHit(h));
|
||||
}
|
||||
|
||||
async getLoggingStatus(clusterId?: string): Promise<{ available: boolean; deployed: boolean }> {
|
||||
const deployed = await this.isDeployed(clusterId);
|
||||
return { available: deployed, deployed };
|
||||
async getLoggingStatus(
|
||||
clusterId?: string,
|
||||
): Promise<{ available: boolean; deployed: boolean; recovering?: boolean; message?: string }> {
|
||||
let deployed = await this.isDeployed(clusterId);
|
||||
if (!deployed && this.shouldAutoPortForward()) {
|
||||
deployed = await this.waitForLoggingStack(8_000);
|
||||
}
|
||||
if (!deployed) {
|
||||
return {
|
||||
available: false,
|
||||
deployed: false,
|
||||
message: 'Central logging is not deployed. Ask an administrator to deploy Elasticsearch.',
|
||||
};
|
||||
}
|
||||
|
||||
if (this.shouldAutoPortForward() && !(await this.probeElasticsearch())) {
|
||||
void this.ensureLocalElasticsearchAccess();
|
||||
}
|
||||
|
||||
if (await this.probeElasticsearch()) {
|
||||
return { available: true, deployed: true };
|
||||
}
|
||||
|
||||
return {
|
||||
available: false,
|
||||
deployed: true,
|
||||
recovering: this.shouldAutoPortForward(),
|
||||
message: this.shouldAutoPortForward()
|
||||
? 'Reconnecting to Elasticsearch after cluster or API restart. This usually takes under a minute.'
|
||||
: `Elasticsearch is running in the cluster, but this backend cannot reach it. ${this.localElasticsearchHint()}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -100,7 +100,12 @@ export class HelmService {
|
||||
*/
|
||||
async installLoggingStack(
|
||||
kubeconfig: string,
|
||||
values: { elasticPassword: string; fluentbitPassword: string; kibanaSystemPassword: string },
|
||||
values: {
|
||||
elasticPassword: string;
|
||||
fluentbitPassword: string;
|
||||
kibanaSystemPassword: string;
|
||||
images?: { elasticsearch?: string; kibana?: string };
|
||||
},
|
||||
): Promise<{ stdout: string; stderr: string }> {
|
||||
return this.installOrUpgradeFromChart(
|
||||
'cloudhost-logging',
|
||||
@@ -110,6 +115,16 @@ export class HelmService {
|
||||
elasticPassword: values.elasticPassword,
|
||||
fluentbitPassword: values.fluentbitPassword,
|
||||
kibanaSystemPassword: values.kibanaSystemPassword,
|
||||
...(values.images?.elasticsearch || values.images?.kibana
|
||||
? {
|
||||
images: {
|
||||
...(values.images.elasticsearch
|
||||
? { elasticsearch: values.images.elasticsearch }
|
||||
: {}),
|
||||
...(values.images.kibana ? { kibana: values.images.kibana } : {}),
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
kubeconfig,
|
||||
{ wait: true, timeout: '10m' },
|
||||
|
||||
@@ -303,6 +303,9 @@ export class KubernetesService implements OnModuleInit {
|
||||
logPaths: app.logPaths || [],
|
||||
ownerId: app.userId,
|
||||
applicationId: app.id,
|
||||
elasticPassword: this.configService.get<string>('elasticsearch.password') || 'CloudHost2024!Secure',
|
||||
fluentbitPassword: this.configService.get<string>('elasticsearch.fluentbitPassword') || 'FluentBit2024!Writer',
|
||||
kibanaPassword: this.configService.get<string>('elasticsearch.kibanaPassword') || 'Kibana2024!System',
|
||||
},
|
||||
changeCause: `Deploy ${imageUri} at ${new Date().toISOString()}`,
|
||||
};
|
||||
@@ -602,8 +605,9 @@ export class KubernetesService implements OnModuleInit {
|
||||
manifests.rabbitmq = true;
|
||||
}
|
||||
|
||||
// 3.7 Create Fluent Bit ConfigMap if Elasticsearch is enabled
|
||||
// 3.7 Logging: credentials secret + Fluent Bit config
|
||||
if (context.enableElasticsearch) {
|
||||
await this.ensureElasticsearchCredentialsSecret(coreApi, context.namespace);
|
||||
await this.createFluentBitConfigMap(coreApi, context);
|
||||
manifests.fluentBitConfig = true;
|
||||
}
|
||||
@@ -874,6 +878,7 @@ export class KubernetesService implements OnModuleInit {
|
||||
// Add log volume mount if Elasticsearch is enabled
|
||||
if (ctx.enableElasticsearch) {
|
||||
appContainer.volumeMounts.push({ name: 'app-logs', mountPath: '/var/log/app' });
|
||||
this.applyLoggingCommandWrapper(appContainer, ctx.runtime);
|
||||
}
|
||||
|
||||
containers.push(appContainer);
|
||||
@@ -943,21 +948,97 @@ export class KubernetesService implements OnModuleInit {
|
||||
return ['/var/www/html/storage/logs/*.log', '/var/log/app/*.log'];
|
||||
case AppRuntime.PHP:
|
||||
return ['/var/www/html/storage/logs/*.log', '/var/log/php/*.log', '/var/log/app/*.log'];
|
||||
case AppRuntime.DJANGO:
|
||||
return ['/app/logs/*.log', '/var/log/app/*.log'];
|
||||
case AppRuntime.PYTHON:
|
||||
return ['/app/logs/*.log', '/var/log/app/*.log'];
|
||||
case AppRuntime.NODEJS:
|
||||
return ['/app/logs/*.log', '/var/log/app/*.log'];
|
||||
case AppRuntime.GO:
|
||||
return ['/app/logs/*.log', '/var/log/app/*.log'];
|
||||
case AppRuntime.DOTNET:
|
||||
return ['/app/logs/*.log', '/var/log/app/*.log'];
|
||||
default:
|
||||
// Node/Go/Python/.NET log to stdout — captured into /var/log/app/app.log at runtime
|
||||
return ['/var/log/app/*.log'];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Redirect stdout/stderr into the shared log volume so Fluent Bit can tail them.
|
||||
*/
|
||||
private applyLoggingCommandWrapper(container: any, runtime: string): void {
|
||||
const startCmd = this.getRuntimeStartCommand(runtime);
|
||||
if (!startCmd) return;
|
||||
container.command = ['sh', '-c'];
|
||||
container.args = [`mkdir -p /var/log/app && (${startCmd}) >> /var/log/app/app.log 2>&1`];
|
||||
}
|
||||
|
||||
/** Shell command that mirrors CloudHost-generated image ENTRYPOINT/CMD per runtime. */
|
||||
private getRuntimeStartCommand(runtime: string): string | null {
|
||||
switch (runtime) {
|
||||
case AppRuntime.NODEJS:
|
||||
return (
|
||||
'if [ -f /app/.mode ] && [ "$(cat /app/.mode)" = "standalone" ] && [ -f server.js ]; ' +
|
||||
'then node server.js; else npm start; fi'
|
||||
);
|
||||
case AppRuntime.GO:
|
||||
return './main';
|
||||
case AppRuntime.PYTHON:
|
||||
return (
|
||||
'if [ -f main.py ]; then ' +
|
||||
'if grep -qi fastapi main.py; then exec uvicorn main:app --host 0.0.0.0 --port ${PORT:-3000}; ' +
|
||||
'elif grep -qi flask main.py; then exec gunicorn -w 4 -b 0.0.0.0:${PORT:-3000} main:app; ' +
|
||||
'else exec python main.py; fi; ' +
|
||||
'elif [ -f app.py ]; then ' +
|
||||
'if grep -qi fastapi app.py; then exec uvicorn app:app --host 0.0.0.0 --port ${PORT:-3000}; ' +
|
||||
'elif grep -qi flask app.py; then exec gunicorn -w 4 -b 0.0.0.0:${PORT:-3000} app:app; ' +
|
||||
'else exec python app.py; fi; ' +
|
||||
'else exec gunicorn -w 4 -b 0.0.0.0:${PORT:-3000} app:app; fi'
|
||||
);
|
||||
case AppRuntime.DJANGO:
|
||||
return 'python manage.py runserver 0.0.0.0:${PORT:-8000}';
|
||||
case AppRuntime.DOTNET:
|
||||
return (
|
||||
'DLL=$(find . -maxdepth 1 -name "*.dll" ! -name "*.deps.dll" ! -name "*.runtimeconfig.dll" | head -1) ' +
|
||||
'&& dotnet "$DLL"'
|
||||
);
|
||||
case AppRuntime.WORDPRESS:
|
||||
case AppRuntime.LARAVEL:
|
||||
case AppRuntime.PHP:
|
||||
return null;
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Replicate logging credentials into the app namespace for Fluent Bit sidecars. */
|
||||
private async ensureElasticsearchCredentialsSecret(
|
||||
coreApi: k8s.CoreV1Api,
|
||||
namespace: string,
|
||||
): Promise<void> {
|
||||
const name = 'elasticsearch-credentials';
|
||||
const stringData = {
|
||||
ELASTIC_PASSWORD: this.configService.get<string>('elasticsearch.password') || 'CloudHost2024!Secure',
|
||||
FLUENTBIT_PASSWORD: this.configService.get<string>('elasticsearch.fluentbitPassword') || 'FluentBit2024!Writer',
|
||||
KIBANA_SYSTEM_PASSWORD: this.configService.get<string>('elasticsearch.kibanaPassword') || 'Kibana2024!System',
|
||||
};
|
||||
|
||||
try {
|
||||
await coreApi.readNamespacedSecret(name, namespace);
|
||||
await coreApi.replaceNamespacedSecret(name, namespace, {
|
||||
apiVersion: 'v1',
|
||||
kind: 'Secret',
|
||||
metadata: { name, namespace },
|
||||
type: 'Opaque',
|
||||
stringData,
|
||||
});
|
||||
} catch (err: any) {
|
||||
if (err.statusCode === 404 || err.body?.code === 404) {
|
||||
await coreApi.createNamespacedSecret(namespace, {
|
||||
apiVersion: 'v1',
|
||||
kind: 'Secret',
|
||||
metadata: { name, namespace },
|
||||
type: 'Opaque',
|
||||
stringData,
|
||||
});
|
||||
this.logger.log(`Created ${name} secret in ${namespace}`);
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build Fluent Bit configuration for log collection
|
||||
*/
|
||||
@@ -3755,6 +3836,56 @@ export class KubernetesService implements OnModuleInit {
|
||||
return bytes / (1024 * 1024 * 1024);
|
||||
}
|
||||
|
||||
/**
|
||||
* Expand a named PVC (Redis, RabbitMQ, or other optional service volumes).
|
||||
*/
|
||||
async resizeNamedPvc(
|
||||
app: Application,
|
||||
pvcName: string,
|
||||
newSize: string,
|
||||
label: string,
|
||||
): Promise<{ success: boolean; message: string }> {
|
||||
const { coreApi } = await this.getK8sClient(app.clusterId);
|
||||
const namespace = `user-${app.userId.split('-')[0]}`;
|
||||
|
||||
try {
|
||||
const pvc = await coreApi.readNamespacedPersistentVolumeClaim(pvcName, namespace);
|
||||
const currentSize = pvc.body.spec?.resources?.requests?.storage || '1Gi';
|
||||
const parseGi = (s: string) => parseInt(String(s).replace(/Gi/i, ''), 10) || 0;
|
||||
|
||||
if (parseGi(newSize) <= parseGi(currentSize)) {
|
||||
return {
|
||||
success: false,
|
||||
message: `New size (${newSize}) must be larger than current size (${currentSize})`,
|
||||
};
|
||||
}
|
||||
|
||||
await this.patchPvcStorageSize(coreApi, pvcName, namespace, newSize);
|
||||
this.logger.log(`Expanded ${pvcName} from ${currentSize} to ${newSize}`);
|
||||
return { success: true, message: `${label} storage expanded from ${currentSize} to ${newSize}` };
|
||||
} catch (e: any) {
|
||||
this.logger.error(`Failed to resize ${pvcName}: ${e.message}`);
|
||||
return {
|
||||
success: false,
|
||||
message: e.body?.message || e.message || `Failed to resize ${label} storage`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async resizeRedisStoragePvc(app: Application, newSize: string): Promise<{ success: boolean; message: string }> {
|
||||
if (!app.enableRedis) {
|
||||
return { success: false, message: 'Redis is not enabled for this application' };
|
||||
}
|
||||
return this.resizeNamedPvc(app, `${app.name}-redis-data`, newSize, 'Redis');
|
||||
}
|
||||
|
||||
async resizeRabbitmqStoragePvc(app: Application, newSize: string): Promise<{ success: boolean; message: string }> {
|
||||
if (!app.enableRabbitmq) {
|
||||
return { success: false, message: 'RabbitMQ is not enabled for this application' };
|
||||
}
|
||||
return this.resizeNamedPvc(app, `${app.name}-rabbitmq-data`, newSize, 'RabbitMQ');
|
||||
}
|
||||
|
||||
/**
|
||||
* Resize app storage PVC (all app types).
|
||||
*/
|
||||
|
||||
Reference in New Issue
Block a user