Improve logging recovery, resource scaling, and app deploy logging.
Auto-reconnect Elasticsearch port-forward after cluster or API restarts, poll log status in the UI, and apply storage changes through billing upgrade for all workloads. Add Redis/RabbitMQ PVC resize, Helm ES credentials for Fluent Bit, and fix deploy progress overlay behavior. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
keyhan
@@ -303,6 +303,9 @@ export class KubernetesService implements OnModuleInit {
|
||||
logPaths: app.logPaths || [],
|
||||
ownerId: app.userId,
|
||||
applicationId: app.id,
|
||||
elasticPassword: this.configService.get<string>('elasticsearch.password') || 'CloudHost2024!Secure',
|
||||
fluentbitPassword: this.configService.get<string>('elasticsearch.fluentbitPassword') || 'FluentBit2024!Writer',
|
||||
kibanaPassword: this.configService.get<string>('elasticsearch.kibanaPassword') || 'Kibana2024!System',
|
||||
},
|
||||
changeCause: `Deploy ${imageUri} at ${new Date().toISOString()}`,
|
||||
};
|
||||
@@ -602,8 +605,9 @@ export class KubernetesService implements OnModuleInit {
|
||||
manifests.rabbitmq = true;
|
||||
}
|
||||
|
||||
// 3.7 Create Fluent Bit ConfigMap if Elasticsearch is enabled
|
||||
// 3.7 Logging: credentials secret + Fluent Bit config
|
||||
if (context.enableElasticsearch) {
|
||||
await this.ensureElasticsearchCredentialsSecret(coreApi, context.namespace);
|
||||
await this.createFluentBitConfigMap(coreApi, context);
|
||||
manifests.fluentBitConfig = true;
|
||||
}
|
||||
@@ -874,6 +878,7 @@ export class KubernetesService implements OnModuleInit {
|
||||
// Add log volume mount if Elasticsearch is enabled
|
||||
if (ctx.enableElasticsearch) {
|
||||
appContainer.volumeMounts.push({ name: 'app-logs', mountPath: '/var/log/app' });
|
||||
this.applyLoggingCommandWrapper(appContainer, ctx.runtime);
|
||||
}
|
||||
|
||||
containers.push(appContainer);
|
||||
@@ -943,21 +948,97 @@ export class KubernetesService implements OnModuleInit {
|
||||
return ['/var/www/html/storage/logs/*.log', '/var/log/app/*.log'];
|
||||
case AppRuntime.PHP:
|
||||
return ['/var/www/html/storage/logs/*.log', '/var/log/php/*.log', '/var/log/app/*.log'];
|
||||
case AppRuntime.DJANGO:
|
||||
return ['/app/logs/*.log', '/var/log/app/*.log'];
|
||||
case AppRuntime.PYTHON:
|
||||
return ['/app/logs/*.log', '/var/log/app/*.log'];
|
||||
case AppRuntime.NODEJS:
|
||||
return ['/app/logs/*.log', '/var/log/app/*.log'];
|
||||
case AppRuntime.GO:
|
||||
return ['/app/logs/*.log', '/var/log/app/*.log'];
|
||||
case AppRuntime.DOTNET:
|
||||
return ['/app/logs/*.log', '/var/log/app/*.log'];
|
||||
default:
|
||||
// Node/Go/Python/.NET log to stdout — captured into /var/log/app/app.log at runtime
|
||||
return ['/var/log/app/*.log'];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Redirect stdout/stderr into the shared log volume so Fluent Bit can tail them.
|
||||
*/
|
||||
private applyLoggingCommandWrapper(container: any, runtime: string): void {
|
||||
const startCmd = this.getRuntimeStartCommand(runtime);
|
||||
if (!startCmd) return;
|
||||
container.command = ['sh', '-c'];
|
||||
container.args = [`mkdir -p /var/log/app && (${startCmd}) >> /var/log/app/app.log 2>&1`];
|
||||
}
|
||||
|
||||
/** Shell command that mirrors CloudHost-generated image ENTRYPOINT/CMD per runtime. */
|
||||
private getRuntimeStartCommand(runtime: string): string | null {
|
||||
switch (runtime) {
|
||||
case AppRuntime.NODEJS:
|
||||
return (
|
||||
'if [ -f /app/.mode ] && [ "$(cat /app/.mode)" = "standalone" ] && [ -f server.js ]; ' +
|
||||
'then node server.js; else npm start; fi'
|
||||
);
|
||||
case AppRuntime.GO:
|
||||
return './main';
|
||||
case AppRuntime.PYTHON:
|
||||
return (
|
||||
'if [ -f main.py ]; then ' +
|
||||
'if grep -qi fastapi main.py; then exec uvicorn main:app --host 0.0.0.0 --port ${PORT:-3000}; ' +
|
||||
'elif grep -qi flask main.py; then exec gunicorn -w 4 -b 0.0.0.0:${PORT:-3000} main:app; ' +
|
||||
'else exec python main.py; fi; ' +
|
||||
'elif [ -f app.py ]; then ' +
|
||||
'if grep -qi fastapi app.py; then exec uvicorn app:app --host 0.0.0.0 --port ${PORT:-3000}; ' +
|
||||
'elif grep -qi flask app.py; then exec gunicorn -w 4 -b 0.0.0.0:${PORT:-3000} app:app; ' +
|
||||
'else exec python app.py; fi; ' +
|
||||
'else exec gunicorn -w 4 -b 0.0.0.0:${PORT:-3000} app:app; fi'
|
||||
);
|
||||
case AppRuntime.DJANGO:
|
||||
return 'python manage.py runserver 0.0.0.0:${PORT:-8000}';
|
||||
case AppRuntime.DOTNET:
|
||||
return (
|
||||
'DLL=$(find . -maxdepth 1 -name "*.dll" ! -name "*.deps.dll" ! -name "*.runtimeconfig.dll" | head -1) ' +
|
||||
'&& dotnet "$DLL"'
|
||||
);
|
||||
case AppRuntime.WORDPRESS:
|
||||
case AppRuntime.LARAVEL:
|
||||
case AppRuntime.PHP:
|
||||
return null;
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Replicate logging credentials into the app namespace for Fluent Bit sidecars. */
|
||||
private async ensureElasticsearchCredentialsSecret(
|
||||
coreApi: k8s.CoreV1Api,
|
||||
namespace: string,
|
||||
): Promise<void> {
|
||||
const name = 'elasticsearch-credentials';
|
||||
const stringData = {
|
||||
ELASTIC_PASSWORD: this.configService.get<string>('elasticsearch.password') || 'CloudHost2024!Secure',
|
||||
FLUENTBIT_PASSWORD: this.configService.get<string>('elasticsearch.fluentbitPassword') || 'FluentBit2024!Writer',
|
||||
KIBANA_SYSTEM_PASSWORD: this.configService.get<string>('elasticsearch.kibanaPassword') || 'Kibana2024!System',
|
||||
};
|
||||
|
||||
try {
|
||||
await coreApi.readNamespacedSecret(name, namespace);
|
||||
await coreApi.replaceNamespacedSecret(name, namespace, {
|
||||
apiVersion: 'v1',
|
||||
kind: 'Secret',
|
||||
metadata: { name, namespace },
|
||||
type: 'Opaque',
|
||||
stringData,
|
||||
});
|
||||
} catch (err: any) {
|
||||
if (err.statusCode === 404 || err.body?.code === 404) {
|
||||
await coreApi.createNamespacedSecret(namespace, {
|
||||
apiVersion: 'v1',
|
||||
kind: 'Secret',
|
||||
metadata: { name, namespace },
|
||||
type: 'Opaque',
|
||||
stringData,
|
||||
});
|
||||
this.logger.log(`Created ${name} secret in ${namespace}`);
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build Fluent Bit configuration for log collection
|
||||
*/
|
||||
@@ -3755,6 +3836,56 @@ export class KubernetesService implements OnModuleInit {
|
||||
return bytes / (1024 * 1024 * 1024);
|
||||
}
|
||||
|
||||
/**
|
||||
* Expand a named PVC (Redis, RabbitMQ, or other optional service volumes).
|
||||
*/
|
||||
async resizeNamedPvc(
|
||||
app: Application,
|
||||
pvcName: string,
|
||||
newSize: string,
|
||||
label: string,
|
||||
): Promise<{ success: boolean; message: string }> {
|
||||
const { coreApi } = await this.getK8sClient(app.clusterId);
|
||||
const namespace = `user-${app.userId.split('-')[0]}`;
|
||||
|
||||
try {
|
||||
const pvc = await coreApi.readNamespacedPersistentVolumeClaim(pvcName, namespace);
|
||||
const currentSize = pvc.body.spec?.resources?.requests?.storage || '1Gi';
|
||||
const parseGi = (s: string) => parseInt(String(s).replace(/Gi/i, ''), 10) || 0;
|
||||
|
||||
if (parseGi(newSize) <= parseGi(currentSize)) {
|
||||
return {
|
||||
success: false,
|
||||
message: `New size (${newSize}) must be larger than current size (${currentSize})`,
|
||||
};
|
||||
}
|
||||
|
||||
await this.patchPvcStorageSize(coreApi, pvcName, namespace, newSize);
|
||||
this.logger.log(`Expanded ${pvcName} from ${currentSize} to ${newSize}`);
|
||||
return { success: true, message: `${label} storage expanded from ${currentSize} to ${newSize}` };
|
||||
} catch (e: any) {
|
||||
this.logger.error(`Failed to resize ${pvcName}: ${e.message}`);
|
||||
return {
|
||||
success: false,
|
||||
message: e.body?.message || e.message || `Failed to resize ${label} storage`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async resizeRedisStoragePvc(app: Application, newSize: string): Promise<{ success: boolean; message: string }> {
|
||||
if (!app.enableRedis) {
|
||||
return { success: false, message: 'Redis is not enabled for this application' };
|
||||
}
|
||||
return this.resizeNamedPvc(app, `${app.name}-redis-data`, newSize, 'Redis');
|
||||
}
|
||||
|
||||
async resizeRabbitmqStoragePvc(app: Application, newSize: string): Promise<{ success: boolean; message: string }> {
|
||||
if (!app.enableRabbitmq) {
|
||||
return { success: false, message: 'RabbitMQ is not enabled for this application' };
|
||||
}
|
||||
return this.resizeNamedPvc(app, `${app.name}-rabbitmq-data`, newSize, 'RabbitMQ');
|
||||
}
|
||||
|
||||
/**
|
||||
* Resize app storage PVC (all app types).
|
||||
*/
|
||||
|
||||
Reference in New Issue
Block a user