abrban/cloud-host
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add unified logs platform with Helm-managed central Elasticsearch.

Browse Source 
Deploy cloudhost-logging on cluster registration, ship app and optional service logs to ES with owner isolation, and fix Kibana 8.12 auth via kibana_system.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
keyhan
2026-05-15 15:56:33 +03:30
parent 2303985d0c
commit 35dd771f63
31 changed files with 1657 additions and 938 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/kubernetes/elasticsearch.service.ts
+315 -315
View File
@@ -1,21 +1,53 @@
import { Injectable, Logger } from '@nestjs/common';
import { Injectable, Logger, ServiceUnavailableException, Inject, forwardRef } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import * as k8s from '@kubernetes/client-node';
import * as crypto from 'crypto';
import { ClustersService } from '../clusters/clusters.service';
import { HelmService, LOGGING_HELM_NAMESPACE, LOGGING_HELM_RELEASE } from './helm.service';
interface ElasticsearchCredentials {
username: string;
password: string;
}
interface LogEntry {
export interface LogSearchFilters {
applicationId?: string;
applicationName?: string;
workload?: string;
level?: string;
from?: string;
to?: string;
search?: string;
page?: number;
limit?: number;
}
export interface NormalizedLogEntry {
id: string;
timestamp: string;
level: string;
message: string;
app: string;
namespace: string;
[key: string]: any;
applicationId?: string;
applicationName?: string;
workload?: string;
namespace?: string;
pod?: string;
}
export interface LogSearchResult {
hits: NormalizedLogEntry[];
total: number;
page: number;
limit: number;
}
export interface LogStatsResult {
total: number;
errors: number;
warnings: number;
byLevel: Record<string, number>;
byWorkload: Record<string, number>;
period: string;
}
/**
@@ -33,13 +65,17 @@ export class ElasticsearchService {
// Default credentials - should be overridden via env in production
private readonly ELASTIC_PASSWORD: string;
private readonly FLUENTBIT_PASSWORD: string;
private readonly KIBANA_SYSTEM_PASSWORD: string;
constructor(
@Inject(forwardRef(() => ClustersService))
private clustersService: ClustersService,
private configService: ConfigService,
private helmService: HelmService,
) {
this.ELASTIC_PASSWORD = this.configService.get('elasticsearch.password') || 'CloudHost2024!Secure';
this.FLUENTBIT_PASSWORD = this.configService.get('elasticsearch.fluentbitPassword') || 'FluentBit2024!Writer';
this.KIBANA_SYSTEM_PASSWORD = this.configService.get('elasticsearch.kibanaPassword') || 'Kibana2024!System';
}
private async getK8sClients(clusterId?: string) {
@@ -133,25 +169,20 @@ export class ElasticsearchService {
}
/**
* Deploy central Elasticsearch + Kibana stack
* This should be called once per cluster by admin
* Deploy central Elasticsearch + Kibana stack via Helm.
*/
async deploy(clusterId?: string): Promise<{ esPassword: string; kibanaUrl: string }> {
const { coreApi, appsApi } = await this.getK8sClients(clusterId);
const { cluster } = await this.getK8sClients(clusterId);
// 1. Create logging namespace
await this.ensureNamespace(coreApi);
await this.helmService.installLoggingStack(cluster.kubeconfig, {
elasticPassword: this.ELASTIC_PASSWORD,
fluentbitPassword: this.FLUENTBIT_PASSWORD,
kibanaSystemPassword: this.KIBANA_SYSTEM_PASSWORD,
});
// 2. Create credentials secret
await this.createCredentialsSecret(coreApi);
// 3. Deploy Elasticsearch
await this.deployElasticsearch(coreApi, appsApi);
// 4. Deploy Kibana
await this.deployKibana(coreApi, appsApi);
this.logger.log('Central Elasticsearch stack deployed successfully');
this.logger.log(
`Central logging stack deployed via Helm (${LOGGING_HELM_RELEASE} in ${LOGGING_HELM_NAMESPACE})`,
);
return {
esPassword: this.ELASTIC_PASSWORD,
@@ -160,300 +191,22 @@ export class ElasticsearchService {
}
/**
* Undeploy Elasticsearch stack
* Undeploy Elasticsearch stack (Helm release; PVC retained by chart policy).
*/
async undeploy(clusterId?: string): Promise<void> {
const { coreApi, appsApi } = await this.getK8sClients(clusterId);
const { cluster } = await this.getK8sClients(clusterId);
try {
// Delete Kibana
await appsApi.deleteNamespacedDeployment(this.KIBANA_NAME, this.ES_NAMESPACE);
await coreApi.deleteNamespacedService(this.KIBANA_NAME, this.ES_NAMESPACE);
this.logger.log('Kibana deleted');
await this.helmService.uninstall(LOGGING_HELM_RELEASE, LOGGING_HELM_NAMESPACE, cluster.kubeconfig);
this.logger.log('Elasticsearch stack undeployed via Helm (PVC preserved)');
} catch (e: any) {
if (e?.response?.statusCode !== 404) {
this.logger.warn(`Failed to delete Kibana: ${e.message}`);
const msg = e?.message || String(e);
if (msg.includes('not found')) {
this.logger.log('Logging Helm release not found — nothing to undeploy');
return;
}
throw e;
}
try {
// Delete Elasticsearch
await appsApi.deleteNamespacedStatefulSet(this.ES_NAME, this.ES_NAMESPACE);
await coreApi.deleteNamespacedService(this.ES_NAME, this.ES_NAMESPACE);
this.logger.log('Elasticsearch deleted');
} catch (e: any) {
if (e?.response?.statusCode !== 404) {
this.logger.warn(`Failed to delete Elasticsearch: ${e.message}`);
}
}
// Note: PVC is kept intentionally to preserve data
this.logger.log('Elasticsearch stack undeployed (PVC preserved)');
}
private async ensureNamespace(coreApi: k8s.CoreV1Api): Promise<void> {
try {
await coreApi.readNamespace(this.ES_NAMESPACE);
} catch {
await coreApi.createNamespace({
metadata: {
name: this.ES_NAMESPACE,
labels: { 'app.kubernetes.io/managed-by': 'cloudhost' },
},
});
this.logger.log(`Created namespace: ${this.ES_NAMESPACE}`);
}
}
private async createCredentialsSecret(coreApi: k8s.CoreV1Api): Promise<void> {
const secret = {
apiVersion: 'v1',
kind: 'Secret',
metadata: {
name: 'elasticsearch-credentials',
namespace: this.ES_NAMESPACE,
},
type: 'Opaque',
data: {
ELASTIC_PASSWORD: Buffer.from(this.ELASTIC_PASSWORD).toString('base64'),
FLUENTBIT_PASSWORD: Buffer.from(this.FLUENTBIT_PASSWORD).toString('base64'),
},
};
try {
await coreApi.replaceNamespacedSecret('elasticsearch-credentials', this.ES_NAMESPACE, secret);
} catch {
await coreApi.createNamespacedSecret(this.ES_NAMESPACE, secret);
}
}
private async deployElasticsearch(
coreApi: k8s.CoreV1Api,
appsApi: k8s.AppsV1Api,
): Promise<void> {
// PVC for Elasticsearch data
const pvc: k8s.V1PersistentVolumeClaim = {
apiVersion: 'v1',
kind: 'PersistentVolumeClaim',
metadata: { name: `${this.ES_NAME}-data`, namespace: this.ES_NAMESPACE },
spec: {
accessModes: ['ReadWriteOnce'],
resources: { requests: { storage: '50Gi' } },
},
};
try {
await coreApi.readNamespacedPersistentVolumeClaim(pvc.metadata!.name!, this.ES_NAMESPACE);
} catch {
await coreApi.createNamespacedPersistentVolumeClaim(this.ES_NAMESPACE, pvc);
}
// Base64 encoded auth header
const authHeader = Buffer.from(`elastic:${this.ELASTIC_PASSWORD}`).toString('base64');
// StatefulSet for Elasticsearch
const statefulSet: k8s.V1StatefulSet = {
apiVersion: 'apps/v1',
kind: 'StatefulSet',
metadata: {
name: this.ES_NAME,
namespace: this.ES_NAMESPACE,
labels: { app: this.ES_NAME },
},
spec: {
serviceName: this.ES_NAME,
replicas: 1,
selector: { matchLabels: { app: this.ES_NAME } },
template: {
metadata: { labels: { app: this.ES_NAME } },
spec: {
securityContext: { fsGroup: 1000 },
initContainers: [
{
name: 'fix-permissions',
image: 'busybox:1.36',
command: ['sh', '-c', 'chown -R 1000:1000 /usr/share/elasticsearch/data'],
securityContext: { runAsUser: 0, privileged: true },
volumeMounts: [{ name: 'es-data', mountPath: '/usr/share/elasticsearch/data' }],
},
{
name: 'increase-vm-max-map',
image: 'busybox:1.36',
command: ['sysctl', '-w', 'vm.max_map_count=262144'],
securityContext: { privileged: true },
},
],
containers: [
{
name: 'elasticsearch',
image: 'docker.elastic.co/elasticsearch/elasticsearch:8.12.0',
ports: [
{ containerPort: 9200, name: 'http' },
{ containerPort: 9300, name: 'transport' },
],
env: [
{ name: 'discovery.type', value: 'single-node' },
{ name: 'xpack.security.enabled', value: 'true' },
{ name: 'xpack.security.http.ssl.enabled', value: 'false' },
{ name: 'xpack.security.transport.ssl.enabled', value: 'false' },
{
name: 'ELASTIC_PASSWORD',
valueFrom: {
secretKeyRef: { name: 'elasticsearch-credentials', key: 'ELASTIC_PASSWORD' },
},
},
{ name: 'ES_JAVA_OPTS', value: '-Xms1g -Xmx1g' },
{ name: 'cluster.name', value: 'cloudhost-logs' },
{ name: 'bootstrap.memory_lock', value: 'false' },
],
resources: {
requests: { cpu: '500m', memory: '2Gi' },
limits: { cpu: '2000m', memory: '4Gi' },
},
volumeMounts: [{ name: 'es-data', mountPath: '/usr/share/elasticsearch/data' }],
readinessProbe: {
httpGet: {
path: '/_cluster/health?local=true',
port: 9200 as any,
httpHeaders: [{ name: 'Authorization', value: `Basic ${authHeader}` }],
},
initialDelaySeconds: 30,
periodSeconds: 10,
timeoutSeconds: 5,
},
livenessProbe: {
httpGet: {
path: '/_cluster/health?local=true',
port: 9200 as any,
httpHeaders: [{ name: 'Authorization', value: `Basic ${authHeader}` }],
},
initialDelaySeconds: 60,
periodSeconds: 30,
timeoutSeconds: 10,
},
},
],
volumes: [
{
name: 'es-data',
persistentVolumeClaim: { claimName: `${this.ES_NAME}-data` },
},
],
},
},
},
};
try {
await appsApi.replaceNamespacedStatefulSet(this.ES_NAME, this.ES_NAMESPACE, statefulSet);
} catch {
await appsApi.createNamespacedStatefulSet(this.ES_NAMESPACE, statefulSet);
}
// Service for Elasticsearch
const service: k8s.V1Service = {
apiVersion: 'v1',
kind: 'Service',
metadata: { name: this.ES_NAME, namespace: this.ES_NAMESPACE },
spec: {
selector: { app: this.ES_NAME },
ports: [
{ port: 9200, targetPort: 9200 as any, name: 'http' },
{ port: 9300, targetPort: 9300 as any, name: 'transport' },
],
type: 'ClusterIP',
},
};
try {
await coreApi.replaceNamespacedService(this.ES_NAME, this.ES_NAMESPACE, service);
} catch {
await coreApi.createNamespacedService(this.ES_NAMESPACE, service);
}
this.logger.log('Elasticsearch deployed');
}
private async deployKibana(
coreApi: k8s.CoreV1Api,
appsApi: k8s.AppsV1Api,
): Promise<void> {
const deployment: k8s.V1Deployment = {
apiVersion: 'apps/v1',
kind: 'Deployment',
metadata: {
name: this.KIBANA_NAME,
namespace: this.ES_NAMESPACE,
labels: { app: this.KIBANA_NAME },
},
spec: {
replicas: 1,
selector: { matchLabels: { app: this.KIBANA_NAME } },
template: {
metadata: { labels: { app: this.KIBANA_NAME } },
spec: {
containers: [
{
name: 'kibana',
image: 'docker.elastic.co/kibana/kibana:8.12.0',
ports: [{ containerPort: 5601 }],
env: [
{ name: 'ELASTICSEARCH_HOSTS', value: `http://${this.ES_NAME}:9200` },
{ name: 'ELASTICSEARCH_USERNAME', value: 'elastic' },
{
name: 'ELASTICSEARCH_PASSWORD',
valueFrom: {
secretKeyRef: { name: 'elasticsearch-credentials', key: 'ELASTIC_PASSWORD' },
},
},
{ name: 'SERVER_NAME', value: 'kibana' },
{ name: 'XPACK_SECURITY_ENABLED', value: 'true' },
],
resources: {
requests: { cpu: '200m', memory: '512Mi' },
limits: { cpu: '1000m', memory: '1Gi' },
},
readinessProbe: {
httpGet: { path: '/api/status', port: 5601 as any },
initialDelaySeconds: 30,
periodSeconds: 10,
},
livenessProbe: {
httpGet: { path: '/api/status', port: 5601 as any },
initialDelaySeconds: 60,
periodSeconds: 30,
},
},
],
},
},
},
};
try {
await appsApi.replaceNamespacedDeployment(this.KIBANA_NAME, this.ES_NAMESPACE, deployment);
} catch {
await appsApi.createNamespacedDeployment(this.ES_NAMESPACE, deployment);
}
const service: k8s.V1Service = {
apiVersion: 'v1',
kind: 'Service',
metadata: { name: this.KIBANA_NAME, namespace: this.ES_NAMESPACE },
spec: {
selector: { app: this.KIBANA_NAME },
ports: [{ port: 5601, targetPort: 5601 as any }],
type: 'ClusterIP',
},
};
try {
await coreApi.replaceNamespacedService(this.KIBANA_NAME, this.ES_NAMESPACE, service);
} catch {
await coreApi.createNamespacedService(this.ES_NAMESPACE, service);
}
this.logger.log('Kibana deployed');
}
/**
@@ -506,20 +259,267 @@ export class ElasticsearchService {
}
/**
* Build Elasticsearch query to filter logs by user/owner
* Users can only see logs from their own applications
* Build must clauses for user log isolation (new + legacy fields).
*/
getUserLogsQuery(userId: string): { query: { bool: { must: any } } } {
buildUserLogMustClauses(userId: string, filters: LogSearchFilters = {}): any[] {
const userPrefix = userId.split('-')[0];
const namespace = `user-${userPrefix}`;
const must: any[] = [
{
bool: {
should: [
{ term: { ownerId: userId } },
{ term: { 'ownerId.keyword': userId } },
{ term: { namespace } },
{ term: { 'namespace.keyword': namespace } },
],
minimum_should_match: 1,
},
},
];
if (filters.applicationId) {
must.push({
bool: {
should: [
{ term: { applicationId: filters.applicationId } },
{ term: { 'applicationId.keyword': filters.applicationId } },
],
minimum_should_match: 1,
},
});
}
if (filters.applicationName) {
must.push({
bool: {
should: [
{ term: { applicationName: filters.applicationName } },
{ term: { 'applicationName.keyword': filters.applicationName } },
{ term: { app: filters.applicationName } },
{ term: { 'app.keyword': filters.applicationName } },
],
minimum_should_match: 1,
},
});
}
if (filters.workload) {
must.push({
bool: {
should: [
{ term: { workload: filters.workload } },
{ term: { 'workload.keyword': filters.workload } },
],
minimum_should_match: 1,
},
});
}
if (filters.level) {
must.push({
bool: {
should: [
{ term: { level: filters.level.toLowerCase() } },
{ term: { 'level.keyword': filters.level.toLowerCase() } },
],
minimum_should_match: 1,
},
});
}
if (filters.from || filters.to) {
const rangeFilter: any = { range: { '@timestamp': {} } };
if (filters.from) rangeFilter.range['@timestamp'].gte = filters.from;
if (filters.to) rangeFilter.range['@timestamp'].lte = filters.to;
must.push(rangeFilter);
}
if (filters.search) {
must.push({
multi_match: {
query: filters.search,
fields: ['message', 'log', 'msg', 'error.message'],
type: 'phrase_prefix',
},
});
}
return must;
}
getUserLogsQuery(userId: string): { query: { bool: { must: any[] } } } {
return {
query: {
bool: {
must: {
term: {
'kubernetes.labels.owner': userId,
},
},
must: this.buildUserLogMustClauses(userId),
},
},
};
}
getUserIndexPattern(userId: string): string {
return `logs-user-${userId.split('-')[0]}-*`;
}
private async esRequest(path: string, body: unknown, clusterId?: string): Promise<any> {
const deployed = await this.isDeployed(clusterId);
if (!deployed) {
throw new ServiceUnavailableException(
'Central logging is not configured. Ask an administrator to deploy Elasticsearch.',
);
}
const conn = this.getConnectionInfo();
const url = `http://${conn.host}:${conn.port}${path}`;
const auth = Buffer.from(`${conn.username}:${conn.password}`).toString('base64');
const response = await fetch(url, {
method: body === undefined ? 'GET' : 'POST',
headers: {
'Content-Type': 'application/json',
Authorization: `Basic ${auth}`,
},
body: body === undefined ? undefined : JSON.stringify(body),
});
if (!response.ok) {
const text = await response.text();
this.logger.warn(`Elasticsearch request failed: ${response.status} ${text}`);
throw new ServiceUnavailableException('Failed to query log storage');
}
return response.json();
}
private normalizeHit(hit: any): NormalizedLogEntry {
const src = hit._source || {};
const message =
src.message ||
src.log ||
src.msg ||
(typeof src.error === 'string' ? src.error : src.error?.message) ||
'';
return {
id: hit._id || '',
timestamp: src['@timestamp'] || src.timestamp || new Date().toISOString(),
level: (src.level || 'info').toString().toLowerCase(),
message: typeof message === 'string' ? message : JSON.stringify(message),
applicationId: src.applicationId,
applicationName: src.applicationName || src.app,
workload: src.workload || 'app',
namespace: src.namespace,
pod: src.kubernetes?.pod_name || src.pod_name,
};
}
async searchLogs(userId: string, filters: LogSearchFilters, clusterId?: string): Promise<LogSearchResult> {
const page = filters.page || 1;
const limit = Math.min(filters.limit || 100, 1000);
const from = (page - 1) * limit;
const body = {
query: { bool: { must: this.buildUserLogMustClauses(userId, filters) } },
sort: [{ '@timestamp': 'desc' }],
from,
size: limit,
};
const index = this.getUserIndexPattern(userId);
const result = await this.esRequest(`/${index}/_search`, body, clusterId);
const hits = result.hits?.hits || [];
return {
hits: hits.map((h: any) => this.normalizeHit(h)),
total: result.hits?.total?.value ?? result.hits?.total ?? hits.length,
page,
limit,
};
}
async searchLogStats(
userId: string,
filters: { applicationId?: string; applicationName?: string; workload?: string; period?: string },
clusterId?: string,
): Promise<LogStatsResult> {
const periodMap: Record<string, string> = {
'1h': 'now-1h',
'6h': 'now-6h',
'24h': 'now-24h',
'7d': 'now-7d',
};
const period = filters.period || '24h';
const timeRange = periodMap[period] || 'now-24h';
const must = this.buildUserLogMustClauses(userId, {
applicationId: filters.applicationId,
applicationName: filters.applicationName,
workload: filters.workload,
});
must.push({ range: { '@timestamp': { gte: timeRange } } });
const body = {
query: { bool: { must } },
size: 0,
aggs: {
by_level: { terms: { field: 'level.keyword', size: 10, missing: 'unknown' } },
by_workload: { terms: { field: 'workload.keyword', size: 10, missing: 'app' } },
error_count: { filter: { term: { 'level.keyword': 'error' } } },
warn_count: { filter: { term: { 'level.keyword': 'warn' } } },
},
};
const index = this.getUserIndexPattern(userId);
const result = await this.esRequest(`/${index}/_search`, body, clusterId);
const byLevel: Record<string, number> = {};
for (const bucket of result.aggregations?.by_level?.buckets || []) {
byLevel[bucket.key] = bucket.doc_count;
}
const byWorkload: Record<string, number> = {};
for (const bucket of result.aggregations?.by_workload?.buckets || []) {
byWorkload[bucket.key] = bucket.doc_count;
}
const errors = result.aggregations?.error_count?.doc_count || 0;
const warnings = result.aggregations?.warn_count?.doc_count || 0;
const total = Object.values(byLevel).reduce((a, b) => a + b, 0);
return { total, errors, warnings, byLevel, byWorkload, period };
}
async searchRecentErrors(
userId: string,
filters: { applicationId?: string; applicationName?: string; workload?: string; hours?: number; limit?: number },
clusterId?: string,
): Promise<NormalizedLogEntry[]> {
const hours = filters.hours || 24;
const limit = Math.min(filters.limit || 50, 500);
const must = this.buildUserLogMustClauses(userId, {
applicationId: filters.applicationId,
applicationName: filters.applicationName,
workload: filters.workload,
level: 'error',
});
must.push({ range: { '@timestamp': { gte: `now-${hours}h` } } });
const body = {
query: { bool: { must } },
sort: [{ '@timestamp': 'desc' }],
size: limit,
};
const index = this.getUserIndexPattern(userId);
const result = await this.esRequest(`/${index}/_search`, body, clusterId);
return (result.hits?.hits || []).map((h: any) => this.normalizeHit(h));
}
async getLoggingStatus(clusterId?: string): Promise<{ available: boolean; deployed: boolean }> {
const deployed = await this.isDeployed(clusterId);
return { available: deployed, deployed };
}
}
backend/src/kubernetes/helm.service.ts
+84 -10
View File
@@ -25,25 +25,99 @@ export interface HelmRevision {
description: string;
}
export const LOGGING_HELM_RELEASE = 'cloudhost-logging';
export const LOGGING_HELM_NAMESPACE = 'logging';
export interface HelmInstallOptions {
wait?: boolean;
timeout?: string;
}
@Injectable()
export class HelmService {
private readonly logger = new Logger(HelmService.name);
private readonly chartPath: string;
private readonly appChartPath: string;
constructor() {
// In production (dist/kubernetes/), __dirname resolves to dist/kubernetes
// so we go up two levels to project root, then into helm/
// In Docker, the helm/ dir is copied alongside dist/ at /app/helm/
this.appChartPath = this.resolveChartPath('cloudhost-app');
}
private resolveChartPath(chartName: string): string {
const candidates = [
path.resolve(__dirname, '..', '..', 'helm', 'cloudhost-app'),
path.resolve(process.cwd(), 'helm', 'cloudhost-app'),
path.resolve(__dirname, '..', '..', 'helm', chartName),
path.resolve(process.cwd(), 'helm', chartName),
];
this.chartPath = candidates.find((p) => fs.existsSync(p)) || candidates[0];
return candidates.find((p) => fs.existsSync(p)) || candidates[0];
}
/**
* Install or upgrade a Helm release.
* Equivalent to: helm upgrade --install <release> <chart> -n <ns> --create-namespace -f <values>
* Install or upgrade a Helm release from a named chart directory.
*/
async installOrUpgradeFromChart(
chartName: string,
releaseName: string,
namespace: string,
values: Record<string, any>,
kubeconfig: string,
options: HelmInstallOptions = {},
): Promise<{ stdout: string; stderr: string }> {
const chartPath = this.resolveChartPath(chartName);
const kubeconfigFile = await this.writeTempKubeconfig(kubeconfig);
const valuesFile = await this.writeTempValues(values);
const wait = options.wait !== false;
const timeout = options.timeout || '5m';
try {
const args = [
'upgrade', '--install',
releaseName,
chartPath,
'--namespace', namespace,
'--create-namespace',
'--values', valuesFile,
'--history-max', '10',
'--kubeconfig', kubeconfigFile,
];
if (wait) {
args.push('--wait', '--timeout', timeout);
}
this.logger.log(`Helm install/upgrade: ${releaseName} (${chartName}) in ${namespace}`);
const result = await execFileAsync('helm', args, { timeout: 660_000 });
this.logger.log(`Helm release ${releaseName} installed/upgraded successfully`);
return result;
} catch (error: any) {
this.logger.error(`Helm install/upgrade failed for ${releaseName}: ${error.stderr || error.message}`);
throw new Error(`Helm install/upgrade failed: ${error.stderr || error.message}`);
} finally {
this.cleanupTempFiles(kubeconfigFile, valuesFile);
}
}
/**
* Install or upgrade the central logging stack (Elasticsearch + Kibana).
*/
async installLoggingStack(
kubeconfig: string,
values: { elasticPassword: string; fluentbitPassword: string; kibanaSystemPassword: string },
): Promise<{ stdout: string; stderr: string }> {
return this.installOrUpgradeFromChart(
'cloudhost-logging',
LOGGING_HELM_RELEASE,
LOGGING_HELM_NAMESPACE,
{
elasticPassword: values.elasticPassword,
fluentbitPassword: values.fluentbitPassword,
kibanaSystemPassword: values.kibanaSystemPassword,
},
kubeconfig,
{ wait: true, timeout: '10m' },
);
}
/**
* Install or upgrade a user application Helm release.
*/
async installOrUpgrade(
releaseName: string,
@@ -58,7 +132,7 @@ export class HelmService {
const args = [
'upgrade', '--install',
releaseName,
this.chartPath,
this.appChartPath,
'--namespace', namespace,
'--create-namespace',
'--values', valuesFile,
backend/src/kubernetes/kubernetes.module.ts
+3 -1
View File
@@ -1,13 +1,15 @@
import { Module, forwardRef } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { KubernetesService } from './kubernetes.service';
import { HelmService } from './helm.service';
import { ElasticsearchService } from './elasticsearch.service';
import { ElasticsearchController } from './elasticsearch.controller';
import { LogsController } from './logs.controller';
import { ClustersModule } from '../clusters/clusters.module';
import { Application } from '../applications/entities/application.entity';
@Module({
imports: [forwardRef(() => ClustersModule)],
imports: [forwardRef(() => ClustersModule), TypeOrmModule.forFeature([Application])],
controllers: [ElasticsearchController, LogsController],
providers: [KubernetesService, HelmService, ElasticsearchService],
exports: [KubernetesService, HelmService, ElasticsearchService],
backend/src/kubernetes/kubernetes.service.ts
+163 -2
View File
@@ -41,6 +41,8 @@ interface ManifestContext {
enableElasticsearch: boolean;
elasticsearchVersion: string;
logPaths: string[];
ownerId: string;
applicationId: string;
}
type StorageUsageSlice = {
@@ -183,6 +185,8 @@ export class KubernetesService implements OnModuleInit {
elasticsearch: {
enabled: app.enableElasticsearch || false,
logPaths: app.logPaths || [],
ownerId: app.userId,
applicationId: app.id,
},
changeCause: `Deploy ${imageUri} at ${new Date().toISOString()}`,
};
@@ -247,6 +251,8 @@ export class KubernetesService implements OnModuleInit {
enableElasticsearch: app.enableElasticsearch || false,
elasticsearchVersion: app.elasticsearchVersion || '8.12',
logPaths: app.logPaths || [],
ownerId: app.userId,
applicationId: app.id,
};
await this.applyIngress(networkingApi, ctx, customDomain);
this.logger.log(`Updated ingress for ${app.name} via K8s API (customDomain: ${customDomain || 'none'})`);
@@ -305,6 +311,8 @@ export class KubernetesService implements OnModuleInit {
enableElasticsearch: app.enableElasticsearch || false,
elasticsearchVersion: app.elasticsearchVersion || '8.12',
logPaths: app.logPaths || [],
ownerId: app.userId,
applicationId: app.id,
};
const manifests: Record<string, any> = {};
@@ -689,7 +697,15 @@ export class KubernetesService implements OnModuleInit {
/**
* Build Fluent Bit configuration for log collection
*/
private buildFluentBitConfig(appName: string, namespace: string, runtime: string, customLogPaths?: string[]): string {
private buildFluentBitConfig(
appName: string,
namespace: string,
runtime: string,
ownerId: string,
applicationId: string,
workload: string,
customLogPaths?: string[],
): string {
const logPaths = customLogPaths && customLogPaths.length > 0
? customLogPaths
: this.getDefaultLogPaths(runtime);
@@ -714,8 +730,12 @@ export class KubernetesService implements OnModuleInit {
Name record_modifier
Match *
Record app ${appName}
Record applicationName ${appName}
Record namespace ${namespace}
Record runtime ${runtime}
Record ownerId ${ownerId}
Record applicationId ${applicationId}
Record workload ${workload}
[FILTER]
Name parser
@@ -763,7 +783,15 @@ export class KubernetesService implements OnModuleInit {
labels: { app: ctx.appName },
},
data: {
'fluent-bit.conf': this.buildFluentBitConfig(ctx.appName, ctx.namespace, ctx.runtime, customLogPaths),
'fluent-bit.conf': this.buildFluentBitConfig(
ctx.appName,
ctx.namespace,
ctx.runtime,
ctx.ownerId,
ctx.applicationId,
'app',
customLogPaths,
),
'parsers.conf': `
[PARSER]
Name json
@@ -788,6 +816,127 @@ export class KubernetesService implements OnModuleInit {
this.logger.log(`Created Fluent Bit ConfigMap for ${ctx.appName}`);
}
private buildWorkloadFluentBitConfig(
ctx: ManifestContext,
workload: 'redis' | 'rabbitmq' | 'database',
resourceName: string,
): string {
const logGlob = `/var/log/pods/*${resourceName}*/*/*.log`;
return `
[SERVICE]
Flush 5
Daemon Off
Log_Level info
Parsers_File /fluent-bit/etc/parsers.conf
[INPUT]
Name tail
Path ${logGlob}
Tag ${workload}.${resourceName}
Refresh_Interval 5
Mem_Buf_Limit 5MB
Skip_Long_Lines On
Parser docker
[FILTER]
Name record_modifier
Match *
Record app ${ctx.appName}
Record applicationName ${ctx.appName}
Record namespace ${ctx.namespace}
Record ownerId ${ctx.ownerId}
Record applicationId ${ctx.applicationId}
Record workload ${workload}
[OUTPUT]
Name es
Match *
Host \${ES_HOST}
Port \${ES_PORT}
HTTP_User elastic
HTTP_Passwd \${ES_PASSWORD}
Index logs-${ctx.namespace}-${ctx.appName}
Logstash_Format On
Logstash_Prefix logs-${ctx.namespace}
Suppress_Type_Name On
tls Off
Retry_Limit 3
`;
}
private async attachWorkloadLogShipper(
coreApi: k8s.CoreV1Api,
ctx: ManifestContext,
workload: 'redis' | 'rabbitmq' | 'database',
resourceName: string,
): Promise<{ containers: k8s.V1Container[]; volumes: k8s.V1Volume[] }> {
if (!ctx.enableElasticsearch) {
return { containers: [], volumes: [] };
}
const configMapName = `${resourceName}-log-shipper-config`;
const configMap = {
apiVersion: 'v1',
kind: 'ConfigMap',
metadata: {
name: configMapName,
namespace: ctx.namespace,
labels: { app: resourceName, 'cloudhost.io/log-shipper': 'true' },
},
data: {
'fluent-bit.conf': this.buildWorkloadFluentBitConfig(ctx, workload, resourceName),
'parsers.conf': `
[PARSER]
Name docker
Format json
Time_Key time
Time_Format %Y-%m-%dT%H:%M:%S.%L
`,
},
};
try {
await coreApi.replaceNamespacedConfigMap(configMapName, ctx.namespace, configMap);
} catch {
await coreApi.createNamespacedConfigMap(ctx.namespace, configMap);
}
return {
containers: [
{
name: 'log-shipper',
image: 'fluent/fluent-bit:2.2',
resources: {
requests: { cpu: '10m', memory: '32Mi' },
limits: { cpu: '50m', memory: '64Mi' },
},
volumeMounts: [
{ name: 'varlogpods', mountPath: '/var/log/pods', readOnly: true },
{ name: 'log-shipper-config', mountPath: '/fluent-bit/etc' },
],
env: [
{ name: 'ES_HOST', value: 'elasticsearch.logging.svc.cluster.local' },
{ name: 'ES_PORT', value: '9200' },
{
name: 'ES_PASSWORD',
valueFrom: {
secretKeyRef: {
name: 'elasticsearch-credentials',
key: 'ELASTIC_PASSWORD',
optional: true,
},
},
},
],
},
],
volumes: [
{ name: 'varlogpods', hostPath: { path: '/var/log/pods', type: 'Directory' } },
{ name: 'log-shipper-config', configMap: { name: configMapName } },
],
};
}
private async applyService(coreApi: k8s.CoreV1Api, ctx: ManifestContext): Promise<any> {
const service: k8s.V1Service = {
apiVersion: 'v1',
@@ -953,6 +1102,8 @@ export class KubernetesService implements OnModuleInit {
throw new Error(`Unsupported database type: ${dbType}`);
}
const dbLogShipper = await this.attachWorkloadLogShipper(coreApi, ctx, 'database', dbName);
const dbDeployment: k8s.V1Deployment = {
apiVersion: 'apps/v1',
kind: 'Deployment',
@@ -977,9 +1128,11 @@ export class KubernetesService implements OnModuleInit {
readinessProbe,
livenessProbe,
},
...dbLogShipper.containers,
],
volumes: [
{ name: 'db-storage', persistentVolumeClaim: { claimName: dbName } },
...dbLogShipper.volumes,
],
},
},
@@ -1091,6 +1244,8 @@ export class KubernetesService implements OnModuleInit {
await coreApi.createNamespacedSecret(ctx.namespace, redisSecret);
}
const logShipper = await this.attachWorkloadLogShipper(coreApi, ctx, 'redis', redisName);
// Create Redis Deployment
const redisDeployment: k8s.V1Deployment = {
apiVersion: 'apps/v1',
@@ -1134,12 +1289,14 @@ export class KubernetesService implements OnModuleInit {
periodSeconds: 20,
},
},
...logShipper.containers,
],
volumes: [
{
name: 'redis-data',
persistentVolumeClaim: { claimName: `${redisName}-data` },
},
...logShipper.volumes,
],
},
},
@@ -1205,6 +1362,8 @@ export class KubernetesService implements OnModuleInit {
await coreApi.createNamespacedSecret(ctx.namespace, rabbitSecret);
}
const rabbitLogShipper = await this.attachWorkloadLogShipper(coreApi, ctx, 'rabbitmq', rabbitName);
// Create RabbitMQ Deployment
const rabbitDeployment: k8s.V1Deployment = {
apiVersion: 'apps/v1',
@@ -1258,12 +1417,14 @@ export class KubernetesService implements OnModuleInit {
timeoutSeconds: 10,
},
},
...rabbitLogShipper.containers,
],
volumes: [
{
name: 'rabbitmq-data',
persistentVolumeClaim: { claimName: `${rabbitName}-data` },
},
...rabbitLogShipper.volumes,
],
},
},
backend/src/kubernetes/logs.controller.ts
+123 -304
View File
@@ -15,36 +15,69 @@ import {
ApiResponse,
} from '@nestjs/swagger';
import { AuthGuard } from '@nestjs/passport';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { ElasticsearchService } from './elasticsearch.service';
import { Application } from '../applications/entities/application.entity';
import { RolesGuard } from '../common/guards/roles.guard';
import { Roles } from '../common/decorators/roles.decorator';
import { UserRole } from '../common/enums';
interface AuthenticatedRequest {
user: {
sub: string;
id: string;
email: string;
role: string;
};
}
@ApiTags('Logs')
@ApiBearerAuth()
@Controller('logs')
@UseGuards(AuthGuard('jwt'))
@UseGuards(AuthGuard('jwt'), RolesGuard)
export class LogsController {
constructor(private readonly esService: ElasticsearchService) {}
constructor(
private readonly esService: ElasticsearchService,
@InjectRepository(Application)
private readonly appsRepo: Repository<Application>,
) {}
private async resolveAppFilters(
userId: string,
appId?: string,
allowStaff = false,
): Promise<{ applicationId?: string; applicationName?: string }> {
if (!appId) return {};
const where = allowStaff ? { id: appId } : { id: appId, userId };
const app = await this.appsRepo.findOne({ where });
if (!app) throw new NotFoundException('Application not found');
return { applicationId: app.id, applicationName: app.name };
}
private isStaff(role: string): boolean {
return role === UserRole.ADMIN || role === UserRole.TECHNICAL;
}
@Get('status')
@ApiOperation({ summary: 'Check if central logging is available' })
async getStatus() {
return this.esService.getLoggingStatus();
}
@Get()
@ApiOperation({ summary: 'Get logs for authenticated user\'s applications' })
@ApiQuery({ name: 'appId', required: false, description: 'Filter by application ID' })
@ApiQuery({ name: 'level', required: false, description: 'Filter by log level (error, warn, info, debug)' })
@ApiQuery({ name: 'from', required: false, description: 'Start time (ISO 8601 format)' })
@ApiQuery({ name: 'to', required: false, description: 'End time (ISO 8601 format)' })
@ApiQuery({ name: 'search', required: false, description: 'Full-text search in log messages' })
@ApiQuery({ name: 'page', required: false, description: 'Page number (default: 1)' })
@ApiQuery({ name: 'limit', required: false, description: 'Results per page (default: 100, max: 1000)' })
@ApiResponse({ status: 200, description: 'User logs' })
@ApiResponse({ status: 400, description: 'Invalid query parameters' })
@ApiOperation({ summary: 'Get logs for authenticated user applications' })
@ApiQuery({ name: 'appId', required: false })
@ApiQuery({ name: 'workload', required: false, description: 'app | redis | rabbitmq | database' })
@ApiQuery({ name: 'level', required: false })
@ApiQuery({ name: 'from', required: false })
@ApiQuery({ name: 'to', required: false })
@ApiQuery({ name: 'search', required: false })
@ApiQuery({ name: 'page', required: false })
@ApiQuery({ name: 'limit', required: false })
async getUserLogs(
@Request() req: AuthenticatedRequest,
@Query('appId') appId?: string,
@Query('workload') workload?: string,
@Query('level') level?: string,
@Query('from') from?: string,
@Query('to') to?: string,
@@ -52,352 +85,138 @@ export class LogsController {
@Query('page') page?: string,
@Query('limit') limit?: string,
) {
const userId = req.user.sub;
const pageNum = parseInt(page || '1', 10);
const limitNum = Math.min(parseInt(limit || '100', 10), 1000);
const offset = (pageNum - 1) * limitNum;
const userId = req.user.id;
// Validate log level
if (level && !['error', 'warn', 'info', 'debug', 'trace'].includes(level.toLowerCase())) {
throw new BadRequestException('Invalid log level. Use: error, warn, info, debug, or trace');
throw new BadRequestException('Invalid log level');
}
// Validate date formats
if (from && isNaN(Date.parse(from))) {
throw new BadRequestException('Invalid "from" date format. Use ISO 8601 format.');
throw new BadRequestException('Invalid "from" date format');
}
if (to && isNaN(Date.parse(to))) {
throw new BadRequestException('Invalid "to" date format. Use ISO 8601 format.');
throw new BadRequestException('Invalid "to" date format');
}
if (workload && !['app', 'redis', 'rabbitmq', 'database'].includes(workload)) {
throw new BadRequestException('Invalid workload');
}
// Build Elasticsearch query
const baseQuery = this.esService.getUserLogsQuery(userId);
const must: any[] = [baseQuery.query.bool.must];
const appFilters = await this.resolveAppFilters(userId, appId, this.isStaff(req.user.role));
// Add application filter
if (appId) {
must.push({
term: { 'kubernetes.labels.app': appId },
});
}
// Add level filter
if (level) {
must.push({
term: { level: level.toLowerCase() },
});
}
// Add time range filter
if (from || to) {
const rangeFilter: any = { range: { '@timestamp': {} } };
if (from) rangeFilter.range['@timestamp'].gte = from;
if (to) rangeFilter.range['@timestamp'].lte = to;
must.push(rangeFilter);
}
// Add full-text search
if (search) {
must.push({
multi_match: {
query: search,
fields: ['message', 'log', 'msg', 'error.message'],
type: 'phrase_prefix',
},
});
}
const query = {
query: {
bool: {
must,
},
},
sort: [{ '@timestamp': 'desc' }],
from: offset,
size: limitNum,
};
return {
query,
meta: {
page: pageNum,
limit: limitNum,
userId,
filters: {
appId: appId || null,
level: level || null,
from: from || null,
to: to || null,
search: search || null,
},
},
usage: {
description: 'Execute this query against Elasticsearch to get logs',
endpoint: 'POST /logs-*/_search',
note: 'Use the Elasticsearch endpoint provided by admin to execute queries',
},
};
return this.esService.searchLogs(userId, {
...appFilters,
workload,
level: level?.toLowerCase(),
from,
to,
search,
page: parseInt(page || '1', 10),
limit: Math.min(parseInt(limit || '100', 10), 1000),
});
}
@Get('stream')
@ApiOperation({ summary: 'Get live log stream query for user\'s applications' })
@ApiQuery({ name: 'appId', required: false, description: 'Filter by application ID' })
@ApiResponse({ status: 200, description: 'Stream query configuration' })
async getStreamConfig(
@ApiOperation({ summary: 'Recent logs for live tail (last 5 minutes)' })
@ApiQuery({ name: 'appId', required: false })
@ApiQuery({ name: 'workload', required: false })
async getStream(
@Request() req: AuthenticatedRequest,
@Query('appId') appId?: string,
@Query('workload') workload?: string,
) {
const userId = req.user.sub;
const userId = req.user.id;
const appFilters = await this.resolveAppFilters(userId, appId, this.isStaff(req.user.role));
const fiveMinAgo = new Date(Date.now() - 5 * 60 * 1000).toISOString();
// Build query for streaming
const baseQuery = this.esService.getUserLogsQuery(userId);
const must: any[] = [baseQuery.query.bool.must];
if (appId) {
must.push({
term: { 'kubernetes.labels.app': appId },
});
}
// Add time filter for last 5 minutes
must.push({
range: {
'@timestamp': {
gte: 'now-5m',
},
},
return this.esService.searchLogs(userId, {
...appFilters,
workload,
from: fiveMinAgo,
limit: 100,
page: 1,
});
const query = {
query: {
bool: {
must,
},
},
sort: [{ '@timestamp': 'asc' }],
size: 100,
};
return {
query,
meta: {
userId,
appId: appId || 'all',
refreshInterval: '5s',
},
usage: {
description: 'Poll this query every 5 seconds to get new logs',
note: 'Use search_after for efficient pagination in streaming mode',
},
};
}
@Get('stats')
@ApiOperation({ summary: 'Get log statistics for user\'s applications' })
@ApiQuery({ name: 'appId', required: false, description: 'Filter by application ID' })
@ApiQuery({ name: 'period', required: false, description: 'Time period: 1h, 6h, 24h, 7d (default: 24h)' })
@ApiResponse({ status: 200, description: 'Log statistics' })
@ApiOperation({ summary: 'Log statistics for user applications' })
@ApiQuery({ name: 'appId', required: false })
@ApiQuery({ name: 'workload', required: false })
@ApiQuery({ name: 'period', required: false })
async getLogStats(
@Request() req: AuthenticatedRequest,
@Query('appId') appId?: string,
@Query('workload') workload?: string,
@Query('period') period?: string,
) {
const userId = req.user.sub;
// Convert period to time range
const periodMap: Record<string, string> = {
'1h': 'now-1h',
'6h': 'now-6h',
'24h': 'now-24h',
'7d': 'now-7d',
};
const timeRange = periodMap[period || '24h'] || 'now-24h';
// Build aggregation query
const baseQuery = this.esService.getUserLogsQuery(userId);
const must: any[] = [baseQuery.query.bool.must];
if (appId) {
must.push({
term: { 'kubernetes.labels.app': appId },
});
}
must.push({
range: {
'@timestamp': {
gte: timeRange,
},
},
const userId = req.user.id;
const appFilters = await this.resolveAppFilters(userId, appId, this.isStaff(req.user.role));
return this.esService.searchLogStats(userId, {
...appFilters,
workload,
period: period || '24h',
});
const aggregationQuery = {
query: {
bool: {
must,
},
},
size: 0,
aggs: {
by_level: {
terms: {
field: 'level',
size: 10,
},
},
by_app: {
terms: {
field: 'kubernetes.labels.app',
size: 50,
},
},
over_time: {
date_histogram: {
field: '@timestamp',
fixed_interval: period === '1h' ? '5m' : period === '6h' ? '30m' : '1h',
},
aggs: {
by_level: {
terms: {
field: 'level',
size: 5,
},
},
},
},
error_count: {
filter: {
term: { level: 'error' },
},
},
warn_count: {
filter: {
term: { level: 'warn' },
},
},
},
};
return {
query: aggregationQuery,
meta: {
userId,
appId: appId || 'all',
period: period || '24h',
timeRange,
},
usage: {
description: 'Execute this aggregation query to get log statistics',
endpoint: 'POST /logs-*/_search',
},
};
}
@Get('errors')
@ApiOperation({ summary: 'Get recent errors for user\'s applications' })
@ApiQuery({ name: 'appId', required: false, description: 'Filter by application ID' })
@ApiQuery({ name: 'hours', required: false, description: 'Hours to look back (default: 24)' })
@ApiQuery({ name: 'limit', required: false, description: 'Max errors to return (default: 50)' })
@ApiResponse({ status: 200, description: 'Recent errors' })
@ApiOperation({ summary: 'Recent error logs' })
@ApiQuery({ name: 'appId', required: false })
@ApiQuery({ name: 'workload', required: false })
@ApiQuery({ name: 'hours', required: false })
@ApiQuery({ name: 'limit', required: false })
async getRecentErrors(
@Request() req: AuthenticatedRequest,
@Query('appId') appId?: string,
@Query('workload') workload?: string,
@Query('hours') hours?: string,
@Query('limit') limit?: string,
) {
const userId = req.user.sub;
const hoursNum = parseInt(hours || '24', 10);
const limitNum = Math.min(parseInt(limit || '50', 10), 500);
// Build error query
const baseQuery = this.esService.getUserLogsQuery(userId);
const must: any[] = [baseQuery.query.bool.must];
if (appId) {
must.push({
term: { 'kubernetes.labels.app': appId },
});
}
must.push({
term: { level: 'error' },
const userId = req.user.id;
const appFilters = await this.resolveAppFilters(userId, appId, this.isStaff(req.user.role));
const hits = await this.esService.searchRecentErrors(userId, {
...appFilters,
workload,
hours: parseInt(hours || '24', 10),
limit: Math.min(parseInt(limit || '50', 10), 500),
});
must.push({
range: {
'@timestamp': {
gte: `now-${hoursNum}h`,
},
},
});
const query = {
query: {
bool: {
must,
},
},
sort: [{ '@timestamp': 'desc' }],
size: limitNum,
_source: ['@timestamp', 'message', 'log', 'error', 'kubernetes.labels.app', 'kubernetes.pod_name'],
};
return {
query,
meta: {
userId,
appId: appId || 'all',
lookbackHours: hoursNum,
limit: limitNum,
},
usage: {
description: 'Execute this query to get recent errors',
endpoint: 'POST /logs-*/_search',
},
};
return { hits, total: hits.length };
}
@Get('kibana-url')
@ApiOperation({ summary: 'Get Kibana URL for user\'s application logs' })
@ApiQuery({ name: 'appId', required: false, description: 'Application ID to filter' })
@ApiResponse({ status: 200, description: 'Kibana discovery URL' })
@Roles(UserRole.ADMIN, UserRole.TECHNICAL)
@ApiOperation({ summary: 'Kibana access info (admin/technical only)' })
@ApiQuery({ name: 'appId', required: false })
async getKibanaUrl(
@Request() req: AuthenticatedRequest,
@Query('appId') appId?: string,
) {
const userId = req.user.sub;
const userId = req.user.id;
const connInfo = this.esService.getConnectionInfo();
const appFilters = appId
? await this.resolveAppFilters(userId, appId, true)
: {};
const filters: Array<{
meta: { key: string; negate: boolean };
query: { match_phrase: Record<string, string> };
}> = [
{
meta: { key: 'kubernetes.labels.owner', negate: false },
query: { match_phrase: { 'kubernetes.labels.owner': userId } },
},
];
if (appId) {
filters.push({
meta: { key: 'kubernetes.labels.app', negate: false },
query: { match_phrase: { 'kubernetes.labels.app': appId } },
});
const filterParts: string[] = [];
if (appFilters.applicationName) {
filterParts.push(`applicationName:${appFilters.applicationName}`);
}
filterParts.push(`namespace:user-${userId.split('-')[0]}`);
const rison = encodeURIComponent(JSON.stringify(filters));
const kibanaHost = connInfo.host.replace('elasticsearch', 'kibana');
const query = filterParts.length > 0 ? filterParts.join(' AND ') : '*';
return {
kibana: {
baseUrl: `http://${connInfo.host.replace('elasticsearch', 'kibana')}:5601`,
discoverUrl: `/app/discover#/?_g=(time:(from:now-24h,to:now))&_a=(filters:!${rison})`,
note: 'Access Kibana through your cluster ingress or port-forward',
internalUrl: `http://${kibanaHost}:5601`,
discoverHint: query,
note: 'Use kubectl port-forward from the admin clusters page. Not exposed to end users.',
},
portForward: {
command: 'kubectl port-forward svc/kibana 5601:5601 -n logging',
localUrl: 'http://localhost:5601',
},
credentials: {
username: 'elastic',
note: 'Password is configured in platform settings / elasticsearch deploy output',
},
};
}
}