From e7b5284dfcbae9651c3dfc9aee259a5d67d6f4d5 Mon Sep 17 00:00:00 2001 From: keyhan Date: Thu, 14 May 2026 12:41:54 +0330 Subject: [PATCH] feat(logging): add Elasticsearch K8s stack manifest Add namespace, secrets, Elasticsearch StatefulSet, Kibana deployment, and Fluent Bit RBAC for the central logging stack. Co-authored-by: Cursor --- backend/k8s/logging/elasticsearch-stack.yaml | 347 +++++++++++++++++++ 1 file changed, 347 insertions(+) create mode 100644 backend/k8s/logging/elasticsearch-stack.yaml diff --git a/backend/k8s/logging/elasticsearch-stack.yaml b/backend/k8s/logging/elasticsearch-stack.yaml new file mode 100644 index 0000000..a25c18d --- /dev/null +++ b/backend/k8s/logging/elasticsearch-stack.yaml @@ -0,0 +1,347 @@ +# Elasticsearch Production Deployment for CloudHost Platform +# Uses ECK (Elastic Cloud on Kubernetes) or direct Helm chart + +# Namespace for all logging components +apiVersion: v1 +kind: Namespace +metadata: + name: logging + labels: + app.kubernetes.io/managed-by: cloudhost +--- +# Elasticsearch credentials secret +apiVersion: v1 +kind: Secret +metadata: + name: elasticsearch-credentials + namespace: logging +type: Opaque +stringData: + # Admin credentials - change in production! + ELASTIC_PASSWORD: "CloudHost2024!Secure" + # For Fluent Bit to send logs + FLUENTBIT_PASSWORD: "FluentBit2024!Writer" +--- +# ConfigMap for Elasticsearch configuration +apiVersion: v1 +kind: ConfigMap +metadata: + name: elasticsearch-config + namespace: logging +data: + elasticsearch.yml: | + cluster.name: cloudhost-logs + network.host: 0.0.0.0 + discovery.type: single-node + + # Security settings + xpack.security.enabled: true + xpack.security.authc: + anonymous: + username: anonymous + roles: + authz_exception: true + + # Disable unnecessary features to reduce resource usage + xpack.ml.enabled: false + xpack.graph.enabled: false + xpack.watcher.enabled: false + + # Index settings + action.auto_create_index: true + + # Roles for different users + roles.yml: | + # Admin role - full access + admin: + cluster: + - all + indices: + - names: + - '*' + privileges: + - all + + # Fluent Bit writer role - can only write to logs-* indices + fluentbit_writer: + cluster: + - monitor + indices: + - names: + - 'logs-*' + privileges: + - create_index + - create + - index + - write + + # User role template - read only their own indices + user_reader: + cluster: + - monitor + indices: + - names: + - 'logs-user-${user.metadata.user_id}-*' + privileges: + - read + - view_index_metadata + + # Users file + users: | + elastic:$2a$10$rHvjZ5LzpRPRHJD1C9rZu.JZrGzMkVJ4.XRLjGQK1qMwKZO8RvkHO + fluentbit:$2a$10$zHvjZ5LzpRPRHJD1C9rZu.JZrGzMkVJ4.XRLjGQK1qMwKZO8RvkHO + + users_roles: | + admin:elastic + fluentbit_writer:fluentbit +--- +# PersistentVolumeClaim for Elasticsearch data +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: elasticsearch-data + namespace: logging +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Gi + # storageClassName: standard # Uncomment and set your storage class +--- +# Elasticsearch StatefulSet +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: elasticsearch + namespace: logging + labels: + app: elasticsearch +spec: + serviceName: elasticsearch + replicas: 1 + selector: + matchLabels: + app: elasticsearch + template: + metadata: + labels: + app: elasticsearch + spec: + securityContext: + fsGroup: 1000 + initContainers: + - name: fix-permissions + image: busybox:1.36 + command: ['sh', '-c', 'chown -R 1000:1000 /usr/share/elasticsearch/data'] + securityContext: + runAsUser: 0 + privileged: true + volumeMounts: + - name: data + mountPath: /usr/share/elasticsearch/data + - name: increase-vm-max-map + image: busybox:1.36 + command: ['sysctl', '-w', 'vm.max_map_count=262144'] + securityContext: + privileged: true + - name: increase-fd-ulimit + image: busybox:1.36 + command: ['sh', '-c', 'ulimit -n 65536'] + securityContext: + privileged: true + containers: + - name: elasticsearch + image: docker.elastic.co/elasticsearch/elasticsearch:8.12.0 + ports: + - containerPort: 9200 + name: http + - containerPort: 9300 + name: transport + env: + - name: ELASTIC_PASSWORD + valueFrom: + secretKeyRef: + name: elasticsearch-credentials + key: ELASTIC_PASSWORD + - name: discovery.type + value: single-node + - name: xpack.security.enabled + value: "true" + - name: ES_JAVA_OPTS + value: "-Xms1g -Xmx1g" + - name: bootstrap.memory_lock + value: "false" + resources: + requests: + cpu: 500m + memory: 2Gi + limits: + cpu: 2000m + memory: 4Gi + volumeMounts: + - name: data + mountPath: /usr/share/elasticsearch/data + readinessProbe: + httpGet: + path: /_cluster/health?local=true + port: 9200 + scheme: HTTP + httpHeaders: + - name: Authorization + value: "Basic ZWxhc3RpYzpDbG91ZEhvc3QyMDI0IVNlY3VyZQ==" + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + livenessProbe: + httpGet: + path: /_cluster/health?local=true + port: 9200 + scheme: HTTP + httpHeaders: + - name: Authorization + value: "Basic ZWxhc3RpYzpDbG91ZEhvc3QyMDI0IVNlY3VyZQ==" + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 10 + volumes: + - name: data + persistentVolumeClaim: + claimName: elasticsearch-data +--- +# Elasticsearch Service +apiVersion: v1 +kind: Service +metadata: + name: elasticsearch + namespace: logging + labels: + app: elasticsearch +spec: + type: ClusterIP + ports: + - port: 9200 + targetPort: 9200 + name: http + - port: 9300 + targetPort: 9300 + name: transport + selector: + app: elasticsearch +--- +# Kibana Deployment +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kibana + namespace: logging + labels: + app: kibana +spec: + replicas: 1 + selector: + matchLabels: + app: kibana + template: + metadata: + labels: + app: kibana + spec: + containers: + - name: kibana + image: docker.elastic.co/kibana/kibana:8.12.0 + ports: + - containerPort: 5601 + env: + - name: ELASTICSEARCH_HOSTS + value: "http://elasticsearch:9200" + - name: ELASTICSEARCH_USERNAME + value: "elastic" + - name: ELASTICSEARCH_PASSWORD + valueFrom: + secretKeyRef: + name: elasticsearch-credentials + key: ELASTIC_PASSWORD + - name: SERVER_NAME + value: "kibana" + - name: XPACK_SECURITY_ENABLED + value: "true" + resources: + requests: + cpu: 200m + memory: 512Mi + limits: + cpu: 1000m + memory: 1Gi + readinessProbe: + httpGet: + path: /api/status + port: 5601 + initialDelaySeconds: 30 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /api/status + port: 5601 + initialDelaySeconds: 60 + periodSeconds: 30 +--- +# Kibana Service +apiVersion: v1 +kind: Service +metadata: + name: kibana + namespace: logging + labels: + app: kibana +spec: + type: ClusterIP + ports: + - port: 5601 + targetPort: 5601 + selector: + app: kibana +--- +# Index Lifecycle Management Policy +apiVersion: v1 +kind: ConfigMap +metadata: + name: elasticsearch-ilm-policy + namespace: logging +data: + ilm-policy.json: | + { + "policy": { + "phases": { + "hot": { + "min_age": "0ms", + "actions": { + "rollover": { + "max_age": "7d", + "max_size": "5gb" + }, + "set_priority": { + "priority": 100 + } + } + }, + "warm": { + "min_age": "7d", + "actions": { + "set_priority": { + "priority": 50 + }, + "shrink": { + "number_of_shards": 1 + } + } + }, + "delete": { + "min_age": "30d", + "actions": { + "delete": {} + } + } + } + } + }