cloud-host/backend/k8s/logging/elasticsearch-stack.yaml

# Elasticsearch Production Deployment for CloudHost Platform
# Uses ECK (Elastic Cloud on Kubernetes) or direct Helm chart

# Namespace for all logging components
apiVersion: v1
kind: Namespace
metadata:
  name: logging
  labels:
    app.kubernetes.io/managed-by: cloudhost
---
# Elasticsearch credentials secret
apiVersion: v1
kind: Secret
metadata:
  name: elasticsearch-credentials
  namespace: logging
type: Opaque
stringData:
  # Admin credentials - change in production!
  ELASTIC_PASSWORD: "CloudHost2024!Secure"
  # For Fluent Bit to send logs
  FLUENTBIT_PASSWORD: "FluentBit2024!Writer"
---
# ConfigMap for Elasticsearch configuration
apiVersion: v1
kind: ConfigMap
metadata:
  name: elasticsearch-config
  namespace: logging
data:
  elasticsearch.yml: |
    cluster.name: cloudhost-logs
    network.host: 0.0.0.0
    discovery.type: single-node
    
    # Security settings
    xpack.security.enabled: true
    xpack.security.authc:
      anonymous:
        username: anonymous
        roles: 
        authz_exception: true
    
    # Disable unnecessary features to reduce resource usage
    xpack.ml.enabled: false
    xpack.graph.enabled: false
    xpack.watcher.enabled: false
    
    # Index settings
    action.auto_create_index: true
    
  # Roles for different users
  roles.yml: |
    # Admin role - full access
    admin:
      cluster:
        - all
      indices:
        - names:
            - '*'
          privileges:
            - all
    
    # Fluent Bit writer role - can only write to logs-* indices
    fluentbit_writer:
      cluster:
        - monitor
      indices:
        - names:
            - 'logs-*'
          privileges:
            - create_index
            - create
            - index
            - write
    
    # User role template - read only their own indices
    user_reader:
      cluster:
        - monitor
      indices:
        - names:
            - 'logs-user-${user.metadata.user_id}-*'
          privileges:
            - read
            - view_index_metadata
  
  # Users file
  users: |
    elastic:$2a$10$rHvjZ5LzpRPRHJD1C9rZu.JZrGzMkVJ4.XRLjGQK1qMwKZO8RvkHO
    fluentbit:$2a$10$zHvjZ5LzpRPRHJD1C9rZu.JZrGzMkVJ4.XRLjGQK1qMwKZO8RvkHO
  
  users_roles: |
    admin:elastic
    fluentbit_writer:fluentbit
---
# PersistentVolumeClaim for Elasticsearch data
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: elasticsearch-data
  namespace: logging
spec:
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 50Gi
  # storageClassName: standard  # Uncomment and set your storage class
---
# Elasticsearch StatefulSet
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: elasticsearch
  namespace: logging
  labels:
    app: elasticsearch
spec:
  serviceName: elasticsearch
  replicas: 1
  selector:
    matchLabels:
      app: elasticsearch
  template:
    metadata:
      labels:
        app: elasticsearch
    spec:
      securityContext:
        fsGroup: 1000
      initContainers:
        - name: fix-permissions
          image: busybox:1.36
          command: ['sh', '-c', 'chown -R 1000:1000 /usr/share/elasticsearch/data']
          securityContext:
            runAsUser: 0
            privileged: true
          volumeMounts:
            - name: data
              mountPath: /usr/share/elasticsearch/data
        - name: increase-vm-max-map
          image: busybox:1.36
          command: ['sysctl', '-w', 'vm.max_map_count=262144']
          securityContext:
            privileged: true
        - name: increase-fd-ulimit
          image: busybox:1.36
          command: ['sh', '-c', 'ulimit -n 65536']
          securityContext:
            privileged: true
      containers:
        - name: elasticsearch
          image: docker.elastic.co/elasticsearch/elasticsearch:8.12.0
          ports:
            - containerPort: 9200
              name: http
            - containerPort: 9300
              name: transport
          env:
            - name: ELASTIC_PASSWORD
              valueFrom:
                secretKeyRef:
                  name: elasticsearch-credentials
                  key: ELASTIC_PASSWORD
            - name: discovery.type
              value: single-node
            - name: xpack.security.enabled
              value: "true"
            - name: ES_JAVA_OPTS
              value: "-Xms1g -Xmx1g"
            - name: bootstrap.memory_lock
              value: "false"
          resources:
            requests:
              cpu: 500m
              memory: 2Gi
            limits:
              cpu: 2000m
              memory: 4Gi
          volumeMounts:
            - name: data
              mountPath: /usr/share/elasticsearch/data
          readinessProbe:
            httpGet:
              path: /_cluster/health?local=true
              port: 9200
              scheme: HTTP
              httpHeaders:
                - name: Authorization
                  value: "Basic ZWxhc3RpYzpDbG91ZEhvc3QyMDI0IVNlY3VyZQ=="
            initialDelaySeconds: 30
            periodSeconds: 10
            timeoutSeconds: 5
          livenessProbe:
            httpGet:
              path: /_cluster/health?local=true
              port: 9200
              scheme: HTTP
              httpHeaders:
                - name: Authorization
                  value: "Basic ZWxhc3RpYzpDbG91ZEhvc3QyMDI0IVNlY3VyZQ=="
            initialDelaySeconds: 60
            periodSeconds: 30
            timeoutSeconds: 10
      volumes:
        - name: data
          persistentVolumeClaim:
            claimName: elasticsearch-data
---
# Elasticsearch Service
apiVersion: v1
kind: Service
metadata:
  name: elasticsearch
  namespace: logging
  labels:
    app: elasticsearch
spec:
  type: ClusterIP
  ports:
    - port: 9200
      targetPort: 9200
      name: http
    - port: 9300
      targetPort: 9300
      name: transport
  selector:
    app: elasticsearch
---
# Kibana Deployment
apiVersion: apps/v1
kind: Deployment
metadata:
  name: kibana
  namespace: logging
  labels:
    app: kibana
spec:
  replicas: 1
  selector:
    matchLabels:
      app: kibana
  template:
    metadata:
      labels:
        app: kibana
    spec:
      containers:
        - name: kibana
          image: docker.elastic.co/kibana/kibana:8.12.0
          ports:
            - containerPort: 5601
          env:
            - name: ELASTICSEARCH_HOSTS
              value: "http://elasticsearch:9200"
            - name: ELASTICSEARCH_USERNAME
              value: "elastic"
            - name: ELASTICSEARCH_PASSWORD
              valueFrom:
                secretKeyRef:
                  name: elasticsearch-credentials
                  key: ELASTIC_PASSWORD
            - name: SERVER_NAME
              value: "kibana"
            - name: XPACK_SECURITY_ENABLED
              value: "true"
          resources:
            requests:
              cpu: 200m
              memory: 512Mi
            limits:
              cpu: 1000m
              memory: 1Gi
          readinessProbe:
            httpGet:
              path: /api/status
              port: 5601
            initialDelaySeconds: 30
            periodSeconds: 10
          livenessProbe:
            httpGet:
              path: /api/status
              port: 5601
            initialDelaySeconds: 60
            periodSeconds: 30
---
# Kibana Service
apiVersion: v1
kind: Service
metadata:
  name: kibana
  namespace: logging
  labels:
    app: kibana
spec:
  type: ClusterIP
  ports:
    - port: 5601
      targetPort: 5601
  selector:
    app: kibana
---
# Index Lifecycle Management Policy
apiVersion: v1
kind: ConfigMap
metadata:
  name: elasticsearch-ilm-policy
  namespace: logging
data:
  ilm-policy.json: |
    {
      "policy": {
        "phases": {
          "hot": {
            "min_age": "0ms",
            "actions": {
              "rollover": {
                "max_age": "7d",
                "max_size": "5gb"
              },
              "set_priority": {
                "priority": 100
              }
            }
          },
          "warm": {
            "min_age": "7d",
            "actions": {
              "set_priority": {
                "priority": 50
              },
              "shrink": {
                "number_of_shards": 1
              }
            }
          },
          "delete": {
            "min_age": "30d",
            "actions": {
              "delete": {}
            }
          }
        }
      }
    }