feat(logging): add Elasticsearch K8s stack manifest
Add namespace, secrets, Elasticsearch StatefulSet, Kibana deployment, and Fluent Bit RBAC for the central logging stack. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
keyhan
@@ -0,0 +1,347 @@
|
||||
# Elasticsearch Production Deployment for CloudHost Platform
|
||||
# Uses ECK (Elastic Cloud on Kubernetes) or direct Helm chart
|
||||
|
||||
# Namespace for all logging components
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: logging
|
||||
labels:
|
||||
app.kubernetes.io/managed-by: cloudhost
|
||||
---
|
||||
# Elasticsearch credentials secret
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: elasticsearch-credentials
|
||||
namespace: logging
|
||||
type: Opaque
|
||||
stringData:
|
||||
# Admin credentials - change in production!
|
||||
ELASTIC_PASSWORD: "CloudHost2024!Secure"
|
||||
# For Fluent Bit to send logs
|
||||
FLUENTBIT_PASSWORD: "FluentBit2024!Writer"
|
||||
---
|
||||
# ConfigMap for Elasticsearch configuration
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: elasticsearch-config
|
||||
namespace: logging
|
||||
data:
|
||||
elasticsearch.yml: |
|
||||
cluster.name: cloudhost-logs
|
||||
network.host: 0.0.0.0
|
||||
discovery.type: single-node
|
||||
|
||||
# Security settings
|
||||
xpack.security.enabled: true
|
||||
xpack.security.authc:
|
||||
anonymous:
|
||||
username: anonymous
|
||||
roles:
|
||||
authz_exception: true
|
||||
|
||||
# Disable unnecessary features to reduce resource usage
|
||||
xpack.ml.enabled: false
|
||||
xpack.graph.enabled: false
|
||||
xpack.watcher.enabled: false
|
||||
|
||||
# Index settings
|
||||
action.auto_create_index: true
|
||||
|
||||
# Roles for different users
|
||||
roles.yml: |
|
||||
# Admin role - full access
|
||||
admin:
|
||||
cluster:
|
||||
- all
|
||||
indices:
|
||||
- names:
|
||||
- '*'
|
||||
privileges:
|
||||
- all
|
||||
|
||||
# Fluent Bit writer role - can only write to logs-* indices
|
||||
fluentbit_writer:
|
||||
cluster:
|
||||
- monitor
|
||||
indices:
|
||||
- names:
|
||||
- 'logs-*'
|
||||
privileges:
|
||||
- create_index
|
||||
- create
|
||||
- index
|
||||
- write
|
||||
|
||||
# User role template - read only their own indices
|
||||
user_reader:
|
||||
cluster:
|
||||
- monitor
|
||||
indices:
|
||||
- names:
|
||||
- 'logs-user-${user.metadata.user_id}-*'
|
||||
privileges:
|
||||
- read
|
||||
- view_index_metadata
|
||||
|
||||
# Users file
|
||||
users: |
|
||||
elastic:$2a$10$rHvjZ5LzpRPRHJD1C9rZu.JZrGzMkVJ4.XRLjGQK1qMwKZO8RvkHO
|
||||
fluentbit:$2a$10$zHvjZ5LzpRPRHJD1C9rZu.JZrGzMkVJ4.XRLjGQK1qMwKZO8RvkHO
|
||||
|
||||
users_roles: |
|
||||
admin:elastic
|
||||
fluentbit_writer:fluentbit
|
||||
---
|
||||
# PersistentVolumeClaim for Elasticsearch data
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: elasticsearch-data
|
||||
namespace: logging
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 50Gi
|
||||
# storageClassName: standard # Uncomment and set your storage class
|
||||
---
|
||||
# Elasticsearch StatefulSet
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
name: elasticsearch
|
||||
namespace: logging
|
||||
labels:
|
||||
app: elasticsearch
|
||||
spec:
|
||||
serviceName: elasticsearch
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: elasticsearch
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: elasticsearch
|
||||
spec:
|
||||
securityContext:
|
||||
fsGroup: 1000
|
||||
initContainers:
|
||||
- name: fix-permissions
|
||||
image: busybox:1.36
|
||||
command: ['sh', '-c', 'chown -R 1000:1000 /usr/share/elasticsearch/data']
|
||||
securityContext:
|
||||
runAsUser: 0
|
||||
privileged: true
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /usr/share/elasticsearch/data
|
||||
- name: increase-vm-max-map
|
||||
image: busybox:1.36
|
||||
command: ['sysctl', '-w', 'vm.max_map_count=262144']
|
||||
securityContext:
|
||||
privileged: true
|
||||
- name: increase-fd-ulimit
|
||||
image: busybox:1.36
|
||||
command: ['sh', '-c', 'ulimit -n 65536']
|
||||
securityContext:
|
||||
privileged: true
|
||||
containers:
|
||||
- name: elasticsearch
|
||||
image: docker.elastic.co/elasticsearch/elasticsearch:8.12.0
|
||||
ports:
|
||||
- containerPort: 9200
|
||||
name: http
|
||||
- containerPort: 9300
|
||||
name: transport
|
||||
env:
|
||||
- name: ELASTIC_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: elasticsearch-credentials
|
||||
key: ELASTIC_PASSWORD
|
||||
- name: discovery.type
|
||||
value: single-node
|
||||
- name: xpack.security.enabled
|
||||
value: "true"
|
||||
- name: ES_JAVA_OPTS
|
||||
value: "-Xms1g -Xmx1g"
|
||||
- name: bootstrap.memory_lock
|
||||
value: "false"
|
||||
resources:
|
||||
requests:
|
||||
cpu: 500m
|
||||
memory: 2Gi
|
||||
limits:
|
||||
cpu: 2000m
|
||||
memory: 4Gi
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /usr/share/elasticsearch/data
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /_cluster/health?local=true
|
||||
port: 9200
|
||||
scheme: HTTP
|
||||
httpHeaders:
|
||||
- name: Authorization
|
||||
value: "Basic ZWxhc3RpYzpDbG91ZEhvc3QyMDI0IVNlY3VyZQ=="
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /_cluster/health?local=true
|
||||
port: 9200
|
||||
scheme: HTTP
|
||||
httpHeaders:
|
||||
- name: Authorization
|
||||
value: "Basic ZWxhc3RpYzpDbG91ZEhvc3QyMDI0IVNlY3VyZQ=="
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 30
|
||||
timeoutSeconds: 10
|
||||
volumes:
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: elasticsearch-data
|
||||
---
|
||||
# Elasticsearch Service
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: elasticsearch
|
||||
namespace: logging
|
||||
labels:
|
||||
app: elasticsearch
|
||||
spec:
|
||||
type: ClusterIP
|
||||
ports:
|
||||
- port: 9200
|
||||
targetPort: 9200
|
||||
name: http
|
||||
- port: 9300
|
||||
targetPort: 9300
|
||||
name: transport
|
||||
selector:
|
||||
app: elasticsearch
|
||||
---
|
||||
# Kibana Deployment
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: kibana
|
||||
namespace: logging
|
||||
labels:
|
||||
app: kibana
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: kibana
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: kibana
|
||||
spec:
|
||||
containers:
|
||||
- name: kibana
|
||||
image: docker.elastic.co/kibana/kibana:8.12.0
|
||||
ports:
|
||||
- containerPort: 5601
|
||||
env:
|
||||
- name: ELASTICSEARCH_HOSTS
|
||||
value: "http://elasticsearch:9200"
|
||||
- name: ELASTICSEARCH_USERNAME
|
||||
value: "elastic"
|
||||
- name: ELASTICSEARCH_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: elasticsearch-credentials
|
||||
key: ELASTIC_PASSWORD
|
||||
- name: SERVER_NAME
|
||||
value: "kibana"
|
||||
- name: XPACK_SECURITY_ENABLED
|
||||
value: "true"
|
||||
resources:
|
||||
requests:
|
||||
cpu: 200m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: 1000m
|
||||
memory: 1Gi
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /api/status
|
||||
port: 5601
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /api/status
|
||||
port: 5601
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 30
|
||||
---
|
||||
# Kibana Service
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: kibana
|
||||
namespace: logging
|
||||
labels:
|
||||
app: kibana
|
||||
spec:
|
||||
type: ClusterIP
|
||||
ports:
|
||||
- port: 5601
|
||||
targetPort: 5601
|
||||
selector:
|
||||
app: kibana
|
||||
---
|
||||
# Index Lifecycle Management Policy
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: elasticsearch-ilm-policy
|
||||
namespace: logging
|
||||
data:
|
||||
ilm-policy.json: |
|
||||
{
|
||||
"policy": {
|
||||
"phases": {
|
||||
"hot": {
|
||||
"min_age": "0ms",
|
||||
"actions": {
|
||||
"rollover": {
|
||||
"max_age": "7d",
|
||||
"max_size": "5gb"
|
||||
},
|
||||
"set_priority": {
|
||||
"priority": 100
|
||||
}
|
||||
}
|
||||
},
|
||||
"warm": {
|
||||
"min_age": "7d",
|
||||
"actions": {
|
||||
"set_priority": {
|
||||
"priority": 50
|
||||
},
|
||||
"shrink": {
|
||||
"number_of_shards": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
"delete": {
|
||||
"min_age": "30d",
|
||||
"actions": {
|
||||
"delete": {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user