Add unified logs platform with Helm-managed central Elasticsearch.
Deploy cloudhost-logging on cluster registration, ship app and optional service logs to ES with owner isolation, and fix Kibana 8.12 auth via kibana_system. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
keyhan
@@ -0,0 +1,46 @@
|
||||
{{/*
|
||||
Optional log-shipper sidecar for Redis / RabbitMQ / Database pods.
|
||||
Requires .workloadName, .workloadType (redis|rabbitmq|database), and root context .
|
||||
*/}}
|
||||
{{- define "cloudhost-app.logShipperContainers" -}}
|
||||
{{- if .root.Values.elasticsearch.enabled }}
|
||||
- name: log-shipper
|
||||
image: fluent/fluent-bit:2.2
|
||||
resources:
|
||||
requests:
|
||||
cpu: "10m"
|
||||
memory: "32Mi"
|
||||
limits:
|
||||
cpu: "50m"
|
||||
memory: "64Mi"
|
||||
volumeMounts:
|
||||
- name: varlogpods
|
||||
mountPath: /var/log/pods
|
||||
readOnly: true
|
||||
- name: log-shipper-config
|
||||
mountPath: /fluent-bit/etc
|
||||
env:
|
||||
- name: ES_HOST
|
||||
value: "elasticsearch.logging.svc.cluster.local"
|
||||
- name: ES_PORT
|
||||
value: "9200"
|
||||
- name: ES_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: elasticsearch-credentials
|
||||
key: ELASTIC_PASSWORD
|
||||
optional: true
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "cloudhost-app.logShipperVolumes" -}}
|
||||
{{- if .root.Values.elasticsearch.enabled }}
|
||||
- name: varlogpods
|
||||
hostPath:
|
||||
path: /var/log/pods
|
||||
type: Directory
|
||||
- name: log-shipper-config
|
||||
configMap:
|
||||
name: {{ .workloadName }}-log-shipper-config
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -138,8 +138,10 @@ spec:
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 10
|
||||
failureThreshold: 5
|
||||
{{- include "cloudhost-app.logShipperContainers" (dict "root" . "workloadName" $dbName "workloadType" "database") | nindent 8 }}
|
||||
volumes:
|
||||
- name: db-storage
|
||||
persistentVolumeClaim:
|
||||
claimName: {{ $dbName }}
|
||||
{{- include "cloudhost-app.logShipperVolumes" (dict "root" . "workloadName" $dbName) | nindent 8 }}
|
||||
{{- end }}
|
||||
|
||||
@@ -31,8 +31,12 @@ data:
|
||||
Name record_modifier
|
||||
Match *
|
||||
Record app {{ $name }}
|
||||
Record applicationName {{ $name }}
|
||||
Record namespace {{ $ns }}
|
||||
Record runtime {{ .Values.app.runtime }}
|
||||
Record ownerId {{ .Values.elasticsearch.ownerId }}
|
||||
Record applicationId {{ .Values.elasticsearch.applicationId }}
|
||||
Record workload app
|
||||
|
||||
[FILTER]
|
||||
Name parser
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
{{- if .Values.elasticsearch.enabled }}
|
||||
{{- $name := include "cloudhost-app.name" . -}}
|
||||
{{- $ns := include "cloudhost-app.namespace" . -}}
|
||||
{{- if .Values.redis.enabled }}
|
||||
{{- $redisName := printf "%s-redis" $name -}}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ $redisName }}-log-shipper-config
|
||||
namespace: {{ $ns }}
|
||||
data:
|
||||
fluent-bit.conf: |
|
||||
[SERVICE]
|
||||
Flush 5
|
||||
Daemon Off
|
||||
Log_Level info
|
||||
Parsers_File /fluent-bit/etc/parsers.conf
|
||||
[INPUT]
|
||||
Name tail
|
||||
Path /var/log/pods/*{{ $redisName }}*/*/*.log
|
||||
Tag redis.{{ $redisName }}
|
||||
Refresh_Interval 5
|
||||
Parser docker
|
||||
[FILTER]
|
||||
Name record_modifier
|
||||
Match *
|
||||
Record app {{ $name }}
|
||||
Record applicationName {{ $name }}
|
||||
Record namespace {{ $ns }}
|
||||
Record ownerId {{ .Values.elasticsearch.ownerId }}
|
||||
Record applicationId {{ .Values.elasticsearch.applicationId }}
|
||||
Record workload redis
|
||||
[OUTPUT]
|
||||
Name es
|
||||
Match *
|
||||
Host ${ES_HOST}
|
||||
Port ${ES_PORT}
|
||||
HTTP_User elastic
|
||||
HTTP_Passwd ${ES_PASSWORD}
|
||||
Index logs-{{ $ns }}-{{ $name }}
|
||||
Logstash_Format On
|
||||
Logstash_Prefix logs-{{ $ns }}
|
||||
Suppress_Type_Name On
|
||||
tls Off
|
||||
parsers.conf: |
|
||||
[PARSER]
|
||||
Name docker
|
||||
Format json
|
||||
Time_Key time
|
||||
Time_Format %Y-%m-%dT%H:%M:%S.%L
|
||||
{{- end }}
|
||||
{{- if .Values.rabbitmq.enabled }}
|
||||
{{- $rabbitName := printf "%s-rabbitmq" $name -}}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ $rabbitName }}-log-shipper-config
|
||||
namespace: {{ $ns }}
|
||||
data:
|
||||
fluent-bit.conf: |
|
||||
[SERVICE]
|
||||
Flush 5
|
||||
Daemon Off
|
||||
Log_Level info
|
||||
Parsers_File /fluent-bit/etc/parsers.conf
|
||||
[INPUT]
|
||||
Name tail
|
||||
Path /var/log/pods/*{{ $rabbitName }}*/*/*.log
|
||||
Tag rabbitmq.{{ $rabbitName }}
|
||||
Refresh_Interval 5
|
||||
Parser docker
|
||||
[FILTER]
|
||||
Name record_modifier
|
||||
Match *
|
||||
Record app {{ $name }}
|
||||
Record applicationName {{ $name }}
|
||||
Record namespace {{ $ns }}
|
||||
Record ownerId {{ .Values.elasticsearch.ownerId }}
|
||||
Record applicationId {{ .Values.elasticsearch.applicationId }}
|
||||
Record workload rabbitmq
|
||||
[OUTPUT]
|
||||
Name es
|
||||
Match *
|
||||
Host ${ES_HOST}
|
||||
Port ${ES_PORT}
|
||||
HTTP_User elastic
|
||||
HTTP_Passwd ${ES_PASSWORD}
|
||||
Index logs-{{ $ns }}-{{ $name }}
|
||||
Logstash_Format On
|
||||
Logstash_Prefix logs-{{ $ns }}
|
||||
Suppress_Type_Name On
|
||||
tls Off
|
||||
parsers.conf: |
|
||||
[PARSER]
|
||||
Name docker
|
||||
Format json
|
||||
Time_Key time
|
||||
Time_Format %Y-%m-%dT%H:%M:%S.%L
|
||||
{{- end }}
|
||||
{{- if .Values.database.enabled }}
|
||||
{{- $dbName := include "cloudhost-app.dbDeploymentName" . -}}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ $dbName }}-log-shipper-config
|
||||
namespace: {{ $ns }}
|
||||
data:
|
||||
fluent-bit.conf: |
|
||||
[SERVICE]
|
||||
Flush 5
|
||||
Daemon Off
|
||||
Log_Level info
|
||||
Parsers_File /fluent-bit/etc/parsers.conf
|
||||
[INPUT]
|
||||
Name tail
|
||||
Path /var/log/pods/*{{ $dbName }}*/*/*.log
|
||||
Tag database.{{ $dbName }}
|
||||
Refresh_Interval 5
|
||||
Parser docker
|
||||
[FILTER]
|
||||
Name record_modifier
|
||||
Match *
|
||||
Record app {{ $name }}
|
||||
Record applicationName {{ $name }}
|
||||
Record namespace {{ $ns }}
|
||||
Record ownerId {{ .Values.elasticsearch.ownerId }}
|
||||
Record applicationId {{ .Values.elasticsearch.applicationId }}
|
||||
Record workload database
|
||||
[OUTPUT]
|
||||
Name es
|
||||
Match *
|
||||
Host ${ES_HOST}
|
||||
Port ${ES_PORT}
|
||||
HTTP_User elastic
|
||||
HTTP_Passwd ${ES_PASSWORD}
|
||||
Index logs-{{ $ns }}-{{ $name }}
|
||||
Logstash_Format On
|
||||
Logstash_Prefix logs-{{ $ns }}
|
||||
Suppress_Type_Name On
|
||||
tls Off
|
||||
parsers.conf: |
|
||||
[PARSER]
|
||||
Name docker
|
||||
Format json
|
||||
Time_Key time
|
||||
Time_Format %Y-%m-%dT%H:%M:%S.%L
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -94,10 +94,12 @@ spec:
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 30
|
||||
timeoutSeconds: 10
|
||||
{{- include "cloudhost-app.logShipperContainers" (dict "root" . "workloadName" $rabbitName "workloadType" "rabbitmq") | nindent 8 }}
|
||||
volumes:
|
||||
- name: rabbitmq-data
|
||||
persistentVolumeClaim:
|
||||
claimName: {{ $rabbitName }}-data
|
||||
{{- include "cloudhost-app.logShipperVolumes" (dict "root" . "workloadName" $rabbitName) | nindent 8 }}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
|
||||
@@ -84,10 +84,12 @@ spec:
|
||||
command: ["redis-cli", "ping"]
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 20
|
||||
{{- include "cloudhost-app.logShipperContainers" (dict "root" . "workloadName" $redisName "workloadType" "redis") | nindent 8 }}
|
||||
volumes:
|
||||
- name: redis-data
|
||||
persistentVolumeClaim:
|
||||
claimName: {{ $redisName }}-data
|
||||
{{- include "cloudhost-app.logShipperVolumes" (dict "root" . "workloadName" $redisName) | nindent 8 }}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
|
||||
@@ -78,6 +78,8 @@ rabbitmq:
|
||||
elasticsearch:
|
||||
enabled: false
|
||||
logPaths: []
|
||||
ownerId: ""
|
||||
applicationId: ""
|
||||
|
||||
# ── Change metadata ─────────────────────────────────────
|
||||
changeCause: ""
|
||||
|
||||
Reference in New Issue
Block a user