diff --git a/services/logging/fluent-bit-helmrelease.yaml b/services/logging/fluent-bit-helmrelease.yaml index 30952c1..72fa958 100644 --- a/services/logging/fluent-bit-helmrelease.yaml +++ b/services/logging/fluent-bit-helmrelease.yaml @@ -34,9 +34,9 @@ spec: hostPath: path: /var/log/journal - name: fluentbit-state - hostPath: - path: /var/lib/fluent-bit - type: DirectoryOrCreate + emptyDir: + medium: Memory + sizeLimit: 64Mi extraVolumeMounts: - name: runlogjournal mountPath: /run/log/journal diff --git a/services/logging/kustomization.yaml b/services/logging/kustomization.yaml index a4e0bab..3f3a25e 100644 --- a/services/logging/kustomization.yaml +++ b/services/logging/kustomization.yaml @@ -5,6 +5,8 @@ resources: - namespace.yaml - opensearch-dashboards-objects.yaml - opensearch-observability-objects.yaml + - node-log-rotation-serviceaccount.yaml + - node-log-rotation-script.yaml - opensearch-pvc.yaml - opensearch-helmrelease.yaml - opensearch-dashboards-helmrelease.yaml @@ -15,5 +17,6 @@ resources: - opensearch-observability-setup-job.yaml - opensearch-prune-cronjob.yaml - fluent-bit-helmrelease.yaml + - node-log-rotation-daemonset.yaml - oauth2-proxy.yaml - ingress.yaml diff --git a/services/logging/node-log-rotation-daemonset.yaml b/services/logging/node-log-rotation-daemonset.yaml new file mode 100644 index 0000000..f6a672c --- /dev/null +++ b/services/logging/node-log-rotation-daemonset.yaml @@ -0,0 +1,49 @@ +# services/logging/node-log-rotation-daemonset.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: node-log-rotation + namespace: logging +spec: + selector: + matchLabels: + app: node-log-rotation + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + app: node-log-rotation + spec: + serviceAccountName: node-log-rotation + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + nodeSelector: + hardware: rpi5 + containers: + - name: node-log-rotation + image: bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131 + command: ["/usr/bin/env", "bash"] + args: ["/scripts/node_log_rotation.sh"] + securityContext: + privileged: true + runAsUser: 0 + volumeMounts: + - name: host-root + mountPath: /host + - name: script + mountPath: /scripts + readOnly: true + volumes: + - name: host-root + hostPath: + path: / + - name: script + configMap: + name: node-log-rotation-script + defaultMode: 0555 diff --git a/services/logging/node-log-rotation-script.yaml b/services/logging/node-log-rotation-script.yaml new file mode 100644 index 0000000..14fac87 --- /dev/null +++ b/services/logging/node-log-rotation-script.yaml @@ -0,0 +1,56 @@ +# services/logging/node-log-rotation-script.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: node-log-rotation-script + namespace: logging +data: + node_log_rotation.sh: | + #!/usr/bin/env bash + set -euo pipefail + + changed=0 + journald_changed=0 + k3s_changed=0 + k3s_agent_changed=0 + + journald_dropin="/host/etc/systemd/journald.conf.d/99-logging.conf" + k3s_dropin="/host/etc/systemd/system/k3s.service.d/99-logging.conf" + k3s_agent_dropin="/host/etc/systemd/system/k3s-agent.service.d/99-logging.conf" + + if [ ! -f "${journald_dropin}" ]; then + mkdir -p "$(dirname "${journald_dropin}")" + printf "[Journal]\nStorage=volatile\nRuntimeMaxUse=200M\nRuntimeKeepFree=512M\nMaxFileSec=1h\n" > "${journald_dropin}" + changed=1 + journald_changed=1 + fi + + if [ -f "/host/etc/systemd/system/k3s.service" ] && [ ! -f "${k3s_dropin}" ]; then + mkdir -p "$(dirname "${k3s_dropin}")" + printf "[Service]\nEnvironment=\"K3S_KUBELET_ARG=container-log-max-size=10Mi\"\nEnvironment=\"K3S_KUBELET_ARG=container-log-max-files=2\"\n" > "${k3s_dropin}" + changed=1 + k3s_changed=1 + fi + + if [ -f "/host/etc/systemd/system/k3s-agent.service" ] && [ ! -f "${k3s_agent_dropin}" ]; then + mkdir -p "$(dirname "${k3s_agent_dropin}")" + printf "[Service]\nEnvironment=\"K3S_KUBELET_ARG=container-log-max-size=10Mi\"\nEnvironment=\"K3S_KUBELET_ARG=container-log-max-files=2\"\n" > "${k3s_agent_dropin}" + changed=1 + k3s_agent_changed=1 + fi + + if [ "${changed}" -eq 1 ]; then + sleep "$(( (RANDOM % 300) + 10 ))" + chroot /host /bin/systemctl daemon-reload + if [ "${journald_changed}" -eq 1 ]; then + chroot /host /bin/systemctl restart systemd-journald + fi + if [ "${k3s_changed}" -eq 1 ]; then + chroot /host /bin/systemctl restart k3s + fi + if [ "${k3s_agent_changed}" -eq 1 ]; then + chroot /host /bin/systemctl restart k3s-agent + fi + fi + + sleep infinity diff --git a/services/logging/node-log-rotation-serviceaccount.yaml b/services/logging/node-log-rotation-serviceaccount.yaml new file mode 100644 index 0000000..68fc463 --- /dev/null +++ b/services/logging/node-log-rotation-serviceaccount.yaml @@ -0,0 +1,6 @@ +# services/logging/node-log-rotation-serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: node-log-rotation + namespace: logging