logging: add rpi5 log retention tuning

services/logging/fluent-bit-helmrelease.yaml

@@ -34,9 +34,9 @@ spec:
         hostPath:
           path: /var/log/journal
       - name: fluentbit-state
-        hostPath:
-          path: /var/lib/fluent-bit
-          type: DirectoryOrCreate
+        emptyDir:
+          medium: Memory
+          sizeLimit: 64Mi
     extraVolumeMounts:
       - name: runlogjournal
         mountPath: /run/log/journal

services/logging/kustomization.yaml

@@ -5,6 +5,8 @@ resources:
   - namespace.yaml
   - opensearch-dashboards-objects.yaml
   - opensearch-observability-objects.yaml
+  - node-log-rotation-serviceaccount.yaml
+  - node-log-rotation-script.yaml
   - opensearch-pvc.yaml
   - opensearch-helmrelease.yaml
   - opensearch-dashboards-helmrelease.yaml
@@ -15,5 +17,6 @@ resources:
   - opensearch-observability-setup-job.yaml
   - opensearch-prune-cronjob.yaml
   - fluent-bit-helmrelease.yaml
+  - node-log-rotation-daemonset.yaml
   - oauth2-proxy.yaml
   - ingress.yaml

services/logging/node-log-rotation-daemonset.yaml

@@ -0,0 +1,49 @@
+# services/logging/node-log-rotation-daemonset.yaml
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: node-log-rotation
+  namespace: logging
+spec:
+  selector:
+    matchLabels:
+      app: node-log-rotation
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: node-log-rotation
+    spec:
+      serviceAccountName: node-log-rotation
+      tolerations:
+        - key: node-role.kubernetes.io/control-plane
+          operator: Exists
+          effect: NoSchedule
+        - key: node-role.kubernetes.io/master
+          operator: Exists
+          effect: NoSchedule
+      nodeSelector:
+        hardware: rpi5
+      containers:
+        - name: node-log-rotation
+          image: bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131
+          command: ["/usr/bin/env", "bash"]
+          args: ["/scripts/node_log_rotation.sh"]
+          securityContext:
+            privileged: true
+            runAsUser: 0
+          volumeMounts:
+            - name: host-root
+              mountPath: /host
+            - name: script
+              mountPath: /scripts
+              readOnly: true
+      volumes:
+        - name: host-root
+          hostPath:
+            path: /
+        - name: script
+          configMap:
+            name: node-log-rotation-script
+            defaultMode: 0555

services/logging/node-log-rotation-script.yaml

@@ -0,0 +1,56 @@
+# services/logging/node-log-rotation-script.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: node-log-rotation-script
+  namespace: logging
+data:
+  node_log_rotation.sh: |
+    #!/usr/bin/env bash
+    set -euo pipefail
+
+    changed=0
+    journald_changed=0
+    k3s_changed=0
+    k3s_agent_changed=0
+
+    journald_dropin="/host/etc/systemd/journald.conf.d/99-logging.conf"
+    k3s_dropin="/host/etc/systemd/system/k3s.service.d/99-logging.conf"
+    k3s_agent_dropin="/host/etc/systemd/system/k3s-agent.service.d/99-logging.conf"
+
+    if [ ! -f "${journald_dropin}" ]; then
+      mkdir -p "$(dirname "${journald_dropin}")"
+      printf "[Journal]\nStorage=volatile\nRuntimeMaxUse=200M\nRuntimeKeepFree=512M\nMaxFileSec=1h\n" > "${journald_dropin}"
+      changed=1
+      journald_changed=1
+    fi
+
+    if [ -f "/host/etc/systemd/system/k3s.service" ] && [ ! -f "${k3s_dropin}" ]; then
+      mkdir -p "$(dirname "${k3s_dropin}")"
+      printf "[Service]\nEnvironment=\"K3S_KUBELET_ARG=container-log-max-size=10Mi\"\nEnvironment=\"K3S_KUBELET_ARG=container-log-max-files=2\"\n" > "${k3s_dropin}"
+      changed=1
+      k3s_changed=1
+    fi
+
+    if [ -f "/host/etc/systemd/system/k3s-agent.service" ] && [ ! -f "${k3s_agent_dropin}" ]; then
+      mkdir -p "$(dirname "${k3s_agent_dropin}")"
+      printf "[Service]\nEnvironment=\"K3S_KUBELET_ARG=container-log-max-size=10Mi\"\nEnvironment=\"K3S_KUBELET_ARG=container-log-max-files=2\"\n" > "${k3s_agent_dropin}"
+      changed=1
+      k3s_agent_changed=1
+    fi
+
+    if [ "${changed}" -eq 1 ]; then
+      sleep "$(( (RANDOM % 300) + 10 ))"
+      chroot /host /bin/systemctl daemon-reload
+      if [ "${journald_changed}" -eq 1 ]; then
+        chroot /host /bin/systemctl restart systemd-journald
+      fi
+      if [ "${k3s_changed}" -eq 1 ]; then
+        chroot /host /bin/systemctl restart k3s
+      fi
+      if [ "${k3s_agent_changed}" -eq 1 ]; then
+        chroot /host /bin/systemctl restart k3s-agent
+      fi
+    fi
+
+    sleep infinity

services/logging/node-log-rotation-serviceaccount.yaml

@@ -0,0 +1,6 @@
+# services/logging/node-log-rotation-serviceaccount.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: node-log-rotation
+  namespace: logging