logging: tune rpi4 image gc and rpi5 prune

services/logging/kustomization.yaml

@@ -7,6 +7,10 @@ resources:
   - opensearch-observability-objects.yaml
   - node-log-rotation-serviceaccount.yaml
   - node-log-rotation-script.yaml
+  - node-image-gc-rpi4-serviceaccount.yaml
+  - node-image-gc-rpi4-script.yaml
+  - node-image-prune-rpi5-serviceaccount.yaml
+  - node-image-prune-rpi5-script.yaml
   - opensearch-pvc.yaml
   - opensearch-helmrelease.yaml
   - opensearch-dashboards-helmrelease.yaml
@@ -18,5 +22,7 @@ resources:
   - opensearch-prune-cronjob.yaml
   - fluent-bit-helmrelease.yaml
   - node-log-rotation-daemonset.yaml
+  - node-image-gc-rpi4-daemonset.yaml
+  - node-image-prune-rpi5-daemonset.yaml
   - oauth2-proxy.yaml
   - ingress.yaml

services/logging/node-image-gc-rpi4-daemonset.yaml

@@ -0,0 +1,49 @@
+# services/logging/node-image-gc-rpi4-daemonset.yaml
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: node-image-gc-rpi4
+  namespace: logging
+spec:
+  selector:
+    matchLabels:
+      app: node-image-gc-rpi4
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: node-image-gc-rpi4
+    spec:
+      serviceAccountName: node-image-gc-rpi4
+      tolerations:
+        - key: node-role.kubernetes.io/control-plane
+          operator: Exists
+          effect: NoSchedule
+        - key: node-role.kubernetes.io/master
+          operator: Exists
+          effect: NoSchedule
+      nodeSelector:
+        hardware: rpi4
+      containers:
+        - name: node-image-gc-rpi4
+          image: bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131
+          command: ["/usr/bin/env", "bash"]
+          args: ["/scripts/node_image_gc_rpi4.sh"]
+          securityContext:
+            privileged: true
+            runAsUser: 0
+          volumeMounts:
+            - name: host-root
+              mountPath: /host
+            - name: script
+              mountPath: /scripts
+              readOnly: true
+      volumes:
+        - name: host-root
+          hostPath:
+            path: /
+        - name: script
+          configMap:
+            name: node-image-gc-rpi4-script
+            defaultMode: 0555

services/logging/node-image-gc-rpi4-script.yaml

@@ -0,0 +1,44 @@
+# services/logging/node-image-gc-rpi4-script.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: node-image-gc-rpi4-script
+  namespace: logging
+data:
+  node_image_gc_rpi4.sh: |
+    #!/usr/bin/env bash
+    set -euo pipefail
+
+    changed=0
+    k3s_changed=0
+    k3s_agent_changed=0
+
+    k3s_dropin="/host/etc/systemd/system/k3s.service.d/98-image-gc.conf"
+    k3s_agent_dropin="/host/etc/systemd/system/k3s-agent.service.d/98-image-gc.conf"
+
+    if [ -f "/host/etc/systemd/system/k3s.service" ] && [ ! -f "${k3s_dropin}" ]; then
+      mkdir -p "$(dirname "${k3s_dropin}")"
+      printf "[Service]\nEnvironment=\"K3S_KUBELET_ARG=image-gc-high-threshold=70\"\nEnvironment=\"K3S_KUBELET_ARG=image-gc-low-threshold=60\"\nEnvironment=\"K3S_KUBELET_ARG=image-gc-minimum-available=5Gi\"\n" > "${k3s_dropin}"
+      changed=1
+      k3s_changed=1
+    fi
+
+    if [ -f "/host/etc/systemd/system/k3s-agent.service" ] && [ ! -f "${k3s_agent_dropin}" ]; then
+      mkdir -p "$(dirname "${k3s_agent_dropin}")"
+      printf "[Service]\nEnvironment=\"K3S_KUBELET_ARG=image-gc-high-threshold=70\"\nEnvironment=\"K3S_KUBELET_ARG=image-gc-low-threshold=60\"\nEnvironment=\"K3S_KUBELET_ARG=image-gc-minimum-available=5Gi\"\n" > "${k3s_agent_dropin}"
+      changed=1
+      k3s_agent_changed=1
+    fi
+
+    if [ "${changed}" -eq 1 ]; then
+      sleep "$(( (RANDOM % 300) + 10 ))"
+      chroot /host /bin/systemctl daemon-reload
+      if [ "${k3s_changed}" -eq 1 ]; then
+        chroot /host /bin/systemctl restart k3s
+      fi
+      if [ "${k3s_agent_changed}" -eq 1 ]; then
+        chroot /host /bin/systemctl restart k3s-agent
+      fi
+    fi
+
+    sleep infinity

services/logging/node-image-gc-rpi4-serviceaccount.yaml

@@ -0,0 +1,6 @@
+# services/logging/node-image-gc-rpi4-serviceaccount.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: node-image-gc-rpi4
+  namespace: logging

services/logging/node-image-prune-rpi5-daemonset.yaml

@@ -0,0 +1,49 @@
+# services/logging/node-image-prune-rpi5-daemonset.yaml
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: node-image-prune-rpi5
+  namespace: logging
+spec:
+  selector:
+    matchLabels:
+      app: node-image-prune-rpi5
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: node-image-prune-rpi5
+    spec:
+      serviceAccountName: node-image-prune-rpi5
+      tolerations:
+        - key: node-role.kubernetes.io/control-plane
+          operator: Exists
+          effect: NoSchedule
+        - key: node-role.kubernetes.io/master
+          operator: Exists
+          effect: NoSchedule
+      nodeSelector:
+        hardware: rpi5
+      containers:
+        - name: node-image-prune-rpi5
+          image: bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131
+          command: ["/usr/bin/env", "bash"]
+          args: ["/scripts/node_image_prune_rpi5.sh"]
+          securityContext:
+            privileged: true
+            runAsUser: 0
+          volumeMounts:
+            - name: host-root
+              mountPath: /host
+            - name: script
+              mountPath: /scripts
+              readOnly: true
+      volumes:
+        - name: host-root
+          hostPath:
+            path: /
+        - name: script
+          configMap:
+            name: node-image-prune-rpi5-script
+            defaultMode: 0555

services/logging/node-image-prune-rpi5-script.yaml

@@ -0,0 +1,34 @@
+# services/logging/node-image-prune-rpi5-script.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: node-image-prune-rpi5-script
+  namespace: logging
+data:
+  node_image_prune_rpi5.sh: |
+    #!/usr/bin/env bash
+    set -euo pipefail
+
+    threshold=70
+
+    sleep "$(( (RANDOM % 300) + 10 ))"
+
+    while true; do
+      usage=$(df -P /host | awk 'NR==2 {gsub(/%/,"",$5); print $5}')
+      if [ -z "${usage}" ]; then
+        sleep 1800
+        continue
+      fi
+
+      if [ "${usage}" -ge "${threshold}" ]; then
+        chroot /host /bin/sh -c '
+          if command -v crictl >/dev/null 2>&1; then
+            crictl --runtime-endpoint=unix:///run/k3s/containerd/containerd.sock rmi --prune || true
+          elif [ -x /usr/local/bin/crictl ]; then
+            /usr/local/bin/crictl --runtime-endpoint=unix:///run/k3s/containerd/containerd.sock rmi --prune || true
+          fi
+        '
+      fi
+
+      sleep 21600
+    done

services/logging/node-image-prune-rpi5-serviceaccount.yaml

@@ -0,0 +1,6 @@
+# services/logging/node-image-prune-rpi5-serviceaccount.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: node-image-prune-rpi5
+  namespace: logging