From 7e96995866d2fc6eddc55514f3a008ba8ff023b6 Mon Sep 17 00:00:00 2001
From: jenkins <brad@bstein.dev>
Date: Sat, 3 Oct 2026 01:25:25 -0500
Subject: [PATCH] logging: leave image and runtime log rotation to kubelet

---
 services/logging/kustomization.yaml           | 16 ---
 .../logging/node-image-gc-rpi4-daemonset.yaml | 49 ----------
 .../node-image-gc-rpi4-serviceaccount.yaml    |  6 --
 .../node-image-prune-rpi5-daemonset.yaml      | 49 ----------
 .../node-image-prune-rpi5-serviceaccount.yaml |  6 --
 .../logging/scripts/node_image_gc_rpi4.sh     | 36 -------
 .../logging/scripts/node_image_prune_rpi5.sh  | 26 -----
 services/logging/scripts/node_log_rotation.sh | 98 ++-----------------
 8 files changed, 7 insertions(+), 279 deletions(-)
 delete mode 100644 services/logging/node-image-gc-rpi4-daemonset.yaml
 delete mode 100644 services/logging/node-image-gc-rpi4-serviceaccount.yaml
 delete mode 100644 services/logging/node-image-prune-rpi5-daemonset.yaml
 delete mode 100644 services/logging/node-image-prune-rpi5-serviceaccount.yaml
 delete mode 100644 services/logging/scripts/node_image_gc_rpi4.sh
 delete mode 100644 services/logging/scripts/node_image_prune_rpi5.sh

diff --git a/services/logging/kustomization.yaml b/services/logging/kustomization.yaml
index cfed11f9..01778d88 100644
--- a/services/logging/kustomization.yaml
+++ b/services/logging/kustomization.yaml
@@ -7,8 +7,6 @@ resources:
   - opensearch-dashboards-objects.yaml
   - opensearch-observability-objects.yaml
   - node-log-rotation-serviceaccount.yaml
-  - node-image-gc-rpi4-serviceaccount.yaml
-  - node-image-prune-rpi5-serviceaccount.yaml
   - vault-serviceaccount.yaml
   - secretproviderclass.yaml
   - opensearch-pvc.yaml
@@ -23,8 +21,6 @@ resources:
   - opensearch-prune-cronjob.yaml
   - fluent-bit-helmrelease.yaml
   - node-log-rotation-daemonset.yaml
-  - node-image-gc-rpi4-daemonset.yaml
-  - node-image-prune-rpi5-daemonset.yaml
   - oauth2-proxy.yaml
   - vault-sync-deployment.yaml
   - ingress.yaml
@@ -36,18 +32,6 @@ configMapGenerator:
       - node_log_rotation.sh=scripts/node_log_rotation.sh
     options:
       disableNameSuffixHash: true
-  - name: node-image-gc-rpi4-script
-    namespace: logging
-    files:
-      - node_image_gc_rpi4.sh=scripts/node_image_gc_rpi4.sh
-    options:
-      disableNameSuffixHash: true
-  - name: node-image-prune-rpi5-script
-    namespace: logging
-    files:
-      - node_image_prune_rpi5.sh=scripts/node_image_prune_rpi5.sh
-    options:
-      disableNameSuffixHash: true
   - name: opensearch-prune-script
     namespace: logging
     files:
diff --git a/services/logging/node-image-gc-rpi4-daemonset.yaml b/services/logging/node-image-gc-rpi4-daemonset.yaml
deleted file mode 100644
index 70bace55..00000000
--- a/services/logging/node-image-gc-rpi4-daemonset.yaml
+++ /dev/null
@@ -1,49 +0,0 @@
-# services/logging/node-image-gc-rpi4-daemonset.yaml
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: node-image-gc-rpi4
-  namespace: logging
-spec:
-  selector:
-    matchLabels:
-      app: node-image-gc-rpi4
-  updateStrategy:
-    type: RollingUpdate
-  template:
-    metadata:
-      labels:
-        app: node-image-gc-rpi4
-    spec:
-      serviceAccountName: node-image-gc-rpi4
-      tolerations:
-        - key: node-role.kubernetes.io/control-plane
-          operator: Exists
-          effect: NoSchedule
-        - key: node-role.kubernetes.io/master
-          operator: Exists
-          effect: NoSchedule
-      nodeSelector:
-        hardware: rpi4
-      containers:
-        - name: node-image-gc-rpi4
-          image: bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131
-          command: ["/usr/bin/env", "bash"]
-          args: ["/scripts/node_image_gc_rpi4.sh"]
-          securityContext:
-            privileged: true
-            runAsUser: 0
-          volumeMounts:
-            - name: host-root
-              mountPath: /host
-            - name: script
-              mountPath: /scripts
-              readOnly: true
-      volumes:
-        - name: host-root
-          hostPath:
-            path: /
-        - name: script
-          configMap:
-            name: node-image-gc-rpi4-script
-            defaultMode: 0555
diff --git a/services/logging/node-image-gc-rpi4-serviceaccount.yaml b/services/logging/node-image-gc-rpi4-serviceaccount.yaml
deleted file mode 100644
index ec1eb412..00000000
--- a/services/logging/node-image-gc-rpi4-serviceaccount.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-# services/logging/node-image-gc-rpi4-serviceaccount.yaml
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: node-image-gc-rpi4
-  namespace: logging
diff --git a/services/logging/node-image-prune-rpi5-daemonset.yaml b/services/logging/node-image-prune-rpi5-daemonset.yaml
deleted file mode 100644
index 63fa471f..00000000
--- a/services/logging/node-image-prune-rpi5-daemonset.yaml
+++ /dev/null
@@ -1,49 +0,0 @@
-# services/logging/node-image-prune-rpi5-daemonset.yaml
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: node-image-prune-rpi5
-  namespace: logging
-spec:
-  selector:
-    matchLabels:
-      app: node-image-prune-rpi5
-  updateStrategy:
-    type: RollingUpdate
-  template:
-    metadata:
-      labels:
-        app: node-image-prune-rpi5
-    spec:
-      serviceAccountName: node-image-prune-rpi5
-      tolerations:
-        - key: node-role.kubernetes.io/control-plane
-          operator: Exists
-          effect: NoSchedule
-        - key: node-role.kubernetes.io/master
-          operator: Exists
-          effect: NoSchedule
-      nodeSelector:
-        hardware: rpi5
-      containers:
-        - name: node-image-prune-rpi5
-          image: bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131
-          command: ["/usr/bin/env", "bash"]
-          args: ["/scripts/node_image_prune_rpi5.sh"]
-          securityContext:
-            privileged: true
-            runAsUser: 0
-          volumeMounts:
-            - name: host-root
-              mountPath: /host
-            - name: script
-              mountPath: /scripts
-              readOnly: true
-      volumes:
-        - name: host-root
-          hostPath:
-            path: /
-        - name: script
-          configMap:
-            name: node-image-prune-rpi5-script
-            defaultMode: 0555
diff --git a/services/logging/node-image-prune-rpi5-serviceaccount.yaml b/services/logging/node-image-prune-rpi5-serviceaccount.yaml
deleted file mode 100644
index 938c78ac..00000000
--- a/services/logging/node-image-prune-rpi5-serviceaccount.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-# services/logging/node-image-prune-rpi5-serviceaccount.yaml
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: node-image-prune-rpi5
-  namespace: logging
diff --git a/services/logging/scripts/node_image_gc_rpi4.sh b/services/logging/scripts/node_image_gc_rpi4.sh
deleted file mode 100644
index 81f27b14..00000000
--- a/services/logging/scripts/node_image_gc_rpi4.sh
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-changed=0
-k3s_changed=0
-k3s_agent_changed=0
-
-k3s_dropin="/host/etc/systemd/system/k3s.service.d/98-image-gc.conf"
-k3s_agent_dropin="/host/etc/systemd/system/k3s-agent.service.d/98-image-gc.conf"
-
-if [ -f "/host/etc/systemd/system/k3s.service" ] && [ ! -f "${k3s_dropin}" ]; then
-  mkdir -p "$(dirname "${k3s_dropin}")"
-  printf "[Service]\nEnvironment=\"K3S_KUBELET_ARG=image-gc-high-threshold=70\"\nEnvironment=\"K3S_KUBELET_ARG=image-gc-low-threshold=60\"\nEnvironment=\"K3S_KUBELET_ARG=image-gc-minimum-available=5Gi\"\n" > "${k3s_dropin}"
-  changed=1
-  k3s_changed=1
-fi
-
-if [ -f "/host/etc/systemd/system/k3s-agent.service" ] && [ ! -f "${k3s_agent_dropin}" ]; then
-  mkdir -p "$(dirname "${k3s_agent_dropin}")"
-  printf "[Service]\nEnvironment=\"K3S_KUBELET_ARG=image-gc-high-threshold=70\"\nEnvironment=\"K3S_KUBELET_ARG=image-gc-low-threshold=60\"\nEnvironment=\"K3S_KUBELET_ARG=image-gc-minimum-available=5Gi\"\n" > "${k3s_agent_dropin}"
-  changed=1
-  k3s_agent_changed=1
-fi
-
-if [ "${changed}" -eq 1 ]; then
-  sleep "$(( (RANDOM % 300) + 10 ))"
-  chroot /host /bin/systemctl daemon-reload
-  if [ "${k3s_changed}" -eq 1 ]; then
-    chroot /host /bin/systemctl restart k3s
-  fi
-  if [ "${k3s_agent_changed}" -eq 1 ]; then
-    chroot /host /bin/systemctl restart k3s-agent
-  fi
-fi
-
-sleep infinity
diff --git a/services/logging/scripts/node_image_prune_rpi5.sh b/services/logging/scripts/node_image_prune_rpi5.sh
deleted file mode 100644
index eb54b778..00000000
--- a/services/logging/scripts/node_image_prune_rpi5.sh
+++ /dev/null
@@ -1,26 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-threshold=70
-
-sleep "$(( (RANDOM % 300) + 10 ))"
-
-while true; do
-  usage=$(df -P /host | awk 'NR==2 {gsub(/%/,"",$5); print $5}')
-  if [ -z "${usage}" ]; then
-    sleep 1800
-    continue
-  fi
-
-  if [ "${usage}" -ge "${threshold}" ]; then
-    chroot /host /bin/sh -c '
-      if command -v crictl >/dev/null 2>&1; then
-        crictl --runtime-endpoint=unix:///run/k3s/containerd/containerd.sock rmi --prune || true
-      elif [ -x /usr/local/bin/crictl ]; then
-        /usr/local/bin/crictl --runtime-endpoint=unix:///run/k3s/containerd/containerd.sock rmi --prune || true
-      fi
-    '
-  fi
-
-  sleep 21600
-done
diff --git a/services/logging/scripts/node_log_rotation.sh b/services/logging/scripts/node_log_rotation.sh
index 73f4dd0f..2d1e713f 100644
--- a/services/logging/scripts/node_log_rotation.sh
+++ b/services/logging/scripts/node_log_rotation.sh
@@ -1,102 +1,18 @@
 #!/usr/bin/env bash
+# Kubelet owns image and container-log rotation; this guard only protects
+# constrained log mounts and caps the host journal. It never restarts k3s.
 set -euo pipefail
 
-changed=0
-journald_changed=0
-k3s_changed=0
-k3s_agent_changed=0
-
 journald_dropin="/host/etc/systemd/journald.conf.d/99-logging.conf"
-k3s_dropin="/host/etc/systemd/system/k3s.service.d/99-logging.conf"
-k3s_agent_dropin="/host/etc/systemd/system/k3s-agent.service.d/99-logging.conf"
-k3s_image_gc_dropin="/host/etc/systemd/system/k3s.service.d/98-image-gc.conf"
-k3s_agent_image_gc_dropin="/host/etc/systemd/system/k3s-agent.service.d/98-image-gc.conf"
-
-ensure_dropin() {
-  local path="$1"
-  local owner="$2"
-  local new_content="$3"
-  local current=""
-  if [ -f "${path}" ]; then
-    current="$(cat "${path}" || true)"
-  fi
-  if [ "${current}" != "${new_content}" ]; then
-    mkdir -p "$(dirname "${path}")"
-    printf "%s\n" "${new_content}" > "${path}"
-    changed=1
-    case "${owner}" in
-      journald)
-        journald_changed=1
-        ;;
-      k3s)
-        k3s_changed=1
-        ;;
-      k3s-agent)
-        k3s_agent_changed=1
-        ;;
-    esac
-  fi
-}
-
-ensure_dropin \
-  "${journald_dropin}" \
-  "journald" \
-  "[Journal]
+expected="[Journal]
 Storage=volatile
 RuntimeMaxUse=200M
 RuntimeKeepFree=512M
 MaxFileSec=1h"
-
-if [ -f "/host/etc/systemd/system/k3s.service" ]; then
-  ensure_dropin \
-    "${k3s_dropin}" \
-    "k3s" \
-    "[Service]
-Environment=\"K3S_KUBELET_ARG=container-log-max-size=10Mi\"
-Environment=\"K3S_KUBELET_ARG=container-log-max-files=2\""
-fi
-
-if [ -f "/host/etc/systemd/system/k3s.service" ]; then
-  ensure_dropin \
-    "${k3s_image_gc_dropin}" \
-    "k3s" \
-    "[Service]
-Environment=\"K3S_KUBELET_ARG=image-gc-high-threshold=65\"
-Environment=\"K3S_KUBELET_ARG=image-gc-low-threshold=50\"
-Environment=\"K3S_KUBELET_ARG=image-gc-minimum-available=8Gi\""
-fi
-
-if [ -f "/host/etc/systemd/system/k3s-agent.service" ]; then
-  ensure_dropin \
-    "${k3s_agent_dropin}" \
-    "k3s-agent" \
-    "[Service]
-Environment=\"K3S_KUBELET_ARG=container-log-max-size=10Mi\"
-Environment=\"K3S_KUBELET_ARG=container-log-max-files=2\""
-fi
-
-if [ -f "/host/etc/systemd/system/k3s-agent.service" ]; then
-  ensure_dropin \
-    "${k3s_agent_image_gc_dropin}" \
-    "k3s-agent" \
-    "[Service]
-Environment=\"K3S_KUBELET_ARG=image-gc-high-threshold=65\"
-Environment=\"K3S_KUBELET_ARG=image-gc-low-threshold=50\"
-Environment=\"K3S_KUBELET_ARG=image-gc-minimum-available=8Gi\""
-fi
-
-if [ "${changed}" -eq 1 ]; then
-  sleep "$(( (RANDOM % 300) + 10 ))"
-  chroot /host /bin/systemctl daemon-reload
-  if [ "${journald_changed}" -eq 1 ]; then
-    chroot /host /bin/systemctl restart systemd-journald
-  fi
-  if [ "${k3s_changed}" -eq 1 ]; then
-    chroot /host /bin/systemctl restart k3s
-  fi
-  if [ "${k3s_agent_changed}" -eq 1 ]; then
-    chroot /host /bin/systemctl restart k3s-agent
-  fi
+if [[ ! -f "$journald_dropin" ]] || [[ $(cat "$journald_dropin") != "$expected" ]]; then
+  mkdir -p "$(dirname "$journald_dropin")"
+  printf '%s\n' "$expected" > "$journald_dropin"
+  chroot /host /bin/systemctl restart systemd-journald
 fi
 
 trim_constrained_pod_logs() {