logging: trim oversized rotated pod logs on constrained nodes

services/logging/node-log-rotation-daemonset.yaml

@@ -12,6 +12,8 @@ spec:
     type: RollingUpdate
   template:
     metadata:
+      annotations:
+        logging.bstein.dev/node-log-rotation-rev: "2026-04-27-1"
       labels:
         app: node-log-rotation
     spec:

services/logging/scripts/node_log_rotation.sh

@@ -99,4 +99,24 @@ if [ "${changed}" -eq 1 ]; then
   fi
 fi
 
-sleep infinity
+trim_constrained_pod_logs() {
+  local base usage
+  for base in /host/mnt/astraios/var/log /host/var/log.hdd; do
+    if [ ! -d "${base}/pods" ]; then
+      continue
+    fi
+    usage="$(df -P "${base}" | awk 'NR==2 {gsub(/%/, "", $5); print $5}')"
+    if [ -z "${usage}" ] || [ "${usage}" -lt 85 ]; then
+      continue
+    fi
+    find "${base}/pods" -type f -name '[1-9]*.log' -size +1M -print -exec truncate -s 0 {} \; 2>/dev/null || true
+    if [ -d "${base}/containers" ]; then
+      find "${base}/containers" -xtype l -print -delete 2>/dev/null || true
+    fi
+  done
+}
+
+while true; do
+  trim_constrained_pod_logs
+  sleep 600
+done