From 2b5c7ca10bd1a0a4ce98096d1353498d95324c0c Mon Sep 17 00:00:00 2001
From: Brad Stein <Brad.Stein@gmail.com>
Date: Mon, 27 Apr 2026 16:31:57 -0300
Subject: [PATCH] logging: trim oversized rotated pod logs on constrained nodes

---
 .../logging/node-log-rotation-daemonset.yaml  |  2 ++
 services/logging/scripts/node_log_rotation.sh | 22 ++++++++++++++++++-
 2 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/services/logging/node-log-rotation-daemonset.yaml b/services/logging/node-log-rotation-daemonset.yaml
index b7753c36..42afe529 100644
--- a/services/logging/node-log-rotation-daemonset.yaml
+++ b/services/logging/node-log-rotation-daemonset.yaml
@@ -12,6 +12,8 @@ spec:
     type: RollingUpdate
   template:
     metadata:
+      annotations:
+        logging.bstein.dev/node-log-rotation-rev: "2026-04-27-1"
       labels:
         app: node-log-rotation
     spec:
diff --git a/services/logging/scripts/node_log_rotation.sh b/services/logging/scripts/node_log_rotation.sh
index c12847e0..e1ab2de1 100644
--- a/services/logging/scripts/node_log_rotation.sh
+++ b/services/logging/scripts/node_log_rotation.sh
@@ -99,4 +99,24 @@ if [ "${changed}" -eq 1 ]; then
   fi
 fi
 
-sleep infinity
+trim_constrained_pod_logs() {
+  local base usage
+  for base in /host/mnt/astraios/var/log /host/var/log.hdd; do
+    if [ ! -d "${base}/pods" ]; then
+      continue
+    fi
+    usage="$(df -P "${base}" | awk 'NR==2 {gsub(/%/, "", $5); print $5}')"
+    if [ -z "${usage}" ] || [ "${usage}" -lt 85 ]; then
+      continue
+    fi
+    find "${base}/pods" -type f -name '[1-9]*.log' -size +1M -print -exec truncate -s 0 {} \; 2>/dev/null || true
+    if [ -d "${base}/containers" ]; then
+      find "${base}/containers" -xtype l -print -delete 2>/dev/null || true
+    fi
+  done
+}
+
+while true; do
+  trim_constrained_pod_logs
+  sleep 600
+done