atlas-iac/services/logging/scripts/node_log_rotation.sh

#!/usr/bin/env bash
# Kubelet owns image and container-log rotation; this guard only protects
# constrained log mounts and caps the host journal. It never restarts k3s.
set -euo pipefail

journald_dropin="/host/etc/systemd/journald.conf.d/99-logging.conf"
expected="[Journal]
Storage=volatile
RuntimeMaxUse=200M
RuntimeKeepFree=512M
MaxFileSec=1h"
if [[ ! -f "$journald_dropin" ]] || [[ $(cat "$journald_dropin") != "$expected" ]]; then
  mkdir -p "$(dirname "$journald_dropin")"
  printf '%s\n' "$expected" > "$journald_dropin"
  chroot /host /bin/systemctl restart systemd-journald
fi

trim_constrained_pod_logs() {
  local base usage
  for base in /host/mnt/astraios/var/log /host/var/log.hdd; do
    if [ ! -d "${base}/pods" ]; then
      continue
    fi
    usage="$(df -P "${base}" | awk 'NR==2 {gsub(/%/, "", $5); print $5}')"
    if [ -z "${usage}" ] || [ "${usage}" -lt 75 ]; then
      continue
    fi
    find "${base}/pods" -type f \( -name '[1-9]*.log' -o -name '*.log.20*' \) -size +1M -print -exec truncate -s 0 {} \; 2>/dev/null || true
    # Some nodes keep active container logs on tiny zram-backed /var/log.
    # Trim noisy live logs before kubelet loses the ability to create pods.
    find "${base}/pods" -type f -name '0.log' -size +1M -print -exec truncate -s 1M {} \; 2>/dev/null || true
    if [ -d "${base}/containers" ]; then
      find "${base}/containers" -xtype l -print -delete 2>/dev/null || true
    fi
  done
}

while true; do
  trim_constrained_pod_logs
  sleep 600
done