titan-iac/services/logging/scripts/node_log_rotation.sh

#!/usr/bin/env bash
set -euo pipefail

changed=0
journald_changed=0
k3s_changed=0
k3s_agent_changed=0

journald_dropin="/host/etc/systemd/journald.conf.d/99-logging.conf"
k3s_dropin="/host/etc/systemd/system/k3s.service.d/99-logging.conf"
k3s_agent_dropin="/host/etc/systemd/system/k3s-agent.service.d/99-logging.conf"
k3s_image_gc_dropin="/host/etc/systemd/system/k3s.service.d/98-image-gc.conf"
k3s_agent_image_gc_dropin="/host/etc/systemd/system/k3s-agent.service.d/98-image-gc.conf"

ensure_dropin() {
  local path="$1"
  local owner="$2"
  local new_content="$3"
  local current=""
  if [ -f "${path}" ]; then
    current="$(cat "${path}" || true)"
  fi
  if [ "${current}" != "${new_content}" ]; then
    mkdir -p "$(dirname "${path}")"
    printf "%s\n" "${new_content}" > "${path}"
    changed=1
    case "${owner}" in
      journald)
        journald_changed=1
        ;;
      k3s)
        k3s_changed=1
        ;;
      k3s-agent)
        k3s_agent_changed=1
        ;;
    esac
  fi
}

ensure_dropin \
  "${journald_dropin}" \
  "journald" \
  "[Journal]
Storage=volatile
RuntimeMaxUse=200M
RuntimeKeepFree=512M
MaxFileSec=1h"

if [ -f "/host/etc/systemd/system/k3s.service" ]; then
  ensure_dropin \
    "${k3s_dropin}" \
    "k3s" \
    "[Service]
Environment=\"K3S_KUBELET_ARG=container-log-max-size=10Mi\"
Environment=\"K3S_KUBELET_ARG=container-log-max-files=2\""
fi

if [ -f "/host/etc/systemd/system/k3s.service" ]; then
  ensure_dropin \
    "${k3s_image_gc_dropin}" \
    "k3s" \
    "[Service]
Environment=\"K3S_KUBELET_ARG=image-gc-high-threshold=65\"
Environment=\"K3S_KUBELET_ARG=image-gc-low-threshold=50\"
Environment=\"K3S_KUBELET_ARG=image-gc-minimum-available=8Gi\""
fi

if [ -f "/host/etc/systemd/system/k3s-agent.service" ]; then
  ensure_dropin \
    "${k3s_agent_dropin}" \
    "k3s-agent" \
    "[Service]
Environment=\"K3S_KUBELET_ARG=container-log-max-size=10Mi\"
Environment=\"K3S_KUBELET_ARG=container-log-max-files=2\""
fi

if [ -f "/host/etc/systemd/system/k3s-agent.service" ]; then
  ensure_dropin \
    "${k3s_agent_image_gc_dropin}" \
    "k3s-agent" \
    "[Service]
Environment=\"K3S_KUBELET_ARG=image-gc-high-threshold=65\"
Environment=\"K3S_KUBELET_ARG=image-gc-low-threshold=50\"
Environment=\"K3S_KUBELET_ARG=image-gc-minimum-available=8Gi\""
fi

if [ "${changed}" -eq 1 ]; then
  sleep "$(( (RANDOM % 300) + 10 ))"
  chroot /host /bin/systemctl daemon-reload
  if [ "${journald_changed}" -eq 1 ]; then
    chroot /host /bin/systemctl restart systemd-journald
  fi
  if [ "${k3s_changed}" -eq 1 ]; then
    chroot /host /bin/systemctl restart k3s
  fi
  if [ "${k3s_agent_changed}" -eq 1 ]; then
    chroot /host /bin/systemctl restart k3s-agent
  fi
fi

trim_constrained_pod_logs() {
  local base usage
  for base in /host/mnt/astraios/var/log /host/var/log.hdd; do
    if [ ! -d "${base}/pods" ]; then
      continue
    fi
    usage="$(df -P "${base}" | awk 'NR==2 {gsub(/%/, "", $5); print $5}')"
    if [ -z "${usage}" ] || [ "${usage}" -lt 75 ]; then
      continue
    fi
    find "${base}/pods" -type f \( -name '[1-9]*.log' -o -name '*.log.20*' \) -size +1M -print -exec truncate -s 0 {} \; 2>/dev/null || true
    if [ -d "${base}/containers" ]; then
      find "${base}/containers" -xtype l -print -delete 2>/dev/null || true
    fi
  done
}

while true; do
  trim_constrained_pod_logs
  sleep 600
done