From 04a737e156d253e44c13057a3c29d3773ab70363 Mon Sep 17 00:00:00 2001
From: jenkins <brad@bstein.dev>
Date: Wed, 30 Sep 2026 11:53:55 -0500
Subject: [PATCH] maintenance: restore titan-22 overlay after USB disconnect

---
 .../titan-22-link-keeper-daemonset.yaml       | 43 +++++++++
 .../tests/test_titan22_overlay_recovery.py    | 88 +++++++++++++++++++
 2 files changed, 131 insertions(+)
 create mode 100644 testing/tests/test_titan22_overlay_recovery.py

diff --git a/services/maintenance/node-ops/titan-22-link-keeper-daemonset.yaml b/services/maintenance/node-ops/titan-22-link-keeper-daemonset.yaml
index e956e7df..5279a3fc 100644
--- a/services/maintenance/node-ops/titan-22-link-keeper-daemonset.yaml
+++ b/services/maintenance/node-ops/titan-22-link-keeper-daemonset.yaml
@@ -142,6 +142,49 @@ spec:
                   # k3s embeds kubelet; the standalone Debian unit only crash-loops.
                   systemctl disable --now kubelet.service >/dev/null 2>&1 || true
 
+                  recover_overlay() {
+                    # A USB disconnect removes its VXLAN child without stopping k3s.
+                    # Host /run survives helper restarts but resets on the next boot.
+                    state_dir="$1"
+                    now="$2"
+                    mkdir -p "${state_dir}"
+                    if ip link show flannel.1 >/dev/null 2>&1; then
+                      rm -f "${state_dir}/missing-since"
+                      return
+                    fi
+                    systemctl is-active --quiet k3s-agent || return 0
+                    [ "${now}" -ge 300 ] || return 0
+                    if [ ! -f "${state_dir}/missing-since" ]; then
+                      printf "%s\n" "${now}" >"${state_dir}/missing-since"
+                      echo "pod overlay missing; checking again before recovery"
+                      return
+                    fi
+                    missing_since="$(cat "${state_dir}/missing-since")"
+                    [ "$((now - missing_since))" -ge 60 ] || return 0
+                    attempts=0
+                    last_restart=0
+                    [ ! -f "${state_dir}/attempts" ] || attempts="$(cat "${state_dir}/attempts")"
+                    [ ! -f "${state_dir}/last-restart" ] || last_restart="$(cat "${state_dir}/last-restart")"
+                    if [ "${attempts}" -ge 3 ]; then
+                      echo "pod overlay recovery exhausted for this boot; operator review required"
+                      return
+                    fi
+                    if [ "${attempts}" -gt 0 ] && [ "$((now - last_restart))" -lt 900 ]; then
+                      return
+                    fi
+                    # Record before restarting so helper interruption cannot cause a loop.
+                    printf "%s\n" "$((attempts + 1))" >"${state_dir}/attempts"
+                    printf "%s\n" "${now}" >"${state_dir}/last-restart"
+                    echo "restarting k3s-agent to restore missing pod overlay"
+                    systemctl --no-block try-restart k3s-agent
+                  }
+
+                  if [ "$(cat "/sys/class/net/${usb_if}/carrier" 2>/dev/null)" = "1" ] &&
+                    ip -4 address show dev "${usb_if}" | grep -q " ${canonical_ip}/24"; then
+                    read -r uptime_seconds unused </proc/uptime
+                    recover_overlay /run/titan-22-link-keeper "${uptime_seconds%%.*}"
+                  fi
+
                   ethtool "${usb_if}" | sed -n "1,45p" || true
                   ip -br address show "${usb_if}" || true
                   ip -d link show flannel.1 | sed -n "1,3p" || true
diff --git a/testing/tests/test_titan22_overlay_recovery.py b/testing/tests/test_titan22_overlay_recovery.py
new file mode 100644
index 00000000..e7160595
--- /dev/null
+++ b/testing/tests/test_titan22_overlay_recovery.py
@@ -0,0 +1,88 @@
+"""Exercise the deployed overlay recovery logic without touching host services."""
+
+from pathlib import Path
+import subprocess
+import textwrap
+
+import yaml
+
+
+MANIFEST = (
+    Path(__file__).resolve().parents[2]
+    / "services/maintenance/node-ops/titan-22-link-keeper-daemonset.yaml"
+)
+
+
+def run_recovery(tmp_path, scenario):
+    """Run the manifest function with fake network/service commands; return actions."""
+    doc = yaml.safe_load(MANIFEST.read_text())
+    script = doc["spec"]["template"]["spec"]["containers"][0]["command"][-1]
+    start = script.index("recover_overlay() {")
+    end = script.index('\n    if [ "$(cat "/sys/class/net/', start)
+    function = textwrap.dedent(script[start:end])
+    harness = r'''
+set -eu
+overlay=missing
+active=yes
+ip() { [ "$overlay" = present ]; }
+systemctl() {
+  if [ "$1" = is-active ]; then
+    [ "$active" = yes ]
+  else
+    printf "%s\n" "$*" >>actions
+  fi
+}
+'''
+    subprocess.run(["sh", "-c", harness + function + "\n" + scenario],
+                   cwd=tmp_path, check=True, capture_output=True, text=True)
+    actions = tmp_path / "actions"
+    return actions.read_text().splitlines() if actions.exists() else []
+
+
+def test_missing_overlay_needs_two_observations(tmp_path):
+    """Ignore one missing observation and wait the full grace interval."""
+    assert run_recovery(tmp_path, """
+recover_overlay state 1000
+recover_overlay state 1059
+test ! -f actions
+recover_overlay state 1060
+""") == ["--no-block try-restart k3s-agent"]
+
+
+def test_restart_cooldown_and_boot_limit(tmp_path):
+    """Persistent failure cannot restart the node agent continuously."""
+    assert run_recovery(tmp_path, """
+recover_overlay state 1000
+recover_overlay state 1060
+recover_overlay state 1959
+recover_overlay state 1960
+recover_overlay state 2860
+recover_overlay state 3760
+recover_overlay state 10000
+""") == ["--no-block try-restart k3s-agent"] * 3
+
+
+def test_healthy_overlay_resets_observation(tmp_path):
+    """A new outage must get its own observation grace interval."""
+    assert run_recovery(tmp_path, """
+recover_overlay state 1000
+overlay=present
+recover_overlay state 1060
+test ! -f state/missing-since
+overlay=missing
+recover_overlay state 2000
+test ! -f actions
+recover_overlay state 2060
+""") == ["--no-block try-restart k3s-agent"]
+
+
+def test_startup_and_inactive_service_are_not_restarted(tmp_path):
+    """Allow normal boot and leave intentionally stopped agents alone."""
+    assert run_recovery(tmp_path, """
+recover_overlay state 100
+test ! -f state/missing-since
+active=no
+recover_overlay state 1000 || true
+recover_overlay state 2000 || true
+test ! -f state/missing-since
+""") == []