"""Exercise the deployed overlay recovery logic without touching host services.""" from pathlib import Path import subprocess import textwrap import yaml MANIFEST = ( Path(__file__).resolve().parents[2] / "services/maintenance/node-ops/titan-22-link-keeper-daemonset.yaml" ) def run_recovery(tmp_path, scenario): """Run the manifest function with fake network/service commands; return actions.""" doc = yaml.safe_load(MANIFEST.read_text()) script = doc["spec"]["template"]["spec"]["containers"][0]["command"][-1] start = script.index("recover_overlay() {") end = script.index('\n if [ "$(cat "/sys/class/net/', start) function = textwrap.dedent(script[start:end]) harness = r''' set -eu overlay=missing active=yes ip() { [ "$overlay" = present ]; } systemctl() { if [ "$1" = is-active ]; then [ "$active" = yes ] else printf "%s\n" "$*" >>actions fi } ''' subprocess.run(["sh", "-c", harness + function + "\n" + scenario], cwd=tmp_path, check=True, capture_output=True, text=True) actions = tmp_path / "actions" return actions.read_text().splitlines() if actions.exists() else [] def test_missing_overlay_needs_two_observations(tmp_path): """Ignore one missing observation and wait the full grace interval.""" assert run_recovery(tmp_path, """ recover_overlay state 1000 recover_overlay state 1059 test ! -f actions recover_overlay state 1060 """) == ["--no-block try-restart k3s-agent"] def test_restart_cooldown_and_boot_limit(tmp_path): """Persistent failure cannot restart the node agent continuously.""" assert run_recovery(tmp_path, """ recover_overlay state 1000 recover_overlay state 1060 recover_overlay state 1959 recover_overlay state 1960 recover_overlay state 2860 recover_overlay state 3760 recover_overlay state 10000 """) == ["--no-block try-restart k3s-agent"] * 3 def test_healthy_overlay_resets_observation(tmp_path): """A new outage must get its own observation grace interval.""" assert run_recovery(tmp_path, """ recover_overlay state 1000 overlay=present recover_overlay state 1060 test ! -f state/missing-since overlay=missing recover_overlay state 2000 test ! -f actions recover_overlay state 2060 """) == ["--no-block try-restart k3s-agent"] def test_startup_and_inactive_service_are_not_restarted(tmp_path): """Allow normal boot and leave intentionally stopped agents alone.""" assert run_recovery(tmp_path, """ recover_overlay state 100 test ! -f state/missing-since active=no recover_overlay state 1000 || true recover_overlay state 2000 || true test ! -f state/missing-since """) == []