04a737e156
Some checks failed
Tests / Declarative: Post Actions failed: 43, skipped: 89, passed: 4272
89 lines
2.6 KiB
Python
89 lines
2.6 KiB
Python
"""Exercise the deployed overlay recovery logic without touching host services."""
|
|
|
|
from pathlib import Path
|
|
import subprocess
|
|
import textwrap
|
|
|
|
import yaml
|
|
|
|
|
|
MANIFEST = (
|
|
Path(__file__).resolve().parents[2]
|
|
/ "services/maintenance/node-ops/titan-22-link-keeper-daemonset.yaml"
|
|
)
|
|
|
|
|
|
def run_recovery(tmp_path, scenario):
|
|
"""Run the manifest function with fake network/service commands; return actions."""
|
|
doc = yaml.safe_load(MANIFEST.read_text())
|
|
script = doc["spec"]["template"]["spec"]["containers"][0]["command"][-1]
|
|
start = script.index("recover_overlay() {")
|
|
end = script.index('\n if [ "$(cat "/sys/class/net/', start)
|
|
function = textwrap.dedent(script[start:end])
|
|
harness = r'''
|
|
set -eu
|
|
overlay=missing
|
|
active=yes
|
|
ip() { [ "$overlay" = present ]; }
|
|
systemctl() {
|
|
if [ "$1" = is-active ]; then
|
|
[ "$active" = yes ]
|
|
else
|
|
printf "%s\n" "$*" >>actions
|
|
fi
|
|
}
|
|
'''
|
|
subprocess.run(["sh", "-c", harness + function + "\n" + scenario],
|
|
cwd=tmp_path, check=True, capture_output=True, text=True)
|
|
actions = tmp_path / "actions"
|
|
return actions.read_text().splitlines() if actions.exists() else []
|
|
|
|
|
|
def test_missing_overlay_needs_two_observations(tmp_path):
|
|
"""Ignore one missing observation and wait the full grace interval."""
|
|
assert run_recovery(tmp_path, """
|
|
recover_overlay state 1000
|
|
recover_overlay state 1059
|
|
test ! -f actions
|
|
recover_overlay state 1060
|
|
""") == ["--no-block try-restart k3s-agent"]
|
|
|
|
|
|
def test_restart_cooldown_and_boot_limit(tmp_path):
|
|
"""Persistent failure cannot restart the node agent continuously."""
|
|
assert run_recovery(tmp_path, """
|
|
recover_overlay state 1000
|
|
recover_overlay state 1060
|
|
recover_overlay state 1959
|
|
recover_overlay state 1960
|
|
recover_overlay state 2860
|
|
recover_overlay state 3760
|
|
recover_overlay state 10000
|
|
""") == ["--no-block try-restart k3s-agent"] * 3
|
|
|
|
|
|
def test_healthy_overlay_resets_observation(tmp_path):
|
|
"""A new outage must get its own observation grace interval."""
|
|
assert run_recovery(tmp_path, """
|
|
recover_overlay state 1000
|
|
overlay=present
|
|
recover_overlay state 1060
|
|
test ! -f state/missing-since
|
|
overlay=missing
|
|
recover_overlay state 2000
|
|
test ! -f actions
|
|
recover_overlay state 2060
|
|
""") == ["--no-block try-restart k3s-agent"]
|
|
|
|
|
|
def test_startup_and_inactive_service_are_not_restarted(tmp_path):
|
|
"""Allow normal boot and leave intentionally stopped agents alone."""
|
|
assert run_recovery(tmp_path, """
|
|
recover_overlay state 100
|
|
test ! -f state/missing-since
|
|
active=no
|
|
recover_overlay state 1000 || true
|
|
recover_overlay state 2000 || true
|
|
test ! -f state/missing-since
|
|
""") == []
|