390 lines
13 KiB
Python
390 lines
13 KiB
Python
"""Linux process-owner coverage for the Hermes CLI lane supervisor."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
import shutil
|
|
import signal
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
import yaml
|
|
|
|
|
|
ROOT = Path(__file__).parents[2]
|
|
HERMES = ROOT / "services/hermes"
|
|
SCRIPT = HERMES / "scripts/cli_lane_supervisor.py"
|
|
SPEC = importlib.util.spec_from_file_location("cli_lane_supervisor", SCRIPT)
|
|
assert SPEC and SPEC.loader
|
|
supervisor = importlib.util.module_from_spec(SPEC)
|
|
sys.modules[SPEC.name] = supervisor
|
|
SPEC.loader.exec_module(supervisor)
|
|
|
|
|
|
def _run(command: list[str], state_path: Path, *, grace: float = 0.1) -> int:
|
|
owner = supervisor.ChildSupervisor(
|
|
command,
|
|
state_path,
|
|
grace_seconds=grace,
|
|
poll_seconds=0.01,
|
|
)
|
|
return owner.run()
|
|
|
|
|
|
def _state(path: Path) -> dict:
|
|
return json.loads(path.read_text(encoding="utf-8"))
|
|
|
|
|
|
def _python(code: str) -> list[str]:
|
|
return [sys.executable, "-c", code]
|
|
|
|
|
|
def test_manifest_installs_supervisor_as_pid_one_boundary():
|
|
deployment = yaml.safe_load((HERMES / "agent-deployment.yaml").read_text())
|
|
containers = {
|
|
item["name"]: item
|
|
for item in deployment["spec"]["template"]["spec"]["containers"]
|
|
}
|
|
startup = containers["cli-lane-runner"]["args"][0]
|
|
kustomization = (HERMES / "kustomization.yaml").read_text()
|
|
source = SCRIPT.read_text()
|
|
|
|
assert "exec /opt/hermes/.venv/bin/python /opt/coordinator/cli_lane_supervisor.py --" in startup
|
|
assert "/opt/hermes/.venv/bin/python /opt/coordinator/cli_lane_runner.py" in startup
|
|
assert "cli_lane_supervisor.py=scripts/cli_lane_supervisor.py" in kustomization
|
|
assert "signal.signal(signal.SIGCHLD" not in source
|
|
assert len(source.splitlines()) < 500
|
|
|
|
|
|
@pytest.mark.parametrize("exit_code", [0, 7])
|
|
def test_normal_exit_preserves_status_stream_and_bounded_state(
|
|
tmp_path: Path,
|
|
capfd,
|
|
exit_code: int,
|
|
):
|
|
state_path = tmp_path / "process-state.json"
|
|
|
|
result = _run(
|
|
_python(f"import sys; print('provider-stream-ok'); sys.exit({exit_code})"),
|
|
state_path,
|
|
)
|
|
|
|
captured = capfd.readouterr()
|
|
document = _state(state_path)
|
|
assert result == exit_code
|
|
assert "provider-stream-ok" in captured.out
|
|
assert document == {
|
|
"active_children": 0,
|
|
"adopted_children": 0,
|
|
"escalated": False,
|
|
"orphaned_children_total": 0,
|
|
"phase": "exited",
|
|
"reaped_children_total": 1,
|
|
"reaped_orphans_total": 0,
|
|
"runner_exit_code": exit_code,
|
|
"termination_signal": None,
|
|
"updated_at": document["updated_at"],
|
|
}
|
|
assert state_path.stat().st_mode & 0o777 == 0o600
|
|
assert "provider-stream-ok" not in state_path.read_text()
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("binary", "arguments"),
|
|
[("git", ["git", "--version"]), ("ssh", ["ssh", "-V"])],
|
|
)
|
|
def test_provider_session_orphans_are_adopted_and_reaped_while_runner_lives(
|
|
tmp_path: Path,
|
|
binary: str,
|
|
arguments: list[str],
|
|
):
|
|
executable = shutil.which(binary)
|
|
assert executable
|
|
state_path = tmp_path / f"{binary}.json"
|
|
provider_code = (
|
|
"import os,time\n"
|
|
"child=os.fork()\n"
|
|
"if child == 0:\n"
|
|
" time.sleep(0.05)\n"
|
|
f" os.execv({executable!r}, {arguments!r})\n"
|
|
"os._exit(0)\n"
|
|
)
|
|
runner_code = (
|
|
"import subprocess,sys,time\n"
|
|
f"provider=subprocess.Popen([sys.executable,'-c',{provider_code!r}],"
|
|
"start_new_session=True)\n"
|
|
"provider.wait()\n"
|
|
"time.sleep(0.25)\n"
|
|
)
|
|
|
|
assert _run(_python(runner_code), state_path) == 0
|
|
|
|
document = _state(state_path)
|
|
assert document["active_children"] == 0
|
|
assert document["orphaned_children_total"] >= 1
|
|
assert document["reaped_orphans_total"] >= 1
|
|
assert document["runner_exit_code"] == 0
|
|
|
|
|
|
def test_concurrent_provider_orphans_have_one_non_competing_wait_owner(
|
|
tmp_path: Path,
|
|
):
|
|
state_path = tmp_path / "concurrent.json"
|
|
provider_code = (
|
|
"import os,time\n"
|
|
"child=os.fork()\n"
|
|
"if child == 0:\n"
|
|
" time.sleep(0.05)\n"
|
|
" os.execl('/bin/true','true')\n"
|
|
"os._exit(0)\n"
|
|
)
|
|
runner_code = (
|
|
"import subprocess,sys,time\n"
|
|
f"code={provider_code!r}\n"
|
|
"providers=[subprocess.Popen([sys.executable,'-c',code],"
|
|
"start_new_session=True) for _ in range(4)]\n"
|
|
"statuses=[item.wait() for item in providers]\n"
|
|
"assert statuses == [0,0,0,0]\n"
|
|
"time.sleep(0.25)\n"
|
|
)
|
|
|
|
assert _run(_python(runner_code), state_path) == 0
|
|
|
|
document = _state(state_path)
|
|
assert document["orphaned_children_total"] >= 4
|
|
assert document["reaped_orphans_total"] >= 4
|
|
assert document["reaped_children_total"] >= 5
|
|
|
|
|
|
def test_runner_exit_terminates_and_reaps_detached_helper_before_restart(
|
|
tmp_path: Path,
|
|
):
|
|
state_path = tmp_path / "restart.json"
|
|
helper_pid_path = tmp_path / "helper.pid"
|
|
runner_code = (
|
|
"import os,signal,time\n"
|
|
"child=os.fork()\n"
|
|
"if child == 0:\n"
|
|
" os.setsid()\n"
|
|
" signal.signal(signal.SIGTERM,signal.SIG_IGN)\n"
|
|
" while True: time.sleep(1)\n"
|
|
f"open({str(helper_pid_path)!r},'w').write(str(child))\n"
|
|
)
|
|
|
|
assert _run(_python(runner_code), state_path, grace=0.05) == 0
|
|
|
|
helper_pid = int(helper_pid_path.read_text())
|
|
document = _state(state_path)
|
|
assert not Path(f"/proc/{helper_pid}").exists()
|
|
assert document["escalated"] is True
|
|
assert document["orphaned_children_total"] >= 1
|
|
assert document["reaped_orphans_total"] >= 1
|
|
assert document["runner_exit_code"] == 0
|
|
|
|
|
|
def test_sigterm_cancellation_escalates_to_sigkill_and_reaps_nested_sessions(
|
|
tmp_path: Path,
|
|
):
|
|
state_path = tmp_path / "cancel.json"
|
|
helper_pid_path = tmp_path / "cancel-helper.pid"
|
|
runner_code = (
|
|
"import os,signal,time\n"
|
|
"signal.signal(signal.SIGTERM,signal.SIG_IGN)\n"
|
|
"child=os.fork()\n"
|
|
"if child == 0:\n"
|
|
" os.setsid()\n"
|
|
" signal.signal(signal.SIGTERM,signal.SIG_IGN)\n"
|
|
" while True: time.sleep(1)\n"
|
|
f"open({str(helper_pid_path)!r},'w').write(str(child))\n"
|
|
"time.sleep(0.1)\n"
|
|
"os.kill(os.getppid(),signal.SIGTERM)\n"
|
|
"while True: time.sleep(1)\n"
|
|
)
|
|
|
|
assert _run(_python(runner_code), state_path, grace=0.05) == 137
|
|
|
|
helper_pid = int(helper_pid_path.read_text())
|
|
document = _state(state_path)
|
|
assert not Path(f"/proc/{helper_pid}").exists()
|
|
assert document["termination_signal"] == signal.SIGTERM
|
|
assert document["escalated"] is True
|
|
assert document["runner_exit_code"] == 137
|
|
assert document["active_children"] == 0
|
|
|
|
|
|
def test_provider_cancellation_leaves_no_unowned_helper(tmp_path: Path):
|
|
state_path = tmp_path / "provider-cancel.json"
|
|
helper_ready = tmp_path / "provider-helper.ready"
|
|
provider_code = (
|
|
"import os,signal,time\n"
|
|
"child=os.fork()\n"
|
|
"if child == 0:\n"
|
|
" os.setsid()\n"
|
|
" signal.signal(signal.SIGTERM,signal.SIG_IGN)\n"
|
|
f" open({str(helper_ready)!r},'w').write('ready')\n"
|
|
" while True: time.sleep(1)\n"
|
|
"while True: time.sleep(1)\n"
|
|
)
|
|
runner_code = (
|
|
"import os,pathlib,signal,subprocess,sys,time\n"
|
|
f"provider=subprocess.Popen([sys.executable,'-c',{provider_code!r}],"
|
|
"start_new_session=True)\n"
|
|
f"ready=pathlib.Path({str(helper_ready)!r})\n"
|
|
"while not ready.exists(): time.sleep(0.01)\n"
|
|
"os.killpg(provider.pid,signal.SIGTERM)\n"
|
|
"provider.wait()\n"
|
|
)
|
|
|
|
assert _run(_python(runner_code), state_path, grace=0.05) == 0
|
|
|
|
document = _state(state_path)
|
|
assert document["active_children"] == 0
|
|
assert document["orphaned_children_total"] >= 1
|
|
assert document["reaped_orphans_total"] >= 1
|
|
|
|
|
|
def test_observability_failure_does_not_mask_runner_status(capfd):
|
|
result = _run(_python("raise SystemExit(3)"), Path("/proc/not-writable/state"))
|
|
|
|
assert result == 3
|
|
assert capfd.readouterr().err.count("process state unavailable") == 1
|
|
|
|
|
|
def test_exec_failure_is_generic_and_preserves_127(tmp_path: Path, capfd):
|
|
result = _run(["/definitely/missing/runner"], tmp_path / "exec.json")
|
|
|
|
assert result == 127
|
|
assert "runner exec failed" in capfd.readouterr().err
|
|
assert _state(tmp_path / "exec.json")["runner_exit_code"] == 127
|
|
|
|
|
|
def test_helpers_cover_invalid_input_bounds_and_process_identity(
|
|
monkeypatch,
|
|
capfd,
|
|
):
|
|
monkeypatch.setenv("SUPERVISOR_FLOAT", "invalid")
|
|
assert supervisor._bounded_float("SUPERVISOR_FLOAT", 2.0, 1.0, 3.0) == 2.0
|
|
monkeypatch.setenv("SUPERVISOR_FLOAT", "99")
|
|
assert supervisor._bounded_float("SUPERVISOR_FLOAT", 2.0, 1.0, 3.0) == 3.0
|
|
monkeypatch.setenv("SUPERVISOR_FLOAT", "-1")
|
|
assert supervisor._bounded_float("SUPERVISOR_FLOAT", 2.0, 1.0, 3.0) == 1.0
|
|
assert supervisor.main([]) == supervisor.EXIT_USAGE
|
|
assert supervisor.main(["--", "relative-runner"]) == supervisor.EXIT_USAGE
|
|
assert "usage:" in capfd.readouterr().err
|
|
|
|
current = supervisor._process_record(os.getpid())
|
|
assert current and current.parent > 0 and current.started > 0
|
|
assert supervisor._same_process(os.getpid(), current.started)
|
|
assert not supervisor._same_process(os.getpid(), current.started + 1)
|
|
records = {
|
|
10: supervisor.ProcessRecord(1, 10, 1, "S"),
|
|
11: supervisor.ProcessRecord(10, 11, 2, "S"),
|
|
12: supervisor.ProcessRecord(11, 12, 3, "Z"),
|
|
20: supervisor.ProcessRecord(1, 20, 4, "S"),
|
|
}
|
|
assert set(supervisor._descendants(10, records)) == {11, 12}
|
|
assert supervisor._decode_wait_status((signal.SIGSTOP << 8) | 0x7F) == 1
|
|
|
|
|
|
def test_main_builds_bounded_supervisor_without_exposing_configuration(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
):
|
|
observed = {}
|
|
|
|
class FakeSupervisor:
|
|
def __init__(self, command, state_path, **kwargs):
|
|
observed.update(command=command, state_path=state_path, kwargs=kwargs)
|
|
|
|
@staticmethod
|
|
def run():
|
|
return 9
|
|
|
|
state_path = tmp_path / "configured.json"
|
|
monkeypatch.setattr(supervisor, "ChildSupervisor", FakeSupervisor)
|
|
monkeypatch.setenv("HERMES_CLI_PROCESS_STATE_PATH", str(state_path))
|
|
monkeypatch.setenv("HERMES_CLI_PROCESS_GRACE_SECONDS", "999")
|
|
monkeypatch.setenv("HERMES_CLI_PROCESS_POLL_SECONDS", "0")
|
|
|
|
assert supervisor.main(["--", "/runner", "argument"]) == 9
|
|
assert observed == {
|
|
"command": ["/runner", "argument"],
|
|
"state_path": state_path,
|
|
"kwargs": {"grace_seconds": 30.0, "poll_seconds": 0.01},
|
|
}
|
|
|
|
|
|
def test_signal_forwarding_ignores_stale_and_disappeared_processes(monkeypatch):
|
|
owner = supervisor.ChildSupervisor(["runner"], Path("state"))
|
|
owner.requested_signal = signal.SIGTERM
|
|
owner._handle_control_signal(signal.SIGINT, None)
|
|
assert owner.requested_signal == signal.SIGTERM
|
|
|
|
groups = []
|
|
processes = []
|
|
tree = {
|
|
41: supervisor.ProcessRecord(1, 41, 1, "S"),
|
|
42: supervisor.ProcessRecord(1, 42, 2, "S"),
|
|
}
|
|
monkeypatch.setattr(supervisor.os, "getpgrp", lambda: 42)
|
|
|
|
def missing_group(group, _signal):
|
|
groups.append(group)
|
|
raise ProcessLookupError
|
|
|
|
def denied_process(pid, _signal):
|
|
processes.append(pid)
|
|
raise PermissionError
|
|
|
|
monkeypatch.setattr(supervisor.os, "killpg", missing_group)
|
|
monkeypatch.setattr(
|
|
supervisor,
|
|
"_same_process",
|
|
lambda pid, _started: pid == 42,
|
|
)
|
|
monkeypatch.setattr(supervisor.os, "kill", denied_process)
|
|
|
|
owner._signal_tree(tree, signal.SIGKILL)
|
|
|
|
assert groups == [41]
|
|
assert processes == [42]
|
|
|
|
|
|
def test_orphan_identity_memory_is_bounded_by_current_adoptions(monkeypatch):
|
|
owner = supervisor.ChildSupervisor(["runner"], Path("state"))
|
|
owner.pid = 10
|
|
owner.root_pid = 11
|
|
root = supervisor.ProcessRecord(10, 11, 1, "S")
|
|
orphan = supervisor.ProcessRecord(10, 12, 2, "Z")
|
|
|
|
owner._observe({11: root, 12: orphan})
|
|
owner._observe({11: root, 12: orphan})
|
|
assert owner.counts.orphaned_total == 1
|
|
assert owner._active_orphans == {12: 2}
|
|
|
|
owner._observe({11: root})
|
|
assert owner._active_orphans == {}
|
|
waits = iter([(99, 0), (0, 0)])
|
|
monkeypatch.setattr(supervisor.os, "waitpid", lambda *_args: next(waits))
|
|
owner._reap()
|
|
|
|
assert owner.counts.orphaned_total == 2
|
|
assert owner.counts.reaped_orphans_total == 1
|
|
assert owner._active_orphans == {}
|
|
|
|
|
|
def test_subreaper_setup_failure_is_explicit(monkeypatch):
|
|
class FailedPrctl:
|
|
@staticmethod
|
|
def prctl(*_args):
|
|
return -1
|
|
|
|
monkeypatch.setattr(supervisor.ctypes, "CDLL", lambda *_args, **_kwargs: FailedPrctl())
|
|
monkeypatch.setattr(supervisor.ctypes, "get_errno", lambda: 22)
|
|
|
|
with pytest.raises(OSError, match="subreaper"):
|
|
supervisor._enable_subreaper()
|