"""Linux process-owner coverage for the Hermes CLI lane supervisor.""" from __future__ import annotations import importlib.util import json import os import shutil import signal import sys from pathlib import Path import pytest import yaml ROOT = Path(__file__).parents[2] HERMES = ROOT / "services/hermes" SCRIPT = HERMES / "scripts/cli_lane_supervisor.py" SPEC = importlib.util.spec_from_file_location("cli_lane_supervisor", SCRIPT) assert SPEC and SPEC.loader supervisor = importlib.util.module_from_spec(SPEC) sys.modules[SPEC.name] = supervisor SPEC.loader.exec_module(supervisor) def _run(command: list[str], state_path: Path, *, grace: float = 0.1) -> int: owner = supervisor.ChildSupervisor( command, state_path, grace_seconds=grace, poll_seconds=0.01, ) return owner.run() def _state(path: Path) -> dict: return json.loads(path.read_text(encoding="utf-8")) def _python(code: str) -> list[str]: return [sys.executable, "-c", code] def test_manifest_installs_supervisor_as_pid_one_boundary(): deployment = yaml.safe_load((HERMES / "agent-deployment.yaml").read_text()) containers = { item["name"]: item for item in deployment["spec"]["template"]["spec"]["containers"] } startup = containers["cli-lane-runner"]["args"][0] kustomization = (HERMES / "kustomization.yaml").read_text() source = SCRIPT.read_text() assert "exec /opt/hermes/.venv/bin/python /opt/coordinator/cli_lane_supervisor.py --" in startup assert "/opt/hermes/.venv/bin/python /opt/coordinator/cli_lane_runner.py" in startup assert "cli_lane_supervisor.py=scripts/cli_lane_supervisor.py" in kustomization assert "signal.signal(signal.SIGCHLD" not in source assert len(source.splitlines()) < 500 @pytest.mark.parametrize("exit_code", [0, 7]) def test_normal_exit_preserves_status_stream_and_bounded_state( tmp_path: Path, capfd, exit_code: int, ): state_path = tmp_path / "process-state.json" result = _run( _python(f"import sys; print('provider-stream-ok'); sys.exit({exit_code})"), state_path, ) captured = capfd.readouterr() document = _state(state_path) assert result == exit_code assert "provider-stream-ok" in captured.out assert document == { "active_children": 0, "adopted_children": 0, "escalated": False, "orphaned_children_total": 0, "phase": "exited", "reaped_children_total": 1, "reaped_orphans_total": 0, "runner_exit_code": exit_code, "termination_signal": None, "updated_at": document["updated_at"], } assert state_path.stat().st_mode & 0o777 == 0o600 assert "provider-stream-ok" not in state_path.read_text() @pytest.mark.parametrize( ("binary", "arguments"), [("git", ["git", "--version"]), ("ssh", ["ssh", "-V"])], ) def test_provider_session_orphans_are_adopted_and_reaped_while_runner_lives( tmp_path: Path, binary: str, arguments: list[str], ): executable = shutil.which(binary) assert executable state_path = tmp_path / f"{binary}.json" provider_code = ( "import os,time\n" "child=os.fork()\n" "if child == 0:\n" " time.sleep(0.05)\n" f" os.execv({executable!r}, {arguments!r})\n" "os._exit(0)\n" ) runner_code = ( "import subprocess,sys,time\n" f"provider=subprocess.Popen([sys.executable,'-c',{provider_code!r}]," "start_new_session=True)\n" "provider.wait()\n" "time.sleep(0.25)\n" ) assert _run(_python(runner_code), state_path) == 0 document = _state(state_path) assert document["active_children"] == 0 assert document["orphaned_children_total"] >= 1 assert document["reaped_orphans_total"] >= 1 assert document["runner_exit_code"] == 0 def test_concurrent_provider_orphans_have_one_non_competing_wait_owner( tmp_path: Path, ): state_path = tmp_path / "concurrent.json" provider_code = ( "import os,time\n" "child=os.fork()\n" "if child == 0:\n" " time.sleep(0.05)\n" " os.execl('/bin/true','true')\n" "os._exit(0)\n" ) runner_code = ( "import subprocess,sys,time\n" f"code={provider_code!r}\n" "providers=[subprocess.Popen([sys.executable,'-c',code]," "start_new_session=True) for _ in range(4)]\n" "statuses=[item.wait() for item in providers]\n" "assert statuses == [0,0,0,0]\n" "time.sleep(0.25)\n" ) assert _run(_python(runner_code), state_path) == 0 document = _state(state_path) assert document["orphaned_children_total"] >= 4 assert document["reaped_orphans_total"] >= 4 assert document["reaped_children_total"] >= 5 def test_runner_exit_terminates_and_reaps_detached_helper_before_restart( tmp_path: Path, ): state_path = tmp_path / "restart.json" helper_pid_path = tmp_path / "helper.pid" runner_code = ( "import os,signal,time\n" "child=os.fork()\n" "if child == 0:\n" " os.setsid()\n" " signal.signal(signal.SIGTERM,signal.SIG_IGN)\n" " while True: time.sleep(1)\n" f"open({str(helper_pid_path)!r},'w').write(str(child))\n" ) assert _run(_python(runner_code), state_path, grace=0.05) == 0 helper_pid = int(helper_pid_path.read_text()) document = _state(state_path) assert not Path(f"/proc/{helper_pid}").exists() assert document["escalated"] is True assert document["orphaned_children_total"] >= 1 assert document["reaped_orphans_total"] >= 1 assert document["runner_exit_code"] == 0 def test_sigterm_cancellation_escalates_to_sigkill_and_reaps_nested_sessions( tmp_path: Path, ): state_path = tmp_path / "cancel.json" helper_pid_path = tmp_path / "cancel-helper.pid" runner_code = ( "import os,signal,time\n" "signal.signal(signal.SIGTERM,signal.SIG_IGN)\n" "child=os.fork()\n" "if child == 0:\n" " os.setsid()\n" " signal.signal(signal.SIGTERM,signal.SIG_IGN)\n" " while True: time.sleep(1)\n" f"open({str(helper_pid_path)!r},'w').write(str(child))\n" "time.sleep(0.1)\n" "os.kill(os.getppid(),signal.SIGTERM)\n" "while True: time.sleep(1)\n" ) assert _run(_python(runner_code), state_path, grace=0.05) == 137 helper_pid = int(helper_pid_path.read_text()) document = _state(state_path) assert not Path(f"/proc/{helper_pid}").exists() assert document["termination_signal"] == signal.SIGTERM assert document["escalated"] is True assert document["runner_exit_code"] == 137 assert document["active_children"] == 0 def test_provider_cancellation_leaves_no_unowned_helper(tmp_path: Path): state_path = tmp_path / "provider-cancel.json" helper_ready = tmp_path / "provider-helper.ready" provider_code = ( "import os,signal,time\n" "child=os.fork()\n" "if child == 0:\n" " os.setsid()\n" " signal.signal(signal.SIGTERM,signal.SIG_IGN)\n" f" open({str(helper_ready)!r},'w').write('ready')\n" " while True: time.sleep(1)\n" "while True: time.sleep(1)\n" ) runner_code = ( "import os,pathlib,signal,subprocess,sys,time\n" f"provider=subprocess.Popen([sys.executable,'-c',{provider_code!r}]," "start_new_session=True)\n" f"ready=pathlib.Path({str(helper_ready)!r})\n" "while not ready.exists(): time.sleep(0.01)\n" "os.killpg(provider.pid,signal.SIGTERM)\n" "provider.wait()\n" ) assert _run(_python(runner_code), state_path, grace=0.05) == 0 document = _state(state_path) assert document["active_children"] == 0 assert document["orphaned_children_total"] >= 1 assert document["reaped_orphans_total"] >= 1 def test_observability_failure_does_not_mask_runner_status(capfd): result = _run(_python("raise SystemExit(3)"), Path("/proc/not-writable/state")) assert result == 3 assert capfd.readouterr().err.count("process state unavailable") == 1 def test_exec_failure_is_generic_and_preserves_127(tmp_path: Path, capfd): result = _run(["/definitely/missing/runner"], tmp_path / "exec.json") assert result == 127 assert "runner exec failed" in capfd.readouterr().err assert _state(tmp_path / "exec.json")["runner_exit_code"] == 127 def test_helpers_cover_invalid_input_bounds_and_process_identity( monkeypatch, capfd, ): monkeypatch.setenv("SUPERVISOR_FLOAT", "invalid") assert supervisor._bounded_float("SUPERVISOR_FLOAT", 2.0, 1.0, 3.0) == 2.0 monkeypatch.setenv("SUPERVISOR_FLOAT", "99") assert supervisor._bounded_float("SUPERVISOR_FLOAT", 2.0, 1.0, 3.0) == 3.0 monkeypatch.setenv("SUPERVISOR_FLOAT", "-1") assert supervisor._bounded_float("SUPERVISOR_FLOAT", 2.0, 1.0, 3.0) == 1.0 assert supervisor.main([]) == supervisor.EXIT_USAGE assert supervisor.main(["--", "relative-runner"]) == supervisor.EXIT_USAGE assert "usage:" in capfd.readouterr().err current = supervisor._process_record(os.getpid()) assert current and current.parent > 0 and current.started > 0 assert supervisor._same_process(os.getpid(), current.started) assert not supervisor._same_process(os.getpid(), current.started + 1) records = { 10: supervisor.ProcessRecord(1, 10, 1, "S"), 11: supervisor.ProcessRecord(10, 11, 2, "S"), 12: supervisor.ProcessRecord(11, 12, 3, "Z"), 20: supervisor.ProcessRecord(1, 20, 4, "S"), } assert set(supervisor._descendants(10, records)) == {11, 12} assert supervisor._decode_wait_status((signal.SIGSTOP << 8) | 0x7F) == 1 def test_main_builds_bounded_supervisor_without_exposing_configuration( tmp_path: Path, monkeypatch, ): observed = {} class FakeSupervisor: def __init__(self, command, state_path, **kwargs): observed.update(command=command, state_path=state_path, kwargs=kwargs) @staticmethod def run(): return 9 state_path = tmp_path / "configured.json" monkeypatch.setattr(supervisor, "ChildSupervisor", FakeSupervisor) monkeypatch.setenv("HERMES_CLI_PROCESS_STATE_PATH", str(state_path)) monkeypatch.setenv("HERMES_CLI_PROCESS_GRACE_SECONDS", "999") monkeypatch.setenv("HERMES_CLI_PROCESS_POLL_SECONDS", "0") assert supervisor.main(["--", "/runner", "argument"]) == 9 assert observed == { "command": ["/runner", "argument"], "state_path": state_path, "kwargs": {"grace_seconds": 30.0, "poll_seconds": 0.01}, } def test_signal_forwarding_ignores_stale_and_disappeared_processes(monkeypatch): owner = supervisor.ChildSupervisor(["runner"], Path("state")) owner.requested_signal = signal.SIGTERM owner._handle_control_signal(signal.SIGINT, None) assert owner.requested_signal == signal.SIGTERM groups = [] processes = [] tree = { 41: supervisor.ProcessRecord(1, 41, 1, "S"), 42: supervisor.ProcessRecord(1, 42, 2, "S"), } monkeypatch.setattr(supervisor.os, "getpgrp", lambda: 42) def missing_group(group, _signal): groups.append(group) raise ProcessLookupError def denied_process(pid, _signal): processes.append(pid) raise PermissionError monkeypatch.setattr(supervisor.os, "killpg", missing_group) monkeypatch.setattr( supervisor, "_same_process", lambda pid, _started: pid == 42, ) monkeypatch.setattr(supervisor.os, "kill", denied_process) owner._signal_tree(tree, signal.SIGKILL) assert groups == [41] assert processes == [42] def test_orphan_identity_memory_is_bounded_by_current_adoptions(monkeypatch): owner = supervisor.ChildSupervisor(["runner"], Path("state")) owner.pid = 10 owner.root_pid = 11 root = supervisor.ProcessRecord(10, 11, 1, "S") orphan = supervisor.ProcessRecord(10, 12, 2, "Z") owner._observe({11: root, 12: orphan}) owner._observe({11: root, 12: orphan}) assert owner.counts.orphaned_total == 1 assert owner._active_orphans == {12: 2} owner._observe({11: root}) assert owner._active_orphans == {} waits = iter([(99, 0), (0, 0)]) monkeypatch.setattr(supervisor.os, "waitpid", lambda *_args: next(waits)) owner._reap() assert owner.counts.orphaned_total == 2 assert owner.counts.reaped_orphans_total == 1 assert owner._active_orphans == {} def test_subreaper_setup_failure_is_explicit(monkeypatch): class FailedPrctl: @staticmethod def prctl(*_args): return -1 monkeypatch.setattr(supervisor.ctypes, "CDLL", lambda *_args, **_kwargs: FailedPrctl()) monkeypatch.setattr(supervisor.ctypes, "get_errno", lambda: 22) with pytest.raises(OSError, match="subreaper"): supervisor._enable_subreaper()