"""Process lifecycle and provider artifact edge coverage for CLI workers.""" from __future__ import annotations import json import os import signal import subprocess import sys from pathlib import Path from testing.tests.test_hermes_cli_support import _completed_result, lanes def test_stream_process_enforces_runtime_and_lease_boundaries( tmp_path: Path, ): timed_out = lanes.stream_process( [sys.executable, "-c", "import time; time.sleep(30)"], provider="codex", cwd=tmp_path, env=dict(os.environ), log_path=tmp_path / "timeout.log", state={}, state_file=tmp_path / "timeout-state.json", heartbeat=lambda _note: True, max_runtime=-1, ) assert "maximum runtime" in timed_out.output assert timed_out.returncode != 0 lease_lost = lanes.stream_process( [sys.executable, "-c", "import time; time.sleep(30)"], provider="claude", cwd=tmp_path, env=dict(os.environ), log_path=tmp_path / "lease.log", state={}, state_file=tmp_path / "lease-state.json", heartbeat=lambda _note: False, max_runtime=60, ) assert "lease was lost" in lease_lost.output assert lease_lost.returncode != 0 def test_stream_process_bounds_retained_line_history(tmp_path: Path): result = lanes.stream_process( [ sys.executable, "-c", "import sys; [sys.stdout.write(f'{n}\\n') for n in range(4010)]", ], provider="codex", cwd=tmp_path, env=dict(os.environ), log_path=tmp_path / "bounded.log", state={}, state_file=tmp_path / "state.json", heartbeat=lambda _note: True, max_runtime=60, ) assert result.returncode == 0 assert "4009" in result.output def test_process_identity_and_signal_failures_are_bounded(monkeypatch): assert lanes._process_record(999999999) is None monkeypatch.setattr( lanes, "_process_identity_matches", lambda _pid, _start: True, ) monkeypatch.setattr( lanes.os, "killpg", lambda *_args: (_ for _ in ()).throw(ProcessLookupError()), ) monkeypatch.setattr( lanes.os, "kill", lambda *_args: (_ for _ in ()).throw(ProcessLookupError()), ) lanes._signal_worker_tree(10, {11: (12, 13)}, signal.SIGTERM) signals = [] monkeypatch.setattr(lanes, "_process_identity_matches", lambda *_args: False) monkeypatch.setattr(lanes.os, "killpg", lambda pid, sig: signals.append((pid, sig))) monkeypatch.setattr( lanes.os, "kill", lambda *_args: (_ for _ in ()).throw(AssertionError("stale PID signaled")), ) lanes._signal_worker_tree(20, {21: (22, 23)}, signal.SIGKILL) assert signals == [(20, signal.SIGKILL)] def test_descendant_snapshot_follows_multiple_generations(monkeypatch): entries = [ Path("/proc/self"), Path("/proc/10"), Path("/proc/11"), Path("/proc/12"), Path("/proc/13"), ] monkeypatch.setattr(lanes.Path, "iterdir", lambda _path: iter(entries)) records = { 10: (1, 10, 100), 11: (10, 11, 101), 12: (11, 12, 102), } monkeypatch.setattr(lanes, "_process_record", lambda pid: records.get(pid)) assert lanes._descendant_processes(10) == {11: (11, 101), 12: (12, 102)} def test_claude_missing_unstarted_session_does_not_mark_started( tmp_path: Path, monkeypatch, ): state = {"claude_session_id": "synthetic-session"} state_file = tmp_path / "state.json" monkeypatch.setattr( lanes, "stream_process", lambda *_args, **_kwargs: lanes.ProcessResult( 1, lanes.NO_CLAUDE_SESSION, None, False ), ) route = lanes.Route("claude", "model", "high", "profile", "test", "test", 1, ()) result = lanes.run_provider( route, "work", tmp_path, state, state_file, tmp_path / "log", lambda _note: True, 60, ) assert result.returncode == 1 assert "claude_started" not in state def test_terminate_escalates_after_term_timeout(monkeypatch): class Process: pid = 321 waits = 0 @classmethod def poll(cls): return None @classmethod def wait(cls, timeout): cls.waits += 1 if cls.waits == 1: raise subprocess.TimeoutExpired("worker", timeout) return 0 signals = [] monkeypatch.setattr(lanes, "_descendant_processes", lambda _pid: {}) monkeypatch.setattr( lanes, "_signal_worker_tree", lambda _pid, _descendants, sig: signals.append(sig), ) lanes._terminate_worker_process(Process()) assert signals == [signal.SIGTERM, signal.SIGKILL] assert Process.waits == 2 def test_codex_result_collision_and_chmod_failure_preserve_result( tmp_path: Path, monkeypatch, ): state = {"run_id": 7, "result_sequence": 0} state_file = tmp_path / "task.json" first = lanes._result_path(state_file, 7, 1) first.write_text("prior", encoding="utf-8") result_paths = [] def stream(command, **_kwargs): result_path = Path(command[command.index("-o") + 1]) result_path.write_text(json.dumps(_completed_result("new result")), encoding="utf-8") result_paths.append(result_path) return lanes.ProcessResult(0, "", None, False) monkeypatch.setattr(lanes, "stream_process", stream) monkeypatch.setattr( lanes.Path, "chmod", lambda *_args, **_kwargs: (_ for _ in ()).throw(OSError("readonly")), ) route = lanes.Route("codex", "gpt", "high", "p", "c", "r", 1, ()) result = lanes.run_provider( route, "work", tmp_path, state, state_file, tmp_path / "log", lambda _note: True, 60, ) assert result.structured["summary"] == "new result" assert result_paths[0].name.endswith("provider-2.result.json") def test_codex_missing_thread_skips_colliding_restart_result( tmp_path: Path, monkeypatch, ): state = {"run_id": 8, "codex_thread_id": "missing"} state_file = tmp_path / "task.json" second = lanes._result_path(state_file, 8, 2) second.write_text("collision", encoding="utf-8") calls = [] def stream(command, **_kwargs): calls.append(command) if len(calls) == 1: return lanes.ProcessResult(1, lanes.NO_CODEX_THREAD, None, False) return lanes.ProcessResult(0, "", None, False) monkeypatch.setattr(lanes, "stream_process", stream) route = lanes.Route("codex", "gpt", "high", "p", "c", "r", 1, ()) lanes.run_provider( route, "work", tmp_path, state, state_file, tmp_path / "log", lambda _note: True, 60, ) restart = Path(calls[1][calls[1].index("-o") + 1]) assert restart.name.endswith("provider-3.result.json") def test_stream_process_trims_the_retained_line_window(tmp_path: Path, monkeypatch): """Streamed chatter keeps only the newest bounded output window.""" monkeypatch.setattr(lanes, "_descendant_processes", lambda _pid: {}) monkeypatch.setattr(lanes, "_terminate_worker_process", lambda *_a, **_k: None) read_fd, write_fd = os.pipe() os.write(write_fd, "".join(f"{index}\n" for index in range(4100)).encode()) os.close(write_fd) polls = [] class ChatteringProcess: pid = 4243 returncode = 0 stdout = os.fdopen(read_fd, "r") def poll(self): polls.append(True) return None if len(polls) < 4300 else 0 monkeypatch.setattr( lanes.subprocess, "Popen", lambda *_args, **_kwargs: ChatteringProcess(), ) heartbeats = [] result = lanes.stream_process( ["ignored"], provider="codex", cwd=tmp_path, env=dict(os.environ), log_path=tmp_path / "chatter.log", state={}, state_file=tmp_path / "chatter-state.json", heartbeat=lambda note: heartbeats.append(note) or True, max_runtime=60, ) assert result.returncode == 0 assert heartbeats published = result.output.splitlines() assert len(published) == 4000 assert published[0] == "100" assert published[-1] == "4099" def test_stream_process_parses_output_left_after_worker_exit( tmp_path: Path, monkeypatch, ): """Buffered final output from an already-exited worker is still parsed.""" payload = json.dumps({"result": json.dumps(_completed_result("buffered"))}) read_fd, write_fd = os.pipe() os.write(write_fd, f"{payload}\n".encode()) os.close(write_fd) fake = type( "ExitedProcess", (), { "pid": 4242, "returncode": 0, "poll": lambda self: 0, "stdout": os.fdopen(read_fd, "r"), }, )() monkeypatch.setattr(lanes.subprocess, "Popen", lambda *_args, **_kwargs: fake) monkeypatch.setattr(lanes, "_terminate_worker_process", lambda *_a, **_k: None) result = lanes.stream_process( ["ignored"], provider="codex", cwd=tmp_path, env={}, log_path=tmp_path / "buffered.log", state={}, state_file=tmp_path / "buffered-state.json", heartbeat=lambda _note: True, max_runtime=60, ) assert result.returncode == 0 assert result.structured and result.structured["summary"] == "buffered" assert payload in (tmp_path / "buffered.log").read_text(encoding="utf-8") def test_descendant_snapshot_ignores_processes_that_vanish(monkeypatch): entries = [Path("/proc/10"), Path("/proc/11"), Path("/proc/13")] monkeypatch.setattr(lanes.Path, "iterdir", lambda _path: iter(entries)) records = {10: (1, 10, 100), 11: (10, 11, 101)} monkeypatch.setattr(lanes, "_process_record", lambda pid: records.get(pid)) assert lanes._descendant_processes(10) == {11: (11, 101)} def test_signal_tree_targets_only_identity_verified_descendants(monkeypatch): """Reused PIDs are never signaled; verified groups and PIDs both are.""" records = {10: (1, 20, 100)} monkeypatch.setattr(lanes, "_process_record", lambda pid: records.get(pid)) group_signals = [] pid_signals = [] monkeypatch.setattr( lanes.os, "killpg", lambda group, sig: group_signals.append((group, sig)), ) monkeypatch.setattr( lanes.os, "kill", lambda pid, sig: pid_signals.append((pid, sig)), ) lanes._signal_worker_tree( 5, {10: (20, 100), 11: (21, 999)}, signal.SIGTERM, ) assert sorted(group_signals) == [(5, signal.SIGTERM), (20, signal.SIGTERM)] assert pid_signals == [(10, signal.SIGTERM)] def test_claude_worker_without_any_recoverable_session_stays_unstarted( tmp_path: Path, monkeypatch, ): """A session lost on resume and restart never marks the lane as started.""" state = {"claude_started": True, "claude_session_id": "session-lost"} state_file = tmp_path / "state.json" commands = [] def stream(command, **_kwargs): commands.append(command) return lanes.ProcessResult( 1, f"{lanes.NO_CLAUDE_SESSION} session-lost", None, False, ) monkeypatch.setattr(lanes, "stream_process", stream) persisted = [] monkeypatch.setattr( sys.modules["cli_lane_provider"], "atomic_json", lambda path, value: persisted.append(dict(value)), ) route = lanes.Route("claude", "claude-fable-5", "high", "p", "c", "r", 1, ()) result = lanes.run_provider( route, "prompt", tmp_path, state, state_file, tmp_path / "worker.log", lambda _note: True, 60, ) assert result.returncode == 1 assert len(commands) == 2 assert "--resume" in commands[0] assert "--session-id" in commands[1] assert len(persisted) == 1