174 lines
5.1 KiB
Python
174 lines
5.1 KiB
Python
"""Process lifecycle and provider artifact edge coverage for CLI workers."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
from testing.tests.test_hermes_cli_support import _completed_result, lanes
|
|
|
|
|
|
def test_stream_process_enforces_runtime_and_lease_boundaries(
|
|
tmp_path: Path,
|
|
):
|
|
timed_out = lanes.stream_process(
|
|
[sys.executable, "-c", "import time; time.sleep(30)"],
|
|
provider="codex",
|
|
cwd=tmp_path,
|
|
env=dict(os.environ),
|
|
log_path=tmp_path / "timeout.log",
|
|
state={},
|
|
state_file=tmp_path / "timeout-state.json",
|
|
heartbeat=lambda _note: True,
|
|
max_runtime=-1,
|
|
)
|
|
assert "maximum runtime" in timed_out.output
|
|
assert timed_out.returncode != 0
|
|
|
|
lease_lost = lanes.stream_process(
|
|
[sys.executable, "-c", "import time; time.sleep(30)"],
|
|
provider="claude",
|
|
cwd=tmp_path,
|
|
env=dict(os.environ),
|
|
log_path=tmp_path / "lease.log",
|
|
state={},
|
|
state_file=tmp_path / "lease-state.json",
|
|
heartbeat=lambda _note: False,
|
|
max_runtime=60,
|
|
)
|
|
assert "lease was lost" in lease_lost.output
|
|
assert lease_lost.returncode != 0
|
|
|
|
|
|
def test_process_identity_and_signal_failures_are_bounded(monkeypatch):
|
|
assert lanes._process_record(999999999) is None
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"_process_identity_matches",
|
|
lambda _pid, _start: True,
|
|
)
|
|
monkeypatch.setattr(
|
|
lanes.os,
|
|
"killpg",
|
|
lambda *_args: (_ for _ in ()).throw(ProcessLookupError()),
|
|
)
|
|
monkeypatch.setattr(
|
|
lanes.os,
|
|
"kill",
|
|
lambda *_args: (_ for _ in ()).throw(ProcessLookupError()),
|
|
)
|
|
lanes._signal_worker_tree(10, {11: (12, 13)}, signal.SIGTERM)
|
|
|
|
|
|
def test_descendant_snapshot_follows_multiple_generations(monkeypatch):
|
|
entries = [Path("/proc/self"), Path("/proc/10"), Path("/proc/11"), Path("/proc/12")]
|
|
monkeypatch.setattr(lanes.Path, "iterdir", lambda _path: iter(entries))
|
|
records = {
|
|
10: (1, 10, 100),
|
|
11: (10, 11, 101),
|
|
12: (11, 12, 102),
|
|
}
|
|
monkeypatch.setattr(lanes, "_process_record", lambda pid: records.get(pid))
|
|
assert lanes._descendant_processes(10) == {11: (11, 101), 12: (12, 102)}
|
|
|
|
|
|
def test_terminate_escalates_after_term_timeout(monkeypatch):
|
|
class Process:
|
|
pid = 321
|
|
waits = 0
|
|
|
|
@classmethod
|
|
def poll(cls):
|
|
return None
|
|
|
|
@classmethod
|
|
def wait(cls, timeout):
|
|
cls.waits += 1
|
|
if cls.waits == 1:
|
|
raise subprocess.TimeoutExpired("worker", timeout)
|
|
return 0
|
|
|
|
signals = []
|
|
monkeypatch.setattr(lanes, "_descendant_processes", lambda _pid: {})
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"_signal_worker_tree",
|
|
lambda _pid, _descendants, sig: signals.append(sig),
|
|
)
|
|
lanes._terminate_worker_process(Process())
|
|
assert signals == [signal.SIGTERM, signal.SIGKILL]
|
|
assert Process.waits == 2
|
|
|
|
|
|
def test_codex_result_collision_and_chmod_failure_preserve_result(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
):
|
|
state = {"run_id": 7, "result_sequence": 0}
|
|
state_file = tmp_path / "task.json"
|
|
first = lanes._result_path(state_file, 7, 1)
|
|
first.write_text("prior", encoding="utf-8")
|
|
result_paths = []
|
|
|
|
def stream(command, **_kwargs):
|
|
result_path = Path(command[command.index("-o") + 1])
|
|
result_path.write_text(json.dumps(_completed_result("new result")), encoding="utf-8")
|
|
result_paths.append(result_path)
|
|
return lanes.ProcessResult(0, "", None, False)
|
|
|
|
monkeypatch.setattr(lanes, "stream_process", stream)
|
|
monkeypatch.setattr(
|
|
lanes.Path,
|
|
"chmod",
|
|
lambda *_args, **_kwargs: (_ for _ in ()).throw(OSError("readonly")),
|
|
)
|
|
route = lanes.Route("codex", "gpt", "high", "p", "c", "r", 1, ())
|
|
result = lanes.run_provider(
|
|
route,
|
|
"work",
|
|
tmp_path,
|
|
state,
|
|
state_file,
|
|
tmp_path / "log",
|
|
lambda _note: True,
|
|
60,
|
|
)
|
|
assert result.structured["summary"] == "new result"
|
|
assert result_paths[0].name.endswith("provider-2.result.json")
|
|
|
|
|
|
def test_codex_missing_thread_skips_colliding_restart_result(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
):
|
|
state = {"run_id": 8, "codex_thread_id": "missing"}
|
|
state_file = tmp_path / "task.json"
|
|
second = lanes._result_path(state_file, 8, 2)
|
|
second.write_text("collision", encoding="utf-8")
|
|
calls = []
|
|
|
|
def stream(command, **_kwargs):
|
|
calls.append(command)
|
|
if len(calls) == 1:
|
|
return lanes.ProcessResult(1, lanes.NO_CODEX_THREAD, None, False)
|
|
return lanes.ProcessResult(0, "", None, False)
|
|
|
|
monkeypatch.setattr(lanes, "stream_process", stream)
|
|
route = lanes.Route("codex", "gpt", "high", "p", "c", "r", 1, ())
|
|
lanes.run_provider(
|
|
route,
|
|
"work",
|
|
tmp_path,
|
|
state,
|
|
state_file,
|
|
tmp_path / "log",
|
|
lambda _note: True,
|
|
60,
|
|
)
|
|
restart = Path(calls[1][calls[1].index("-o") + 1])
|
|
assert restart.name.endswith("provider-3.result.json")
|