302 lines
9.0 KiB
Python
302 lines
9.0 KiB
Python
|
"""Codex, Claude, and worker-process session behavior."""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
from testing.tests.test_hermes_cli_support import (
|
||
|
|
Path,
|
||
|
|
json,
|
||
|
|
lanes,
|
||
|
|
os,
|
||
|
|
signal,
|
||
|
|
sys,
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def test_claude_session_is_reserved_before_first_process(tmp_path: Path, monkeypatch):
|
||
|
|
state: dict = {}
|
||
|
|
state_file = tmp_path / "state.json"
|
||
|
|
observed = {}
|
||
|
|
|
||
|
|
def fake_stream(command, **kwargs):
|
||
|
|
observed["command"] = command
|
||
|
|
observed["state"] = json.loads(state_file.read_text())
|
||
|
|
return lanes.ProcessResult(0, "", {"status": "completed"}, False)
|
||
|
|
|
||
|
|
monkeypatch.setattr(lanes, "stream_process", fake_stream)
|
||
|
|
route = lanes.Route("claude", "claude-opus-5", "high", "claude-high", "jetson", "vote", 1, ())
|
||
|
|
|
||
|
|
lanes.run_provider(
|
||
|
|
route,
|
||
|
|
"Do the work.",
|
||
|
|
tmp_path,
|
||
|
|
state,
|
||
|
|
state_file,
|
||
|
|
tmp_path / "worker.log",
|
||
|
|
lambda _note: True,
|
||
|
|
60,
|
||
|
|
)
|
||
|
|
|
||
|
|
reserved = observed["state"]["claude_session_id"]
|
||
|
|
assert reserved
|
||
|
|
assert ["--session-id", reserved] == observed["command"][-4:-2]
|
||
|
|
|
||
|
|
|
||
|
|
def test_claude_recovery_uses_verified_error_state_machine(tmp_path: Path, monkeypatch):
|
||
|
|
state = {"claude_session_id": "d4f91f59-88cb-43d0-a64f-f3284688fe9e", "claude_started": True}
|
||
|
|
state_file = tmp_path / "state.json"
|
||
|
|
calls = []
|
||
|
|
|
||
|
|
def fake_stream(command, **kwargs):
|
||
|
|
calls.append(command)
|
||
|
|
if len(calls) == 1:
|
||
|
|
return lanes.ProcessResult(
|
||
|
|
1,
|
||
|
|
"No conversation found with session ID: d4f91f59-88cb-43d0-a64f-f3284688fe9e",
|
||
|
|
None,
|
||
|
|
False,
|
||
|
|
)
|
||
|
|
return lanes.ProcessResult(0, "", {"status": "completed"}, False)
|
||
|
|
|
||
|
|
monkeypatch.setattr(lanes, "stream_process", fake_stream)
|
||
|
|
route = lanes.Route("claude", "claude-opus-5", "high", "claude-high", "jetson", "vote", 1, ())
|
||
|
|
lanes.run_provider(route, "Continue.", tmp_path, state, state_file, tmp_path / "log", lambda _: True, 60)
|
||
|
|
|
||
|
|
assert "--resume" in calls[0]
|
||
|
|
assert "--session-id" in calls[1]
|
||
|
|
assert calls[0][calls[0].index("--resume") + 1] == calls[1][calls[1].index("--session-id") + 1]
|
||
|
|
|
||
|
|
|
||
|
|
def test_claude_collision_never_double_starts_a_reserved_session(tmp_path: Path, monkeypatch):
|
||
|
|
state = {"claude_session_id": "24927cee-f3da-4bcf-b624-602c22860155"}
|
||
|
|
state_file = tmp_path / "state.json"
|
||
|
|
calls = []
|
||
|
|
|
||
|
|
def fake_stream(command, **kwargs):
|
||
|
|
calls.append(command)
|
||
|
|
if len(calls) == 1:
|
||
|
|
return lanes.ProcessResult(1, "Session ID already in use", None, False)
|
||
|
|
return lanes.ProcessResult(0, "", {"status": "completed"}, False)
|
||
|
|
|
||
|
|
monkeypatch.setattr(lanes, "stream_process", fake_stream)
|
||
|
|
route = lanes.Route("claude", "claude-opus-5", "high", "claude-high", "jetson", "vote", 1, ())
|
||
|
|
lanes.run_provider(route, "Continue.", tmp_path, state, state_file, tmp_path / "log", lambda _: True, 60)
|
||
|
|
|
||
|
|
assert "--session-id" in calls[0]
|
||
|
|
assert "--resume" in calls[1]
|
||
|
|
assert calls[0][calls[0].index("--session-id") + 1] == calls[1][calls[1].index("--resume") + 1]
|
||
|
|
|
||
|
|
|
||
|
|
def test_codex_thread_started_is_persisted_before_completion(tmp_path: Path):
|
||
|
|
state: dict = {}
|
||
|
|
state_file = tmp_path / "state.json"
|
||
|
|
|
||
|
|
lanes._event_payload(
|
||
|
|
"codex",
|
||
|
|
json.dumps({"type": "thread.started", "thread_id": "thread-123"}),
|
||
|
|
state,
|
||
|
|
state_file,
|
||
|
|
)
|
||
|
|
|
||
|
|
assert json.loads(state_file.read_text())["codex_thread_id"] == "thread-123"
|
||
|
|
|
||
|
|
|
||
|
|
def test_codex_missing_server_thread_restarts_fresh(tmp_path: Path, monkeypatch):
|
||
|
|
state = {"codex_thread_id": "failed-thread"}
|
||
|
|
state_file = tmp_path / "state.json"
|
||
|
|
calls = []
|
||
|
|
|
||
|
|
def fake_stream(command, **kwargs):
|
||
|
|
calls.append(command)
|
||
|
|
if len(calls) == 1:
|
||
|
|
return lanes.ProcessResult(
|
||
|
|
1,
|
||
|
|
"thread/resume failed: no rollout found for thread id failed-thread",
|
||
|
|
None,
|
||
|
|
False,
|
||
|
|
)
|
||
|
|
return lanes.ProcessResult(0, "", {"status": "completed"}, False)
|
||
|
|
|
||
|
|
monkeypatch.setattr(lanes, "stream_process", fake_stream)
|
||
|
|
route = lanes.Route(
|
||
|
|
"codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, ()
|
||
|
|
)
|
||
|
|
|
||
|
|
result = lanes.run_provider(
|
||
|
|
route,
|
||
|
|
"Continue.",
|
||
|
|
tmp_path,
|
||
|
|
state,
|
||
|
|
state_file,
|
||
|
|
tmp_path / "log",
|
||
|
|
lambda _: True,
|
||
|
|
60,
|
||
|
|
)
|
||
|
|
|
||
|
|
assert result.returncode == 0
|
||
|
|
assert "resume" in calls[0]
|
||
|
|
assert "resume" not in calls[1]
|
||
|
|
assert "codex_thread_id" not in json.loads(state_file.read_text())
|
||
|
|
|
||
|
|
|
||
|
|
def test_codex_result_files_are_unique_and_prior_turns_are_retained(
|
||
|
|
tmp_path: Path,
|
||
|
|
monkeypatch,
|
||
|
|
):
|
||
|
|
state = {"run_id": 17}
|
||
|
|
state_file = tmp_path / "task.json"
|
||
|
|
result_paths = []
|
||
|
|
|
||
|
|
def fake_stream(command, **_kwargs):
|
||
|
|
result_path = Path(command[command.index("-o") + 1])
|
||
|
|
result_path.write_text(
|
||
|
|
json.dumps(
|
||
|
|
{
|
||
|
|
"status": "completed",
|
||
|
|
"summary": f"turn {len(result_paths) + 1}",
|
||
|
|
"changed_files": [],
|
||
|
|
"tests_run": [],
|
||
|
|
"artifacts": [],
|
||
|
|
"findings": [],
|
||
|
|
"blockers": [],
|
||
|
|
}
|
||
|
|
),
|
||
|
|
encoding="utf-8",
|
||
|
|
)
|
||
|
|
result_paths.append(result_path)
|
||
|
|
return lanes.ProcessResult(0, "", None, False)
|
||
|
|
|
||
|
|
monkeypatch.setattr(lanes, "stream_process", fake_stream)
|
||
|
|
route = lanes.Route(
|
||
|
|
"codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, ()
|
||
|
|
)
|
||
|
|
|
||
|
|
first = lanes.run_provider(
|
||
|
|
route,
|
||
|
|
"First turn.",
|
||
|
|
tmp_path,
|
||
|
|
state,
|
||
|
|
state_file,
|
||
|
|
tmp_path / "worker.log",
|
||
|
|
lambda _note: True,
|
||
|
|
60,
|
||
|
|
)
|
||
|
|
second = lanes.run_provider(
|
||
|
|
route,
|
||
|
|
"Second turn.",
|
||
|
|
tmp_path,
|
||
|
|
state,
|
||
|
|
state_file,
|
||
|
|
tmp_path / "worker.log",
|
||
|
|
lambda _note: True,
|
||
|
|
60,
|
||
|
|
)
|
||
|
|
|
||
|
|
assert first.structured["summary"] == "turn 1"
|
||
|
|
assert second.structured["summary"] == "turn 2"
|
||
|
|
assert len(set(result_paths)) == 2
|
||
|
|
assert all(path.exists() for path in result_paths)
|
||
|
|
assert all(path.stat().st_mode & 0o777 == 0o600 for path in result_paths)
|
||
|
|
|
||
|
|
|
||
|
|
def test_successful_process_text_cannot_masquerade_as_capacity_failure(tmp_path: Path):
|
||
|
|
result = lanes.stream_process(
|
||
|
|
[sys.executable, "-c", "print('authentication work completed')"],
|
||
|
|
provider="codex",
|
||
|
|
cwd=tmp_path,
|
||
|
|
env=dict(os.environ),
|
||
|
|
log_path=tmp_path / "worker.log",
|
||
|
|
state={},
|
||
|
|
state_file=tmp_path / "state.json",
|
||
|
|
heartbeat=lambda _note: True,
|
||
|
|
max_runtime=60,
|
||
|
|
)
|
||
|
|
|
||
|
|
assert result.returncode == 0
|
||
|
|
assert result.capacity_failure is False
|
||
|
|
|
||
|
|
|
||
|
|
def test_worker_process_uses_isolated_process_session(monkeypatch, tmp_path: Path):
|
||
|
|
original_popen = lanes.subprocess.Popen
|
||
|
|
calls = []
|
||
|
|
|
||
|
|
def recording_popen(*args, **kwargs):
|
||
|
|
calls.append(kwargs)
|
||
|
|
return original_popen(*args, **kwargs)
|
||
|
|
|
||
|
|
monkeypatch.setattr(lanes.subprocess, "Popen", recording_popen)
|
||
|
|
result = lanes.stream_process(
|
||
|
|
[sys.executable, "-c", "print('done')"],
|
||
|
|
provider="codex",
|
||
|
|
cwd=tmp_path,
|
||
|
|
env=dict(os.environ),
|
||
|
|
log_path=tmp_path / "worker.log",
|
||
|
|
state={},
|
||
|
|
state_file=tmp_path / "state.json",
|
||
|
|
heartbeat=lambda _note: True,
|
||
|
|
max_runtime=60,
|
||
|
|
)
|
||
|
|
|
||
|
|
assert result.returncode == 0
|
||
|
|
assert calls[0]["start_new_session"] is True
|
||
|
|
|
||
|
|
|
||
|
|
def test_worker_process_group_is_killed_after_leader_exits(monkeypatch):
|
||
|
|
class FinishedProcess:
|
||
|
|
pid = 4321
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def poll():
|
||
|
|
return 0
|
||
|
|
|
||
|
|
signals = []
|
||
|
|
monkeypatch.setattr(lanes.os, "killpg", lambda pid, sig: signals.append((pid, sig)))
|
||
|
|
|
||
|
|
lanes._terminate_worker_process(FinishedProcess())
|
||
|
|
|
||
|
|
assert signals == [(4321, signal.SIGKILL)]
|
||
|
|
|
||
|
|
|
||
|
|
def test_worker_terminal_descendants_in_separate_groups_are_killed(monkeypatch):
|
||
|
|
class RunningProcess:
|
||
|
|
pid = 4321
|
||
|
|
running = True
|
||
|
|
|
||
|
|
@classmethod
|
||
|
|
def poll(cls):
|
||
|
|
return None if cls.running else 0
|
||
|
|
|
||
|
|
@classmethod
|
||
|
|
def wait(cls, timeout):
|
||
|
|
assert timeout == 10
|
||
|
|
cls.running = False
|
||
|
|
return 0
|
||
|
|
|
||
|
|
group_signals = []
|
||
|
|
process_signals = []
|
||
|
|
monkeypatch.setattr(
|
||
|
|
lanes,
|
||
|
|
"_descendant_processes",
|
||
|
|
lambda _pid: {5000: (5000, 12345)},
|
||
|
|
)
|
||
|
|
monkeypatch.setattr(lanes, "_process_identity_matches", lambda _pid, _start: True)
|
||
|
|
monkeypatch.setattr(
|
||
|
|
lanes.os,
|
||
|
|
"killpg",
|
||
|
|
lambda process_group, sig: group_signals.append((process_group, sig)),
|
||
|
|
)
|
||
|
|
monkeypatch.setattr(
|
||
|
|
lanes.os,
|
||
|
|
"kill",
|
||
|
|
lambda pid, sig: process_signals.append((pid, sig)),
|
||
|
|
)
|
||
|
|
|
||
|
|
lanes._terminate_worker_process(RunningProcess())
|
||
|
|
|
||
|
|
assert set(group_signals) == {
|
||
|
|
(4321, signal.SIGTERM),
|
||
|
|
(5000, signal.SIGTERM),
|
||
|
|
(4321, signal.SIGKILL),
|
||
|
|
(5000, signal.SIGKILL),
|
||
|
|
}
|
||
|
|
assert process_signals == [(5000, signal.SIGTERM), (5000, signal.SIGKILL)]
|