"""Terminal-result validation and exact-run finalization.""" from __future__ import annotations from testing.tests.test_hermes_cli_support import ( Path, SimpleNamespace, _completed_result, errno, json, lanes, nullcontext, pytest, sys, ) def test_terminal_recovery_can_complete_the_exact_latest_ended_run( tmp_path: Path, monkeypatch, ): monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") state_file = lanes.state_path("cassandra", "t_ended") pending, _record = lanes._write_terminal_record( state_file, board="cassandra", task_id="t_ended", run_id=17, structured=_completed_result("accepted before legacy block"), summary="accepted before legacy block", metadata={}, ) task = SimpleNamespace( id="t_ended", status="blocked", result=None, current_run_id=None, assignee="cli-auto", ) guards = [] class Connection: def close(self): return None def complete_task(_conn, _task_id, **kwargs): guards.append(kwargs) task.status = "done" task.result = kwargs["result"] return True fake_db = SimpleNamespace( scoped_current_board=lambda _board: nullcontext(), connect=lambda board: Connection(), get_task=lambda _conn, _task_id: task, complete_task=complete_task, ) monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) assert lanes.recover_pending_finalizations() == 1 assert guards[0]["replay_ended_run_id"] == 17 assert "expected_run_id" not in guards[0] assert task.status == "done" assert not pending.exists() def test_terminal_recovery_completes_an_unchanged_triage_exact_run( tmp_path: Path, monkeypatch, ): monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") pending, record = lanes._write_terminal_record( lanes.state_path("cassandra", "t_triage_ended"), board="cassandra", task_id="t_triage_ended", run_id=18, structured=_completed_result("accepted before loop-breaker triage"), summary="accepted before loop-breaker triage", metadata={}, ) task = SimpleNamespace( id="t_triage_ended", status="triage", result=None, current_run_id=None, completed_run_id=None, assignee="cli-auto", ) guards = [] class Connection: def close(self): return None def complete_task(_conn, _task_id, **kwargs): guards.append(kwargs) assert kwargs == { "result": record["result"], "summary": record["summary"], "metadata": {}, "replay_ended_run_id": 18, } task.status = "done" task.result = kwargs["result"] task.completed_run_id = 18 return True fake_db = SimpleNamespace( scoped_current_board=lambda _board: nullcontext(), connect=lambda board: Connection(), get_task=lambda _conn, _task_id: task, complete_task=complete_task, ) monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) assert lanes.recover_pending_finalizations() == 1 assert len(guards) == 1 assert task.status == "done" assert task.completed_run_id == 18 assert not pending.exists() assert lanes.recover_pending_finalizations() == 0 assert len(guards) == 1 def test_complete_exception_after_journal_is_replayable_and_never_blocks( tmp_path: Path, monkeypatch, ): state_root = tmp_path / "cli-lanes" monkeypatch.setattr(lanes, "STATE_ROOT", state_root) task = SimpleNamespace( id="t_crash", status="running", result=None, current_run_id=51, assignee="cli-auto", max_runtime_seconds=60, ) completion_raises = {"value": True} blocks = [] comments = [] class Connection: def close(self): return None def complete_task(_conn, _task_id, **kwargs): if completion_raises["value"]: raise RuntimeError("crash between journal and commit") task.status = "done" task.result = kwargs["result"] task.current_run_id = None return True fake_db = SimpleNamespace( scoped_current_board=lambda _board: nullcontext(), connect=lambda board: Connection(), get_task=lambda _conn, _task_id: task, worker_log_path=lambda _task_id, board: tmp_path / "worker.log", _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_crash"), set_branch_name=lambda *_args: None, set_workspace_path=lambda *_args: None, build_worker_context=lambda *_args: "Finish safely.", heartbeat_worker=lambda *_args, **_kwargs: True, add_comment=lambda _conn, _task_id, _author, body: comments.append(body), complete_task=complete_task, block_task=lambda *_args, **kwargs: blocks.append(kwargs), ) monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) monkeypatch.setattr( lanes, "select_route", lambda *_args, **_kwargs: lanes.Route( "codex", "gpt-5.6-sol", "high", "codex-high", "test", "test", 1, () ), ) monkeypatch.setattr( lanes, "run_provider", lambda *_args, **_kwargs: lanes.ProcessResult( 0, "", _completed_result("crash-safe result"), False ), ) lanes.execute_claim("cassandra", "t_crash") pending = list((state_root / "cassandra").glob("*.terminal.pending.json")) assert len(pending) == 1 assert blocks == [] assert any("RuntimeError" in body for body in comments) completion_raises["value"] = False assert lanes.recover_pending_finalizations() == 1 assert task.status == "done" assert not pending[0].exists() def test_terminal_replace_then_directory_fsync_enospc_replays_after_restart( tmp_path: Path, monkeypatch, ): state_root = tmp_path / "cli-lanes" monkeypatch.setattr(lanes, "STATE_ROOT", state_root) task = SimpleNamespace( id="t_enospc", status="running", result=None, current_run_id=52, assignee="cli-auto", max_runtime_seconds=60, ) blocks = [] comments = [] completions = [] class Connection: def close(self): return None def complete_task(_conn, _task_id, **kwargs): completions.append(kwargs) task.status = "done" task.result = kwargs["result"] task.current_run_id = None return True fake_db = SimpleNamespace( scoped_current_board=lambda _board: nullcontext(), connect=lambda board: Connection(), get_task=lambda _conn, _task_id: task, worker_log_path=lambda _task_id, board: tmp_path / "worker.log", _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_enospc"), set_branch_name=lambda *_args: None, set_workspace_path=lambda *_args: None, build_worker_context=lambda *_args: "Finish without losing the result.", heartbeat_worker=lambda *_args, **_kwargs: True, add_comment=lambda _conn, _task_id, _author, body: comments.append(body), complete_task=complete_task, block_task=lambda *_args, **kwargs: blocks.append(kwargs), ) monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) monkeypatch.setattr( lanes, "select_route", lambda *_args, **_kwargs: lanes.Route( "codex", "gpt-5.6-sol", "high", "codex-high", "test", "test", 1, () ), ) monkeypatch.setattr( lanes, "run_provider", lambda *_args, **_kwargs: lanes.ProcessResult( 0, "", _completed_result("persisted before ENOSPC"), False ), ) real_replace = lanes.os.replace real_fsync_directory = lanes._fsync_directory terminal_replaced = {"value": False} fail_once = {"value": True} def replace_then_mark(source, destination): real_replace(source, destination) if str(destination).endswith(".terminal.pending.json"): terminal_replaced["value"] = True def fail_after_terminal_replace(directory): if terminal_replaced["value"] and fail_once["value"]: fail_once["value"] = False raise OSError(errno.ENOSPC, "no space after terminal rename") real_fsync_directory(directory) monkeypatch.setattr(lanes.os, "replace", replace_then_mark) monkeypatch.setattr(lanes, "_fsync_directory", fail_after_terminal_replace) lanes.execute_claim("cassandra", "t_enospc") pending = list((state_root / "cassandra").glob("*.terminal.pending.json")) assert terminal_replaced["value"] is True assert fail_once["value"] is False assert len(pending) == 1 assert blocks == [] assert completions == [] assert task.status == "running" assert any("terminal replayable=True" in body for body in comments) # A restarted runner sees the exact pending journal and completes the run. assert lanes.recover_pending_finalizations() == 1 assert len(completions) == 1 assert task.status == "done" assert not pending[0].exists() committed = list((state_root / "cassandra").glob("*.terminal.committed.json")) assert len(committed) == 1 def test_terminal_payload_cannot_select_a_different_board( tmp_path: Path, monkeypatch, ): state_root = tmp_path / "cli-lanes" monkeypatch.setattr(lanes, "STATE_ROOT", state_root) path = lanes._terminal_path(lanes.state_path("alpha", "t_alpha"), 3) lanes.atomic_json( path, { "board": "beta", "task_id": "t_beta", "expected_run_id": 9, "result": json.dumps(_completed_result()), "summary": "forged", "metadata": {}, "kanban_state": "pending", }, ) alpha_task = SimpleNamespace( id="t_alpha", status="running", current_run_id=3, assignee="cli-auto" ) opened = [] reclaimed = [] class Connection: def close(self): return None fake_db = SimpleNamespace( scoped_current_board=lambda _board: nullcontext(), connect=lambda board: (opened.append(board) or Connection()), get_task=lambda _conn, task_id: alpha_task if task_id == "t_alpha" else None, reclaim_task=lambda _conn, task_id, **_kwargs: (reclaimed.append(task_id) or True), ) monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) assert lanes.recover_pending_finalizations() == 0 assert opened == ["alpha"] assert reclaimed == ["t_alpha"] assert not path.exists() quarantined = list((state_root / "alpha/quarantine").glob("*.quarantine")) assert len(quarantined) == 1 assert quarantined[0].stat().st_mode & 0o777 == 0o600 def test_terminal_finalize_rereads_journal_after_a_path_swap( tmp_path: Path, monkeypatch, ): monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") state_file = lanes.state_path("alpha", "t_alpha") path, safe_record = lanes._write_terminal_record( state_file, board="alpha", task_id="t_alpha", run_id=4, structured=_completed_result("safe"), summary="safe", metadata={}, ) forged = dict(safe_record) forged.update({"board": "beta", "task_id": "t_beta", "expected_run_id": 7}) lanes.atomic_json(path, forged) opened = [] fake_db = SimpleNamespace( scoped_current_board=lambda _board: nullcontext(), connect=lambda board: (opened.append(board) or pytest.fail("must not open DB")), ) assert lanes._finalize_terminal_record(fake_db, path, safe_record) == "foreign" assert opened == [] @pytest.mark.parametrize("payload", [b"", b'{"board":"cassandra"']) def test_malformed_exact_run_journal_is_quarantined_and_reclaimed( tmp_path: Path, monkeypatch, payload: bytes, ): state_root = tmp_path / "cli-lanes" monkeypatch.setattr(lanes, "STATE_ROOT", state_root) path = lanes._terminal_path(lanes.state_path("cassandra", "t_partial"), 12) path.parent.mkdir(parents=True) path.write_bytes(payload) task = SimpleNamespace( id="t_partial", status="running", current_run_id=12, assignee="cli-auto" ) reclaimed = [] class Connection: def close(self): return None fake_db = SimpleNamespace( scoped_current_board=lambda _board: nullcontext(), connect=lambda board: Connection(), get_task=lambda _conn, _task_id: task, reclaim_task=lambda *_args, **kwargs: (reclaimed.append(kwargs) or True), ) monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) assert lanes.recover_pending_finalizations() == 0 assert reclaimed == [{ "reason": ( "terminal journal recovery failed (malformed-payload); " "exact run may retry" ), "expected_run_id": 12, }] assert not path.exists() assert lanes._has_pending_finalization("cassandra", "t_partial", 12) is False quarantined = list((state_root / "cassandra/quarantine").glob("*.quarantine")) assert len(quarantined) == 1 assert quarantined[0].stat().st_mode & 0o777 == 0o600