"""Board scanning, claiming, orphan recovery, and replay guards.""" from __future__ import annotations from testing.tests.test_hermes_cli_support import ( Path, SimpleNamespace, _completed_result, json, lanes, nullcontext, pytest, sys, ) def test_unassigned_ready_task_is_persistently_routed_to_auto_lane(monkeypatch): task = SimpleNamespace(id="t_auto", assignee=None, status="ready") assigned = [] class Connection: def close(self): return None def assign_task(_conn, task_id, profile): assigned.append((task_id, profile)) task.assignee = profile return True fake_db = SimpleNamespace( list_boards=lambda include_archived=False: [{"slug": "cassandra"}], scoped_current_board=lambda _board: nullcontext(), connect=lambda board: Connection(), recompute_ready=lambda _conn: None, list_tasks=lambda _conn: [task], assign_task=assign_task, get_task=lambda _conn, _task_id: task, claim_task=lambda _conn, _task_id, **_kwargs: task, ) monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) assert lanes.claim_ready(set(), 1) == [("cassandra", "t_auto")] assert assigned == [("t_auto", "cli-auto")] def test_corrupt_board_is_quarantined_without_stopping_healthy_lanes(monkeypatch, capsys): class CorruptBoardError(Exception): pass task = SimpleNamespace(id="t_healthy", assignee="cli-auto", status="ready") class Connection: def close(self): return None def connect(*, board): if board == "cassandra": raise CorruptBoardError("integrity_check failed") return Connection() fake_db = SimpleNamespace( KanbanDbCorruptError=CorruptBoardError, list_boards=lambda include_archived=False: [ {"slug": "cassandra"}, {"slug": "healthy"}, ], scoped_current_board=lambda _board: nullcontext(), connect=connect, recompute_ready=lambda _conn: None, list_tasks=lambda _conn: [task], claim_task=lambda _conn, _task_id, **_kwargs: task, ) monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) lanes.BOARD_CORRUPTION_ERRORS.clear() assert lanes.claim_ready(set(), 1) == [("healthy", "t_healthy")] assert "temporarily skipping Kanban board 'cassandra'" in capsys.readouterr().err def test_transient_board_scan_failure_does_not_stop_healthy_lanes(monkeypatch, capsys): task = SimpleNamespace(id="t_healthy", assignee="cli-auto", status="ready") class Connection: def __init__(self, board): self.board = board def close(self): return None def recompute_ready(connection): if connection.board == "cassandra": raise lanes.sqlite3.OperationalError("disk I/O error") fake_db = SimpleNamespace( list_boards=lambda include_archived=False: [ {"slug": "cassandra"}, {"slug": "healthy"}, ], scoped_current_board=lambda _board: nullcontext(), connect=lambda board: Connection(board), recompute_ready=recompute_ready, list_tasks=lambda _conn: [task], claim_task=lambda _conn, _task_id, **_kwargs: task, ) monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) lanes.BOARD_CORRUPTION_ERRORS.clear() assert lanes.claim_ready(set(), 1) == [("healthy", "t_healthy")] error = capsys.readouterr().err assert "temporarily skipping Kanban board 'cassandra'" in error assert "storage OperationalError: disk I/O error" in error def test_board_call_retries_storage_faults_on_fresh_connections(): connections = [] class Connection: def __init__(self): self.closed = False def close(self): self.closed = True def connect(*, board): assert board == "cassandra" connection = Connection() connections.append(connection) return connection attempts = [] def operation(_connection): attempts.append(1) if len(attempts) < 3: raise lanes.sqlite3.OperationalError("disk I/O error") return "healthy" fake_db = SimpleNamespace( scoped_current_board=lambda _board: nullcontext(), connect=connect, ) lanes.BOARD_CORRUPTION_ERRORS.clear() assert lanes._board_call(fake_db, "cassandra", operation) == "healthy" assert len(connections) == 3 assert all(connection.closed for connection in connections) def test_accepted_result_survives_failed_finalization_and_replays_exact_run( tmp_path: Path, monkeypatch, ): state_root = tmp_path / "cli-lanes" monkeypatch.setattr(lanes, "STATE_ROOT", state_root) task = SimpleNamespace( id="t_terminal", status="running", result=None, current_run_id=41, assignee="cli-auto", max_runtime_seconds=60, ) completions = [] comments = [] permit_completion = {"value": False} class Connection: def close(self): return None def complete_task(_conn, _task_id, **kwargs): completions.append(kwargs) if not permit_completion["value"]: return False task.status = "done" task.result = kwargs["result"] task.completed_run_id = kwargs["expected_run_id"] task.current_run_id = None return True fake_db = SimpleNamespace( scoped_current_board=lambda _board: nullcontext(), connect=lambda board: Connection(), get_task=lambda _conn, _task_id: task, worker_log_path=lambda _task_id, board: tmp_path / "worker.log", _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_terminal"), set_branch_name=lambda *_args: None, set_workspace_path=lambda *_args: None, build_worker_context=lambda *_args: "Finish and verify the objective.", heartbeat_worker=lambda *_args, **_kwargs: True, add_comment=lambda _conn, _task_id, _author, body: comments.append(body), complete_task=complete_task, block_task=lambda *_args, **_kwargs: pytest.fail( "an accepted journaled result must not be converted into a block" ), ) monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) monkeypatch.setattr( lanes, "select_route", lambda *_args, **_kwargs: lanes.Route( "codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, () ), ) structured = { "status": "completed", "summary": "Verified exact terminal result.", "changed_files": [], "tests_run": ["pytest: passed"], "artifacts": [], "findings": [], "blockers": [], } monkeypatch.setattr( lanes, "run_provider", lambda *_args, **_kwargs: lanes.ProcessResult(0, "", structured, False), ) lanes.execute_claim("cassandra", "t_terminal") candidates = list((state_root / "cassandra").glob("*.candidate-*.json")) terminals = list((state_root / "cassandra").glob("*.terminal.pending.json")) assert len(candidates) == 1 assert json.loads(candidates[0].read_text())["structured"] == structured assert len(terminals) == 1 assert json.loads(terminals[0].read_text())["kanban_state"] == "pending" assert task.status == "running" assert any("durably journaled" in body for body in comments) permit_completion["value"] = True assert lanes.recover_pending_finalizations() == 1 assert task.status == "done" assert len(completions) == 2 assert not terminals[0].exists() committed = list( (state_root / "cassandra").glob("*.terminal.committed.json") ) assert len(committed) == 1 terminal = json.loads(committed[0].read_text()) assert terminal["kanban_state"] == "committed" assert completions[-1]["result"] == terminal["result"] def test_restart_does_not_reclaim_an_exact_run_awaiting_finalization( tmp_path: Path, monkeypatch, ): state_root = tmp_path / "cli-lanes" monkeypatch.setattr(lanes, "STATE_ROOT", state_root) state_file = lanes.state_path("cassandra", "t_terminal") terminal_path, _record = lanes._write_terminal_record( state_file, board="cassandra", task_id="t_terminal", run_id=8, structured={ "status": "completed", "summary": "done", "changed_files": [], "tests_run": [], "artifacts": [], "findings": [], "blockers": [], }, summary="done", metadata={}, ) assert terminal_path.exists() task = SimpleNamespace( id="t_terminal", status="running", assignee="cli-auto", current_run_id=8, ) reclaimed = [] class Connection: def close(self): return None fake_db = SimpleNamespace( list_boards=lambda include_archived=False: [{"slug": "cassandra"}], scoped_current_board=lambda _board: nullcontext(), connect=lambda board: Connection(), list_tasks=lambda _conn: [task], reclaim_task=lambda *_args, **_kwargs: reclaimed.append(True), ) monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0) lanes.recover_orphans() assert reclaimed == [] def test_orphan_recovery_passes_the_scanned_run_as_atomic_reclaim_guard( tmp_path: Path, monkeypatch, ): monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") tasks = [ SimpleNamespace( id="t_guarded", status="running", assignee="cli-auto", current_run_id=91, claim_lock="direct-cli-lane", ), SimpleNamespace( id="t_no_run", status="running", assignee="cli-auto", current_run_id=None, ), ] reclaimed = [] class Connection: def close(self): return None fake_db = SimpleNamespace( list_boards=lambda include_archived=False: [{"slug": "cassandra"}], scoped_current_board=lambda _board: nullcontext(), connect=lambda board: Connection(), list_tasks=lambda _conn: tasks, reclaim_task=lambda _conn, task_id, **kwargs: ( reclaimed.append((task_id, kwargs)) or True ), ) monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0) monkeypatch.setattr(lanes.supervisor_state, "get_child", lambda *_args: None) lanes.recover_orphans() assert reclaimed == [ ( "t_guarded", { "reason": "direct CLI lane restarted; provider session will resume", "expected_run_id": 91, }, ) ] def test_prepared_evidence_without_pending_still_pins_the_exact_run( tmp_path: Path, monkeypatch, ): monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") pending, record = lanes._write_terminal_record( lanes.state_path("cassandra", "t_prepared_pin"), board="cassandra", task_id="t_prepared_pin", run_id=14, structured=_completed_result("accepted and prepared"), summary="accepted and prepared", metadata={}, ) identity = lanes._terminal_identity(pending) assert identity is not None lanes._persist_prepared_evidence(identity, record) pending.unlink() assert lanes._has_pending_finalization("cassandra", "t_prepared_pin", 14) is True def test_terminal_replay_never_crosses_into_a_replacement_run( tmp_path: Path, monkeypatch, ): state_root = tmp_path / "cli-lanes" monkeypatch.setattr(lanes, "STATE_ROOT", state_root) state_file = lanes.state_path("cassandra", "t_terminal") terminal_path, _record = lanes._write_terminal_record( state_file, board="cassandra", task_id="t_terminal", run_id=8, structured={ "status": "completed", "summary": "old run", "changed_files": [], "tests_run": [], "artifacts": [], "findings": [], "blockers": [], }, summary="old run", metadata={}, ) replacement = SimpleNamespace( id="t_terminal", status="running", result=None, current_run_id=9, ) completions = [] class Connection: def close(self): return None fake_db = SimpleNamespace( scoped_current_board=lambda _board: nullcontext(), connect=lambda board: Connection(), get_task=lambda _conn, _task_id: replacement, complete_task=lambda *_args, **_kwargs: completions.append(True), ) monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) assert lanes.recover_pending_finalizations() == 0 assert completions == [] assert not terminal_path.exists() conflicts = list((state_root / "cassandra").glob("*.terminal.conflict-*.json")) assert len(conflicts) == 1 conflict = json.loads(conflicts[0].read_text(encoding="utf-8")) assert conflict["result"] == _record["result"] assert conflict["kanban_state"] == "conflict"