"""Dispatcher startup, degraded-board, and worker-loop regressions.""" from __future__ import annotations import runpy import sys from contextlib import nullcontext from pathlib import Path from types import SimpleNamespace import pytest from testing.tests.test_hermes_cli_support import SCRIPTS, lanes def test_board_slug_and_connection_failure_paths(monkeypatch): assert lanes._board_slug(SimpleNamespace(slug="cassandra")) == "cassandra" assert lanes._board_slug(SimpleNamespace(id="fallback")) == "fallback" failures = [] monkeypatch.setattr( lanes, "_record_board_access_error", lambda board, error: failures.append((board, str(error))), ) db = SimpleNamespace( connect=lambda **_kwargs: (_ for _ in ()).throw(OSError("offline")) ) assert lanes._connect_healthy_board(db, "broken") is None assert failures == [("broken", "offline")] def test_orphan_recovery_bounds_registry_and_per_board_failures( monkeypatch, ): failures = [] registry_db = SimpleNamespace( complete_task=lambda *_args, expected_run_id=None, **_kwargs: True, reclaim_task=lambda *_args, expected_run_id=None, **_kwargs: True, list_boards=lambda **_kwargs: (_ for _ in ()).throw(OSError("registry")), ) monkeypatch.setitem( sys.modules, "hermes_cli", SimpleNamespace(kanban_db=registry_db), ) monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0) monkeypatch.setattr( lanes, "_record_board_access_error", lambda board, error: failures.append((board, str(error))), ) lanes.recover_orphans() assert failures == [("board-registry", "registry")] class Connection: def close(self): failures.append(("close", "yes")) board_db = SimpleNamespace( complete_task=registry_db.complete_task, reclaim_task=registry_db.reclaim_task, list_boards=lambda **_kwargs: [{"slug": ""}, {"slug": "broken"}], scoped_current_board=lambda _board: nullcontext(), connect=lambda board: Connection(), list_tasks=lambda _conn: (_ for _ in ()).throw(OSError("board read")), ) monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=board_db)) lanes.recover_orphans() assert ("broken", "board read") in failures assert ("close", "yes") in failures def test_claim_ready_handles_zero_limit_empty_boards_and_claim_errors(monkeypatch): monkeypatch.setitem( sys.modules, "hermes_cli", SimpleNamespace(kanban_db=SimpleNamespace()), ) assert lanes.claim_ready(set(), 0) == [] class Connection: def close(self): return None task = SimpleNamespace( id="t_claim", status="ready", assignee="cli-auto", ) db = SimpleNamespace( list_boards=lambda **_kwargs: [{"slug": ""}, {"slug": "cassandra"}], scoped_current_board=lambda _board: nullcontext(), connect=lambda board: Connection(), recompute_ready=lambda _conn: None, list_tasks=lambda _conn: [task], claim_task=lambda *_args, **_kwargs: (_ for _ in ()).throw( RuntimeError("claim raced") ), ) monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db)) assert lanes.claim_ready(set(), 1) == [] def test_claim_ready_skips_an_unreadable_board_and_claims_a_healthy_fallback(tmp_path, monkeypatch): """A metadata failure on one board cannot starve healthy ready cards.""" root = tmp_path / "boards" (root / "broken").mkdir(parents=True) (root / "soteria").mkdir() (root / "archived").mkdir() for board in ("broken", "soteria", "archived"): (root / board / "kanban.db").touch() failures = [] task = SimpleNamespace(id="t_soteria", status="ready", assignee="cli-auto") class Connection: def __init__(self, board): self.board = board def close(self): return None def connect(*, board): if board == "broken": raise PermissionError("board database unreadable") return Connection(board) db = SimpleNamespace( list_boards=lambda **_kwargs: (_ for _ in ()).throw(PermissionError("probe metadata")), boards_root=lambda: root, scoped_current_board=lambda _board: nullcontext(), read_board_metadata=lambda board: {"slug": board, "archived": board == "archived"}, connect=connect, recompute_ready=lambda _conn: None, list_tasks=lambda conn: [task] if conn.board == "soteria" else [], claim_task=lambda _conn, task_id, **_kwargs: task_id, ) monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db)) monkeypatch.setattr(lanes, "_record_board_access_error", lambda board, error: failures.append(board)) assert lanes.claim_ready(set(), 1) == [("soteria", "t_soteria")] assert failures.count("board-registry") == 1 and failures.count("broken") == 1 def test_direct_lane_reserves_signed_continuations_for_the_mediator(monkeypatch): task = SimpleNamespace(id="t_continue") monkeypatch.setattr(lanes.supervisor_state, "get_child", lambda *_args: {"kind": "repair"}) assert lanes._direct_lane_eligible("soteria", task) is False monkeypatch.setattr(lanes.supervisor_state, "get_child", lambda *_args: None) assert lanes._direct_lane_eligible("soteria", task) is True monkeypatch.setattr(lanes.supervisor_state, "get_child", lambda *_args: (_ for _ in ()).throw(OSError("state"))) assert lanes._direct_lane_eligible("soteria", task) is False def test_direct_lane_owned_workspace_policy_precedes_unassigned_mutation(monkeypatch): task = SimpleNamespace(id="t_pathless", workspace_path="", status="ready", assignee="") monkeypatch.setattr(lanes, "OWNED_WORKSPACES_ONLY", True) assert lanes._direct_lane_eligible("soteria", task) is False assigned = [] db = SimpleNamespace( list_boards=lambda **_kwargs: [{"slug": "soteria"}], scoped_current_board=lambda _board: nullcontext(), connect=lambda **_kwargs: SimpleNamespace(close=lambda: None), recompute_ready=lambda _conn: None, list_tasks=lambda _conn: [task], assign_task=lambda *_args: assigned.append(True), ) monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db)) assert lanes.claim_ready(set(), 1, lanes._direct_lane_eligible) == [] assert assigned == [] class _Future: def __init__(self): self.polls = 0 def done(self): self.polls += 1 return self.polls >= 1 def result(self): raise RuntimeError("worker failed") class _Pool: def __init__(self, max_workers): self.max_workers = max_workers self.future = _Future() def __enter__(self): return self def __exit__(self, *_args): return False def submit(self, function, board, task_id): assert callable(function) assert (board, task_id) == ("cassandra", "t_loop") return self.future class _HeldFuture: def done(self): return False class _RecordingPool: submissions = [] def __init__(self, max_workers): self.max_workers = max_workers def __enter__(self): return self def __exit__(self, *_args): return False def submit(self, _function, board, task_id): self.submissions.append((board, task_id)) return _HeldFuture() def test_ready_dispatch_loop_submits_and_reaps_failed_workers( tmp_path: Path, monkeypatch, capsys, ): db = SimpleNamespace() monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db)) monkeypatch.setattr(lanes, "RESULT_SCHEMA_PATH", tmp_path / "schema.json") monkeypatch.setattr(lanes, "recover_orphans", lambda: None) monkeypatch.setattr(lanes, "start_metrics_server", lambda port=None, **_kwargs: None) ready = lanes.KanbanCapabilities(True, True, True) monkeypatch.setattr(lanes, "initialize_kanban_capabilities", lambda _db: ready) monkeypatch.setattr(lanes, "refresh_kanban_capabilities", lambda _db: ready) monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0) monkeypatch.setattr(lanes, "maybe_gc_lane_artifacts", lambda: 0) claims = [[("cassandra", "t_loop")], []] monkeypatch.setattr(lanes, "claim_ready", lambda *_args: claims.pop(0)) monkeypatch.setattr( lanes.concurrent.futures, "ThreadPoolExecutor", _Pool, ) sleeps = [] def stop_second_loop(_seconds): sleeps.append(True) if len(sleeps) == 2: raise RuntimeError("stop loop") monkeypatch.setattr(lanes.time, "sleep", stop_second_loop) with pytest.raises(RuntimeError, match="stop loop"): lanes.main() assert "worker future failed: worker failed" in capsys.readouterr().err def test_deferred_dispatch_health_never_claims_new_work( tmp_path: Path, monkeypatch, ): db = SimpleNamespace() monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db)) monkeypatch.setattr(lanes, "RESULT_SCHEMA_PATH", tmp_path / "schema.json") monkeypatch.setattr(lanes, "recover_orphans", lambda: None) monkeypatch.setattr(lanes, "start_metrics_server", lambda port=None, **_kwargs: None) deferred = lanes.KanbanCapabilities(True, False, False) monkeypatch.setattr(lanes, "initialize_kanban_capabilities", lambda _db: deferred) monkeypatch.setattr(lanes, "refresh_kanban_capabilities", lambda _db: deferred) monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0) monkeypatch.setattr(lanes, "maybe_gc_lane_artifacts", lambda: 0) monkeypatch.setattr( lanes, "claim_ready", lambda *_args: pytest.fail("deferred startup must not claim work"), ) monkeypatch.setattr(lanes.concurrent.futures, "ThreadPoolExecutor", _Pool) monkeypatch.setattr( lanes.time, "sleep", lambda _seconds: (_ for _ in ()).throw(RuntimeError("stop loop")), ) with pytest.raises(RuntimeError, match="stop loop"): lanes.main() @pytest.mark.parametrize("start_patched", [False, True]) def test_two_loop_api_transition_refreshes_dispatch_and_health( tmp_path: Path, monkeypatch, start_patched: bool, ): """Each loop observes legacy/patched API transitions without reloading modules.""" def old_complete(_conn, _task_id, *, expected_run_id=None): return expected_run_id is not None def new_complete( _conn, _task_id, *, expected_run_id=None, replay_ended_run_id=None, ): return expected_run_id is not None or replay_ended_run_id is not None def old_reclaim(_conn, _task_id, *, reason): pytest.fail(f"unguarded legacy reclaim invoked: {reason}") def new_reclaim(_conn, _task_id, *, reason, expected_run_id=None): return bool(reason and expected_run_id) db = SimpleNamespace( complete_task=new_complete if start_patched else old_complete, reclaim_task=new_reclaim if start_patched else old_reclaim, ) monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db)) monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") monkeypatch.setattr(lanes, "RESULT_SCHEMA_PATH", tmp_path / "schema.json") recoveries = [] monkeypatch.setattr(lanes, "recover_orphans", lambda: recoveries.append("recover")) monkeypatch.setattr(lanes, "start_metrics_server", lambda port=None, **_kwargs: None) monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0) monkeypatch.setattr(lanes, "maybe_gc_lane_artifacts", lambda: 0) claim_states = [] def claim(_active, _limit, _eligible=None): ready = lanes.kanban_capabilities(db).ready claim_states.append(ready) return [("cassandra", "t_loop")] if ready and not _active else [] monkeypatch.setattr(lanes, "claim_ready", claim) _RecordingPool.submissions = [] monkeypatch.setattr(lanes.concurrent.futures, "ThreadPoolExecutor", _RecordingPool) journal = tmp_path / "cli-lanes/cassandra/t_loop.terminal.json" journal.parent.mkdir(parents=True) journal.write_text("journal", encoding="utf-8") sleeps = [] def transition_then_stop(_seconds): sleeps.append(True) if len(sleeps) == 1: db.complete_task = old_complete if start_patched else new_complete db.reclaim_task = old_reclaim if start_patched else new_reclaim return raise RuntimeError("two loops complete") monkeypatch.setattr(lanes.time, "sleep", transition_then_stop) with pytest.raises(RuntimeError, match="two loops complete"): lanes.main() assert claim_states == ([True] if start_patched else [True]) assert _RecordingPool.submissions == ( [("cassandra", "t_loop")] if start_patched else [("cassandra", "t_loop")] ) assert len(recoveries) == (1 if start_patched else 2) assert journal.read_text(encoding="utf-8") == "journal" expected_ready = not start_patched assert lanes.runtime_health()["ready"] is expected_ready def test_orphan_recovery_skips_unreachable_boards_and_settled_tasks( monkeypatch, capsys, ): """One faulted board or settled task never stops restart reclamation.""" running = SimpleNamespace( id="t_run", status="running", current_run_id=4, assignee="cli-auto", claim_lock="direct-cli-lane", ) pool_running = SimpleNamespace( id="t_pool", status="running", current_run_id=5, assignee="cli-auto", claim_lock="execution-pool", ) settled = SimpleNamespace( id="t_done", status="done", current_run_id=None, assignee="cli-auto", ) reclaims = [] def connect(*, board): if board == "broken": raise OSError("volume stall") return SimpleNamespace(close=lambda: None) db = SimpleNamespace( list_boards=lambda include_archived=False: [ {"slug": "broken"}, {"slug": "healthy"}, ], scoped_current_board=lambda _board: nullcontext(), connect=connect, list_tasks=lambda _conn: [settled, running, pool_running], reclaim_task=lambda _conn, task_id, **kwargs: reclaims.append( (task_id, kwargs["expected_run_id"]) ), ) monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db)) monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0) monkeypatch.setattr(lanes, "_has_pending_finalization", lambda *_args: False) monkeypatch.setattr(lanes.supervisor_state, "get_child", lambda *_args: None) lanes.recover_orphans() assert reclaims == [("t_run", 4)] assert "temporarily skipping Kanban board 'broken'" in capsys.readouterr().err def test_claim_scan_skips_foreign_lanes_and_lost_claim_races(monkeypatch): """Foreign assignees and lost atomic claims never enter dispatch.""" foreign = SimpleNamespace(id="t_manual", assignee="human", status="ready") lost = SimpleNamespace(id="t_lost", assignee="cli-auto", status="ready") won = SimpleNamespace(id="t_won", assignee="cli-auto", status="ready") def claim_task(_conn, task_id, **_kwargs): return None if task_id == "t_lost" else SimpleNamespace(id=task_id) db = SimpleNamespace( list_boards=lambda include_archived=False: [{"slug": "cassandra"}], scoped_current_board=lambda _board: nullcontext(), connect=lambda board: SimpleNamespace(close=lambda: None), recompute_ready=lambda _conn: None, list_tasks=lambda _conn: [foreign, lost, won], claim_task=claim_task, ) monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db)) assert lanes.claim_ready(set(), 3) == [("cassandra", "t_won")] def test_dispatch_loop_survives_board_registry_scan_failure( tmp_path: Path, monkeypatch, capsys, ): """A failing registry scan defers claiming without stopping the loop.""" db = SimpleNamespace() monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db)) monkeypatch.setattr(lanes, "RESULT_SCHEMA_PATH", tmp_path / "schema.json") monkeypatch.setattr(lanes, "recover_orphans", lambda: None) monkeypatch.setattr(lanes, "start_metrics_server", lambda port=None, **_kwargs: None) ready = lanes.KanbanCapabilities(True, True, True) monkeypatch.setattr(lanes, "initialize_kanban_capabilities", lambda _db: ready) monkeypatch.setattr(lanes, "refresh_kanban_capabilities", lambda _db: ready) monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0) monkeypatch.setattr(lanes, "maybe_gc_lane_artifacts", lambda: 0) def failing_claim(_active, _limit): raise RuntimeError("registry scan failed") monkeypatch.setattr(lanes, "claim_ready", failing_claim) monkeypatch.setattr( lanes.time, "sleep", lambda _seconds: (_ for _ in ()).throw(RuntimeError("stop loop")), ) with pytest.raises(RuntimeError, match="stop loop"): lanes.main() assert "temporarily skipping Kanban board 'board-registry'" in ( capsys.readouterr().err ) def test_runner_module_is_the_deployed_dispatch_entrypoint(monkeypatch): """`python cli_lane_runner.py` hands control to the dispatch loop.""" calls = [] monkeypatch.setattr(lanes, "main", lambda: calls.append(True) or 0) with pytest.raises(SystemExit) as excinfo: runpy.run_path(str(SCRIPTS / "cli_lane_runner.py"), run_name="__main__") assert excinfo.value.code == 0 assert calls == [True]