atlas-iac/testing/tests/test_hermes_cli_dispatch_runtime.py

"""Dispatcher startup, degraded-board, and worker-loop regressions."""

from __future__ import annotations

import sys
from contextlib import nullcontext
from pathlib import Path
from types import SimpleNamespace

import pytest

from testing.tests.test_hermes_cli_support import lanes


def test_board_slug_and_connection_failure_paths(monkeypatch):
    assert lanes._board_slug(SimpleNamespace(slug="cassandra")) == "cassandra"
    assert lanes._board_slug(SimpleNamespace(id="fallback")) == "fallback"
    failures = []
    monkeypatch.setattr(
        lanes,
        "_record_board_access_error",
        lambda board, error: failures.append((board, str(error))),
    )
    db = SimpleNamespace(
        connect=lambda **_kwargs: (_ for _ in ()).throw(OSError("offline"))
    )
    assert lanes._connect_healthy_board(db, "broken") is None
    assert failures == [("broken", "offline")]


def test_orphan_recovery_bounds_registry_and_per_board_failures(
    monkeypatch,
):
    failures = []
    registry_db = SimpleNamespace(
        complete_task=lambda *_args, expected_run_id=None, **_kwargs: True,
        reclaim_task=lambda *_args, expected_run_id=None, **_kwargs: True,
        list_boards=lambda **_kwargs: (_ for _ in ()).throw(OSError("registry")),
    )
    monkeypatch.setitem(
        sys.modules,
        "hermes_cli",
        SimpleNamespace(kanban_db=registry_db),
    )
    monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0)
    monkeypatch.setattr(
        lanes,
        "_record_board_access_error",
        lambda board, error: failures.append((board, str(error))),
    )
    lanes.recover_orphans()
    assert failures == [("board-registry", "registry")]

    class Connection:
        def close(self):
            failures.append(("close", "yes"))

    board_db = SimpleNamespace(
        complete_task=registry_db.complete_task,
        reclaim_task=registry_db.reclaim_task,
        list_boards=lambda **_kwargs: [{"slug": ""}, {"slug": "broken"}],
        scoped_current_board=lambda _board: nullcontext(),
        connect=lambda board: Connection(),
        list_tasks=lambda _conn: (_ for _ in ()).throw(OSError("board read")),
    )
    monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=board_db))
    lanes.recover_orphans()
    assert ("broken", "board read") in failures
    assert ("close", "yes") in failures


def test_claim_ready_handles_zero_limit_empty_boards_and_claim_errors(monkeypatch):
    monkeypatch.setitem(
        sys.modules,
        "hermes_cli",
        SimpleNamespace(kanban_db=SimpleNamespace()),
    )
    assert lanes.claim_ready(set(), 0) == []

    class Connection:
        def close(self):
            return None

    task = SimpleNamespace(
        id="t_claim",
        status="ready",
        assignee="cli-auto",
    )
    db = SimpleNamespace(
        list_boards=lambda **_kwargs: [{"slug": ""}, {"slug": "cassandra"}],
        scoped_current_board=lambda _board: nullcontext(),
        connect=lambda board: Connection(),
        recompute_ready=lambda _conn: None,
        list_tasks=lambda _conn: [task],
        claim_task=lambda *_args, **_kwargs: (_ for _ in ()).throw(
            RuntimeError("claim raced")
        ),
    )
    monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db))
    assert lanes.claim_ready(set(), 1) == []


class _Future:
    def __init__(self):
        self.polls = 0

    def done(self):
        self.polls += 1
        return self.polls >= 1

    def result(self):
        raise RuntimeError("worker failed")


class _Pool:
    def __init__(self, max_workers):
        self.max_workers = max_workers
        self.future = _Future()

    def __enter__(self):
        return self

    def __exit__(self, *_args):
        return False

    def submit(self, function, board, task_id):
        assert callable(function)
        assert (board, task_id) == ("cassandra", "t_loop")
        return self.future


def test_ready_dispatch_loop_submits_and_reaps_failed_workers(
    tmp_path: Path,
    monkeypatch,
    capsys,
):
    db = SimpleNamespace()
    monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db))
    monkeypatch.setattr(lanes, "RESULT_SCHEMA_PATH", tmp_path / "schema.json")
    monkeypatch.setattr(lanes, "recover_orphans", lambda: None)
    monkeypatch.setattr(
        lanes,
        "initialize_kanban_capabilities",
        lambda _db: lanes.KanbanCapabilities(True, True, True),
    )
    monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0)
    monkeypatch.setattr(lanes, "maybe_gc_lane_artifacts", lambda: 0)
    claims = [[("cassandra", "t_loop")], []]
    monkeypatch.setattr(lanes, "claim_ready", lambda *_args: claims.pop(0))
    monkeypatch.setattr(
        lanes.concurrent.futures,
        "ThreadPoolExecutor",
        _Pool,
    )
    sleeps = []

    def stop_second_loop(_seconds):
        sleeps.append(True)
        if len(sleeps) == 2:
            raise RuntimeError("stop loop")

    monkeypatch.setattr(lanes.time, "sleep", stop_second_loop)

    with pytest.raises(RuntimeError, match="stop loop"):
        lanes.main()

    assert "worker future failed: worker failed" in capsys.readouterr().err


def test_deferred_dispatch_health_never_claims_new_work(
    tmp_path: Path,
    monkeypatch,
):
    db = SimpleNamespace()
    monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db))
    monkeypatch.setattr(lanes, "RESULT_SCHEMA_PATH", tmp_path / "schema.json")
    monkeypatch.setattr(lanes, "recover_orphans", lambda: None)
    monkeypatch.setattr(
        lanes,
        "initialize_kanban_capabilities",
        lambda _db: lanes.KanbanCapabilities(True, False, False),
    )
    monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0)
    monkeypatch.setattr(lanes, "maybe_gc_lane_artifacts", lambda: 0)
    monkeypatch.setattr(
        lanes,
        "claim_ready",
        lambda *_args: pytest.fail("deferred startup must not claim work"),
    )
    monkeypatch.setattr(lanes.concurrent.futures, "ThreadPoolExecutor", _Pool)
    monkeypatch.setattr(
        lanes.time,
        "sleep",
        lambda _seconds: (_ for _ in ()).throw(RuntimeError("stop loop")),
    )
    with pytest.raises(RuntimeError, match="stop loop"):
        lanes.main()