atlas-iac/testing/tests/test_hermes_cli_quota_failover.py

"""Fail-closed manual lanes, bounded failover, and router-outage blocking."""

from __future__ import annotations

from testing.tests.test_hermes_cli_support import (
    Path,
    SimpleNamespace,
    json,
    lanes,
    nullcontext,
    sys,
)


COMPLETED_RESULT = {
    "status": "completed",
    "summary": "Work finished with evidence.",
    "changed_files": ["src/a.py"],
    "tests_run": ["pytest full: passed"],
    "artifacts": [],
    "findings": [],
    "blockers": [],
}


class _Connection:
    def close(self):
        return None


def _lane_board(tmp_path: Path, task, comments: list, calls: list) -> SimpleNamespace:
    return SimpleNamespace(
        scoped_current_board=lambda _board: nullcontext(),
        connect=lambda board: _Connection(),
        get_task=lambda _conn, _task_id: task,
        worker_log_path=lambda _task_id, board: tmp_path / "worker.log",
        _resolve_worktree_workspace=lambda _task, board: (tmp_path, f"wt/{task.id}"),
        set_branch_name=lambda *_args: None,
        set_workspace_path=lambda *_args: None,
        build_worker_context=lambda *_args: "Finish the assigned card.",
        heartbeat_worker=lambda *_args, **_kwargs: True,
        add_comment=lambda _conn, _task_id, _author, body: comments.append(body),
        complete_task=lambda *_args, **kwargs: (
            calls.append(("complete", kwargs)) or True
        ),
        block_task=lambda *_args, **kwargs: calls.append(("block", kwargs)),
    )


def _isolate_lane(tmp_path: Path, monkeypatch, board) -> dict[str, Path]:
    monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=board))
    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
    monkeypatch.setattr(lanes, "DATA_ROOT", tmp_path)
    health_paths = {
        "codex": tmp_path / "provider-health/codex.json",
        "claude": tmp_path / "provider-health/claude.json",
    }
    monkeypatch.setattr(lanes, "PROVIDER_HEALTH_PATHS", health_paths)
    monkeypatch.setattr(lanes, "fetch_quota_snapshot", lambda *_a, **_k: {})
    return health_paths


def _route(provider: str, effort: str):
    model = "gpt-5.6-sol" if provider == "codex" else "claude-opus-5"
    return lanes.Route(
        provider, model, effort, f"{provider}-{effort}", "switchyard-manual", "r", 1, ()
    )


def _run_manual_fail_closed(tmp_path, monkeypatch, assignee, provider, output):
    task = SimpleNamespace(
        id="t_manual",
        status="running",
        result=None,
        current_run_id=31,
        assignee=assignee,
        max_runtime_seconds=60,
    )
    comments: list = []
    calls: list = []
    board = _lane_board(tmp_path, task, comments, calls)
    health_paths = _isolate_lane(tmp_path, monkeypatch, board)
    route_calls: list = []
    monkeypatch.setattr(
        lanes,
        "select_route",
        lambda _prompt, lane, **kwargs: (
            route_calls.append((lane, kwargs)) or _route(provider, "high")
        ),
    )
    runs: list = []
    monkeypatch.setattr(
        lanes,
        "run_provider",
        lambda *_args, **_kwargs: (
            runs.append(provider) or lanes.ProcessResult(1, output, None, True)
        ),
    )

    lanes.execute_claim("cassandra", "t_manual")
    return calls, comments, route_calls, runs, health_paths


def test_manual_codex_assignee_fails_closed_without_provider_switch(
    tmp_path: Path, monkeypatch
):
    calls, _comments, route_calls, runs, health_paths = _run_manual_fail_closed(
        tmp_path, monkeypatch, "cli-codex-high", "codex", "HTTP 429 rate limit hit"
    )

    assert runs == ["codex"]
    assert [lane for lane, _ in route_calls] == ["cli-codex-high"]
    kind, kwargs = calls[-1]
    assert kind == "block" and kwargs["kind"] == "transient"
    assert "Manually pinned provider codex" in kwargs["reason"]
    assert "cli-codex-high" in kwargs["reason"]
    assert "rate-limit" in kwargs["reason"]
    health = json.loads(health_paths["codex"].read_text())
    assert health["state"] == "capacity-limited"
    assert not health_paths["claude"].exists()


def test_manual_claude_assignee_fails_closed_without_provider_switch(
    tmp_path: Path, monkeypatch
):
    calls, _comments, route_calls, runs, health_paths = _run_manual_fail_closed(
        tmp_path, monkeypatch, "cli-claude-low", "claude", "usage limit exhausted"
    )

    assert runs == ["claude"]
    assert [lane for lane, _ in route_calls] == ["cli-claude-low"]
    kind, kwargs = calls[-1]
    assert kind == "block" and kwargs["kind"] == "transient"
    assert "Manually pinned provider claude" in kwargs["reason"]
    assert "quota" in kwargs["reason"]
    health = json.loads(health_paths["claude"].read_text())
    assert health["state"] == "capacity-limited"
    assert not health_paths["codex"].exists()


def test_auto_failover_preserves_effort_and_records_cooldown(
    tmp_path: Path, monkeypatch
):
    task = SimpleNamespace(
        id="t_auto",
        status="running",
        result=None,
        current_run_id=32,
        assignee="cli-auto",
        max_runtime_seconds=120,
    )
    comments: list = []
    calls: list = []
    board = _lane_board(tmp_path, task, comments, calls)
    health_paths = _isolate_lane(tmp_path, monkeypatch, board)
    route_calls: list = []

    def select_route(_prompt, lane, **kwargs):
        route_calls.append((lane, kwargs))
        assert lane == "cli-auto"
        if not kwargs.get("exclude_provider"):
            return lanes.Route(
                "codex", "gpt-5.6-terra", "xhigh", "codex-xhigh",
                "switchyard-classifier", "vote", 1, (),
            )
        # Automatic reclassification at the retry boundary: this must remain
        # a Jetson classifier decision, not a hardcoded manual alternate.
        return lanes.Route(
            "claude", "claude-opus-5", "xhigh", "claude-xhigh",
            "switchyard-classifier", "vote", 1, (),
        )

    monkeypatch.setattr(lanes, "select_route", select_route)
    fallbacks: list = []
    monkeypatch.setattr(
        lanes,
        "record_provider_fallback",
        lambda source, target, reason: fallbacks.append((source, target, reason)),
    )
    reports = [
        lanes.ProcessResult(1, "You have hit your usage limit.", None, True),
        lanes.ProcessResult(0, "done", dict(COMPLETED_RESULT), False),
    ]
    monkeypatch.setattr(
        lanes, "run_provider", lambda *_args, **_kwargs: reports.pop(0)
    )

    lanes.execute_claim("cassandra", "t_auto")

    assert reports == []
    assert calls and calls[0][0] == "complete"
    # Exactly two Switchyard selections: the initial classification and the
    # automatic retry with an explicit failed-provider exclusion. No third
    # (manual pin) call, because the classifier already preserved effort.
    assert len(route_calls) == 2
    initial_lane, initial_kwargs = route_calls[0]
    assert initial_lane == "cli-auto"
    assert not initial_kwargs.get("exclude_provider")
    retry_lane, retry_kwargs = route_calls[1]
    assert retry_lane == "cli-auto"
    assert retry_kwargs["exclude_provider"] == "codex"
    assert "quota" in retry_kwargs["exclude_reason"]
    assert fallbacks == [("codex", "claude", "quota")]
    assert any(
        "Provider fallback: codex -> claude after a quota failure" in item
        for item in comments
    )
    codex_health = json.loads(health_paths["codex"].read_text())
    assert codex_health["state"] == "capacity-limited"
    assert codex_health["cooldown_until"] == codex_health["failed_at"] + 300.0
    claude_health = json.loads(health_paths["claude"].read_text())
    assert claude_health["state"] == "available"


def test_auto_failover_escalates_when_classifier_downgrades_effort(
    tmp_path: Path, monkeypatch
):
    task = SimpleNamespace(
        id="t_auto_escalate",
        status="running",
        result=None,
        current_run_id=39,
        assignee="cli-auto",
        max_runtime_seconds=120,
    )
    comments: list = []
    calls: list = []
    board = _lane_board(tmp_path, task, comments, calls)
    health_paths = _isolate_lane(tmp_path, monkeypatch, board)
    route_calls: list = []

    def select_route(_prompt, lane, **kwargs):
        route_calls.append((lane, kwargs))
        if lane == "cli-auto" and not kwargs.get("exclude_provider"):
            return lanes.Route(
                "codex", "gpt-5.6-terra", "high", "codex-high",
                "switchyard-classifier", "vote", 1, (),
            )
        if lane == "cli-auto" and kwargs.get("exclude_provider") == "codex":
            # Jetson reclassifies but picks a lower effort than the original
            # route; the lane must never let capacity failover downgrade it.
            return lanes.Route(
                "claude", "claude-haiku-4-5", "low", "claude-low",
                "switchyard-classifier", "vote", 1, (),
            )
        assert lane == "cli-claude-high"
        return _route("claude", "high")

    monkeypatch.setattr(lanes, "select_route", select_route)
    fallbacks: list = []
    monkeypatch.setattr(
        lanes,
        "record_provider_fallback",
        lambda source, target, reason: fallbacks.append((source, target, reason)),
    )
    reports = [
        lanes.ProcessResult(1, "You have hit your usage limit.", None, True),
        lanes.ProcessResult(0, "done", dict(COMPLETED_RESULT), False),
    ]
    monkeypatch.setattr(
        lanes, "run_provider", lambda *_args, **_kwargs: reports.pop(0)
    )

    lanes.execute_claim("cassandra", "t_auto_escalate")

    assert reports == []
    assert calls and calls[0][0] == "complete"
    assert [lane for lane, _ in route_calls] == [
        "cli-auto",
        "cli-auto",
        "cli-claude-high",
    ]
    # The provider fallback is recorded against the final, effort-preserved
    # selection, not the transient low-effort classification.
    assert fallbacks == [("codex", "claude", "quota")]
    assert any(
        "escalated to the original high" in item
        for item in comments
    )
    assert any(
        "Provider fallback: codex -> claude after a quota failure" in item
        for item in comments
    )
    codex_health = json.loads(health_paths["codex"].read_text())
    assert codex_health["state"] == "capacity-limited"
    claude_health = json.loads(health_paths["claude"].read_text())
    assert claude_health["state"] == "available"


def test_bare_forbidden_auth_blip_fails_over_and_records_auth_cooldown(
    tmp_path: Path, monkeypatch
):
    task = SimpleNamespace(
        id="t_forbidden",
        status="running",
        result=None,
        current_run_id=38,
        assignee="cli-auto",
        max_runtime_seconds=120,
    )
    comments: list = []
    calls: list = []
    board = _lane_board(tmp_path, task, comments, calls)
    health_paths = _isolate_lane(tmp_path, monkeypatch, board)

    def select_route(_prompt, lane, **kwargs):
        assert lane == "cli-auto"
        return (
            _route("claude", "high")
            if kwargs.get("exclude_provider") == "codex"
            else _route("codex", "high")
        )

    monkeypatch.setattr(lanes, "select_route", select_route)
    fallbacks: list = []
    monkeypatch.setattr(
        lanes,
        "record_provider_fallback",
        lambda source, target, reason: fallbacks.append((source, target, reason)),
    )
    # The failure surfaces only as "403 Forbidden" with none of the words the
    # capacity gate previously keyed on (authentication/oauth/token expired).
    reports = [
        lanes.ProcessResult(1, "403 Forbidden", None, True),
        lanes.ProcessResult(0, "done", dict(COMPLETED_RESULT), False),
    ]
    monkeypatch.setattr(lanes, "run_provider", lambda *_args, **_kwargs: reports.pop(0))

    lanes.execute_claim("cassandra", "t_forbidden")

    # (a) failed over to the other provider and completed.
    assert reports == []
    assert calls and calls[0][0] == "complete"
    # (c) the fallback reason classifies as auth in metrics.
    assert fallbacks == [("codex", "claude", "auth")]
    assert any(
        "Provider fallback: codex -> claude after a auth failure" in item
        for item in comments
    )
    # (b) an auth cooldown was recorded (authenticated:false, longer window).
    codex_health = json.loads(health_paths["codex"].read_text())
    assert codex_health["state"] == "unavailable"
    assert codex_health["authenticated"] is False
    assert codex_health["failure_reason"] == "auth"
    assert codex_health["cooldown_until"] == codex_health["failed_at"] + 3600.0


def test_bare_forbidden_auth_blip_still_fails_closed_on_manual_lane(
    tmp_path: Path, monkeypatch
):
    calls, _comments, route_calls, runs, health_paths = _run_manual_fail_closed(
        tmp_path, monkeypatch, "cli-codex-high", "codex", "403 Forbidden"
    )

    # Fail-closed is unchanged: the pinned provider is not switched.
    assert runs == ["codex"]
    assert [lane for lane, _ in route_calls] == ["cli-codex-high"]
    kind, kwargs = calls[-1]
    assert kind == "block" and kwargs["kind"] == "transient"
    assert "Manually pinned provider codex" in kwargs["reason"]
    assert "auth" in kwargs["reason"]
    codex_health = json.loads(health_paths["codex"].read_text())
    assert codex_health["state"] == "unavailable"
    assert codex_health["authenticated"] is False


def test_double_capacity_failure_blocks_transient_with_both_reasons(
    tmp_path: Path, monkeypatch
):
    task = SimpleNamespace(
        id="t_double",
        status="running",
        result=None,
        current_run_id=33,
        assignee="cli-auto",
        max_runtime_seconds=120,
    )
    comments: list = []
    calls: list = []
    board = _lane_board(tmp_path, task, comments, calls)
    health_paths = _isolate_lane(tmp_path, monkeypatch, board)

    def select_route(_prompt, lane, **kwargs):
        assert lane == "cli-auto"
        return (
            _route("claude", "high")
            if kwargs.get("exclude_provider") == "codex"
            else _route("codex", "high")
        )

    monkeypatch.setattr(lanes, "select_route", select_route)
    decisions: list = []
    monkeypatch.setattr(
        lanes,
        "record_route_decision",
        lambda provider, effort, classifier, outcome: decisions.append(
            (provider, effort, outcome)
        ),
    )
    reports = [
        lanes.ProcessResult(1, "usage limit reached for this window", None, True),
        lanes.ProcessResult(1, "invalid oauth token, authentication failed", None, True),
    ]
    runs: list = []
    monkeypatch.setattr(
        lanes,
        "run_provider",
        lambda *_args, **_kwargs: runs.append(1) or reports.pop(0),
    )

    lanes.execute_claim("cassandra", "t_double")

    assert len(runs) == 2
    kind, kwargs = calls[-1]
    assert kind == "block" and kwargs["kind"] == "transient"
    assert "Both hosted providers failed" in kwargs["reason"]
    assert "codex: quota" in kwargs["reason"]
    assert "claude: auth" in kwargs["reason"]
    assert "never falls to a metered or local path" in kwargs["reason"]
    assert decisions == [
        ("codex", "high", "capacity-failure"),
        ("claude", "high", "capacity-failure"),
    ]
    assert json.loads(health_paths["codex"].read_text())["state"] == "capacity-limited"
    claude_health = json.loads(health_paths["claude"].read_text())
    assert claude_health["state"] == "unavailable"
    assert claude_health["authenticated"] is False


def test_soft_quota_exclusion_steers_new_auto_work(tmp_path: Path, monkeypatch):
    task = SimpleNamespace(
        id="t_quota",
        status="running",
        result=None,
        current_run_id=37,
        assignee="cli-auto",
        max_runtime_seconds=60,
    )
    comments: list = []
    calls: list = []
    board = _lane_board(tmp_path, task, comments, calls)
    _isolate_lane(tmp_path, monkeypatch, board)
    monkeypatch.setattr(
        lanes,
        "fetch_quota_snapshot",
        lambda *_a, **_k: {
            "codex": lanes.ProviderQuota(10.0, None),
            "claude": lanes.ProviderQuota(60.0, None),
        },
    )
    route_calls: list = []
    monkeypatch.setattr(
        lanes,
        "select_route",
        lambda _prompt, lane, **kwargs: (
            route_calls.append((lane, kwargs)) or _route("claude", "high")
        ),
    )
    monkeypatch.setattr(
        lanes,
        "run_provider",
        lambda *_args, **_kwargs: lanes.ProcessResult(
            0, "done", dict(COMPLETED_RESULT), False
        ),
    )

    lanes.execute_claim("cassandra", "t_quota")

    assert calls and calls[0][0] == "complete"
    lane, kwargs = route_calls[0]
    assert lane == "cli-auto"
    assert kwargs["exclude_provider"] == "codex"
    assert kwargs["exclude_reason"] == "is below its remaining-quota routing threshold"
    assert any("Quota guard: codex" in item for item in comments)
    assert any(
        "Provider routing guard (quota) excluded codex" in item for item in comments
    )