"""Fail-closed manual lanes, bounded failover, and router-outage blocking.""" from __future__ import annotations from testing.tests.test_hermes_cli_support import ( Path, SimpleNamespace, json, lanes, nullcontext, sys, ) COMPLETED_RESULT = { "status": "completed", "summary": "Work finished with evidence.", "changed_files": ["src/a.py"], "tests_run": ["pytest full: passed"], "artifacts": [], "findings": [], "blockers": [], } class _Connection: def close(self): return None def _lane_board(tmp_path: Path, task, comments: list, calls: list) -> SimpleNamespace: return SimpleNamespace( scoped_current_board=lambda _board: nullcontext(), connect=lambda board: _Connection(), get_task=lambda _conn, _task_id: task, worker_log_path=lambda _task_id, board: tmp_path / "worker.log", _resolve_worktree_workspace=lambda _task, board: (tmp_path, f"wt/{task.id}"), set_branch_name=lambda *_args: None, set_workspace_path=lambda *_args: None, build_worker_context=lambda *_args: "Finish the assigned card.", heartbeat_worker=lambda *_args, **_kwargs: True, add_comment=lambda _conn, _task_id, _author, body: comments.append(body), complete_task=lambda *_args, **kwargs: ( calls.append(("complete", kwargs)) or True ), block_task=lambda *_args, **kwargs: calls.append(("block", kwargs)), ) def _isolate_lane(tmp_path: Path, monkeypatch, board) -> dict[str, Path]: monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=board)) monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") monkeypatch.setattr(lanes, "DATA_ROOT", tmp_path) health_paths = { "codex": tmp_path / "provider-health/codex.json", "claude": tmp_path / "provider-health/claude.json", } monkeypatch.setattr(lanes, "PROVIDER_HEALTH_PATHS", health_paths) monkeypatch.setattr(lanes, "fetch_quota_snapshot", lambda *_a, **_k: {}) return health_paths def _route(provider: str, effort: str): model = "gpt-5.6-sol" if provider == "codex" else "claude-opus-5" return lanes.Route( provider, model, effort, f"{provider}-{effort}", "switchyard-manual", "r", 1, () ) def _run_manual_fail_closed(tmp_path, monkeypatch, assignee, provider, output): task = SimpleNamespace( id="t_manual", status="running", result=None, current_run_id=31, assignee=assignee, max_runtime_seconds=60, ) comments: list = [] calls: list = [] board = _lane_board(tmp_path, task, comments, calls) health_paths = _isolate_lane(tmp_path, monkeypatch, board) route_calls: list = [] monkeypatch.setattr( lanes, "select_route", lambda _prompt, lane, **kwargs: ( route_calls.append((lane, kwargs)) or _route(provider, "high") ), ) runs: list = [] monkeypatch.setattr( lanes, "run_provider", lambda *_args, **_kwargs: ( runs.append(provider) or lanes.ProcessResult(1, output, None, True) ), ) lanes.execute_claim("cassandra", "t_manual") return calls, comments, route_calls, runs, health_paths def test_manual_codex_assignee_fails_closed_without_provider_switch( tmp_path: Path, monkeypatch ): calls, _comments, route_calls, runs, health_paths = _run_manual_fail_closed( tmp_path, monkeypatch, "cli-codex-high", "codex", "HTTP 429 rate limit hit" ) assert runs == ["codex"] assert [lane for lane, _ in route_calls] == ["cli-codex-high"] kind, kwargs = calls[-1] assert kind == "block" and kwargs["kind"] == "transient" assert "Manually pinned provider codex" in kwargs["reason"] assert "cli-codex-high" in kwargs["reason"] assert "rate-limit" in kwargs["reason"] health = json.loads(health_paths["codex"].read_text()) assert health["state"] == "capacity-limited" assert not health_paths["claude"].exists() def test_manual_claude_assignee_fails_closed_without_provider_switch( tmp_path: Path, monkeypatch ): calls, _comments, route_calls, runs, health_paths = _run_manual_fail_closed( tmp_path, monkeypatch, "cli-claude-low", "claude", "usage limit exhausted" ) assert runs == ["claude"] assert [lane for lane, _ in route_calls] == ["cli-claude-low"] kind, kwargs = calls[-1] assert kind == "block" and kwargs["kind"] == "transient" assert "Manually pinned provider claude" in kwargs["reason"] assert "quota" in kwargs["reason"] health = json.loads(health_paths["claude"].read_text()) assert health["state"] == "capacity-limited" assert not health_paths["codex"].exists() def test_auto_failover_preserves_effort_and_records_cooldown( tmp_path: Path, monkeypatch ): task = SimpleNamespace( id="t_auto", status="running", result=None, current_run_id=32, assignee="cli-auto", max_runtime_seconds=120, ) comments: list = [] calls: list = [] board = _lane_board(tmp_path, task, comments, calls) health_paths = _isolate_lane(tmp_path, monkeypatch, board) route_calls: list = [] def select_route(_prompt, lane, **kwargs): route_calls.append((lane, kwargs)) assert lane == "cli-auto" if not kwargs.get("exclude_provider"): return lanes.Route( "codex", "gpt-5.6-terra", "xhigh", "codex-xhigh", "switchyard-classifier", "vote", 1, (), ) # Automatic reclassification at the retry boundary: this must remain # a Jetson classifier decision, not a hardcoded manual alternate. return lanes.Route( "claude", "claude-opus-5", "xhigh", "claude-xhigh", "switchyard-classifier", "vote", 1, (), ) monkeypatch.setattr(lanes, "select_route", select_route) fallbacks: list = [] monkeypatch.setattr( lanes, "record_provider_fallback", lambda source, target, reason: fallbacks.append((source, target, reason)), ) reports = [ lanes.ProcessResult(1, "You have hit your usage limit.", None, True), lanes.ProcessResult(0, "done", dict(COMPLETED_RESULT), False), ] monkeypatch.setattr( lanes, "run_provider", lambda *_args, **_kwargs: reports.pop(0) ) lanes.execute_claim("cassandra", "t_auto") assert reports == [] assert calls and calls[0][0] == "complete" # Exactly two Switchyard selections: the initial classification and the # automatic retry with an explicit failed-provider exclusion. No third # (manual pin) call, because the classifier already preserved effort. assert len(route_calls) == 2 initial_lane, initial_kwargs = route_calls[0] assert initial_lane == "cli-auto" assert not initial_kwargs.get("exclude_provider") retry_lane, retry_kwargs = route_calls[1] assert retry_lane == "cli-auto" assert retry_kwargs["exclude_provider"] == "codex" assert "quota" in retry_kwargs["exclude_reason"] assert fallbacks == [("codex", "claude", "quota")] assert any( "Provider fallback: codex -> claude after a quota failure" in item for item in comments ) codex_health = json.loads(health_paths["codex"].read_text()) assert codex_health["state"] == "capacity-limited" assert codex_health["cooldown_until"] == codex_health["failed_at"] + 300.0 claude_health = json.loads(health_paths["claude"].read_text()) assert claude_health["state"] == "available" def test_auto_failover_escalates_when_classifier_downgrades_effort( tmp_path: Path, monkeypatch ): task = SimpleNamespace( id="t_auto_escalate", status="running", result=None, current_run_id=39, assignee="cli-auto", max_runtime_seconds=120, ) comments: list = [] calls: list = [] board = _lane_board(tmp_path, task, comments, calls) health_paths = _isolate_lane(tmp_path, monkeypatch, board) route_calls: list = [] def select_route(_prompt, lane, **kwargs): route_calls.append((lane, kwargs)) if lane == "cli-auto" and not kwargs.get("exclude_provider"): return lanes.Route( "codex", "gpt-5.6-terra", "high", "codex-high", "switchyard-classifier", "vote", 1, (), ) if lane == "cli-auto" and kwargs.get("exclude_provider") == "codex": # Jetson reclassifies but picks a lower effort than the original # route; the lane must never let capacity failover downgrade it. return lanes.Route( "claude", "claude-haiku-4-5", "low", "claude-low", "switchyard-classifier", "vote", 1, (), ) assert lane == "cli-claude-high" return _route("claude", "high") monkeypatch.setattr(lanes, "select_route", select_route) fallbacks: list = [] monkeypatch.setattr( lanes, "record_provider_fallback", lambda source, target, reason: fallbacks.append((source, target, reason)), ) reports = [ lanes.ProcessResult(1, "You have hit your usage limit.", None, True), lanes.ProcessResult(0, "done", dict(COMPLETED_RESULT), False), ] monkeypatch.setattr( lanes, "run_provider", lambda *_args, **_kwargs: reports.pop(0) ) lanes.execute_claim("cassandra", "t_auto_escalate") assert reports == [] assert calls and calls[0][0] == "complete" assert [lane for lane, _ in route_calls] == [ "cli-auto", "cli-auto", "cli-claude-high", ] # The provider fallback is recorded against the final, effort-preserved # selection, not the transient low-effort classification. assert fallbacks == [("codex", "claude", "quota")] assert any( "escalated to the original high" in item for item in comments ) assert any( "Provider fallback: codex -> claude after a quota failure" in item for item in comments ) codex_health = json.loads(health_paths["codex"].read_text()) assert codex_health["state"] == "capacity-limited" claude_health = json.loads(health_paths["claude"].read_text()) assert claude_health["state"] == "available" def test_bare_forbidden_auth_blip_fails_over_and_records_auth_cooldown( tmp_path: Path, monkeypatch ): task = SimpleNamespace( id="t_forbidden", status="running", result=None, current_run_id=38, assignee="cli-auto", max_runtime_seconds=120, ) comments: list = [] calls: list = [] board = _lane_board(tmp_path, task, comments, calls) health_paths = _isolate_lane(tmp_path, monkeypatch, board) def select_route(_prompt, lane, **kwargs): assert lane == "cli-auto" return ( _route("claude", "high") if kwargs.get("exclude_provider") == "codex" else _route("codex", "high") ) monkeypatch.setattr(lanes, "select_route", select_route) fallbacks: list = [] monkeypatch.setattr( lanes, "record_provider_fallback", lambda source, target, reason: fallbacks.append((source, target, reason)), ) # The failure surfaces only as "403 Forbidden" with none of the words the # capacity gate previously keyed on (authentication/oauth/token expired). reports = [ lanes.ProcessResult(1, "403 Forbidden", None, True), lanes.ProcessResult(0, "done", dict(COMPLETED_RESULT), False), ] monkeypatch.setattr(lanes, "run_provider", lambda *_args, **_kwargs: reports.pop(0)) lanes.execute_claim("cassandra", "t_forbidden") # (a) failed over to the other provider and completed. assert reports == [] assert calls and calls[0][0] == "complete" # (c) the fallback reason classifies as auth in metrics. assert fallbacks == [("codex", "claude", "auth")] assert any( "Provider fallback: codex -> claude after a auth failure" in item for item in comments ) # (b) an auth cooldown was recorded (authenticated:false, longer window). codex_health = json.loads(health_paths["codex"].read_text()) assert codex_health["state"] == "unavailable" assert codex_health["authenticated"] is False assert codex_health["failure_reason"] == "auth" assert codex_health["cooldown_until"] == codex_health["failed_at"] + 3600.0 def test_bare_forbidden_auth_blip_still_fails_closed_on_manual_lane( tmp_path: Path, monkeypatch ): calls, _comments, route_calls, runs, health_paths = _run_manual_fail_closed( tmp_path, monkeypatch, "cli-codex-high", "codex", "403 Forbidden" ) # Fail-closed is unchanged: the pinned provider is not switched. assert runs == ["codex"] assert [lane for lane, _ in route_calls] == ["cli-codex-high"] kind, kwargs = calls[-1] assert kind == "block" and kwargs["kind"] == "transient" assert "Manually pinned provider codex" in kwargs["reason"] assert "auth" in kwargs["reason"] codex_health = json.loads(health_paths["codex"].read_text()) assert codex_health["state"] == "unavailable" assert codex_health["authenticated"] is False def test_double_capacity_failure_blocks_transient_with_both_reasons( tmp_path: Path, monkeypatch ): task = SimpleNamespace( id="t_double", status="running", result=None, current_run_id=33, assignee="cli-auto", max_runtime_seconds=120, ) comments: list = [] calls: list = [] board = _lane_board(tmp_path, task, comments, calls) health_paths = _isolate_lane(tmp_path, monkeypatch, board) def select_route(_prompt, lane, **kwargs): assert lane == "cli-auto" return ( _route("claude", "high") if kwargs.get("exclude_provider") == "codex" else _route("codex", "high") ) monkeypatch.setattr(lanes, "select_route", select_route) decisions: list = [] monkeypatch.setattr( lanes, "record_route_decision", lambda provider, effort, classifier, outcome: decisions.append( (provider, effort, outcome) ), ) reports = [ lanes.ProcessResult(1, "usage limit reached for this window", None, True), lanes.ProcessResult(1, "invalid oauth token, authentication failed", None, True), ] runs: list = [] monkeypatch.setattr( lanes, "run_provider", lambda *_args, **_kwargs: runs.append(1) or reports.pop(0), ) lanes.execute_claim("cassandra", "t_double") assert len(runs) == 2 kind, kwargs = calls[-1] assert kind == "block" and kwargs["kind"] == "transient" assert "Both hosted providers failed" in kwargs["reason"] assert "codex: quota" in kwargs["reason"] assert "claude: auth" in kwargs["reason"] assert "never falls to a metered or local path" in kwargs["reason"] assert decisions == [ ("codex", "high", "capacity-failure"), ("claude", "high", "capacity-failure"), ] assert json.loads(health_paths["codex"].read_text())["state"] == "capacity-limited" claude_health = json.loads(health_paths["claude"].read_text()) assert claude_health["state"] == "unavailable" assert claude_health["authenticated"] is False def test_soft_quota_exclusion_steers_new_auto_work(tmp_path: Path, monkeypatch): task = SimpleNamespace( id="t_quota", status="running", result=None, current_run_id=37, assignee="cli-auto", max_runtime_seconds=60, ) comments: list = [] calls: list = [] board = _lane_board(tmp_path, task, comments, calls) _isolate_lane(tmp_path, monkeypatch, board) monkeypatch.setattr( lanes, "fetch_quota_snapshot", lambda *_a, **_k: { "codex": lanes.ProviderQuota(10.0, None), "claude": lanes.ProviderQuota(60.0, None), }, ) route_calls: list = [] monkeypatch.setattr( lanes, "select_route", lambda _prompt, lane, **kwargs: ( route_calls.append((lane, kwargs)) or _route("claude", "high") ), ) monkeypatch.setattr( lanes, "run_provider", lambda *_args, **_kwargs: lanes.ProcessResult( 0, "done", dict(COMPLETED_RESULT), False ), ) lanes.execute_claim("cassandra", "t_quota") assert calls and calls[0][0] == "complete" lane, kwargs = route_calls[0] assert lane == "cli-auto" assert kwargs["exclude_provider"] == "codex" assert kwargs["exclude_reason"] == "is below its remaining-quota routing threshold" assert any("Quota guard: codex" in item for item in comments) assert any( "Provider routing guard (quota) excluded codex" in item for item in comments )