a5d0abb678
CAPACITY_PATTERN (the gate that sets result.capacity_failure) lacked the bare unauthorized/forbidden/401/403 signals that classify_capacity_failure already recognizes, so an auth blip surfacing only as "403 Forbidden" was blocked as capability instead of transient: no auto failover, no health cooldown. Add 401|403|unauthorized|forbidden to the gate so it matches the classifier; reason classification still distinguishes auth from quota/rate-limit/transport. Tests: an auto card failing with only "403 Forbidden" now fails over to the other provider, records an auth cooldown (authenticated:false), and classifies the fallback reason as auth in metrics; a manual card with the same failure still fails closed as transient. Router-outage tests moved to test_hermes_cli_router_outage.py to keep both files <500 LOC. Based on PR #15 (fix/hermes-result-decomposition-reliability). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
381 lines
13 KiB
Python
381 lines
13 KiB
Python
"""Fail-closed manual lanes, bounded failover, and router-outage blocking."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from testing.tests.test_hermes_cli_support import (
|
|
Path,
|
|
SimpleNamespace,
|
|
json,
|
|
lanes,
|
|
nullcontext,
|
|
sys,
|
|
)
|
|
|
|
|
|
COMPLETED_RESULT = {
|
|
"status": "completed",
|
|
"summary": "Work finished with evidence.",
|
|
"changed_files": ["src/a.py"],
|
|
"tests_run": ["pytest full: passed"],
|
|
"artifacts": [],
|
|
"findings": [],
|
|
"blockers": [],
|
|
}
|
|
|
|
|
|
class _Connection:
|
|
def close(self):
|
|
return None
|
|
|
|
|
|
def _lane_board(tmp_path: Path, task, comments: list, calls: list) -> SimpleNamespace:
|
|
return SimpleNamespace(
|
|
scoped_current_board=lambda _board: nullcontext(),
|
|
connect=lambda board: _Connection(),
|
|
get_task=lambda _conn, _task_id: task,
|
|
worker_log_path=lambda _task_id, board: tmp_path / "worker.log",
|
|
_resolve_worktree_workspace=lambda _task, board: (tmp_path, f"wt/{task.id}"),
|
|
set_branch_name=lambda *_args: None,
|
|
set_workspace_path=lambda *_args: None,
|
|
build_worker_context=lambda *_args: "Finish the assigned card.",
|
|
heartbeat_worker=lambda *_args, **_kwargs: True,
|
|
add_comment=lambda _conn, _task_id, _author, body: comments.append(body),
|
|
complete_task=lambda *_args, **kwargs: (
|
|
calls.append(("complete", kwargs)) or True
|
|
),
|
|
block_task=lambda *_args, **kwargs: calls.append(("block", kwargs)),
|
|
)
|
|
|
|
|
|
def _isolate_lane(tmp_path: Path, monkeypatch, board) -> dict[str, Path]:
|
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=board))
|
|
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
|
|
monkeypatch.setattr(lanes, "DATA_ROOT", tmp_path)
|
|
health_paths = {
|
|
"codex": tmp_path / "provider-health/codex.json",
|
|
"claude": tmp_path / "provider-health/claude.json",
|
|
}
|
|
monkeypatch.setattr(lanes, "PROVIDER_HEALTH_PATHS", health_paths)
|
|
monkeypatch.setattr(lanes, "fetch_quota_snapshot", lambda *_a, **_k: {})
|
|
return health_paths
|
|
|
|
|
|
def _route(provider: str, effort: str):
|
|
model = "gpt-5.6-sol" if provider == "codex" else "claude-opus-5"
|
|
return lanes.Route(
|
|
provider, model, effort, f"{provider}-{effort}", "switchyard-manual", "r", 1, ()
|
|
)
|
|
|
|
|
|
def _run_manual_fail_closed(tmp_path, monkeypatch, assignee, provider, output):
|
|
task = SimpleNamespace(
|
|
id="t_manual",
|
|
status="running",
|
|
result=None,
|
|
current_run_id=31,
|
|
assignee=assignee,
|
|
max_runtime_seconds=60,
|
|
)
|
|
comments: list = []
|
|
calls: list = []
|
|
board = _lane_board(tmp_path, task, comments, calls)
|
|
health_paths = _isolate_lane(tmp_path, monkeypatch, board)
|
|
route_calls: list = []
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"select_route",
|
|
lambda _prompt, lane, **kwargs: (
|
|
route_calls.append((lane, kwargs)) or _route(provider, "high")
|
|
),
|
|
)
|
|
runs: list = []
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"run_provider",
|
|
lambda *_args, **_kwargs: (
|
|
runs.append(provider) or lanes.ProcessResult(1, output, None, True)
|
|
),
|
|
)
|
|
|
|
lanes.execute_claim("cassandra", "t_manual")
|
|
return calls, comments, route_calls, runs, health_paths
|
|
|
|
|
|
def test_manual_codex_assignee_fails_closed_without_provider_switch(
|
|
tmp_path: Path, monkeypatch
|
|
):
|
|
calls, _comments, route_calls, runs, health_paths = _run_manual_fail_closed(
|
|
tmp_path, monkeypatch, "cli-codex-high", "codex", "HTTP 429 rate limit hit"
|
|
)
|
|
|
|
assert runs == ["codex"]
|
|
assert [lane for lane, _ in route_calls] == ["cli-codex-high"]
|
|
kind, kwargs = calls[-1]
|
|
assert kind == "block" and kwargs["kind"] == "transient"
|
|
assert "Manually pinned provider codex" in kwargs["reason"]
|
|
assert "cli-codex-high" in kwargs["reason"]
|
|
assert "rate-limit" in kwargs["reason"]
|
|
health = json.loads(health_paths["codex"].read_text())
|
|
assert health["state"] == "capacity-limited"
|
|
assert not health_paths["claude"].exists()
|
|
|
|
|
|
def test_manual_claude_assignee_fails_closed_without_provider_switch(
|
|
tmp_path: Path, monkeypatch
|
|
):
|
|
calls, _comments, route_calls, runs, health_paths = _run_manual_fail_closed(
|
|
tmp_path, monkeypatch, "cli-claude-low", "claude", "usage limit exhausted"
|
|
)
|
|
|
|
assert runs == ["claude"]
|
|
assert [lane for lane, _ in route_calls] == ["cli-claude-low"]
|
|
kind, kwargs = calls[-1]
|
|
assert kind == "block" and kwargs["kind"] == "transient"
|
|
assert "Manually pinned provider claude" in kwargs["reason"]
|
|
assert "quota" in kwargs["reason"]
|
|
health = json.loads(health_paths["claude"].read_text())
|
|
assert health["state"] == "capacity-limited"
|
|
assert not health_paths["codex"].exists()
|
|
|
|
|
|
def test_auto_failover_preserves_effort_and_records_cooldown(
|
|
tmp_path: Path, monkeypatch
|
|
):
|
|
task = SimpleNamespace(
|
|
id="t_auto",
|
|
status="running",
|
|
result=None,
|
|
current_run_id=32,
|
|
assignee="cli-auto",
|
|
max_runtime_seconds=120,
|
|
)
|
|
comments: list = []
|
|
calls: list = []
|
|
board = _lane_board(tmp_path, task, comments, calls)
|
|
health_paths = _isolate_lane(tmp_path, monkeypatch, board)
|
|
route_calls: list = []
|
|
|
|
def select_route(_prompt, lane, **kwargs):
|
|
route_calls.append((lane, kwargs))
|
|
if lane == "cli-auto":
|
|
return lanes.Route(
|
|
"codex", "gpt-5.6-terra", "xhigh", "codex-xhigh",
|
|
"switchyard-classifier", "vote", 1, (),
|
|
)
|
|
assert lane == "cli-claude-xhigh"
|
|
return _route("claude", "xhigh")
|
|
|
|
monkeypatch.setattr(lanes, "select_route", select_route)
|
|
fallbacks: list = []
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"record_provider_fallback",
|
|
lambda source, target, reason: fallbacks.append((source, target, reason)),
|
|
)
|
|
reports = [
|
|
lanes.ProcessResult(1, "You have hit your usage limit.", None, True),
|
|
lanes.ProcessResult(0, "done", dict(COMPLETED_RESULT), False),
|
|
]
|
|
monkeypatch.setattr(
|
|
lanes, "run_provider", lambda *_args, **_kwargs: reports.pop(0)
|
|
)
|
|
|
|
lanes.execute_claim("cassandra", "t_auto")
|
|
|
|
assert reports == []
|
|
assert calls and calls[0][0] == "complete"
|
|
assert fallbacks == [("codex", "claude", "quota")]
|
|
assert any(
|
|
"Provider fallback: codex -> claude after a quota failure" in item
|
|
for item in comments
|
|
)
|
|
codex_health = json.loads(health_paths["codex"].read_text())
|
|
assert codex_health["state"] == "capacity-limited"
|
|
assert codex_health["cooldown_until"] == codex_health["failed_at"] + 300.0
|
|
claude_health = json.loads(health_paths["claude"].read_text())
|
|
assert claude_health["state"] == "available"
|
|
|
|
|
|
def test_bare_forbidden_auth_blip_fails_over_and_records_auth_cooldown(
|
|
tmp_path: Path, monkeypatch
|
|
):
|
|
task = SimpleNamespace(
|
|
id="t_forbidden",
|
|
status="running",
|
|
result=None,
|
|
current_run_id=38,
|
|
assignee="cli-auto",
|
|
max_runtime_seconds=120,
|
|
)
|
|
comments: list = []
|
|
calls: list = []
|
|
board = _lane_board(tmp_path, task, comments, calls)
|
|
health_paths = _isolate_lane(tmp_path, monkeypatch, board)
|
|
|
|
def select_route(_prompt, lane, **_kwargs):
|
|
return (
|
|
_route("codex", "high") if lane == "cli-auto" else _route("claude", "high")
|
|
)
|
|
|
|
monkeypatch.setattr(lanes, "select_route", select_route)
|
|
fallbacks: list = []
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"record_provider_fallback",
|
|
lambda source, target, reason: fallbacks.append((source, target, reason)),
|
|
)
|
|
# The failure surfaces only as "403 Forbidden" with none of the words the
|
|
# capacity gate previously keyed on (authentication/oauth/token expired).
|
|
reports = [
|
|
lanes.ProcessResult(1, "403 Forbidden", None, True),
|
|
lanes.ProcessResult(0, "done", dict(COMPLETED_RESULT), False),
|
|
]
|
|
monkeypatch.setattr(lanes, "run_provider", lambda *_args, **_kwargs: reports.pop(0))
|
|
|
|
lanes.execute_claim("cassandra", "t_forbidden")
|
|
|
|
# (a) failed over to the other provider and completed.
|
|
assert reports == []
|
|
assert calls and calls[0][0] == "complete"
|
|
# (c) the fallback reason classifies as auth in metrics.
|
|
assert fallbacks == [("codex", "claude", "auth")]
|
|
assert any(
|
|
"Provider fallback: codex -> claude after a auth failure" in item
|
|
for item in comments
|
|
)
|
|
# (b) an auth cooldown was recorded (authenticated:false, longer window).
|
|
codex_health = json.loads(health_paths["codex"].read_text())
|
|
assert codex_health["state"] == "unavailable"
|
|
assert codex_health["authenticated"] is False
|
|
assert codex_health["failure_reason"] == "auth"
|
|
assert codex_health["cooldown_until"] == codex_health["failed_at"] + 3600.0
|
|
|
|
|
|
def test_bare_forbidden_auth_blip_still_fails_closed_on_manual_lane(
|
|
tmp_path: Path, monkeypatch
|
|
):
|
|
calls, _comments, route_calls, runs, health_paths = _run_manual_fail_closed(
|
|
tmp_path, monkeypatch, "cli-codex-high", "codex", "403 Forbidden"
|
|
)
|
|
|
|
# Fail-closed is unchanged: the pinned provider is not switched.
|
|
assert runs == ["codex"]
|
|
assert [lane for lane, _ in route_calls] == ["cli-codex-high"]
|
|
kind, kwargs = calls[-1]
|
|
assert kind == "block" and kwargs["kind"] == "transient"
|
|
assert "Manually pinned provider codex" in kwargs["reason"]
|
|
assert "auth" in kwargs["reason"]
|
|
codex_health = json.loads(health_paths["codex"].read_text())
|
|
assert codex_health["state"] == "unavailable"
|
|
assert codex_health["authenticated"] is False
|
|
|
|
|
|
def test_double_capacity_failure_blocks_transient_with_both_reasons(
|
|
tmp_path: Path, monkeypatch
|
|
):
|
|
task = SimpleNamespace(
|
|
id="t_double",
|
|
status="running",
|
|
result=None,
|
|
current_run_id=33,
|
|
assignee="cli-auto",
|
|
max_runtime_seconds=120,
|
|
)
|
|
comments: list = []
|
|
calls: list = []
|
|
board = _lane_board(tmp_path, task, comments, calls)
|
|
health_paths = _isolate_lane(tmp_path, monkeypatch, board)
|
|
|
|
def select_route(_prompt, lane, **_kwargs):
|
|
return (
|
|
_route("codex", "high") if lane == "cli-auto" else _route("claude", "high")
|
|
)
|
|
|
|
monkeypatch.setattr(lanes, "select_route", select_route)
|
|
decisions: list = []
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"record_route_decision",
|
|
lambda provider, effort, classifier, outcome: decisions.append(
|
|
(provider, effort, outcome)
|
|
),
|
|
)
|
|
reports = [
|
|
lanes.ProcessResult(1, "usage limit reached for this window", None, True),
|
|
lanes.ProcessResult(1, "invalid oauth token, authentication failed", None, True),
|
|
]
|
|
runs: list = []
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"run_provider",
|
|
lambda *_args, **_kwargs: runs.append(1) or reports.pop(0),
|
|
)
|
|
|
|
lanes.execute_claim("cassandra", "t_double")
|
|
|
|
assert len(runs) == 2
|
|
kind, kwargs = calls[-1]
|
|
assert kind == "block" and kwargs["kind"] == "transient"
|
|
assert "Both hosted providers failed" in kwargs["reason"]
|
|
assert "codex: quota" in kwargs["reason"]
|
|
assert "claude: auth" in kwargs["reason"]
|
|
assert "never falls to a metered or local path" in kwargs["reason"]
|
|
assert decisions == [
|
|
("codex", "high", "capacity-failure"),
|
|
("claude", "high", "capacity-failure"),
|
|
]
|
|
assert json.loads(health_paths["codex"].read_text())["state"] == "capacity-limited"
|
|
claude_health = json.loads(health_paths["claude"].read_text())
|
|
assert claude_health["state"] == "unavailable"
|
|
assert claude_health["authenticated"] is False
|
|
|
|
|
|
def test_soft_quota_exclusion_steers_new_auto_work(tmp_path: Path, monkeypatch):
|
|
task = SimpleNamespace(
|
|
id="t_quota",
|
|
status="running",
|
|
result=None,
|
|
current_run_id=37,
|
|
assignee="cli-auto",
|
|
max_runtime_seconds=60,
|
|
)
|
|
comments: list = []
|
|
calls: list = []
|
|
board = _lane_board(tmp_path, task, comments, calls)
|
|
_isolate_lane(tmp_path, monkeypatch, board)
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"fetch_quota_snapshot",
|
|
lambda *_a, **_k: {
|
|
"codex": lanes.ProviderQuota(10.0, None),
|
|
"claude": lanes.ProviderQuota(60.0, None),
|
|
},
|
|
)
|
|
route_calls: list = []
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"select_route",
|
|
lambda _prompt, lane, **kwargs: (
|
|
route_calls.append((lane, kwargs)) or _route("claude", "high")
|
|
),
|
|
)
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"run_provider",
|
|
lambda *_args, **_kwargs: lanes.ProcessResult(
|
|
0, "done", dict(COMPLETED_RESULT), False
|
|
),
|
|
)
|
|
|
|
lanes.execute_claim("cassandra", "t_quota")
|
|
|
|
assert calls and calls[0][0] == "complete"
|
|
lane, kwargs = route_calls[0]
|
|
assert lane == "cli-auto"
|
|
assert kwargs["exclude_provider"] == "codex"
|
|
assert kwargs["exclude_reason"] == "is below its remaining-quota routing threshold"
|
|
assert any("Quota guard: codex" in item for item in comments)
|
|
assert any(
|
|
"Provider routing guard (quota) excluded codex" in item for item in comments
|
|
)
|