titan/atlas-iac
3
0
Fork 0
Code Issues Pull Requests 9 Actions Packages Projects Releases Wiki Activity

Merge pull request 'fix(hermes): cli-auto capacity failover uses automatic Switchyard reclassification' (#32) from fix/cli-auto-failover-effort into main

Browse Source 
Reviewed-on: atlas/titan-iac#32
Reviewed-by: bstein <bstein@noreply.scm.bstein.dev>
This commit is contained in:
bstein 2026-08-21 23:13:08 +00:00
commit 0841681a06
5 changed files with 164 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
services/hermes/scripts/cli_lane_failover.py
View File

@ -9,7 +9,7 @@ from pathlib import Path
from typing import Any, Callable
from cli_lane_board import _board_call
from cli_lane_config import ProcessResult, Route
from cli_lane_config import EFFORTS, ProcessResult, Route
from cli_lane_health import classify_capacity_failure, record_provider_failure
from cli_lane_metrics import (
record_provider_fallback,
@ -117,20 +117,47 @@ def capacity_failover(
+ "\n\nRouting boundary: the first provider failed from capacity/authentication. "
+ "Select the alternate hosted provider at an appropriate effort."
)
alternate = "claude" if route.provider == "codex" else "codex"
fallback = _routed_or_blocked(
kanban_db,
board,
task_id,
run_id,
lambda: select_route(retry_context, f"cli-{alternate}-{route.effort}"),
lambda: select_route(
retry_context,
assignee,
exclude_provider=route.provider,
exclude_reason=f"hit a {failure_class} failure at this boundary",
),
)
if fallback is None:
return FailoverOutcome(route, result, candidate_file, route.provider, True)
if EFFORTS.index(fallback.effort) < EFFORTS.index(route.effort):
# Never let a capacity-triggered reclassification downgrade effort:
# re-pin the classifier's chosen (healthy) provider at the original
# floor, whether the downgrade came from the classifier itself or
# from its own excluded-provider health guard.
preserved = _routed_or_blocked(
kanban_db,
board,
task_id,
run_id,
lambda: select_route(
retry_context, f"cli-{fallback.provider}-{route.effort}"
),
)
if preserved is None:
return FailoverOutcome(route, result, candidate_file, route.provider, True)
comment(
f"Effort preserved: Switchyard classification chose {fallback.effort} "
f"for {fallback.provider}; escalated to the original {route.effort} "
"floor so capacity failover never downgrades a safety task.",
)
fallback = preserved
record_provider_fallback(route.provider, fallback.provider, failure_class)
comment(
f"Provider fallback: {route.provider} -> {fallback.provider} "
f"after a {failure_class} failure; Jetson reclassified the retry boundary.",
f"after a {failure_class} failure; Jetson reclassified the retry boundary "
f"(classifier={fallback.classifier}).",
)
fallback_result = run_provider(
fallback,

22
testing/tests/test_hermes_cli_fallback.py
View File

@ -53,6 +53,15 @@ def test_goal_card_continues_after_local_judge_rejects_progress(
)
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
monkeypatch.setattr(
lanes,
"PROVIDER_HEALTH_PATHS",
{
"codex": tmp_path / "provider-health/codex.json",
"claude": tmp_path / "provider-health/claude.json",
},
)
monkeypatch.setattr(lanes, "fetch_quota_snapshot", lambda *_a, **_k: {})
lanes.atomic_json(
lanes.state_path("cassandra", "t_goal"),
{"goal_rejections": ["prior incomplete report"]},
@ -185,6 +194,15 @@ def test_capacity_fallback_preserves_first_claude_structured_response(
)
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
monkeypatch.setattr(
lanes,
"PROVIDER_HEALTH_PATHS",
{
"codex": tmp_path / "provider-health/codex.json",
"claude": tmp_path / "provider-health/claude.json",
},
)
monkeypatch.setattr(lanes, "fetch_quota_snapshot", lambda *_a, **_k: {})
claude = lanes.Route(
"claude", "claude-fable-5", "high", "claude-high", "test", "test", 1, ()
)
@ -194,8 +212,8 @@ def test_capacity_fallback_preserves_first_claude_structured_response(
monkeypatch.setattr(
lanes,
"select_route",
lambda _prompt, assignee, **_kwargs: codex
if assignee == "cli-codex-high"
lambda _prompt, assignee, **kwargs: codex
if kwargs.get("exclude_provider") == "claude"
else claude,
)
reports = [

111
testing/tests/test_hermes_cli_quota_failover.py
View File

@ -157,13 +157,18 @@ def test_auto_failover_preserves_effort_and_records_cooldown(
def select_route(_prompt, lane, **kwargs):
route_calls.append((lane, kwargs))
if lane == "cli-auto":
assert lane == "cli-auto"
if not kwargs.get("exclude_provider"):
return lanes.Route(
"codex", "gpt-5.6-terra", "xhigh", "codex-xhigh",
"switchyard-classifier", "vote", 1, (),
)
assert lane == "cli-claude-xhigh"
return _route("claude", "xhigh")
# Automatic reclassification at the retry boundary: this must remain
# a Jetson classifier decision, not a hardcoded manual alternate.
return lanes.Route(
"claude", "claude-opus-5", "xhigh", "claude-xhigh",
"switchyard-classifier", "vote", 1, (),
)
monkeypatch.setattr(lanes, "select_route", select_route)
fallbacks: list = []
@ -184,6 +189,17 @@ def test_auto_failover_preserves_effort_and_records_cooldown(
assert reports == []
assert calls and calls[0][0] == "complete"
# Exactly two Switchyard selections: the initial classification and the
# automatic retry with an explicit failed-provider exclusion. No third
# (manual pin) call, because the classifier already preserved effort.
assert len(route_calls) == 2
initial_lane, initial_kwargs = route_calls[0]
assert initial_lane == "cli-auto"
assert not initial_kwargs.get("exclude_provider")
retry_lane, retry_kwargs = route_calls[1]
assert retry_lane == "cli-auto"
assert retry_kwargs["exclude_provider"] == "codex"
assert "quota" in retry_kwargs["exclude_reason"]
assert fallbacks == [("codex", "claude", "quota")]
assert any(
"Provider fallback: codex -> claude after a quota failure" in item
@ -196,6 +212,81 @@ def test_auto_failover_preserves_effort_and_records_cooldown(
assert claude_health["state"] == "available"
def test_auto_failover_escalates_when_classifier_downgrades_effort(
tmp_path: Path, monkeypatch
):
task = SimpleNamespace(
id="t_auto_escalate",
status="running",
result=None,
current_run_id=39,
assignee="cli-auto",
max_runtime_seconds=120,
)
comments: list = []
calls: list = []
board = _lane_board(tmp_path, task, comments, calls)
health_paths = _isolate_lane(tmp_path, monkeypatch, board)
route_calls: list = []
def select_route(_prompt, lane, **kwargs):
route_calls.append((lane, kwargs))
if lane == "cli-auto" and not kwargs.get("exclude_provider"):
return lanes.Route(
"codex", "gpt-5.6-terra", "high", "codex-high",
"switchyard-classifier", "vote", 1, (),
)
if lane == "cli-auto" and kwargs.get("exclude_provider") == "codex":
# Jetson reclassifies but picks a lower effort than the original
# route; the lane must never let capacity failover downgrade it.
return lanes.Route(
"claude", "claude-haiku-4-5", "low", "claude-low",
"switchyard-classifier", "vote", 1, (),
)
assert lane == "cli-claude-high"
return _route("claude", "high")
monkeypatch.setattr(lanes, "select_route", select_route)
fallbacks: list = []
monkeypatch.setattr(
lanes,
"record_provider_fallback",
lambda source, target, reason: fallbacks.append((source, target, reason)),
)
reports = [
lanes.ProcessResult(1, "You have hit your usage limit.", None, True),
lanes.ProcessResult(0, "done", dict(COMPLETED_RESULT), False),
]
monkeypatch.setattr(
lanes, "run_provider", lambda *_args, **_kwargs: reports.pop(0)
)
lanes.execute_claim("cassandra", "t_auto_escalate")
assert reports == []
assert calls and calls[0][0] == "complete"
assert [lane for lane, _ in route_calls] == [
"cli-auto",
"cli-auto",
"cli-claude-high",
]
# The provider fallback is recorded against the final, effort-preserved
# selection, not the transient low-effort classification.
assert fallbacks == [("codex", "claude", "quota")]
assert any(
"escalated to the original high" in item
for item in comments
)
assert any(
"Provider fallback: codex -> claude after a quota failure" in item
for item in comments
)
codex_health = json.loads(health_paths["codex"].read_text())
assert codex_health["state"] == "capacity-limited"
claude_health = json.loads(health_paths["claude"].read_text())
assert claude_health["state"] == "available"
def test_bare_forbidden_auth_blip_fails_over_and_records_auth_cooldown(
tmp_path: Path, monkeypatch
):
@ -212,9 +303,12 @@ def test_bare_forbidden_auth_blip_fails_over_and_records_auth_cooldown(
board = _lane_board(tmp_path, task, comments, calls)
health_paths = _isolate_lane(tmp_path, monkeypatch, board)
def select_route(_prompt, lane, **_kwargs):
def select_route(_prompt, lane, **kwargs):
assert lane == "cli-auto"
return (
_route("codex", "high") if lane == "cli-auto" else _route("claude", "high")
_route("claude", "high")
if kwargs.get("exclude_provider") == "codex"
else _route("codex", "high")
)
monkeypatch.setattr(lanes, "select_route", select_route)
@ -286,9 +380,12 @@ def test_double_capacity_failure_blocks_transient_with_both_reasons(
board = _lane_board(tmp_path, task, comments, calls)
health_paths = _isolate_lane(tmp_path, monkeypatch, board)
def select_route(_prompt, lane, **_kwargs):
def select_route(_prompt, lane, **kwargs):
assert lane == "cli-auto"
return (
_route("codex", "high") if lane == "cli-auto" else _route("claude", "high")
_route("claude", "high")
if kwargs.get("exclude_provider") == "codex"
else _route("codex", "high")
)
monkeypatch.setattr(lanes, "select_route", select_route)

4
testing/tests/test_hermes_cli_review_lane.py
View File

@ -182,6 +182,10 @@ class _Lane:
def _select_route(self, _prompt, assignee, **kwargs):
self.routes.append((assignee, kwargs))
if assignee == "cli-auto":
if kwargs.get("exclude_provider") == "claude":
return self._route("codex")
return self._route("claude")
if assignee.startswith("cli-codex"):
return self._route("codex")
return self._route("claude")

8
testing/tests/test_hermes_cli_router_outage.py
View File

@ -74,9 +74,9 @@ def test_router_outage_during_fallback_selection_blocks_transient(
_isolate_lane(tmp_path, monkeypatch, board)
selections: list = []
def select_route(_prompt, lane, **_kwargs):
def select_route(_prompt, lane, **kwargs):
selections.append(lane)
if lane == "cli-auto":
if lane == "cli-auto" and not kwargs.get("exclude_provider"):
return _route("codex", "medium")
raise RuntimeError("Switchyard worker routing failed: refused")
@ -89,7 +89,9 @@ def test_router_outage_during_fallback_selection_blocks_transient(
lanes.execute_claim("cassandra", "t_router_fb")
assert selections == ["cli-auto", "cli-claude-medium"]
# The automatic retry re-classifies via Switchyard (same "cli-auto" lane)
# with an explicit failed-provider exclusion, not a hardcoded manual lane.
assert selections == ["cli-auto", "cli-auto"]
kind, kwargs = calls[-1]
assert kind == "block" and kwargs["kind"] == "transient"
assert "Switchyard route selection is unavailable" in kwargs["reason"]