fix(hermes): cli-auto capacity failover uses automatic Switchyard reclassification

Capacity/auth/quota failover for cli-auto previously called select_route
with a hardcoded manual lane (cli-{alternate}-{effort}), so the retry
boundary was classified as switchyard-manual instead of going through
Jetson automatic classification, making the routing evidence misleading.

Now the retry calls select_route(context, "cli-auto", exclude_provider=...)
so the boundary stays automatically classified with an explicit
failed-provider exclusion. If the classifier reclassifies to a lower
effort than the original route, the lane re-pins the chosen provider at
the original effort floor so a capacity failure never silently downgrades
a high/xhigh task. Manual lanes (assignee != cli-auto) remain unchanged
and still fail closed without switching providers.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Hermes Agent 2026-08-21 05:04:55 +00:00
parent 25fcda010f
commit d579c08cbe
5 changed files with 164 additions and 16 deletions

View File

@ -9,7 +9,7 @@ from pathlib import Path
from typing import Any, Callable from typing import Any, Callable
from cli_lane_board import _board_call from cli_lane_board import _board_call
from cli_lane_config import ProcessResult, Route from cli_lane_config import EFFORTS, ProcessResult, Route
from cli_lane_health import classify_capacity_failure, record_provider_failure from cli_lane_health import classify_capacity_failure, record_provider_failure
from cli_lane_metrics import ( from cli_lane_metrics import (
record_provider_fallback, record_provider_fallback,
@ -117,20 +117,47 @@ def capacity_failover(
+ "\n\nRouting boundary: the first provider failed from capacity/authentication. " + "\n\nRouting boundary: the first provider failed from capacity/authentication. "
+ "Select the alternate hosted provider at an appropriate effort." + "Select the alternate hosted provider at an appropriate effort."
) )
alternate = "claude" if route.provider == "codex" else "codex"
fallback = _routed_or_blocked( fallback = _routed_or_blocked(
kanban_db, kanban_db,
board, board,
task_id, task_id,
run_id, run_id,
lambda: select_route(retry_context, f"cli-{alternate}-{route.effort}"), lambda: select_route(
retry_context,
assignee,
exclude_provider=route.provider,
exclude_reason=f"hit a {failure_class} failure at this boundary",
),
) )
if fallback is None: if fallback is None:
return FailoverOutcome(route, result, candidate_file, route.provider, True) return FailoverOutcome(route, result, candidate_file, route.provider, True)
if EFFORTS.index(fallback.effort) < EFFORTS.index(route.effort):
# Never let a capacity-triggered reclassification downgrade effort:
# re-pin the classifier's chosen (healthy) provider at the original
# floor, whether the downgrade came from the classifier itself or
# from its own excluded-provider health guard.
preserved = _routed_or_blocked(
kanban_db,
board,
task_id,
run_id,
lambda: select_route(
retry_context, f"cli-{fallback.provider}-{route.effort}"
),
)
if preserved is None:
return FailoverOutcome(route, result, candidate_file, route.provider, True)
comment(
f"Effort preserved: Switchyard classification chose {fallback.effort} "
f"for {fallback.provider}; escalated to the original {route.effort} "
"floor so capacity failover never downgrades a safety task.",
)
fallback = preserved
record_provider_fallback(route.provider, fallback.provider, failure_class) record_provider_fallback(route.provider, fallback.provider, failure_class)
comment( comment(
f"Provider fallback: {route.provider} -> {fallback.provider} " f"Provider fallback: {route.provider} -> {fallback.provider} "
f"after a {failure_class} failure; Jetson reclassified the retry boundary.", f"after a {failure_class} failure; Jetson reclassified the retry boundary "
f"(classifier={fallback.classifier}).",
) )
fallback_result = run_provider( fallback_result = run_provider(
fallback, fallback,

View File

@ -53,6 +53,15 @@ def test_goal_card_continues_after_local_judge_rejects_progress(
) )
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
monkeypatch.setattr(
lanes,
"PROVIDER_HEALTH_PATHS",
{
"codex": tmp_path / "provider-health/codex.json",
"claude": tmp_path / "provider-health/claude.json",
},
)
monkeypatch.setattr(lanes, "fetch_quota_snapshot", lambda *_a, **_k: {})
lanes.atomic_json( lanes.atomic_json(
lanes.state_path("cassandra", "t_goal"), lanes.state_path("cassandra", "t_goal"),
{"goal_rejections": ["prior incomplete report"]}, {"goal_rejections": ["prior incomplete report"]},
@ -185,6 +194,15 @@ def test_capacity_fallback_preserves_first_claude_structured_response(
) )
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
monkeypatch.setattr(
lanes,
"PROVIDER_HEALTH_PATHS",
{
"codex": tmp_path / "provider-health/codex.json",
"claude": tmp_path / "provider-health/claude.json",
},
)
monkeypatch.setattr(lanes, "fetch_quota_snapshot", lambda *_a, **_k: {})
claude = lanes.Route( claude = lanes.Route(
"claude", "claude-fable-5", "high", "claude-high", "test", "test", 1, () "claude", "claude-fable-5", "high", "claude-high", "test", "test", 1, ()
) )
@ -194,8 +212,8 @@ def test_capacity_fallback_preserves_first_claude_structured_response(
monkeypatch.setattr( monkeypatch.setattr(
lanes, lanes,
"select_route", "select_route",
lambda _prompt, assignee, **_kwargs: codex lambda _prompt, assignee, **kwargs: codex
if assignee == "cli-codex-high" if kwargs.get("exclude_provider") == "claude"
else claude, else claude,
) )
reports = [ reports = [

View File

@ -157,13 +157,18 @@ def test_auto_failover_preserves_effort_and_records_cooldown(
def select_route(_prompt, lane, **kwargs): def select_route(_prompt, lane, **kwargs):
route_calls.append((lane, kwargs)) route_calls.append((lane, kwargs))
if lane == "cli-auto": assert lane == "cli-auto"
if not kwargs.get("exclude_provider"):
return lanes.Route( return lanes.Route(
"codex", "gpt-5.6-terra", "xhigh", "codex-xhigh", "codex", "gpt-5.6-terra", "xhigh", "codex-xhigh",
"switchyard-classifier", "vote", 1, (), "switchyard-classifier", "vote", 1, (),
) )
assert lane == "cli-claude-xhigh" # Automatic reclassification at the retry boundary: this must remain
return _route("claude", "xhigh") # a Jetson classifier decision, not a hardcoded manual alternate.
return lanes.Route(
"claude", "claude-opus-5", "xhigh", "claude-xhigh",
"switchyard-classifier", "vote", 1, (),
)
monkeypatch.setattr(lanes, "select_route", select_route) monkeypatch.setattr(lanes, "select_route", select_route)
fallbacks: list = [] fallbacks: list = []
@ -184,6 +189,17 @@ def test_auto_failover_preserves_effort_and_records_cooldown(
assert reports == [] assert reports == []
assert calls and calls[0][0] == "complete" assert calls and calls[0][0] == "complete"
# Exactly two Switchyard selections: the initial classification and the
# automatic retry with an explicit failed-provider exclusion. No third
# (manual pin) call, because the classifier already preserved effort.
assert len(route_calls) == 2
initial_lane, initial_kwargs = route_calls[0]
assert initial_lane == "cli-auto"
assert not initial_kwargs.get("exclude_provider")
retry_lane, retry_kwargs = route_calls[1]
assert retry_lane == "cli-auto"
assert retry_kwargs["exclude_provider"] == "codex"
assert "quota" in retry_kwargs["exclude_reason"]
assert fallbacks == [("codex", "claude", "quota")] assert fallbacks == [("codex", "claude", "quota")]
assert any( assert any(
"Provider fallback: codex -> claude after a quota failure" in item "Provider fallback: codex -> claude after a quota failure" in item
@ -196,6 +212,81 @@ def test_auto_failover_preserves_effort_and_records_cooldown(
assert claude_health["state"] == "available" assert claude_health["state"] == "available"
def test_auto_failover_escalates_when_classifier_downgrades_effort(
tmp_path: Path, monkeypatch
):
task = SimpleNamespace(
id="t_auto_escalate",
status="running",
result=None,
current_run_id=39,
assignee="cli-auto",
max_runtime_seconds=120,
)
comments: list = []
calls: list = []
board = _lane_board(tmp_path, task, comments, calls)
health_paths = _isolate_lane(tmp_path, monkeypatch, board)
route_calls: list = []
def select_route(_prompt, lane, **kwargs):
route_calls.append((lane, kwargs))
if lane == "cli-auto" and not kwargs.get("exclude_provider"):
return lanes.Route(
"codex", "gpt-5.6-terra", "high", "codex-high",
"switchyard-classifier", "vote", 1, (),
)
if lane == "cli-auto" and kwargs.get("exclude_provider") == "codex":
# Jetson reclassifies but picks a lower effort than the original
# route; the lane must never let capacity failover downgrade it.
return lanes.Route(
"claude", "claude-haiku-4-5", "low", "claude-low",
"switchyard-classifier", "vote", 1, (),
)
assert lane == "cli-claude-high"
return _route("claude", "high")
monkeypatch.setattr(lanes, "select_route", select_route)
fallbacks: list = []
monkeypatch.setattr(
lanes,
"record_provider_fallback",
lambda source, target, reason: fallbacks.append((source, target, reason)),
)
reports = [
lanes.ProcessResult(1, "You have hit your usage limit.", None, True),
lanes.ProcessResult(0, "done", dict(COMPLETED_RESULT), False),
]
monkeypatch.setattr(
lanes, "run_provider", lambda *_args, **_kwargs: reports.pop(0)
)
lanes.execute_claim("cassandra", "t_auto_escalate")
assert reports == []
assert calls and calls[0][0] == "complete"
assert [lane for lane, _ in route_calls] == [
"cli-auto",
"cli-auto",
"cli-claude-high",
]
# The provider fallback is recorded against the final, effort-preserved
# selection, not the transient low-effort classification.
assert fallbacks == [("codex", "claude", "quota")]
assert any(
"escalated to the original high" in item
for item in comments
)
assert any(
"Provider fallback: codex -> claude after a quota failure" in item
for item in comments
)
codex_health = json.loads(health_paths["codex"].read_text())
assert codex_health["state"] == "capacity-limited"
claude_health = json.loads(health_paths["claude"].read_text())
assert claude_health["state"] == "available"
def test_bare_forbidden_auth_blip_fails_over_and_records_auth_cooldown( def test_bare_forbidden_auth_blip_fails_over_and_records_auth_cooldown(
tmp_path: Path, monkeypatch tmp_path: Path, monkeypatch
): ):
@ -212,9 +303,12 @@ def test_bare_forbidden_auth_blip_fails_over_and_records_auth_cooldown(
board = _lane_board(tmp_path, task, comments, calls) board = _lane_board(tmp_path, task, comments, calls)
health_paths = _isolate_lane(tmp_path, monkeypatch, board) health_paths = _isolate_lane(tmp_path, monkeypatch, board)
def select_route(_prompt, lane, **_kwargs): def select_route(_prompt, lane, **kwargs):
assert lane == "cli-auto"
return ( return (
_route("codex", "high") if lane == "cli-auto" else _route("claude", "high") _route("claude", "high")
if kwargs.get("exclude_provider") == "codex"
else _route("codex", "high")
) )
monkeypatch.setattr(lanes, "select_route", select_route) monkeypatch.setattr(lanes, "select_route", select_route)
@ -286,9 +380,12 @@ def test_double_capacity_failure_blocks_transient_with_both_reasons(
board = _lane_board(tmp_path, task, comments, calls) board = _lane_board(tmp_path, task, comments, calls)
health_paths = _isolate_lane(tmp_path, monkeypatch, board) health_paths = _isolate_lane(tmp_path, monkeypatch, board)
def select_route(_prompt, lane, **_kwargs): def select_route(_prompt, lane, **kwargs):
assert lane == "cli-auto"
return ( return (
_route("codex", "high") if lane == "cli-auto" else _route("claude", "high") _route("claude", "high")
if kwargs.get("exclude_provider") == "codex"
else _route("codex", "high")
) )
monkeypatch.setattr(lanes, "select_route", select_route) monkeypatch.setattr(lanes, "select_route", select_route)

View File

@ -182,6 +182,10 @@ class _Lane:
def _select_route(self, _prompt, assignee, **kwargs): def _select_route(self, _prompt, assignee, **kwargs):
self.routes.append((assignee, kwargs)) self.routes.append((assignee, kwargs))
if assignee == "cli-auto":
if kwargs.get("exclude_provider") == "claude":
return self._route("codex")
return self._route("claude")
if assignee.startswith("cli-codex"): if assignee.startswith("cli-codex"):
return self._route("codex") return self._route("codex")
return self._route("claude") return self._route("claude")

View File

@ -74,9 +74,9 @@ def test_router_outage_during_fallback_selection_blocks_transient(
_isolate_lane(tmp_path, monkeypatch, board) _isolate_lane(tmp_path, monkeypatch, board)
selections: list = [] selections: list = []
def select_route(_prompt, lane, **_kwargs): def select_route(_prompt, lane, **kwargs):
selections.append(lane) selections.append(lane)
if lane == "cli-auto": if lane == "cli-auto" and not kwargs.get("exclude_provider"):
return _route("codex", "medium") return _route("codex", "medium")
raise RuntimeError("Switchyard worker routing failed: refused") raise RuntimeError("Switchyard worker routing failed: refused")
@ -89,7 +89,9 @@ def test_router_outage_during_fallback_selection_blocks_transient(
lanes.execute_claim("cassandra", "t_router_fb") lanes.execute_claim("cassandra", "t_router_fb")
assert selections == ["cli-auto", "cli-claude-medium"] # The automatic retry re-classifies via Switchyard (same "cli-auto" lane)
# with an explicit failed-provider exclusion, not a hardcoded manual lane.
assert selections == ["cli-auto", "cli-auto"]
kind, kwargs = calls[-1] kind, kwargs = calls[-1]
assert kind == "block" and kwargs["kind"] == "transient" assert kind == "block" and kwargs["kind"] == "transient"
assert "Switchyard route selection is unavailable" in kwargs["reason"] assert "Switchyard route selection is unavailable" in kwargs["reason"]