From d579c08cbe069ef3b65d116cdf91a8d0c593bd57 Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Fri, 21 Aug 2026 05:04:55 +0000 Subject: [PATCH] fix(hermes): cli-auto capacity failover uses automatic Switchyard reclassification Capacity/auth/quota failover for cli-auto previously called select_route with a hardcoded manual lane (cli-{alternate}-{effort}), so the retry boundary was classified as switchyard-manual instead of going through Jetson automatic classification, making the routing evidence misleading. Now the retry calls select_route(context, "cli-auto", exclude_provider=...) so the boundary stays automatically classified with an explicit failed-provider exclusion. If the classifier reclassifies to a lower effort than the original route, the lane re-pins the chosen provider at the original effort floor so a capacity failure never silently downgrades a high/xhigh task. Manual lanes (assignee != cli-auto) remain unchanged and still fail closed without switching providers. Co-Authored-By: Claude Sonnet 5 --- services/hermes/scripts/cli_lane_failover.py | 35 +++++- testing/tests/test_hermes_cli_fallback.py | 22 +++- .../tests/test_hermes_cli_quota_failover.py | 111 ++++++++++++++++-- testing/tests/test_hermes_cli_review_lane.py | 4 + .../tests/test_hermes_cli_router_outage.py | 8 +- 5 files changed, 164 insertions(+), 16 deletions(-) diff --git a/services/hermes/scripts/cli_lane_failover.py b/services/hermes/scripts/cli_lane_failover.py index 0f6becdc..5cbcae9a 100644 --- a/services/hermes/scripts/cli_lane_failover.py +++ b/services/hermes/scripts/cli_lane_failover.py @@ -9,7 +9,7 @@ from pathlib import Path from typing import Any, Callable from cli_lane_board import _board_call -from cli_lane_config import ProcessResult, Route +from cli_lane_config import EFFORTS, ProcessResult, Route from cli_lane_health import classify_capacity_failure, record_provider_failure from cli_lane_metrics import ( record_provider_fallback, @@ -117,20 +117,47 @@ def capacity_failover( + "\n\nRouting boundary: the first provider failed from capacity/authentication. " + "Select the alternate hosted provider at an appropriate effort." ) - alternate = "claude" if route.provider == "codex" else "codex" fallback = _routed_or_blocked( kanban_db, board, task_id, run_id, - lambda: select_route(retry_context, f"cli-{alternate}-{route.effort}"), + lambda: select_route( + retry_context, + assignee, + exclude_provider=route.provider, + exclude_reason=f"hit a {failure_class} failure at this boundary", + ), ) if fallback is None: return FailoverOutcome(route, result, candidate_file, route.provider, True) + if EFFORTS.index(fallback.effort) < EFFORTS.index(route.effort): + # Never let a capacity-triggered reclassification downgrade effort: + # re-pin the classifier's chosen (healthy) provider at the original + # floor, whether the downgrade came from the classifier itself or + # from its own excluded-provider health guard. + preserved = _routed_or_blocked( + kanban_db, + board, + task_id, + run_id, + lambda: select_route( + retry_context, f"cli-{fallback.provider}-{route.effort}" + ), + ) + if preserved is None: + return FailoverOutcome(route, result, candidate_file, route.provider, True) + comment( + f"Effort preserved: Switchyard classification chose {fallback.effort} " + f"for {fallback.provider}; escalated to the original {route.effort} " + "floor so capacity failover never downgrades a safety task.", + ) + fallback = preserved record_provider_fallback(route.provider, fallback.provider, failure_class) comment( f"Provider fallback: {route.provider} -> {fallback.provider} " - f"after a {failure_class} failure; Jetson reclassified the retry boundary.", + f"after a {failure_class} failure; Jetson reclassified the retry boundary " + f"(classifier={fallback.classifier}).", ) fallback_result = run_provider( fallback, diff --git a/testing/tests/test_hermes_cli_fallback.py b/testing/tests/test_hermes_cli_fallback.py index 2818ca71..242a0503 100644 --- a/testing/tests/test_hermes_cli_fallback.py +++ b/testing/tests/test_hermes_cli_fallback.py @@ -53,6 +53,15 @@ def test_goal_card_continues_after_local_judge_rejects_progress( ) monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + monkeypatch.setattr( + lanes, + "PROVIDER_HEALTH_PATHS", + { + "codex": tmp_path / "provider-health/codex.json", + "claude": tmp_path / "provider-health/claude.json", + }, + ) + monkeypatch.setattr(lanes, "fetch_quota_snapshot", lambda *_a, **_k: {}) lanes.atomic_json( lanes.state_path("cassandra", "t_goal"), {"goal_rejections": ["prior incomplete report"]}, @@ -185,6 +194,15 @@ def test_capacity_fallback_preserves_first_claude_structured_response( ) monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + monkeypatch.setattr( + lanes, + "PROVIDER_HEALTH_PATHS", + { + "codex": tmp_path / "provider-health/codex.json", + "claude": tmp_path / "provider-health/claude.json", + }, + ) + monkeypatch.setattr(lanes, "fetch_quota_snapshot", lambda *_a, **_k: {}) claude = lanes.Route( "claude", "claude-fable-5", "high", "claude-high", "test", "test", 1, () ) @@ -194,8 +212,8 @@ def test_capacity_fallback_preserves_first_claude_structured_response( monkeypatch.setattr( lanes, "select_route", - lambda _prompt, assignee, **_kwargs: codex - if assignee == "cli-codex-high" + lambda _prompt, assignee, **kwargs: codex + if kwargs.get("exclude_provider") == "claude" else claude, ) reports = [ diff --git a/testing/tests/test_hermes_cli_quota_failover.py b/testing/tests/test_hermes_cli_quota_failover.py index ff503609..0a77848d 100644 --- a/testing/tests/test_hermes_cli_quota_failover.py +++ b/testing/tests/test_hermes_cli_quota_failover.py @@ -157,13 +157,18 @@ def test_auto_failover_preserves_effort_and_records_cooldown( def select_route(_prompt, lane, **kwargs): route_calls.append((lane, kwargs)) - if lane == "cli-auto": + assert lane == "cli-auto" + if not kwargs.get("exclude_provider"): return lanes.Route( "codex", "gpt-5.6-terra", "xhigh", "codex-xhigh", "switchyard-classifier", "vote", 1, (), ) - assert lane == "cli-claude-xhigh" - return _route("claude", "xhigh") + # Automatic reclassification at the retry boundary: this must remain + # a Jetson classifier decision, not a hardcoded manual alternate. + return lanes.Route( + "claude", "claude-opus-5", "xhigh", "claude-xhigh", + "switchyard-classifier", "vote", 1, (), + ) monkeypatch.setattr(lanes, "select_route", select_route) fallbacks: list = [] @@ -184,6 +189,17 @@ def test_auto_failover_preserves_effort_and_records_cooldown( assert reports == [] assert calls and calls[0][0] == "complete" + # Exactly two Switchyard selections: the initial classification and the + # automatic retry with an explicit failed-provider exclusion. No third + # (manual pin) call, because the classifier already preserved effort. + assert len(route_calls) == 2 + initial_lane, initial_kwargs = route_calls[0] + assert initial_lane == "cli-auto" + assert not initial_kwargs.get("exclude_provider") + retry_lane, retry_kwargs = route_calls[1] + assert retry_lane == "cli-auto" + assert retry_kwargs["exclude_provider"] == "codex" + assert "quota" in retry_kwargs["exclude_reason"] assert fallbacks == [("codex", "claude", "quota")] assert any( "Provider fallback: codex -> claude after a quota failure" in item @@ -196,6 +212,81 @@ def test_auto_failover_preserves_effort_and_records_cooldown( assert claude_health["state"] == "available" +def test_auto_failover_escalates_when_classifier_downgrades_effort( + tmp_path: Path, monkeypatch +): + task = SimpleNamespace( + id="t_auto_escalate", + status="running", + result=None, + current_run_id=39, + assignee="cli-auto", + max_runtime_seconds=120, + ) + comments: list = [] + calls: list = [] + board = _lane_board(tmp_path, task, comments, calls) + health_paths = _isolate_lane(tmp_path, monkeypatch, board) + route_calls: list = [] + + def select_route(_prompt, lane, **kwargs): + route_calls.append((lane, kwargs)) + if lane == "cli-auto" and not kwargs.get("exclude_provider"): + return lanes.Route( + "codex", "gpt-5.6-terra", "high", "codex-high", + "switchyard-classifier", "vote", 1, (), + ) + if lane == "cli-auto" and kwargs.get("exclude_provider") == "codex": + # Jetson reclassifies but picks a lower effort than the original + # route; the lane must never let capacity failover downgrade it. + return lanes.Route( + "claude", "claude-haiku-4-5", "low", "claude-low", + "switchyard-classifier", "vote", 1, (), + ) + assert lane == "cli-claude-high" + return _route("claude", "high") + + monkeypatch.setattr(lanes, "select_route", select_route) + fallbacks: list = [] + monkeypatch.setattr( + lanes, + "record_provider_fallback", + lambda source, target, reason: fallbacks.append((source, target, reason)), + ) + reports = [ + lanes.ProcessResult(1, "You have hit your usage limit.", None, True), + lanes.ProcessResult(0, "done", dict(COMPLETED_RESULT), False), + ] + monkeypatch.setattr( + lanes, "run_provider", lambda *_args, **_kwargs: reports.pop(0) + ) + + lanes.execute_claim("cassandra", "t_auto_escalate") + + assert reports == [] + assert calls and calls[0][0] == "complete" + assert [lane for lane, _ in route_calls] == [ + "cli-auto", + "cli-auto", + "cli-claude-high", + ] + # The provider fallback is recorded against the final, effort-preserved + # selection, not the transient low-effort classification. + assert fallbacks == [("codex", "claude", "quota")] + assert any( + "escalated to the original high" in item + for item in comments + ) + assert any( + "Provider fallback: codex -> claude after a quota failure" in item + for item in comments + ) + codex_health = json.loads(health_paths["codex"].read_text()) + assert codex_health["state"] == "capacity-limited" + claude_health = json.loads(health_paths["claude"].read_text()) + assert claude_health["state"] == "available" + + def test_bare_forbidden_auth_blip_fails_over_and_records_auth_cooldown( tmp_path: Path, monkeypatch ): @@ -212,9 +303,12 @@ def test_bare_forbidden_auth_blip_fails_over_and_records_auth_cooldown( board = _lane_board(tmp_path, task, comments, calls) health_paths = _isolate_lane(tmp_path, monkeypatch, board) - def select_route(_prompt, lane, **_kwargs): + def select_route(_prompt, lane, **kwargs): + assert lane == "cli-auto" return ( - _route("codex", "high") if lane == "cli-auto" else _route("claude", "high") + _route("claude", "high") + if kwargs.get("exclude_provider") == "codex" + else _route("codex", "high") ) monkeypatch.setattr(lanes, "select_route", select_route) @@ -286,9 +380,12 @@ def test_double_capacity_failure_blocks_transient_with_both_reasons( board = _lane_board(tmp_path, task, comments, calls) health_paths = _isolate_lane(tmp_path, monkeypatch, board) - def select_route(_prompt, lane, **_kwargs): + def select_route(_prompt, lane, **kwargs): + assert lane == "cli-auto" return ( - _route("codex", "high") if lane == "cli-auto" else _route("claude", "high") + _route("claude", "high") + if kwargs.get("exclude_provider") == "codex" + else _route("codex", "high") ) monkeypatch.setattr(lanes, "select_route", select_route) diff --git a/testing/tests/test_hermes_cli_review_lane.py b/testing/tests/test_hermes_cli_review_lane.py index 69e84a16..133c5337 100644 --- a/testing/tests/test_hermes_cli_review_lane.py +++ b/testing/tests/test_hermes_cli_review_lane.py @@ -182,6 +182,10 @@ class _Lane: def _select_route(self, _prompt, assignee, **kwargs): self.routes.append((assignee, kwargs)) + if assignee == "cli-auto": + if kwargs.get("exclude_provider") == "claude": + return self._route("codex") + return self._route("claude") if assignee.startswith("cli-codex"): return self._route("codex") return self._route("claude") diff --git a/testing/tests/test_hermes_cli_router_outage.py b/testing/tests/test_hermes_cli_router_outage.py index b2d7b01d..0c4e46a6 100644 --- a/testing/tests/test_hermes_cli_router_outage.py +++ b/testing/tests/test_hermes_cli_router_outage.py @@ -74,9 +74,9 @@ def test_router_outage_during_fallback_selection_blocks_transient( _isolate_lane(tmp_path, monkeypatch, board) selections: list = [] - def select_route(_prompt, lane, **_kwargs): + def select_route(_prompt, lane, **kwargs): selections.append(lane) - if lane == "cli-auto": + if lane == "cli-auto" and not kwargs.get("exclude_provider"): return _route("codex", "medium") raise RuntimeError("Switchyard worker routing failed: refused") @@ -89,7 +89,9 @@ def test_router_outage_during_fallback_selection_blocks_transient( lanes.execute_claim("cassandra", "t_router_fb") - assert selections == ["cli-auto", "cli-claude-medium"] + # The automatic retry re-classifies via Switchyard (same "cli-auto" lane) + # with an explicit failed-provider exclusion, not a hardcoded manual lane. + assert selections == ["cli-auto", "cli-auto"] kind, kwargs = calls[-1] assert kind == "block" and kwargs["kind"] == "transient" assert "Switchyard route selection is unavailable" in kwargs["reason"]