From 320927bd3f0479bcd9af54ad9ffaa459f9ab91f1 Mon Sep 17 00:00:00 2001 From: jenkins Date: Sun, 9 Aug 2026 03:51:03 -0300 Subject: [PATCH] fix(hermes): expose actual routed provider --- services/hermes/agent-deployment.yaml | 4 +- .../hermes/plugins/auto-router/__init__.py | 69 +++++++++++++++++-- testing/tests/test_hermes_auto_router.py | 63 ++++++++++++++++- 3 files changed, 128 insertions(+), 8 deletions(-) diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml index a306a16b7..0070f4606 100644 --- a/services/hermes/agent-deployment.yaml +++ b/services/hermes/agent-deployment.yaml @@ -24,7 +24,7 @@ spec: ai.bstein.dev/execution: Herdr-supervised Codex and Claude Code ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available - ai.bstein.dev/config-rev: "20260809-jetson-auto-router" + ai.bstein.dev/config-rev: "20260809-route-outcomes" vault.hashicorp.com/agent-inject: "true" vault.hashicorp.com/role: hermes-agent vault.hashicorp.com/agent-inject-secret-anthropic-token: kv/data/atlas/hermes/agent-tokens @@ -511,7 +511,7 @@ spec: type: RuntimeDefault volumeMounts: - {name: home, mountPath: /opt/data} - - {name: provider-auth, mountPath: /shared-auth, readOnly: true} + - {name: provider-auth, mountPath: /shared-auth} - {name: coordinator, mountPath: /opt/coordinator, readOnly: true} - {name: auth-patch, mountPath: /opt/hermes/hermes_cli/auth.py, subPath: auth.py} - {name: coordinator, mountPath: /opt/data/home/.local/bin/herdr-dispatch, subPath: herdr_dispatch.py, readOnly: true} diff --git a/services/hermes/plugins/auto-router/__init__.py b/services/hermes/plugins/auto-router/__init__.py index 85f207aa2..b0361c221 100644 --- a/services/hermes/plugins/auto-router/__init__.py +++ b/services/hermes/plugins/auto-router/__init__.py @@ -392,6 +392,55 @@ def _record_plan(policy: dict[str, Any], plan: dict[str, Any]) -> None: _write_policy(policy) +def _runtime_agent(ctx: Any) -> Any | None: + """Return the active agent without assuming a single CLI lifecycle.""" + cli = getattr(ctx._manager, "_cli_ref", None) + return getattr(cli, "agent", None) if cli is not None else None + + +def _post_turn_route(ctx: Any, **kwargs: Any) -> None: + """Persist and surface the provider/model that completed the routed turn.""" + policy = _current_policy() + last = policy.get("last_decision") + if not isinstance(last, dict) or not last: + return + + agent = _runtime_agent(ctx) + actual_provider = str(getattr(agent, "provider", "") or "") + actual_model = str( + kwargs.get("model") or getattr(agent, "model", "") or "" + ) + if not actual_provider or not actual_model: + return + + target_provider = str(last.get("provider") or "") + target_model = str(last.get("model") or "") + fallback_used = ( + actual_provider != target_provider or actual_model != target_model + ) + last.update( + { + "actual_provider": actual_provider, + "actual_model": actual_model, + "fallback_used": fallback_used, + "completed_at": datetime.now(timezone.utc).isoformat(), + } + ) + policy["last_decision"] = last + _write_policy(policy) + + emit = getattr(agent, "_emit_status", None) + if not callable(emit): + return + if fallback_used: + emit( + f"FALLBACK USED → {actual_provider}/{actual_model} · requested " + f"{target_provider}/{target_model}" + ) + else: + emit(f"ROUTE USED → {actual_provider}/{actual_model}") + + def _pre_turn_route(ctx: Any, **kwargs: Any) -> None: """Apply the persistent AUTO or manual route before prompt construction.""" policy = _current_policy() @@ -423,14 +472,14 @@ def _pre_turn_route(ctx: Any, **kwargs: Any) -> None: if callable(emit): if plan["classifier"] == "manual": emit( - f"MANUAL → {plan['provider']}/{plan['model']} · " - f"{plan['effort']}" + f"MANUAL target → {plan['provider']}/{plan['model']} · " + f"{plan['effort']} · automatic capacity fallback remains enabled" ) else: source = "Jetson" if plan["classifier"] == "jetson" else "fast fallback" emit( - f"AUTO → {plan['provider']}/{plan['model']} · {plan['effort']} " - f"({source})" + f"AUTO target → {plan['provider']}/{plan['model']} · " + f"{plan['effort']} ({source}) · automatic capacity fallback enabled" ) @@ -443,15 +492,24 @@ def _status_text(ctx: Any) -> str: current = f"{cli.provider}/{cli.model} at {effort}" last = policy.get("last_decision") or {} last_text = "none yet" + outcome_text = "none yet" if last: last_text = ( f"{last.get('provider')}/{last.get('model')} at {last.get('effort')} " f"via {last.get('classifier')}" ) + actual_provider = last.get("actual_provider") + actual_model = last.get("actual_model") + if actual_provider and actual_model: + prefix = "fallback" if last.get("fallback_used") else "target completed" + outcome_text = f"{prefix}: {actual_provider}/{actual_model}" + else: + outcome_text = "pending" return ( f"Route mode: {policy['mode'].upper()}\n" f"Current runtime: {current}\n" - f"Last routed turn: {last_text}\n" + f"Last requested route: {last_text}\n" + f"Last actual outcome: {outcome_text}\n" "Commands: /route auto | /route manual " " [model] | /route status" ) @@ -499,6 +557,7 @@ def _route_command(ctx: Any, raw_args: str) -> str: def register(ctx: Any) -> None: """Register the pre-turn router and its explicit override command.""" ctx.register_hook("pre_turn_route", lambda **kwargs: _pre_turn_route(ctx, **kwargs)) + ctx.register_hook("post_llm_call", lambda **kwargs: _post_turn_route(ctx, **kwargs)) ctx.register_command( "route", lambda raw_args: _route_command(ctx, raw_args), diff --git a/testing/tests/test_hermes_auto_router.py b/testing/tests/test_hermes_auto_router.py index 13a832ef5..df9ff8ac2 100644 --- a/testing/tests/test_hermes_auto_router.py +++ b/testing/tests/test_hermes_auto_router.py @@ -136,4 +136,65 @@ def test_manual_policy_is_reapplied_on_every_non_command_turn(monkeypatch): assert plans[0]["profile"] == "claude-medium" assert plans[0]["model"] == "claude-sonnet-5" - assert agent.message.startswith("MANUAL") + assert agent.message.startswith("MANUAL target") + + +def test_post_turn_records_and_announces_capacity_fallback(monkeypatch): + policy = { + "mode": "auto", + "last_decision": { + "provider": "anthropic", + "model": "claude-sonnet-5", + "effort": "medium", + "classifier": "jetson", + }, + } + written = [] + monkeypatch.setattr(router, "_current_policy", lambda: policy) + monkeypatch.setattr(router, "_write_policy", lambda value: written.append(value)) + + class Agent: + provider = "openai-codex" + model = "gpt-5.6-terra" + + def _emit_status(self, message): + self.message = message + + agent = Agent() + cli = type("CLI", (), {"agent": agent})() + manager = type("Manager", (), {"_cli_ref": cli})() + ctx = type("Context", (), {"_manager": manager})() + + router._post_turn_route(ctx, model="gpt-5.6-terra") + + outcome = written[-1]["last_decision"] + assert outcome["fallback_used"] is True + assert outcome["actual_provider"] == "openai-codex" + assert outcome["actual_model"] == "gpt-5.6-terra" + assert agent.message.startswith("FALLBACK USED") + + +def test_status_distinguishes_requested_route_from_actual_outcome(monkeypatch): + monkeypatch.setattr( + router, + "_current_policy", + lambda: { + "mode": "auto", + "last_decision": { + "provider": "anthropic", + "model": "claude-sonnet-5", + "effort": "medium", + "classifier": "jetson", + "actual_provider": "openai-codex", + "actual_model": "gpt-5.6-terra", + "fallback_used": True, + }, + }, + ) + manager = type("Manager", (), {"_cli_ref": None})() + ctx = type("Context", (), {"_manager": manager})() + + status = router._status_text(ctx) + + assert "Last requested route: anthropic/claude-sonnet-5" in status + assert "Last actual outcome: fallback: openai-codex/gpt-5.6-terra" in status