fix(hermes): expose actual routed provider
This commit is contained in:
jenkins
parent
026657fdca
commit
320927bd3f
@ -24,7 +24,7 @@ spec:
|
||||
ai.bstein.dev/execution: Herdr-supervised Codex and Claude Code
|
||||
ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback
|
||||
ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available
|
||||
ai.bstein.dev/config-rev: "20260809-jetson-auto-router"
|
||||
ai.bstein.dev/config-rev: "20260809-route-outcomes"
|
||||
vault.hashicorp.com/agent-inject: "true"
|
||||
vault.hashicorp.com/role: hermes-agent
|
||||
vault.hashicorp.com/agent-inject-secret-anthropic-token: kv/data/atlas/hermes/agent-tokens
|
||||
@ -511,7 +511,7 @@ spec:
|
||||
type: RuntimeDefault
|
||||
volumeMounts:
|
||||
- {name: home, mountPath: /opt/data}
|
||||
- {name: provider-auth, mountPath: /shared-auth, readOnly: true}
|
||||
- {name: provider-auth, mountPath: /shared-auth}
|
||||
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
|
||||
- {name: auth-patch, mountPath: /opt/hermes/hermes_cli/auth.py, subPath: auth.py}
|
||||
- {name: coordinator, mountPath: /opt/data/home/.local/bin/herdr-dispatch, subPath: herdr_dispatch.py, readOnly: true}
|
||||
|
||||
@ -392,6 +392,55 @@ def _record_plan(policy: dict[str, Any], plan: dict[str, Any]) -> None:
|
||||
_write_policy(policy)
|
||||
|
||||
|
||||
def _runtime_agent(ctx: Any) -> Any | None:
|
||||
"""Return the active agent without assuming a single CLI lifecycle."""
|
||||
cli = getattr(ctx._manager, "_cli_ref", None)
|
||||
return getattr(cli, "agent", None) if cli is not None else None
|
||||
|
||||
|
||||
def _post_turn_route(ctx: Any, **kwargs: Any) -> None:
|
||||
"""Persist and surface the provider/model that completed the routed turn."""
|
||||
policy = _current_policy()
|
||||
last = policy.get("last_decision")
|
||||
if not isinstance(last, dict) or not last:
|
||||
return
|
||||
|
||||
agent = _runtime_agent(ctx)
|
||||
actual_provider = str(getattr(agent, "provider", "") or "")
|
||||
actual_model = str(
|
||||
kwargs.get("model") or getattr(agent, "model", "") or ""
|
||||
)
|
||||
if not actual_provider or not actual_model:
|
||||
return
|
||||
|
||||
target_provider = str(last.get("provider") or "")
|
||||
target_model = str(last.get("model") or "")
|
||||
fallback_used = (
|
||||
actual_provider != target_provider or actual_model != target_model
|
||||
)
|
||||
last.update(
|
||||
{
|
||||
"actual_provider": actual_provider,
|
||||
"actual_model": actual_model,
|
||||
"fallback_used": fallback_used,
|
||||
"completed_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
)
|
||||
policy["last_decision"] = last
|
||||
_write_policy(policy)
|
||||
|
||||
emit = getattr(agent, "_emit_status", None)
|
||||
if not callable(emit):
|
||||
return
|
||||
if fallback_used:
|
||||
emit(
|
||||
f"FALLBACK USED → {actual_provider}/{actual_model} · requested "
|
||||
f"{target_provider}/{target_model}"
|
||||
)
|
||||
else:
|
||||
emit(f"ROUTE USED → {actual_provider}/{actual_model}")
|
||||
|
||||
|
||||
def _pre_turn_route(ctx: Any, **kwargs: Any) -> None:
|
||||
"""Apply the persistent AUTO or manual route before prompt construction."""
|
||||
policy = _current_policy()
|
||||
@ -423,14 +472,14 @@ def _pre_turn_route(ctx: Any, **kwargs: Any) -> None:
|
||||
if callable(emit):
|
||||
if plan["classifier"] == "manual":
|
||||
emit(
|
||||
f"MANUAL → {plan['provider']}/{plan['model']} · "
|
||||
f"{plan['effort']}"
|
||||
f"MANUAL target → {plan['provider']}/{plan['model']} · "
|
||||
f"{plan['effort']} · automatic capacity fallback remains enabled"
|
||||
)
|
||||
else:
|
||||
source = "Jetson" if plan["classifier"] == "jetson" else "fast fallback"
|
||||
emit(
|
||||
f"AUTO → {plan['provider']}/{plan['model']} · {plan['effort']} "
|
||||
f"({source})"
|
||||
f"AUTO target → {plan['provider']}/{plan['model']} · "
|
||||
f"{plan['effort']} ({source}) · automatic capacity fallback enabled"
|
||||
)
|
||||
|
||||
|
||||
@ -443,15 +492,24 @@ def _status_text(ctx: Any) -> str:
|
||||
current = f"{cli.provider}/{cli.model} at {effort}"
|
||||
last = policy.get("last_decision") or {}
|
||||
last_text = "none yet"
|
||||
outcome_text = "none yet"
|
||||
if last:
|
||||
last_text = (
|
||||
f"{last.get('provider')}/{last.get('model')} at {last.get('effort')} "
|
||||
f"via {last.get('classifier')}"
|
||||
)
|
||||
actual_provider = last.get("actual_provider")
|
||||
actual_model = last.get("actual_model")
|
||||
if actual_provider and actual_model:
|
||||
prefix = "fallback" if last.get("fallback_used") else "target completed"
|
||||
outcome_text = f"{prefix}: {actual_provider}/{actual_model}"
|
||||
else:
|
||||
outcome_text = "pending"
|
||||
return (
|
||||
f"Route mode: {policy['mode'].upper()}\n"
|
||||
f"Current runtime: {current}\n"
|
||||
f"Last routed turn: {last_text}\n"
|
||||
f"Last requested route: {last_text}\n"
|
||||
f"Last actual outcome: {outcome_text}\n"
|
||||
"Commands: /route auto | /route manual <codex|claude> "
|
||||
"<low|medium|high|xhigh> [model] | /route status"
|
||||
)
|
||||
@ -499,6 +557,7 @@ def _route_command(ctx: Any, raw_args: str) -> str:
|
||||
def register(ctx: Any) -> None:
|
||||
"""Register the pre-turn router and its explicit override command."""
|
||||
ctx.register_hook("pre_turn_route", lambda **kwargs: _pre_turn_route(ctx, **kwargs))
|
||||
ctx.register_hook("post_llm_call", lambda **kwargs: _post_turn_route(ctx, **kwargs))
|
||||
ctx.register_command(
|
||||
"route",
|
||||
lambda raw_args: _route_command(ctx, raw_args),
|
||||
|
||||
@ -136,4 +136,65 @@ def test_manual_policy_is_reapplied_on_every_non_command_turn(monkeypatch):
|
||||
|
||||
assert plans[0]["profile"] == "claude-medium"
|
||||
assert plans[0]["model"] == "claude-sonnet-5"
|
||||
assert agent.message.startswith("MANUAL")
|
||||
assert agent.message.startswith("MANUAL target")
|
||||
|
||||
|
||||
def test_post_turn_records_and_announces_capacity_fallback(monkeypatch):
|
||||
policy = {
|
||||
"mode": "auto",
|
||||
"last_decision": {
|
||||
"provider": "anthropic",
|
||||
"model": "claude-sonnet-5",
|
||||
"effort": "medium",
|
||||
"classifier": "jetson",
|
||||
},
|
||||
}
|
||||
written = []
|
||||
monkeypatch.setattr(router, "_current_policy", lambda: policy)
|
||||
monkeypatch.setattr(router, "_write_policy", lambda value: written.append(value))
|
||||
|
||||
class Agent:
|
||||
provider = "openai-codex"
|
||||
model = "gpt-5.6-terra"
|
||||
|
||||
def _emit_status(self, message):
|
||||
self.message = message
|
||||
|
||||
agent = Agent()
|
||||
cli = type("CLI", (), {"agent": agent})()
|
||||
manager = type("Manager", (), {"_cli_ref": cli})()
|
||||
ctx = type("Context", (), {"_manager": manager})()
|
||||
|
||||
router._post_turn_route(ctx, model="gpt-5.6-terra")
|
||||
|
||||
outcome = written[-1]["last_decision"]
|
||||
assert outcome["fallback_used"] is True
|
||||
assert outcome["actual_provider"] == "openai-codex"
|
||||
assert outcome["actual_model"] == "gpt-5.6-terra"
|
||||
assert agent.message.startswith("FALLBACK USED")
|
||||
|
||||
|
||||
def test_status_distinguishes_requested_route_from_actual_outcome(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
router,
|
||||
"_current_policy",
|
||||
lambda: {
|
||||
"mode": "auto",
|
||||
"last_decision": {
|
||||
"provider": "anthropic",
|
||||
"model": "claude-sonnet-5",
|
||||
"effort": "medium",
|
||||
"classifier": "jetson",
|
||||
"actual_provider": "openai-codex",
|
||||
"actual_model": "gpt-5.6-terra",
|
||||
"fallback_used": True,
|
||||
},
|
||||
},
|
||||
)
|
||||
manager = type("Manager", (), {"_cli_ref": None})()
|
||||
ctx = type("Context", (), {"_manager": manager})()
|
||||
|
||||
status = router._status_text(ctx)
|
||||
|
||||
assert "Last requested route: anthropic/claude-sonnet-5" in status
|
||||
assert "Last actual outcome: fallback: openai-codex/gpt-5.6-terra" in status
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user