"""Regression tests for durable Hermes worker-route model resolution.""" from __future__ import annotations import importlib.util import json import sys import pytest from pathlib import Path SCRIPTS = Path(__file__).parents[2] / "services/hermes/scripts" sys.path.insert(0, str(SCRIPTS)) SCRIPT = SCRIPTS / "cli_lane_runner.py" SPEC = importlib.util.spec_from_file_location("cli_lane_runner_routing_test", SCRIPT) assert SPEC and SPEC.loader lanes = importlib.util.module_from_spec(SPEC) sys.modules[SPEC.name] = lanes SPEC.loader.exec_module(lanes) class SwitchyardResponse: """Return a tier header and independently resolved broker content.""" def __init__( self, selected: str, resolved_target: str, rationale: str = "test route" ): self.headers = { "x-model-router-selected-model": selected, "x-model-router-rationale": rationale, } self.resolved_target = resolved_target def __enter__(self): return self def __exit__(self, *_args): return False def read(self) -> bytes: return json.dumps( { "model": self.headers["x-model-router-selected-model"], "choices": [ {"message": {"content": self.resolved_target}} ], } ).encode() def test_switchyard_tier_is_resolved_before_invoking_codex(): """Codex receives the exact account-visible ID, not a tier shorthand.""" route = lanes.select_route( "Repair the failing implementation.", "cli-auto", open_request=lambda *_args, **_kwargs: SwitchyardResponse( "worker/codex/terra/xhigh", "worker/codex/gpt-5.6-terra/xhigh", ), ) assert (route.provider, route.model, route.effort) == ( "codex", "gpt-5.6-terra", "xhigh", ) def test_mismatched_broker_resolution_cannot_change_route_decision(): """Resolved content cannot change Switchyard's provider or effort.""" route = lanes.select_route( "Repair it.", "cli-auto", open_request=lambda *_args, **_kwargs: SwitchyardResponse( "worker/codex/terra/xhigh", "worker/claude/claude-opus-5/high", ), ) assert (route.provider, route.model, route.effort) == ( "codex", "terra", "xhigh", ) def test_generic_claude_worker_keeps_a_catalog_advertised_alias(): """The native CLI receives its provider alias after generic AUTO routing.""" route = lanes.select_route( "Review it.", "cli-auto", open_request=lambda *_args, **_kwargs: SwitchyardResponse( "worker/claude/auto/medium", "worker/claude/sonnet/medium" ), catalog_model_allowed=lambda provider, model: provider == "claude" and model == "sonnet", ) assert (route.provider, route.model, route.effort) == ("claude", "sonnet", "medium") def test_generic_claude_worker_rejects_an_unadvertised_alias(): """A forged worker receipt cannot pass an arbitrary native Claude alias.""" with pytest.raises(RuntimeError, match="current provider model"): lanes.select_route( "Review it.", "cli-auto", open_request=lambda *_args, **_kwargs: SwitchyardResponse( "worker/claude/auto/medium", "worker/claude/not-advertised/medium" ), catalog_model_allowed=lambda *_args: False, ) @pytest.mark.parametrize( ("effort", "capability"), [("low", "economy"), ("medium", "balanced"), ("high", "advanced"), ("xhigh", "advanced")], ) def test_legacy_auto_derives_capability_from_its_effort(effort: str, capability: str): """Legacy AUTO keeps its documented effort-to-capability compatibility map.""" route = lanes.select_route( "Legacy worker route.", "cli-auto", open_request=lambda *_args, **_kwargs: SwitchyardResponse( f"worker/codex/auto/{effort}", f"worker/codex/gpt-5.6-sol/{effort}" ), ) assert route.capability == capability def test_unresolved_capability_selector_never_reaches_native_cli(): """A broker receipt must resolve every `auto-*` selector to a real model.""" with pytest.raises(RuntimeError, match="current provider model"): lanes.select_route( "Exceptional ambiguity.", "cli-auto", open_request=lambda *_args, **_kwargs: SwitchyardResponse( "worker/codex/auto-frontier/high", "worker/codex/auto-frontier/high" ), ) def test_health_guard_preserves_capability_and_effort_on_alternate_provider(): """An excluded provider cannot erase the selected frontier floor.""" calls = [] def request(request, **_kwargs): calls.append(json.loads(request.data)) if len(calls) == 1: return SwitchyardResponse( "worker/codex/auto-frontier/high", "worker/codex/gpt-6-astra/high" ) return SwitchyardResponse( "worker/claude/auto-frontier/high", "worker/claude/claude-fable-5/high" ) route = lanes.select_route( "Exceptional coupled architecture.", "cli-auto", exclude_provider="codex", open_request=request, ) assert calls[1]["model"] == "atlas/worker/manual/claude/frontier/high" assert (route.provider, route.capability, route.effort) == ("claude", "frontier", "high") def test_health_guard_rejects_a_capability_downgrade(): """A manually guarded retry must satisfy the original capability floor.""" responses = iter( [ SwitchyardResponse( "worker/codex/auto-frontier/high", "worker/codex/gpt-6-astra/high" ), SwitchyardResponse( "worker/claude/auto-advanced/high", "worker/claude/claude-opus-5/high" ), ] ) with pytest.raises(RuntimeError, match="preserve the requested frontier/high"): lanes.select_route( "Exceptional coupled architecture.", "cli-auto", exclude_provider="codex", open_request=lambda *_args, **_kwargs: next(responses), ) def test_switchyard_request_fallback_reselects_the_original_floor(): """A frontier timeout cannot return Sol merely because both use xhigh.""" calls = [] def request(request, **_kwargs): calls.append(json.loads(request.data)) if len(calls) == 1: return SwitchyardResponse( "worker/codex/auto-advanced/xhigh", "worker/codex/gpt-5.6-sol/xhigh", ( "worker/codex/auto-frontier/xhigh was unavailable; fell back to " "worker/codex/auto-advanced/xhigh" ), ) return SwitchyardResponse( "worker/claude/auto-frontier/xhigh", "worker/claude/claude-astra-6/xhigh", ) route = lanes.select_route( "Resolve exceptional coupled architecture ambiguity.", "cli-auto", open_request=request, catalog_model_allowed=lambda provider, model: ( provider == "claude" and model == "claude-astra-6" ), ) assert calls[1]["model"] == "atlas/worker/manual/claude/frontier/xhigh" assert (route.provider, route.capability, route.effort) == ( "claude", "frontier", "xhigh" ) assert route.classifier == "switchyard-classifier-fallback-floor-guard" def test_worker_route_keeps_sol_xhigh_distinct_from_astra_high(): """Capability arrives from the selector rather than being inferred from effort.""" advanced = lanes.select_route( "Ordinary complex implementation.", "cli-auto", open_request=lambda *_args, **_kwargs: SwitchyardResponse( "worker/codex/auto-advanced/xhigh", "worker/codex/gpt-5.6-sol/xhigh", ), ) frontier = lanes.select_route( "Resolve exceptional coupled architecture ambiguity.", "cli-auto", open_request=lambda *_args, **_kwargs: SwitchyardResponse( "worker/codex/auto-frontier/high", "worker/codex/gpt-6-astra/high", ), ) assert (advanced.model, advanced.capability, advanced.effort) == ( "gpt-5.6-sol", "advanced", "xhigh" ) assert (frontier.model, frontier.capability, frontier.effort) == ( "gpt-6-astra", "frontier", "high" ) def test_capability_manual_override_uses_exact_switchyard_route(): """A manual capability floor is represented separately from reasoning effort.""" captured = [] def request(request, **_kwargs): captured.append(json.loads(request.data)) return SwitchyardResponse( "worker/codex/auto-frontier/high", "worker/codex/gpt-6-astra/high", ) route = lanes.select_route("Review the coupled architecture.", "cli-codex-frontier-high", open_request=request) assert captured[0]["model"] == "atlas/worker/manual/codex/frontier/high" assert route.capability == "frontier"