267 lines
8.9 KiB
Python
267 lines
8.9 KiB
Python
"""Regression tests for durable Hermes worker-route model resolution."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import json
|
|
import sys
|
|
|
|
import pytest
|
|
from pathlib import Path
|
|
|
|
|
|
SCRIPTS = Path(__file__).parents[2] / "services/hermes/scripts"
|
|
sys.path.insert(0, str(SCRIPTS))
|
|
SCRIPT = SCRIPTS / "cli_lane_runner.py"
|
|
SPEC = importlib.util.spec_from_file_location("cli_lane_runner_routing_test", SCRIPT)
|
|
assert SPEC and SPEC.loader
|
|
lanes = importlib.util.module_from_spec(SPEC)
|
|
sys.modules[SPEC.name] = lanes
|
|
SPEC.loader.exec_module(lanes)
|
|
|
|
|
|
class SwitchyardResponse:
|
|
"""Return a tier header and independently resolved broker content."""
|
|
|
|
def __init__(
|
|
self, selected: str, resolved_target: str, rationale: str = "test route"
|
|
):
|
|
self.headers = {
|
|
"x-model-router-selected-model": selected,
|
|
"x-model-router-rationale": rationale,
|
|
}
|
|
self.resolved_target = resolved_target
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *_args):
|
|
return False
|
|
|
|
def read(self) -> bytes:
|
|
return json.dumps(
|
|
{
|
|
"model": self.headers["x-model-router-selected-model"],
|
|
"choices": [
|
|
{"message": {"content": self.resolved_target}}
|
|
],
|
|
}
|
|
).encode()
|
|
|
|
|
|
def test_switchyard_tier_is_resolved_before_invoking_codex():
|
|
"""Codex receives the exact account-visible ID, not a tier shorthand."""
|
|
route = lanes.select_route(
|
|
"Repair the failing implementation.",
|
|
"cli-auto",
|
|
open_request=lambda *_args, **_kwargs: SwitchyardResponse(
|
|
"worker/codex/terra/xhigh",
|
|
"worker/codex/gpt-5.6-terra/xhigh",
|
|
),
|
|
)
|
|
|
|
assert (route.provider, route.model, route.effort) == (
|
|
"codex",
|
|
"gpt-5.6-terra",
|
|
"xhigh",
|
|
)
|
|
|
|
|
|
def test_mismatched_broker_resolution_cannot_change_route_decision():
|
|
"""Resolved content cannot change Switchyard's provider or effort."""
|
|
route = lanes.select_route(
|
|
"Repair it.",
|
|
"cli-auto",
|
|
open_request=lambda *_args, **_kwargs: SwitchyardResponse(
|
|
"worker/codex/terra/xhigh",
|
|
"worker/claude/claude-opus-5/high",
|
|
),
|
|
)
|
|
|
|
assert (route.provider, route.model, route.effort) == (
|
|
"codex",
|
|
"terra",
|
|
"xhigh",
|
|
)
|
|
|
|
|
|
def test_generic_claude_worker_keeps_a_catalog_advertised_alias():
|
|
"""The native CLI receives its provider alias after generic AUTO routing."""
|
|
route = lanes.select_route(
|
|
"Review it.",
|
|
"cli-auto",
|
|
open_request=lambda *_args, **_kwargs: SwitchyardResponse(
|
|
"worker/claude/auto/medium", "worker/claude/sonnet/medium"
|
|
),
|
|
catalog_model_allowed=lambda provider, model: provider == "claude" and model == "sonnet",
|
|
)
|
|
assert (route.provider, route.model, route.effort) == ("claude", "sonnet", "medium")
|
|
|
|
|
|
def test_generic_claude_worker_rejects_an_unadvertised_alias():
|
|
"""A forged worker receipt cannot pass an arbitrary native Claude alias."""
|
|
with pytest.raises(RuntimeError, match="current provider model"):
|
|
lanes.select_route(
|
|
"Review it.",
|
|
"cli-auto",
|
|
open_request=lambda *_args, **_kwargs: SwitchyardResponse(
|
|
"worker/claude/auto/medium", "worker/claude/not-advertised/medium"
|
|
),
|
|
catalog_model_allowed=lambda *_args: False,
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("effort", "capability"),
|
|
[("low", "economy"), ("medium", "balanced"), ("high", "advanced"), ("xhigh", "advanced")],
|
|
)
|
|
def test_legacy_auto_derives_capability_from_its_effort(effort: str, capability: str):
|
|
"""Legacy AUTO keeps its documented effort-to-capability compatibility map."""
|
|
route = lanes.select_route(
|
|
"Legacy worker route.",
|
|
"cli-auto",
|
|
open_request=lambda *_args, **_kwargs: SwitchyardResponse(
|
|
f"worker/codex/auto/{effort}", f"worker/codex/gpt-5.6-sol/{effort}"
|
|
),
|
|
)
|
|
|
|
assert route.capability == capability
|
|
|
|
|
|
def test_unresolved_capability_selector_never_reaches_native_cli():
|
|
"""A broker receipt must resolve every `auto-*` selector to a real model."""
|
|
with pytest.raises(RuntimeError, match="current provider model"):
|
|
lanes.select_route(
|
|
"Exceptional ambiguity.",
|
|
"cli-auto",
|
|
open_request=lambda *_args, **_kwargs: SwitchyardResponse(
|
|
"worker/codex/auto-frontier/high", "worker/codex/auto-frontier/high"
|
|
),
|
|
)
|
|
|
|
|
|
def test_health_guard_preserves_capability_and_effort_on_alternate_provider():
|
|
"""An excluded provider cannot erase the selected frontier floor."""
|
|
calls = []
|
|
|
|
def request(request, **_kwargs):
|
|
calls.append(json.loads(request.data))
|
|
if len(calls) == 1:
|
|
return SwitchyardResponse(
|
|
"worker/codex/auto-frontier/high", "worker/codex/gpt-6-astra/high"
|
|
)
|
|
return SwitchyardResponse(
|
|
"worker/claude/auto-frontier/high", "worker/claude/claude-fable-5/high"
|
|
)
|
|
|
|
route = lanes.select_route(
|
|
"Exceptional coupled architecture.",
|
|
"cli-auto",
|
|
exclude_provider="codex",
|
|
open_request=request,
|
|
)
|
|
|
|
assert calls[1]["model"] == "atlas/worker/manual/claude/frontier/high"
|
|
assert (route.provider, route.capability, route.effort) == ("claude", "frontier", "high")
|
|
|
|
|
|
def test_health_guard_rejects_a_capability_downgrade():
|
|
"""A manually guarded retry must satisfy the original capability floor."""
|
|
responses = iter(
|
|
[
|
|
SwitchyardResponse(
|
|
"worker/codex/auto-frontier/high", "worker/codex/gpt-6-astra/high"
|
|
),
|
|
SwitchyardResponse(
|
|
"worker/claude/auto-advanced/high", "worker/claude/claude-opus-5/high"
|
|
),
|
|
]
|
|
)
|
|
with pytest.raises(RuntimeError, match="preserve the requested frontier/high"):
|
|
lanes.select_route(
|
|
"Exceptional coupled architecture.",
|
|
"cli-auto",
|
|
exclude_provider="codex",
|
|
open_request=lambda *_args, **_kwargs: next(responses),
|
|
)
|
|
|
|
|
|
def test_switchyard_request_fallback_reselects_the_original_floor():
|
|
"""A frontier timeout cannot return Sol merely because both use xhigh."""
|
|
calls = []
|
|
|
|
def request(request, **_kwargs):
|
|
calls.append(json.loads(request.data))
|
|
if len(calls) == 1:
|
|
return SwitchyardResponse(
|
|
"worker/codex/auto-advanced/xhigh",
|
|
"worker/codex/gpt-5.6-sol/xhigh",
|
|
(
|
|
"worker/codex/auto-frontier/xhigh was unavailable; fell back to "
|
|
"worker/codex/auto-advanced/xhigh"
|
|
),
|
|
)
|
|
return SwitchyardResponse(
|
|
"worker/claude/auto-frontier/xhigh",
|
|
"worker/claude/claude-astra-6/xhigh",
|
|
)
|
|
|
|
route = lanes.select_route(
|
|
"Resolve exceptional coupled architecture ambiguity.",
|
|
"cli-auto",
|
|
open_request=request,
|
|
catalog_model_allowed=lambda provider, model: (
|
|
provider == "claude" and model == "claude-astra-6"
|
|
),
|
|
)
|
|
|
|
assert calls[1]["model"] == "atlas/worker/manual/claude/frontier/xhigh"
|
|
assert (route.provider, route.capability, route.effort) == (
|
|
"claude", "frontier", "xhigh"
|
|
)
|
|
assert route.classifier == "switchyard-classifier-fallback-floor-guard"
|
|
|
|
|
|
def test_worker_route_keeps_sol_xhigh_distinct_from_astra_high():
|
|
"""Capability arrives from the selector rather than being inferred from effort."""
|
|
advanced = lanes.select_route(
|
|
"Ordinary complex implementation.",
|
|
"cli-auto",
|
|
open_request=lambda *_args, **_kwargs: SwitchyardResponse(
|
|
"worker/codex/auto-advanced/xhigh",
|
|
"worker/codex/gpt-5.6-sol/xhigh",
|
|
),
|
|
)
|
|
frontier = lanes.select_route(
|
|
"Resolve exceptional coupled architecture ambiguity.",
|
|
"cli-auto",
|
|
open_request=lambda *_args, **_kwargs: SwitchyardResponse(
|
|
"worker/codex/auto-frontier/high",
|
|
"worker/codex/gpt-6-astra/high",
|
|
),
|
|
)
|
|
|
|
assert (advanced.model, advanced.capability, advanced.effort) == (
|
|
"gpt-5.6-sol", "advanced", "xhigh"
|
|
)
|
|
assert (frontier.model, frontier.capability, frontier.effort) == (
|
|
"gpt-6-astra", "frontier", "high"
|
|
)
|
|
|
|
|
|
def test_capability_manual_override_uses_exact_switchyard_route():
|
|
"""A manual capability floor is represented separately from reasoning effort."""
|
|
captured = []
|
|
|
|
def request(request, **_kwargs):
|
|
captured.append(json.loads(request.data))
|
|
return SwitchyardResponse(
|
|
"worker/codex/auto-frontier/high",
|
|
"worker/codex/gpt-6-astra/high",
|
|
)
|
|
|
|
route = lanes.select_route("Review the coupled architecture.", "cli-codex-frontier-high", open_request=request)
|
|
|
|
assert captured[0]["model"] == "atlas/worker/manual/codex/frontier/high"
|
|
assert route.capability == "frontier"
|