72 lines
2.3 KiB
Python
72 lines
2.3 KiB
Python
|
"""Provider-neutral fallback contracts for Hermes automatic routing."""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import tomllib
|
||
|
|
|
||
|
|
from testing.tests.test_hermes_chat_support import HERMES, _documents
|
||
|
|
|
||
|
|
|
||
|
|
def _config() -> dict:
|
||
|
|
manifest = _documents(HERMES / "switchyard-configmap.yaml")[0]
|
||
|
|
return tomllib.loads(manifest["data"]["routes.toml"])
|
||
|
|
|
||
|
|
|
||
|
|
def _provider(target_id: str) -> str | None:
|
||
|
|
parts = target_id.split("/")
|
||
|
|
if len(parts) >= 3 and parts[0] in {"route", "worker"}:
|
||
|
|
return parts[1]
|
||
|
|
return None
|
||
|
|
|
||
|
|
|
||
|
|
def test_every_classifier_selects_from_both_hosted_providers():
|
||
|
|
"""No automatic classifier may narrow its candidate set to one provider."""
|
||
|
|
config = _config()
|
||
|
|
targets = config["targets"]
|
||
|
|
classifiers = {
|
||
|
|
name: route
|
||
|
|
for name, route in config["routes"].items()
|
||
|
|
if route["type"] == "llm_classifier"
|
||
|
|
}
|
||
|
|
|
||
|
|
assert classifiers
|
||
|
|
for route in classifiers.values():
|
||
|
|
providers = {
|
||
|
|
_provider(targets[name]["id"])
|
||
|
|
for name in route["targets"]
|
||
|
|
}
|
||
|
|
assert {"codex", "claude"} <= providers
|
||
|
|
|
||
|
|
|
||
|
|
def test_classifier_failure_uses_an_even_cross_provider_pool():
|
||
|
|
"""The required Switchyard default must name a neutral pool, not a provider."""
|
||
|
|
config = _config()
|
||
|
|
targets = config["targets"]
|
||
|
|
routes = config["routes"]
|
||
|
|
routes_by_id = {route["id"]: route for route in routes.values()}
|
||
|
|
|
||
|
|
for route in routes.values():
|
||
|
|
if route["type"] != "llm_classifier":
|
||
|
|
continue
|
||
|
|
fallback_target_name = route["default_target"]
|
||
|
|
assert fallback_target_name in route["targets"]
|
||
|
|
assert fallback_target_name.startswith("neutral_")
|
||
|
|
fallback_target = targets[fallback_target_name]
|
||
|
|
assert fallback_target["llm_client"] == "neutral_pool"
|
||
|
|
|
||
|
|
pool = routes_by_id[fallback_target["id"]]
|
||
|
|
assert pool["type"] == "random"
|
||
|
|
assert pool["weights"] == [1.0, 1.0]
|
||
|
|
pool_providers = [
|
||
|
|
_provider(targets[name]["id"])
|
||
|
|
for name in pool["targets"]
|
||
|
|
]
|
||
|
|
assert sorted(pool_providers) == ["claude", "codex"]
|
||
|
|
|
||
|
|
neutral_client = config["llm_clients"]["neutral_pool"]
|
||
|
|
assert neutral_client == {
|
||
|
|
"format": "openai_chat",
|
||
|
|
"base_url": "http://127.0.0.1:9005/v1",
|
||
|
|
"max_retries": 0,
|
||
|
|
}
|