From 3da6a101740d206e03fc68db3a2da53383f328e3 Mon Sep 17 00:00:00 2001 From: jenkins Date: Sat, 22 Aug 2026 23:54:40 -0300 Subject: [PATCH] hermes: remove provider-biased auto fallbacks --- services/hermes/switchyard-configmap.yaml | 94 +++++++++++++++++-- services/hermes/switchyard-deployment.yaml | 2 +- .../test_hermes_switchyard_provider_pool.py | 71 ++++++++++++++ 3 files changed, 156 insertions(+), 11 deletions(-) create mode 100644 testing/tests/test_hermes_switchyard_provider_pool.py diff --git a/services/hermes/switchyard-configmap.yaml b/services/hermes/switchyard-configmap.yaml index ae3d6436..a1421281 100644 --- a/services/hermes/switchyard-configmap.yaml +++ b/services/hermes/switchyard-configmap.yaml @@ -81,6 +81,11 @@ data: base_url = "http://127.0.0.1:9007/v1" max_retries = 0 + [llm_clients.neutral_pool] + format = "openai_chat" + base_url = "http://127.0.0.1:9005/v1" + max_retries = 0 + [targets.classifier] id = "qwen2.5:14b-instruct-q4_0" llm_client = "classifier" @@ -345,6 +350,75 @@ data: id = "worker/claude/opus/xhigh" llm_client = "worker_decision" + # Switchyard 0.2.0 requires one default_target for every custom classifier. + # These targets recurse once into unbiased random routes whose candidates are + # split evenly across providers. A classifier outage therefore falls back to + # a pool, never directly to either Codex/OpenAI or Claude/Anthropic. + [targets.neutral_fast_pool] + id = "atlas/fallback/fast" + llm_client = "neutral_pool" + + [targets.neutral_balanced_pool] + id = "atlas/fallback/balanced" + llm_client = "neutral_pool" + + [targets.neutral_deep_pool] + id = "atlas/fallback/deep" + llm_client = "neutral_pool" + + [targets.neutral_maximum_pool] + id = "atlas/fallback/maximum" + llm_client = "neutral_pool" + + [targets.neutral_worker_maximum_pool] + id = "atlas/worker/fallback/maximum" + llm_client = "neutral_pool" + + [routes.fallback_fast] + id = "atlas/fallback/fast" + type = "random" + targets = ["codex_terra_medium", "claude_fable_medium"] + weights = [1.0, 1.0] + context_window = 272000 + tool_calling = true + reasoning = true + + [routes.fallback_balanced] + id = "atlas/fallback/balanced" + type = "random" + targets = ["codex_terra_high", "claude_sonnet_high"] + weights = [1.0, 1.0] + context_window = 272000 + tool_calling = true + reasoning = true + + [routes.fallback_deep] + id = "atlas/fallback/deep" + type = "random" + targets = ["codex_sol_high", "claude_opus_high"] + weights = [1.0, 1.0] + context_window = 272000 + tool_calling = true + reasoning = true + + [routes.fallback_maximum] + id = "atlas/fallback/maximum" + type = "random" + targets = ["codex_sol_xhigh", "claude_opus_xhigh"] + weights = [1.0, 1.0] + context_window = 272000 + tool_calling = true + reasoning = true + + [routes.fallback_worker_maximum] + id = "atlas/worker/fallback/maximum" + type = "random" + targets = ["worker_codex_sol_xhigh", "worker_claude_opus_xhigh"] + weights = [1.0, 1.0] + context_window = 272000 + tool_calling = false + reasoning = true + [routes.auto_fast] id = "atlas/auto/fast" type = "llm_classifier" @@ -352,8 +426,8 @@ data: classifier_target = "classifier" # Switchyard falls through this list after a request-local target failure. # Keep both xhigh providers first so recovery can escalate, never downgrade. - targets = ["codex_sol_xhigh", "claude_opus_xhigh", "claude_fable_xhigh", "codex_sol_high", "claude_sonnet_high", "claude_opus_high", "claude_fable_high", "codex_terra_high", "codex_sol_medium", "claude_sonnet_medium", "claude_fable_medium", "codex_terra_medium", "codex_terra_low", "codex_luna_low", "claude_haiku_low", "claude_fable_low"] - default_target = "codex_terra_medium" + targets = ["codex_sol_xhigh", "claude_opus_xhigh", "claude_fable_xhigh", "codex_sol_high", "claude_sonnet_high", "claude_opus_high", "claude_fable_high", "codex_terra_high", "codex_sol_medium", "claude_sonnet_medium", "claude_fable_medium", "codex_terra_medium", "codex_terra_low", "codex_luna_low", "claude_haiku_low", "claude_fable_low", "neutral_fast_pool"] + default_target = "neutral_fast_pool" session_affinity = false recent_turn_window = 4 context_window = 272000 @@ -430,8 +504,8 @@ data: classifier_target = "classifier" # Switchyard falls through this list after a request-local target failure. # Keep both xhigh providers first so recovery can escalate, never downgrade. - targets = ["codex_sol_xhigh", "claude_opus_xhigh", "claude_fable_xhigh", "codex_sol_high", "claude_sonnet_high", "claude_opus_high", "claude_fable_high", "codex_terra_high", "codex_sol_medium", "claude_sonnet_medium", "claude_fable_medium", "codex_terra_medium", "codex_terra_low", "codex_luna_low", "claude_haiku_low", "claude_fable_low"] - default_target = "codex_terra_medium" + targets = ["codex_sol_xhigh", "claude_opus_xhigh", "claude_fable_xhigh", "codex_sol_high", "claude_sonnet_high", "claude_opus_high", "claude_fable_high", "codex_terra_high", "codex_sol_medium", "claude_sonnet_medium", "claude_fable_medium", "codex_terra_medium", "codex_terra_low", "codex_luna_low", "claude_haiku_low", "claude_fable_low", "neutral_balanced_pool"] + default_target = "neutral_balanced_pool" session_affinity = false recent_turn_window = 4 context_window = 272000 @@ -509,8 +583,8 @@ data: classifier_target = "classifier" # Switchyard falls through this list after a request-local target failure. # Keep both xhigh providers first so recovery can escalate, never downgrade. - targets = ["claude_opus_xhigh", "codex_sol_xhigh", "claude_fable_xhigh", "claude_sonnet_high", "codex_sol_high", "claude_opus_high", "claude_fable_high", "codex_terra_high", "claude_sonnet_medium", "claude_fable_medium", "codex_sol_medium", "codex_terra_medium"] - default_target = "claude_sonnet_high" + targets = ["claude_opus_xhigh", "codex_sol_xhigh", "claude_fable_xhigh", "claude_sonnet_high", "codex_sol_high", "claude_opus_high", "claude_fable_high", "codex_terra_high", "claude_sonnet_medium", "claude_fable_medium", "codex_sol_medium", "codex_terra_medium", "neutral_deep_pool"] + default_target = "neutral_deep_pool" session_affinity = false recent_turn_window = 6 context_window = 272000 @@ -578,8 +652,8 @@ data: classifier_target = "classifier" # Switchyard falls through this list after a request-local target failure. # Keep both xhigh providers first so recovery can escalate, never downgrade. - targets = ["codex_sol_xhigh", "claude_opus_xhigh", "claude_fable_xhigh", "codex_sol_high", "claude_opus_high", "claude_sonnet_high", "claude_fable_high", "codex_terra_high"] - default_target = "codex_sol_high" + targets = ["codex_sol_xhigh", "claude_opus_xhigh", "claude_fable_xhigh", "codex_sol_high", "claude_opus_high", "claude_sonnet_high", "claude_fable_high", "codex_terra_high", "neutral_maximum_pool"] + default_target = "neutral_maximum_pool" session_affinity = false recent_turn_window = 6 context_window = 272000 @@ -639,8 +713,8 @@ data: type = "llm_classifier" mode = "custom" classifier_target = "classifier" - targets = ["worker_codex_luna_low", "worker_codex_luna_medium", "worker_codex_luna_high", "worker_codex_luna_xhigh", "worker_codex_terra_low", "worker_codex_terra_medium", "worker_codex_terra_high", "worker_codex_terra_xhigh", "worker_codex_sol_low", "worker_codex_sol_medium", "worker_codex_sol_high", "worker_codex_sol_xhigh", "worker_claude_haiku_low", "worker_claude_haiku_medium", "worker_claude_haiku_high", "worker_claude_haiku_xhigh", "worker_claude_fable_low", "worker_claude_fable_medium", "worker_claude_fable_high", "worker_claude_fable_xhigh", "worker_claude_sonnet_low", "worker_claude_sonnet_medium", "worker_claude_sonnet_high", "worker_claude_sonnet_xhigh", "worker_claude_opus_low", "worker_claude_opus_medium", "worker_claude_opus_high", "worker_claude_opus_xhigh"] - default_target = "worker_codex_sol_high" + targets = ["worker_codex_luna_low", "worker_codex_luna_medium", "worker_codex_luna_high", "worker_codex_luna_xhigh", "worker_codex_terra_low", "worker_codex_terra_medium", "worker_codex_terra_high", "worker_codex_terra_xhigh", "worker_codex_sol_low", "worker_codex_sol_medium", "worker_codex_sol_high", "worker_codex_sol_xhigh", "worker_claude_haiku_low", "worker_claude_haiku_medium", "worker_claude_haiku_high", "worker_claude_haiku_xhigh", "worker_claude_fable_low", "worker_claude_fable_medium", "worker_claude_fable_high", "worker_claude_fable_xhigh", "worker_claude_sonnet_low", "worker_claude_sonnet_medium", "worker_claude_sonnet_high", "worker_claude_sonnet_xhigh", "worker_claude_opus_low", "worker_claude_opus_medium", "worker_claude_opus_high", "worker_claude_opus_xhigh", "neutral_worker_maximum_pool"] + default_target = "neutral_worker_maximum_pool" session_affinity = false recent_turn_window = 6 context_window = 272000 diff --git a/services/hermes/switchyard-deployment.yaml b/services/hermes/switchyard-deployment.yaml index 456d0a1a..8839ffed 100644 --- a/services/hermes/switchyard-deployment.yaml +++ b/services/hermes/switchyard-deployment.yaml @@ -22,7 +22,7 @@ spec: labels: app: hermes-switchyard annotations: - ai.bstein.dev/config-rev: "20260815-runtime-access-boundary" + ai.bstein.dev/config-rev: "20260823-provider-neutral-pools" prometheus.io/scrape: "true" prometheus.io/port: "9005" prometheus.io/path: /metrics diff --git a/testing/tests/test_hermes_switchyard_provider_pool.py b/testing/tests/test_hermes_switchyard_provider_pool.py new file mode 100644 index 00000000..97aca224 --- /dev/null +++ b/testing/tests/test_hermes_switchyard_provider_pool.py @@ -0,0 +1,71 @@ +"""Provider-neutral fallback contracts for Hermes automatic routing.""" + +from __future__ import annotations + +import tomllib + +from testing.tests.test_hermes_chat_support import HERMES, _documents + + +def _config() -> dict: + manifest = _documents(HERMES / "switchyard-configmap.yaml")[0] + return tomllib.loads(manifest["data"]["routes.toml"]) + + +def _provider(target_id: str) -> str | None: + parts = target_id.split("/") + if len(parts) >= 3 and parts[0] in {"route", "worker"}: + return parts[1] + return None + + +def test_every_classifier_selects_from_both_hosted_providers(): + """No automatic classifier may narrow its candidate set to one provider.""" + config = _config() + targets = config["targets"] + classifiers = { + name: route + for name, route in config["routes"].items() + if route["type"] == "llm_classifier" + } + + assert classifiers + for route in classifiers.values(): + providers = { + _provider(targets[name]["id"]) + for name in route["targets"] + } + assert {"codex", "claude"} <= providers + + +def test_classifier_failure_uses_an_even_cross_provider_pool(): + """The required Switchyard default must name a neutral pool, not a provider.""" + config = _config() + targets = config["targets"] + routes = config["routes"] + routes_by_id = {route["id"]: route for route in routes.values()} + + for route in routes.values(): + if route["type"] != "llm_classifier": + continue + fallback_target_name = route["default_target"] + assert fallback_target_name in route["targets"] + assert fallback_target_name.startswith("neutral_") + fallback_target = targets[fallback_target_name] + assert fallback_target["llm_client"] == "neutral_pool" + + pool = routes_by_id[fallback_target["id"]] + assert pool["type"] == "random" + assert pool["weights"] == [1.0, 1.0] + pool_providers = [ + _provider(targets[name]["id"]) + for name in pool["targets"] + ] + assert sorted(pool_providers) == ["claude", "codex"] + + neutral_client = config["llm_clients"]["neutral_pool"] + assert neutral_client == { + "format": "openai_chat", + "base_url": "http://127.0.0.1:9005/v1", + "max_retries": 0, + }