hermes: remove provider-biased auto fallbacks
This commit is contained in:
jenkins
parent
a7e09f9719
commit
3da6a10174
@ -81,6 +81,11 @@ data:
|
||||
base_url = "http://127.0.0.1:9007/v1"
|
||||
max_retries = 0
|
||||
|
||||
[llm_clients.neutral_pool]
|
||||
format = "openai_chat"
|
||||
base_url = "http://127.0.0.1:9005/v1"
|
||||
max_retries = 0
|
||||
|
||||
[targets.classifier]
|
||||
id = "qwen2.5:14b-instruct-q4_0"
|
||||
llm_client = "classifier"
|
||||
@ -345,6 +350,75 @@ data:
|
||||
id = "worker/claude/opus/xhigh"
|
||||
llm_client = "worker_decision"
|
||||
|
||||
# Switchyard 0.2.0 requires one default_target for every custom classifier.
|
||||
# These targets recurse once into unbiased random routes whose candidates are
|
||||
# split evenly across providers. A classifier outage therefore falls back to
|
||||
# a pool, never directly to either Codex/OpenAI or Claude/Anthropic.
|
||||
[targets.neutral_fast_pool]
|
||||
id = "atlas/fallback/fast"
|
||||
llm_client = "neutral_pool"
|
||||
|
||||
[targets.neutral_balanced_pool]
|
||||
id = "atlas/fallback/balanced"
|
||||
llm_client = "neutral_pool"
|
||||
|
||||
[targets.neutral_deep_pool]
|
||||
id = "atlas/fallback/deep"
|
||||
llm_client = "neutral_pool"
|
||||
|
||||
[targets.neutral_maximum_pool]
|
||||
id = "atlas/fallback/maximum"
|
||||
llm_client = "neutral_pool"
|
||||
|
||||
[targets.neutral_worker_maximum_pool]
|
||||
id = "atlas/worker/fallback/maximum"
|
||||
llm_client = "neutral_pool"
|
||||
|
||||
[routes.fallback_fast]
|
||||
id = "atlas/fallback/fast"
|
||||
type = "random"
|
||||
targets = ["codex_terra_medium", "claude_fable_medium"]
|
||||
weights = [1.0, 1.0]
|
||||
context_window = 272000
|
||||
tool_calling = true
|
||||
reasoning = true
|
||||
|
||||
[routes.fallback_balanced]
|
||||
id = "atlas/fallback/balanced"
|
||||
type = "random"
|
||||
targets = ["codex_terra_high", "claude_sonnet_high"]
|
||||
weights = [1.0, 1.0]
|
||||
context_window = 272000
|
||||
tool_calling = true
|
||||
reasoning = true
|
||||
|
||||
[routes.fallback_deep]
|
||||
id = "atlas/fallback/deep"
|
||||
type = "random"
|
||||
targets = ["codex_sol_high", "claude_opus_high"]
|
||||
weights = [1.0, 1.0]
|
||||
context_window = 272000
|
||||
tool_calling = true
|
||||
reasoning = true
|
||||
|
||||
[routes.fallback_maximum]
|
||||
id = "atlas/fallback/maximum"
|
||||
type = "random"
|
||||
targets = ["codex_sol_xhigh", "claude_opus_xhigh"]
|
||||
weights = [1.0, 1.0]
|
||||
context_window = 272000
|
||||
tool_calling = true
|
||||
reasoning = true
|
||||
|
||||
[routes.fallback_worker_maximum]
|
||||
id = "atlas/worker/fallback/maximum"
|
||||
type = "random"
|
||||
targets = ["worker_codex_sol_xhigh", "worker_claude_opus_xhigh"]
|
||||
weights = [1.0, 1.0]
|
||||
context_window = 272000
|
||||
tool_calling = false
|
||||
reasoning = true
|
||||
|
||||
[routes.auto_fast]
|
||||
id = "atlas/auto/fast"
|
||||
type = "llm_classifier"
|
||||
@ -352,8 +426,8 @@ data:
|
||||
classifier_target = "classifier"
|
||||
# Switchyard falls through this list after a request-local target failure.
|
||||
# Keep both xhigh providers first so recovery can escalate, never downgrade.
|
||||
targets = ["codex_sol_xhigh", "claude_opus_xhigh", "claude_fable_xhigh", "codex_sol_high", "claude_sonnet_high", "claude_opus_high", "claude_fable_high", "codex_terra_high", "codex_sol_medium", "claude_sonnet_medium", "claude_fable_medium", "codex_terra_medium", "codex_terra_low", "codex_luna_low", "claude_haiku_low", "claude_fable_low"]
|
||||
default_target = "codex_terra_medium"
|
||||
targets = ["codex_sol_xhigh", "claude_opus_xhigh", "claude_fable_xhigh", "codex_sol_high", "claude_sonnet_high", "claude_opus_high", "claude_fable_high", "codex_terra_high", "codex_sol_medium", "claude_sonnet_medium", "claude_fable_medium", "codex_terra_medium", "codex_terra_low", "codex_luna_low", "claude_haiku_low", "claude_fable_low", "neutral_fast_pool"]
|
||||
default_target = "neutral_fast_pool"
|
||||
session_affinity = false
|
||||
recent_turn_window = 4
|
||||
context_window = 272000
|
||||
@ -430,8 +504,8 @@ data:
|
||||
classifier_target = "classifier"
|
||||
# Switchyard falls through this list after a request-local target failure.
|
||||
# Keep both xhigh providers first so recovery can escalate, never downgrade.
|
||||
targets = ["codex_sol_xhigh", "claude_opus_xhigh", "claude_fable_xhigh", "codex_sol_high", "claude_sonnet_high", "claude_opus_high", "claude_fable_high", "codex_terra_high", "codex_sol_medium", "claude_sonnet_medium", "claude_fable_medium", "codex_terra_medium", "codex_terra_low", "codex_luna_low", "claude_haiku_low", "claude_fable_low"]
|
||||
default_target = "codex_terra_medium"
|
||||
targets = ["codex_sol_xhigh", "claude_opus_xhigh", "claude_fable_xhigh", "codex_sol_high", "claude_sonnet_high", "claude_opus_high", "claude_fable_high", "codex_terra_high", "codex_sol_medium", "claude_sonnet_medium", "claude_fable_medium", "codex_terra_medium", "codex_terra_low", "codex_luna_low", "claude_haiku_low", "claude_fable_low", "neutral_balanced_pool"]
|
||||
default_target = "neutral_balanced_pool"
|
||||
session_affinity = false
|
||||
recent_turn_window = 4
|
||||
context_window = 272000
|
||||
@ -509,8 +583,8 @@ data:
|
||||
classifier_target = "classifier"
|
||||
# Switchyard falls through this list after a request-local target failure.
|
||||
# Keep both xhigh providers first so recovery can escalate, never downgrade.
|
||||
targets = ["claude_opus_xhigh", "codex_sol_xhigh", "claude_fable_xhigh", "claude_sonnet_high", "codex_sol_high", "claude_opus_high", "claude_fable_high", "codex_terra_high", "claude_sonnet_medium", "claude_fable_medium", "codex_sol_medium", "codex_terra_medium"]
|
||||
default_target = "claude_sonnet_high"
|
||||
targets = ["claude_opus_xhigh", "codex_sol_xhigh", "claude_fable_xhigh", "claude_sonnet_high", "codex_sol_high", "claude_opus_high", "claude_fable_high", "codex_terra_high", "claude_sonnet_medium", "claude_fable_medium", "codex_sol_medium", "codex_terra_medium", "neutral_deep_pool"]
|
||||
default_target = "neutral_deep_pool"
|
||||
session_affinity = false
|
||||
recent_turn_window = 6
|
||||
context_window = 272000
|
||||
@ -578,8 +652,8 @@ data:
|
||||
classifier_target = "classifier"
|
||||
# Switchyard falls through this list after a request-local target failure.
|
||||
# Keep both xhigh providers first so recovery can escalate, never downgrade.
|
||||
targets = ["codex_sol_xhigh", "claude_opus_xhigh", "claude_fable_xhigh", "codex_sol_high", "claude_opus_high", "claude_sonnet_high", "claude_fable_high", "codex_terra_high"]
|
||||
default_target = "codex_sol_high"
|
||||
targets = ["codex_sol_xhigh", "claude_opus_xhigh", "claude_fable_xhigh", "codex_sol_high", "claude_opus_high", "claude_sonnet_high", "claude_fable_high", "codex_terra_high", "neutral_maximum_pool"]
|
||||
default_target = "neutral_maximum_pool"
|
||||
session_affinity = false
|
||||
recent_turn_window = 6
|
||||
context_window = 272000
|
||||
@ -639,8 +713,8 @@ data:
|
||||
type = "llm_classifier"
|
||||
mode = "custom"
|
||||
classifier_target = "classifier"
|
||||
targets = ["worker_codex_luna_low", "worker_codex_luna_medium", "worker_codex_luna_high", "worker_codex_luna_xhigh", "worker_codex_terra_low", "worker_codex_terra_medium", "worker_codex_terra_high", "worker_codex_terra_xhigh", "worker_codex_sol_low", "worker_codex_sol_medium", "worker_codex_sol_high", "worker_codex_sol_xhigh", "worker_claude_haiku_low", "worker_claude_haiku_medium", "worker_claude_haiku_high", "worker_claude_haiku_xhigh", "worker_claude_fable_low", "worker_claude_fable_medium", "worker_claude_fable_high", "worker_claude_fable_xhigh", "worker_claude_sonnet_low", "worker_claude_sonnet_medium", "worker_claude_sonnet_high", "worker_claude_sonnet_xhigh", "worker_claude_opus_low", "worker_claude_opus_medium", "worker_claude_opus_high", "worker_claude_opus_xhigh"]
|
||||
default_target = "worker_codex_sol_high"
|
||||
targets = ["worker_codex_luna_low", "worker_codex_luna_medium", "worker_codex_luna_high", "worker_codex_luna_xhigh", "worker_codex_terra_low", "worker_codex_terra_medium", "worker_codex_terra_high", "worker_codex_terra_xhigh", "worker_codex_sol_low", "worker_codex_sol_medium", "worker_codex_sol_high", "worker_codex_sol_xhigh", "worker_claude_haiku_low", "worker_claude_haiku_medium", "worker_claude_haiku_high", "worker_claude_haiku_xhigh", "worker_claude_fable_low", "worker_claude_fable_medium", "worker_claude_fable_high", "worker_claude_fable_xhigh", "worker_claude_sonnet_low", "worker_claude_sonnet_medium", "worker_claude_sonnet_high", "worker_claude_sonnet_xhigh", "worker_claude_opus_low", "worker_claude_opus_medium", "worker_claude_opus_high", "worker_claude_opus_xhigh", "neutral_worker_maximum_pool"]
|
||||
default_target = "neutral_worker_maximum_pool"
|
||||
session_affinity = false
|
||||
recent_turn_window = 6
|
||||
context_window = 272000
|
||||
|
||||
@ -22,7 +22,7 @@ spec:
|
||||
labels:
|
||||
app: hermes-switchyard
|
||||
annotations:
|
||||
ai.bstein.dev/config-rev: "20260815-runtime-access-boundary"
|
||||
ai.bstein.dev/config-rev: "20260823-provider-neutral-pools"
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/port: "9005"
|
||||
prometheus.io/path: /metrics
|
||||
|
||||
71
testing/tests/test_hermes_switchyard_provider_pool.py
Normal file
71
testing/tests/test_hermes_switchyard_provider_pool.py
Normal file
@ -0,0 +1,71 @@
|
||||
"""Provider-neutral fallback contracts for Hermes automatic routing."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import tomllib
|
||||
|
||||
from testing.tests.test_hermes_chat_support import HERMES, _documents
|
||||
|
||||
|
||||
def _config() -> dict:
|
||||
manifest = _documents(HERMES / "switchyard-configmap.yaml")[0]
|
||||
return tomllib.loads(manifest["data"]["routes.toml"])
|
||||
|
||||
|
||||
def _provider(target_id: str) -> str | None:
|
||||
parts = target_id.split("/")
|
||||
if len(parts) >= 3 and parts[0] in {"route", "worker"}:
|
||||
return parts[1]
|
||||
return None
|
||||
|
||||
|
||||
def test_every_classifier_selects_from_both_hosted_providers():
|
||||
"""No automatic classifier may narrow its candidate set to one provider."""
|
||||
config = _config()
|
||||
targets = config["targets"]
|
||||
classifiers = {
|
||||
name: route
|
||||
for name, route in config["routes"].items()
|
||||
if route["type"] == "llm_classifier"
|
||||
}
|
||||
|
||||
assert classifiers
|
||||
for route in classifiers.values():
|
||||
providers = {
|
||||
_provider(targets[name]["id"])
|
||||
for name in route["targets"]
|
||||
}
|
||||
assert {"codex", "claude"} <= providers
|
||||
|
||||
|
||||
def test_classifier_failure_uses_an_even_cross_provider_pool():
|
||||
"""The required Switchyard default must name a neutral pool, not a provider."""
|
||||
config = _config()
|
||||
targets = config["targets"]
|
||||
routes = config["routes"]
|
||||
routes_by_id = {route["id"]: route for route in routes.values()}
|
||||
|
||||
for route in routes.values():
|
||||
if route["type"] != "llm_classifier":
|
||||
continue
|
||||
fallback_target_name = route["default_target"]
|
||||
assert fallback_target_name in route["targets"]
|
||||
assert fallback_target_name.startswith("neutral_")
|
||||
fallback_target = targets[fallback_target_name]
|
||||
assert fallback_target["llm_client"] == "neutral_pool"
|
||||
|
||||
pool = routes_by_id[fallback_target["id"]]
|
||||
assert pool["type"] == "random"
|
||||
assert pool["weights"] == [1.0, 1.0]
|
||||
pool_providers = [
|
||||
_provider(targets[name]["id"])
|
||||
for name in pool["targets"]
|
||||
]
|
||||
assert sorted(pool_providers) == ["claude", "codex"]
|
||||
|
||||
neutral_client = config["llm_clients"]["neutral_pool"]
|
||||
assert neutral_client == {
|
||||
"format": "openai_chat",
|
||||
"base_url": "http://127.0.0.1:9005/v1",
|
||||
"max_retries": 0,
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user