From 3af729849e927963618bc996ffa84227672b6525 Mon Sep 17 00:00:00 2001 From: jenkins Date: Wed, 12 Aug 2026 02:52:27 -0300 Subject: [PATCH] hermes: keep quality routes on hosted models --- services/hermes/switchyard-configmap.yaml | 46 +++++++++++----------- services/hermes/switchyard-deployment.yaml | 2 +- testing/tests/test_hermes_chat_quality.py | 10 +++++ 3 files changed, 35 insertions(+), 23 deletions(-) diff --git a/services/hermes/switchyard-configmap.yaml b/services/hermes/switchyard-configmap.yaml index 4f401d205..a80d61e0b 100644 --- a/services/hermes/switchyard-configmap.yaml +++ b/services/hermes/switchyard-configmap.yaml @@ -303,7 +303,7 @@ data: classifier_target = "classifier" # Switchyard falls through this list after a request-local target failure. # Keep both xhigh providers first so recovery can escalate, never downgrade. - targets = ["claude_opus_xhigh", "codex_sol_xhigh", "claude_sonnet_high", "codex_sol_high", "claude_opus_high", "codex_terra_high", "claude_sonnet_medium", "codex_sol_medium", "codex_terra_medium", "local_qwen_medium", "claude_haiku_low", "codex_luna_low", "codex_terra_low", "local_qwen_low"] + targets = ["claude_opus_xhigh", "codex_sol_xhigh", "claude_sonnet_high", "codex_sol_high", "claude_opus_high", "codex_terra_high", "claude_sonnet_medium", "codex_sol_medium", "codex_terra_medium", "claude_haiku_low", "codex_luna_low", "codex_terra_low"] default_target = "claude_sonnet_high" session_affinity = false recent_turn_window = 12 @@ -331,23 +331,24 @@ data: the fastest target at the safety floor, never below it. An explicit available provider or model wins at the safety floor. - 3. Use local Qwen only at low or medium for low-risk summaries, formatting, - lookup, and continuity. Prefer Codex for implementation, debugging, tests, - commands, and repository work. Prefer Claude for diagnosis, architecture, - ambiguity, synthesis, risk analysis, adversarial analysis, and independent - review. Anthropic means Claude; OpenAI means Codex. Never select a provider - stated to be unavailable, failed, exhausted, rate-limited, or out of - capacity; use the other provider at the same floor. + 3. This quality-first triage route must use a hosted Codex or Claude target. + The local Qwen model remains the routing classifier, but its 32K context and + tool protocol are not eligible for foreground triage execution. Prefer + Codex for implementation, debugging, tests, commands, and repository work. + Prefer Claude for diagnosis, architecture, ambiguity, synthesis, risk + analysis, adversarial analysis, and independent review. Anthropic means + Claude; OpenAI means Codex. Never select a provider stated to be + unavailable, failed, exhausted, rate-limited, or out of capacity; use the + other provider at the same floor. 4. Map exactly: Codex low=codex_luna_low, medium=codex_terra_medium, high=codex_sol_high, xhigh=codex_sol_xhigh; Claude low=claude_haiku_low, medium=claude_sonnet_medium, high=claude_sonnet_high, - xhigh=claude_opus_xhigh; local low=local_qwen_low and local - medium=local_qwen_medium. Re-evaluate every boundary and resolve "continue" + xhigh=claude_opus_xhigh. Re-evaluate every boundary and resolve "continue" or "do it" from recent context. """ response_schema = ''' - {"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["claude_sonnet_high","codex_sol_high","codex_terra_medium","claude_sonnet_medium","codex_terra_high","codex_sol_medium","claude_opus_high","codex_sol_xhigh","claude_opus_xhigh","codex_luna_low","claude_haiku_low","local_qwen_medium","local_qwen_low","codex_terra_low"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false} + {"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["claude_sonnet_high","codex_sol_high","codex_terra_medium","claude_sonnet_medium","codex_terra_high","codex_sol_medium","claude_opus_high","codex_sol_xhigh","claude_opus_xhigh","codex_luna_low","claude_haiku_low","codex_terra_low"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false} ''' [routes.auto_deep.policy] @@ -361,7 +362,7 @@ data: classifier_target = "classifier" # Switchyard falls through this list after a request-local target failure. # Keep both xhigh providers first so recovery can escalate, never downgrade. - targets = ["codex_sol_xhigh", "claude_opus_xhigh", "codex_sol_high", "claude_opus_high", "claude_sonnet_high", "codex_terra_high", "codex_sol_medium", "claude_sonnet_medium", "codex_terra_medium", "local_qwen_medium", "codex_luna_low", "claude_haiku_low", "codex_terra_low", "local_qwen_low"] + targets = ["codex_sol_xhigh", "claude_opus_xhigh", "codex_sol_high", "claude_opus_high", "claude_sonnet_high", "codex_terra_high", "codex_sol_medium", "claude_sonnet_medium", "codex_terra_medium", "codex_luna_low", "claude_haiku_low", "codex_terra_low"] default_target = "codex_sol_high" session_affinity = false recent_turn_window = 16 @@ -389,23 +390,24 @@ data: the fastest target at the safety floor, never below it. An explicit available provider or model wins at the safety floor. - 3. Use local Qwen only at low or medium for low-risk lookup, formatting, and - continuity. Prefer Codex for implementation, debugging, tests, commands, - and repository changes. Prefer Claude for architecture, ambiguity, - synthesis, risk analysis, adversarial analysis, and independent review. - Anthropic means Claude; OpenAI means Codex. Never select a provider stated - to be unavailable, failed, exhausted, rate-limited, or out of capacity; use - the other provider at the same floor. + 3. This owner-only engineering route must use a hosted Codex or Claude + target. The local Qwen model remains the routing classifier, but its 32K + context and tool protocol are not eligible for foreground agent execution. + Prefer Codex for implementation, debugging, tests, commands, and repository + changes. Prefer Claude for architecture, ambiguity, synthesis, risk + analysis, adversarial analysis, and independent review. Anthropic means + Claude; OpenAI means Codex. Never select a provider stated to be + unavailable, failed, exhausted, rate-limited, or out of capacity; use the + other provider at the same floor. 4. Map exactly: Codex low=codex_luna_low, medium=codex_terra_medium, high=codex_sol_high, xhigh=codex_sol_xhigh; Claude low=claude_haiku_low, medium=claude_sonnet_medium, high=claude_sonnet_high, - xhigh=claude_opus_xhigh; local low=local_qwen_low and local - medium=local_qwen_medium. Re-evaluate every boundary and resolve "continue" + xhigh=claude_opus_xhigh. Re-evaluate every boundary and resolve "continue" or "do it" from recent context. """ response_schema = ''' - {"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["codex_sol_high","claude_opus_high","claude_sonnet_high","codex_terra_high","codex_sol_xhigh","claude_opus_xhigh","codex_terra_medium","claude_sonnet_medium","codex_sol_medium","codex_luna_low","claude_haiku_low","local_qwen_medium","local_qwen_low","codex_terra_low"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false} + {"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["codex_sol_high","claude_opus_high","claude_sonnet_high","codex_terra_high","codex_sol_xhigh","claude_opus_xhigh","codex_terra_medium","claude_sonnet_medium","codex_sol_medium","codex_luna_low","claude_haiku_low","codex_terra_low"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false} ''' [routes.auto_maximum.policy] diff --git a/services/hermes/switchyard-deployment.yaml b/services/hermes/switchyard-deployment.yaml index 213602cd6..7b5ecd365 100644 --- a/services/hermes/switchyard-deployment.yaml +++ b/services/hermes/switchyard-deployment.yaml @@ -22,7 +22,7 @@ spec: labels: app: hermes-switchyard annotations: - ai.bstein.dev/config-rev: "20260812-auto-response-budget" + ai.bstein.dev/config-rev: "20260812-quality-route-local-exclusion" prometheus.io/scrape: "true" prometheus.io/port: "9005" prometheus.io/path: /metrics diff --git a/testing/tests/test_hermes_chat_quality.py b/testing/tests/test_hermes_chat_quality.py index da808acc3..58c5d771f 100644 --- a/testing/tests/test_hermes_chat_quality.py +++ b/testing/tests/test_hermes_chat_quality.py @@ -855,6 +855,16 @@ def test_local_flux_runtime_and_gpu_handoff_are_flux_managed(): leading_targets = set(routes[route_name]["targets"][:2]) assert leading_targets == {"codex_sol_xhigh", "claude_opus_xhigh"} assert "max_output_tokens" not in routes[route_name] + for route_name in ("auto_deep", "auto_maximum"): + targets = routes[route_name]["targets"] + selector_targets = routes[route_name]["response_schema"] + assert not any(target.startswith("local_") for target in targets) + assert "local_qwen" not in selector_targets + assert "not eligible for foreground" in routes[route_name]["prompt"] + for route_name in ("auto_fast", "auto_balanced", "manual_local_qwen"): + assert any( + target.startswith("local_") for target in routes[route_name]["targets"] + ) assert "max_output_tokens" not in routes["worker_auto_maximum"] model_gate = _documents(HERMES / "model-gate-configmap.yaml")[0]["data"][ "model_gate.py"