diff --git a/services/hermes/switchyard-configmap.yaml b/services/hermes/switchyard-configmap.yaml index d189a3060..af19736a5 100644 --- a/services/hermes/switchyard-configmap.yaml +++ b/services/hermes/switchyard-configmap.yaml @@ -191,7 +191,7 @@ data: classifier_target = "classifier" # Switchyard falls through this list after a request-local target failure. # Keep both xhigh providers first so recovery can escalate, never downgrade. - targets = ["codex_sol_xhigh", "claude_opus_xhigh", "codex_sol_high", "claude_sonnet_high", "claude_opus_high", "codex_terra_high", "codex_sol_medium", "claude_sonnet_medium", "codex_terra_medium", "local_qwen_medium", "codex_terra_low", "codex_luna_low", "claude_haiku_low", "local_qwen_low"] + targets = ["codex_sol_xhigh", "claude_opus_xhigh", "codex_sol_high", "claude_sonnet_high", "claude_opus_high", "codex_terra_high", "codex_sol_medium", "claude_sonnet_medium", "codex_terra_medium", "codex_terra_low", "codex_luna_low", "claude_haiku_low"] default_target = "codex_terra_medium" session_affinity = false recent_turn_window = 4 @@ -225,29 +225,30 @@ data: 3. Route image creation and editing before general provider preference. When image-generation/edit tools are available and the user asks to create, transform, restore, colorize, or continue editing an image: select - local_qwen_medium when local generation is requested; select - codex_terra_medium when OpenAI or hosted generation is requested; otherwise - choose either of those two medium targets. Do not select a Claude target for - an image-tool boundary: Anthropic supplies the conversation model, not an - image backend, and Claude capacity must not block image execution. + codex_terra_medium. The image tool—not the conversational model—honors the + user's local, OpenAI/hosted, or AUTO image-backend choice. Do not select a + local Qwen or Claude target for an image-tool boundary: local Qwen cannot + reliably carry the full Hermes tool context, and Anthropic supplies the + conversation model rather than either configured image backend. - 4. Use local Qwen only at low or medium for low-risk conversation, - formatting, lookup, and continuity. Prefer Codex for implementation, - debugging, tests, commands, and repository work. Prefer Claude for - architecture, ambiguity, synthesis, risk analysis, adversarial analysis, - and independent review. Anthropic means Claude; OpenAI means Codex. Never - select a provider stated to be unavailable, failed, exhausted, rate-limited, - or out of capacity; use the other provider at the same floor. + 4. Local Qwen is the mandatory routing classifier, but its 8K context and + tool protocol are not eligible for foreground Hermes execution. Prefer + Codex for implementation, debugging, tests, commands, repository work, and + image-tool invocation. Prefer Claude for architecture, ambiguity, + synthesis, risk analysis, adversarial analysis, and independent review. + Anthropic means Claude; OpenAI means Codex. Never select a provider stated + to be unavailable, failed, exhausted, rate-limited, or out of capacity; use + the other hosted provider at the same floor. 5. Map exactly: Codex low=codex_luna_low, medium=codex_terra_medium, high=codex_sol_high, xhigh=codex_sol_xhigh; Claude low=claude_haiku_low, medium=claude_sonnet_medium, high=claude_sonnet_high, - xhigh=claude_opus_xhigh; local low=local_qwen_low and local - medium=local_qwen_medium. Re-evaluate every boundary and resolve "continue" - or "do it" from recent context. + xhigh=claude_opus_xhigh. Re-evaluate every boundary and resolve "continue" + or "do it" from recent context. The manual local route remains available + only when the user explicitly selects it. """ response_schema = ''' - {"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["codex_terra_medium","claude_sonnet_medium","codex_luna_low","claude_haiku_low","local_qwen_low","local_qwen_medium","codex_terra_low","codex_terra_high","codex_sol_medium","codex_sol_high","codex_sol_xhigh","claude_sonnet_high","claude_opus_high","claude_opus_xhigh"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false} + {"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["codex_terra_medium","claude_sonnet_medium","codex_luna_low","claude_haiku_low","codex_terra_low","codex_terra_high","codex_sol_medium","codex_sol_high","codex_sol_xhigh","claude_sonnet_high","claude_opus_high","claude_opus_xhigh"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false} ''' [routes.auto_fast.policy] @@ -261,7 +262,7 @@ data: classifier_target = "classifier" # Switchyard falls through this list after a request-local target failure. # Keep both xhigh providers first so recovery can escalate, never downgrade. - targets = ["codex_sol_xhigh", "claude_opus_xhigh", "codex_sol_high", "claude_sonnet_high", "claude_opus_high", "codex_terra_high", "codex_sol_medium", "claude_sonnet_medium", "codex_terra_medium", "local_qwen_medium", "codex_terra_low", "codex_luna_low", "claude_haiku_low", "local_qwen_low"] + targets = ["codex_sol_xhigh", "claude_opus_xhigh", "codex_sol_high", "claude_sonnet_high", "claude_opus_high", "codex_terra_high", "codex_sol_medium", "claude_sonnet_medium", "codex_terra_medium", "codex_terra_low", "codex_luna_low", "claude_haiku_low"] default_target = "codex_terra_medium" session_affinity = false recent_turn_window = 4 @@ -296,29 +297,30 @@ data: 3. Route image creation and editing before general provider preference. When image-generation/edit tools are available and the user asks to create, transform, restore, colorize, or continue editing an image: select - local_qwen_medium when local generation is requested; select - codex_terra_medium when OpenAI or hosted generation is requested; otherwise - choose either of those two medium targets. Do not select a Claude target for - an image-tool boundary: Anthropic supplies the conversation model, not an - image backend, and Claude capacity must not block image execution. + codex_terra_medium. The image tool—not the conversational model—honors the + user's local, OpenAI/hosted, or AUTO image-backend choice. Do not select a + local Qwen or Claude target for an image-tool boundary: local Qwen cannot + reliably carry the full Hermes tool context, and Anthropic supplies the + conversation model rather than either configured image backend. - 4. Use local Qwen only at low or medium for low-risk conversation, - formatting, lookup, and continuity. Prefer Codex for implementation, - debugging, tests, commands, and repository work. Prefer Claude for - architecture, ambiguity, synthesis, risk analysis, adversarial analysis, - and independent review. Anthropic means Claude; OpenAI means Codex. Never - select a provider stated to be unavailable, failed, exhausted, rate-limited, - or out of capacity; use the other provider at the same floor. + 4. Local Qwen is the mandatory routing classifier, but its 8K context and + tool protocol are not eligible for foreground Hermes execution. Prefer + Codex for implementation, debugging, tests, commands, repository work, and + image-tool invocation. Prefer Claude for architecture, ambiguity, + synthesis, risk analysis, adversarial analysis, and independent review. + Anthropic means Claude; OpenAI means Codex. Never select a provider stated + to be unavailable, failed, exhausted, rate-limited, or out of capacity; use + the other hosted provider at the same floor. 5. Map exactly: Codex low=codex_luna_low, medium=codex_terra_medium, high=codex_sol_high, xhigh=codex_sol_xhigh; Claude low=claude_haiku_low, medium=claude_sonnet_medium, high=claude_sonnet_high, - xhigh=claude_opus_xhigh; local low=local_qwen_low and local - medium=local_qwen_medium. Re-evaluate every boundary and resolve "continue" - or "do it" from recent context. + xhigh=claude_opus_xhigh. Re-evaluate every boundary and resolve "continue" + or "do it" from recent context. The manual local route remains available + only when the user explicitly selects it. """ response_schema = ''' - {"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["codex_terra_medium","claude_sonnet_medium","codex_luna_low","claude_haiku_low","local_qwen_low","local_qwen_medium","codex_terra_low","codex_terra_high","codex_sol_medium","codex_sol_high","codex_sol_xhigh","claude_sonnet_high","claude_opus_high","claude_opus_xhigh"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false} + {"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["codex_terra_medium","claude_sonnet_medium","codex_luna_low","claude_haiku_low","codex_terra_low","codex_terra_high","codex_sol_medium","codex_sol_high","codex_sol_xhigh","claude_sonnet_high","claude_opus_high","claude_opus_xhigh"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false} ''' [routes.auto_balanced.policy] @@ -365,7 +367,7 @@ data: available provider or model wins at the safety floor. 3. This quality-first triage route must use a hosted Codex or Claude target. - The local Qwen model remains the routing classifier, but its 32K context and + The local Qwen model remains the routing classifier, but its 8K context and tool protocol are not eligible for foreground triage execution. Prefer Codex for implementation, debugging, tests, commands, and repository work. Prefer Claude for diagnosis, architecture, ambiguity, synthesis, risk @@ -429,7 +431,7 @@ data: available provider or model wins at the safety floor. 3. This owner-only engineering route must use a hosted Codex or Claude - target. The local Qwen model remains the routing classifier, but its 32K + target. The local Qwen model remains the routing classifier, but its 8K context and tool protocol are not eligible for foreground agent execution. Prefer Codex for implementation, debugging, tests, commands, and repository changes. Prefer Claude for architecture, ambiguity, synthesis, risk @@ -574,8 +576,8 @@ data: [routes.manual_codex_luna] id = "atlas/manual/codex/luna" type = "random" - targets = ["codex_luna_low", "claude_haiku_low", "codex_terra_medium", "claude_sonnet_medium", "local_qwen_low"] - weights = [1, 0, 0, 0, 0] + targets = ["codex_luna_low", "claude_haiku_low", "codex_terra_medium", "claude_sonnet_medium"] + weights = [1, 0, 0, 0] context_window = 272000 tool_calling = true reasoning = true @@ -583,8 +585,8 @@ data: [routes.manual_codex_terra] id = "atlas/manual/codex/terra" type = "random" - targets = ["codex_terra_medium", "claude_sonnet_medium", "codex_sol_high", "claude_sonnet_high", "local_qwen_medium"] - weights = [1, 0, 0, 0, 0] + targets = ["codex_terra_medium", "claude_sonnet_medium", "codex_sol_high", "claude_sonnet_high"] + weights = [1, 0, 0, 0] context_window = 272000 tool_calling = true reasoning = true @@ -592,8 +594,8 @@ data: [routes.manual_codex_sol] id = "atlas/manual/codex/sol" type = "random" - targets = ["codex_sol_high", "claude_opus_high", "claude_sonnet_high", "codex_terra_high", "local_qwen_medium"] - weights = [1, 0, 0, 0, 0] + targets = ["codex_sol_high", "claude_opus_high", "claude_sonnet_high", "codex_terra_high"] + weights = [1, 0, 0, 0] context_window = 272000 tool_calling = true reasoning = true @@ -601,8 +603,8 @@ data: [routes.manual_claude_haiku] id = "atlas/manual/claude/haiku" type = "random" - targets = ["claude_haiku_low", "codex_luna_low", "claude_sonnet_medium", "codex_terra_medium", "local_qwen_low"] - weights = [1, 0, 0, 0, 0] + targets = ["claude_haiku_low", "codex_luna_low", "claude_sonnet_medium", "codex_terra_medium"] + weights = [1, 0, 0, 0] context_window = 272000 tool_calling = true reasoning = true @@ -610,8 +612,8 @@ data: [routes.manual_claude_sonnet] id = "atlas/manual/claude/sonnet" type = "random" - targets = ["claude_sonnet_high", "codex_sol_high", "claude_opus_high", "codex_terra_high", "local_qwen_medium"] - weights = [1, 0, 0, 0, 0] + targets = ["claude_sonnet_high", "codex_sol_high", "claude_opus_high", "codex_terra_high"] + weights = [1, 0, 0, 0] context_window = 272000 tool_calling = true reasoning = true @@ -619,8 +621,8 @@ data: [routes.manual_claude_opus] id = "atlas/manual/claude/opus" type = "random" - targets = ["claude_opus_high", "codex_sol_high", "claude_sonnet_high", "codex_terra_high", "local_qwen_medium"] - weights = [1, 0, 0, 0, 0] + targets = ["claude_opus_high", "codex_sol_high", "claude_sonnet_high", "codex_terra_high"] + weights = [1, 0, 0, 0] context_window = 272000 tool_calling = true reasoning = true diff --git a/services/hermes/switchyard-deployment.yaml b/services/hermes/switchyard-deployment.yaml index ad0b769bc..4e3090f42 100644 --- a/services/hermes/switchyard-deployment.yaml +++ b/services/hermes/switchyard-deployment.yaml @@ -22,7 +22,7 @@ spec: labels: app: hermes-switchyard annotations: - ai.bstein.dev/config-rev: "20260812-classifier-context" + ai.bstein.dev/config-rev: "20260812-hosted-foreground" prometheus.io/scrape: "true" prometheus.io/port: "9005" prometheus.io/path: /metrics diff --git a/testing/tests/test_hermes_chat_quality.py b/testing/tests/test_hermes_chat_quality.py index d2c6a780e..de3abd328 100644 --- a/testing/tests/test_hermes_chat_quality.py +++ b/testing/tests/test_hermes_chat_quality.py @@ -1135,22 +1135,37 @@ def test_local_flux_runtime_and_gpu_handoff_are_flux_managed(): leading_targets = set(routes[route_name]["targets"][:2]) assert leading_targets == {"codex_sol_xhigh", "claude_opus_xhigh"} assert "max_output_tokens" not in routes[route_name] - for route_name in ("auto_deep", "auto_maximum"): + for route_name in ("auto_fast", "auto_balanced", "auto_deep", "auto_maximum"): targets = routes[route_name]["targets"] selector_targets = routes[route_name]["response_schema"] assert not any(target.startswith("local_") for target in targets) - assert not any(target.endswith("_low") for target in targets) assert "local_qwen" not in selector_targets - assert "_low" not in selector_targets assert "not eligible for foreground" in routes[route_name]["prompt"] - for route_name in ("auto_fast", "auto_balanced", "manual_local_qwen"): - assert any( + for route_name in ("auto_deep", "auto_maximum"): + targets = routes[route_name]["targets"] + selector_targets = routes[route_name]["response_schema"] + assert not any(target.endswith("_low") for target in targets) + assert "_low" not in selector_targets + assert any( + target.startswith("local_") + for target in routes["manual_local_qwen"]["targets"] + ) + for route_name in ( + "manual_codex_luna", + "manual_codex_terra", + "manual_codex_sol", + "manual_claude_haiku", + "manual_claude_sonnet", + "manual_claude_opus", + ): + assert not any( target.startswith("local_") for target in routes[route_name]["targets"] ) for route_name in ("auto_fast", "auto_balanced"): prompt = routes[route_name]["prompt"] - assert "Route image creation and editing before general provider" in prompt - assert "Do not select a Claude target for" in prompt + assert "image tool—not the conversational model" in prompt + assert "Do not select a" in prompt + assert "local Qwen or Claude target" in prompt assert "max_output_tokens" not in routes["worker_auto_maximum"] model_gate = _documents(HERMES / "model-gate-configmap.yaml")[0]["data"][ "model_gate.py"