From 9441753b12b359a5e51662c4afeadd5036e07953 Mon Sep 17 00:00:00 2001 From: jenkins Date: Wed, 12 Aug 2026 00:27:21 -0300 Subject: [PATCH] hermes: preserve AUTO quality during fallback --- services/hermes/switchyard-configmap.yaml | 16 ++++++++++++---- services/hermes/switchyard-deployment.yaml | 2 +- testing/tests/test_hermes_chat_quality.py | 5 +++++ 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/services/hermes/switchyard-configmap.yaml b/services/hermes/switchyard-configmap.yaml index 06410ae1f..6e8e4c045 100644 --- a/services/hermes/switchyard-configmap.yaml +++ b/services/hermes/switchyard-configmap.yaml @@ -186,7 +186,9 @@ data: type = "llm_classifier" mode = "custom" classifier_target = "classifier" - targets = ["codex_terra_medium", "claude_sonnet_medium", "codex_luna_low", "claude_haiku_low", "local_qwen_low", "local_qwen_medium", "codex_terra_low", "codex_terra_high", "codex_sol_medium", "codex_sol_high", "codex_sol_xhigh", "claude_sonnet_high", "claude_opus_high", "claude_opus_xhigh"] + # Switchyard falls through this list after a request-local target failure. + # Keep both xhigh providers first so recovery can escalate, never downgrade. + targets = ["codex_sol_xhigh", "claude_opus_xhigh", "codex_sol_high", "claude_sonnet_high", "claude_opus_high", "codex_terra_high", "codex_sol_medium", "claude_sonnet_medium", "codex_terra_medium", "local_qwen_medium", "codex_terra_low", "codex_luna_low", "claude_haiku_low", "local_qwen_low"] default_target = "codex_terra_medium" max_output_tokens = 96 session_affinity = false @@ -242,7 +244,9 @@ data: type = "llm_classifier" mode = "custom" classifier_target = "classifier" - targets = ["codex_terra_medium", "claude_sonnet_medium", "codex_luna_low", "claude_haiku_low", "local_qwen_low", "local_qwen_medium", "codex_terra_low", "codex_terra_high", "codex_sol_medium", "codex_sol_high", "codex_sol_xhigh", "claude_sonnet_high", "claude_opus_high", "claude_opus_xhigh"] + # Switchyard falls through this list after a request-local target failure. + # Keep both xhigh providers first so recovery can escalate, never downgrade. + targets = ["codex_sol_xhigh", "claude_opus_xhigh", "codex_sol_high", "claude_sonnet_high", "claude_opus_high", "codex_terra_high", "codex_sol_medium", "claude_sonnet_medium", "codex_terra_medium", "local_qwen_medium", "codex_terra_low", "codex_luna_low", "claude_haiku_low", "local_qwen_low"] default_target = "codex_terra_medium" max_output_tokens = 96 session_affinity = false @@ -299,7 +303,9 @@ data: type = "llm_classifier" mode = "custom" classifier_target = "classifier" - targets = ["claude_sonnet_high", "codex_sol_high", "codex_terra_medium", "claude_sonnet_medium", "codex_terra_high", "codex_sol_medium", "claude_opus_high", "codex_sol_xhigh", "claude_opus_xhigh", "codex_luna_low", "claude_haiku_low", "local_qwen_medium", "local_qwen_low", "codex_terra_low"] + # Switchyard falls through this list after a request-local target failure. + # Keep both xhigh providers first so recovery can escalate, never downgrade. + targets = ["claude_opus_xhigh", "codex_sol_xhigh", "claude_sonnet_high", "codex_sol_high", "claude_opus_high", "codex_terra_high", "claude_sonnet_medium", "codex_sol_medium", "codex_terra_medium", "local_qwen_medium", "claude_haiku_low", "codex_luna_low", "codex_terra_low", "local_qwen_low"] default_target = "claude_sonnet_high" max_output_tokens = 96 session_affinity = false @@ -356,7 +362,9 @@ data: type = "llm_classifier" mode = "custom" classifier_target = "classifier" - targets = ["codex_sol_high", "claude_opus_high", "claude_sonnet_high", "codex_terra_high", "codex_sol_xhigh", "claude_opus_xhigh", "codex_terra_medium", "claude_sonnet_medium", "codex_sol_medium", "codex_luna_low", "claude_haiku_low", "local_qwen_medium", "local_qwen_low", "codex_terra_low"] + # Switchyard falls through this list after a request-local target failure. + # Keep both xhigh providers first so recovery can escalate, never downgrade. + targets = ["codex_sol_xhigh", "claude_opus_xhigh", "codex_sol_high", "claude_opus_high", "claude_sonnet_high", "codex_terra_high", "codex_sol_medium", "claude_sonnet_medium", "codex_terra_medium", "local_qwen_medium", "codex_luna_low", "claude_haiku_low", "codex_terra_low", "local_qwen_low"] default_target = "codex_sol_high" max_output_tokens = 96 session_affinity = false diff --git a/services/hermes/switchyard-deployment.yaml b/services/hermes/switchyard-deployment.yaml index f9ea46cb8..39eec47ac 100644 --- a/services/hermes/switchyard-deployment.yaml +++ b/services/hermes/switchyard-deployment.yaml @@ -22,7 +22,7 @@ spec: labels: app: hermes-switchyard annotations: - ai.bstein.dev/config-rev: "20260811-switchyard-authority-v10" + ai.bstein.dev/config-rev: "20260811-switchyard-authority-v11" prometheus.io/scrape: "true" prometheus.io/port: "9005" prometheus.io/path: /metrics diff --git a/testing/tests/test_hermes_chat_quality.py b/testing/tests/test_hermes_chat_quality.py index d22b7fea4..8e0cf8113 100644 --- a/testing/tests/test_hermes_chat_quality.py +++ b/testing/tests/test_hermes_chat_quality.py @@ -7,6 +7,7 @@ import importlib.util import json import sys import time +import tomllib from pathlib import Path from types import SimpleNamespace @@ -777,6 +778,10 @@ def test_local_flux_runtime_and_gpu_handoff_are_flux_managed(): assert switchyard.count("Claude low=claude_haiku_low") == 4 assert switchyard.count('Treat "think hard"') == 4 assert switchyard.count("Never choose below the") >= 5 + routes = tomllib.loads(switchyard)["routes"] + for route_name in ("auto_fast", "auto_balanced", "auto_deep", "auto_maximum"): + leading_targets = set(routes[route_name]["targets"][:2]) + assert leading_targets == {"codex_sol_xhigh", "claude_opus_xhigh"} model_gate = _documents(HERMES / "model-gate-configmap.yaml")[0]["data"][ "model_gate.py" ]