From d81778013fb29ee00dfd2ff230bafb122271fba9 Mon Sep 17 00:00:00 2001 From: jenkins Date: Tue, 11 Aug 2026 23:47:18 -0300 Subject: [PATCH] hermes: enforce worker routing quality floors --- services/hermes/switchyard-configmap.yaml | 64 ++++++++++++++++++---- services/hermes/switchyard-deployment.yaml | 2 +- testing/tests/test_hermes_chat_quality.py | 4 ++ 3 files changed, 59 insertions(+), 11 deletions(-) diff --git a/services/hermes/switchyard-configmap.yaml b/services/hermes/switchyard-configmap.yaml index ad438875c..da6c8eff8 100644 --- a/services/hermes/switchyard-configmap.yaml +++ b/services/hermes/switchyard-configmap.yaml @@ -336,16 +336,60 @@ data: tool_calling = false reasoning = true prompt = """ - You are the sole routing authority for one durable engineering-worker - launch. Choose exactly one configured worker target from the current - objective, acceptance criteria, risk, and any prior-provider failure in the - request. Prefer Codex for implementation, debugging, tests, and repository - changes. Prefer Claude for architecture, ambiguity, synthesis, adversarial - analysis, and independent review. Use low only for mechanical bounded work, - medium for ordinary work, high for difficult or consequential work, and - xhigh for critical security, migration, destructive-risk, or independent - final review. Never exceed xhigh. If the request says a provider has failed - or exhausted capacity, do not select that provider. + You are the deterministic routing authority for one durable engineering + worker launch. Choose exactly one configured target. Apply these steps in + order. + + 1. Set a mandatory minimum effort floor from the whole objective: + - xhigh: critical security work; a risky production migration; destructive + or data-loss risk; or an independent final/release review of consequential + changes. + - high: difficult debugging, consequential implementation, adversarial + review, architecture with major tradeoffs, or uncertain production work. + - medium: ordinary implementation, tests, analysis, or bounded architecture + work. + - low: only mechanical, reversible, tightly bounded work such as a typo, + formatting, or a simple lookup. + Never choose below the floor and never exceed xhigh. + + 2. Determine provider availability before provider preference: + - Anthropic and Claude name the same provider. If either is failed, + unavailable, exhausted, rate-limited, or out of capacity, Claude is + unavailable. + - OpenAI and Codex name the same provider. If either is failed, unavailable, + exhausted, rate-limited, or out of capacity, Codex is unavailable. + - Never select an unavailable provider. Use the available provider at the + same effort floor. + + 3. Choose the provider for the dominant action when both are available: + - Codex: implementation, debugging, tests, commands, and repository changes. + - Claude: architecture, ambiguity, synthesis, risk analysis, adversarial + analysis, and independent review. + A final independent review is Claude; implementing review findings is Codex. + + 4. Map provider and effort exactly: + Codex low=worker_codex_luna_low; + Codex medium=worker_codex_terra_medium; + Codex high=worker_codex_sol_high; + Codex xhigh=worker_codex_sol_xhigh. + Claude low=worker_claude_haiku_low; + Claude medium=worker_claude_sonnet_medium; + Claude high=worker_claude_sonnet_high; + Claude xhigh=worker_claude_opus_xhigh. + + Examples: + Critical security migration final review -> worker_claude_opus_xhigh. + Implement critical security review fixes -> worker_codex_sol_xhigh. + Difficult intermittent production failure -> worker_codex_sol_high. + Ordinary component design -> worker_claude_sonnet_medium. + One spelling correction -> worker_codex_luna_low. + Anthropic exhausted + critical independent final review -> + worker_codex_sol_xhigh. + OpenAI exhausted + difficult repository implementation -> + worker_claude_sonnet_high. + + Before responding, verify the provider is available and effort is not below + the floor. Return only the required decision object. """ response_schema = ''' {"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["worker_codex_sol_high","worker_claude_sonnet_high","worker_codex_sol_xhigh","worker_claude_opus_xhigh","worker_codex_terra_medium","worker_claude_sonnet_medium","worker_codex_luna_low","worker_claude_haiku_low"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false} diff --git a/services/hermes/switchyard-deployment.yaml b/services/hermes/switchyard-deployment.yaml index 0d23c8097..5b9a4ef7b 100644 --- a/services/hermes/switchyard-deployment.yaml +++ b/services/hermes/switchyard-deployment.yaml @@ -19,7 +19,7 @@ spec: labels: app: hermes-switchyard annotations: - ai.bstein.dev/config-rev: "20260811-switchyard-authority-v8" + ai.bstein.dev/config-rev: "20260811-switchyard-authority-v9" prometheus.io/scrape: "true" prometheus.io/port: "9005" prometheus.io/path: /metrics diff --git a/testing/tests/test_hermes_chat_quality.py b/testing/tests/test_hermes_chat_quality.py index 780cc68b1..dfae39062 100644 --- a/testing/tests/test_hermes_chat_quality.py +++ b/testing/tests/test_hermes_chat_quality.py @@ -769,6 +769,10 @@ def test_local_flux_runtime_and_gpu_handoff_are_flux_managed(): assert 'id = "qwen2.5:14b-instruct-q4_0"' in switchyard assert "qwen2.5:3b-instruct-q4_0" not in switchyard assert "route/local/qwen2.5-14b/medium" in switchyard + assert "Codex xhigh=worker_codex_sol_xhigh" in switchyard + assert "Claude xhigh=worker_claude_opus_xhigh" in switchyard + assert "Anthropic and Claude name the same provider" in switchyard + assert "OpenAI and Codex name the same provider" in switchyard model_gate = _documents(HERMES / "model-gate-configmap.yaml")[0]["data"][ "model_gate.py" ]