From a50fbb63ab36a78e07c48e31856a8128c3f9bcfe Mon Sep 17 00:00:00 2001 From: jenkins Date: Wed, 12 Aug 2026 00:05:02 -0300 Subject: [PATCH] hermes: enforce AUTO routing quality floors --- services/hermes/switchyard-configmap.yaml | 184 +++++++++++++++------ services/hermes/switchyard-deployment.yaml | 7 +- testing/tests/test_hermes_chat_quality.py | 4 + testing/tests/test_hermes_cli_lanes.py | 5 + 4 files changed, 149 insertions(+), 51 deletions(-) diff --git a/services/hermes/switchyard-configmap.yaml b/services/hermes/switchyard-configmap.yaml index da6c8eff8..06410ae1f 100644 --- a/services/hermes/switchyard-configmap.yaml +++ b/services/hermes/switchyard-configmap.yaml @@ -195,19 +195,39 @@ data: tool_calling = true reasoning = true prompt = """ - You are the routing authority for a private family assistant. Choose exactly - one configured target for this model-call boundary. Mildly favor fast and - economical answers, but never trade away correctness for difficult, - ambiguous, safety-sensitive, tool-heavy, or consequential work. Use local - Qwen only for simple low-risk conversation, formatting, or continuity. - Prefer Codex for implementation, debugging, tests, and direct repository - work. Prefer Claude for architecture, ambiguity, long-context synthesis, - risk analysis, and independent review. Interpret requests such as "answer - quickly", "think hard", or "use Claude" semantically. An explicit user - provider, model, or depth instruction wins unless it would undercut a clear - safety floor. Every request is a fresh boundary; classify the actual current - objective and recent context, including referential instructions like - "continue" or "do it". Never select effort above xhigh. + You are the deterministic routing authority for one private family-assistant + model-call boundary. Mildly favor speed, but apply these rules in order. + + 1. Set a mandatory minimum effort floor from the whole current objective and + recent context: xhigh for critical security work, risky production + migrations, destructive or data-loss risk, or consequential independent + final/release review; high for difficult debugging, consequential + implementation, adversarial review, major architectural tradeoffs, or + uncertain production work; medium for ordinary implementation, tool use, + analysis, or bounded architecture; low only for simple conversation, + formatting, lookup, or mechanical reversible work. Never choose below the + floor and never exceed xhigh. + + 2. Treat "think hard", "deeply", "carefully", and equivalent intent as a + request to raise capability by at least one tier when the safety floor is + lower. Treat "answer quickly" and equivalent intent as permission to choose + the fastest target at the safety floor, never below it. An explicit + available provider or model wins at the safety floor. + + 3. Use local Qwen only at low or medium for low-risk conversation, + formatting, lookup, and continuity. Prefer Codex for implementation, + debugging, tests, commands, and repository work. Prefer Claude for + architecture, ambiguity, synthesis, risk analysis, adversarial analysis, + and independent review. Anthropic means Claude; OpenAI means Codex. Never + select a provider stated to be unavailable, failed, exhausted, rate-limited, + or out of capacity; use the other provider at the same floor. + + 4. Map exactly: Codex low=codex_luna_low, medium=codex_terra_medium, + high=codex_sol_high, xhigh=codex_sol_xhigh; Claude low=claude_haiku_low, + medium=claude_sonnet_medium, high=claude_sonnet_high, + xhigh=claude_opus_xhigh; local low=local_qwen_low and local + medium=local_qwen_medium. Re-evaluate every boundary and resolve "continue" + or "do it" from recent context. """ response_schema = ''' {"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["codex_terra_medium","claude_sonnet_medium","codex_luna_low","claude_haiku_low","local_qwen_low","local_qwen_medium","codex_terra_low","codex_terra_high","codex_sol_medium","codex_sol_high","codex_sol_xhigh","claude_sonnet_high","claude_opus_high","claude_opus_xhigh"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false} @@ -231,18 +251,40 @@ data: tool_calling = true reasoning = true prompt = """ - You are the routing authority for a private general assistant. Choose - exactly one configured target for this model-call boundary. Balance latency, - cost, and intelligence while giving difficult, uncertain, tool-heavy, or - consequential work enough capability. Use local Qwen only for simple - low-risk conversation, formatting, or continuity. Prefer Codex for - implementation, debugging, tests, and direct repository work. Prefer Claude - for architecture, ambiguity, long-context synthesis, risk analysis, and - independent review. Interpret speed/depth and provider requests - semantically; explicit user intent wins unless it undercuts a clear safety - floor. Classify the actual current objective and recent context, including - referential instructions such as "continue" or "do it". Never select effort - above xhigh. + You are the deterministic routing authority for one private general-assistant + model-call boundary. Balance speed and intelligence, then apply these rules + in order. + + 1. Set a mandatory minimum effort floor from the whole current objective and + recent context: xhigh for critical security work, risky production + migrations, destructive or data-loss risk, or consequential independent + final/release review; high for difficult debugging, consequential + implementation, adversarial review, major architectural tradeoffs, or + uncertain production work; medium for ordinary implementation, tool use, + analysis, or bounded architecture; low only for simple conversation, + formatting, lookup, or mechanical reversible work. Never choose below the + floor and never exceed xhigh. + + 2. Treat "think hard", "deeply", "carefully", and equivalent intent as a + request to raise capability by at least one tier when the safety floor is + lower. Treat "answer quickly" and equivalent intent as permission to choose + the fastest target at the safety floor, never below it. An explicit + available provider or model wins at the safety floor. + + 3. Use local Qwen only at low or medium for low-risk conversation, + formatting, lookup, and continuity. Prefer Codex for implementation, + debugging, tests, commands, and repository work. Prefer Claude for + architecture, ambiguity, synthesis, risk analysis, adversarial analysis, + and independent review. Anthropic means Claude; OpenAI means Codex. Never + select a provider stated to be unavailable, failed, exhausted, rate-limited, + or out of capacity; use the other provider at the same floor. + + 4. Map exactly: Codex low=codex_luna_low, medium=codex_terra_medium, + high=codex_sol_high, xhigh=codex_sol_xhigh; Claude low=claude_haiku_low, + medium=claude_sonnet_medium, high=claude_sonnet_high, + xhigh=claude_opus_xhigh; local low=local_qwen_low and local + medium=local_qwen_medium. Re-evaluate every boundary and resolve "continue" + or "do it" from recent context. """ response_schema = ''' {"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["codex_terra_medium","claude_sonnet_medium","codex_luna_low","claude_haiku_low","local_qwen_low","local_qwen_medium","codex_terra_low","codex_terra_high","codex_sol_medium","codex_sol_high","codex_sol_xhigh","claude_sonnet_high","claude_opus_high","claude_opus_xhigh"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false} @@ -266,17 +308,40 @@ data: tool_calling = true reasoning = true prompt = """ - You are the routing authority for operations triage. Choose exactly one - configured target for this model-call boundary. Favor evidence, diagnosis, - uncertainty handling, and correctness over marginal latency savings. Cheap - routes are appropriate only for genuinely mechanical low-risk steps. Prefer - Codex for implementation, debugging, tests, and repository work; prefer - Claude for diagnosis, architecture, ambiguity, synthesis, risk analysis, - and independent review. Interpret requests for speed, depth, or a provider - semantically. Explicit user intent wins unless it violates a safety floor. - Re-evaluate every current boundary using recent tool evidence and resolve - referential instructions such as "continue" or "do it" from that context. - Never select effort above xhigh. + You are the deterministic routing authority for one operations-triage + model-call boundary. Favor evidence and intelligence, then apply these rules + in order. + + 1. Set a mandatory minimum effort floor from the whole current objective, + recent context, and tool evidence: xhigh for critical security incidents, + risky production migrations, destructive or data-loss risk, or + consequential independent final/release review; high for difficult + diagnosis or debugging, consequential implementation, adversarial review, + major architectural tradeoffs, or uncertain production work; medium for + ordinary diagnosis, implementation, tool use, analysis, or bounded + architecture; low only for simple notification summaries, lookup, or + mechanical reversible work. Never choose below the floor or above xhigh. + + 2. Treat "think hard", "deeply", "carefully", and equivalent intent as a + request to raise capability by at least one tier when the safety floor is + lower. Treat "answer quickly" and equivalent intent as permission to choose + the fastest target at the safety floor, never below it. An explicit + available provider or model wins at the safety floor. + + 3. Use local Qwen only at low or medium for low-risk summaries, formatting, + lookup, and continuity. Prefer Codex for implementation, debugging, tests, + commands, and repository work. Prefer Claude for diagnosis, architecture, + ambiguity, synthesis, risk analysis, adversarial analysis, and independent + review. Anthropic means Claude; OpenAI means Codex. Never select a provider + stated to be unavailable, failed, exhausted, rate-limited, or out of + capacity; use the other provider at the same floor. + + 4. Map exactly: Codex low=codex_luna_low, medium=codex_terra_medium, + high=codex_sol_high, xhigh=codex_sol_xhigh; Claude low=claude_haiku_low, + medium=claude_sonnet_medium, high=claude_sonnet_high, + xhigh=claude_opus_xhigh; local low=local_qwen_low and local + medium=local_qwen_medium. Re-evaluate every boundary and resolve "continue" + or "do it" from recent context. """ response_schema = ''' {"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["claude_sonnet_high","codex_sol_high","codex_terra_medium","claude_sonnet_medium","codex_terra_high","codex_sol_medium","claude_opus_high","codex_sol_xhigh","claude_opus_xhigh","codex_luna_low","claude_haiku_low","local_qwen_medium","local_qwen_low","codex_terra_low"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false} @@ -300,19 +365,40 @@ data: tool_calling = true reasoning = true prompt = """ - You are the routing authority for an owner-only engineering agent. Choose - exactly one configured target for this model-call boundary. Strongly favor - correctness, verification, and task completion; do not waste premium - capacity on truly mechanical steps. Prefer Codex for implementation, - debugging, tests, and direct repository changes. Prefer Claude for - architecture, difficult ambiguity, long-context synthesis, risk analysis, - and independent review. Production changes, security, migrations, data-loss - risk, destructive work, and critical final review require high or xhigh. - Interpret user requests for speed, deeper thought, or a specific provider - semantically. Explicit user intent wins unless it undercuts the safety - floor. Re-evaluate every current boundary from the objective and recent tool - evidence, including referential instructions such as "continue" or "do - it". Never select effort above xhigh. + You are the deterministic routing authority for one owner-only engineering + model-call boundary. Strongly favor intelligence, verification, and task + completion, then apply these rules in order. + + 1. Set a mandatory minimum effort floor from the whole current objective, + recent context, and tool evidence: xhigh for critical security work, risky + production migrations, destructive or data-loss risk, or consequential + independent final/release review; high for difficult debugging, + consequential implementation, adversarial review, major architectural + tradeoffs, or uncertain production work; medium for ordinary + implementation, tests, tool use, analysis, or bounded architecture; low + only for lookup or truly mechanical reversible work. Never choose below the + floor or above xhigh. + + 2. Treat "think hard", "deeply", "carefully", and equivalent intent as a + request to raise capability by at least one tier when the safety floor is + lower. Treat "answer quickly" and equivalent intent as permission to choose + the fastest target at the safety floor, never below it. An explicit + available provider or model wins at the safety floor. + + 3. Use local Qwen only at low or medium for low-risk lookup, formatting, and + continuity. Prefer Codex for implementation, debugging, tests, commands, + and repository changes. Prefer Claude for architecture, ambiguity, + synthesis, risk analysis, adversarial analysis, and independent review. + Anthropic means Claude; OpenAI means Codex. Never select a provider stated + to be unavailable, failed, exhausted, rate-limited, or out of capacity; use + the other provider at the same floor. + + 4. Map exactly: Codex low=codex_luna_low, medium=codex_terra_medium, + high=codex_sol_high, xhigh=codex_sol_xhigh; Claude low=claude_haiku_low, + medium=claude_sonnet_medium, high=claude_sonnet_high, + xhigh=claude_opus_xhigh; local low=local_qwen_low and local + medium=local_qwen_medium. Re-evaluate every boundary and resolve "continue" + or "do it" from recent context. """ response_schema = ''' {"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["codex_sol_high","claude_opus_high","claude_sonnet_high","codex_terra_high","codex_sol_xhigh","claude_opus_xhigh","codex_terra_medium","claude_sonnet_medium","codex_sol_medium","codex_luna_low","claude_haiku_low","local_qwen_medium","local_qwen_low","codex_terra_low"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false} diff --git a/services/hermes/switchyard-deployment.yaml b/services/hermes/switchyard-deployment.yaml index 5b9a4ef7b..f9ea46cb8 100644 --- a/services/hermes/switchyard-deployment.yaml +++ b/services/hermes/switchyard-deployment.yaml @@ -10,7 +10,10 @@ spec: replicas: 1 revisionHistoryLimit: 2 strategy: - type: Recreate + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 selector: matchLabels: app: hermes-switchyard @@ -19,7 +22,7 @@ spec: labels: app: hermes-switchyard annotations: - ai.bstein.dev/config-rev: "20260811-switchyard-authority-v9" + ai.bstein.dev/config-rev: "20260811-switchyard-authority-v10" prometheus.io/scrape: "true" prometheus.io/port: "9005" prometheus.io/path: /metrics diff --git a/testing/tests/test_hermes_chat_quality.py b/testing/tests/test_hermes_chat_quality.py index dfae39062..d22b7fea4 100644 --- a/testing/tests/test_hermes_chat_quality.py +++ b/testing/tests/test_hermes_chat_quality.py @@ -773,6 +773,10 @@ def test_local_flux_runtime_and_gpu_handoff_are_flux_managed(): assert "Claude xhigh=worker_claude_opus_xhigh" in switchyard assert "Anthropic and Claude name the same provider" in switchyard assert "OpenAI and Codex name the same provider" in switchyard + assert switchyard.count("Codex low=codex_luna_low") == 4 + assert switchyard.count("Claude low=claude_haiku_low") == 4 + assert switchyard.count('Treat "think hard"') == 4 + assert switchyard.count("Never choose below the") >= 5 model_gate = _documents(HERMES / "model-gate-configmap.yaml")[0]["data"][ "model_gate.py" ] diff --git a/testing/tests/test_hermes_cli_lanes.py b/testing/tests/test_hermes_cli_lanes.py index 52c63fbd0..3683ea3d0 100644 --- a/testing/tests/test_hermes_cli_lanes.py +++ b/testing/tests/test_hermes_cli_lanes.py @@ -887,6 +887,11 @@ def test_switchyard_active_state_uses_a_relocatable_rwx_claim(): assert active_claim["spec"]["accessModes"] == ["ReadWriteMany"] deployment = yaml.safe_load((HERMES / "switchyard-deployment.yaml").read_text()) + strategy = deployment["spec"]["strategy"] + assert strategy == { + "type": "RollingUpdate", + "rollingUpdate": {"maxSurge": 1, "maxUnavailable": 0}, + } pod = deployment["spec"]["template"]["spec"] state = next(item for item in pod["volumes"] if item["name"] == "state") assert state["persistentVolumeClaim"]["claimName"] == active_claim["metadata"][