diff --git a/services/hermes/switchyard-configmap.yaml b/services/hermes/switchyard-configmap.yaml index 8398b7b5..449e9c56 100644 --- a/services/hermes/switchyard-configmap.yaml +++ b/services/hermes/switchyard-configmap.yaml @@ -793,7 +793,7 @@ data: 3. Route image creation and editing before general provider preference. When image-generation/edit tools are available and the user asks to create, transform, restore, colorize, or continue editing an image: select - codex_auto_medium. The image tool—not the conversational model—honors the + codex_auto_balanced_medium. The image tool—not the conversational model—honors the user's local, OpenAI/hosted, or AUTO image-backend choice. Do not select a local Qwen or Claude target for an image-tool boundary: local Qwen cannot reliably carry the full Hermes tool context, and Anthropic supplies the @@ -809,7 +809,7 @@ data: the other hosted provider at the same floor. 5. Choose the provider AUTO target at the exact effort floor. AUTO resolves - an account-visible economy, balanced, or advanced general-purpose model + an account-visible economy, balanced, advanced, or frontier general-purpose model from provider metadata. It uses comparable provider cost data only to break ties between adequate models. Re-evaluate every boundary and resolve "continue" or "do it" from recent context. The manual local route remains @@ -880,7 +880,7 @@ data: 3. Route image creation and editing before general provider preference. When image-generation/edit tools are available and the user asks to create, transform, restore, colorize, or continue editing an image: select - codex_auto_medium. The image tool—not the conversational model—honors the + codex_auto_balanced_medium. The image tool—not the conversational model—honors the user's local, OpenAI/hosted, or AUTO image-backend choice. Do not select a local Qwen or Claude target for an image-tool boundary: local Qwen cannot reliably carry the full Hermes tool context, and Anthropic supplies the @@ -896,7 +896,7 @@ data: the other hosted provider at the same floor. 5. Choose the provider AUTO target at the exact effort floor. AUTO resolves - an account-visible economy, balanced, or advanced general-purpose model + an account-visible economy, balanced, advanced, or frontier general-purpose model from provider metadata. It uses comparable provider cost data only to break ties between adequate models. Re-evaluate every boundary and resolve "continue" or "do it" from recent context. The manual local route remains @@ -973,7 +973,7 @@ data: other provider at the same floor. 4. Choose a provider AUTO target at the exact effort floor. AUTO resolves - an account-visible balanced or advanced general-purpose model from provider + an account-visible balanced, advanced, or frontier general-purpose model from provider metadata; economy targets are intentionally unavailable on this route. Re-evaluate every boundary and resolve "continue" or "do it" from recent context. @@ -1043,7 +1043,7 @@ data: other provider at the same floor. 4. Choose a provider AUTO target at high or xhigh only. AUTO resolves an - account-visible advanced general-purpose model from provider metadata. + account-visible advanced or frontier general-purpose model from provider metadata. Medium and low targets are intentionally unavailable on this route. Re-evaluate every boundary and resolve "continue" or "do it" from recent context. @@ -1121,21 +1121,21 @@ data: A final independent review is Claude; implementing review findings is Codex. 4. Choose an available provider AUTO target at the exact effort floor. - AUTO resolves the current account-visible economy, balanced, or advanced + AUTO resolves the current account-visible economy, balanced, advanced, or frontier general-purpose model. It uses comparable provider cost data only to break ties among models that already satisfy the capability requirement. Do not infer a tier from an unfamiliar model name or unadvertised effort support. Examples: - Critical security migration final review -> worker_claude_auto_xhigh. - Implement critical security review fixes -> worker_codex_auto_xhigh. - Difficult intermittent production failure -> worker_codex_auto_high. - Ordinary component design -> worker_claude_auto_medium. - One spelling correction -> worker_codex_auto_low. + Critical security migration final review -> worker_claude_auto_frontier_xhigh. + Implement critical security review fixes -> worker_codex_auto_advanced_xhigh. + Difficult intermittent production failure -> worker_codex_auto_advanced_high. + Ordinary component design -> worker_claude_auto_balanced_medium. + One spelling correction -> worker_codex_auto_economy_low. Anthropic exhausted + critical independent final review -> - worker_codex_auto_xhigh. + worker_codex_auto_frontier_xhigh. OpenAI exhausted + difficult repository implementation -> - worker_claude_auto_high. + worker_claude_auto_advanced_high. Before responding, verify the provider is available and effort is not below the floor. Return only the required decision object. diff --git a/services/hermes/switchyard-deployment.yaml b/services/hermes/switchyard-deployment.yaml index 7e5be545..88f29b9b 100644 --- a/services/hermes/switchyard-deployment.yaml +++ b/services/hermes/switchyard-deployment.yaml @@ -22,7 +22,7 @@ spec: labels: app: hermes-switchyard annotations: - ai.bstein.dev/config-rev: "20260913-capability-effort-v3" + ai.bstein.dev/config-rev: "20260913-capability-effort-v4" prometheus.io/scrape: "true" prometheus.io/port: "9005" prometheus.io/path: /metrics diff --git a/testing/tests/test_hermes_chat_routing_runtime.py b/testing/tests/test_hermes_chat_routing_runtime.py index dad16160..c4d971a1 100644 --- a/testing/tests/test_hermes_chat_routing_runtime.py +++ b/testing/tests/test_hermes_chat_routing_runtime.py @@ -4,6 +4,7 @@ from __future__ import annotations import importlib.util import json +import re import sqlite3 import sys import tomllib @@ -411,7 +412,7 @@ def test_local_flux_runtime_and_gpu_handoff_are_flux_managed(): assert "route/local/qwen2.5-14b/medium" in switchyard assert "Anthropic and Claude name the same provider" in switchyard assert "OpenAI and Codex name the same provider" in switchyard - assert "account-visible economy, balanced, or advanced" in switchyard + assert "account-visible economy, balanced, advanced, or frontier" in switchyard assert "Choose across every configured Codex and Claude family" not in switchyard assert switchyard.count('Treat "think hard"') == 4 assert switchyard.count("Never choose below the") >= 5 @@ -526,3 +527,32 @@ def test_local_flux_runtime_and_gpu_handoff_are_flux_managed(): "model_gate.py" ] assert "qwen2.5:14b-instruct-q4_0" in model_gate + + +def test_classifier_prompt_examples_are_allowed_and_schema_valid(): + """Classifier prompt examples must name configured capability targets.""" + switchyard = _documents(HERMES / "switchyard-configmap.yaml")[0]["data"][ + "routes.toml" + ] + routes = tomllib.loads(switchyard)["routes"] + target_pattern = re.compile( + r"\b(?:worker_)?(?:codex|claude)_auto_[a-z0-9_]+\b" + ) + classifier_count = 0 + example_count = 0 + for route in routes.values(): + if route.get("type") != "llm_classifier": + continue + classifier_count += 1 + examples = target_pattern.findall(route["prompt"]) + example_count += len(examples) + schema = json.loads(route["response_schema"]) + pattern = schema["properties"]["decision"]["properties"]["target"][ + "pattern" + ] + allowed_targets = set(route["targets"]) + assert all(target in allowed_targets for target in examples), route["id"] + assert all(re.fullmatch(pattern, target) for target in examples), route["id"] + + assert classifier_count >= 5 + assert example_count >= 8