diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml index 907ed1fe..1cdd8d8f 100644 --- a/services/hermes/agent-deployment.yaml +++ b/services/hermes/agent-deployment.yaml @@ -25,7 +25,7 @@ spec: ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback ai.bstein.dev/placement: primary amd64 accelerator titan-22; arm64 rpi5 fleet fallback; storage-backbone nodes excluded - ai.bstein.dev/config-rev: "20260912-dashboard-bootstrap-visibility" + ai.bstein.dev/config-rev: "20260913-provider-model-catalog-v2" prometheus.io/scrape: "true" prometheus.io/path: /metrics prometheus.io/port: "9010" @@ -475,11 +475,11 @@ spec: # which self-heals after the pod has run on the other architecture. case "$(uname -m)" in x86_64) nodearch=x64 ;; aarch64) nodearch=arm64 ;; *) nodearch=unknown ;; esac codex_native="${tools}/lib/node_modules/@openai/codex-linux-${nodearch}" - if [ ! -f "${tools}/.cli-versions-0.147.0-2.1.226-${arch}" ] || [ ! -d "${codex_native}" ]; then + if [ ! -f "${tools}/.cli-versions-0.154.0-2.1.226-${arch}" ] || [ ! -d "${codex_native}" ]; then npm install --global --omit=dev --no-audit --no-fund --prefix "${tools}" \ - @openai/codex@0.147.0 \ + @openai/codex@0.154.0 \ @anthropic-ai/claude-code@2.1.226 - touch "${tools}/.cli-versions-0.147.0-2.1.226-${arch}" + touch "${tools}/.cli-versions-0.154.0-2.1.226-${arch}" fi kubectl_version="$("${tools}/bin/kubectl" version --client --output=json 2>/dev/null || true)" case "${kubectl_version}" in *\"gitVersion\":\"v1.33.3\"*) kubectl_ready=1 ;; *) kubectl_ready=0 ;; esac diff --git a/services/hermes/chat-configmap.yaml b/services/hermes/chat-configmap.yaml index cfe0284a..7a9e9db2 100644 --- a/services/hermes/chat-configmap.yaml +++ b/services/hermes/chat-configmap.yaml @@ -61,9 +61,11 @@ data: atlas/auto/balanced: {provider: atlas-switchyard, model: atlas/auto/balanced} atlas/auto/deep: {provider: atlas-switchyard, model: atlas/auto/deep} atlas/auto/maximum: {provider: atlas-switchyard, model: atlas/auto/maximum} + atlas/manual/codex/auto: {provider: atlas-switchyard, model: atlas/manual/codex/auto} atlas/manual/codex/luna: {provider: atlas-switchyard, model: atlas/manual/codex/luna} atlas/manual/codex/terra: {provider: atlas-switchyard, model: atlas/manual/codex/terra} atlas/manual/codex/sol: {provider: atlas-switchyard, model: atlas/manual/codex/sol} + atlas/manual/claude/auto: {provider: atlas-switchyard, model: atlas/manual/claude/auto} atlas/manual/claude/haiku: {provider: atlas-switchyard, model: atlas/manual/claude/haiku} atlas/manual/claude/fable: {provider: atlas-switchyard, model: atlas/manual/claude/fable} atlas/manual/claude/sonnet: {provider: atlas-switchyard, model: atlas/manual/claude/sonnet} diff --git a/services/hermes/chat-statefulset.yaml b/services/hermes/chat-statefulset.yaml index 5fbe1a75..b87f72a6 100644 --- a/services/hermes/chat-statefulset.yaml +++ b/services/hermes/chat-statefulset.yaml @@ -29,7 +29,7 @@ spec: ai.bstein.dev/router-wire-contract: ollama-numeric-keepalive ai.bstein.dev/isolation: one Hermes process and PVC per Keycloak subject ai.bstein.dev/model-policy: uniform automatic policy with per-user overrides - ai.bstein.dev/config-rev: "20260816-telegram-topics" + ai.bstein.dev/config-rev: "20260913-provider-model-catalog-v2" ai.bstein.dev/hux-config-rev: "20260824-hux-v1" vault.hashicorp.com/agent-inject: "true" vault.hashicorp.com/role: hermes-chat diff --git a/services/hermes/kustomization.yaml b/services/hermes/kustomization.yaml index 54f46d04..2e25b8f6 100644 --- a/services/hermes/kustomization.yaml +++ b/services/hermes/kustomization.yaml @@ -141,6 +141,9 @@ configMapGenerator: - worker_route_broker.py=scripts/worker_route_broker.py - hermes_coordinator.py=scripts/hermes_coordinator.py - hermes_model_routing.py=scripts/hermes_model_routing.py + - provider_model_catalog.py=scripts/provider_model_catalog.py + - provider_model_discovery.py=scripts/provider_model_discovery.py + - claude_model_discovery.py=scripts/claude_model_discovery.py - hermes_stt_client.py=scripts/hermes_stt_client.py - image_broker.py=scripts/image_broker.py - install_agent_tools.sh=scripts/install_agent_tools.sh diff --git a/services/hermes/plugins/auto-router/__init__.py b/services/hermes/plugins/auto-router/__init__.py index 3231a02f..6fb42914 100644 --- a/services/hermes/plugins/auto-router/__init__.py +++ b/services/hermes/plugins/auto-router/__init__.py @@ -51,10 +51,12 @@ MANUAL_ROUTES = frozenset( "atlas/manual/codex/luna", "atlas/manual/codex/terra", "atlas/manual/codex/sol", + "atlas/manual/codex/auto", "atlas/manual/claude/haiku", "atlas/manual/claude/fable", "atlas/manual/claude/sonnet", "atlas/manual/claude/opus", + "atlas/manual/claude/auto", "atlas/manual/local/qwen-14b", } ) @@ -69,8 +71,8 @@ EXACT_MANUAL_ROUTES = frozenset( ) ALL_REQUEST_ROUTES = ALL_ROUTES | EXACT_MANUAL_ROUTES PROVIDER_DEFAULT = { - "codex": "atlas/manual/codex/terra", - "claude": "atlas/manual/claude/sonnet", + "codex": "atlas/manual/codex/auto", + "claude": "atlas/manual/claude/auto", "local": "atlas/manual/local/qwen-14b", } diff --git a/services/hermes/scripts/claude_model_discovery.py b/services/hermes/scripts/claude_model_discovery.py new file mode 100644 index 00000000..4e75580c --- /dev/null +++ b/services/hermes/scripts/claude_model_discovery.py @@ -0,0 +1,280 @@ +#!/usr/bin/env python3 +"""Discover the current Claude subscription model catalog without inference. + +Claude Code exposes the account-visible ``ModelInfo[]`` through its Agent SDK +``initialize`` control response. This module speaks that documented stream-JSON +control protocol directly instead of treating authentication status or a cached +model list as a live catalog. It intentionally retains only non-secret +ModelInfo metadata. +""" + +from __future__ import annotations + +import json +import os +import signal +import subprocess +import uuid +from dataclasses import dataclass +from typing import Any, Mapping, Sequence + + +DEFAULT_CLAUDE_BIN = "/opt/data/tools/bin/claude" +DEFAULT_TIMEOUT_SECONDS = 30.0 +MAX_MODELS = 256 +EFFORT_LEVELS = frozenset({"low", "medium", "high", "xhigh", "max"}) + + +@dataclass(frozen=True) +class ClaudeModel: + """Safe metadata for one model advertised by the native Claude CLI.""" + + value: str + resolved_model: str | None + display_name: str + description: str + supports_effort: bool + supported_effort_levels: tuple[str, ...] + supports_adaptive_thinking: bool + supports_fast_mode: bool + supports_auto_mode: bool + disabled: bool + + def as_dict(self) -> dict[str, Any]: + """Serialize only ModelInfo metadata, preserving Claude's alias value.""" + return { + "value": self.value, + "resolvedModel": self.resolved_model, + "displayName": self.display_name, + "description": self.description, + "supportsEffort": self.supports_effort, + "supportedEffortLevels": list(self.supported_effort_levels), + "supportsAdaptiveThinking": self.supports_adaptive_thinking, + "supportsFastMode": self.supports_fast_mode, + "supportsAutoMode": self.supports_auto_mode, + "disabled": self.disabled, + } + + +@dataclass(frozen=True) +class ClaudeModelDiscovery: + """Result of one bounded, subscription-authenticated catalog probe.""" + + models: tuple[ClaudeModel, ...] + live: bool + connected: bool + state: str + error_code: str | None = None + + +def _native_environment(base: Mapping[str, str] | None = None) -> dict[str, str]: + """Keep mounted OAuth configuration while preventing API-key billing lanes.""" + environment = dict(os.environ if base is None else base) + environment.pop("ANTHROPIC_API_KEY", None) + environment.pop("CLAUDE_API_KEY", None) + return environment + + +def _command(claude_bin: str) -> list[str]: + """Build a no-inference CLI invocation with all tools and MCP disabled.""" + return [ + claude_bin, + "--input-format", + "stream-json", + "--output-format", + "stream-json", + "--verbose", + "--print", + "--no-session-persistence", + "--safe-mode", + "--tools", + "", + "--strict-mcp-config", + "--mcp-config", + '{"mcpServers":{}}', + ] + + +def _initialize_request() -> str: + """Build the Claude Agent SDK initialize request for ModelInfo discovery.""" + return json.dumps( + { + "type": "control_request", + "request_id": f"hermes-model-catalog-{uuid.uuid4()}", + "request": {"subtype": "initialize", "hooks": {}}, + }, + separators=(",", ":"), + ) + + +def _stop_process(process: subprocess.Popen[str]) -> None: + """End a timed-out CLI process and its process group without leaking output.""" + if process.poll() is not None: + return + try: + os.killpg(process.pid, signal.SIGTERM) + except (OSError, ProcessLookupError): + process.terminate() + try: + process.communicate(timeout=2) + return + except subprocess.TimeoutExpired: + pass + try: + os.killpg(process.pid, signal.SIGKILL) + except (OSError, ProcessLookupError): + process.kill() + try: + process.communicate(timeout=2) + except subprocess.TimeoutExpired: + pass + + +def _run_initialize( + command: Sequence[str], environment: Mapping[str, str], timeout_seconds: float +) -> tuple[str | None, str | None]: + """Run initialize once, returning stream output or a non-sensitive error code.""" + try: + process = subprocess.Popen( + list(command), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, + env=dict(environment), + start_new_session=True, + ) + except OSError: + return None, "spawn-failed" + try: + request = _initialize_request() + "\n" + stdout, _ = process.communicate(request, timeout=timeout_seconds) + except subprocess.TimeoutExpired: + _stop_process(process) + return None, "timeout" + except OSError: + _stop_process(process) + return None, "transport-error" + if process.returncode != 0: + return None, "cli-error" + return stdout, None + + +def _text(value: Any, *, maximum: int = 1024) -> str: + """Return bounded string metadata and reject non-string protocol values.""" + return value[:maximum] if isinstance(value, str) else "" + + +def _model_info(value: Any) -> ClaudeModel | None: + """Validate one ModelInfo record without inventing aliases or identifiers.""" + if not isinstance(value, dict): + return None + alias = _text(value.get("value")) + if not alias: + return None + resolved_model = _text(value.get("resolvedModel")) or None + efforts = value.get("supportedEffortLevels") + supported_effort_levels = ( + tuple( + level + for level in efforts + if isinstance(level, str) and level in EFFORT_LEVELS + ) + if isinstance(efforts, list) + else () + ) + return ClaudeModel( + value=alias, + resolved_model=resolved_model, + display_name=_text(value.get("displayName")), + description=_text(value.get("description")), + supports_effort=value.get("supportsEffort") is True, + supported_effort_levels=supported_effort_levels, + supports_adaptive_thinking=value.get("supportsAdaptiveThinking") is True, + supports_fast_mode=value.get("supportsFastMode") is True, + supports_auto_mode=value.get("supportsAutoMode") is True, + disabled=value.get("disabled") is True, + ) + + +def _parse_initialize_output(stdout: str) -> ClaudeModelDiscovery: + """Extract all ModelInfo pages present in initialize control responses. + + Claude Code 2.1.226 returns its complete ``models`` array in one response. + The stream parser still gathers every valid initialize response so a future + paged/multi-frame implementation is not silently truncated. The protocol + currently specifies no continuation request, so this function never + fabricates one. + """ + models: list[ClaudeModel] = [] + seen: set[tuple[str, str | None]] = set() + received_models = False + first_party = False + for line in stdout.splitlines(): + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(event, dict) or event.get("type") != "control_response": + continue + response = event.get("response") + if not isinstance(response, dict) or response.get("subtype") != "success": + continue + payload = response.get("response") + if not isinstance(payload, dict) or not isinstance(payload.get("models"), list): + continue + received_models = True + account = payload.get("account") + if isinstance(account, dict) and account.get("apiProvider") == "firstParty": + first_party = True + for raw_model in payload["models"]: + model = _model_info(raw_model) + if model is None or len(models) >= MAX_MODELS: + continue + key = (model.value, model.resolved_model) + if key not in seen: + seen.add(key) + models.append(model) + if not received_models: + return ClaudeModelDiscovery( + (), False, False, "protocol-error", "protocol-error" + ) + if not first_party: + return ClaudeModelDiscovery( + tuple(models), False, False, "not-subscription", "not-first-party" + ) + if not models: + return ClaudeModelDiscovery( + (), True, True, "connected-subscription", "empty-model-catalog" + ) + return ClaudeModelDiscovery(tuple(models), True, True, "connected-subscription") + + +def discover_claude_subscription_models( + *, + claude_bin: str | None = None, + timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS, + environment: Mapping[str, str] | None = None, +) -> ClaudeModelDiscovery: + """Return current subscription models using only Claude's initialize protocol. + + A successful native authentication-status command or a locally cached model + list is deliberately insufficient: ``live`` is true only after a valid, + first-party initialize response supplies the account-visible ``models``. + """ + binary = ( + claude_bin + or os.environ.get("HERMES_CLAUDE_NATIVE_BIN") + or DEFAULT_CLAUDE_BIN + ) + if timeout_seconds <= 0: + return ClaudeModelDiscovery( + (), False, False, "invalid-request", "invalid-timeout" + ) + stdout, error_code = _run_initialize( + _command(binary), _native_environment(environment), timeout_seconds + ) + if error_code: + state = "unavailable" if error_code == "spawn-failed" else "degraded" + return ClaudeModelDiscovery((), False, False, state, error_code) + return _parse_initialize_output(stdout or "") diff --git a/services/hermes/scripts/claude_oauth_broker.py b/services/hermes/scripts/claude_oauth_broker.py index f0eeca4c..7d31f156 100644 --- a/services/hermes/scripts/claude_oauth_broker.py +++ b/services/hermes/scripts/claude_oauth_broker.py @@ -67,6 +67,15 @@ _auth_probe_value: dict[str, Any] = {} HEALTH_POLL_SECONDS: Final = max( 30, int(os.environ.get("HERMES_CLAUDE_HEALTH_POLL_SECONDS", "60")) ) + + +def _catalog_contains(model: str) -> bool: + """Accept a provider-advertised Claude alias without trusting arbitrary IDs.""" + catalog = load_catalog() + providers = catalog.get("providers", {}) if isinstance(catalog, dict) else {} + claude = providers.get("claude", {}) if isinstance(providers, dict) else {} + models = claude.get("models", []) if isinstance(claude, dict) else [] + return isinstance(model, str) and model in models def _read_secret(file_env_name: str) -> str: """Read a secret only from its runtime-mounted file.""" path = os.environ.get(file_env_name, "").strip() @@ -108,7 +117,7 @@ def _route(model: str, payload: dict[str, Any]) -> tuple[str, str]: output_config = payload.get("output_config") if isinstance(output_config, dict) and output_config.get("effort") in EFFORTS: effort = str(output_config["effort"]) - if not model.startswith("claude-"): + if not model.startswith("claude-") and not _catalog_contains(model): raise ValueError("unsupported Claude model") return model, effort @@ -626,7 +635,7 @@ class Handler(BaseHTTPRequestHandler): models = sorted( model for model in raw_models - if isinstance(model, str) and model.startswith("claude-") + if isinstance(model, str) ) self._json(200, {"data": [{"id": model, "type": "model"} for model in models]}) diff --git a/services/hermes/scripts/cli_lane_routing.py b/services/hermes/scripts/cli_lane_routing.py index 63819981..e0b82bdd 100644 --- a/services/hermes/scripts/cli_lane_routing.py +++ b/services/hermes/scripts/cli_lane_routing.py @@ -19,6 +19,7 @@ from cli_lane_config import ( Route, ) from cli_lane_files import load_json +from routing_catalog import catalog_contains def parse_assignee(assignee: str) -> tuple[str | None, str | None]: @@ -107,6 +108,7 @@ def select_route( exclude_reason: str | None = None, switchyard_url: str = SWITCHYARD_URL, open_request: Callable[..., Any] = urllib.request.urlopen, + catalog_model_allowed: Callable[[str, str], bool] = catalog_contains, ) -> Route: """Ask Switchyard to select one native CLI worker at this boundary.""" started = time.monotonic() @@ -149,19 +151,26 @@ def select_route( # Switchyard preserves the stable tier target in the selection header and # top-level response model. The worker broker's assistant content contains # the steward-resolved provider model required by the native CLI. + resolved_valid = False try: response_document = json.loads(response_body) resolved_target = str(response_document["choices"][0]["message"]["content"] or "") resolved_provider, resolved_model, resolved_effort = _decode_worker_target(resolved_target) required_prefix = "gpt-" if provider == "codex" else "claude-" - if ( + resolved_valid = ( resolved_provider == provider and resolved_effort == effort - and resolved_model.startswith(required_prefix) - ): + and ( + resolved_model.startswith(required_prefix) + or (provider == "claude" and catalog_model_allowed(provider, resolved_model)) + ) + ) + if resolved_valid: model = resolved_model except (AttributeError, IndexError, KeyError, RuntimeError, TypeError, ValueError, json.JSONDecodeError): - pass + resolved_valid = False + if model == "auto" and not resolved_valid: + raise RuntimeError("Switchyard did not return a current provider model") if exclude_provider and provider == exclude_provider: alternate = "claude" if provider == "codex" else "codex" guarded = select_route( @@ -169,6 +178,7 @@ def select_route( f"cli-{alternate}-{effort}", switchyard_url=switchyard_url, open_request=open_request, + catalog_model_allowed=catalog_model_allowed, ) return Route( provider=guarded.provider, diff --git a/services/hermes/scripts/codex_broker.py b/services/hermes/scripts/codex_broker.py index bc88b26b..1e12dddd 100644 --- a/services/hermes/scripts/codex_broker.py +++ b/services/hermes/scripts/codex_broker.py @@ -18,7 +18,7 @@ from typing import Any, Iterable import httpx -from routing_catalog import resolve_route +from routing_catalog import load_catalog, resolve_route HOST = os.environ.get("HERMES_CODEX_BROKER_LISTEN_HOST", "0.0.0.0") @@ -580,13 +580,20 @@ class Handler(BaseHTTPRequestHandler): self._json(200, {"ok": True, "provider": "openai-codex"}) return if self.path in {"/models", "/v1/models"}: + catalog = load_catalog() + providers = catalog.get("providers", {}) if isinstance(catalog, dict) else {} + codex = providers.get("codex", {}) if isinstance(providers, dict) else {} + discovered = codex.get("models", []) if isinstance(codex, dict) else [] + models = { + model for model in discovered if isinstance(model, str) and model.startswith("gpt-") + } or FALLBACK_ALLOWED_MODELS self._json( 200, { "object": "list", "data": [ {"id": model, "object": "model", "owned_by": "openai-codex"} - for model in sorted(FALLBACK_ALLOWED_MODELS) + for model in sorted(models) ], }, ) diff --git a/services/hermes/scripts/hermes_model_routing.py b/services/hermes/scripts/hermes_model_routing.py index d4b87259..297c882e 100644 --- a/services/hermes/scripts/hermes_model_routing.py +++ b/services/hermes/scripts/hermes_model_routing.py @@ -6,26 +6,29 @@ from __future__ import annotations import copy import json import os -import re import shutil import subprocess import time -from dataclasses import dataclass from pathlib import Path from typing import Any, Iterable import yaml +from provider_model_catalog import ( + CAPABILITY_TIERS, + EFFORTS, + EFFORT_TIERS, + Catalog, + model_records, + model_version, + select_tier_model, + unique_models, +) +from provider_model_discovery import discover_claude_models, discover_codex_models + CODEX_BASELINE = "gpt-5.6-terra" CLAUDE_BASELINE = "claude-opus-5" -CLAUDE_SUBSCRIPTION_MODELS = ( - "claude-haiku-4-5-20251001", - "claude-fable-5", - "claude-sonnet-5", - "claude-opus-5", -) -EFFORTS = ("low", "medium", "high", "xhigh") ATLAS_FALLBACK = { "provider": "custom", "model": "qwen2.5:14b-instruct-q4_0", @@ -38,166 +41,46 @@ SWITCHYARD_PROVIDER = "atlas-switchyard" SWITCHYARD_API = "http://hermes-switchyard.hermes.svc.cluster.local:9005/v1" SWITCHYARD_AUTO_ROUTE = "atlas/auto/maximum" ROUTING_CATALOG_PATH = os.environ.get("HERMES_ROUTING_CATALOG_PATH", "").strip() -MANAGED_ENV_KEYS = { - "GIT_ASKPASS", - "GIT_TERMINAL_PROMPT", -} +MANAGED_ENV_KEYS = {"GIT_ASKPASS", "GIT_TERMINAL_PROMPT"} RUNTIME_SECRET_ENV_KEYS = { - "ANTHROPIC_API_KEY", - "API_SERVER_KEY", - "CLAUDE_API_KEY", - "CLAUDE_CODE_OAUTH_TOKEN", - "GITEA_TOKEN", - "GITEA_USERNAME", - "HERMES_IMAGE_BROKER_KEY", - "OPENAI_API_KEY", + "ANTHROPIC_API_KEY", "API_SERVER_KEY", "CLAUDE_API_KEY", "CLAUDE_CODE_OAUTH_TOKEN", + "GITEA_TOKEN", "GITEA_USERNAME", "HERMES_IMAGE_BROKER_KEY", "OPENAI_API_KEY", } -@dataclass(frozen=True) -class Catalog: - """Non-secret provider discovery result.""" - - provider: str - models: list[str] - live: bool - connected: bool - state: str +def choose_codex_model(models: Iterable[str], current: str = CODEX_BASELINE, *, balanced: bool = False) -> str: + """Compatibility helper selecting a declared advanced or balanced Codex tier.""" + effort = "medium" if balanced else "xhigh" + tier = "balanced" if balanced else "advanced" + return select_tier_model("codex", models, {}, tier, effort, current)[0] -def _unique_models(models: Iterable[str]) -> list[str]: - """Return normalized model IDs without changing provider order.""" - seen: set[str] = set() - result: list[str] = [] - for model in models: - value = str(model or "").strip() - if value and value.lower() not in seen: - result.append(value) - seen.add(value.lower()) - return result - - -def model_version(model: str) -> tuple[int, ...]: - """Extract a sortable model version while ignoring dated aliases.""" - value = re.sub(r"-\d{8}$", "", model.lower()) - groups = re.findall(r"\d+(?:\.\d+)*", value) - parts: list[int] = [] - for group in groups: - parts.extend(int(piece) for piece in group.split(".")) - return tuple(parts) or (0,) - - -def _codex_quality(model: str) -> int: - """Rank known Codex capability tiers for delegated implementation work.""" - value = model.lower() - if "sol" in value: - return 60 - if "terra" in value: - return 50 - if "codex" in value and "spark" not in value: - return 45 - if re.fullmatch(r"gpt-\d+(?:\.\d+)*", value): - return 40 - if "luna" in value: - return 20 - if "mini" in value or "nano" in value: - return 10 - if "spark" in value: - return 5 - return 0 - - -def choose_codex_model( - models: Iterable[str], current: str = CODEX_BASELINE, *, balanced: bool = False -) -> str: - """Choose a current full Codex model, retaining current when unknown.""" - candidates = [m for m in _unique_models(models) if m.lower().startswith("gpt-")] - full = [m for m in candidates if _codex_quality(m) >= 40] - pool = full or candidates - if not pool: - return current - newest = max(model_version(model) for model in pool) - latest = [model for model in pool if model_version(model) == newest] - if balanced: - terra = [model for model in latest if "terra" in model.lower()] - if terra: - return max(terra, key=lambda model: (_codex_quality(model), model)) - return max(latest, key=lambda model: (_codex_quality(model), model)) - - -def _choose_by_hints( - models: Iterable[str], hints: tuple[str, ...], current: str, prefix: str -) -> str: - """Pick the newest model in the first available capability class.""" - candidates = [ - model - for model in _unique_models(models) - if model.lower().startswith(prefix) - ] - if not candidates: - return current - for hint in hints: - tier = [model for model in candidates if hint in model.lower()] - if tier: - return max(tier, key=lambda model: (model_version(model), model)) - return max(candidates, key=lambda model: (model_version(model), model)) - - -def choose_codex_for_effort( - models: Iterable[str], effort: str, current: str = CODEX_BASELINE -) -> str: - """Choose the account-visible Codex tier for an allowed effort level.""" - hints = { - "low": ("luna", "mini", "spark", "terra", "sol", "codex"), - "medium": ("terra", "codex", "sol", "luna", "mini", "spark"), - "high": ("sol", "codex", "terra", "luna", "mini", "spark"), - "xhigh": ("sol", "codex", "terra", "luna", "mini", "spark"), - } +def choose_codex_for_effort(models: Iterable[str], effort: str, current: str = CODEX_BASELINE) -> str: + """Compatibility helper selecting an effort's generic Codex capability tier.""" if effort not in EFFORTS: raise ValueError(f"unsupported effort: {effort}") - return _choose_by_hints(models, hints[effort], current, "gpt-") - - -def _claude_quality(model: str) -> int: - """Rank known Claude capability tiers for architecture and review work.""" - value = model.lower() - if "opus" in value: - return 40 - if "sonnet" in value: - return 30 - if "fable" in value: - return 20 - if "haiku" in value: - return 10 - return 0 + return select_tier_model("codex", models, {}, EFFORT_TIERS[effort], effort, current)[0] def choose_claude_model(models: Iterable[str], current: str = CLAUDE_BASELINE) -> str: - """Choose the newest full Claude reasoning model visible to the account.""" - candidates = [m for m in _unique_models(models) if m.lower().startswith("claude-")] - full = [m for m in candidates if _claude_quality(m) >= 30] - pool = full or candidates - if not pool: - return current - return max( - pool, - key=lambda model: (model_version(model), _claude_quality(model), model), - ) + """Compatibility helper selecting the declared advanced Claude tier.""" + return select_tier_model("claude", models, {}, "advanced", "xhigh", current)[0] def choose_claude_for_effort( models: Iterable[str], effort: str, current: str = CLAUDE_BASELINE ) -> str: - """Choose the account-visible Claude tier for an allowed effort level.""" - hints = { - "low": ("haiku", "fable", "sonnet", "opus"), - "medium": ("sonnet", "fable", "opus", "haiku"), - "high": ("opus", "sonnet", "fable", "haiku"), - "xhigh": ("opus", "sonnet", "fable", "haiku"), - } + """Compatibility helper selecting an effort's generic Claude capability tier.""" if effort not in EFFORTS: raise ValueError(f"unsupported effort: {effort}") - return _choose_by_hints(models, hints[effort], current, "claude-") + return select_tier_model("claude", models, {}, EFFORT_TIERS[effort], effort, current)[0] + + +# The coordinator retains these private spellings for compact call sites while +# provider_model_catalog owns the metadata policy and its independent tests. +_unique_models = unique_models +_model_records = model_records +_select_tier_model = select_tier_model def _read_yaml(path: Path) -> dict[str, Any]: @@ -245,34 +128,42 @@ def _read_json(path: Path) -> dict[str, Any]: def _previous_provider_models( previous: dict[str, Any], provider: str -) -> tuple[dict[str, str], dict[str, str], list[str]]: - """Return last-known-good effort, tier, and catalog values for a provider.""" +) -> tuple[dict[str, str], dict[str, str], list[str], dict[str, dict[str, Any]]]: + """Return last-known-good effort, selector, models, and metadata values.""" providers = previous.get("providers", {}) record = providers.get(provider, {}) if isinstance(providers, dict) else {} if not isinstance(record, dict): - return {}, {}, [] + return {}, {}, [], {} resolved = record.get("resolved", {}) tiers = record.get("tiers", {}) models = record.get("models", []) + metadata = record.get("model_metadata", {}) return ( dict(resolved) if isinstance(resolved, dict) else {}, dict(tiers) if isinstance(tiers, dict) else {}, _unique_models(models if isinstance(models, list) else []), + { + str(model): dict(details) + for model, details in metadata.items() + if isinstance(model, str) and isinstance(details, dict) + } + if isinstance(metadata, dict) + else {}, ) def build_routing_catalog( codex: Catalog, claude: Catalog, previous: dict[str, Any] | None = None ) -> dict[str, Any]: - """Build a current catalog while retaining working routes during outages.""" + """Build a current catalog while retaining only outage-safe known routes.""" previous = previous or {} + checked_at = int(time.time()) providers: dict[str, Any] = {} specifications = ( ( "codex", codex, - choose_codex_for_effort, - {"luna": "low", "terra": "medium", "sol": "high"}, + {"luna": "economy", "terra": "balanced", "sol": "advanced"}, { "low": "gpt-5.6-luna", "medium": "gpt-5.6-terra", @@ -283,12 +174,11 @@ def build_routing_catalog( ( "claude", claude, - choose_claude_for_effort, { - "haiku": "low", - "fable": "medium", - "sonnet": "high", - "opus": "xhigh", + "haiku": "economy", + "fable": "advanced", + "sonnet": "balanced", + "opus": "advanced", }, { "low": "claude-haiku-4-5-20251001", @@ -298,27 +188,64 @@ def build_routing_catalog( }, ), ) - for name, discovered, chooser, tier_efforts, defaults in specifications: - old_resolved, old_tiers, old_models = _previous_provider_models(previous, name) + for name, discovered, legacy_selectors, defaults in specifications: + old_resolved, old_tiers, old_models, old_metadata = _previous_provider_models( + previous, name + ) + old_providers = previous.get("providers", {}) + old_record = old_providers.get(name, {}) if isinstance(old_providers, dict) else {} + previous_success = ( + int(old_record.get("last_success_at")) + if isinstance(old_record, dict) and isinstance(old_record.get("last_success_at"), int) + else None + ) source_models = discovered.models if discovered.live else old_models + source_metadata = discovered.metadata if discovered.live else old_metadata resolved: dict[str, str] = {} + observations: dict[str, Any] = {} for effort in EFFORTS: + # A successful provider list is authoritative: a retired model + # must not survive it merely because it was last known-good. current = str(old_resolved.get(effort) or defaults[effort]) - resolved[effort] = ( - chooser(source_models, effort, current) - if source_models - else current + selected, observed = _select_tier_model( + name, + source_models, + source_metadata, + EFFORT_TIERS[effort], + effort, + current, + allow_current_fallback=not discovered.live, ) + resolved[effort] = selected + observations.update(observed) tiers: dict[str, str] = {} - for tier, effort in tier_efforts.items(): - current = str(old_tiers.get(tier) or defaults[effort]) - tier_matches = [model for model in source_models if tier in model.lower()] - tiers[tier] = ( - max(tier_matches, key=lambda model: (model_version(model), model)) - if tier_matches - else (resolved[effort] if discovered.live else current) + for capability in CAPABILITY_TIERS: + effort = next( + value for value, target in EFFORT_TIERS.items() if target == capability + ) + current = str(old_tiers.get(capability) or "") + tiers[capability], observed = _select_tier_model( + name, source_models, source_metadata, capability, effort, current, + allow_current_fallback=not discovered.live, + ) + observations.update(observed) + for selector, capability in legacy_selectors.items(): + # Explicit historical family picks are not generic capability + # requests. They must remain exact or become unavailable, never + # silently change to a newer family such as Astra. + exact = [model for model in source_models if selector in model.lower()] + tiers[selector] = ( + exact[0] + if exact + else ("" if discovered.live else str(old_tiers.get(selector) or "")) ) providers[name] = { + "provenance": ( + "live-account" if discovered.live + else ("last-known-good" if old_models else "bootstrap-fallback") + ), + "checked_at": checked_at, + "last_success_at": checked_at if discovered.live else previous_success, "state": discovered.state, "connected": discovered.connected, "live": discovered.live, @@ -327,12 +254,14 @@ def build_routing_catalog( if discovered.live else (old_models or discovered.models) ), + "model_metadata": source_metadata, + "candidates": observations, "resolved": resolved, "tiers": tiers, } return { - "schema_version": 1, - "updated_at": int(time.time()), + "schema_version": 2, + "updated_at": checked_at, "providers": providers, } @@ -385,19 +314,6 @@ def _update_profile_env(path: Path, source: dict[str, str]) -> None: _atomic_write(path, "\n".join(kept).rstrip() + "\n", 0o600) -def _existing_model(config: dict[str, Any], provider: str, default: str) -> str: - """Return the last configured model for a provider or a safe baseline.""" - model_cfg = config.get("model", {}) - if isinstance(model_cfg, dict) and model_cfg.get("provider") == provider: - return str(model_cfg.get("model") or model_cfg.get("default") or default) - fallbacks = config.get("fallback_providers", []) - if isinstance(fallbacks, list): - for fallback in fallbacks: - if isinstance(fallback, dict) and fallback.get("provider") == provider: - return str(fallback.get("model") or default) - return default - - def codex_cli_authenticated() -> bool: """Return whether the installed Codex CLI has a usable local login.""" codex = shutil.which("codex") @@ -420,127 +336,6 @@ def codex_cli_authenticated() -> bool: return False -def discover_codex_models() -> Catalog: - """Use the authenticated Codex endpoint; catalogs are status-only fallback.""" - token = "" - try: - from hermes_cli.auth import resolve_codex_runtime_credentials - - credentials = resolve_codex_runtime_credentials(refresh_if_expiring=True) or {} - token = str(credentials.get("api_key") or "").strip() - except Exception: - token = "" - live: list[str] = [] - if token: - try: - from hermes_cli.codex_models import _fetch_models_from_api - - live = _unique_models(_fetch_models_from_api(token)) - except Exception: - live = [] - if live: - return Catalog("openai-codex", live, True, True, "connected") - try: - from hermes_cli.models import provider_model_ids - - known = _unique_models(provider_model_ids("openai-codex", force_refresh=True)) - except Exception: - known = [] - # The owner agent deliberately uses the local Codex app-server and the - # Codex CLI's ChatGPT login. That credential is not exported into Hermes' - # bearer-token store, so API discovery above may be empty even though the - # runtime is fully authenticated. Treat a successful CLI status check as a - # connected, degraded catalog: routing may use Codex while retaining the - # last-known working model names until app-server discovery is available. - if codex_cli_authenticated(): - return Catalog( - "openai-codex", - known, - False, - True, - "connected-cli", - ) - state = "degraded" if token else "not-configured" - return Catalog("openai-codex", known, False, bool(token), state) - - -def discover_claude_models() -> Catalog: - """Use the native subscription login before any API-key catalog fallback.""" - claude = shutil.which("claude") or os.environ.get("HERMES_CLAUDE_BIN", "") - if claude: - try: - environment = os.environ.copy() - environment.pop("ANTHROPIC_API_KEY", None) - environment.pop("CLAUDE_API_KEY", None) - status = subprocess.run( - [claude, "auth", "status"], - capture_output=True, - check=False, - text=True, - timeout=15, - env=environment, - ) - detail = json.loads(status.stdout) if status.stdout.strip() else {} - if ( - status.returncode == 0 - and isinstance(detail, dict) - and detail.get("loggedIn") is True - and detail.get("apiProvider") == "firstParty" - ): - try: - from hermes_cli.models import provider_model_ids - - known = _unique_models( - provider_model_ids("anthropic", force_refresh=True) - ) - except Exception: - known = [] - return Catalog( - "anthropic", - _unique_models((*CLAUDE_SUBSCRIPTION_MODELS, *known)), - True, - True, - "connected-subscription", - ) - except (OSError, subprocess.SubprocessError, ValueError, json.JSONDecodeError): - pass - try: - from hermes_cli.models import provider_model_ids - - known = _unique_models(provider_model_ids("anthropic", force_refresh=True)) - except Exception: - known = [] - return Catalog("anthropic", known, False, False, "not-configured") - - -def _profile_config( - base: dict[str, Any], - primary_provider: str, - primary_model: str, - fallback: dict[str, str], - effort: str, -) -> dict[str, Any]: - """Derive a worker configuration with an explicit cross-provider fallback.""" - config = copy.deepcopy(base) - config["model"] = { - "provider": primary_provider, - "default": primary_model, - "model": primary_model, - "openai_runtime": "codex_app_server", - } - config["fallback_providers"] = [fallback] - # Local OpenAI-compatible runtimes top out at the equivalent of high. - # Never silently downgrade an explicitly xhigh task after both hosted - # provider lanes are exhausted. - if effort != "xhigh": - config["fallback_providers"].append(copy.deepcopy(ATLAS_FALLBACK)) - agent = config.setdefault("agent", {}) - if isinstance(agent, dict): - agent["reasoning_effort"] = effort - config["toolsets"] = [] - return config - - def _switchyard_profile_config( base: dict[str, Any], route: str, effort: str ) -> dict[str, Any]: @@ -626,18 +421,8 @@ def configure_routes( for effort in EFFORTS: codex_name = f"codex-{effort}" claude_name = f"claude-{effort}" - codex_route = { - "low": "atlas/manual/codex/luna", - "medium": "atlas/manual/codex/terra", - "high": "atlas/manual/codex/sol", - "xhigh": "atlas/manual/codex/sol", - }[effort] - claude_route = { - "low": "atlas/manual/claude/haiku", - "medium": "atlas/manual/claude/sonnet", - "high": "atlas/manual/claude/sonnet", - "xhigh": "atlas/manual/claude/opus", - }[effort] + codex_route = f"atlas/manual/codex/auto/{effort}" + claude_route = f"atlas/manual/claude/auto/{effort}" _write_profile( root, codex_name, diff --git a/services/hermes/scripts/provider_model_catalog.py b/services/hermes/scripts/provider_model_catalog.py new file mode 100644 index 00000000..776ebfe3 --- /dev/null +++ b/services/hermes/scripts/provider_model_catalog.py @@ -0,0 +1,302 @@ +#!/usr/bin/env python3 +"""Normalize provider model metadata into safe Hermes routing candidates.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import Any, Iterable + + +EFFORTS = ("low", "medium", "high", "xhigh") +CAPABILITY_TIERS = ("economy", "balanced", "advanced") +EFFORT_TIERS = { + "low": "economy", + "medium": "balanced", + "high": "advanced", + "xhigh": "advanced", +} +SPECIALIST_MODEL_WORDS = frozenset( + {"audio", "embedding", "image", "moderation", "realtime", "speech", "transcrib", "tts"} +) + + +@dataclass(frozen=True) +class Catalog: + """Non-secret provider discovery result.""" + + provider: str + models: list[str] + live: bool + connected: bool + state: str + metadata: dict[str, dict[str, Any]] = field(default_factory=dict) + + +def unique_models(models: Iterable[str]) -> list[str]: + """Return normalized model IDs without changing provider order.""" + seen: set[str] = set() + result: list[str] = [] + for model in models: + value = str(model or "").strip() + if value and value.lower() not in seen: + result.append(value) + seen.add(value.lower()) + return result + + +def model_id(value: Any) -> str: + """Extract a provider model ID from either a string or model-list record.""" + if isinstance(value, str): + return value.strip() + if not isinstance(value, dict): + return "" + for key in ("id", "model", "model_id", "name", "value"): + candidate = value.get(key) + if isinstance(candidate, str) and candidate.strip(): + return candidate.strip() + return "" + + +def public_metadata(value: Any) -> dict[str, Any]: + """Keep provider routing fields while excluding unrelated response data.""" + if not isinstance(value, dict): + return {} + allowed = { + "capabilities", "capability", "capability_tier", "cost", "cost_rank", + "created_at", "default_reasoning_effort", "description", "display_name", "displayName", + "disabled", "enabled", "hidden", "input_modalities", "inputModalities", "isDefault", + "is_default", "is_general_purpose", "isGeneralPurpose", "max_input_tokens", "max_tokens", + "output_modalities", "outputModalities", "pricing", "status", + "modelSpecialty", "resolvedModel", "supported_reasoning_efforts", "supportedReasoningEfforts", "supportedEffortLevels", + "supportsAdaptiveThinking", "supportsEffort", "tier", "upgrade", + "verified", "visibility", + } + return {key: value[key] for key in allowed if key in value} + + +def model_records(values: Iterable[Any]) -> tuple[list[str], dict[str, dict[str, Any]]]: + """Normalize provider records into ordered IDs and safe per-model metadata.""" + models: list[str] = [] + metadata: dict[str, dict[str, Any]] = {} + for value in values: + model = model_id(value) + if not model: + continue + models.append(model) + if isinstance(value, dict): + metadata[model] = public_metadata(value) + unique = unique_models(models) + return unique, {model: metadata[model] for model in unique if model in metadata} + + +def merge_records( + *records: tuple[list[str], dict[str, dict[str, Any]]] +) -> tuple[list[str], dict[str, dict[str, Any]]]: + """Merge ordered discovery sources, preferring metadata from later sources.""" + models: list[str] = [] + metadata: dict[str, dict[str, Any]] = {} + for source_models, source_metadata in records: + for model in source_models: + if model not in models: + models.append(model) + details = source_metadata.get(model) + if details: + metadata[model] = {**metadata.get(model, {}), **details} + return unique_models(models), metadata + + +def model_version(model: str) -> tuple[int, ...]: + """Extract a sortable model version while ignoring dated aliases.""" + value = re.sub(r"-\d{8}$", "", model.lower()) + groups = re.findall(r"\d+(?:\.\d+)*", value) + parts: list[int] = [] + for group in groups: + parts.extend(int(piece) for piece in group.split(".")) + return tuple(parts) or (0,) + + +def _text_metadata(metadata: dict[str, Any]) -> str: + values: list[str] = [] + for key in ("description", "display_name", "capability", "tier"): + value = metadata.get(key) + if isinstance(value, str): + values.append(value.lower()) + capabilities = metadata.get("capabilities") + if isinstance(capabilities, list): + values.extend(str(value).lower() for value in capabilities) + return " ".join(values) + + +def metadata_tier(metadata: dict[str, Any]) -> tuple[str | None, str]: + """Classify a model only from provider-declared capability information.""" + for key in ("capability_tier", "tier", "capability"): + value = metadata.get(key) + if isinstance(value, str): + normalized = value.strip().lower() + if normalized in {"economy", "cheap", "fast", "low_cost"}: + return "economy", f"provider {key}={value!r}" + if normalized in {"balanced", "standard", "mid", "medium"}: + return "balanced", f"provider {key}={value!r}" + if normalized in {"advanced", "strongest", "premium", "high"}: + return "advanced", f"provider {key}={value!r}" + text = _text_metadata(metadata) + if any(term in text for term in ("strongest", "most capable", "most intelligent", "complex tasks")): + return "advanced", "provider description marks it strongest/complex" + if any(term in text for term in ("balanced", "general purpose", "everyday tasks", "routine tasks")): + return "balanced", "provider description marks it balanced/general-purpose" + if any(term in text for term in ("lowest cost", "low cost", "cheapest", "fastest", "fast and affordable")): + return "economy", "provider description marks it low-cost/fast" + return None, "provider metadata does not declare a capability tier" + + +def legacy_tier(provider: str, model: str) -> tuple[str | None, str]: + """Migrate deployed pre-metadata families without predicting new IDs.""" + known = { + "codex": {"luna": "economy", "terra": "balanced", "sol": "advanced"}, + "claude": { + "haiku": "economy", "sonnet": "balanced", "opus": "advanced", + "fable": "advanced", + }, + } + for marker, tier in known[provider].items(): + if marker in model.lower(): + return tier, "legacy deployed-family compatibility" + return None, "no provider capability metadata" + + +def _model_cost(metadata: dict[str, Any]) -> tuple[str, float] | None: + """Return a cost only with its provider-declared comparable unit.""" + for key in ("cost_rank", "cost"): + rank = metadata.get(key) + if not isinstance(rank, bool) and isinstance(rank, (int, float)) and rank >= 0: + return "ordinal", float(rank) + pricing = metadata.get("pricing") + sources = [pricing, metadata] if isinstance(pricing, dict) else [metadata] + for source in sources: + input_cost = source.get("input_cost_per_million") + output_cost = source.get("output_cost_per_million") + if ( + not isinstance(input_cost, bool) + and not isinstance(output_cost, bool) + and isinstance(input_cost, (int, float)) + and isinstance(output_cost, (int, float)) + and input_cost >= 0 + and output_cost >= 0 + ): + return "per-million", float(input_cost) + float(output_cost) + return None + + +def _comparable_costs( + pool: Iterable[tuple[str, dict[str, Any], str]] +) -> dict[str, float]: + """Use cost only when every adequate candidate advertises one shared unit.""" + records = [(model, _model_cost(details)) for model, details, _ in pool] + units = {cost[0] for _, cost in records if cost is not None} + if len(units) != 1 or any(cost is None for _, cost in records): + return {} + return {model: cost[1] for model, cost in records if cost is not None} + +def _candidate_status( + provider: str, model: str, metadata: dict[str, Any], effort: str +) -> tuple[bool, str]: + if metadata.get("hidden") is True: + return False, "provider marks model hidden" + if str(metadata.get("visibility") or "").lower() in {"hidden", "internal", "private"}: + return False, "provider marks model non-public" + if metadata.get("enabled") is False or metadata.get("disabled") is True: + return False, "provider marks model disabled" + if str(metadata.get("status") or "").lower() in {"deprecated", "disabled", "retired"}: + return False, "provider marks model unavailable" + if metadata.get("is_general_purpose") is False or metadata.get("isGeneralPurpose") is False: + return False, "provider marks model specialist" + if metadata.get("modelSpecialty") not in (None, "", False): + return False, "provider marks model specialist" + modalities = metadata.get("input_modalities", metadata.get("inputModalities")) + if isinstance(modalities, list) and modalities and "text" not in {str(item).lower() for item in modalities}: + return False, "provider record is not text-capable" + if "specialist" in _text_metadata(metadata): + return False, "provider description identifies a specialist model" + if any(word in model.lower() for word in SPECIALIST_MODEL_WORDS): + return False, "model ID identifies a specialist model" + if metadata.get("supportsEffort") is False: + return False, "provider does not advertise effort support" + supported = metadata.get( + "supported_reasoning_efforts", + metadata.get("supportedReasoningEfforts", metadata.get("supportedEffortLevels")), + ) + supported_efforts = { + str(item.get("reasoningEffort") or item.get("effort") or "").lower() + if isinstance(item, dict) + else str(item).lower() + for item in supported + } if isinstance(supported, list) else set() + if supported_efforts and effort not in supported_efforts: + return False, f"provider does not support {effort} effort" + if provider == "codex" and not model.lower().startswith("gpt-"): + return False, "model ID is outside the Codex general route namespace" + return True, "eligible" + + +def select_tier_model( + provider: str, models: Iterable[str], metadata: dict[str, dict[str, Any]], + desired: str, effort: str, current: str, *, allow_current_fallback: bool = True, +) -> tuple[str, dict[str, Any]]: + """Select by declared tier and cost, retaining last-known-good only on outage.""" + candidates: list[tuple[str, dict[str, Any], str]] = [] + observations: dict[str, Any] = {} + for model in unique_models(models): + details = metadata.get(model, {}) + eligible, reason = _candidate_status(provider, model, details, effort) + tier, tier_reason = metadata_tier(details) + if tier is None: + tier, legacy_reason = legacy_tier(provider, model) + tier_reason = legacy_reason if tier else tier_reason + cost = _model_cost(details) + observations[model] = { + "eligible": eligible, + "reason": reason if not eligible else tier_reason, + "tier": tier, + "cost": {"unit": cost[0], "value": cost[1]} if cost else None, + } + if eligible and tier: + candidates.append((model, details, tier)) + exact = [item for item in candidates if item[2] == desired] + tier_order = {"economy": 0, "balanced": 1, "advanced": 2} + # When an economy model cannot honor the requested effort, use the nearest + # higher declared tier. A difficult request never falls to a lower tier. + higher = [ + item for item in candidates + if tier_order[item[2]] > tier_order[desired] + ] + nearest_higher = min( + (tier_order[item[2]] for item in higher), default=None + ) + pool = exact or ( + [item for item in higher if tier_order[item[2]] == nearest_higher] + if nearest_higher is not None else [] + ) + if not pool: + return (current if allow_current_fallback else ""), observations + comparable_costs = _comparable_costs(pool) + + def key(item: tuple[str, dict[str, Any], str]) -> tuple[int, int, float, int, int, tuple[int, ...], str]: + model, details, tier = item + text = _text_metadata(details) + strongest = any(term in text for term in ("strongest", "most capable", "most intelligent")) + return ( + 0 if tier == desired else 1, + 0 if strongest else 1, + comparable_costs.get(model, float("inf")), + # A provider moving its default is a current release signal. It + # supersedes an old advanced route, while current protects equal + # economy/balanced candidates where no comparable cost is known. + 0 if desired == "advanced" and ( + details.get("isDefault") is True or details.get("is_default") is True + ) else 1, + 0 if model == current else 1, + tuple(-part for part in model_version(model)), + model, + ) + return min(pool, key=key)[0], observations diff --git a/services/hermes/scripts/provider_model_discovery.py b/services/hermes/scripts/provider_model_discovery.py new file mode 100644 index 00000000..c5a68bb3 --- /dev/null +++ b/services/hermes/scripts/provider_model_discovery.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python3 +"""Discover account-visible provider models without leaking credentials.""" + +from __future__ import annotations + +import json +import os +import select +import shutil +import subprocess +import time +from typing import Any +from urllib.error import HTTPError, URLError +from urllib.parse import urlencode +from urllib.request import Request, urlopen + +from provider_model_catalog import Catalog, model_records, unique_models + + +def codex_cli_authenticated() -> bool: + """Return whether the installed Codex CLI has a usable local login.""" + codex = shutil.which("codex") + if not codex: + return False + try: + status = subprocess.run( + [codex, "login", "status"], capture_output=True, check=False, + text=True, timeout=10, + ) + except (OSError, subprocess.SubprocessError): + return False + detail = f"{status.stdout}\n{status.stderr}".lower() + return status.returncode == 0 and ( + "logged in" in detail or "authenticated" in detail + ) + +def _codex_app_server_records(codex: str) -> tuple[list[str], dict[str, dict[str, Any]]] | None: + """Read the authenticated app-server ``model/list`` response when available.""" + try: + process = subprocess.Popen( + [codex, "app-server"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=False, + ) + except OSError: + return None + if process.stdin is None or process.stdout is None: + process.terminate() + return None + buffer = b"" + + def send(message: dict[str, Any]) -> None: + assert process.stdin is not None + process.stdin.write(json.dumps(message, separators=(",", ":")).encode() + b"\n") + process.stdin.flush() + + def response(request_id: int, deadline: float) -> dict[str, Any] | None: + nonlocal buffer + assert process.stdout is not None + while time.monotonic() < deadline: + readable, _, _ = select.select([process.stdout.fileno()], [], [], 0.25) + if not readable: + continue + chunk = os.read(process.stdout.fileno(), 65536) + if not chunk: + return None + buffer += chunk + while b"\n" in buffer: + line, buffer = buffer.split(b"\n", 1) + try: + document = json.loads(line) + except (TypeError, ValueError, json.JSONDecodeError): + continue + if document.get("id") == request_id and isinstance(document.get("result"), dict): + return document["result"] + return None + + try: + send({"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {"clientInfo": {"name": "hermes-catalog", "version": "1"}, "capabilities": {}}}) + if response(1, time.monotonic() + 12) is None: + return None + send({"jsonrpc": "2.0", "method": "initialized", "params": {}}) + values: list[Any] = [] + cursor: str | None = None + request_id = 2 + while True: + params: dict[str, Any] = {"includeHidden": False, "limit": 100} + if cursor: + params["cursor"] = cursor + send({"jsonrpc": "2.0", "id": request_id, "method": "model/list", "params": params}) + result = response(request_id, time.monotonic() + 12) + if result is None: + return None + page = result.get("models", result.get("data", [])) + if not isinstance(page, list): + return None + values.extend(page) + cursor = result.get("nextCursor") or result.get("next_cursor") + if not isinstance(cursor, str) or not cursor: + return model_records(values) + request_id += 1 + except (OSError, ValueError, json.JSONDecodeError): + pass + finally: + process.terminate() + try: + process.wait(timeout=2) + except subprocess.TimeoutExpired: + process.kill() + return None + + +def _anthropic_api_records() -> tuple[list[str], dict[str, dict[str, Any]]] | None: + """List current Anthropic account models, including documented capabilities.""" + token = os.environ.get("ANTHROPIC_API_KEY", "").strip() + if not token: + return None + request = Request( + "https://api.anthropic.com/v1/models?" + urlencode({"limit": 100}), + headers={"x-api-key": token, "anthropic-version": "2023-06-01"}, + ) + try: + with urlopen(request, timeout=15) as response: + document = json.load(response) + except (HTTPError, URLError, OSError, TimeoutError, ValueError): + return None + values = document.get("data", []) if isinstance(document, dict) else [] + return model_records(values if isinstance(values, list) else []) + + +def discover_codex_models() -> Catalog: + """Use the authenticated Codex endpoint; catalogs are status-only fallback.""" + token = "" + try: + from hermes_cli.auth import resolve_codex_runtime_credentials + + credentials = resolve_codex_runtime_credentials(refresh_if_expiring=True) or {} + token = str(credentials.get("api_key") or "").strip() + except Exception: + token = "" + api_records: tuple[list[str], dict[str, dict[str, Any]]] | None = None + if token: + try: + from hermes_cli.codex_models import _fetch_models_from_api + + api_records = model_records(_fetch_models_from_api(token)) + except Exception: + api_records = None + if api_records is not None: + live, live_metadata = api_records + return Catalog("openai-codex", live, True, True, "connected-api", live_metadata) + codex = shutil.which("codex") + if codex: + app_records = _codex_app_server_records(codex) + if app_records is not None: + app_models, app_metadata = app_records + return Catalog("openai-codex", app_models, True, True, "connected-app-server", app_metadata) + try: + from hermes_cli.models import provider_model_ids + + known, known_metadata = model_records( + provider_model_ids("openai-codex", force_refresh=True) + ) + except Exception: + known, known_metadata = [], {} + # The owner agent deliberately uses the local Codex app-server and the + # Codex CLI's ChatGPT login. That credential is not exported into Hermes' + # bearer-token store, so API discovery above may be empty even though the + # runtime is fully authenticated. Treat a successful CLI status check as a + # connected, degraded catalog: routing may use Codex while retaining the + # last-known working model names until app-server discovery is available. + if codex_cli_authenticated(): + return Catalog( + "openai-codex", + known, + False, + True, + "connected-cli", + known_metadata, + ) + state = "degraded" if token else "not-configured" + return Catalog("openai-codex", known, False, bool(token), state, known_metadata) + + +def discover_claude_models() -> Catalog: + """Discover subscription models first, retaining API discovery as fallback.""" + try: + from claude_model_discovery import discover_claude_subscription_models + + subscription = discover_claude_subscription_models() + except Exception: + subscription = None + if subscription is not None and subscription.live: + records = [model.as_dict() for model in subscription.models] + models, metadata = model_records(records) + return Catalog( + "anthropic", models, True, subscription.connected, + subscription.state, metadata, + ) + api_records = _anthropic_api_records() + if api_records is not None: + live_models, live_metadata = api_records + return Catalog("anthropic", live_models, True, True, "connected-api", live_metadata) + try: + from hermes_cli.models import provider_model_ids + + known, known_metadata = model_records( + provider_model_ids("anthropic", force_refresh=True) + ) + except Exception: + known, known_metadata = [], {} + if subscription is not None: + return Catalog( + "anthropic", known, False, subscription.connected, + subscription.state, known_metadata, + ) + return Catalog("anthropic", known, False, False, "not-configured", known_metadata) diff --git a/services/hermes/scripts/routing_catalog.py b/services/hermes/scripts/routing_catalog.py index 8c7cc5ef..a1ced43f 100644 --- a/services/hermes/scripts/routing_catalog.py +++ b/services/hermes/scripts/routing_catalog.py @@ -51,6 +51,15 @@ def load_catalog(path: Path = CATALOG_PATH) -> dict[str, Any]: return value if isinstance(value, dict) else {} +def catalog_contains(provider: str, model: str, catalog: dict[str, Any] | None = None) -> bool: + """Return whether a provider's stewarded current catalog advertises a model.""" + document = catalog if catalog is not None else load_catalog() + providers = document.get("providers", {}) if isinstance(document, dict) else {} + record = providers.get(provider, {}) if isinstance(providers, dict) else {} + models = record.get("models", []) if isinstance(record, dict) else [] + return isinstance(model, str) and model in models + + def resolve_model( provider: str, selector: str, @@ -61,10 +70,16 @@ def resolve_model( if provider not in DEFAULTS or effort not in DEFAULTS[provider]: raise ValueError("unsupported provider or effort") prefix = PREFIXES[provider] + document = catalog if catalog is not None else load_catalog() if selector.startswith(prefix): + if catalog_contains(provider, selector, document): + return selector + providers = document.get("providers", {}) if isinstance(document, dict) else {} + record = providers.get(provider, {}) if isinstance(providers, dict) else {} + if isinstance(record, dict) and record.get("live") is True: + raise ValueError(f"model is not in the current {provider} catalog") return selector - document = catalog if catalog is not None else load_catalog() providers = document.get("providers", {}) if isinstance(document, dict) else {} record = providers.get(provider, {}) if isinstance(providers, dict) else {} if not isinstance(record, dict): @@ -72,7 +87,23 @@ def resolve_model( mapping_name = "resolved" if selector == "auto" else "tiers" mapping = record.get(mapping_name, {}) candidate = mapping.get(effort if selector == "auto" else selector, "") if isinstance(mapping, dict) else "" - if not isinstance(candidate, str) or not candidate.startswith(prefix): + if ( + selector != "auto" + and isinstance(candidate, str) + and candidate + and selector not in candidate.lower() + ): + candidate = "" + advertised = record.get("models", []) if isinstance(record, dict) else [] + candidate_allowed = isinstance(candidate, str) and ( + candidate.startswith(prefix) + or (provider == "claude" and candidate in advertised) + ) + if not candidate_allowed: + # A fresh account list is authoritative. Do not turn a deliberately + # unresolved/retired selector into a stale baseline model. + if record.get("live") is True: + raise ValueError(f"no current {provider} model for selector {selector!r}") candidate = ( DEFAULTS[provider][effort] if selector == "auto" diff --git a/services/hermes/switchyard-configmap.yaml b/services/hermes/switchyard-configmap.yaml index a1421281..86496cac 100644 --- a/services/hermes/switchyard-configmap.yaml +++ b/services/hermes/switchyard-configmap.yaml @@ -158,6 +158,26 @@ data: llm_client = "codex_xhigh" extra_body = { reasoning = { effort = "xhigh" } } + [targets.codex_auto_low] + id = "route/codex/auto/low" + llm_client = "codex_low" + extra_body = { reasoning = { effort = "low" } } + + [targets.codex_auto_medium] + id = "route/codex/auto/medium" + llm_client = "codex_medium" + extra_body = { reasoning = { effort = "medium" } } + + [targets.codex_auto_high] + id = "route/codex/auto/high" + llm_client = "codex_high" + extra_body = { reasoning = { effort = "high" } } + + [targets.codex_auto_xhigh] + id = "route/codex/auto/xhigh" + llm_client = "codex_xhigh" + extra_body = { reasoning = { effort = "xhigh" } } + [targets.claude_haiku_low] id = "route/claude/haiku/low" llm_client = "claude_low" @@ -238,6 +258,26 @@ data: llm_client = "claude_xhigh" extra_body = { thinking = { type = "adaptive" }, output_config = { effort = "xhigh" } } + [targets.claude_auto_low] + id = "route/claude/auto/low" + llm_client = "claude_low" + extra_body = { thinking = { type = "adaptive" }, output_config = { effort = "low" } } + + [targets.claude_auto_medium] + id = "route/claude/auto/medium" + llm_client = "claude_medium" + extra_body = { thinking = { type = "adaptive" }, output_config = { effort = "medium" } } + + [targets.claude_auto_high] + id = "route/claude/auto/high" + llm_client = "claude_high" + extra_body = { thinking = { type = "adaptive" }, output_config = { effort = "high" } } + + [targets.claude_auto_xhigh] + id = "route/claude/auto/xhigh" + llm_client = "claude_xhigh" + extra_body = { thinking = { type = "adaptive" }, output_config = { effort = "xhigh" } } + [targets.worker_codex_luna_low] id = "worker/codex/luna/low" llm_client = "worker_decision" @@ -350,6 +390,38 @@ data: id = "worker/claude/opus/xhigh" llm_client = "worker_decision" + [targets.worker_codex_auto_low] + id = "worker/codex/auto/low" + llm_client = "worker_decision" + + [targets.worker_codex_auto_medium] + id = "worker/codex/auto/medium" + llm_client = "worker_decision" + + [targets.worker_codex_auto_high] + id = "worker/codex/auto/high" + llm_client = "worker_decision" + + [targets.worker_codex_auto_xhigh] + id = "worker/codex/auto/xhigh" + llm_client = "worker_decision" + + [targets.worker_claude_auto_low] + id = "worker/claude/auto/low" + llm_client = "worker_decision" + + [targets.worker_claude_auto_medium] + id = "worker/claude/auto/medium" + llm_client = "worker_decision" + + [targets.worker_claude_auto_high] + id = "worker/claude/auto/high" + llm_client = "worker_decision" + + [targets.worker_claude_auto_xhigh] + id = "worker/claude/auto/xhigh" + llm_client = "worker_decision" + # Switchyard 0.2.0 requires one default_target for every custom classifier. # These targets recurse once into unbiased random routes whose candidates are # split evenly across providers. A classifier outage therefore falls back to @@ -377,7 +449,7 @@ data: [routes.fallback_fast] id = "atlas/fallback/fast" type = "random" - targets = ["codex_terra_medium", "claude_fable_medium"] + targets = ["codex_auto_medium", "claude_auto_medium"] weights = [1.0, 1.0] context_window = 272000 tool_calling = true @@ -386,7 +458,7 @@ data: [routes.fallback_balanced] id = "atlas/fallback/balanced" type = "random" - targets = ["codex_terra_high", "claude_sonnet_high"] + targets = ["codex_auto_high", "claude_auto_high"] weights = [1.0, 1.0] context_window = 272000 tool_calling = true @@ -395,7 +467,7 @@ data: [routes.fallback_deep] id = "atlas/fallback/deep" type = "random" - targets = ["codex_sol_high", "claude_opus_high"] + targets = ["codex_auto_high", "claude_auto_high"] weights = [1.0, 1.0] context_window = 272000 tool_calling = true @@ -404,7 +476,7 @@ data: [routes.fallback_maximum] id = "atlas/fallback/maximum" type = "random" - targets = ["codex_sol_xhigh", "claude_opus_xhigh"] + targets = ["codex_auto_xhigh", "claude_auto_xhigh"] weights = [1.0, 1.0] context_window = 272000 tool_calling = true @@ -413,7 +485,7 @@ data: [routes.fallback_worker_maximum] id = "atlas/worker/fallback/maximum" type = "random" - targets = ["worker_codex_sol_xhigh", "worker_claude_opus_xhigh"] + targets = ["worker_codex_auto_xhigh", "worker_claude_auto_xhigh"] weights = [1.0, 1.0] context_window = 272000 tool_calling = false @@ -426,7 +498,7 @@ data: classifier_target = "classifier" # Switchyard falls through this list after a request-local target failure. # Keep both xhigh providers first so recovery can escalate, never downgrade. - targets = ["codex_sol_xhigh", "claude_opus_xhigh", "claude_fable_xhigh", "codex_sol_high", "claude_sonnet_high", "claude_opus_high", "claude_fable_high", "codex_terra_high", "codex_sol_medium", "claude_sonnet_medium", "claude_fable_medium", "codex_terra_medium", "codex_terra_low", "codex_luna_low", "claude_haiku_low", "claude_fable_low", "neutral_fast_pool"] + targets = ["codex_auto_xhigh", "claude_auto_xhigh", "codex_auto_high", "claude_auto_high", "codex_auto_medium", "claude_auto_medium", "codex_auto_low", "claude_auto_low", "neutral_fast_pool"] default_target = "neutral_fast_pool" session_affinity = false recent_turn_window = 4 @@ -449,10 +521,10 @@ data: If tools are present or the boundary may emit a tool call, the floor is medium even when the user's wording is short. Filesystem, shell, repository, cluster, browser, and image-generation/edit operations are - tool work; never route those boundaries to Luna, Haiku, or local-low. + tool work; never route those boundaries to economy or local-low targets. A failing test, failed tool plan, contradicted result, rejected review, or incomplete evidence means the previous quality mark was missed. Raise the - next boundary by at least one effort tier and prefer a stronger family or + next boundary by at least one effort tier and prefer an available advanced target or the other hosted provider; repeated misses require xhigh. Never repeat a failed lower-capability plan unchanged. @@ -465,7 +537,7 @@ data: 3. Route image creation and editing before general provider preference. When image-generation/edit tools are available and the user asks to create, transform, restore, colorize, or continue editing an image: select - codex_terra_medium. The image tool—not the conversational model—honors the + codex_auto_medium. The image tool—not the conversational model—honors the user's local, OpenAI/hosted, or AUTO image-backend choice. Do not select a local Qwen or Claude target for an image-tool boundary: local Qwen cannot reliably carry the full Hermes tool context, and Anthropic supplies the @@ -480,17 +552,15 @@ data: to be unavailable, failed, exhausted, rate-limited, or out of capacity; use the other hosted provider at the same floor. - 5. Choose across the complete family catalog. Codex options are Luna, - Terra, and SOL at low through xhigh. Claude options are Haiku, Fable, - Sonnet, and Opus at low through xhigh. Prefer Fable for concise writing, - synthesis, and instruction-following where its capability fits the effort - floor; use Sonnet or Opus for deeper analysis and review. Re-evaluate every - boundary and resolve "continue" - or "do it" from recent context. The manual local route remains available - only when the user explicitly selects it. + 5. Choose the provider AUTO target at the exact effort floor. AUTO resolves + an account-visible economy, balanced, or advanced general-purpose model + from provider metadata. It uses comparable provider cost data only to break + ties between adequate models. Re-evaluate every boundary and resolve + "continue" or "do it" from recent context. The manual local route remains + available only when the user explicitly selects it. """ response_schema = ''' - {"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["codex_terra_medium","claude_sonnet_medium","claude_fable_medium","codex_luna_low","claude_haiku_low","claude_fable_low","codex_terra_low","codex_terra_high","claude_fable_high","codex_sol_medium","codex_sol_high","codex_sol_xhigh","claude_sonnet_high","claude_opus_high","claude_fable_xhigh","claude_opus_xhigh"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false} + {"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["codex_auto_low","codex_auto_medium","codex_auto_high","codex_auto_xhigh","claude_auto_low","claude_auto_medium","claude_auto_high","claude_auto_xhigh"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false} ''' [routes.auto_fast.policy] @@ -504,7 +574,7 @@ data: classifier_target = "classifier" # Switchyard falls through this list after a request-local target failure. # Keep both xhigh providers first so recovery can escalate, never downgrade. - targets = ["codex_sol_xhigh", "claude_opus_xhigh", "claude_fable_xhigh", "codex_sol_high", "claude_sonnet_high", "claude_opus_high", "claude_fable_high", "codex_terra_high", "codex_sol_medium", "claude_sonnet_medium", "claude_fable_medium", "codex_terra_medium", "codex_terra_low", "codex_luna_low", "claude_haiku_low", "claude_fable_low", "neutral_balanced_pool"] + targets = ["codex_auto_xhigh", "claude_auto_xhigh", "codex_auto_high", "claude_auto_high", "codex_auto_medium", "claude_auto_medium", "codex_auto_low", "claude_auto_low", "neutral_balanced_pool"] default_target = "neutral_balanced_pool" session_affinity = false recent_turn_window = 4 @@ -528,10 +598,10 @@ data: If tools are present or the boundary may emit a tool call, the floor is medium even when the user's wording is short. Filesystem, shell, repository, cluster, browser, and image-generation/edit operations are - tool work; never route those boundaries to Luna, Haiku, or local-low. + tool work; never route those boundaries to economy or local-low targets. A failing test, failed tool plan, contradicted result, rejected review, or incomplete evidence means the previous quality mark was missed. Raise the - next boundary by at least one effort tier and prefer a stronger family or + next boundary by at least one effort tier and prefer an available advanced target or the other hosted provider; repeated misses require xhigh. Never repeat a failed lower-capability plan unchanged. @@ -544,7 +614,7 @@ data: 3. Route image creation and editing before general provider preference. When image-generation/edit tools are available and the user asks to create, transform, restore, colorize, or continue editing an image: select - codex_terra_medium. The image tool—not the conversational model—honors the + codex_auto_medium. The image tool—not the conversational model—honors the user's local, OpenAI/hosted, or AUTO image-backend choice. Do not select a local Qwen or Claude target for an image-tool boundary: local Qwen cannot reliably carry the full Hermes tool context, and Anthropic supplies the @@ -559,17 +629,15 @@ data: to be unavailable, failed, exhausted, rate-limited, or out of capacity; use the other hosted provider at the same floor. - 5. Choose across the complete family catalog. Codex options are Luna, - Terra, and SOL at low through xhigh. Claude options are Haiku, Fable, - Sonnet, and Opus at low through xhigh. Prefer Fable for concise writing, - synthesis, and instruction-following where its capability fits the effort - floor; use Sonnet or Opus for deeper analysis and review. Re-evaluate every - boundary and resolve "continue" - or "do it" from recent context. The manual local route remains available - only when the user explicitly selects it. + 5. Choose the provider AUTO target at the exact effort floor. AUTO resolves + an account-visible economy, balanced, or advanced general-purpose model + from provider metadata. It uses comparable provider cost data only to break + ties between adequate models. Re-evaluate every boundary and resolve + "continue" or "do it" from recent context. The manual local route remains + available only when the user explicitly selects it. """ response_schema = ''' - {"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["codex_terra_medium","claude_sonnet_medium","claude_fable_medium","codex_luna_low","claude_haiku_low","claude_fable_low","codex_terra_low","codex_terra_high","claude_fable_high","codex_sol_medium","codex_sol_high","codex_sol_xhigh","claude_sonnet_high","claude_opus_high","claude_fable_xhigh","claude_opus_xhigh"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false} + {"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["codex_auto_low","codex_auto_medium","codex_auto_high","codex_auto_xhigh","claude_auto_low","claude_auto_medium","claude_auto_high","claude_auto_xhigh"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false} ''' [routes.auto_balanced.policy] @@ -583,7 +651,7 @@ data: classifier_target = "classifier" # Switchyard falls through this list after a request-local target failure. # Keep both xhigh providers first so recovery can escalate, never downgrade. - targets = ["claude_opus_xhigh", "codex_sol_xhigh", "claude_fable_xhigh", "claude_sonnet_high", "codex_sol_high", "claude_opus_high", "claude_fable_high", "codex_terra_high", "claude_sonnet_medium", "claude_fable_medium", "codex_sol_medium", "codex_terra_medium", "neutral_deep_pool"] + targets = ["claude_auto_xhigh", "codex_auto_xhigh", "claude_auto_high", "codex_auto_high", "claude_auto_medium", "codex_auto_medium", "neutral_deep_pool"] default_target = "neutral_deep_pool" session_affinity = false recent_turn_window = 6 @@ -607,10 +675,10 @@ data: If tools are present or the boundary may emit a tool call, the floor is medium even when the user's wording is short. Filesystem, shell, repository, cluster, browser, and image-generation/edit operations are - tool work; never route those boundaries to Luna or Haiku. + tool work; never route those boundaries to economy targets. A failing test, failed tool plan, contradicted result, rejected review, or incomplete evidence means the previous quality mark was missed. Raise the - next boundary by at least one effort tier and prefer a stronger family or + next boundary by at least one effort tier and prefer an available advanced target or the other hosted provider; repeated misses require xhigh. Never repeat a failed lower-capability plan unchanged. @@ -630,15 +698,14 @@ data: unavailable, failed, exhausted, rate-limited, or out of capacity; use the other provider at the same floor. - 4. Choose across Codex Terra/SOL and Claude Fable/Sonnet/Opus at medium - through xhigh. Prefer Fable for writing and compact synthesis where it - clears the quality floor; use Sonnet or Opus for deeper diagnosis and - review. Low-tier targets are intentionally unavailable on - this route. Re-evaluate every boundary and resolve "continue" or "do it" - from recent context. + 4. Choose a provider AUTO target at the exact effort floor. AUTO resolves + an account-visible balanced or advanced general-purpose model from provider + metadata; economy targets are intentionally unavailable on this route. + Re-evaluate every boundary and resolve "continue" or "do it" from recent + context. """ response_schema = ''' - {"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["claude_sonnet_high","claude_fable_high","codex_sol_high","codex_terra_medium","claude_sonnet_medium","claude_fable_medium","codex_terra_high","codex_sol_medium","claude_opus_high","codex_sol_xhigh","claude_fable_xhigh","claude_opus_xhigh"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false} + {"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["claude_auto_medium","claude_auto_high","claude_auto_xhigh","codex_auto_medium","codex_auto_high","codex_auto_xhigh"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false} ''' [routes.auto_deep.policy] @@ -652,7 +719,7 @@ data: classifier_target = "classifier" # Switchyard falls through this list after a request-local target failure. # Keep both xhigh providers first so recovery can escalate, never downgrade. - targets = ["codex_sol_xhigh", "claude_opus_xhigh", "claude_fable_xhigh", "codex_sol_high", "claude_opus_high", "claude_sonnet_high", "claude_fable_high", "codex_terra_high", "neutral_maximum_pool"] + targets = ["codex_auto_xhigh", "claude_auto_xhigh", "codex_auto_high", "claude_auto_high", "neutral_maximum_pool"] default_target = "neutral_maximum_pool" session_affinity = false recent_turn_window = 6 @@ -674,7 +741,7 @@ data: Never choose below the floor or above xhigh. A failing test, failed tool plan, contradicted result, rejected review, or incomplete evidence raises the next boundary to xhigh and should switch to - the strongest suitable family or other hosted provider. Never repeat a + the strongest suitable available target or other hosted provider. Never repeat a failed high-capability plan unchanged. 2. Treat "think hard", "deeply", "carefully", and equivalent intent as a @@ -693,15 +760,14 @@ data: unavailable, failed, exhausted, rate-limited, or out of capacity; use the other provider at the same floor. - 4. Choose across Codex Terra/SOL and Claude Fable/Sonnet/Opus at high or - xhigh only. Prefer Fable for writing and compact synthesis where it clears - the quality floor; use Sonnet or Opus for deeper analysis and independent - review. Medium and low targets are intentionally unavailable on this - route. Re-evaluate every boundary and resolve "continue" or "do it" from - recent context. + 4. Choose a provider AUTO target at high or xhigh only. AUTO resolves an + account-visible advanced general-purpose model from provider metadata. + Medium and low targets are intentionally unavailable on this route. + Re-evaluate every boundary and resolve "continue" or "do it" from recent + context. """ response_schema = ''' - {"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["codex_sol_high","claude_opus_high","claude_sonnet_high","claude_fable_high","codex_terra_high","codex_sol_xhigh","claude_opus_xhigh","claude_fable_xhigh"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false} + {"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["codex_auto_high","codex_auto_xhigh","claude_auto_high","claude_auto_xhigh"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false} ''' [routes.auto_maximum.policy] @@ -713,7 +779,7 @@ data: type = "llm_classifier" mode = "custom" classifier_target = "classifier" - targets = ["worker_codex_luna_low", "worker_codex_luna_medium", "worker_codex_luna_high", "worker_codex_luna_xhigh", "worker_codex_terra_low", "worker_codex_terra_medium", "worker_codex_terra_high", "worker_codex_terra_xhigh", "worker_codex_sol_low", "worker_codex_sol_medium", "worker_codex_sol_high", "worker_codex_sol_xhigh", "worker_claude_haiku_low", "worker_claude_haiku_medium", "worker_claude_haiku_high", "worker_claude_haiku_xhigh", "worker_claude_fable_low", "worker_claude_fable_medium", "worker_claude_fable_high", "worker_claude_fable_xhigh", "worker_claude_sonnet_low", "worker_claude_sonnet_medium", "worker_claude_sonnet_high", "worker_claude_sonnet_xhigh", "worker_claude_opus_low", "worker_claude_opus_medium", "worker_claude_opus_high", "worker_claude_opus_xhigh", "neutral_worker_maximum_pool"] + targets = ["worker_codex_auto_low", "worker_codex_auto_medium", "worker_codex_auto_high", "worker_codex_auto_xhigh", "worker_claude_auto_low", "worker_claude_auto_medium", "worker_claude_auto_high", "worker_claude_auto_xhigh", "neutral_worker_maximum_pool"] default_target = "neutral_worker_maximum_pool" session_affinity = false recent_turn_window = 6 @@ -738,7 +804,7 @@ data: Never choose below the floor and never exceed xhigh. If the objective records a failing test, failed attempt, contradicted result, rejected review, or incomplete evidence from a previous worker, - raise effort by at least one tier and prefer a stronger family or the other + raise effort by at least one tier and prefer the available advanced target or the other hosted provider. Repeated quality misses require xhigh; never launch the same failed lower-capability plan unchanged. @@ -757,30 +823,28 @@ data: analysis, and independent review. A final independent review is Claude; implementing review findings is Codex. - 4. Choose across every configured Codex and Claude family at the exact - effort floor. Codex Luna is the economical tier, Terra is balanced, and - SOL is the deepest implementation tier. Claude Haiku is the economical - tier, Fable is preferred for concise writing and synthesis, Sonnet is - balanced, and Opus is the deepest analysis and review tier. Every family - supports low, medium, high, and xhigh; choose the cheapest family that - clears the objective's quality floor without lowering its effort. + 4. Choose an available provider AUTO target at the exact effort floor. + AUTO resolves the current account-visible economy, balanced, or advanced + general-purpose model. It uses comparable provider cost data only to break + ties among models that already satisfy the capability requirement. Do not + infer a tier from an unfamiliar model name or unadvertised effort support. Examples: - Critical security migration final review -> worker_claude_opus_xhigh. - Implement critical security review fixes -> worker_codex_sol_xhigh. - Difficult intermittent production failure -> worker_codex_sol_high. - Ordinary component design -> worker_claude_sonnet_medium. - One spelling correction -> worker_codex_luna_low. + Critical security migration final review -> worker_claude_auto_xhigh. + Implement critical security review fixes -> worker_codex_auto_xhigh. + Difficult intermittent production failure -> worker_codex_auto_high. + Ordinary component design -> worker_claude_auto_medium. + One spelling correction -> worker_codex_auto_low. Anthropic exhausted + critical independent final review -> - worker_codex_sol_xhigh. + worker_codex_auto_xhigh. OpenAI exhausted + difficult repository implementation -> - worker_claude_sonnet_high. + worker_claude_auto_high. Before responding, verify the provider is available and effort is not below the floor. Return only the required decision object. """ response_schema = ''' - {"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["worker_codex_luna_low","worker_codex_luna_medium","worker_codex_luna_high","worker_codex_luna_xhigh","worker_codex_terra_low","worker_codex_terra_medium","worker_codex_terra_high","worker_codex_terra_xhigh","worker_codex_sol_low","worker_codex_sol_medium","worker_codex_sol_high","worker_codex_sol_xhigh","worker_claude_haiku_low","worker_claude_haiku_medium","worker_claude_haiku_high","worker_claude_haiku_xhigh","worker_claude_fable_low","worker_claude_fable_medium","worker_claude_fable_high","worker_claude_fable_xhigh","worker_claude_sonnet_low","worker_claude_sonnet_medium","worker_claude_sonnet_high","worker_claude_sonnet_xhigh","worker_claude_opus_low","worker_claude_opus_medium","worker_claude_opus_high","worker_claude_opus_xhigh"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false} + {"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["worker_codex_auto_low","worker_codex_auto_medium","worker_codex_auto_high","worker_codex_auto_xhigh","worker_claude_auto_low","worker_claude_auto_medium","worker_claude_auto_high","worker_claude_auto_xhigh"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false} ''' [routes.worker_auto_maximum.policy] @@ -790,42 +854,98 @@ data: [routes.worker_manual_codex_low] id = "atlas/worker/manual/codex/low" type = "random" - targets = ["worker_codex_luna_low"] + targets = ["worker_codex_auto_low"] [routes.worker_manual_codex_medium] id = "atlas/worker/manual/codex/medium" type = "random" - targets = ["worker_codex_terra_medium"] + targets = ["worker_codex_auto_medium"] [routes.worker_manual_codex_high] id = "atlas/worker/manual/codex/high" type = "random" - targets = ["worker_codex_sol_high"] + targets = ["worker_codex_auto_high"] [routes.worker_manual_codex_xhigh] id = "atlas/worker/manual/codex/xhigh" type = "random" - targets = ["worker_codex_sol_xhigh"] + targets = ["worker_codex_auto_xhigh"] [routes.worker_manual_claude_low] id = "atlas/worker/manual/claude/low" type = "random" - targets = ["worker_claude_haiku_low"] + targets = ["worker_claude_auto_low"] [routes.worker_manual_claude_medium] id = "atlas/worker/manual/claude/medium" type = "random" - targets = ["worker_claude_sonnet_medium"] + targets = ["worker_claude_auto_medium"] [routes.worker_manual_claude_high] id = "atlas/worker/manual/claude/high" type = "random" - targets = ["worker_claude_sonnet_high"] + targets = ["worker_claude_auto_high"] [routes.worker_manual_claude_xhigh] id = "atlas/worker/manual/claude/xhigh" type = "random" - targets = ["worker_claude_opus_xhigh"] + targets = ["worker_claude_auto_xhigh"] + + [routes.manual_codex_auto] + id = "atlas/manual/codex/auto" + type = "random" + targets = ["codex_auto_medium"] + context_window = 272000 + tool_calling = true + reasoning = true + + [routes.manual_codex_auto_low] + id = "atlas/manual/codex/auto/low" + type = "random" + targets = ["codex_auto_low"] + + [routes.manual_codex_auto_medium] + id = "atlas/manual/codex/auto/medium" + type = "random" + targets = ["codex_auto_medium"] + + [routes.manual_codex_auto_high] + id = "atlas/manual/codex/auto/high" + type = "random" + targets = ["codex_auto_high"] + + [routes.manual_codex_auto_xhigh] + id = "atlas/manual/codex/auto/xhigh" + type = "random" + targets = ["codex_auto_xhigh"] + + [routes.manual_claude_auto] + id = "atlas/manual/claude/auto" + type = "random" + targets = ["claude_auto_medium"] + context_window = 272000 + tool_calling = true + reasoning = true + + [routes.manual_claude_auto_low] + id = "atlas/manual/claude/auto/low" + type = "random" + targets = ["claude_auto_low"] + + [routes.manual_claude_auto_medium] + id = "atlas/manual/claude/auto/medium" + type = "random" + targets = ["claude_auto_medium"] + + [routes.manual_claude_auto_high] + id = "atlas/manual/claude/auto/high" + type = "random" + targets = ["claude_auto_high"] + + [routes.manual_claude_auto_xhigh] + id = "atlas/manual/claude/auto/xhigh" + type = "random" + targets = ["claude_auto_xhigh"] # A zero-weight target is not selected initially. Switchyard still walks # this ordered list after 403/408/429/5xx, timeout, or transport failure. diff --git a/services/hermes/switchyard-deployment.yaml b/services/hermes/switchyard-deployment.yaml index 4b259aa2..412c8195 100644 --- a/services/hermes/switchyard-deployment.yaml +++ b/services/hermes/switchyard-deployment.yaml @@ -22,7 +22,7 @@ spec: labels: app: hermes-switchyard annotations: - ai.bstein.dev/config-rev: "20260823-voice-route-preflight-v1" + ai.bstein.dev/config-rev: "20260913-provider-model-catalog-v2" prometheus.io/scrape: "true" prometheus.io/port: "9005" prometheus.io/path: /metrics diff --git a/testing/fixtures/hermes/codex-0.154-visible-models.json b/testing/fixtures/hermes/codex-0.154-visible-models.json new file mode 100644 index 00000000..1bd6e7a8 --- /dev/null +++ b/testing/fixtures/hermes/codex-0.154-visible-models.json @@ -0,0 +1,227 @@ +[ + { + "defaultReasoningEffort": "medium", + "description": "Our most capable model for complex, demanding work.", + "displayName": "GPT-6-Astra", + "hidden": false, + "id": "gpt-6-astra", + "inputModalities": [ + "text", + "image" + ], + "isDefault": true, + "model": "gpt-6-astra", + "modelSpecialty": null, + "supportedReasoningEfforts": [ + { + "description": "Fast responses with lighter reasoning", + "reasoningEffort": "low" + }, + { + "description": "Balances speed and reasoning depth for everyday tasks", + "reasoningEffort": "medium" + }, + { + "description": "Greater reasoning depth for complex problems", + "reasoningEffort": "high" + }, + { + "description": "Extra high reasoning depth for complex problems", + "reasoningEffort": "xhigh" + }, + { + "description": "Maximum reasoning depth for the hardest problems", + "reasoningEffort": "max" + }, + { + "description": "Maximum reasoning with automatic task delegation", + "reasoningEffort": "ultra" + } + ], + "upgrade": null + }, + { + "defaultReasoningEffort": "low", + "description": "Reliable agentic workhorse for everyday tasks.", + "displayName": "GPT-5.6-Sol", + "hidden": false, + "id": "gpt-5.6-sol", + "inputModalities": [ + "text", + "image" + ], + "isDefault": false, + "model": "gpt-5.6-sol", + "modelSpecialty": null, + "supportedReasoningEfforts": [ + { + "description": "Fast responses with lighter reasoning", + "reasoningEffort": "low" + }, + { + "description": "Balances speed and reasoning depth for everyday tasks", + "reasoningEffort": "medium" + }, + { + "description": "Greater reasoning depth for complex problems", + "reasoningEffort": "high" + }, + { + "description": "Extra high reasoning depth for complex problems", + "reasoningEffort": "xhigh" + }, + { + "description": "Maximum reasoning depth for the hardest problems", + "reasoningEffort": "max" + }, + { + "description": "Maximum reasoning with automatic task delegation", + "reasoningEffort": "ultra" + } + ], + "upgrade": null + }, + { + "defaultReasoningEffort": "medium", + "description": "Balanced agentic coding model for everyday work.", + "displayName": "GPT-5.6-Terra", + "hidden": false, + "id": "gpt-5.6-terra", + "inputModalities": [ + "text", + "image" + ], + "isDefault": false, + "model": "gpt-5.6-terra", + "modelSpecialty": null, + "supportedReasoningEfforts": [ + { + "description": "Fast responses with lighter reasoning", + "reasoningEffort": "low" + }, + { + "description": "Balances speed and reasoning depth for everyday tasks", + "reasoningEffort": "medium" + }, + { + "description": "Greater reasoning depth for complex problems", + "reasoningEffort": "high" + }, + { + "description": "Extra high reasoning depth for complex problems", + "reasoningEffort": "xhigh" + }, + { + "description": "Maximum reasoning depth for the hardest problems", + "reasoningEffort": "max" + }, + { + "description": "Maximum reasoning with automatic task delegation", + "reasoningEffort": "ultra" + } + ], + "upgrade": null + }, + { + "defaultReasoningEffort": "medium", + "description": "Fast and affordable agentic coding model.", + "displayName": "GPT-5.6-Luna", + "hidden": false, + "id": "gpt-5.6-luna", + "inputModalities": [ + "text", + "image" + ], + "isDefault": false, + "model": "gpt-5.6-luna", + "modelSpecialty": null, + "supportedReasoningEfforts": [ + { + "description": "Fast responses with lighter reasoning", + "reasoningEffort": "low" + }, + { + "description": "Balances speed and reasoning depth for everyday tasks", + "reasoningEffort": "medium" + }, + { + "description": "Greater reasoning depth for complex problems", + "reasoningEffort": "high" + }, + { + "description": "Extra high reasoning depth for complex problems", + "reasoningEffort": "xhigh" + }, + { + "description": "Maximum reasoning depth for the hardest problems", + "reasoningEffort": "max" + } + ], + "upgrade": null + }, + { + "defaultReasoningEffort": "medium", + "description": "Proven previous-generation model for coding and general work.", + "displayName": "GPT-5.5", + "hidden": false, + "id": "gpt-5.5", + "inputModalities": [ + "text", + "image" + ], + "isDefault": false, + "model": "gpt-5.5", + "modelSpecialty": null, + "supportedReasoningEfforts": [ + { + "description": "Fast responses with lighter reasoning", + "reasoningEffort": "low" + }, + { + "description": "Balances speed and reasoning depth for everyday tasks", + "reasoningEffort": "medium" + }, + { + "description": "Greater reasoning depth for complex problems", + "reasoningEffort": "high" + }, + { + "description": "Extra high reasoning depth for complex problems", + "reasoningEffort": "xhigh" + } + ], + "upgrade": null + }, + { + "defaultReasoningEffort": "high", + "description": "Ultra-fast coding model.", + "displayName": "GPT-5.3-Codex-Spark", + "hidden": false, + "id": "gpt-5.3-codex-spark", + "inputModalities": [ + "text" + ], + "isDefault": false, + "model": "gpt-5.3-codex-spark", + "modelSpecialty": null, + "supportedReasoningEfforts": [ + { + "description": "Fast responses with lighter reasoning", + "reasoningEffort": "low" + }, + { + "description": "Balances speed and reasoning depth for everyday tasks", + "reasoningEffort": "medium" + }, + { + "description": "Greater reasoning depth for complex problems", + "reasoningEffort": "high" + }, + { + "description": "Extra high reasoning depth for complex problems", + "reasoningEffort": "xhigh" + } + ], + "upgrade": null + } +] diff --git a/testing/tests/test_claude_model_discovery.py b/testing/tests/test_claude_model_discovery.py new file mode 100644 index 00000000..4586a5ae --- /dev/null +++ b/testing/tests/test_claude_model_discovery.py @@ -0,0 +1,106 @@ +"""Contract tests for native Claude subscription model discovery.""" + +from __future__ import annotations + +import importlib +import json +import sys +from pathlib import Path + + +SCRIPTS = Path(__file__).parents[2] / "services/hermes/scripts" +sys.path.insert(0, str(SCRIPTS)) +discovery = importlib.import_module("claude_model_discovery") + + +def _initialize_event(*, provider: str = "firstParty") -> str: + """Build representative non-secret CLI initialization output.""" + return json.dumps( + { + "type": "control_response", + "response": { + "subtype": "success", + "request_id": "catalog", + "response": { + "account": { + "apiProvider": provider, + "email": "discard-me@example.test", + }, + "models": [ + { + "value": "opus", + "resolvedModel": "claude-opus-5", + "displayName": "Opus", + "description": "Most capable", + "supportsEffort": True, + "supportedEffortLevels": [ + "low", + "high", + "xhigh", + "unknown", + ], + "supportsAdaptiveThinking": True, + }, + { + "value": "haiku", + "resolvedModel": "claude-haiku-4-5-20251001", + "displayName": "Haiku", + "description": "Fast", + }, + ], + }, + }, + } + ) + + +def test_initialize_parses_live_subscription_models_without_account_data(): + """Literal aliases and the CLI-advertised effort set survive discovery.""" + result = discovery._parse_initialize_output(_initialize_event()) + + assert result.live is True + assert result.connected is True + assert result.error_code is None + assert [model.value for model in result.models] == ["opus", "haiku"] + assert result.models[0].resolved_model == "claude-opus-5" + assert result.models[0].supported_effort_levels == ("low", "high", "xhigh") + assert "email" not in result.models[0].as_dict() + rendered_models = json.dumps([model.as_dict() for model in result.models]) + assert "discard-me@example.test" not in rendered_models + + +def test_initialize_rejects_non_subscription_catalogs(): + """An API-key or third-party account cannot become a subscription route.""" + result = discovery._parse_initialize_output(_initialize_event(provider="apiKey")) + + assert result.live is False + assert result.connected is False + assert result.error_code == "not-first-party" + + +def test_discovery_never_uses_api_key_environment(monkeypatch): + """The native call inherits OAuth config but strips metered API-key variables.""" + observed: dict[str, object] = {} + + def fake_run(command, environment, timeout_seconds): + observed["command"] = command + observed["environment"] = environment + observed["timeout"] = timeout_seconds + return _initialize_event(), None + + monkeypatch.setattr(discovery, "_run_initialize", fake_run) + result = discovery.discover_claude_subscription_models( + claude_bin="/native/claude", + timeout_seconds=7, + environment={ + "CLAUDE_CONFIG_DIR": "/mounted/claude", + "ANTHROPIC_API_KEY": "secret", + "CLAUDE_API_KEY": "secret", + }, + ) + + assert result.live is True + assert observed["environment"] == {"CLAUDE_CONFIG_DIR": "/mounted/claude"} + assert observed["command"][:2] == ["/native/claude", "--input-format"] + assert "--safe-mode" in observed["command"] + assert "" in observed["command"] diff --git a/testing/tests/test_hermes_chat_routing_runtime.py b/testing/tests/test_hermes_chat_routing_runtime.py index 6a81d625..18a688be 100644 --- a/testing/tests/test_hermes_chat_routing_runtime.py +++ b/testing/tests/test_hermes_chat_routing_runtime.py @@ -411,8 +411,8 @@ def test_local_flux_runtime_and_gpu_handoff_are_flux_managed(): assert "route/local/qwen2.5-14b/medium" in switchyard assert "Anthropic and Claude name the same provider" in switchyard assert "OpenAI and Codex name the same provider" in switchyard - assert "Choose across every configured Codex and Claude family" in switchyard - assert "Claude Fable" in switchyard + assert "account-visible economy, balanced, or advanced" in switchyard + assert "Choose across every configured Codex and Claude family" not in switchyard assert switchyard.count('Treat "think hard"') == 4 assert switchyard.count("Never choose below the") >= 5 switchyard_config = tomllib.loads(switchyard) @@ -420,13 +420,13 @@ def test_local_flux_runtime_and_gpu_handoff_are_flux_managed(): configured_targets = switchyard_config["targets"] for route_name in ("auto_fast", "auto_balanced", "auto_deep", "auto_maximum"): leading_targets = set(routes[route_name]["targets"][:2]) - assert leading_targets == {"codex_sol_xhigh", "claude_opus_xhigh"} + assert leading_targets == {"codex_auto_xhigh", "claude_auto_xhigh"} assert "max_output_tokens" not in routes[route_name] for route_name in ("auto_fast", "auto_balanced", "auto_deep", "auto_maximum"): targets = routes[route_name]["targets"] selector_targets = routes[route_name]["response_schema"] assert not any(target.startswith("local_") for target in targets) - assert any("fable" in target for target in targets) + assert all("_auto_" in target or target.startswith("neutral_") for target in targets) assert "local_qwen" not in selector_targets assert "not eligible for foreground" in routes[route_name]["prompt"] for route_name in ("auto_deep", "auto_maximum"): @@ -470,10 +470,11 @@ def test_local_flux_runtime_and_gpu_handoff_are_flux_managed(): assert route["id"] == f"atlas/manual/{provider}/{family}/{effort}" assert route["targets"][0] == f"{provider}_{family}_{effort}" worker_target = f"worker_{provider}_{family}_{effort}" - assert worker_target in routes["worker_auto_maximum"]["targets"] assert configured_targets[worker_target]["id"] == ( f"worker/{provider}/{family}/{effort}" ) + assert all("_auto_" in target or target.startswith("neutral_") + for target in routes["worker_auto_maximum"]["targets"]) for route_name in ("auto_fast", "auto_balanced"): prompt = routes[route_name]["prompt"] assert "image tool—not the conversational model" in prompt diff --git a/testing/tests/test_hermes_cli_lane_routing.py b/testing/tests/test_hermes_cli_lane_routing.py index 49935d54..df93a5c6 100644 --- a/testing/tests/test_hermes_cli_lane_routing.py +++ b/testing/tests/test_hermes_cli_lane_routing.py @@ -5,6 +5,8 @@ from __future__ import annotations import importlib.util import json import sys + +import pytest from pathlib import Path @@ -79,3 +81,29 @@ def test_mismatched_broker_resolution_cannot_change_route_decision(): "terra", "xhigh", ) + + +def test_generic_claude_worker_keeps_a_catalog_advertised_alias(): + """The native CLI receives its provider alias after generic AUTO routing.""" + route = lanes.select_route( + "Review it.", + "cli-auto", + open_request=lambda *_args, **_kwargs: SwitchyardResponse( + "worker/claude/auto/medium", "worker/claude/sonnet/medium" + ), + catalog_model_allowed=lambda provider, model: provider == "claude" and model == "sonnet", + ) + assert (route.provider, route.model, route.effort) == ("claude", "sonnet", "medium") + + +def test_generic_claude_worker_rejects_an_unadvertised_alias(): + """A forged worker receipt cannot pass an arbitrary native Claude alias.""" + with pytest.raises(RuntimeError, match="current provider model"): + lanes.select_route( + "Review it.", + "cli-auto", + open_request=lambda *_args, **_kwargs: SwitchyardResponse( + "worker/claude/auto/medium", "worker/claude/not-advertised/medium" + ), + catalog_model_allowed=lambda *_args: False, + ) diff --git a/testing/tests/test_hermes_coordinator.py b/testing/tests/test_hermes_coordinator.py index 683c0201..391dc0a6 100644 --- a/testing/tests/test_hermes_coordinator.py +++ b/testing/tests/test_hermes_coordinator.py @@ -24,10 +24,10 @@ def test_model_version_and_quality_selection_handle_new_and_small_models(): codex = ["gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.7-luna", "gpt-5.3-codex-spark"] assert routing.choose_codex_model(codex) == "gpt-5.6-sol" assert routing.choose_codex_model(codex, balanced=True) == "gpt-5.6-terra" - assert routing.choose_codex_model(codex + ["gpt-5.7-terra"]) == "gpt-5.7-terra" + assert routing.choose_codex_model(codex + ["gpt-5.7-terra"]) == "gpt-5.6-sol" claude = ["claude-opus-4.8", "claude-haiku-5", "claude-sonnet-5"] - assert routing.choose_claude_model(claude) == "claude-sonnet-5" + assert routing.choose_claude_model(claude) == "claude-opus-4.8" assert routing.choose_codex_for_effort(codex, "low") == "gpt-5.7-luna" assert routing.choose_codex_for_effort(codex, "medium") == "gpt-5.6-terra" @@ -89,6 +89,284 @@ def test_dynamic_catalog_resolves_new_models_and_preserves_last_known_good(): ) +def test_dynamic_catalog_uses_metadata_for_unfamiliar_models_and_efforts(): + """Future IDs route by declared capability, price, and effort support.""" + codex = routing.Catalog( + "openai-codex", + [ + "gpt-6.2-orbit", + "gpt-6.2-balanced", + "gpt-6.2-quick", + "gpt-6.2-high-only", + "gpt-6.2-xhigh", + ], + True, + True, + "connected", + { + "gpt-6.2-orbit": { + "description": "A strongest upgrade for difficult, complex tasks", + "upgrade": "strongest", + "supported_reasoning_efforts": ["low", "medium", "high", "xhigh"], + "cost": 10, + }, + "gpt-6.2-balanced": { + "capability_tier": "balanced", + "description": "General purpose model for everyday tasks", + "supported_reasoning_efforts": ["low", "medium", "high", "xhigh"], + "cost": 2, + }, + "gpt-6.2-quick": { + "capability_tier": "economy", + "description": "Fast and affordable general purpose model", + "supported_reasoning_efforts": ["low", "medium"], + "cost": 1, + }, + "gpt-6.2-high-only": { + "capability_tier": "advanced", + "supported_reasoning_efforts": ["high"], + "cost": 1, + }, + "gpt-6.2-xhigh": { + "capability_tier": "advanced", + "supported_reasoning_efforts": ["xhigh"], + "cost": 2, + }, + }, + ) + claude = routing.Catalog("anthropic", [], True, True, "connected") + + catalog = routing.build_routing_catalog(codex, claude) + resolved = catalog["providers"]["codex"]["resolved"] + + assert resolved["xhigh"] == "gpt-6.2-orbit" + assert resolved["low"] == "gpt-6.2-quick" + assert resolved["medium"] == "gpt-6.2-balanced" + assert resolved["high"] == "gpt-6.2-orbit" + assert "gpt-6.2-high-only" in catalog["providers"]["codex"]["candidates"] + assert catalog["providers"]["codex"]["candidates"]["gpt-6.2-high-only"]["tier"] == "advanced" + + +def test_catalog_accepts_codex_effort_records_and_generalist_image_input(): + """Codex app-server effort records and multimodal generalists stay routable.""" + codex = routing.Catalog( + "openai-codex", + ["gpt-7-nova"], + True, + True, + "connected-app-server", + { + "gpt-7-nova": { + "description": "Strongest general purpose model with text and image inputs", + "supportedReasoningEfforts": [ + {"reasoningEffort": "low"}, + {"reasoningEffort": "medium"}, + {"reasoningEffort": "high"}, + {"reasoningEffort": "xhigh"}, + ], + } + }, + ) + catalog = routing.build_routing_catalog( + codex, routing.Catalog("anthropic", [], True, True, "connected") + ) + + assert catalog["providers"]["codex"]["resolved"]["xhigh"] == "gpt-7-nova" + + +def test_dynamic_catalog_excludes_hidden_internal_and_specialist_records(): + """Provider listings must not make private or modality-only products routable.""" + models = [ + "gpt-6.2-hidden", + "gpt-6.2-internal", + "gpt-6.2-image", + "gpt-6.2-public", + ] + metadata = { + "gpt-6.2-hidden": { + "hidden": True, + "capability_tier": "advanced", + }, + "gpt-6.2-internal": { + "visibility": "internal", + "description": "Internal evaluation model", + "capability_tier": "advanced", + }, + "gpt-6.2-image": { + "description": "Image generation specialist", + "capability_tier": "advanced", + }, + "gpt-6.2-public": { + "description": "Strongest general purpose model", + "capability_tier": "advanced", + }, + } + codex = routing.Catalog( + "openai-codex", models, True, True, "connected", metadata + ) + claude = routing.Catalog("anthropic", [], True, True, "connected") + + catalog = routing.build_routing_catalog(codex, claude) + provider = catalog["providers"]["codex"] + + assert provider["resolved"]["xhigh"] == "gpt-6.2-public" + assert provider["candidates"]["gpt-6.2-hidden"]["eligible"] is False + assert provider["candidates"]["gpt-6.2-internal"]["eligible"] is False + assert provider["candidates"]["gpt-6.2-image"]["eligible"] is False + + +def test_live_codex_metadata_selects_luna_terra_and_astra_by_effort(): + """Actual Codex model/list fields route generic effort without name guesses.""" + fixture = Path(__file__).parents[1] / "fixtures/hermes/codex-0.154-visible-models.json" + records = json.loads(fixture.read_text(encoding="utf-8")) + models, metadata = routing.model_records(records) + catalog = routing.build_routing_catalog( + routing.Catalog("openai-codex", models, True, True, "connected", metadata), + routing.Catalog("anthropic", [], True, True, "connected"), + ) + assert catalog["providers"]["codex"]["resolved"] == { + "low": "gpt-5.6-luna", + "medium": "gpt-5.6-terra", + "high": "gpt-6-astra", + "xhigh": "gpt-6-astra", + } + + +def test_live_default_flagship_replaces_previous_advanced_route(): + """A provider's current default wins over an equally strongest old model.""" + previous = { + "providers": {"codex": {"models": ["gpt-astra"], "resolved": { + effort: "gpt-astra" for effort in routing.EFFORTS + }, "model_metadata": {"gpt-astra": { + "description": "Most capable for complex work", "isDefault": False, + }}}} + } + metadata = { + "gpt-astra": {"description": "Most capable for complex work", "isDefault": False}, + "gpt-nova": {"description": "Most capable for complex work", "isDefault": True}, + } + catalog = routing.build_routing_catalog( + routing.Catalog("openai-codex", list(metadata), True, True, "connected", metadata), + routing.Catalog("anthropic", [], True, True, "connected"), previous, + ) + assert catalog["providers"]["codex"]["resolved"]["xhigh"] == "gpt-nova" + + +def test_live_claude_effort_gap_uses_nearest_higher_declared_tier(): + """An effortless economy alias cannot silently take a stronger route.""" + metadata = { + "haiku": {"description": "Fastest for quick answers", "supportsEffort": False}, + "sonnet": { + "description": "Efficient for routine tasks", "supportsEffort": True, + "supportedEffortLevels": ["low", "medium", "high", "xhigh"], + }, + "opus": { + "description": "Best for everyday, complex tasks", "supportsEffort": True, + "supportedEffortLevels": ["low", "medium", "high", "xhigh"], + }, + "fable": { + "description": "Most capable for your hardest and longest-running tasks", + "supportsEffort": True, + "supportedEffortLevels": ["low", "medium", "high", "xhigh"], + }, + } + catalog = routing.build_routing_catalog( + routing.Catalog("openai-codex", [], True, True, "connected"), + routing.Catalog("anthropic", list(metadata), True, True, "connected", metadata), + ) + assert catalog["providers"]["claude"]["resolved"] == { + "low": "sonnet", "medium": "sonnet", "high": "fable", "xhigh": "fable", + } + assert catalog["providers"]["claude"]["candidates"]["haiku"]["eligible"] is False + + +def test_ambiguous_live_catalog_exposes_an_unresolved_route(): + """An unfamiliar live model cannot revive a retired LKG model.""" + previous = { + "providers": { + "codex": { + "models": ["gpt-5.6-sol"], + "resolved": {effort: "gpt-5.6-sol" for effort in routing.EFFORTS}, + "tiers": {"advanced": "gpt-5.6-sol"}, + "model_metadata": {}, + } + } + } + codex = routing.Catalog( + "openai-codex", ["gpt-7.0-mystery"], True, True, "connected" + ) + claude = routing.Catalog("anthropic", [], False, True, "degraded") + + catalog = routing.build_routing_catalog(codex, claude, previous) + + assert catalog["providers"]["codex"]["resolved"]["xhigh"] == "" + assert catalog["providers"]["codex"]["candidates"]["gpt-7.0-mystery"]["tier"] is None + + +def test_live_removal_clears_legacy_selector_instead_of_mapping_to_new_family(): + """A live authoritative list cannot revive removed legacy selectors.""" + previous = { + "providers": { + "codex": { + "models": ["gpt-5.6-sol"], + "resolved": {effort: "gpt-5.6-sol" for effort in routing.EFFORTS}, + "tiers": {"sol": "gpt-5.6-sol", "advanced": "gpt-5.6-sol"}, + } + } + } + codex = routing.Catalog( + "openai-codex", + ["gpt-6.2-astra"], + True, + True, + "connected", + {"gpt-6.2-astra": {"capability_tier": "advanced"}}, + ) + claude = routing.Catalog("anthropic", [], False, True, "degraded") + + catalog = routing.build_routing_catalog(codex, claude, previous) + + assert catalog["providers"]["codex"]["tiers"]["sol"] == "" + assert catalog["providers"]["codex"]["resolved"]["xhigh"] == "gpt-6.2-astra" + + +def test_degraded_catalog_preserves_lkg_after_live_removal(): + """A provider outage retains the prior route and does not treat it as removal.""" + previous = { + "providers": { + "codex": { + "models": ["gpt-5.6-sol"], + "resolved": {effort: "gpt-5.6-sol" for effort in routing.EFFORTS}, + "tiers": {"sol": "gpt-5.6-sol"}, + "model_metadata": {}, + } + } + } + codex = routing.Catalog("openai-codex", [], False, True, "degraded") + claude = routing.Catalog("anthropic", [], False, True, "degraded") + + catalog = routing.build_routing_catalog(codex, claude, previous) + + assert catalog["providers"]["codex"]["resolved"]["xhigh"] == "gpt-5.6-sol" + assert catalog["providers"]["codex"]["tiers"]["sol"] == "gpt-5.6-sol" + + +def test_explicit_legacy_selector_never_maps_to_astra(): + """The old sol selector is exact and cannot silently follow a new family.""" + catalog = { + "providers": { + "codex": { + "tiers": {"sol": "gpt-6.2-astra"}, + "resolved": {"xhigh": "gpt-6.2-astra"}, + } + } + } + + assert catalog_resolver.resolve_model("codex", "sol", "xhigh", catalog) == ( + "gpt-5.6-sol" + ) + + def test_live_catalog_never_routes_to_a_removed_model_tier(): """A live provider catalog must replace a retired tier with a live model.""" previous = { @@ -116,9 +394,8 @@ def test_live_catalog_never_routes_to_a_removed_model_tier(): current = routing.build_routing_catalog(codex, claude, previous) - assert current["providers"]["codex"]["tiers"]["luna"] == "gpt-5.7-terra" - assert current["providers"]["codex"]["tiers"]["luna"] in codex.models - assert current["providers"]["claude"]["tiers"]["opus"] == "claude-sonnet-6" + assert current["providers"]["codex"]["tiers"]["luna"] == "" + assert current["providers"]["claude"]["tiers"]["opus"] == "" def test_switchyard_targets_have_unique_upstream_identities(): @@ -218,17 +495,17 @@ def test_configure_routes_keeps_every_profile_on_switchyard(tmp_path: Path): } assert root["fallback_providers"] == [] assert root["toolsets"] == ["kanban"] - assert codex_profile["model"]["model"] == "atlas/manual/codex/sol" + assert codex_profile["model"]["model"] == "atlas/manual/codex/auto/high" assert codex_profile["model"]["provider"] == "atlas-switchyard" - assert claude_profile["model"]["model"] == "atlas/manual/claude/sonnet" + assert claude_profile["model"]["model"] == "atlas/manual/claude/auto/high" assert claude_profile["model"]["provider"] == "atlas-switchyard" assert codex_profile["fallback_providers"] == [] assert claude_profile["fallback_providers"] == [] assert codex_profile["toolsets"] == [] assert codex_profile["agent"]["reasoning_effort"] == "high" assert codex_xhigh_profile["fallback_providers"] == [] - assert routes["codex-xhigh"] == ["atlas/manual/codex/sol"] - assert routes["claude-xhigh"] == ["atlas/manual/claude/opus"] + assert routes["codex-xhigh"] == ["atlas/manual/codex/auto/xhigh"] + assert routes["claude-xhigh"] == ["atlas/manual/claude/auto/xhigh"] assert routes["synthesis-xhigh"] == ["atlas/auto/maximum"] assert routes["coordinator"] == ["atlas/auto/maximum"] assert all( @@ -285,7 +562,7 @@ def test_degraded_refresh_preserves_switchyard_worker_preference(tmp_path: Path) worker = yaml.safe_load( (tmp_path / "profiles/codex-high/config.yaml").read_text(encoding="utf-8") ) - assert worker["model"]["model"] == "atlas/manual/codex/sol" + assert worker["model"]["model"] == "atlas/manual/codex/auto/high" assert worker["model"]["provider"] == "atlas-switchyard"