hermes: resolve difficulty routes from live model catalogs
This commit is contained in:
parent
2ab737f8f8
commit
d141b33a7d
@ -25,7 +25,7 @@ spec:
|
||||
ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers
|
||||
ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback
|
||||
ai.bstein.dev/placement: primary amd64 accelerator titan-22; arm64 rpi5 fleet fallback; storage-backbone nodes excluded
|
||||
ai.bstein.dev/config-rev: "20260912-dashboard-bootstrap-visibility"
|
||||
ai.bstein.dev/config-rev: "20260913-provider-model-catalog-v2"
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/path: /metrics
|
||||
prometheus.io/port: "9010"
|
||||
@ -475,11 +475,11 @@ spec:
|
||||
# which self-heals after the pod has run on the other architecture.
|
||||
case "$(uname -m)" in x86_64) nodearch=x64 ;; aarch64) nodearch=arm64 ;; *) nodearch=unknown ;; esac
|
||||
codex_native="${tools}/lib/node_modules/@openai/codex-linux-${nodearch}"
|
||||
if [ ! -f "${tools}/.cli-versions-0.147.0-2.1.226-${arch}" ] || [ ! -d "${codex_native}" ]; then
|
||||
if [ ! -f "${tools}/.cli-versions-0.154.0-2.1.226-${arch}" ] || [ ! -d "${codex_native}" ]; then
|
||||
npm install --global --omit=dev --no-audit --no-fund --prefix "${tools}" \
|
||||
@openai/codex@0.147.0 \
|
||||
@openai/codex@0.154.0 \
|
||||
@anthropic-ai/claude-code@2.1.226
|
||||
touch "${tools}/.cli-versions-0.147.0-2.1.226-${arch}"
|
||||
touch "${tools}/.cli-versions-0.154.0-2.1.226-${arch}"
|
||||
fi
|
||||
kubectl_version="$("${tools}/bin/kubectl" version --client --output=json 2>/dev/null || true)"
|
||||
case "${kubectl_version}" in *\"gitVersion\":\"v1.33.3\"*) kubectl_ready=1 ;; *) kubectl_ready=0 ;; esac
|
||||
|
||||
@ -61,9 +61,11 @@ data:
|
||||
atlas/auto/balanced: {provider: atlas-switchyard, model: atlas/auto/balanced}
|
||||
atlas/auto/deep: {provider: atlas-switchyard, model: atlas/auto/deep}
|
||||
atlas/auto/maximum: {provider: atlas-switchyard, model: atlas/auto/maximum}
|
||||
atlas/manual/codex/auto: {provider: atlas-switchyard, model: atlas/manual/codex/auto}
|
||||
atlas/manual/codex/luna: {provider: atlas-switchyard, model: atlas/manual/codex/luna}
|
||||
atlas/manual/codex/terra: {provider: atlas-switchyard, model: atlas/manual/codex/terra}
|
||||
atlas/manual/codex/sol: {provider: atlas-switchyard, model: atlas/manual/codex/sol}
|
||||
atlas/manual/claude/auto: {provider: atlas-switchyard, model: atlas/manual/claude/auto}
|
||||
atlas/manual/claude/haiku: {provider: atlas-switchyard, model: atlas/manual/claude/haiku}
|
||||
atlas/manual/claude/fable: {provider: atlas-switchyard, model: atlas/manual/claude/fable}
|
||||
atlas/manual/claude/sonnet: {provider: atlas-switchyard, model: atlas/manual/claude/sonnet}
|
||||
|
||||
@ -29,7 +29,7 @@ spec:
|
||||
ai.bstein.dev/router-wire-contract: ollama-numeric-keepalive
|
||||
ai.bstein.dev/isolation: one Hermes process and PVC per Keycloak subject
|
||||
ai.bstein.dev/model-policy: uniform automatic policy with per-user overrides
|
||||
ai.bstein.dev/config-rev: "20260816-telegram-topics"
|
||||
ai.bstein.dev/config-rev: "20260913-provider-model-catalog-v2"
|
||||
ai.bstein.dev/hux-config-rev: "20260824-hux-v1"
|
||||
vault.hashicorp.com/agent-inject: "true"
|
||||
vault.hashicorp.com/role: hermes-chat
|
||||
|
||||
@ -141,6 +141,9 @@ configMapGenerator:
|
||||
- worker_route_broker.py=scripts/worker_route_broker.py
|
||||
- hermes_coordinator.py=scripts/hermes_coordinator.py
|
||||
- hermes_model_routing.py=scripts/hermes_model_routing.py
|
||||
- provider_model_catalog.py=scripts/provider_model_catalog.py
|
||||
- provider_model_discovery.py=scripts/provider_model_discovery.py
|
||||
- claude_model_discovery.py=scripts/claude_model_discovery.py
|
||||
- hermes_stt_client.py=scripts/hermes_stt_client.py
|
||||
- image_broker.py=scripts/image_broker.py
|
||||
- install_agent_tools.sh=scripts/install_agent_tools.sh
|
||||
|
||||
@ -51,10 +51,12 @@ MANUAL_ROUTES = frozenset(
|
||||
"atlas/manual/codex/luna",
|
||||
"atlas/manual/codex/terra",
|
||||
"atlas/manual/codex/sol",
|
||||
"atlas/manual/codex/auto",
|
||||
"atlas/manual/claude/haiku",
|
||||
"atlas/manual/claude/fable",
|
||||
"atlas/manual/claude/sonnet",
|
||||
"atlas/manual/claude/opus",
|
||||
"atlas/manual/claude/auto",
|
||||
"atlas/manual/local/qwen-14b",
|
||||
}
|
||||
)
|
||||
@ -69,8 +71,8 @@ EXACT_MANUAL_ROUTES = frozenset(
|
||||
)
|
||||
ALL_REQUEST_ROUTES = ALL_ROUTES | EXACT_MANUAL_ROUTES
|
||||
PROVIDER_DEFAULT = {
|
||||
"codex": "atlas/manual/codex/terra",
|
||||
"claude": "atlas/manual/claude/sonnet",
|
||||
"codex": "atlas/manual/codex/auto",
|
||||
"claude": "atlas/manual/claude/auto",
|
||||
"local": "atlas/manual/local/qwen-14b",
|
||||
}
|
||||
|
||||
|
||||
280
services/hermes/scripts/claude_model_discovery.py
Normal file
280
services/hermes/scripts/claude_model_discovery.py
Normal file
@ -0,0 +1,280 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Discover the current Claude subscription model catalog without inference.
|
||||
|
||||
Claude Code exposes the account-visible ``ModelInfo[]`` through its Agent SDK
|
||||
``initialize`` control response. This module speaks that documented stream-JSON
|
||||
control protocol directly instead of treating authentication status or a cached
|
||||
model list as a live catalog. It intentionally retains only non-secret
|
||||
ModelInfo metadata.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Mapping, Sequence
|
||||
|
||||
|
||||
DEFAULT_CLAUDE_BIN = "/opt/data/tools/bin/claude"
|
||||
DEFAULT_TIMEOUT_SECONDS = 30.0
|
||||
MAX_MODELS = 256
|
||||
EFFORT_LEVELS = frozenset({"low", "medium", "high", "xhigh", "max"})
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ClaudeModel:
|
||||
"""Safe metadata for one model advertised by the native Claude CLI."""
|
||||
|
||||
value: str
|
||||
resolved_model: str | None
|
||||
display_name: str
|
||||
description: str
|
||||
supports_effort: bool
|
||||
supported_effort_levels: tuple[str, ...]
|
||||
supports_adaptive_thinking: bool
|
||||
supports_fast_mode: bool
|
||||
supports_auto_mode: bool
|
||||
disabled: bool
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
"""Serialize only ModelInfo metadata, preserving Claude's alias value."""
|
||||
return {
|
||||
"value": self.value,
|
||||
"resolvedModel": self.resolved_model,
|
||||
"displayName": self.display_name,
|
||||
"description": self.description,
|
||||
"supportsEffort": self.supports_effort,
|
||||
"supportedEffortLevels": list(self.supported_effort_levels),
|
||||
"supportsAdaptiveThinking": self.supports_adaptive_thinking,
|
||||
"supportsFastMode": self.supports_fast_mode,
|
||||
"supportsAutoMode": self.supports_auto_mode,
|
||||
"disabled": self.disabled,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ClaudeModelDiscovery:
|
||||
"""Result of one bounded, subscription-authenticated catalog probe."""
|
||||
|
||||
models: tuple[ClaudeModel, ...]
|
||||
live: bool
|
||||
connected: bool
|
||||
state: str
|
||||
error_code: str | None = None
|
||||
|
||||
|
||||
def _native_environment(base: Mapping[str, str] | None = None) -> dict[str, str]:
|
||||
"""Keep mounted OAuth configuration while preventing API-key billing lanes."""
|
||||
environment = dict(os.environ if base is None else base)
|
||||
environment.pop("ANTHROPIC_API_KEY", None)
|
||||
environment.pop("CLAUDE_API_KEY", None)
|
||||
return environment
|
||||
|
||||
|
||||
def _command(claude_bin: str) -> list[str]:
|
||||
"""Build a no-inference CLI invocation with all tools and MCP disabled."""
|
||||
return [
|
||||
claude_bin,
|
||||
"--input-format",
|
||||
"stream-json",
|
||||
"--output-format",
|
||||
"stream-json",
|
||||
"--verbose",
|
||||
"--print",
|
||||
"--no-session-persistence",
|
||||
"--safe-mode",
|
||||
"--tools",
|
||||
"",
|
||||
"--strict-mcp-config",
|
||||
"--mcp-config",
|
||||
'{"mcpServers":{}}',
|
||||
]
|
||||
|
||||
|
||||
def _initialize_request() -> str:
|
||||
"""Build the Claude Agent SDK initialize request for ModelInfo discovery."""
|
||||
return json.dumps(
|
||||
{
|
||||
"type": "control_request",
|
||||
"request_id": f"hermes-model-catalog-{uuid.uuid4()}",
|
||||
"request": {"subtype": "initialize", "hooks": {}},
|
||||
},
|
||||
separators=(",", ":"),
|
||||
)
|
||||
|
||||
|
||||
def _stop_process(process: subprocess.Popen[str]) -> None:
|
||||
"""End a timed-out CLI process and its process group without leaking output."""
|
||||
if process.poll() is not None:
|
||||
return
|
||||
try:
|
||||
os.killpg(process.pid, signal.SIGTERM)
|
||||
except (OSError, ProcessLookupError):
|
||||
process.terminate()
|
||||
try:
|
||||
process.communicate(timeout=2)
|
||||
return
|
||||
except subprocess.TimeoutExpired:
|
||||
pass
|
||||
try:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
except (OSError, ProcessLookupError):
|
||||
process.kill()
|
||||
try:
|
||||
process.communicate(timeout=2)
|
||||
except subprocess.TimeoutExpired:
|
||||
pass
|
||||
|
||||
|
||||
def _run_initialize(
|
||||
command: Sequence[str], environment: Mapping[str, str], timeout_seconds: float
|
||||
) -> tuple[str | None, str | None]:
|
||||
"""Run initialize once, returning stream output or a non-sensitive error code."""
|
||||
try:
|
||||
process = subprocess.Popen(
|
||||
list(command),
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
text=True,
|
||||
env=dict(environment),
|
||||
start_new_session=True,
|
||||
)
|
||||
except OSError:
|
||||
return None, "spawn-failed"
|
||||
try:
|
||||
request = _initialize_request() + "\n"
|
||||
stdout, _ = process.communicate(request, timeout=timeout_seconds)
|
||||
except subprocess.TimeoutExpired:
|
||||
_stop_process(process)
|
||||
return None, "timeout"
|
||||
except OSError:
|
||||
_stop_process(process)
|
||||
return None, "transport-error"
|
||||
if process.returncode != 0:
|
||||
return None, "cli-error"
|
||||
return stdout, None
|
||||
|
||||
|
||||
def _text(value: Any, *, maximum: int = 1024) -> str:
|
||||
"""Return bounded string metadata and reject non-string protocol values."""
|
||||
return value[:maximum] if isinstance(value, str) else ""
|
||||
|
||||
|
||||
def _model_info(value: Any) -> ClaudeModel | None:
|
||||
"""Validate one ModelInfo record without inventing aliases or identifiers."""
|
||||
if not isinstance(value, dict):
|
||||
return None
|
||||
alias = _text(value.get("value"))
|
||||
if not alias:
|
||||
return None
|
||||
resolved_model = _text(value.get("resolvedModel")) or None
|
||||
efforts = value.get("supportedEffortLevels")
|
||||
supported_effort_levels = (
|
||||
tuple(
|
||||
level
|
||||
for level in efforts
|
||||
if isinstance(level, str) and level in EFFORT_LEVELS
|
||||
)
|
||||
if isinstance(efforts, list)
|
||||
else ()
|
||||
)
|
||||
return ClaudeModel(
|
||||
value=alias,
|
||||
resolved_model=resolved_model,
|
||||
display_name=_text(value.get("displayName")),
|
||||
description=_text(value.get("description")),
|
||||
supports_effort=value.get("supportsEffort") is True,
|
||||
supported_effort_levels=supported_effort_levels,
|
||||
supports_adaptive_thinking=value.get("supportsAdaptiveThinking") is True,
|
||||
supports_fast_mode=value.get("supportsFastMode") is True,
|
||||
supports_auto_mode=value.get("supportsAutoMode") is True,
|
||||
disabled=value.get("disabled") is True,
|
||||
)
|
||||
|
||||
|
||||
def _parse_initialize_output(stdout: str) -> ClaudeModelDiscovery:
|
||||
"""Extract all ModelInfo pages present in initialize control responses.
|
||||
|
||||
Claude Code 2.1.226 returns its complete ``models`` array in one response.
|
||||
The stream parser still gathers every valid initialize response so a future
|
||||
paged/multi-frame implementation is not silently truncated. The protocol
|
||||
currently specifies no continuation request, so this function never
|
||||
fabricates one.
|
||||
"""
|
||||
models: list[ClaudeModel] = []
|
||||
seen: set[tuple[str, str | None]] = set()
|
||||
received_models = False
|
||||
first_party = False
|
||||
for line in stdout.splitlines():
|
||||
try:
|
||||
event = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if not isinstance(event, dict) or event.get("type") != "control_response":
|
||||
continue
|
||||
response = event.get("response")
|
||||
if not isinstance(response, dict) or response.get("subtype") != "success":
|
||||
continue
|
||||
payload = response.get("response")
|
||||
if not isinstance(payload, dict) or not isinstance(payload.get("models"), list):
|
||||
continue
|
||||
received_models = True
|
||||
account = payload.get("account")
|
||||
if isinstance(account, dict) and account.get("apiProvider") == "firstParty":
|
||||
first_party = True
|
||||
for raw_model in payload["models"]:
|
||||
model = _model_info(raw_model)
|
||||
if model is None or len(models) >= MAX_MODELS:
|
||||
continue
|
||||
key = (model.value, model.resolved_model)
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
models.append(model)
|
||||
if not received_models:
|
||||
return ClaudeModelDiscovery(
|
||||
(), False, False, "protocol-error", "protocol-error"
|
||||
)
|
||||
if not first_party:
|
||||
return ClaudeModelDiscovery(
|
||||
tuple(models), False, False, "not-subscription", "not-first-party"
|
||||
)
|
||||
if not models:
|
||||
return ClaudeModelDiscovery(
|
||||
(), True, True, "connected-subscription", "empty-model-catalog"
|
||||
)
|
||||
return ClaudeModelDiscovery(tuple(models), True, True, "connected-subscription")
|
||||
|
||||
|
||||
def discover_claude_subscription_models(
|
||||
*,
|
||||
claude_bin: str | None = None,
|
||||
timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS,
|
||||
environment: Mapping[str, str] | None = None,
|
||||
) -> ClaudeModelDiscovery:
|
||||
"""Return current subscription models using only Claude's initialize protocol.
|
||||
|
||||
A successful native authentication-status command or a locally cached model
|
||||
list is deliberately insufficient: ``live`` is true only after a valid,
|
||||
first-party initialize response supplies the account-visible ``models``.
|
||||
"""
|
||||
binary = (
|
||||
claude_bin
|
||||
or os.environ.get("HERMES_CLAUDE_NATIVE_BIN")
|
||||
or DEFAULT_CLAUDE_BIN
|
||||
)
|
||||
if timeout_seconds <= 0:
|
||||
return ClaudeModelDiscovery(
|
||||
(), False, False, "invalid-request", "invalid-timeout"
|
||||
)
|
||||
stdout, error_code = _run_initialize(
|
||||
_command(binary), _native_environment(environment), timeout_seconds
|
||||
)
|
||||
if error_code:
|
||||
state = "unavailable" if error_code == "spawn-failed" else "degraded"
|
||||
return ClaudeModelDiscovery((), False, False, state, error_code)
|
||||
return _parse_initialize_output(stdout or "")
|
||||
@ -67,6 +67,15 @@ _auth_probe_value: dict[str, Any] = {}
|
||||
HEALTH_POLL_SECONDS: Final = max(
|
||||
30, int(os.environ.get("HERMES_CLAUDE_HEALTH_POLL_SECONDS", "60"))
|
||||
)
|
||||
|
||||
|
||||
def _catalog_contains(model: str) -> bool:
|
||||
"""Accept a provider-advertised Claude alias without trusting arbitrary IDs."""
|
||||
catalog = load_catalog()
|
||||
providers = catalog.get("providers", {}) if isinstance(catalog, dict) else {}
|
||||
claude = providers.get("claude", {}) if isinstance(providers, dict) else {}
|
||||
models = claude.get("models", []) if isinstance(claude, dict) else []
|
||||
return isinstance(model, str) and model in models
|
||||
def _read_secret(file_env_name: str) -> str:
|
||||
"""Read a secret only from its runtime-mounted file."""
|
||||
path = os.environ.get(file_env_name, "").strip()
|
||||
@ -108,7 +117,7 @@ def _route(model: str, payload: dict[str, Any]) -> tuple[str, str]:
|
||||
output_config = payload.get("output_config")
|
||||
if isinstance(output_config, dict) and output_config.get("effort") in EFFORTS:
|
||||
effort = str(output_config["effort"])
|
||||
if not model.startswith("claude-"):
|
||||
if not model.startswith("claude-") and not _catalog_contains(model):
|
||||
raise ValueError("unsupported Claude model")
|
||||
return model, effort
|
||||
|
||||
@ -626,7 +635,7 @@ class Handler(BaseHTTPRequestHandler):
|
||||
models = sorted(
|
||||
model
|
||||
for model in raw_models
|
||||
if isinstance(model, str) and model.startswith("claude-")
|
||||
if isinstance(model, str)
|
||||
)
|
||||
self._json(200, {"data": [{"id": model, "type": "model"} for model in models]})
|
||||
|
||||
|
||||
@ -19,6 +19,7 @@ from cli_lane_config import (
|
||||
Route,
|
||||
)
|
||||
from cli_lane_files import load_json
|
||||
from routing_catalog import catalog_contains
|
||||
|
||||
|
||||
def parse_assignee(assignee: str) -> tuple[str | None, str | None]:
|
||||
@ -107,6 +108,7 @@ def select_route(
|
||||
exclude_reason: str | None = None,
|
||||
switchyard_url: str = SWITCHYARD_URL,
|
||||
open_request: Callable[..., Any] = urllib.request.urlopen,
|
||||
catalog_model_allowed: Callable[[str, str], bool] = catalog_contains,
|
||||
) -> Route:
|
||||
"""Ask Switchyard to select one native CLI worker at this boundary."""
|
||||
started = time.monotonic()
|
||||
@ -149,19 +151,26 @@ def select_route(
|
||||
# Switchyard preserves the stable tier target in the selection header and
|
||||
# top-level response model. The worker broker's assistant content contains
|
||||
# the steward-resolved provider model required by the native CLI.
|
||||
resolved_valid = False
|
||||
try:
|
||||
response_document = json.loads(response_body)
|
||||
resolved_target = str(response_document["choices"][0]["message"]["content"] or "")
|
||||
resolved_provider, resolved_model, resolved_effort = _decode_worker_target(resolved_target)
|
||||
required_prefix = "gpt-" if provider == "codex" else "claude-"
|
||||
if (
|
||||
resolved_valid = (
|
||||
resolved_provider == provider
|
||||
and resolved_effort == effort
|
||||
and resolved_model.startswith(required_prefix)
|
||||
):
|
||||
and (
|
||||
resolved_model.startswith(required_prefix)
|
||||
or (provider == "claude" and catalog_model_allowed(provider, resolved_model))
|
||||
)
|
||||
)
|
||||
if resolved_valid:
|
||||
model = resolved_model
|
||||
except (AttributeError, IndexError, KeyError, RuntimeError, TypeError, ValueError, json.JSONDecodeError):
|
||||
pass
|
||||
resolved_valid = False
|
||||
if model == "auto" and not resolved_valid:
|
||||
raise RuntimeError("Switchyard did not return a current provider model")
|
||||
if exclude_provider and provider == exclude_provider:
|
||||
alternate = "claude" if provider == "codex" else "codex"
|
||||
guarded = select_route(
|
||||
@ -169,6 +178,7 @@ def select_route(
|
||||
f"cli-{alternate}-{effort}",
|
||||
switchyard_url=switchyard_url,
|
||||
open_request=open_request,
|
||||
catalog_model_allowed=catalog_model_allowed,
|
||||
)
|
||||
return Route(
|
||||
provider=guarded.provider,
|
||||
|
||||
@ -18,7 +18,7 @@ from typing import Any, Iterable
|
||||
|
||||
import httpx
|
||||
|
||||
from routing_catalog import resolve_route
|
||||
from routing_catalog import load_catalog, resolve_route
|
||||
|
||||
|
||||
HOST = os.environ.get("HERMES_CODEX_BROKER_LISTEN_HOST", "0.0.0.0")
|
||||
@ -580,13 +580,20 @@ class Handler(BaseHTTPRequestHandler):
|
||||
self._json(200, {"ok": True, "provider": "openai-codex"})
|
||||
return
|
||||
if self.path in {"/models", "/v1/models"}:
|
||||
catalog = load_catalog()
|
||||
providers = catalog.get("providers", {}) if isinstance(catalog, dict) else {}
|
||||
codex = providers.get("codex", {}) if isinstance(providers, dict) else {}
|
||||
discovered = codex.get("models", []) if isinstance(codex, dict) else []
|
||||
models = {
|
||||
model for model in discovered if isinstance(model, str) and model.startswith("gpt-")
|
||||
} or FALLBACK_ALLOWED_MODELS
|
||||
self._json(
|
||||
200,
|
||||
{
|
||||
"object": "list",
|
||||
"data": [
|
||||
{"id": model, "object": "model", "owned_by": "openai-codex"}
|
||||
for model in sorted(FALLBACK_ALLOWED_MODELS)
|
||||
for model in sorted(models)
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
@ -6,26 +6,29 @@ from __future__ import annotations
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
import yaml
|
||||
|
||||
from provider_model_catalog import (
|
||||
CAPABILITY_TIERS,
|
||||
EFFORTS,
|
||||
EFFORT_TIERS,
|
||||
Catalog,
|
||||
model_records,
|
||||
model_version,
|
||||
select_tier_model,
|
||||
unique_models,
|
||||
)
|
||||
from provider_model_discovery import discover_claude_models, discover_codex_models
|
||||
|
||||
|
||||
CODEX_BASELINE = "gpt-5.6-terra"
|
||||
CLAUDE_BASELINE = "claude-opus-5"
|
||||
CLAUDE_SUBSCRIPTION_MODELS = (
|
||||
"claude-haiku-4-5-20251001",
|
||||
"claude-fable-5",
|
||||
"claude-sonnet-5",
|
||||
"claude-opus-5",
|
||||
)
|
||||
EFFORTS = ("low", "medium", "high", "xhigh")
|
||||
ATLAS_FALLBACK = {
|
||||
"provider": "custom",
|
||||
"model": "qwen2.5:14b-instruct-q4_0",
|
||||
@ -38,166 +41,46 @@ SWITCHYARD_PROVIDER = "atlas-switchyard"
|
||||
SWITCHYARD_API = "http://hermes-switchyard.hermes.svc.cluster.local:9005/v1"
|
||||
SWITCHYARD_AUTO_ROUTE = "atlas/auto/maximum"
|
||||
ROUTING_CATALOG_PATH = os.environ.get("HERMES_ROUTING_CATALOG_PATH", "").strip()
|
||||
MANAGED_ENV_KEYS = {
|
||||
"GIT_ASKPASS",
|
||||
"GIT_TERMINAL_PROMPT",
|
||||
}
|
||||
MANAGED_ENV_KEYS = {"GIT_ASKPASS", "GIT_TERMINAL_PROMPT"}
|
||||
RUNTIME_SECRET_ENV_KEYS = {
|
||||
"ANTHROPIC_API_KEY",
|
||||
"API_SERVER_KEY",
|
||||
"CLAUDE_API_KEY",
|
||||
"CLAUDE_CODE_OAUTH_TOKEN",
|
||||
"GITEA_TOKEN",
|
||||
"GITEA_USERNAME",
|
||||
"HERMES_IMAGE_BROKER_KEY",
|
||||
"OPENAI_API_KEY",
|
||||
"ANTHROPIC_API_KEY", "API_SERVER_KEY", "CLAUDE_API_KEY", "CLAUDE_CODE_OAUTH_TOKEN",
|
||||
"GITEA_TOKEN", "GITEA_USERNAME", "HERMES_IMAGE_BROKER_KEY", "OPENAI_API_KEY",
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Catalog:
|
||||
"""Non-secret provider discovery result."""
|
||||
|
||||
provider: str
|
||||
models: list[str]
|
||||
live: bool
|
||||
connected: bool
|
||||
state: str
|
||||
def choose_codex_model(models: Iterable[str], current: str = CODEX_BASELINE, *, balanced: bool = False) -> str:
|
||||
"""Compatibility helper selecting a declared advanced or balanced Codex tier."""
|
||||
effort = "medium" if balanced else "xhigh"
|
||||
tier = "balanced" if balanced else "advanced"
|
||||
return select_tier_model("codex", models, {}, tier, effort, current)[0]
|
||||
|
||||
|
||||
def _unique_models(models: Iterable[str]) -> list[str]:
|
||||
"""Return normalized model IDs without changing provider order."""
|
||||
seen: set[str] = set()
|
||||
result: list[str] = []
|
||||
for model in models:
|
||||
value = str(model or "").strip()
|
||||
if value and value.lower() not in seen:
|
||||
result.append(value)
|
||||
seen.add(value.lower())
|
||||
return result
|
||||
|
||||
|
||||
def model_version(model: str) -> tuple[int, ...]:
|
||||
"""Extract a sortable model version while ignoring dated aliases."""
|
||||
value = re.sub(r"-\d{8}$", "", model.lower())
|
||||
groups = re.findall(r"\d+(?:\.\d+)*", value)
|
||||
parts: list[int] = []
|
||||
for group in groups:
|
||||
parts.extend(int(piece) for piece in group.split("."))
|
||||
return tuple(parts) or (0,)
|
||||
|
||||
|
||||
def _codex_quality(model: str) -> int:
|
||||
"""Rank known Codex capability tiers for delegated implementation work."""
|
||||
value = model.lower()
|
||||
if "sol" in value:
|
||||
return 60
|
||||
if "terra" in value:
|
||||
return 50
|
||||
if "codex" in value and "spark" not in value:
|
||||
return 45
|
||||
if re.fullmatch(r"gpt-\d+(?:\.\d+)*", value):
|
||||
return 40
|
||||
if "luna" in value:
|
||||
return 20
|
||||
if "mini" in value or "nano" in value:
|
||||
return 10
|
||||
if "spark" in value:
|
||||
return 5
|
||||
return 0
|
||||
|
||||
|
||||
def choose_codex_model(
|
||||
models: Iterable[str], current: str = CODEX_BASELINE, *, balanced: bool = False
|
||||
) -> str:
|
||||
"""Choose a current full Codex model, retaining current when unknown."""
|
||||
candidates = [m for m in _unique_models(models) if m.lower().startswith("gpt-")]
|
||||
full = [m for m in candidates if _codex_quality(m) >= 40]
|
||||
pool = full or candidates
|
||||
if not pool:
|
||||
return current
|
||||
newest = max(model_version(model) for model in pool)
|
||||
latest = [model for model in pool if model_version(model) == newest]
|
||||
if balanced:
|
||||
terra = [model for model in latest if "terra" in model.lower()]
|
||||
if terra:
|
||||
return max(terra, key=lambda model: (_codex_quality(model), model))
|
||||
return max(latest, key=lambda model: (_codex_quality(model), model))
|
||||
|
||||
|
||||
def _choose_by_hints(
|
||||
models: Iterable[str], hints: tuple[str, ...], current: str, prefix: str
|
||||
) -> str:
|
||||
"""Pick the newest model in the first available capability class."""
|
||||
candidates = [
|
||||
model
|
||||
for model in _unique_models(models)
|
||||
if model.lower().startswith(prefix)
|
||||
]
|
||||
if not candidates:
|
||||
return current
|
||||
for hint in hints:
|
||||
tier = [model for model in candidates if hint in model.lower()]
|
||||
if tier:
|
||||
return max(tier, key=lambda model: (model_version(model), model))
|
||||
return max(candidates, key=lambda model: (model_version(model), model))
|
||||
|
||||
|
||||
def choose_codex_for_effort(
|
||||
models: Iterable[str], effort: str, current: str = CODEX_BASELINE
|
||||
) -> str:
|
||||
"""Choose the account-visible Codex tier for an allowed effort level."""
|
||||
hints = {
|
||||
"low": ("luna", "mini", "spark", "terra", "sol", "codex"),
|
||||
"medium": ("terra", "codex", "sol", "luna", "mini", "spark"),
|
||||
"high": ("sol", "codex", "terra", "luna", "mini", "spark"),
|
||||
"xhigh": ("sol", "codex", "terra", "luna", "mini", "spark"),
|
||||
}
|
||||
def choose_codex_for_effort(models: Iterable[str], effort: str, current: str = CODEX_BASELINE) -> str:
|
||||
"""Compatibility helper selecting an effort's generic Codex capability tier."""
|
||||
if effort not in EFFORTS:
|
||||
raise ValueError(f"unsupported effort: {effort}")
|
||||
return _choose_by_hints(models, hints[effort], current, "gpt-")
|
||||
|
||||
|
||||
def _claude_quality(model: str) -> int:
|
||||
"""Rank known Claude capability tiers for architecture and review work."""
|
||||
value = model.lower()
|
||||
if "opus" in value:
|
||||
return 40
|
||||
if "sonnet" in value:
|
||||
return 30
|
||||
if "fable" in value:
|
||||
return 20
|
||||
if "haiku" in value:
|
||||
return 10
|
||||
return 0
|
||||
return select_tier_model("codex", models, {}, EFFORT_TIERS[effort], effort, current)[0]
|
||||
|
||||
|
||||
def choose_claude_model(models: Iterable[str], current: str = CLAUDE_BASELINE) -> str:
|
||||
"""Choose the newest full Claude reasoning model visible to the account."""
|
||||
candidates = [m for m in _unique_models(models) if m.lower().startswith("claude-")]
|
||||
full = [m for m in candidates if _claude_quality(m) >= 30]
|
||||
pool = full or candidates
|
||||
if not pool:
|
||||
return current
|
||||
return max(
|
||||
pool,
|
||||
key=lambda model: (model_version(model), _claude_quality(model), model),
|
||||
)
|
||||
"""Compatibility helper selecting the declared advanced Claude tier."""
|
||||
return select_tier_model("claude", models, {}, "advanced", "xhigh", current)[0]
|
||||
|
||||
|
||||
def choose_claude_for_effort(
|
||||
models: Iterable[str], effort: str, current: str = CLAUDE_BASELINE
|
||||
) -> str:
|
||||
"""Choose the account-visible Claude tier for an allowed effort level."""
|
||||
hints = {
|
||||
"low": ("haiku", "fable", "sonnet", "opus"),
|
||||
"medium": ("sonnet", "fable", "opus", "haiku"),
|
||||
"high": ("opus", "sonnet", "fable", "haiku"),
|
||||
"xhigh": ("opus", "sonnet", "fable", "haiku"),
|
||||
}
|
||||
"""Compatibility helper selecting an effort's generic Claude capability tier."""
|
||||
if effort not in EFFORTS:
|
||||
raise ValueError(f"unsupported effort: {effort}")
|
||||
return _choose_by_hints(models, hints[effort], current, "claude-")
|
||||
return select_tier_model("claude", models, {}, EFFORT_TIERS[effort], effort, current)[0]
|
||||
|
||||
|
||||
# The coordinator retains these private spellings for compact call sites while
|
||||
# provider_model_catalog owns the metadata policy and its independent tests.
|
||||
_unique_models = unique_models
|
||||
_model_records = model_records
|
||||
_select_tier_model = select_tier_model
|
||||
|
||||
|
||||
def _read_yaml(path: Path) -> dict[str, Any]:
|
||||
@ -245,34 +128,42 @@ def _read_json(path: Path) -> dict[str, Any]:
|
||||
|
||||
def _previous_provider_models(
|
||||
previous: dict[str, Any], provider: str
|
||||
) -> tuple[dict[str, str], dict[str, str], list[str]]:
|
||||
"""Return last-known-good effort, tier, and catalog values for a provider."""
|
||||
) -> tuple[dict[str, str], dict[str, str], list[str], dict[str, dict[str, Any]]]:
|
||||
"""Return last-known-good effort, selector, models, and metadata values."""
|
||||
providers = previous.get("providers", {})
|
||||
record = providers.get(provider, {}) if isinstance(providers, dict) else {}
|
||||
if not isinstance(record, dict):
|
||||
return {}, {}, []
|
||||
return {}, {}, [], {}
|
||||
resolved = record.get("resolved", {})
|
||||
tiers = record.get("tiers", {})
|
||||
models = record.get("models", [])
|
||||
metadata = record.get("model_metadata", {})
|
||||
return (
|
||||
dict(resolved) if isinstance(resolved, dict) else {},
|
||||
dict(tiers) if isinstance(tiers, dict) else {},
|
||||
_unique_models(models if isinstance(models, list) else []),
|
||||
{
|
||||
str(model): dict(details)
|
||||
for model, details in metadata.items()
|
||||
if isinstance(model, str) and isinstance(details, dict)
|
||||
}
|
||||
if isinstance(metadata, dict)
|
||||
else {},
|
||||
)
|
||||
|
||||
|
||||
def build_routing_catalog(
|
||||
codex: Catalog, claude: Catalog, previous: dict[str, Any] | None = None
|
||||
) -> dict[str, Any]:
|
||||
"""Build a current catalog while retaining working routes during outages."""
|
||||
"""Build a current catalog while retaining only outage-safe known routes."""
|
||||
previous = previous or {}
|
||||
checked_at = int(time.time())
|
||||
providers: dict[str, Any] = {}
|
||||
specifications = (
|
||||
(
|
||||
"codex",
|
||||
codex,
|
||||
choose_codex_for_effort,
|
||||
{"luna": "low", "terra": "medium", "sol": "high"},
|
||||
{"luna": "economy", "terra": "balanced", "sol": "advanced"},
|
||||
{
|
||||
"low": "gpt-5.6-luna",
|
||||
"medium": "gpt-5.6-terra",
|
||||
@ -283,12 +174,11 @@ def build_routing_catalog(
|
||||
(
|
||||
"claude",
|
||||
claude,
|
||||
choose_claude_for_effort,
|
||||
{
|
||||
"haiku": "low",
|
||||
"fable": "medium",
|
||||
"sonnet": "high",
|
||||
"opus": "xhigh",
|
||||
"haiku": "economy",
|
||||
"fable": "advanced",
|
||||
"sonnet": "balanced",
|
||||
"opus": "advanced",
|
||||
},
|
||||
{
|
||||
"low": "claude-haiku-4-5-20251001",
|
||||
@ -298,27 +188,64 @@ def build_routing_catalog(
|
||||
},
|
||||
),
|
||||
)
|
||||
for name, discovered, chooser, tier_efforts, defaults in specifications:
|
||||
old_resolved, old_tiers, old_models = _previous_provider_models(previous, name)
|
||||
for name, discovered, legacy_selectors, defaults in specifications:
|
||||
old_resolved, old_tiers, old_models, old_metadata = _previous_provider_models(
|
||||
previous, name
|
||||
)
|
||||
old_providers = previous.get("providers", {})
|
||||
old_record = old_providers.get(name, {}) if isinstance(old_providers, dict) else {}
|
||||
previous_success = (
|
||||
int(old_record.get("last_success_at"))
|
||||
if isinstance(old_record, dict) and isinstance(old_record.get("last_success_at"), int)
|
||||
else None
|
||||
)
|
||||
source_models = discovered.models if discovered.live else old_models
|
||||
source_metadata = discovered.metadata if discovered.live else old_metadata
|
||||
resolved: dict[str, str] = {}
|
||||
observations: dict[str, Any] = {}
|
||||
for effort in EFFORTS:
|
||||
# A successful provider list is authoritative: a retired model
|
||||
# must not survive it merely because it was last known-good.
|
||||
current = str(old_resolved.get(effort) or defaults[effort])
|
||||
resolved[effort] = (
|
||||
chooser(source_models, effort, current)
|
||||
if source_models
|
||||
else current
|
||||
selected, observed = _select_tier_model(
|
||||
name,
|
||||
source_models,
|
||||
source_metadata,
|
||||
EFFORT_TIERS[effort],
|
||||
effort,
|
||||
current,
|
||||
allow_current_fallback=not discovered.live,
|
||||
)
|
||||
resolved[effort] = selected
|
||||
observations.update(observed)
|
||||
tiers: dict[str, str] = {}
|
||||
for tier, effort in tier_efforts.items():
|
||||
current = str(old_tiers.get(tier) or defaults[effort])
|
||||
tier_matches = [model for model in source_models if tier in model.lower()]
|
||||
tiers[tier] = (
|
||||
max(tier_matches, key=lambda model: (model_version(model), model))
|
||||
if tier_matches
|
||||
else (resolved[effort] if discovered.live else current)
|
||||
for capability in CAPABILITY_TIERS:
|
||||
effort = next(
|
||||
value for value, target in EFFORT_TIERS.items() if target == capability
|
||||
)
|
||||
current = str(old_tiers.get(capability) or "")
|
||||
tiers[capability], observed = _select_tier_model(
|
||||
name, source_models, source_metadata, capability, effort, current,
|
||||
allow_current_fallback=not discovered.live,
|
||||
)
|
||||
observations.update(observed)
|
||||
for selector, capability in legacy_selectors.items():
|
||||
# Explicit historical family picks are not generic capability
|
||||
# requests. They must remain exact or become unavailable, never
|
||||
# silently change to a newer family such as Astra.
|
||||
exact = [model for model in source_models if selector in model.lower()]
|
||||
tiers[selector] = (
|
||||
exact[0]
|
||||
if exact
|
||||
else ("" if discovered.live else str(old_tiers.get(selector) or ""))
|
||||
)
|
||||
providers[name] = {
|
||||
"provenance": (
|
||||
"live-account" if discovered.live
|
||||
else ("last-known-good" if old_models else "bootstrap-fallback")
|
||||
),
|
||||
"checked_at": checked_at,
|
||||
"last_success_at": checked_at if discovered.live else previous_success,
|
||||
"state": discovered.state,
|
||||
"connected": discovered.connected,
|
||||
"live": discovered.live,
|
||||
@ -327,12 +254,14 @@ def build_routing_catalog(
|
||||
if discovered.live
|
||||
else (old_models or discovered.models)
|
||||
),
|
||||
"model_metadata": source_metadata,
|
||||
"candidates": observations,
|
||||
"resolved": resolved,
|
||||
"tiers": tiers,
|
||||
}
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"updated_at": int(time.time()),
|
||||
"schema_version": 2,
|
||||
"updated_at": checked_at,
|
||||
"providers": providers,
|
||||
}
|
||||
|
||||
@ -385,19 +314,6 @@ def _update_profile_env(path: Path, source: dict[str, str]) -> None:
|
||||
_atomic_write(path, "\n".join(kept).rstrip() + "\n", 0o600)
|
||||
|
||||
|
||||
def _existing_model(config: dict[str, Any], provider: str, default: str) -> str:
|
||||
"""Return the last configured model for a provider or a safe baseline."""
|
||||
model_cfg = config.get("model", {})
|
||||
if isinstance(model_cfg, dict) and model_cfg.get("provider") == provider:
|
||||
return str(model_cfg.get("model") or model_cfg.get("default") or default)
|
||||
fallbacks = config.get("fallback_providers", [])
|
||||
if isinstance(fallbacks, list):
|
||||
for fallback in fallbacks:
|
||||
if isinstance(fallback, dict) and fallback.get("provider") == provider:
|
||||
return str(fallback.get("model") or default)
|
||||
return default
|
||||
|
||||
|
||||
def codex_cli_authenticated() -> bool:
|
||||
"""Return whether the installed Codex CLI has a usable local login."""
|
||||
codex = shutil.which("codex")
|
||||
@ -420,127 +336,6 @@ def codex_cli_authenticated() -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def discover_codex_models() -> Catalog:
|
||||
"""Use the authenticated Codex endpoint; catalogs are status-only fallback."""
|
||||
token = ""
|
||||
try:
|
||||
from hermes_cli.auth import resolve_codex_runtime_credentials
|
||||
|
||||
credentials = resolve_codex_runtime_credentials(refresh_if_expiring=True) or {}
|
||||
token = str(credentials.get("api_key") or "").strip()
|
||||
except Exception:
|
||||
token = ""
|
||||
live: list[str] = []
|
||||
if token:
|
||||
try:
|
||||
from hermes_cli.codex_models import _fetch_models_from_api
|
||||
|
||||
live = _unique_models(_fetch_models_from_api(token))
|
||||
except Exception:
|
||||
live = []
|
||||
if live:
|
||||
return Catalog("openai-codex", live, True, True, "connected")
|
||||
try:
|
||||
from hermes_cli.models import provider_model_ids
|
||||
|
||||
known = _unique_models(provider_model_ids("openai-codex", force_refresh=True))
|
||||
except Exception:
|
||||
known = []
|
||||
# The owner agent deliberately uses the local Codex app-server and the
|
||||
# Codex CLI's ChatGPT login. That credential is not exported into Hermes'
|
||||
# bearer-token store, so API discovery above may be empty even though the
|
||||
# runtime is fully authenticated. Treat a successful CLI status check as a
|
||||
# connected, degraded catalog: routing may use Codex while retaining the
|
||||
# last-known working model names until app-server discovery is available.
|
||||
if codex_cli_authenticated():
|
||||
return Catalog(
|
||||
"openai-codex",
|
||||
known,
|
||||
False,
|
||||
True,
|
||||
"connected-cli",
|
||||
)
|
||||
state = "degraded" if token else "not-configured"
|
||||
return Catalog("openai-codex", known, False, bool(token), state)
|
||||
|
||||
|
||||
def discover_claude_models() -> Catalog:
|
||||
"""Use the native subscription login before any API-key catalog fallback."""
|
||||
claude = shutil.which("claude") or os.environ.get("HERMES_CLAUDE_BIN", "")
|
||||
if claude:
|
||||
try:
|
||||
environment = os.environ.copy()
|
||||
environment.pop("ANTHROPIC_API_KEY", None)
|
||||
environment.pop("CLAUDE_API_KEY", None)
|
||||
status = subprocess.run(
|
||||
[claude, "auth", "status"],
|
||||
capture_output=True,
|
||||
check=False,
|
||||
text=True,
|
||||
timeout=15,
|
||||
env=environment,
|
||||
)
|
||||
detail = json.loads(status.stdout) if status.stdout.strip() else {}
|
||||
if (
|
||||
status.returncode == 0
|
||||
and isinstance(detail, dict)
|
||||
and detail.get("loggedIn") is True
|
||||
and detail.get("apiProvider") == "firstParty"
|
||||
):
|
||||
try:
|
||||
from hermes_cli.models import provider_model_ids
|
||||
|
||||
known = _unique_models(
|
||||
provider_model_ids("anthropic", force_refresh=True)
|
||||
)
|
||||
except Exception:
|
||||
known = []
|
||||
return Catalog(
|
||||
"anthropic",
|
||||
_unique_models((*CLAUDE_SUBSCRIPTION_MODELS, *known)),
|
||||
True,
|
||||
True,
|
||||
"connected-subscription",
|
||||
)
|
||||
except (OSError, subprocess.SubprocessError, ValueError, json.JSONDecodeError):
|
||||
pass
|
||||
try:
|
||||
from hermes_cli.models import provider_model_ids
|
||||
|
||||
known = _unique_models(provider_model_ids("anthropic", force_refresh=True))
|
||||
except Exception:
|
||||
known = []
|
||||
return Catalog("anthropic", known, False, False, "not-configured")
|
||||
|
||||
|
||||
def _profile_config(
|
||||
base: dict[str, Any],
|
||||
primary_provider: str,
|
||||
primary_model: str,
|
||||
fallback: dict[str, str],
|
||||
effort: str,
|
||||
) -> dict[str, Any]:
|
||||
"""Derive a worker configuration with an explicit cross-provider fallback."""
|
||||
config = copy.deepcopy(base)
|
||||
config["model"] = {
|
||||
"provider": primary_provider,
|
||||
"default": primary_model,
|
||||
"model": primary_model,
|
||||
"openai_runtime": "codex_app_server",
|
||||
}
|
||||
config["fallback_providers"] = [fallback]
|
||||
# Local OpenAI-compatible runtimes top out at the equivalent of high.
|
||||
# Never silently downgrade an explicitly xhigh task after both hosted
|
||||
# provider lanes are exhausted.
|
||||
if effort != "xhigh":
|
||||
config["fallback_providers"].append(copy.deepcopy(ATLAS_FALLBACK))
|
||||
agent = config.setdefault("agent", {})
|
||||
if isinstance(agent, dict):
|
||||
agent["reasoning_effort"] = effort
|
||||
config["toolsets"] = []
|
||||
return config
|
||||
|
||||
|
||||
def _switchyard_profile_config(
|
||||
base: dict[str, Any], route: str, effort: str
|
||||
) -> dict[str, Any]:
|
||||
@ -626,18 +421,8 @@ def configure_routes(
|
||||
for effort in EFFORTS:
|
||||
codex_name = f"codex-{effort}"
|
||||
claude_name = f"claude-{effort}"
|
||||
codex_route = {
|
||||
"low": "atlas/manual/codex/luna",
|
||||
"medium": "atlas/manual/codex/terra",
|
||||
"high": "atlas/manual/codex/sol",
|
||||
"xhigh": "atlas/manual/codex/sol",
|
||||
}[effort]
|
||||
claude_route = {
|
||||
"low": "atlas/manual/claude/haiku",
|
||||
"medium": "atlas/manual/claude/sonnet",
|
||||
"high": "atlas/manual/claude/sonnet",
|
||||
"xhigh": "atlas/manual/claude/opus",
|
||||
}[effort]
|
||||
codex_route = f"atlas/manual/codex/auto/{effort}"
|
||||
claude_route = f"atlas/manual/claude/auto/{effort}"
|
||||
_write_profile(
|
||||
root,
|
||||
codex_name,
|
||||
|
||||
302
services/hermes/scripts/provider_model_catalog.py
Normal file
302
services/hermes/scripts/provider_model_catalog.py
Normal file
@ -0,0 +1,302 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Normalize provider model metadata into safe Hermes routing candidates."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Iterable
|
||||
|
||||
|
||||
EFFORTS = ("low", "medium", "high", "xhigh")
|
||||
CAPABILITY_TIERS = ("economy", "balanced", "advanced")
|
||||
EFFORT_TIERS = {
|
||||
"low": "economy",
|
||||
"medium": "balanced",
|
||||
"high": "advanced",
|
||||
"xhigh": "advanced",
|
||||
}
|
||||
SPECIALIST_MODEL_WORDS = frozenset(
|
||||
{"audio", "embedding", "image", "moderation", "realtime", "speech", "transcrib", "tts"}
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Catalog:
|
||||
"""Non-secret provider discovery result."""
|
||||
|
||||
provider: str
|
||||
models: list[str]
|
||||
live: bool
|
||||
connected: bool
|
||||
state: str
|
||||
metadata: dict[str, dict[str, Any]] = field(default_factory=dict)
|
||||
|
||||
|
||||
def unique_models(models: Iterable[str]) -> list[str]:
|
||||
"""Return normalized model IDs without changing provider order."""
|
||||
seen: set[str] = set()
|
||||
result: list[str] = []
|
||||
for model in models:
|
||||
value = str(model or "").strip()
|
||||
if value and value.lower() not in seen:
|
||||
result.append(value)
|
||||
seen.add(value.lower())
|
||||
return result
|
||||
|
||||
|
||||
def model_id(value: Any) -> str:
|
||||
"""Extract a provider model ID from either a string or model-list record."""
|
||||
if isinstance(value, str):
|
||||
return value.strip()
|
||||
if not isinstance(value, dict):
|
||||
return ""
|
||||
for key in ("id", "model", "model_id", "name", "value"):
|
||||
candidate = value.get(key)
|
||||
if isinstance(candidate, str) and candidate.strip():
|
||||
return candidate.strip()
|
||||
return ""
|
||||
|
||||
|
||||
def public_metadata(value: Any) -> dict[str, Any]:
|
||||
"""Keep provider routing fields while excluding unrelated response data."""
|
||||
if not isinstance(value, dict):
|
||||
return {}
|
||||
allowed = {
|
||||
"capabilities", "capability", "capability_tier", "cost", "cost_rank",
|
||||
"created_at", "default_reasoning_effort", "description", "display_name", "displayName",
|
||||
"disabled", "enabled", "hidden", "input_modalities", "inputModalities", "isDefault",
|
||||
"is_default", "is_general_purpose", "isGeneralPurpose", "max_input_tokens", "max_tokens",
|
||||
"output_modalities", "outputModalities", "pricing", "status",
|
||||
"modelSpecialty", "resolvedModel", "supported_reasoning_efforts", "supportedReasoningEfforts", "supportedEffortLevels",
|
||||
"supportsAdaptiveThinking", "supportsEffort", "tier", "upgrade",
|
||||
"verified", "visibility",
|
||||
}
|
||||
return {key: value[key] for key in allowed if key in value}
|
||||
|
||||
|
||||
def model_records(values: Iterable[Any]) -> tuple[list[str], dict[str, dict[str, Any]]]:
|
||||
"""Normalize provider records into ordered IDs and safe per-model metadata."""
|
||||
models: list[str] = []
|
||||
metadata: dict[str, dict[str, Any]] = {}
|
||||
for value in values:
|
||||
model = model_id(value)
|
||||
if not model:
|
||||
continue
|
||||
models.append(model)
|
||||
if isinstance(value, dict):
|
||||
metadata[model] = public_metadata(value)
|
||||
unique = unique_models(models)
|
||||
return unique, {model: metadata[model] for model in unique if model in metadata}
|
||||
|
||||
|
||||
def merge_records(
|
||||
*records: tuple[list[str], dict[str, dict[str, Any]]]
|
||||
) -> tuple[list[str], dict[str, dict[str, Any]]]:
|
||||
"""Merge ordered discovery sources, preferring metadata from later sources."""
|
||||
models: list[str] = []
|
||||
metadata: dict[str, dict[str, Any]] = {}
|
||||
for source_models, source_metadata in records:
|
||||
for model in source_models:
|
||||
if model not in models:
|
||||
models.append(model)
|
||||
details = source_metadata.get(model)
|
||||
if details:
|
||||
metadata[model] = {**metadata.get(model, {}), **details}
|
||||
return unique_models(models), metadata
|
||||
|
||||
|
||||
def model_version(model: str) -> tuple[int, ...]:
|
||||
"""Extract a sortable model version while ignoring dated aliases."""
|
||||
value = re.sub(r"-\d{8}$", "", model.lower())
|
||||
groups = re.findall(r"\d+(?:\.\d+)*", value)
|
||||
parts: list[int] = []
|
||||
for group in groups:
|
||||
parts.extend(int(piece) for piece in group.split("."))
|
||||
return tuple(parts) or (0,)
|
||||
|
||||
|
||||
def _text_metadata(metadata: dict[str, Any]) -> str:
|
||||
values: list[str] = []
|
||||
for key in ("description", "display_name", "capability", "tier"):
|
||||
value = metadata.get(key)
|
||||
if isinstance(value, str):
|
||||
values.append(value.lower())
|
||||
capabilities = metadata.get("capabilities")
|
||||
if isinstance(capabilities, list):
|
||||
values.extend(str(value).lower() for value in capabilities)
|
||||
return " ".join(values)
|
||||
|
||||
|
||||
def metadata_tier(metadata: dict[str, Any]) -> tuple[str | None, str]:
|
||||
"""Classify a model only from provider-declared capability information."""
|
||||
for key in ("capability_tier", "tier", "capability"):
|
||||
value = metadata.get(key)
|
||||
if isinstance(value, str):
|
||||
normalized = value.strip().lower()
|
||||
if normalized in {"economy", "cheap", "fast", "low_cost"}:
|
||||
return "economy", f"provider {key}={value!r}"
|
||||
if normalized in {"balanced", "standard", "mid", "medium"}:
|
||||
return "balanced", f"provider {key}={value!r}"
|
||||
if normalized in {"advanced", "strongest", "premium", "high"}:
|
||||
return "advanced", f"provider {key}={value!r}"
|
||||
text = _text_metadata(metadata)
|
||||
if any(term in text for term in ("strongest", "most capable", "most intelligent", "complex tasks")):
|
||||
return "advanced", "provider description marks it strongest/complex"
|
||||
if any(term in text for term in ("balanced", "general purpose", "everyday tasks", "routine tasks")):
|
||||
return "balanced", "provider description marks it balanced/general-purpose"
|
||||
if any(term in text for term in ("lowest cost", "low cost", "cheapest", "fastest", "fast and affordable")):
|
||||
return "economy", "provider description marks it low-cost/fast"
|
||||
return None, "provider metadata does not declare a capability tier"
|
||||
|
||||
|
||||
def legacy_tier(provider: str, model: str) -> tuple[str | None, str]:
|
||||
"""Migrate deployed pre-metadata families without predicting new IDs."""
|
||||
known = {
|
||||
"codex": {"luna": "economy", "terra": "balanced", "sol": "advanced"},
|
||||
"claude": {
|
||||
"haiku": "economy", "sonnet": "balanced", "opus": "advanced",
|
||||
"fable": "advanced",
|
||||
},
|
||||
}
|
||||
for marker, tier in known[provider].items():
|
||||
if marker in model.lower():
|
||||
return tier, "legacy deployed-family compatibility"
|
||||
return None, "no provider capability metadata"
|
||||
|
||||
|
||||
def _model_cost(metadata: dict[str, Any]) -> tuple[str, float] | None:
|
||||
"""Return a cost only with its provider-declared comparable unit."""
|
||||
for key in ("cost_rank", "cost"):
|
||||
rank = metadata.get(key)
|
||||
if not isinstance(rank, bool) and isinstance(rank, (int, float)) and rank >= 0:
|
||||
return "ordinal", float(rank)
|
||||
pricing = metadata.get("pricing")
|
||||
sources = [pricing, metadata] if isinstance(pricing, dict) else [metadata]
|
||||
for source in sources:
|
||||
input_cost = source.get("input_cost_per_million")
|
||||
output_cost = source.get("output_cost_per_million")
|
||||
if (
|
||||
not isinstance(input_cost, bool)
|
||||
and not isinstance(output_cost, bool)
|
||||
and isinstance(input_cost, (int, float))
|
||||
and isinstance(output_cost, (int, float))
|
||||
and input_cost >= 0
|
||||
and output_cost >= 0
|
||||
):
|
||||
return "per-million", float(input_cost) + float(output_cost)
|
||||
return None
|
||||
|
||||
|
||||
def _comparable_costs(
|
||||
pool: Iterable[tuple[str, dict[str, Any], str]]
|
||||
) -> dict[str, float]:
|
||||
"""Use cost only when every adequate candidate advertises one shared unit."""
|
||||
records = [(model, _model_cost(details)) for model, details, _ in pool]
|
||||
units = {cost[0] for _, cost in records if cost is not None}
|
||||
if len(units) != 1 or any(cost is None for _, cost in records):
|
||||
return {}
|
||||
return {model: cost[1] for model, cost in records if cost is not None}
|
||||
|
||||
def _candidate_status(
|
||||
provider: str, model: str, metadata: dict[str, Any], effort: str
|
||||
) -> tuple[bool, str]:
|
||||
if metadata.get("hidden") is True:
|
||||
return False, "provider marks model hidden"
|
||||
if str(metadata.get("visibility") or "").lower() in {"hidden", "internal", "private"}:
|
||||
return False, "provider marks model non-public"
|
||||
if metadata.get("enabled") is False or metadata.get("disabled") is True:
|
||||
return False, "provider marks model disabled"
|
||||
if str(metadata.get("status") or "").lower() in {"deprecated", "disabled", "retired"}:
|
||||
return False, "provider marks model unavailable"
|
||||
if metadata.get("is_general_purpose") is False or metadata.get("isGeneralPurpose") is False:
|
||||
return False, "provider marks model specialist"
|
||||
if metadata.get("modelSpecialty") not in (None, "", False):
|
||||
return False, "provider marks model specialist"
|
||||
modalities = metadata.get("input_modalities", metadata.get("inputModalities"))
|
||||
if isinstance(modalities, list) and modalities and "text" not in {str(item).lower() for item in modalities}:
|
||||
return False, "provider record is not text-capable"
|
||||
if "specialist" in _text_metadata(metadata):
|
||||
return False, "provider description identifies a specialist model"
|
||||
if any(word in model.lower() for word in SPECIALIST_MODEL_WORDS):
|
||||
return False, "model ID identifies a specialist model"
|
||||
if metadata.get("supportsEffort") is False:
|
||||
return False, "provider does not advertise effort support"
|
||||
supported = metadata.get(
|
||||
"supported_reasoning_efforts",
|
||||
metadata.get("supportedReasoningEfforts", metadata.get("supportedEffortLevels")),
|
||||
)
|
||||
supported_efforts = {
|
||||
str(item.get("reasoningEffort") or item.get("effort") or "").lower()
|
||||
if isinstance(item, dict)
|
||||
else str(item).lower()
|
||||
for item in supported
|
||||
} if isinstance(supported, list) else set()
|
||||
if supported_efforts and effort not in supported_efforts:
|
||||
return False, f"provider does not support {effort} effort"
|
||||
if provider == "codex" and not model.lower().startswith("gpt-"):
|
||||
return False, "model ID is outside the Codex general route namespace"
|
||||
return True, "eligible"
|
||||
|
||||
|
||||
def select_tier_model(
|
||||
provider: str, models: Iterable[str], metadata: dict[str, dict[str, Any]],
|
||||
desired: str, effort: str, current: str, *, allow_current_fallback: bool = True,
|
||||
) -> tuple[str, dict[str, Any]]:
|
||||
"""Select by declared tier and cost, retaining last-known-good only on outage."""
|
||||
candidates: list[tuple[str, dict[str, Any], str]] = []
|
||||
observations: dict[str, Any] = {}
|
||||
for model in unique_models(models):
|
||||
details = metadata.get(model, {})
|
||||
eligible, reason = _candidate_status(provider, model, details, effort)
|
||||
tier, tier_reason = metadata_tier(details)
|
||||
if tier is None:
|
||||
tier, legacy_reason = legacy_tier(provider, model)
|
||||
tier_reason = legacy_reason if tier else tier_reason
|
||||
cost = _model_cost(details)
|
||||
observations[model] = {
|
||||
"eligible": eligible,
|
||||
"reason": reason if not eligible else tier_reason,
|
||||
"tier": tier,
|
||||
"cost": {"unit": cost[0], "value": cost[1]} if cost else None,
|
||||
}
|
||||
if eligible and tier:
|
||||
candidates.append((model, details, tier))
|
||||
exact = [item for item in candidates if item[2] == desired]
|
||||
tier_order = {"economy": 0, "balanced": 1, "advanced": 2}
|
||||
# When an economy model cannot honor the requested effort, use the nearest
|
||||
# higher declared tier. A difficult request never falls to a lower tier.
|
||||
higher = [
|
||||
item for item in candidates
|
||||
if tier_order[item[2]] > tier_order[desired]
|
||||
]
|
||||
nearest_higher = min(
|
||||
(tier_order[item[2]] for item in higher), default=None
|
||||
)
|
||||
pool = exact or (
|
||||
[item for item in higher if tier_order[item[2]] == nearest_higher]
|
||||
if nearest_higher is not None else []
|
||||
)
|
||||
if not pool:
|
||||
return (current if allow_current_fallback else ""), observations
|
||||
comparable_costs = _comparable_costs(pool)
|
||||
|
||||
def key(item: tuple[str, dict[str, Any], str]) -> tuple[int, int, float, int, int, tuple[int, ...], str]:
|
||||
model, details, tier = item
|
||||
text = _text_metadata(details)
|
||||
strongest = any(term in text for term in ("strongest", "most capable", "most intelligent"))
|
||||
return (
|
||||
0 if tier == desired else 1,
|
||||
0 if strongest else 1,
|
||||
comparable_costs.get(model, float("inf")),
|
||||
# A provider moving its default is a current release signal. It
|
||||
# supersedes an old advanced route, while current protects equal
|
||||
# economy/balanced candidates where no comparable cost is known.
|
||||
0 if desired == "advanced" and (
|
||||
details.get("isDefault") is True or details.get("is_default") is True
|
||||
) else 1,
|
||||
0 if model == current else 1,
|
||||
tuple(-part for part in model_version(model)),
|
||||
model,
|
||||
)
|
||||
return min(pool, key=key)[0], observations
|
||||
219
services/hermes/scripts/provider_model_discovery.py
Normal file
219
services/hermes/scripts/provider_model_discovery.py
Normal file
@ -0,0 +1,219 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Discover account-visible provider models without leaking credentials."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import select
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
from typing import Any
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.parse import urlencode
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
from provider_model_catalog import Catalog, model_records, unique_models
|
||||
|
||||
|
||||
def codex_cli_authenticated() -> bool:
|
||||
"""Return whether the installed Codex CLI has a usable local login."""
|
||||
codex = shutil.which("codex")
|
||||
if not codex:
|
||||
return False
|
||||
try:
|
||||
status = subprocess.run(
|
||||
[codex, "login", "status"], capture_output=True, check=False,
|
||||
text=True, timeout=10,
|
||||
)
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
return False
|
||||
detail = f"{status.stdout}\n{status.stderr}".lower()
|
||||
return status.returncode == 0 and (
|
||||
"logged in" in detail or "authenticated" in detail
|
||||
)
|
||||
|
||||
def _codex_app_server_records(codex: str) -> tuple[list[str], dict[str, dict[str, Any]]] | None:
|
||||
"""Read the authenticated app-server ``model/list`` response when available."""
|
||||
try:
|
||||
process = subprocess.Popen(
|
||||
[codex, "app-server"],
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
text=False,
|
||||
)
|
||||
except OSError:
|
||||
return None
|
||||
if process.stdin is None or process.stdout is None:
|
||||
process.terminate()
|
||||
return None
|
||||
buffer = b""
|
||||
|
||||
def send(message: dict[str, Any]) -> None:
|
||||
assert process.stdin is not None
|
||||
process.stdin.write(json.dumps(message, separators=(",", ":")).encode() + b"\n")
|
||||
process.stdin.flush()
|
||||
|
||||
def response(request_id: int, deadline: float) -> dict[str, Any] | None:
|
||||
nonlocal buffer
|
||||
assert process.stdout is not None
|
||||
while time.monotonic() < deadline:
|
||||
readable, _, _ = select.select([process.stdout.fileno()], [], [], 0.25)
|
||||
if not readable:
|
||||
continue
|
||||
chunk = os.read(process.stdout.fileno(), 65536)
|
||||
if not chunk:
|
||||
return None
|
||||
buffer += chunk
|
||||
while b"\n" in buffer:
|
||||
line, buffer = buffer.split(b"\n", 1)
|
||||
try:
|
||||
document = json.loads(line)
|
||||
except (TypeError, ValueError, json.JSONDecodeError):
|
||||
continue
|
||||
if document.get("id") == request_id and isinstance(document.get("result"), dict):
|
||||
return document["result"]
|
||||
return None
|
||||
|
||||
try:
|
||||
send({"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {"clientInfo": {"name": "hermes-catalog", "version": "1"}, "capabilities": {}}})
|
||||
if response(1, time.monotonic() + 12) is None:
|
||||
return None
|
||||
send({"jsonrpc": "2.0", "method": "initialized", "params": {}})
|
||||
values: list[Any] = []
|
||||
cursor: str | None = None
|
||||
request_id = 2
|
||||
while True:
|
||||
params: dict[str, Any] = {"includeHidden": False, "limit": 100}
|
||||
if cursor:
|
||||
params["cursor"] = cursor
|
||||
send({"jsonrpc": "2.0", "id": request_id, "method": "model/list", "params": params})
|
||||
result = response(request_id, time.monotonic() + 12)
|
||||
if result is None:
|
||||
return None
|
||||
page = result.get("models", result.get("data", []))
|
||||
if not isinstance(page, list):
|
||||
return None
|
||||
values.extend(page)
|
||||
cursor = result.get("nextCursor") or result.get("next_cursor")
|
||||
if not isinstance(cursor, str) or not cursor:
|
||||
return model_records(values)
|
||||
request_id += 1
|
||||
except (OSError, ValueError, json.JSONDecodeError):
|
||||
pass
|
||||
finally:
|
||||
process.terminate()
|
||||
try:
|
||||
process.wait(timeout=2)
|
||||
except subprocess.TimeoutExpired:
|
||||
process.kill()
|
||||
return None
|
||||
|
||||
|
||||
def _anthropic_api_records() -> tuple[list[str], dict[str, dict[str, Any]]] | None:
|
||||
"""List current Anthropic account models, including documented capabilities."""
|
||||
token = os.environ.get("ANTHROPIC_API_KEY", "").strip()
|
||||
if not token:
|
||||
return None
|
||||
request = Request(
|
||||
"https://api.anthropic.com/v1/models?" + urlencode({"limit": 100}),
|
||||
headers={"x-api-key": token, "anthropic-version": "2023-06-01"},
|
||||
)
|
||||
try:
|
||||
with urlopen(request, timeout=15) as response:
|
||||
document = json.load(response)
|
||||
except (HTTPError, URLError, OSError, TimeoutError, ValueError):
|
||||
return None
|
||||
values = document.get("data", []) if isinstance(document, dict) else []
|
||||
return model_records(values if isinstance(values, list) else [])
|
||||
|
||||
|
||||
def discover_codex_models() -> Catalog:
|
||||
"""Use the authenticated Codex endpoint; catalogs are status-only fallback."""
|
||||
token = ""
|
||||
try:
|
||||
from hermes_cli.auth import resolve_codex_runtime_credentials
|
||||
|
||||
credentials = resolve_codex_runtime_credentials(refresh_if_expiring=True) or {}
|
||||
token = str(credentials.get("api_key") or "").strip()
|
||||
except Exception:
|
||||
token = ""
|
||||
api_records: tuple[list[str], dict[str, dict[str, Any]]] | None = None
|
||||
if token:
|
||||
try:
|
||||
from hermes_cli.codex_models import _fetch_models_from_api
|
||||
|
||||
api_records = model_records(_fetch_models_from_api(token))
|
||||
except Exception:
|
||||
api_records = None
|
||||
if api_records is not None:
|
||||
live, live_metadata = api_records
|
||||
return Catalog("openai-codex", live, True, True, "connected-api", live_metadata)
|
||||
codex = shutil.which("codex")
|
||||
if codex:
|
||||
app_records = _codex_app_server_records(codex)
|
||||
if app_records is not None:
|
||||
app_models, app_metadata = app_records
|
||||
return Catalog("openai-codex", app_models, True, True, "connected-app-server", app_metadata)
|
||||
try:
|
||||
from hermes_cli.models import provider_model_ids
|
||||
|
||||
known, known_metadata = model_records(
|
||||
provider_model_ids("openai-codex", force_refresh=True)
|
||||
)
|
||||
except Exception:
|
||||
known, known_metadata = [], {}
|
||||
# The owner agent deliberately uses the local Codex app-server and the
|
||||
# Codex CLI's ChatGPT login. That credential is not exported into Hermes'
|
||||
# bearer-token store, so API discovery above may be empty even though the
|
||||
# runtime is fully authenticated. Treat a successful CLI status check as a
|
||||
# connected, degraded catalog: routing may use Codex while retaining the
|
||||
# last-known working model names until app-server discovery is available.
|
||||
if codex_cli_authenticated():
|
||||
return Catalog(
|
||||
"openai-codex",
|
||||
known,
|
||||
False,
|
||||
True,
|
||||
"connected-cli",
|
||||
known_metadata,
|
||||
)
|
||||
state = "degraded" if token else "not-configured"
|
||||
return Catalog("openai-codex", known, False, bool(token), state, known_metadata)
|
||||
|
||||
|
||||
def discover_claude_models() -> Catalog:
|
||||
"""Discover subscription models first, retaining API discovery as fallback."""
|
||||
try:
|
||||
from claude_model_discovery import discover_claude_subscription_models
|
||||
|
||||
subscription = discover_claude_subscription_models()
|
||||
except Exception:
|
||||
subscription = None
|
||||
if subscription is not None and subscription.live:
|
||||
records = [model.as_dict() for model in subscription.models]
|
||||
models, metadata = model_records(records)
|
||||
return Catalog(
|
||||
"anthropic", models, True, subscription.connected,
|
||||
subscription.state, metadata,
|
||||
)
|
||||
api_records = _anthropic_api_records()
|
||||
if api_records is not None:
|
||||
live_models, live_metadata = api_records
|
||||
return Catalog("anthropic", live_models, True, True, "connected-api", live_metadata)
|
||||
try:
|
||||
from hermes_cli.models import provider_model_ids
|
||||
|
||||
known, known_metadata = model_records(
|
||||
provider_model_ids("anthropic", force_refresh=True)
|
||||
)
|
||||
except Exception:
|
||||
known, known_metadata = [], {}
|
||||
if subscription is not None:
|
||||
return Catalog(
|
||||
"anthropic", known, False, subscription.connected,
|
||||
subscription.state, known_metadata,
|
||||
)
|
||||
return Catalog("anthropic", known, False, False, "not-configured", known_metadata)
|
||||
@ -51,6 +51,15 @@ def load_catalog(path: Path = CATALOG_PATH) -> dict[str, Any]:
|
||||
return value if isinstance(value, dict) else {}
|
||||
|
||||
|
||||
def catalog_contains(provider: str, model: str, catalog: dict[str, Any] | None = None) -> bool:
|
||||
"""Return whether a provider's stewarded current catalog advertises a model."""
|
||||
document = catalog if catalog is not None else load_catalog()
|
||||
providers = document.get("providers", {}) if isinstance(document, dict) else {}
|
||||
record = providers.get(provider, {}) if isinstance(providers, dict) else {}
|
||||
models = record.get("models", []) if isinstance(record, dict) else []
|
||||
return isinstance(model, str) and model in models
|
||||
|
||||
|
||||
def resolve_model(
|
||||
provider: str,
|
||||
selector: str,
|
||||
@ -61,10 +70,16 @@ def resolve_model(
|
||||
if provider not in DEFAULTS or effort not in DEFAULTS[provider]:
|
||||
raise ValueError("unsupported provider or effort")
|
||||
prefix = PREFIXES[provider]
|
||||
document = catalog if catalog is not None else load_catalog()
|
||||
if selector.startswith(prefix):
|
||||
if catalog_contains(provider, selector, document):
|
||||
return selector
|
||||
providers = document.get("providers", {}) if isinstance(document, dict) else {}
|
||||
record = providers.get(provider, {}) if isinstance(providers, dict) else {}
|
||||
if isinstance(record, dict) and record.get("live") is True:
|
||||
raise ValueError(f"model is not in the current {provider} catalog")
|
||||
return selector
|
||||
|
||||
document = catalog if catalog is not None else load_catalog()
|
||||
providers = document.get("providers", {}) if isinstance(document, dict) else {}
|
||||
record = providers.get(provider, {}) if isinstance(providers, dict) else {}
|
||||
if not isinstance(record, dict):
|
||||
@ -72,7 +87,23 @@ def resolve_model(
|
||||
mapping_name = "resolved" if selector == "auto" else "tiers"
|
||||
mapping = record.get(mapping_name, {})
|
||||
candidate = mapping.get(effort if selector == "auto" else selector, "") if isinstance(mapping, dict) else ""
|
||||
if not isinstance(candidate, str) or not candidate.startswith(prefix):
|
||||
if (
|
||||
selector != "auto"
|
||||
and isinstance(candidate, str)
|
||||
and candidate
|
||||
and selector not in candidate.lower()
|
||||
):
|
||||
candidate = ""
|
||||
advertised = record.get("models", []) if isinstance(record, dict) else []
|
||||
candidate_allowed = isinstance(candidate, str) and (
|
||||
candidate.startswith(prefix)
|
||||
or (provider == "claude" and candidate in advertised)
|
||||
)
|
||||
if not candidate_allowed:
|
||||
# A fresh account list is authoritative. Do not turn a deliberately
|
||||
# unresolved/retired selector into a stale baseline model.
|
||||
if record.get("live") is True:
|
||||
raise ValueError(f"no current {provider} model for selector {selector!r}")
|
||||
candidate = (
|
||||
DEFAULTS[provider][effort]
|
||||
if selector == "auto"
|
||||
|
||||
@ -158,6 +158,26 @@ data:
|
||||
llm_client = "codex_xhigh"
|
||||
extra_body = { reasoning = { effort = "xhigh" } }
|
||||
|
||||
[targets.codex_auto_low]
|
||||
id = "route/codex/auto/low"
|
||||
llm_client = "codex_low"
|
||||
extra_body = { reasoning = { effort = "low" } }
|
||||
|
||||
[targets.codex_auto_medium]
|
||||
id = "route/codex/auto/medium"
|
||||
llm_client = "codex_medium"
|
||||
extra_body = { reasoning = { effort = "medium" } }
|
||||
|
||||
[targets.codex_auto_high]
|
||||
id = "route/codex/auto/high"
|
||||
llm_client = "codex_high"
|
||||
extra_body = { reasoning = { effort = "high" } }
|
||||
|
||||
[targets.codex_auto_xhigh]
|
||||
id = "route/codex/auto/xhigh"
|
||||
llm_client = "codex_xhigh"
|
||||
extra_body = { reasoning = { effort = "xhigh" } }
|
||||
|
||||
[targets.claude_haiku_low]
|
||||
id = "route/claude/haiku/low"
|
||||
llm_client = "claude_low"
|
||||
@ -238,6 +258,26 @@ data:
|
||||
llm_client = "claude_xhigh"
|
||||
extra_body = { thinking = { type = "adaptive" }, output_config = { effort = "xhigh" } }
|
||||
|
||||
[targets.claude_auto_low]
|
||||
id = "route/claude/auto/low"
|
||||
llm_client = "claude_low"
|
||||
extra_body = { thinking = { type = "adaptive" }, output_config = { effort = "low" } }
|
||||
|
||||
[targets.claude_auto_medium]
|
||||
id = "route/claude/auto/medium"
|
||||
llm_client = "claude_medium"
|
||||
extra_body = { thinking = { type = "adaptive" }, output_config = { effort = "medium" } }
|
||||
|
||||
[targets.claude_auto_high]
|
||||
id = "route/claude/auto/high"
|
||||
llm_client = "claude_high"
|
||||
extra_body = { thinking = { type = "adaptive" }, output_config = { effort = "high" } }
|
||||
|
||||
[targets.claude_auto_xhigh]
|
||||
id = "route/claude/auto/xhigh"
|
||||
llm_client = "claude_xhigh"
|
||||
extra_body = { thinking = { type = "adaptive" }, output_config = { effort = "xhigh" } }
|
||||
|
||||
[targets.worker_codex_luna_low]
|
||||
id = "worker/codex/luna/low"
|
||||
llm_client = "worker_decision"
|
||||
@ -350,6 +390,38 @@ data:
|
||||
id = "worker/claude/opus/xhigh"
|
||||
llm_client = "worker_decision"
|
||||
|
||||
[targets.worker_codex_auto_low]
|
||||
id = "worker/codex/auto/low"
|
||||
llm_client = "worker_decision"
|
||||
|
||||
[targets.worker_codex_auto_medium]
|
||||
id = "worker/codex/auto/medium"
|
||||
llm_client = "worker_decision"
|
||||
|
||||
[targets.worker_codex_auto_high]
|
||||
id = "worker/codex/auto/high"
|
||||
llm_client = "worker_decision"
|
||||
|
||||
[targets.worker_codex_auto_xhigh]
|
||||
id = "worker/codex/auto/xhigh"
|
||||
llm_client = "worker_decision"
|
||||
|
||||
[targets.worker_claude_auto_low]
|
||||
id = "worker/claude/auto/low"
|
||||
llm_client = "worker_decision"
|
||||
|
||||
[targets.worker_claude_auto_medium]
|
||||
id = "worker/claude/auto/medium"
|
||||
llm_client = "worker_decision"
|
||||
|
||||
[targets.worker_claude_auto_high]
|
||||
id = "worker/claude/auto/high"
|
||||
llm_client = "worker_decision"
|
||||
|
||||
[targets.worker_claude_auto_xhigh]
|
||||
id = "worker/claude/auto/xhigh"
|
||||
llm_client = "worker_decision"
|
||||
|
||||
# Switchyard 0.2.0 requires one default_target for every custom classifier.
|
||||
# These targets recurse once into unbiased random routes whose candidates are
|
||||
# split evenly across providers. A classifier outage therefore falls back to
|
||||
@ -377,7 +449,7 @@ data:
|
||||
[routes.fallback_fast]
|
||||
id = "atlas/fallback/fast"
|
||||
type = "random"
|
||||
targets = ["codex_terra_medium", "claude_fable_medium"]
|
||||
targets = ["codex_auto_medium", "claude_auto_medium"]
|
||||
weights = [1.0, 1.0]
|
||||
context_window = 272000
|
||||
tool_calling = true
|
||||
@ -386,7 +458,7 @@ data:
|
||||
[routes.fallback_balanced]
|
||||
id = "atlas/fallback/balanced"
|
||||
type = "random"
|
||||
targets = ["codex_terra_high", "claude_sonnet_high"]
|
||||
targets = ["codex_auto_high", "claude_auto_high"]
|
||||
weights = [1.0, 1.0]
|
||||
context_window = 272000
|
||||
tool_calling = true
|
||||
@ -395,7 +467,7 @@ data:
|
||||
[routes.fallback_deep]
|
||||
id = "atlas/fallback/deep"
|
||||
type = "random"
|
||||
targets = ["codex_sol_high", "claude_opus_high"]
|
||||
targets = ["codex_auto_high", "claude_auto_high"]
|
||||
weights = [1.0, 1.0]
|
||||
context_window = 272000
|
||||
tool_calling = true
|
||||
@ -404,7 +476,7 @@ data:
|
||||
[routes.fallback_maximum]
|
||||
id = "atlas/fallback/maximum"
|
||||
type = "random"
|
||||
targets = ["codex_sol_xhigh", "claude_opus_xhigh"]
|
||||
targets = ["codex_auto_xhigh", "claude_auto_xhigh"]
|
||||
weights = [1.0, 1.0]
|
||||
context_window = 272000
|
||||
tool_calling = true
|
||||
@ -413,7 +485,7 @@ data:
|
||||
[routes.fallback_worker_maximum]
|
||||
id = "atlas/worker/fallback/maximum"
|
||||
type = "random"
|
||||
targets = ["worker_codex_sol_xhigh", "worker_claude_opus_xhigh"]
|
||||
targets = ["worker_codex_auto_xhigh", "worker_claude_auto_xhigh"]
|
||||
weights = [1.0, 1.0]
|
||||
context_window = 272000
|
||||
tool_calling = false
|
||||
@ -426,7 +498,7 @@ data:
|
||||
classifier_target = "classifier"
|
||||
# Switchyard falls through this list after a request-local target failure.
|
||||
# Keep both xhigh providers first so recovery can escalate, never downgrade.
|
||||
targets = ["codex_sol_xhigh", "claude_opus_xhigh", "claude_fable_xhigh", "codex_sol_high", "claude_sonnet_high", "claude_opus_high", "claude_fable_high", "codex_terra_high", "codex_sol_medium", "claude_sonnet_medium", "claude_fable_medium", "codex_terra_medium", "codex_terra_low", "codex_luna_low", "claude_haiku_low", "claude_fable_low", "neutral_fast_pool"]
|
||||
targets = ["codex_auto_xhigh", "claude_auto_xhigh", "codex_auto_high", "claude_auto_high", "codex_auto_medium", "claude_auto_medium", "codex_auto_low", "claude_auto_low", "neutral_fast_pool"]
|
||||
default_target = "neutral_fast_pool"
|
||||
session_affinity = false
|
||||
recent_turn_window = 4
|
||||
@ -449,10 +521,10 @@ data:
|
||||
If tools are present or the boundary may emit a tool call, the floor is
|
||||
medium even when the user's wording is short. Filesystem, shell,
|
||||
repository, cluster, browser, and image-generation/edit operations are
|
||||
tool work; never route those boundaries to Luna, Haiku, or local-low.
|
||||
tool work; never route those boundaries to economy or local-low targets.
|
||||
A failing test, failed tool plan, contradicted result, rejected review, or
|
||||
incomplete evidence means the previous quality mark was missed. Raise the
|
||||
next boundary by at least one effort tier and prefer a stronger family or
|
||||
next boundary by at least one effort tier and prefer an available advanced target or
|
||||
the other hosted provider; repeated misses require xhigh. Never repeat a
|
||||
failed lower-capability plan unchanged.
|
||||
|
||||
@ -465,7 +537,7 @@ data:
|
||||
3. Route image creation and editing before general provider preference.
|
||||
When image-generation/edit tools are available and the user asks to create,
|
||||
transform, restore, colorize, or continue editing an image: select
|
||||
codex_terra_medium. The image tool—not the conversational model—honors the
|
||||
codex_auto_medium. The image tool—not the conversational model—honors the
|
||||
user's local, OpenAI/hosted, or AUTO image-backend choice. Do not select a
|
||||
local Qwen or Claude target for an image-tool boundary: local Qwen cannot
|
||||
reliably carry the full Hermes tool context, and Anthropic supplies the
|
||||
@ -480,17 +552,15 @@ data:
|
||||
to be unavailable, failed, exhausted, rate-limited, or out of capacity; use
|
||||
the other hosted provider at the same floor.
|
||||
|
||||
5. Choose across the complete family catalog. Codex options are Luna,
|
||||
Terra, and SOL at low through xhigh. Claude options are Haiku, Fable,
|
||||
Sonnet, and Opus at low through xhigh. Prefer Fable for concise writing,
|
||||
synthesis, and instruction-following where its capability fits the effort
|
||||
floor; use Sonnet or Opus for deeper analysis and review. Re-evaluate every
|
||||
boundary and resolve "continue"
|
||||
or "do it" from recent context. The manual local route remains available
|
||||
only when the user explicitly selects it.
|
||||
5. Choose the provider AUTO target at the exact effort floor. AUTO resolves
|
||||
an account-visible economy, balanced, or advanced general-purpose model
|
||||
from provider metadata. It uses comparable provider cost data only to break
|
||||
ties between adequate models. Re-evaluate every boundary and resolve
|
||||
"continue" or "do it" from recent context. The manual local route remains
|
||||
available only when the user explicitly selects it.
|
||||
"""
|
||||
response_schema = '''
|
||||
{"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["codex_terra_medium","claude_sonnet_medium","claude_fable_medium","codex_luna_low","claude_haiku_low","claude_fable_low","codex_terra_low","codex_terra_high","claude_fable_high","codex_sol_medium","codex_sol_high","codex_sol_xhigh","claude_sonnet_high","claude_opus_high","claude_fable_xhigh","claude_opus_xhigh"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false}
|
||||
{"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["codex_auto_low","codex_auto_medium","codex_auto_high","codex_auto_xhigh","claude_auto_low","claude_auto_medium","claude_auto_high","claude_auto_xhigh"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false}
|
||||
'''
|
||||
|
||||
[routes.auto_fast.policy]
|
||||
@ -504,7 +574,7 @@ data:
|
||||
classifier_target = "classifier"
|
||||
# Switchyard falls through this list after a request-local target failure.
|
||||
# Keep both xhigh providers first so recovery can escalate, never downgrade.
|
||||
targets = ["codex_sol_xhigh", "claude_opus_xhigh", "claude_fable_xhigh", "codex_sol_high", "claude_sonnet_high", "claude_opus_high", "claude_fable_high", "codex_terra_high", "codex_sol_medium", "claude_sonnet_medium", "claude_fable_medium", "codex_terra_medium", "codex_terra_low", "codex_luna_low", "claude_haiku_low", "claude_fable_low", "neutral_balanced_pool"]
|
||||
targets = ["codex_auto_xhigh", "claude_auto_xhigh", "codex_auto_high", "claude_auto_high", "codex_auto_medium", "claude_auto_medium", "codex_auto_low", "claude_auto_low", "neutral_balanced_pool"]
|
||||
default_target = "neutral_balanced_pool"
|
||||
session_affinity = false
|
||||
recent_turn_window = 4
|
||||
@ -528,10 +598,10 @@ data:
|
||||
If tools are present or the boundary may emit a tool call, the floor is
|
||||
medium even when the user's wording is short. Filesystem, shell,
|
||||
repository, cluster, browser, and image-generation/edit operations are
|
||||
tool work; never route those boundaries to Luna, Haiku, or local-low.
|
||||
tool work; never route those boundaries to economy or local-low targets.
|
||||
A failing test, failed tool plan, contradicted result, rejected review, or
|
||||
incomplete evidence means the previous quality mark was missed. Raise the
|
||||
next boundary by at least one effort tier and prefer a stronger family or
|
||||
next boundary by at least one effort tier and prefer an available advanced target or
|
||||
the other hosted provider; repeated misses require xhigh. Never repeat a
|
||||
failed lower-capability plan unchanged.
|
||||
|
||||
@ -544,7 +614,7 @@ data:
|
||||
3. Route image creation and editing before general provider preference.
|
||||
When image-generation/edit tools are available and the user asks to create,
|
||||
transform, restore, colorize, or continue editing an image: select
|
||||
codex_terra_medium. The image tool—not the conversational model—honors the
|
||||
codex_auto_medium. The image tool—not the conversational model—honors the
|
||||
user's local, OpenAI/hosted, or AUTO image-backend choice. Do not select a
|
||||
local Qwen or Claude target for an image-tool boundary: local Qwen cannot
|
||||
reliably carry the full Hermes tool context, and Anthropic supplies the
|
||||
@ -559,17 +629,15 @@ data:
|
||||
to be unavailable, failed, exhausted, rate-limited, or out of capacity; use
|
||||
the other hosted provider at the same floor.
|
||||
|
||||
5. Choose across the complete family catalog. Codex options are Luna,
|
||||
Terra, and SOL at low through xhigh. Claude options are Haiku, Fable,
|
||||
Sonnet, and Opus at low through xhigh. Prefer Fable for concise writing,
|
||||
synthesis, and instruction-following where its capability fits the effort
|
||||
floor; use Sonnet or Opus for deeper analysis and review. Re-evaluate every
|
||||
boundary and resolve "continue"
|
||||
or "do it" from recent context. The manual local route remains available
|
||||
only when the user explicitly selects it.
|
||||
5. Choose the provider AUTO target at the exact effort floor. AUTO resolves
|
||||
an account-visible economy, balanced, or advanced general-purpose model
|
||||
from provider metadata. It uses comparable provider cost data only to break
|
||||
ties between adequate models. Re-evaluate every boundary and resolve
|
||||
"continue" or "do it" from recent context. The manual local route remains
|
||||
available only when the user explicitly selects it.
|
||||
"""
|
||||
response_schema = '''
|
||||
{"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["codex_terra_medium","claude_sonnet_medium","claude_fable_medium","codex_luna_low","claude_haiku_low","claude_fable_low","codex_terra_low","codex_terra_high","claude_fable_high","codex_sol_medium","codex_sol_high","codex_sol_xhigh","claude_sonnet_high","claude_opus_high","claude_fable_xhigh","claude_opus_xhigh"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false}
|
||||
{"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["codex_auto_low","codex_auto_medium","codex_auto_high","codex_auto_xhigh","claude_auto_low","claude_auto_medium","claude_auto_high","claude_auto_xhigh"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false}
|
||||
'''
|
||||
|
||||
[routes.auto_balanced.policy]
|
||||
@ -583,7 +651,7 @@ data:
|
||||
classifier_target = "classifier"
|
||||
# Switchyard falls through this list after a request-local target failure.
|
||||
# Keep both xhigh providers first so recovery can escalate, never downgrade.
|
||||
targets = ["claude_opus_xhigh", "codex_sol_xhigh", "claude_fable_xhigh", "claude_sonnet_high", "codex_sol_high", "claude_opus_high", "claude_fable_high", "codex_terra_high", "claude_sonnet_medium", "claude_fable_medium", "codex_sol_medium", "codex_terra_medium", "neutral_deep_pool"]
|
||||
targets = ["claude_auto_xhigh", "codex_auto_xhigh", "claude_auto_high", "codex_auto_high", "claude_auto_medium", "codex_auto_medium", "neutral_deep_pool"]
|
||||
default_target = "neutral_deep_pool"
|
||||
session_affinity = false
|
||||
recent_turn_window = 6
|
||||
@ -607,10 +675,10 @@ data:
|
||||
If tools are present or the boundary may emit a tool call, the floor is
|
||||
medium even when the user's wording is short. Filesystem, shell,
|
||||
repository, cluster, browser, and image-generation/edit operations are
|
||||
tool work; never route those boundaries to Luna or Haiku.
|
||||
tool work; never route those boundaries to economy targets.
|
||||
A failing test, failed tool plan, contradicted result, rejected review, or
|
||||
incomplete evidence means the previous quality mark was missed. Raise the
|
||||
next boundary by at least one effort tier and prefer a stronger family or
|
||||
next boundary by at least one effort tier and prefer an available advanced target or
|
||||
the other hosted provider; repeated misses require xhigh. Never repeat a
|
||||
failed lower-capability plan unchanged.
|
||||
|
||||
@ -630,15 +698,14 @@ data:
|
||||
unavailable, failed, exhausted, rate-limited, or out of capacity; use the
|
||||
other provider at the same floor.
|
||||
|
||||
4. Choose across Codex Terra/SOL and Claude Fable/Sonnet/Opus at medium
|
||||
through xhigh. Prefer Fable for writing and compact synthesis where it
|
||||
clears the quality floor; use Sonnet or Opus for deeper diagnosis and
|
||||
review. Low-tier targets are intentionally unavailable on
|
||||
this route. Re-evaluate every boundary and resolve "continue" or "do it"
|
||||
from recent context.
|
||||
4. Choose a provider AUTO target at the exact effort floor. AUTO resolves
|
||||
an account-visible balanced or advanced general-purpose model from provider
|
||||
metadata; economy targets are intentionally unavailable on this route.
|
||||
Re-evaluate every boundary and resolve "continue" or "do it" from recent
|
||||
context.
|
||||
"""
|
||||
response_schema = '''
|
||||
{"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["claude_sonnet_high","claude_fable_high","codex_sol_high","codex_terra_medium","claude_sonnet_medium","claude_fable_medium","codex_terra_high","codex_sol_medium","claude_opus_high","codex_sol_xhigh","claude_fable_xhigh","claude_opus_xhigh"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false}
|
||||
{"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["claude_auto_medium","claude_auto_high","claude_auto_xhigh","codex_auto_medium","codex_auto_high","codex_auto_xhigh"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false}
|
||||
'''
|
||||
|
||||
[routes.auto_deep.policy]
|
||||
@ -652,7 +719,7 @@ data:
|
||||
classifier_target = "classifier"
|
||||
# Switchyard falls through this list after a request-local target failure.
|
||||
# Keep both xhigh providers first so recovery can escalate, never downgrade.
|
||||
targets = ["codex_sol_xhigh", "claude_opus_xhigh", "claude_fable_xhigh", "codex_sol_high", "claude_opus_high", "claude_sonnet_high", "claude_fable_high", "codex_terra_high", "neutral_maximum_pool"]
|
||||
targets = ["codex_auto_xhigh", "claude_auto_xhigh", "codex_auto_high", "claude_auto_high", "neutral_maximum_pool"]
|
||||
default_target = "neutral_maximum_pool"
|
||||
session_affinity = false
|
||||
recent_turn_window = 6
|
||||
@ -674,7 +741,7 @@ data:
|
||||
Never choose below the floor or above xhigh.
|
||||
A failing test, failed tool plan, contradicted result, rejected review, or
|
||||
incomplete evidence raises the next boundary to xhigh and should switch to
|
||||
the strongest suitable family or other hosted provider. Never repeat a
|
||||
the strongest suitable available target or other hosted provider. Never repeat a
|
||||
failed high-capability plan unchanged.
|
||||
|
||||
2. Treat "think hard", "deeply", "carefully", and equivalent intent as a
|
||||
@ -693,15 +760,14 @@ data:
|
||||
unavailable, failed, exhausted, rate-limited, or out of capacity; use the
|
||||
other provider at the same floor.
|
||||
|
||||
4. Choose across Codex Terra/SOL and Claude Fable/Sonnet/Opus at high or
|
||||
xhigh only. Prefer Fable for writing and compact synthesis where it clears
|
||||
the quality floor; use Sonnet or Opus for deeper analysis and independent
|
||||
review. Medium and low targets are intentionally unavailable on this
|
||||
route. Re-evaluate every boundary and resolve "continue" or "do it" from
|
||||
recent context.
|
||||
4. Choose a provider AUTO target at high or xhigh only. AUTO resolves an
|
||||
account-visible advanced general-purpose model from provider metadata.
|
||||
Medium and low targets are intentionally unavailable on this route.
|
||||
Re-evaluate every boundary and resolve "continue" or "do it" from recent
|
||||
context.
|
||||
"""
|
||||
response_schema = '''
|
||||
{"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["codex_sol_high","claude_opus_high","claude_sonnet_high","claude_fable_high","codex_terra_high","codex_sol_xhigh","claude_opus_xhigh","claude_fable_xhigh"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false}
|
||||
{"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["codex_auto_high","codex_auto_xhigh","claude_auto_high","claude_auto_xhigh"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false}
|
||||
'''
|
||||
|
||||
[routes.auto_maximum.policy]
|
||||
@ -713,7 +779,7 @@ data:
|
||||
type = "llm_classifier"
|
||||
mode = "custom"
|
||||
classifier_target = "classifier"
|
||||
targets = ["worker_codex_luna_low", "worker_codex_luna_medium", "worker_codex_luna_high", "worker_codex_luna_xhigh", "worker_codex_terra_low", "worker_codex_terra_medium", "worker_codex_terra_high", "worker_codex_terra_xhigh", "worker_codex_sol_low", "worker_codex_sol_medium", "worker_codex_sol_high", "worker_codex_sol_xhigh", "worker_claude_haiku_low", "worker_claude_haiku_medium", "worker_claude_haiku_high", "worker_claude_haiku_xhigh", "worker_claude_fable_low", "worker_claude_fable_medium", "worker_claude_fable_high", "worker_claude_fable_xhigh", "worker_claude_sonnet_low", "worker_claude_sonnet_medium", "worker_claude_sonnet_high", "worker_claude_sonnet_xhigh", "worker_claude_opus_low", "worker_claude_opus_medium", "worker_claude_opus_high", "worker_claude_opus_xhigh", "neutral_worker_maximum_pool"]
|
||||
targets = ["worker_codex_auto_low", "worker_codex_auto_medium", "worker_codex_auto_high", "worker_codex_auto_xhigh", "worker_claude_auto_low", "worker_claude_auto_medium", "worker_claude_auto_high", "worker_claude_auto_xhigh", "neutral_worker_maximum_pool"]
|
||||
default_target = "neutral_worker_maximum_pool"
|
||||
session_affinity = false
|
||||
recent_turn_window = 6
|
||||
@ -738,7 +804,7 @@ data:
|
||||
Never choose below the floor and never exceed xhigh.
|
||||
If the objective records a failing test, failed attempt, contradicted
|
||||
result, rejected review, or incomplete evidence from a previous worker,
|
||||
raise effort by at least one tier and prefer a stronger family or the other
|
||||
raise effort by at least one tier and prefer the available advanced target or the other
|
||||
hosted provider. Repeated quality misses require xhigh; never launch the
|
||||
same failed lower-capability plan unchanged.
|
||||
|
||||
@ -757,30 +823,28 @@ data:
|
||||
analysis, and independent review.
|
||||
A final independent review is Claude; implementing review findings is Codex.
|
||||
|
||||
4. Choose across every configured Codex and Claude family at the exact
|
||||
effort floor. Codex Luna is the economical tier, Terra is balanced, and
|
||||
SOL is the deepest implementation tier. Claude Haiku is the economical
|
||||
tier, Fable is preferred for concise writing and synthesis, Sonnet is
|
||||
balanced, and Opus is the deepest analysis and review tier. Every family
|
||||
supports low, medium, high, and xhigh; choose the cheapest family that
|
||||
clears the objective's quality floor without lowering its effort.
|
||||
4. Choose an available provider AUTO target at the exact effort floor.
|
||||
AUTO resolves the current account-visible economy, balanced, or advanced
|
||||
general-purpose model. It uses comparable provider cost data only to break
|
||||
ties among models that already satisfy the capability requirement. Do not
|
||||
infer a tier from an unfamiliar model name or unadvertised effort support.
|
||||
|
||||
Examples:
|
||||
Critical security migration final review -> worker_claude_opus_xhigh.
|
||||
Implement critical security review fixes -> worker_codex_sol_xhigh.
|
||||
Difficult intermittent production failure -> worker_codex_sol_high.
|
||||
Ordinary component design -> worker_claude_sonnet_medium.
|
||||
One spelling correction -> worker_codex_luna_low.
|
||||
Critical security migration final review -> worker_claude_auto_xhigh.
|
||||
Implement critical security review fixes -> worker_codex_auto_xhigh.
|
||||
Difficult intermittent production failure -> worker_codex_auto_high.
|
||||
Ordinary component design -> worker_claude_auto_medium.
|
||||
One spelling correction -> worker_codex_auto_low.
|
||||
Anthropic exhausted + critical independent final review ->
|
||||
worker_codex_sol_xhigh.
|
||||
worker_codex_auto_xhigh.
|
||||
OpenAI exhausted + difficult repository implementation ->
|
||||
worker_claude_sonnet_high.
|
||||
worker_claude_auto_high.
|
||||
|
||||
Before responding, verify the provider is available and effort is not below
|
||||
the floor. Return only the required decision object.
|
||||
"""
|
||||
response_schema = '''
|
||||
{"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["worker_codex_luna_low","worker_codex_luna_medium","worker_codex_luna_high","worker_codex_luna_xhigh","worker_codex_terra_low","worker_codex_terra_medium","worker_codex_terra_high","worker_codex_terra_xhigh","worker_codex_sol_low","worker_codex_sol_medium","worker_codex_sol_high","worker_codex_sol_xhigh","worker_claude_haiku_low","worker_claude_haiku_medium","worker_claude_haiku_high","worker_claude_haiku_xhigh","worker_claude_fable_low","worker_claude_fable_medium","worker_claude_fable_high","worker_claude_fable_xhigh","worker_claude_sonnet_low","worker_claude_sonnet_medium","worker_claude_sonnet_high","worker_claude_sonnet_xhigh","worker_claude_opus_low","worker_claude_opus_medium","worker_claude_opus_high","worker_claude_opus_xhigh"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false}
|
||||
{"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["worker_codex_auto_low","worker_codex_auto_medium","worker_codex_auto_high","worker_codex_auto_xhigh","worker_claude_auto_low","worker_claude_auto_medium","worker_claude_auto_high","worker_claude_auto_xhigh"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false}
|
||||
'''
|
||||
|
||||
[routes.worker_auto_maximum.policy]
|
||||
@ -790,42 +854,98 @@ data:
|
||||
[routes.worker_manual_codex_low]
|
||||
id = "atlas/worker/manual/codex/low"
|
||||
type = "random"
|
||||
targets = ["worker_codex_luna_low"]
|
||||
targets = ["worker_codex_auto_low"]
|
||||
|
||||
[routes.worker_manual_codex_medium]
|
||||
id = "atlas/worker/manual/codex/medium"
|
||||
type = "random"
|
||||
targets = ["worker_codex_terra_medium"]
|
||||
targets = ["worker_codex_auto_medium"]
|
||||
|
||||
[routes.worker_manual_codex_high]
|
||||
id = "atlas/worker/manual/codex/high"
|
||||
type = "random"
|
||||
targets = ["worker_codex_sol_high"]
|
||||
targets = ["worker_codex_auto_high"]
|
||||
|
||||
[routes.worker_manual_codex_xhigh]
|
||||
id = "atlas/worker/manual/codex/xhigh"
|
||||
type = "random"
|
||||
targets = ["worker_codex_sol_xhigh"]
|
||||
targets = ["worker_codex_auto_xhigh"]
|
||||
|
||||
[routes.worker_manual_claude_low]
|
||||
id = "atlas/worker/manual/claude/low"
|
||||
type = "random"
|
||||
targets = ["worker_claude_haiku_low"]
|
||||
targets = ["worker_claude_auto_low"]
|
||||
|
||||
[routes.worker_manual_claude_medium]
|
||||
id = "atlas/worker/manual/claude/medium"
|
||||
type = "random"
|
||||
targets = ["worker_claude_sonnet_medium"]
|
||||
targets = ["worker_claude_auto_medium"]
|
||||
|
||||
[routes.worker_manual_claude_high]
|
||||
id = "atlas/worker/manual/claude/high"
|
||||
type = "random"
|
||||
targets = ["worker_claude_sonnet_high"]
|
||||
targets = ["worker_claude_auto_high"]
|
||||
|
||||
[routes.worker_manual_claude_xhigh]
|
||||
id = "atlas/worker/manual/claude/xhigh"
|
||||
type = "random"
|
||||
targets = ["worker_claude_opus_xhigh"]
|
||||
targets = ["worker_claude_auto_xhigh"]
|
||||
|
||||
[routes.manual_codex_auto]
|
||||
id = "atlas/manual/codex/auto"
|
||||
type = "random"
|
||||
targets = ["codex_auto_medium"]
|
||||
context_window = 272000
|
||||
tool_calling = true
|
||||
reasoning = true
|
||||
|
||||
[routes.manual_codex_auto_low]
|
||||
id = "atlas/manual/codex/auto/low"
|
||||
type = "random"
|
||||
targets = ["codex_auto_low"]
|
||||
|
||||
[routes.manual_codex_auto_medium]
|
||||
id = "atlas/manual/codex/auto/medium"
|
||||
type = "random"
|
||||
targets = ["codex_auto_medium"]
|
||||
|
||||
[routes.manual_codex_auto_high]
|
||||
id = "atlas/manual/codex/auto/high"
|
||||
type = "random"
|
||||
targets = ["codex_auto_high"]
|
||||
|
||||
[routes.manual_codex_auto_xhigh]
|
||||
id = "atlas/manual/codex/auto/xhigh"
|
||||
type = "random"
|
||||
targets = ["codex_auto_xhigh"]
|
||||
|
||||
[routes.manual_claude_auto]
|
||||
id = "atlas/manual/claude/auto"
|
||||
type = "random"
|
||||
targets = ["claude_auto_medium"]
|
||||
context_window = 272000
|
||||
tool_calling = true
|
||||
reasoning = true
|
||||
|
||||
[routes.manual_claude_auto_low]
|
||||
id = "atlas/manual/claude/auto/low"
|
||||
type = "random"
|
||||
targets = ["claude_auto_low"]
|
||||
|
||||
[routes.manual_claude_auto_medium]
|
||||
id = "atlas/manual/claude/auto/medium"
|
||||
type = "random"
|
||||
targets = ["claude_auto_medium"]
|
||||
|
||||
[routes.manual_claude_auto_high]
|
||||
id = "atlas/manual/claude/auto/high"
|
||||
type = "random"
|
||||
targets = ["claude_auto_high"]
|
||||
|
||||
[routes.manual_claude_auto_xhigh]
|
||||
id = "atlas/manual/claude/auto/xhigh"
|
||||
type = "random"
|
||||
targets = ["claude_auto_xhigh"]
|
||||
|
||||
# A zero-weight target is not selected initially. Switchyard still walks
|
||||
# this ordered list after 403/408/429/5xx, timeout, or transport failure.
|
||||
|
||||
@ -22,7 +22,7 @@ spec:
|
||||
labels:
|
||||
app: hermes-switchyard
|
||||
annotations:
|
||||
ai.bstein.dev/config-rev: "20260823-voice-route-preflight-v1"
|
||||
ai.bstein.dev/config-rev: "20260913-provider-model-catalog-v2"
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/port: "9005"
|
||||
prometheus.io/path: /metrics
|
||||
|
||||
227
testing/fixtures/hermes/codex-0.154-visible-models.json
Normal file
227
testing/fixtures/hermes/codex-0.154-visible-models.json
Normal file
@ -0,0 +1,227 @@
|
||||
[
|
||||
{
|
||||
"defaultReasoningEffort": "medium",
|
||||
"description": "Our most capable model for complex, demanding work.",
|
||||
"displayName": "GPT-6-Astra",
|
||||
"hidden": false,
|
||||
"id": "gpt-6-astra",
|
||||
"inputModalities": [
|
||||
"text",
|
||||
"image"
|
||||
],
|
||||
"isDefault": true,
|
||||
"model": "gpt-6-astra",
|
||||
"modelSpecialty": null,
|
||||
"supportedReasoningEfforts": [
|
||||
{
|
||||
"description": "Fast responses with lighter reasoning",
|
||||
"reasoningEffort": "low"
|
||||
},
|
||||
{
|
||||
"description": "Balances speed and reasoning depth for everyday tasks",
|
||||
"reasoningEffort": "medium"
|
||||
},
|
||||
{
|
||||
"description": "Greater reasoning depth for complex problems",
|
||||
"reasoningEffort": "high"
|
||||
},
|
||||
{
|
||||
"description": "Extra high reasoning depth for complex problems",
|
||||
"reasoningEffort": "xhigh"
|
||||
},
|
||||
{
|
||||
"description": "Maximum reasoning depth for the hardest problems",
|
||||
"reasoningEffort": "max"
|
||||
},
|
||||
{
|
||||
"description": "Maximum reasoning with automatic task delegation",
|
||||
"reasoningEffort": "ultra"
|
||||
}
|
||||
],
|
||||
"upgrade": null
|
||||
},
|
||||
{
|
||||
"defaultReasoningEffort": "low",
|
||||
"description": "Reliable agentic workhorse for everyday tasks.",
|
||||
"displayName": "GPT-5.6-Sol",
|
||||
"hidden": false,
|
||||
"id": "gpt-5.6-sol",
|
||||
"inputModalities": [
|
||||
"text",
|
||||
"image"
|
||||
],
|
||||
"isDefault": false,
|
||||
"model": "gpt-5.6-sol",
|
||||
"modelSpecialty": null,
|
||||
"supportedReasoningEfforts": [
|
||||
{
|
||||
"description": "Fast responses with lighter reasoning",
|
||||
"reasoningEffort": "low"
|
||||
},
|
||||
{
|
||||
"description": "Balances speed and reasoning depth for everyday tasks",
|
||||
"reasoningEffort": "medium"
|
||||
},
|
||||
{
|
||||
"description": "Greater reasoning depth for complex problems",
|
||||
"reasoningEffort": "high"
|
||||
},
|
||||
{
|
||||
"description": "Extra high reasoning depth for complex problems",
|
||||
"reasoningEffort": "xhigh"
|
||||
},
|
||||
{
|
||||
"description": "Maximum reasoning depth for the hardest problems",
|
||||
"reasoningEffort": "max"
|
||||
},
|
||||
{
|
||||
"description": "Maximum reasoning with automatic task delegation",
|
||||
"reasoningEffort": "ultra"
|
||||
}
|
||||
],
|
||||
"upgrade": null
|
||||
},
|
||||
{
|
||||
"defaultReasoningEffort": "medium",
|
||||
"description": "Balanced agentic coding model for everyday work.",
|
||||
"displayName": "GPT-5.6-Terra",
|
||||
"hidden": false,
|
||||
"id": "gpt-5.6-terra",
|
||||
"inputModalities": [
|
||||
"text",
|
||||
"image"
|
||||
],
|
||||
"isDefault": false,
|
||||
"model": "gpt-5.6-terra",
|
||||
"modelSpecialty": null,
|
||||
"supportedReasoningEfforts": [
|
||||
{
|
||||
"description": "Fast responses with lighter reasoning",
|
||||
"reasoningEffort": "low"
|
||||
},
|
||||
{
|
||||
"description": "Balances speed and reasoning depth for everyday tasks",
|
||||
"reasoningEffort": "medium"
|
||||
},
|
||||
{
|
||||
"description": "Greater reasoning depth for complex problems",
|
||||
"reasoningEffort": "high"
|
||||
},
|
||||
{
|
||||
"description": "Extra high reasoning depth for complex problems",
|
||||
"reasoningEffort": "xhigh"
|
||||
},
|
||||
{
|
||||
"description": "Maximum reasoning depth for the hardest problems",
|
||||
"reasoningEffort": "max"
|
||||
},
|
||||
{
|
||||
"description": "Maximum reasoning with automatic task delegation",
|
||||
"reasoningEffort": "ultra"
|
||||
}
|
||||
],
|
||||
"upgrade": null
|
||||
},
|
||||
{
|
||||
"defaultReasoningEffort": "medium",
|
||||
"description": "Fast and affordable agentic coding model.",
|
||||
"displayName": "GPT-5.6-Luna",
|
||||
"hidden": false,
|
||||
"id": "gpt-5.6-luna",
|
||||
"inputModalities": [
|
||||
"text",
|
||||
"image"
|
||||
],
|
||||
"isDefault": false,
|
||||
"model": "gpt-5.6-luna",
|
||||
"modelSpecialty": null,
|
||||
"supportedReasoningEfforts": [
|
||||
{
|
||||
"description": "Fast responses with lighter reasoning",
|
||||
"reasoningEffort": "low"
|
||||
},
|
||||
{
|
||||
"description": "Balances speed and reasoning depth for everyday tasks",
|
||||
"reasoningEffort": "medium"
|
||||
},
|
||||
{
|
||||
"description": "Greater reasoning depth for complex problems",
|
||||
"reasoningEffort": "high"
|
||||
},
|
||||
{
|
||||
"description": "Extra high reasoning depth for complex problems",
|
||||
"reasoningEffort": "xhigh"
|
||||
},
|
||||
{
|
||||
"description": "Maximum reasoning depth for the hardest problems",
|
||||
"reasoningEffort": "max"
|
||||
}
|
||||
],
|
||||
"upgrade": null
|
||||
},
|
||||
{
|
||||
"defaultReasoningEffort": "medium",
|
||||
"description": "Proven previous-generation model for coding and general work.",
|
||||
"displayName": "GPT-5.5",
|
||||
"hidden": false,
|
||||
"id": "gpt-5.5",
|
||||
"inputModalities": [
|
||||
"text",
|
||||
"image"
|
||||
],
|
||||
"isDefault": false,
|
||||
"model": "gpt-5.5",
|
||||
"modelSpecialty": null,
|
||||
"supportedReasoningEfforts": [
|
||||
{
|
||||
"description": "Fast responses with lighter reasoning",
|
||||
"reasoningEffort": "low"
|
||||
},
|
||||
{
|
||||
"description": "Balances speed and reasoning depth for everyday tasks",
|
||||
"reasoningEffort": "medium"
|
||||
},
|
||||
{
|
||||
"description": "Greater reasoning depth for complex problems",
|
||||
"reasoningEffort": "high"
|
||||
},
|
||||
{
|
||||
"description": "Extra high reasoning depth for complex problems",
|
||||
"reasoningEffort": "xhigh"
|
||||
}
|
||||
],
|
||||
"upgrade": null
|
||||
},
|
||||
{
|
||||
"defaultReasoningEffort": "high",
|
||||
"description": "Ultra-fast coding model.",
|
||||
"displayName": "GPT-5.3-Codex-Spark",
|
||||
"hidden": false,
|
||||
"id": "gpt-5.3-codex-spark",
|
||||
"inputModalities": [
|
||||
"text"
|
||||
],
|
||||
"isDefault": false,
|
||||
"model": "gpt-5.3-codex-spark",
|
||||
"modelSpecialty": null,
|
||||
"supportedReasoningEfforts": [
|
||||
{
|
||||
"description": "Fast responses with lighter reasoning",
|
||||
"reasoningEffort": "low"
|
||||
},
|
||||
{
|
||||
"description": "Balances speed and reasoning depth for everyday tasks",
|
||||
"reasoningEffort": "medium"
|
||||
},
|
||||
{
|
||||
"description": "Greater reasoning depth for complex problems",
|
||||
"reasoningEffort": "high"
|
||||
},
|
||||
{
|
||||
"description": "Extra high reasoning depth for complex problems",
|
||||
"reasoningEffort": "xhigh"
|
||||
}
|
||||
],
|
||||
"upgrade": null
|
||||
}
|
||||
]
|
||||
106
testing/tests/test_claude_model_discovery.py
Normal file
106
testing/tests/test_claude_model_discovery.py
Normal file
@ -0,0 +1,106 @@
|
||||
"""Contract tests for native Claude subscription model discovery."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
SCRIPTS = Path(__file__).parents[2] / "services/hermes/scripts"
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
discovery = importlib.import_module("claude_model_discovery")
|
||||
|
||||
|
||||
def _initialize_event(*, provider: str = "firstParty") -> str:
|
||||
"""Build representative non-secret CLI initialization output."""
|
||||
return json.dumps(
|
||||
{
|
||||
"type": "control_response",
|
||||
"response": {
|
||||
"subtype": "success",
|
||||
"request_id": "catalog",
|
||||
"response": {
|
||||
"account": {
|
||||
"apiProvider": provider,
|
||||
"email": "discard-me@example.test",
|
||||
},
|
||||
"models": [
|
||||
{
|
||||
"value": "opus",
|
||||
"resolvedModel": "claude-opus-5",
|
||||
"displayName": "Opus",
|
||||
"description": "Most capable",
|
||||
"supportsEffort": True,
|
||||
"supportedEffortLevels": [
|
||||
"low",
|
||||
"high",
|
||||
"xhigh",
|
||||
"unknown",
|
||||
],
|
||||
"supportsAdaptiveThinking": True,
|
||||
},
|
||||
{
|
||||
"value": "haiku",
|
||||
"resolvedModel": "claude-haiku-4-5-20251001",
|
||||
"displayName": "Haiku",
|
||||
"description": "Fast",
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_initialize_parses_live_subscription_models_without_account_data():
|
||||
"""Literal aliases and the CLI-advertised effort set survive discovery."""
|
||||
result = discovery._parse_initialize_output(_initialize_event())
|
||||
|
||||
assert result.live is True
|
||||
assert result.connected is True
|
||||
assert result.error_code is None
|
||||
assert [model.value for model in result.models] == ["opus", "haiku"]
|
||||
assert result.models[0].resolved_model == "claude-opus-5"
|
||||
assert result.models[0].supported_effort_levels == ("low", "high", "xhigh")
|
||||
assert "email" not in result.models[0].as_dict()
|
||||
rendered_models = json.dumps([model.as_dict() for model in result.models])
|
||||
assert "discard-me@example.test" not in rendered_models
|
||||
|
||||
|
||||
def test_initialize_rejects_non_subscription_catalogs():
|
||||
"""An API-key or third-party account cannot become a subscription route."""
|
||||
result = discovery._parse_initialize_output(_initialize_event(provider="apiKey"))
|
||||
|
||||
assert result.live is False
|
||||
assert result.connected is False
|
||||
assert result.error_code == "not-first-party"
|
||||
|
||||
|
||||
def test_discovery_never_uses_api_key_environment(monkeypatch):
|
||||
"""The native call inherits OAuth config but strips metered API-key variables."""
|
||||
observed: dict[str, object] = {}
|
||||
|
||||
def fake_run(command, environment, timeout_seconds):
|
||||
observed["command"] = command
|
||||
observed["environment"] = environment
|
||||
observed["timeout"] = timeout_seconds
|
||||
return _initialize_event(), None
|
||||
|
||||
monkeypatch.setattr(discovery, "_run_initialize", fake_run)
|
||||
result = discovery.discover_claude_subscription_models(
|
||||
claude_bin="/native/claude",
|
||||
timeout_seconds=7,
|
||||
environment={
|
||||
"CLAUDE_CONFIG_DIR": "/mounted/claude",
|
||||
"ANTHROPIC_API_KEY": "secret",
|
||||
"CLAUDE_API_KEY": "secret",
|
||||
},
|
||||
)
|
||||
|
||||
assert result.live is True
|
||||
assert observed["environment"] == {"CLAUDE_CONFIG_DIR": "/mounted/claude"}
|
||||
assert observed["command"][:2] == ["/native/claude", "--input-format"]
|
||||
assert "--safe-mode" in observed["command"]
|
||||
assert "" in observed["command"]
|
||||
@ -411,8 +411,8 @@ def test_local_flux_runtime_and_gpu_handoff_are_flux_managed():
|
||||
assert "route/local/qwen2.5-14b/medium" in switchyard
|
||||
assert "Anthropic and Claude name the same provider" in switchyard
|
||||
assert "OpenAI and Codex name the same provider" in switchyard
|
||||
assert "Choose across every configured Codex and Claude family" in switchyard
|
||||
assert "Claude Fable" in switchyard
|
||||
assert "account-visible economy, balanced, or advanced" in switchyard
|
||||
assert "Choose across every configured Codex and Claude family" not in switchyard
|
||||
assert switchyard.count('Treat "think hard"') == 4
|
||||
assert switchyard.count("Never choose below the") >= 5
|
||||
switchyard_config = tomllib.loads(switchyard)
|
||||
@ -420,13 +420,13 @@ def test_local_flux_runtime_and_gpu_handoff_are_flux_managed():
|
||||
configured_targets = switchyard_config["targets"]
|
||||
for route_name in ("auto_fast", "auto_balanced", "auto_deep", "auto_maximum"):
|
||||
leading_targets = set(routes[route_name]["targets"][:2])
|
||||
assert leading_targets == {"codex_sol_xhigh", "claude_opus_xhigh"}
|
||||
assert leading_targets == {"codex_auto_xhigh", "claude_auto_xhigh"}
|
||||
assert "max_output_tokens" not in routes[route_name]
|
||||
for route_name in ("auto_fast", "auto_balanced", "auto_deep", "auto_maximum"):
|
||||
targets = routes[route_name]["targets"]
|
||||
selector_targets = routes[route_name]["response_schema"]
|
||||
assert not any(target.startswith("local_") for target in targets)
|
||||
assert any("fable" in target for target in targets)
|
||||
assert all("_auto_" in target or target.startswith("neutral_") for target in targets)
|
||||
assert "local_qwen" not in selector_targets
|
||||
assert "not eligible for foreground" in routes[route_name]["prompt"]
|
||||
for route_name in ("auto_deep", "auto_maximum"):
|
||||
@ -470,10 +470,11 @@ def test_local_flux_runtime_and_gpu_handoff_are_flux_managed():
|
||||
assert route["id"] == f"atlas/manual/{provider}/{family}/{effort}"
|
||||
assert route["targets"][0] == f"{provider}_{family}_{effort}"
|
||||
worker_target = f"worker_{provider}_{family}_{effort}"
|
||||
assert worker_target in routes["worker_auto_maximum"]["targets"]
|
||||
assert configured_targets[worker_target]["id"] == (
|
||||
f"worker/{provider}/{family}/{effort}"
|
||||
)
|
||||
assert all("_auto_" in target or target.startswith("neutral_")
|
||||
for target in routes["worker_auto_maximum"]["targets"])
|
||||
for route_name in ("auto_fast", "auto_balanced"):
|
||||
prompt = routes[route_name]["prompt"]
|
||||
assert "image tool—not the conversational model" in prompt
|
||||
|
||||
@ -5,6 +5,8 @@ from __future__ import annotations
|
||||
import importlib.util
|
||||
import json
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
@ -79,3 +81,29 @@ def test_mismatched_broker_resolution_cannot_change_route_decision():
|
||||
"terra",
|
||||
"xhigh",
|
||||
)
|
||||
|
||||
|
||||
def test_generic_claude_worker_keeps_a_catalog_advertised_alias():
|
||||
"""The native CLI receives its provider alias after generic AUTO routing."""
|
||||
route = lanes.select_route(
|
||||
"Review it.",
|
||||
"cli-auto",
|
||||
open_request=lambda *_args, **_kwargs: SwitchyardResponse(
|
||||
"worker/claude/auto/medium", "worker/claude/sonnet/medium"
|
||||
),
|
||||
catalog_model_allowed=lambda provider, model: provider == "claude" and model == "sonnet",
|
||||
)
|
||||
assert (route.provider, route.model, route.effort) == ("claude", "sonnet", "medium")
|
||||
|
||||
|
||||
def test_generic_claude_worker_rejects_an_unadvertised_alias():
|
||||
"""A forged worker receipt cannot pass an arbitrary native Claude alias."""
|
||||
with pytest.raises(RuntimeError, match="current provider model"):
|
||||
lanes.select_route(
|
||||
"Review it.",
|
||||
"cli-auto",
|
||||
open_request=lambda *_args, **_kwargs: SwitchyardResponse(
|
||||
"worker/claude/auto/medium", "worker/claude/not-advertised/medium"
|
||||
),
|
||||
catalog_model_allowed=lambda *_args: False,
|
||||
)
|
||||
|
||||
@ -24,10 +24,10 @@ def test_model_version_and_quality_selection_handle_new_and_small_models():
|
||||
codex = ["gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.7-luna", "gpt-5.3-codex-spark"]
|
||||
assert routing.choose_codex_model(codex) == "gpt-5.6-sol"
|
||||
assert routing.choose_codex_model(codex, balanced=True) == "gpt-5.6-terra"
|
||||
assert routing.choose_codex_model(codex + ["gpt-5.7-terra"]) == "gpt-5.7-terra"
|
||||
assert routing.choose_codex_model(codex + ["gpt-5.7-terra"]) == "gpt-5.6-sol"
|
||||
|
||||
claude = ["claude-opus-4.8", "claude-haiku-5", "claude-sonnet-5"]
|
||||
assert routing.choose_claude_model(claude) == "claude-sonnet-5"
|
||||
assert routing.choose_claude_model(claude) == "claude-opus-4.8"
|
||||
|
||||
assert routing.choose_codex_for_effort(codex, "low") == "gpt-5.7-luna"
|
||||
assert routing.choose_codex_for_effort(codex, "medium") == "gpt-5.6-terra"
|
||||
@ -89,6 +89,284 @@ def test_dynamic_catalog_resolves_new_models_and_preserves_last_known_good():
|
||||
)
|
||||
|
||||
|
||||
def test_dynamic_catalog_uses_metadata_for_unfamiliar_models_and_efforts():
|
||||
"""Future IDs route by declared capability, price, and effort support."""
|
||||
codex = routing.Catalog(
|
||||
"openai-codex",
|
||||
[
|
||||
"gpt-6.2-orbit",
|
||||
"gpt-6.2-balanced",
|
||||
"gpt-6.2-quick",
|
||||
"gpt-6.2-high-only",
|
||||
"gpt-6.2-xhigh",
|
||||
],
|
||||
True,
|
||||
True,
|
||||
"connected",
|
||||
{
|
||||
"gpt-6.2-orbit": {
|
||||
"description": "A strongest upgrade for difficult, complex tasks",
|
||||
"upgrade": "strongest",
|
||||
"supported_reasoning_efforts": ["low", "medium", "high", "xhigh"],
|
||||
"cost": 10,
|
||||
},
|
||||
"gpt-6.2-balanced": {
|
||||
"capability_tier": "balanced",
|
||||
"description": "General purpose model for everyday tasks",
|
||||
"supported_reasoning_efforts": ["low", "medium", "high", "xhigh"],
|
||||
"cost": 2,
|
||||
},
|
||||
"gpt-6.2-quick": {
|
||||
"capability_tier": "economy",
|
||||
"description": "Fast and affordable general purpose model",
|
||||
"supported_reasoning_efforts": ["low", "medium"],
|
||||
"cost": 1,
|
||||
},
|
||||
"gpt-6.2-high-only": {
|
||||
"capability_tier": "advanced",
|
||||
"supported_reasoning_efforts": ["high"],
|
||||
"cost": 1,
|
||||
},
|
||||
"gpt-6.2-xhigh": {
|
||||
"capability_tier": "advanced",
|
||||
"supported_reasoning_efforts": ["xhigh"],
|
||||
"cost": 2,
|
||||
},
|
||||
},
|
||||
)
|
||||
claude = routing.Catalog("anthropic", [], True, True, "connected")
|
||||
|
||||
catalog = routing.build_routing_catalog(codex, claude)
|
||||
resolved = catalog["providers"]["codex"]["resolved"]
|
||||
|
||||
assert resolved["xhigh"] == "gpt-6.2-orbit"
|
||||
assert resolved["low"] == "gpt-6.2-quick"
|
||||
assert resolved["medium"] == "gpt-6.2-balanced"
|
||||
assert resolved["high"] == "gpt-6.2-orbit"
|
||||
assert "gpt-6.2-high-only" in catalog["providers"]["codex"]["candidates"]
|
||||
assert catalog["providers"]["codex"]["candidates"]["gpt-6.2-high-only"]["tier"] == "advanced"
|
||||
|
||||
|
||||
def test_catalog_accepts_codex_effort_records_and_generalist_image_input():
|
||||
"""Codex app-server effort records and multimodal generalists stay routable."""
|
||||
codex = routing.Catalog(
|
||||
"openai-codex",
|
||||
["gpt-7-nova"],
|
||||
True,
|
||||
True,
|
||||
"connected-app-server",
|
||||
{
|
||||
"gpt-7-nova": {
|
||||
"description": "Strongest general purpose model with text and image inputs",
|
||||
"supportedReasoningEfforts": [
|
||||
{"reasoningEffort": "low"},
|
||||
{"reasoningEffort": "medium"},
|
||||
{"reasoningEffort": "high"},
|
||||
{"reasoningEffort": "xhigh"},
|
||||
],
|
||||
}
|
||||
},
|
||||
)
|
||||
catalog = routing.build_routing_catalog(
|
||||
codex, routing.Catalog("anthropic", [], True, True, "connected")
|
||||
)
|
||||
|
||||
assert catalog["providers"]["codex"]["resolved"]["xhigh"] == "gpt-7-nova"
|
||||
|
||||
|
||||
def test_dynamic_catalog_excludes_hidden_internal_and_specialist_records():
|
||||
"""Provider listings must not make private or modality-only products routable."""
|
||||
models = [
|
||||
"gpt-6.2-hidden",
|
||||
"gpt-6.2-internal",
|
||||
"gpt-6.2-image",
|
||||
"gpt-6.2-public",
|
||||
]
|
||||
metadata = {
|
||||
"gpt-6.2-hidden": {
|
||||
"hidden": True,
|
||||
"capability_tier": "advanced",
|
||||
},
|
||||
"gpt-6.2-internal": {
|
||||
"visibility": "internal",
|
||||
"description": "Internal evaluation model",
|
||||
"capability_tier": "advanced",
|
||||
},
|
||||
"gpt-6.2-image": {
|
||||
"description": "Image generation specialist",
|
||||
"capability_tier": "advanced",
|
||||
},
|
||||
"gpt-6.2-public": {
|
||||
"description": "Strongest general purpose model",
|
||||
"capability_tier": "advanced",
|
||||
},
|
||||
}
|
||||
codex = routing.Catalog(
|
||||
"openai-codex", models, True, True, "connected", metadata
|
||||
)
|
||||
claude = routing.Catalog("anthropic", [], True, True, "connected")
|
||||
|
||||
catalog = routing.build_routing_catalog(codex, claude)
|
||||
provider = catalog["providers"]["codex"]
|
||||
|
||||
assert provider["resolved"]["xhigh"] == "gpt-6.2-public"
|
||||
assert provider["candidates"]["gpt-6.2-hidden"]["eligible"] is False
|
||||
assert provider["candidates"]["gpt-6.2-internal"]["eligible"] is False
|
||||
assert provider["candidates"]["gpt-6.2-image"]["eligible"] is False
|
||||
|
||||
|
||||
def test_live_codex_metadata_selects_luna_terra_and_astra_by_effort():
|
||||
"""Actual Codex model/list fields route generic effort without name guesses."""
|
||||
fixture = Path(__file__).parents[1] / "fixtures/hermes/codex-0.154-visible-models.json"
|
||||
records = json.loads(fixture.read_text(encoding="utf-8"))
|
||||
models, metadata = routing.model_records(records)
|
||||
catalog = routing.build_routing_catalog(
|
||||
routing.Catalog("openai-codex", models, True, True, "connected", metadata),
|
||||
routing.Catalog("anthropic", [], True, True, "connected"),
|
||||
)
|
||||
assert catalog["providers"]["codex"]["resolved"] == {
|
||||
"low": "gpt-5.6-luna",
|
||||
"medium": "gpt-5.6-terra",
|
||||
"high": "gpt-6-astra",
|
||||
"xhigh": "gpt-6-astra",
|
||||
}
|
||||
|
||||
|
||||
def test_live_default_flagship_replaces_previous_advanced_route():
|
||||
"""A provider's current default wins over an equally strongest old model."""
|
||||
previous = {
|
||||
"providers": {"codex": {"models": ["gpt-astra"], "resolved": {
|
||||
effort: "gpt-astra" for effort in routing.EFFORTS
|
||||
}, "model_metadata": {"gpt-astra": {
|
||||
"description": "Most capable for complex work", "isDefault": False,
|
||||
}}}}
|
||||
}
|
||||
metadata = {
|
||||
"gpt-astra": {"description": "Most capable for complex work", "isDefault": False},
|
||||
"gpt-nova": {"description": "Most capable for complex work", "isDefault": True},
|
||||
}
|
||||
catalog = routing.build_routing_catalog(
|
||||
routing.Catalog("openai-codex", list(metadata), True, True, "connected", metadata),
|
||||
routing.Catalog("anthropic", [], True, True, "connected"), previous,
|
||||
)
|
||||
assert catalog["providers"]["codex"]["resolved"]["xhigh"] == "gpt-nova"
|
||||
|
||||
|
||||
def test_live_claude_effort_gap_uses_nearest_higher_declared_tier():
|
||||
"""An effortless economy alias cannot silently take a stronger route."""
|
||||
metadata = {
|
||||
"haiku": {"description": "Fastest for quick answers", "supportsEffort": False},
|
||||
"sonnet": {
|
||||
"description": "Efficient for routine tasks", "supportsEffort": True,
|
||||
"supportedEffortLevels": ["low", "medium", "high", "xhigh"],
|
||||
},
|
||||
"opus": {
|
||||
"description": "Best for everyday, complex tasks", "supportsEffort": True,
|
||||
"supportedEffortLevels": ["low", "medium", "high", "xhigh"],
|
||||
},
|
||||
"fable": {
|
||||
"description": "Most capable for your hardest and longest-running tasks",
|
||||
"supportsEffort": True,
|
||||
"supportedEffortLevels": ["low", "medium", "high", "xhigh"],
|
||||
},
|
||||
}
|
||||
catalog = routing.build_routing_catalog(
|
||||
routing.Catalog("openai-codex", [], True, True, "connected"),
|
||||
routing.Catalog("anthropic", list(metadata), True, True, "connected", metadata),
|
||||
)
|
||||
assert catalog["providers"]["claude"]["resolved"] == {
|
||||
"low": "sonnet", "medium": "sonnet", "high": "fable", "xhigh": "fable",
|
||||
}
|
||||
assert catalog["providers"]["claude"]["candidates"]["haiku"]["eligible"] is False
|
||||
|
||||
|
||||
def test_ambiguous_live_catalog_exposes_an_unresolved_route():
|
||||
"""An unfamiliar live model cannot revive a retired LKG model."""
|
||||
previous = {
|
||||
"providers": {
|
||||
"codex": {
|
||||
"models": ["gpt-5.6-sol"],
|
||||
"resolved": {effort: "gpt-5.6-sol" for effort in routing.EFFORTS},
|
||||
"tiers": {"advanced": "gpt-5.6-sol"},
|
||||
"model_metadata": {},
|
||||
}
|
||||
}
|
||||
}
|
||||
codex = routing.Catalog(
|
||||
"openai-codex", ["gpt-7.0-mystery"], True, True, "connected"
|
||||
)
|
||||
claude = routing.Catalog("anthropic", [], False, True, "degraded")
|
||||
|
||||
catalog = routing.build_routing_catalog(codex, claude, previous)
|
||||
|
||||
assert catalog["providers"]["codex"]["resolved"]["xhigh"] == ""
|
||||
assert catalog["providers"]["codex"]["candidates"]["gpt-7.0-mystery"]["tier"] is None
|
||||
|
||||
|
||||
def test_live_removal_clears_legacy_selector_instead_of_mapping_to_new_family():
|
||||
"""A live authoritative list cannot revive removed legacy selectors."""
|
||||
previous = {
|
||||
"providers": {
|
||||
"codex": {
|
||||
"models": ["gpt-5.6-sol"],
|
||||
"resolved": {effort: "gpt-5.6-sol" for effort in routing.EFFORTS},
|
||||
"tiers": {"sol": "gpt-5.6-sol", "advanced": "gpt-5.6-sol"},
|
||||
}
|
||||
}
|
||||
}
|
||||
codex = routing.Catalog(
|
||||
"openai-codex",
|
||||
["gpt-6.2-astra"],
|
||||
True,
|
||||
True,
|
||||
"connected",
|
||||
{"gpt-6.2-astra": {"capability_tier": "advanced"}},
|
||||
)
|
||||
claude = routing.Catalog("anthropic", [], False, True, "degraded")
|
||||
|
||||
catalog = routing.build_routing_catalog(codex, claude, previous)
|
||||
|
||||
assert catalog["providers"]["codex"]["tiers"]["sol"] == ""
|
||||
assert catalog["providers"]["codex"]["resolved"]["xhigh"] == "gpt-6.2-astra"
|
||||
|
||||
|
||||
def test_degraded_catalog_preserves_lkg_after_live_removal():
|
||||
"""A provider outage retains the prior route and does not treat it as removal."""
|
||||
previous = {
|
||||
"providers": {
|
||||
"codex": {
|
||||
"models": ["gpt-5.6-sol"],
|
||||
"resolved": {effort: "gpt-5.6-sol" for effort in routing.EFFORTS},
|
||||
"tiers": {"sol": "gpt-5.6-sol"},
|
||||
"model_metadata": {},
|
||||
}
|
||||
}
|
||||
}
|
||||
codex = routing.Catalog("openai-codex", [], False, True, "degraded")
|
||||
claude = routing.Catalog("anthropic", [], False, True, "degraded")
|
||||
|
||||
catalog = routing.build_routing_catalog(codex, claude, previous)
|
||||
|
||||
assert catalog["providers"]["codex"]["resolved"]["xhigh"] == "gpt-5.6-sol"
|
||||
assert catalog["providers"]["codex"]["tiers"]["sol"] == "gpt-5.6-sol"
|
||||
|
||||
|
||||
def test_explicit_legacy_selector_never_maps_to_astra():
|
||||
"""The old sol selector is exact and cannot silently follow a new family."""
|
||||
catalog = {
|
||||
"providers": {
|
||||
"codex": {
|
||||
"tiers": {"sol": "gpt-6.2-astra"},
|
||||
"resolved": {"xhigh": "gpt-6.2-astra"},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert catalog_resolver.resolve_model("codex", "sol", "xhigh", catalog) == (
|
||||
"gpt-5.6-sol"
|
||||
)
|
||||
|
||||
|
||||
def test_live_catalog_never_routes_to_a_removed_model_tier():
|
||||
"""A live provider catalog must replace a retired tier with a live model."""
|
||||
previous = {
|
||||
@ -116,9 +394,8 @@ def test_live_catalog_never_routes_to_a_removed_model_tier():
|
||||
|
||||
current = routing.build_routing_catalog(codex, claude, previous)
|
||||
|
||||
assert current["providers"]["codex"]["tiers"]["luna"] == "gpt-5.7-terra"
|
||||
assert current["providers"]["codex"]["tiers"]["luna"] in codex.models
|
||||
assert current["providers"]["claude"]["tiers"]["opus"] == "claude-sonnet-6"
|
||||
assert current["providers"]["codex"]["tiers"]["luna"] == ""
|
||||
assert current["providers"]["claude"]["tiers"]["opus"] == ""
|
||||
|
||||
|
||||
def test_switchyard_targets_have_unique_upstream_identities():
|
||||
@ -218,17 +495,17 @@ def test_configure_routes_keeps_every_profile_on_switchyard(tmp_path: Path):
|
||||
}
|
||||
assert root["fallback_providers"] == []
|
||||
assert root["toolsets"] == ["kanban"]
|
||||
assert codex_profile["model"]["model"] == "atlas/manual/codex/sol"
|
||||
assert codex_profile["model"]["model"] == "atlas/manual/codex/auto/high"
|
||||
assert codex_profile["model"]["provider"] == "atlas-switchyard"
|
||||
assert claude_profile["model"]["model"] == "atlas/manual/claude/sonnet"
|
||||
assert claude_profile["model"]["model"] == "atlas/manual/claude/auto/high"
|
||||
assert claude_profile["model"]["provider"] == "atlas-switchyard"
|
||||
assert codex_profile["fallback_providers"] == []
|
||||
assert claude_profile["fallback_providers"] == []
|
||||
assert codex_profile["toolsets"] == []
|
||||
assert codex_profile["agent"]["reasoning_effort"] == "high"
|
||||
assert codex_xhigh_profile["fallback_providers"] == []
|
||||
assert routes["codex-xhigh"] == ["atlas/manual/codex/sol"]
|
||||
assert routes["claude-xhigh"] == ["atlas/manual/claude/opus"]
|
||||
assert routes["codex-xhigh"] == ["atlas/manual/codex/auto/xhigh"]
|
||||
assert routes["claude-xhigh"] == ["atlas/manual/claude/auto/xhigh"]
|
||||
assert routes["synthesis-xhigh"] == ["atlas/auto/maximum"]
|
||||
assert routes["coordinator"] == ["atlas/auto/maximum"]
|
||||
assert all(
|
||||
@ -285,7 +562,7 @@ def test_degraded_refresh_preserves_switchyard_worker_preference(tmp_path: Path)
|
||||
worker = yaml.safe_load(
|
||||
(tmp_path / "profiles/codex-high/config.yaml").read_text(encoding="utf-8")
|
||||
)
|
||||
assert worker["model"]["model"] == "atlas/manual/codex/sol"
|
||||
assert worker["model"]["model"] == "atlas/manual/codex/auto/high"
|
||||
assert worker["model"]["provider"] == "atlas-switchyard"
|
||||
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user