hermes: resolve difficulty routes from live model catalogs
This commit is contained in:
parent
2ab737f8f8
commit
d141b33a7d
@ -25,7 +25,7 @@ spec:
|
|||||||
ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers
|
ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers
|
||||||
ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback
|
ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback
|
||||||
ai.bstein.dev/placement: primary amd64 accelerator titan-22; arm64 rpi5 fleet fallback; storage-backbone nodes excluded
|
ai.bstein.dev/placement: primary amd64 accelerator titan-22; arm64 rpi5 fleet fallback; storage-backbone nodes excluded
|
||||||
ai.bstein.dev/config-rev: "20260912-dashboard-bootstrap-visibility"
|
ai.bstein.dev/config-rev: "20260913-provider-model-catalog-v2"
|
||||||
prometheus.io/scrape: "true"
|
prometheus.io/scrape: "true"
|
||||||
prometheus.io/path: /metrics
|
prometheus.io/path: /metrics
|
||||||
prometheus.io/port: "9010"
|
prometheus.io/port: "9010"
|
||||||
@ -475,11 +475,11 @@ spec:
|
|||||||
# which self-heals after the pod has run on the other architecture.
|
# which self-heals after the pod has run on the other architecture.
|
||||||
case "$(uname -m)" in x86_64) nodearch=x64 ;; aarch64) nodearch=arm64 ;; *) nodearch=unknown ;; esac
|
case "$(uname -m)" in x86_64) nodearch=x64 ;; aarch64) nodearch=arm64 ;; *) nodearch=unknown ;; esac
|
||||||
codex_native="${tools}/lib/node_modules/@openai/codex-linux-${nodearch}"
|
codex_native="${tools}/lib/node_modules/@openai/codex-linux-${nodearch}"
|
||||||
if [ ! -f "${tools}/.cli-versions-0.147.0-2.1.226-${arch}" ] || [ ! -d "${codex_native}" ]; then
|
if [ ! -f "${tools}/.cli-versions-0.154.0-2.1.226-${arch}" ] || [ ! -d "${codex_native}" ]; then
|
||||||
npm install --global --omit=dev --no-audit --no-fund --prefix "${tools}" \
|
npm install --global --omit=dev --no-audit --no-fund --prefix "${tools}" \
|
||||||
@openai/codex@0.147.0 \
|
@openai/codex@0.154.0 \
|
||||||
@anthropic-ai/claude-code@2.1.226
|
@anthropic-ai/claude-code@2.1.226
|
||||||
touch "${tools}/.cli-versions-0.147.0-2.1.226-${arch}"
|
touch "${tools}/.cli-versions-0.154.0-2.1.226-${arch}"
|
||||||
fi
|
fi
|
||||||
kubectl_version="$("${tools}/bin/kubectl" version --client --output=json 2>/dev/null || true)"
|
kubectl_version="$("${tools}/bin/kubectl" version --client --output=json 2>/dev/null || true)"
|
||||||
case "${kubectl_version}" in *\"gitVersion\":\"v1.33.3\"*) kubectl_ready=1 ;; *) kubectl_ready=0 ;; esac
|
case "${kubectl_version}" in *\"gitVersion\":\"v1.33.3\"*) kubectl_ready=1 ;; *) kubectl_ready=0 ;; esac
|
||||||
|
|||||||
@ -61,9 +61,11 @@ data:
|
|||||||
atlas/auto/balanced: {provider: atlas-switchyard, model: atlas/auto/balanced}
|
atlas/auto/balanced: {provider: atlas-switchyard, model: atlas/auto/balanced}
|
||||||
atlas/auto/deep: {provider: atlas-switchyard, model: atlas/auto/deep}
|
atlas/auto/deep: {provider: atlas-switchyard, model: atlas/auto/deep}
|
||||||
atlas/auto/maximum: {provider: atlas-switchyard, model: atlas/auto/maximum}
|
atlas/auto/maximum: {provider: atlas-switchyard, model: atlas/auto/maximum}
|
||||||
|
atlas/manual/codex/auto: {provider: atlas-switchyard, model: atlas/manual/codex/auto}
|
||||||
atlas/manual/codex/luna: {provider: atlas-switchyard, model: atlas/manual/codex/luna}
|
atlas/manual/codex/luna: {provider: atlas-switchyard, model: atlas/manual/codex/luna}
|
||||||
atlas/manual/codex/terra: {provider: atlas-switchyard, model: atlas/manual/codex/terra}
|
atlas/manual/codex/terra: {provider: atlas-switchyard, model: atlas/manual/codex/terra}
|
||||||
atlas/manual/codex/sol: {provider: atlas-switchyard, model: atlas/manual/codex/sol}
|
atlas/manual/codex/sol: {provider: atlas-switchyard, model: atlas/manual/codex/sol}
|
||||||
|
atlas/manual/claude/auto: {provider: atlas-switchyard, model: atlas/manual/claude/auto}
|
||||||
atlas/manual/claude/haiku: {provider: atlas-switchyard, model: atlas/manual/claude/haiku}
|
atlas/manual/claude/haiku: {provider: atlas-switchyard, model: atlas/manual/claude/haiku}
|
||||||
atlas/manual/claude/fable: {provider: atlas-switchyard, model: atlas/manual/claude/fable}
|
atlas/manual/claude/fable: {provider: atlas-switchyard, model: atlas/manual/claude/fable}
|
||||||
atlas/manual/claude/sonnet: {provider: atlas-switchyard, model: atlas/manual/claude/sonnet}
|
atlas/manual/claude/sonnet: {provider: atlas-switchyard, model: atlas/manual/claude/sonnet}
|
||||||
|
|||||||
@ -29,7 +29,7 @@ spec:
|
|||||||
ai.bstein.dev/router-wire-contract: ollama-numeric-keepalive
|
ai.bstein.dev/router-wire-contract: ollama-numeric-keepalive
|
||||||
ai.bstein.dev/isolation: one Hermes process and PVC per Keycloak subject
|
ai.bstein.dev/isolation: one Hermes process and PVC per Keycloak subject
|
||||||
ai.bstein.dev/model-policy: uniform automatic policy with per-user overrides
|
ai.bstein.dev/model-policy: uniform automatic policy with per-user overrides
|
||||||
ai.bstein.dev/config-rev: "20260816-telegram-topics"
|
ai.bstein.dev/config-rev: "20260913-provider-model-catalog-v2"
|
||||||
ai.bstein.dev/hux-config-rev: "20260824-hux-v1"
|
ai.bstein.dev/hux-config-rev: "20260824-hux-v1"
|
||||||
vault.hashicorp.com/agent-inject: "true"
|
vault.hashicorp.com/agent-inject: "true"
|
||||||
vault.hashicorp.com/role: hermes-chat
|
vault.hashicorp.com/role: hermes-chat
|
||||||
|
|||||||
@ -141,6 +141,9 @@ configMapGenerator:
|
|||||||
- worker_route_broker.py=scripts/worker_route_broker.py
|
- worker_route_broker.py=scripts/worker_route_broker.py
|
||||||
- hermes_coordinator.py=scripts/hermes_coordinator.py
|
- hermes_coordinator.py=scripts/hermes_coordinator.py
|
||||||
- hermes_model_routing.py=scripts/hermes_model_routing.py
|
- hermes_model_routing.py=scripts/hermes_model_routing.py
|
||||||
|
- provider_model_catalog.py=scripts/provider_model_catalog.py
|
||||||
|
- provider_model_discovery.py=scripts/provider_model_discovery.py
|
||||||
|
- claude_model_discovery.py=scripts/claude_model_discovery.py
|
||||||
- hermes_stt_client.py=scripts/hermes_stt_client.py
|
- hermes_stt_client.py=scripts/hermes_stt_client.py
|
||||||
- image_broker.py=scripts/image_broker.py
|
- image_broker.py=scripts/image_broker.py
|
||||||
- install_agent_tools.sh=scripts/install_agent_tools.sh
|
- install_agent_tools.sh=scripts/install_agent_tools.sh
|
||||||
|
|||||||
@ -51,10 +51,12 @@ MANUAL_ROUTES = frozenset(
|
|||||||
"atlas/manual/codex/luna",
|
"atlas/manual/codex/luna",
|
||||||
"atlas/manual/codex/terra",
|
"atlas/manual/codex/terra",
|
||||||
"atlas/manual/codex/sol",
|
"atlas/manual/codex/sol",
|
||||||
|
"atlas/manual/codex/auto",
|
||||||
"atlas/manual/claude/haiku",
|
"atlas/manual/claude/haiku",
|
||||||
"atlas/manual/claude/fable",
|
"atlas/manual/claude/fable",
|
||||||
"atlas/manual/claude/sonnet",
|
"atlas/manual/claude/sonnet",
|
||||||
"atlas/manual/claude/opus",
|
"atlas/manual/claude/opus",
|
||||||
|
"atlas/manual/claude/auto",
|
||||||
"atlas/manual/local/qwen-14b",
|
"atlas/manual/local/qwen-14b",
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
@ -69,8 +71,8 @@ EXACT_MANUAL_ROUTES = frozenset(
|
|||||||
)
|
)
|
||||||
ALL_REQUEST_ROUTES = ALL_ROUTES | EXACT_MANUAL_ROUTES
|
ALL_REQUEST_ROUTES = ALL_ROUTES | EXACT_MANUAL_ROUTES
|
||||||
PROVIDER_DEFAULT = {
|
PROVIDER_DEFAULT = {
|
||||||
"codex": "atlas/manual/codex/terra",
|
"codex": "atlas/manual/codex/auto",
|
||||||
"claude": "atlas/manual/claude/sonnet",
|
"claude": "atlas/manual/claude/auto",
|
||||||
"local": "atlas/manual/local/qwen-14b",
|
"local": "atlas/manual/local/qwen-14b",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
280
services/hermes/scripts/claude_model_discovery.py
Normal file
280
services/hermes/scripts/claude_model_discovery.py
Normal file
@ -0,0 +1,280 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Discover the current Claude subscription model catalog without inference.
|
||||||
|
|
||||||
|
Claude Code exposes the account-visible ``ModelInfo[]`` through its Agent SDK
|
||||||
|
``initialize`` control response. This module speaks that documented stream-JSON
|
||||||
|
control protocol directly instead of treating authentication status or a cached
|
||||||
|
model list as a live catalog. It intentionally retains only non-secret
|
||||||
|
ModelInfo metadata.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import signal
|
||||||
|
import subprocess
|
||||||
|
import uuid
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import Any, Mapping, Sequence
|
||||||
|
|
||||||
|
|
||||||
|
DEFAULT_CLAUDE_BIN = "/opt/data/tools/bin/claude"
|
||||||
|
DEFAULT_TIMEOUT_SECONDS = 30.0
|
||||||
|
MAX_MODELS = 256
|
||||||
|
EFFORT_LEVELS = frozenset({"low", "medium", "high", "xhigh", "max"})
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class ClaudeModel:
|
||||||
|
"""Safe metadata for one model advertised by the native Claude CLI."""
|
||||||
|
|
||||||
|
value: str
|
||||||
|
resolved_model: str | None
|
||||||
|
display_name: str
|
||||||
|
description: str
|
||||||
|
supports_effort: bool
|
||||||
|
supported_effort_levels: tuple[str, ...]
|
||||||
|
supports_adaptive_thinking: bool
|
||||||
|
supports_fast_mode: bool
|
||||||
|
supports_auto_mode: bool
|
||||||
|
disabled: bool
|
||||||
|
|
||||||
|
def as_dict(self) -> dict[str, Any]:
|
||||||
|
"""Serialize only ModelInfo metadata, preserving Claude's alias value."""
|
||||||
|
return {
|
||||||
|
"value": self.value,
|
||||||
|
"resolvedModel": self.resolved_model,
|
||||||
|
"displayName": self.display_name,
|
||||||
|
"description": self.description,
|
||||||
|
"supportsEffort": self.supports_effort,
|
||||||
|
"supportedEffortLevels": list(self.supported_effort_levels),
|
||||||
|
"supportsAdaptiveThinking": self.supports_adaptive_thinking,
|
||||||
|
"supportsFastMode": self.supports_fast_mode,
|
||||||
|
"supportsAutoMode": self.supports_auto_mode,
|
||||||
|
"disabled": self.disabled,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class ClaudeModelDiscovery:
|
||||||
|
"""Result of one bounded, subscription-authenticated catalog probe."""
|
||||||
|
|
||||||
|
models: tuple[ClaudeModel, ...]
|
||||||
|
live: bool
|
||||||
|
connected: bool
|
||||||
|
state: str
|
||||||
|
error_code: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def _native_environment(base: Mapping[str, str] | None = None) -> dict[str, str]:
|
||||||
|
"""Keep mounted OAuth configuration while preventing API-key billing lanes."""
|
||||||
|
environment = dict(os.environ if base is None else base)
|
||||||
|
environment.pop("ANTHROPIC_API_KEY", None)
|
||||||
|
environment.pop("CLAUDE_API_KEY", None)
|
||||||
|
return environment
|
||||||
|
|
||||||
|
|
||||||
|
def _command(claude_bin: str) -> list[str]:
|
||||||
|
"""Build a no-inference CLI invocation with all tools and MCP disabled."""
|
||||||
|
return [
|
||||||
|
claude_bin,
|
||||||
|
"--input-format",
|
||||||
|
"stream-json",
|
||||||
|
"--output-format",
|
||||||
|
"stream-json",
|
||||||
|
"--verbose",
|
||||||
|
"--print",
|
||||||
|
"--no-session-persistence",
|
||||||
|
"--safe-mode",
|
||||||
|
"--tools",
|
||||||
|
"",
|
||||||
|
"--strict-mcp-config",
|
||||||
|
"--mcp-config",
|
||||||
|
'{"mcpServers":{}}',
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _initialize_request() -> str:
|
||||||
|
"""Build the Claude Agent SDK initialize request for ModelInfo discovery."""
|
||||||
|
return json.dumps(
|
||||||
|
{
|
||||||
|
"type": "control_request",
|
||||||
|
"request_id": f"hermes-model-catalog-{uuid.uuid4()}",
|
||||||
|
"request": {"subtype": "initialize", "hooks": {}},
|
||||||
|
},
|
||||||
|
separators=(",", ":"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _stop_process(process: subprocess.Popen[str]) -> None:
|
||||||
|
"""End a timed-out CLI process and its process group without leaking output."""
|
||||||
|
if process.poll() is not None:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
os.killpg(process.pid, signal.SIGTERM)
|
||||||
|
except (OSError, ProcessLookupError):
|
||||||
|
process.terminate()
|
||||||
|
try:
|
||||||
|
process.communicate(timeout=2)
|
||||||
|
return
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
pass
|
||||||
|
try:
|
||||||
|
os.killpg(process.pid, signal.SIGKILL)
|
||||||
|
except (OSError, ProcessLookupError):
|
||||||
|
process.kill()
|
||||||
|
try:
|
||||||
|
process.communicate(timeout=2)
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def _run_initialize(
|
||||||
|
command: Sequence[str], environment: Mapping[str, str], timeout_seconds: float
|
||||||
|
) -> tuple[str | None, str | None]:
|
||||||
|
"""Run initialize once, returning stream output or a non-sensitive error code."""
|
||||||
|
try:
|
||||||
|
process = subprocess.Popen(
|
||||||
|
list(command),
|
||||||
|
stdin=subprocess.PIPE,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.DEVNULL,
|
||||||
|
text=True,
|
||||||
|
env=dict(environment),
|
||||||
|
start_new_session=True,
|
||||||
|
)
|
||||||
|
except OSError:
|
||||||
|
return None, "spawn-failed"
|
||||||
|
try:
|
||||||
|
request = _initialize_request() + "\n"
|
||||||
|
stdout, _ = process.communicate(request, timeout=timeout_seconds)
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
_stop_process(process)
|
||||||
|
return None, "timeout"
|
||||||
|
except OSError:
|
||||||
|
_stop_process(process)
|
||||||
|
return None, "transport-error"
|
||||||
|
if process.returncode != 0:
|
||||||
|
return None, "cli-error"
|
||||||
|
return stdout, None
|
||||||
|
|
||||||
|
|
||||||
|
def _text(value: Any, *, maximum: int = 1024) -> str:
|
||||||
|
"""Return bounded string metadata and reject non-string protocol values."""
|
||||||
|
return value[:maximum] if isinstance(value, str) else ""
|
||||||
|
|
||||||
|
|
||||||
|
def _model_info(value: Any) -> ClaudeModel | None:
|
||||||
|
"""Validate one ModelInfo record without inventing aliases or identifiers."""
|
||||||
|
if not isinstance(value, dict):
|
||||||
|
return None
|
||||||
|
alias = _text(value.get("value"))
|
||||||
|
if not alias:
|
||||||
|
return None
|
||||||
|
resolved_model = _text(value.get("resolvedModel")) or None
|
||||||
|
efforts = value.get("supportedEffortLevels")
|
||||||
|
supported_effort_levels = (
|
||||||
|
tuple(
|
||||||
|
level
|
||||||
|
for level in efforts
|
||||||
|
if isinstance(level, str) and level in EFFORT_LEVELS
|
||||||
|
)
|
||||||
|
if isinstance(efforts, list)
|
||||||
|
else ()
|
||||||
|
)
|
||||||
|
return ClaudeModel(
|
||||||
|
value=alias,
|
||||||
|
resolved_model=resolved_model,
|
||||||
|
display_name=_text(value.get("displayName")),
|
||||||
|
description=_text(value.get("description")),
|
||||||
|
supports_effort=value.get("supportsEffort") is True,
|
||||||
|
supported_effort_levels=supported_effort_levels,
|
||||||
|
supports_adaptive_thinking=value.get("supportsAdaptiveThinking") is True,
|
||||||
|
supports_fast_mode=value.get("supportsFastMode") is True,
|
||||||
|
supports_auto_mode=value.get("supportsAutoMode") is True,
|
||||||
|
disabled=value.get("disabled") is True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_initialize_output(stdout: str) -> ClaudeModelDiscovery:
|
||||||
|
"""Extract all ModelInfo pages present in initialize control responses.
|
||||||
|
|
||||||
|
Claude Code 2.1.226 returns its complete ``models`` array in one response.
|
||||||
|
The stream parser still gathers every valid initialize response so a future
|
||||||
|
paged/multi-frame implementation is not silently truncated. The protocol
|
||||||
|
currently specifies no continuation request, so this function never
|
||||||
|
fabricates one.
|
||||||
|
"""
|
||||||
|
models: list[ClaudeModel] = []
|
||||||
|
seen: set[tuple[str, str | None]] = set()
|
||||||
|
received_models = False
|
||||||
|
first_party = False
|
||||||
|
for line in stdout.splitlines():
|
||||||
|
try:
|
||||||
|
event = json.loads(line)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
continue
|
||||||
|
if not isinstance(event, dict) or event.get("type") != "control_response":
|
||||||
|
continue
|
||||||
|
response = event.get("response")
|
||||||
|
if not isinstance(response, dict) or response.get("subtype") != "success":
|
||||||
|
continue
|
||||||
|
payload = response.get("response")
|
||||||
|
if not isinstance(payload, dict) or not isinstance(payload.get("models"), list):
|
||||||
|
continue
|
||||||
|
received_models = True
|
||||||
|
account = payload.get("account")
|
||||||
|
if isinstance(account, dict) and account.get("apiProvider") == "firstParty":
|
||||||
|
first_party = True
|
||||||
|
for raw_model in payload["models"]:
|
||||||
|
model = _model_info(raw_model)
|
||||||
|
if model is None or len(models) >= MAX_MODELS:
|
||||||
|
continue
|
||||||
|
key = (model.value, model.resolved_model)
|
||||||
|
if key not in seen:
|
||||||
|
seen.add(key)
|
||||||
|
models.append(model)
|
||||||
|
if not received_models:
|
||||||
|
return ClaudeModelDiscovery(
|
||||||
|
(), False, False, "protocol-error", "protocol-error"
|
||||||
|
)
|
||||||
|
if not first_party:
|
||||||
|
return ClaudeModelDiscovery(
|
||||||
|
tuple(models), False, False, "not-subscription", "not-first-party"
|
||||||
|
)
|
||||||
|
if not models:
|
||||||
|
return ClaudeModelDiscovery(
|
||||||
|
(), True, True, "connected-subscription", "empty-model-catalog"
|
||||||
|
)
|
||||||
|
return ClaudeModelDiscovery(tuple(models), True, True, "connected-subscription")
|
||||||
|
|
||||||
|
|
||||||
|
def discover_claude_subscription_models(
|
||||||
|
*,
|
||||||
|
claude_bin: str | None = None,
|
||||||
|
timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS,
|
||||||
|
environment: Mapping[str, str] | None = None,
|
||||||
|
) -> ClaudeModelDiscovery:
|
||||||
|
"""Return current subscription models using only Claude's initialize protocol.
|
||||||
|
|
||||||
|
A successful native authentication-status command or a locally cached model
|
||||||
|
list is deliberately insufficient: ``live`` is true only after a valid,
|
||||||
|
first-party initialize response supplies the account-visible ``models``.
|
||||||
|
"""
|
||||||
|
binary = (
|
||||||
|
claude_bin
|
||||||
|
or os.environ.get("HERMES_CLAUDE_NATIVE_BIN")
|
||||||
|
or DEFAULT_CLAUDE_BIN
|
||||||
|
)
|
||||||
|
if timeout_seconds <= 0:
|
||||||
|
return ClaudeModelDiscovery(
|
||||||
|
(), False, False, "invalid-request", "invalid-timeout"
|
||||||
|
)
|
||||||
|
stdout, error_code = _run_initialize(
|
||||||
|
_command(binary), _native_environment(environment), timeout_seconds
|
||||||
|
)
|
||||||
|
if error_code:
|
||||||
|
state = "unavailable" if error_code == "spawn-failed" else "degraded"
|
||||||
|
return ClaudeModelDiscovery((), False, False, state, error_code)
|
||||||
|
return _parse_initialize_output(stdout or "")
|
||||||
@ -67,6 +67,15 @@ _auth_probe_value: dict[str, Any] = {}
|
|||||||
HEALTH_POLL_SECONDS: Final = max(
|
HEALTH_POLL_SECONDS: Final = max(
|
||||||
30, int(os.environ.get("HERMES_CLAUDE_HEALTH_POLL_SECONDS", "60"))
|
30, int(os.environ.get("HERMES_CLAUDE_HEALTH_POLL_SECONDS", "60"))
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _catalog_contains(model: str) -> bool:
|
||||||
|
"""Accept a provider-advertised Claude alias without trusting arbitrary IDs."""
|
||||||
|
catalog = load_catalog()
|
||||||
|
providers = catalog.get("providers", {}) if isinstance(catalog, dict) else {}
|
||||||
|
claude = providers.get("claude", {}) if isinstance(providers, dict) else {}
|
||||||
|
models = claude.get("models", []) if isinstance(claude, dict) else []
|
||||||
|
return isinstance(model, str) and model in models
|
||||||
def _read_secret(file_env_name: str) -> str:
|
def _read_secret(file_env_name: str) -> str:
|
||||||
"""Read a secret only from its runtime-mounted file."""
|
"""Read a secret only from its runtime-mounted file."""
|
||||||
path = os.environ.get(file_env_name, "").strip()
|
path = os.environ.get(file_env_name, "").strip()
|
||||||
@ -108,7 +117,7 @@ def _route(model: str, payload: dict[str, Any]) -> tuple[str, str]:
|
|||||||
output_config = payload.get("output_config")
|
output_config = payload.get("output_config")
|
||||||
if isinstance(output_config, dict) and output_config.get("effort") in EFFORTS:
|
if isinstance(output_config, dict) and output_config.get("effort") in EFFORTS:
|
||||||
effort = str(output_config["effort"])
|
effort = str(output_config["effort"])
|
||||||
if not model.startswith("claude-"):
|
if not model.startswith("claude-") and not _catalog_contains(model):
|
||||||
raise ValueError("unsupported Claude model")
|
raise ValueError("unsupported Claude model")
|
||||||
return model, effort
|
return model, effort
|
||||||
|
|
||||||
@ -626,7 +635,7 @@ class Handler(BaseHTTPRequestHandler):
|
|||||||
models = sorted(
|
models = sorted(
|
||||||
model
|
model
|
||||||
for model in raw_models
|
for model in raw_models
|
||||||
if isinstance(model, str) and model.startswith("claude-")
|
if isinstance(model, str)
|
||||||
)
|
)
|
||||||
self._json(200, {"data": [{"id": model, "type": "model"} for model in models]})
|
self._json(200, {"data": [{"id": model, "type": "model"} for model in models]})
|
||||||
|
|
||||||
|
|||||||
@ -19,6 +19,7 @@ from cli_lane_config import (
|
|||||||
Route,
|
Route,
|
||||||
)
|
)
|
||||||
from cli_lane_files import load_json
|
from cli_lane_files import load_json
|
||||||
|
from routing_catalog import catalog_contains
|
||||||
|
|
||||||
|
|
||||||
def parse_assignee(assignee: str) -> tuple[str | None, str | None]:
|
def parse_assignee(assignee: str) -> tuple[str | None, str | None]:
|
||||||
@ -107,6 +108,7 @@ def select_route(
|
|||||||
exclude_reason: str | None = None,
|
exclude_reason: str | None = None,
|
||||||
switchyard_url: str = SWITCHYARD_URL,
|
switchyard_url: str = SWITCHYARD_URL,
|
||||||
open_request: Callable[..., Any] = urllib.request.urlopen,
|
open_request: Callable[..., Any] = urllib.request.urlopen,
|
||||||
|
catalog_model_allowed: Callable[[str, str], bool] = catalog_contains,
|
||||||
) -> Route:
|
) -> Route:
|
||||||
"""Ask Switchyard to select one native CLI worker at this boundary."""
|
"""Ask Switchyard to select one native CLI worker at this boundary."""
|
||||||
started = time.monotonic()
|
started = time.monotonic()
|
||||||
@ -149,19 +151,26 @@ def select_route(
|
|||||||
# Switchyard preserves the stable tier target in the selection header and
|
# Switchyard preserves the stable tier target in the selection header and
|
||||||
# top-level response model. The worker broker's assistant content contains
|
# top-level response model. The worker broker's assistant content contains
|
||||||
# the steward-resolved provider model required by the native CLI.
|
# the steward-resolved provider model required by the native CLI.
|
||||||
|
resolved_valid = False
|
||||||
try:
|
try:
|
||||||
response_document = json.loads(response_body)
|
response_document = json.loads(response_body)
|
||||||
resolved_target = str(response_document["choices"][0]["message"]["content"] or "")
|
resolved_target = str(response_document["choices"][0]["message"]["content"] or "")
|
||||||
resolved_provider, resolved_model, resolved_effort = _decode_worker_target(resolved_target)
|
resolved_provider, resolved_model, resolved_effort = _decode_worker_target(resolved_target)
|
||||||
required_prefix = "gpt-" if provider == "codex" else "claude-"
|
required_prefix = "gpt-" if provider == "codex" else "claude-"
|
||||||
if (
|
resolved_valid = (
|
||||||
resolved_provider == provider
|
resolved_provider == provider
|
||||||
and resolved_effort == effort
|
and resolved_effort == effort
|
||||||
and resolved_model.startswith(required_prefix)
|
and (
|
||||||
):
|
resolved_model.startswith(required_prefix)
|
||||||
|
or (provider == "claude" and catalog_model_allowed(provider, resolved_model))
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if resolved_valid:
|
||||||
model = resolved_model
|
model = resolved_model
|
||||||
except (AttributeError, IndexError, KeyError, RuntimeError, TypeError, ValueError, json.JSONDecodeError):
|
except (AttributeError, IndexError, KeyError, RuntimeError, TypeError, ValueError, json.JSONDecodeError):
|
||||||
pass
|
resolved_valid = False
|
||||||
|
if model == "auto" and not resolved_valid:
|
||||||
|
raise RuntimeError("Switchyard did not return a current provider model")
|
||||||
if exclude_provider and provider == exclude_provider:
|
if exclude_provider and provider == exclude_provider:
|
||||||
alternate = "claude" if provider == "codex" else "codex"
|
alternate = "claude" if provider == "codex" else "codex"
|
||||||
guarded = select_route(
|
guarded = select_route(
|
||||||
@ -169,6 +178,7 @@ def select_route(
|
|||||||
f"cli-{alternate}-{effort}",
|
f"cli-{alternate}-{effort}",
|
||||||
switchyard_url=switchyard_url,
|
switchyard_url=switchyard_url,
|
||||||
open_request=open_request,
|
open_request=open_request,
|
||||||
|
catalog_model_allowed=catalog_model_allowed,
|
||||||
)
|
)
|
||||||
return Route(
|
return Route(
|
||||||
provider=guarded.provider,
|
provider=guarded.provider,
|
||||||
|
|||||||
@ -18,7 +18,7 @@ from typing import Any, Iterable
|
|||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
|
|
||||||
from routing_catalog import resolve_route
|
from routing_catalog import load_catalog, resolve_route
|
||||||
|
|
||||||
|
|
||||||
HOST = os.environ.get("HERMES_CODEX_BROKER_LISTEN_HOST", "0.0.0.0")
|
HOST = os.environ.get("HERMES_CODEX_BROKER_LISTEN_HOST", "0.0.0.0")
|
||||||
@ -580,13 +580,20 @@ class Handler(BaseHTTPRequestHandler):
|
|||||||
self._json(200, {"ok": True, "provider": "openai-codex"})
|
self._json(200, {"ok": True, "provider": "openai-codex"})
|
||||||
return
|
return
|
||||||
if self.path in {"/models", "/v1/models"}:
|
if self.path in {"/models", "/v1/models"}:
|
||||||
|
catalog = load_catalog()
|
||||||
|
providers = catalog.get("providers", {}) if isinstance(catalog, dict) else {}
|
||||||
|
codex = providers.get("codex", {}) if isinstance(providers, dict) else {}
|
||||||
|
discovered = codex.get("models", []) if isinstance(codex, dict) else []
|
||||||
|
models = {
|
||||||
|
model for model in discovered if isinstance(model, str) and model.startswith("gpt-")
|
||||||
|
} or FALLBACK_ALLOWED_MODELS
|
||||||
self._json(
|
self._json(
|
||||||
200,
|
200,
|
||||||
{
|
{
|
||||||
"object": "list",
|
"object": "list",
|
||||||
"data": [
|
"data": [
|
||||||
{"id": model, "object": "model", "owned_by": "openai-codex"}
|
{"id": model, "object": "model", "owned_by": "openai-codex"}
|
||||||
for model in sorted(FALLBACK_ALLOWED_MODELS)
|
for model in sorted(models)
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|||||||
@ -6,26 +6,29 @@ from __future__ import annotations
|
|||||||
import copy
|
import copy
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import re
|
|
||||||
import shutil
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
import time
|
import time
|
||||||
from dataclasses import dataclass
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Iterable
|
from typing import Any, Iterable
|
||||||
|
|
||||||
import yaml
|
import yaml
|
||||||
|
|
||||||
|
from provider_model_catalog import (
|
||||||
|
CAPABILITY_TIERS,
|
||||||
|
EFFORTS,
|
||||||
|
EFFORT_TIERS,
|
||||||
|
Catalog,
|
||||||
|
model_records,
|
||||||
|
model_version,
|
||||||
|
select_tier_model,
|
||||||
|
unique_models,
|
||||||
|
)
|
||||||
|
from provider_model_discovery import discover_claude_models, discover_codex_models
|
||||||
|
|
||||||
|
|
||||||
CODEX_BASELINE = "gpt-5.6-terra"
|
CODEX_BASELINE = "gpt-5.6-terra"
|
||||||
CLAUDE_BASELINE = "claude-opus-5"
|
CLAUDE_BASELINE = "claude-opus-5"
|
||||||
CLAUDE_SUBSCRIPTION_MODELS = (
|
|
||||||
"claude-haiku-4-5-20251001",
|
|
||||||
"claude-fable-5",
|
|
||||||
"claude-sonnet-5",
|
|
||||||
"claude-opus-5",
|
|
||||||
)
|
|
||||||
EFFORTS = ("low", "medium", "high", "xhigh")
|
|
||||||
ATLAS_FALLBACK = {
|
ATLAS_FALLBACK = {
|
||||||
"provider": "custom",
|
"provider": "custom",
|
||||||
"model": "qwen2.5:14b-instruct-q4_0",
|
"model": "qwen2.5:14b-instruct-q4_0",
|
||||||
@ -38,166 +41,46 @@ SWITCHYARD_PROVIDER = "atlas-switchyard"
|
|||||||
SWITCHYARD_API = "http://hermes-switchyard.hermes.svc.cluster.local:9005/v1"
|
SWITCHYARD_API = "http://hermes-switchyard.hermes.svc.cluster.local:9005/v1"
|
||||||
SWITCHYARD_AUTO_ROUTE = "atlas/auto/maximum"
|
SWITCHYARD_AUTO_ROUTE = "atlas/auto/maximum"
|
||||||
ROUTING_CATALOG_PATH = os.environ.get("HERMES_ROUTING_CATALOG_PATH", "").strip()
|
ROUTING_CATALOG_PATH = os.environ.get("HERMES_ROUTING_CATALOG_PATH", "").strip()
|
||||||
MANAGED_ENV_KEYS = {
|
MANAGED_ENV_KEYS = {"GIT_ASKPASS", "GIT_TERMINAL_PROMPT"}
|
||||||
"GIT_ASKPASS",
|
|
||||||
"GIT_TERMINAL_PROMPT",
|
|
||||||
}
|
|
||||||
RUNTIME_SECRET_ENV_KEYS = {
|
RUNTIME_SECRET_ENV_KEYS = {
|
||||||
"ANTHROPIC_API_KEY",
|
"ANTHROPIC_API_KEY", "API_SERVER_KEY", "CLAUDE_API_KEY", "CLAUDE_CODE_OAUTH_TOKEN",
|
||||||
"API_SERVER_KEY",
|
"GITEA_TOKEN", "GITEA_USERNAME", "HERMES_IMAGE_BROKER_KEY", "OPENAI_API_KEY",
|
||||||
"CLAUDE_API_KEY",
|
|
||||||
"CLAUDE_CODE_OAUTH_TOKEN",
|
|
||||||
"GITEA_TOKEN",
|
|
||||||
"GITEA_USERNAME",
|
|
||||||
"HERMES_IMAGE_BROKER_KEY",
|
|
||||||
"OPENAI_API_KEY",
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
def choose_codex_model(models: Iterable[str], current: str = CODEX_BASELINE, *, balanced: bool = False) -> str:
|
||||||
class Catalog:
|
"""Compatibility helper selecting a declared advanced or balanced Codex tier."""
|
||||||
"""Non-secret provider discovery result."""
|
effort = "medium" if balanced else "xhigh"
|
||||||
|
tier = "balanced" if balanced else "advanced"
|
||||||
provider: str
|
return select_tier_model("codex", models, {}, tier, effort, current)[0]
|
||||||
models: list[str]
|
|
||||||
live: bool
|
|
||||||
connected: bool
|
|
||||||
state: str
|
|
||||||
|
|
||||||
|
|
||||||
def _unique_models(models: Iterable[str]) -> list[str]:
|
def choose_codex_for_effort(models: Iterable[str], effort: str, current: str = CODEX_BASELINE) -> str:
|
||||||
"""Return normalized model IDs without changing provider order."""
|
"""Compatibility helper selecting an effort's generic Codex capability tier."""
|
||||||
seen: set[str] = set()
|
|
||||||
result: list[str] = []
|
|
||||||
for model in models:
|
|
||||||
value = str(model or "").strip()
|
|
||||||
if value and value.lower() not in seen:
|
|
||||||
result.append(value)
|
|
||||||
seen.add(value.lower())
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
def model_version(model: str) -> tuple[int, ...]:
|
|
||||||
"""Extract a sortable model version while ignoring dated aliases."""
|
|
||||||
value = re.sub(r"-\d{8}$", "", model.lower())
|
|
||||||
groups = re.findall(r"\d+(?:\.\d+)*", value)
|
|
||||||
parts: list[int] = []
|
|
||||||
for group in groups:
|
|
||||||
parts.extend(int(piece) for piece in group.split("."))
|
|
||||||
return tuple(parts) or (0,)
|
|
||||||
|
|
||||||
|
|
||||||
def _codex_quality(model: str) -> int:
|
|
||||||
"""Rank known Codex capability tiers for delegated implementation work."""
|
|
||||||
value = model.lower()
|
|
||||||
if "sol" in value:
|
|
||||||
return 60
|
|
||||||
if "terra" in value:
|
|
||||||
return 50
|
|
||||||
if "codex" in value and "spark" not in value:
|
|
||||||
return 45
|
|
||||||
if re.fullmatch(r"gpt-\d+(?:\.\d+)*", value):
|
|
||||||
return 40
|
|
||||||
if "luna" in value:
|
|
||||||
return 20
|
|
||||||
if "mini" in value or "nano" in value:
|
|
||||||
return 10
|
|
||||||
if "spark" in value:
|
|
||||||
return 5
|
|
||||||
return 0
|
|
||||||
|
|
||||||
|
|
||||||
def choose_codex_model(
|
|
||||||
models: Iterable[str], current: str = CODEX_BASELINE, *, balanced: bool = False
|
|
||||||
) -> str:
|
|
||||||
"""Choose a current full Codex model, retaining current when unknown."""
|
|
||||||
candidates = [m for m in _unique_models(models) if m.lower().startswith("gpt-")]
|
|
||||||
full = [m for m in candidates if _codex_quality(m) >= 40]
|
|
||||||
pool = full or candidates
|
|
||||||
if not pool:
|
|
||||||
return current
|
|
||||||
newest = max(model_version(model) for model in pool)
|
|
||||||
latest = [model for model in pool if model_version(model) == newest]
|
|
||||||
if balanced:
|
|
||||||
terra = [model for model in latest if "terra" in model.lower()]
|
|
||||||
if terra:
|
|
||||||
return max(terra, key=lambda model: (_codex_quality(model), model))
|
|
||||||
return max(latest, key=lambda model: (_codex_quality(model), model))
|
|
||||||
|
|
||||||
|
|
||||||
def _choose_by_hints(
|
|
||||||
models: Iterable[str], hints: tuple[str, ...], current: str, prefix: str
|
|
||||||
) -> str:
|
|
||||||
"""Pick the newest model in the first available capability class."""
|
|
||||||
candidates = [
|
|
||||||
model
|
|
||||||
for model in _unique_models(models)
|
|
||||||
if model.lower().startswith(prefix)
|
|
||||||
]
|
|
||||||
if not candidates:
|
|
||||||
return current
|
|
||||||
for hint in hints:
|
|
||||||
tier = [model for model in candidates if hint in model.lower()]
|
|
||||||
if tier:
|
|
||||||
return max(tier, key=lambda model: (model_version(model), model))
|
|
||||||
return max(candidates, key=lambda model: (model_version(model), model))
|
|
||||||
|
|
||||||
|
|
||||||
def choose_codex_for_effort(
|
|
||||||
models: Iterable[str], effort: str, current: str = CODEX_BASELINE
|
|
||||||
) -> str:
|
|
||||||
"""Choose the account-visible Codex tier for an allowed effort level."""
|
|
||||||
hints = {
|
|
||||||
"low": ("luna", "mini", "spark", "terra", "sol", "codex"),
|
|
||||||
"medium": ("terra", "codex", "sol", "luna", "mini", "spark"),
|
|
||||||
"high": ("sol", "codex", "terra", "luna", "mini", "spark"),
|
|
||||||
"xhigh": ("sol", "codex", "terra", "luna", "mini", "spark"),
|
|
||||||
}
|
|
||||||
if effort not in EFFORTS:
|
if effort not in EFFORTS:
|
||||||
raise ValueError(f"unsupported effort: {effort}")
|
raise ValueError(f"unsupported effort: {effort}")
|
||||||
return _choose_by_hints(models, hints[effort], current, "gpt-")
|
return select_tier_model("codex", models, {}, EFFORT_TIERS[effort], effort, current)[0]
|
||||||
|
|
||||||
|
|
||||||
def _claude_quality(model: str) -> int:
|
|
||||||
"""Rank known Claude capability tiers for architecture and review work."""
|
|
||||||
value = model.lower()
|
|
||||||
if "opus" in value:
|
|
||||||
return 40
|
|
||||||
if "sonnet" in value:
|
|
||||||
return 30
|
|
||||||
if "fable" in value:
|
|
||||||
return 20
|
|
||||||
if "haiku" in value:
|
|
||||||
return 10
|
|
||||||
return 0
|
|
||||||
|
|
||||||
|
|
||||||
def choose_claude_model(models: Iterable[str], current: str = CLAUDE_BASELINE) -> str:
|
def choose_claude_model(models: Iterable[str], current: str = CLAUDE_BASELINE) -> str:
|
||||||
"""Choose the newest full Claude reasoning model visible to the account."""
|
"""Compatibility helper selecting the declared advanced Claude tier."""
|
||||||
candidates = [m for m in _unique_models(models) if m.lower().startswith("claude-")]
|
return select_tier_model("claude", models, {}, "advanced", "xhigh", current)[0]
|
||||||
full = [m for m in candidates if _claude_quality(m) >= 30]
|
|
||||||
pool = full or candidates
|
|
||||||
if not pool:
|
|
||||||
return current
|
|
||||||
return max(
|
|
||||||
pool,
|
|
||||||
key=lambda model: (model_version(model), _claude_quality(model), model),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def choose_claude_for_effort(
|
def choose_claude_for_effort(
|
||||||
models: Iterable[str], effort: str, current: str = CLAUDE_BASELINE
|
models: Iterable[str], effort: str, current: str = CLAUDE_BASELINE
|
||||||
) -> str:
|
) -> str:
|
||||||
"""Choose the account-visible Claude tier for an allowed effort level."""
|
"""Compatibility helper selecting an effort's generic Claude capability tier."""
|
||||||
hints = {
|
|
||||||
"low": ("haiku", "fable", "sonnet", "opus"),
|
|
||||||
"medium": ("sonnet", "fable", "opus", "haiku"),
|
|
||||||
"high": ("opus", "sonnet", "fable", "haiku"),
|
|
||||||
"xhigh": ("opus", "sonnet", "fable", "haiku"),
|
|
||||||
}
|
|
||||||
if effort not in EFFORTS:
|
if effort not in EFFORTS:
|
||||||
raise ValueError(f"unsupported effort: {effort}")
|
raise ValueError(f"unsupported effort: {effort}")
|
||||||
return _choose_by_hints(models, hints[effort], current, "claude-")
|
return select_tier_model("claude", models, {}, EFFORT_TIERS[effort], effort, current)[0]
|
||||||
|
|
||||||
|
|
||||||
|
# The coordinator retains these private spellings for compact call sites while
|
||||||
|
# provider_model_catalog owns the metadata policy and its independent tests.
|
||||||
|
_unique_models = unique_models
|
||||||
|
_model_records = model_records
|
||||||
|
_select_tier_model = select_tier_model
|
||||||
|
|
||||||
|
|
||||||
def _read_yaml(path: Path) -> dict[str, Any]:
|
def _read_yaml(path: Path) -> dict[str, Any]:
|
||||||
@ -245,34 +128,42 @@ def _read_json(path: Path) -> dict[str, Any]:
|
|||||||
|
|
||||||
def _previous_provider_models(
|
def _previous_provider_models(
|
||||||
previous: dict[str, Any], provider: str
|
previous: dict[str, Any], provider: str
|
||||||
) -> tuple[dict[str, str], dict[str, str], list[str]]:
|
) -> tuple[dict[str, str], dict[str, str], list[str], dict[str, dict[str, Any]]]:
|
||||||
"""Return last-known-good effort, tier, and catalog values for a provider."""
|
"""Return last-known-good effort, selector, models, and metadata values."""
|
||||||
providers = previous.get("providers", {})
|
providers = previous.get("providers", {})
|
||||||
record = providers.get(provider, {}) if isinstance(providers, dict) else {}
|
record = providers.get(provider, {}) if isinstance(providers, dict) else {}
|
||||||
if not isinstance(record, dict):
|
if not isinstance(record, dict):
|
||||||
return {}, {}, []
|
return {}, {}, [], {}
|
||||||
resolved = record.get("resolved", {})
|
resolved = record.get("resolved", {})
|
||||||
tiers = record.get("tiers", {})
|
tiers = record.get("tiers", {})
|
||||||
models = record.get("models", [])
|
models = record.get("models", [])
|
||||||
|
metadata = record.get("model_metadata", {})
|
||||||
return (
|
return (
|
||||||
dict(resolved) if isinstance(resolved, dict) else {},
|
dict(resolved) if isinstance(resolved, dict) else {},
|
||||||
dict(tiers) if isinstance(tiers, dict) else {},
|
dict(tiers) if isinstance(tiers, dict) else {},
|
||||||
_unique_models(models if isinstance(models, list) else []),
|
_unique_models(models if isinstance(models, list) else []),
|
||||||
|
{
|
||||||
|
str(model): dict(details)
|
||||||
|
for model, details in metadata.items()
|
||||||
|
if isinstance(model, str) and isinstance(details, dict)
|
||||||
|
}
|
||||||
|
if isinstance(metadata, dict)
|
||||||
|
else {},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def build_routing_catalog(
|
def build_routing_catalog(
|
||||||
codex: Catalog, claude: Catalog, previous: dict[str, Any] | None = None
|
codex: Catalog, claude: Catalog, previous: dict[str, Any] | None = None
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
"""Build a current catalog while retaining working routes during outages."""
|
"""Build a current catalog while retaining only outage-safe known routes."""
|
||||||
previous = previous or {}
|
previous = previous or {}
|
||||||
|
checked_at = int(time.time())
|
||||||
providers: dict[str, Any] = {}
|
providers: dict[str, Any] = {}
|
||||||
specifications = (
|
specifications = (
|
||||||
(
|
(
|
||||||
"codex",
|
"codex",
|
||||||
codex,
|
codex,
|
||||||
choose_codex_for_effort,
|
{"luna": "economy", "terra": "balanced", "sol": "advanced"},
|
||||||
{"luna": "low", "terra": "medium", "sol": "high"},
|
|
||||||
{
|
{
|
||||||
"low": "gpt-5.6-luna",
|
"low": "gpt-5.6-luna",
|
||||||
"medium": "gpt-5.6-terra",
|
"medium": "gpt-5.6-terra",
|
||||||
@ -283,12 +174,11 @@ def build_routing_catalog(
|
|||||||
(
|
(
|
||||||
"claude",
|
"claude",
|
||||||
claude,
|
claude,
|
||||||
choose_claude_for_effort,
|
|
||||||
{
|
{
|
||||||
"haiku": "low",
|
"haiku": "economy",
|
||||||
"fable": "medium",
|
"fable": "advanced",
|
||||||
"sonnet": "high",
|
"sonnet": "balanced",
|
||||||
"opus": "xhigh",
|
"opus": "advanced",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"low": "claude-haiku-4-5-20251001",
|
"low": "claude-haiku-4-5-20251001",
|
||||||
@ -298,27 +188,64 @@ def build_routing_catalog(
|
|||||||
},
|
},
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
for name, discovered, chooser, tier_efforts, defaults in specifications:
|
for name, discovered, legacy_selectors, defaults in specifications:
|
||||||
old_resolved, old_tiers, old_models = _previous_provider_models(previous, name)
|
old_resolved, old_tiers, old_models, old_metadata = _previous_provider_models(
|
||||||
|
previous, name
|
||||||
|
)
|
||||||
|
old_providers = previous.get("providers", {})
|
||||||
|
old_record = old_providers.get(name, {}) if isinstance(old_providers, dict) else {}
|
||||||
|
previous_success = (
|
||||||
|
int(old_record.get("last_success_at"))
|
||||||
|
if isinstance(old_record, dict) and isinstance(old_record.get("last_success_at"), int)
|
||||||
|
else None
|
||||||
|
)
|
||||||
source_models = discovered.models if discovered.live else old_models
|
source_models = discovered.models if discovered.live else old_models
|
||||||
|
source_metadata = discovered.metadata if discovered.live else old_metadata
|
||||||
resolved: dict[str, str] = {}
|
resolved: dict[str, str] = {}
|
||||||
|
observations: dict[str, Any] = {}
|
||||||
for effort in EFFORTS:
|
for effort in EFFORTS:
|
||||||
|
# A successful provider list is authoritative: a retired model
|
||||||
|
# must not survive it merely because it was last known-good.
|
||||||
current = str(old_resolved.get(effort) or defaults[effort])
|
current = str(old_resolved.get(effort) or defaults[effort])
|
||||||
resolved[effort] = (
|
selected, observed = _select_tier_model(
|
||||||
chooser(source_models, effort, current)
|
name,
|
||||||
if source_models
|
source_models,
|
||||||
else current
|
source_metadata,
|
||||||
|
EFFORT_TIERS[effort],
|
||||||
|
effort,
|
||||||
|
current,
|
||||||
|
allow_current_fallback=not discovered.live,
|
||||||
)
|
)
|
||||||
|
resolved[effort] = selected
|
||||||
|
observations.update(observed)
|
||||||
tiers: dict[str, str] = {}
|
tiers: dict[str, str] = {}
|
||||||
for tier, effort in tier_efforts.items():
|
for capability in CAPABILITY_TIERS:
|
||||||
current = str(old_tiers.get(tier) or defaults[effort])
|
effort = next(
|
||||||
tier_matches = [model for model in source_models if tier in model.lower()]
|
value for value, target in EFFORT_TIERS.items() if target == capability
|
||||||
tiers[tier] = (
|
)
|
||||||
max(tier_matches, key=lambda model: (model_version(model), model))
|
current = str(old_tiers.get(capability) or "")
|
||||||
if tier_matches
|
tiers[capability], observed = _select_tier_model(
|
||||||
else (resolved[effort] if discovered.live else current)
|
name, source_models, source_metadata, capability, effort, current,
|
||||||
|
allow_current_fallback=not discovered.live,
|
||||||
|
)
|
||||||
|
observations.update(observed)
|
||||||
|
for selector, capability in legacy_selectors.items():
|
||||||
|
# Explicit historical family picks are not generic capability
|
||||||
|
# requests. They must remain exact or become unavailable, never
|
||||||
|
# silently change to a newer family such as Astra.
|
||||||
|
exact = [model for model in source_models if selector in model.lower()]
|
||||||
|
tiers[selector] = (
|
||||||
|
exact[0]
|
||||||
|
if exact
|
||||||
|
else ("" if discovered.live else str(old_tiers.get(selector) or ""))
|
||||||
)
|
)
|
||||||
providers[name] = {
|
providers[name] = {
|
||||||
|
"provenance": (
|
||||||
|
"live-account" if discovered.live
|
||||||
|
else ("last-known-good" if old_models else "bootstrap-fallback")
|
||||||
|
),
|
||||||
|
"checked_at": checked_at,
|
||||||
|
"last_success_at": checked_at if discovered.live else previous_success,
|
||||||
"state": discovered.state,
|
"state": discovered.state,
|
||||||
"connected": discovered.connected,
|
"connected": discovered.connected,
|
||||||
"live": discovered.live,
|
"live": discovered.live,
|
||||||
@ -327,12 +254,14 @@ def build_routing_catalog(
|
|||||||
if discovered.live
|
if discovered.live
|
||||||
else (old_models or discovered.models)
|
else (old_models or discovered.models)
|
||||||
),
|
),
|
||||||
|
"model_metadata": source_metadata,
|
||||||
|
"candidates": observations,
|
||||||
"resolved": resolved,
|
"resolved": resolved,
|
||||||
"tiers": tiers,
|
"tiers": tiers,
|
||||||
}
|
}
|
||||||
return {
|
return {
|
||||||
"schema_version": 1,
|
"schema_version": 2,
|
||||||
"updated_at": int(time.time()),
|
"updated_at": checked_at,
|
||||||
"providers": providers,
|
"providers": providers,
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -385,19 +314,6 @@ def _update_profile_env(path: Path, source: dict[str, str]) -> None:
|
|||||||
_atomic_write(path, "\n".join(kept).rstrip() + "\n", 0o600)
|
_atomic_write(path, "\n".join(kept).rstrip() + "\n", 0o600)
|
||||||
|
|
||||||
|
|
||||||
def _existing_model(config: dict[str, Any], provider: str, default: str) -> str:
|
|
||||||
"""Return the last configured model for a provider or a safe baseline."""
|
|
||||||
model_cfg = config.get("model", {})
|
|
||||||
if isinstance(model_cfg, dict) and model_cfg.get("provider") == provider:
|
|
||||||
return str(model_cfg.get("model") or model_cfg.get("default") or default)
|
|
||||||
fallbacks = config.get("fallback_providers", [])
|
|
||||||
if isinstance(fallbacks, list):
|
|
||||||
for fallback in fallbacks:
|
|
||||||
if isinstance(fallback, dict) and fallback.get("provider") == provider:
|
|
||||||
return str(fallback.get("model") or default)
|
|
||||||
return default
|
|
||||||
|
|
||||||
|
|
||||||
def codex_cli_authenticated() -> bool:
|
def codex_cli_authenticated() -> bool:
|
||||||
"""Return whether the installed Codex CLI has a usable local login."""
|
"""Return whether the installed Codex CLI has a usable local login."""
|
||||||
codex = shutil.which("codex")
|
codex = shutil.which("codex")
|
||||||
@ -420,127 +336,6 @@ def codex_cli_authenticated() -> bool:
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
def discover_codex_models() -> Catalog:
|
|
||||||
"""Use the authenticated Codex endpoint; catalogs are status-only fallback."""
|
|
||||||
token = ""
|
|
||||||
try:
|
|
||||||
from hermes_cli.auth import resolve_codex_runtime_credentials
|
|
||||||
|
|
||||||
credentials = resolve_codex_runtime_credentials(refresh_if_expiring=True) or {}
|
|
||||||
token = str(credentials.get("api_key") or "").strip()
|
|
||||||
except Exception:
|
|
||||||
token = ""
|
|
||||||
live: list[str] = []
|
|
||||||
if token:
|
|
||||||
try:
|
|
||||||
from hermes_cli.codex_models import _fetch_models_from_api
|
|
||||||
|
|
||||||
live = _unique_models(_fetch_models_from_api(token))
|
|
||||||
except Exception:
|
|
||||||
live = []
|
|
||||||
if live:
|
|
||||||
return Catalog("openai-codex", live, True, True, "connected")
|
|
||||||
try:
|
|
||||||
from hermes_cli.models import provider_model_ids
|
|
||||||
|
|
||||||
known = _unique_models(provider_model_ids("openai-codex", force_refresh=True))
|
|
||||||
except Exception:
|
|
||||||
known = []
|
|
||||||
# The owner agent deliberately uses the local Codex app-server and the
|
|
||||||
# Codex CLI's ChatGPT login. That credential is not exported into Hermes'
|
|
||||||
# bearer-token store, so API discovery above may be empty even though the
|
|
||||||
# runtime is fully authenticated. Treat a successful CLI status check as a
|
|
||||||
# connected, degraded catalog: routing may use Codex while retaining the
|
|
||||||
# last-known working model names until app-server discovery is available.
|
|
||||||
if codex_cli_authenticated():
|
|
||||||
return Catalog(
|
|
||||||
"openai-codex",
|
|
||||||
known,
|
|
||||||
False,
|
|
||||||
True,
|
|
||||||
"connected-cli",
|
|
||||||
)
|
|
||||||
state = "degraded" if token else "not-configured"
|
|
||||||
return Catalog("openai-codex", known, False, bool(token), state)
|
|
||||||
|
|
||||||
|
|
||||||
def discover_claude_models() -> Catalog:
|
|
||||||
"""Use the native subscription login before any API-key catalog fallback."""
|
|
||||||
claude = shutil.which("claude") or os.environ.get("HERMES_CLAUDE_BIN", "")
|
|
||||||
if claude:
|
|
||||||
try:
|
|
||||||
environment = os.environ.copy()
|
|
||||||
environment.pop("ANTHROPIC_API_KEY", None)
|
|
||||||
environment.pop("CLAUDE_API_KEY", None)
|
|
||||||
status = subprocess.run(
|
|
||||||
[claude, "auth", "status"],
|
|
||||||
capture_output=True,
|
|
||||||
check=False,
|
|
||||||
text=True,
|
|
||||||
timeout=15,
|
|
||||||
env=environment,
|
|
||||||
)
|
|
||||||
detail = json.loads(status.stdout) if status.stdout.strip() else {}
|
|
||||||
if (
|
|
||||||
status.returncode == 0
|
|
||||||
and isinstance(detail, dict)
|
|
||||||
and detail.get("loggedIn") is True
|
|
||||||
and detail.get("apiProvider") == "firstParty"
|
|
||||||
):
|
|
||||||
try:
|
|
||||||
from hermes_cli.models import provider_model_ids
|
|
||||||
|
|
||||||
known = _unique_models(
|
|
||||||
provider_model_ids("anthropic", force_refresh=True)
|
|
||||||
)
|
|
||||||
except Exception:
|
|
||||||
known = []
|
|
||||||
return Catalog(
|
|
||||||
"anthropic",
|
|
||||||
_unique_models((*CLAUDE_SUBSCRIPTION_MODELS, *known)),
|
|
||||||
True,
|
|
||||||
True,
|
|
||||||
"connected-subscription",
|
|
||||||
)
|
|
||||||
except (OSError, subprocess.SubprocessError, ValueError, json.JSONDecodeError):
|
|
||||||
pass
|
|
||||||
try:
|
|
||||||
from hermes_cli.models import provider_model_ids
|
|
||||||
|
|
||||||
known = _unique_models(provider_model_ids("anthropic", force_refresh=True))
|
|
||||||
except Exception:
|
|
||||||
known = []
|
|
||||||
return Catalog("anthropic", known, False, False, "not-configured")
|
|
||||||
|
|
||||||
|
|
||||||
def _profile_config(
|
|
||||||
base: dict[str, Any],
|
|
||||||
primary_provider: str,
|
|
||||||
primary_model: str,
|
|
||||||
fallback: dict[str, str],
|
|
||||||
effort: str,
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
"""Derive a worker configuration with an explicit cross-provider fallback."""
|
|
||||||
config = copy.deepcopy(base)
|
|
||||||
config["model"] = {
|
|
||||||
"provider": primary_provider,
|
|
||||||
"default": primary_model,
|
|
||||||
"model": primary_model,
|
|
||||||
"openai_runtime": "codex_app_server",
|
|
||||||
}
|
|
||||||
config["fallback_providers"] = [fallback]
|
|
||||||
# Local OpenAI-compatible runtimes top out at the equivalent of high.
|
|
||||||
# Never silently downgrade an explicitly xhigh task after both hosted
|
|
||||||
# provider lanes are exhausted.
|
|
||||||
if effort != "xhigh":
|
|
||||||
config["fallback_providers"].append(copy.deepcopy(ATLAS_FALLBACK))
|
|
||||||
agent = config.setdefault("agent", {})
|
|
||||||
if isinstance(agent, dict):
|
|
||||||
agent["reasoning_effort"] = effort
|
|
||||||
config["toolsets"] = []
|
|
||||||
return config
|
|
||||||
|
|
||||||
|
|
||||||
def _switchyard_profile_config(
|
def _switchyard_profile_config(
|
||||||
base: dict[str, Any], route: str, effort: str
|
base: dict[str, Any], route: str, effort: str
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
@ -626,18 +421,8 @@ def configure_routes(
|
|||||||
for effort in EFFORTS:
|
for effort in EFFORTS:
|
||||||
codex_name = f"codex-{effort}"
|
codex_name = f"codex-{effort}"
|
||||||
claude_name = f"claude-{effort}"
|
claude_name = f"claude-{effort}"
|
||||||
codex_route = {
|
codex_route = f"atlas/manual/codex/auto/{effort}"
|
||||||
"low": "atlas/manual/codex/luna",
|
claude_route = f"atlas/manual/claude/auto/{effort}"
|
||||||
"medium": "atlas/manual/codex/terra",
|
|
||||||
"high": "atlas/manual/codex/sol",
|
|
||||||
"xhigh": "atlas/manual/codex/sol",
|
|
||||||
}[effort]
|
|
||||||
claude_route = {
|
|
||||||
"low": "atlas/manual/claude/haiku",
|
|
||||||
"medium": "atlas/manual/claude/sonnet",
|
|
||||||
"high": "atlas/manual/claude/sonnet",
|
|
||||||
"xhigh": "atlas/manual/claude/opus",
|
|
||||||
}[effort]
|
|
||||||
_write_profile(
|
_write_profile(
|
||||||
root,
|
root,
|
||||||
codex_name,
|
codex_name,
|
||||||
|
|||||||
302
services/hermes/scripts/provider_model_catalog.py
Normal file
302
services/hermes/scripts/provider_model_catalog.py
Normal file
@ -0,0 +1,302 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Normalize provider model metadata into safe Hermes routing candidates."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import re
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from typing import Any, Iterable
|
||||||
|
|
||||||
|
|
||||||
|
EFFORTS = ("low", "medium", "high", "xhigh")
|
||||||
|
CAPABILITY_TIERS = ("economy", "balanced", "advanced")
|
||||||
|
EFFORT_TIERS = {
|
||||||
|
"low": "economy",
|
||||||
|
"medium": "balanced",
|
||||||
|
"high": "advanced",
|
||||||
|
"xhigh": "advanced",
|
||||||
|
}
|
||||||
|
SPECIALIST_MODEL_WORDS = frozenset(
|
||||||
|
{"audio", "embedding", "image", "moderation", "realtime", "speech", "transcrib", "tts"}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Catalog:
|
||||||
|
"""Non-secret provider discovery result."""
|
||||||
|
|
||||||
|
provider: str
|
||||||
|
models: list[str]
|
||||||
|
live: bool
|
||||||
|
connected: bool
|
||||||
|
state: str
|
||||||
|
metadata: dict[str, dict[str, Any]] = field(default_factory=dict)
|
||||||
|
|
||||||
|
|
||||||
|
def unique_models(models: Iterable[str]) -> list[str]:
|
||||||
|
"""Return normalized model IDs without changing provider order."""
|
||||||
|
seen: set[str] = set()
|
||||||
|
result: list[str] = []
|
||||||
|
for model in models:
|
||||||
|
value = str(model or "").strip()
|
||||||
|
if value and value.lower() not in seen:
|
||||||
|
result.append(value)
|
||||||
|
seen.add(value.lower())
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def model_id(value: Any) -> str:
|
||||||
|
"""Extract a provider model ID from either a string or model-list record."""
|
||||||
|
if isinstance(value, str):
|
||||||
|
return value.strip()
|
||||||
|
if not isinstance(value, dict):
|
||||||
|
return ""
|
||||||
|
for key in ("id", "model", "model_id", "name", "value"):
|
||||||
|
candidate = value.get(key)
|
||||||
|
if isinstance(candidate, str) and candidate.strip():
|
||||||
|
return candidate.strip()
|
||||||
|
return ""
|
||||||
|
|
||||||
|
|
||||||
|
def public_metadata(value: Any) -> dict[str, Any]:
|
||||||
|
"""Keep provider routing fields while excluding unrelated response data."""
|
||||||
|
if not isinstance(value, dict):
|
||||||
|
return {}
|
||||||
|
allowed = {
|
||||||
|
"capabilities", "capability", "capability_tier", "cost", "cost_rank",
|
||||||
|
"created_at", "default_reasoning_effort", "description", "display_name", "displayName",
|
||||||
|
"disabled", "enabled", "hidden", "input_modalities", "inputModalities", "isDefault",
|
||||||
|
"is_default", "is_general_purpose", "isGeneralPurpose", "max_input_tokens", "max_tokens",
|
||||||
|
"output_modalities", "outputModalities", "pricing", "status",
|
||||||
|
"modelSpecialty", "resolvedModel", "supported_reasoning_efforts", "supportedReasoningEfforts", "supportedEffortLevels",
|
||||||
|
"supportsAdaptiveThinking", "supportsEffort", "tier", "upgrade",
|
||||||
|
"verified", "visibility",
|
||||||
|
}
|
||||||
|
return {key: value[key] for key in allowed if key in value}
|
||||||
|
|
||||||
|
|
||||||
|
def model_records(values: Iterable[Any]) -> tuple[list[str], dict[str, dict[str, Any]]]:
|
||||||
|
"""Normalize provider records into ordered IDs and safe per-model metadata."""
|
||||||
|
models: list[str] = []
|
||||||
|
metadata: dict[str, dict[str, Any]] = {}
|
||||||
|
for value in values:
|
||||||
|
model = model_id(value)
|
||||||
|
if not model:
|
||||||
|
continue
|
||||||
|
models.append(model)
|
||||||
|
if isinstance(value, dict):
|
||||||
|
metadata[model] = public_metadata(value)
|
||||||
|
unique = unique_models(models)
|
||||||
|
return unique, {model: metadata[model] for model in unique if model in metadata}
|
||||||
|
|
||||||
|
|
||||||
|
def merge_records(
|
||||||
|
*records: tuple[list[str], dict[str, dict[str, Any]]]
|
||||||
|
) -> tuple[list[str], dict[str, dict[str, Any]]]:
|
||||||
|
"""Merge ordered discovery sources, preferring metadata from later sources."""
|
||||||
|
models: list[str] = []
|
||||||
|
metadata: dict[str, dict[str, Any]] = {}
|
||||||
|
for source_models, source_metadata in records:
|
||||||
|
for model in source_models:
|
||||||
|
if model not in models:
|
||||||
|
models.append(model)
|
||||||
|
details = source_metadata.get(model)
|
||||||
|
if details:
|
||||||
|
metadata[model] = {**metadata.get(model, {}), **details}
|
||||||
|
return unique_models(models), metadata
|
||||||
|
|
||||||
|
|
||||||
|
def model_version(model: str) -> tuple[int, ...]:
|
||||||
|
"""Extract a sortable model version while ignoring dated aliases."""
|
||||||
|
value = re.sub(r"-\d{8}$", "", model.lower())
|
||||||
|
groups = re.findall(r"\d+(?:\.\d+)*", value)
|
||||||
|
parts: list[int] = []
|
||||||
|
for group in groups:
|
||||||
|
parts.extend(int(piece) for piece in group.split("."))
|
||||||
|
return tuple(parts) or (0,)
|
||||||
|
|
||||||
|
|
||||||
|
def _text_metadata(metadata: dict[str, Any]) -> str:
|
||||||
|
values: list[str] = []
|
||||||
|
for key in ("description", "display_name", "capability", "tier"):
|
||||||
|
value = metadata.get(key)
|
||||||
|
if isinstance(value, str):
|
||||||
|
values.append(value.lower())
|
||||||
|
capabilities = metadata.get("capabilities")
|
||||||
|
if isinstance(capabilities, list):
|
||||||
|
values.extend(str(value).lower() for value in capabilities)
|
||||||
|
return " ".join(values)
|
||||||
|
|
||||||
|
|
||||||
|
def metadata_tier(metadata: dict[str, Any]) -> tuple[str | None, str]:
|
||||||
|
"""Classify a model only from provider-declared capability information."""
|
||||||
|
for key in ("capability_tier", "tier", "capability"):
|
||||||
|
value = metadata.get(key)
|
||||||
|
if isinstance(value, str):
|
||||||
|
normalized = value.strip().lower()
|
||||||
|
if normalized in {"economy", "cheap", "fast", "low_cost"}:
|
||||||
|
return "economy", f"provider {key}={value!r}"
|
||||||
|
if normalized in {"balanced", "standard", "mid", "medium"}:
|
||||||
|
return "balanced", f"provider {key}={value!r}"
|
||||||
|
if normalized in {"advanced", "strongest", "premium", "high"}:
|
||||||
|
return "advanced", f"provider {key}={value!r}"
|
||||||
|
text = _text_metadata(metadata)
|
||||||
|
if any(term in text for term in ("strongest", "most capable", "most intelligent", "complex tasks")):
|
||||||
|
return "advanced", "provider description marks it strongest/complex"
|
||||||
|
if any(term in text for term in ("balanced", "general purpose", "everyday tasks", "routine tasks")):
|
||||||
|
return "balanced", "provider description marks it balanced/general-purpose"
|
||||||
|
if any(term in text for term in ("lowest cost", "low cost", "cheapest", "fastest", "fast and affordable")):
|
||||||
|
return "economy", "provider description marks it low-cost/fast"
|
||||||
|
return None, "provider metadata does not declare a capability tier"
|
||||||
|
|
||||||
|
|
||||||
|
def legacy_tier(provider: str, model: str) -> tuple[str | None, str]:
|
||||||
|
"""Migrate deployed pre-metadata families without predicting new IDs."""
|
||||||
|
known = {
|
||||||
|
"codex": {"luna": "economy", "terra": "balanced", "sol": "advanced"},
|
||||||
|
"claude": {
|
||||||
|
"haiku": "economy", "sonnet": "balanced", "opus": "advanced",
|
||||||
|
"fable": "advanced",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for marker, tier in known[provider].items():
|
||||||
|
if marker in model.lower():
|
||||||
|
return tier, "legacy deployed-family compatibility"
|
||||||
|
return None, "no provider capability metadata"
|
||||||
|
|
||||||
|
|
||||||
|
def _model_cost(metadata: dict[str, Any]) -> tuple[str, float] | None:
|
||||||
|
"""Return a cost only with its provider-declared comparable unit."""
|
||||||
|
for key in ("cost_rank", "cost"):
|
||||||
|
rank = metadata.get(key)
|
||||||
|
if not isinstance(rank, bool) and isinstance(rank, (int, float)) and rank >= 0:
|
||||||
|
return "ordinal", float(rank)
|
||||||
|
pricing = metadata.get("pricing")
|
||||||
|
sources = [pricing, metadata] if isinstance(pricing, dict) else [metadata]
|
||||||
|
for source in sources:
|
||||||
|
input_cost = source.get("input_cost_per_million")
|
||||||
|
output_cost = source.get("output_cost_per_million")
|
||||||
|
if (
|
||||||
|
not isinstance(input_cost, bool)
|
||||||
|
and not isinstance(output_cost, bool)
|
||||||
|
and isinstance(input_cost, (int, float))
|
||||||
|
and isinstance(output_cost, (int, float))
|
||||||
|
and input_cost >= 0
|
||||||
|
and output_cost >= 0
|
||||||
|
):
|
||||||
|
return "per-million", float(input_cost) + float(output_cost)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _comparable_costs(
|
||||||
|
pool: Iterable[tuple[str, dict[str, Any], str]]
|
||||||
|
) -> dict[str, float]:
|
||||||
|
"""Use cost only when every adequate candidate advertises one shared unit."""
|
||||||
|
records = [(model, _model_cost(details)) for model, details, _ in pool]
|
||||||
|
units = {cost[0] for _, cost in records if cost is not None}
|
||||||
|
if len(units) != 1 or any(cost is None for _, cost in records):
|
||||||
|
return {}
|
||||||
|
return {model: cost[1] for model, cost in records if cost is not None}
|
||||||
|
|
||||||
|
def _candidate_status(
|
||||||
|
provider: str, model: str, metadata: dict[str, Any], effort: str
|
||||||
|
) -> tuple[bool, str]:
|
||||||
|
if metadata.get("hidden") is True:
|
||||||
|
return False, "provider marks model hidden"
|
||||||
|
if str(metadata.get("visibility") or "").lower() in {"hidden", "internal", "private"}:
|
||||||
|
return False, "provider marks model non-public"
|
||||||
|
if metadata.get("enabled") is False or metadata.get("disabled") is True:
|
||||||
|
return False, "provider marks model disabled"
|
||||||
|
if str(metadata.get("status") or "").lower() in {"deprecated", "disabled", "retired"}:
|
||||||
|
return False, "provider marks model unavailable"
|
||||||
|
if metadata.get("is_general_purpose") is False or metadata.get("isGeneralPurpose") is False:
|
||||||
|
return False, "provider marks model specialist"
|
||||||
|
if metadata.get("modelSpecialty") not in (None, "", False):
|
||||||
|
return False, "provider marks model specialist"
|
||||||
|
modalities = metadata.get("input_modalities", metadata.get("inputModalities"))
|
||||||
|
if isinstance(modalities, list) and modalities and "text" not in {str(item).lower() for item in modalities}:
|
||||||
|
return False, "provider record is not text-capable"
|
||||||
|
if "specialist" in _text_metadata(metadata):
|
||||||
|
return False, "provider description identifies a specialist model"
|
||||||
|
if any(word in model.lower() for word in SPECIALIST_MODEL_WORDS):
|
||||||
|
return False, "model ID identifies a specialist model"
|
||||||
|
if metadata.get("supportsEffort") is False:
|
||||||
|
return False, "provider does not advertise effort support"
|
||||||
|
supported = metadata.get(
|
||||||
|
"supported_reasoning_efforts",
|
||||||
|
metadata.get("supportedReasoningEfforts", metadata.get("supportedEffortLevels")),
|
||||||
|
)
|
||||||
|
supported_efforts = {
|
||||||
|
str(item.get("reasoningEffort") or item.get("effort") or "").lower()
|
||||||
|
if isinstance(item, dict)
|
||||||
|
else str(item).lower()
|
||||||
|
for item in supported
|
||||||
|
} if isinstance(supported, list) else set()
|
||||||
|
if supported_efforts and effort not in supported_efforts:
|
||||||
|
return False, f"provider does not support {effort} effort"
|
||||||
|
if provider == "codex" and not model.lower().startswith("gpt-"):
|
||||||
|
return False, "model ID is outside the Codex general route namespace"
|
||||||
|
return True, "eligible"
|
||||||
|
|
||||||
|
|
||||||
|
def select_tier_model(
|
||||||
|
provider: str, models: Iterable[str], metadata: dict[str, dict[str, Any]],
|
||||||
|
desired: str, effort: str, current: str, *, allow_current_fallback: bool = True,
|
||||||
|
) -> tuple[str, dict[str, Any]]:
|
||||||
|
"""Select by declared tier and cost, retaining last-known-good only on outage."""
|
||||||
|
candidates: list[tuple[str, dict[str, Any], str]] = []
|
||||||
|
observations: dict[str, Any] = {}
|
||||||
|
for model in unique_models(models):
|
||||||
|
details = metadata.get(model, {})
|
||||||
|
eligible, reason = _candidate_status(provider, model, details, effort)
|
||||||
|
tier, tier_reason = metadata_tier(details)
|
||||||
|
if tier is None:
|
||||||
|
tier, legacy_reason = legacy_tier(provider, model)
|
||||||
|
tier_reason = legacy_reason if tier else tier_reason
|
||||||
|
cost = _model_cost(details)
|
||||||
|
observations[model] = {
|
||||||
|
"eligible": eligible,
|
||||||
|
"reason": reason if not eligible else tier_reason,
|
||||||
|
"tier": tier,
|
||||||
|
"cost": {"unit": cost[0], "value": cost[1]} if cost else None,
|
||||||
|
}
|
||||||
|
if eligible and tier:
|
||||||
|
candidates.append((model, details, tier))
|
||||||
|
exact = [item for item in candidates if item[2] == desired]
|
||||||
|
tier_order = {"economy": 0, "balanced": 1, "advanced": 2}
|
||||||
|
# When an economy model cannot honor the requested effort, use the nearest
|
||||||
|
# higher declared tier. A difficult request never falls to a lower tier.
|
||||||
|
higher = [
|
||||||
|
item for item in candidates
|
||||||
|
if tier_order[item[2]] > tier_order[desired]
|
||||||
|
]
|
||||||
|
nearest_higher = min(
|
||||||
|
(tier_order[item[2]] for item in higher), default=None
|
||||||
|
)
|
||||||
|
pool = exact or (
|
||||||
|
[item for item in higher if tier_order[item[2]] == nearest_higher]
|
||||||
|
if nearest_higher is not None else []
|
||||||
|
)
|
||||||
|
if not pool:
|
||||||
|
return (current if allow_current_fallback else ""), observations
|
||||||
|
comparable_costs = _comparable_costs(pool)
|
||||||
|
|
||||||
|
def key(item: tuple[str, dict[str, Any], str]) -> tuple[int, int, float, int, int, tuple[int, ...], str]:
|
||||||
|
model, details, tier = item
|
||||||
|
text = _text_metadata(details)
|
||||||
|
strongest = any(term in text for term in ("strongest", "most capable", "most intelligent"))
|
||||||
|
return (
|
||||||
|
0 if tier == desired else 1,
|
||||||
|
0 if strongest else 1,
|
||||||
|
comparable_costs.get(model, float("inf")),
|
||||||
|
# A provider moving its default is a current release signal. It
|
||||||
|
# supersedes an old advanced route, while current protects equal
|
||||||
|
# economy/balanced candidates where no comparable cost is known.
|
||||||
|
0 if desired == "advanced" and (
|
||||||
|
details.get("isDefault") is True or details.get("is_default") is True
|
||||||
|
) else 1,
|
||||||
|
0 if model == current else 1,
|
||||||
|
tuple(-part for part in model_version(model)),
|
||||||
|
model,
|
||||||
|
)
|
||||||
|
return min(pool, key=key)[0], observations
|
||||||
219
services/hermes/scripts/provider_model_discovery.py
Normal file
219
services/hermes/scripts/provider_model_discovery.py
Normal file
@ -0,0 +1,219 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Discover account-visible provider models without leaking credentials."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import select
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import time
|
||||||
|
from typing import Any
|
||||||
|
from urllib.error import HTTPError, URLError
|
||||||
|
from urllib.parse import urlencode
|
||||||
|
from urllib.request import Request, urlopen
|
||||||
|
|
||||||
|
from provider_model_catalog import Catalog, model_records, unique_models
|
||||||
|
|
||||||
|
|
||||||
|
def codex_cli_authenticated() -> bool:
|
||||||
|
"""Return whether the installed Codex CLI has a usable local login."""
|
||||||
|
codex = shutil.which("codex")
|
||||||
|
if not codex:
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
status = subprocess.run(
|
||||||
|
[codex, "login", "status"], capture_output=True, check=False,
|
||||||
|
text=True, timeout=10,
|
||||||
|
)
|
||||||
|
except (OSError, subprocess.SubprocessError):
|
||||||
|
return False
|
||||||
|
detail = f"{status.stdout}\n{status.stderr}".lower()
|
||||||
|
return status.returncode == 0 and (
|
||||||
|
"logged in" in detail or "authenticated" in detail
|
||||||
|
)
|
||||||
|
|
||||||
|
def _codex_app_server_records(codex: str) -> tuple[list[str], dict[str, dict[str, Any]]] | None:
|
||||||
|
"""Read the authenticated app-server ``model/list`` response when available."""
|
||||||
|
try:
|
||||||
|
process = subprocess.Popen(
|
||||||
|
[codex, "app-server"],
|
||||||
|
stdin=subprocess.PIPE,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.DEVNULL,
|
||||||
|
text=False,
|
||||||
|
)
|
||||||
|
except OSError:
|
||||||
|
return None
|
||||||
|
if process.stdin is None or process.stdout is None:
|
||||||
|
process.terminate()
|
||||||
|
return None
|
||||||
|
buffer = b""
|
||||||
|
|
||||||
|
def send(message: dict[str, Any]) -> None:
|
||||||
|
assert process.stdin is not None
|
||||||
|
process.stdin.write(json.dumps(message, separators=(",", ":")).encode() + b"\n")
|
||||||
|
process.stdin.flush()
|
||||||
|
|
||||||
|
def response(request_id: int, deadline: float) -> dict[str, Any] | None:
|
||||||
|
nonlocal buffer
|
||||||
|
assert process.stdout is not None
|
||||||
|
while time.monotonic() < deadline:
|
||||||
|
readable, _, _ = select.select([process.stdout.fileno()], [], [], 0.25)
|
||||||
|
if not readable:
|
||||||
|
continue
|
||||||
|
chunk = os.read(process.stdout.fileno(), 65536)
|
||||||
|
if not chunk:
|
||||||
|
return None
|
||||||
|
buffer += chunk
|
||||||
|
while b"\n" in buffer:
|
||||||
|
line, buffer = buffer.split(b"\n", 1)
|
||||||
|
try:
|
||||||
|
document = json.loads(line)
|
||||||
|
except (TypeError, ValueError, json.JSONDecodeError):
|
||||||
|
continue
|
||||||
|
if document.get("id") == request_id and isinstance(document.get("result"), dict):
|
||||||
|
return document["result"]
|
||||||
|
return None
|
||||||
|
|
||||||
|
try:
|
||||||
|
send({"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {"clientInfo": {"name": "hermes-catalog", "version": "1"}, "capabilities": {}}})
|
||||||
|
if response(1, time.monotonic() + 12) is None:
|
||||||
|
return None
|
||||||
|
send({"jsonrpc": "2.0", "method": "initialized", "params": {}})
|
||||||
|
values: list[Any] = []
|
||||||
|
cursor: str | None = None
|
||||||
|
request_id = 2
|
||||||
|
while True:
|
||||||
|
params: dict[str, Any] = {"includeHidden": False, "limit": 100}
|
||||||
|
if cursor:
|
||||||
|
params["cursor"] = cursor
|
||||||
|
send({"jsonrpc": "2.0", "id": request_id, "method": "model/list", "params": params})
|
||||||
|
result = response(request_id, time.monotonic() + 12)
|
||||||
|
if result is None:
|
||||||
|
return None
|
||||||
|
page = result.get("models", result.get("data", []))
|
||||||
|
if not isinstance(page, list):
|
||||||
|
return None
|
||||||
|
values.extend(page)
|
||||||
|
cursor = result.get("nextCursor") or result.get("next_cursor")
|
||||||
|
if not isinstance(cursor, str) or not cursor:
|
||||||
|
return model_records(values)
|
||||||
|
request_id += 1
|
||||||
|
except (OSError, ValueError, json.JSONDecodeError):
|
||||||
|
pass
|
||||||
|
finally:
|
||||||
|
process.terminate()
|
||||||
|
try:
|
||||||
|
process.wait(timeout=2)
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
process.kill()
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _anthropic_api_records() -> tuple[list[str], dict[str, dict[str, Any]]] | None:
|
||||||
|
"""List current Anthropic account models, including documented capabilities."""
|
||||||
|
token = os.environ.get("ANTHROPIC_API_KEY", "").strip()
|
||||||
|
if not token:
|
||||||
|
return None
|
||||||
|
request = Request(
|
||||||
|
"https://api.anthropic.com/v1/models?" + urlencode({"limit": 100}),
|
||||||
|
headers={"x-api-key": token, "anthropic-version": "2023-06-01"},
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with urlopen(request, timeout=15) as response:
|
||||||
|
document = json.load(response)
|
||||||
|
except (HTTPError, URLError, OSError, TimeoutError, ValueError):
|
||||||
|
return None
|
||||||
|
values = document.get("data", []) if isinstance(document, dict) else []
|
||||||
|
return model_records(values if isinstance(values, list) else [])
|
||||||
|
|
||||||
|
|
||||||
|
def discover_codex_models() -> Catalog:
|
||||||
|
"""Use the authenticated Codex endpoint; catalogs are status-only fallback."""
|
||||||
|
token = ""
|
||||||
|
try:
|
||||||
|
from hermes_cli.auth import resolve_codex_runtime_credentials
|
||||||
|
|
||||||
|
credentials = resolve_codex_runtime_credentials(refresh_if_expiring=True) or {}
|
||||||
|
token = str(credentials.get("api_key") or "").strip()
|
||||||
|
except Exception:
|
||||||
|
token = ""
|
||||||
|
api_records: tuple[list[str], dict[str, dict[str, Any]]] | None = None
|
||||||
|
if token:
|
||||||
|
try:
|
||||||
|
from hermes_cli.codex_models import _fetch_models_from_api
|
||||||
|
|
||||||
|
api_records = model_records(_fetch_models_from_api(token))
|
||||||
|
except Exception:
|
||||||
|
api_records = None
|
||||||
|
if api_records is not None:
|
||||||
|
live, live_metadata = api_records
|
||||||
|
return Catalog("openai-codex", live, True, True, "connected-api", live_metadata)
|
||||||
|
codex = shutil.which("codex")
|
||||||
|
if codex:
|
||||||
|
app_records = _codex_app_server_records(codex)
|
||||||
|
if app_records is not None:
|
||||||
|
app_models, app_metadata = app_records
|
||||||
|
return Catalog("openai-codex", app_models, True, True, "connected-app-server", app_metadata)
|
||||||
|
try:
|
||||||
|
from hermes_cli.models import provider_model_ids
|
||||||
|
|
||||||
|
known, known_metadata = model_records(
|
||||||
|
provider_model_ids("openai-codex", force_refresh=True)
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
known, known_metadata = [], {}
|
||||||
|
# The owner agent deliberately uses the local Codex app-server and the
|
||||||
|
# Codex CLI's ChatGPT login. That credential is not exported into Hermes'
|
||||||
|
# bearer-token store, so API discovery above may be empty even though the
|
||||||
|
# runtime is fully authenticated. Treat a successful CLI status check as a
|
||||||
|
# connected, degraded catalog: routing may use Codex while retaining the
|
||||||
|
# last-known working model names until app-server discovery is available.
|
||||||
|
if codex_cli_authenticated():
|
||||||
|
return Catalog(
|
||||||
|
"openai-codex",
|
||||||
|
known,
|
||||||
|
False,
|
||||||
|
True,
|
||||||
|
"connected-cli",
|
||||||
|
known_metadata,
|
||||||
|
)
|
||||||
|
state = "degraded" if token else "not-configured"
|
||||||
|
return Catalog("openai-codex", known, False, bool(token), state, known_metadata)
|
||||||
|
|
||||||
|
|
||||||
|
def discover_claude_models() -> Catalog:
|
||||||
|
"""Discover subscription models first, retaining API discovery as fallback."""
|
||||||
|
try:
|
||||||
|
from claude_model_discovery import discover_claude_subscription_models
|
||||||
|
|
||||||
|
subscription = discover_claude_subscription_models()
|
||||||
|
except Exception:
|
||||||
|
subscription = None
|
||||||
|
if subscription is not None and subscription.live:
|
||||||
|
records = [model.as_dict() for model in subscription.models]
|
||||||
|
models, metadata = model_records(records)
|
||||||
|
return Catalog(
|
||||||
|
"anthropic", models, True, subscription.connected,
|
||||||
|
subscription.state, metadata,
|
||||||
|
)
|
||||||
|
api_records = _anthropic_api_records()
|
||||||
|
if api_records is not None:
|
||||||
|
live_models, live_metadata = api_records
|
||||||
|
return Catalog("anthropic", live_models, True, True, "connected-api", live_metadata)
|
||||||
|
try:
|
||||||
|
from hermes_cli.models import provider_model_ids
|
||||||
|
|
||||||
|
known, known_metadata = model_records(
|
||||||
|
provider_model_ids("anthropic", force_refresh=True)
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
known, known_metadata = [], {}
|
||||||
|
if subscription is not None:
|
||||||
|
return Catalog(
|
||||||
|
"anthropic", known, False, subscription.connected,
|
||||||
|
subscription.state, known_metadata,
|
||||||
|
)
|
||||||
|
return Catalog("anthropic", known, False, False, "not-configured", known_metadata)
|
||||||
@ -51,6 +51,15 @@ def load_catalog(path: Path = CATALOG_PATH) -> dict[str, Any]:
|
|||||||
return value if isinstance(value, dict) else {}
|
return value if isinstance(value, dict) else {}
|
||||||
|
|
||||||
|
|
||||||
|
def catalog_contains(provider: str, model: str, catalog: dict[str, Any] | None = None) -> bool:
|
||||||
|
"""Return whether a provider's stewarded current catalog advertises a model."""
|
||||||
|
document = catalog if catalog is not None else load_catalog()
|
||||||
|
providers = document.get("providers", {}) if isinstance(document, dict) else {}
|
||||||
|
record = providers.get(provider, {}) if isinstance(providers, dict) else {}
|
||||||
|
models = record.get("models", []) if isinstance(record, dict) else []
|
||||||
|
return isinstance(model, str) and model in models
|
||||||
|
|
||||||
|
|
||||||
def resolve_model(
|
def resolve_model(
|
||||||
provider: str,
|
provider: str,
|
||||||
selector: str,
|
selector: str,
|
||||||
@ -61,10 +70,16 @@ def resolve_model(
|
|||||||
if provider not in DEFAULTS or effort not in DEFAULTS[provider]:
|
if provider not in DEFAULTS or effort not in DEFAULTS[provider]:
|
||||||
raise ValueError("unsupported provider or effort")
|
raise ValueError("unsupported provider or effort")
|
||||||
prefix = PREFIXES[provider]
|
prefix = PREFIXES[provider]
|
||||||
|
document = catalog if catalog is not None else load_catalog()
|
||||||
if selector.startswith(prefix):
|
if selector.startswith(prefix):
|
||||||
|
if catalog_contains(provider, selector, document):
|
||||||
|
return selector
|
||||||
|
providers = document.get("providers", {}) if isinstance(document, dict) else {}
|
||||||
|
record = providers.get(provider, {}) if isinstance(providers, dict) else {}
|
||||||
|
if isinstance(record, dict) and record.get("live") is True:
|
||||||
|
raise ValueError(f"model is not in the current {provider} catalog")
|
||||||
return selector
|
return selector
|
||||||
|
|
||||||
document = catalog if catalog is not None else load_catalog()
|
|
||||||
providers = document.get("providers", {}) if isinstance(document, dict) else {}
|
providers = document.get("providers", {}) if isinstance(document, dict) else {}
|
||||||
record = providers.get(provider, {}) if isinstance(providers, dict) else {}
|
record = providers.get(provider, {}) if isinstance(providers, dict) else {}
|
||||||
if not isinstance(record, dict):
|
if not isinstance(record, dict):
|
||||||
@ -72,7 +87,23 @@ def resolve_model(
|
|||||||
mapping_name = "resolved" if selector == "auto" else "tiers"
|
mapping_name = "resolved" if selector == "auto" else "tiers"
|
||||||
mapping = record.get(mapping_name, {})
|
mapping = record.get(mapping_name, {})
|
||||||
candidate = mapping.get(effort if selector == "auto" else selector, "") if isinstance(mapping, dict) else ""
|
candidate = mapping.get(effort if selector == "auto" else selector, "") if isinstance(mapping, dict) else ""
|
||||||
if not isinstance(candidate, str) or not candidate.startswith(prefix):
|
if (
|
||||||
|
selector != "auto"
|
||||||
|
and isinstance(candidate, str)
|
||||||
|
and candidate
|
||||||
|
and selector not in candidate.lower()
|
||||||
|
):
|
||||||
|
candidate = ""
|
||||||
|
advertised = record.get("models", []) if isinstance(record, dict) else []
|
||||||
|
candidate_allowed = isinstance(candidate, str) and (
|
||||||
|
candidate.startswith(prefix)
|
||||||
|
or (provider == "claude" and candidate in advertised)
|
||||||
|
)
|
||||||
|
if not candidate_allowed:
|
||||||
|
# A fresh account list is authoritative. Do not turn a deliberately
|
||||||
|
# unresolved/retired selector into a stale baseline model.
|
||||||
|
if record.get("live") is True:
|
||||||
|
raise ValueError(f"no current {provider} model for selector {selector!r}")
|
||||||
candidate = (
|
candidate = (
|
||||||
DEFAULTS[provider][effort]
|
DEFAULTS[provider][effort]
|
||||||
if selector == "auto"
|
if selector == "auto"
|
||||||
|
|||||||
@ -158,6 +158,26 @@ data:
|
|||||||
llm_client = "codex_xhigh"
|
llm_client = "codex_xhigh"
|
||||||
extra_body = { reasoning = { effort = "xhigh" } }
|
extra_body = { reasoning = { effort = "xhigh" } }
|
||||||
|
|
||||||
|
[targets.codex_auto_low]
|
||||||
|
id = "route/codex/auto/low"
|
||||||
|
llm_client = "codex_low"
|
||||||
|
extra_body = { reasoning = { effort = "low" } }
|
||||||
|
|
||||||
|
[targets.codex_auto_medium]
|
||||||
|
id = "route/codex/auto/medium"
|
||||||
|
llm_client = "codex_medium"
|
||||||
|
extra_body = { reasoning = { effort = "medium" } }
|
||||||
|
|
||||||
|
[targets.codex_auto_high]
|
||||||
|
id = "route/codex/auto/high"
|
||||||
|
llm_client = "codex_high"
|
||||||
|
extra_body = { reasoning = { effort = "high" } }
|
||||||
|
|
||||||
|
[targets.codex_auto_xhigh]
|
||||||
|
id = "route/codex/auto/xhigh"
|
||||||
|
llm_client = "codex_xhigh"
|
||||||
|
extra_body = { reasoning = { effort = "xhigh" } }
|
||||||
|
|
||||||
[targets.claude_haiku_low]
|
[targets.claude_haiku_low]
|
||||||
id = "route/claude/haiku/low"
|
id = "route/claude/haiku/low"
|
||||||
llm_client = "claude_low"
|
llm_client = "claude_low"
|
||||||
@ -238,6 +258,26 @@ data:
|
|||||||
llm_client = "claude_xhigh"
|
llm_client = "claude_xhigh"
|
||||||
extra_body = { thinking = { type = "adaptive" }, output_config = { effort = "xhigh" } }
|
extra_body = { thinking = { type = "adaptive" }, output_config = { effort = "xhigh" } }
|
||||||
|
|
||||||
|
[targets.claude_auto_low]
|
||||||
|
id = "route/claude/auto/low"
|
||||||
|
llm_client = "claude_low"
|
||||||
|
extra_body = { thinking = { type = "adaptive" }, output_config = { effort = "low" } }
|
||||||
|
|
||||||
|
[targets.claude_auto_medium]
|
||||||
|
id = "route/claude/auto/medium"
|
||||||
|
llm_client = "claude_medium"
|
||||||
|
extra_body = { thinking = { type = "adaptive" }, output_config = { effort = "medium" } }
|
||||||
|
|
||||||
|
[targets.claude_auto_high]
|
||||||
|
id = "route/claude/auto/high"
|
||||||
|
llm_client = "claude_high"
|
||||||
|
extra_body = { thinking = { type = "adaptive" }, output_config = { effort = "high" } }
|
||||||
|
|
||||||
|
[targets.claude_auto_xhigh]
|
||||||
|
id = "route/claude/auto/xhigh"
|
||||||
|
llm_client = "claude_xhigh"
|
||||||
|
extra_body = { thinking = { type = "adaptive" }, output_config = { effort = "xhigh" } }
|
||||||
|
|
||||||
[targets.worker_codex_luna_low]
|
[targets.worker_codex_luna_low]
|
||||||
id = "worker/codex/luna/low"
|
id = "worker/codex/luna/low"
|
||||||
llm_client = "worker_decision"
|
llm_client = "worker_decision"
|
||||||
@ -350,6 +390,38 @@ data:
|
|||||||
id = "worker/claude/opus/xhigh"
|
id = "worker/claude/opus/xhigh"
|
||||||
llm_client = "worker_decision"
|
llm_client = "worker_decision"
|
||||||
|
|
||||||
|
[targets.worker_codex_auto_low]
|
||||||
|
id = "worker/codex/auto/low"
|
||||||
|
llm_client = "worker_decision"
|
||||||
|
|
||||||
|
[targets.worker_codex_auto_medium]
|
||||||
|
id = "worker/codex/auto/medium"
|
||||||
|
llm_client = "worker_decision"
|
||||||
|
|
||||||
|
[targets.worker_codex_auto_high]
|
||||||
|
id = "worker/codex/auto/high"
|
||||||
|
llm_client = "worker_decision"
|
||||||
|
|
||||||
|
[targets.worker_codex_auto_xhigh]
|
||||||
|
id = "worker/codex/auto/xhigh"
|
||||||
|
llm_client = "worker_decision"
|
||||||
|
|
||||||
|
[targets.worker_claude_auto_low]
|
||||||
|
id = "worker/claude/auto/low"
|
||||||
|
llm_client = "worker_decision"
|
||||||
|
|
||||||
|
[targets.worker_claude_auto_medium]
|
||||||
|
id = "worker/claude/auto/medium"
|
||||||
|
llm_client = "worker_decision"
|
||||||
|
|
||||||
|
[targets.worker_claude_auto_high]
|
||||||
|
id = "worker/claude/auto/high"
|
||||||
|
llm_client = "worker_decision"
|
||||||
|
|
||||||
|
[targets.worker_claude_auto_xhigh]
|
||||||
|
id = "worker/claude/auto/xhigh"
|
||||||
|
llm_client = "worker_decision"
|
||||||
|
|
||||||
# Switchyard 0.2.0 requires one default_target for every custom classifier.
|
# Switchyard 0.2.0 requires one default_target for every custom classifier.
|
||||||
# These targets recurse once into unbiased random routes whose candidates are
|
# These targets recurse once into unbiased random routes whose candidates are
|
||||||
# split evenly across providers. A classifier outage therefore falls back to
|
# split evenly across providers. A classifier outage therefore falls back to
|
||||||
@ -377,7 +449,7 @@ data:
|
|||||||
[routes.fallback_fast]
|
[routes.fallback_fast]
|
||||||
id = "atlas/fallback/fast"
|
id = "atlas/fallback/fast"
|
||||||
type = "random"
|
type = "random"
|
||||||
targets = ["codex_terra_medium", "claude_fable_medium"]
|
targets = ["codex_auto_medium", "claude_auto_medium"]
|
||||||
weights = [1.0, 1.0]
|
weights = [1.0, 1.0]
|
||||||
context_window = 272000
|
context_window = 272000
|
||||||
tool_calling = true
|
tool_calling = true
|
||||||
@ -386,7 +458,7 @@ data:
|
|||||||
[routes.fallback_balanced]
|
[routes.fallback_balanced]
|
||||||
id = "atlas/fallback/balanced"
|
id = "atlas/fallback/balanced"
|
||||||
type = "random"
|
type = "random"
|
||||||
targets = ["codex_terra_high", "claude_sonnet_high"]
|
targets = ["codex_auto_high", "claude_auto_high"]
|
||||||
weights = [1.0, 1.0]
|
weights = [1.0, 1.0]
|
||||||
context_window = 272000
|
context_window = 272000
|
||||||
tool_calling = true
|
tool_calling = true
|
||||||
@ -395,7 +467,7 @@ data:
|
|||||||
[routes.fallback_deep]
|
[routes.fallback_deep]
|
||||||
id = "atlas/fallback/deep"
|
id = "atlas/fallback/deep"
|
||||||
type = "random"
|
type = "random"
|
||||||
targets = ["codex_sol_high", "claude_opus_high"]
|
targets = ["codex_auto_high", "claude_auto_high"]
|
||||||
weights = [1.0, 1.0]
|
weights = [1.0, 1.0]
|
||||||
context_window = 272000
|
context_window = 272000
|
||||||
tool_calling = true
|
tool_calling = true
|
||||||
@ -404,7 +476,7 @@ data:
|
|||||||
[routes.fallback_maximum]
|
[routes.fallback_maximum]
|
||||||
id = "atlas/fallback/maximum"
|
id = "atlas/fallback/maximum"
|
||||||
type = "random"
|
type = "random"
|
||||||
targets = ["codex_sol_xhigh", "claude_opus_xhigh"]
|
targets = ["codex_auto_xhigh", "claude_auto_xhigh"]
|
||||||
weights = [1.0, 1.0]
|
weights = [1.0, 1.0]
|
||||||
context_window = 272000
|
context_window = 272000
|
||||||
tool_calling = true
|
tool_calling = true
|
||||||
@ -413,7 +485,7 @@ data:
|
|||||||
[routes.fallback_worker_maximum]
|
[routes.fallback_worker_maximum]
|
||||||
id = "atlas/worker/fallback/maximum"
|
id = "atlas/worker/fallback/maximum"
|
||||||
type = "random"
|
type = "random"
|
||||||
targets = ["worker_codex_sol_xhigh", "worker_claude_opus_xhigh"]
|
targets = ["worker_codex_auto_xhigh", "worker_claude_auto_xhigh"]
|
||||||
weights = [1.0, 1.0]
|
weights = [1.0, 1.0]
|
||||||
context_window = 272000
|
context_window = 272000
|
||||||
tool_calling = false
|
tool_calling = false
|
||||||
@ -426,7 +498,7 @@ data:
|
|||||||
classifier_target = "classifier"
|
classifier_target = "classifier"
|
||||||
# Switchyard falls through this list after a request-local target failure.
|
# Switchyard falls through this list after a request-local target failure.
|
||||||
# Keep both xhigh providers first so recovery can escalate, never downgrade.
|
# Keep both xhigh providers first so recovery can escalate, never downgrade.
|
||||||
targets = ["codex_sol_xhigh", "claude_opus_xhigh", "claude_fable_xhigh", "codex_sol_high", "claude_sonnet_high", "claude_opus_high", "claude_fable_high", "codex_terra_high", "codex_sol_medium", "claude_sonnet_medium", "claude_fable_medium", "codex_terra_medium", "codex_terra_low", "codex_luna_low", "claude_haiku_low", "claude_fable_low", "neutral_fast_pool"]
|
targets = ["codex_auto_xhigh", "claude_auto_xhigh", "codex_auto_high", "claude_auto_high", "codex_auto_medium", "claude_auto_medium", "codex_auto_low", "claude_auto_low", "neutral_fast_pool"]
|
||||||
default_target = "neutral_fast_pool"
|
default_target = "neutral_fast_pool"
|
||||||
session_affinity = false
|
session_affinity = false
|
||||||
recent_turn_window = 4
|
recent_turn_window = 4
|
||||||
@ -449,10 +521,10 @@ data:
|
|||||||
If tools are present or the boundary may emit a tool call, the floor is
|
If tools are present or the boundary may emit a tool call, the floor is
|
||||||
medium even when the user's wording is short. Filesystem, shell,
|
medium even when the user's wording is short. Filesystem, shell,
|
||||||
repository, cluster, browser, and image-generation/edit operations are
|
repository, cluster, browser, and image-generation/edit operations are
|
||||||
tool work; never route those boundaries to Luna, Haiku, or local-low.
|
tool work; never route those boundaries to economy or local-low targets.
|
||||||
A failing test, failed tool plan, contradicted result, rejected review, or
|
A failing test, failed tool plan, contradicted result, rejected review, or
|
||||||
incomplete evidence means the previous quality mark was missed. Raise the
|
incomplete evidence means the previous quality mark was missed. Raise the
|
||||||
next boundary by at least one effort tier and prefer a stronger family or
|
next boundary by at least one effort tier and prefer an available advanced target or
|
||||||
the other hosted provider; repeated misses require xhigh. Never repeat a
|
the other hosted provider; repeated misses require xhigh. Never repeat a
|
||||||
failed lower-capability plan unchanged.
|
failed lower-capability plan unchanged.
|
||||||
|
|
||||||
@ -465,7 +537,7 @@ data:
|
|||||||
3. Route image creation and editing before general provider preference.
|
3. Route image creation and editing before general provider preference.
|
||||||
When image-generation/edit tools are available and the user asks to create,
|
When image-generation/edit tools are available and the user asks to create,
|
||||||
transform, restore, colorize, or continue editing an image: select
|
transform, restore, colorize, or continue editing an image: select
|
||||||
codex_terra_medium. The image tool—not the conversational model—honors the
|
codex_auto_medium. The image tool—not the conversational model—honors the
|
||||||
user's local, OpenAI/hosted, or AUTO image-backend choice. Do not select a
|
user's local, OpenAI/hosted, or AUTO image-backend choice. Do not select a
|
||||||
local Qwen or Claude target for an image-tool boundary: local Qwen cannot
|
local Qwen or Claude target for an image-tool boundary: local Qwen cannot
|
||||||
reliably carry the full Hermes tool context, and Anthropic supplies the
|
reliably carry the full Hermes tool context, and Anthropic supplies the
|
||||||
@ -480,17 +552,15 @@ data:
|
|||||||
to be unavailable, failed, exhausted, rate-limited, or out of capacity; use
|
to be unavailable, failed, exhausted, rate-limited, or out of capacity; use
|
||||||
the other hosted provider at the same floor.
|
the other hosted provider at the same floor.
|
||||||
|
|
||||||
5. Choose across the complete family catalog. Codex options are Luna,
|
5. Choose the provider AUTO target at the exact effort floor. AUTO resolves
|
||||||
Terra, and SOL at low through xhigh. Claude options are Haiku, Fable,
|
an account-visible economy, balanced, or advanced general-purpose model
|
||||||
Sonnet, and Opus at low through xhigh. Prefer Fable for concise writing,
|
from provider metadata. It uses comparable provider cost data only to break
|
||||||
synthesis, and instruction-following where its capability fits the effort
|
ties between adequate models. Re-evaluate every boundary and resolve
|
||||||
floor; use Sonnet or Opus for deeper analysis and review. Re-evaluate every
|
"continue" or "do it" from recent context. The manual local route remains
|
||||||
boundary and resolve "continue"
|
available only when the user explicitly selects it.
|
||||||
or "do it" from recent context. The manual local route remains available
|
|
||||||
only when the user explicitly selects it.
|
|
||||||
"""
|
"""
|
||||||
response_schema = '''
|
response_schema = '''
|
||||||
{"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["codex_terra_medium","claude_sonnet_medium","claude_fable_medium","codex_luna_low","claude_haiku_low","claude_fable_low","codex_terra_low","codex_terra_high","claude_fable_high","codex_sol_medium","codex_sol_high","codex_sol_xhigh","claude_sonnet_high","claude_opus_high","claude_fable_xhigh","claude_opus_xhigh"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false}
|
{"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["codex_auto_low","codex_auto_medium","codex_auto_high","codex_auto_xhigh","claude_auto_low","claude_auto_medium","claude_auto_high","claude_auto_xhigh"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false}
|
||||||
'''
|
'''
|
||||||
|
|
||||||
[routes.auto_fast.policy]
|
[routes.auto_fast.policy]
|
||||||
@ -504,7 +574,7 @@ data:
|
|||||||
classifier_target = "classifier"
|
classifier_target = "classifier"
|
||||||
# Switchyard falls through this list after a request-local target failure.
|
# Switchyard falls through this list after a request-local target failure.
|
||||||
# Keep both xhigh providers first so recovery can escalate, never downgrade.
|
# Keep both xhigh providers first so recovery can escalate, never downgrade.
|
||||||
targets = ["codex_sol_xhigh", "claude_opus_xhigh", "claude_fable_xhigh", "codex_sol_high", "claude_sonnet_high", "claude_opus_high", "claude_fable_high", "codex_terra_high", "codex_sol_medium", "claude_sonnet_medium", "claude_fable_medium", "codex_terra_medium", "codex_terra_low", "codex_luna_low", "claude_haiku_low", "claude_fable_low", "neutral_balanced_pool"]
|
targets = ["codex_auto_xhigh", "claude_auto_xhigh", "codex_auto_high", "claude_auto_high", "codex_auto_medium", "claude_auto_medium", "codex_auto_low", "claude_auto_low", "neutral_balanced_pool"]
|
||||||
default_target = "neutral_balanced_pool"
|
default_target = "neutral_balanced_pool"
|
||||||
session_affinity = false
|
session_affinity = false
|
||||||
recent_turn_window = 4
|
recent_turn_window = 4
|
||||||
@ -528,10 +598,10 @@ data:
|
|||||||
If tools are present or the boundary may emit a tool call, the floor is
|
If tools are present or the boundary may emit a tool call, the floor is
|
||||||
medium even when the user's wording is short. Filesystem, shell,
|
medium even when the user's wording is short. Filesystem, shell,
|
||||||
repository, cluster, browser, and image-generation/edit operations are
|
repository, cluster, browser, and image-generation/edit operations are
|
||||||
tool work; never route those boundaries to Luna, Haiku, or local-low.
|
tool work; never route those boundaries to economy or local-low targets.
|
||||||
A failing test, failed tool plan, contradicted result, rejected review, or
|
A failing test, failed tool plan, contradicted result, rejected review, or
|
||||||
incomplete evidence means the previous quality mark was missed. Raise the
|
incomplete evidence means the previous quality mark was missed. Raise the
|
||||||
next boundary by at least one effort tier and prefer a stronger family or
|
next boundary by at least one effort tier and prefer an available advanced target or
|
||||||
the other hosted provider; repeated misses require xhigh. Never repeat a
|
the other hosted provider; repeated misses require xhigh. Never repeat a
|
||||||
failed lower-capability plan unchanged.
|
failed lower-capability plan unchanged.
|
||||||
|
|
||||||
@ -544,7 +614,7 @@ data:
|
|||||||
3. Route image creation and editing before general provider preference.
|
3. Route image creation and editing before general provider preference.
|
||||||
When image-generation/edit tools are available and the user asks to create,
|
When image-generation/edit tools are available and the user asks to create,
|
||||||
transform, restore, colorize, or continue editing an image: select
|
transform, restore, colorize, or continue editing an image: select
|
||||||
codex_terra_medium. The image tool—not the conversational model—honors the
|
codex_auto_medium. The image tool—not the conversational model—honors the
|
||||||
user's local, OpenAI/hosted, or AUTO image-backend choice. Do not select a
|
user's local, OpenAI/hosted, or AUTO image-backend choice. Do not select a
|
||||||
local Qwen or Claude target for an image-tool boundary: local Qwen cannot
|
local Qwen or Claude target for an image-tool boundary: local Qwen cannot
|
||||||
reliably carry the full Hermes tool context, and Anthropic supplies the
|
reliably carry the full Hermes tool context, and Anthropic supplies the
|
||||||
@ -559,17 +629,15 @@ data:
|
|||||||
to be unavailable, failed, exhausted, rate-limited, or out of capacity; use
|
to be unavailable, failed, exhausted, rate-limited, or out of capacity; use
|
||||||
the other hosted provider at the same floor.
|
the other hosted provider at the same floor.
|
||||||
|
|
||||||
5. Choose across the complete family catalog. Codex options are Luna,
|
5. Choose the provider AUTO target at the exact effort floor. AUTO resolves
|
||||||
Terra, and SOL at low through xhigh. Claude options are Haiku, Fable,
|
an account-visible economy, balanced, or advanced general-purpose model
|
||||||
Sonnet, and Opus at low through xhigh. Prefer Fable for concise writing,
|
from provider metadata. It uses comparable provider cost data only to break
|
||||||
synthesis, and instruction-following where its capability fits the effort
|
ties between adequate models. Re-evaluate every boundary and resolve
|
||||||
floor; use Sonnet or Opus for deeper analysis and review. Re-evaluate every
|
"continue" or "do it" from recent context. The manual local route remains
|
||||||
boundary and resolve "continue"
|
available only when the user explicitly selects it.
|
||||||
or "do it" from recent context. The manual local route remains available
|
|
||||||
only when the user explicitly selects it.
|
|
||||||
"""
|
"""
|
||||||
response_schema = '''
|
response_schema = '''
|
||||||
{"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["codex_terra_medium","claude_sonnet_medium","claude_fable_medium","codex_luna_low","claude_haiku_low","claude_fable_low","codex_terra_low","codex_terra_high","claude_fable_high","codex_sol_medium","codex_sol_high","codex_sol_xhigh","claude_sonnet_high","claude_opus_high","claude_fable_xhigh","claude_opus_xhigh"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false}
|
{"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["codex_auto_low","codex_auto_medium","codex_auto_high","codex_auto_xhigh","claude_auto_low","claude_auto_medium","claude_auto_high","claude_auto_xhigh"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false}
|
||||||
'''
|
'''
|
||||||
|
|
||||||
[routes.auto_balanced.policy]
|
[routes.auto_balanced.policy]
|
||||||
@ -583,7 +651,7 @@ data:
|
|||||||
classifier_target = "classifier"
|
classifier_target = "classifier"
|
||||||
# Switchyard falls through this list after a request-local target failure.
|
# Switchyard falls through this list after a request-local target failure.
|
||||||
# Keep both xhigh providers first so recovery can escalate, never downgrade.
|
# Keep both xhigh providers first so recovery can escalate, never downgrade.
|
||||||
targets = ["claude_opus_xhigh", "codex_sol_xhigh", "claude_fable_xhigh", "claude_sonnet_high", "codex_sol_high", "claude_opus_high", "claude_fable_high", "codex_terra_high", "claude_sonnet_medium", "claude_fable_medium", "codex_sol_medium", "codex_terra_medium", "neutral_deep_pool"]
|
targets = ["claude_auto_xhigh", "codex_auto_xhigh", "claude_auto_high", "codex_auto_high", "claude_auto_medium", "codex_auto_medium", "neutral_deep_pool"]
|
||||||
default_target = "neutral_deep_pool"
|
default_target = "neutral_deep_pool"
|
||||||
session_affinity = false
|
session_affinity = false
|
||||||
recent_turn_window = 6
|
recent_turn_window = 6
|
||||||
@ -607,10 +675,10 @@ data:
|
|||||||
If tools are present or the boundary may emit a tool call, the floor is
|
If tools are present or the boundary may emit a tool call, the floor is
|
||||||
medium even when the user's wording is short. Filesystem, shell,
|
medium even when the user's wording is short. Filesystem, shell,
|
||||||
repository, cluster, browser, and image-generation/edit operations are
|
repository, cluster, browser, and image-generation/edit operations are
|
||||||
tool work; never route those boundaries to Luna or Haiku.
|
tool work; never route those boundaries to economy targets.
|
||||||
A failing test, failed tool plan, contradicted result, rejected review, or
|
A failing test, failed tool plan, contradicted result, rejected review, or
|
||||||
incomplete evidence means the previous quality mark was missed. Raise the
|
incomplete evidence means the previous quality mark was missed. Raise the
|
||||||
next boundary by at least one effort tier and prefer a stronger family or
|
next boundary by at least one effort tier and prefer an available advanced target or
|
||||||
the other hosted provider; repeated misses require xhigh. Never repeat a
|
the other hosted provider; repeated misses require xhigh. Never repeat a
|
||||||
failed lower-capability plan unchanged.
|
failed lower-capability plan unchanged.
|
||||||
|
|
||||||
@ -630,15 +698,14 @@ data:
|
|||||||
unavailable, failed, exhausted, rate-limited, or out of capacity; use the
|
unavailable, failed, exhausted, rate-limited, or out of capacity; use the
|
||||||
other provider at the same floor.
|
other provider at the same floor.
|
||||||
|
|
||||||
4. Choose across Codex Terra/SOL and Claude Fable/Sonnet/Opus at medium
|
4. Choose a provider AUTO target at the exact effort floor. AUTO resolves
|
||||||
through xhigh. Prefer Fable for writing and compact synthesis where it
|
an account-visible balanced or advanced general-purpose model from provider
|
||||||
clears the quality floor; use Sonnet or Opus for deeper diagnosis and
|
metadata; economy targets are intentionally unavailable on this route.
|
||||||
review. Low-tier targets are intentionally unavailable on
|
Re-evaluate every boundary and resolve "continue" or "do it" from recent
|
||||||
this route. Re-evaluate every boundary and resolve "continue" or "do it"
|
context.
|
||||||
from recent context.
|
|
||||||
"""
|
"""
|
||||||
response_schema = '''
|
response_schema = '''
|
||||||
{"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["claude_sonnet_high","claude_fable_high","codex_sol_high","codex_terra_medium","claude_sonnet_medium","claude_fable_medium","codex_terra_high","codex_sol_medium","claude_opus_high","codex_sol_xhigh","claude_fable_xhigh","claude_opus_xhigh"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false}
|
{"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["claude_auto_medium","claude_auto_high","claude_auto_xhigh","codex_auto_medium","codex_auto_high","codex_auto_xhigh"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false}
|
||||||
'''
|
'''
|
||||||
|
|
||||||
[routes.auto_deep.policy]
|
[routes.auto_deep.policy]
|
||||||
@ -652,7 +719,7 @@ data:
|
|||||||
classifier_target = "classifier"
|
classifier_target = "classifier"
|
||||||
# Switchyard falls through this list after a request-local target failure.
|
# Switchyard falls through this list after a request-local target failure.
|
||||||
# Keep both xhigh providers first so recovery can escalate, never downgrade.
|
# Keep both xhigh providers first so recovery can escalate, never downgrade.
|
||||||
targets = ["codex_sol_xhigh", "claude_opus_xhigh", "claude_fable_xhigh", "codex_sol_high", "claude_opus_high", "claude_sonnet_high", "claude_fable_high", "codex_terra_high", "neutral_maximum_pool"]
|
targets = ["codex_auto_xhigh", "claude_auto_xhigh", "codex_auto_high", "claude_auto_high", "neutral_maximum_pool"]
|
||||||
default_target = "neutral_maximum_pool"
|
default_target = "neutral_maximum_pool"
|
||||||
session_affinity = false
|
session_affinity = false
|
||||||
recent_turn_window = 6
|
recent_turn_window = 6
|
||||||
@ -674,7 +741,7 @@ data:
|
|||||||
Never choose below the floor or above xhigh.
|
Never choose below the floor or above xhigh.
|
||||||
A failing test, failed tool plan, contradicted result, rejected review, or
|
A failing test, failed tool plan, contradicted result, rejected review, or
|
||||||
incomplete evidence raises the next boundary to xhigh and should switch to
|
incomplete evidence raises the next boundary to xhigh and should switch to
|
||||||
the strongest suitable family or other hosted provider. Never repeat a
|
the strongest suitable available target or other hosted provider. Never repeat a
|
||||||
failed high-capability plan unchanged.
|
failed high-capability plan unchanged.
|
||||||
|
|
||||||
2. Treat "think hard", "deeply", "carefully", and equivalent intent as a
|
2. Treat "think hard", "deeply", "carefully", and equivalent intent as a
|
||||||
@ -693,15 +760,14 @@ data:
|
|||||||
unavailable, failed, exhausted, rate-limited, or out of capacity; use the
|
unavailable, failed, exhausted, rate-limited, or out of capacity; use the
|
||||||
other provider at the same floor.
|
other provider at the same floor.
|
||||||
|
|
||||||
4. Choose across Codex Terra/SOL and Claude Fable/Sonnet/Opus at high or
|
4. Choose a provider AUTO target at high or xhigh only. AUTO resolves an
|
||||||
xhigh only. Prefer Fable for writing and compact synthesis where it clears
|
account-visible advanced general-purpose model from provider metadata.
|
||||||
the quality floor; use Sonnet or Opus for deeper analysis and independent
|
Medium and low targets are intentionally unavailable on this route.
|
||||||
review. Medium and low targets are intentionally unavailable on this
|
Re-evaluate every boundary and resolve "continue" or "do it" from recent
|
||||||
route. Re-evaluate every boundary and resolve "continue" or "do it" from
|
context.
|
||||||
recent context.
|
|
||||||
"""
|
"""
|
||||||
response_schema = '''
|
response_schema = '''
|
||||||
{"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["codex_sol_high","claude_opus_high","claude_sonnet_high","claude_fable_high","codex_terra_high","codex_sol_xhigh","claude_opus_xhigh","claude_fable_xhigh"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false}
|
{"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["codex_auto_high","codex_auto_xhigh","claude_auto_high","claude_auto_xhigh"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false}
|
||||||
'''
|
'''
|
||||||
|
|
||||||
[routes.auto_maximum.policy]
|
[routes.auto_maximum.policy]
|
||||||
@ -713,7 +779,7 @@ data:
|
|||||||
type = "llm_classifier"
|
type = "llm_classifier"
|
||||||
mode = "custom"
|
mode = "custom"
|
||||||
classifier_target = "classifier"
|
classifier_target = "classifier"
|
||||||
targets = ["worker_codex_luna_low", "worker_codex_luna_medium", "worker_codex_luna_high", "worker_codex_luna_xhigh", "worker_codex_terra_low", "worker_codex_terra_medium", "worker_codex_terra_high", "worker_codex_terra_xhigh", "worker_codex_sol_low", "worker_codex_sol_medium", "worker_codex_sol_high", "worker_codex_sol_xhigh", "worker_claude_haiku_low", "worker_claude_haiku_medium", "worker_claude_haiku_high", "worker_claude_haiku_xhigh", "worker_claude_fable_low", "worker_claude_fable_medium", "worker_claude_fable_high", "worker_claude_fable_xhigh", "worker_claude_sonnet_low", "worker_claude_sonnet_medium", "worker_claude_sonnet_high", "worker_claude_sonnet_xhigh", "worker_claude_opus_low", "worker_claude_opus_medium", "worker_claude_opus_high", "worker_claude_opus_xhigh", "neutral_worker_maximum_pool"]
|
targets = ["worker_codex_auto_low", "worker_codex_auto_medium", "worker_codex_auto_high", "worker_codex_auto_xhigh", "worker_claude_auto_low", "worker_claude_auto_medium", "worker_claude_auto_high", "worker_claude_auto_xhigh", "neutral_worker_maximum_pool"]
|
||||||
default_target = "neutral_worker_maximum_pool"
|
default_target = "neutral_worker_maximum_pool"
|
||||||
session_affinity = false
|
session_affinity = false
|
||||||
recent_turn_window = 6
|
recent_turn_window = 6
|
||||||
@ -738,7 +804,7 @@ data:
|
|||||||
Never choose below the floor and never exceed xhigh.
|
Never choose below the floor and never exceed xhigh.
|
||||||
If the objective records a failing test, failed attempt, contradicted
|
If the objective records a failing test, failed attempt, contradicted
|
||||||
result, rejected review, or incomplete evidence from a previous worker,
|
result, rejected review, or incomplete evidence from a previous worker,
|
||||||
raise effort by at least one tier and prefer a stronger family or the other
|
raise effort by at least one tier and prefer the available advanced target or the other
|
||||||
hosted provider. Repeated quality misses require xhigh; never launch the
|
hosted provider. Repeated quality misses require xhigh; never launch the
|
||||||
same failed lower-capability plan unchanged.
|
same failed lower-capability plan unchanged.
|
||||||
|
|
||||||
@ -757,30 +823,28 @@ data:
|
|||||||
analysis, and independent review.
|
analysis, and independent review.
|
||||||
A final independent review is Claude; implementing review findings is Codex.
|
A final independent review is Claude; implementing review findings is Codex.
|
||||||
|
|
||||||
4. Choose across every configured Codex and Claude family at the exact
|
4. Choose an available provider AUTO target at the exact effort floor.
|
||||||
effort floor. Codex Luna is the economical tier, Terra is balanced, and
|
AUTO resolves the current account-visible economy, balanced, or advanced
|
||||||
SOL is the deepest implementation tier. Claude Haiku is the economical
|
general-purpose model. It uses comparable provider cost data only to break
|
||||||
tier, Fable is preferred for concise writing and synthesis, Sonnet is
|
ties among models that already satisfy the capability requirement. Do not
|
||||||
balanced, and Opus is the deepest analysis and review tier. Every family
|
infer a tier from an unfamiliar model name or unadvertised effort support.
|
||||||
supports low, medium, high, and xhigh; choose the cheapest family that
|
|
||||||
clears the objective's quality floor without lowering its effort.
|
|
||||||
|
|
||||||
Examples:
|
Examples:
|
||||||
Critical security migration final review -> worker_claude_opus_xhigh.
|
Critical security migration final review -> worker_claude_auto_xhigh.
|
||||||
Implement critical security review fixes -> worker_codex_sol_xhigh.
|
Implement critical security review fixes -> worker_codex_auto_xhigh.
|
||||||
Difficult intermittent production failure -> worker_codex_sol_high.
|
Difficult intermittent production failure -> worker_codex_auto_high.
|
||||||
Ordinary component design -> worker_claude_sonnet_medium.
|
Ordinary component design -> worker_claude_auto_medium.
|
||||||
One spelling correction -> worker_codex_luna_low.
|
One spelling correction -> worker_codex_auto_low.
|
||||||
Anthropic exhausted + critical independent final review ->
|
Anthropic exhausted + critical independent final review ->
|
||||||
worker_codex_sol_xhigh.
|
worker_codex_auto_xhigh.
|
||||||
OpenAI exhausted + difficult repository implementation ->
|
OpenAI exhausted + difficult repository implementation ->
|
||||||
worker_claude_sonnet_high.
|
worker_claude_auto_high.
|
||||||
|
|
||||||
Before responding, verify the provider is available and effort is not below
|
Before responding, verify the provider is available and effort is not below
|
||||||
the floor. Return only the required decision object.
|
the floor. Return only the required decision object.
|
||||||
"""
|
"""
|
||||||
response_schema = '''
|
response_schema = '''
|
||||||
{"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["worker_codex_luna_low","worker_codex_luna_medium","worker_codex_luna_high","worker_codex_luna_xhigh","worker_codex_terra_low","worker_codex_terra_medium","worker_codex_terra_high","worker_codex_terra_xhigh","worker_codex_sol_low","worker_codex_sol_medium","worker_codex_sol_high","worker_codex_sol_xhigh","worker_claude_haiku_low","worker_claude_haiku_medium","worker_claude_haiku_high","worker_claude_haiku_xhigh","worker_claude_fable_low","worker_claude_fable_medium","worker_claude_fable_high","worker_claude_fable_xhigh","worker_claude_sonnet_low","worker_claude_sonnet_medium","worker_claude_sonnet_high","worker_claude_sonnet_xhigh","worker_claude_opus_low","worker_claude_opus_medium","worker_claude_opus_high","worker_claude_opus_xhigh"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false}
|
{"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["worker_codex_auto_low","worker_codex_auto_medium","worker_codex_auto_high","worker_codex_auto_xhigh","worker_claude_auto_low","worker_claude_auto_medium","worker_claude_auto_high","worker_claude_auto_xhigh"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false}
|
||||||
'''
|
'''
|
||||||
|
|
||||||
[routes.worker_auto_maximum.policy]
|
[routes.worker_auto_maximum.policy]
|
||||||
@ -790,42 +854,98 @@ data:
|
|||||||
[routes.worker_manual_codex_low]
|
[routes.worker_manual_codex_low]
|
||||||
id = "atlas/worker/manual/codex/low"
|
id = "atlas/worker/manual/codex/low"
|
||||||
type = "random"
|
type = "random"
|
||||||
targets = ["worker_codex_luna_low"]
|
targets = ["worker_codex_auto_low"]
|
||||||
|
|
||||||
[routes.worker_manual_codex_medium]
|
[routes.worker_manual_codex_medium]
|
||||||
id = "atlas/worker/manual/codex/medium"
|
id = "atlas/worker/manual/codex/medium"
|
||||||
type = "random"
|
type = "random"
|
||||||
targets = ["worker_codex_terra_medium"]
|
targets = ["worker_codex_auto_medium"]
|
||||||
|
|
||||||
[routes.worker_manual_codex_high]
|
[routes.worker_manual_codex_high]
|
||||||
id = "atlas/worker/manual/codex/high"
|
id = "atlas/worker/manual/codex/high"
|
||||||
type = "random"
|
type = "random"
|
||||||
targets = ["worker_codex_sol_high"]
|
targets = ["worker_codex_auto_high"]
|
||||||
|
|
||||||
[routes.worker_manual_codex_xhigh]
|
[routes.worker_manual_codex_xhigh]
|
||||||
id = "atlas/worker/manual/codex/xhigh"
|
id = "atlas/worker/manual/codex/xhigh"
|
||||||
type = "random"
|
type = "random"
|
||||||
targets = ["worker_codex_sol_xhigh"]
|
targets = ["worker_codex_auto_xhigh"]
|
||||||
|
|
||||||
[routes.worker_manual_claude_low]
|
[routes.worker_manual_claude_low]
|
||||||
id = "atlas/worker/manual/claude/low"
|
id = "atlas/worker/manual/claude/low"
|
||||||
type = "random"
|
type = "random"
|
||||||
targets = ["worker_claude_haiku_low"]
|
targets = ["worker_claude_auto_low"]
|
||||||
|
|
||||||
[routes.worker_manual_claude_medium]
|
[routes.worker_manual_claude_medium]
|
||||||
id = "atlas/worker/manual/claude/medium"
|
id = "atlas/worker/manual/claude/medium"
|
||||||
type = "random"
|
type = "random"
|
||||||
targets = ["worker_claude_sonnet_medium"]
|
targets = ["worker_claude_auto_medium"]
|
||||||
|
|
||||||
[routes.worker_manual_claude_high]
|
[routes.worker_manual_claude_high]
|
||||||
id = "atlas/worker/manual/claude/high"
|
id = "atlas/worker/manual/claude/high"
|
||||||
type = "random"
|
type = "random"
|
||||||
targets = ["worker_claude_sonnet_high"]
|
targets = ["worker_claude_auto_high"]
|
||||||
|
|
||||||
[routes.worker_manual_claude_xhigh]
|
[routes.worker_manual_claude_xhigh]
|
||||||
id = "atlas/worker/manual/claude/xhigh"
|
id = "atlas/worker/manual/claude/xhigh"
|
||||||
type = "random"
|
type = "random"
|
||||||
targets = ["worker_claude_opus_xhigh"]
|
targets = ["worker_claude_auto_xhigh"]
|
||||||
|
|
||||||
|
[routes.manual_codex_auto]
|
||||||
|
id = "atlas/manual/codex/auto"
|
||||||
|
type = "random"
|
||||||
|
targets = ["codex_auto_medium"]
|
||||||
|
context_window = 272000
|
||||||
|
tool_calling = true
|
||||||
|
reasoning = true
|
||||||
|
|
||||||
|
[routes.manual_codex_auto_low]
|
||||||
|
id = "atlas/manual/codex/auto/low"
|
||||||
|
type = "random"
|
||||||
|
targets = ["codex_auto_low"]
|
||||||
|
|
||||||
|
[routes.manual_codex_auto_medium]
|
||||||
|
id = "atlas/manual/codex/auto/medium"
|
||||||
|
type = "random"
|
||||||
|
targets = ["codex_auto_medium"]
|
||||||
|
|
||||||
|
[routes.manual_codex_auto_high]
|
||||||
|
id = "atlas/manual/codex/auto/high"
|
||||||
|
type = "random"
|
||||||
|
targets = ["codex_auto_high"]
|
||||||
|
|
||||||
|
[routes.manual_codex_auto_xhigh]
|
||||||
|
id = "atlas/manual/codex/auto/xhigh"
|
||||||
|
type = "random"
|
||||||
|
targets = ["codex_auto_xhigh"]
|
||||||
|
|
||||||
|
[routes.manual_claude_auto]
|
||||||
|
id = "atlas/manual/claude/auto"
|
||||||
|
type = "random"
|
||||||
|
targets = ["claude_auto_medium"]
|
||||||
|
context_window = 272000
|
||||||
|
tool_calling = true
|
||||||
|
reasoning = true
|
||||||
|
|
||||||
|
[routes.manual_claude_auto_low]
|
||||||
|
id = "atlas/manual/claude/auto/low"
|
||||||
|
type = "random"
|
||||||
|
targets = ["claude_auto_low"]
|
||||||
|
|
||||||
|
[routes.manual_claude_auto_medium]
|
||||||
|
id = "atlas/manual/claude/auto/medium"
|
||||||
|
type = "random"
|
||||||
|
targets = ["claude_auto_medium"]
|
||||||
|
|
||||||
|
[routes.manual_claude_auto_high]
|
||||||
|
id = "atlas/manual/claude/auto/high"
|
||||||
|
type = "random"
|
||||||
|
targets = ["claude_auto_high"]
|
||||||
|
|
||||||
|
[routes.manual_claude_auto_xhigh]
|
||||||
|
id = "atlas/manual/claude/auto/xhigh"
|
||||||
|
type = "random"
|
||||||
|
targets = ["claude_auto_xhigh"]
|
||||||
|
|
||||||
# A zero-weight target is not selected initially. Switchyard still walks
|
# A zero-weight target is not selected initially. Switchyard still walks
|
||||||
# this ordered list after 403/408/429/5xx, timeout, or transport failure.
|
# this ordered list after 403/408/429/5xx, timeout, or transport failure.
|
||||||
|
|||||||
@ -22,7 +22,7 @@ spec:
|
|||||||
labels:
|
labels:
|
||||||
app: hermes-switchyard
|
app: hermes-switchyard
|
||||||
annotations:
|
annotations:
|
||||||
ai.bstein.dev/config-rev: "20260823-voice-route-preflight-v1"
|
ai.bstein.dev/config-rev: "20260913-provider-model-catalog-v2"
|
||||||
prometheus.io/scrape: "true"
|
prometheus.io/scrape: "true"
|
||||||
prometheus.io/port: "9005"
|
prometheus.io/port: "9005"
|
||||||
prometheus.io/path: /metrics
|
prometheus.io/path: /metrics
|
||||||
|
|||||||
227
testing/fixtures/hermes/codex-0.154-visible-models.json
Normal file
227
testing/fixtures/hermes/codex-0.154-visible-models.json
Normal file
@ -0,0 +1,227 @@
|
|||||||
|
[
|
||||||
|
{
|
||||||
|
"defaultReasoningEffort": "medium",
|
||||||
|
"description": "Our most capable model for complex, demanding work.",
|
||||||
|
"displayName": "GPT-6-Astra",
|
||||||
|
"hidden": false,
|
||||||
|
"id": "gpt-6-astra",
|
||||||
|
"inputModalities": [
|
||||||
|
"text",
|
||||||
|
"image"
|
||||||
|
],
|
||||||
|
"isDefault": true,
|
||||||
|
"model": "gpt-6-astra",
|
||||||
|
"modelSpecialty": null,
|
||||||
|
"supportedReasoningEfforts": [
|
||||||
|
{
|
||||||
|
"description": "Fast responses with lighter reasoning",
|
||||||
|
"reasoningEffort": "low"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"description": "Balances speed and reasoning depth for everyday tasks",
|
||||||
|
"reasoningEffort": "medium"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"description": "Greater reasoning depth for complex problems",
|
||||||
|
"reasoningEffort": "high"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"description": "Extra high reasoning depth for complex problems",
|
||||||
|
"reasoningEffort": "xhigh"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"description": "Maximum reasoning depth for the hardest problems",
|
||||||
|
"reasoningEffort": "max"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"description": "Maximum reasoning with automatic task delegation",
|
||||||
|
"reasoningEffort": "ultra"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"upgrade": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"defaultReasoningEffort": "low",
|
||||||
|
"description": "Reliable agentic workhorse for everyday tasks.",
|
||||||
|
"displayName": "GPT-5.6-Sol",
|
||||||
|
"hidden": false,
|
||||||
|
"id": "gpt-5.6-sol",
|
||||||
|
"inputModalities": [
|
||||||
|
"text",
|
||||||
|
"image"
|
||||||
|
],
|
||||||
|
"isDefault": false,
|
||||||
|
"model": "gpt-5.6-sol",
|
||||||
|
"modelSpecialty": null,
|
||||||
|
"supportedReasoningEfforts": [
|
||||||
|
{
|
||||||
|
"description": "Fast responses with lighter reasoning",
|
||||||
|
"reasoningEffort": "low"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"description": "Balances speed and reasoning depth for everyday tasks",
|
||||||
|
"reasoningEffort": "medium"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"description": "Greater reasoning depth for complex problems",
|
||||||
|
"reasoningEffort": "high"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"description": "Extra high reasoning depth for complex problems",
|
||||||
|
"reasoningEffort": "xhigh"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"description": "Maximum reasoning depth for the hardest problems",
|
||||||
|
"reasoningEffort": "max"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"description": "Maximum reasoning with automatic task delegation",
|
||||||
|
"reasoningEffort": "ultra"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"upgrade": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"defaultReasoningEffort": "medium",
|
||||||
|
"description": "Balanced agentic coding model for everyday work.",
|
||||||
|
"displayName": "GPT-5.6-Terra",
|
||||||
|
"hidden": false,
|
||||||
|
"id": "gpt-5.6-terra",
|
||||||
|
"inputModalities": [
|
||||||
|
"text",
|
||||||
|
"image"
|
||||||
|
],
|
||||||
|
"isDefault": false,
|
||||||
|
"model": "gpt-5.6-terra",
|
||||||
|
"modelSpecialty": null,
|
||||||
|
"supportedReasoningEfforts": [
|
||||||
|
{
|
||||||
|
"description": "Fast responses with lighter reasoning",
|
||||||
|
"reasoningEffort": "low"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"description": "Balances speed and reasoning depth for everyday tasks",
|
||||||
|
"reasoningEffort": "medium"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"description": "Greater reasoning depth for complex problems",
|
||||||
|
"reasoningEffort": "high"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"description": "Extra high reasoning depth for complex problems",
|
||||||
|
"reasoningEffort": "xhigh"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"description": "Maximum reasoning depth for the hardest problems",
|
||||||
|
"reasoningEffort": "max"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"description": "Maximum reasoning with automatic task delegation",
|
||||||
|
"reasoningEffort": "ultra"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"upgrade": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"defaultReasoningEffort": "medium",
|
||||||
|
"description": "Fast and affordable agentic coding model.",
|
||||||
|
"displayName": "GPT-5.6-Luna",
|
||||||
|
"hidden": false,
|
||||||
|
"id": "gpt-5.6-luna",
|
||||||
|
"inputModalities": [
|
||||||
|
"text",
|
||||||
|
"image"
|
||||||
|
],
|
||||||
|
"isDefault": false,
|
||||||
|
"model": "gpt-5.6-luna",
|
||||||
|
"modelSpecialty": null,
|
||||||
|
"supportedReasoningEfforts": [
|
||||||
|
{
|
||||||
|
"description": "Fast responses with lighter reasoning",
|
||||||
|
"reasoningEffort": "low"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"description": "Balances speed and reasoning depth for everyday tasks",
|
||||||
|
"reasoningEffort": "medium"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"description": "Greater reasoning depth for complex problems",
|
||||||
|
"reasoningEffort": "high"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"description": "Extra high reasoning depth for complex problems",
|
||||||
|
"reasoningEffort": "xhigh"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"description": "Maximum reasoning depth for the hardest problems",
|
||||||
|
"reasoningEffort": "max"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"upgrade": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"defaultReasoningEffort": "medium",
|
||||||
|
"description": "Proven previous-generation model for coding and general work.",
|
||||||
|
"displayName": "GPT-5.5",
|
||||||
|
"hidden": false,
|
||||||
|
"id": "gpt-5.5",
|
||||||
|
"inputModalities": [
|
||||||
|
"text",
|
||||||
|
"image"
|
||||||
|
],
|
||||||
|
"isDefault": false,
|
||||||
|
"model": "gpt-5.5",
|
||||||
|
"modelSpecialty": null,
|
||||||
|
"supportedReasoningEfforts": [
|
||||||
|
{
|
||||||
|
"description": "Fast responses with lighter reasoning",
|
||||||
|
"reasoningEffort": "low"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"description": "Balances speed and reasoning depth for everyday tasks",
|
||||||
|
"reasoningEffort": "medium"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"description": "Greater reasoning depth for complex problems",
|
||||||
|
"reasoningEffort": "high"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"description": "Extra high reasoning depth for complex problems",
|
||||||
|
"reasoningEffort": "xhigh"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"upgrade": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"defaultReasoningEffort": "high",
|
||||||
|
"description": "Ultra-fast coding model.",
|
||||||
|
"displayName": "GPT-5.3-Codex-Spark",
|
||||||
|
"hidden": false,
|
||||||
|
"id": "gpt-5.3-codex-spark",
|
||||||
|
"inputModalities": [
|
||||||
|
"text"
|
||||||
|
],
|
||||||
|
"isDefault": false,
|
||||||
|
"model": "gpt-5.3-codex-spark",
|
||||||
|
"modelSpecialty": null,
|
||||||
|
"supportedReasoningEfforts": [
|
||||||
|
{
|
||||||
|
"description": "Fast responses with lighter reasoning",
|
||||||
|
"reasoningEffort": "low"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"description": "Balances speed and reasoning depth for everyday tasks",
|
||||||
|
"reasoningEffort": "medium"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"description": "Greater reasoning depth for complex problems",
|
||||||
|
"reasoningEffort": "high"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"description": "Extra high reasoning depth for complex problems",
|
||||||
|
"reasoningEffort": "xhigh"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"upgrade": null
|
||||||
|
}
|
||||||
|
]
|
||||||
106
testing/tests/test_claude_model_discovery.py
Normal file
106
testing/tests/test_claude_model_discovery.py
Normal file
@ -0,0 +1,106 @@
|
|||||||
|
"""Contract tests for native Claude subscription model discovery."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import importlib
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
SCRIPTS = Path(__file__).parents[2] / "services/hermes/scripts"
|
||||||
|
sys.path.insert(0, str(SCRIPTS))
|
||||||
|
discovery = importlib.import_module("claude_model_discovery")
|
||||||
|
|
||||||
|
|
||||||
|
def _initialize_event(*, provider: str = "firstParty") -> str:
|
||||||
|
"""Build representative non-secret CLI initialization output."""
|
||||||
|
return json.dumps(
|
||||||
|
{
|
||||||
|
"type": "control_response",
|
||||||
|
"response": {
|
||||||
|
"subtype": "success",
|
||||||
|
"request_id": "catalog",
|
||||||
|
"response": {
|
||||||
|
"account": {
|
||||||
|
"apiProvider": provider,
|
||||||
|
"email": "discard-me@example.test",
|
||||||
|
},
|
||||||
|
"models": [
|
||||||
|
{
|
||||||
|
"value": "opus",
|
||||||
|
"resolvedModel": "claude-opus-5",
|
||||||
|
"displayName": "Opus",
|
||||||
|
"description": "Most capable",
|
||||||
|
"supportsEffort": True,
|
||||||
|
"supportedEffortLevels": [
|
||||||
|
"low",
|
||||||
|
"high",
|
||||||
|
"xhigh",
|
||||||
|
"unknown",
|
||||||
|
],
|
||||||
|
"supportsAdaptiveThinking": True,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"value": "haiku",
|
||||||
|
"resolvedModel": "claude-haiku-4-5-20251001",
|
||||||
|
"displayName": "Haiku",
|
||||||
|
"description": "Fast",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_initialize_parses_live_subscription_models_without_account_data():
|
||||||
|
"""Literal aliases and the CLI-advertised effort set survive discovery."""
|
||||||
|
result = discovery._parse_initialize_output(_initialize_event())
|
||||||
|
|
||||||
|
assert result.live is True
|
||||||
|
assert result.connected is True
|
||||||
|
assert result.error_code is None
|
||||||
|
assert [model.value for model in result.models] == ["opus", "haiku"]
|
||||||
|
assert result.models[0].resolved_model == "claude-opus-5"
|
||||||
|
assert result.models[0].supported_effort_levels == ("low", "high", "xhigh")
|
||||||
|
assert "email" not in result.models[0].as_dict()
|
||||||
|
rendered_models = json.dumps([model.as_dict() for model in result.models])
|
||||||
|
assert "discard-me@example.test" not in rendered_models
|
||||||
|
|
||||||
|
|
||||||
|
def test_initialize_rejects_non_subscription_catalogs():
|
||||||
|
"""An API-key or third-party account cannot become a subscription route."""
|
||||||
|
result = discovery._parse_initialize_output(_initialize_event(provider="apiKey"))
|
||||||
|
|
||||||
|
assert result.live is False
|
||||||
|
assert result.connected is False
|
||||||
|
assert result.error_code == "not-first-party"
|
||||||
|
|
||||||
|
|
||||||
|
def test_discovery_never_uses_api_key_environment(monkeypatch):
|
||||||
|
"""The native call inherits OAuth config but strips metered API-key variables."""
|
||||||
|
observed: dict[str, object] = {}
|
||||||
|
|
||||||
|
def fake_run(command, environment, timeout_seconds):
|
||||||
|
observed["command"] = command
|
||||||
|
observed["environment"] = environment
|
||||||
|
observed["timeout"] = timeout_seconds
|
||||||
|
return _initialize_event(), None
|
||||||
|
|
||||||
|
monkeypatch.setattr(discovery, "_run_initialize", fake_run)
|
||||||
|
result = discovery.discover_claude_subscription_models(
|
||||||
|
claude_bin="/native/claude",
|
||||||
|
timeout_seconds=7,
|
||||||
|
environment={
|
||||||
|
"CLAUDE_CONFIG_DIR": "/mounted/claude",
|
||||||
|
"ANTHROPIC_API_KEY": "secret",
|
||||||
|
"CLAUDE_API_KEY": "secret",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result.live is True
|
||||||
|
assert observed["environment"] == {"CLAUDE_CONFIG_DIR": "/mounted/claude"}
|
||||||
|
assert observed["command"][:2] == ["/native/claude", "--input-format"]
|
||||||
|
assert "--safe-mode" in observed["command"]
|
||||||
|
assert "" in observed["command"]
|
||||||
@ -411,8 +411,8 @@ def test_local_flux_runtime_and_gpu_handoff_are_flux_managed():
|
|||||||
assert "route/local/qwen2.5-14b/medium" in switchyard
|
assert "route/local/qwen2.5-14b/medium" in switchyard
|
||||||
assert "Anthropic and Claude name the same provider" in switchyard
|
assert "Anthropic and Claude name the same provider" in switchyard
|
||||||
assert "OpenAI and Codex name the same provider" in switchyard
|
assert "OpenAI and Codex name the same provider" in switchyard
|
||||||
assert "Choose across every configured Codex and Claude family" in switchyard
|
assert "account-visible economy, balanced, or advanced" in switchyard
|
||||||
assert "Claude Fable" in switchyard
|
assert "Choose across every configured Codex and Claude family" not in switchyard
|
||||||
assert switchyard.count('Treat "think hard"') == 4
|
assert switchyard.count('Treat "think hard"') == 4
|
||||||
assert switchyard.count("Never choose below the") >= 5
|
assert switchyard.count("Never choose below the") >= 5
|
||||||
switchyard_config = tomllib.loads(switchyard)
|
switchyard_config = tomllib.loads(switchyard)
|
||||||
@ -420,13 +420,13 @@ def test_local_flux_runtime_and_gpu_handoff_are_flux_managed():
|
|||||||
configured_targets = switchyard_config["targets"]
|
configured_targets = switchyard_config["targets"]
|
||||||
for route_name in ("auto_fast", "auto_balanced", "auto_deep", "auto_maximum"):
|
for route_name in ("auto_fast", "auto_balanced", "auto_deep", "auto_maximum"):
|
||||||
leading_targets = set(routes[route_name]["targets"][:2])
|
leading_targets = set(routes[route_name]["targets"][:2])
|
||||||
assert leading_targets == {"codex_sol_xhigh", "claude_opus_xhigh"}
|
assert leading_targets == {"codex_auto_xhigh", "claude_auto_xhigh"}
|
||||||
assert "max_output_tokens" not in routes[route_name]
|
assert "max_output_tokens" not in routes[route_name]
|
||||||
for route_name in ("auto_fast", "auto_balanced", "auto_deep", "auto_maximum"):
|
for route_name in ("auto_fast", "auto_balanced", "auto_deep", "auto_maximum"):
|
||||||
targets = routes[route_name]["targets"]
|
targets = routes[route_name]["targets"]
|
||||||
selector_targets = routes[route_name]["response_schema"]
|
selector_targets = routes[route_name]["response_schema"]
|
||||||
assert not any(target.startswith("local_") for target in targets)
|
assert not any(target.startswith("local_") for target in targets)
|
||||||
assert any("fable" in target for target in targets)
|
assert all("_auto_" in target or target.startswith("neutral_") for target in targets)
|
||||||
assert "local_qwen" not in selector_targets
|
assert "local_qwen" not in selector_targets
|
||||||
assert "not eligible for foreground" in routes[route_name]["prompt"]
|
assert "not eligible for foreground" in routes[route_name]["prompt"]
|
||||||
for route_name in ("auto_deep", "auto_maximum"):
|
for route_name in ("auto_deep", "auto_maximum"):
|
||||||
@ -470,10 +470,11 @@ def test_local_flux_runtime_and_gpu_handoff_are_flux_managed():
|
|||||||
assert route["id"] == f"atlas/manual/{provider}/{family}/{effort}"
|
assert route["id"] == f"atlas/manual/{provider}/{family}/{effort}"
|
||||||
assert route["targets"][0] == f"{provider}_{family}_{effort}"
|
assert route["targets"][0] == f"{provider}_{family}_{effort}"
|
||||||
worker_target = f"worker_{provider}_{family}_{effort}"
|
worker_target = f"worker_{provider}_{family}_{effort}"
|
||||||
assert worker_target in routes["worker_auto_maximum"]["targets"]
|
|
||||||
assert configured_targets[worker_target]["id"] == (
|
assert configured_targets[worker_target]["id"] == (
|
||||||
f"worker/{provider}/{family}/{effort}"
|
f"worker/{provider}/{family}/{effort}"
|
||||||
)
|
)
|
||||||
|
assert all("_auto_" in target or target.startswith("neutral_")
|
||||||
|
for target in routes["worker_auto_maximum"]["targets"])
|
||||||
for route_name in ("auto_fast", "auto_balanced"):
|
for route_name in ("auto_fast", "auto_balanced"):
|
||||||
prompt = routes[route_name]["prompt"]
|
prompt = routes[route_name]["prompt"]
|
||||||
assert "image tool—not the conversational model" in prompt
|
assert "image tool—not the conversational model" in prompt
|
||||||
|
|||||||
@ -5,6 +5,8 @@ from __future__ import annotations
|
|||||||
import importlib.util
|
import importlib.util
|
||||||
import json
|
import json
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
|
import pytest
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
@ -79,3 +81,29 @@ def test_mismatched_broker_resolution_cannot_change_route_decision():
|
|||||||
"terra",
|
"terra",
|
||||||
"xhigh",
|
"xhigh",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_generic_claude_worker_keeps_a_catalog_advertised_alias():
|
||||||
|
"""The native CLI receives its provider alias after generic AUTO routing."""
|
||||||
|
route = lanes.select_route(
|
||||||
|
"Review it.",
|
||||||
|
"cli-auto",
|
||||||
|
open_request=lambda *_args, **_kwargs: SwitchyardResponse(
|
||||||
|
"worker/claude/auto/medium", "worker/claude/sonnet/medium"
|
||||||
|
),
|
||||||
|
catalog_model_allowed=lambda provider, model: provider == "claude" and model == "sonnet",
|
||||||
|
)
|
||||||
|
assert (route.provider, route.model, route.effort) == ("claude", "sonnet", "medium")
|
||||||
|
|
||||||
|
|
||||||
|
def test_generic_claude_worker_rejects_an_unadvertised_alias():
|
||||||
|
"""A forged worker receipt cannot pass an arbitrary native Claude alias."""
|
||||||
|
with pytest.raises(RuntimeError, match="current provider model"):
|
||||||
|
lanes.select_route(
|
||||||
|
"Review it.",
|
||||||
|
"cli-auto",
|
||||||
|
open_request=lambda *_args, **_kwargs: SwitchyardResponse(
|
||||||
|
"worker/claude/auto/medium", "worker/claude/not-advertised/medium"
|
||||||
|
),
|
||||||
|
catalog_model_allowed=lambda *_args: False,
|
||||||
|
)
|
||||||
|
|||||||
@ -24,10 +24,10 @@ def test_model_version_and_quality_selection_handle_new_and_small_models():
|
|||||||
codex = ["gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.7-luna", "gpt-5.3-codex-spark"]
|
codex = ["gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.7-luna", "gpt-5.3-codex-spark"]
|
||||||
assert routing.choose_codex_model(codex) == "gpt-5.6-sol"
|
assert routing.choose_codex_model(codex) == "gpt-5.6-sol"
|
||||||
assert routing.choose_codex_model(codex, balanced=True) == "gpt-5.6-terra"
|
assert routing.choose_codex_model(codex, balanced=True) == "gpt-5.6-terra"
|
||||||
assert routing.choose_codex_model(codex + ["gpt-5.7-terra"]) == "gpt-5.7-terra"
|
assert routing.choose_codex_model(codex + ["gpt-5.7-terra"]) == "gpt-5.6-sol"
|
||||||
|
|
||||||
claude = ["claude-opus-4.8", "claude-haiku-5", "claude-sonnet-5"]
|
claude = ["claude-opus-4.8", "claude-haiku-5", "claude-sonnet-5"]
|
||||||
assert routing.choose_claude_model(claude) == "claude-sonnet-5"
|
assert routing.choose_claude_model(claude) == "claude-opus-4.8"
|
||||||
|
|
||||||
assert routing.choose_codex_for_effort(codex, "low") == "gpt-5.7-luna"
|
assert routing.choose_codex_for_effort(codex, "low") == "gpt-5.7-luna"
|
||||||
assert routing.choose_codex_for_effort(codex, "medium") == "gpt-5.6-terra"
|
assert routing.choose_codex_for_effort(codex, "medium") == "gpt-5.6-terra"
|
||||||
@ -89,6 +89,284 @@ def test_dynamic_catalog_resolves_new_models_and_preserves_last_known_good():
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_dynamic_catalog_uses_metadata_for_unfamiliar_models_and_efforts():
|
||||||
|
"""Future IDs route by declared capability, price, and effort support."""
|
||||||
|
codex = routing.Catalog(
|
||||||
|
"openai-codex",
|
||||||
|
[
|
||||||
|
"gpt-6.2-orbit",
|
||||||
|
"gpt-6.2-balanced",
|
||||||
|
"gpt-6.2-quick",
|
||||||
|
"gpt-6.2-high-only",
|
||||||
|
"gpt-6.2-xhigh",
|
||||||
|
],
|
||||||
|
True,
|
||||||
|
True,
|
||||||
|
"connected",
|
||||||
|
{
|
||||||
|
"gpt-6.2-orbit": {
|
||||||
|
"description": "A strongest upgrade for difficult, complex tasks",
|
||||||
|
"upgrade": "strongest",
|
||||||
|
"supported_reasoning_efforts": ["low", "medium", "high", "xhigh"],
|
||||||
|
"cost": 10,
|
||||||
|
},
|
||||||
|
"gpt-6.2-balanced": {
|
||||||
|
"capability_tier": "balanced",
|
||||||
|
"description": "General purpose model for everyday tasks",
|
||||||
|
"supported_reasoning_efforts": ["low", "medium", "high", "xhigh"],
|
||||||
|
"cost": 2,
|
||||||
|
},
|
||||||
|
"gpt-6.2-quick": {
|
||||||
|
"capability_tier": "economy",
|
||||||
|
"description": "Fast and affordable general purpose model",
|
||||||
|
"supported_reasoning_efforts": ["low", "medium"],
|
||||||
|
"cost": 1,
|
||||||
|
},
|
||||||
|
"gpt-6.2-high-only": {
|
||||||
|
"capability_tier": "advanced",
|
||||||
|
"supported_reasoning_efforts": ["high"],
|
||||||
|
"cost": 1,
|
||||||
|
},
|
||||||
|
"gpt-6.2-xhigh": {
|
||||||
|
"capability_tier": "advanced",
|
||||||
|
"supported_reasoning_efforts": ["xhigh"],
|
||||||
|
"cost": 2,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
claude = routing.Catalog("anthropic", [], True, True, "connected")
|
||||||
|
|
||||||
|
catalog = routing.build_routing_catalog(codex, claude)
|
||||||
|
resolved = catalog["providers"]["codex"]["resolved"]
|
||||||
|
|
||||||
|
assert resolved["xhigh"] == "gpt-6.2-orbit"
|
||||||
|
assert resolved["low"] == "gpt-6.2-quick"
|
||||||
|
assert resolved["medium"] == "gpt-6.2-balanced"
|
||||||
|
assert resolved["high"] == "gpt-6.2-orbit"
|
||||||
|
assert "gpt-6.2-high-only" in catalog["providers"]["codex"]["candidates"]
|
||||||
|
assert catalog["providers"]["codex"]["candidates"]["gpt-6.2-high-only"]["tier"] == "advanced"
|
||||||
|
|
||||||
|
|
||||||
|
def test_catalog_accepts_codex_effort_records_and_generalist_image_input():
|
||||||
|
"""Codex app-server effort records and multimodal generalists stay routable."""
|
||||||
|
codex = routing.Catalog(
|
||||||
|
"openai-codex",
|
||||||
|
["gpt-7-nova"],
|
||||||
|
True,
|
||||||
|
True,
|
||||||
|
"connected-app-server",
|
||||||
|
{
|
||||||
|
"gpt-7-nova": {
|
||||||
|
"description": "Strongest general purpose model with text and image inputs",
|
||||||
|
"supportedReasoningEfforts": [
|
||||||
|
{"reasoningEffort": "low"},
|
||||||
|
{"reasoningEffort": "medium"},
|
||||||
|
{"reasoningEffort": "high"},
|
||||||
|
{"reasoningEffort": "xhigh"},
|
||||||
|
],
|
||||||
|
}
|
||||||
|
},
|
||||||
|
)
|
||||||
|
catalog = routing.build_routing_catalog(
|
||||||
|
codex, routing.Catalog("anthropic", [], True, True, "connected")
|
||||||
|
)
|
||||||
|
|
||||||
|
assert catalog["providers"]["codex"]["resolved"]["xhigh"] == "gpt-7-nova"
|
||||||
|
|
||||||
|
|
||||||
|
def test_dynamic_catalog_excludes_hidden_internal_and_specialist_records():
|
||||||
|
"""Provider listings must not make private or modality-only products routable."""
|
||||||
|
models = [
|
||||||
|
"gpt-6.2-hidden",
|
||||||
|
"gpt-6.2-internal",
|
||||||
|
"gpt-6.2-image",
|
||||||
|
"gpt-6.2-public",
|
||||||
|
]
|
||||||
|
metadata = {
|
||||||
|
"gpt-6.2-hidden": {
|
||||||
|
"hidden": True,
|
||||||
|
"capability_tier": "advanced",
|
||||||
|
},
|
||||||
|
"gpt-6.2-internal": {
|
||||||
|
"visibility": "internal",
|
||||||
|
"description": "Internal evaluation model",
|
||||||
|
"capability_tier": "advanced",
|
||||||
|
},
|
||||||
|
"gpt-6.2-image": {
|
||||||
|
"description": "Image generation specialist",
|
||||||
|
"capability_tier": "advanced",
|
||||||
|
},
|
||||||
|
"gpt-6.2-public": {
|
||||||
|
"description": "Strongest general purpose model",
|
||||||
|
"capability_tier": "advanced",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
codex = routing.Catalog(
|
||||||
|
"openai-codex", models, True, True, "connected", metadata
|
||||||
|
)
|
||||||
|
claude = routing.Catalog("anthropic", [], True, True, "connected")
|
||||||
|
|
||||||
|
catalog = routing.build_routing_catalog(codex, claude)
|
||||||
|
provider = catalog["providers"]["codex"]
|
||||||
|
|
||||||
|
assert provider["resolved"]["xhigh"] == "gpt-6.2-public"
|
||||||
|
assert provider["candidates"]["gpt-6.2-hidden"]["eligible"] is False
|
||||||
|
assert provider["candidates"]["gpt-6.2-internal"]["eligible"] is False
|
||||||
|
assert provider["candidates"]["gpt-6.2-image"]["eligible"] is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_live_codex_metadata_selects_luna_terra_and_astra_by_effort():
|
||||||
|
"""Actual Codex model/list fields route generic effort without name guesses."""
|
||||||
|
fixture = Path(__file__).parents[1] / "fixtures/hermes/codex-0.154-visible-models.json"
|
||||||
|
records = json.loads(fixture.read_text(encoding="utf-8"))
|
||||||
|
models, metadata = routing.model_records(records)
|
||||||
|
catalog = routing.build_routing_catalog(
|
||||||
|
routing.Catalog("openai-codex", models, True, True, "connected", metadata),
|
||||||
|
routing.Catalog("anthropic", [], True, True, "connected"),
|
||||||
|
)
|
||||||
|
assert catalog["providers"]["codex"]["resolved"] == {
|
||||||
|
"low": "gpt-5.6-luna",
|
||||||
|
"medium": "gpt-5.6-terra",
|
||||||
|
"high": "gpt-6-astra",
|
||||||
|
"xhigh": "gpt-6-astra",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_live_default_flagship_replaces_previous_advanced_route():
|
||||||
|
"""A provider's current default wins over an equally strongest old model."""
|
||||||
|
previous = {
|
||||||
|
"providers": {"codex": {"models": ["gpt-astra"], "resolved": {
|
||||||
|
effort: "gpt-astra" for effort in routing.EFFORTS
|
||||||
|
}, "model_metadata": {"gpt-astra": {
|
||||||
|
"description": "Most capable for complex work", "isDefault": False,
|
||||||
|
}}}}
|
||||||
|
}
|
||||||
|
metadata = {
|
||||||
|
"gpt-astra": {"description": "Most capable for complex work", "isDefault": False},
|
||||||
|
"gpt-nova": {"description": "Most capable for complex work", "isDefault": True},
|
||||||
|
}
|
||||||
|
catalog = routing.build_routing_catalog(
|
||||||
|
routing.Catalog("openai-codex", list(metadata), True, True, "connected", metadata),
|
||||||
|
routing.Catalog("anthropic", [], True, True, "connected"), previous,
|
||||||
|
)
|
||||||
|
assert catalog["providers"]["codex"]["resolved"]["xhigh"] == "gpt-nova"
|
||||||
|
|
||||||
|
|
||||||
|
def test_live_claude_effort_gap_uses_nearest_higher_declared_tier():
|
||||||
|
"""An effortless economy alias cannot silently take a stronger route."""
|
||||||
|
metadata = {
|
||||||
|
"haiku": {"description": "Fastest for quick answers", "supportsEffort": False},
|
||||||
|
"sonnet": {
|
||||||
|
"description": "Efficient for routine tasks", "supportsEffort": True,
|
||||||
|
"supportedEffortLevels": ["low", "medium", "high", "xhigh"],
|
||||||
|
},
|
||||||
|
"opus": {
|
||||||
|
"description": "Best for everyday, complex tasks", "supportsEffort": True,
|
||||||
|
"supportedEffortLevels": ["low", "medium", "high", "xhigh"],
|
||||||
|
},
|
||||||
|
"fable": {
|
||||||
|
"description": "Most capable for your hardest and longest-running tasks",
|
||||||
|
"supportsEffort": True,
|
||||||
|
"supportedEffortLevels": ["low", "medium", "high", "xhigh"],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
catalog = routing.build_routing_catalog(
|
||||||
|
routing.Catalog("openai-codex", [], True, True, "connected"),
|
||||||
|
routing.Catalog("anthropic", list(metadata), True, True, "connected", metadata),
|
||||||
|
)
|
||||||
|
assert catalog["providers"]["claude"]["resolved"] == {
|
||||||
|
"low": "sonnet", "medium": "sonnet", "high": "fable", "xhigh": "fable",
|
||||||
|
}
|
||||||
|
assert catalog["providers"]["claude"]["candidates"]["haiku"]["eligible"] is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_ambiguous_live_catalog_exposes_an_unresolved_route():
|
||||||
|
"""An unfamiliar live model cannot revive a retired LKG model."""
|
||||||
|
previous = {
|
||||||
|
"providers": {
|
||||||
|
"codex": {
|
||||||
|
"models": ["gpt-5.6-sol"],
|
||||||
|
"resolved": {effort: "gpt-5.6-sol" for effort in routing.EFFORTS},
|
||||||
|
"tiers": {"advanced": "gpt-5.6-sol"},
|
||||||
|
"model_metadata": {},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
codex = routing.Catalog(
|
||||||
|
"openai-codex", ["gpt-7.0-mystery"], True, True, "connected"
|
||||||
|
)
|
||||||
|
claude = routing.Catalog("anthropic", [], False, True, "degraded")
|
||||||
|
|
||||||
|
catalog = routing.build_routing_catalog(codex, claude, previous)
|
||||||
|
|
||||||
|
assert catalog["providers"]["codex"]["resolved"]["xhigh"] == ""
|
||||||
|
assert catalog["providers"]["codex"]["candidates"]["gpt-7.0-mystery"]["tier"] is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_live_removal_clears_legacy_selector_instead_of_mapping_to_new_family():
|
||||||
|
"""A live authoritative list cannot revive removed legacy selectors."""
|
||||||
|
previous = {
|
||||||
|
"providers": {
|
||||||
|
"codex": {
|
||||||
|
"models": ["gpt-5.6-sol"],
|
||||||
|
"resolved": {effort: "gpt-5.6-sol" for effort in routing.EFFORTS},
|
||||||
|
"tiers": {"sol": "gpt-5.6-sol", "advanced": "gpt-5.6-sol"},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
codex = routing.Catalog(
|
||||||
|
"openai-codex",
|
||||||
|
["gpt-6.2-astra"],
|
||||||
|
True,
|
||||||
|
True,
|
||||||
|
"connected",
|
||||||
|
{"gpt-6.2-astra": {"capability_tier": "advanced"}},
|
||||||
|
)
|
||||||
|
claude = routing.Catalog("anthropic", [], False, True, "degraded")
|
||||||
|
|
||||||
|
catalog = routing.build_routing_catalog(codex, claude, previous)
|
||||||
|
|
||||||
|
assert catalog["providers"]["codex"]["tiers"]["sol"] == ""
|
||||||
|
assert catalog["providers"]["codex"]["resolved"]["xhigh"] == "gpt-6.2-astra"
|
||||||
|
|
||||||
|
|
||||||
|
def test_degraded_catalog_preserves_lkg_after_live_removal():
|
||||||
|
"""A provider outage retains the prior route and does not treat it as removal."""
|
||||||
|
previous = {
|
||||||
|
"providers": {
|
||||||
|
"codex": {
|
||||||
|
"models": ["gpt-5.6-sol"],
|
||||||
|
"resolved": {effort: "gpt-5.6-sol" for effort in routing.EFFORTS},
|
||||||
|
"tiers": {"sol": "gpt-5.6-sol"},
|
||||||
|
"model_metadata": {},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
codex = routing.Catalog("openai-codex", [], False, True, "degraded")
|
||||||
|
claude = routing.Catalog("anthropic", [], False, True, "degraded")
|
||||||
|
|
||||||
|
catalog = routing.build_routing_catalog(codex, claude, previous)
|
||||||
|
|
||||||
|
assert catalog["providers"]["codex"]["resolved"]["xhigh"] == "gpt-5.6-sol"
|
||||||
|
assert catalog["providers"]["codex"]["tiers"]["sol"] == "gpt-5.6-sol"
|
||||||
|
|
||||||
|
|
||||||
|
def test_explicit_legacy_selector_never_maps_to_astra():
|
||||||
|
"""The old sol selector is exact and cannot silently follow a new family."""
|
||||||
|
catalog = {
|
||||||
|
"providers": {
|
||||||
|
"codex": {
|
||||||
|
"tiers": {"sol": "gpt-6.2-astra"},
|
||||||
|
"resolved": {"xhigh": "gpt-6.2-astra"},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
assert catalog_resolver.resolve_model("codex", "sol", "xhigh", catalog) == (
|
||||||
|
"gpt-5.6-sol"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_live_catalog_never_routes_to_a_removed_model_tier():
|
def test_live_catalog_never_routes_to_a_removed_model_tier():
|
||||||
"""A live provider catalog must replace a retired tier with a live model."""
|
"""A live provider catalog must replace a retired tier with a live model."""
|
||||||
previous = {
|
previous = {
|
||||||
@ -116,9 +394,8 @@ def test_live_catalog_never_routes_to_a_removed_model_tier():
|
|||||||
|
|
||||||
current = routing.build_routing_catalog(codex, claude, previous)
|
current = routing.build_routing_catalog(codex, claude, previous)
|
||||||
|
|
||||||
assert current["providers"]["codex"]["tiers"]["luna"] == "gpt-5.7-terra"
|
assert current["providers"]["codex"]["tiers"]["luna"] == ""
|
||||||
assert current["providers"]["codex"]["tiers"]["luna"] in codex.models
|
assert current["providers"]["claude"]["tiers"]["opus"] == ""
|
||||||
assert current["providers"]["claude"]["tiers"]["opus"] == "claude-sonnet-6"
|
|
||||||
|
|
||||||
|
|
||||||
def test_switchyard_targets_have_unique_upstream_identities():
|
def test_switchyard_targets_have_unique_upstream_identities():
|
||||||
@ -218,17 +495,17 @@ def test_configure_routes_keeps_every_profile_on_switchyard(tmp_path: Path):
|
|||||||
}
|
}
|
||||||
assert root["fallback_providers"] == []
|
assert root["fallback_providers"] == []
|
||||||
assert root["toolsets"] == ["kanban"]
|
assert root["toolsets"] == ["kanban"]
|
||||||
assert codex_profile["model"]["model"] == "atlas/manual/codex/sol"
|
assert codex_profile["model"]["model"] == "atlas/manual/codex/auto/high"
|
||||||
assert codex_profile["model"]["provider"] == "atlas-switchyard"
|
assert codex_profile["model"]["provider"] == "atlas-switchyard"
|
||||||
assert claude_profile["model"]["model"] == "atlas/manual/claude/sonnet"
|
assert claude_profile["model"]["model"] == "atlas/manual/claude/auto/high"
|
||||||
assert claude_profile["model"]["provider"] == "atlas-switchyard"
|
assert claude_profile["model"]["provider"] == "atlas-switchyard"
|
||||||
assert codex_profile["fallback_providers"] == []
|
assert codex_profile["fallback_providers"] == []
|
||||||
assert claude_profile["fallback_providers"] == []
|
assert claude_profile["fallback_providers"] == []
|
||||||
assert codex_profile["toolsets"] == []
|
assert codex_profile["toolsets"] == []
|
||||||
assert codex_profile["agent"]["reasoning_effort"] == "high"
|
assert codex_profile["agent"]["reasoning_effort"] == "high"
|
||||||
assert codex_xhigh_profile["fallback_providers"] == []
|
assert codex_xhigh_profile["fallback_providers"] == []
|
||||||
assert routes["codex-xhigh"] == ["atlas/manual/codex/sol"]
|
assert routes["codex-xhigh"] == ["atlas/manual/codex/auto/xhigh"]
|
||||||
assert routes["claude-xhigh"] == ["atlas/manual/claude/opus"]
|
assert routes["claude-xhigh"] == ["atlas/manual/claude/auto/xhigh"]
|
||||||
assert routes["synthesis-xhigh"] == ["atlas/auto/maximum"]
|
assert routes["synthesis-xhigh"] == ["atlas/auto/maximum"]
|
||||||
assert routes["coordinator"] == ["atlas/auto/maximum"]
|
assert routes["coordinator"] == ["atlas/auto/maximum"]
|
||||||
assert all(
|
assert all(
|
||||||
@ -285,7 +562,7 @@ def test_degraded_refresh_preserves_switchyard_worker_preference(tmp_path: Path)
|
|||||||
worker = yaml.safe_load(
|
worker = yaml.safe_load(
|
||||||
(tmp_path / "profiles/codex-high/config.yaml").read_text(encoding="utf-8")
|
(tmp_path / "profiles/codex-high/config.yaml").read_text(encoding="utf-8")
|
||||||
)
|
)
|
||||||
assert worker["model"]["model"] == "atlas/manual/codex/sol"
|
assert worker["model"]["model"] == "atlas/manual/codex/auto/high"
|
||||||
assert worker["model"]["provider"] == "atlas-switchyard"
|
assert worker["model"]["provider"] == "atlas-switchyard"
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user