463 lines
20 KiB
Python
463 lines
20 KiB
Python
#!/usr/bin/env python3
|
|
"""Normalize provider model metadata into safe Hermes routing candidates."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from dataclasses import dataclass, field
|
|
from typing import Any, Iterable
|
|
|
|
from model_evaluation_evidence import EVALUATION_VERSION, metadata_fingerprint
|
|
|
|
EFFORTS = ("low", "medium", "high", "xhigh")
|
|
CAPABILITY_ROLES = ("economy", "balanced", "advanced", "frontier")
|
|
# Retain the old name for callers while catalog schema 3 calls these roles.
|
|
CAPABILITY_TIERS = CAPABILITY_ROLES
|
|
EFFORT_TIERS = {
|
|
"low": "economy",
|
|
"medium": "balanced",
|
|
"high": "advanced",
|
|
"xhigh": "advanced",
|
|
}
|
|
REVIEWED_CAPABILITY_POLICY = {
|
|
"codex": {
|
|
"gpt-5.6-luna": "economy",
|
|
"gpt-5.6-terra": "balanced",
|
|
"gpt-5.6-sol": "advanced",
|
|
"gpt-6-astra": "frontier",
|
|
},
|
|
"claude": {
|
|
"haiku": "economy",
|
|
"sonnet": "balanced",
|
|
"opus": "advanced",
|
|
"fable": "frontier",
|
|
"claude-haiku-4-5": "economy",
|
|
"claude-sonnet-5": "balanced",
|
|
"claude-opus-5": "advanced",
|
|
"claude-fable-5": "frontier",
|
|
"claude-fable-5[1m]": "frontier",
|
|
"claude-opus-5[1m]": "advanced",
|
|
},
|
|
}
|
|
REVIEWED_POLICY_PROVENANCE = "reviewed-current-account-catalog"
|
|
LEGACY_DEPLOYED_MODEL_ROLES = {
|
|
"codex": {
|
|
"gpt-5.6-luna": "economy",
|
|
"gpt-5.6-terra": "balanced",
|
|
"gpt-5.6-sol": "advanced",
|
|
},
|
|
"claude": {
|
|
"haiku": "economy",
|
|
"sonnet": "balanced",
|
|
"opus": "advanced",
|
|
"fable": "frontier",
|
|
"claude-haiku-4-5": "economy",
|
|
"claude-haiku-4-5-20251001": "economy",
|
|
"claude-haiku-5": "economy",
|
|
"claude-sonnet-5": "balanced",
|
|
"claude-opus-4.8": "advanced",
|
|
"claude-opus-5": "advanced",
|
|
"claude-fable-5": "frontier",
|
|
"claude-fable-5[1m]": "frontier",
|
|
"claude-opus-5[1m]": "advanced",
|
|
},
|
|
}
|
|
LEGACY_SELECTOR_MODELS = {
|
|
"codex": {
|
|
"luna": frozenset({"gpt-5.6-luna"}),
|
|
"terra": frozenset({"gpt-5.6-terra"}),
|
|
"sol": frozenset({"gpt-5.6-sol"}),
|
|
},
|
|
"claude": {
|
|
"haiku": frozenset({"haiku", "claude-haiku-4-5", "claude-haiku-4-5-20251001", "claude-haiku-5"}),
|
|
"sonnet": frozenset({"sonnet", "claude-sonnet-5"}),
|
|
"opus": frozenset({"opus", "claude-opus-4.8", "claude-opus-5", "claude-opus-5[1m]"}),
|
|
"fable": frozenset({"fable", "claude-fable-5", "claude-fable-5[1m]"}),
|
|
},
|
|
}
|
|
SPECIALIST_MODEL_WORDS = frozenset(
|
|
{"audio", "embedding", "image", "moderation", "realtime", "speech", "transcrib", "tts"}
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Catalog:
|
|
"""Non-secret provider discovery result."""
|
|
|
|
provider: str
|
|
models: list[str]
|
|
live: bool
|
|
connected: bool
|
|
state: str
|
|
metadata: dict[str, dict[str, Any]] = field(default_factory=dict)
|
|
|
|
|
|
def unique_models(models: Iterable[str]) -> list[str]:
|
|
"""Return normalized model IDs without changing provider order."""
|
|
seen: set[str] = set()
|
|
result: list[str] = []
|
|
for model in models:
|
|
value = str(model or "").strip()
|
|
if value and value.lower() not in seen:
|
|
result.append(value)
|
|
seen.add(value.lower())
|
|
return result
|
|
|
|
|
|
def model_id(value: Any) -> str:
|
|
"""Extract a provider model ID from either a string or model-list record."""
|
|
if isinstance(value, str):
|
|
return value.strip()
|
|
if not isinstance(value, dict):
|
|
return ""
|
|
for key in ("id", "model", "model_id", "name", "value"):
|
|
candidate = value.get(key)
|
|
if isinstance(candidate, str) and candidate.strip():
|
|
return candidate.strip()
|
|
return ""
|
|
|
|
|
|
def public_metadata(value: Any) -> dict[str, Any]:
|
|
"""Keep provider routing fields while excluding unrelated response data."""
|
|
if not isinstance(value, dict):
|
|
return {}
|
|
allowed = {
|
|
"capabilities", "capability", "capability_tier", "cost", "cost_rank",
|
|
"created_at", "default_reasoning_effort", "description", "display_name", "displayName",
|
|
"disabled", "enabled", "hidden", "input_modalities", "inputModalities", "isDefault",
|
|
"is_default", "is_general_purpose", "isGeneralPurpose", "max_input_tokens", "max_tokens",
|
|
"output_modalities", "outputModalities", "pricing", "status",
|
|
"modelSpecialty", "resolvedModel", "supported_reasoning_efforts", "supportedReasoningEfforts", "supportedEffortLevels",
|
|
"supportsAdaptiveThinking", "supportsEffort", "tier", "upgrade",
|
|
"verified", "visibility",
|
|
}
|
|
return {key: value[key] for key in allowed if key in value}
|
|
|
|
|
|
def model_records(values: Iterable[Any]) -> tuple[list[str], dict[str, dict[str, Any]]]:
|
|
"""Normalize provider records into ordered IDs and safe per-model metadata."""
|
|
models: list[str] = []
|
|
metadata: dict[str, dict[str, Any]] = {}
|
|
for value in values:
|
|
model = model_id(value)
|
|
if not model:
|
|
continue
|
|
models.append(model)
|
|
if isinstance(value, dict):
|
|
metadata[model] = public_metadata(value)
|
|
unique = unique_models(models)
|
|
return unique, {model: metadata[model] for model in unique if model in metadata}
|
|
|
|
|
|
def merge_records(
|
|
*records: tuple[list[str], dict[str, dict[str, Any]]]
|
|
) -> tuple[list[str], dict[str, dict[str, Any]]]:
|
|
"""Merge ordered discovery sources, preferring metadata from later sources."""
|
|
models: list[str] = []
|
|
metadata: dict[str, dict[str, Any]] = {}
|
|
for source_models, source_metadata in records:
|
|
for model in source_models:
|
|
if model not in models:
|
|
models.append(model)
|
|
details = source_metadata.get(model)
|
|
if details:
|
|
metadata[model] = {**metadata.get(model, {}), **details}
|
|
return unique_models(models), metadata
|
|
|
|
|
|
def model_version(model: str) -> tuple[int, ...]:
|
|
"""Extract a sortable model version while ignoring dated aliases."""
|
|
value = re.sub(r"-\d{8}$", "", model.lower())
|
|
groups = re.findall(r"\d+(?:\.\d+)*", value)
|
|
parts: list[int] = []
|
|
for group in groups:
|
|
parts.extend(int(piece) for piece in group.split("."))
|
|
return tuple(parts) or (0,)
|
|
|
|
|
|
def _text_metadata(metadata: dict[str, Any]) -> str:
|
|
values: list[str] = []
|
|
for key in ("description", "display_name", "capability", "tier"):
|
|
value = metadata.get(key)
|
|
if isinstance(value, str):
|
|
values.append(value.lower())
|
|
capabilities = metadata.get("capabilities")
|
|
if isinstance(capabilities, list):
|
|
values.extend(str(value).lower() for value in capabilities)
|
|
return " ".join(values)
|
|
|
|
|
|
def proposed_candidate_role(
|
|
provider: str, model: str, metadata: dict[str, Any]
|
|
) -> tuple[str | None, str]:
|
|
"""Propose an evaluation role from provider metadata, never a model ID."""
|
|
for key in ("capability_tier", "tier", "capability"):
|
|
value = metadata.get(key)
|
|
if isinstance(value, str):
|
|
normalized = value.strip().lower()
|
|
if normalized in {"economy", "cheap", "fast", "low_cost"}:
|
|
return "economy", f"provider {key}={value!r}"
|
|
if normalized in {"balanced", "standard", "mid", "medium"}:
|
|
return "balanced", f"provider {key}={value!r}"
|
|
if normalized in {"advanced", "premium", "high"}:
|
|
return "advanced", f"provider {key}={value!r}"
|
|
if normalized in {"frontier", "flagship", "strongest", "most_capable"}:
|
|
return "frontier", f"provider {key}={value!r}"
|
|
text = _text_metadata(metadata)
|
|
if any(term in text for term in ("most capable", "most intelligent", "state of the art", "frontier")):
|
|
return "frontier", "provider description marks it provider-declared frontier"
|
|
if "strongest" in text and not any(term in text for term in ("near-frontier", "fastest")):
|
|
return "frontier", "provider description marks it provider-declared strongest"
|
|
if any(term in text for term in ("complex tasks", "hardest tasks", "demanding work")):
|
|
return "advanced", "provider description marks it advanced"
|
|
if any(term in text for term in ("balanced", "general purpose", "everyday tasks", "routine tasks")):
|
|
return "balanced", "provider description marks it balanced/general-purpose"
|
|
if any(term in text for term in ("lowest cost", "low cost", "cheapest", "fastest", "fast and affordable")):
|
|
return "economy", "provider description marks it low-cost/fast"
|
|
return None, "provider metadata does not declare a capability tier"
|
|
|
|
|
|
def metadata_tier(metadata: dict[str, Any]) -> tuple[str | None, str]:
|
|
"""Compatibility wrapper for provider-declared candidate classification."""
|
|
return proposed_candidate_role("", "", metadata)
|
|
|
|
|
|
def legacy_tier(provider: str, model: str) -> tuple[str | None, str]:
|
|
"""Migrate deployed pre-metadata families without predicting new IDs."""
|
|
tier = LEGACY_DEPLOYED_MODEL_ROLES.get(provider, {}).get(model.strip().lower())
|
|
if tier is not None:
|
|
return tier, "legacy reviewed-model compatibility"
|
|
return None, "no reviewed provider capability metadata"
|
|
|
|
|
|
def legacy_selector_matches(provider: str, selector: str, model: str) -> bool:
|
|
"""Match a historical selector only to its reviewed, exact model aliases."""
|
|
return model.strip().lower() in LEGACY_SELECTOR_MODELS.get(provider, {}).get(selector, ())
|
|
|
|
|
|
def capability_role(
|
|
provider: str, model: str, metadata: dict[str, Any]
|
|
) -> tuple[str | None, str]:
|
|
"""Return an active role from reviewed policy or verified evaluation."""
|
|
reviewed = REVIEWED_CAPABILITY_POLICY.get(provider, {}).get(model.strip().lower())
|
|
if reviewed:
|
|
return reviewed, REVIEWED_POLICY_PROVENANCE
|
|
# Claude Code exposes executable aliases (for example ``opus[1m]``) and
|
|
# separately attests their concrete native model. Admit an alias only
|
|
# when that attested value is itself an exact reviewed model ID.
|
|
resolved = metadata.get("resolvedModel") if provider == "claude" else None
|
|
if isinstance(resolved, str):
|
|
reviewed = REVIEWED_CAPABILITY_POLICY["claude"].get(resolved.strip().lower())
|
|
if reviewed:
|
|
return reviewed, "reviewed native Claude alias"
|
|
evaluated = metadata.get("evaluated_capability_role")
|
|
if isinstance(evaluated, str) and evaluated in CAPABILITY_ROLES:
|
|
return evaluated, str(metadata.get("evaluation_provenance") or "verified evaluation")
|
|
proposed, reason = proposed_candidate_role(provider, model, metadata)
|
|
if proposed:
|
|
return None, f"candidate pending evaluation: {reason}"
|
|
return None, reason
|
|
|
|
|
|
def _model_cost(metadata: dict[str, Any]) -> tuple[str, float] | None:
|
|
"""Return a cost only with its provider-declared comparable unit."""
|
|
for key in ("cost_rank", "cost"):
|
|
rank = metadata.get(key)
|
|
if not isinstance(rank, bool) and isinstance(rank, (int, float)) and rank >= 0:
|
|
return "ordinal", float(rank)
|
|
pricing = metadata.get("pricing")
|
|
sources = [pricing, metadata] if isinstance(pricing, dict) else [metadata]
|
|
for source in sources:
|
|
input_cost = source.get("input_cost_per_million")
|
|
output_cost = source.get("output_cost_per_million")
|
|
if (
|
|
not isinstance(input_cost, bool)
|
|
and not isinstance(output_cost, bool)
|
|
and isinstance(input_cost, (int, float))
|
|
and isinstance(output_cost, (int, float))
|
|
and input_cost >= 0
|
|
and output_cost >= 0
|
|
):
|
|
return "per-million", float(input_cost) + float(output_cost)
|
|
return None
|
|
|
|
|
|
def _comparable_costs(
|
|
pool: Iterable[tuple[str, dict[str, Any], str]]
|
|
) -> dict[str, float]:
|
|
"""Use cost only when every adequate candidate advertises one shared unit."""
|
|
records = [(model, _model_cost(details)) for model, details, _ in pool]
|
|
units = {cost[0] for _, cost in records if cost is not None}
|
|
if len(units) != 1 or any(cost is None for _, cost in records):
|
|
return {}
|
|
return {model: cost[1] for model, cost in records if cost is not None}
|
|
|
|
def supported_efforts(metadata: dict[str, Any]) -> frozenset[str]:
|
|
"""Normalize provider effort spellings without implying unsupported levels."""
|
|
supported = metadata.get(
|
|
"supported_reasoning_efforts",
|
|
metadata.get("supportedReasoningEfforts", metadata.get("supportedEffortLevels")),
|
|
)
|
|
if not isinstance(supported, list):
|
|
return frozenset()
|
|
values = {
|
|
str(item.get("reasoningEffort") or item.get("effort") or item.get("level") or "").lower()
|
|
if isinstance(item, dict) else str(item).lower()
|
|
for item in supported
|
|
}
|
|
return frozenset(value for value in values if value in EFFORTS)
|
|
|
|
|
|
def is_eligible(
|
|
provider: str, model: str, metadata: dict[str, Any], effort: str | None
|
|
) -> tuple[bool, str]:
|
|
"""Check public/general-purpose status and optionally exact effort support."""
|
|
if metadata.get("hidden") is True:
|
|
return False, "provider marks model hidden"
|
|
if str(metadata.get("visibility") or "").lower() in {"hidden", "internal", "private"}:
|
|
return False, "provider marks model non-public"
|
|
if metadata.get("enabled") is False or metadata.get("disabled") is True:
|
|
return False, "provider marks model disabled"
|
|
if str(metadata.get("status") or "").lower() in {"deprecated", "disabled", "retired"}:
|
|
return False, "provider marks model unavailable"
|
|
if metadata.get("is_general_purpose") is False or metadata.get("isGeneralPurpose") is False:
|
|
return False, "provider marks model specialist"
|
|
if metadata.get("modelSpecialty") not in (None, "", False):
|
|
return False, "provider marks model specialist"
|
|
modalities = metadata.get("input_modalities", metadata.get("inputModalities"))
|
|
if isinstance(modalities, list) and modalities and "text" not in {str(item).lower() for item in modalities}:
|
|
return False, "provider record is not text-capable"
|
|
if "specialist" in _text_metadata(metadata):
|
|
return False, "provider description identifies a specialist model"
|
|
if any(word in model.lower() for word in SPECIALIST_MODEL_WORDS):
|
|
return False, "model ID identifies a specialist model"
|
|
if effort is not None and metadata.get("supportsEffort") is False:
|
|
return False, "provider does not advertise effort support"
|
|
advertised_efforts = supported_efforts(metadata)
|
|
if effort is not None and advertised_efforts and effort not in advertised_efforts:
|
|
return False, f"provider does not support {effort} effort"
|
|
if provider == "codex" and not model.lower().startswith("gpt-"):
|
|
return False, "model ID is outside the Codex general route namespace"
|
|
return True, "eligible"
|
|
|
|
|
|
_candidate_status = is_eligible
|
|
|
|
|
|
def capability_pool(
|
|
provider: str, models: Iterable[str], metadata: dict[str, dict[str, Any]], role: str
|
|
) -> list[str]:
|
|
"""Return all public, general-purpose candidates for one capability role."""
|
|
if role not in CAPABILITY_ROLES:
|
|
return []
|
|
result: list[str] = []
|
|
for model in unique_models(models):
|
|
eligible, _ = is_eligible(provider, model, metadata.get(model, {}), None)
|
|
classified, _ = capability_role(provider, model, metadata.get(model, {}))
|
|
if eligible and classified == role:
|
|
result.append(model)
|
|
return result
|
|
|
|
|
|
def apply_verified_evaluations(
|
|
provider: str, models: Iterable[str], metadata: dict[str, dict[str, Any]],
|
|
evaluations: dict[str, Any] | None,
|
|
) -> dict[str, dict[str, Any]]:
|
|
"""Add only current verified evidence for still-advertised model records."""
|
|
advertised = set(unique_models(models))
|
|
result = {model: dict(metadata.get(model, {})) for model in advertised}
|
|
records = evaluations.get(provider, {}) if isinstance(evaluations, dict) else {}
|
|
if not isinstance(records, dict):
|
|
return result
|
|
for model, record in records.items():
|
|
if not isinstance(model, str) or model not in advertised or not isinstance(record, dict):
|
|
continue
|
|
if not (
|
|
record.get("proposed_role") in CAPABILITY_ROLES
|
|
and record.get("result") == "pass"
|
|
and record.get("role_fit") == "verified"
|
|
and record.get("eval_version") == EVALUATION_VERSION
|
|
and record.get("metadata_fingerprint") == metadata_fingerprint(model, result[model])
|
|
):
|
|
continue
|
|
result[model]["evaluated_capability_role"] = record["proposed_role"]
|
|
result[model]["evaluation_provenance"] = "bounded-representative-eval-v1"
|
|
return result
|
|
|
|
|
|
def select_tier_model(
|
|
provider: str, models: Iterable[str], metadata: dict[str, dict[str, Any]],
|
|
desired: str, effort: str, current: str, *, allow_current_fallback: bool = True,
|
|
legacy_compat: bool = False,
|
|
) -> tuple[str, dict[str, Any]]:
|
|
"""Select within one role, retaining current only during discovery outages."""
|
|
if desired not in CAPABILITY_ROLES:
|
|
raise ValueError(f"unsupported capability role: {desired}")
|
|
candidates: list[tuple[str, dict[str, Any], str]] = []
|
|
exact_declared = False
|
|
exact_available = False
|
|
observations: dict[str, Any] = {}
|
|
for model in unique_models(models):
|
|
details = metadata.get(model, {})
|
|
eligible, reason = is_eligible(provider, model, details, effort)
|
|
tier, tier_reason = capability_role(provider, model, details)
|
|
if tier is None and legacy_compat:
|
|
tier, tier_reason = legacy_tier(provider, model)
|
|
if tier == desired:
|
|
exact_declared = True
|
|
exact_available = exact_available or is_eligible(
|
|
provider, model, details, None
|
|
)[0]
|
|
proposed, proposed_reason = proposed_candidate_role(provider, model, details)
|
|
cost = _model_cost(details)
|
|
observations[model] = {
|
|
"eligible": eligible,
|
|
"reason": reason if not eligible else tier_reason,
|
|
"tier": tier,
|
|
"proposed_role": proposed,
|
|
"proposed_reason": proposed_reason,
|
|
"cost": {"unit": cost[0], "value": cost[1]} if cost else None,
|
|
}
|
|
if eligible and tier:
|
|
candidates.append((model, details, tier))
|
|
exact = [item for item in candidates if item[2] == desired]
|
|
tier_order = {role: index for index, role in enumerate(CAPABILITY_ROLES)}
|
|
# Economy/balanced may use the nearest adequate non-frontier role when a
|
|
# provider cannot express an effort. Advanced and frontier remain strict:
|
|
# an automatic hard-task route must not quietly promote to Astra or demote.
|
|
higher = [
|
|
item for item in candidates
|
|
if desired in {"economy", "balanced"}
|
|
and tier_order[item[2]] > tier_order[desired]
|
|
and item[2] != "frontier"
|
|
]
|
|
nearest_higher = min(
|
|
(tier_order[item[2]] for item in higher), default=None
|
|
)
|
|
# A fresh catalog that still declares the desired role but marks every
|
|
# member unavailable is a provider-side availability signal. Do not turn
|
|
# that into a more capable route; a caller may retain last-known-good only
|
|
# when its whole discovery read was unavailable. Promotion remains valid
|
|
# when no such role exists, or when an available exact role lacks only the
|
|
# requested effort support.
|
|
permit_higher = not exact_declared or exact_available
|
|
pool = exact or (
|
|
[item for item in higher if tier_order[item[2]] == nearest_higher]
|
|
if permit_higher and nearest_higher is not None else []
|
|
)
|
|
if not pool:
|
|
return (current if allow_current_fallback else ""), observations
|
|
comparable_costs = _comparable_costs(pool)
|
|
|
|
def key(item: tuple[str, dict[str, Any], str]) -> tuple[int, float, int, str]:
|
|
model, details, tier = item
|
|
return (
|
|
0 if tier == desired else 1,
|
|
comparable_costs.get(model, float("inf")),
|
|
# A verified evaluation establishes role fit; cost decides between
|
|
# adequate peers when provider metadata supplies one common unit.
|
|
0 if model == current else 1,
|
|
model,
|
|
)
|
|
return min(pool, key=key)[0], observations
|