#!/usr/bin/env python3 """Normalize provider model metadata into safe Hermes routing candidates.""" from __future__ import annotations import re from dataclasses import dataclass, field from typing import Any, Iterable from model_evaluation_evidence import EVALUATION_VERSION, metadata_fingerprint EFFORTS = ("low", "medium", "high", "xhigh") CAPABILITY_ROLES = ("economy", "balanced", "advanced", "frontier") # Retain the old name for callers while catalog schema 3 calls these roles. CAPABILITY_TIERS = CAPABILITY_ROLES EFFORT_TIERS = { "low": "economy", "medium": "balanced", "high": "advanced", "xhigh": "advanced", } REVIEWED_CAPABILITY_POLICY = { "codex": { "gpt-5.6-luna": "economy", "gpt-5.6-terra": "balanced", "gpt-5.6-sol": "advanced", "gpt-6-astra": "frontier", }, "claude": { "haiku": "economy", "sonnet": "balanced", "opus": "advanced", "fable": "frontier", "claude-haiku-4-5": "economy", "claude-sonnet-5": "balanced", "claude-opus-5": "advanced", "claude-fable-5": "frontier", "claude-fable-5[1m]": "frontier", "claude-opus-5[1m]": "advanced", }, } REVIEWED_POLICY_PROVENANCE = "reviewed-current-account-catalog" LEGACY_DEPLOYED_MODEL_ROLES = { "codex": { "gpt-5.6-luna": "economy", "gpt-5.6-terra": "balanced", "gpt-5.6-sol": "advanced", }, "claude": { "haiku": "economy", "sonnet": "balanced", "opus": "advanced", "fable": "frontier", "claude-haiku-4-5": "economy", "claude-haiku-4-5-20251001": "economy", "claude-haiku-5": "economy", "claude-sonnet-5": "balanced", "claude-opus-4.8": "advanced", "claude-opus-5": "advanced", "claude-fable-5": "frontier", "claude-fable-5[1m]": "frontier", "claude-opus-5[1m]": "advanced", }, } LEGACY_SELECTOR_MODELS = { "codex": { "luna": frozenset({"gpt-5.6-luna"}), "terra": frozenset({"gpt-5.6-terra"}), "sol": frozenset({"gpt-5.6-sol"}), }, "claude": { "haiku": frozenset({"haiku", "claude-haiku-4-5", "claude-haiku-4-5-20251001", "claude-haiku-5"}), "sonnet": frozenset({"sonnet", "claude-sonnet-5"}), "opus": frozenset({"opus", "claude-opus-4.8", "claude-opus-5", "claude-opus-5[1m]"}), "fable": frozenset({"fable", "claude-fable-5", "claude-fable-5[1m]"}), }, } SPECIALIST_MODEL_WORDS = frozenset( {"audio", "embedding", "image", "moderation", "realtime", "speech", "transcrib", "tts"} ) @dataclass(frozen=True) class Catalog: """Non-secret provider discovery result.""" provider: str models: list[str] live: bool connected: bool state: str metadata: dict[str, dict[str, Any]] = field(default_factory=dict) def unique_models(models: Iterable[str]) -> list[str]: """Return normalized model IDs without changing provider order.""" seen: set[str] = set() result: list[str] = [] for model in models: value = str(model or "").strip() if value and value.lower() not in seen: result.append(value) seen.add(value.lower()) return result def model_id(value: Any) -> str: """Extract a provider model ID from either a string or model-list record.""" if isinstance(value, str): return value.strip() if not isinstance(value, dict): return "" for key in ("id", "model", "model_id", "name", "value"): candidate = value.get(key) if isinstance(candidate, str) and candidate.strip(): return candidate.strip() return "" def public_metadata(value: Any) -> dict[str, Any]: """Keep provider routing fields while excluding unrelated response data.""" if not isinstance(value, dict): return {} allowed = { "capabilities", "capability", "capability_tier", "cost", "cost_rank", "created_at", "default_reasoning_effort", "description", "display_name", "displayName", "disabled", "enabled", "hidden", "input_modalities", "inputModalities", "isDefault", "is_default", "is_general_purpose", "isGeneralPurpose", "max_input_tokens", "max_tokens", "output_modalities", "outputModalities", "pricing", "status", "modelSpecialty", "resolvedModel", "supported_reasoning_efforts", "supportedReasoningEfforts", "supportedEffortLevels", "supportsAdaptiveThinking", "supportsEffort", "tier", "upgrade", "verified", "visibility", } return {key: value[key] for key in allowed if key in value} def model_records(values: Iterable[Any]) -> tuple[list[str], dict[str, dict[str, Any]]]: """Normalize provider records into ordered IDs and safe per-model metadata.""" models: list[str] = [] metadata: dict[str, dict[str, Any]] = {} for value in values: model = model_id(value) if not model: continue models.append(model) if isinstance(value, dict): metadata[model] = public_metadata(value) unique = unique_models(models) return unique, {model: metadata[model] for model in unique if model in metadata} def merge_records( *records: tuple[list[str], dict[str, dict[str, Any]]] ) -> tuple[list[str], dict[str, dict[str, Any]]]: """Merge ordered discovery sources, preferring metadata from later sources.""" models: list[str] = [] metadata: dict[str, dict[str, Any]] = {} for source_models, source_metadata in records: for model in source_models: if model not in models: models.append(model) details = source_metadata.get(model) if details: metadata[model] = {**metadata.get(model, {}), **details} return unique_models(models), metadata def model_version(model: str) -> tuple[int, ...]: """Extract a sortable model version while ignoring dated aliases.""" value = re.sub(r"-\d{8}$", "", model.lower()) groups = re.findall(r"\d+(?:\.\d+)*", value) parts: list[int] = [] for group in groups: parts.extend(int(piece) for piece in group.split(".")) return tuple(parts) or (0,) def _text_metadata(metadata: dict[str, Any]) -> str: values: list[str] = [] for key in ("description", "display_name", "capability", "tier"): value = metadata.get(key) if isinstance(value, str): values.append(value.lower()) capabilities = metadata.get("capabilities") if isinstance(capabilities, list): values.extend(str(value).lower() for value in capabilities) return " ".join(values) def proposed_candidate_role( provider: str, model: str, metadata: dict[str, Any] ) -> tuple[str | None, str]: """Propose an evaluation role from provider metadata, never a model ID.""" for key in ("capability_tier", "tier", "capability"): value = metadata.get(key) if isinstance(value, str): normalized = value.strip().lower() if normalized in {"economy", "cheap", "fast", "low_cost"}: return "economy", f"provider {key}={value!r}" if normalized in {"balanced", "standard", "mid", "medium"}: return "balanced", f"provider {key}={value!r}" if normalized in {"advanced", "premium", "high"}: return "advanced", f"provider {key}={value!r}" if normalized in {"frontier", "flagship", "strongest", "most_capable"}: return "frontier", f"provider {key}={value!r}" text = _text_metadata(metadata) if any(term in text for term in ("most capable", "most intelligent", "state of the art", "frontier")): return "frontier", "provider description marks it provider-declared frontier" if "strongest" in text and not any(term in text for term in ("near-frontier", "fastest")): return "frontier", "provider description marks it provider-declared strongest" if any(term in text for term in ("complex tasks", "hardest tasks", "demanding work")): return "advanced", "provider description marks it advanced" if any(term in text for term in ("balanced", "general purpose", "everyday tasks", "routine tasks")): return "balanced", "provider description marks it balanced/general-purpose" if any(term in text for term in ("lowest cost", "low cost", "cheapest", "fastest", "fast and affordable")): return "economy", "provider description marks it low-cost/fast" return None, "provider metadata does not declare a capability tier" def metadata_tier(metadata: dict[str, Any]) -> tuple[str | None, str]: """Compatibility wrapper for provider-declared candidate classification.""" return proposed_candidate_role("", "", metadata) def legacy_tier(provider: str, model: str) -> tuple[str | None, str]: """Migrate deployed pre-metadata families without predicting new IDs.""" tier = LEGACY_DEPLOYED_MODEL_ROLES.get(provider, {}).get(model.strip().lower()) if tier is not None: return tier, "legacy reviewed-model compatibility" return None, "no reviewed provider capability metadata" def legacy_selector_matches(provider: str, selector: str, model: str) -> bool: """Match a historical selector only to its reviewed, exact model aliases.""" return model.strip().lower() in LEGACY_SELECTOR_MODELS.get(provider, {}).get(selector, ()) def capability_role( provider: str, model: str, metadata: dict[str, Any] ) -> tuple[str | None, str]: """Return an active role from reviewed policy or verified evaluation.""" reviewed = REVIEWED_CAPABILITY_POLICY.get(provider, {}).get(model.strip().lower()) if reviewed: return reviewed, REVIEWED_POLICY_PROVENANCE # Claude Code exposes executable aliases (for example ``opus[1m]``) and # separately attests their concrete native model. Admit an alias only # when that attested value is itself an exact reviewed model ID. resolved = metadata.get("resolvedModel") if provider == "claude" else None if isinstance(resolved, str): reviewed = REVIEWED_CAPABILITY_POLICY["claude"].get(resolved.strip().lower()) if reviewed: return reviewed, "reviewed native Claude alias" evaluated = metadata.get("evaluated_capability_role") if isinstance(evaluated, str) and evaluated in CAPABILITY_ROLES: return evaluated, str(metadata.get("evaluation_provenance") or "verified evaluation") proposed, reason = proposed_candidate_role(provider, model, metadata) if proposed: return None, f"candidate pending evaluation: {reason}" return None, reason def _model_cost(metadata: dict[str, Any]) -> tuple[str, float] | None: """Return a cost only with its provider-declared comparable unit.""" for key in ("cost_rank", "cost"): rank = metadata.get(key) if not isinstance(rank, bool) and isinstance(rank, (int, float)) and rank >= 0: return "ordinal", float(rank) pricing = metadata.get("pricing") sources = [pricing, metadata] if isinstance(pricing, dict) else [metadata] for source in sources: input_cost = source.get("input_cost_per_million") output_cost = source.get("output_cost_per_million") if ( not isinstance(input_cost, bool) and not isinstance(output_cost, bool) and isinstance(input_cost, (int, float)) and isinstance(output_cost, (int, float)) and input_cost >= 0 and output_cost >= 0 ): return "per-million", float(input_cost) + float(output_cost) return None def _comparable_costs( pool: Iterable[tuple[str, dict[str, Any], str]] ) -> dict[str, float]: """Use cost only when every adequate candidate advertises one shared unit.""" records = [(model, _model_cost(details)) for model, details, _ in pool] units = {cost[0] for _, cost in records if cost is not None} if len(units) != 1 or any(cost is None for _, cost in records): return {} return {model: cost[1] for model, cost in records if cost is not None} def supported_efforts(metadata: dict[str, Any]) -> frozenset[str]: """Normalize provider effort spellings without implying unsupported levels.""" supported = metadata.get( "supported_reasoning_efforts", metadata.get("supportedReasoningEfforts", metadata.get("supportedEffortLevels")), ) if not isinstance(supported, list): return frozenset() values = { str(item.get("reasoningEffort") or item.get("effort") or item.get("level") or "").lower() if isinstance(item, dict) else str(item).lower() for item in supported } return frozenset(value for value in values if value in EFFORTS) def is_eligible( provider: str, model: str, metadata: dict[str, Any], effort: str | None ) -> tuple[bool, str]: """Check public/general-purpose status and optionally exact effort support.""" if metadata.get("hidden") is True: return False, "provider marks model hidden" if str(metadata.get("visibility") or "").lower() in {"hidden", "internal", "private"}: return False, "provider marks model non-public" if metadata.get("enabled") is False or metadata.get("disabled") is True: return False, "provider marks model disabled" if str(metadata.get("status") or "").lower() in {"deprecated", "disabled", "retired"}: return False, "provider marks model unavailable" if metadata.get("is_general_purpose") is False or metadata.get("isGeneralPurpose") is False: return False, "provider marks model specialist" if metadata.get("modelSpecialty") not in (None, "", False): return False, "provider marks model specialist" modalities = metadata.get("input_modalities", metadata.get("inputModalities")) if isinstance(modalities, list) and modalities and "text" not in {str(item).lower() for item in modalities}: return False, "provider record is not text-capable" if "specialist" in _text_metadata(metadata): return False, "provider description identifies a specialist model" if any(word in model.lower() for word in SPECIALIST_MODEL_WORDS): return False, "model ID identifies a specialist model" if effort is not None and metadata.get("supportsEffort") is False: return False, "provider does not advertise effort support" advertised_efforts = supported_efforts(metadata) if effort is not None and advertised_efforts and effort not in advertised_efforts: return False, f"provider does not support {effort} effort" if provider == "codex" and not model.lower().startswith("gpt-"): return False, "model ID is outside the Codex general route namespace" return True, "eligible" _candidate_status = is_eligible def capability_pool( provider: str, models: Iterable[str], metadata: dict[str, dict[str, Any]], role: str ) -> list[str]: """Return all public, general-purpose candidates for one capability role.""" if role not in CAPABILITY_ROLES: return [] result: list[str] = [] for model in unique_models(models): eligible, _ = is_eligible(provider, model, metadata.get(model, {}), None) classified, _ = capability_role(provider, model, metadata.get(model, {})) if eligible and classified == role: result.append(model) return result def apply_verified_evaluations( provider: str, models: Iterable[str], metadata: dict[str, dict[str, Any]], evaluations: dict[str, Any] | None, ) -> dict[str, dict[str, Any]]: """Add only current verified evidence for still-advertised model records.""" advertised = set(unique_models(models)) result = {model: dict(metadata.get(model, {})) for model in advertised} records = evaluations.get(provider, {}) if isinstance(evaluations, dict) else {} if not isinstance(records, dict): return result for model, record in records.items(): if not isinstance(model, str) or model not in advertised or not isinstance(record, dict): continue if not ( record.get("proposed_role") in CAPABILITY_ROLES and record.get("result") == "pass" and record.get("role_fit") == "verified" and record.get("eval_version") == EVALUATION_VERSION and record.get("metadata_fingerprint") == metadata_fingerprint(model, result[model]) ): continue result[model]["evaluated_capability_role"] = record["proposed_role"] result[model]["evaluation_provenance"] = "bounded-representative-eval-v1" return result def select_tier_model( provider: str, models: Iterable[str], metadata: dict[str, dict[str, Any]], desired: str, effort: str, current: str, *, allow_current_fallback: bool = True, legacy_compat: bool = False, ) -> tuple[str, dict[str, Any]]: """Select within one role, retaining current only during discovery outages.""" if desired not in CAPABILITY_ROLES: raise ValueError(f"unsupported capability role: {desired}") candidates: list[tuple[str, dict[str, Any], str]] = [] exact_declared = False exact_available = False observations: dict[str, Any] = {} for model in unique_models(models): details = metadata.get(model, {}) eligible, reason = is_eligible(provider, model, details, effort) tier, tier_reason = capability_role(provider, model, details) if tier is None and legacy_compat: tier, tier_reason = legacy_tier(provider, model) if tier == desired: exact_declared = True exact_available = exact_available or is_eligible( provider, model, details, None )[0] proposed, proposed_reason = proposed_candidate_role(provider, model, details) cost = _model_cost(details) observations[model] = { "eligible": eligible, "reason": reason if not eligible else tier_reason, "tier": tier, "proposed_role": proposed, "proposed_reason": proposed_reason, "cost": {"unit": cost[0], "value": cost[1]} if cost else None, } if eligible and tier: candidates.append((model, details, tier)) exact = [item for item in candidates if item[2] == desired] tier_order = {role: index for index, role in enumerate(CAPABILITY_ROLES)} # Economy/balanced may use the nearest adequate non-frontier role when a # provider cannot express an effort. Advanced and frontier remain strict: # an automatic hard-task route must not quietly promote to Astra or demote. higher = [ item for item in candidates if desired in {"economy", "balanced"} and tier_order[item[2]] > tier_order[desired] and item[2] != "frontier" ] nearest_higher = min( (tier_order[item[2]] for item in higher), default=None ) # A fresh catalog that still declares the desired role but marks every # member unavailable is a provider-side availability signal. Do not turn # that into a more capable route; a caller may retain last-known-good only # when its whole discovery read was unavailable. Promotion remains valid # when no such role exists, or when an available exact role lacks only the # requested effort support. permit_higher = not exact_declared or exact_available pool = exact or ( [item for item in higher if tier_order[item[2]] == nearest_higher] if permit_higher and nearest_higher is not None else [] ) if not pool: return (current if allow_current_fallback else ""), observations comparable_costs = _comparable_costs(pool) def key(item: tuple[str, dict[str, Any], str]) -> tuple[int, float, int, str]: model, details, tier = item return ( 0 if tier == desired else 1, comparable_costs.get(model, float("inf")), # A verified evaluation establishes role fit; cost decides between # adequate peers when provider metadata supplies one common unit. 0 if model == current else 1, model, ) return min(pool, key=key)[0], observations