39 lines
1.5 KiB
Python
39 lines
1.5 KiB
Python
#!/usr/bin/env python3
|
|
"""Evaluate discovered candidates without interrupting established routes."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from hermes_model_routing import write_routing_catalog
|
|
from model_capability_evaluator import evaluate_catalog_candidates
|
|
from routing_catalog import load_catalog
|
|
|
|
|
|
def refresh_model_evaluations(root: Path, codex: Any, claude: Any) -> dict[str, Any]:
|
|
"""Refresh evidence and publish eligible routes; keep the provisional catalog on error."""
|
|
path = Path(os.environ.get("HERMES_ROUTING_CATALOG_PATH") or root / "routing-catalog.json")
|
|
catalog = load_catalog(path)
|
|
if not catalog:
|
|
return {"state": "awaiting-catalog"}
|
|
try:
|
|
evidence = evaluate_catalog_candidates(
|
|
catalog, evidence_path=path.with_name("model-evaluations.json")
|
|
)
|
|
write_routing_catalog(path, codex, claude)
|
|
except (OSError, ValueError, RuntimeError) as error:
|
|
# Provider and evidence-store outages must leave established routes usable.
|
|
return {"state": "deferred", "error_type": type(error).__name__}
|
|
records = [
|
|
record
|
|
for provider in evidence.get("evaluations", {}).values()
|
|
for record in provider.values()
|
|
]
|
|
return {
|
|
"state": "ready",
|
|
"verified": sum(record.get("role_fit") == "verified" for record in records),
|
|
"pending": sum(record.get("role_fit") != "verified" for record in records),
|
|
}
|