96 lines
4.1 KiB
Python
96 lines
4.1 KiB
Python
|
"""Integration checks for model admission during a coordinator refresh."""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import json
|
||
|
|
import sys
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
sys.path.insert(0, str(Path(__file__).parents[2] / "services/hermes/scripts"))
|
||
|
|
|
||
|
|
import model_catalog_refresh as refresh
|
||
|
|
from hermes_model_routing import Catalog, write_routing_catalog
|
||
|
|
from model_evaluation_evidence import EVALUATION_VERSION, metadata_fingerprint, write_store
|
||
|
|
from routing_catalog import load_catalog
|
||
|
|
|
||
|
|
|
||
|
|
def providers():
|
||
|
|
"""Return a reviewed model and one newly advertised candidate."""
|
||
|
|
metadata = {
|
||
|
|
"gpt-10-lattice": {
|
||
|
|
"description": "Built for difficult complex tasks",
|
||
|
|
"supported_reasoning_efforts": ["high", "xhigh"],
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return (
|
||
|
|
Catalog("openai-codex", ["gpt-5.6-sol", "gpt-10-lattice"], True, True, "connected", metadata),
|
||
|
|
Catalog("anthropic", [], True, True, "connected"),
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def test_refresh_admits_new_model_only_after_evidence_is_persisted(tmp_path, monkeypatch):
|
||
|
|
codex, claude = providers()
|
||
|
|
path = tmp_path / "catalog.json"
|
||
|
|
monkeypatch.setenv("HERMES_ROUTING_CATALOG_PATH", str(path))
|
||
|
|
provisional = write_routing_catalog(path, codex, claude)
|
||
|
|
assert provisional["providers"]["codex"]["capability_pools"]["advanced"] == ["gpt-5.6-sol"]
|
||
|
|
|
||
|
|
def evaluate(catalog, *, evidence_path):
|
||
|
|
assert "gpt-10-lattice" in catalog["providers"]["codex"]["models"]
|
||
|
|
record = {
|
||
|
|
"provider": "codex", "model": "gpt-10-lattice",
|
||
|
|
"metadata_fingerprint": metadata_fingerprint("gpt-10-lattice", codex.metadata["gpt-10-lattice"]),
|
||
|
|
"eval_version": EVALUATION_VERSION, "proposed_role": "advanced",
|
||
|
|
"result": "pass", "role_fit": "verified",
|
||
|
|
}
|
||
|
|
write_store(evidence_path, {"schema_version": 1, "evaluations": {"codex:gpt-10-lattice": record}})
|
||
|
|
return {"evaluations": {"codex": {"gpt-10-lattice": record}}}
|
||
|
|
|
||
|
|
monkeypatch.setattr(refresh, "evaluate_catalog_candidates", evaluate)
|
||
|
|
assert refresh.refresh_model_evaluations(tmp_path, codex, claude) == {
|
||
|
|
"state": "ready", "verified": 1, "pending": 0,
|
||
|
|
}
|
||
|
|
pools = load_catalog(path)["providers"]["codex"]["capability_pools"]
|
||
|
|
assert pools["advanced"] == ["gpt-5.6-sol", "gpt-10-lattice"]
|
||
|
|
assert json.loads(path.read_text())["schema_version"] == 3
|
||
|
|
|
||
|
|
|
||
|
|
def test_evidence_store_failure_preserves_provisional_routes(tmp_path, monkeypatch):
|
||
|
|
codex, claude = providers()
|
||
|
|
path = tmp_path / "catalog.json"
|
||
|
|
monkeypatch.setenv("HERMES_ROUTING_CATALOG_PATH", str(path))
|
||
|
|
write_routing_catalog(path, codex, claude)
|
||
|
|
original = path.read_bytes()
|
||
|
|
|
||
|
|
def unavailable(*args, **kwargs):
|
||
|
|
raise OSError("store unavailable")
|
||
|
|
|
||
|
|
monkeypatch.setattr(refresh, "evaluate_catalog_candidates", unavailable)
|
||
|
|
assert refresh.refresh_model_evaluations(tmp_path, codex, claude) == {
|
||
|
|
"state": "deferred", "error_type": "OSError",
|
||
|
|
}
|
||
|
|
assert path.read_bytes() == original
|
||
|
|
|
||
|
|
|
||
|
|
def test_writer_ignores_stale_or_retired_evidence(tmp_path, monkeypatch):
|
||
|
|
"""Evidence cannot admit a changed or removed model after a live refresh."""
|
||
|
|
codex, claude = providers()
|
||
|
|
path = tmp_path / "catalog.json"
|
||
|
|
monkeypatch.setenv("HERMES_ROUTING_CATALOG_PATH", str(path))
|
||
|
|
write_store(path.with_name("model-evaluations.json"), {"schema_version": 1, "evaluations": {
|
||
|
|
"codex:gpt-10-lattice": {
|
||
|
|
"provider": "codex", "model": "gpt-10-lattice", "proposed_role": "advanced",
|
||
|
|
"result": "pass", "role_fit": "verified", "eval_version": EVALUATION_VERSION,
|
||
|
|
"metadata_fingerprint": "stale",
|
||
|
|
},
|
||
|
|
"codex:gpt-retired": {
|
||
|
|
"provider": "codex", "model": "gpt-retired", "proposed_role": "advanced",
|
||
|
|
"result": "pass", "role_fit": "verified", "eval_version": EVALUATION_VERSION,
|
||
|
|
"metadata_fingerprint": metadata_fingerprint("gpt-retired", {}),
|
||
|
|
},
|
||
|
|
}})
|
||
|
|
catalog = write_routing_catalog(path, codex, claude)
|
||
|
|
provider = catalog["providers"]["codex"]
|
||
|
|
assert provider["capability_pools"]["advanced"] == ["gpt-5.6-sol"]
|
||
|
|
assert "gpt-retired" not in provider["model_metadata"]
|