atlas-iac/testing/tests/test_model_capability_evaluator.py

"""Bounded evidence contracts for dynamically discovered provider models."""

from __future__ import annotations

import json
import sys
from pathlib import Path

import pytest


SCRIPTS = Path(__file__).parents[2] / "services" / "hermes" / "scripts"
sys.path.insert(0, str(SCRIPTS))

import model_capability_evaluator as evaluator  # noqa: E402
from hermes_model_routing import Catalog, write_routing_catalog  # noqa: E402
from model_evaluation_evidence import (  # noqa: E402
    TRANSIENT_RETRY_SECONDS,
    accepted_outcomes,
    load_store,
    retry_due,
    unavailable_record,
)
from routing_catalog import load_catalog  # noqa: E402


class Replies:
    """Record literal broker requests and return exact deterministic answers."""

    def __init__(self, role="advanced") -> None:
        self.answers = iter(evaluator._probe_cases(role))
        self.calls: list[tuple[str, str, str, str]] = []

    def invoke(self, provider: str, model: str, effort: str, prompt: str):
        self.calls.append((provider, model, effort, prompt))
        _kind, _prompt, decision, invariants, checks = next(self.answers)
        return evaluator.ProbeReply(
            text=json.dumps(
                {"decision": decision, "invariants": sorted(invariants), "checks": sorted(checks)}
            ),
            input_tokens=3,
            output_tokens=4,
            latency_ms=9,
        )


class Unavailable:
    """Simulate a broker condition that must remain pending, not poor quality."""

    def __init__(self) -> None:
        self.calls = 0

    def invoke(self, *_args):
        self.calls += 1
        raise evaluator.ProbeTransportError("timeout")


def catalog(*models: str) -> dict:
    """Return a live catalog whose models are unknown to reviewed policy."""
    return {
        "providers": {
            "codex": {
                "live": True,
                "models": list(models),
                "model_metadata": {
                    model: {
                        "description": "Built for difficult complex tasks",
                        "supported_reasoning_efforts": ["high", "xhigh"],
                    }
                    for model in models
                },
            }
        }
    }


def test_eval_uses_literal_model_and_persists_only_sanitized_results(tmp_path):
    transport = Replies()
    path = tmp_path / "model-evaluations.json"

    result = evaluator.evaluate_catalog_candidates(
        catalog("gpt-9-aurora"), evidence_path=path, transport=transport, now=100
    )

    record = result["evaluations"]["codex"]["gpt-9-aurora"]
    assert record["result"] == "pass"
    assert record["role_fit"] == "verified"
    assert record["proposed_role"] == "advanced"
    assert record["attempted_efforts"] == ["high"]
    assert record["tokens"] == {"input": 6, "output": 8, "total": 14}
    assert record["latency_ms"] == 18
    assert "prompt" not in record and "response" not in record
    assert [call[:3] for call in transport.calls] == [
        ("codex", "gpt-9-aurora", "high"),
        ("codex", "gpt-9-aurora", "high"),
    ]
    assert all("auto" not in call[1] and "route/" not in call[1] for call in transport.calls)


def test_success_is_cached_by_metadata_fingerprint_and_eval_version(tmp_path):
    path = tmp_path / "model-evaluations.json"
    evaluator.evaluate_catalog_candidates(
        catalog("gpt-9-aurora"), evidence_path=path, transport=Replies(), now=100
    )
    cached = Replies()

    evaluator.evaluate_catalog_candidates(
        catalog("gpt-9-aurora"), evidence_path=path, transport=cached, now=101
    )

    assert cached.calls == []


def test_admitted_catalog_refresh_keeps_a_verified_evaluation_cached(tmp_path):
    """Derived admission metadata cannot invalidate the original fingerprint."""
    path = tmp_path / "catalog.json"
    metadata = {
        "gpt-9-aurora": {
            "description": "Built for difficult complex tasks",
            "supported_reasoning_efforts": ["high", "xhigh"],
        }
    }
    codex = Catalog("openai-codex", ["gpt-5.6-sol", "gpt-9-aurora"], True, True, "connected", metadata)
    claude = Catalog("anthropic", [], True, True, "connected")
    write_routing_catalog(path, codex, claude)
    evaluator.evaluate_catalog_candidates(
        load_catalog(path), evidence_path=path.with_name("model-evaluations.json"),
        transport=Replies(), now=100,
    )
    admitted = write_routing_catalog(path, codex, claude)
    assert "gpt-9-aurora" in admitted["providers"]["codex"]["capability_pools"]["advanced"]
    cached = Replies()
    evaluator.evaluate_catalog_candidates(
        load_catalog(path), evidence_path=path.with_name("model-evaluations.json"),
        transport=cached, now=101,
    )
    assert cached.calls == []


def test_stale_success_is_not_reused_when_provider_metadata_changes(tmp_path):
    path = tmp_path / "model-evaluations.json"
    evaluator.evaluate_catalog_candidates(
        catalog("gpt-9-aurora"), evidence_path=path, transport=Replies(), now=100
    )
    changed = {
        "providers": {
            "codex": {
                "live": True,
                "models": ["gpt-9-aurora"],
                "model_metadata": {
                    "gpt-9-aurora": {
                        "description": "The frontier most capable state of the art model",
                        "supported_reasoning_efforts": ["xhigh"],
                    }
                },
            }
        }
    }
    refreshed = Replies("frontier")

    result = evaluator.evaluate_catalog_candidates(
        changed, evidence_path=path, transport=refreshed, now=101
    )

    assert len(refreshed.calls) == 2
    assert result["evaluations"]["codex"]["gpt-9-aurora"]["proposed_role"] == "frontier"


def test_refresh_is_capped_and_transport_failure_is_not_quality_failure(tmp_path):
    path = tmp_path / "model-evaluations.json"
    unavailable = Unavailable()
    result = evaluator.evaluate_catalog_candidates(
        catalog("gpt-9-a", "gpt-9-b", "gpt-9-c"),
        evidence_path=path,
        transport=unavailable,
        now=100,
    )

    records = result["evaluations"]["codex"]
    assert unavailable.calls == 2
    assert set(records) == {"gpt-9-a", "gpt-9-b"}
    assert all(record["result"] == "unavailable" for record in records.values())
    assert all(record["role_fit"] == "pending" for record in records.values())
    assert all(record["failure_class"] == "timeout" for record in records.values())
    retry = Unavailable()
    evaluator.evaluate_catalog_candidates(
        catalog("gpt-9-a", "gpt-9-b"), evidence_path=path, transport=retry, now=101
    )
    assert retry.calls == 0


def test_loopback_transport_failure_retries_before_the_next_hourly_refresh():
    """A broker startup race does not suppress a future model for six hours."""
    record = unavailable_record(
        provider="claude",
        model="unknown",
        fingerprint="fingerprint",
        role="advanced",
        efforts=["high"],
        failure_class="network",
        now=100,
    )

    assert record["retry_after"] == 100 + TRANSIENT_RETRY_SECONDS
    assert retry_due(record, 100 + TRANSIENT_RETRY_SECONDS - 1) is False
    assert retry_due(record, 100 + TRANSIENT_RETRY_SECONDS) is True


def test_only_verified_acceptance_outcomes_are_retained(tmp_path):
    path = tmp_path / "model-evaluations.json"
    result = evaluator.evaluate_catalog_candidates(
        catalog("gpt-9-aurora"),
        evidence_path=path,
        transport=Replies(),
        now=100,
        observed_outcomes=(
            {"provider": "codex", "model": "gpt-9-aurora", "evidence_kind": "acceptance", "verified": True, "accepted": True, "evidence_id": "run-1"},
            {"provider": "codex", "model": "gpt-9-aurora", "evidence_kind": "acceptance", "verified": False, "accepted": False, "evidence_id": "untrusted"},
        ),
    )

    assert result["accepted_outcomes"] == [
        {"provider": "codex", "model": "gpt-9-aurora", "accepted": True, "evidence_id": "run-1"}
    ]
    record = load_store(path)["evaluations"]["codex:gpt-9-aurora"]
    assert record["observed_acceptance"] == {"accepted": True, "evidence_id": "run-1"}
    assert accepted_outcomes(({"provider": "codex", "model": "auto", "verified": True, "accepted": True},)) == []


def test_provider_description_and_supported_effort_levels_form_a_candidate(tmp_path):
    transport = Replies("frontier")
    document = {
        "providers": {
            "codex": {
                "live": True,
                "models": ["gpt-10-vertex"],
                "model_metadata": {
                    "gpt-10-vertex": {
                        "description": "Our frontier model for the most capable state of the art reasoning",
                        "supportedEffortLevels": [{"level": "xhigh"}],
                    }
                },
            }
        }
    }
    result = evaluator.evaluate_catalog_candidates(
        document, evidence_path=tmp_path / "evidence.json", transport=transport, now=100
    )

    record = result["evaluations"]["codex"]["gpt-10-vertex"]
    assert record["proposed_role"] == "frontier"
    assert record["attempted_efforts"] == ["xhigh"]
    assert record["result"] == "pass"


def test_malformed_output_and_reviewed_policy_records_do_not_promote(tmp_path):
    class Malformed:
        def invoke(self, *_args):
            return evaluator.ProbeReply(text="[]")

    result = evaluator.evaluate_catalog_candidates(
        catalog("gpt-9-aurora", "gpt-5.6-sol"),
        evidence_path=tmp_path / "evidence.json",
        transport=Malformed(),
        now=100,
    )

    record = result["evaluations"]["codex"]["gpt-9-aurora"]
    assert record["result"] == "unavailable"
    assert record["role_fit"] == "pending"
    assert record["failure_class"] == "invalid_response"
    assert "gpt-5.6-sol" not in result["evaluations"]["codex"]


def test_reviewed_native_claude_alias_does_not_spend_an_evaluation(tmp_path):
    """A first-party alias resolving to reviewed Opus is already classified."""
    document = {
        "providers": {
            "claude": {
                "live": True,
                "models": ["opus[1m]"],
                "model_metadata": {
                    "opus[1m]": {
                        "resolvedModel": "claude-opus-5[1m]",
                        "supportsEffort": True,
                        "supportedEffortLevels": ["high", "xhigh"],
                    }
                },
            }
        }
    }
    transport = Unavailable()

    result = evaluator.evaluate_catalog_candidates(
        document, evidence_path=tmp_path / "evidence.json", transport=transport, now=100
    )

    assert transport.calls == 0
    assert result["evaluations"]["claude"] == {}


def test_native_broker_envelopes_keep_literal_models_and_account_usage(monkeypatch):
    class Response:
        def __init__(self, body):
            self.body = body

        def __enter__(self):
            return self

        def __exit__(self, *_args):
            return False

        def read(self, _limit):
            return self.body

    seen = []
    codex_body = json.dumps(
        {
            "output": [{"content": [{"type": "output_text", "text": '{"answer":"ok"}'}]}],
            "usage": {"input_tokens": 7, "output_tokens": 999},
        }
    ).encode()
    claude_body = json.dumps(
        {
            "content": [{"type": "text", "text": '{"answer":"ok"}'}],
            "usage": {"input_tokens": 8, "output_tokens": 9},
        }
    ).encode()

    def fake_urlopen(request, timeout):
        seen.append((request, timeout))
        return Response(codex_body if "responses" in request.full_url else claude_body)

    monkeypatch.setattr(evaluator, "urlopen", fake_urlopen)
    transport = evaluator.BrokerHttpTransport(key="private")
    codex = transport.invoke("codex", "gpt-10-vertex", "xhigh", "probe")
    claude = transport.invoke("claude", "claude-next", "high", "probe")

    assert codex.output_tokens == 999  # Usage is recorded, never treated as a hard cutoff.
    assert claude.input_tokens == 8
    codex_payload = json.loads(seen[0][0].data)
    claude_payload = json.loads(seen[1][0].data)
    assert codex_payload["model"] == "gpt-10-vertex"
    assert codex_payload["reasoning"] == {"effort": "xhigh"}
    assert claude_payload["model"] == "claude-next"
    assert claude_payload["output_config"] == {"effort": "high"}
    assert all(request.get_header("Authorization") == "Bearer private" for request, _ in seen)
    assert all(timeout == evaluator.REQUEST_TIMEOUT_SECONDS for _, timeout in seen)


def test_clipped_broker_body_is_inconclusive_not_a_quality_mismatch(monkeypatch):
    class Response:
        def __enter__(self):
            return self

        def __exit__(self, *_args):
            return False

        def read(self, limit):
            return b"x" * limit

    monkeypatch.setattr(evaluator, "urlopen", lambda *_args, **_kwargs: Response())
    transport = evaluator.BrokerHttpTransport(key="private")

    with pytest.raises(evaluator.ProbeTransportError, match="truncated"):
        transport.invoke("codex", "gpt-10-vertex", "high", "probe")


def test_probe_score_requires_correct_finite_ids_and_rejects_unsafe_extras():
    kind, _prompt, decision, invariants, checks = evaluator._probe_cases("advanced")[0]
    good = json.dumps({"decision": decision, "invariants": sorted(invariants), "checks": sorted(checks)})
    wrong = json.dumps({"decision": "D2", "invariants": sorted(invariants), "checks": sorted(checks)})
    unsafe_extra = json.dumps(
        {"decision": decision, "invariants": sorted(invariants | {"I3"}), "checks": sorted(checks)}
    )

    assert evaluator._score(kind, good, decision, invariants, checks) is True
    assert evaluator._score(kind, wrong, decision, invariants, checks) is False
    assert evaluator._score(kind, unsafe_extra, decision, invariants, checks) is False
    assert evaluator._score(kind, "[]", decision, invariants, checks) is None