atlas-iac/services/hermes/scripts/model_evaluation_evidence.py

#!/usr/bin/env python3
"""Persist bounded, non-secret evidence for provider model evaluations."""

from __future__ import annotations

import hashlib
import json
import os
from pathlib import Path
from typing import Any, Iterable, Mapping


SCHEMA_VERSION = 1
EVALUATION_VERSION = "capability-v1"
RETRY_SECONDS = 6 * 60 * 60
TRANSIENT_RETRY_SECONDS = 15 * 60
FAILURE_CLASSES = frozenset(
    {
        "auth", "network", "timeout", "rate_limited", "provider",
        "invalid_response", "truncated",
    }
)


def metadata_fingerprint(model: str, metadata: Mapping[str, Any]) -> str:
    """Bind cached evidence to the advertised, non-secret model metadata."""
    # These two fields are injected only after this fingerprint admits a
    # candidate. Excluding them keeps an accepted cache record stable across
    # the publish/evaluate/publish refresh cycle.
    source = {
        key: value
        for key, value in metadata.items()
        if key not in {"evaluated_capability_role", "evaluation_provenance"}
    }
    value = json.dumps(
        {"model": model, "metadata": source},
        sort_keys=True,
        separators=(",", ":"),
        default=str,
    )
    return hashlib.sha256(value.encode("utf-8")).hexdigest()


def empty_store() -> dict[str, Any]:
    """Return the stable on-disk envelope used by the evaluator."""
    return {"schema_version": SCHEMA_VERSION, "evaluations": {}}


def load_store(path: Path) -> dict[str, Any]:
    """Load valid evidence, refusing malformed or incompatible state."""
    try:
        value = json.loads(path.read_text(encoding="utf-8"))
    except (OSError, TypeError, ValueError, json.JSONDecodeError):
        return empty_store()
    if not isinstance(value, dict) or value.get("schema_version") != SCHEMA_VERSION:
        return empty_store()
    evaluations = value.get("evaluations")
    return value if isinstance(evaluations, dict) else empty_store()


def write_store(path: Path, store: Mapping[str, Any]) -> None:
    """Atomically write evidence without retaining provider response text."""
    path.parent.mkdir(parents=True, exist_ok=True)
    content = json.dumps(dict(store), indent=2, sort_keys=True) + "\n"
    temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp")
    temporary.write_text(content, encoding="utf-8")
    os.chmod(temporary, 0o644)
    os.replace(temporary, path)


def record_key(provider: str, model: str) -> str:
    """Return a collision-safe provider/model identity for JSON storage."""
    return f"{provider}:{model}"


def reusable_success(
    record: Mapping[str, Any] | None, fingerprint: str, *, version: str = EVALUATION_VERSION
) -> bool:
    """Accept a cache hit only for the identical metadata and eval version."""
    return bool(
        isinstance(record, Mapping)
        and record.get("metadata_fingerprint") == fingerprint
        and record.get("eval_version") == version
        and record.get("result") == "pass"
        and record.get("role_fit") == "verified"
    )


def retry_due(record: Mapping[str, Any] | None, now: int) -> bool:
    """Avoid repeatedly spending a refresh on one non-passing model."""
    if not isinstance(record, Mapping) or record.get("result") == "pass":
        return True
    retry_after = record.get("retry_after")
    return not isinstance(retry_after, int) or now >= retry_after


def unavailable_record(
    *, provider: str, model: str, fingerprint: str, role: str, efforts: list[str],
    failure_class: str, now: int, input_tokens: int = 0, output_tokens: int = 0,
    latency_ms: int = 0, version: str = EVALUATION_VERSION,
) -> dict[str, Any]:
    """Represent transport trouble without treating it as model quality evidence."""
    if failure_class not in FAILURE_CLASSES:
        failure_class = "provider"
    # The steward starts alongside its loopback brokers. Retry a local startup
    # race on the next hourly refresh, while keeping provider failures backoffed.
    retry_seconds = TRANSIENT_RETRY_SECONDS if failure_class == "network" else RETRY_SECONDS
    return {
        "provider": provider,
        "model": model,
        "metadata_fingerprint": fingerprint,
        "eval_version": version,
        "proposed_role": role,
        "attempted_efforts": efforts,
        "result": "unavailable",
        "role_fit": "pending",
        "failure_class": failure_class,
        "last_attempt_at": now,
        "retry_after": now + retry_seconds,
        "tokens": {
            "input": max(0, input_tokens),
            "output": max(0, output_tokens),
            "total": max(0, input_tokens) + max(0, output_tokens),
        },
        "latency_ms": max(0, latency_ms),
    }


def accepted_outcomes(
    observations: Iterable[Mapping[str, Any]],
) -> list[dict[str, Any]]:
    """Keep only externally verified acceptance outcomes with literal model IDs.

    This deliberately ignores user preference, inferred quality, and incomplete
    task reports. Callers may merge the returned summaries into model evidence
    once they have a retained acceptance evaluator result.
    """
    accepted: list[dict[str, Any]] = []
    for item in observations:
        if not isinstance(item, Mapping):
            continue
        provider = item.get("provider")
        model = item.get("model")
        if (
            item.get("evidence_kind") != "acceptance"
            or item.get("verified") is not True
            or item.get("accepted") not in {True, False}
            or not isinstance(provider, str)
            or not provider
            or not isinstance(model, str)
            or not model
        ):
            continue
        accepted.append(
            {
                "provider": provider,
                "model": model,
                "accepted": item["accepted"],
                "evidence_id": str(item.get("evidence_id") or "")[:128],
            }
        )
    return accepted