405 lines
19 KiB
Python
405 lines
19 KiB
Python
#!/usr/bin/env python3
|
|
"""Run small deterministic model-fit probes through Hermes' native brokers.
|
|
|
|
The probes establish only a narrow routing signal. They do not claim a model is
|
|
generally intelligent, replace provider declarations, or promote a route by
|
|
model name. Transport failures remain pending availability, never quality loss.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import time
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Any, Mapping, Protocol
|
|
from urllib.error import HTTPError, URLError
|
|
from urllib.request import Request, urlopen
|
|
|
|
from model_evaluation_evidence import (
|
|
EVALUATION_VERSION,
|
|
accepted_outcomes,
|
|
load_store,
|
|
metadata_fingerprint,
|
|
record_key,
|
|
retry_due,
|
|
reusable_success,
|
|
unavailable_record,
|
|
write_store,
|
|
)
|
|
from provider_model_catalog import (
|
|
CAPABILITY_ROLES,
|
|
EFFORTS,
|
|
capability_role,
|
|
is_eligible,
|
|
proposed_candidate_role,
|
|
supported_efforts,
|
|
)
|
|
|
|
|
|
MAX_MODELS_PER_REFRESH = 2
|
|
MAX_CALLS_PER_MODEL = 3
|
|
REQUEST_TIMEOUT_SECONDS = 25
|
|
MAX_OUTPUT_TOKENS = 160
|
|
MAX_RESPONSE_BYTES = 16 * 1024
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ProbeReply:
|
|
"""One sanitized broker response used by the deterministic checker."""
|
|
|
|
text: str
|
|
input_tokens: int = 0
|
|
output_tokens: int = 0
|
|
latency_ms: int = 0
|
|
|
|
|
|
class LiteralModelTransport(Protocol):
|
|
"""Invoke one literal, provider-advertised model without route selection."""
|
|
|
|
def invoke(self, provider: str, model: str, effort: str, prompt: str) -> ProbeReply:
|
|
"""Return one response or raise a classified transport exception."""
|
|
|
|
|
|
class ProbeTransportError(RuntimeError):
|
|
"""A safe transport classification that must not become a quality verdict."""
|
|
|
|
def __init__(self, failure_class: str) -> None:
|
|
super().__init__(failure_class)
|
|
self.failure_class = failure_class
|
|
|
|
|
|
def _read_secret() -> str:
|
|
path = Path(os.environ.get("HERMES_MODEL_EVAL_KEY_FILE", "/runtime-access/chat-relay-key"))
|
|
try:
|
|
return path.read_text(encoding="utf-8").strip()
|
|
except OSError:
|
|
return ""
|
|
|
|
|
|
def _failure_class(status: int | None, error: BaseException | None = None) -> str:
|
|
if status in {401, 403}:
|
|
return "auth"
|
|
if status == 429:
|
|
return "rate_limited"
|
|
if status is not None and status >= 500:
|
|
return "provider"
|
|
if isinstance(error, TimeoutError):
|
|
return "timeout"
|
|
if isinstance(error, URLError):
|
|
return "network"
|
|
return "invalid_response" if status and 400 <= status < 500 else "provider"
|
|
|
|
|
|
class BrokerHttpTransport:
|
|
"""Use the local credential-isolating native brokers with fixed model IDs."""
|
|
|
|
def __init__(self, key: str | None = None) -> None:
|
|
self.key = key if key is not None else _read_secret()
|
|
self.codex_endpoint = os.environ.get(
|
|
"HERMES_MODEL_EVAL_CODEX_ENDPOINT", "http://127.0.0.1:9003/v1/responses"
|
|
)
|
|
self.claude_endpoint = os.environ.get(
|
|
"HERMES_MODEL_EVAL_CLAUDE_ENDPOINT", "http://127.0.0.1:9006/v1/messages"
|
|
)
|
|
|
|
def invoke(self, provider: str, model: str, effort: str, prompt: str) -> ProbeReply:
|
|
if not self.key:
|
|
raise ProbeTransportError("auth")
|
|
if provider == "codex":
|
|
endpoint = self.codex_endpoint
|
|
payload = {
|
|
"model": model,
|
|
"input": prompt,
|
|
"stream": False,
|
|
"store": False,
|
|
"reasoning": {"effort": effort},
|
|
# The subscription broker deliberately drops this unsupported
|
|
# upstream field. The prompt and local response-size limit
|
|
# still bound the evaluation exchange without misclassifying
|
|
# a longer provider reasoning trace as poor quality.
|
|
"max_output_tokens": MAX_OUTPUT_TOKENS,
|
|
}
|
|
elif provider == "claude":
|
|
endpoint = self.claude_endpoint
|
|
payload = {
|
|
"model": model,
|
|
"max_tokens": MAX_OUTPUT_TOKENS,
|
|
"stream": False,
|
|
"system": "Return only the requested JSON. Tools are unavailable for this probe.",
|
|
"messages": [{"role": "user", "content": prompt}],
|
|
"output_config": {"effort": effort},
|
|
}
|
|
else:
|
|
raise ProbeTransportError("provider")
|
|
request = Request(
|
|
endpoint,
|
|
data=json.dumps(payload, separators=(",", ":")).encode("utf-8"),
|
|
headers={"Content-Type": "application/json", "Authorization": f"Bearer {self.key}"},
|
|
method="POST",
|
|
)
|
|
started = time.monotonic()
|
|
try:
|
|
with urlopen(request, timeout=REQUEST_TIMEOUT_SECONDS) as response:
|
|
body = response.read(MAX_RESPONSE_BYTES + 1)
|
|
if len(body) > MAX_RESPONSE_BYTES:
|
|
raise ProbeTransportError("truncated")
|
|
document = json.loads(body.decode("utf-8"))
|
|
except HTTPError as exc:
|
|
raise ProbeTransportError(_failure_class(exc.code, exc)) from exc
|
|
except (URLError, TimeoutError, OSError) as exc:
|
|
raise ProbeTransportError(_failure_class(None, exc)) from exc
|
|
except (TypeError, ValueError, json.JSONDecodeError) as exc:
|
|
raise ProbeTransportError("invalid_response") from exc
|
|
if not isinstance(document, Mapping):
|
|
raise ProbeTransportError("invalid_response")
|
|
text, input_tokens, output_tokens = _response_text(provider, document)
|
|
return ProbeReply(
|
|
text=text,
|
|
input_tokens=input_tokens,
|
|
output_tokens=output_tokens,
|
|
latency_ms=int((time.monotonic() - started) * 1000),
|
|
)
|
|
|
|
|
|
def _response_text(provider: str, document: Mapping[str, Any]) -> tuple[str, int, int]:
|
|
"""Extract final text and usage from broker-specific response envelopes."""
|
|
usage = document.get("usage") if isinstance(document.get("usage"), Mapping) else {}
|
|
if provider == "claude":
|
|
content = document.get("content")
|
|
text = "".join(
|
|
str(item.get("text") or "") for item in content
|
|
if isinstance(item, Mapping) and item.get("type") == "text"
|
|
) if isinstance(content, list) else ""
|
|
return text, _token_count(usage.get("input_tokens")), _token_count(usage.get("output_tokens"))
|
|
output = document.get("output")
|
|
text = ""
|
|
if isinstance(output, list):
|
|
for item in output:
|
|
if not isinstance(item, Mapping):
|
|
continue
|
|
for content in item.get("content", []):
|
|
if isinstance(content, Mapping) and content.get("type") in {"output_text", "text"}:
|
|
text += str(content.get("text") or "")
|
|
return text, _token_count(usage.get("input_tokens")), _token_count(usage.get("output_tokens"))
|
|
|
|
|
|
def _token_count(value: Any) -> int:
|
|
"""Return a non-negative broker usage count without trusting response types."""
|
|
return value if isinstance(value, int) and value >= 0 else 0
|
|
|
|
|
|
def _efforts(metadata: Mapping[str, Any], role: str) -> list[str]:
|
|
"""Return the one provider-advertised effort used for this role's probes."""
|
|
advertised = supported_efforts(dict(metadata))
|
|
available = [effort for effort in EFFORTS if effort in advertised]
|
|
target = {"economy": "low", "balanced": "medium", "advanced": "high", "frontier": "xhigh"}[role]
|
|
return [target] if target in available else available[:1]
|
|
|
|
|
|
def _probe_cases(role: str) -> tuple[tuple[str, str, str, set[str], set[str]], ...]:
|
|
"""Return role-sized finite-answer contracts without prose interpretation."""
|
|
prefix = (
|
|
"Return only JSON {\"decision\":\"D#\",\"invariants\":[\"I#\"],"
|
|
"\"checks\":[\"C#\"]}. Select IDs exactly; do not run code, call tools, or add prose. "
|
|
)
|
|
cases = {
|
|
"economy": (
|
|
("idempotency", "A message consumer receives event E twice after a retry and must create one invoice. "
|
|
"Decisions: D1 record E durably before the invoice effect; D2 create invoice then record E. "
|
|
"Invariants: I1 event_id is unique before side effect; I2 retry is best effort. "
|
|
"Checks: C1 concurrent duplicate E makes one invoice; C2 one normal E makes an invoice.",
|
|
"D1", {"I1"}, {"C1"}),
|
|
("cache", "A user permission cache is keyed by user ID. Policy revision changes from 7 to 8, "
|
|
"revoking access. Decisions: D1 revalidate revision before allow; D2 use cached allow until TTL. "
|
|
"Invariants: I1 key includes policy revision; I2 TTL is under one hour. "
|
|
"Checks: C1 revision change returns deny; C2 cache hit returns prior allow.",
|
|
"D1", {"I1"}, {"C1"}),
|
|
),
|
|
"balanced": (
|
|
("cache", "A permission cache uses (user_id, policy_revision). A request reads revision 7, "
|
|
"then revocation commits revision 8 before a handler uses its lookup. Decisions: D1 compare current "
|
|
"revision before allow; D2 trust the earlier cache read. Invariants: I1 allow requires matching current "
|
|
"revision; I2 revocation is eventually consistent. Checks: C1 revocation between read/use denies; "
|
|
"C2 a cache read before revocation permits.", "D1", {"I1"}, {"C1"}),
|
|
("idempotency", "A webhook retries while the first request commits. The database has a unique event_id "
|
|
"constraint. Decisions: D1 use insert conflict as same-event result; D2 retry the side effect. "
|
|
"Invariants: I1 side effect follows successful unique event insert; I2 last writer wins. "
|
|
"Checks: C1 concurrent duplicates create one effect; C2 two different IDs create one effect.",
|
|
"D1", {"I1"}, {"C1"}),
|
|
),
|
|
"advanced": (
|
|
("outbox", "Payment P must charge once and publish receipt once despite a crash after DB commit. "
|
|
"Decisions: D1 atomically commit payment state, unique payment event, and outbox, then relay idempotently; "
|
|
"D2 charge then publish directly. Invariants: I1 unique payment event prevents another charge; "
|
|
"I2 outbox commits with payment state; I3 relay retry alone guarantees exactly once. Checks: "
|
|
"C1 crash/replay gives one charge and eventually one idempotent receipt; C2 crash/replay sends no receipt.",
|
|
"D1", {"I1", "I2"}, {"C1"}),
|
|
("state", "Two workers process the same order. A lease can expire while worker A is paused. "
|
|
"Decisions: D1 compare-and-set expected version; D2 overwrite by latest wall clock. Invariants: "
|
|
"I1 transition requires current version; I2 lease holder may always write. Checks: C1 paused stale worker "
|
|
"cannot overwrite committed state; C2 two writers eventually agree without rejection.", "D1", {"I1"}, {"C1"}),
|
|
),
|
|
"frontier": (
|
|
("fencing", "Worker A holds lease token 10 and pauses. Lease token 11 goes to B, which commits. "
|
|
"A resumes. Decisions: D1 storage rejects lower fencing tokens; D2 A writes if its clock says lease valid. "
|
|
"Invariants: I1 every write carries monotonically increasing fence token; I2 lease expiry is enough. "
|
|
"Checks: C1 token 10 cannot overwrite token 11; C2 a valid clock always permits write.", "D1", {"I1"}, {"C1"}),
|
|
("saga", "Reserve inventory then charge payment across services. The reservation expires before charge reply. "
|
|
"Decisions: D1 compensate/refund or re-reserve before shipping; D2 ship because payment succeeded. "
|
|
"Invariants: I1 ship requires durable active reservation and payment confirmation; I2 payment confirmation "
|
|
"alone permits shipping. Checks: C1 delayed charge after expiry does not ship; C2 delayed charge always ships.",
|
|
"D1", {"I1"}, {"C1"}),
|
|
),
|
|
}
|
|
return tuple(
|
|
(kind, prefix + prompt, decision, invariants, checks)
|
|
for kind, prompt, decision, invariants, checks in cases[role]
|
|
)
|
|
|
|
|
|
def _score(
|
|
_kind: str, text: str, decision: str, invariants: set[str], checks: set[str]
|
|
) -> bool | None:
|
|
"""Validate structured, exact evidence fields without interpreting prose."""
|
|
try:
|
|
answer = json.loads(text)
|
|
except (TypeError, ValueError, json.JSONDecodeError):
|
|
return None
|
|
if not isinstance(answer, Mapping) or not isinstance(answer.get("decision"), str):
|
|
return None
|
|
observed_invariants = answer.get("invariants")
|
|
observed_checks = answer.get("checks")
|
|
if not isinstance(observed_invariants, list) or not isinstance(observed_checks, list):
|
|
return None
|
|
if not all(isinstance(value, str) for value in observed_invariants + observed_checks):
|
|
return None
|
|
normalized_invariants = set(observed_invariants)
|
|
normalized_checks = set(observed_checks)
|
|
return (
|
|
answer["decision"] == decision
|
|
and invariants == normalized_invariants
|
|
and checks == normalized_checks
|
|
)
|
|
|
|
|
|
def _candidate_records(catalog: Mapping[str, Any]) -> list[tuple[str, str, Mapping[str, Any], str]]:
|
|
providers = catalog.get("providers") if isinstance(catalog.get("providers"), Mapping) else {}
|
|
candidates: list[tuple[str, str, Mapping[str, Any], str]] = []
|
|
for provider in ("codex", "claude"):
|
|
record = providers.get(provider)
|
|
if not isinstance(record, Mapping) or record.get("live") is not True:
|
|
continue
|
|
metadata = record.get("model_metadata") if isinstance(record.get("model_metadata"), Mapping) else {}
|
|
models = record.get("models") if isinstance(record.get("models"), list) else []
|
|
for model in models:
|
|
details = metadata.get(model, {}) if isinstance(metadata.get(model), Mapping) else {}
|
|
eligible, _ = is_eligible(provider, model, dict(details), None) if isinstance(model, str) else (False, "")
|
|
role, provenance = proposed_candidate_role(provider, model, dict(details)) if isinstance(model, str) else (None, "")
|
|
reviewed, _ = capability_role(provider, model, dict(details)) if isinstance(model, str) else (None, "")
|
|
if isinstance(model, str) and eligible and not reviewed and role in CAPABILITY_ROLES and provenance:
|
|
candidates.append((provider, model, details, role))
|
|
return candidates
|
|
|
|
|
|
def evaluate_catalog_candidates(
|
|
catalog: Mapping[str, Any], *, evidence_path: Path = Path("/routing-catalog/model-evaluations.json"),
|
|
transport: LiteralModelTransport | None = None, now: int | None = None,
|
|
observed_outcomes: tuple[Mapping[str, Any], ...] = (),
|
|
) -> dict[str, Any]:
|
|
"""Evaluate at most two new candidates and retain cached/pending evidence.
|
|
|
|
Only a passing result says the two small probes fit a provider-proposed role.
|
|
Every other outcome remains pending; route selection owns any later promotion.
|
|
"""
|
|
checked_at = int(time.time()) if now is None else now
|
|
store = load_store(evidence_path)
|
|
records = store["evaluations"]
|
|
selected = 0
|
|
active_transport = transport or BrokerHttpTransport()
|
|
for provider, model, metadata, role in _candidate_records(catalog):
|
|
key = record_key(provider, model)
|
|
fingerprint = metadata_fingerprint(model, metadata)
|
|
previous = records.get(key) if isinstance(records.get(key), Mapping) else None
|
|
if reusable_success(previous, fingerprint) or not retry_due(previous, checked_at):
|
|
continue
|
|
if selected >= MAX_MODELS_PER_REFRESH:
|
|
break
|
|
selected += 1
|
|
efforts = _efforts(metadata, role)
|
|
if not efforts:
|
|
records[key] = unavailable_record(
|
|
provider=provider, model=model, fingerprint=fingerprint, role=role,
|
|
efforts=[], failure_class="provider", now=checked_at,
|
|
)
|
|
continue
|
|
replies: list[ProbeReply] = []
|
|
try:
|
|
cases = _probe_cases(role)
|
|
for kind, prompt, decision, invariants, checks in cases[:MAX_CALLS_PER_MODEL]:
|
|
reply = active_transport.invoke(provider, model, efforts[0], prompt)
|
|
replies.append(reply)
|
|
score = _score(kind, reply.text, decision, invariants, checks)
|
|
if score is None:
|
|
raise ProbeTransportError("invalid_response")
|
|
if not score:
|
|
break
|
|
except ProbeTransportError as exc:
|
|
records[key] = unavailable_record(
|
|
provider=provider, model=model, fingerprint=fingerprint, role=role,
|
|
efforts=efforts, failure_class=exc.failure_class, now=checked_at,
|
|
input_tokens=sum(reply.input_tokens for reply in replies),
|
|
output_tokens=sum(reply.output_tokens for reply in replies),
|
|
latency_ms=sum(reply.latency_ms for reply in replies),
|
|
)
|
|
continue
|
|
except (OSError, TimeoutError, URLError):
|
|
records[key] = unavailable_record(
|
|
provider=provider, model=model, fingerprint=fingerprint, role=role,
|
|
efforts=efforts, failure_class="network", now=checked_at,
|
|
input_tokens=sum(reply.input_tokens for reply in replies),
|
|
output_tokens=sum(reply.output_tokens for reply in replies),
|
|
latency_ms=sum(reply.latency_ms for reply in replies),
|
|
)
|
|
continue
|
|
passed = len(replies) == len(cases) and all(
|
|
_score(kind, reply.text, decision, invariants, checks) is True
|
|
for (kind, _prompt, decision, invariants, checks), reply in zip(cases, replies)
|
|
)
|
|
records[key] = {
|
|
"provider": provider,
|
|
"model": model,
|
|
"metadata_fingerprint": fingerprint,
|
|
"eval_version": EVALUATION_VERSION,
|
|
"proposed_role": role,
|
|
"attempted_efforts": efforts,
|
|
"result": "pass" if passed else "quality_mismatch",
|
|
"role_fit": "verified" if passed else "pending",
|
|
"failure_class": None,
|
|
"last_attempt_at": checked_at,
|
|
"retry_after": None if passed else checked_at + 6 * 60 * 60,
|
|
"tokens": {
|
|
"input": sum(reply.input_tokens for reply in replies),
|
|
"output": sum(reply.output_tokens for reply in replies),
|
|
"total": sum(reply.input_tokens + reply.output_tokens for reply in replies),
|
|
},
|
|
"latency_ms": sum(reply.latency_ms for reply in replies),
|
|
}
|
|
store["updated_at"] = checked_at
|
|
write_store(evidence_path, store)
|
|
reliable_outcomes = accepted_outcomes(observed_outcomes)
|
|
for observation in reliable_outcomes:
|
|
record = records.get(record_key(observation["provider"], observation["model"]))
|
|
if isinstance(record, dict):
|
|
record["observed_acceptance"] = {
|
|
"accepted": observation["accepted"], "evidence_id": observation["evidence_id"],
|
|
}
|
|
if reliable_outcomes:
|
|
write_store(evidence_path, store)
|
|
evaluations: dict[str, dict[str, Any]] = {"codex": {}, "claude": {}}
|
|
for value in records.values():
|
|
if isinstance(value, Mapping) and value.get("provider") in evaluations and isinstance(value.get("model"), str):
|
|
evaluations[str(value["provider"])][value["model"]] = dict(value)
|
|
return {"evaluations": evaluations, "accepted_outcomes": reliable_outcomes}
|