414 lines
14 KiB
Python
414 lines
14 KiB
Python
"""Contracts for the JSON-shaped handoff acceptance evaluators."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import datetime as dt
|
|
import json
|
|
|
|
import pytest
|
|
|
|
from testing.tests.test_hermes_handoff_support import (
|
|
load_handoff_module,
|
|
outcome,
|
|
spec,
|
|
step,
|
|
)
|
|
|
|
evaluators = load_handoff_module("hermes_handoff_evaluators")
|
|
model = load_handoff_module("hermes_handoff_model")
|
|
|
|
NOW = dt.datetime(2026, 8, 17, 12, 0, 0, tzinfo=dt.timezone.utc)
|
|
|
|
|
|
def evaluate(built, payload):
|
|
body = payload if isinstance(payload, str) else json.dumps(payload)
|
|
return evaluators.evaluate(built, {"s": outcome(stdout=body)})
|
|
|
|
|
|
def one(rule: str, expect: dict):
|
|
return spec(rule, {"step": "s", **expect}, steps=(step("s"),))
|
|
|
|
|
|
def test_json_field_compares_paths_against_expected_values() -> None:
|
|
built = one(
|
|
"json_field", {"fields": {"authenticated": True, "transport": ["a", "b"]}}
|
|
)
|
|
|
|
assert (
|
|
evaluate(built, {"authenticated": True, "transport": "b"}).status == model.PASS
|
|
)
|
|
|
|
wrong = evaluate(built, {"authenticated": False, "transport": "c"})
|
|
assert wrong.status == model.FAIL
|
|
assert "authenticated=False" in wrong.reason and "transport='c'" in wrong.reason
|
|
|
|
assert evaluate(built, {"authenticated": True}).status == model.NOT_RUN
|
|
|
|
|
|
def test_json_field_skips_rather_than_guesses_on_unusable_output() -> None:
|
|
built = one("json_field", {"fields": {"a": 1}})
|
|
|
|
assert evaluate(built, "not json").status == model.NOT_RUN
|
|
failed = evaluators.evaluate(built, {"s": outcome(returncode=1)})
|
|
assert failed.status == model.NOT_RUN
|
|
assert evaluate(built, '{"a": 1, "a": 1}').status == model.NOT_RUN
|
|
assert evaluate(built, '{"a": NaN}').status == model.NOT_RUN
|
|
|
|
|
|
def test_json_equality_does_not_confuse_booleans_and_integers() -> None:
|
|
built = one("json_field", {"fields": {"enabled": True}})
|
|
assert evaluate(built, {"enabled": 1}).status == model.FAIL
|
|
array = one("all_of_field", {"fields": {"enabled": False}})
|
|
assert evaluate(array, [{"enabled": 0}]).status == model.FAIL
|
|
|
|
|
|
def test_json_numeric_enforces_inclusive_bounds_and_rejects_non_numbers() -> None:
|
|
built = one("json_numeric", {"fields": {"latency_ms": {"min": 1, "max": 1000}}})
|
|
|
|
assert evaluate(built, {"latency_ms": 500}).status == model.PASS
|
|
assert evaluate(built, {"latency_ms": 1}).status == model.PASS
|
|
|
|
low = evaluate(built, {"latency_ms": 0})
|
|
assert low.status == model.FAIL and "below minimum" in low.reason
|
|
|
|
high = evaluate(built, {"latency_ms": 5000})
|
|
assert high.status == model.FAIL and "above maximum" in high.reason
|
|
|
|
assert evaluate(built, {"latency_ms": True}).status == model.NOT_RUN
|
|
assert evaluate(built, {"latency_ms": "fast"}).status == model.NOT_RUN
|
|
assert evaluate(built, {}).status == model.NOT_RUN
|
|
|
|
|
|
def test_json_recent_treats_stale_evidence_as_a_failure() -> None:
|
|
built = one("json_recent", {"now": NOW, "fields": {"checked_at": 3600}})
|
|
|
|
fresh = evaluate(built, {"checked_at": "2026-08-17T11:30:00Z"})
|
|
assert (
|
|
fresh.status == model.PASS
|
|
and fresh.evidence["checked_at"]["age_seconds"] == 1800
|
|
)
|
|
|
|
stale = evaluate(built, {"checked_at": "2026-08-16T00:00:00Z"})
|
|
assert stale.status == model.FAIL and "limit 3600s" in stale.reason
|
|
|
|
naive = evaluate(built, {"checked_at": "2026-08-17T11:30:00"})
|
|
assert naive.status == model.NOT_RUN
|
|
assert evaluate(built, {"checked_at": "2026-08-17T12:00:01Z"}).status == model.FAIL
|
|
|
|
assert evaluate(built, {"checked_at": "yesterday"}).status == model.NOT_RUN
|
|
assert evaluate(built, {}).status == model.NOT_RUN
|
|
invalid_clock = one("json_recent", {"now": "now", "fields": {"checked_at": 1}})
|
|
assert (
|
|
evaluate(invalid_clock, {"checked_at": "2026-08-17T11:30:00Z"}).status
|
|
== model.NOT_RUN
|
|
)
|
|
|
|
|
|
def test_all_of_field_selects_keys_and_reports_every_mismatch() -> None:
|
|
built = one(
|
|
"all_of_field",
|
|
{"key_field": "number", "keys": [14, 15], "fields": {"merged": True}},
|
|
)
|
|
payload = [
|
|
{"number": 14, "merged": True},
|
|
{"number": 15, "merged": False},
|
|
{"number": 16, "merged": False},
|
|
]
|
|
|
|
evaluation = evaluate(built, payload)
|
|
|
|
assert evaluation.status == model.FAIL
|
|
assert "15: merged=False" in evaluation.reason
|
|
assert "16" not in evaluation.reason
|
|
assert evaluation.evidence["examined"] == [14, 15]
|
|
|
|
|
|
def test_all_of_field_skips_when_an_expected_key_is_absent_or_nothing_matched() -> None:
|
|
keyed = one(
|
|
"all_of_field",
|
|
{"key_field": "number", "keys": [14], "fields": {"merged": True}},
|
|
)
|
|
absent = evaluate(keyed, [{"number": 99, "merged": True}])
|
|
assert absent.status == model.NOT_RUN and "expected keys absent" in absent.reason
|
|
|
|
unkeyed = one("all_of_field", {"fields": {"status": ["done"]}})
|
|
assert evaluate(unkeyed, []).status == model.NOT_RUN
|
|
assert evaluate(unkeyed, {"not": "an array"}).status == model.NOT_RUN
|
|
|
|
|
|
def test_all_of_field_skips_when_the_named_array_path_is_absent() -> None:
|
|
built = spec(
|
|
"all_of_field",
|
|
{"step": "s", "array": "items", "fields": {"a": 1}},
|
|
steps=(step("s"),),
|
|
)
|
|
|
|
evaluation = evaluate(built, {"other": []})
|
|
|
|
assert evaluation.status == model.NOT_RUN
|
|
assert "array path is absent" in evaluation.reason
|
|
|
|
|
|
def test_all_of_field_treats_an_absent_field_as_a_mismatch_not_a_crash() -> None:
|
|
built = one("all_of_field", {"key_field": "name", "fields": {"suspended": False}})
|
|
|
|
evaluation = evaluate(built, [{"name": "a"}])
|
|
|
|
assert evaluation.status == model.FAIL
|
|
assert "suspended=None" in evaluation.reason
|
|
|
|
|
|
def test_all_of_field_honours_exemptions_and_non_empty_paths() -> None:
|
|
exempted = one(
|
|
"all_of_field",
|
|
{
|
|
"key_field": "name",
|
|
"exempt_keys": ["parked"],
|
|
"fields": {"suspended": False},
|
|
},
|
|
)
|
|
payload = [
|
|
{"name": "parked", "suspended": True},
|
|
{"name": "live", "suspended": False},
|
|
]
|
|
assert evaluate(exempted, payload).status == model.PASS
|
|
|
|
non_empty = one("all_of_field", {"key_field": "id", "non_empty": ["title"]})
|
|
assert evaluate(non_empty, [{"id": "a", "title": "work"}]).status == model.PASS
|
|
blank = evaluate(non_empty, [{"id": "a", "title": ""}, {"id": "b"}])
|
|
assert blank.status == model.FAIL and "is empty" in blank.reason
|
|
|
|
|
|
def test_json_record_captures_named_paths_or_skips() -> None:
|
|
built = one(
|
|
"json_record", {"record": {"numbers": "[].number", "refs": "[].head.ref"}}
|
|
)
|
|
payload = [
|
|
{"number": 14, "head": {"ref": "a"}},
|
|
{"number": 15, "head": {"ref": "b"}},
|
|
]
|
|
|
|
evaluation = evaluate(built, payload)
|
|
assert evaluation.status == model.PASS
|
|
assert evaluation.evidence == {"numbers": [14, 15], "refs": ["a", "b"]}
|
|
|
|
missing = evaluate(built, [{"number": 14}])
|
|
assert missing.status == model.NOT_RUN and "absent" in missing.reason
|
|
|
|
|
|
def routing_log(*records: dict) -> str:
|
|
return "\n".join(json.dumps(record) for record in records)
|
|
|
|
|
|
def routing_spec(**extra):
|
|
return spec(
|
|
"routing_evidence",
|
|
{
|
|
"step": "s",
|
|
"providers": ("codex", "claude"),
|
|
"efforts": ("high", "xhigh"),
|
|
"lanes": ("route", "worker"),
|
|
"require_fallback_evidence": True,
|
|
**extra,
|
|
},
|
|
steps=(step("s"),),
|
|
)
|
|
|
|
|
|
FULL_LOG = routing_log(
|
|
{"ts": "2026-08-17T11:00:00Z", "model": "route/codex/sol/high"},
|
|
{"ts": "2026-08-17T11:30:00Z", "model": "route/claude/opus/xhigh"},
|
|
{
|
|
"ts": "2026-08-17T11:45:00Z",
|
|
"model": "worker/codex/sol/xhigh",
|
|
"fallback_reason": "unavailable",
|
|
},
|
|
{"ts": "2026-08-17T11:50:00Z", "model": "qwen2.5:14b", "tier": "classifier"},
|
|
)
|
|
|
|
|
|
def test_routing_evidence_summarises_lanes_providers_efforts_and_fallbacks() -> None:
|
|
evaluation = evaluate(routing_spec(), FULL_LOG)
|
|
|
|
assert evaluation.status == model.PASS
|
|
assert evaluation.evidence["providers"] == ["claude", "codex"]
|
|
assert evaluation.evidence["efforts"] == ["high", "xhigh"]
|
|
assert evaluation.evidence["lanes"] == ["route", "worker"]
|
|
assert evaluation.evidence["fallback_reasons"] == ["unavailable"]
|
|
assert evaluation.evidence["records"] == 4 and evaluation.evidence["routed"] == 3
|
|
|
|
|
|
def test_routing_evidence_names_exactly_what_the_window_did_not_show() -> None:
|
|
partial = routing_log(
|
|
{"ts": "2026-08-17T11:00:00Z", "model": "route/codex/sol/high"}
|
|
)
|
|
|
|
evaluation = evaluate(routing_spec(), partial)
|
|
|
|
assert evaluation.status == model.FAIL
|
|
assert "provider:claude" in evaluation.reason
|
|
assert "effort:xhigh" in evaluation.reason
|
|
assert "lane:worker" in evaluation.reason
|
|
assert "fallback:none-observed" in evaluation.reason
|
|
|
|
|
|
def test_routing_evidence_rejects_a_clipped_tail_and_empty_input() -> None:
|
|
clipped = '{"ts": "2026-08-17T11:0\n' + FULL_LOG
|
|
|
|
assert evaluate(routing_spec(), clipped).status == model.NOT_RUN
|
|
assert evaluate(routing_spec(), "").status == model.NOT_RUN
|
|
assert evaluate(routing_spec(), "garbage\n").status == model.NOT_RUN
|
|
|
|
|
|
def test_routing_evidence_can_require_freshness() -> None:
|
|
fresh = evaluate(routing_spec(max_age_seconds=3600, now=NOW), FULL_LOG)
|
|
assert fresh.status == model.PASS
|
|
assert max(fresh.evidence["age_seconds"].values()) <= 3600
|
|
|
|
stale = evaluate(
|
|
routing_spec(max_age_seconds=60, now=NOW + dt.timedelta(days=2)), FULL_LOG
|
|
)
|
|
assert stale.status == model.FAIL and "freshness" in stale.reason
|
|
|
|
undated = routing_log(
|
|
{"model": "route/codex/sol/high"}, {"model": "route/claude/opus/xhigh"}
|
|
)
|
|
unknown = evaluate(routing_spec(max_age_seconds=60, now=NOW), undated)
|
|
assert unknown.status == model.NOT_RUN
|
|
|
|
|
|
def test_routing_freshness_is_required_per_provider_not_only_globally() -> None:
|
|
rows = routing_log(
|
|
{"ts": "2026-08-15T11:00:00Z", "model": "route/claude/opus/high"},
|
|
{"ts": "2026-08-17T11:59:00Z", "model": "route/codex/sol/xhigh"},
|
|
{
|
|
"ts": "2026-08-17T11:59:00Z",
|
|
"model": "worker/codex/sol/high",
|
|
"fallback_reason": "down",
|
|
},
|
|
)
|
|
evaluation = evaluate(routing_spec(max_age_seconds=3600, now=NOW), rows)
|
|
assert evaluation.status == model.FAIL
|
|
assert "freshness:provider:claude" in evaluation.reason
|
|
missing_dimension = evaluate(
|
|
routing_spec(max_age_seconds=3600, now=NOW),
|
|
routing_log(
|
|
{"ts": "2026-08-17T11:59:00Z", "model": "route/codex/sol/high"},
|
|
{
|
|
"ts": "2026-08-17T11:59:00Z",
|
|
"model": "worker/codex/sol/xhigh",
|
|
"fallback_reason": "down",
|
|
},
|
|
),
|
|
)
|
|
assert missing_dimension.status == model.FAIL
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"body",
|
|
[
|
|
'["not-an-object"]',
|
|
'{"ts":"2026-08-17T12:00:01Z","model":"route/codex/sol/high"}',
|
|
'{"ts":"2026-08-17T11:00:00Z"}',
|
|
'{"ts":"2026-08-17T11:00:00Z","model":"bad"}',
|
|
'{"ts":"2026-08-17T11:00:00Z","model":"route/codex/sol/high","fallback_reason":0}',
|
|
],
|
|
)
|
|
def test_routing_rows_fail_closed_on_structural_defects(body: str) -> None:
|
|
assert evaluate(routing_spec(now=NOW), body).status in {model.FAIL, model.NOT_RUN}
|
|
invalid_clock = routing_spec(now="now")
|
|
assert evaluate(invalid_clock, FULL_LOG).status == model.NOT_RUN
|
|
|
|
|
|
def flux_spec(*expected: str):
|
|
return spec(
|
|
"flux_health",
|
|
{"step": "s", "expected_suspensions": expected},
|
|
steps=(step("s"),),
|
|
)
|
|
|
|
|
|
def test_flux_health_separates_parked_objects_from_broken_ones() -> None:
|
|
rows = (
|
|
"flux-system/core\t1\t1\tfalse\tReady=True,Healthy=True,\n"
|
|
"flux-system/parked\t1\t1\ttrue\tReady=False,\n"
|
|
"flux-system/broken\t1\t1\tfalse\tReady=False,Healthy=False,\n"
|
|
)
|
|
|
|
evaluation = evaluators.evaluate(
|
|
flux_spec("flux-system/parked"), {"s": outcome(stdout=rows)}
|
|
)
|
|
|
|
assert evaluation.status == model.FAIL
|
|
assert evaluation.evidence["unhealthy"] == ["flux-system/broken"]
|
|
assert evaluation.evidence["unexpected_suspensions"] == []
|
|
assert "unhealthy: flux-system/broken" in evaluation.reason
|
|
|
|
|
|
def test_flux_health_flags_a_suspension_nobody_declared() -> None:
|
|
rows = "flux-system/core\t1\t1\tfalse\tReady=True,\nflux-system/parked\t1\t1\ttrue\tReady=False,\n"
|
|
|
|
evaluation = evaluators.evaluate(flux_spec(), {"s": outcome(stdout=rows)})
|
|
|
|
assert evaluation.status == model.FAIL
|
|
assert evaluation.evidence["unexpected_suspensions"] == ["flux-system/parked"]
|
|
|
|
|
|
def test_flux_health_fails_closed_on_in_flight_or_unknown_conditions() -> None:
|
|
rows = (
|
|
"flux-system/reconciling\t1\t1\tfalse\tReconciling=True,Ready=False,\n"
|
|
"flux-system/degraded\t1\t1\tfalse\tReady=False,Healthy=True,\n"
|
|
"flux-system/waiting\t1\t1\tfalse\tReady=Unknown,\n"
|
|
)
|
|
|
|
evaluation = evaluators.evaluate(flux_spec(), {"s": outcome(stdout=rows)})
|
|
|
|
assert evaluation.status == model.FAIL
|
|
assert evaluation.evidence["total"] == 3
|
|
|
|
|
|
def test_blank_rows_in_a_projection_are_ignored_by_both_log_parsers() -> None:
|
|
padded = "\n\n" + FULL_LOG + "\n\n"
|
|
assert evaluate(routing_spec(), padded).status == model.PASS
|
|
|
|
rows = "\n\nflux-system/core\t1\t1\tfalse\tReady=True,\n\n"
|
|
evaluation = evaluators.evaluate(flux_spec(), {"s": outcome(stdout=rows)})
|
|
assert evaluation.status == model.PASS and evaluation.evidence["total"] == 1
|
|
|
|
|
|
def test_flux_health_skips_on_an_empty_or_failed_projection() -> None:
|
|
assert (
|
|
evaluators.evaluate(flux_spec(), {"s": outcome(stdout="")}).status
|
|
== model.NOT_RUN
|
|
)
|
|
assert (
|
|
evaluators.evaluate(flux_spec(), {"s": outcome(returncode=1)}).status
|
|
== model.NOT_RUN
|
|
)
|
|
malformed = "flux-system/core\t1\t0\tfalse\tReady=True,\n"
|
|
assert (
|
|
evaluators.evaluate(flux_spec(), {"s": outcome(stdout=malformed)}).status
|
|
== model.FAIL
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"row",
|
|
[
|
|
"too\tfew",
|
|
"BAD NAME\t1\t1\tfalse\tReady=True,",
|
|
"flux-system/core\tx\t1\tfalse\tReady=True,",
|
|
"flux-system/core\t1\t1\tmaybe\tReady=True,",
|
|
"flux-system/core\t1\t1\tfalse\tbroken,",
|
|
"flux-system/core\t1\t1\tfalse\tReady=Yes,",
|
|
"flux-system/core\t1\t1\tfalse\tReady=True,Ready=True,",
|
|
],
|
|
)
|
|
def test_flux_rows_reject_malformed_columns_names_generations_and_conditions(
|
|
row: str,
|
|
) -> None:
|
|
assert (
|
|
evaluators.evaluate(flux_spec(), {"s": outcome(stdout=row)}).status
|
|
== model.NOT_RUN
|
|
)
|