atlas-iac/testing/tests/test_hermes_handoff_json_rules.py

"""Contracts for the JSON-shaped handoff acceptance evaluators."""

from __future__ import annotations

import datetime as dt
import json

from testing.tests.test_hermes_handoff_support import load_handoff_module, outcome, spec, step

evaluators = load_handoff_module("hermes_handoff_evaluators")
model = load_handoff_module("hermes_handoff_model")

NOW = dt.datetime(2026, 8, 17, 12, 0, 0, tzinfo=dt.timezone.utc)


def evaluate(built, payload):
    body = payload if isinstance(payload, str) else json.dumps(payload)
    return evaluators.evaluate(built, {"s": outcome(stdout=body)})


def one(rule: str, expect: dict):
    return spec(rule, {"step": "s", **expect}, steps=(step("s"),))


def test_json_field_compares_paths_against_expected_values() -> None:
    built = one("json_field", {"fields": {"authenticated": True, "transport": ["a", "b"]}})

    assert evaluate(built, {"authenticated": True, "transport": "b"}).status == model.PASS

    wrong = evaluate(built, {"authenticated": False, "transport": "c"})
    assert wrong.status == model.FAIL
    assert "authenticated=False" in wrong.reason and "transport='c'" in wrong.reason

    assert evaluate(built, {"authenticated": True}).status == model.NOT_RUN


def test_json_field_skips_rather_than_guesses_on_unusable_output() -> None:
    built = one("json_field", {"fields": {"a": 1}})

    assert evaluate(built, "not json").status == model.NOT_RUN
    failed = evaluators.evaluate(built, {"s": outcome(returncode=1)})
    assert failed.status == model.NOT_RUN


def test_json_numeric_enforces_inclusive_bounds_and_rejects_non_numbers() -> None:
    built = one("json_numeric", {"fields": {"latency_ms": {"min": 1, "max": 1000}}})

    assert evaluate(built, {"latency_ms": 500}).status == model.PASS
    assert evaluate(built, {"latency_ms": 1}).status == model.PASS

    low = evaluate(built, {"latency_ms": 0})
    assert low.status == model.FAIL and "below minimum" in low.reason

    high = evaluate(built, {"latency_ms": 5000})
    assert high.status == model.FAIL and "above maximum" in high.reason

    assert evaluate(built, {"latency_ms": True}).status == model.NOT_RUN
    assert evaluate(built, {"latency_ms": "fast"}).status == model.NOT_RUN
    assert evaluate(built, {}).status == model.NOT_RUN


def test_json_recent_treats_stale_evidence_as_a_failure() -> None:
    built = one("json_recent", {"now": NOW, "fields": {"checked_at": 3600}})

    fresh = evaluate(built, {"checked_at": "2026-08-17T11:30:00Z"})
    assert fresh.status == model.PASS and fresh.evidence["checked_at"]["age_seconds"] == 1800

    stale = evaluate(built, {"checked_at": "2026-08-16T00:00:00Z"})
    assert stale.status == model.FAIL and "limit 3600s" in stale.reason

    naive = evaluate(built, {"checked_at": "2026-08-17T11:30:00"})
    assert naive.status == model.PASS

    assert evaluate(built, {"checked_at": "yesterday"}).status == model.NOT_RUN
    assert evaluate(built, {}).status == model.NOT_RUN


def test_all_of_field_selects_keys_and_reports_every_mismatch() -> None:
    built = one("all_of_field", {"key_field": "number", "keys": [14, 15], "fields": {"merged": True}})
    payload = [
        {"number": 14, "merged": True},
        {"number": 15, "merged": False},
        {"number": 16, "merged": False},
    ]

    evaluation = evaluate(built, payload)

    assert evaluation.status == model.FAIL
    assert "15: merged=False" in evaluation.reason
    assert "16" not in evaluation.reason
    assert evaluation.evidence["examined"] == [14, 15]


def test_all_of_field_skips_when_an_expected_key_is_absent_or_nothing_matched() -> None:
    keyed = one("all_of_field", {"key_field": "number", "keys": [14], "fields": {"merged": True}})
    absent = evaluate(keyed, [{"number": 99, "merged": True}])
    assert absent.status == model.NOT_RUN and "expected keys absent" in absent.reason

    unkeyed = one("all_of_field", {"fields": {"status": ["done"]}})
    assert evaluate(unkeyed, []).status == model.NOT_RUN
    assert evaluate(unkeyed, {"not": "an array"}).status == model.NOT_RUN


def test_all_of_field_skips_when_the_named_array_path_is_absent() -> None:
    built = spec(
        "all_of_field", {"step": "s", "array": "items", "fields": {"a": 1}}, steps=(step("s"),)
    )

    evaluation = evaluate(built, {"other": []})

    assert evaluation.status == model.NOT_RUN
    assert "array path is absent" in evaluation.reason


def test_all_of_field_treats_an_absent_field_as_a_mismatch_not_a_crash() -> None:
    built = one("all_of_field", {"key_field": "name", "fields": {"suspended": False}})

    evaluation = evaluate(built, [{"name": "a"}])

    assert evaluation.status == model.FAIL
    assert "suspended=None" in evaluation.reason


def test_all_of_field_honours_exemptions_and_non_empty_paths() -> None:
    exempted = one(
        "all_of_field",
        {"key_field": "name", "exempt_keys": ["parked"], "fields": {"suspended": False}},
    )
    payload = [{"name": "parked", "suspended": True}, {"name": "live", "suspended": False}]
    assert evaluate(exempted, payload).status == model.PASS

    non_empty = one("all_of_field", {"key_field": "id", "non_empty": ["title"]})
    assert evaluate(non_empty, [{"id": "a", "title": "work"}]).status == model.PASS
    blank = evaluate(non_empty, [{"id": "a", "title": ""}, {"id": "b"}])
    assert blank.status == model.FAIL and "is empty" in blank.reason


def test_json_record_captures_named_paths_or_skips() -> None:
    built = one("json_record", {"record": {"numbers": "[].number", "refs": "[].head.ref"}})
    payload = [{"number": 14, "head": {"ref": "a"}}, {"number": 15, "head": {"ref": "b"}}]

    evaluation = evaluate(built, payload)
    assert evaluation.status == model.PASS
    assert evaluation.evidence == {"numbers": [14, 15], "refs": ["a", "b"]}

    missing = evaluate(built, [{"number": 14}])
    assert missing.status == model.NOT_RUN and "absent" in missing.reason


def routing_log(*records: dict) -> str:
    return "\n".join(json.dumps(record) for record in records)


def routing_spec(**extra):
    return spec(
        "routing_evidence",
        {
            "step": "s",
            "providers": ("codex", "claude"),
            "efforts": ("high", "xhigh"),
            "lanes": ("route", "worker"),
            "require_fallback_evidence": True,
            **extra,
        },
        steps=(step("s"),),
    )


FULL_LOG = routing_log(
    {"ts": "2026-08-17T11:00:00Z", "model": "route/codex/sol/high"},
    {"ts": "2026-08-17T11:30:00Z", "model": "route/claude/opus/xhigh"},
    {"ts": "2026-08-17T11:45:00Z", "model": "worker/codex/sol/xhigh", "fallback_reason": "unavailable"},
    {"ts": "2026-08-17T11:50:00Z", "model": "qwen2.5:14b", "tier": "classifier"},
)


def test_routing_evidence_summarises_lanes_providers_efforts_and_fallbacks() -> None:
    evaluation = evaluate(routing_spec(), FULL_LOG)

    assert evaluation.status == model.PASS
    assert evaluation.evidence["providers"] == ["claude", "codex"]
    assert evaluation.evidence["efforts"] == ["high", "xhigh"]
    assert evaluation.evidence["lanes"] == ["route", "worker"]
    assert evaluation.evidence["fallback_reasons"] == ["unavailable"]
    assert evaluation.evidence["records"] == 4 and evaluation.evidence["routed"] == 3


def test_routing_evidence_names_exactly_what_the_window_did_not_show() -> None:
    partial = routing_log({"ts": "2026-08-17T11:00:00Z", "model": "route/codex/sol/high"})

    evaluation = evaluate(routing_spec(), partial)

    assert evaluation.status == model.FAIL
    assert "provider:claude" in evaluation.reason
    assert "effort:xhigh" in evaluation.reason
    assert "lane:worker" in evaluation.reason
    assert "fallback:none-observed" in evaluation.reason


def test_routing_evidence_tolerates_a_clipped_tail_but_not_an_empty_one() -> None:
    clipped = '{"ts": "2026-08-17T11:0\n' + FULL_LOG

    assert evaluate(routing_spec(), clipped).status == model.PASS
    assert evaluate(routing_spec(), "").status == model.NOT_RUN
    assert evaluate(routing_spec(), "garbage\n").status == model.NOT_RUN


def test_routing_evidence_can_require_freshness() -> None:
    fresh = evaluate(routing_spec(max_age_seconds=3600, now=NOW), FULL_LOG)
    assert fresh.status == model.PASS and fresh.evidence["age_seconds"] == 600

    stale = evaluate(
        routing_spec(max_age_seconds=60, now=NOW + dt.timedelta(days=2)), FULL_LOG
    )
    assert stale.status == model.FAIL and "freshness" in stale.reason

    undated = routing_log({"model": "route/codex/sol/high"}, {"model": "route/claude/opus/xhigh"})
    unknown = evaluate(routing_spec(max_age_seconds=60, now=NOW), undated)
    assert unknown.status == model.NOT_RUN


def flux_spec(*expected: str):
    return spec(
        "flux_health",
        {"step": "s", "expected_suspensions": expected},
        steps=(step("s"),),
    )


def test_flux_health_separates_parked_objects_from_broken_ones() -> None:
    rows = (
        "flux-system/core\tfalse\tReady=True,Healthy=True,\n"
        "flux-system/parked\ttrue\tReady=False,\n"
        "flux-system/broken\tfalse\tReady=False,Healthy=False,\n"
    )

    evaluation = evaluators.evaluate(flux_spec("flux-system/parked"), {"s": outcome(stdout=rows)})

    assert evaluation.status == model.FAIL
    assert evaluation.evidence["unhealthy"] == ["flux-system/broken"]
    assert evaluation.evidence["unexpected_suspensions"] == []
    assert "unhealthy: flux-system/broken" in evaluation.reason


def test_flux_health_flags_a_suspension_nobody_declared() -> None:
    rows = "flux-system/core\tfalse\tReady=True,\nflux-system/parked\ttrue\tReady=False,\n"

    evaluation = evaluators.evaluate(flux_spec(), {"s": outcome(stdout=rows)})

    assert evaluation.status == model.FAIL
    assert evaluation.evidence["unexpected_suspensions"] == ["flux-system/parked"]


def test_flux_health_does_not_call_an_in_flight_reconcile_unhealthy() -> None:
    """Flux drops Ready to False for the duration of a reconcile."""
    rows = (
        "flux-system/reconciling\tfalse\tReconciling=True,Ready=False,\n"
        "flux-system/degraded\tfalse\tReady=False,Healthy=True,\n"
        "flux-system/waiting\t\tReady=Unknown,\n"
    )

    evaluation = evaluators.evaluate(flux_spec(), {"s": outcome(stdout=rows)})

    assert evaluation.status == model.PASS
    assert evaluation.evidence["total"] == 3


def test_blank_rows_in_a_projection_are_ignored_by_both_log_parsers() -> None:
    padded = "\n\n" + FULL_LOG + "\n\n"
    assert evaluate(routing_spec(), padded).status == model.PASS

    rows = "\n\nflux-system/core\tfalse\tReady=True,\n\n"
    evaluation = evaluators.evaluate(flux_spec(), {"s": outcome(stdout=rows)})
    assert evaluation.status == model.PASS and evaluation.evidence["total"] == 1


def test_flux_health_skips_on_an_empty_or_failed_projection() -> None:
    assert evaluators.evaluate(flux_spec(), {"s": outcome(stdout="")}).status == model.NOT_RUN
    assert evaluators.evaluate(flux_spec(), {"s": outcome(returncode=1)}).status == model.NOT_RUN