atlas-iac/scripts/ops/hermes_handoff_evaluators.py

#!/usr/bin/env python3
"""Text-shaped evaluators for the handoff acceptance harness.

These rules cover evidence that arrives as a refusal, an exit status, or a list
of names. The ``denied`` rule is the load-bearing one: it refuses to classify on
an authorization review alone — ``kubectl auth can-i`` reports what RBAC says,
not what the API server does — so a real request must have been made and
refused before it will pass.

Importing this module also registers the JSON-shaped rules, so a caller only has
to import one place to have the full catalog vocabulary available.
"""

from __future__ import annotations

import re
from typing import Any

from hermes_handoff_exec import Outcome
from hermes_handoff_json_rules import evaluate_flux_health  # noqa: F401  (registers rules)
from hermes_handoff_model import ATTEMPT, FAIL, NOT_RUN, PASS, REVIEW, CheckSpec
from hermes_handoff_rules import (
    EVALUATORS,
    Evaluation,
    evaluate,
    evaluator,
    lines,
    looks_denied,
    missing_steps,
    step_keys,
)

__all__ = ["EVALUATORS", "Evaluation", "evaluate", "evaluator"]


@evaluator("denied")
def evaluate_denied(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
    """Pass only when the review says no and a real request was refused."""
    blocked = missing_steps(spec, outcomes)
    if blocked:
        return blocked
    attempts = step_keys(spec, ATTEMPT)
    if not attempts:
        return Evaluation(NOT_RUN, "deny checks require a real attempt, not only a review")
    evidence: dict[str, Any] = {}
    for key in step_keys(spec, REVIEW):
        verdict = outcomes[key].stdout.strip().lower()
        evidence[key] = verdict
        if verdict != "no":
            return Evaluation(
                FAIL, f"authorization review {key} reported {verdict or 'nothing'}", evidence
            )
    for key in attempts:
        outcome = outcomes[key]
        if outcome.ok:
            return Evaluation(FAIL, f"attempt {key} succeeded but must be refused", evidence)
        if not looks_denied(outcome):
            return Evaluation(
                NOT_RUN,
                f"attempt {key} failed without a recognisable refusal (rc={outcome.returncode})",
                evidence,
            )
        evidence[key] = "refused"
    return Evaluation(PASS, "review and live attempt were both refused", evidence)


@evaluator("allowed")
def evaluate_allowed(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
    """Pass when every required step succeeded and every review said yes."""
    blocked = missing_steps(spec, outcomes)
    if blocked:
        return blocked
    evidence: dict[str, Any] = {}
    for step in spec.steps:
        outcome = outcomes[step.key]
        if step.kind == REVIEW:
            verdict = outcome.stdout.strip().lower()
            evidence[step.key] = verdict
            if verdict != "yes":
                return Evaluation(
                    FAIL,
                    f"authorization review {step.key} reported {verdict or 'nothing'}",
                    evidence,
                )
            continue
        if not outcome.ok:
            return Evaluation(FAIL, f"step {step.key} failed (rc={outcome.returncode})", evidence)
        evidence[step.key] = "succeeded"
    return Evaluation(PASS, "required access is present", evidence)


def _matched(observed: set[str], patterns: tuple[str, ...]) -> set[str]:
    return {name for pattern in patterns for name in observed if pattern in name}


def _line_step(spec: CheckSpec, outcomes: dict[str, Outcome]) -> tuple[set[str], str] | Evaluation:
    key = spec.expect["step"]
    outcome = outcomes[key]
    if not outcome.ok:
        return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})")
    return (set(lines(outcome)), key)


@evaluator("names_absent")
def evaluate_names_absent(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
    """Pass when no forbidden name or substring appears in a step's lines."""
    blocked = missing_steps(spec, outcomes)
    if blocked:
        return blocked
    resolved = _line_step(spec, outcomes)
    if isinstance(resolved, Evaluation):
        return resolved
    observed, _ = resolved
    offenders = sorted(
        (observed & set(spec.expect.get("names", ())))
        | _matched(observed, tuple(spec.expect.get("contains", ())))
    )
    evidence = {"observed_count": len(observed), "offenders": offenders}
    if offenders:
        return Evaluation(FAIL, f"forbidden names present: {', '.join(offenders)}", evidence)
    return Evaluation(PASS, "no forbidden names are present", evidence)


@evaluator("names_present")
def evaluate_names_present(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
    """Pass when every required name, and every required substring, appears."""
    blocked = missing_steps(spec, outcomes)
    if blocked:
        return blocked
    resolved = _line_step(spec, outcomes)
    if isinstance(resolved, Evaluation):
        return resolved
    observed, _ = resolved
    missing = sorted(set(spec.expect.get("names", ())) - observed)
    missing += sorted(
        pattern
        for pattern in spec.expect.get("contains", ())
        if not any(pattern in name for name in observed)
    )
    evidence = {"observed_count": len(observed), "missing": missing}
    if missing:
        return Evaluation(FAIL, f"required names absent: {', '.join(missing)}", evidence)
    if not observed:
        return Evaluation(NOT_RUN, "step produced no lines to examine", evidence)
    return Evaluation(PASS, "every required name is present", evidence)


@evaluator("stdout_matches")
def evaluate_stdout_matches(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
    """Compare a step's trimmed stdout against an expected value or set."""
    blocked = missing_steps(spec, outcomes)
    if blocked:
        return blocked
    key = spec.expect["step"]
    outcome = outcomes[key]
    if not outcome.ok:
        return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})")
    actual = outcome.stdout.strip()
    expected = spec.expect["equals"]
    allowed = expected if isinstance(expected, list) else [expected]
    evidence = {"observed": actual}
    if actual not in allowed:
        return Evaluation(FAIL, f"observed {actual!r}, expected one of {allowed!r}", evidence)
    return Evaluation(PASS, "output matches the expected value", evidence)


@evaluator("distinct_count")
def evaluate_distinct_count(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
    """Pass when a step yields enough distinct rows — and, optionally, exactly N.

    Distinctness is the point for placement checks: three worker pods pinned to
    one node satisfy a replica count and defeat the isolation the replica count
    was standing in for.
    """
    blocked = missing_steps(spec, outcomes)
    if blocked:
        return blocked
    key = spec.expect["step"]
    outcome = outcomes[key]
    if not outcome.ok:
        return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})")
    values = lines(outcome)
    distinct = sorted(set(values))
    minimum = spec.expect["minimum"]
    evidence = {"total": len(values), "distinct": len(distinct), "values": distinct[:16]}
    if not values:
        return Evaluation(NOT_RUN, f"step {key} produced no rows to count", evidence)
    if len(distinct) < minimum:
        return Evaluation(
            FAIL, f"{len(distinct)} distinct values, expected at least {minimum}", evidence
        )
    total_equals = spec.expect.get("total_equals")
    if total_equals is not None and len(values) != total_equals:
        return Evaluation(FAIL, f"{len(values)} rows, expected exactly {total_equals}", evidence)
    return Evaluation(PASS, f"{len(distinct)} distinct values observed", evidence)


@evaluator("lines_match")
def evaluate_lines_match(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
    """Pass when every data row of a tabular listing matches a shape.

    Some surfaces only speak tables. A blank leading column is exactly what the
    session-render regression looked like, so asserting the shape of each row is
    a real check rather than a proxy for one.
    """
    blocked = missing_steps(spec, outcomes)
    if blocked:
        return blocked
    key = spec.expect["step"]
    outcome = outcomes[key]
    if not outcome.ok:
        return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})")
    pattern = re.compile(spec.expect["pattern"])
    rows = outcome.stdout.splitlines()[spec.expect.get("skip", 0) :]
    rows = [row for row in rows if row.strip()]
    offenders = [row[:60] for row in rows if not pattern.search(row)]
    evidence = {"rows": len(rows), "offenders": offenders[:8]}
    if len(rows) < spec.expect.get("minimum", 1):
        return Evaluation(
            NOT_RUN, f"step {key} produced {len(rows)} data rows, expected at least "
            f"{spec.expect.get('minimum', 1)}", evidence
        )
    if offenders:
        return Evaluation(FAIL, f"{len(offenders)} row(s) do not match the expected shape", evidence)
    return Evaluation(PASS, f"all {len(rows)} rows match the expected shape", evidence)


@evaluator("vantages_agree")
def evaluate_vantages_agree(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
    """Pass only when two independent observations report the same fact.

    A disagreement is a finding in its own right: it usually means the manifest
    on record and the workload actually running have drifted apart, and either
    one alone would have looked clean.
    """
    blocked = missing_steps(spec, outcomes)
    if blocked:
        return blocked
    left_key, right_key = spec.expect["steps"]
    observations: dict[str, list[str]] = {}
    for key in (left_key, right_key):
        outcome = outcomes[key]
        if not outcome.ok:
            return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})")
        observations[key] = sorted(lines(outcome))
    evidence = {key: value[:16] for key, value in observations.items()}
    if observations[left_key] != observations[right_key]:
        return Evaluation(FAIL, f"vantages disagree: {left_key} and {right_key} differ", evidence)
    if not observations[left_key]:
        return Evaluation(NOT_RUN, "both vantages returned nothing to compare", evidence)
    return Evaluation(PASS, "both vantages report the same state", evidence)


@evaluator("not_armed")
def evaluate_not_armed(spec: CheckSpec, _outcomes: dict[str, Outcome]) -> Evaluation:
    """Report the fixed NOT_RUN a default, read-only run must produce."""
    return Evaluation(
        NOT_RUN,
        spec.expect.get("reason", "ephemeral mutation mode is not armed"),
        {"armed": False},
    )


# Two observations of the same fact agree whether or not they came from
# different vantages; the dual-vantage case is just the important one.
EVALUATORS["steps_agree"] = evaluate_vantages_agree