#!/usr/bin/env python3 """Text-shaped evaluators for the handoff acceptance harness. These rules cover evidence that arrives as a refusal, an exit status, or a list of names. The ``denied`` rule is the load-bearing one: it refuses to classify on an authorization review alone — ``kubectl auth can-i`` reports what RBAC says, not what the API server does — so a real request must have been made and refused before it will pass. An absence is only evidence when the probe observed something, so a name step that exits 0 with no lines is ``NOT_RUN``: drift produces silence, not a pass. Importing this module also registers the JSON-shaped rules, so a caller only has to import one place to have the full catalog vocabulary available. """ from __future__ import annotations import re from typing import Any from hermes_handoff_exec import Outcome from hermes_handoff_json_rules import evaluate_flux_health # noqa: F401 (registers rules) from hermes_handoff_model import ATTEMPT, FAIL, NOT_RUN, PASS, REVIEW, CheckSpec from hermes_handoff_rules import ( EVALUATORS, Evaluation, dotted, evaluate, evaluator, exact_equal, lines, looks_denied, missing_steps, step_keys, strict_json, ) __all__ = ["EVALUATORS", "Evaluation", "evaluate", "evaluator"] @evaluator("denied") def evaluate_denied(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: """Pass only when the review says no and a real request was refused.""" blocked = missing_steps(spec, outcomes) if blocked: return blocked attempts = step_keys(spec, ATTEMPT) if not attempts: return Evaluation( NOT_RUN, "deny checks require a real attempt, not only a review" ) evidence: dict[str, Any] = {} for key in step_keys(spec, REVIEW): verdict = outcomes[key].stdout.strip().lower() evidence[key] = verdict if verdict != "no": return Evaluation( FAIL, f"authorization review {key} reported {verdict or 'nothing'}", evidence, ) for key in attempts: outcome = outcomes[key] if outcome.ok: return Evaluation( FAIL, f"attempt {key} succeeded but must be refused", evidence ) if not looks_denied(outcome): return Evaluation( NOT_RUN, f"attempt {key} failed without a recognisable refusal (rc={outcome.returncode})", evidence, ) evidence[key] = "refused" return Evaluation(PASS, "review and live attempt were both refused", evidence) @evaluator("review_denied") def evaluate_review_denied(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: """Pass only when every side-effect-free authorization review says no.""" blocked = missing_steps(spec, outcomes) if blocked: return blocked reviews = step_keys(spec, REVIEW) if not reviews: return Evaluation(NOT_RUN, "a denial check requires an authorization review") evidence: dict[str, str] = {} for key in reviews: verdict = outcomes[key].stdout.strip().lower() evidence[key] = verdict if verdict != "no": return Evaluation( FAIL, f"authorization review {key} reported {verdict or 'nothing'}", evidence, ) return Evaluation(PASS, "every authorization review was denied", evidence) @evaluator("allowed") def evaluate_allowed(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: """Pass when every required step succeeded and every review said yes.""" blocked = missing_steps(spec, outcomes) if blocked: return blocked evidence: dict[str, Any] = {} for step in spec.steps: outcome = outcomes[step.key] if step.kind == REVIEW: verdict = outcome.stdout.strip().lower() evidence[step.key] = verdict if verdict != "yes": return Evaluation( FAIL, f"authorization review {step.key} reported {verdict or 'nothing'}", evidence, ) continue if not outcome.ok: return Evaluation( FAIL, f"step {step.key} failed (rc={outcome.returncode})", evidence ) evidence[step.key] = "succeeded" return Evaluation(PASS, "required access is present", evidence) def _matched(observed: set[str], patterns: tuple[str, ...]) -> set[str]: return {name for pattern in patterns for name in observed if pattern in name} def _line_step( spec: CheckSpec, outcomes: dict[str, Outcome] ) -> tuple[set[str], str] | Evaluation: """Return a step's distinct lines, or the NOT_RUN that silence must produce.""" key = spec.expect["step"] outcome = outcomes[key] if not outcome.ok: return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})") observed = set(lines(outcome)) if not observed: return Evaluation(NOT_RUN, f"step {key} exited 0 with no lines to examine") return (observed, key) @evaluator("names_absent") def evaluate_names_absent(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: """Pass when a step observed names and none of them is forbidden.""" blocked = missing_steps(spec, outcomes) if blocked: return blocked resolved = _line_step(spec, outcomes) if isinstance(resolved, Evaluation): return resolved observed, _ = resolved offenders = sorted( (observed & set(spec.expect.get("names", ()))) | _matched(observed, tuple(spec.expect.get("contains", ()))) ) evidence = {"observed_count": len(observed), "offenders": offenders} if offenders: return Evaluation( FAIL, f"forbidden names present: {', '.join(offenders)}", evidence ) return Evaluation(PASS, "no forbidden names are present", evidence) @evaluator("names_present") def evaluate_names_present(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: """Pass when every required name, and every required substring, appears.""" blocked = missing_steps(spec, outcomes) if blocked: return blocked resolved = _line_step(spec, outcomes) if isinstance(resolved, Evaluation): return resolved observed, _ = resolved missing = sorted(set(spec.expect.get("names", ())) - observed) missing += sorted( pattern for pattern in spec.expect.get("contains", ()) if not any(pattern in name for name in observed) ) evidence = {"observed_count": len(observed), "missing": missing} if missing: return Evaluation( FAIL, f"required names absent: {', '.join(missing)}", evidence ) return Evaluation(PASS, "every required name is present", evidence) @evaluator("stdout_matches") def evaluate_stdout_matches( spec: CheckSpec, outcomes: dict[str, Outcome] ) -> Evaluation: """Compare a step's trimmed stdout against an expected value or set.""" blocked = missing_steps(spec, outcomes) if blocked: return blocked key = spec.expect["step"] outcome = outcomes[key] if not outcome.ok: return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})") actual = outcome.stdout.strip() expected = spec.expect["equals"] allowed = expected if isinstance(expected, list) else [expected] evidence = {"observed": actual} if actual not in allowed: return Evaluation( FAIL, f"observed {actual!r}, expected one of {allowed!r}", evidence ) return Evaluation(PASS, "output matches the expected value", evidence) @evaluator("distinct_count") def evaluate_distinct_count( spec: CheckSpec, outcomes: dict[str, Outcome] ) -> Evaluation: """Pass when a step yields enough distinct rows — and, optionally, exactly N. Distinctness is the point for placement checks: three worker pods pinned to one node satisfy a replica count and defeat the isolation the replica count was standing in for. """ blocked = missing_steps(spec, outcomes) if blocked: return blocked key = spec.expect["step"] outcome = outcomes[key] if not outcome.ok: return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})") values = lines(outcome) distinct = sorted(set(values)) minimum = spec.expect["minimum"] evidence = { "total": len(values), "distinct": len(distinct), "values": distinct[:16], } if not values: return Evaluation(NOT_RUN, f"step {key} produced no rows to count", evidence) if len(distinct) < minimum: return Evaluation( FAIL, f"{len(distinct)} distinct values, expected at least {minimum}", evidence, ) total_equals = spec.expect.get("total_equals") if total_equals is not None and len(values) != total_equals: return Evaluation( FAIL, f"{len(values)} rows, expected exactly {total_equals}", evidence ) return Evaluation(PASS, f"{len(distinct)} distinct values observed", evidence) @evaluator("lines_match") def evaluate_lines_match(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: """Pass when every data row of a tabular listing matches a shape. Some surfaces only speak tables. A blank leading column is exactly what the session-render regression looked like, so asserting the shape of each row is a real check rather than a proxy for one. """ blocked = missing_steps(spec, outcomes) if blocked: return blocked key = spec.expect["step"] outcome = outcomes[key] if not outcome.ok: return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})") pattern = re.compile(spec.expect["pattern"]) rows = outcome.stdout.splitlines()[spec.expect.get("skip", 0) :] rows = [row for row in rows if row.strip()] offenders = [row[:60] for row in rows if not pattern.search(row)] evidence = {"rows": len(rows), "offenders": offenders[:8]} if len(rows) < spec.expect.get("minimum", 1): return Evaluation( NOT_RUN, f"step {key} produced {len(rows)} data rows, expected at least " f"{spec.expect.get('minimum', 1)}", evidence, ) if offenders: return Evaluation( FAIL, f"{len(offenders)} row(s) do not match the expected shape", evidence ) return Evaluation(PASS, f"all {len(rows)} rows match the expected shape", evidence) @evaluator("vantages_agree") def evaluate_vantages_agree( spec: CheckSpec, outcomes: dict[str, Outcome] ) -> Evaluation: """Pass only when two independent observations report the same fact. A disagreement is a finding in its own right: it usually means the manifest on record and the workload actually running have drifted apart, and either one alone would have looked clean. """ blocked = missing_steps(spec, outcomes) if blocked: return blocked left_key, right_key = spec.expect["steps"] observations: dict[str, list[str]] = {} for key in (left_key, right_key): outcome = outcomes[key] if not outcome.ok: return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})") observations[key] = sorted(lines(outcome)) evidence = {key: value[:16] for key, value in observations.items()} if observations[left_key] != observations[right_key]: return Evaluation( FAIL, f"vantages disagree: {left_key} and {right_key} differ", evidence ) if not observations[left_key]: return Evaluation( NOT_RUN, "both vantages returned nothing to compare", evidence ) return Evaluation(PASS, "both vantages report the same state", evidence) @evaluator("not_armed") def evaluate_not_armed(spec: CheckSpec, _outcomes: dict[str, Outcome]) -> Evaluation: """Report the fixed NOT_RUN a default, read-only run must produce.""" reason = spec.expect.get("reason", "ephemeral mutation mode is not armed") return Evaluation(NOT_RUN, reason, {"armed": False}) # This validator is one compact, linear state machine so every exit observes # only fully validated state. # fmt: off @evaluator("pool_topology") def evaluate_pool_topology(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: """Require exactly three Ready ordinal workers on distinct nodes and claims.""" blocked = missing_steps(spec, outcomes) if blocked: return blocked state = outcomes[spec.expect["state_step"]].stdout.strip().split("\t") if len(state) != 4: return Evaluation(NOT_RUN, "StatefulSet projection is malformed") expected = spec.expect.get("replicas", 3) try: replicas, ready, current = (int(value) for value in state[:3]) except ValueError: return Evaluation(NOT_RUN, "StatefulSet replica fields are malformed") evidence: dict[str, Any] = {"replicas": replicas, "ready": ready, "current": current, "automountServiceAccountToken": state[3]} if (replicas, ready, current, state[3].lower()) != (expected, expected, expected, "false"): return Evaluation(FAIL, "worker replica or tokenless state does not match the contract", evidence) workers: dict[int, tuple[str, str]] = {} names: list[str] = [] for number, row in enumerate(lines(outcomes[spec.expect["worker_step"]]), 1): columns = row.split("\t") if len(columns) != 5: return Evaluation(NOT_RUN, f"worker row {number} is malformed", evidence) name, node, phase, ready_state, claims = columns match = re.fullmatch(rf"{re.escape(spec.expect['name'])}-(\d+)", name) claim_list = [claim for claim in claims.split(",") if claim] if (not match or not node or phase != "Running" or ready_state != "True" or len(claim_list) != 1): return Evaluation(FAIL, f"worker row {number} is not a Ready isolated worker", evidence) ordinal = int(match.group(1)) if ordinal in workers: return Evaluation(FAIL, f"worker ordinal {ordinal} is duplicated", evidence) workers[ordinal] = (node, claim_list[0]) names.append(name) evidence.update({"workers": sorted(names), "nodes": sorted({v[0] for v in workers.values()}), "claims": sorted({v[1] for v in workers.values()})}) if set(workers) != set(range(expected)): return Evaluation(FAIL, f"worker ordinals are not exactly 0..{expected - 1}", evidence) if (len({v[0] for v in workers.values()}) != expected or len({v[1] for v in workers.values()}) != expected): return Evaluation(FAIL, "workers do not occupy distinct nodes and claims", evidence) return Evaluation(PASS, "three Ready workers have distinct nodes, ordinals, and claims", evidence) # fmt: on _SHA_RE = re.compile(r"[0-9a-f]{40}\Z") _DIGEST_RE = re.compile(r"sha256:[0-9a-f]{64}\Z") _IMAGE_RE = re.compile( r"[^@\s]+:git-([0-9a-f]{40})-build-([1-9][0-9]*)@(sha256:[0-9a-f]{64})\Z" ) def _strict_object(outcome: Outcome, label: str) -> dict[str, Any] | Evaluation: try: payload = strict_json(outcome.stdout) except ValueError as exc: return Evaluation(NOT_RUN, f"{label} JSON is malformed: {exc}") if not isinstance(payload, dict): return Evaluation(NOT_RUN, f"{label} JSON is not an object") return payload # fmt: off @evaluator("release_lineage") def evaluate_release_lineage(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: """Atomically bind Git, PR, image, build, Flux, and running revision evidence.""" blocked = missing_steps(spec, outcomes) if blocked: return blocked expected = spec.expect main_sha = expected["main_sha"] head_sha = expected["head_sha"] image = expected["image"] build_sha = expected["build_sha"] revision = str(expected["deployment_revision"]) digest = image.rsplit("@", 1)[-1] image_match = _IMAGE_RE.fullmatch(image) if not (_SHA_RE.fullmatch(main_sha) and _SHA_RE.fullmatch(head_sha) and _SHA_RE.fullmatch(build_sha) and _DIGEST_RE.fullmatch(digest) and image_match and image_match.group(1) == build_sha and build_sha == main_sha): return Evaluation(NOT_RUN, "release lineage expectations are malformed") remote = outcomes["remote-main"].stdout.strip().split() if remote != [main_sha, "refs/heads/main"]: return Evaluation(FAIL, "remote main does not match the fixed expected SHA", {"remote_main": remote[:2]}) pr = _strict_object(outcomes["reviewed-pr"], "reviewed PR") if isinstance(pr, Evaluation): return pr fields = { "number": expected["pr_number"], "state": "closed", "merged": True, "base.ref": "main", "head.ref": expected["head_ref"], "head.sha": head_sha, } for path, wanted in fields.items(): try: actual = dotted(pr, path) except (KeyError, ValueError, IndexError): return Evaluation(NOT_RUN, f"reviewed PR is missing {path}") if not exact_equal(actual, wanted): return Evaluation(FAIL, f"reviewed PR {path} does not match", {path: actual}) if outcomes["build-source"].stdout.strip() != build_sha: return Evaluation(FAIL, "build source SHA does not match the fixed release build SHA") ancestry = outcomes["merge-ancestry"] if not ancestry.ok: status = FAIL if ancestry.returncode == 1 else NOT_RUN return Evaluation(status, "reviewed head is not a proven ancestor of remote main") deployment = outcomes["deployment"].stdout.strip().split("\t") if len(deployment) != 6: return Evaluation(NOT_RUN, "Deployment projection is malformed") generation, observed, replicas, ready, deployed_revision, deployed_image = deployment try: current = (int(generation) > 0 and generation == observed and int(replicas) > 0 and replicas == ready) except ValueError: return Evaluation(NOT_RUN, "Deployment numeric fields are malformed") if not current or (deployed_revision, deployed_image) != (revision, image): return Evaluation(FAIL, "Deployment lineage does not match the release contract") replica_hash = "" for number, row in enumerate(lines(outcomes["replicasets"]), 1): columns = row.split("\t") if len(columns) != 6: return Evaluation(NOT_RUN, f"ReplicaSet row {number} is malformed") _, rs_revision, rs_ready, rs_available, rs_image, rs_hash = columns if rs_revision != revision: continue if (rs_ready != replicas or rs_available != replicas or rs_image != image or not rs_hash): return Evaluation(FAIL, "active ReplicaSet does not match the Deployment lineage") if replica_hash: return Evaluation(FAIL, "more than one ReplicaSet claims the running revision") replica_hash = rs_hash if not replica_hash: return Evaluation(FAIL, "running Deployment revision has no matching ReplicaSet") pod_rows = lines(outcomes["pods"]) if len(pod_rows) != int(replicas): return Evaluation(FAIL, "running pod count does not match the Deployment") pods: list[str] = [] for number, row in enumerate(pod_rows, 1): columns = row.split("\t") if len(columns) != 6: return Evaluation(NOT_RUN, f"pod row {number} is malformed") name, phase, ready_state, pod_image, image_id, pod_hash = columns if phase != "Running" or ready_state != "True": return Evaluation(FAIL, f"pod {name or number} is not Ready") if (pod_image, pod_hash) != (image, replica_hash) or not image_id.endswith(f"@{digest}"): return Evaluation(FAIL, f"pod {name or number} lineage does not match") pods.append(name) flux = outcomes["flux"].stdout.strip().split("\t") if len(flux) != 6: return Evaluation(NOT_RUN, "Flux lineage projection is malformed") name, generation, observed, suspended, ready_state, applied = flux if (not name or generation != observed or suspended.lower() == "true" or ready_state != "True"): return Evaluation(FAIL, "Flux source is not current and Ready") match = re.search(r"sha1:([0-9a-f]{40})\Z", applied) if not match or match.group(1) != main_sha: return Evaluation(FAIL, "Flux applied revision is not the expected remote main") return Evaluation(PASS, "Git, PR, image, build, Flux, and running revision are bound", {"main_sha": main_sha, "head_sha": head_sha, "image_digest": digest, "build_sha": build_sha, "deployment_revision": revision, "pods": pods}) # fmt: on # Two observations of the same fact agree whether or not they came from # different vantages; the dual-vantage case is just the important one. EVALUATORS["steps_agree"] = evaluate_vantages_agree