Hermes Agent
8f00545828
Decides whether the Hermes platform handoff is fit to release, and refuses to round an absence of evidence up to a pass. The harness is read-only by default and classifies 71 checks PASS / FAIL / NOT_RUN / NOT_APPLICABLE. Any mandatory FAIL or NOT_RUN is NO_GO, and so is a harness-level problem: an unreachable vantage, a catalog entry whose evidence no longer exists, an expired deadline, or an evaluator that raised. Evidence comes from two vantages that cannot cover for each other: an external read-only operator kubeconfig, and the Hermes agent probing itself from inside its own pod. Before any check runs, the harness asks each vantage who it is and stops if they are the same principal, because dual-vantage evidence from one identity is a restatement rather than a corroboration. `--as` is rejected for every operator-side command and reachable only as the inner command of a `kubectl exec`, so impersonation can never stand in for a real self-probe. A deny check needs a live refused request, not only an authorization review. Two safety properties are structural rather than conventional, enforced where an argv becomes a subprocess: the default mode mutates nothing (mutating verbs require a server dry run; there is deliberately no live TokenRequest probe, because a successful one would mint a real credential), and no probe can pull a credential value into a report (no vault/sops/curl, secrets readable only with -o name, environment probes list names, shell only through frozen reviewed templates). Captures are bounded before they are screened, and the rendered report is re-screened before it is written. Mutation lives behind a separate arming flag with an exact confirmation phrase, a caller-supplied unique ref, a preflight that refuses a protected push target before any network call, and a cleanup whose verification is itself mandatory. A default run reports those four checks NOT_RUN. The catalog is declarative so a reviewer reads what is asserted rather than how it is plumbed, and so structural properties can be proven over every entry before a run. Catalog drift surfaces as NOT_RUN, which stops the release. docs/hermes_full_handoff_acceptance.md carries the merge order for PRs #14-#18 on top of the merged #13 baseline, the image build and Flux rollout, the rollback point for each step, the go/no-go checklist, and the limits that are asserted rather than exercised. Validation: 295 handoff tests pass with 100% line coverage on all 15 new modules; the full unit suite is 647 passed with two failures that reproduce unchanged on origin/main; Ruff, py_compile, kustomize render, and a diff credential screen are clean; a live read-only run against Atlas returns NO_GO for the pre-merge cluster with no unscreened fields in the report.
202 lines
8.0 KiB
Python
202 lines
8.0 KiB
Python
"""Contracts for the text-shaped handoff acceptance evaluators."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from testing.tests.test_hermes_handoff_support import load_handoff_module, outcome, spec, step
|
|
|
|
evaluators = load_handoff_module("hermes_handoff_evaluators")
|
|
model = load_handoff_module("hermes_handoff_model")
|
|
|
|
REVIEW = model.REVIEW
|
|
ATTEMPT = model.ATTEMPT
|
|
FORBIDDEN = outcome(stderr="Error from server (Forbidden): pods is forbidden", returncode=1)
|
|
|
|
|
|
def deny_spec() -> object:
|
|
return spec(
|
|
"denied",
|
|
steps=(step("review", kind=REVIEW), step("attempt", kind=ATTEMPT)),
|
|
)
|
|
|
|
|
|
def evaluate(built, outcomes):
|
|
return evaluators.evaluate(built, outcomes)
|
|
|
|
|
|
def test_a_deny_check_passes_only_on_a_review_no_and_a_live_refusal() -> None:
|
|
evaluation = evaluate(deny_spec(), {"review": outcome(stdout="no"), "attempt": FORBIDDEN})
|
|
|
|
assert evaluation.status == model.PASS
|
|
assert evaluation.evidence == {"review": "no", "attempt": "refused"}
|
|
|
|
|
|
def test_a_deny_check_fails_when_the_review_says_the_authority_exists() -> None:
|
|
evaluation = evaluate(deny_spec(), {"review": outcome(stdout="yes"), "attempt": FORBIDDEN})
|
|
|
|
assert evaluation.status == model.FAIL
|
|
assert "reported yes" in evaluation.reason
|
|
|
|
|
|
def test_a_deny_check_fails_when_the_forbidden_request_actually_succeeds() -> None:
|
|
evaluation = evaluate(deny_spec(), {"review": outcome(stdout="no"), "attempt": outcome()})
|
|
|
|
assert evaluation.status == model.FAIL
|
|
assert "must be refused" in evaluation.reason
|
|
|
|
|
|
def test_a_request_that_merely_broke_is_not_evidence_of_a_refusal() -> None:
|
|
broken = outcome(stderr="connection refused", returncode=7)
|
|
evaluation = evaluate(deny_spec(), {"review": outcome(stdout="no"), "attempt": broken})
|
|
|
|
assert evaluation.status == model.NOT_RUN
|
|
assert "recognisable refusal" in evaluation.reason
|
|
|
|
|
|
def test_a_deny_check_without_a_live_attempt_refuses_to_classify() -> None:
|
|
"""An authorization review states policy; only an attempt proves enforcement."""
|
|
review_only = spec("denied", steps=(step("review", kind=REVIEW),))
|
|
|
|
evaluation = evaluate(review_only, {"review": outcome(stdout="no")})
|
|
|
|
assert evaluation.status == model.NOT_RUN
|
|
assert "require a real attempt" in evaluation.reason
|
|
|
|
|
|
def test_an_allow_check_needs_every_step_to_succeed_and_every_review_to_say_yes() -> None:
|
|
built = spec("allowed", steps=(step("review", kind=REVIEW), step("read")))
|
|
|
|
assert evaluate(built, {"review": outcome(stdout="yes"), "read": outcome()}).status == model.PASS
|
|
|
|
denied = evaluate(built, {"review": outcome(stdout="no"), "read": outcome()})
|
|
assert denied.status == model.FAIL and "reported no" in denied.reason
|
|
|
|
failed = evaluate(built, {"review": outcome(stdout="yes"), "read": outcome(returncode=1)})
|
|
assert failed.status == model.FAIL and "step read failed" in failed.reason
|
|
|
|
|
|
def test_names_absent_flags_exact_names_and_substrings() -> None:
|
|
built = spec(
|
|
"names_absent",
|
|
{"step": "env", "names": ("GITEA_TOKEN",), "contains": ("API_KEY",)},
|
|
steps=(step("env"),),
|
|
)
|
|
|
|
clean = evaluate(built, {"env": outcome(stdout="PATH\nHOME\n")})
|
|
assert clean.status == model.PASS and clean.evidence["observed_count"] == 2
|
|
|
|
leaked = evaluate(built, {"env": outcome(stdout="PATH\nGITEA_TOKEN\nOPENAI_API_KEY\n")})
|
|
assert leaked.status == model.FAIL
|
|
assert leaked.evidence["offenders"] == ["GITEA_TOKEN", "OPENAI_API_KEY"]
|
|
|
|
unreadable = evaluate(built, {"env": outcome(returncode=1)})
|
|
assert unreadable.status == model.NOT_RUN
|
|
|
|
|
|
def test_names_present_requires_names_and_substrings_and_rejects_a_silent_empty() -> None:
|
|
built = spec(
|
|
"names_present",
|
|
{"step": "init", "names": ("patch-web-session-activity",), "contains": ("@sha256:",)},
|
|
steps=(step("init"),),
|
|
)
|
|
|
|
good = outcome(stdout="patch-web-session-activity\nimage@sha256:abc\n")
|
|
assert evaluate(built, {"init": good}).status == model.PASS
|
|
|
|
partial = evaluate(built, {"init": outcome(stdout="patch-web-session-activity\n")})
|
|
assert partial.status == model.FAIL and "@sha256:" in partial.reason
|
|
|
|
empty = spec("names_present", {"step": "init"}, steps=(step("init"),))
|
|
assert evaluate(empty, {"init": outcome(stdout="")}).status == model.NOT_RUN
|
|
|
|
|
|
def test_stdout_matches_accepts_a_value_or_a_set_of_values() -> None:
|
|
single = spec("stdout_matches", {"step": "s", "equals": "locked"}, steps=(step("s"),))
|
|
assert evaluate(single, {"s": outcome(stdout=" locked \n")}).status == model.PASS
|
|
|
|
mismatch = evaluate(single, {"s": outcome(stdout="unlocked")})
|
|
assert mismatch.status == model.FAIL and mismatch.evidence["observed"] == "unlocked"
|
|
|
|
several = spec(
|
|
"stdout_matches", {"step": "s", "equals": ["600 root:root", "absent"]}, steps=(step("s"),)
|
|
)
|
|
assert evaluate(several, {"s": outcome(stdout="absent")}).status == model.PASS
|
|
|
|
assert evaluate(single, {"s": outcome(returncode=1)}).status == model.NOT_RUN
|
|
|
|
|
|
def test_distinct_count_separates_replica_count_from_actual_spread() -> None:
|
|
built = spec(
|
|
"distinct_count", {"step": "n", "minimum": 3, "total_equals": 3}, steps=(step("n"),)
|
|
)
|
|
|
|
spread = evaluate(built, {"n": outcome(stdout="titan-05\ntitan-06\ntitan-07\n")})
|
|
assert spread.status == model.PASS and spread.evidence["distinct"] == 3
|
|
|
|
stacked = evaluate(built, {"n": outcome(stdout="titan-05\ntitan-05\ntitan-05\n")})
|
|
assert stacked.status == model.FAIL and "distinct values" in stacked.reason
|
|
|
|
extra = spec("distinct_count", {"step": "n", "minimum": 1, "total_equals": 2}, steps=(step("n"),))
|
|
over = evaluate(extra, {"n": outcome(stdout="a\nb\nc\n")})
|
|
assert over.status == model.FAIL and "expected exactly 2" in over.reason
|
|
|
|
assert evaluate(built, {"n": outcome(stdout="")}).status == model.NOT_RUN
|
|
assert evaluate(built, {"n": outcome(returncode=1)}).status == model.NOT_RUN
|
|
|
|
|
|
def test_vantages_agree_reports_a_mismatch_as_its_own_finding() -> None:
|
|
built = spec(
|
|
"vantages_agree",
|
|
{"steps": ("operator", "self")},
|
|
steps=(step("operator", vantage="operator"), step("self")),
|
|
)
|
|
|
|
agree = evaluate(
|
|
built,
|
|
{"operator": outcome(stdout="ns/a\nns/b\n"), "self": outcome(stdout="ns/b\nns/a\n")},
|
|
)
|
|
assert agree.status == model.PASS
|
|
|
|
differ = evaluate(
|
|
built, {"operator": outcome(stdout="ns/a\nns/b\n"), "self": outcome(stdout="ns/a\n")}
|
|
)
|
|
assert differ.status == model.FAIL and "disagree" in differ.reason
|
|
|
|
empty = evaluate(built, {"operator": outcome(stdout=""), "self": outcome(stdout="")})
|
|
assert empty.status == model.NOT_RUN
|
|
|
|
broken = evaluate(built, {"operator": outcome(returncode=1), "self": outcome()})
|
|
assert broken.status == model.NOT_RUN
|
|
|
|
|
|
def test_steps_agree_is_the_same_rule_under_a_neutral_name() -> None:
|
|
assert evaluators.EVALUATORS["steps_agree"] is evaluators.EVALUATORS["vantages_agree"]
|
|
|
|
|
|
def test_lines_match_catches_a_blank_leading_column() -> None:
|
|
built = spec(
|
|
"lines_match",
|
|
{"step": "s", "pattern": r"^\S.*\S$", "skip": 2, "minimum": 1},
|
|
steps=(step("s"),),
|
|
)
|
|
table = "Preview Src ID\n-------\nwork on task cli 2026\n"
|
|
|
|
assert evaluate(built, {"s": outcome(stdout=table)}).status == model.PASS
|
|
|
|
blank = "Preview Src ID\n-------\n cli 2026\n"
|
|
failed = evaluate(built, {"s": outcome(stdout=blank)})
|
|
assert failed.status == model.FAIL and failed.evidence["offenders"]
|
|
|
|
assert evaluate(built, {"s": outcome(stdout="No sessions found.\n")}).status == model.NOT_RUN
|
|
assert evaluate(built, {"s": outcome(returncode=1)}).status == model.NOT_RUN
|
|
|
|
|
|
def test_not_armed_is_the_fixed_skip_a_default_run_must_produce() -> None:
|
|
built = spec("not_armed", {"reason": "ephemeral mutation mode is not armed"})
|
|
|
|
evaluation = evaluate(built, {})
|
|
|
|
assert evaluation.status == model.NOT_RUN
|
|
assert evaluation.evidence == {"armed": False}
|
|
assert evaluation.reason == "ephemeral mutation mode is not armed"
|
|
assert evaluate(spec("not_armed"), {}).reason == "ephemeral mutation mode is not armed"
|