atlas-iac/testing/tests/test_hermes_handoff_ephemeral.py
Hermes Agent d208a89d9d fix(hermes): close the zero-evidence fail-open in absence checks
evaluate_names_absent returned PASS when its step exited 0 with no output,
so five mandatory checks - the ones asserting that provider API keys, forge
credentials, a cluster-admin binding, and shared coordinator state are
absent - could report a pass on no evidence and turn a NO_GO into a GO.
Both name rules now resolve their step through one guard in _line_step, so
zero observations are NOT_RUN. Regressions pin all five real catalog specs
plus both reachable silence paths: a POSIX pipeline whose status comes from
its last stage, and a drifted kubectl -o jsonpath. The pool claim projection
emits one <volume>=<claim> line per template volume so a volume without a
PVC still counts as an observation rather than reading as drift.

Also closes the review's reachable hardening and evidence defects:

- pin Gitea paths to atlas/titan-iac on an exact segment boundary and
  reject relative segments, including percent-encoded ones
- forbid impersonation structurally in every mode and vantage; the inner
  command of kubectl exec is re-checked rather than exempted, and
  validate_catalog no longer guards only the operator vantage
- drop flux and helm from the binary allowlist; they had no pinned release
  digest, so no allowlisted binary can now be admitted that the executor
  would refuse to attest
- remove the inert --concurrency and --expect-telegram-sessions flags and
  the dead concurrency bound; Telegram continuity stays mandatory
- read the ephemeral pull index page by page, treat the create response as
  an authoritative source for the pull number, close every number either
  source names, and surface residue_ref plus exact manual_cleanup commands
  when creation is uncertain
- keep executable_path and executable_sha256 on unrecorded bulk-evidence
  steps so withholding bytes never withholds binary attestation
- revert the repo-wide hygiene legacy-exception mechanism; the contract
  change here is purely additive and the four pre-existing over-cap files
  are left to the canonical contract change in PR #14/#15
- correct the runbook ruff format scope so the documented command passes

Split hermes_handoff_arming.py out of hermes_handoff_ephemeral.py to keep
both modules under the 500-line cap. All 16 handoff modules hold at least
95% line and branch coverage; the mutation gate is 13/13.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 16:24:37 +00:00

439 lines
16 KiB
Python

"""Contracts for the strictly bounded, self-cleaning armed mode."""
from __future__ import annotations
import json
import subprocess
from pathlib import Path
import pytest
from testing.tests.test_hermes_handoff_support import (
FakeClock,
FakeSpawn,
load_handoff_module,
)
ephemeral = load_handoff_module("hermes_handoff_ephemeral")
arming = load_handoff_module("hermes_handoff_arming")
exec_module = load_handoff_module("hermes_handoff_exec")
policy = load_handoff_module("hermes_handoff_policy")
model = load_handoff_module("hermes_handoff_model")
TOKEN = "acceptance-20260817a"
HEAD = "8f005458282269ba5c07941814e4237f5d4cf3ac"
MAIN = "1" * 40
VANTAGE = exec_module.operator_vantage()
def request(**overrides):
fields = {
"repo": policy.EXPECTED_REPO,
"remote": policy.EXPECTED_REMOTE,
"token": TOKEN,
"confirmation": ephemeral.CONFIRMATION,
"base": policy.EXPECTED_BASE,
"expected_head": HEAD,
"expected_base_sha": MAIN,
}
fields.update(overrides)
return ephemeral.ArmRequest(**fields)
def completed(stdout: str = "", stderr: str = "", returncode: int = 0):
return subprocess.CompletedProcess(
args=[], returncode=returncode, stdout=stdout, stderr=stderr
)
def armed_runner(results: list, mode=policy.ARMED):
spawn = FakeSpawn(results)
runner = exec_module.Runner(
mode=mode,
clock=FakeClock(),
spawn=spawn,
environ={},
deadline_seconds=3600,
attestor=lambda command, _environment: exec_module.Attestation(
command, "a" * 64
),
)
return runner, spawn
def by_id(results: list) -> dict:
return {result.spec.id: result for result in results}
def pr_payload(state="open", number=42):
return {
"number": number,
"state": state,
"draft": True,
"merged": False,
"base": {"ref": "main", "sha": MAIN},
"head": {"ref": request().ref, "sha": HEAD},
}
def linked_worktree(tmp_path: Path) -> Path:
root = tmp_path / "worktree"
common = tmp_path / "repo.git"
gitdir = common / "worktrees" / "acceptance"
gitdir.mkdir(parents=True)
(root).mkdir()
(root / ".git").write_text(f"gitdir: {gitdir}\n", encoding="utf-8")
(gitdir / "commondir").write_text("../..\n", encoding="utf-8")
(gitdir / "HEAD").write_text(
f"ref: refs/heads/{arming.EXPECTED_HEAD_REF}\n", encoding="utf-8"
)
ref = common / "refs" / "heads" / arming.EXPECTED_HEAD_REF
ref.parent.mkdir(parents=True)
ref.write_text(f"{HEAD}\n", encoding="utf-8")
remote_ref = common / "refs" / "remotes" / "origin" / "main"
remote_ref.parent.mkdir(parents=True)
remote_ref.write_text(f"{MAIN}\n", encoding="utf-8")
(common / "config").write_text(
f'[remote "origin"]\n\turl = {arming.EXPECTED_REMOTE_URL}\n',
encoding="utf-8",
)
return root
@pytest.mark.parametrize(
"ref", ["main", "master", "refs/heads/main", "HEAD", "feature/x", ""]
)
def test_only_the_unique_ephemeral_ref_is_allowed(ref: str) -> None:
with pytest.raises(ephemeral.ArmingError):
ephemeral.assert_push_target_allowed(ref)
ephemeral.assert_push_target_allowed(request().ref)
@pytest.mark.parametrize(
"overrides",
[
{"confirmation": "wrong"},
{"repo": "atlas/other"},
{"remote": "upstream"},
{"base": "develop"},
{"token": "short"},
{"expected_head": "bad"},
{"expected_base_sha": "bad"},
],
)
def test_preflight_rejects_bad_inputs_before_any_runner_exists(
tmp_path: Path, overrides: dict
) -> None:
with pytest.raises(ephemeral.ArmingError):
ephemeral.preflight(
request(**overrides), policy.EXPECTED_REPO, linked_worktree(tmp_path)
)
def test_preflight_attests_the_actual_linked_worktree(tmp_path: Path) -> None:
ephemeral.preflight(request(), policy.EXPECTED_REPO, linked_worktree(tmp_path))
root = linked_worktree(tmp_path / "wrong")
(Path((root / ".git").read_text().split(": ", 1)[1].strip()) / "HEAD").write_text(
"ref: refs/heads/other\n", encoding="utf-8"
)
with pytest.raises(ephemeral.ArmingError, match="existing PR #19"):
ephemeral.preflight(request(), policy.EXPECTED_REPO, root)
def test_worktree_attestation_rejects_each_malformed_local_source(
tmp_path: Path,
) -> None:
roots = []
empty = tmp_path / "empty"
empty.mkdir()
roots.append(empty)
bad_marker = linked_worktree(tmp_path / "marker")
(bad_marker / ".git").write_text("not-a-gitdir\n", encoding="utf-8")
roots.append(bad_marker)
wrong_sha = linked_worktree(tmp_path / "sha")
gitdir = Path((wrong_sha / ".git").read_text().split(": ", 1)[1].strip())
common = (gitdir / (gitdir / "commondir").read_text().strip()).resolve()
(common / "refs" / "heads" / arming.EXPECTED_HEAD_REF).write_text("0" * 40)
roots.append(wrong_sha)
wrong_main = linked_worktree(tmp_path / "main")
gitdir = Path((wrong_main / ".git").read_text().split(": ", 1)[1].strip())
common = (gitdir / (gitdir / "commondir").read_text().strip()).resolve()
(common / "refs" / "remotes" / "origin" / "main").write_text("0" * 40)
roots.append(wrong_main)
bad_config = linked_worktree(tmp_path / "config")
gitdir = Path((bad_config / ".git").read_text().split(": ", 1)[1].strip())
common = (gitdir / (gitdir / "commondir").read_text().strip()).resolve()
(common / "config").write_text("[broken\n", encoding="utf-8")
roots.append(bad_config)
wrong_remote = linked_worktree(tmp_path / "remote")
gitdir = Path((wrong_remote / ".git").read_text().split(": ", 1)[1].strip())
common = (gitdir / (gitdir / "commondir").read_text().strip()).resolve()
(common / "config").write_text('[remote "origin"]\nurl=https://example.invalid/x\n')
roots.append(wrong_remote)
for root in roots:
with pytest.raises((ephemeral.ArmingError, FileNotFoundError)):
ephemeral.preflight(request(), policy.EXPECTED_REPO, root)
oversized = tmp_path / "oversized"
oversized.write_text("x" * 5, encoding="utf-8")
with pytest.raises(ephemeral.ArmingError):
ephemeral.bounded_read(oversized, 4)
def test_guard_failure_stops_before_runner_calls(monkeypatch) -> None:
monkeypatch.setattr(ephemeral, "assert_push_target_allowed", lambda _ref: None)
runner, spawn = armed_runner([])
results = by_id(ephemeral.run_armed(runner, VANTAGE, request()))
assert results["ephemeral.protected-branch-refusal"].status == model.FAIL
assert all(
result.status in {model.FAIL, model.NOT_RUN} for result in results.values()
)
assert spawn.calls == []
def test_full_armed_run_validates_and_removes_exact_artifacts() -> None:
ref_line = f"{HEAD}\trefs/heads/{request().ref}\n"
runner, spawn = armed_runner(
[
completed(),
completed(stdout=f"{HEAD}\n"),
completed(),
completed(stdout=ref_line),
completed(stdout=json.dumps(pr_payload())),
completed(stdout=json.dumps([pr_payload()])),
completed(stdout=json.dumps(pr_payload(state="closed"))),
completed(),
completed(),
completed(stdout=json.dumps(pr_payload(state="closed"))),
]
)
results = by_id(ephemeral.run_armed(runner, VANTAGE, request()))
assert all(result.status == model.PASS for result in results.values())
push = next(call for call in spawn.calls if call["argv"][1] == "push")
assert push["argv"][-1] == f"HEAD:refs/heads/{request().ref}"
assert all(result.spec.mandatory for result in results.values())
def test_existing_or_unverifiable_ref_never_reaches_a_write() -> None:
for first in (completed(stdout="abc\trefs/heads/x\n"), completed(returncode=128)):
runner, spawn = armed_runner([first])
results = by_id(ephemeral.run_armed(runner, VANTAGE, request()))
assert results["ephemeral.feature-branch-push"].status == model.FAIL
assert not any(
call["argv"][1] in {"push", "POST", "PATCH", "DELETE"}
for call in spawn.calls
)
def test_failed_or_ambiguous_create_is_discovered_and_cleanup_still_runs() -> None:
ref_line = f"{HEAD}\trefs/heads/{request().ref}\n"
runner, _ = armed_runner(
[
completed(),
completed(stdout=f"{HEAD}\n"),
completed(),
completed(stdout=ref_line),
completed(returncode=1),
completed(stdout=json.dumps([pr_payload()])),
completed(stdout=json.dumps(pr_payload(state="closed"))),
completed(),
completed(),
completed(stdout=json.dumps(pr_payload(state="closed"))),
]
)
results = by_id(ephemeral.run_armed(runner, VANTAGE, request()))
assert results["ephemeral.draft-pull-request"].status == model.PASS
assert results["ephemeral.cleanup-verified"].status == model.PASS
def test_pull_request_parsers_reject_every_malformed_or_ambiguous_shape() -> None:
assert ephemeral._pr_problems([], request()) == ["pull request is not an object"]
malformed = pr_payload()
malformed.update({"number": True, "draft": "yes", "base": None})
malformed["head"] = {"ref": "other", "sha": "bad"}
problems = ephemeral._pr_problems(malformed, request())
assert {"number is malformed", "draft does not match", "base is malformed"} <= set(
problems
)
assert ephemeral._created_number(
exec_module.Outcome((), "x", returncode=1), request()
) == (0, "pull-request creation failed: exit status 1")
for body in ("not-json", "{}"):
number, error = ephemeral._created_number(
exec_module.Outcome((), "x", 0, body), request()
)
assert number == 0 and error
assert ephemeral._created_number(
exec_module.Outcome((), "x", 0, json.dumps(pr_payload())), request()
) == (42, "")
for rows in ([None], [{"head": {"ref": "other"}, "base": {}}]):
numbers, error = ephemeral._match_pulls(rows, request())
assert numbers == [] and error
invalid = pr_payload()
invalid["draft"] = False
assert ephemeral._match_pulls([invalid], request())[1]
assert "discovered 0" in ephemeral._match_pulls([], request())[1]
@pytest.mark.parametrize(
"delete_result,remaining,closed",
[
(completed(returncode=1), completed(), pr_payload(state="closed")),
(completed(), completed(returncode=128), pr_payload(state="closed")),
(completed(), completed(), {}),
(completed(), completed(), pr_payload(state="open")),
],
)
def test_cleanup_fails_closed_on_partial_or_malformed_evidence(
delete_result, remaining, closed
) -> None:
ref_line = f"{HEAD}\trefs/heads/{request().ref}\n"
runner, _ = armed_runner(
[
completed(),
completed(stdout=f"{HEAD}\n"),
completed(),
completed(stdout=ref_line),
completed(),
completed(stdout=json.dumps([pr_payload()])),
completed(stdout=json.dumps(pr_payload(state="closed"))),
delete_result,
remaining,
completed(stdout=json.dumps(closed)),
]
)
cleanup = by_id(ephemeral.run_armed(runner, VANTAGE, request()))[
"ephemeral.cleanup-verified"
]
assert cleanup.status == model.FAIL
def test_cleanup_fails_when_close_or_verification_commands_are_bad_json_or_failures() -> (
None
):
api = "/api/v1/repos/atlas/titan-iac"
for scripted in (
[completed(returncode=1), completed(), completed(), completed(returncode=1)],
[completed(stdout="bad"), completed(), completed(), completed(stdout="bad")],
[
completed(stdout=json.dumps(pr_payload())),
completed(),
completed(),
completed(stdout=json.dumps(pr_payload(state="closed"))),
],
):
runner, _ = armed_runner(scripted)
result = ephemeral._cleanup(runner, VANTAGE, request(), api, [42])
assert result.status == model.FAIL
def test_read_only_runner_cannot_reach_push() -> None:
runner, spawn = armed_runner(
[completed(), completed(stdout=f"{HEAD}\n"), completed(), completed()],
mode=policy.READ_ONLY,
)
results = by_id(ephemeral.run_armed(runner, VANTAGE, request()))
assert results["ephemeral.feature-branch-push"].status == model.FAIL
assert not any(call["argv"][1] == "push" for call in spawn.calls)
def push_prelude(ref_line: str) -> list:
"""The four scripted results that get an armed run to a verified push."""
return [
completed(),
completed(stdout=f"{HEAD}\n"),
completed(),
completed(stdout=ref_line),
]
def test_pull_request_discovery_reads_every_page() -> None:
"""A full first page must not end discovery: the match can be on page two."""
ref_line = f"{HEAD}\trefs/heads/{request().ref}\n"
filler = [
{
"number": index,
"state": "open",
"draft": True,
"merged": False,
"base": {"ref": "main", "sha": MAIN},
"head": {"ref": "feature/other", "sha": HEAD},
}
for index in range(1, ephemeral.DISCOVERY_PAGE_SIZE + 1)
]
runner, spawn = armed_runner(
[
*push_prelude(ref_line),
completed(stdout=json.dumps(pr_payload())),
completed(stdout=json.dumps(filler)),
completed(stdout=json.dumps([pr_payload()])),
completed(stdout=json.dumps(pr_payload(state="closed"))),
completed(),
completed(),
completed(stdout=json.dumps(pr_payload(state="closed"))),
]
)
results = by_id(ephemeral.run_armed(runner, VANTAGE, request()))
assert results["ephemeral.draft-pull-request"].status == model.PASS
pages = [call["argv"][2] for call in spawn.calls if "pulls?state=all" in str(call)]
assert any("page=1" in page for page in pages)
assert any("page=2" in page for page in pages)
def test_uncertain_discovery_still_closes_the_created_pull_and_names_residue() -> None:
"""The create response names the pull, so an unusable index cannot orphan it."""
ref_line = f"{HEAD}\trefs/heads/{request().ref}\n"
runner, spawn = armed_runner(
[
*push_prelude(ref_line),
completed(stdout=json.dumps(pr_payload())),
completed(returncode=1),
completed(stdout=json.dumps(pr_payload(state="closed"))),
completed(),
completed(),
completed(stdout=json.dumps(pr_payload(state="closed"))),
]
)
results = by_id(ephemeral.run_armed(runner, VANTAGE, request()))
draft = results["ephemeral.draft-pull-request"]
assert draft.status == model.FAIL
assert draft.evidence["pull_requests"] == [42]
assert draft.evidence["residue_ref"] == request().ref
assert any("pulls/42" in line for line in draft.evidence["manual_cleanup"])
assert any("branches/" in line for line in draft.evidence["manual_cleanup"])
# The pull the harness created is closed, not left behind.
assert any(
call["argv"][1] == "PATCH" and call["argv"][2].endswith("/pulls/42")
for call in spawn.calls
)
assert results["ephemeral.cleanup-verified"].status == model.FAIL
def test_a_wholly_unidentifiable_create_names_the_ref_for_manual_cleanup() -> None:
"""When neither source names a pull, the operator gets the exact ref to sweep."""
ref_line = f"{HEAD}\trefs/heads/{request().ref}\n"
runner, _ = armed_runner(
[
*push_prelude(ref_line),
completed(stdout="not-json"),
completed(returncode=1),
completed(),
completed(),
]
)
draft = by_id(ephemeral.run_armed(runner, VANTAGE, request()))[
"ephemeral.draft-pull-request"
]
assert draft.status == model.FAIL
assert draft.evidence["pull_requests"] == []
assert draft.evidence["residue_ref"] == request().ref
assert any(
request().ref in line and "close by hand" in line
for line in draft.evidence["manual_cleanup"]
)