evaluate_names_absent returned PASS when its step exited 0 with no output, so five mandatory checks - the ones asserting that provider API keys, forge credentials, a cluster-admin binding, and shared coordinator state are absent - could report a pass on no evidence and turn a NO_GO into a GO. Both name rules now resolve their step through one guard in _line_step, so zero observations are NOT_RUN. Regressions pin all five real catalog specs plus both reachable silence paths: a POSIX pipeline whose status comes from its last stage, and a drifted kubectl -o jsonpath. The pool claim projection emits one <volume>=<claim> line per template volume so a volume without a PVC still counts as an observation rather than reading as drift. Also closes the review's reachable hardening and evidence defects: - pin Gitea paths to atlas/titan-iac on an exact segment boundary and reject relative segments, including percent-encoded ones - forbid impersonation structurally in every mode and vantage; the inner command of kubectl exec is re-checked rather than exempted, and validate_catalog no longer guards only the operator vantage - drop flux and helm from the binary allowlist; they had no pinned release digest, so no allowlisted binary can now be admitted that the executor would refuse to attest - remove the inert --concurrency and --expect-telegram-sessions flags and the dead concurrency bound; Telegram continuity stays mandatory - read the ephemeral pull index page by page, treat the create response as an authoritative source for the pull number, close every number either source names, and surface residue_ref plus exact manual_cleanup commands when creation is uncertain - keep executable_path and executable_sha256 on unrecorded bulk-evidence steps so withholding bytes never withholds binary attestation - revert the repo-wide hygiene legacy-exception mechanism; the contract change here is purely additive and the four pre-existing over-cap files are left to the canonical contract change in PR #14/#15 - correct the runbook ruff format scope so the documented command passes Split hermes_handoff_arming.py out of hermes_handoff_ephemeral.py to keep both modules under the 500-line cap. All 16 handoff modules hold at least 95% line and branch coverage; the mutation gate is 13/13. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
439 lines
16 KiB
Python
439 lines
16 KiB
Python
"""Contracts for the strictly bounded, self-cleaning armed mode."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import subprocess
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from testing.tests.test_hermes_handoff_support import (
|
|
FakeClock,
|
|
FakeSpawn,
|
|
load_handoff_module,
|
|
)
|
|
|
|
ephemeral = load_handoff_module("hermes_handoff_ephemeral")
|
|
arming = load_handoff_module("hermes_handoff_arming")
|
|
exec_module = load_handoff_module("hermes_handoff_exec")
|
|
policy = load_handoff_module("hermes_handoff_policy")
|
|
model = load_handoff_module("hermes_handoff_model")
|
|
|
|
TOKEN = "acceptance-20260817a"
|
|
HEAD = "8f005458282269ba5c07941814e4237f5d4cf3ac"
|
|
MAIN = "1" * 40
|
|
VANTAGE = exec_module.operator_vantage()
|
|
|
|
|
|
def request(**overrides):
|
|
fields = {
|
|
"repo": policy.EXPECTED_REPO,
|
|
"remote": policy.EXPECTED_REMOTE,
|
|
"token": TOKEN,
|
|
"confirmation": ephemeral.CONFIRMATION,
|
|
"base": policy.EXPECTED_BASE,
|
|
"expected_head": HEAD,
|
|
"expected_base_sha": MAIN,
|
|
}
|
|
fields.update(overrides)
|
|
return ephemeral.ArmRequest(**fields)
|
|
|
|
|
|
def completed(stdout: str = "", stderr: str = "", returncode: int = 0):
|
|
return subprocess.CompletedProcess(
|
|
args=[], returncode=returncode, stdout=stdout, stderr=stderr
|
|
)
|
|
|
|
|
|
def armed_runner(results: list, mode=policy.ARMED):
|
|
spawn = FakeSpawn(results)
|
|
runner = exec_module.Runner(
|
|
mode=mode,
|
|
clock=FakeClock(),
|
|
spawn=spawn,
|
|
environ={},
|
|
deadline_seconds=3600,
|
|
attestor=lambda command, _environment: exec_module.Attestation(
|
|
command, "a" * 64
|
|
),
|
|
)
|
|
return runner, spawn
|
|
|
|
|
|
def by_id(results: list) -> dict:
|
|
return {result.spec.id: result for result in results}
|
|
|
|
|
|
def pr_payload(state="open", number=42):
|
|
return {
|
|
"number": number,
|
|
"state": state,
|
|
"draft": True,
|
|
"merged": False,
|
|
"base": {"ref": "main", "sha": MAIN},
|
|
"head": {"ref": request().ref, "sha": HEAD},
|
|
}
|
|
|
|
|
|
def linked_worktree(tmp_path: Path) -> Path:
|
|
root = tmp_path / "worktree"
|
|
common = tmp_path / "repo.git"
|
|
gitdir = common / "worktrees" / "acceptance"
|
|
gitdir.mkdir(parents=True)
|
|
(root).mkdir()
|
|
(root / ".git").write_text(f"gitdir: {gitdir}\n", encoding="utf-8")
|
|
(gitdir / "commondir").write_text("../..\n", encoding="utf-8")
|
|
(gitdir / "HEAD").write_text(
|
|
f"ref: refs/heads/{arming.EXPECTED_HEAD_REF}\n", encoding="utf-8"
|
|
)
|
|
ref = common / "refs" / "heads" / arming.EXPECTED_HEAD_REF
|
|
ref.parent.mkdir(parents=True)
|
|
ref.write_text(f"{HEAD}\n", encoding="utf-8")
|
|
remote_ref = common / "refs" / "remotes" / "origin" / "main"
|
|
remote_ref.parent.mkdir(parents=True)
|
|
remote_ref.write_text(f"{MAIN}\n", encoding="utf-8")
|
|
(common / "config").write_text(
|
|
f'[remote "origin"]\n\turl = {arming.EXPECTED_REMOTE_URL}\n',
|
|
encoding="utf-8",
|
|
)
|
|
return root
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"ref", ["main", "master", "refs/heads/main", "HEAD", "feature/x", ""]
|
|
)
|
|
def test_only_the_unique_ephemeral_ref_is_allowed(ref: str) -> None:
|
|
with pytest.raises(ephemeral.ArmingError):
|
|
ephemeral.assert_push_target_allowed(ref)
|
|
ephemeral.assert_push_target_allowed(request().ref)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"overrides",
|
|
[
|
|
{"confirmation": "wrong"},
|
|
{"repo": "atlas/other"},
|
|
{"remote": "upstream"},
|
|
{"base": "develop"},
|
|
{"token": "short"},
|
|
{"expected_head": "bad"},
|
|
{"expected_base_sha": "bad"},
|
|
],
|
|
)
|
|
def test_preflight_rejects_bad_inputs_before_any_runner_exists(
|
|
tmp_path: Path, overrides: dict
|
|
) -> None:
|
|
with pytest.raises(ephemeral.ArmingError):
|
|
ephemeral.preflight(
|
|
request(**overrides), policy.EXPECTED_REPO, linked_worktree(tmp_path)
|
|
)
|
|
|
|
|
|
def test_preflight_attests_the_actual_linked_worktree(tmp_path: Path) -> None:
|
|
ephemeral.preflight(request(), policy.EXPECTED_REPO, linked_worktree(tmp_path))
|
|
root = linked_worktree(tmp_path / "wrong")
|
|
(Path((root / ".git").read_text().split(": ", 1)[1].strip()) / "HEAD").write_text(
|
|
"ref: refs/heads/other\n", encoding="utf-8"
|
|
)
|
|
with pytest.raises(ephemeral.ArmingError, match="existing PR #19"):
|
|
ephemeral.preflight(request(), policy.EXPECTED_REPO, root)
|
|
|
|
|
|
def test_worktree_attestation_rejects_each_malformed_local_source(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
roots = []
|
|
empty = tmp_path / "empty"
|
|
empty.mkdir()
|
|
roots.append(empty)
|
|
bad_marker = linked_worktree(tmp_path / "marker")
|
|
(bad_marker / ".git").write_text("not-a-gitdir\n", encoding="utf-8")
|
|
roots.append(bad_marker)
|
|
wrong_sha = linked_worktree(tmp_path / "sha")
|
|
gitdir = Path((wrong_sha / ".git").read_text().split(": ", 1)[1].strip())
|
|
common = (gitdir / (gitdir / "commondir").read_text().strip()).resolve()
|
|
(common / "refs" / "heads" / arming.EXPECTED_HEAD_REF).write_text("0" * 40)
|
|
roots.append(wrong_sha)
|
|
wrong_main = linked_worktree(tmp_path / "main")
|
|
gitdir = Path((wrong_main / ".git").read_text().split(": ", 1)[1].strip())
|
|
common = (gitdir / (gitdir / "commondir").read_text().strip()).resolve()
|
|
(common / "refs" / "remotes" / "origin" / "main").write_text("0" * 40)
|
|
roots.append(wrong_main)
|
|
bad_config = linked_worktree(tmp_path / "config")
|
|
gitdir = Path((bad_config / ".git").read_text().split(": ", 1)[1].strip())
|
|
common = (gitdir / (gitdir / "commondir").read_text().strip()).resolve()
|
|
(common / "config").write_text("[broken\n", encoding="utf-8")
|
|
roots.append(bad_config)
|
|
wrong_remote = linked_worktree(tmp_path / "remote")
|
|
gitdir = Path((wrong_remote / ".git").read_text().split(": ", 1)[1].strip())
|
|
common = (gitdir / (gitdir / "commondir").read_text().strip()).resolve()
|
|
(common / "config").write_text('[remote "origin"]\nurl=https://example.invalid/x\n')
|
|
roots.append(wrong_remote)
|
|
for root in roots:
|
|
with pytest.raises((ephemeral.ArmingError, FileNotFoundError)):
|
|
ephemeral.preflight(request(), policy.EXPECTED_REPO, root)
|
|
oversized = tmp_path / "oversized"
|
|
oversized.write_text("x" * 5, encoding="utf-8")
|
|
with pytest.raises(ephemeral.ArmingError):
|
|
ephemeral.bounded_read(oversized, 4)
|
|
|
|
|
|
def test_guard_failure_stops_before_runner_calls(monkeypatch) -> None:
|
|
monkeypatch.setattr(ephemeral, "assert_push_target_allowed", lambda _ref: None)
|
|
runner, spawn = armed_runner([])
|
|
results = by_id(ephemeral.run_armed(runner, VANTAGE, request()))
|
|
assert results["ephemeral.protected-branch-refusal"].status == model.FAIL
|
|
assert all(
|
|
result.status in {model.FAIL, model.NOT_RUN} for result in results.values()
|
|
)
|
|
assert spawn.calls == []
|
|
|
|
|
|
def test_full_armed_run_validates_and_removes_exact_artifacts() -> None:
|
|
ref_line = f"{HEAD}\trefs/heads/{request().ref}\n"
|
|
runner, spawn = armed_runner(
|
|
[
|
|
completed(),
|
|
completed(stdout=f"{HEAD}\n"),
|
|
completed(),
|
|
completed(stdout=ref_line),
|
|
completed(stdout=json.dumps(pr_payload())),
|
|
completed(stdout=json.dumps([pr_payload()])),
|
|
completed(stdout=json.dumps(pr_payload(state="closed"))),
|
|
completed(),
|
|
completed(),
|
|
completed(stdout=json.dumps(pr_payload(state="closed"))),
|
|
]
|
|
)
|
|
results = by_id(ephemeral.run_armed(runner, VANTAGE, request()))
|
|
assert all(result.status == model.PASS for result in results.values())
|
|
push = next(call for call in spawn.calls if call["argv"][1] == "push")
|
|
assert push["argv"][-1] == f"HEAD:refs/heads/{request().ref}"
|
|
assert all(result.spec.mandatory for result in results.values())
|
|
|
|
|
|
def test_existing_or_unverifiable_ref_never_reaches_a_write() -> None:
|
|
for first in (completed(stdout="abc\trefs/heads/x\n"), completed(returncode=128)):
|
|
runner, spawn = armed_runner([first])
|
|
results = by_id(ephemeral.run_armed(runner, VANTAGE, request()))
|
|
assert results["ephemeral.feature-branch-push"].status == model.FAIL
|
|
assert not any(
|
|
call["argv"][1] in {"push", "POST", "PATCH", "DELETE"}
|
|
for call in spawn.calls
|
|
)
|
|
|
|
|
|
def test_failed_or_ambiguous_create_is_discovered_and_cleanup_still_runs() -> None:
|
|
ref_line = f"{HEAD}\trefs/heads/{request().ref}\n"
|
|
runner, _ = armed_runner(
|
|
[
|
|
completed(),
|
|
completed(stdout=f"{HEAD}\n"),
|
|
completed(),
|
|
completed(stdout=ref_line),
|
|
completed(returncode=1),
|
|
completed(stdout=json.dumps([pr_payload()])),
|
|
completed(stdout=json.dumps(pr_payload(state="closed"))),
|
|
completed(),
|
|
completed(),
|
|
completed(stdout=json.dumps(pr_payload(state="closed"))),
|
|
]
|
|
)
|
|
results = by_id(ephemeral.run_armed(runner, VANTAGE, request()))
|
|
assert results["ephemeral.draft-pull-request"].status == model.PASS
|
|
assert results["ephemeral.cleanup-verified"].status == model.PASS
|
|
|
|
|
|
def test_pull_request_parsers_reject_every_malformed_or_ambiguous_shape() -> None:
|
|
assert ephemeral._pr_problems([], request()) == ["pull request is not an object"]
|
|
malformed = pr_payload()
|
|
malformed.update({"number": True, "draft": "yes", "base": None})
|
|
malformed["head"] = {"ref": "other", "sha": "bad"}
|
|
problems = ephemeral._pr_problems(malformed, request())
|
|
assert {"number is malformed", "draft does not match", "base is malformed"} <= set(
|
|
problems
|
|
)
|
|
assert ephemeral._created_number(
|
|
exec_module.Outcome((), "x", returncode=1), request()
|
|
) == (0, "pull-request creation failed: exit status 1")
|
|
for body in ("not-json", "{}"):
|
|
number, error = ephemeral._created_number(
|
|
exec_module.Outcome((), "x", 0, body), request()
|
|
)
|
|
assert number == 0 and error
|
|
assert ephemeral._created_number(
|
|
exec_module.Outcome((), "x", 0, json.dumps(pr_payload())), request()
|
|
) == (42, "")
|
|
for rows in ([None], [{"head": {"ref": "other"}, "base": {}}]):
|
|
numbers, error = ephemeral._match_pulls(rows, request())
|
|
assert numbers == [] and error
|
|
invalid = pr_payload()
|
|
invalid["draft"] = False
|
|
assert ephemeral._match_pulls([invalid], request())[1]
|
|
assert "discovered 0" in ephemeral._match_pulls([], request())[1]
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"delete_result,remaining,closed",
|
|
[
|
|
(completed(returncode=1), completed(), pr_payload(state="closed")),
|
|
(completed(), completed(returncode=128), pr_payload(state="closed")),
|
|
(completed(), completed(), {}),
|
|
(completed(), completed(), pr_payload(state="open")),
|
|
],
|
|
)
|
|
def test_cleanup_fails_closed_on_partial_or_malformed_evidence(
|
|
delete_result, remaining, closed
|
|
) -> None:
|
|
ref_line = f"{HEAD}\trefs/heads/{request().ref}\n"
|
|
runner, _ = armed_runner(
|
|
[
|
|
completed(),
|
|
completed(stdout=f"{HEAD}\n"),
|
|
completed(),
|
|
completed(stdout=ref_line),
|
|
completed(),
|
|
completed(stdout=json.dumps([pr_payload()])),
|
|
completed(stdout=json.dumps(pr_payload(state="closed"))),
|
|
delete_result,
|
|
remaining,
|
|
completed(stdout=json.dumps(closed)),
|
|
]
|
|
)
|
|
cleanup = by_id(ephemeral.run_armed(runner, VANTAGE, request()))[
|
|
"ephemeral.cleanup-verified"
|
|
]
|
|
assert cleanup.status == model.FAIL
|
|
|
|
|
|
def test_cleanup_fails_when_close_or_verification_commands_are_bad_json_or_failures() -> (
|
|
None
|
|
):
|
|
api = "/api/v1/repos/atlas/titan-iac"
|
|
for scripted in (
|
|
[completed(returncode=1), completed(), completed(), completed(returncode=1)],
|
|
[completed(stdout="bad"), completed(), completed(), completed(stdout="bad")],
|
|
[
|
|
completed(stdout=json.dumps(pr_payload())),
|
|
completed(),
|
|
completed(),
|
|
completed(stdout=json.dumps(pr_payload(state="closed"))),
|
|
],
|
|
):
|
|
runner, _ = armed_runner(scripted)
|
|
result = ephemeral._cleanup(runner, VANTAGE, request(), api, [42])
|
|
assert result.status == model.FAIL
|
|
|
|
|
|
def test_read_only_runner_cannot_reach_push() -> None:
|
|
runner, spawn = armed_runner(
|
|
[completed(), completed(stdout=f"{HEAD}\n"), completed(), completed()],
|
|
mode=policy.READ_ONLY,
|
|
)
|
|
results = by_id(ephemeral.run_armed(runner, VANTAGE, request()))
|
|
assert results["ephemeral.feature-branch-push"].status == model.FAIL
|
|
assert not any(call["argv"][1] == "push" for call in spawn.calls)
|
|
|
|
|
|
def push_prelude(ref_line: str) -> list:
|
|
"""The four scripted results that get an armed run to a verified push."""
|
|
return [
|
|
completed(),
|
|
completed(stdout=f"{HEAD}\n"),
|
|
completed(),
|
|
completed(stdout=ref_line),
|
|
]
|
|
|
|
|
|
def test_pull_request_discovery_reads_every_page() -> None:
|
|
"""A full first page must not end discovery: the match can be on page two."""
|
|
ref_line = f"{HEAD}\trefs/heads/{request().ref}\n"
|
|
filler = [
|
|
{
|
|
"number": index,
|
|
"state": "open",
|
|
"draft": True,
|
|
"merged": False,
|
|
"base": {"ref": "main", "sha": MAIN},
|
|
"head": {"ref": "feature/other", "sha": HEAD},
|
|
}
|
|
for index in range(1, ephemeral.DISCOVERY_PAGE_SIZE + 1)
|
|
]
|
|
runner, spawn = armed_runner(
|
|
[
|
|
*push_prelude(ref_line),
|
|
completed(stdout=json.dumps(pr_payload())),
|
|
completed(stdout=json.dumps(filler)),
|
|
completed(stdout=json.dumps([pr_payload()])),
|
|
completed(stdout=json.dumps(pr_payload(state="closed"))),
|
|
completed(),
|
|
completed(),
|
|
completed(stdout=json.dumps(pr_payload(state="closed"))),
|
|
]
|
|
)
|
|
|
|
results = by_id(ephemeral.run_armed(runner, VANTAGE, request()))
|
|
|
|
assert results["ephemeral.draft-pull-request"].status == model.PASS
|
|
pages = [call["argv"][2] for call in spawn.calls if "pulls?state=all" in str(call)]
|
|
assert any("page=1" in page for page in pages)
|
|
assert any("page=2" in page for page in pages)
|
|
|
|
|
|
def test_uncertain_discovery_still_closes_the_created_pull_and_names_residue() -> None:
|
|
"""The create response names the pull, so an unusable index cannot orphan it."""
|
|
ref_line = f"{HEAD}\trefs/heads/{request().ref}\n"
|
|
runner, spawn = armed_runner(
|
|
[
|
|
*push_prelude(ref_line),
|
|
completed(stdout=json.dumps(pr_payload())),
|
|
completed(returncode=1),
|
|
completed(stdout=json.dumps(pr_payload(state="closed"))),
|
|
completed(),
|
|
completed(),
|
|
completed(stdout=json.dumps(pr_payload(state="closed"))),
|
|
]
|
|
)
|
|
|
|
results = by_id(ephemeral.run_armed(runner, VANTAGE, request()))
|
|
draft = results["ephemeral.draft-pull-request"]
|
|
|
|
assert draft.status == model.FAIL
|
|
assert draft.evidence["pull_requests"] == [42]
|
|
assert draft.evidence["residue_ref"] == request().ref
|
|
assert any("pulls/42" in line for line in draft.evidence["manual_cleanup"])
|
|
assert any("branches/" in line for line in draft.evidence["manual_cleanup"])
|
|
# The pull the harness created is closed, not left behind.
|
|
assert any(
|
|
call["argv"][1] == "PATCH" and call["argv"][2].endswith("/pulls/42")
|
|
for call in spawn.calls
|
|
)
|
|
assert results["ephemeral.cleanup-verified"].status == model.FAIL
|
|
|
|
|
|
def test_a_wholly_unidentifiable_create_names_the_ref_for_manual_cleanup() -> None:
|
|
"""When neither source names a pull, the operator gets the exact ref to sweep."""
|
|
ref_line = f"{HEAD}\trefs/heads/{request().ref}\n"
|
|
runner, _ = armed_runner(
|
|
[
|
|
*push_prelude(ref_line),
|
|
completed(stdout="not-json"),
|
|
completed(returncode=1),
|
|
completed(),
|
|
completed(),
|
|
]
|
|
)
|
|
|
|
draft = by_id(ephemeral.run_armed(runner, VANTAGE, request()))[
|
|
"ephemeral.draft-pull-request"
|
|
]
|
|
|
|
assert draft.status == model.FAIL
|
|
assert draft.evidence["pull_requests"] == []
|
|
assert draft.evidence["residue_ref"] == request().ref
|
|
assert any(
|
|
request().ref in line and "close by hand" in line
|
|
for line in draft.evidence["manual_cleanup"]
|
|
)
|