"""End-to-end contracts for the Hermes full-handoff acceptance harness CLI.""" from __future__ import annotations import json import subprocess import pytest from testing.tests.test_hermes_handoff_support import FakeClock, load_handoff_module acceptance = load_handoff_module("hermes_handoff_acceptance") catalog = load_handoff_module("hermes_handoff_catalog") exec_module = load_handoff_module("hermes_handoff_exec") model = load_handoff_module("hermes_handoff_model") policy = load_handoff_module("hermes_handoff_policy") harness_run = load_handoff_module("hermes_handoff_run") def completed(stdout: str = "", stderr: str = "", returncode: int = 0): return subprocess.CompletedProcess(args=[], returncode=returncode, stdout=stdout, stderr=stderr) class ScriptedCluster: """A ``subprocess.run`` stand-in that answers by command shape. Real runs issue a couple of hundred commands; scripting them positionally would make every test a brittle ordering puzzle. Matching on the command line keeps each test about the behaviour it is checking. """ def __init__(self, rules: list[tuple[str, object]], default=None) -> None: self.rules = rules self.default = default or completed(stderr="not found", returncode=1) self.calls: list[list[str]] = [] def __call__(self, argv, **_kwargs): self.calls.append(list(argv)) line = " ".join(argv) for fragment, response in self.rules: if fragment in line: return response return self.default def whoami(username: str): return completed(stdout=json.dumps({"status": {"userInfo": {"username": username}}})) HEALTHY_CLUSTER = [ ("auth whoami", whoami("kubernetes-admin")), ("get pods --selector app=hermes-agent", completed(stdout="pod/hermes-agent-1\n")), ("get pods --selector", completed(stdout="pod/other-1\n")), ("get pod/hermes-chat-tenant-0", completed(stdout="pod/hermes-chat-tenant-0\n")), ] def run_cli(monkeypatch, argv, rules): cluster = ScriptedCluster(rules) clock = FakeClock() def build_runner(**kwargs): return exec_module.Runner(clock=clock, spawn=cluster, environ={}, **kwargs) monkeypatch.setattr(acceptance, "Runner", build_runner) return (acceptance.main(argv), cluster) def read_report(path): return json.loads(path.read_text(encoding="utf-8")) def test_the_parser_exposes_the_documented_controls() -> None: arguments = acceptance.build_parser().parse_args( [ "--namespace", "hermes", "--context", "atlas-operator", "--dependency-pr", "14", "--dependency-pr", "15", "--expected-suspension", "flux-system/parked", "--node-count", "21", "--pool-worker-env", "HERMES_POOL_LEASE_SECONDS", "--expect-telegram-sessions", ] ) targets = acceptance.targets_from_args(arguments, None) assert targets.dependency_pull_requests == (14, 15) assert targets.expected_suspensions == ("flux-system/parked",) assert targets.node_count == 21 assert targets.pool_worker_env == ("HERMES_POOL_LEASE_SECONDS",) assert targets.expect_telegram_sessions is True def test_defaults_fall_back_to_the_shipped_targets() -> None: arguments = acceptance.build_parser().parse_args([]) targets = acceptance.targets_from_args(arguments, None) assert targets.dependency_pull_requests == catalog.Targets.dependency_pull_requests assert targets.expected_suspensions == () assert targets.repo == catalog.Targets.repo def test_a_default_run_is_read_only_and_never_pushes_or_posts(monkeypatch, tmp_path) -> None: output = tmp_path / "report.json" status, cluster = run_cli( monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER ) report = read_report(output) assert status == acceptance.EXIT_NO_GO assert report["mode"] == policy.READ_ONLY assert not any("push" in call for call in cluster.calls) assert not any("POST" in call for call in cluster.calls) assert not any("DELETE" in call for call in cluster.calls) writes = [ call for call in cluster.calls if {"patch", "create", "delete", "apply", "replace"} & set(call) and "can-i" not in call ] assert writes assert all("--dry-run=server" in call for call in writes) def test_mutating_checks_report_not_run_until_the_mode_is_armed(monkeypatch, tmp_path) -> None: output = tmp_path / "report.json" run_cli(monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER) ephemeral = { check["id"]: check for check in read_report(output)["checks"] if check["group"] == "ephemeral" } assert len(ephemeral) == 4 for check in ephemeral.values(): assert check["status"] == model.NOT_RUN assert check["mandatory"] is False assert check["scope"] == model.EPHEMERAL def test_a_zero_state_cluster_is_no_go_with_every_check_accounted_for( monkeypatch, tmp_path ) -> None: """Nothing observed must never read as nothing wrong.""" output = tmp_path / "report.json" status, _ = run_cli( monkeypatch, ["--output", str(output), "--json-only"], [("auth whoami", whoami("kubernetes-admin"))], ) report = read_report(output) assert status == acceptance.EXIT_NO_GO assert report["decision"] == model.NO_GO assert report["counts"][model.PASS] == 0 assert sum(report["counts"].values()) == len(report["checks"]) assert any("in-pod Hermes vantage is unavailable" in error for error in report["harness_errors"]) def test_partial_failure_is_reported_per_check_rather_than_aborting( monkeypatch, tmp_path ) -> None: output = tmp_path / "report.json" rules = [ *HEALTHY_CLUSTER, ("auth can-i", completed(stdout="no\n", returncode=1)), ("get namespaces", completed(stdout="namespace/hermes\n")), ("merge-base", completed()), ] status, _ = run_cli(monkeypatch, ["--output", str(output), "--json-only"], rules) report = read_report(output) statuses = {check["status"] for check in report["checks"]} assert status == acceptance.EXIT_NO_GO assert model.PASS in statuses and model.NOT_RUN in statuses assert report["blocking"] def test_a_vantage_collision_is_a_no_go_even_with_clean_checks(monkeypatch, tmp_path) -> None: output = tmp_path / "report.json" rules = [ ("exec --namespace hermes hermes-agent-1", whoami("system:serviceaccount:hermes:hermes-agent")), ("auth whoami", whoami("system:serviceaccount:hermes:hermes-agent")), ("get pods --selector app=hermes-agent", completed(stdout="pod/hermes-agent-1\n")), ("get pods --selector", completed(stdout="pod/other-1\n")), ("get pod/hermes-chat-tenant-0", completed(stdout="pod/hermes-chat-tenant-0\n")), ] status, _ = run_cli(monkeypatch, ["--output", str(output), "--json-only"], rules) report = read_report(output) assert status == acceptance.EXIT_NO_GO assert any("same principal" in error for error in report["harness_errors"]) def test_an_exhausted_deadline_leaves_the_remaining_checks_not_run(monkeypatch, tmp_path) -> None: output = tmp_path / "report.json" status, _ = run_cli( monkeypatch, ["--output", str(output), "--json-only", "--deadline", "0"], HEALTHY_CLUSTER ) report = read_report(output) assert status == acceptance.EXIT_NO_GO assert report["counts"][model.NOT_RUN] > 0 assert any("deadline expired" in error for error in report["harness_errors"]) def test_arming_with_a_wrong_confirmation_stops_before_anything_runs( monkeypatch, tmp_path ) -> None: output = tmp_path / "report.json" status, cluster = run_cli( monkeypatch, [ "--output", str(output), "--json-only", "--arm-ephemeral-push", "--confirm", "please", "--ephemeral-token", "acceptance-20260817a", ], HEALTHY_CLUSTER, ) report = read_report(output) assert status == acceptance.EXIT_NO_GO assert "arming refused" in report["harness_errors"][0] assert not any("push" in call for call in cluster.calls) def test_arming_against_a_protected_push_target_is_refused(monkeypatch, tmp_path) -> None: output = tmp_path / "report.json" status, cluster = run_cli( monkeypatch, [ "--output", str(output), "--json-only", "--arm-ephemeral-push", "--confirm", acceptance.CONFIRMATION, "--ephemeral-token", "main", ], HEALTHY_CLUSTER, ) assert status == acceptance.EXIT_NO_GO assert "arming refused" in read_report(output)["harness_errors"][0] assert not any("push" in call for call in cluster.calls) def test_an_armed_run_pushes_one_ephemeral_ref_and_verifies_its_removal( monkeypatch, tmp_path ) -> None: output = tmp_path / "report.json" rules = [ ("git ls-remote origin refs/heads/ephemeral", completed(stdout="")), ("git push", completed()), ("POST /api/v1/repos/atlas/titan-iac/pulls", completed(stdout=json.dumps({"number": 9}))), ("PATCH", completed()), ("DELETE", completed()), ("GET /api/v1/repos/atlas/titan-iac/pulls/9", completed(stdout='{"state": "closed"}')), *HEALTHY_CLUSTER, ] status, cluster = run_cli( monkeypatch, [ "--output", str(output), "--json-only", "--arm-ephemeral-push", "--confirm", acceptance.CONFIRMATION, "--ephemeral-token", "acceptance-20260817a", ], rules, ) report = read_report(output) ephemeral = {check["id"]: check for check in report["checks"] if check["group"] == "ephemeral"} assert report["mode"] == policy.ARMED assert status == acceptance.EXIT_NO_GO # the rest of the cluster is still unproven assert ephemeral["ephemeral.feature-branch-push"]["status"] == model.PASS assert ephemeral["ephemeral.draft-pull-request"]["status"] == model.PASS assert ephemeral["ephemeral.cleanup-verified"]["status"] == model.PASS assert ephemeral["ephemeral.protected-branch-refusal"]["status"] == model.PASS pushes = [call for call in cluster.calls if "push" in call] assert len(pushes) == 1 assert pushes[0][-1].startswith("HEAD:refs/heads/ephemeral/hermes-handoff-acceptance/") assert pushes[0][0] == "kubectl" and "exec" in pushes[0] # pushed as the worker, not the operator assert any("DELETE" in call for call in cluster.calls) def test_the_report_is_written_to_stdout_when_no_output_path_is_given( monkeypatch, capsys ) -> None: status, _ = run_cli(monkeypatch, ["--json-only"], HEALTHY_CLUSTER) payload = json.loads(capsys.readouterr().out) assert status == acceptance.EXIT_NO_GO assert payload["harness"] == model.HARNESS def test_the_human_summary_accompanies_the_json_unless_suppressed( monkeypatch, capsys, tmp_path ) -> None: output = tmp_path / "report.json" run_cli(monkeypatch, ["--output", str(output)], HEALTHY_CLUSTER) captured = capsys.readouterr() assert model.HARNESS in captured.err assert "vantage operator" in captured.err assert captured.out == "" def test_a_report_that_fails_its_own_screening_is_replaced_not_published(tmp_path) -> None: """The last barrier: never write an artifact nobody re-read.""" output = tmp_path / "report.json" arguments = acceptance.build_parser().parse_args(["--output", str(output), "--json-only"]) report = model.Report(mode="read-only", started_at="2026-08-17T00:00:00Z") report.harness_errors = ["leaked ghp_ABCDEFGHIJKLMNOPQRSTUV1234"] monkey = report.as_dict report.as_dict = lambda *_args, **_kwargs: { # type: ignore[method-assign] **monkey(), "leak": "token: ghp_ABCDEFGHIJKLMNOPQRSTUV1234", } offenders = acceptance.emit(report, arguments) payload = read_report(output) assert offenders == ["$.leak"] assert payload["decision"] == model.NO_GO assert "leak" not in payload assert "failed its own credential screening" in payload["harness_errors"][0] def test_a_screening_failure_forces_no_go_from_the_entry_point(monkeypatch, tmp_path) -> None: """The exit status must follow the refusal to publish, not the check tally.""" output = tmp_path / "report.json" monkeypatch.setattr(acceptance, "unscreened_fields", lambda _payload: ["$.checks[0].stdout"]) monkeypatch.setattr(acceptance, "build_catalog", lambda _targets: []) monkeypatch.setattr(harness_run, "vantage_problems", lambda _records: []) status, _ = run_cli(monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER) assert status == acceptance.EXIT_NO_GO assert read_report(output)["decision"] == model.NO_GO def test_no_recorded_command_output_carries_a_credential_shape(monkeypatch, tmp_path) -> None: output = tmp_path / "report.json" leaky = completed(stdout="ANTHROPIC_API_KEY=sk-ant-abcdefghijklmnopqrstuvwxyz\n") run_cli( monkeypatch, ["--output", str(output), "--json-only"], [*HEALTHY_CLUSTER, ("exec", leaky)], ) assert model.unscreened_fields(read_report(output)) == [] assert "sk-ant-abcdefghijklmnopqrstuvwxyz" not in output.read_text(encoding="utf-8") @pytest.mark.parametrize("flag", ["--help"]) def test_the_help_text_describes_the_read_only_default(flag: str, capsys) -> None: with pytest.raises(SystemExit): acceptance.main([flag]) assert "Read-only by default" in capsys.readouterr().out def test_go_is_reachable_when_nothing_mandatory_is_outstanding(monkeypatch, tmp_path) -> None: """Exercise the success exit path without pretending the cluster is clean.""" output = tmp_path / "report.json" monkeypatch.setattr(harness_run, "build_catalog", lambda _targets: []) monkeypatch.setattr(acceptance, "build_catalog", lambda _targets: []) monkeypatch.setattr(harness_run, "vantage_problems", lambda _records: []) status, _ = run_cli(monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER) assert status == acceptance.EXIT_GO assert read_report(output)["decision"] == model.GO