"""End-to-end contracts for the Hermes full-handoff acceptance harness CLI.""" from __future__ import annotations import json import subprocess import pytest from testing.tests.test_hermes_handoff_support import FakeClock, load_handoff_module acceptance = load_handoff_module("hermes_handoff_acceptance") catalog = load_handoff_module("hermes_handoff_catalog") exec_module = load_handoff_module("hermes_handoff_exec") model = load_handoff_module("hermes_handoff_model") policy = load_handoff_module("hermes_handoff_policy") harness_run = load_handoff_module("hermes_handoff_run") def completed(stdout: str = "", stderr: str = "", returncode: int = 0): return subprocess.CompletedProcess( args=[], returncode=returncode, stdout=stdout, stderr=stderr ) class ScriptedCluster: """A ``subprocess.run`` stand-in that answers by command shape. Real runs issue a couple of hundred commands; scripting them positionally would make every test a brittle ordering puzzle. Matching on the command line keeps each test about the behaviour it is checking. """ def __init__(self, rules: list[tuple[str, object]], default=None) -> None: self.rules = rules self.default = default or completed(stderr="not found", returncode=1) self.calls: list[list[str]] = [] def __call__(self, argv, **_kwargs): self.calls.append(list(argv)) line = " ".join(argv) for fragment, response in self.rules: if fragment in line: return response return self.default def whoami(username: str): return completed(stdout=username) REVIEWED = "8f005458282269ba5c07941814e4237f5d4cf3ac" SHA = "1" * 40 VALID_ARGS = [ "--remote-main-sha", SHA, "--reviewed-head-sha", REVIEWED, "--agent-image", f"registry.example/hermes:git-{'1' * 40}-build-7@sha256:{'2' * 64}", "--build-sha", SHA, "--deployment-revision", "1", "--chat-config-revision", "chat-rev", *[ item for number in catalog.Targets().dependency_pull_requests for item in ("--dependency-head", f"{number}={SHA}") ], ] HEALTHY_CLUSTER = [ ( "exec --namespace hermes hermes-agent-1", whoami("system:serviceaccount:hermes:hermes-agent"), ), ("auth whoami", whoami("kubernetes-admin")), ("get pods --selector app=hermes-agent", completed(stdout="pod/hermes-agent-1\n")), ("get pods --selector", completed(stdout="pod/other-1\n")), ("get pod/hermes-chat-tenant-0", completed(stdout="pod/hermes-chat-tenant-0\n")), ] def run_cli(monkeypatch, argv, rules): cluster = ScriptedCluster(rules) clock = FakeClock() def build_runner(**kwargs): return exec_module.Runner( clock=clock, spawn=cluster, environ={}, attestor=lambda command, _environment: exec_module.Attestation( command, "a" * 64 ), **kwargs, ) monkeypatch.setattr(acceptance, "Runner", build_runner) return (acceptance.main([*VALID_ARGS, *argv]), cluster) def read_report(path): return json.loads(path.read_text(encoding="utf-8")) def test_the_parser_exposes_the_documented_controls() -> None: arguments = acceptance.build_parser().parse_args( [ "--namespace", "hermes", "--context", "atlas-operator", "--dependency-pr", "14", "--dependency-pr", "15", "--expected-suspension", "flux-system/parked", "--node-count", "21", "--pool-worker-env", "HERMES_POOL_LEASE_SECONDS", ] ) targets = acceptance.targets_from_args(arguments, None) assert targets.dependency_pull_requests == (14, 15) assert targets.expected_suspensions == ("flux-system/parked",) assert targets.node_count == 21 assert targets.pool_worker_env == ("HERMES_POOL_LEASE_SECONDS",) assert not hasattr(arguments, "concurrency") assert not hasattr(arguments, "expect_telegram_sessions") def test_defaults_fall_back_to_the_shipped_targets() -> None: arguments = acceptance.build_parser().parse_args([]) targets = acceptance.targets_from_args(arguments, None) assert targets.dependency_pull_requests == catalog.Targets.dependency_pull_requests assert targets.expected_suspensions == () assert targets.repo == catalog.Targets.repo @pytest.mark.parametrize("value", ["bad", "x=" + "1" * 40, "0=" + "1" * 40, "1=BAD"]) def test_dependency_heads_require_a_positive_pr_and_lowercase_sha(value: str) -> None: with pytest.raises(Exception): acceptance.dependency_head(value) def test_a_default_run_is_read_only_and_never_pushes_or_posts( monkeypatch, tmp_path ) -> None: output = tmp_path / "report.json" status, cluster = run_cli( monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER ) report = read_report(output) assert status == acceptance.EXIT_NO_GO assert report["mode"] == policy.READ_ONLY assert not any("push" in call for call in cluster.calls) assert not any("POST" in call for call in cluster.calls) assert not any("DELETE" in call for call in cluster.calls) writes = [ call for call in cluster.calls if {"patch", "create", "delete", "apply", "replace"} & set(call) and "can-i" not in call ] assert writes == [] def test_mutating_checks_report_not_run_until_the_mode_is_armed( monkeypatch, tmp_path ) -> None: output = tmp_path / "report.json" run_cli(monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER) ephemeral = { check["id"]: check for check in read_report(output)["checks"] if check["group"] == "ephemeral" } assert len(ephemeral) == 4 for check in ephemeral.values(): assert check["status"] == model.NOT_RUN assert check["mandatory"] is False assert check["scope"] == model.EPHEMERAL def test_a_zero_state_cluster_is_no_go_with_every_check_accounted_for( monkeypatch, tmp_path ) -> None: """Nothing observed must never read as nothing wrong.""" output = tmp_path / "report.json" status, _ = run_cli( monkeypatch, ["--output", str(output), "--json-only"], [("auth whoami", whoami("kubernetes-admin"))], ) report = read_report(output) assert status == acceptance.EXIT_NO_GO assert report["decision"] == model.NO_GO assert report["counts"][model.PASS] == 0 assert sum(report["counts"].values()) == len(report["checks"]) assert any( "in-pod Hermes vantage is unavailable" in error for error in report["harness_errors"] ) def test_partial_failure_is_reported_per_check_rather_than_aborting( monkeypatch, tmp_path ) -> None: output = tmp_path / "report.json" rules = [ *HEALTHY_CLUSTER, ("auth can-i", completed(stdout="no\n", returncode=1)), ("get namespaces", completed(stdout="namespace/hermes\n")), ("merge-base", completed()), ] status, _ = run_cli(monkeypatch, ["--output", str(output), "--json-only"], rules) report = read_report(output) statuses = {check["status"] for check in report["checks"]} assert status == acceptance.EXIT_NO_GO assert model.PASS in statuses and model.NOT_RUN in statuses assert report["blocking"] def test_a_vantage_collision_is_a_no_go_even_with_clean_checks( monkeypatch, tmp_path ) -> None: output = tmp_path / "report.json" rules = [ ( "exec --namespace hermes hermes-agent-1", whoami("system:serviceaccount:hermes:hermes-agent"), ), ("auth whoami", whoami("system:serviceaccount:hermes:hermes-agent")), ( "get pods --selector app=hermes-agent", completed(stdout="pod/hermes-agent-1\n"), ), ("get pods --selector", completed(stdout="pod/other-1\n")), ( "get pod/hermes-chat-tenant-0", completed(stdout="pod/hermes-chat-tenant-0\n"), ), ] status, _ = run_cli(monkeypatch, ["--output", str(output), "--json-only"], rules) report = read_report(output) assert status == acceptance.EXIT_NO_GO assert any("same principal" in error for error in report["harness_errors"]) def test_an_invalid_deadline_stops_before_any_subprocess(monkeypatch, tmp_path) -> None: output = tmp_path / "report.json" status, _ = run_cli( monkeypatch, ["--output", str(output), "--json-only", "--deadline", "0"], HEALTHY_CLUSTER, ) report = read_report(output) assert status == acceptance.EXIT_USAGE assert report["counts"][model.NOT_RUN] > 0 assert any("invalid execution bound" in error for error in report["harness_errors"]) def test_arming_with_a_wrong_confirmation_stops_before_anything_runs( monkeypatch, tmp_path ) -> None: output = tmp_path / "report.json" status, cluster = run_cli( monkeypatch, [ "--output", str(output), "--json-only", "--arm-ephemeral-push", "--confirm", "please", "--ephemeral-token", "acceptance-20260817a", ], HEALTHY_CLUSTER, ) report = read_report(output) assert status == acceptance.EXIT_NO_GO assert "arming refused" in report["harness_errors"][0] assert not any("push" in call for call in cluster.calls) def test_arming_against_a_protected_push_target_is_refused( monkeypatch, tmp_path ) -> None: output = tmp_path / "report.json" status, cluster = run_cli( monkeypatch, [ "--output", str(output), "--json-only", "--arm-ephemeral-push", "--confirm", acceptance.CONFIRMATION, "--ephemeral-token", "main", ], HEALTHY_CLUSTER, ) assert status == acceptance.EXIT_NO_GO assert "arming refused" in read_report(output)["harness_errors"][0] assert not any("push" in call for call in cluster.calls) def test_an_armed_run_does_not_mutate_until_read_only_acceptance_is_go( monkeypatch, tmp_path ) -> None: output = tmp_path / "report.json" monkeypatch.setattr(acceptance, "preflight", lambda *_args: None) rules = [ ("git ls-remote origin refs/heads/ephemeral", completed(stdout="")), ("git push", completed()), ( "POST /api/v1/repos/atlas/titan-iac/pulls", completed(stdout=json.dumps({"number": 9})), ), ("PATCH", completed()), ("DELETE", completed()), ( "read /api/v1/repos/atlas/titan-iac/pulls/9", completed(stdout='{"state": "closed"}'), ), *HEALTHY_CLUSTER, ] status, cluster = run_cli( monkeypatch, [ "--output", str(output), "--json-only", "--arm-ephemeral-push", "--confirm", acceptance.CONFIRMATION, "--ephemeral-token", "acceptance-20260817a", ], rules, ) report = read_report(output) assert report["mode"] == policy.ARMED assert status == acceptance.EXIT_NO_GO assert any( "read-only acceptance did not GO" in error for error in report["harness_errors"] ) pushes = [call for call in cluster.calls if "push" in call] assert pushes == [] assert not any("DELETE" in call for call in cluster.calls) def test_the_report_is_written_to_stdout_when_no_output_path_is_given( monkeypatch, capsys ) -> None: status, _ = run_cli(monkeypatch, ["--json-only"], HEALTHY_CLUSTER) payload = json.loads(capsys.readouterr().out) assert status == acceptance.EXIT_NO_GO assert payload["harness"] == model.HARNESS def test_the_human_summary_accompanies_the_json_unless_suppressed( monkeypatch, capsys, tmp_path ) -> None: output = tmp_path / "report.json" run_cli(monkeypatch, ["--output", str(output)], HEALTHY_CLUSTER) captured = capsys.readouterr() assert model.HARNESS in captured.err assert "vantage operator" in captured.err assert captured.out == "" def test_a_report_that_fails_its_own_screening_is_replaced_not_published( tmp_path, ) -> None: """The last barrier: never write an artifact nobody re-read.""" output = tmp_path / "report.json" arguments = acceptance.build_parser().parse_args( ["--output", str(output), "--json-only"] ) report = model.Report(mode="read-only", started_at="2026-08-17T00:00:00Z") report.harness_errors = ["leaked ghp_ABCDEFGHIJKLMNOPQRSTUV1234"] monkey = report.as_dict report.as_dict = lambda *_args, **_kwargs: { # type: ignore[method-assign] **monkey(), "leak": "token: ghp_ABCDEFGHIJKLMNOPQRSTUV1234", } offenders = acceptance.emit(report, arguments) payload = read_report(output) assert offenders == ["$.leak"] assert payload["decision"] == model.NO_GO assert "leak" not in payload assert "failed its own credential screening" in payload["harness_errors"][0] def test_a_screening_failure_forces_no_go_from_the_entry_point( monkeypatch, tmp_path ) -> None: """The exit status must follow the refusal to publish, not the check tally.""" output = tmp_path / "report.json" monkeypatch.setattr( acceptance, "unscreened_fields", lambda _payload: ["$.checks[0].stdout"] ) monkeypatch.setattr(acceptance, "build_catalog", lambda _targets: []) monkeypatch.setattr(harness_run, "vantage_problems", lambda _records: []) status, _ = run_cli( monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER ) assert status == acceptance.EXIT_NO_GO assert read_report(output)["decision"] == model.NO_GO def test_no_recorded_command_output_carries_a_credential_shape( monkeypatch, tmp_path ) -> None: output = tmp_path / "report.json" leaky = completed(stdout="ANTHROPIC_API_KEY=sk-ant-abcdefghijklmnopqrstuvwxyz\n") run_cli( monkeypatch, ["--output", str(output), "--json-only"], [*HEALTHY_CLUSTER, ("exec", leaky)], ) assert model.unscreened_fields(read_report(output)) == [] assert "sk-ant-abcdefghijklmnopqrstuvwxyz" not in output.read_text(encoding="utf-8") @pytest.mark.parametrize("flag", ["--help"]) def test_the_help_text_describes_the_read_only_default(flag: str, capsys) -> None: with pytest.raises(SystemExit): acceptance.main([flag]) assert "Read-only by default" in capsys.readouterr().out