atlas-iac/testing/tests/test_hermes_handoff_acceptance.py

"""End-to-end contracts for the Hermes full-handoff acceptance harness CLI."""

from __future__ import annotations

import json
import subprocess

import pytest

from testing.tests.test_hermes_handoff_support import FakeClock, load_handoff_module

acceptance = load_handoff_module("hermes_handoff_acceptance")
catalog = load_handoff_module("hermes_handoff_catalog")
exec_module = load_handoff_module("hermes_handoff_exec")
model = load_handoff_module("hermes_handoff_model")
policy = load_handoff_module("hermes_handoff_policy")
harness_run = load_handoff_module("hermes_handoff_run")


def completed(stdout: str = "", stderr: str = "", returncode: int = 0):
    return subprocess.CompletedProcess(
        args=[], returncode=returncode, stdout=stdout, stderr=stderr
    )


class ScriptedCluster:
    """A ``subprocess.run`` stand-in that answers by command shape.

    Real runs issue a couple of hundred commands; scripting them positionally
    would make every test a brittle ordering puzzle. Matching on the command
    line keeps each test about the behaviour it is checking.
    """

    def __init__(self, rules: list[tuple[str, object]], default=None) -> None:
        self.rules = rules
        self.default = default or completed(stderr="not found", returncode=1)
        self.calls: list[list[str]] = []

    def __call__(self, argv, **_kwargs):
        self.calls.append(list(argv))
        line = " ".join(argv)
        for fragment, response in self.rules:
            if fragment in line:
                return response
        return self.default


def whoami(username: str):
    return completed(stdout=username)


REVIEWED = "8f005458282269ba5c07941814e4237f5d4cf3ac"
SHA = "1" * 40
VALID_ARGS = [
    "--remote-main-sha",
    SHA,
    "--reviewed-head-sha",
    REVIEWED,
    "--agent-image",
    f"registry.example/hermes:git-{'1' * 40}-build-7@sha256:{'2' * 64}",
    "--build-sha",
    SHA,
    "--deployment-revision",
    "1",
    "--chat-config-revision",
    "chat-rev",
    *[
        item
        for number in catalog.Targets().dependency_pull_requests
        for item in ("--dependency-head", f"{number}={SHA}")
    ],
]


HEALTHY_CLUSTER = [
    (
        "exec --namespace hermes hermes-agent-1",
        whoami("system:serviceaccount:hermes:hermes-agent"),
    ),
    ("auth whoami", whoami("kubernetes-admin")),
    ("get pods --selector app=hermes-agent", completed(stdout="pod/hermes-agent-1\n")),
    ("get pods --selector", completed(stdout="pod/other-1\n")),
    ("get pod/hermes-chat-tenant-0", completed(stdout="pod/hermes-chat-tenant-0\n")),
]


def run_cli(monkeypatch, argv, rules):
    cluster = ScriptedCluster(rules)
    clock = FakeClock()

    def build_runner(**kwargs):
        return exec_module.Runner(
            clock=clock,
            spawn=cluster,
            environ={},
            attestor=lambda command, _environment: exec_module.Attestation(
                command, "a" * 64
            ),
            **kwargs,
        )

    monkeypatch.setattr(acceptance, "Runner", build_runner)
    return (acceptance.main([*VALID_ARGS, *argv]), cluster)


def read_report(path):
    return json.loads(path.read_text(encoding="utf-8"))


def test_the_parser_exposes_the_documented_controls() -> None:
    arguments = acceptance.build_parser().parse_args(
        [
            "--namespace",
            "hermes",
            "--context",
            "atlas-operator",
            "--dependency-pr",
            "14",
            "--dependency-pr",
            "15",
            "--expected-suspension",
            "flux-system/parked",
            "--node-count",
            "21",
            "--pool-worker-env",
            "HERMES_POOL_LEASE_SECONDS",
            "--expect-telegram-sessions",
        ]
    )

    targets = acceptance.targets_from_args(arguments, None)

    assert targets.dependency_pull_requests == (14, 15)
    assert targets.expected_suspensions == ("flux-system/parked",)
    assert targets.node_count == 21
    assert targets.pool_worker_env == ("HERMES_POOL_LEASE_SECONDS",)
    assert targets.expect_telegram_sessions is True


def test_defaults_fall_back_to_the_shipped_targets() -> None:
    arguments = acceptance.build_parser().parse_args([])
    targets = acceptance.targets_from_args(arguments, None)

    assert targets.dependency_pull_requests == catalog.Targets.dependency_pull_requests
    assert targets.expected_suspensions == ()
    assert targets.repo == catalog.Targets.repo


@pytest.mark.parametrize("value", ["bad", "x=" + "1" * 40, "0=" + "1" * 40, "1=BAD"])
def test_dependency_heads_require_a_positive_pr_and_lowercase_sha(value: str) -> None:
    with pytest.raises(Exception):
        acceptance.dependency_head(value)


def test_a_default_run_is_read_only_and_never_pushes_or_posts(
    monkeypatch, tmp_path
) -> None:
    output = tmp_path / "report.json"
    status, cluster = run_cli(
        monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER
    )
    report = read_report(output)

    assert status == acceptance.EXIT_NO_GO
    assert report["mode"] == policy.READ_ONLY
    assert not any("push" in call for call in cluster.calls)
    assert not any("POST" in call for call in cluster.calls)
    assert not any("DELETE" in call for call in cluster.calls)
    writes = [
        call
        for call in cluster.calls
        if {"patch", "create", "delete", "apply", "replace"} & set(call)
        and "can-i" not in call
    ]
    assert writes == []


def test_mutating_checks_report_not_run_until_the_mode_is_armed(
    monkeypatch, tmp_path
) -> None:
    output = tmp_path / "report.json"
    run_cli(monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER)

    ephemeral = {
        check["id"]: check
        for check in read_report(output)["checks"]
        if check["group"] == "ephemeral"
    }

    assert len(ephemeral) == 4
    for check in ephemeral.values():
        assert check["status"] == model.NOT_RUN
        assert check["mandatory"] is False
        assert check["scope"] == model.EPHEMERAL


def test_a_zero_state_cluster_is_no_go_with_every_check_accounted_for(
    monkeypatch, tmp_path
) -> None:
    """Nothing observed must never read as nothing wrong."""
    output = tmp_path / "report.json"
    status, _ = run_cli(
        monkeypatch,
        ["--output", str(output), "--json-only"],
        [("auth whoami", whoami("kubernetes-admin"))],
    )
    report = read_report(output)

    assert status == acceptance.EXIT_NO_GO
    assert report["decision"] == model.NO_GO
    assert report["counts"][model.PASS] == 0
    assert sum(report["counts"].values()) == len(report["checks"])
    assert any(
        "in-pod Hermes vantage is unavailable" in error
        for error in report["harness_errors"]
    )


def test_partial_failure_is_reported_per_check_rather_than_aborting(
    monkeypatch, tmp_path
) -> None:
    output = tmp_path / "report.json"
    rules = [
        *HEALTHY_CLUSTER,
        ("auth can-i", completed(stdout="no\n", returncode=1)),
        ("get namespaces", completed(stdout="namespace/hermes\n")),
        ("merge-base", completed()),
    ]
    status, _ = run_cli(monkeypatch, ["--output", str(output), "--json-only"], rules)
    report = read_report(output)

    statuses = {check["status"] for check in report["checks"]}

    assert status == acceptance.EXIT_NO_GO
    assert model.PASS in statuses and model.NOT_RUN in statuses
    assert report["blocking"]


def test_a_vantage_collision_is_a_no_go_even_with_clean_checks(
    monkeypatch, tmp_path
) -> None:
    output = tmp_path / "report.json"
    rules = [
        (
            "exec --namespace hermes hermes-agent-1",
            whoami("system:serviceaccount:hermes:hermes-agent"),
        ),
        ("auth whoami", whoami("system:serviceaccount:hermes:hermes-agent")),
        (
            "get pods --selector app=hermes-agent",
            completed(stdout="pod/hermes-agent-1\n"),
        ),
        ("get pods --selector", completed(stdout="pod/other-1\n")),
        (
            "get pod/hermes-chat-tenant-0",
            completed(stdout="pod/hermes-chat-tenant-0\n"),
        ),
    ]
    status, _ = run_cli(monkeypatch, ["--output", str(output), "--json-only"], rules)
    report = read_report(output)

    assert status == acceptance.EXIT_NO_GO
    assert any("same principal" in error for error in report["harness_errors"])


def test_an_invalid_deadline_stops_before_any_subprocess(monkeypatch, tmp_path) -> None:
    output = tmp_path / "report.json"
    status, _ = run_cli(
        monkeypatch,
        ["--output", str(output), "--json-only", "--deadline", "0"],
        HEALTHY_CLUSTER,
    )
    report = read_report(output)

    assert status == acceptance.EXIT_USAGE
    assert report["counts"][model.NOT_RUN] > 0
    assert any("invalid execution bound" in error for error in report["harness_errors"])


def test_arming_with_a_wrong_confirmation_stops_before_anything_runs(
    monkeypatch, tmp_path
) -> None:
    output = tmp_path / "report.json"
    status, cluster = run_cli(
        monkeypatch,
        [
            "--output",
            str(output),
            "--json-only",
            "--arm-ephemeral-push",
            "--confirm",
            "please",
            "--ephemeral-token",
            "acceptance-20260817a",
        ],
        HEALTHY_CLUSTER,
    )
    report = read_report(output)

    assert status == acceptance.EXIT_NO_GO
    assert "arming refused" in report["harness_errors"][0]
    assert not any("push" in call for call in cluster.calls)


def test_arming_against_a_protected_push_target_is_refused(
    monkeypatch, tmp_path
) -> None:
    output = tmp_path / "report.json"
    status, cluster = run_cli(
        monkeypatch,
        [
            "--output",
            str(output),
            "--json-only",
            "--arm-ephemeral-push",
            "--confirm",
            acceptance.CONFIRMATION,
            "--ephemeral-token",
            "main",
        ],
        HEALTHY_CLUSTER,
    )

    assert status == acceptance.EXIT_NO_GO
    assert "arming refused" in read_report(output)["harness_errors"][0]
    assert not any("push" in call for call in cluster.calls)


def test_an_armed_run_does_not_mutate_until_read_only_acceptance_is_go(
    monkeypatch, tmp_path
) -> None:
    output = tmp_path / "report.json"
    monkeypatch.setattr(acceptance, "preflight", lambda *_args: None)
    rules = [
        ("git ls-remote origin refs/heads/ephemeral", completed(stdout="")),
        ("git push", completed()),
        (
            "POST /api/v1/repos/atlas/titan-iac/pulls",
            completed(stdout=json.dumps({"number": 9})),
        ),
        ("PATCH", completed()),
        ("DELETE", completed()),
        (
            "GET /api/v1/repos/atlas/titan-iac/pulls/9",
            completed(stdout='{"state": "closed"}'),
        ),
        *HEALTHY_CLUSTER,
    ]
    status, cluster = run_cli(
        monkeypatch,
        [
            "--output",
            str(output),
            "--json-only",
            "--arm-ephemeral-push",
            "--confirm",
            acceptance.CONFIRMATION,
            "--ephemeral-token",
            "acceptance-20260817a",
        ],
        rules,
    )
    report = read_report(output)
    assert report["mode"] == policy.ARMED
    assert status == acceptance.EXIT_NO_GO
    assert any(
        "read-only acceptance did not GO" in error for error in report["harness_errors"]
    )
    pushes = [call for call in cluster.calls if "push" in call]
    assert pushes == []
    assert not any("DELETE" in call for call in cluster.calls)


def test_the_report_is_written_to_stdout_when_no_output_path_is_given(
    monkeypatch, capsys
) -> None:
    status, _ = run_cli(monkeypatch, ["--json-only"], HEALTHY_CLUSTER)
    payload = json.loads(capsys.readouterr().out)

    assert status == acceptance.EXIT_NO_GO
    assert payload["harness"] == model.HARNESS


def test_the_human_summary_accompanies_the_json_unless_suppressed(
    monkeypatch, capsys, tmp_path
) -> None:
    output = tmp_path / "report.json"
    run_cli(monkeypatch, ["--output", str(output)], HEALTHY_CLUSTER)
    captured = capsys.readouterr()

    assert model.HARNESS in captured.err
    assert "vantage operator" in captured.err
    assert captured.out == ""


def test_a_report_that_fails_its_own_screening_is_replaced_not_published(
    tmp_path,
) -> None:
    """The last barrier: never write an artifact nobody re-read."""
    output = tmp_path / "report.json"
    arguments = acceptance.build_parser().parse_args(
        ["--output", str(output), "--json-only"]
    )
    report = model.Report(mode="read-only", started_at="2026-08-17T00:00:00Z")
    report.harness_errors = ["leaked ghp_ABCDEFGHIJKLMNOPQRSTUV1234"]

    monkey = report.as_dict
    report.as_dict = lambda *_args, **_kwargs: {  # type: ignore[method-assign]
        **monkey(),
        "leak": "token: ghp_ABCDEFGHIJKLMNOPQRSTUV1234",
    }
    offenders = acceptance.emit(report, arguments)
    payload = read_report(output)

    assert offenders == ["$.leak"]
    assert payload["decision"] == model.NO_GO
    assert "leak" not in payload
    assert "failed its own credential screening" in payload["harness_errors"][0]


def test_a_screening_failure_forces_no_go_from_the_entry_point(
    monkeypatch, tmp_path
) -> None:
    """The exit status must follow the refusal to publish, not the check tally."""
    output = tmp_path / "report.json"
    monkeypatch.setattr(
        acceptance, "unscreened_fields", lambda _payload: ["$.checks[0].stdout"]
    )
    monkeypatch.setattr(acceptance, "build_catalog", lambda _targets: [])
    monkeypatch.setattr(harness_run, "vantage_problems", lambda _records: [])

    status, _ = run_cli(
        monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER
    )

    assert status == acceptance.EXIT_NO_GO
    assert read_report(output)["decision"] == model.NO_GO


def test_no_recorded_command_output_carries_a_credential_shape(
    monkeypatch, tmp_path
) -> None:
    output = tmp_path / "report.json"
    leaky = completed(stdout="ANTHROPIC_API_KEY=sk-ant-abcdefghijklmnopqrstuvwxyz\n")
    run_cli(
        monkeypatch,
        ["--output", str(output), "--json-only"],
        [*HEALTHY_CLUSTER, ("exec", leaky)],
    )

    assert model.unscreened_fields(read_report(output)) == []
    assert "sk-ant-abcdefghijklmnopqrstuvwxyz" not in output.read_text(encoding="utf-8")


@pytest.mark.parametrize("flag", ["--help"])
def test_the_help_text_describes_the_read_only_default(flag: str, capsys) -> None:
    with pytest.raises(SystemExit):
        acceptance.main([flag])

    assert "Read-only by default" in capsys.readouterr().out