atlas-iac/testing/tests/test_hermes_handoff_acceptance.py

"""End-to-end contracts for the Hermes full-handoff acceptance harness CLI."""

from __future__ import annotations

import json
import subprocess

import pytest

from testing.tests.test_hermes_handoff_support import FakeClock, load_handoff_module

acceptance = load_handoff_module("hermes_handoff_acceptance")
catalog = load_handoff_module("hermes_handoff_catalog")
exec_module = load_handoff_module("hermes_handoff_exec")
model = load_handoff_module("hermes_handoff_model")
policy = load_handoff_module("hermes_handoff_policy")
harness_run = load_handoff_module("hermes_handoff_run")


def completed(stdout: str = "", stderr: str = "", returncode: int = 0):
    return subprocess.CompletedProcess(args=[], returncode=returncode, stdout=stdout, stderr=stderr)


class ScriptedCluster:
    """A ``subprocess.run`` stand-in that answers by command shape.

    Real runs issue a couple of hundred commands; scripting them positionally
    would make every test a brittle ordering puzzle. Matching on the command
    line keeps each test about the behaviour it is checking.
    """

    def __init__(self, rules: list[tuple[str, object]], default=None) -> None:
        self.rules = rules
        self.default = default or completed(stderr="not found", returncode=1)
        self.calls: list[list[str]] = []

    def __call__(self, argv, **_kwargs):
        self.calls.append(list(argv))
        line = " ".join(argv)
        for fragment, response in self.rules:
            if fragment in line:
                return response
        return self.default


def whoami(username: str):
    return completed(stdout=json.dumps({"status": {"userInfo": {"username": username}}}))


HEALTHY_CLUSTER = [
    ("auth whoami", whoami("kubernetes-admin")),
    ("get pods --selector app=hermes-agent", completed(stdout="pod/hermes-agent-1\n")),
    ("get pods --selector", completed(stdout="pod/other-1\n")),
    ("get pod/hermes-chat-tenant-0", completed(stdout="pod/hermes-chat-tenant-0\n")),
]


def run_cli(monkeypatch, argv, rules):
    cluster = ScriptedCluster(rules)
    clock = FakeClock()

    def build_runner(**kwargs):
        return exec_module.Runner(clock=clock, spawn=cluster, environ={}, **kwargs)

    monkeypatch.setattr(acceptance, "Runner", build_runner)
    return (acceptance.main(argv), cluster)


def read_report(path):
    return json.loads(path.read_text(encoding="utf-8"))


def test_the_parser_exposes_the_documented_controls() -> None:
    arguments = acceptance.build_parser().parse_args(
        [
            "--namespace",
            "hermes",
            "--context",
            "atlas-operator",
            "--dependency-pr",
            "14",
            "--dependency-pr",
            "15",
            "--expected-suspension",
            "flux-system/parked",
            "--node-count",
            "21",
            "--pool-worker-env",
            "HERMES_POOL_LEASE_SECONDS",
            "--expect-telegram-sessions",
        ]
    )

    targets = acceptance.targets_from_args(arguments, None)

    assert targets.dependency_pull_requests == (14, 15)
    assert targets.expected_suspensions == ("flux-system/parked",)
    assert targets.node_count == 21
    assert targets.pool_worker_env == ("HERMES_POOL_LEASE_SECONDS",)
    assert targets.expect_telegram_sessions is True


def test_defaults_fall_back_to_the_shipped_targets() -> None:
    arguments = acceptance.build_parser().parse_args([])
    targets = acceptance.targets_from_args(arguments, None)

    assert targets.dependency_pull_requests == catalog.Targets.dependency_pull_requests
    assert targets.expected_suspensions == ()
    assert targets.repo == catalog.Targets.repo


def test_a_default_run_is_read_only_and_never_pushes_or_posts(monkeypatch, tmp_path) -> None:
    output = tmp_path / "report.json"
    status, cluster = run_cli(
        monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER
    )
    report = read_report(output)

    assert status == acceptance.EXIT_NO_GO
    assert report["mode"] == policy.READ_ONLY
    assert not any("push" in call for call in cluster.calls)
    assert not any("POST" in call for call in cluster.calls)
    assert not any("DELETE" in call for call in cluster.calls)
    writes = [
        call
        for call in cluster.calls
        if {"patch", "create", "delete", "apply", "replace"} & set(call) and "can-i" not in call
    ]
    assert writes
    assert all("--dry-run=server" in call for call in writes)


def test_mutating_checks_report_not_run_until_the_mode_is_armed(monkeypatch, tmp_path) -> None:
    output = tmp_path / "report.json"
    run_cli(monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER)

    ephemeral = {
        check["id"]: check
        for check in read_report(output)["checks"]
        if check["group"] == "ephemeral"
    }

    assert len(ephemeral) == 4
    for check in ephemeral.values():
        assert check["status"] == model.NOT_RUN
        assert check["mandatory"] is False
        assert check["scope"] == model.EPHEMERAL


def test_a_zero_state_cluster_is_no_go_with_every_check_accounted_for(
    monkeypatch, tmp_path
) -> None:
    """Nothing observed must never read as nothing wrong."""
    output = tmp_path / "report.json"
    status, _ = run_cli(
        monkeypatch,
        ["--output", str(output), "--json-only"],
        [("auth whoami", whoami("kubernetes-admin"))],
    )
    report = read_report(output)

    assert status == acceptance.EXIT_NO_GO
    assert report["decision"] == model.NO_GO
    assert report["counts"][model.PASS] == 0
    assert sum(report["counts"].values()) == len(report["checks"])
    assert any("in-pod Hermes vantage is unavailable" in error for error in report["harness_errors"])


def test_partial_failure_is_reported_per_check_rather_than_aborting(
    monkeypatch, tmp_path
) -> None:
    output = tmp_path / "report.json"
    rules = [
        *HEALTHY_CLUSTER,
        ("auth can-i", completed(stdout="no\n", returncode=1)),
        ("get namespaces", completed(stdout="namespace/hermes\n")),
        ("merge-base", completed()),
    ]
    status, _ = run_cli(monkeypatch, ["--output", str(output), "--json-only"], rules)
    report = read_report(output)

    statuses = {check["status"] for check in report["checks"]}

    assert status == acceptance.EXIT_NO_GO
    assert model.PASS in statuses and model.NOT_RUN in statuses
    assert report["blocking"]


def test_a_vantage_collision_is_a_no_go_even_with_clean_checks(monkeypatch, tmp_path) -> None:
    output = tmp_path / "report.json"
    rules = [
        ("exec --namespace hermes hermes-agent-1", whoami("system:serviceaccount:hermes:hermes-agent")),
        ("auth whoami", whoami("system:serviceaccount:hermes:hermes-agent")),
        ("get pods --selector app=hermes-agent", completed(stdout="pod/hermes-agent-1\n")),
        ("get pods --selector", completed(stdout="pod/other-1\n")),
        ("get pod/hermes-chat-tenant-0", completed(stdout="pod/hermes-chat-tenant-0\n")),
    ]
    status, _ = run_cli(monkeypatch, ["--output", str(output), "--json-only"], rules)
    report = read_report(output)

    assert status == acceptance.EXIT_NO_GO
    assert any("same principal" in error for error in report["harness_errors"])


def test_an_exhausted_deadline_leaves_the_remaining_checks_not_run(monkeypatch, tmp_path) -> None:
    output = tmp_path / "report.json"
    status, _ = run_cli(
        monkeypatch, ["--output", str(output), "--json-only", "--deadline", "0"], HEALTHY_CLUSTER
    )
    report = read_report(output)

    assert status == acceptance.EXIT_NO_GO
    assert report["counts"][model.NOT_RUN] > 0
    assert any("deadline expired" in error for error in report["harness_errors"])


def test_arming_with_a_wrong_confirmation_stops_before_anything_runs(
    monkeypatch, tmp_path
) -> None:
    output = tmp_path / "report.json"
    status, cluster = run_cli(
        monkeypatch,
        [
            "--output",
            str(output),
            "--json-only",
            "--arm-ephemeral-push",
            "--confirm",
            "please",
            "--ephemeral-token",
            "acceptance-20260817a",
        ],
        HEALTHY_CLUSTER,
    )
    report = read_report(output)

    assert status == acceptance.EXIT_NO_GO
    assert "arming refused" in report["harness_errors"][0]
    assert not any("push" in call for call in cluster.calls)


def test_arming_against_a_protected_push_target_is_refused(monkeypatch, tmp_path) -> None:
    output = tmp_path / "report.json"
    status, cluster = run_cli(
        monkeypatch,
        [
            "--output",
            str(output),
            "--json-only",
            "--arm-ephemeral-push",
            "--confirm",
            acceptance.CONFIRMATION,
            "--ephemeral-token",
            "main",
        ],
        HEALTHY_CLUSTER,
    )

    assert status == acceptance.EXIT_NO_GO
    assert "arming refused" in read_report(output)["harness_errors"][0]
    assert not any("push" in call for call in cluster.calls)


def test_an_armed_run_pushes_one_ephemeral_ref_and_verifies_its_removal(
    monkeypatch, tmp_path
) -> None:
    output = tmp_path / "report.json"
    rules = [
        ("git ls-remote origin refs/heads/ephemeral", completed(stdout="")),
        ("git push", completed()),
        ("POST /api/v1/repos/atlas/titan-iac/pulls", completed(stdout=json.dumps({"number": 9}))),
        ("PATCH", completed()),
        ("DELETE", completed()),
        ("GET /api/v1/repos/atlas/titan-iac/pulls/9", completed(stdout='{"state": "closed"}')),
        *HEALTHY_CLUSTER,
    ]
    status, cluster = run_cli(
        monkeypatch,
        [
            "--output",
            str(output),
            "--json-only",
            "--arm-ephemeral-push",
            "--confirm",
            acceptance.CONFIRMATION,
            "--ephemeral-token",
            "acceptance-20260817a",
        ],
        rules,
    )
    report = read_report(output)
    ephemeral = {check["id"]: check for check in report["checks"] if check["group"] == "ephemeral"}

    assert report["mode"] == policy.ARMED
    assert status == acceptance.EXIT_NO_GO  # the rest of the cluster is still unproven
    assert ephemeral["ephemeral.feature-branch-push"]["status"] == model.PASS
    assert ephemeral["ephemeral.draft-pull-request"]["status"] == model.PASS
    assert ephemeral["ephemeral.cleanup-verified"]["status"] == model.PASS
    assert ephemeral["ephemeral.protected-branch-refusal"]["status"] == model.PASS

    pushes = [call for call in cluster.calls if "push" in call]
    assert len(pushes) == 1
    assert pushes[0][-1].startswith("HEAD:refs/heads/ephemeral/hermes-handoff-acceptance/")
    assert pushes[0][0] == "kubectl" and "exec" in pushes[0]  # pushed as the worker, not the operator
    assert any("DELETE" in call for call in cluster.calls)


def test_the_report_is_written_to_stdout_when_no_output_path_is_given(
    monkeypatch, capsys
) -> None:
    status, _ = run_cli(monkeypatch, ["--json-only"], HEALTHY_CLUSTER)
    payload = json.loads(capsys.readouterr().out)

    assert status == acceptance.EXIT_NO_GO
    assert payload["harness"] == model.HARNESS


def test_the_human_summary_accompanies_the_json_unless_suppressed(
    monkeypatch, capsys, tmp_path
) -> None:
    output = tmp_path / "report.json"
    run_cli(monkeypatch, ["--output", str(output)], HEALTHY_CLUSTER)
    captured = capsys.readouterr()

    assert model.HARNESS in captured.err
    assert "vantage operator" in captured.err
    assert captured.out == ""


def test_a_report_that_fails_its_own_screening_is_replaced_not_published(tmp_path) -> None:
    """The last barrier: never write an artifact nobody re-read."""
    output = tmp_path / "report.json"
    arguments = acceptance.build_parser().parse_args(["--output", str(output), "--json-only"])
    report = model.Report(mode="read-only", started_at="2026-08-17T00:00:00Z")
    report.harness_errors = ["leaked ghp_ABCDEFGHIJKLMNOPQRSTUV1234"]

    monkey = report.as_dict
    report.as_dict = lambda *_args, **_kwargs: {  # type: ignore[method-assign]
        **monkey(),
        "leak": "token: ghp_ABCDEFGHIJKLMNOPQRSTUV1234",
    }
    offenders = acceptance.emit(report, arguments)
    payload = read_report(output)

    assert offenders == ["$.leak"]
    assert payload["decision"] == model.NO_GO
    assert "leak" not in payload
    assert "failed its own credential screening" in payload["harness_errors"][0]


def test_a_screening_failure_forces_no_go_from_the_entry_point(monkeypatch, tmp_path) -> None:
    """The exit status must follow the refusal to publish, not the check tally."""
    output = tmp_path / "report.json"
    monkeypatch.setattr(acceptance, "unscreened_fields", lambda _payload: ["$.checks[0].stdout"])
    monkeypatch.setattr(acceptance, "build_catalog", lambda _targets: [])
    monkeypatch.setattr(harness_run, "vantage_problems", lambda _records: [])

    status, _ = run_cli(monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER)

    assert status == acceptance.EXIT_NO_GO
    assert read_report(output)["decision"] == model.NO_GO


def test_no_recorded_command_output_carries_a_credential_shape(monkeypatch, tmp_path) -> None:
    output = tmp_path / "report.json"
    leaky = completed(stdout="ANTHROPIC_API_KEY=sk-ant-abcdefghijklmnopqrstuvwxyz\n")
    run_cli(
        monkeypatch,
        ["--output", str(output), "--json-only"],
        [*HEALTHY_CLUSTER, ("exec", leaky)],
    )

    assert model.unscreened_fields(read_report(output)) == []
    assert "sk-ant-abcdefghijklmnopqrstuvwxyz" not in output.read_text(encoding="utf-8")


@pytest.mark.parametrize("flag", ["--help"])
def test_the_help_text_describes_the_read_only_default(flag: str, capsys) -> None:
    with pytest.raises(SystemExit):
        acceptance.main([flag])

    assert "Read-only by default" in capsys.readouterr().out


def test_go_is_reachable_when_nothing_mandatory_is_outstanding(monkeypatch, tmp_path) -> None:
    """Exercise the success exit path without pretending the cluster is clean."""
    output = tmp_path / "report.json"
    monkeypatch.setattr(harness_run, "build_catalog", lambda _targets: [])
    monkeypatch.setattr(acceptance, "build_catalog", lambda _targets: [])
    monkeypatch.setattr(harness_run, "vantage_problems", lambda _records: [])

    status, _ = run_cli(monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER)

    assert status == acceptance.EXIT_GO
    assert read_report(output)["decision"] == model.GO