atlas-iac/testing/tests/test_hermes_handoff_run.py

"""Contracts for handoff acceptance orchestration: vantages, catalog, report."""

from __future__ import annotations

import datetime as dt
import subprocess
from dataclasses import replace

from testing.tests.test_hermes_handoff_support import (
    FakeClock,
    FakeSpawn,
    load_handoff_module,
    step,
)

harness_run = load_handoff_module("hermes_handoff_run")
catalog = load_handoff_module("hermes_handoff_catalog")
exec_module = load_handoff_module("hermes_handoff_exec")
model = load_handoff_module("hermes_handoff_model")

NOW = dt.datetime(2026, 8, 17, 12, 0, 0, tzinfo=dt.timezone.utc)
TARGETS = catalog.Targets(now=NOW)


def completed(stdout: str = "", stderr: str = "", returncode: int = 0):
    return subprocess.CompletedProcess(
        args=[], returncode=returncode, stdout=stdout, stderr=stderr
    )


def whoami(username: str) -> object:
    return completed(stdout=username)


def runner_for(results: list, **kwargs):
    return exec_module.Runner(
        clock=FakeClock(),
        spawn=FakeSpawn(results),
        environ={},
        deadline_seconds=3600,
        attestor=lambda command, _environment: exec_module.Attestation(
            command, "a" * 64
        ),
        **kwargs,
    )


def check(
    rule: str,
    expect: dict | None = None,
    steps=(),
    identifier="test.check",
    mandatory=True,
):
    return model.CheckSpec(
        id=identifier,
        title="t",
        group="g",
        rule=rule,
        steps=tuple(steps),
        expect=expect or {},
        mandatory=mandatory,
    )


def test_the_shipped_catalog_is_valid_and_covers_every_group() -> None:
    specs = harness_run.build_catalog(TARGETS)
    assert harness_run.validate_catalog(specs) == []
    groups = {spec.group for spec in specs}
    assert {
        "baseline",
        "identity",
        "routing",
        "scopes",
        "access-denied",
        "gitops",
    } <= groups
    assert {
        "forge",
        "nodes",
        "build",
        "reliability",
        "pool",
        "surfaces",
        "ephemeral",
    } <= groups


def test_catalog_validation_names_every_structural_problem() -> None:
    broken = [
        check("no-such-rule", identifier="a.dup"),
        check("allowed", identifier="a.dup"),
        check("allowed", steps=(step("s", vantage="nowhere"),), identifier="b.vantage"),
        check("allowed", steps=(step("s", kind="guess"),), identifier="c.kind"),
        check(
            "denied", steps=(step("s", kind=model.REVIEW),), identifier="d.review-only"
        ),
        model.CheckSpec(
            id="e.impersonation",
            title="t",
            group="g",
            rule="allowed",
            steps=(
                model.Step(
                    key="s",
                    vantage=catalog.OPERATOR,
                    argv=("kubectl", "get", "ns", "--as", "x"),
                ),
            ),
        ),
    ]

    problems = " | ".join(harness_run.validate_catalog(broken))

    assert "duplicate check id" in problems
    assert "unknown rule" in problems
    assert "unknown vantage" in problems
    assert "unknown kind" in problems
    assert "needs a real attempt" in problems
    assert "impersonates from the operator vantage" in problems


def test_target_validation_rejects_every_vacuous_or_unbound_release_input() -> None:
    broken = replace(
        TARGETS,
        repo="atlas/other",
        remote="upstream",
        reviewed_pr_number=20,
        reviewed_head_ref="other",
        baseline_commit="bad",
        remote_main_sha="bad",
        reviewed_head_sha="bad",
        build_sha="bad",
        agent_image="latest",
        deployment_revision="0",
        node_count=1,
        pool_replicas=1,
        chat_config_revision="",
        pool_worker_env=(),
        dependency_heads=((14, "bad"), (14, "bad")),
        max_evidence_age_seconds=0,
    )
    problems = harness_run.validate_targets(broken)
    assert len(problems) >= 12
    sha = "1" * 40
    valid = replace(
        TARGETS,
        remote_main_sha=sha,
        reviewed_head_sha=sha,
        build_sha=sha,
        agent_image=f"registry.example/hermes:git-{sha}-build-7@sha256:{'2' * 64}",
        deployment_revision="1",
        chat_config_revision="rev",
        dependency_heads=tuple(
            (number, sha) for number in TARGETS.dependency_pull_requests
        ),
    )
    assert harness_run.validate_targets(valid) == []
    wrong_image_source = replace(
        valid,
        agent_image=f"registry.example/hermes:git-{'2' * 40}-build-7@sha256:{'2' * 64}",
    )
    assert "image tag/digest" in " ".join(
        harness_run.validate_targets(wrong_image_source)
    )
    wrong_build = replace(valid, build_sha="2" * 40)
    assert "build SHA" in " ".join(harness_run.validate_targets(wrong_build))
    malformed_heads = replace(
        valid,
        dependency_heads=tuple(
            (number, "bad") for number in valid.dependency_pull_requests
        ),
    )
    assert "dependency heads" in " ".join(harness_run.validate_targets(malformed_heads))


def test_identity_resolution_extracts_only_the_username() -> None:
    runner = runner_for([whoami("system:serviceaccount:hermes:hermes-agent")])
    username, detail = harness_run.resolve_identity(
        runner, exec_module.operator_vantage()
    )
    assert (username, detail) == ("system:serviceaccount:hermes:hermes-agent", "")

    failed = harness_run.resolve_identity(
        runner_for([completed(stderr="no access", returncode=1)]),
        exec_module.operator_vantage(),
    )
    assert failed[0] == "" and "no access" in failed[1]

    unparsable = harness_run.resolve_identity(
        runner_for([completed(stdout="two names")]), exec_module.operator_vantage()
    )
    assert unparsable[0] == "" and "no username" in unparsable[1]

    nameless = harness_run.resolve_identity(
        runner_for([completed(stdout="")]), exec_module.operator_vantage()
    )
    assert nameless[0] == "" and "no username" in nameless[1]


def test_vantages_resolve_to_running_pods_and_record_what_is_missing() -> None:
    runner = runner_for(
        [
            whoami("kubernetes-admin"),
            completed(stdout="pod/hermes-agent-2\npod/hermes-agent-1\n"),
            whoami("system:serviceaccount:hermes:hermes-agent"),
            completed(stdout="pod/hermes-switchyard-1\n"),
            completed(stderr="no pods", returncode=1),
            completed(stdout="pod/hermes-chat-tenant-0\n"),
        ]
    )

    vantages, records = harness_run.resolve_vantages(runner, TARGETS)
    by_name = {record.name: record for record in records}

    assert set(vantages) == {
        catalog.OPERATOR,
        catalog.SELF,
        catalog.SWITCHYARD,
        catalog.CHAT,
    }
    assert by_name[catalog.SELF].identity == "system:serviceaccount:hermes:hermes-agent"
    assert "hermes-agent-1" in by_name[catalog.SELF].description  # deterministic pick
    assert by_name[catalog.NODE].available is False
    assert by_name[catalog.CHAT].available is True


def test_an_unreachable_operator_stops_vantage_resolution_immediately() -> None:
    runner = runner_for([completed(stderr="connection refused", returncode=1)])

    vantages, records = harness_run.resolve_vantages(runner, TARGETS)

    assert set(vantages) == {catalog.OPERATOR}
    assert records[0].available is False
    assert harness_run.vantage_problems(records)[0].startswith(
        "the operator vantage is unavailable"
    )


def test_a_missing_chat_pod_is_recorded_rather_than_assumed_present() -> None:
    runner = runner_for(
        [
            whoami("kubernetes-admin"),
            completed(stdout="pod/hermes-agent-1\n"),
            whoami("system:serviceaccount:hermes:hermes-agent"),
            completed(stdout="pod/hermes-switchyard-1\n"),
            completed(stdout="pod/hermes-node-1\n"),
            completed(stderr="pods 'hermes-chat-tenant-0' not found", returncode=1),
        ]
    )

    vantages, records = harness_run.resolve_vantages(runner, TARGETS)
    chat = next(record for record in records if record.name == catalog.CHAT)

    assert catalog.CHAT not in vantages
    assert chat.available is False and "not found" in chat.detail


def test_a_selector_that_matches_no_running_pod_leaves_the_vantage_absent() -> None:
    runner = runner_for(
        [
            whoami("kubernetes-admin"),
            completed(stdout="\n"),
            completed(stdout="\n"),
            completed(stdout="\n"),
            completed(stdout="\n"),
        ]
    )

    vantages, records = harness_run.resolve_vantages(runner, TARGETS)
    by_name = {record.name: record for record in records}

    assert set(vantages) == {catalog.OPERATOR}
    assert "no running pod matches" in by_name[catalog.SELF].detail


def test_two_vantages_that_are_the_same_principal_invalidate_the_evidence() -> None:
    """Corroboration needs two identities; one identity twice is a restatement."""
    records = [
        model.VantageRecord(
            name=catalog.OPERATOR, description="d", identity="same", available=True
        ),
        model.VantageRecord(
            name=catalog.SELF, description="d", identity="same", available=True
        ),
        model.VantageRecord(name=catalog.SWITCHYARD, description="d", available=True),
        model.VantageRecord(name=catalog.CHAT, description="d", available=True),
    ]

    problems = harness_run.vantage_problems(records)

    assert len(problems) == 1
    assert "same principal" in problems[0]


def test_vantage_problems_flag_a_missing_or_anonymous_self_probe() -> None:
    operator = model.VantageRecord(
        name=catalog.OPERATOR, description="d", identity="admin", available=True
    )

    absent = harness_run.vantage_problems([operator])
    assert "in-pod Hermes vantage is unavailable" in absent[0]

    anonymous = harness_run.vantage_problems(
        [
            operator,
            model.VantageRecord(name=catalog.SELF, description="d", available=True),
        ]
    )
    assert "did not report an identity" in anonymous[0]

    clean = harness_run.vantage_problems(
        [
            operator,
            model.VantageRecord(
                name=catalog.SELF, description="d", identity="agent", available=True
            ),
            model.VantageRecord(
                name=catalog.SWITCHYARD, description="d", available=True
            ),
            model.VantageRecord(name=catalog.CHAT, description="d", available=True),
        ]
    )
    assert clean == []


def test_a_check_whose_vantage_is_absent_is_not_run() -> None:
    spec = check("allowed", steps=(step("s", vantage=catalog.NODE),))

    result = harness_run.run_check(runner_for([]), spec, {})

    assert result.status == model.NOT_RUN
    assert "vantage node is unavailable" in result.reason
    assert result.outcomes[0].error == "vantage node is unavailable"


def test_validate_catalog_rejects_impersonation_from_every_vantage() -> None:
    """The operator vantage was the only one guarded; now none is exempt."""
    for vantage in (catalog.OPERATOR, catalog.SELF, catalog.CHAT):
        spec = check(
            "stdout_matches",
            {"step": "s", "equals": "x"},
            steps=(
                model.Step(
                    key="s",
                    vantage=vantage,
                    argv=("kubectl", "get", "ns", "-o", "name", "--as", "system:admin"),
                ),
            ),
        )
        problems = harness_run.validate_catalog([spec])
        assert any("impersonates" in problem for problem in problems)
        assert any(vantage in problem for problem in problems)


def test_bulk_evidence_is_evaluated_but_replaced_before_it_is_recorded() -> None:
    spec = check(
        "stdout_matches",
        {"step": "s", "equals": "locked"},
        steps=(
            model.Step(
                key="s",
                vantage=catalog.OPERATOR,
                argv=("kubectl", "version"),
                record=False,
            ),
        ),
    )
    runner = runner_for([completed(stdout="locked", stderr="noise")])

    result = harness_run.run_check(
        runner, spec, {catalog.OPERATOR: exec_module.operator_vantage()}
    )

    assert result.status == model.PASS
    assert result.outcomes[0].stdout == harness_run.UNRECORDED
    assert result.outcomes[0].stderr == harness_run.UNRECORDED
    # Withholding bulk bytes must not withhold the binary that produced them.
    recorded = result.outcomes[0]
    assert recorded.executable_path
    assert recorded.executable_sha256
    assert recorded.executable_path == "kubectl"
    assert recorded.executable_sha256 == "a" * 64
    assert recorded.as_dict()["executable_sha256"] == recorded.executable_sha256


def test_an_expired_deadline_leaves_the_report_saying_so() -> None:
    clock = FakeClock()
    runner = exec_module.Runner(
        clock=clock,
        spawn=FakeSpawn([]),
        environ={},
        deadline_seconds=1.0,
        attestor=lambda command, _environment: exec_module.Attestation(
            command, "a" * 64
        ),
    )
    clock.advance(5.0)
    spec = check("allowed", steps=(step("s", vantage=catalog.OPERATOR),))

    report = harness_run.build_report(
        runner,
        TARGETS,
        [spec],
        {catalog.OPERATOR: exec_module.operator_vantage()},
        [],
        "read-only",
        "2026-08-17T00:00:00Z",
    )

    assert report.results[0].status == model.NOT_RUN
    assert any("deadline expired" in error for error in report.harness_errors)
    assert report.decision == model.NO_GO


def test_the_report_carries_the_configured_baseline_and_sorts_its_checks() -> None:
    runner = runner_for([completed(stdout="ok"), completed(stdout="ok")])
    specs = [
        check(
            "allowed", steps=(step("s", vantage=catalog.OPERATOR),), identifier="z.last"
        ),
        check(
            "allowed",
            steps=(step("s", vantage=catalog.OPERATOR),),
            identifier="a.first",
        ),
    ]
    records = [
        model.VantageRecord(
            name=catalog.OPERATOR, description="d", identity="admin", available=True
        ),
        model.VantageRecord(
            name=catalog.SELF, description="d", identity="agent", available=True
        ),
        model.VantageRecord(name=catalog.SWITCHYARD, description="d", available=True),
        model.VantageRecord(name=catalog.CHAT, description="d", available=True),
    ]

    report = harness_run.build_report(
        runner,
        TARGETS,
        specs,
        {catalog.OPERATOR: exec_module.operator_vantage()},
        records,
        "read-only",
        "2026-08-17T00:00:00Z",
    )

    assert [result.spec.id for result in report.results] == ["a.first", "z.last"]
    assert report.baseline["baseline_commit"] == TARGETS.baseline_commit
    assert report.baseline["dependency_pull_requests"] == list(
        TARGETS.dependency_pull_requests
    )
    assert report.decision == model.GO
    assert report.finished_at


def test_armed_results_replace_the_matching_catalog_placeholders() -> None:
    placeholder = check(
        "not_armed", identifier="ephemeral.feature-branch-push", mandatory=False
    )
    armed = model.CheckResult(
        spec=model.CheckSpec(
            id="ephemeral.feature-branch-push",
            title="t",
            group="ephemeral",
            rule="armed",
        ),
        status=model.PASS,
        reason="pushed",
    )

    report = harness_run.build_report(
        runner_for([]),
        TARGETS,
        [placeholder],
        {},
        [],
        "ephemeral-armed",
        "2026-08-17T00:00:00Z",
        [armed],
    )

    assert len(report.results) == 1
    assert report.results[0].status == model.PASS


def test_an_unstartable_run_reports_no_go_with_a_named_reason() -> None:
    report = harness_run.unavailable_report(
        "read-only", "2026-08-17T00:00:00Z", "arming refused: bad phrase", TARGETS
    )

    assert report.decision == model.NO_GO
    assert report.results[0].spec.id == "harness.startup"
    assert report.results[0].status == model.NOT_RUN
    assert report.harness_errors == ["arming refused: bad phrase"]