atlas-iac/testing/tests/test_hermes_handoff_run.py

"""Contracts for handoff acceptance orchestration: vantages, catalog, report."""

from __future__ import annotations

import datetime as dt
import subprocess

from testing.tests.test_hermes_handoff_support import (
    FakeClock,
    FakeSpawn,
    load_handoff_module,
    step,
)

harness_run = load_handoff_module("hermes_handoff_run")
catalog = load_handoff_module("hermes_handoff_catalog")
exec_module = load_handoff_module("hermes_handoff_exec")
model = load_handoff_module("hermes_handoff_model")

NOW = dt.datetime(2026, 8, 17, 12, 0, 0, tzinfo=dt.timezone.utc)
TARGETS = catalog.Targets(now=NOW)


def completed(stdout: str = "", stderr: str = "", returncode: int = 0):
    return subprocess.CompletedProcess(args=[], returncode=returncode, stdout=stdout, stderr=stderr)


def whoami(username: str) -> object:
    return completed(stdout=f'{{"status": {{"userInfo": {{"username": "{username}"}}}}}}')


def runner_for(results: list, **kwargs):
    return exec_module.Runner(
        clock=FakeClock(), spawn=FakeSpawn(results), environ={}, deadline_seconds=3600, **kwargs
    )


def check(rule: str, expect: dict | None = None, steps=(), identifier="test.check", mandatory=True):
    return model.CheckSpec(
        id=identifier,
        title="t",
        group="g",
        rule=rule,
        steps=tuple(steps),
        expect=expect or {},
        mandatory=mandatory,
    )


def test_the_shipped_catalog_is_valid_and_covers_every_group() -> None:
    specs = harness_run.build_catalog(TARGETS)
    assert harness_run.validate_catalog(specs) == []
    groups = {spec.group for spec in specs}
    assert {"baseline", "identity", "routing", "scopes", "access-denied", "gitops"} <= groups
    assert {"forge", "nodes", "build", "reliability", "pool", "surfaces", "ephemeral"} <= groups


def test_catalog_validation_names_every_structural_problem() -> None:
    broken = [
        check("no-such-rule", identifier="a.dup"),
        check("allowed", identifier="a.dup"),
        check("allowed", steps=(step("s", vantage="nowhere"),), identifier="b.vantage"),
        check("allowed", steps=(step("s", kind="guess"),), identifier="c.kind"),
        check("denied", steps=(step("s", kind=model.REVIEW),), identifier="d.review-only"),
        model.CheckSpec(
            id="e.impersonation",
            title="t",
            group="g",
            rule="allowed",
            steps=(
                model.Step(key="s", vantage=catalog.OPERATOR, argv=("kubectl", "get", "ns", "--as", "x")),
            ),
        ),
    ]

    problems = " | ".join(harness_run.validate_catalog(broken))

    assert "duplicate check id" in problems
    assert "unknown rule" in problems
    assert "unknown vantage" in problems
    assert "unknown kind" in problems
    assert "needs a real attempt" in problems
    assert "impersonates from the operator vantage" in problems


def test_identity_resolution_extracts_only_the_username() -> None:
    runner = runner_for([whoami("system:serviceaccount:hermes:hermes-agent")])
    username, detail = harness_run.resolve_identity(runner, exec_module.operator_vantage())
    assert (username, detail) == ("system:serviceaccount:hermes:hermes-agent", "")

    failed = harness_run.resolve_identity(
        runner_for([completed(stderr="no access", returncode=1)]), exec_module.operator_vantage()
    )
    assert failed[0] == "" and "no access" in failed[1]

    unparsable = harness_run.resolve_identity(
        runner_for([completed(stdout="nope")]), exec_module.operator_vantage()
    )
    assert unparsable[0] == "" and "unparsable identity" in unparsable[1]

    nameless = harness_run.resolve_identity(
        runner_for([completed(stdout='{"status": {}}')]), exec_module.operator_vantage()
    )
    assert nameless[0] == "" and "no username" in nameless[1]


def test_vantages_resolve_to_running_pods_and_record_what_is_missing() -> None:
    runner = runner_for(
        [
            whoami("kubernetes-admin"),
            completed(stdout="pod/hermes-agent-2\npod/hermes-agent-1\n"),
            whoami("system:serviceaccount:hermes:hermes-agent"),
            completed(stdout="pod/hermes-switchyard-1\n"),
            completed(stderr="no pods", returncode=1),
            completed(stdout="pod/hermes-chat-tenant-0\n"),
        ]
    )

    vantages, records = harness_run.resolve_vantages(runner, TARGETS)
    by_name = {record.name: record for record in records}

    assert set(vantages) == {catalog.OPERATOR, catalog.SELF, catalog.SWITCHYARD, catalog.CHAT}
    assert by_name[catalog.SELF].identity == "system:serviceaccount:hermes:hermes-agent"
    assert "hermes-agent-1" in by_name[catalog.SELF].description  # deterministic pick
    assert by_name[catalog.NODE].available is False
    assert by_name[catalog.CHAT].available is True


def test_an_unreachable_operator_stops_vantage_resolution_immediately() -> None:
    runner = runner_for([completed(stderr="connection refused", returncode=1)])

    vantages, records = harness_run.resolve_vantages(runner, TARGETS)

    assert set(vantages) == {catalog.OPERATOR}
    assert records[0].available is False
    assert harness_run.vantage_problems(records)[0].startswith("the operator vantage is unavailable")


def test_a_missing_chat_pod_is_recorded_rather_than_assumed_present() -> None:
    runner = runner_for(
        [
            whoami("kubernetes-admin"),
            completed(stdout="pod/hermes-agent-1\n"),
            whoami("system:serviceaccount:hermes:hermes-agent"),
            completed(stdout="pod/hermes-switchyard-1\n"),
            completed(stdout="pod/hermes-node-1\n"),
            completed(stderr="pods 'hermes-chat-tenant-0' not found", returncode=1),
        ]
    )

    vantages, records = harness_run.resolve_vantages(runner, TARGETS)
    chat = next(record for record in records if record.name == catalog.CHAT)

    assert catalog.CHAT not in vantages
    assert chat.available is False and "not found" in chat.detail


def test_a_selector_that_matches_no_running_pod_leaves_the_vantage_absent() -> None:
    runner = runner_for(
        [
            whoami("kubernetes-admin"),
            completed(stdout="\n"),
            completed(stdout="\n"),
            completed(stdout="\n"),
            completed(stdout="\n"),
        ]
    )

    vantages, records = harness_run.resolve_vantages(runner, TARGETS)
    by_name = {record.name: record for record in records}

    assert set(vantages) == {catalog.OPERATOR}
    assert "no running pod matches" in by_name[catalog.SELF].detail


def test_two_vantages_that_are_the_same_principal_invalidate_the_evidence() -> None:
    """Corroboration needs two identities; one identity twice is a restatement."""
    records = [
        model.VantageRecord(name=catalog.OPERATOR, description="d", identity="same", available=True),
        model.VantageRecord(name=catalog.SELF, description="d", identity="same", available=True),
    ]

    problems = harness_run.vantage_problems(records)

    assert len(problems) == 1
    assert "same principal" in problems[0]


def test_vantage_problems_flag_a_missing_or_anonymous_self_probe() -> None:
    operator = model.VantageRecord(
        name=catalog.OPERATOR, description="d", identity="admin", available=True
    )

    absent = harness_run.vantage_problems([operator])
    assert "in-pod Hermes vantage is unavailable" in absent[0]

    anonymous = harness_run.vantage_problems(
        [operator, model.VantageRecord(name=catalog.SELF, description="d", available=True)]
    )
    assert "did not report an identity" in anonymous[0]

    clean = harness_run.vantage_problems(
        [
            operator,
            model.VantageRecord(
                name=catalog.SELF, description="d", identity="agent", available=True
            ),
        ]
    )
    assert clean == []


def test_a_check_whose_vantage_is_absent_is_not_run() -> None:
    spec = check("allowed", steps=(step("s", vantage=catalog.NODE),))

    result = harness_run.run_check(runner_for([]), spec, {})

    assert result.status == model.NOT_RUN
    assert "vantage node is unavailable" in result.reason
    assert result.outcomes[0].error == "vantage node is unavailable"


def test_bulk_evidence_is_evaluated_but_replaced_before_it_is_recorded() -> None:
    spec = check(
        "stdout_matches",
        {"step": "s", "equals": "locked"},
        steps=(model.Step(key="s", vantage=catalog.OPERATOR, argv=("kubectl", "version"), record=False),),
    )
    runner = runner_for([completed(stdout="locked", stderr="noise")])

    result = harness_run.run_check(runner, spec, {catalog.OPERATOR: exec_module.operator_vantage()})

    assert result.status == model.PASS
    assert result.outcomes[0].stdout == harness_run.UNRECORDED
    assert result.outcomes[0].stderr == harness_run.UNRECORDED


def test_an_expired_deadline_leaves_the_report_saying_so() -> None:
    clock = FakeClock()
    runner = exec_module.Runner(clock=clock, spawn=FakeSpawn([]), environ={}, deadline_seconds=1.0)
    clock.advance(5.0)
    spec = check("allowed", steps=(step("s", vantage=catalog.OPERATOR),))

    report = harness_run.build_report(
        runner,
        TARGETS,
        [spec],
        {catalog.OPERATOR: exec_module.operator_vantage()},
        [],
        "read-only",
        "2026-08-17T00:00:00Z",
    )

    assert report.results[0].status == model.NOT_RUN
    assert any("deadline expired" in error for error in report.harness_errors)
    assert report.decision == model.NO_GO


def test_the_report_carries_the_configured_baseline_and_sorts_its_checks() -> None:
    runner = runner_for([completed(stdout="ok"), completed(stdout="ok")])
    specs = [
        check("allowed", steps=(step("s", vantage=catalog.OPERATOR),), identifier="z.last"),
        check("allowed", steps=(step("s", vantage=catalog.OPERATOR),), identifier="a.first"),
    ]
    records = [
        model.VantageRecord(name=catalog.OPERATOR, description="d", identity="admin", available=True),
        model.VantageRecord(name=catalog.SELF, description="d", identity="agent", available=True),
    ]

    report = harness_run.build_report(
        runner,
        TARGETS,
        specs,
        {catalog.OPERATOR: exec_module.operator_vantage()},
        records,
        "read-only",
        "2026-08-17T00:00:00Z",
    )

    assert [result.spec.id for result in report.results] == ["a.first", "z.last"]
    assert report.baseline["baseline_commit"] == TARGETS.baseline_commit
    assert report.baseline["dependency_pull_requests"] == list(TARGETS.dependency_pull_requests)
    assert report.decision == model.GO
    assert report.finished_at


def test_armed_results_replace_the_matching_catalog_placeholders() -> None:
    placeholder = check("not_armed", identifier="ephemeral.feature-branch-push", mandatory=False)
    armed = model.CheckResult(
        spec=model.CheckSpec(
            id="ephemeral.feature-branch-push", title="t", group="ephemeral", rule="armed"
        ),
        status=model.PASS,
        reason="pushed",
    )

    report = harness_run.build_report(
        runner_for([]),
        TARGETS,
        [placeholder],
        {},
        [],
        "ephemeral-armed",
        "2026-08-17T00:00:00Z",
        [armed],
    )

    assert len(report.results) == 1
    assert report.results[0].status == model.PASS


def test_an_unstartable_run_reports_no_go_with_a_named_reason() -> None:
    report = harness_run.unavailable_report(
        "read-only", "2026-08-17T00:00:00Z", "arming refused: bad phrase", TARGETS
    )

    assert report.decision == model.NO_GO
    assert report.results[0].spec.id == "harness.startup"
    assert report.results[0].status == model.NOT_RUN
    assert report.harness_errors == ["arming refused: bad phrase"]