atlas-iac/scripts/ops/hermes_handoff_run.py

#!/usr/bin/env python3
"""Orchestration for the Hermes full-handoff acceptance harness.

Three things happen before any check runs, and each of them can stop the run on
its own:

* the catalog is validated structurally — every rule exists, every deny check
  carries a real attempt, no operator-side step can impersonate;
* each vantage is resolved and asked who it is; and
* the operator and in-pod identities are compared. If they are the same
  principal there is only one vantage, the dual-vantage evidence would be a
  restatement rather than a corroboration, and the run is a NO_GO.

Everything after that is uniform: run a check's steps against its vantage, hand
the outcomes to the named evaluator, and record the classification. A vantage
that could not be resolved yields ``NOT_RUN`` for every check that needs it,
which is exactly the fail-closed answer.
"""

from __future__ import annotations

import json
from collections.abc import Iterable, Mapping, Sequence

from hermes_handoff_catalog import (
    CHAT,
    NODE,
    OPERATOR,
    SELF,
    SWITCHYARD,
    VANTAGE_NAMES,
    Targets,
)
from hermes_handoff_checks_access import access_checks
from hermes_handoff_checks_delivery import delivery_checks
from hermes_handoff_checks_platform import platform_checks
from hermes_handoff_checks_workers import worker_checks
from hermes_handoff_evaluators import EVALUATORS, evaluate
from hermes_handoff_exec import Outcome, Runner, Vantage, operator_vantage, pod_vantage
from hermes_handoff_model import (
    ATTEMPT,
    EPHEMERAL,
    NOT_RUN,
    STEP_KINDS,
    CheckResult,
    CheckSpec,
    Report,
    VantageRecord,
    utc_now,
)
from hermes_handoff_policy import IMPERSONATION_ARGS

UNRECORDED = "[output not recorded]"


def build_catalog(targets: Targets) -> list[CheckSpec]:
    """Return the whole acceptance catalog for a set of targets."""
    return [
        *platform_checks(targets),
        *access_checks(targets),
        *delivery_checks(targets),
        *worker_checks(targets),
    ]


def validate_catalog(specs: Sequence[CheckSpec]) -> list[str]:
    """Return the structural problems that make a catalog unsafe to run."""
    problems: list[str] = []
    seen: set[str] = set()
    for spec in specs:
        if spec.id in seen:
            problems.append(f"{spec.id}: duplicate check id")
        seen.add(spec.id)
        if spec.rule not in EVALUATORS and spec.scope != EPHEMERAL:
            problems.append(f"{spec.id}: unknown rule {spec.rule!r}")
        for step in spec.steps:
            if step.vantage not in VANTAGE_NAMES:
                problems.append(f"{spec.id}: step {step.key} names unknown vantage {step.vantage!r}")
            if step.kind not in STEP_KINDS:
                problems.append(f"{spec.id}: step {step.key} has unknown kind {step.kind!r}")
            if step.vantage == OPERATOR and set(step.argv) & set(IMPERSONATION_ARGS):
                problems.append(f"{spec.id}: step {step.key} impersonates from the operator vantage")
        if spec.rule == "denied" and not any(step.kind == ATTEMPT for step in spec.steps):
            problems.append(f"{spec.id}: a deny check needs a real attempt, not only a review")
    return problems


def resolve_identity(runner: Runner, vantage: Vantage) -> tuple[str, str]:
    """Return the authenticated username of a vantage and any failure detail."""
    outcome = runner.run(("kubectl", "auth", "whoami", "-o", "json"), vantage)
    if not outcome.ok:
        return ("", outcome.error or outcome.combined[:200])
    try:
        payload = json.loads(outcome.stdout)
    except ValueError as exc:
        return ("", f"unparsable identity: {exc}")
    username = ((payload.get("status") or {}).get("userInfo") or {}).get("username")
    if not isinstance(username, str) or not username:
        return ("", "identity response carried no username")
    return (username, "")


def _first_pod(runner: Runner, operator: Vantage, namespace: str, selector: str) -> tuple[str, str]:
    outcome = runner.run(
        (
            "kubectl",
            "--namespace",
            namespace,
            "get",
            "pods",
            "--selector",
            selector,
            "--field-selector",
            "status.phase=Running",
            "-o",
            "name",
        ),
        operator,
    )
    if not outcome.ok:
        return ("", outcome.error or outcome.combined[:200])
    names = [line.strip().removeprefix("pod/") for line in outcome.stdout.splitlines() if line.strip()]
    if not names:
        return ("", f"no running pod matches {selector}")
    return (sorted(names)[0], "")


def resolve_vantages(
    runner: Runner, targets: Targets, kubeconfig: str | None = None, context: str | None = None
) -> tuple[dict[str, Vantage], list[VantageRecord]]:
    """Resolve every vantage the catalog names, recording what each one is."""
    operator = operator_vantage(kubeconfig, context)
    username, detail = resolve_identity(runner, operator)
    records = [
        VantageRecord(
            name=OPERATOR,
            description=operator.description or "external read-only operator",
            identity=username,
            available=bool(username),
            detail=detail,
        )
    ]
    vantages: dict[str, Vantage] = {OPERATOR: operator}
    if not username:
        return (vantages, records)

    for name, selector, container, description in (
        (SELF, f"app={targets.agent_deployment}", targets.agent_container, "the Hermes agent itself"),
        (
            SWITCHYARD,
            f"app={targets.switchyard_deployment}",
            targets.switchyard_container,
            "the Switchyard routing evidence log",
        ),
        (NODE, f"app={targets.node_daemonset}", targets.node_container, "a node hardening probe"),
    ):
        pod, failure = _first_pod(runner, operator, targets.namespace, selector)
        if not pod:
            records.append(
                VantageRecord(name=name, description=description, available=False, detail=failure)
            )
            continue
        vantage = pod_vantage(targets.namespace, pod, container, operator)
        identity, identity_detail = resolve_identity(runner, vantage) if name == SELF else ("", "")
        records.append(
            VantageRecord(
                name=name,
                description=f"{description} ({targets.namespace}/{pod})",
                identity=identity,
                available=True,
                detail=identity_detail,
            )
        )
        vantages[name] = vantage

    chat_pod = f"{targets.chat_statefulset}-{targets.chat_ordinal}"
    present = runner.run(
        ("kubectl", "--namespace", targets.namespace, "get", f"pod/{chat_pod}", "-o", "name"),
        operator,
    )
    if present.ok and present.stdout.strip():
        vantages[CHAT] = pod_vantage(targets.namespace, chat_pod, targets.chat_container, operator)
        records.append(
            VantageRecord(
                name=CHAT,
                description=f"the chat surface ({targets.namespace}/{chat_pod})",
                available=True,
            )
        )
    else:
        records.append(
            VantageRecord(
                name=CHAT,
                description=f"the chat surface ({targets.namespace}/{chat_pod})",
                available=False,
                detail=present.error or present.combined[:200] or "chat tenant pod not found",
            )
        )
    return (vantages, records)


def vantage_problems(records: Iterable[VantageRecord]) -> list[str]:
    """Return the identity problems that invalidate dual-vantage evidence."""
    by_name = {record.name: record for record in records}
    operator = by_name.get(OPERATOR)
    hermes = by_name.get(SELF)
    problems: list[str] = []
    if operator is None or not operator.available:
        problems.append("the operator vantage is unavailable; no evidence can be collected")
        return problems
    if hermes is None or not hermes.available:
        problems.append("the in-pod Hermes vantage is unavailable; self-probe evidence is missing")
        return problems
    if not hermes.identity:
        problems.append("the in-pod vantage did not report an identity")
    elif operator.identity == hermes.identity:
        problems.append(
            "the operator and in-pod vantages authenticate as the same principal "
            f"({operator.identity}); run the harness with a separate operator kubeconfig"
        )
    return problems


def run_check(
    runner: Runner, spec: CheckSpec, vantages: Mapping[str, Vantage]
) -> CheckResult:
    """Run one check's steps and classify the result."""
    outcomes: dict[str, Outcome] = {}
    recorded: list[Outcome] = []
    for step in spec.steps:
        vantage = vantages.get(step.vantage)
        if vantage is None:
            outcome = Outcome(
                argv=step.argv, vantage=step.vantage, error=f"vantage {step.vantage} is unavailable"
            )
        else:
            outcome = runner.run(step.argv, vantage, step.max_bytes)
        outcomes[step.key] = outcome
        recorded.append(
            outcome
            if step.record
            else Outcome(
                argv=outcome.argv,
                vantage=outcome.vantage,
                returncode=outcome.returncode,
                stdout=UNRECORDED if outcome.stdout else "",
                stderr=UNRECORDED if outcome.stderr else "",
                truncated=outcome.truncated,
                duration_ms=outcome.duration_ms,
                error=outcome.error,
            )
        )
    evaluation = evaluate(spec, outcomes)
    return CheckResult(
        spec=spec,
        status=evaluation.status,
        reason=evaluation.reason,
        evidence=evaluation.evidence,
        outcomes=recorded,
    )


def run_catalog(
    runner: Runner, specs: Sequence[CheckSpec], vantages: Mapping[str, Vantage]
) -> list[CheckResult]:
    """Run every check in the catalog.

    Nothing is skipped once the deadline passes: the runner returns a
    deadline outcome for each remaining command, which classifies as
    ``NOT_RUN``, so a truncated sweep is visible in the report rather than
    silently shorter.
    """
    return [run_check(runner, spec, vantages) for spec in specs]


def build_report(
    runner: Runner,
    targets: Targets,
    specs: Sequence[CheckSpec],
    vantages: Mapping[str, Vantage],
    records: Sequence[VantageRecord],
    mode: str,
    started_at: str,
    extra_results: Sequence[CheckResult] = (),
) -> Report:
    """Run the catalog and assemble the report, including structural problems."""
    problems = validate_catalog(specs) + vantage_problems(records)
    report = Report(
        mode=mode,
        started_at=started_at,
        baseline={
            "namespace": targets.namespace,
            "baseline_commit": targets.baseline_commit,
            "dependency_pull_requests": list(targets.dependency_pull_requests),
            "expected_suspensions": list(targets.expected_suspensions),
            "node_count": targets.node_count,
            "pool_replicas": targets.pool_replicas,
        },
        vantages=list(records),
        harness_errors=problems,
    )
    armed_ids = {result.spec.id for result in extra_results}
    report.results = [
        *run_catalog(runner, [spec for spec in specs if spec.id not in armed_ids], vantages),
        *extra_results,
    ]
    report.results.sort(key=lambda result: result.spec.id)
    if runner.remaining_seconds <= 0:
        report.harness_errors.append("the run deadline expired before every check completed")
    report.finished_at = utc_now()
    return report


def unavailable_report(mode: str, started_at: str, reason: str, targets: Targets) -> Report:
    """Return a NO_GO report for a run that could not start at all."""
    return Report(
        mode=mode,
        started_at=started_at,
        finished_at=utc_now(),
        baseline={"namespace": targets.namespace},
        harness_errors=[reason],
        results=[
            CheckResult(
                spec=CheckSpec(
                    id="harness.startup",
                    title="The harness could start and resolve its vantages",
                    group="harness",
                    rule="allowed",
                ),
                status=NOT_RUN,
                reason=reason,
            )
        ],
    )