atlas-iac/scripts/ops/hermes_handoff_exec.py

#!/usr/bin/env python3
"""Bounded, screened command execution for the handoff acceptance harness.

Every probe runs through :class:`Runner`, which owns the three budgets a
long-running acceptance sweep needs: a per-command timeout, a whole-run
deadline, and a per-capture byte budget. Exhausting the deadline does not
silently shorten the sweep — the remaining probes report ``NOT_RUN``, which is a
NO_GO, so a slow cluster can never be mistaken for a clean one.

Subprocesses never inherit the caller's environment. A probe that echoed
`$ANTHROPIC_API_KEY` would otherwise be one typo away from a leak, so the child
environment is rebuilt from an allowlist of names whose values the harness
already needs to reach the cluster and the forge.
"""

from __future__ import annotations

import os
import shlex
import subprocess
import time
from collections.abc import Callable, Mapping, Sequence
from dataclasses import dataclass, field

from hermes_handoff_policy import READ_ONLY, PolicyError, check_argv
from hermes_handoff_redaction import DEFAULT_MAX_BYTES, redact, safe_text

DEFAULT_COMMAND_TIMEOUT = 25.0
DEFAULT_RUN_DEADLINE = 900.0

# Names whose values the harness needs to reach the cluster and the forge. The
# values are handed to the child process and are never recorded.
ENV_PASSTHROUGH = (
    "GIT_ASKPASS",
    "GITEA_BASE_URL",
    "HERMES_HOME",
    "HOME",
    "JENKINS_BASE_URL",
    "KUBECONFIG",
    "LANG",
    "PATH",
    "REQUESTS_CA_BUNDLE",
    "SSL_CERT_FILE",
    "VAULT_ADDR",
)

DEADLINE_ERROR = "deadline-exceeded"
TIMEOUT_ERROR = "timeout"

OPERATOR = "operator"
SELF = "self"

# Tools that accept a `--context` flag directly after the binary name.
CONTEXT_AWARE_BINARIES = ("kubectl", "flux", "helm")


@dataclass(frozen=True)
class Outcome:
    """The screened, bounded result of one probe command."""

    argv: tuple[str, ...]
    vantage: str
    returncode: int | None = None
    stdout: str = ""
    stderr: str = ""
    truncated: bool = False
    duration_ms: int = 0
    error: str | None = None

    @property
    def ok(self) -> bool:
        """Report whether the command ran to completion and succeeded."""
        return self.error is None and self.returncode == 0

    @property
    def ran(self) -> bool:
        """Report whether the command reached the target at all."""
        return self.error is None

    @property
    def combined(self) -> str:
        """Return stdout and stderr joined, for message matching."""
        return "\n".join(part for part in (self.stdout, self.stderr) if part)

    def as_dict(self) -> dict[str, object]:
        """Return a JSON-serialisable view of the outcome."""
        return {
            "command": redact(shlex.join(self.argv)),
            "vantage": self.vantage,
            "returncode": self.returncode,
            "stdout": self.stdout,
            "stderr": self.stderr,
            "truncated": self.truncated,
            "duration_ms": self.duration_ms,
            "error": self.error,
        }


@dataclass(frozen=True)
class Vantage:
    """A named identity and transport a probe can be observed from.

    ``prefix`` is how the in-pod vantage is reached: a ``kubectl exec`` into the
    target container, so the inner command authenticates as the pod's own
    service account instead of as the operator. ``inserts`` carries per-binary
    flags — the operator's `--context`, which must not follow the command into
    the pod.
    """

    name: str
    prefix: tuple[str, ...] = ()
    inserts: Mapping[str, tuple[str, ...]] = field(default_factory=dict)
    env: Mapping[str, str] = field(default_factory=dict)
    description: str = ""

    def wrap(self, argv: Sequence[str]) -> tuple[str, ...]:
        """Return ``argv`` addressed to this vantage."""
        command = tuple(argv)
        if command:
            extra = self.inserts.get(command[0].rsplit("/", 1)[-1])
            if extra:
                command = (command[0], *extra, *command[1:])
        return (*self.prefix, *command)


def build_environment(vantage: Vantage, source: Mapping[str, str] | None = None) -> dict[str, str]:
    """Return the child environment: allow-listed names plus vantage overrides."""
    origin = os.environ if source is None else source
    environment = {name: origin[name] for name in ENV_PASSTHROUGH if name in origin}
    environment.setdefault("LC_ALL", "C")
    environment.update(vantage.env)
    return environment


def operator_vantage(kubeconfig: str | None = None, context: str | None = None) -> Vantage:
    """Return the external read-only operator vantage."""
    inserts = (
        {binary: ("--context", context) for binary in CONTEXT_AWARE_BINARIES} if context else {}
    )
    detail = " ".join(
        part for part in (f"context={context}" if context else "", "kubeconfig-pinned" if kubeconfig else "") if part
    )
    return Vantage(
        name=OPERATOR,
        inserts=inserts,
        env={"KUBECONFIG": kubeconfig} if kubeconfig else {},
        description=f"operator {detail}".strip(),
    )


def pod_vantage(
    namespace: str,
    pod: str,
    container: str,
    operator: Vantage | None = None,
) -> Vantage:
    """Return the in-pod vantage: the inner command authenticates as the pod."""
    reach = operator.inserts.get("kubectl", ()) if operator else ()
    return Vantage(
        name=SELF,
        prefix=(
            "kubectl",
            *reach,
            "exec",
            "--namespace",
            namespace,
            pod,
            "--container",
            container,
            "--",
        ),
        env=dict(operator.env) if operator else {},
        description=f"in-pod {namespace}/{pod}[{container}]",
    )


class Runner:
    """Run probe commands under policy, timeout, deadline, and byte budgets."""

    def __init__(
        self,
        mode: str = READ_ONLY,
        command_timeout: float = DEFAULT_COMMAND_TIMEOUT,
        deadline_seconds: float = DEFAULT_RUN_DEADLINE,
        max_bytes: int = DEFAULT_MAX_BYTES,
        clock: Callable[[], float] = time.monotonic,
        spawn: Callable[..., subprocess.CompletedProcess] = subprocess.run,
        environ: Mapping[str, str] | None = None,
    ) -> None:
        self.mode = mode
        self.command_timeout = command_timeout
        self.max_bytes = max_bytes
        self._clock = clock
        self._spawn = spawn
        self._environ = environ
        self._deadline_at = clock() + deadline_seconds
        self.commands_run = 0

    @property
    def remaining_seconds(self) -> float:
        """Return the seconds left before the whole-run deadline."""
        return self._deadline_at - self._clock()

    def run(
        self, argv: Sequence[str], vantage: Vantage, max_bytes: int | None = None
    ) -> Outcome:
        """Run one command, returning a screened outcome and never raising."""
        budget_bytes = self.max_bytes if max_bytes is None else max_bytes
        addressed = vantage.wrap(argv)
        try:
            check_argv(addressed, self.mode)
        except PolicyError as exc:
            return Outcome(argv=addressed, vantage=vantage.name, error=f"policy: {exc}")

        budget = min(self.command_timeout, self.remaining_seconds)
        if budget <= 0:
            return Outcome(argv=addressed, vantage=vantage.name, error=DEADLINE_ERROR)

        started = self._clock()
        self.commands_run += 1
        try:
            completed = self._spawn(
                list(addressed),
                capture_output=True,
                text=True,
                timeout=budget,
                check=False,
                env=build_environment(vantage, self._environ),
            )
        except subprocess.TimeoutExpired:
            return Outcome(
                argv=addressed,
                vantage=vantage.name,
                error=TIMEOUT_ERROR,
                duration_ms=int((self._clock() - started) * 1000),
            )
        except OSError as exc:
            return Outcome(
                argv=addressed, vantage=vantage.name, error=f"spawn: {exc.strerror or exc}"
            )

        stdout, cut_out = safe_text(completed.stdout or "", budget_bytes)
        stderr, cut_err = safe_text(completed.stderr or "", budget_bytes)
        return Outcome(
            argv=addressed,
            vantage=vantage.name,
            returncode=completed.returncode,
            stdout=stdout,
            stderr=stderr,
            truncated=cut_out or cut_err,
            duration_ms=int((self._clock() - started) * 1000),
        )