atlas-iac/scripts/ops/hermes_handoff_checks_delivery.py

#!/usr/bin/env python3
"""Acceptance checks for forge authority, node accounts, images, and regressions.

What the platform is allowed to do outside its own namespace: which forge
authority a worker holds, what a node account is and is not, how an image is
built and pinned, and whether the fixes the release depends on are actually
running rather than merely merged.

Where a property has no side-effect-free live probe — a merge that must never be
attempted, a branch protection the harness identity cannot read — the entry says
so in its rationale and asserts the nearest fact that *is* observable, rather
than quietly asserting nothing.
"""

from __future__ import annotations

from hermes_handoff_catalog import (
    FORGE_CREDENTIAL_NAMES,
    INIT_NAME_PROJECTION,
    NODE,
    NODE_NAME_PROJECTION,
    OPERATOR,
    SELF,
    Targets,
    check,
    step,
)
from hermes_handoff_model import ATTEMPT, CheckSpec
from hermes_handoff_policy import GITEA_CLIENT, shell

HOST_ETC = "/host-etc"
HOST_HOME = "/host-home"


def _forge(targets: Targets) -> list[CheckSpec]:
    api = f"/api/v1/repos/{targets.repo}"
    return [
        check(
            "forge.worker-holds-no-git-credential-variable",
            "No forge credential is exported into the worker process",
            "forge",
            "names_absent",
            [step("env", SELF, *shell("env_names"))],
            {"step": "env", "names": FORGE_CREDENTIAL_NAMES},
            rationale="Authentication reaches Git through an askpass helper that reads runtime state, never through the environment.",
        ),
        check(
            "forge.worker-holds-no-credential-file",
            "The worker home has no stored Git credential file",
            "forge",
            "stdout_matches",
            [step("credentials", SELF, *shell("path_readable", path="/opt/data/.git-credentials"))],
            {"step": "credentials", "equals": "unreadable"},
        ),
        check(
            "forge.identity-is-not-an-administrator",
            "The forge identity the worker uses holds no administrative rights",
            "forge",
            "json_field",
            [step("user", SELF, GITEA_CLIENT, "GET", "/api/v1/user")],
            {"step": "user", "fields": {"is_admin": False}},
        ),
        check(
            "forge.repository-authority-is-push-only",
            "The worker can push a branch but cannot administer the repository",
            "forge",
            "json_field",
            [step("repo", SELF, GITEA_CLIENT, "GET", api)],
            {
                "step": "repo",
                "fields": {
                    "permissions.admin": False,
                    "permissions.push": True,
                    "permissions.pull": True,
                },
            },
            rationale="Merge, approve, close, and protection changes all require authority this identity is proven not to hold.",
        ),
        check(
            "forge.administrative-route-is-refused",
            "An administrative forge route is refused for this identity",
            "forge",
            "denied",
            [
                step(
                    "attempt",
                    SELF,
                    GITEA_CLIENT,
                    "GET",
                    f"{api}/branch_protections",
                    kind=ATTEMPT,
                )
            ],
            rationale="A live refusal, not a configuration reading. The same lack of authority is what blocks a merge route.",
        ),
        check(
            "forge.clone-and-fetch-still-work",
            "The worker can still read the repository it is expected to work in",
            "forge",
            "allowed",
            [
                step(
                    "ls-remote",
                    SELF,
                    *shell(
                        "git_ls_remote",
                        askpass=targets.git_askpass,
                        url=f"https://scm.bstein.dev/{targets.repo}.git",
                    ),
                )
            ],
        ),
    ]


def _nodes(targets: Targets) -> list[CheckSpec]:
    account = targets.node_account
    return [
        check(
            "nodes.account-password-is-locked",
            "The dedicated node account has no usable password",
            "nodes",
            "stdout_matches",
            [
                step(
                    "lock",
                    NODE,
                    *shell("account_lock_state", account=account, path=f"{HOST_ETC}/shadow"),
                )
            ],
            {"step": "lock", "equals": "locked"},
            rationale="The probe reports the lock marker only; the password hash is never read.",
        ),
        check(
            "nodes.account-identity-is-dedicated",
            "The node account is the dedicated Hermes identity, not a shared one",
            "nodes",
            "stdout_matches",
            [
                step(
                    "identity",
                    NODE,
                    *shell("account_identity", account=account, path=f"{HOST_ETC}/passwd"),
                )
            ],
            {"step": "identity", "equals": targets.node_account_identity},
        ),
        check(
            "nodes.account-holds-no-supplementary-groups",
            "The node account joins no supplementary or administrative group",
            "nodes",
            "stdout_matches",
            [
                step(
                    "groups",
                    NODE,
                    *shell("account_groups", account=account, path=f"{HOST_ETC}/group"),
                )
            ],
            {"step": "groups", "equals": ""},
            rationale="sudo, docker, adm, and disk are all group grants; an empty membership list rules out every one of them.",
        ),
        check(
            "nodes.account-has-no-sudo-policy",
            "No sudoers fragment names the node account",
            "nodes",
            "stdout_matches",
            [step("sudoers", NODE, *shell("path_mode", path=f"{HOST_ETC}/sudoers.d/{account}"))],
            {"step": "sudoers", "equals": "absent"},
        ),
        check(
            "nodes.account-holds-a-single-authorized-key",
            "Exactly one authorized key is installed for the node account",
            "nodes",
            "stdout_matches",
            [
                step(
                    "keys",
                    NODE,
                    *shell("line_count", path=f"{HOST_HOME}/{account}/.ssh/authorized_keys"),
                )
            ],
            {"step": "keys", "equals": "1"},
            rationale="A count, never the key material. A second key is how a human identity re-enters an account that is meant to be dedicated.",
        ),
        *[
            check(
                f"nodes.{label}-is-root-only",
                f"{title} remains unreadable to the node account",
                "nodes",
                "stdout_matches",
                [step("mode", NODE, *shell("path_mode", path=path))],
                {"step": "mode", "equals": expected},
            )
            for label, title, path, expected in (
                (
                    "k3s-credentials",
                    "The K3s admin kubeconfig",
                    f"{HOST_ETC}/rancher/k3s/k3s.yaml",
                    ["600 root:root", "absent"],
                ),
                ("k3s-state", "K3s server state", "/host-k3s", ["700 root:root", "absent"]),
                ("kubelet-state", "Kubelet state", "/host-kubelet", ["700 root:root", "750 root:root", "absent"]),
                ("container-runtime", "The container runtime socket directory", "/host-run-containerd", ["710 root:root", "700 root:root", "absent"]),
            )
        ],
        check(
            "nodes.hardening-covers-every-node",
            "The node account audit runs on every node in the cluster",
            "nodes",
            "distinct_count",
            [
                step(
                    "coverage",
                    OPERATOR,
                    "kubectl",
                    "--namespace",
                    targets.namespace,
                    "get",
                    "pods",
                    "--selector",
                    f"app={targets.node_daemonset}",
                    "-o",
                    NODE_NAME_PROJECTION,
                )
            ],
            {"step": "coverage", "minimum": targets.node_count},
            rationale="A hardened fleet with one unenrolled node is an unhardened fleet.",
        ),
    ]


def _build(targets: Targets) -> list[CheckSpec]:
    pipeline = f"{targets.remote}/main:{targets.builder_pipeline}"
    return [
        check(
            "build.builder-service-account-is-tokenless",
            "The image builder service account mounts no API token",
            "build",
            "json_field",
            [
                step(
                    "serviceaccount",
                    OPERATOR,
                    "kubectl",
                    "--namespace",
                    targets.builder_namespace,
                    "get",
                    "serviceaccount",
                    targets.builder_serviceaccount,
                    "-o",
                    "json",
                )
            ],
            {"step": "serviceaccount", "fields": {"automountServiceAccountToken": False}},
        ),
        check(
            "build.builder-runs-unprivileged-with-exact-capabilities",
            "The build pod is unprivileged and adds only the expected capabilities",
            "build",
            "names_present",
            [step("pipeline", OPERATOR, "git", "show", pipeline, record=False)],
            {
                "step": "pipeline",
                "names": (
                    targets.builder_capabilities,
                    "privileged: false",
                    "allowPrivilegeEscalation: false",
                    "automountServiceAccountToken: false",
                    'drop: ["ALL"]',
                ),
            },
            rationale="Exact lines, not a substring sweep: a widened capability set has to change one of these lines to take effect.",
        ),
        check(
            "build.harbor-immutability-rule-is-applied",
            "The Harbor immutable-tag rule for the agent image has been applied",
            "build",
            "json_numeric",
            [
                step(
                    "job",
                    OPERATOR,
                    "kubectl",
                    "--namespace",
                    targets.harbor_namespace,
                    "get",
                    "job",
                    targets.harbor_immutability_job,
                    "-o",
                    "json",
                )
            ],
            {"step": "job", "fields": {"status.succeeded": {"min": 1}}},
            rationale="The job verifies the rule it creates, so a completed run is evidence the rule matched the expected contract.",
        ),
        check(
            "build.deployed-image-is-digest-pinned",
            "The running agent image is pinned by digest, not by a mutable tag",
            "build",
            "names_present",
            [
                step(
                    "image",
                    OPERATOR,
                    "kubectl",
                    "--namespace",
                    targets.namespace,
                    "get",
                    f"deploy/{targets.agent_deployment}",
                    "-o",
                    'jsonpath={range .spec.template.spec.containers[*]}{.image}{"\\n"}{end}',
                )
            ],
            {"step": "image", "contains": ("@sha256:",)},
        ),
    ]


def _reliability(targets: Targets) -> list[CheckSpec]:
    return [
        check(
            "reliability.finalization-and-replay-patches-are-deployed",
            "The agent workload runs the finalization, replay, and session patches",
            "reliability",
            "names_present",
            [
                step(
                    "init",
                    OPERATOR,
                    "kubectl",
                    "--namespace",
                    targets.namespace,
                    "get",
                    f"deploy/{targets.agent_deployment}",
                    "-o",
                    INIT_NAME_PROJECTION,
                )
            ],
            {"step": "init", "names": targets.agent_init_containers},
            rationale="The regression suites live in the repository; this asserts the fixes they cover are actually running.",
        ),
        check(
            "reliability.regression-suites-are-present-on-main",
            "The decomposition and worker-recovery regression suites are on main",
            "reliability",
            "names_present",
            [
                step(
                    "suites",
                    OPERATOR,
                    "git",
                    "ls-tree",
                    "--name-only",
                    f"{targets.remote}/main",
                    "testing/tests/",
                )
            ],
            {
                "step": "suites",
                "names": (
                    "testing/tests/test_hermes_cli_lanes.py",
                    "testing/tests/test_hermes_coordinator.py",
                    "testing/tests/test_hermes_worker_recovery.py",
                ),
            },
        ),
    ]


def delivery_checks(targets: Targets) -> list[CheckSpec]:
    """Return the forge, node, build, and reliability checks."""
    return [*_forge(targets), *_nodes(targets), *_build(targets), *_reliability(targets)]