atlas-iac/scripts/ops/hermes_handoff_checks_workers.py

#!/usr/bin/env python3
"""Acceptance checks for the distributed worker pool and the visible surfaces.

A pool is only distributed if its workers occupy distinct nodes and distinct
volumes and leave state ownership with the coordinator; a replica count alone
satisfies none of that. A surface is only working if the work it did is visible
in it, which is why the session and Kanban checks look at the same data the UI
renders rather than at whether a pod is Running.

The ephemeral entries are placeholders in a default run. They report NOT_RUN
until the mutation mode is separately armed, and the armed mode replaces them
with real, mandatory results.
"""

from __future__ import annotations

from hermes_handoff_catalog import (
    CHAT,
    INIT_NAME_PROJECTION,
    NODE_NAME_PROJECTION,
    OPERATOR,
    SELF,
    Targets,
    check,
)
from hermes_handoff_catalog import step as step
from hermes_handoff_model import EPHEMERAL, CheckSpec

KANBAN_STATUSES = [
    "archived",
    "blocked",
    "done",
    "ready",
    "review",
    "running",
    "scheduled",
    "todo",
    "triage",
]


def _pool(targets: Targets) -> list[CheckSpec]:
    selector = targets.pool_selector
    return [
        check(
            "pool.three-workers-are-ready",
            "The distributed execution pool has its full complement of ready workers",
            "pool",
            "json_numeric",
            [
                step(
                    "statefulset",
                    OPERATOR,
                    "kubectl",
                    "--namespace",
                    targets.namespace,
                    "get",
                    f"statefulset/{targets.pool_statefulset}",
                    "-o",
                    "json",
                )
            ],
            {
                "step": "statefulset",
                "fields": {
                    "status.readyReplicas": {"min": targets.pool_replicas},
                    "spec.replicas": {"min": targets.pool_replicas},
                },
            },
        ),
        check(
            "pool.workers-occupy-distinct-nodes",
            "Each pool worker runs on its own node",
            "pool",
            "distinct_count",
            [
                step(
                    "nodes",
                    OPERATOR,
                    "kubectl",
                    "--namespace",
                    targets.namespace,
                    "get",
                    "pods",
                    "--selector",
                    selector,
                    "-o",
                    NODE_NAME_PROJECTION,
                )
            ],
            {
                "step": "nodes",
                "minimum": targets.pool_replicas,
                "total_equals": targets.pool_replicas,
            },
            rationale="Three replicas co-scheduled onto one node satisfy a replica count and defeat the isolation it stands for.",
        ),
        check(
            "pool.workers-own-distinct-volumes",
            "Each pool worker has its own durable workspace",
            "pool",
            "distinct_count",
            [
                step(
                    "claims",
                    OPERATOR,
                    "kubectl",
                    "--namespace",
                    targets.namespace,
                    "get",
                    "persistentvolumeclaims",
                    "--selector",
                    selector,
                    "-o",
                    "name",
                )
            ],
            {"step": "claims", "minimum": targets.pool_replicas},
        ),
        check(
            "pool.workers-are-tokenless",
            "Pool workers mount no Kubernetes API token",
            "pool",
            "json_field",
            [
                step(
                    "statefulset",
                    OPERATOR,
                    "kubectl",
                    "--namespace",
                    targets.namespace,
                    "get",
                    f"statefulset/{targets.pool_statefulset}",
                    "-o",
                    "json",
                )
            ],
            {
                "step": "statefulset",
                "fields": {"spec.template.spec.automountServiceAccountToken": False},
            },
            rationale="A fenced execution worker needs no cluster authority; holding a token is the failure, not the exception.",
        ),
        check(
            "pool.coordinator-retains-sole-state-ownership",
            "No pool worker mounts the coordinator's durable state",
            "pool",
            "names_absent",
            [
                step(
                    "claims",
                    OPERATOR,
                    "kubectl",
                    "--namespace",
                    targets.namespace,
                    "get",
                    f"statefulset/{targets.pool_statefulset}",
                    "-o",
                    'jsonpath={range .spec.template.spec.volumes[*]}{.persistentVolumeClaim.claimName}{"\\n"}{end}',
                )
            ],
            {"step": "claims", "names": targets.coordinator_claims},
            rationale="Two writers on one SQLite state file is the duplicate-work and lost-result failure this pool exists to avoid.",
        ),
        *_pool_env(targets),
    ]


def _pool_env(targets: Targets) -> list[CheckSpec]:
    """Return the assignment-safety check only when a site declares what to expect.

    A ``names_present`` assertion over an empty expectation would pass on any
    workload at all, which is worse than not asserting: it reads as coverage.
    """
    if not targets.pool_worker_env:
        return []
    return [
        check(
            "pool.assignment-safety-knobs-are-configured",
            "Pool workers carry the lease and de-duplication configuration",
            "pool",
            "names_present",
            [
                step(
                    "env",
                    OPERATOR,
                    "kubectl",
                    "--namespace",
                    targets.namespace,
                    "get",
                    f"statefulset/{targets.pool_statefulset}",
                    "-o",
                    'jsonpath={range .spec.template.spec.containers[*]}{range .env[*]}{.name}{"\\n"}{end}{end}',
                )
            ],
            {"step": "env", "names": targets.pool_worker_env},
            rationale="Deployed configuration only; the stale-claim and duplicate-dispatch behaviour itself is covered by the pool's own regression suite.",
        )
    ]


def _surfaces(targets: Targets) -> list[CheckSpec]:
    checks = [
        check(
            "surfaces.kanban-activity-is-visible",
            "Durable Kanban activity is visible to the agent surface",
            "surfaces",
            "all_of_field",
            [
                step(
                    "tasks",
                    SELF,
                    "hermes",
                    "kanban",
                    "list",
                    "--json",
                    record=False,
                    max_bytes=targets.listing_bytes,
                )
            ],
            {"step": "tasks", "key_field": "id", "fields": {"status": KANBAN_STATUSES}},
            rationale="Non-empty and well-formed. An empty board is NOT_RUN, because it cannot distinguish a quiet week from a broken read.",
        ),
        check(
            "surfaces.durable-sessions-render-without-blank-rows",
            "Durable sessions are listed with a preview, a source, and an id",
            "surfaces",
            "lines_match",
            [step("sessions", SELF, "hermes", "sessions", "list", "--limit", "20", record=False)],
            {"step": "sessions", "pattern": r"^\S.*\S$", "skip": 2, "minimum": 1},
            rationale="A row whose leading column is blank is the session-render regression seen from the data side.",
        ),
        check(
            "surfaces.agent-ui-session-patches-are-deployed",
            "The agent surface runs the session-activity rendering patches",
            "surfaces",
            "names_present",
            [
                step(
                    "init",
                    OPERATOR,
                    "kubectl",
                    "--namespace",
                    targets.namespace,
                    "get",
                    f"deploy/{targets.agent_deployment}",
                    "-o",
                    INIT_NAME_PROJECTION,
                )
            ],
            {"step": "init", "names": ("patch-web-session-activity", "patch-api-server-sessions")},
            rationale="Blank session rows were a rendering regression; this asserts the patches that fixed it are in the running pod.",
        ),
        check(
            "surfaces.chat-session-patches-are-deployed",
            "The chat surface runs the API-session continuity patches",
            "surfaces",
            "names_present",
            [
                step(
                    "init",
                    OPERATOR,
                    "kubectl",
                    "--namespace",
                    targets.namespace,
                    "get",
                    f"statefulset/{targets.chat_statefulset}",
                    "-o",
                    INIT_NAME_PROJECTION,
                )
            ],
            {"step": "init", "names": targets.chat_init_containers},
        ),
        check(
            "surfaces.chat-telegram-topic-state-is-durable",
            "Telegram topic and tenant state is held on durable storage",
            "surfaces",
            "names_present",
            [
                step(
                    "claims",
                    OPERATOR,
                    "kubectl",
                    "--namespace",
                    targets.namespace,
                    "get",
                    f"deploy/{targets.chat_router_deployment}",
                    "-o",
                    'jsonpath={range .spec.template.spec.volumes[*]}{.persistentVolumeClaim.claimName}{"\\n"}{end}',
                )
            ],
            {"step": "claims", "contains": ("state",)},
            rationale="Topic selection survives a router restart only if the tenant state file is on a claim rather than in the pod.",
        ),
    ]
    if targets.expect_telegram_sessions:
        checks.append(
            check(
                "surfaces.chat-telegram-sessions-are-continuous",
                "The selected chat tenant still holds durable Telegram-sourced sessions",
                "surfaces",
                "lines_match",
                [
                    step(
                        "telegram",
                        CHAT,
                        "hermes",
                        "sessions",
                        "list",
                        "--source",
                        "telegram",
                        "--limit",
                        "20",
                        record=False,
                    )
                ],
                {"step": "telegram", "pattern": r"^\S.*\S$", "skip": 2, "minimum": 1},
                rationale="Session continuity is per tenant and depends on there having been Telegram traffic, so it is asserted only when the operator names the tenant that carries the identity.",
            )
        )
    if targets.chat_config_revision:
        checks.append(
            check(
                "surfaces.chat-runs-the-telegram-topic-revision",
                "The chat surface runs the durable Telegram topic configuration",
                "surfaces",
                "stdout_matches",
                [
                    step(
                        "revision",
                        OPERATOR,
                        "kubectl",
                        "--namespace",
                        targets.namespace,
                        "get",
                        f"statefulset/{targets.chat_statefulset}",
                        "-o",
                        "jsonpath={.spec.template.metadata.annotations.ai\\.bstein\\.dev/config-rev}",
                    )
                ],
                {"step": "revision", "equals": targets.chat_config_revision},
                rationale="Telegram topic continuity is carried by the config revision the pod template pins.",
            )
        )
    return checks


def _ephemeral() -> list[CheckSpec]:
    return [
        check(
            f"ephemeral.{name}",
            title,
            "ephemeral",
            "not_armed",
            [],
            {"reason": reason},
            mandatory=False,
            scope=EPHEMERAL,
            rationale="Mutating checks are separately armed. A default run reports them NOT_RUN and never touches the forge.",
        )
        for name, title, reason in (
            (
                "feature-branch-push",
                "A unique ephemeral feature branch can be pushed and removed",
                "ephemeral mutation mode is not armed",
            ),
            (
                "draft-pull-request",
                "A draft pull request can be opened and closed against the ephemeral branch",
                "ephemeral mutation mode is not armed",
            ),
            (
                "protected-branch-refusal",
                "Arming against a protected branch is refused before any network call",
                "ephemeral mutation mode is not armed",
            ),
            (
                "cleanup-verified",
                "Every ephemeral ref and pull request is removed and the removal is verified",
                "ephemeral mutation mode is not armed",
            ),
        )
    ]


def worker_checks(targets: Targets) -> list[CheckSpec]:
    """Return the pool, surface, and ephemeral-placeholder checks."""
    return [*_pool(targets), *_surfaces(targets), *_ephemeral()]