atlas-iac/scripts/ops/hermes_handoff_checks_platform.py

#!/usr/bin/env python3
"""Acceptance checks for the release baseline, provider identity, and routing.

These are the facts a handoff rests on before any authority question is asked:
that the tree under test is the merged one, that both providers authenticate
through the subscriptions they are supposed to, that Switchyard actually decided
the routes it claims to, and that chat, agent, and triage are still three
distinct scopes rather than one surface wearing three hostnames.
"""

from __future__ import annotations

from hermes_handoff_catalog import (
    CLAUDE_HEALTH,
    CLAUDE_HEALTH_FIELDS,
    CODEX_HEALTH,
    CODEX_HEALTH_FIELDS,
    ENV_NAME_PROJECTION,
    OPERATOR,
    PROVIDER_API_KEY_NAMES,
    ROUTING_LOG,
    SELF,
    SWITCHYARD,
    Targets,
    check,
    step,
)
from hermes_handoff_model import CheckSpec
from hermes_handoff_policy import GITEA_CLIENT, projection, shell


def _baseline(targets: Targets) -> list[CheckSpec]:
    api = f"/api/v1/repos/{targets.repo}"
    dependency_heads = dict(targets.dependency_heads)
    checks = [
        check(
            "baseline.local-origin-main-is-current",
            "The local origin/main ref equals the fixed remote-main SHA",
            "baseline",
            "stdout_matches",
            [step("main", OPERATOR, "git", "rev-parse", f"{targets.remote}/main")],
            {"step": "main", "equals": targets.remote_main_sha},
            rationale="The ancestry check is meaningful only after the local tracking ref is bound to the remote observation.",
        ),
        check(
            "baseline.origin-main-descends-merged-work",
            "origin/main descends from the merged Hermes isolation baseline",
            "baseline",
            "allowed",
            [
                step(
                    "ancestor",
                    OPERATOR,
                    "git",
                    "merge-base",
                    "--is-ancestor",
                    targets.baseline_commit,
                    f"{targets.remote}/main",
                )
            ],
            rationale="Everything downstream assumes the merged worker-isolation baseline is present.",
        ),
        check(
            "baseline.pending-pull-requests-recorded",
            "Exact open pull-request heads are recorded, not assumed",
            "baseline",
            "json_record",
            [
                step(
                    "open",
                    OPERATOR,
                    GITEA_CLIENT,
                    "read",
                    f"{api}/pulls?state=open&limit=50",
                )
            ],
            {
                "step": "open",
                "record": {
                    "numbers": "[].number",
                    "head_refs": "[].head.ref",
                    "head_shas": "[].head.sha",
                    "base_shas": "[].base.sha",
                    "drafts": "[].draft",
                    "mergeable": "[].mergeable",
                },
            },
            rationale="The runbook has to name the heads it was verified against; stale heads are how a release verifies the wrong tree.",
        ),
        check(
            "baseline.operator-checkout-clean",
            "The operator checkout has no uncommitted drift",
            "baseline",
            "stdout_matches",
            [step("status", OPERATOR, "git", "status", "--porcelain")],
            {"step": "status", "equals": ""},
            mandatory=False,
            rationale="Advisory: a dirty checkout does not invalidate cluster evidence, but it does muddy what was compared.",
        ),
    ]
    checks += [
        check(
            f"baseline.dependency-pr-{number}-merged",
            f"Pull request #{number} is merged into main",
            "baseline",
            "json_field",
            [step("pr", OPERATOR, GITEA_CLIENT, "read", f"{api}/pulls/{number}")],
            {
                "step": "pr",
                "fields": {
                    "merged": True,
                    "base.ref": "main",
                    "head.sha": dependency_heads.get(number, ""),
                },
            },
            rationale="This harness certifies the post-merge platform; an unmerged dependency means it is measuring something else.",
        )
        for number in targets.dependency_pull_requests
    ]
    return checks


def _identity(targets: Targets) -> list[CheckSpec]:
    codex = shell("json_fields", path=CODEX_HEALTH, fields=CODEX_HEALTH_FIELDS)
    claude = shell("json_fields", path=CLAUDE_HEALTH, fields=CLAUDE_HEALTH_FIELDS)
    agent = f"deploy/{targets.agent_deployment}"
    return [
        check(
            "identity.codex-is-chatgpt-subscription",
            "Codex authenticates through the ChatGPT subscription, not an API key",
            "identity",
            "json_field",
            [step("health", SELF, *codex)],
            {
                "step": "health",
                "fields": {
                    "authenticated": True,
                    "transport": "codex-chatgpt-subscription",
                    "state": "available",
                },
            },
            rationale="Only allow-listed status fields are read; the credential itself is never opened.",
        ),
        check(
            "identity.codex-evidence-is-fresh",
            "Codex authentication evidence is current",
            "identity",
            "json_recent",
            [step("health", SELF, *codex)],
            {
                "step": "health",
                "now": targets.now,
                "fields": {"checked_at": targets.max_evidence_age_seconds},
            },
            rationale="Stale provider evidence describes a state the release no longer has.",
        ),
        check(
            "identity.claude-is-firstparty-subscription",
            "Claude authenticates as a claude.ai first-party subscription",
            "identity",
            "json_field",
            [step("health", SELF, *claude)],
            {
                "step": "health",
                "fields": {
                    "authenticated": True,
                    "api_provider": "firstParty",
                    "auth_method": "claude.ai",
                    "transport": "claude-code-cli-subscription",
                    "state": "available",
                },
            },
        ),
        check(
            "identity.claude-evidence-is-fresh",
            "Claude authentication evidence is current",
            "identity",
            "json_recent",
            [step("health", SELF, *claude)],
            {
                "step": "health",
                "now": targets.now,
                "fields": {"checked_at": targets.max_evidence_age_seconds},
            },
        ),
        check(
            "identity.no-provider-api-key-in-pod",
            "No provider API-key variable is set in the running agent process",
            "identity",
            "names_absent",
            [step("env", SELF, *shell("env_names"))],
            {
                "step": "env",
                "names": PROVIDER_API_KEY_NAMES,
                "contains": ("API_KEY",),
            },
            rationale="Names only. The probe lists variable names and never their values.",
        ),
        check(
            "identity.no-provider-api-key-in-manifest",
            "No provider API-key variable is declared on the agent workload",
            "identity",
            "names_absent",
            [
                step(
                    "env",
                    OPERATOR,
                    "kubectl",
                    "--namespace",
                    targets.namespace,
                    "get",
                    agent,
                    "-o",
                    ENV_NAME_PROJECTION,
                )
            ],
            {"step": "env", "names": PROVIDER_API_KEY_NAMES, "contains": ("API_KEY",)},
            rationale="The operator counterpart to the in-pod probe: desired state and running state are asserted separately.",
        ),
    ]


def _routing(targets: Targets) -> list[CheckSpec]:
    switchyard = f"deploy/{targets.switchyard_deployment}"
    tail = shell("tail_lines", path=ROUTING_LOG, limit=str(targets.routing_tail_lines))
    routing_step = step(
        "routing", SWITCHYARD, *tail, record=False, max_bytes=targets.routing_tail_bytes
    )
    return [
        check(
            "routing.switchyard-is-available",
            "Switchyard has a ready replica serving every model-call boundary",
            "routing",
            "lines_match",
            [
                step(
                    "deployment",
                    OPERATOR,
                    "kubectl",
                    "--namespace",
                    targets.namespace,
                    "get",
                    switchyard,
                    "-o",
                    projection("jsonpath={.status.readyReplicas}"),
                )
            ],
            {"step": "deployment", "pattern": r"^[1-9][0-9]*$", "minimum": 1},
        ),
        check(
            "routing.provider-effort-and-fallback-evidence",
            "Routing evidence covers both providers, all graded efforts, and a real fallback",
            "routing",
            "routing_evidence",
            [routing_step],
            {
                "step": "routing",
                "providers": ("codex", "claude"),
                "efforts": ("medium", "high", "xhigh"),
                "lanes": ("route", "worker"),
                "require_fallback_evidence": True,
                "max_age_seconds": targets.max_evidence_age_seconds,
                "now": targets.now,
            },
            rationale="The routing log records the decided route per boundary; the tail is parsed here and never pasted into the report.",
        ),
        check(
            "routing.codex-latency-recorded",
            "Codex boundary latency is measured, not assumed",
            "routing",
            "json_numeric",
            [
                step(
                    "health",
                    SELF,
                    *shell(
                        "json_fields", path=CODEX_HEALTH, fields=CODEX_HEALTH_FIELDS
                    ),
                )
            ],
            {"step": "health", "fields": {"latency_ms": {"min": 1, "max": 600_000}}},
        ),
        check(
            "routing.claude-latency-recorded",
            "Claude boundary latency is measured, not assumed",
            "routing",
            "json_numeric",
            [
                step(
                    "health",
                    SELF,
                    *shell(
                        "json_fields", path=CLAUDE_HEALTH, fields=CLAUDE_HEALTH_FIELDS
                    ),
                )
            ],
            {"step": "health", "fields": {"latency_ms": {"min": 1, "max": 600_000}}},
        ),
    ]


def _scope_profile(
    targets: Targets, name: str, workload: str, container: str, expected: str
) -> CheckSpec:
    profile_projection = projection(
        f'jsonpath={{.spec.template.spec.containers[?(@.name=="{container}")]'
        '.env[?(@.name=="HERMES_AUTO_ROUTER_PROFILE")].value}'
    )
    return check(
        f"scopes.{name}-profile-is-distinct",
        f"The {name} surface runs under its own routing profile",
        "scopes",
        "stdout_matches",
        [
            step(
                "profile",
                OPERATOR,
                "kubectl",
                "--namespace",
                targets.namespace,
                "get",
                workload,
                "-o",
                profile_projection,
            )
        ],
        {"step": "profile", "equals": expected},
        rationale="Chat, agent, and triage must not collapse into one scope; each states its own profile.",
    )


def _scopes(targets: Targets) -> list[CheckSpec]:
    return [
        _scope_profile(
            targets,
            "agent",
            f"deploy/{targets.agent_deployment}",
            targets.agent_container,
            "agent",
        ),
        _scope_profile(
            targets,
            "chat",
            f"statefulset/{targets.chat_statefulset}",
            targets.chat_container,
            "chat",
        ),
        _scope_profile(
            targets,
            "triage",
            f"deploy/{targets.triage_deployment}",
            targets.triage_container,
            "triage",
        ),
        check(
            "scopes.surfaces-have-distinct-hosts",
            "Chat, agent, and triage are published on distinct hostnames",
            "scopes",
            "distinct_count",
            [
                step(
                    "hosts",
                    OPERATOR,
                    "kubectl",
                    "--namespace",
                    targets.namespace,
                    "get",
                    "ingress",
                    "-o",
                    projection(
                        'jsonpath={range .items[*]}{range .spec.rules[*]}{.host}{"\\n"}{end}{end}'
                    ),
                )
            ],
            {"step": "hosts", "minimum": 3},
        ),
    ]


def platform_checks(targets: Targets) -> list[CheckSpec]:
    """Return the baseline, identity, routing, and scope checks."""
    return [
        *_baseline(targets),
        *_identity(targets),
        *_routing(targets),
        *_scopes(targets),
    ]