#!/usr/bin/env python3 """Acceptance checks for the distributed worker pool and the visible surfaces. A pool is only distributed if its workers occupy distinct nodes and distinct volumes and leave state ownership with the coordinator; a replica count alone satisfies none of that. A surface is only working if the work it did is visible in it, which is why the session and Kanban checks look at the same data the UI renders rather than at whether a pod is Running. The ephemeral entries are placeholders in a default run. They report NOT_RUN until the mutation mode is separately armed, and the armed mode replaces them with real, mandatory results. """ from __future__ import annotations from hermes_handoff_catalog import ( CHAT, INIT_NAME_PROJECTION, NODE_NAME_PROJECTION, OPERATOR, SELF, Targets, check, ) from hermes_handoff_catalog import step as step from hermes_handoff_model import EPHEMERAL, CheckSpec KANBAN_STATUSES = [ "archived", "blocked", "done", "ready", "review", "running", "scheduled", "todo", "triage", ] def _pool(targets: Targets) -> list[CheckSpec]: selector = targets.pool_selector return [ check( "pool.three-workers-are-ready", "The distributed execution pool has its full complement of ready workers", "pool", "json_numeric", [ step( "statefulset", OPERATOR, "kubectl", "--namespace", targets.namespace, "get", f"statefulset/{targets.pool_statefulset}", "-o", "json", ) ], { "step": "statefulset", "fields": { "status.readyReplicas": {"min": targets.pool_replicas}, "spec.replicas": {"min": targets.pool_replicas}, }, }, ), check( "pool.workers-occupy-distinct-nodes", "Each pool worker runs on its own node", "pool", "distinct_count", [ step( "nodes", OPERATOR, "kubectl", "--namespace", targets.namespace, "get", "pods", "--selector", selector, "-o", NODE_NAME_PROJECTION, ) ], { "step": "nodes", "minimum": targets.pool_replicas, "total_equals": targets.pool_replicas, }, rationale="Three replicas co-scheduled onto one node satisfy a replica count and defeat the isolation it stands for.", ), check( "pool.workers-own-distinct-volumes", "Each pool worker has its own durable workspace", "pool", "distinct_count", [ step( "claims", OPERATOR, "kubectl", "--namespace", targets.namespace, "get", "persistentvolumeclaims", "--selector", selector, "-o", "name", ) ], {"step": "claims", "minimum": targets.pool_replicas}, ), check( "pool.workers-are-tokenless", "Pool workers mount no Kubernetes API token", "pool", "json_field", [ step( "statefulset", OPERATOR, "kubectl", "--namespace", targets.namespace, "get", f"statefulset/{targets.pool_statefulset}", "-o", "json", ) ], { "step": "statefulset", "fields": {"spec.template.spec.automountServiceAccountToken": False}, }, rationale="A fenced execution worker needs no cluster authority; holding a token is the failure, not the exception.", ), check( "pool.coordinator-retains-sole-state-ownership", "No pool worker mounts the coordinator's durable state", "pool", "names_absent", [ step( "claims", OPERATOR, "kubectl", "--namespace", targets.namespace, "get", f"statefulset/{targets.pool_statefulset}", "-o", 'jsonpath={range .spec.template.spec.volumes[*]}{.persistentVolumeClaim.claimName}{"\\n"}{end}', ) ], {"step": "claims", "names": targets.coordinator_claims}, rationale="Two writers on one SQLite state file is the duplicate-work and lost-result failure this pool exists to avoid.", ), *_pool_env(targets), ] def _pool_env(targets: Targets) -> list[CheckSpec]: """Return the assignment-safety check only when a site declares what to expect. A ``names_present`` assertion over an empty expectation would pass on any workload at all, which is worse than not asserting: it reads as coverage. """ if not targets.pool_worker_env: return [] return [ check( "pool.assignment-safety-knobs-are-configured", "Pool workers carry the lease and de-duplication configuration", "pool", "names_present", [ step( "env", OPERATOR, "kubectl", "--namespace", targets.namespace, "get", f"statefulset/{targets.pool_statefulset}", "-o", 'jsonpath={range .spec.template.spec.containers[*]}{range .env[*]}{.name}{"\\n"}{end}{end}', ) ], {"step": "env", "names": targets.pool_worker_env}, rationale="Deployed configuration only; the stale-claim and duplicate-dispatch behaviour itself is covered by the pool's own regression suite.", ) ] def _surfaces(targets: Targets) -> list[CheckSpec]: checks = [ check( "surfaces.kanban-activity-is-visible", "Durable Kanban activity is visible to the agent surface", "surfaces", "all_of_field", [ step( "tasks", SELF, "hermes", "kanban", "list", "--json", record=False, max_bytes=targets.listing_bytes, ) ], {"step": "tasks", "key_field": "id", "fields": {"status": KANBAN_STATUSES}}, rationale="Non-empty and well-formed. An empty board is NOT_RUN, because it cannot distinguish a quiet week from a broken read.", ), check( "surfaces.durable-sessions-render-without-blank-rows", "Durable sessions are listed with a preview, a source, and an id", "surfaces", "lines_match", [step("sessions", SELF, "hermes", "sessions", "list", "--limit", "20", record=False)], {"step": "sessions", "pattern": r"^\S.*\S$", "skip": 2, "minimum": 1}, rationale="A row whose leading column is blank is the session-render regression seen from the data side.", ), check( "surfaces.agent-ui-session-patches-are-deployed", "The agent surface runs the session-activity rendering patches", "surfaces", "names_present", [ step( "init", OPERATOR, "kubectl", "--namespace", targets.namespace, "get", f"deploy/{targets.agent_deployment}", "-o", INIT_NAME_PROJECTION, ) ], {"step": "init", "names": ("patch-web-session-activity", "patch-api-server-sessions")}, rationale="Blank session rows were a rendering regression; this asserts the patches that fixed it are in the running pod.", ), check( "surfaces.chat-session-patches-are-deployed", "The chat surface runs the API-session continuity patches", "surfaces", "names_present", [ step( "init", OPERATOR, "kubectl", "--namespace", targets.namespace, "get", f"statefulset/{targets.chat_statefulset}", "-o", INIT_NAME_PROJECTION, ) ], {"step": "init", "names": targets.chat_init_containers}, ), check( "surfaces.chat-telegram-topic-state-is-durable", "Telegram topic and tenant state is held on durable storage", "surfaces", "names_present", [ step( "claims", OPERATOR, "kubectl", "--namespace", targets.namespace, "get", f"deploy/{targets.chat_router_deployment}", "-o", 'jsonpath={range .spec.template.spec.volumes[*]}{.persistentVolumeClaim.claimName}{"\\n"}{end}', ) ], {"step": "claims", "contains": ("state",)}, rationale="Topic selection survives a router restart only if the tenant state file is on a claim rather than in the pod.", ), ] if targets.expect_telegram_sessions: checks.append( check( "surfaces.chat-telegram-sessions-are-continuous", "The selected chat tenant still holds durable Telegram-sourced sessions", "surfaces", "lines_match", [ step( "telegram", CHAT, "hermes", "sessions", "list", "--source", "telegram", "--limit", "20", record=False, ) ], {"step": "telegram", "pattern": r"^\S.*\S$", "skip": 2, "minimum": 1}, rationale="Session continuity is per tenant and depends on there having been Telegram traffic, so it is asserted only when the operator names the tenant that carries the identity.", ) ) if targets.chat_config_revision: checks.append( check( "surfaces.chat-runs-the-telegram-topic-revision", "The chat surface runs the durable Telegram topic configuration", "surfaces", "stdout_matches", [ step( "revision", OPERATOR, "kubectl", "--namespace", targets.namespace, "get", f"statefulset/{targets.chat_statefulset}", "-o", "jsonpath={.spec.template.metadata.annotations.ai\\.bstein\\.dev/config-rev}", ) ], {"step": "revision", "equals": targets.chat_config_revision}, rationale="Telegram topic continuity is carried by the config revision the pod template pins.", ) ) return checks def _ephemeral() -> list[CheckSpec]: return [ check( f"ephemeral.{name}", title, "ephemeral", "not_armed", [], {"reason": reason}, mandatory=False, scope=EPHEMERAL, rationale="Mutating checks are separately armed. A default run reports them NOT_RUN and never touches the forge.", ) for name, title, reason in ( ( "feature-branch-push", "A unique ephemeral feature branch can be pushed and removed", "ephemeral mutation mode is not armed", ), ( "draft-pull-request", "A draft pull request can be opened and closed against the ephemeral branch", "ephemeral mutation mode is not armed", ), ( "protected-branch-refusal", "Arming against a protected branch is refused before any network call", "ephemeral mutation mode is not armed", ), ( "cleanup-verified", "Every ephemeral ref and pull request is removed and the removal is verified", "ephemeral mutation mode is not armed", ), ) ] def worker_checks(targets: Targets) -> list[CheckSpec]: """Return the pool, surface, and ephemeral-placeholder checks.""" return [*_pool(targets), *_surfaces(targets), *_ephemeral()]