Hermes Agent
8f00545828
Decides whether the Hermes platform handoff is fit to release, and refuses to round an absence of evidence up to a pass. The harness is read-only by default and classifies 71 checks PASS / FAIL / NOT_RUN / NOT_APPLICABLE. Any mandatory FAIL or NOT_RUN is NO_GO, and so is a harness-level problem: an unreachable vantage, a catalog entry whose evidence no longer exists, an expired deadline, or an evaluator that raised. Evidence comes from two vantages that cannot cover for each other: an external read-only operator kubeconfig, and the Hermes agent probing itself from inside its own pod. Before any check runs, the harness asks each vantage who it is and stops if they are the same principal, because dual-vantage evidence from one identity is a restatement rather than a corroboration. `--as` is rejected for every operator-side command and reachable only as the inner command of a `kubectl exec`, so impersonation can never stand in for a real self-probe. A deny check needs a live refused request, not only an authorization review. Two safety properties are structural rather than conventional, enforced where an argv becomes a subprocess: the default mode mutates nothing (mutating verbs require a server dry run; there is deliberately no live TokenRequest probe, because a successful one would mint a real credential), and no probe can pull a credential value into a report (no vault/sops/curl, secrets readable only with -o name, environment probes list names, shell only through frozen reviewed templates). Captures are bounded before they are screened, and the rendered report is re-screened before it is written. Mutation lives behind a separate arming flag with an exact confirmation phrase, a caller-supplied unique ref, a preflight that refuses a protected push target before any network call, and a cleanup whose verification is itself mandatory. A default run reports those four checks NOT_RUN. The catalog is declarative so a reviewer reads what is asserted rather than how it is plumbed, and so structural properties can be proven over every entry before a run. Catalog drift surfaces as NOT_RUN, which stops the release. docs/hermes_full_handoff_acceptance.md carries the merge order for PRs #14-#18 on top of the merged #13 baseline, the image build and Flux rollout, the rollback point for each step, the go/no-go checklist, and the limits that are asserted rather than exercised. Validation: 295 handoff tests pass with 100% line coverage on all 15 new modules; the full unit suite is 647 passed with two failures that reproduce unchanged on origin/main; Ruff, py_compile, kustomize render, and a diff credential screen are clean; a live read-only run against Atlas returns NO_GO for the pre-merge cluster with no unscreened fields in the report.
389 lines
14 KiB
Python
389 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
"""Acceptance checks for the distributed worker pool and the visible surfaces.
|
|
|
|
A pool is only distributed if its workers occupy distinct nodes and distinct
|
|
volumes and leave state ownership with the coordinator; a replica count alone
|
|
satisfies none of that. A surface is only working if the work it did is visible
|
|
in it, which is why the session and Kanban checks look at the same data the UI
|
|
renders rather than at whether a pod is Running.
|
|
|
|
The ephemeral entries are placeholders in a default run. They report NOT_RUN
|
|
until the mutation mode is separately armed, and the armed mode replaces them
|
|
with real, mandatory results.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from hermes_handoff_catalog import (
|
|
CHAT,
|
|
INIT_NAME_PROJECTION,
|
|
NODE_NAME_PROJECTION,
|
|
OPERATOR,
|
|
SELF,
|
|
Targets,
|
|
check,
|
|
)
|
|
from hermes_handoff_catalog import step as step
|
|
from hermes_handoff_model import EPHEMERAL, CheckSpec
|
|
|
|
KANBAN_STATUSES = [
|
|
"archived",
|
|
"blocked",
|
|
"done",
|
|
"ready",
|
|
"review",
|
|
"running",
|
|
"scheduled",
|
|
"todo",
|
|
"triage",
|
|
]
|
|
|
|
|
|
def _pool(targets: Targets) -> list[CheckSpec]:
|
|
selector = targets.pool_selector
|
|
return [
|
|
check(
|
|
"pool.three-workers-are-ready",
|
|
"The distributed execution pool has its full complement of ready workers",
|
|
"pool",
|
|
"json_numeric",
|
|
[
|
|
step(
|
|
"statefulset",
|
|
OPERATOR,
|
|
"kubectl",
|
|
"--namespace",
|
|
targets.namespace,
|
|
"get",
|
|
f"statefulset/{targets.pool_statefulset}",
|
|
"-o",
|
|
"json",
|
|
)
|
|
],
|
|
{
|
|
"step": "statefulset",
|
|
"fields": {
|
|
"status.readyReplicas": {"min": targets.pool_replicas},
|
|
"spec.replicas": {"min": targets.pool_replicas},
|
|
},
|
|
},
|
|
),
|
|
check(
|
|
"pool.workers-occupy-distinct-nodes",
|
|
"Each pool worker runs on its own node",
|
|
"pool",
|
|
"distinct_count",
|
|
[
|
|
step(
|
|
"nodes",
|
|
OPERATOR,
|
|
"kubectl",
|
|
"--namespace",
|
|
targets.namespace,
|
|
"get",
|
|
"pods",
|
|
"--selector",
|
|
selector,
|
|
"-o",
|
|
NODE_NAME_PROJECTION,
|
|
)
|
|
],
|
|
{
|
|
"step": "nodes",
|
|
"minimum": targets.pool_replicas,
|
|
"total_equals": targets.pool_replicas,
|
|
},
|
|
rationale="Three replicas co-scheduled onto one node satisfy a replica count and defeat the isolation it stands for.",
|
|
),
|
|
check(
|
|
"pool.workers-own-distinct-volumes",
|
|
"Each pool worker has its own durable workspace",
|
|
"pool",
|
|
"distinct_count",
|
|
[
|
|
step(
|
|
"claims",
|
|
OPERATOR,
|
|
"kubectl",
|
|
"--namespace",
|
|
targets.namespace,
|
|
"get",
|
|
"persistentvolumeclaims",
|
|
"--selector",
|
|
selector,
|
|
"-o",
|
|
"name",
|
|
)
|
|
],
|
|
{"step": "claims", "minimum": targets.pool_replicas},
|
|
),
|
|
check(
|
|
"pool.workers-are-tokenless",
|
|
"Pool workers mount no Kubernetes API token",
|
|
"pool",
|
|
"json_field",
|
|
[
|
|
step(
|
|
"statefulset",
|
|
OPERATOR,
|
|
"kubectl",
|
|
"--namespace",
|
|
targets.namespace,
|
|
"get",
|
|
f"statefulset/{targets.pool_statefulset}",
|
|
"-o",
|
|
"json",
|
|
)
|
|
],
|
|
{
|
|
"step": "statefulset",
|
|
"fields": {"spec.template.spec.automountServiceAccountToken": False},
|
|
},
|
|
rationale="A fenced execution worker needs no cluster authority; holding a token is the failure, not the exception.",
|
|
),
|
|
check(
|
|
"pool.coordinator-retains-sole-state-ownership",
|
|
"No pool worker mounts the coordinator's durable state",
|
|
"pool",
|
|
"names_absent",
|
|
[
|
|
step(
|
|
"claims",
|
|
OPERATOR,
|
|
"kubectl",
|
|
"--namespace",
|
|
targets.namespace,
|
|
"get",
|
|
f"statefulset/{targets.pool_statefulset}",
|
|
"-o",
|
|
'jsonpath={range .spec.template.spec.volumes[*]}{.persistentVolumeClaim.claimName}{"\\n"}{end}',
|
|
)
|
|
],
|
|
{"step": "claims", "names": targets.coordinator_claims},
|
|
rationale="Two writers on one SQLite state file is the duplicate-work and lost-result failure this pool exists to avoid.",
|
|
),
|
|
*_pool_env(targets),
|
|
]
|
|
|
|
|
|
def _pool_env(targets: Targets) -> list[CheckSpec]:
|
|
"""Return the assignment-safety check only when a site declares what to expect.
|
|
|
|
A ``names_present`` assertion over an empty expectation would pass on any
|
|
workload at all, which is worse than not asserting: it reads as coverage.
|
|
"""
|
|
if not targets.pool_worker_env:
|
|
return []
|
|
return [
|
|
check(
|
|
"pool.assignment-safety-knobs-are-configured",
|
|
"Pool workers carry the lease and de-duplication configuration",
|
|
"pool",
|
|
"names_present",
|
|
[
|
|
step(
|
|
"env",
|
|
OPERATOR,
|
|
"kubectl",
|
|
"--namespace",
|
|
targets.namespace,
|
|
"get",
|
|
f"statefulset/{targets.pool_statefulset}",
|
|
"-o",
|
|
'jsonpath={range .spec.template.spec.containers[*]}{range .env[*]}{.name}{"\\n"}{end}{end}',
|
|
)
|
|
],
|
|
{"step": "env", "names": targets.pool_worker_env},
|
|
rationale="Deployed configuration only; the stale-claim and duplicate-dispatch behaviour itself is covered by the pool's own regression suite.",
|
|
)
|
|
]
|
|
|
|
|
|
def _surfaces(targets: Targets) -> list[CheckSpec]:
|
|
checks = [
|
|
check(
|
|
"surfaces.kanban-activity-is-visible",
|
|
"Durable Kanban activity is visible to the agent surface",
|
|
"surfaces",
|
|
"all_of_field",
|
|
[
|
|
step(
|
|
"tasks",
|
|
SELF,
|
|
"hermes",
|
|
"kanban",
|
|
"list",
|
|
"--json",
|
|
record=False,
|
|
max_bytes=targets.listing_bytes,
|
|
)
|
|
],
|
|
{"step": "tasks", "key_field": "id", "fields": {"status": KANBAN_STATUSES}},
|
|
rationale="Non-empty and well-formed. An empty board is NOT_RUN, because it cannot distinguish a quiet week from a broken read.",
|
|
),
|
|
check(
|
|
"surfaces.durable-sessions-render-without-blank-rows",
|
|
"Durable sessions are listed with a preview, a source, and an id",
|
|
"surfaces",
|
|
"lines_match",
|
|
[step("sessions", SELF, "hermes", "sessions", "list", "--limit", "20", record=False)],
|
|
{"step": "sessions", "pattern": r"^\S.*\S$", "skip": 2, "minimum": 1},
|
|
rationale="A row whose leading column is blank is the session-render regression seen from the data side.",
|
|
),
|
|
check(
|
|
"surfaces.agent-ui-session-patches-are-deployed",
|
|
"The agent surface runs the session-activity rendering patches",
|
|
"surfaces",
|
|
"names_present",
|
|
[
|
|
step(
|
|
"init",
|
|
OPERATOR,
|
|
"kubectl",
|
|
"--namespace",
|
|
targets.namespace,
|
|
"get",
|
|
f"deploy/{targets.agent_deployment}",
|
|
"-o",
|
|
INIT_NAME_PROJECTION,
|
|
)
|
|
],
|
|
{"step": "init", "names": ("patch-web-session-activity", "patch-api-server-sessions")},
|
|
rationale="Blank session rows were a rendering regression; this asserts the patches that fixed it are in the running pod.",
|
|
),
|
|
check(
|
|
"surfaces.chat-session-patches-are-deployed",
|
|
"The chat surface runs the API-session continuity patches",
|
|
"surfaces",
|
|
"names_present",
|
|
[
|
|
step(
|
|
"init",
|
|
OPERATOR,
|
|
"kubectl",
|
|
"--namespace",
|
|
targets.namespace,
|
|
"get",
|
|
f"statefulset/{targets.chat_statefulset}",
|
|
"-o",
|
|
INIT_NAME_PROJECTION,
|
|
)
|
|
],
|
|
{"step": "init", "names": targets.chat_init_containers},
|
|
),
|
|
check(
|
|
"surfaces.chat-telegram-topic-state-is-durable",
|
|
"Telegram topic and tenant state is held on durable storage",
|
|
"surfaces",
|
|
"names_present",
|
|
[
|
|
step(
|
|
"claims",
|
|
OPERATOR,
|
|
"kubectl",
|
|
"--namespace",
|
|
targets.namespace,
|
|
"get",
|
|
f"deploy/{targets.chat_router_deployment}",
|
|
"-o",
|
|
'jsonpath={range .spec.template.spec.volumes[*]}{.persistentVolumeClaim.claimName}{"\\n"}{end}',
|
|
)
|
|
],
|
|
{"step": "claims", "contains": ("state",)},
|
|
rationale="Topic selection survives a router restart only if the tenant state file is on a claim rather than in the pod.",
|
|
),
|
|
]
|
|
if targets.expect_telegram_sessions:
|
|
checks.append(
|
|
check(
|
|
"surfaces.chat-telegram-sessions-are-continuous",
|
|
"The selected chat tenant still holds durable Telegram-sourced sessions",
|
|
"surfaces",
|
|
"lines_match",
|
|
[
|
|
step(
|
|
"telegram",
|
|
CHAT,
|
|
"hermes",
|
|
"sessions",
|
|
"list",
|
|
"--source",
|
|
"telegram",
|
|
"--limit",
|
|
"20",
|
|
record=False,
|
|
)
|
|
],
|
|
{"step": "telegram", "pattern": r"^\S.*\S$", "skip": 2, "minimum": 1},
|
|
rationale="Session continuity is per tenant and depends on there having been Telegram traffic, so it is asserted only when the operator names the tenant that carries the identity.",
|
|
)
|
|
)
|
|
if targets.chat_config_revision:
|
|
checks.append(
|
|
check(
|
|
"surfaces.chat-runs-the-telegram-topic-revision",
|
|
"The chat surface runs the durable Telegram topic configuration",
|
|
"surfaces",
|
|
"stdout_matches",
|
|
[
|
|
step(
|
|
"revision",
|
|
OPERATOR,
|
|
"kubectl",
|
|
"--namespace",
|
|
targets.namespace,
|
|
"get",
|
|
f"statefulset/{targets.chat_statefulset}",
|
|
"-o",
|
|
"jsonpath={.spec.template.metadata.annotations.ai\\.bstein\\.dev/config-rev}",
|
|
)
|
|
],
|
|
{"step": "revision", "equals": targets.chat_config_revision},
|
|
rationale="Telegram topic continuity is carried by the config revision the pod template pins.",
|
|
)
|
|
)
|
|
return checks
|
|
|
|
|
|
def _ephemeral() -> list[CheckSpec]:
|
|
return [
|
|
check(
|
|
f"ephemeral.{name}",
|
|
title,
|
|
"ephemeral",
|
|
"not_armed",
|
|
[],
|
|
{"reason": reason},
|
|
mandatory=False,
|
|
scope=EPHEMERAL,
|
|
rationale="Mutating checks are separately armed. A default run reports them NOT_RUN and never touches the forge.",
|
|
)
|
|
for name, title, reason in (
|
|
(
|
|
"feature-branch-push",
|
|
"A unique ephemeral feature branch can be pushed and removed",
|
|
"ephemeral mutation mode is not armed",
|
|
),
|
|
(
|
|
"draft-pull-request",
|
|
"A draft pull request can be opened and closed against the ephemeral branch",
|
|
"ephemeral mutation mode is not armed",
|
|
),
|
|
(
|
|
"protected-branch-refusal",
|
|
"Arming against a protected branch is refused before any network call",
|
|
"ephemeral mutation mode is not armed",
|
|
),
|
|
(
|
|
"cleanup-verified",
|
|
"Every ephemeral ref and pull request is removed and the removal is verified",
|
|
"ephemeral mutation mode is not armed",
|
|
),
|
|
)
|
|
]
|
|
|
|
|
|
def worker_checks(targets: Targets) -> list[CheckSpec]:
|
|
"""Return the pool, surface, and ephemeral-placeholder checks."""
|
|
return [*_pool(targets), *_surfaces(targets), *_ephemeral()]
|