388 lines
13 KiB
Python
388 lines
13 KiB
Python
#!/usr/bin/env python3
|
|
"""Acceptance checks for the release baseline, provider identity, and routing.
|
|
|
|
These are the facts a handoff rests on before any authority question is asked:
|
|
that the tree under test is the merged one, that both providers authenticate
|
|
through the subscriptions they are supposed to, that Switchyard actually decided
|
|
the routes it claims to, and that chat, agent, and triage are still three
|
|
distinct scopes rather than one surface wearing three hostnames.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from hermes_handoff_catalog import (
|
|
CLAUDE_HEALTH,
|
|
CLAUDE_HEALTH_FIELDS,
|
|
CODEX_HEALTH,
|
|
CODEX_HEALTH_FIELDS,
|
|
ENV_NAME_PROJECTION,
|
|
OPERATOR,
|
|
PROVIDER_API_KEY_NAMES,
|
|
ROUTING_LOG,
|
|
SELF,
|
|
SWITCHYARD,
|
|
Targets,
|
|
check,
|
|
step,
|
|
)
|
|
from hermes_handoff_model import CheckSpec
|
|
from hermes_handoff_policy import GITEA_CLIENT, projection, shell
|
|
|
|
|
|
def _baseline(targets: Targets) -> list[CheckSpec]:
|
|
api = f"/api/v1/repos/{targets.repo}"
|
|
dependency_heads = dict(targets.dependency_heads)
|
|
checks = [
|
|
check(
|
|
"baseline.local-origin-main-is-current",
|
|
"The local origin/main ref equals the fixed remote-main SHA",
|
|
"baseline",
|
|
"stdout_matches",
|
|
[step("main", OPERATOR, "git", "rev-parse", f"{targets.remote}/main")],
|
|
{"step": "main", "equals": targets.remote_main_sha},
|
|
rationale="The ancestry check is meaningful only after the local tracking ref is bound to the remote observation.",
|
|
),
|
|
check(
|
|
"baseline.origin-main-descends-merged-work",
|
|
"origin/main descends from the merged Hermes isolation baseline",
|
|
"baseline",
|
|
"allowed",
|
|
[
|
|
step(
|
|
"ancestor",
|
|
OPERATOR,
|
|
"git",
|
|
"merge-base",
|
|
"--is-ancestor",
|
|
targets.baseline_commit,
|
|
f"{targets.remote}/main",
|
|
)
|
|
],
|
|
rationale="Everything downstream assumes the merged worker-isolation baseline is present.",
|
|
),
|
|
check(
|
|
"baseline.pending-pull-requests-recorded",
|
|
"Exact open pull-request heads are recorded, not assumed",
|
|
"baseline",
|
|
"json_record",
|
|
[
|
|
step(
|
|
"open",
|
|
OPERATOR,
|
|
GITEA_CLIENT,
|
|
"GET",
|
|
f"{api}/pulls?state=open&limit=50",
|
|
)
|
|
],
|
|
{
|
|
"step": "open",
|
|
"record": {
|
|
"numbers": "[].number",
|
|
"head_refs": "[].head.ref",
|
|
"head_shas": "[].head.sha",
|
|
"base_shas": "[].base.sha",
|
|
"drafts": "[].draft",
|
|
"mergeable": "[].mergeable",
|
|
},
|
|
},
|
|
rationale="The runbook has to name the heads it was verified against; stale heads are how a release verifies the wrong tree.",
|
|
),
|
|
check(
|
|
"baseline.operator-checkout-clean",
|
|
"The operator checkout has no uncommitted drift",
|
|
"baseline",
|
|
"stdout_matches",
|
|
[step("status", OPERATOR, "git", "status", "--porcelain")],
|
|
{"step": "status", "equals": ""},
|
|
mandatory=False,
|
|
rationale="Advisory: a dirty checkout does not invalidate cluster evidence, but it does muddy what was compared.",
|
|
),
|
|
]
|
|
checks += [
|
|
check(
|
|
f"baseline.dependency-pr-{number}-merged",
|
|
f"Pull request #{number} is merged into main",
|
|
"baseline",
|
|
"json_field",
|
|
[step("pr", OPERATOR, GITEA_CLIENT, "GET", f"{api}/pulls/{number}")],
|
|
{
|
|
"step": "pr",
|
|
"fields": {
|
|
"merged": True,
|
|
"base.ref": "main",
|
|
"head.sha": dependency_heads.get(number, ""),
|
|
},
|
|
},
|
|
rationale="This harness certifies the post-merge platform; an unmerged dependency means it is measuring something else.",
|
|
)
|
|
for number in targets.dependency_pull_requests
|
|
]
|
|
return checks
|
|
|
|
|
|
def _identity(targets: Targets) -> list[CheckSpec]:
|
|
codex = shell("json_fields", path=CODEX_HEALTH, fields=CODEX_HEALTH_FIELDS)
|
|
claude = shell("json_fields", path=CLAUDE_HEALTH, fields=CLAUDE_HEALTH_FIELDS)
|
|
agent = f"deploy/{targets.agent_deployment}"
|
|
return [
|
|
check(
|
|
"identity.codex-is-chatgpt-subscription",
|
|
"Codex authenticates through the ChatGPT subscription, not an API key",
|
|
"identity",
|
|
"json_field",
|
|
[step("health", SELF, *codex)],
|
|
{
|
|
"step": "health",
|
|
"fields": {
|
|
"authenticated": True,
|
|
"transport": "codex-chatgpt-subscription",
|
|
"state": "available",
|
|
},
|
|
},
|
|
rationale="Only allow-listed status fields are read; the credential itself is never opened.",
|
|
),
|
|
check(
|
|
"identity.codex-evidence-is-fresh",
|
|
"Codex authentication evidence is current",
|
|
"identity",
|
|
"json_recent",
|
|
[step("health", SELF, *codex)],
|
|
{
|
|
"step": "health",
|
|
"now": targets.now,
|
|
"fields": {"checked_at": targets.max_evidence_age_seconds},
|
|
},
|
|
rationale="Stale provider evidence describes a state the release no longer has.",
|
|
),
|
|
check(
|
|
"identity.claude-is-firstparty-subscription",
|
|
"Claude authenticates as a claude.ai first-party subscription",
|
|
"identity",
|
|
"json_field",
|
|
[step("health", SELF, *claude)],
|
|
{
|
|
"step": "health",
|
|
"fields": {
|
|
"authenticated": True,
|
|
"api_provider": "firstParty",
|
|
"auth_method": "claude.ai",
|
|
"transport": "claude-code-cli-subscription",
|
|
"state": "available",
|
|
},
|
|
},
|
|
),
|
|
check(
|
|
"identity.claude-evidence-is-fresh",
|
|
"Claude authentication evidence is current",
|
|
"identity",
|
|
"json_recent",
|
|
[step("health", SELF, *claude)],
|
|
{
|
|
"step": "health",
|
|
"now": targets.now,
|
|
"fields": {"checked_at": targets.max_evidence_age_seconds},
|
|
},
|
|
),
|
|
check(
|
|
"identity.no-provider-api-key-in-pod",
|
|
"No provider API-key variable is set in the running agent process",
|
|
"identity",
|
|
"names_absent",
|
|
[step("env", SELF, *shell("env_names"))],
|
|
{
|
|
"step": "env",
|
|
"names": PROVIDER_API_KEY_NAMES,
|
|
"contains": ("API_KEY",),
|
|
},
|
|
rationale="Names only. The probe lists variable names and never their values.",
|
|
),
|
|
check(
|
|
"identity.no-provider-api-key-in-manifest",
|
|
"No provider API-key variable is declared on the agent workload",
|
|
"identity",
|
|
"names_absent",
|
|
[
|
|
step(
|
|
"env",
|
|
OPERATOR,
|
|
"kubectl",
|
|
"--namespace",
|
|
targets.namespace,
|
|
"get",
|
|
agent,
|
|
"-o",
|
|
ENV_NAME_PROJECTION,
|
|
)
|
|
],
|
|
{"step": "env", "names": PROVIDER_API_KEY_NAMES, "contains": ("API_KEY",)},
|
|
rationale="The operator counterpart to the in-pod probe: desired state and running state are asserted separately.",
|
|
),
|
|
]
|
|
|
|
|
|
def _routing(targets: Targets) -> list[CheckSpec]:
|
|
switchyard = f"deploy/{targets.switchyard_deployment}"
|
|
tail = shell("tail_lines", path=ROUTING_LOG, limit=str(targets.routing_tail_lines))
|
|
routing_step = step(
|
|
"routing", SWITCHYARD, *tail, record=False, max_bytes=targets.routing_tail_bytes
|
|
)
|
|
return [
|
|
check(
|
|
"routing.switchyard-is-available",
|
|
"Switchyard has a ready replica serving every model-call boundary",
|
|
"routing",
|
|
"lines_match",
|
|
[
|
|
step(
|
|
"deployment",
|
|
OPERATOR,
|
|
"kubectl",
|
|
"--namespace",
|
|
targets.namespace,
|
|
"get",
|
|
switchyard,
|
|
"-o",
|
|
projection("jsonpath={.status.readyReplicas}"),
|
|
)
|
|
],
|
|
{"step": "deployment", "pattern": r"^[1-9][0-9]*$", "minimum": 1},
|
|
),
|
|
check(
|
|
"routing.provider-effort-and-fallback-evidence",
|
|
"Routing evidence covers both providers, all graded efforts, and a real fallback",
|
|
"routing",
|
|
"routing_evidence",
|
|
[routing_step],
|
|
{
|
|
"step": "routing",
|
|
"providers": ("codex", "claude"),
|
|
"efforts": ("medium", "high", "xhigh"),
|
|
"lanes": ("route", "worker"),
|
|
"require_fallback_evidence": True,
|
|
"max_age_seconds": targets.max_evidence_age_seconds,
|
|
"now": targets.now,
|
|
},
|
|
rationale="The routing log records the decided route per boundary; the tail is parsed here and never pasted into the report.",
|
|
),
|
|
check(
|
|
"routing.codex-latency-recorded",
|
|
"Codex boundary latency is measured, not assumed",
|
|
"routing",
|
|
"json_numeric",
|
|
[
|
|
step(
|
|
"health",
|
|
SELF,
|
|
*shell(
|
|
"json_fields", path=CODEX_HEALTH, fields=CODEX_HEALTH_FIELDS
|
|
),
|
|
)
|
|
],
|
|
{"step": "health", "fields": {"latency_ms": {"min": 1, "max": 600_000}}},
|
|
),
|
|
check(
|
|
"routing.claude-latency-recorded",
|
|
"Claude boundary latency is measured, not assumed",
|
|
"routing",
|
|
"json_numeric",
|
|
[
|
|
step(
|
|
"health",
|
|
SELF,
|
|
*shell(
|
|
"json_fields", path=CLAUDE_HEALTH, fields=CLAUDE_HEALTH_FIELDS
|
|
),
|
|
)
|
|
],
|
|
{"step": "health", "fields": {"latency_ms": {"min": 1, "max": 600_000}}},
|
|
),
|
|
]
|
|
|
|
|
|
def _scope_profile(
|
|
targets: Targets, name: str, workload: str, container: str, expected: str
|
|
) -> CheckSpec:
|
|
profile_projection = projection(
|
|
f'jsonpath={{.spec.template.spec.containers[?(@.name=="{container}")]'
|
|
'.env[?(@.name=="HERMES_AUTO_ROUTER_PROFILE")].value}'
|
|
)
|
|
return check(
|
|
f"scopes.{name}-profile-is-distinct",
|
|
f"The {name} surface runs under its own routing profile",
|
|
"scopes",
|
|
"stdout_matches",
|
|
[
|
|
step(
|
|
"profile",
|
|
OPERATOR,
|
|
"kubectl",
|
|
"--namespace",
|
|
targets.namespace,
|
|
"get",
|
|
workload,
|
|
"-o",
|
|
profile_projection,
|
|
)
|
|
],
|
|
{"step": "profile", "equals": expected},
|
|
rationale="Chat, agent, and triage must not collapse into one scope; each states its own profile.",
|
|
)
|
|
|
|
|
|
def _scopes(targets: Targets) -> list[CheckSpec]:
|
|
return [
|
|
_scope_profile(
|
|
targets,
|
|
"agent",
|
|
f"deploy/{targets.agent_deployment}",
|
|
targets.agent_container,
|
|
"agent",
|
|
),
|
|
_scope_profile(
|
|
targets,
|
|
"chat",
|
|
f"statefulset/{targets.chat_statefulset}",
|
|
targets.chat_container,
|
|
"chat",
|
|
),
|
|
_scope_profile(
|
|
targets,
|
|
"triage",
|
|
f"deploy/{targets.triage_deployment}",
|
|
targets.triage_container,
|
|
"triage",
|
|
),
|
|
check(
|
|
"scopes.surfaces-have-distinct-hosts",
|
|
"Chat, agent, and triage are published on distinct hostnames",
|
|
"scopes",
|
|
"distinct_count",
|
|
[
|
|
step(
|
|
"hosts",
|
|
OPERATOR,
|
|
"kubectl",
|
|
"--namespace",
|
|
targets.namespace,
|
|
"get",
|
|
"ingress",
|
|
"-o",
|
|
projection(
|
|
'jsonpath={range .items[*]}{range .spec.rules[*]}{.host}{"\\n"}{end}{end}'
|
|
),
|
|
)
|
|
],
|
|
{"step": "hosts", "minimum": 3},
|
|
),
|
|
]
|
|
|
|
|
|
def platform_checks(targets: Targets) -> list[CheckSpec]:
|
|
"""Return the baseline, identity, routing, and scope checks."""
|
|
return [
|
|
*_baseline(targets),
|
|
*_identity(targets),
|
|
*_routing(targets),
|
|
*_scopes(targets),
|
|
]
|