#!/usr/bin/env python3 """Orchestration for the Hermes full-handoff acceptance harness. Three things happen before any check runs, and each of them can stop the run on its own: * the catalog is validated structurally — every rule exists, every live-denial evaluator carries a read-only attempt, no operator-side step can impersonate; * each vantage is resolved and asked who it is; and * the operator and in-pod identities are compared. If they are the same principal there is only one vantage, the dual-vantage evidence would be a restatement rather than a corroboration, and the run is a NO_GO. Everything after that is uniform: run a check's steps against its vantage, hand the outcomes to the named evaluator, and record the classification. A vantage that could not be resolved yields ``NOT_RUN`` for every check that needs it, which is exactly the fail-closed answer. """ from __future__ import annotations import re from collections.abc import Iterable, Mapping, Sequence from hermes_handoff_catalog import ( CHAT, NODE, OPERATOR, SELF, SWITCHYARD, VANTAGE_NAMES, Targets, ) from hermes_handoff_checks_access import access_checks from hermes_handoff_checks_delivery import delivery_checks from hermes_handoff_checks_platform import platform_checks from hermes_handoff_checks_workers import worker_checks from hermes_handoff_evaluators import EVALUATORS, evaluate from hermes_handoff_exec import Outcome, Runner, Vantage, operator_vantage, pod_vantage from hermes_handoff_model import ( ATTEMPT, EPHEMERAL, NOT_RUN, STEP_KINDS, CheckResult, CheckSpec, Report, VantageRecord, utc_now, ) from hermes_handoff_policy import ( EXPECTED_REMOTE, EXPECTED_REPO, IMPERSONATION_ARGS, PolicyError, check_argv, projection, ) UNRECORDED = "[output not recorded]" def build_catalog(targets: Targets) -> list[CheckSpec]: """Return the whole acceptance catalog for a set of targets.""" return [ *platform_checks(targets), *access_checks(targets), *delivery_checks(targets), *worker_checks(targets), ] def validate_catalog(specs: Sequence[CheckSpec]) -> list[str]: """Return the structural problems that make a catalog unsafe to run.""" problems: list[str] = [] if not specs: return ["catalog is empty"] seen: set[str] = set() fake_operator = operator_vantage() fake_pod = pod_vantage("hermes", "acceptance-probe", "probe", fake_operator) for spec in specs: if spec.id in seen: problems.append(f"{spec.id}: duplicate check id") seen.add(spec.id) if spec.rule not in EVALUATORS and spec.scope != EPHEMERAL: problems.append(f"{spec.id}: unknown rule {spec.rule!r}") step_names: set[str] = set() for step in spec.steps: if step.key in step_names: problems.append(f"{spec.id}: duplicate step key {step.key!r}") step_names.add(step.key) if step.vantage not in VANTAGE_NAMES: problems.append( f"{spec.id}: step {step.key} names unknown vantage {step.vantage!r}" ) if step.kind not in STEP_KINDS: problems.append( f"{spec.id}: step {step.key} has unknown kind {step.kind!r}" ) if set(step.argv) & set(IMPERSONATION_ARGS): problems.append( f"{spec.id}: step {step.key} impersonates from the " f"{step.vantage} vantage" ) try: addressed = ( fake_operator if step.vantage == OPERATOR else fake_pod ).wrap(step.argv) check_argv(addressed) except PolicyError as exc: problems.append(f"{spec.id}: step {step.key} violates policy: {exc}") if spec.rule == "denied" and not any( step.kind == ATTEMPT for step in spec.steps ): problems.append( f"{spec.id}: a deny check needs a real attempt, not only a review" ) return problems def validate_targets(targets: Targets) -> list[str]: """Return release-input problems before a runner or network call exists.""" problems: list[str] = [] sha = re.compile(r"[0-9a-f]{40}\Z") image = re.compile( r"[^@\s]+:git-([0-9a-f]{40})-build-[1-9][0-9]*@sha256:[0-9a-f]{64}\Z" ) if targets.repo != EXPECTED_REPO or targets.remote != EXPECTED_REMOTE: problems.append( "repository and remote must be the fixed Titan atlas-iac origin" ) if ( targets.reviewed_pr_number != 19 or targets.reviewed_head_ref != "feature/hermes-full-handoff-acceptance" ): problems.append( "reviewed PR identity must remain fixed to PR #19 and its existing branch" ) for label, value in ( ("baseline commit", targets.baseline_commit), ("remote main SHA", targets.remote_main_sha), ("reviewed head SHA", targets.reviewed_head_sha), ("build SHA", targets.build_sha), ): if not sha.fullmatch(value): problems.append(f"{label} must be an exact lowercase 40-character SHA") image_match = image.fullmatch(targets.agent_image) if not image_match or image_match.group(1) != targets.build_sha: problems.append("agent image tag/digest must bind the exact build SHA") if targets.build_sha != targets.remote_main_sha: problems.append("build SHA must equal the exact release main SHA") if not re.fullmatch(r"[1-9][0-9]{0,8}", str(targets.deployment_revision)): problems.append("deployment revision must be a positive integer string") if targets.node_count != 3 or targets.pool_replicas != 3: problems.append("node and pool counts are fixed at exactly three") if not targets.chat_config_revision.strip(): problems.append( "Telegram continuity and its chat config revision are mandatory" ) if set(targets.pool_worker_env) != {"HERMES_WORKER_NODE", "HERMES_WORKER_ORDINAL"}: problems.append("pool assignment evidence must include worker node and ordinal") heads = dict(targets.dependency_heads) if len(heads) != len(targets.dependency_heads) or set(heads) != set( targets.dependency_pull_requests ): problems.append("every dependency PR must have one exact current head") elif any(not sha.fullmatch(value) for value in heads.values()): problems.append("dependency heads must be exact lowercase 40-character SHAs") if ( not isinstance(targets.max_evidence_age_seconds, int) or targets.max_evidence_age_seconds <= 0 ): problems.append("maximum evidence age must be a positive integer") return problems def resolve_identity(runner: Runner, vantage: Vantage) -> tuple[str, str]: """Return the authenticated username of a vantage and any failure detail.""" outcome = runner.run( ( "kubectl", "auth", "whoami", "-o", projection("jsonpath={.status.userInfo.username}"), ), vantage, ) if not outcome.ok: return ("", outcome.error or outcome.combined[:200]) username = outcome.stdout.strip() if not username or any(character.isspace() for character in username): return ("", "identity response carried no username") return (username, "") def _first_pod( runner: Runner, operator: Vantage, namespace: str, selector: str ) -> tuple[str, str]: outcome = runner.run( ( "kubectl", "--namespace", namespace, "get", "pods", "--selector", selector, "--field-selector", "status.phase=Running", "-o", "name", ), operator, ) if not outcome.ok: return ("", outcome.error or outcome.combined[:200]) names = [ line.strip().removeprefix("pod/") for line in outcome.stdout.splitlines() if line.strip() ] if not names: return ("", f"no running pod matches {selector}") return (sorted(names)[0], "") def resolve_vantages( runner: Runner, targets: Targets, kubeconfig: str | None = None, context: str | None = None, ) -> tuple[dict[str, Vantage], list[VantageRecord]]: """Resolve every vantage the catalog names, recording what each one is.""" operator = operator_vantage(kubeconfig, context) username, detail = resolve_identity(runner, operator) records = [ VantageRecord( name=OPERATOR, description=operator.description or "external read-only operator", identity=username, available=bool(username), detail=detail, ) ] vantages: dict[str, Vantage] = {OPERATOR: operator} if not username: return (vantages, records) for name, selector, container, description in ( ( SELF, f"app={targets.agent_deployment}", targets.agent_container, "the Hermes agent itself", ), ( SWITCHYARD, f"app={targets.switchyard_deployment}", targets.switchyard_container, "the Switchyard routing evidence log", ), ( NODE, f"app={targets.node_daemonset}", targets.node_container, "a node hardening probe", ), ): pod, failure = _first_pod(runner, operator, targets.namespace, selector) if not pod: records.append( VantageRecord( name=name, description=description, available=False, detail=failure ) ) continue vantage = pod_vantage(targets.namespace, pod, container, operator) identity, identity_detail = ( resolve_identity(runner, vantage) if name == SELF else ("", "") ) records.append( VantageRecord( name=name, description=f"{description} ({targets.namespace}/{pod})", identity=identity, available=True, detail=identity_detail, ) ) vantages[name] = vantage chat_pod = f"{targets.chat_statefulset}-{targets.chat_ordinal}" present = runner.run( ( "kubectl", "--namespace", targets.namespace, "get", f"pod/{chat_pod}", "-o", "name", ), operator, ) if present.ok and present.stdout.strip(): vantages[CHAT] = pod_vantage( targets.namespace, chat_pod, targets.chat_container, operator ) records.append( VantageRecord( name=CHAT, description=f"the chat surface ({targets.namespace}/{chat_pod})", available=True, ) ) else: records.append( VantageRecord( name=CHAT, description=f"the chat surface ({targets.namespace}/{chat_pod})", available=False, detail=present.error or present.combined[:200] or "chat tenant pod not found", ) ) return (vantages, records) def vantage_problems(records: Iterable[VantageRecord]) -> list[str]: """Return the identity problems that invalidate dual-vantage evidence.""" by_name = {record.name: record for record in records} operator = by_name.get(OPERATOR) hermes = by_name.get(SELF) problems: list[str] = [] if operator is None or not operator.available: problems.append( "the operator vantage is unavailable; no evidence can be collected" ) return problems if hermes is None or not hermes.available: problems.append( "the in-pod Hermes vantage is unavailable; self-probe evidence is missing" ) return problems if not hermes.identity: problems.append("the in-pod vantage did not report an identity") elif operator.identity == hermes.identity: problems.append( "the operator and in-pod vantages authenticate as the same principal " f"({operator.identity}); run the harness with a separate operator kubeconfig" ) for name, label in ((SWITCHYARD, "Switchyard"), (CHAT, "chat/Telegram")): record = by_name.get(name) if record is None or not record.available: problems.append(f"the required {label} vantage is unavailable") return problems def run_check( runner: Runner, spec: CheckSpec, vantages: Mapping[str, Vantage] ) -> CheckResult: """Run one check's steps and classify the result.""" outcomes: dict[str, Outcome] = {} recorded: list[Outcome] = [] for step in spec.steps: vantage = vantages.get(step.vantage) if vantage is None: outcome = Outcome( argv=step.argv, vantage=step.vantage, error=f"vantage {step.vantage} is unavailable", ) else: outcome = runner.run(step.argv, vantage, step.max_bytes) outcomes[step.key] = outcome recorded.append( outcome if step.record else Outcome( argv=outcome.argv, vantage=outcome.vantage, returncode=outcome.returncode, stdout=UNRECORDED if outcome.stdout else "", stderr=UNRECORDED if outcome.stderr else "", truncated=outcome.truncated, duration_ms=outcome.duration_ms, error=outcome.error, executable_path=outcome.executable_path, executable_sha256=outcome.executable_sha256, ) ) evaluation = evaluate(spec, outcomes) return CheckResult( spec=spec, status=evaluation.status, reason=evaluation.reason, evidence=evaluation.evidence, outcomes=recorded, ) def run_catalog( runner: Runner, specs: Sequence[CheckSpec], vantages: Mapping[str, Vantage] ) -> list[CheckResult]: """Run every check in the catalog. Nothing is skipped once the deadline passes: the runner returns a deadline outcome for each remaining command, which classifies as ``NOT_RUN``, so a truncated sweep is visible in the report rather than silently shorter. """ return [run_check(runner, spec, vantages) for spec in specs] def build_report( runner: Runner, targets: Targets, specs: Sequence[CheckSpec], vantages: Mapping[str, Vantage], records: Sequence[VantageRecord], mode: str, started_at: str, extra_results: Sequence[CheckResult] = (), ) -> Report: """Run the catalog and assemble the report, including structural problems.""" problems = validate_catalog(specs) + vantage_problems(records) report = Report( mode=mode, started_at=started_at, baseline={ "namespace": targets.namespace, "baseline_commit": targets.baseline_commit, "dependency_pull_requests": list(targets.dependency_pull_requests), "expected_suspensions": list(targets.expected_suspensions), "node_count": targets.node_count, "pool_replicas": targets.pool_replicas, }, vantages=list(records), harness_errors=problems, ) armed_ids = {result.spec.id for result in extra_results} runnable = [spec for spec in specs if spec.id not in armed_ids] if problems: report.results = [ CheckResult(spec=spec, status=NOT_RUN, reason="harness preflight failed") for spec in runnable ] report.results.extend(extra_results) else: report.results = [*run_catalog(runner, runnable, vantages), *extra_results] report.results.sort(key=lambda result: result.spec.id) if runner.remaining_seconds <= 0: report.harness_errors.append( "the run deadline expired before every check completed" ) report.finished_at = utc_now() return report def unavailable_report( mode: str, started_at: str, reason: str, targets: Targets ) -> Report: """Return a NO_GO report for a run that could not start at all.""" return Report( mode=mode, started_at=started_at, finished_at=utc_now(), baseline={"namespace": targets.namespace}, harness_errors=[reason], results=[ CheckResult( spec=CheckSpec( id="harness.startup", title="The harness could start and resolve its vantages", group="harness", rule="allowed", ), status=NOT_RUN, reason=reason, ) ], )