Hermes Agent
8f00545828
Decides whether the Hermes platform handoff is fit to release, and refuses to round an absence of evidence up to a pass. The harness is read-only by default and classifies 71 checks PASS / FAIL / NOT_RUN / NOT_APPLICABLE. Any mandatory FAIL or NOT_RUN is NO_GO, and so is a harness-level problem: an unreachable vantage, a catalog entry whose evidence no longer exists, an expired deadline, or an evaluator that raised. Evidence comes from two vantages that cannot cover for each other: an external read-only operator kubeconfig, and the Hermes agent probing itself from inside its own pod. Before any check runs, the harness asks each vantage who it is and stops if they are the same principal, because dual-vantage evidence from one identity is a restatement rather than a corroboration. `--as` is rejected for every operator-side command and reachable only as the inner command of a `kubectl exec`, so impersonation can never stand in for a real self-probe. A deny check needs a live refused request, not only an authorization review. Two safety properties are structural rather than conventional, enforced where an argv becomes a subprocess: the default mode mutates nothing (mutating verbs require a server dry run; there is deliberately no live TokenRequest probe, because a successful one would mint a real credential), and no probe can pull a credential value into a report (no vault/sops/curl, secrets readable only with -o name, environment probes list names, shell only through frozen reviewed templates). Captures are bounded before they are screened, and the rendered report is re-screened before it is written. Mutation lives behind a separate arming flag with an exact confirmation phrase, a caller-supplied unique ref, a preflight that refuses a protected push target before any network call, and a cleanup whose verification is itself mandatory. A default run reports those four checks NOT_RUN. The catalog is declarative so a reviewer reads what is asserted rather than how it is plumbed, and so structural properties can be proven over every entry before a run. Catalog drift surfaces as NOT_RUN, which stops the release. docs/hermes_full_handoff_acceptance.md carries the merge order for PRs #14-#18 on top of the merged #13 baseline, the image build and Flux rollout, the rollback point for each step, the go/no-go checklist, and the limits that are asserted rather than exercised. Validation: 295 handoff tests pass with 100% line coverage on all 15 new modules; the full unit suite is 647 passed with two failures that reproduce unchanged on origin/main; Ruff, py_compile, kustomize render, and a diff credential screen are clean; a live read-only run against Atlas returns NO_GO for the pre-merge cluster with no unscreened fields in the report.
374 lines
13 KiB
Python
374 lines
13 KiB
Python
#!/usr/bin/env python3
|
|
"""Acceptance checks for forge authority, node accounts, images, and regressions.
|
|
|
|
What the platform is allowed to do outside its own namespace: which forge
|
|
authority a worker holds, what a node account is and is not, how an image is
|
|
built and pinned, and whether the fixes the release depends on are actually
|
|
running rather than merely merged.
|
|
|
|
Where a property has no side-effect-free live probe — a merge that must never be
|
|
attempted, a branch protection the harness identity cannot read — the entry says
|
|
so in its rationale and asserts the nearest fact that *is* observable, rather
|
|
than quietly asserting nothing.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from hermes_handoff_catalog import (
|
|
FORGE_CREDENTIAL_NAMES,
|
|
INIT_NAME_PROJECTION,
|
|
NODE,
|
|
NODE_NAME_PROJECTION,
|
|
OPERATOR,
|
|
SELF,
|
|
Targets,
|
|
check,
|
|
step,
|
|
)
|
|
from hermes_handoff_model import ATTEMPT, CheckSpec
|
|
from hermes_handoff_policy import GITEA_CLIENT, shell
|
|
|
|
HOST_ETC = "/host-etc"
|
|
HOST_HOME = "/host-home"
|
|
|
|
|
|
def _forge(targets: Targets) -> list[CheckSpec]:
|
|
api = f"/api/v1/repos/{targets.repo}"
|
|
return [
|
|
check(
|
|
"forge.worker-holds-no-git-credential-variable",
|
|
"No forge credential is exported into the worker process",
|
|
"forge",
|
|
"names_absent",
|
|
[step("env", SELF, *shell("env_names"))],
|
|
{"step": "env", "names": FORGE_CREDENTIAL_NAMES},
|
|
rationale="Authentication reaches Git through an askpass helper that reads runtime state, never through the environment.",
|
|
),
|
|
check(
|
|
"forge.worker-holds-no-credential-file",
|
|
"The worker home has no stored Git credential file",
|
|
"forge",
|
|
"stdout_matches",
|
|
[step("credentials", SELF, *shell("path_readable", path="/opt/data/.git-credentials"))],
|
|
{"step": "credentials", "equals": "unreadable"},
|
|
),
|
|
check(
|
|
"forge.identity-is-not-an-administrator",
|
|
"The forge identity the worker uses holds no administrative rights",
|
|
"forge",
|
|
"json_field",
|
|
[step("user", SELF, GITEA_CLIENT, "GET", "/api/v1/user")],
|
|
{"step": "user", "fields": {"is_admin": False}},
|
|
),
|
|
check(
|
|
"forge.repository-authority-is-push-only",
|
|
"The worker can push a branch but cannot administer the repository",
|
|
"forge",
|
|
"json_field",
|
|
[step("repo", SELF, GITEA_CLIENT, "GET", api)],
|
|
{
|
|
"step": "repo",
|
|
"fields": {
|
|
"permissions.admin": False,
|
|
"permissions.push": True,
|
|
"permissions.pull": True,
|
|
},
|
|
},
|
|
rationale="Merge, approve, close, and protection changes all require authority this identity is proven not to hold.",
|
|
),
|
|
check(
|
|
"forge.administrative-route-is-refused",
|
|
"An administrative forge route is refused for this identity",
|
|
"forge",
|
|
"denied",
|
|
[
|
|
step(
|
|
"attempt",
|
|
SELF,
|
|
GITEA_CLIENT,
|
|
"GET",
|
|
f"{api}/branch_protections",
|
|
kind=ATTEMPT,
|
|
)
|
|
],
|
|
rationale="A live refusal, not a configuration reading. The same lack of authority is what blocks a merge route.",
|
|
),
|
|
check(
|
|
"forge.clone-and-fetch-still-work",
|
|
"The worker can still read the repository it is expected to work in",
|
|
"forge",
|
|
"allowed",
|
|
[
|
|
step(
|
|
"ls-remote",
|
|
SELF,
|
|
*shell(
|
|
"git_ls_remote",
|
|
askpass=targets.git_askpass,
|
|
url=f"https://scm.bstein.dev/{targets.repo}.git",
|
|
),
|
|
)
|
|
],
|
|
),
|
|
]
|
|
|
|
|
|
def _nodes(targets: Targets) -> list[CheckSpec]:
|
|
account = targets.node_account
|
|
return [
|
|
check(
|
|
"nodes.account-password-is-locked",
|
|
"The dedicated node account has no usable password",
|
|
"nodes",
|
|
"stdout_matches",
|
|
[
|
|
step(
|
|
"lock",
|
|
NODE,
|
|
*shell("account_lock_state", account=account, path=f"{HOST_ETC}/shadow"),
|
|
)
|
|
],
|
|
{"step": "lock", "equals": "locked"},
|
|
rationale="The probe reports the lock marker only; the password hash is never read.",
|
|
),
|
|
check(
|
|
"nodes.account-identity-is-dedicated",
|
|
"The node account is the dedicated Hermes identity, not a shared one",
|
|
"nodes",
|
|
"stdout_matches",
|
|
[
|
|
step(
|
|
"identity",
|
|
NODE,
|
|
*shell("account_identity", account=account, path=f"{HOST_ETC}/passwd"),
|
|
)
|
|
],
|
|
{"step": "identity", "equals": targets.node_account_identity},
|
|
),
|
|
check(
|
|
"nodes.account-holds-no-supplementary-groups",
|
|
"The node account joins no supplementary or administrative group",
|
|
"nodes",
|
|
"stdout_matches",
|
|
[
|
|
step(
|
|
"groups",
|
|
NODE,
|
|
*shell("account_groups", account=account, path=f"{HOST_ETC}/group"),
|
|
)
|
|
],
|
|
{"step": "groups", "equals": ""},
|
|
rationale="sudo, docker, adm, and disk are all group grants; an empty membership list rules out every one of them.",
|
|
),
|
|
check(
|
|
"nodes.account-has-no-sudo-policy",
|
|
"No sudoers fragment names the node account",
|
|
"nodes",
|
|
"stdout_matches",
|
|
[step("sudoers", NODE, *shell("path_mode", path=f"{HOST_ETC}/sudoers.d/{account}"))],
|
|
{"step": "sudoers", "equals": "absent"},
|
|
),
|
|
check(
|
|
"nodes.account-holds-a-single-authorized-key",
|
|
"Exactly one authorized key is installed for the node account",
|
|
"nodes",
|
|
"stdout_matches",
|
|
[
|
|
step(
|
|
"keys",
|
|
NODE,
|
|
*shell("line_count", path=f"{HOST_HOME}/{account}/.ssh/authorized_keys"),
|
|
)
|
|
],
|
|
{"step": "keys", "equals": "1"},
|
|
rationale="A count, never the key material. A second key is how a human identity re-enters an account that is meant to be dedicated.",
|
|
),
|
|
*[
|
|
check(
|
|
f"nodes.{label}-is-root-only",
|
|
f"{title} remains unreadable to the node account",
|
|
"nodes",
|
|
"stdout_matches",
|
|
[step("mode", NODE, *shell("path_mode", path=path))],
|
|
{"step": "mode", "equals": expected},
|
|
)
|
|
for label, title, path, expected in (
|
|
(
|
|
"k3s-credentials",
|
|
"The K3s admin kubeconfig",
|
|
f"{HOST_ETC}/rancher/k3s/k3s.yaml",
|
|
["600 root:root", "absent"],
|
|
),
|
|
("k3s-state", "K3s server state", "/host-k3s", ["700 root:root", "absent"]),
|
|
("kubelet-state", "Kubelet state", "/host-kubelet", ["700 root:root", "750 root:root", "absent"]),
|
|
("container-runtime", "The container runtime socket directory", "/host-run-containerd", ["710 root:root", "700 root:root", "absent"]),
|
|
)
|
|
],
|
|
check(
|
|
"nodes.hardening-covers-every-node",
|
|
"The node account audit runs on every node in the cluster",
|
|
"nodes",
|
|
"distinct_count",
|
|
[
|
|
step(
|
|
"coverage",
|
|
OPERATOR,
|
|
"kubectl",
|
|
"--namespace",
|
|
targets.namespace,
|
|
"get",
|
|
"pods",
|
|
"--selector",
|
|
f"app={targets.node_daemonset}",
|
|
"-o",
|
|
NODE_NAME_PROJECTION,
|
|
)
|
|
],
|
|
{"step": "coverage", "minimum": targets.node_count},
|
|
rationale="A hardened fleet with one unenrolled node is an unhardened fleet.",
|
|
),
|
|
]
|
|
|
|
|
|
def _build(targets: Targets) -> list[CheckSpec]:
|
|
pipeline = f"{targets.remote}/main:{targets.builder_pipeline}"
|
|
return [
|
|
check(
|
|
"build.builder-service-account-is-tokenless",
|
|
"The image builder service account mounts no API token",
|
|
"build",
|
|
"json_field",
|
|
[
|
|
step(
|
|
"serviceaccount",
|
|
OPERATOR,
|
|
"kubectl",
|
|
"--namespace",
|
|
targets.builder_namespace,
|
|
"get",
|
|
"serviceaccount",
|
|
targets.builder_serviceaccount,
|
|
"-o",
|
|
"json",
|
|
)
|
|
],
|
|
{"step": "serviceaccount", "fields": {"automountServiceAccountToken": False}},
|
|
),
|
|
check(
|
|
"build.builder-runs-unprivileged-with-exact-capabilities",
|
|
"The build pod is unprivileged and adds only the expected capabilities",
|
|
"build",
|
|
"names_present",
|
|
[step("pipeline", OPERATOR, "git", "show", pipeline, record=False)],
|
|
{
|
|
"step": "pipeline",
|
|
"names": (
|
|
targets.builder_capabilities,
|
|
"privileged: false",
|
|
"allowPrivilegeEscalation: false",
|
|
"automountServiceAccountToken: false",
|
|
'drop: ["ALL"]',
|
|
),
|
|
},
|
|
rationale="Exact lines, not a substring sweep: a widened capability set has to change one of these lines to take effect.",
|
|
),
|
|
check(
|
|
"build.harbor-immutability-rule-is-applied",
|
|
"The Harbor immutable-tag rule for the agent image has been applied",
|
|
"build",
|
|
"json_numeric",
|
|
[
|
|
step(
|
|
"job",
|
|
OPERATOR,
|
|
"kubectl",
|
|
"--namespace",
|
|
targets.harbor_namespace,
|
|
"get",
|
|
"job",
|
|
targets.harbor_immutability_job,
|
|
"-o",
|
|
"json",
|
|
)
|
|
],
|
|
{"step": "job", "fields": {"status.succeeded": {"min": 1}}},
|
|
rationale="The job verifies the rule it creates, so a completed run is evidence the rule matched the expected contract.",
|
|
),
|
|
check(
|
|
"build.deployed-image-is-digest-pinned",
|
|
"The running agent image is pinned by digest, not by a mutable tag",
|
|
"build",
|
|
"names_present",
|
|
[
|
|
step(
|
|
"image",
|
|
OPERATOR,
|
|
"kubectl",
|
|
"--namespace",
|
|
targets.namespace,
|
|
"get",
|
|
f"deploy/{targets.agent_deployment}",
|
|
"-o",
|
|
'jsonpath={range .spec.template.spec.containers[*]}{.image}{"\\n"}{end}',
|
|
)
|
|
],
|
|
{"step": "image", "contains": ("@sha256:",)},
|
|
),
|
|
]
|
|
|
|
|
|
def _reliability(targets: Targets) -> list[CheckSpec]:
|
|
return [
|
|
check(
|
|
"reliability.finalization-and-replay-patches-are-deployed",
|
|
"The agent workload runs the finalization, replay, and session patches",
|
|
"reliability",
|
|
"names_present",
|
|
[
|
|
step(
|
|
"init",
|
|
OPERATOR,
|
|
"kubectl",
|
|
"--namespace",
|
|
targets.namespace,
|
|
"get",
|
|
f"deploy/{targets.agent_deployment}",
|
|
"-o",
|
|
INIT_NAME_PROJECTION,
|
|
)
|
|
],
|
|
{"step": "init", "names": targets.agent_init_containers},
|
|
rationale="The regression suites live in the repository; this asserts the fixes they cover are actually running.",
|
|
),
|
|
check(
|
|
"reliability.regression-suites-are-present-on-main",
|
|
"The decomposition and worker-recovery regression suites are on main",
|
|
"reliability",
|
|
"names_present",
|
|
[
|
|
step(
|
|
"suites",
|
|
OPERATOR,
|
|
"git",
|
|
"ls-tree",
|
|
"--name-only",
|
|
f"{targets.remote}/main",
|
|
"testing/tests/",
|
|
)
|
|
],
|
|
{
|
|
"step": "suites",
|
|
"names": (
|
|
"testing/tests/test_hermes_cli_lanes.py",
|
|
"testing/tests/test_hermes_coordinator.py",
|
|
"testing/tests/test_hermes_worker_recovery.py",
|
|
),
|
|
},
|
|
),
|
|
]
|
|
|
|
|
|
def delivery_checks(targets: Targets) -> list[CheckSpec]:
|
|
"""Return the forge, node, build, and reliability checks."""
|
|
return [*_forge(targets), *_nodes(targets), *_build(targets), *_reliability(targets)]
|