Hermes Agent
8f00545828
Decides whether the Hermes platform handoff is fit to release, and refuses to round an absence of evidence up to a pass. The harness is read-only by default and classifies 71 checks PASS / FAIL / NOT_RUN / NOT_APPLICABLE. Any mandatory FAIL or NOT_RUN is NO_GO, and so is a harness-level problem: an unreachable vantage, a catalog entry whose evidence no longer exists, an expired deadline, or an evaluator that raised. Evidence comes from two vantages that cannot cover for each other: an external read-only operator kubeconfig, and the Hermes agent probing itself from inside its own pod. Before any check runs, the harness asks each vantage who it is and stops if they are the same principal, because dual-vantage evidence from one identity is a restatement rather than a corroboration. `--as` is rejected for every operator-side command and reachable only as the inner command of a `kubectl exec`, so impersonation can never stand in for a real self-probe. A deny check needs a live refused request, not only an authorization review. Two safety properties are structural rather than conventional, enforced where an argv becomes a subprocess: the default mode mutates nothing (mutating verbs require a server dry run; there is deliberately no live TokenRequest probe, because a successful one would mint a real credential), and no probe can pull a credential value into a report (no vault/sops/curl, secrets readable only with -o name, environment probes list names, shell only through frozen reviewed templates). Captures are bounded before they are screened, and the rendered report is re-screened before it is written. Mutation lives behind a separate arming flag with an exact confirmation phrase, a caller-supplied unique ref, a preflight that refuses a protected push target before any network call, and a cleanup whose verification is itself mandatory. A default run reports those four checks NOT_RUN. The catalog is declarative so a reviewer reads what is asserted rather than how it is plumbed, and so structural properties can be proven over every entry before a run. Catalog drift surfaces as NOT_RUN, which stops the release. docs/hermes_full_handoff_acceptance.md carries the merge order for PRs #14-#18 on top of the merged #13 baseline, the image build and Flux rollout, the rollback point for each step, the go/no-go checklist, and the limits that are asserted rather than exercised. Validation: 295 handoff tests pass with 100% line coverage on all 15 new modules; the full unit suite is 647 passed with two failures that reproduce unchanged on origin/main; Ruff, py_compile, kustomize render, and a diff credential screen are clean; a live read-only run against Atlas returns NO_GO for the pre-merge cluster with no unscreened fields in the report.
174 lines
6.1 KiB
Python
174 lines
6.1 KiB
Python
"""Contracts for bounded, screened command execution in the handoff harness."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import subprocess
|
|
|
|
from testing.tests.test_hermes_handoff_support import (
|
|
FakeClock,
|
|
FakeSpawn,
|
|
load_handoff_module,
|
|
)
|
|
|
|
runner_module = load_handoff_module("hermes_handoff_exec")
|
|
policy = load_handoff_module("hermes_handoff_policy")
|
|
|
|
|
|
def completed(stdout: str = "", stderr: str = "", returncode: int = 0):
|
|
return subprocess.CompletedProcess(args=[], returncode=returncode, stdout=stdout, stderr=stderr)
|
|
|
|
|
|
def build(results: list, **kwargs) -> tuple:
|
|
clock = FakeClock()
|
|
spawn = FakeSpawn(results)
|
|
runner = runner_module.Runner(clock=clock, spawn=spawn, environ={}, **kwargs)
|
|
return (runner, spawn, clock)
|
|
|
|
|
|
OPERATOR = runner_module.operator_vantage()
|
|
|
|
|
|
def test_a_successful_command_is_captured_and_screened() -> None:
|
|
runner, spawn, _ = build([completed(stdout="password=a-long-enough-secret\n")])
|
|
|
|
outcome = runner.run(("kubectl", "get", "pods"), OPERATOR)
|
|
|
|
assert outcome.ok and outcome.ran
|
|
assert "[redacted]" in outcome.stdout
|
|
assert outcome.returncode == 0
|
|
assert spawn.calls[0]["argv"] == ["kubectl", "get", "pods"]
|
|
assert runner.commands_run == 1
|
|
|
|
|
|
def test_a_failing_command_is_not_ok_but_still_ran() -> None:
|
|
runner, _, _ = build([completed(stderr="Error from server (Forbidden)", returncode=1)])
|
|
|
|
outcome = runner.run(("kubectl", "get", "pods"), OPERATOR)
|
|
|
|
assert outcome.ran and not outcome.ok
|
|
assert "Forbidden" in outcome.combined
|
|
|
|
|
|
def test_a_policy_violation_never_reaches_a_subprocess() -> None:
|
|
runner, spawn, _ = build([])
|
|
|
|
outcome = runner.run(("kubectl", "delete", "pods", "--all"), OPERATOR)
|
|
|
|
assert outcome.error and outcome.error.startswith("policy:")
|
|
assert not outcome.ran
|
|
assert spawn.calls == []
|
|
assert runner.commands_run == 0
|
|
|
|
|
|
def test_a_timeout_is_recorded_rather_than_raised() -> None:
|
|
runner, _, _ = build([subprocess.TimeoutExpired(cmd="kubectl", timeout=1.0)])
|
|
|
|
outcome = runner.run(("kubectl", "get", "pods"), OPERATOR)
|
|
|
|
assert outcome.error == runner_module.TIMEOUT_ERROR
|
|
assert not outcome.ran
|
|
|
|
|
|
def test_a_missing_binary_is_recorded_rather_than_raised() -> None:
|
|
runner, _, _ = build([FileNotFoundError(2, "No such file or directory")])
|
|
|
|
outcome = runner.run(("kubectl", "get", "pods"), OPERATOR)
|
|
|
|
assert outcome.error and outcome.error.startswith("spawn:")
|
|
|
|
|
|
def test_the_run_deadline_stops_further_probes_without_spawning() -> None:
|
|
runner, spawn, clock = build([completed(stdout="first")], deadline_seconds=10.0)
|
|
|
|
assert runner.run(("kubectl", "get", "pods"), OPERATOR).ok
|
|
clock.advance(11.0)
|
|
second = runner.run(("kubectl", "get", "nodes"), OPERATOR)
|
|
|
|
assert second.error == runner_module.DEADLINE_ERROR
|
|
assert len(spawn.calls) == 1
|
|
assert runner.remaining_seconds < 0
|
|
|
|
|
|
def test_the_command_timeout_is_clamped_by_the_remaining_deadline() -> None:
|
|
runner, spawn, clock = build([completed()], command_timeout=30.0, deadline_seconds=10.0)
|
|
clock.advance(6.0)
|
|
|
|
runner.run(("kubectl", "get", "pods"), OPERATOR)
|
|
|
|
assert spawn.calls[0]["timeout"] == 4.0
|
|
|
|
|
|
def test_output_is_bounded_per_command_and_can_be_raised_per_call() -> None:
|
|
runner, _, _ = build([completed(stdout="y" * 200), completed(stdout="y" * 200)], max_bytes=50)
|
|
|
|
small = runner.run(("kubectl", "get", "pods"), OPERATOR)
|
|
large = runner.run(("kubectl", "get", "pods"), OPERATOR, max_bytes=4096)
|
|
|
|
assert small.truncated
|
|
assert not large.truncated
|
|
|
|
|
|
def test_the_child_environment_is_rebuilt_from_an_allowlist() -> None:
|
|
source = {"PATH": "/bin", "KUBECONFIG": "/tmp/kc", "ANTHROPIC_API_KEY": "leak", "HOME": "/root"}
|
|
environment = runner_module.build_environment(OPERATOR, source)
|
|
|
|
assert environment == {"PATH": "/bin", "KUBECONFIG": "/tmp/kc", "HOME": "/root", "LC_ALL": "C"}
|
|
assert "ANTHROPIC_API_KEY" not in environment
|
|
|
|
|
|
def test_a_vantage_can_override_the_environment_it_needs() -> None:
|
|
vantage = runner_module.operator_vantage(kubeconfig="/tmp/operator.yaml")
|
|
environment = runner_module.build_environment(vantage, {"KUBECONFIG": "/tmp/other"})
|
|
|
|
assert environment["KUBECONFIG"] == "/tmp/operator.yaml"
|
|
assert "kubeconfig-pinned" in vantage.description
|
|
|
|
|
|
def test_the_operator_context_is_inserted_only_for_context_aware_tools() -> None:
|
|
vantage = runner_module.operator_vantage(context="atlas-operator")
|
|
|
|
assert vantage.wrap(("kubectl", "get", "pods"))[:3] == ("kubectl", "--context", "atlas-operator")
|
|
assert vantage.wrap(("git", "status")) == ("git", "status")
|
|
assert vantage.wrap(()) == ()
|
|
|
|
|
|
def test_the_pod_vantage_execs_into_the_container_and_drops_the_context() -> None:
|
|
operator = runner_module.operator_vantage(context="atlas-operator")
|
|
vantage = runner_module.pod_vantage("hermes", "hermes-agent-1", "hermes", operator)
|
|
|
|
wrapped = vantage.wrap(("kubectl", "get", "pods"))
|
|
|
|
assert wrapped[:3] == ("kubectl", "--context", "atlas-operator")
|
|
assert wrapped[3] == "exec"
|
|
assert wrapped[-4:] == ("--", "kubectl", "get", "pods")
|
|
assert vantage.description == "in-pod hermes/hermes-agent-1[hermes]"
|
|
policy.check_argv(wrapped)
|
|
|
|
|
|
def test_a_pod_vantage_without_an_operator_still_addresses_the_pod() -> None:
|
|
vantage = runner_module.pod_vantage("hermes", "pod-a", "hermes")
|
|
|
|
assert vantage.wrap(("kubectl", "version"))[0] == "kubectl"
|
|
assert vantage.env == {}
|
|
|
|
|
|
def test_outcome_serialisation_screens_the_command_line() -> None:
|
|
outcome = runner_module.Outcome(
|
|
argv=("git", "clone", "https://user:a-long-enough-secret@scm.example.dev/x"),
|
|
vantage="operator",
|
|
returncode=0,
|
|
stdout="ok",
|
|
)
|
|
|
|
payload = outcome.as_dict()
|
|
|
|
assert "a-long-enough-secret" not in payload["command"]
|
|
assert payload["vantage"] == "operator"
|
|
assert payload["stdout"] == "ok"
|
|
|
|
|
|
def test_combined_output_joins_only_the_streams_that_carry_text() -> None:
|
|
assert runner_module.Outcome(argv=(), vantage="x", stdout="a", stderr="b").combined == "a\nb"
|
|
assert runner_module.Outcome(argv=(), vantage="x", stdout="a").combined == "a"
|
|
assert runner_module.Outcome(argv=(), vantage="x").combined == ""
|