472 lines
15 KiB
Python
472 lines
15 KiB
Python
"""Contracts for handoff acceptance orchestration: vantages, catalog, report."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import datetime as dt
|
|
import subprocess
|
|
from dataclasses import replace
|
|
|
|
from testing.tests.test_hermes_handoff_support import (
|
|
FakeClock,
|
|
FakeSpawn,
|
|
load_handoff_module,
|
|
step,
|
|
)
|
|
|
|
harness_run = load_handoff_module("hermes_handoff_run")
|
|
catalog = load_handoff_module("hermes_handoff_catalog")
|
|
exec_module = load_handoff_module("hermes_handoff_exec")
|
|
model = load_handoff_module("hermes_handoff_model")
|
|
|
|
NOW = dt.datetime(2026, 8, 17, 12, 0, 0, tzinfo=dt.timezone.utc)
|
|
TARGETS = catalog.Targets(now=NOW)
|
|
|
|
|
|
def completed(stdout: str = "", stderr: str = "", returncode: int = 0):
|
|
return subprocess.CompletedProcess(
|
|
args=[], returncode=returncode, stdout=stdout, stderr=stderr
|
|
)
|
|
|
|
|
|
def whoami(username: str) -> object:
|
|
return completed(stdout=username)
|
|
|
|
|
|
def runner_for(results: list, **kwargs):
|
|
return exec_module.Runner(
|
|
clock=FakeClock(),
|
|
spawn=FakeSpawn(results),
|
|
environ={},
|
|
deadline_seconds=3600,
|
|
attestor=lambda command, _environment: exec_module.Attestation(
|
|
command, "a" * 64
|
|
),
|
|
**kwargs,
|
|
)
|
|
|
|
|
|
def check(
|
|
rule: str,
|
|
expect: dict | None = None,
|
|
steps=(),
|
|
identifier="test.check",
|
|
mandatory=True,
|
|
):
|
|
return model.CheckSpec(
|
|
id=identifier,
|
|
title="t",
|
|
group="g",
|
|
rule=rule,
|
|
steps=tuple(steps),
|
|
expect=expect or {},
|
|
mandatory=mandatory,
|
|
)
|
|
|
|
|
|
def test_the_shipped_catalog_is_valid_and_covers_every_group() -> None:
|
|
specs = harness_run.build_catalog(TARGETS)
|
|
assert harness_run.validate_catalog(specs) == []
|
|
groups = {spec.group for spec in specs}
|
|
assert {
|
|
"baseline",
|
|
"identity",
|
|
"routing",
|
|
"scopes",
|
|
"access-denied",
|
|
"gitops",
|
|
} <= groups
|
|
assert {
|
|
"forge",
|
|
"nodes",
|
|
"build",
|
|
"reliability",
|
|
"pool",
|
|
"surfaces",
|
|
"ephemeral",
|
|
} <= groups
|
|
|
|
|
|
def test_catalog_validation_names_every_structural_problem() -> None:
|
|
broken = [
|
|
check("no-such-rule", identifier="a.dup"),
|
|
check("allowed", identifier="a.dup"),
|
|
check("allowed", steps=(step("s", vantage="nowhere"),), identifier="b.vantage"),
|
|
check("allowed", steps=(step("s", kind="guess"),), identifier="c.kind"),
|
|
check(
|
|
"denied", steps=(step("s", kind=model.REVIEW),), identifier="d.review-only"
|
|
),
|
|
model.CheckSpec(
|
|
id="e.impersonation",
|
|
title="t",
|
|
group="g",
|
|
rule="allowed",
|
|
steps=(
|
|
model.Step(
|
|
key="s",
|
|
vantage=catalog.OPERATOR,
|
|
argv=("kubectl", "get", "ns", "--as", "x"),
|
|
),
|
|
),
|
|
),
|
|
]
|
|
|
|
problems = " | ".join(harness_run.validate_catalog(broken))
|
|
|
|
assert "duplicate check id" in problems
|
|
assert "unknown rule" in problems
|
|
assert "unknown vantage" in problems
|
|
assert "unknown kind" in problems
|
|
assert "needs a real attempt" in problems
|
|
assert "impersonates from the operator vantage" in problems
|
|
|
|
|
|
def test_target_validation_rejects_every_vacuous_or_unbound_release_input() -> None:
|
|
broken = replace(
|
|
TARGETS,
|
|
repo="atlas/other",
|
|
remote="upstream",
|
|
reviewed_pr_number=20,
|
|
reviewed_head_ref="other",
|
|
baseline_commit="bad",
|
|
remote_main_sha="bad",
|
|
reviewed_head_sha="bad",
|
|
build_sha="bad",
|
|
agent_image="latest",
|
|
deployment_revision="0",
|
|
node_count=1,
|
|
pool_replicas=1,
|
|
expect_telegram_sessions=False,
|
|
chat_config_revision="",
|
|
pool_worker_env=(),
|
|
dependency_heads=((14, "bad"), (14, "bad")),
|
|
max_evidence_age_seconds=0,
|
|
)
|
|
problems = harness_run.validate_targets(broken)
|
|
assert len(problems) >= 12
|
|
sha = "1" * 40
|
|
valid = replace(
|
|
TARGETS,
|
|
remote_main_sha=sha,
|
|
reviewed_head_sha=sha,
|
|
build_sha=sha,
|
|
agent_image=f"registry.example/hermes:git-{sha}-build-7@sha256:{'2' * 64}",
|
|
deployment_revision="1",
|
|
chat_config_revision="rev",
|
|
dependency_heads=tuple(
|
|
(number, sha) for number in TARGETS.dependency_pull_requests
|
|
),
|
|
)
|
|
assert harness_run.validate_targets(valid) == []
|
|
wrong_image_source = replace(
|
|
valid,
|
|
agent_image=f"registry.example/hermes:git-{'2' * 40}-build-7@sha256:{'2' * 64}",
|
|
)
|
|
assert "image tag/digest" in " ".join(
|
|
harness_run.validate_targets(wrong_image_source)
|
|
)
|
|
wrong_build = replace(valid, build_sha="2" * 40)
|
|
assert "build SHA" in " ".join(harness_run.validate_targets(wrong_build))
|
|
malformed_heads = replace(
|
|
valid,
|
|
dependency_heads=tuple(
|
|
(number, "bad") for number in valid.dependency_pull_requests
|
|
),
|
|
)
|
|
assert "dependency heads" in " ".join(harness_run.validate_targets(malformed_heads))
|
|
|
|
|
|
def test_identity_resolution_extracts_only_the_username() -> None:
|
|
runner = runner_for([whoami("system:serviceaccount:hermes:hermes-agent")])
|
|
username, detail = harness_run.resolve_identity(
|
|
runner, exec_module.operator_vantage()
|
|
)
|
|
assert (username, detail) == ("system:serviceaccount:hermes:hermes-agent", "")
|
|
|
|
failed = harness_run.resolve_identity(
|
|
runner_for([completed(stderr="no access", returncode=1)]),
|
|
exec_module.operator_vantage(),
|
|
)
|
|
assert failed[0] == "" and "no access" in failed[1]
|
|
|
|
unparsable = harness_run.resolve_identity(
|
|
runner_for([completed(stdout="two names")]), exec_module.operator_vantage()
|
|
)
|
|
assert unparsable[0] == "" and "no username" in unparsable[1]
|
|
|
|
nameless = harness_run.resolve_identity(
|
|
runner_for([completed(stdout="")]), exec_module.operator_vantage()
|
|
)
|
|
assert nameless[0] == "" and "no username" in nameless[1]
|
|
|
|
|
|
def test_vantages_resolve_to_running_pods_and_record_what_is_missing() -> None:
|
|
runner = runner_for(
|
|
[
|
|
whoami("kubernetes-admin"),
|
|
completed(stdout="pod/hermes-agent-2\npod/hermes-agent-1\n"),
|
|
whoami("system:serviceaccount:hermes:hermes-agent"),
|
|
completed(stdout="pod/hermes-switchyard-1\n"),
|
|
completed(stderr="no pods", returncode=1),
|
|
completed(stdout="pod/hermes-chat-tenant-0\n"),
|
|
]
|
|
)
|
|
|
|
vantages, records = harness_run.resolve_vantages(runner, TARGETS)
|
|
by_name = {record.name: record for record in records}
|
|
|
|
assert set(vantages) == {
|
|
catalog.OPERATOR,
|
|
catalog.SELF,
|
|
catalog.SWITCHYARD,
|
|
catalog.CHAT,
|
|
}
|
|
assert by_name[catalog.SELF].identity == "system:serviceaccount:hermes:hermes-agent"
|
|
assert "hermes-agent-1" in by_name[catalog.SELF].description # deterministic pick
|
|
assert by_name[catalog.NODE].available is False
|
|
assert by_name[catalog.CHAT].available is True
|
|
|
|
|
|
def test_an_unreachable_operator_stops_vantage_resolution_immediately() -> None:
|
|
runner = runner_for([completed(stderr="connection refused", returncode=1)])
|
|
|
|
vantages, records = harness_run.resolve_vantages(runner, TARGETS)
|
|
|
|
assert set(vantages) == {catalog.OPERATOR}
|
|
assert records[0].available is False
|
|
assert harness_run.vantage_problems(records)[0].startswith(
|
|
"the operator vantage is unavailable"
|
|
)
|
|
|
|
|
|
def test_a_missing_chat_pod_is_recorded_rather_than_assumed_present() -> None:
|
|
runner = runner_for(
|
|
[
|
|
whoami("kubernetes-admin"),
|
|
completed(stdout="pod/hermes-agent-1\n"),
|
|
whoami("system:serviceaccount:hermes:hermes-agent"),
|
|
completed(stdout="pod/hermes-switchyard-1\n"),
|
|
completed(stdout="pod/hermes-node-1\n"),
|
|
completed(stderr="pods 'hermes-chat-tenant-0' not found", returncode=1),
|
|
]
|
|
)
|
|
|
|
vantages, records = harness_run.resolve_vantages(runner, TARGETS)
|
|
chat = next(record for record in records if record.name == catalog.CHAT)
|
|
|
|
assert catalog.CHAT not in vantages
|
|
assert chat.available is False and "not found" in chat.detail
|
|
|
|
|
|
def test_a_selector_that_matches_no_running_pod_leaves_the_vantage_absent() -> None:
|
|
runner = runner_for(
|
|
[
|
|
whoami("kubernetes-admin"),
|
|
completed(stdout="\n"),
|
|
completed(stdout="\n"),
|
|
completed(stdout="\n"),
|
|
completed(stdout="\n"),
|
|
]
|
|
)
|
|
|
|
vantages, records = harness_run.resolve_vantages(runner, TARGETS)
|
|
by_name = {record.name: record for record in records}
|
|
|
|
assert set(vantages) == {catalog.OPERATOR}
|
|
assert "no running pod matches" in by_name[catalog.SELF].detail
|
|
|
|
|
|
def test_two_vantages_that_are_the_same_principal_invalidate_the_evidence() -> None:
|
|
"""Corroboration needs two identities; one identity twice is a restatement."""
|
|
records = [
|
|
model.VantageRecord(
|
|
name=catalog.OPERATOR, description="d", identity="same", available=True
|
|
),
|
|
model.VantageRecord(
|
|
name=catalog.SELF, description="d", identity="same", available=True
|
|
),
|
|
model.VantageRecord(name=catalog.SWITCHYARD, description="d", available=True),
|
|
model.VantageRecord(name=catalog.CHAT, description="d", available=True),
|
|
]
|
|
|
|
problems = harness_run.vantage_problems(records)
|
|
|
|
assert len(problems) == 1
|
|
assert "same principal" in problems[0]
|
|
|
|
|
|
def test_vantage_problems_flag_a_missing_or_anonymous_self_probe() -> None:
|
|
operator = model.VantageRecord(
|
|
name=catalog.OPERATOR, description="d", identity="admin", available=True
|
|
)
|
|
|
|
absent = harness_run.vantage_problems([operator])
|
|
assert "in-pod Hermes vantage is unavailable" in absent[0]
|
|
|
|
anonymous = harness_run.vantage_problems(
|
|
[
|
|
operator,
|
|
model.VantageRecord(name=catalog.SELF, description="d", available=True),
|
|
]
|
|
)
|
|
assert "did not report an identity" in anonymous[0]
|
|
|
|
clean = harness_run.vantage_problems(
|
|
[
|
|
operator,
|
|
model.VantageRecord(
|
|
name=catalog.SELF, description="d", identity="agent", available=True
|
|
),
|
|
model.VantageRecord(
|
|
name=catalog.SWITCHYARD, description="d", available=True
|
|
),
|
|
model.VantageRecord(name=catalog.CHAT, description="d", available=True),
|
|
]
|
|
)
|
|
assert clean == []
|
|
|
|
|
|
def test_a_check_whose_vantage_is_absent_is_not_run() -> None:
|
|
spec = check("allowed", steps=(step("s", vantage=catalog.NODE),))
|
|
|
|
result = harness_run.run_check(runner_for([]), spec, {})
|
|
|
|
assert result.status == model.NOT_RUN
|
|
assert "vantage node is unavailable" in result.reason
|
|
assert result.outcomes[0].error == "vantage node is unavailable"
|
|
|
|
|
|
def test_bulk_evidence_is_evaluated_but_replaced_before_it_is_recorded() -> None:
|
|
spec = check(
|
|
"stdout_matches",
|
|
{"step": "s", "equals": "locked"},
|
|
steps=(
|
|
model.Step(
|
|
key="s",
|
|
vantage=catalog.OPERATOR,
|
|
argv=("kubectl", "version"),
|
|
record=False,
|
|
),
|
|
),
|
|
)
|
|
runner = runner_for([completed(stdout="locked", stderr="noise")])
|
|
|
|
result = harness_run.run_check(
|
|
runner, spec, {catalog.OPERATOR: exec_module.operator_vantage()}
|
|
)
|
|
|
|
assert result.status == model.PASS
|
|
assert result.outcomes[0].stdout == harness_run.UNRECORDED
|
|
assert result.outcomes[0].stderr == harness_run.UNRECORDED
|
|
|
|
|
|
def test_an_expired_deadline_leaves_the_report_saying_so() -> None:
|
|
clock = FakeClock()
|
|
runner = exec_module.Runner(
|
|
clock=clock,
|
|
spawn=FakeSpawn([]),
|
|
environ={},
|
|
deadline_seconds=1.0,
|
|
attestor=lambda command, _environment: exec_module.Attestation(
|
|
command, "a" * 64
|
|
),
|
|
)
|
|
clock.advance(5.0)
|
|
spec = check("allowed", steps=(step("s", vantage=catalog.OPERATOR),))
|
|
|
|
report = harness_run.build_report(
|
|
runner,
|
|
TARGETS,
|
|
[spec],
|
|
{catalog.OPERATOR: exec_module.operator_vantage()},
|
|
[],
|
|
"read-only",
|
|
"2026-08-17T00:00:00Z",
|
|
)
|
|
|
|
assert report.results[0].status == model.NOT_RUN
|
|
assert any("deadline expired" in error for error in report.harness_errors)
|
|
assert report.decision == model.NO_GO
|
|
|
|
|
|
def test_the_report_carries_the_configured_baseline_and_sorts_its_checks() -> None:
|
|
runner = runner_for([completed(stdout="ok"), completed(stdout="ok")])
|
|
specs = [
|
|
check(
|
|
"allowed", steps=(step("s", vantage=catalog.OPERATOR),), identifier="z.last"
|
|
),
|
|
check(
|
|
"allowed",
|
|
steps=(step("s", vantage=catalog.OPERATOR),),
|
|
identifier="a.first",
|
|
),
|
|
]
|
|
records = [
|
|
model.VantageRecord(
|
|
name=catalog.OPERATOR, description="d", identity="admin", available=True
|
|
),
|
|
model.VantageRecord(
|
|
name=catalog.SELF, description="d", identity="agent", available=True
|
|
),
|
|
model.VantageRecord(name=catalog.SWITCHYARD, description="d", available=True),
|
|
model.VantageRecord(name=catalog.CHAT, description="d", available=True),
|
|
]
|
|
|
|
report = harness_run.build_report(
|
|
runner,
|
|
TARGETS,
|
|
specs,
|
|
{catalog.OPERATOR: exec_module.operator_vantage()},
|
|
records,
|
|
"read-only",
|
|
"2026-08-17T00:00:00Z",
|
|
)
|
|
|
|
assert [result.spec.id for result in report.results] == ["a.first", "z.last"]
|
|
assert report.baseline["baseline_commit"] == TARGETS.baseline_commit
|
|
assert report.baseline["dependency_pull_requests"] == list(
|
|
TARGETS.dependency_pull_requests
|
|
)
|
|
assert report.decision == model.GO
|
|
assert report.finished_at
|
|
|
|
|
|
def test_armed_results_replace_the_matching_catalog_placeholders() -> None:
|
|
placeholder = check(
|
|
"not_armed", identifier="ephemeral.feature-branch-push", mandatory=False
|
|
)
|
|
armed = model.CheckResult(
|
|
spec=model.CheckSpec(
|
|
id="ephemeral.feature-branch-push",
|
|
title="t",
|
|
group="ephemeral",
|
|
rule="armed",
|
|
),
|
|
status=model.PASS,
|
|
reason="pushed",
|
|
)
|
|
|
|
report = harness_run.build_report(
|
|
runner_for([]),
|
|
TARGETS,
|
|
[placeholder],
|
|
{},
|
|
[],
|
|
"ephemeral-armed",
|
|
"2026-08-17T00:00:00Z",
|
|
[armed],
|
|
)
|
|
|
|
assert len(report.results) == 1
|
|
assert report.results[0].status == model.PASS
|
|
|
|
|
|
def test_an_unstartable_run_reports_no_go_with_a_named_reason() -> None:
|
|
report = harness_run.unavailable_report(
|
|
"read-only", "2026-08-17T00:00:00Z", "arming refused: bad phrase", TARGETS
|
|
)
|
|
|
|
assert report.decision == model.NO_GO
|
|
assert report.results[0].spec.id == "harness.startup"
|
|
assert report.results[0].status == model.NOT_RUN
|
|
assert report.harness_errors == ["arming refused: bad phrase"]
|