titan/atlas-iac
3
0
Fork 0
Code Issues Pull Requests 9 Actions Packages Projects Releases Wiki Activity
atlas-iac/testing/tests/test_hermes_hux_release_producer.py
jenkins 469e52fd20 hermes(hux): add the HUX-12 release evidence producer 
A companion package (outside the network-free hux/ service package)
that independently verifies and binds the whole release chain before
any transition: reviewed proposal URL, Jenkins job/build/result and
revision, immutable Harbor tag/digest equality, Flux kustomization and
applied revision with pin containment, desired workload image, every
Ready pod imageID, bounded-age health receipt, and rollback target.
Pure injectable verifier core, HTTPS-only collectors (SA token for the
Kubernetes API), and an evidence-trust driver that posts exactly one
If-Match transition with deterministic idempotency. Rejects stale,
replayed, downgraded, incomplete, cross-workload, mismatched, and
self-asserted evidence. 100% line and branch coverage (71 tests).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BvMSXH8VH2tMWXanb8SJdf
2026-08-24 04:35:12 -03:00

479 lines
31 KiB
Python
Raw Blame History

"""HUX-12 release evidence producer: verifier core, collectors and driver."""
from __future__ import annotations
import io
import json
import sys
import threading
import urllib.error
import urllib.request
from datetime import datetime, timedelta, timezone
from email import message_from_string
from pathlib import Path
from types import SimpleNamespace
import pytest
ROOT = Path(__file__).resolve().parents[2]
FOUNDATION = ROOT / "dockerfiles" / "hermes-hux-foundation"
if str(FOUNDATION) not in sys.path:
sys.path.insert(0, str(FOUNDATION))
import hux_producer as release_producer # noqa: E402
from hux import contracts # noqa: E402
from hux.errors import Invalid # noqa: E402
from hux.http import serve # noqa: E402
from hux.server import build_router # noqa: E402
ALL_ON = ",".join(card["flag"] for card in contracts.load_flags()["cards"])
KEY = "evidence-key-value-that-is-at-least-32-bytes"
SUBJECT = "usr_0123456789abcdef"
SHA_M = "a" * 40
DIGEST = "sha256:" + "d" * 64
OTHER_DIGEST = "sha256:" + "e" * 64
REPO = "registry.bstein.dev/bstein/hermes-webui"
JOB = "https://jenkins.bstein.dev/job/hermes-webui-image"
PREFIX = "https://scm.bstein.dev/atlas/titan-iac/pulls/"
REVIEW = PREFIX + "55"
HEALTH = "https://chat.bstein.dev/healthz"
TAG = f"git-{SHA_M}-build-20-release"
REF = f"{REPO}:{TAG}@{DIGEST}"
FLUX_REV = "main@sha1:" + "b" * 40
WP = {"review_url_prefix": PREFIX, "jenkins_job_url": JOB, "image_repository": REPO, "flux_kustomization": "hermes", "health_url": HEALTH}
POLICY = {"schema": "hux.release_evidence_policy.v1", "max_evidence_age_seconds": 900, "workloads": {"hermes-webui": WP}}
FIXED = datetime(2026, 8, 24, 12, 0, 0, tzinfo=timezone.utc)
NOW_S = "2026-08-24T12:00:00Z"
OLD_S = "2026-08-24T10:00:00Z"
GIT_API = "https://scm.bstein.dev/api/v1/repos/atlas/titan-iac/pulls/55"
JENKINS_API = JOB + "/lastSuccessfulBuild/api/json"
HARBOR_API = f"https://registry.bstein.dev/api/v2.0/projects/bstein/repositories/hermes-webui/artifacts/{TAG}"
KUSTOMIZATION_API = "https://kube.test/apis/kustomize.toolkit.fluxcd.io/v1/namespaces/hermes/kustomizations/hermes"
STS_API = "https://kube.test/apis/apps/v1/namespaces/hermes/statefulsets/hermes-webui"
PODS_API = "https://kube.test/api/v1/namespaces/hermes/pods?labelSelector=app=hermes-webui"
ROUTER_HEADERS = {"X-Hermes-Tenant-Identity": "slot-3", "X-Hux-Subject": SUBJECT, "X-Hux-Surface": "chat", "X-Hux-Relay-Key": "rk"}
CA_PEM = """-----BEGIN CERTIFICATE-----
MIIBgTCCASegAwIBAgIUNjZXQ5S3lIfyo4QEuwB6CmxthPwwCgYIKoZIzj0EAwIw
FjEUMBIGA1UEAwwLaHV4LXRlc3QtY2EwHhcNMjYwODI0MDcyMTQzWhcNMzYwODIx
MDcyMTQzWjAWMRQwEgYDVQQDDAtodXgtdGVzdC1jYTBZMBMGByqGSM49AgEGCCqG
SM49AwEHA0IABKUXVC/hmAAhDxIT8HrMvXH/N67OB6zEFKnaW24bg907WXzKqQO6
eSJzPPznQvzuivQtIG6D+39amDg8utWGERijUzBRMB0GA1UdDgQWBBQcnGVz8rJ+
MJrVZ3geD32daLpJBzAfBgNVHSMEGDAWgBQcnGVz8rJ+MJrVZ3geD32daLpJBzAP
BgNVHRMBAf8EBTADAQH/MAoGCCqGSM49BAMCA0gAMEUCIQDVLn3BP3aUfBVdmzck
xydrFRXApxVn0NpQ5v7DBZvZXwIgKibREmX5TKcB9YZyrHaddGTjfPSfREOhxaQQ
nMKqK1Q=
-----END CERTIFICATE-----
"""
def obs(**fields):
return {"workload": "hermes-webui", "observed_at": NOW_S, **fields}
def rel(state, **evidence):
return {"workload": "hermes-webui", "state": state, "evidence": evidence}
GIT_OBS = obs(commit=SHA_M, committed_at="2026-08-24T11:00:00Z", review_url=REVIEW)
JENKINS_OBS = obs(build_url=JOB + "/20", result="SUCCESS", build_number=20, revision=SHA_M, image_digest=DIGEST)
HARBOR_OBS = obs(repository=REPO, tag=TAG, digest=DIGEST)
FLUX_OBS = obs(name="hermes", applied_revision=FLUX_REV, pin_in_revision=True)
SPEC_OBS = obs(image=REF)
PODS_OBS = obs(pods=[{"name": "p0", "ready": True, "image_id": REPO + "@" + DIGEST}])
HEALTH_OBS = obs(url=HEALTH, status=200, checked_at=NOW_S)
BUILT_EV = {"review_url": REVIEW, "merge_commit": SHA_M, "ci_build_url": JOB + "/20", "image_ref": REF, "image_digest": DIGEST, "harbor_digest": DIGEST}
class FakeResponse:
def __init__(self, status, payload, headers=None):
self.status = status
self._body = payload if isinstance(payload, bytes) else json.dumps(payload).encode()
self.headers = headers or {}
def read(self, limit=None):
return self._body
def __enter__(self):
return self
def __exit__(self, *exc):
return False
class FakeOpener:
def __init__(self, routes):
self.routes = dict(routes)
self.calls = []
def __call__(self, request, timeout, context):
url = request.full_url
self.calls.append((request.get_method(), url))
found = self.routes.get(url)
if found is None:
raise urllib.error.URLError(f"no fake route: {url}")
if isinstance(found, Exception):
raise found
status, payload = found
if status >= 400:
raise urllib.error.HTTPError(url, status, "error", message_from_string(""), io.BytesIO(b"{}"))
return FakeResponse(status, payload)
def good_routes():
return {
GIT_API: (200, {"merged": True, "merge_commit_sha": SHA_M.upper(), "merged_at": "2026-08-24T11:00:00Z"}),
JENKINS_API: (200, {"url": JOB + "/20/", "result": "SUCCESS", "number": 20, "description": f"pushed {DIGEST}",
"actions": [None, {"other": 1}, {"lastBuiltRevision": {"SHA1": SHA_M.upper()}}]}),
HARBOR_API: (200, {"digest": DIGEST, "tags": ["junk", {"name": TAG}]}),
KUSTOMIZATION_API: (200, {"metadata": {"name": "hermes"}, "status": {
"lastAppliedRevision": FLUX_REV, "lastAttemptedRevision": FLUX_REV, "conditions": [{"type": "Ready", "status": "True"}]}}),
STS_API: (200, {"spec": {"template": {"spec": {"containers": ["junk", {"image": "other:1"}, {"image": REF}]}}}}),
PODS_API: (200, {"items": ["junk", {"metadata": {"name": "p0"}, "status": {
"conditions": [{"type": "Ready", "status": "True"}],
"containerStatuses": [{"image": "other:1", "imageID": "x"}, {"image": REF, "imageID": REPO + "@" + DIGEST}]}}]}),
HEALTH: (200, b""),
}
def hybrid(routes):
fake = FakeOpener(routes)
def opener(request, timeout, context):
if request.full_url.startswith("http://127.0.0.1"):
return urllib.request.urlopen(request, timeout=timeout)
return fake(request, timeout, context)
return opener
def secrets(tmp_path):
key = tmp_path / "evidence.key"
key.write_text(KEY + "\n")
key.chmod(0o400)
policy = tmp_path / "policy.json"
policy.write_text(json.dumps(POLICY))
policy.chmod(0o444)
return {"HUX_RELEASE_EVIDENCE_KEY_FILE": str(key), "HUX_RELEASE_EVIDENCE_POLICY_FILE": str(policy)}
def dispatch(router, method, path, body=None, headers=None):
raw = b"" if body is None else json.dumps(body).encode()
response = router.dispatch(method, path, {**ROUTER_HEADERS, **(headers or {})}, raw)
return response.status, response.body
def make_collector(routes, extra=None):
env = {"HUX_PRODUCER_NAMESPACE": "hermes", "HUX_PRODUCER_KUBE_API": "https://kube.test",
"HUX_PRODUCER_POD_SELECTOR": "app=hermes-webui", **(extra or {})}
return release_producer.Collector(env, "hermes-webui", WP, FakeOpener(routes), lambda: FIXED)
@pytest.fixture
def stack(tmp_path):
env = secrets(tmp_path)
router = build_router(tmp_path, {"HUX_FLAGS": ALL_ON, "HUX_ROUTER_KEY": "rk", **env})
server = serve(router, "127.0.0.1", 0)
threading.Thread(target=server.serve_forever, daemon=True).start()
base = f"http://127.0.0.1:{server.server_address[1]}"
_, project = dispatch(router, "POST", "/hux/v1/projects", {"name": "P"})
_, conversation = dispatch(router, "POST", "/hux/v1/conversations", {"title": "C", "project_id": project["id"]})
scope = f"/hux/v1/projects/{project['id']}/conversations/{conversation['id']}/releases"
token = tmp_path / "kube.token"
token.write_text("sa-token\n")
status, created = dispatch(router, "POST", scope, {"workload": "hermes-webui", "commit": "1" * 40, "evidence": {"review_url": REVIEW}},
{"If-Match": "0", "Idempotency-Key": "producer-suite-create"})
assert status == 201
env.update({"HUX_BASE_URL": base, "HUX_TENANT_SLOT": "slot-3", "HUX_PRODUCER_SUBJECT": SUBJECT,
"HUX_PRODUCER_WORKLOAD": "hermes-webui", "HUX_PRODUCER_PROJECT_ID": project["id"],
"HUX_PRODUCER_CONVERSATION_ID": conversation["id"], "HUX_PRODUCER_NAMESPACE": "hermes",
"HUX_PRODUCER_KUBE_API": "https://kube.test", "HUX_PRODUCER_KUBE_TOKEN_FILE": str(token),
"HUX_PRODUCER_POD_SELECTOR": "app=hermes-webui"})
yield {"router": router, "base": base, "scope": scope, "env": env, "release_id": created["release"]["id"]}
server.shutdown()
def producer_client(stack, opener=None):
return release_producer.ProducerClient(stack["base"], "slot-3", SUBJECT, KEY, opener)
def test_full_chain_replay_and_terminal_noop(stack):
routes = good_routes()
client = producer_client(stack)
release_id = stack["release_id"]
_, first_view, _ = client.request("GET", f"{stack['scope']}/{release_id}")
states = []
for _ in range(6):
outcome = release_producer.run_once(stack["env"], opener=hybrid(routes), clock=lambda: FIXED)
assert outcome["changed"] is True and outcome["replayed"] is False
states.append(outcome["state"])
assert states == ["merged", "built", "verified", "deployed", "converged", "live_verified"]
status, view, _ = client.request("GET", f"{stack['scope']}/{release_id}")
assert status == 200 and view["revision"] == 7
evidence = view["release"]["evidence"]
assert evidence["image_ref"] == REF and evidence["pod_digest"] == DIGEST and evidence["health_check"]["status"] == "pass"
assert release_producer.run_once(stack["env"], opener=hybrid(routes), clock=lambda: FIXED) == {"changed": False, "reason": "no advanceable release"}
by_id = release_producer.run_once({**stack["env"], "HUX_PRODUCER_RELEASE_ID": release_id}, opener=hybrid(routes), clock=lambda: FIXED)
assert by_id == {"changed": False, "release_id": release_id, "state": "live_verified", "reason": "release is terminal"}
# A duplicate POST with the producer's deterministic key replays the same entry.
status, replay, replayed = client.request("POST", f"{stack['scope']}/{release_id}/transitions",
{"to": "merged", "evidence": {"merge_commit": SHA_M}},
{"If-Match": "1", "Idempotency-Key": f"producer-{release_id}-merged-1"})
assert status == 200 and replayed is True and replay["release"]["state"] == "merged" and replay["revision"] == 2
# The driver reports a replayed transition as unchanged.
stale_get = lambda request, timeout, context: ( # noqa: E731
FakeResponse(200, first_view) if request.get_method() == "GET" else urllib.request.urlopen(request, timeout=timeout))
stub = SimpleNamespace(git=lambda url: dict(GIT_OBS))
replay_client = release_producer.ProducerClient(stack["base"], "slot-3", SUBJECT, KEY, stale_get)
outcome = release_producer.advance(replay_client, stub, WP, 900, stack["scope"], release_id, FIXED)
assert outcome["changed"] is False and outcome["replayed"] is True and outcome["state"] == "merged"
healthz = release_producer._default_opener(urllib.request.Request(stack["base"] + "/healthz"), 5.0, None)
assert healthz.status == 200
def test_driver_never_posts_when_verification_or_collection_fails(stack):
routes = good_routes()
env, scope, release_id = stack["env"], stack["scope"], stack["release_id"]
client = producer_client(stack)
assert release_producer.run_once(env, opener=hybrid(routes), clock=lambda: FIXED)["state"] == "merged"
bad = dict(routes)
bad[JENKINS_API] = (200, {**routes[JENKINS_API][1], "result": "FAILURE"})
with pytest.raises(Invalid, match="not SUCCESS"):
release_producer.run_once(env, opener=hybrid(bad), clock=lambda: FIXED)
gone = dict(routes)
gone[JENKINS_API] = (503, {})
with pytest.raises(Invalid, match="returned HTTP 503"):
release_producer.run_once(env, opener=hybrid(gone), clock=lambda: FIXED)
collector = release_producer.Collector(env, "hermes-webui", WP, hybrid(routes), lambda: FIXED - timedelta(hours=1))
with pytest.raises(Invalid, match="stale evidence"):
release_producer.advance(client, collector, WP, 900, scope, release_id, FIXED)
status, view, _ = client.request("GET", f"{scope}/{release_id}")
assert status == 200 and view["revision"] == 2 and view["release"]["state"] == "merged"
HAPPY = [
(rel("reviewed", review_url=REVIEW), "merged", {"git": GIT_OBS}, {"merge_commit": SHA_M}),
(rel("merged", merge_commit=SHA_M), "built", {"jenkins": JENKINS_OBS, "harbor": HARBOR_OBS},
{"ci_build_url": JOB + "/20", "image_ref": REF, "image_digest": DIGEST, "harbor_digest": DIGEST}),
(rel("built", **BUILT_EV), "verified", {"harbor": HARBOR_OBS}, {}),
(rel("verified", **BUILT_EV), "deployed", {"flux": FLUX_OBS}, {"flux_revision": FLUX_REV}),
(rel("deployed", **BUILT_EV), "converged", {"workload_spec": SPEC_OBS, "pods": PODS_OBS}, {"pod_digest": DIGEST}),
(rel("converged", **BUILT_EV), "live_verified", {"health": HEALTH_OBS},
{"health_check": {"url": HEALTH, "status": "pass", "at": NOW_S}}),
(rel("deployed", **BUILT_EV), "rolled_back", {"rollback": obs(digest=OTHER_DIGEST)}, {"rollback_target": OTHER_DIGEST}),
]
@pytest.mark.parametrize(("release", "target", "observations", "payload"), HAPPY)
def test_verifier_produces_exact_payloads(release, target, observations, payload):
assert release_producer.evidence_for(release, target, observations, WP, 900, FIXED) == payload
REJECTS = [
(rel("banana"), "merged", {"git": GIT_OBS}, "unknown state"),
(rel("reviewed", review_url=REVIEW), "built", {"jenkins": JENKINS_OBS, "harbor": HARBOR_OBS}, "skip, repeat or downgrade"),
(rel("merged", merge_commit=SHA_M), "merged", {"git": GIT_OBS}, "skip, repeat or downgrade"),
(rel("live_verified", **BUILT_EV), "live_verified", {"health": HEALTH_OBS}, "skip, repeat or downgrade"),
(rel("reviewed", review_url=REVIEW), "rolled_back", {"rollback": obs(digest=OTHER_DIGEST)}, "no deployed image"),
(rel("reviewed", review_url=REVIEW), "merged", {}, "git is missing"),
(rel("reviewed", review_url=REVIEW), "merged", {"git": {**GIT_OBS, "workload": "hermes-agent"}}, "different workload"),
(rel("reviewed", review_url=REVIEW), "merged", {"git": {**GIT_OBS, "observed_at": OLD_S}}, "stale evidence"),
(rel("reviewed", review_url=REVIEW), "merged", {"git": {**GIT_OBS, "observed_at": "2026-08-24T13:00:00Z"}}, "stale evidence"),
(rel("reviewed", review_url=REVIEW), "merged", {"git": {**GIT_OBS, "observed_at": "not-a-time"}}, "RFC 3339"),
(rel("reviewed", review_url=REVIEW), "merged", {"git": obs(commit="short", committed_at=NOW_S, review_url=REVIEW)}, "40-hex"),
(rel("reviewed", review_url=REVIEW), "merged", {"git": {**GIT_OBS, "committed_at": "2026-08-24T13:00:00Z"}}, "in the future"),
(rel("reviewed", review_url=PREFIX + "56"), "merged", {"git": GIT_OBS}, "different review"),
(rel("reviewed", review_url="https://evil.example/pulls/55"), "merged",
{"git": {**GIT_OBS, "review_url": "https://evil.example/pulls/55"}}, "outside the policy review prefix"),
(rel("merged", merge_commit=SHA_M), "built", {"jenkins": JENKINS_OBS}, "harbor is missing"),
(rel("merged", merge_commit=SHA_M), "built", {"jenkins": {**JENKINS_OBS, "build_url": "https://jenkins.bstein.dev/job/other/20"}, "harbor": HARBOR_OBS}, "outside the policy job"),
(rel("merged", merge_commit=SHA_M), "built", {"jenkins": {**JENKINS_OBS, "result": "FAILURE"}, "harbor": HARBOR_OBS}, "not SUCCESS"),
(rel("merged", merge_commit=SHA_M), "built", {"jenkins": {**JENKINS_OBS, "revision": "f" * 40}, "harbor": HARBOR_OBS}, "does not equal the merged commit"),
(rel("merged", merge_commit="zz"), "built", {"jenkins": JENKINS_OBS, "harbor": HARBOR_OBS}, "does not equal the merged commit"),
(rel("merged", merge_commit=SHA_M), "built", {"jenkins": {**JENKINS_OBS, "build_number": True}, "harbor": HARBOR_OBS}, "build_number"),
(rel("merged", merge_commit=SHA_M), "built", {"jenkins": {**JENKINS_OBS, "build_number": 0}, "harbor": HARBOR_OBS}, "build number is invalid"),
(rel("merged", merge_commit=SHA_M), "built", {"jenkins": {**JENKINS_OBS, "build_number": 21}, "harbor": HARBOR_OBS}, "does not bind"),
(rel("merged", merge_commit=SHA_M), "built", {"jenkins": JENKINS_OBS, "harbor": {**HARBOR_OBS, "repository": "registry.bstein.dev/evil/x"}}, "policy image repository"),
(rel("merged", merge_commit=SHA_M), "built", {"jenkins": JENKINS_OBS, "harbor": {**HARBOR_OBS, "digest": "sha256:short"}}, "digest is malformed"),
(rel("merged", merge_commit=SHA_M), "built", {"jenkins": {**JENKINS_OBS, "image_digest": OTHER_DIGEST}, "harbor": HARBOR_OBS}, "advertised image digest"),
(rel("built", **BUILT_EV), "verified", {"harbor": {**HARBOR_OBS, "digest": OTHER_DIGEST}}, "no longer serves"),
(rel("built", review_url=REVIEW, merge_commit=SHA_M), "verified", {"harbor": HARBOR_OBS}, "no longer serves"),
(rel("verified", **BUILT_EV), "deployed", {"flux": {**FLUX_OBS, "name": "other"}}, "name does not match"),
(rel("verified", **BUILT_EV), "deployed", {"flux": {**FLUX_OBS, "applied_revision": "feature@sha1:" + "b" * 40}}, "main@sha1"),
(rel("verified", **BUILT_EV), "deployed", {"flux": {**FLUX_OBS, "pin_in_revision": False}}, "not proven"),
(rel("verified", **BUILT_EV), "deployed", {"flux": {**FLUX_OBS, "pin_in_revision": "true"}}, "not proven"),
(rel("deployed", **BUILT_EV), "converged", {"workload_spec": obs(image=f"{REPO}:latest@{DIGEST}"), "pods": PODS_OBS}, "desired workload image"),
(rel("deployed", review_url=REVIEW), "converged", {"workload_spec": SPEC_OBS, "pods": PODS_OBS}, "desired workload image"),
(rel("deployed", **BUILT_EV), "converged", {"workload_spec": SPEC_OBS, "pods": obs(pods=[])}, "no Ready pods"),
(rel("deployed", **BUILT_EV), "converged", {"workload_spec": SPEC_OBS, "pods": obs(pods=[{"name": "p0", "ready": False, "image_id": REPO + "@" + DIGEST}])}, "not Ready"),
(rel("deployed", **BUILT_EV), "converged", {"workload_spec": SPEC_OBS, "pods": obs(pods=["junk"])}, "not Ready"),
(rel("deployed", **BUILT_EV), "converged", {"workload_spec": SPEC_OBS, "pods": obs(pods=[{"name": "p0", "ready": True, "image_id": REPO + "@" + OTHER_DIGEST}])}, "not running the release digest"),
(rel("deployed", **BUILT_EV), "converged", {"workload_spec": SPEC_OBS, "pods": obs(pods=[{"name": "p0", "ready": True, "image_id": None}])}, "not running the release digest"),
(rel("converged", **BUILT_EV), "live_verified", {"health": {**HEALTH_OBS, "url": "https://evil.example/healthz"}}, "policy health URL"),
(rel("converged", **BUILT_EV), "live_verified", {"health": {**HEALTH_OBS, "status": 503}}, "HTTP 503, not 200"),
(rel("converged", **BUILT_EV), "live_verified", {"health": {**HEALTH_OBS, "status": True}}, "health.status"),
(rel("converged", **BUILT_EV), "live_verified", {"health": {**HEALTH_OBS, "checked_at": OLD_S}}, "stale evidence"),
(rel("converged", **BUILT_EV), "live_verified", {"health": {**HEALTH_OBS, "checked_at": None}}, "RFC 3339"),
(rel("deployed", **BUILT_EV), "rolled_back", {"rollback": obs(digest="bad")}, "rollback target digest"),
(rel("deployed", **BUILT_EV), "rolled_back", {"rollback": obs()}, "rollback.digest"),
]
@pytest.mark.parametrize(("release", "target", "observations", "match"), REJECTS)
def test_verifier_rejects_every_bad_observation_set(release, target, observations, match):
with pytest.raises(Invalid, match=match):
release_producer.evidence_for(release, target, observations, WP, 900, FIXED)
def test_verifier_requires_aware_clock_and_next_target_is_total():
with pytest.raises(Invalid, match="timezone-aware"):
release_producer.evidence_for(rel("reviewed"), "merged", {}, WP, 900, datetime(2026, 8, 24))
assert release_producer.next_target("live_verified") is None
assert release_producer.next_target("reviewed") == "merged"
def test_collectors_extract_only_observed_fields():
collector = make_collector(good_routes())
git = collector.git(REVIEW)
assert git == obs(review_url=REVIEW, commit=SHA_M, committed_at="2026-08-24T11:00:00Z")
jenkins = collector.jenkins()
assert jenkins == obs(build_url=JOB + "/20", result="SUCCESS", build_number=20, revision=SHA_M, image_digest=DIGEST)
assert collector.harbor(TAG) == obs(repository=REPO, tag=TAG, digest=DIGEST)
with pytest.raises(Invalid, match="outside the policy review prefix"):
collector.git("https://evil.example/pulls/55")
with pytest.raises(Invalid, match="outside the policy review prefix"):
collector.git(PREFIX + "55x")
def test_collectors_omit_unproven_fields():
weak = make_collector({
GIT_API: (200, {"merged": False, "merge_commit_sha": SHA_M, "merged_at": "yesterday"}),
JENKINS_API: (200, {"url": 5, "result": None, "number": True, "actions": [{"lastBuiltRevision": {"SHA1": "zz"}}], "description": None}),
HARBOR_API: (200, {"digest": "sha256:short", "tags": []}),
})
assert weak.git(REVIEW) == obs(review_url=REVIEW)
assert weak.jenkins() == obs()
assert weak.harbor(TAG) == obs(repository=REPO)
bad_prefix = release_producer.Collector({}, "hermes-webui", {**WP, "review_url_prefix": "https://scm.bstein.dev/x/"}, FakeOpener({}), lambda: FIXED)
with pytest.raises(Invalid, match="not a proposal URL"):
bad_prefix.git("https://scm.bstein.dev/x/55")
bad_repo = release_producer.Collector({}, "hermes-webui", {**WP, "image_repository": "hermes"}, FakeOpener({}), lambda: FIXED)
with pytest.raises(Invalid, match="host/project/name"):
bad_repo.harbor(TAG)
def test_fetch_is_https_only_bounded_and_fail_closed():
collector = make_collector({HEALTH: (200, b"x" * (release_producer.MAX_RESPONSE_BYTES + 1)), JENKINS_API: (200, b"nonsense")})
with pytest.raises(Invalid, match="must be HTTPS"):
collector._fetch("http://chat.bstein.dev/healthz", "health")
with pytest.raises(Invalid, match="fetch failed"):
collector._fetch("https://unrouted.example/x", "health")
with pytest.raises(Invalid, match="too large"):
collector._fetch(HEALTH, "health")
with pytest.raises(Invalid, match="not JSON"):
collector._json(JENKINS_API, "jenkins")
arrays = make_collector({JENKINS_API: (200, [1, 2])})
with pytest.raises(Invalid, match="not a JSON object"):
arrays._json(JENKINS_API, "jenkins")
down = make_collector({HEALTH: (503, {})})
assert down.health() == obs(url=HEALTH, status=503, checked_at=NOW_S)
def test_tls_context_and_timeout_bounds(tmp_path):
ca = tmp_path / "ca.pem"
ca.write_text(CA_PEM)
assert release_producer._context(str(ca)).check_hostname is True
assert release_producer._context("").check_hostname is True
bad = tmp_path / "bad.pem"
bad.write_text("not a certificate")
with pytest.raises(Invalid, match="CA bundle"):
release_producer._context(str(bad))
assert release_producer._bounded_timeout({}) == 10.0
assert release_producer._bounded_timeout({"HUX_PRODUCER_TIMEOUT_SECONDS": "2.5"}) == 2.5
assert release_producer._bounded_timeout({"HUX_PRODUCER_TIMEOUT_SECONDS": "nan-ish"}) == 10.0
assert release_producer._bounded_timeout({"HUX_PRODUCER_TIMEOUT_SECONDS": "900"}) == 10.0
plain = release_producer.Collector({}, "w", WP)
assert plain.opener is release_producer._default_opener and plain.clock().tzinfo is not None
def test_kubernetes_collectors_fail_closed_on_credentials_and_config(tmp_path):
token = tmp_path / "token"
token.write_text("sa-token\n")
with_token = {"HUX_PRODUCER_KUBE_TOKEN_FILE": str(token)}
with pytest.raises(Invalid, match="credentials are unavailable"):
make_collector(good_routes(), {"HUX_PRODUCER_KUBE_TOKEN_FILE": str(tmp_path / "absent")}).pods()
empty = tmp_path / "empty"
empty.write_text(" \n")
with pytest.raises(Invalid, match="credentials are unavailable"):
make_collector(good_routes(), {"HUX_PRODUCER_KUBE_TOKEN_FILE": str(empty)}).pods()
bad_ns = make_collector(good_routes(), {**with_token, "HUX_PRODUCER_NAMESPACE": "Bad_NS"})
with pytest.raises(Invalid, match="HUX_PRODUCER_NAMESPACE"):
bad_ns.pods()
with pytest.raises(Invalid, match="WORKLOAD_KIND"):
make_collector(good_routes(), {**with_token, "HUX_PRODUCER_WORKLOAD_KIND": "daemonset"}).workload_spec()
no_selector = make_collector(good_routes(), {**with_token, "HUX_PRODUCER_POD_SELECTOR": ""})
with pytest.raises(Invalid, match="POD_SELECTOR"):
no_selector.pods()
def test_kubernetes_collectors_observe_flux_workload_pods_and_rollback(tmp_path):
token = tmp_path / "token"
token.write_text("sa-token\n")
extra = {"HUX_PRODUCER_KUBE_TOKEN_FILE": str(token)}
collector = make_collector(good_routes(), extra)
assert collector.flux(REF) == obs(name="hermes", applied_revision=FLUX_REV, pin_in_revision=True)
assert collector.flux(f"{REPO}:other@{OTHER_DIGEST}")["pin_in_revision"] is False
assert collector.flux("")["pin_in_revision"] is False
assert collector.workload_spec() == obs(image=REF)
assert collector.pods() == obs(pods=[{"name": "p0", "ready": True, "image_id": REPO + "@" + DIGEST}])
assert collector.rollback_target() == obs(digest=DIGEST)
routes = good_routes()
routes[KUSTOMIZATION_API] = (200, {"metadata": {}, "status": {"lastAppliedRevision": None, "conditions": [{"type": "Ready", "status": "False"}]}})
routes[STS_API] = (200, {"spec": {"template": {"spec": {"containers": [{"image": "other:1"}]}}}})
routes[PODS_API] = (200, {"items": [{"metadata": {}, "status": {"conditions": [{"type": "Other"}], "containerStatuses": [{"image": "other:1", "imageID": "x"}]}}]})
drifted = make_collector(routes, extra)
flux = drifted.flux(REF)
assert "applied_revision" not in flux and flux["pin_in_revision"] is False and flux["name"] is None
assert drifted.workload_spec() == obs()
assert drifted.pods() == obs(pods=[{"name": None, "ready": False, "image_id": None}])
assert drifted.rollback_target() == obs()
routes[STS_API] = (200, {"spec": {"template": {"spec": {"containers": [{"image": REPO + ":tag-without-digest"}]}}}})
assert make_collector(routes, extra).rollback_target() == obs()
def test_collect_for_covers_every_target_and_fails_closed():
stub = SimpleNamespace(jenkins=lambda: obs(result="SUCCESS"), rollback_target=lambda: obs(digest=DIGEST))
gathered = release_producer.collect_for(stub, "built", rel("merged", merge_commit=SHA_M))
assert "harbor" not in gathered and gathered["jenkins"]["result"] == "SUCCESS"
assert release_producer.collect_for(stub, "rolled_back", rel("deployed"))["rollback"]["digest"] == DIGEST
with pytest.raises(Invalid, match="no collector plan"):
release_producer.collect_for(stub, "reviewed", rel("reviewed"))
def test_client_requires_loopback_or_https_and_fails_closed():
with pytest.raises(Invalid, match="loopback HTTP or HTTPS"):
release_producer.ProducerClient("http://10.0.0.5:8790", "slot-3", SUBJECT, KEY)
default = release_producer.ProducerClient("", "slot-3", SUBJECT, KEY)
assert default.base == release_producer.DEFAULT_BASE_URL and default.opener is release_producer._default_opener
unreachable = release_producer.ProducerClient("https://hux.test", "slot-3", SUBJECT, KEY, FakeOpener({}))
with pytest.raises(Invalid, match="unreachable"):
unreachable.request("GET", "/x")
odd = release_producer.ProducerClient("https://hux.test", "slot-3", SUBJECT, KEY,
FakeOpener({"https://hux.test/raw": (200, b"nonsense"), "https://hux.test/empty": (200, b"")}))
assert odd.request("GET", "/raw") == (200, None, False)
assert odd.request("GET", "/empty") == (200, None, False)
def test_driver_read_and_transition_failures_are_invalid():
scope = "/hux/v1/projects/p/conversations/c/releases"
stub = SimpleNamespace(git=lambda url: dict(GIT_OBS))
def client_for(routes):
return release_producer.ProducerClient("https://hux.test", "slot-3", SUBJECT, KEY, FakeOpener(routes))
with pytest.raises(Invalid, match="read failed with HTTP 500"):
release_producer.advance(client_for({f"https://hux.test{scope}/rel_1": (500, {})}), stub, WP, 900, scope, "rel_1", FIXED)
with pytest.raises(Invalid, match="carries no revision"):
release_producer.advance(client_for({f"https://hux.test{scope}/rel_1": (200, {"release": rel("reviewed")})}), stub, WP, 900, scope, "rel_1", FIXED)
routes = {f"https://hux.test{scope}/rel_1": (200, {"release": rel("reviewed", review_url=REVIEW), "revision": 1}),
f"https://hux.test{scope}/rel_1/transitions": (409, {})}
with pytest.raises(Invalid, match="rejected with HTTP 409"):
release_producer.advance(client_for(routes), stub, WP, 900, scope, "rel_1", FIXED)
listing = {f"https://hux.test{scope}": (200, {"items": ["junk", {"release": {"id": "rel_x", "workload": "hermes-agent", "state": "reviewed"}},
{"release": {"id": "rel_y", "workload": "hermes-webui", "state": "rolled_back"}},
{"release": {"id": "rel_z", "workload": "hermes-webui", "state": "merged"}}]})}
assert release_producer._pick_release(client_for(listing), scope, "hermes-webui") == "rel_z"
with pytest.raises(Invalid, match="list failed with HTTP 500"):
release_producer._pick_release(client_for({f"https://hux.test{scope}": (500, {})}), scope, "hermes-webui")
@pytest.mark.parametrize("missing", ["HUX_PRODUCER_WORKLOAD", "HUX_TENANT_SLOT", "HUX_PRODUCER_SUBJECT", "HUX_PRODUCER_PROJECT_ID", "HUX_PRODUCER_CONVERSATION_ID"])
def test_run_once_requires_complete_environment(tmp_path, missing):
env = {**secrets(tmp_path), "HUX_PRODUCER_WORKLOAD": "hermes-webui", "HUX_TENANT_SLOT": "slot-3",
"HUX_PRODUCER_SUBJECT": SUBJECT, "HUX_PRODUCER_PROJECT_ID": "prj_1", "HUX_PRODUCER_CONVERSATION_ID": "cnv_1"}
env.pop(missing)
with pytest.raises(Invalid, match=missing):
release_producer.run_once(env, opener=FakeOpener({}))
def test_run_once_rejects_unknown_workload_and_reports_empty_scope(tmp_path):
env = {**secrets(tmp_path), "HUX_PRODUCER_WORKLOAD": "hermes-agent", "HUX_TENANT_SLOT": "slot-3",
"HUX_PRODUCER_SUBJECT": SUBJECT, "HUX_PRODUCER_PROJECT_ID": "prj_1", "HUX_PRODUCER_CONVERSATION_ID": "cnv_1"}
with pytest.raises(Invalid, match="not enabled"):
release_producer.run_once(env, opener=FakeOpener({}))
env["HUX_PRODUCER_WORKLOAD"] = "hermes-webui"
env["HUX_BASE_URL"] = "https://hux.test"
empty = FakeOpener({"https://hux.test/hux/v1/projects/prj_1/conversations/cnv_1/releases": (200, {"items": [], "next": None})})
assert release_producer.run_once(env, opener=empty) == {"changed": False, "reason": "no advanceable release"}
assert empty.calls[0][0] == "GET"
def test_module_and_suite_stay_bounded():
for path in (FOUNDATION / "hux_producer" / "__init__.py", Path(__file__)):
assert len(path.read_text().splitlines()) <= 500
Reference in New Issue View Git Blame Copy Permalink