atlas-iac/testing/tests/test_hermes_hux_release_producer.py

"""HUX-12 release evidence producer: verifier core, collectors and driver."""

from __future__ import annotations

import io
import json
import sys
import threading
import urllib.error
import urllib.request
from datetime import datetime, timedelta, timezone
from email import message_from_string
from pathlib import Path
from types import SimpleNamespace

import pytest

ROOT = Path(__file__).resolve().parents[2]
FOUNDATION = ROOT / "dockerfiles" / "hermes-hux-foundation"
if str(FOUNDATION) not in sys.path:
    sys.path.insert(0, str(FOUNDATION))

import hux_producer as release_producer  # noqa: E402
from hux import contracts  # noqa: E402
from hux.errors import Invalid  # noqa: E402
from hux.http import serve  # noqa: E402
from hux.server import build_router  # noqa: E402

ALL_ON = ",".join(card["flag"] for card in contracts.load_flags()["cards"])
KEY = "evidence-key-value-that-is-at-least-32-bytes"
SUBJECT = "usr_0123456789abcdef"
SHA_M = "a" * 40
DIGEST = "sha256:" + "d" * 64
OTHER_DIGEST = "sha256:" + "e" * 64
REPO = "registry.bstein.dev/bstein/hermes-webui"
JOB = "https://jenkins.bstein.dev/job/hermes-webui-image"
PREFIX = "https://scm.bstein.dev/titan/atlas-iac/pulls/"
REVIEW = PREFIX + "55"
HEALTH = "https://chat.bstein.dev/healthz"
TAG = f"git-{SHA_M}-build-20-release"
REF = f"{REPO}:{TAG}@{DIGEST}"
FLUX_REV = "main@sha1:" + "b" * 40
WP = {"review_url_prefix": PREFIX, "jenkins_job_url": JOB, "image_repository": REPO, "flux_kustomization": "hermes", "health_url": HEALTH}
POLICY = {"schema": "hux.release_evidence_policy.v1", "max_evidence_age_seconds": 900, "workloads": {"hermes-webui": WP}}
FIXED = datetime(2026, 8, 24, 12, 0, 0, tzinfo=timezone.utc)
NOW_S = "2026-08-24T12:00:00Z"
OLD_S = "2026-08-24T10:00:00Z"
GIT_API = "https://scm.bstein.dev/api/v1/repos/titan/atlas-iac/pulls/55"
JENKINS_API = JOB + "/lastSuccessfulBuild/api/json"
HARBOR_API = f"https://registry.bstein.dev/api/v2.0/projects/bstein/repositories/hermes-webui/artifacts/{TAG}"
KUSTOMIZATION_API = "https://kube.test/apis/kustomize.toolkit.fluxcd.io/v1/namespaces/hermes/kustomizations/hermes"
STS_API = "https://kube.test/apis/apps/v1/namespaces/hermes/statefulsets/hermes-webui"
PODS_API = "https://kube.test/api/v1/namespaces/hermes/pods?labelSelector=app=hermes-webui"
ROUTER_HEADERS = {"X-Hermes-Tenant-Identity": "slot-3", "X-Hux-Subject": SUBJECT, "X-Hux-Surface": "chat", "X-Hux-Relay-Key": "rk"}
CA_PEM = """-----BEGIN CERTIFICATE-----
MIIBgTCCASegAwIBAgIUNjZXQ5S3lIfyo4QEuwB6CmxthPwwCgYIKoZIzj0EAwIw
FjEUMBIGA1UEAwwLaHV4LXRlc3QtY2EwHhcNMjYwODI0MDcyMTQzWhcNMzYwODIx
MDcyMTQzWjAWMRQwEgYDVQQDDAtodXgtdGVzdC1jYTBZMBMGByqGSM49AgEGCCqG
SM49AwEHA0IABKUXVC/hmAAhDxIT8HrMvXH/N67OB6zEFKnaW24bg907WXzKqQO6
eSJzPPznQvzuivQtIG6D+39amDg8utWGERijUzBRMB0GA1UdDgQWBBQcnGVz8rJ+
MJrVZ3geD32daLpJBzAfBgNVHSMEGDAWgBQcnGVz8rJ+MJrVZ3geD32daLpJBzAP
BgNVHRMBAf8EBTADAQH/MAoGCCqGSM49BAMCA0gAMEUCIQDVLn3BP3aUfBVdmzck
xydrFRXApxVn0NpQ5v7DBZvZXwIgKibREmX5TKcB9YZyrHaddGTjfPSfREOhxaQQ
nMKqK1Q=
-----END CERTIFICATE-----
"""

def obs(**fields):
    return {"workload": "hermes-webui", "observed_at": NOW_S, **fields}

def rel(state, **evidence):
    return {"workload": "hermes-webui", "state": state, "evidence": evidence}

GIT_OBS = obs(commit=SHA_M, committed_at="2026-08-24T11:00:00Z", review_url=REVIEW)
JENKINS_OBS = obs(build_url=JOB + "/20", result="SUCCESS", build_number=20, revision=SHA_M, image_digest=DIGEST)
HARBOR_OBS = obs(repository=REPO, tag=TAG, digest=DIGEST)
FLUX_OBS = obs(name="hermes", applied_revision=FLUX_REV, pin_in_revision=True)
SPEC_OBS = obs(image=REF)
PODS_OBS = obs(pods=[{"name": "p0", "ready": True, "image_id": REPO + "@" + DIGEST}])
HEALTH_OBS = obs(url=HEALTH, status=200, checked_at=NOW_S)
BUILT_EV = {"review_url": REVIEW, "merge_commit": SHA_M, "ci_build_url": JOB + "/20", "image_ref": REF, "image_digest": DIGEST, "harbor_digest": DIGEST}

class FakeResponse:
    def __init__(self, status, payload, headers=None):
        self.status = status
        self._body = payload if isinstance(payload, bytes) else json.dumps(payload).encode()
        self.headers = headers or {}

    def read(self, limit=None):
        return self._body

    def __enter__(self):
        return self

    def __exit__(self, *exc):
        return False

class FakeOpener:
    def __init__(self, routes):
        self.routes = dict(routes)
        self.calls = []

    def __call__(self, request, timeout, context):
        url = request.full_url
        self.calls.append((request.get_method(), url))
        found = self.routes.get(url)
        if found is None:
            raise urllib.error.URLError(f"no fake route: {url}")
        if isinstance(found, Exception):
            raise found
        status, payload = found
        if status >= 400:
            raise urllib.error.HTTPError(url, status, "error", message_from_string(""), io.BytesIO(b"{}"))
        return FakeResponse(status, payload)

def good_routes():
    return {
        GIT_API: (200, {"merged": True, "merge_commit_sha": SHA_M.upper(), "merged_at": "2026-08-24T11:00:00Z"}),
        JENKINS_API: (200, {"url": JOB + "/20/", "result": "SUCCESS", "number": 20, "description": f"pushed {DIGEST}",
                            "actions": [None, {"other": 1}, {"lastBuiltRevision": {"SHA1": SHA_M.upper()}}]}),
        HARBOR_API: (200, {"digest": DIGEST, "tags": ["junk", {"name": TAG}]}),
        KUSTOMIZATION_API: (200, {"metadata": {"name": "hermes"}, "status": {
            "lastAppliedRevision": FLUX_REV, "lastAttemptedRevision": FLUX_REV, "conditions": [{"type": "Ready", "status": "True"}]}}),
        STS_API: (200, {"spec": {"template": {"spec": {"containers": ["junk", {"image": "other:1"}, {"image": REF}]}}}}),
        PODS_API: (200, {"items": ["junk", {"metadata": {"name": "p0"}, "status": {
            "conditions": [{"type": "Ready", "status": "True"}],
            "containerStatuses": [{"image": "other:1", "imageID": "x"}, {"image": REF, "imageID": REPO + "@" + DIGEST}]}}]}),
        HEALTH: (200, b""),
    }

def hybrid(routes):
    fake = FakeOpener(routes)

    def opener(request, timeout, context):
        if request.full_url.startswith("http://127.0.0.1"):
            return urllib.request.urlopen(request, timeout=timeout)
        return fake(request, timeout, context)
    return opener

def secrets(tmp_path):
    key = tmp_path / "evidence.key"
    key.write_text(KEY + "\n")
    key.chmod(0o400)
    policy = tmp_path / "policy.json"
    policy.write_text(json.dumps(POLICY))
    policy.chmod(0o444)
    return {"HUX_RELEASE_EVIDENCE_KEY_FILE": str(key), "HUX_RELEASE_EVIDENCE_POLICY_FILE": str(policy)}

def dispatch(router, method, path, body=None, headers=None):
    raw = b"" if body is None else json.dumps(body).encode()
    response = router.dispatch(method, path, {**ROUTER_HEADERS, **(headers or {})}, raw)
    return response.status, response.body

def make_collector(routes, extra=None):
    env = {"HUX_PRODUCER_NAMESPACE": "hermes", "HUX_PRODUCER_KUBE_API": "https://kube.test",
           "HUX_PRODUCER_POD_SELECTOR": "app=hermes-webui", **(extra or {})}
    return release_producer.Collector(env, "hermes-webui", WP, FakeOpener(routes), lambda: FIXED)

@pytest.fixture
def stack(tmp_path):
    env = secrets(tmp_path)
    router = build_router(tmp_path, {"HUX_FLAGS": ALL_ON, "HUX_ROUTER_KEY": "rk", **env})
    server = serve(router, "127.0.0.1", 0)
    threading.Thread(target=server.serve_forever, daemon=True).start()
    base = f"http://127.0.0.1:{server.server_address[1]}"
    _, project = dispatch(router, "POST", "/hux/v1/projects", {"name": "P"})
    _, conversation = dispatch(router, "POST", "/hux/v1/conversations", {"title": "C", "project_id": project["id"]})
    scope = f"/hux/v1/projects/{project['id']}/conversations/{conversation['id']}/releases"
    token = tmp_path / "kube.token"
    token.write_text("sa-token\n")
    status, created = dispatch(router, "POST", scope, {"workload": "hermes-webui", "commit": "1" * 40, "evidence": {"review_url": REVIEW}},
                               {"If-Match": "0", "Idempotency-Key": "producer-suite-create"})
    assert status == 201
    env.update({"HUX_BASE_URL": base, "HUX_TENANT_SLOT": "slot-3", "HUX_PRODUCER_SUBJECT": SUBJECT,
                "HUX_PRODUCER_WORKLOAD": "hermes-webui", "HUX_PRODUCER_PROJECT_ID": project["id"],
                "HUX_PRODUCER_CONVERSATION_ID": conversation["id"], "HUX_PRODUCER_NAMESPACE": "hermes",
                "HUX_PRODUCER_KUBE_API": "https://kube.test", "HUX_PRODUCER_KUBE_TOKEN_FILE": str(token),
                "HUX_PRODUCER_POD_SELECTOR": "app=hermes-webui"})
    yield {"router": router, "base": base, "scope": scope, "env": env, "release_id": created["release"]["id"]}
    server.shutdown()

def producer_client(stack, opener=None):
    return release_producer.ProducerClient(stack["base"], "slot-3", SUBJECT, KEY, opener)

def test_full_chain_replay_and_terminal_noop(stack):
    routes = good_routes()
    client = producer_client(stack)
    release_id = stack["release_id"]
    _, first_view, _ = client.request("GET", f"{stack['scope']}/{release_id}")
    states = []
    for _ in range(6):
        outcome = release_producer.run_once(stack["env"], opener=hybrid(routes), clock=lambda: FIXED)
        assert outcome["changed"] is True and outcome["replayed"] is False
        states.append(outcome["state"])
    assert states == ["merged", "built", "verified", "deployed", "converged", "live_verified"]
    status, view, _ = client.request("GET", f"{stack['scope']}/{release_id}")
    assert status == 200 and view["revision"] == 7
    evidence = view["release"]["evidence"]
    assert evidence["image_ref"] == REF and evidence["pod_digest"] == DIGEST and evidence["health_check"]["status"] == "pass"
    assert release_producer.run_once(stack["env"], opener=hybrid(routes), clock=lambda: FIXED) == {"changed": False, "reason": "no advanceable release"}
    by_id = release_producer.run_once({**stack["env"], "HUX_PRODUCER_RELEASE_ID": release_id}, opener=hybrid(routes), clock=lambda: FIXED)
    assert by_id == {"changed": False, "release_id": release_id, "state": "live_verified", "reason": "release is terminal"}
    # A duplicate POST with the producer's deterministic key replays the same entry.
    status, replay, replayed = client.request("POST", f"{stack['scope']}/{release_id}/transitions",
                                              {"to": "merged", "evidence": {"merge_commit": SHA_M}},
                                              {"If-Match": "1", "Idempotency-Key": f"producer-{release_id}-merged-1"})
    assert status == 200 and replayed is True and replay["release"]["state"] == "merged" and replay["revision"] == 2
    # The driver reports a replayed transition as unchanged.
    stale_get = lambda request, timeout, context: (  # noqa: E731
        FakeResponse(200, first_view) if request.get_method() == "GET" else urllib.request.urlopen(request, timeout=timeout))
    stub = SimpleNamespace(git=lambda url: dict(GIT_OBS))
    replay_client = release_producer.ProducerClient(stack["base"], "slot-3", SUBJECT, KEY, stale_get)
    outcome = release_producer.advance(replay_client, stub, WP, 900, stack["scope"], release_id, FIXED)
    assert outcome["changed"] is False and outcome["replayed"] is True and outcome["state"] == "merged"
    healthz = release_producer._default_opener(urllib.request.Request(stack["base"] + "/healthz"), 5.0, None)
    assert healthz.status == 200

def test_driver_never_posts_when_verification_or_collection_fails(stack):
    routes = good_routes()
    env, scope, release_id = stack["env"], stack["scope"], stack["release_id"]
    client = producer_client(stack)
    assert release_producer.run_once(env, opener=hybrid(routes), clock=lambda: FIXED)["state"] == "merged"
    bad = dict(routes)
    bad[JENKINS_API] = (200, {**routes[JENKINS_API][1], "result": "FAILURE"})
    with pytest.raises(Invalid, match="not SUCCESS"):
        release_producer.run_once(env, opener=hybrid(bad), clock=lambda: FIXED)
    gone = dict(routes)
    gone[JENKINS_API] = (503, {})
    with pytest.raises(Invalid, match="returned HTTP 503"):
        release_producer.run_once(env, opener=hybrid(gone), clock=lambda: FIXED)
    collector = release_producer.Collector(env, "hermes-webui", WP, hybrid(routes), lambda: FIXED - timedelta(hours=1))
    with pytest.raises(Invalid, match="stale evidence"):
        release_producer.advance(client, collector, WP, 900, scope, release_id, FIXED)
    status, view, _ = client.request("GET", f"{scope}/{release_id}")
    assert status == 200 and view["revision"] == 2 and view["release"]["state"] == "merged"

HAPPY = [
    (rel("reviewed", review_url=REVIEW), "merged", {"git": GIT_OBS}, {"merge_commit": SHA_M}),
    (rel("merged", merge_commit=SHA_M), "built", {"jenkins": JENKINS_OBS, "harbor": HARBOR_OBS},
     {"ci_build_url": JOB + "/20", "image_ref": REF, "image_digest": DIGEST, "harbor_digest": DIGEST}),
    (rel("built", **BUILT_EV), "verified", {"harbor": HARBOR_OBS}, {}),
    (rel("verified", **BUILT_EV), "deployed", {"flux": FLUX_OBS}, {"flux_revision": FLUX_REV}),
    (rel("deployed", **BUILT_EV), "converged", {"workload_spec": SPEC_OBS, "pods": PODS_OBS}, {"pod_digest": DIGEST}),
    (rel("converged", **BUILT_EV), "live_verified", {"health": HEALTH_OBS},
     {"health_check": {"url": HEALTH, "status": "pass", "at": NOW_S}}),
    (rel("deployed", **BUILT_EV), "rolled_back", {"rollback": obs(digest=OTHER_DIGEST)}, {"rollback_target": OTHER_DIGEST}),
]

@pytest.mark.parametrize(("release", "target", "observations", "payload"), HAPPY)
def test_verifier_produces_exact_payloads(release, target, observations, payload):
    assert release_producer.evidence_for(release, target, observations, WP, 900, FIXED) == payload

REJECTS = [
    (rel("banana"), "merged", {"git": GIT_OBS}, "unknown state"),
    (rel("reviewed", review_url=REVIEW), "built", {"jenkins": JENKINS_OBS, "harbor": HARBOR_OBS}, "skip, repeat or downgrade"),
    (rel("merged", merge_commit=SHA_M), "merged", {"git": GIT_OBS}, "skip, repeat or downgrade"),
    (rel("live_verified", **BUILT_EV), "live_verified", {"health": HEALTH_OBS}, "skip, repeat or downgrade"),
    (rel("reviewed", review_url=REVIEW), "rolled_back", {"rollback": obs(digest=OTHER_DIGEST)}, "no deployed image"),
    (rel("reviewed", review_url=REVIEW), "merged", {}, "git is missing"),
    (rel("reviewed", review_url=REVIEW), "merged", {"git": {**GIT_OBS, "workload": "hermes-agent"}}, "different workload"),
    (rel("reviewed", review_url=REVIEW), "merged", {"git": {**GIT_OBS, "observed_at": OLD_S}}, "stale evidence"),
    (rel("reviewed", review_url=REVIEW), "merged", {"git": {**GIT_OBS, "observed_at": "2026-08-24T13:00:00Z"}}, "stale evidence"),
    (rel("reviewed", review_url=REVIEW), "merged", {"git": {**GIT_OBS, "observed_at": "not-a-time"}}, "RFC 3339"),
    (rel("reviewed", review_url=REVIEW), "merged", {"git": obs(commit="short", committed_at=NOW_S, review_url=REVIEW)}, "40-hex"),
    (rel("reviewed", review_url=REVIEW), "merged", {"git": {**GIT_OBS, "committed_at": "2026-08-24T13:00:00Z"}}, "in the future"),
    (rel("reviewed", review_url=PREFIX + "56"), "merged", {"git": GIT_OBS}, "different review"),
    (rel("reviewed", review_url="https://evil.example/pulls/55"), "merged",
     {"git": {**GIT_OBS, "review_url": "https://evil.example/pulls/55"}}, "outside the policy review prefix"),
    (rel("merged", merge_commit=SHA_M), "built", {"jenkins": JENKINS_OBS}, "harbor is missing"),
    (rel("merged", merge_commit=SHA_M), "built", {"jenkins": {**JENKINS_OBS, "build_url": "https://jenkins.bstein.dev/job/other/20"}, "harbor": HARBOR_OBS}, "outside the policy job"),
    (rel("merged", merge_commit=SHA_M), "built", {"jenkins": {**JENKINS_OBS, "result": "FAILURE"}, "harbor": HARBOR_OBS}, "not SUCCESS"),
    (rel("merged", merge_commit=SHA_M), "built", {"jenkins": {**JENKINS_OBS, "revision": "f" * 40}, "harbor": HARBOR_OBS}, "does not equal the merged commit"),
    (rel("merged", merge_commit="zz"), "built", {"jenkins": JENKINS_OBS, "harbor": HARBOR_OBS}, "does not equal the merged commit"),
    (rel("merged", merge_commit=SHA_M), "built", {"jenkins": {**JENKINS_OBS, "build_number": True}, "harbor": HARBOR_OBS}, "build_number"),
    (rel("merged", merge_commit=SHA_M), "built", {"jenkins": {**JENKINS_OBS, "build_number": 0}, "harbor": HARBOR_OBS}, "build number is invalid"),
    (rel("merged", merge_commit=SHA_M), "built", {"jenkins": {**JENKINS_OBS, "build_number": 21}, "harbor": HARBOR_OBS}, "does not bind"),
    (rel("merged", merge_commit=SHA_M), "built", {"jenkins": JENKINS_OBS, "harbor": {**HARBOR_OBS, "repository": "registry.bstein.dev/evil/x"}}, "policy image repository"),
    (rel("merged", merge_commit=SHA_M), "built", {"jenkins": JENKINS_OBS, "harbor": {**HARBOR_OBS, "digest": "sha256:short"}}, "digest is malformed"),
    (rel("merged", merge_commit=SHA_M), "built", {"jenkins": {**JENKINS_OBS, "image_digest": OTHER_DIGEST}, "harbor": HARBOR_OBS}, "advertised image digest"),
    (rel("built", **BUILT_EV), "verified", {"harbor": {**HARBOR_OBS, "digest": OTHER_DIGEST}}, "no longer serves"),
    (rel("built", review_url=REVIEW, merge_commit=SHA_M), "verified", {"harbor": HARBOR_OBS}, "no longer serves"),
    (rel("verified", **BUILT_EV), "deployed", {"flux": {**FLUX_OBS, "name": "other"}}, "name does not match"),
    (rel("verified", **BUILT_EV), "deployed", {"flux": {**FLUX_OBS, "applied_revision": "feature@sha1:" + "b" * 40}}, "main@sha1"),
    (rel("verified", **BUILT_EV), "deployed", {"flux": {**FLUX_OBS, "pin_in_revision": False}}, "not proven"),
    (rel("verified", **BUILT_EV), "deployed", {"flux": {**FLUX_OBS, "pin_in_revision": "true"}}, "not proven"),
    (rel("deployed", **BUILT_EV), "converged", {"workload_spec": obs(image=f"{REPO}:latest@{DIGEST}"), "pods": PODS_OBS}, "desired workload image"),
    (rel("deployed", review_url=REVIEW), "converged", {"workload_spec": SPEC_OBS, "pods": PODS_OBS}, "desired workload image"),
    (rel("deployed", **BUILT_EV), "converged", {"workload_spec": SPEC_OBS, "pods": obs(pods=[])}, "no Ready pods"),
    (rel("deployed", **BUILT_EV), "converged", {"workload_spec": SPEC_OBS, "pods": obs(pods=[{"name": "p0", "ready": False, "image_id": REPO + "@" + DIGEST}])}, "not Ready"),
    (rel("deployed", **BUILT_EV), "converged", {"workload_spec": SPEC_OBS, "pods": obs(pods=["junk"])}, "not Ready"),
    (rel("deployed", **BUILT_EV), "converged", {"workload_spec": SPEC_OBS, "pods": obs(pods=[{"name": "p0", "ready": True, "image_id": REPO + "@" + OTHER_DIGEST}])}, "not running the release digest"),
    (rel("deployed", **BUILT_EV), "converged", {"workload_spec": SPEC_OBS, "pods": obs(pods=[{"name": "p0", "ready": True, "image_id": None}])}, "not running the release digest"),
    (rel("converged", **BUILT_EV), "live_verified", {"health": {**HEALTH_OBS, "url": "https://evil.example/healthz"}}, "policy health URL"),
    (rel("converged", **BUILT_EV), "live_verified", {"health": {**HEALTH_OBS, "status": 503}}, "HTTP 503, not 200"),
    (rel("converged", **BUILT_EV), "live_verified", {"health": {**HEALTH_OBS, "status": True}}, "health.status"),
    (rel("converged", **BUILT_EV), "live_verified", {"health": {**HEALTH_OBS, "checked_at": OLD_S}}, "stale evidence"),
    (rel("converged", **BUILT_EV), "live_verified", {"health": {**HEALTH_OBS, "checked_at": None}}, "RFC 3339"),
    (rel("deployed", **BUILT_EV), "rolled_back", {"rollback": obs(digest="bad")}, "rollback target digest"),
    (rel("deployed", **BUILT_EV), "rolled_back", {"rollback": obs()}, "rollback.digest"),
]

@pytest.mark.parametrize(("release", "target", "observations", "match"), REJECTS)
def test_verifier_rejects_every_bad_observation_set(release, target, observations, match):
    with pytest.raises(Invalid, match=match):
        release_producer.evidence_for(release, target, observations, WP, 900, FIXED)

def test_verifier_requires_aware_clock_and_next_target_is_total():
    with pytest.raises(Invalid, match="timezone-aware"):
        release_producer.evidence_for(rel("reviewed"), "merged", {}, WP, 900, datetime(2026, 8, 24))
    assert release_producer.next_target("live_verified") is None
    assert release_producer.next_target("reviewed") == "merged"

def test_collectors_extract_only_observed_fields():
    collector = make_collector(good_routes())
    git = collector.git(REVIEW)
    assert git == obs(review_url=REVIEW, commit=SHA_M, committed_at="2026-08-24T11:00:00Z")
    jenkins = collector.jenkins()
    assert jenkins == obs(build_url=JOB + "/20", result="SUCCESS", build_number=20, revision=SHA_M, image_digest=DIGEST)
    assert collector.harbor(TAG) == obs(repository=REPO, tag=TAG, digest=DIGEST)
    with pytest.raises(Invalid, match="outside the policy review prefix"):
        collector.git("https://evil.example/pulls/55")
    with pytest.raises(Invalid, match="outside the policy review prefix"):
        collector.git(PREFIX + "55x")

def test_collectors_omit_unproven_fields():
    weak = make_collector({
        GIT_API: (200, {"merged": False, "merge_commit_sha": SHA_M, "merged_at": "yesterday"}),
        JENKINS_API: (200, {"url": 5, "result": None, "number": True, "actions": [{"lastBuiltRevision": {"SHA1": "zz"}}], "description": None}),
        HARBOR_API: (200, {"digest": "sha256:short", "tags": []}),
    })
    assert weak.git(REVIEW) == obs(review_url=REVIEW)
    assert weak.jenkins() == obs()
    assert weak.harbor(TAG) == obs(repository=REPO)
    bad_prefix = release_producer.Collector({}, "hermes-webui", {**WP, "review_url_prefix": "https://scm.bstein.dev/x/"}, FakeOpener({}), lambda: FIXED)
    with pytest.raises(Invalid, match="not a proposal URL"):
        bad_prefix.git("https://scm.bstein.dev/x/55")
    bad_repo = release_producer.Collector({}, "hermes-webui", {**WP, "image_repository": "hermes"}, FakeOpener({}), lambda: FIXED)
    with pytest.raises(Invalid, match="host/project/name"):
        bad_repo.harbor(TAG)

def test_fetch_is_https_only_bounded_and_fail_closed():
    collector = make_collector({HEALTH: (200, b"x" * (release_producer.MAX_RESPONSE_BYTES + 1)), JENKINS_API: (200, b"nonsense")})
    with pytest.raises(Invalid, match="must be HTTPS"):
        collector._fetch("http://chat.bstein.dev/healthz", "health")
    with pytest.raises(Invalid, match="fetch failed"):
        collector._fetch("https://unrouted.example/x", "health")
    with pytest.raises(Invalid, match="too large"):
        collector._fetch(HEALTH, "health")
    with pytest.raises(Invalid, match="not JSON"):
        collector._json(JENKINS_API, "jenkins")
    arrays = make_collector({JENKINS_API: (200, [1, 2])})
    with pytest.raises(Invalid, match="not a JSON object"):
        arrays._json(JENKINS_API, "jenkins")
    down = make_collector({HEALTH: (503, {})})
    assert down.health() == obs(url=HEALTH, status=503, checked_at=NOW_S)

def test_tls_context_and_timeout_bounds(tmp_path):
    ca = tmp_path / "ca.pem"
    ca.write_text(CA_PEM)
    assert release_producer._context(str(ca)).check_hostname is True
    assert release_producer._context("").check_hostname is True
    bad = tmp_path / "bad.pem"
    bad.write_text("not a certificate")
    with pytest.raises(Invalid, match="CA bundle"):
        release_producer._context(str(bad))
    assert release_producer._bounded_timeout({}) == 10.0
    assert release_producer._bounded_timeout({"HUX_PRODUCER_TIMEOUT_SECONDS": "2.5"}) == 2.5
    assert release_producer._bounded_timeout({"HUX_PRODUCER_TIMEOUT_SECONDS": "nan-ish"}) == 10.0
    assert release_producer._bounded_timeout({"HUX_PRODUCER_TIMEOUT_SECONDS": "900"}) == 10.0
    plain = release_producer.Collector({}, "w", WP)
    assert plain.opener is release_producer._default_opener and plain.clock().tzinfo is not None

def test_kubernetes_collectors_fail_closed_on_credentials_and_config(tmp_path):
    token = tmp_path / "token"
    token.write_text("sa-token\n")
    with_token = {"HUX_PRODUCER_KUBE_TOKEN_FILE": str(token)}
    with pytest.raises(Invalid, match="credentials are unavailable"):
        make_collector(good_routes(), {"HUX_PRODUCER_KUBE_TOKEN_FILE": str(tmp_path / "absent")}).pods()
    empty = tmp_path / "empty"
    empty.write_text("  \n")
    with pytest.raises(Invalid, match="credentials are unavailable"):
        make_collector(good_routes(), {"HUX_PRODUCER_KUBE_TOKEN_FILE": str(empty)}).pods()
    bad_ns = make_collector(good_routes(), {**with_token, "HUX_PRODUCER_NAMESPACE": "Bad_NS"})
    with pytest.raises(Invalid, match="HUX_PRODUCER_NAMESPACE"):
        bad_ns.pods()
    with pytest.raises(Invalid, match="WORKLOAD_KIND"):
        make_collector(good_routes(), {**with_token, "HUX_PRODUCER_WORKLOAD_KIND": "daemonset"}).workload_spec()
    no_selector = make_collector(good_routes(), {**with_token, "HUX_PRODUCER_POD_SELECTOR": ""})
    with pytest.raises(Invalid, match="POD_SELECTOR"):
        no_selector.pods()

def test_kubernetes_collectors_observe_flux_workload_pods_and_rollback(tmp_path):
    token = tmp_path / "token"
    token.write_text("sa-token\n")
    extra = {"HUX_PRODUCER_KUBE_TOKEN_FILE": str(token)}
    collector = make_collector(good_routes(), extra)
    assert collector.flux(REF) == obs(name="hermes", applied_revision=FLUX_REV, pin_in_revision=True)
    assert collector.flux(f"{REPO}:other@{OTHER_DIGEST}")["pin_in_revision"] is False
    assert collector.flux("")["pin_in_revision"] is False
    assert collector.workload_spec() == obs(image=REF)
    assert collector.pods() == obs(pods=[{"name": "p0", "ready": True, "image_id": REPO + "@" + DIGEST}])
    assert collector.rollback_target() == obs(digest=DIGEST)
    routes = good_routes()
    routes[KUSTOMIZATION_API] = (200, {"metadata": {}, "status": {"lastAppliedRevision": None, "conditions": [{"type": "Ready", "status": "False"}]}})
    routes[STS_API] = (200, {"spec": {"template": {"spec": {"containers": [{"image": "other:1"}]}}}})
    routes[PODS_API] = (200, {"items": [{"metadata": {}, "status": {"conditions": [{"type": "Other"}], "containerStatuses": [{"image": "other:1", "imageID": "x"}]}}]})
    drifted = make_collector(routes, extra)
    flux = drifted.flux(REF)
    assert "applied_revision" not in flux and flux["pin_in_revision"] is False and flux["name"] is None
    assert drifted.workload_spec() == obs()
    assert drifted.pods() == obs(pods=[{"name": None, "ready": False, "image_id": None}])
    assert drifted.rollback_target() == obs()
    routes[STS_API] = (200, {"spec": {"template": {"spec": {"containers": [{"image": REPO + ":tag-without-digest"}]}}}})
    assert make_collector(routes, extra).rollback_target() == obs()

def test_collect_for_covers_every_target_and_fails_closed():
    stub = SimpleNamespace(jenkins=lambda: obs(result="SUCCESS"), rollback_target=lambda: obs(digest=DIGEST))
    gathered = release_producer.collect_for(stub, "built", rel("merged", merge_commit=SHA_M))
    assert "harbor" not in gathered and gathered["jenkins"]["result"] == "SUCCESS"
    assert release_producer.collect_for(stub, "rolled_back", rel("deployed"))["rollback"]["digest"] == DIGEST
    with pytest.raises(Invalid, match="no collector plan"):
        release_producer.collect_for(stub, "reviewed", rel("reviewed"))

def test_client_requires_loopback_or_https_and_fails_closed():
    with pytest.raises(Invalid, match="loopback HTTP or HTTPS"):
        release_producer.ProducerClient("http://10.0.0.5:8790", "slot-3", SUBJECT, KEY)
    default = release_producer.ProducerClient("", "slot-3", SUBJECT, KEY)
    assert default.base == release_producer.DEFAULT_BASE_URL and default.opener is release_producer._default_opener
    unreachable = release_producer.ProducerClient("https://hux.test", "slot-3", SUBJECT, KEY, FakeOpener({}))
    with pytest.raises(Invalid, match="unreachable"):
        unreachable.request("GET", "/x")
    odd = release_producer.ProducerClient("https://hux.test", "slot-3", SUBJECT, KEY,
                                          FakeOpener({"https://hux.test/raw": (200, b"nonsense"), "https://hux.test/empty": (200, b"")}))
    assert odd.request("GET", "/raw") == (200, None, False)
    assert odd.request("GET", "/empty") == (200, None, False)

def test_driver_read_and_transition_failures_are_invalid():
    scope = "/hux/v1/projects/p/conversations/c/releases"
    stub = SimpleNamespace(git=lambda url: dict(GIT_OBS))
    def client_for(routes):
        return release_producer.ProducerClient("https://hux.test", "slot-3", SUBJECT, KEY, FakeOpener(routes))
    with pytest.raises(Invalid, match="read failed with HTTP 500"):
        release_producer.advance(client_for({f"https://hux.test{scope}/rel_1": (500, {})}), stub, WP, 900, scope, "rel_1", FIXED)
    with pytest.raises(Invalid, match="carries no revision"):
        release_producer.advance(client_for({f"https://hux.test{scope}/rel_1": (200, {"release": rel("reviewed")})}), stub, WP, 900, scope, "rel_1", FIXED)
    routes = {f"https://hux.test{scope}/rel_1": (200, {"release": rel("reviewed", review_url=REVIEW), "revision": 1}),
              f"https://hux.test{scope}/rel_1/transitions": (409, {})}
    with pytest.raises(Invalid, match="rejected with HTTP 409"):
        release_producer.advance(client_for(routes), stub, WP, 900, scope, "rel_1", FIXED)
    listing = {f"https://hux.test{scope}": (200, {"items": ["junk", {"release": {"id": "rel_x", "workload": "hermes-agent", "state": "reviewed"}},
                                                            {"release": {"id": "rel_y", "workload": "hermes-webui", "state": "rolled_back"}},
                                                            {"release": {"id": "rel_z", "workload": "hermes-webui", "state": "merged"}}]})}
    assert release_producer._pick_release(client_for(listing), scope, "hermes-webui") == "rel_z"
    with pytest.raises(Invalid, match="list failed with HTTP 500"):
        release_producer._pick_release(client_for({f"https://hux.test{scope}": (500, {})}), scope, "hermes-webui")

@pytest.mark.parametrize("missing", ["HUX_PRODUCER_WORKLOAD", "HUX_TENANT_SLOT", "HUX_PRODUCER_SUBJECT", "HUX_PRODUCER_PROJECT_ID", "HUX_PRODUCER_CONVERSATION_ID"])
def test_run_once_requires_complete_environment(tmp_path, missing):
    env = {**secrets(tmp_path), "HUX_PRODUCER_WORKLOAD": "hermes-webui", "HUX_TENANT_SLOT": "slot-3",
           "HUX_PRODUCER_SUBJECT": SUBJECT, "HUX_PRODUCER_PROJECT_ID": "prj_1", "HUX_PRODUCER_CONVERSATION_ID": "cnv_1"}
    env.pop(missing)
    with pytest.raises(Invalid, match=missing):
        release_producer.run_once(env, opener=FakeOpener({}))

def test_run_once_rejects_unknown_workload_and_reports_empty_scope(tmp_path):
    env = {**secrets(tmp_path), "HUX_PRODUCER_WORKLOAD": "hermes-agent", "HUX_TENANT_SLOT": "slot-3",
           "HUX_PRODUCER_SUBJECT": SUBJECT, "HUX_PRODUCER_PROJECT_ID": "prj_1", "HUX_PRODUCER_CONVERSATION_ID": "cnv_1"}
    with pytest.raises(Invalid, match="not enabled"):
        release_producer.run_once(env, opener=FakeOpener({}))
    env["HUX_PRODUCER_WORKLOAD"] = "hermes-webui"
    env["HUX_BASE_URL"] = "https://hux.test"
    empty = FakeOpener({"https://hux.test/hux/v1/projects/prj_1/conversations/cnv_1/releases": (200, {"items": [], "next": None})})
    assert release_producer.run_once(env, opener=empty) == {"changed": False, "reason": "no advanceable release"}
    assert empty.calls[0][0] == "GET"

def test_module_and_suite_stay_bounded():
    for path in (FOUNDATION / "hux_producer" / "__init__.py", Path(__file__)):
        assert len(path.read_text().splitlines()) <= 500