atlas-iac/dockerfiles/hermes-hux-foundation/hux_producer/__init__.py

"""HUX-12 release follow-through evidence producer (companion process).

This package lives OUTSIDE the network-free ``hux`` service package (SO-29):
it is the only component allowed to open outbound connections.  It may import
``hux.errors`` and ``hux.release_security``; ``hux`` never imports it.

A pure observation -> verdict core (``evidence_for``) binds every observation
to the release being advanced and the workload policy, returning the exact
evidence payload for the next legal transition or raising ``Invalid``.  A thin
``Collector`` gathers observations from Jenkins, Harbor, the Kubernetes API
and the health URL over HTTPS with an injectable transport.  The driver
(``advance``/``run_once``) posts at most one verified transition per pass with
If-Match and a deterministic Idempotency-Key; verdicts derive only from passed
observations and the injected clock, and nothing here logs the evidence key.
"""

from __future__ import annotations

import json
import re
import ssl
import urllib.error
import urllib.request
from collections.abc import Callable, Mapping
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from urllib.parse import quote, urlsplit

from hux import release_security
from hux.errors import Invalid

ORDER = ("reviewed", "merged", "built", "verified", "deployed", "converged", "live_verified")
TERMINAL = frozenset({"live_verified", "rolled_back"})
ROLLBACK_STATES = frozenset({"built", "verified", "deployed", "converged", "live_verified"})
COMMIT = re.compile(r"^[0-9a-f]{40}$")
SHA = re.compile(r"^sha256:[0-9a-f]{64}$")
SHA_ANY = re.compile(r"sha256:[0-9a-f]{64}")
TIME = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$")
FLUX_REV = re.compile(r"^main@sha1:[0-9a-f]{40}$")
NAMESPACE = re.compile(r"^[a-z0-9-]{1,63}$")
DEFAULT_BASE_URL = "http://127.0.0.1:8790"
DEFAULT_KUBE_API = "https://kubernetes.default.svc"
DEFAULT_KUBE_TOKEN = "/var/run/secrets/kubernetes.io/serviceaccount/token"  # noqa: S105
MAX_RESPONSE_BYTES = 1024 * 1024
Opener = Callable[[urllib.request.Request, float, ssl.SSLContext | None], Any]

def _check(condition: Any, message: str) -> None:
    """Fail closed with a precise reason unless ``condition`` holds."""
    if not condition:
        raise Invalid(message)

def parse_time(value: Any, label: str) -> datetime:
    """Parse one strict RFC 3339 UTC second-precision timestamp."""
    _check(isinstance(value, str) and TIME.fullmatch(value), f"{label} must be an RFC 3339 UTC timestamp")
    return datetime.strptime(value, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc)

def _fresh(value: Any, now: datetime, max_age: int, label: str) -> str:
    age = (now - parse_time(value, label)).total_seconds()
    _check(0 <= age <= max_age, f"stale evidence: {label} is outside the freshness window")
    return str(value)

def _observation(observations: Mapping[str, Any], name: str, workload: str, now: datetime, max_age: int) -> dict[str, Any]:
    obs = observations.get(name)
    _check(isinstance(obs, dict), f"incomplete observations: {name} is missing")
    _check(obs.get("workload") == workload, f"{name} observation is bound to a different workload")
    _fresh(obs.get("observed_at"), now, max_age, f"{name}.observed_at")
    return obs

def _need(obs: Mapping[str, Any], name: str, field: str, kind: type = str) -> Any:
    value = obs.get(field)
    _check(not isinstance(value, bool) and isinstance(value, kind), f"incomplete observations: {name}.{field} is missing or malformed")
    return value

def _verify_merged(release, observations, wpolicy, max_age, now):
    git = _observation(observations, "git", release["workload"], now, max_age)
    commit = _need(git, "git", "commit")
    _check(COMMIT.fullmatch(commit), "git commit must be an exact 40-hex SHA")
    _check(parse_time(_need(git, "git", "committed_at"), "git.committed_at") <= now, "git commit timestamp is in the future")
    review_url = _need(git, "git", "review_url")
    _check(review_url == release.get("evidence", {}).get("review_url"), "git observation is for a different review than this release")
    _check(review_url.startswith(wpolicy["review_url_prefix"]), "review URL is outside the policy review prefix")
    return {"merge_commit": commit}

def _verify_built(release, observations, wpolicy, max_age, now):
    jenkins = _observation(observations, "jenkins", release["workload"], now, max_age)
    harbor = _observation(observations, "harbor", release["workload"], now, max_age)
    sha = str(release.get("evidence", {}).get("merge_commit", ""))
    build_url, job = _need(jenkins, "jenkins", "build_url"), wpolicy["jenkins_job_url"]
    _check(build_url == job or build_url.startswith(job + "/"), "Jenkins build URL is outside the policy job")
    _check(_need(jenkins, "jenkins", "result") == "SUCCESS", "Jenkins build result is not SUCCESS")
    _check(COMMIT.fullmatch(sha) and _need(jenkins, "jenkins", "revision") == sha, "Jenkins build revision does not equal the merged commit")
    number = _need(jenkins, "jenkins", "build_number", int)
    _check(number > 0, "Jenkins build number is invalid")
    tag = f"git-{sha}-build-{number}-release"
    _check(_need(harbor, "harbor", "tag") == tag, "Harbor tag does not bind the merged commit and Jenkins build")
    _check(_need(harbor, "harbor", "repository") == wpolicy["image_repository"], "Harbor repository does not match the policy image repository")
    digest = _need(harbor, "harbor", "digest")
    _check(SHA.fullmatch(digest), "Harbor digest is malformed")
    _check(_need(jenkins, "jenkins", "image_digest") == digest, "Harbor digest does not equal the advertised image digest")
    ref = f"{wpolicy['image_repository']}:{tag}@{digest}"
    return {"ci_build_url": build_url, "image_ref": ref, "image_digest": digest, "harbor_digest": digest}

def _verify_verified(release, observations, wpolicy, max_age, now):
    harbor = _observation(observations, "harbor", release["workload"], now, max_age)
    evidence = release.get("evidence", {})
    tag = str(evidence.get("image_ref", "")).rpartition("@")[0].rpartition(":")[2]
    same = (_need(harbor, "harbor", "repository") == wpolicy["image_repository"]
            and tag and _need(harbor, "harbor", "tag") == tag
            and _need(harbor, "harbor", "digest") == evidence.get("image_digest"))
    _check(same, "Harbor no longer serves the built tag at the recorded digest")
    return {}

def _verify_deployed(release, observations, wpolicy, max_age, now):
    flux = _observation(observations, "flux", release["workload"], now, max_age)
    _check(_need(flux, "flux", "name") == wpolicy["flux_kustomization"], "Flux kustomization name does not match policy")
    revision = _need(flux, "flux", "applied_revision")
    _check(FLUX_REV.fullmatch(revision), "Flux applied revision must look like main@sha1:<40-hex>")
    _check(flux.get("pin_in_revision") is True, "manifest pin is not proven to be contained in the applied revision")
    return {"flux_revision": revision}

def _verify_converged(release, observations, wpolicy, max_age, now):
    spec = _observation(observations, "workload_spec", release["workload"], now, max_age)
    pods = _observation(observations, "pods", release["workload"], now, max_age)
    evidence = release.get("evidence", {})
    ref, digest = str(evidence.get("image_ref", "")), str(evidence.get("image_digest", ""))
    _check(ref and _need(spec, "workload_spec", "image") == ref, "desired workload image does not equal the built release reference")
    entries = _need(pods, "pods", "pods", list)
    _check(entries, "no Ready pods were observed for the workload")
    for pod in entries:
        _check(isinstance(pod, dict) and pod.get("ready") is True, "a workload pod is not Ready")
        image_id = pod.get("image_id")
        _check(isinstance(image_id, str) and image_id.endswith("@" + digest), "a Ready pod is not running the release digest")
    return {"pod_digest": digest}

def _verify_live(release, observations, wpolicy, max_age, now):
    health = _observation(observations, "health", release["workload"], now, max_age)
    _check(_need(health, "health", "url") == wpolicy["health_url"], "health receipt is not for the policy health URL")
    status = _need(health, "health", "status", int)
    _check(status == 200, f"health check returned HTTP {status}, not 200")
    checked = _fresh(health.get("checked_at"), now, max_age, "health.checked_at")
    return {"health_check": {"url": health["url"], "status": "pass", "at": checked}}

def _verify_rollback(release, observations, wpolicy, max_age, now):
    rollback = _observation(observations, "rollback", release["workload"], now, max_age)
    digest = _need(rollback, "rollback", "digest")
    _check(SHA.fullmatch(digest), "rollback target digest is malformed")
    return {"rollback_target": digest}

_VERIFIERS = {
    "merged": _verify_merged, "built": _verify_built, "verified": _verify_verified, "deployed": _verify_deployed,
    "converged": _verify_converged, "live_verified": _verify_live, "rolled_back": _verify_rollback,
}

def next_target(state: str) -> str | None:
    """The only legal forward transition from ``state``; None when terminal."""
    if state in TERMINAL:
        return None
    _check(state in ORDER, "release is in an unknown state")
    return ORDER[ORDER.index(state) + 1]

def evidence_for(release: Mapping[str, Any], target: str, observations: Mapping[str, Any], wpolicy: Mapping[str, Any], max_age: int, now: datetime) -> dict[str, Any]:
    """Verify observations for exactly one legal transition or raise Invalid."""
    _check(now.tzinfo is not None, "now must be a timezone-aware UTC datetime")
    state = str(release.get("state", ""))
    if target == "rolled_back":
        _check(state in ROLLBACK_STATES, "there is no deployed image to roll back")
    else:
        _check(target == next_target(state), f"transition to {target} would skip, repeat or downgrade from {state}")
    return _VERIFIERS[target](dict(release), observations, wpolicy, int(max_age), now)

def _default_opener(request: urllib.request.Request, timeout: float, context: ssl.SSLContext | None):
    return urllib.request.urlopen(request, timeout=timeout, context=context)  # noqa: S310

def _context(ca_file: str) -> ssl.SSLContext:
    """A verifying TLS context, trusting the given CA bundle when one is mounted."""
    try:
        return ssl.create_default_context(cafile=ca_file or None)
    except (OSError, ssl.SSLError) as error:
        raise Invalid("collector CA bundle is unavailable") from error

def _bounded_timeout(environ: Mapping[str, str]) -> float:
    try:
        value = float(environ.get("HUX_PRODUCER_TIMEOUT_SECONDS", 10.0))
    except (TypeError, ValueError):
        return 10.0
    return value if 0.1 <= value <= 60.0 else 10.0

class Collector:
    """Observation gatherer; every field is observed via the transport, never asserted."""

    def __init__(self, environ: Mapping[str, str], workload: str, wpolicy: Mapping[str, Any], opener: Opener | None = None, clock: Callable[[], datetime] | None = None) -> None:
        self.environ = dict(environ)
        self.workload = workload
        self.wpolicy = dict(wpolicy)
        self.opener = opener or _default_opener
        self.clock = clock or (lambda: datetime.now(timezone.utc))
        self.timeout = _bounded_timeout(environ)

    def _stamp(self, fields: dict[str, Any]) -> dict[str, Any]:
        observed = self.clock().strftime("%Y-%m-%dT%H:%M:%SZ")
        return {"workload": self.workload, "observed_at": observed, **fields}

    def _fetch(self, url: str, label: str, headers: Mapping[str, str] | None = None, context: ssl.SSLContext | None = None) -> tuple[int, bytes]:
        _check(urlsplit(url).scheme == "https", f"{label} observation URL must be HTTPS")
        request = urllib.request.Request(url, headers=dict(headers or {}), method="GET")  # noqa: S310
        try:
            with self.opener(request, self.timeout, context or _context("")) as response:
                status, data = int(response.status), response.read(MAX_RESPONSE_BYTES + 1)
        except urllib.error.HTTPError as error:
            return int(error.code), b""
        except (OSError, ValueError) as error:
            raise Invalid(f"{label} observation fetch failed") from error
        _check(len(data) <= MAX_RESPONSE_BYTES, f"{label} observation response is too large")
        return status, data

    def _json(self, url: str, label: str, headers: Mapping[str, str] | None = None, context: ssl.SSLContext | None = None) -> dict[str, Any]:
        status, data = self._fetch(url, label, headers, context)
        _check(status == 200, f"{label} observation returned HTTP {status}")
        try:
            parsed = json.loads(data)
        except ValueError as error:
            raise Invalid(f"{label} observation is not JSON") from error
        _check(isinstance(parsed, dict), f"{label} observation is not a JSON object")
        return parsed

    def git(self, review_url: str) -> dict[str, Any]:
        """Observe the merged proposal named by the release's recorded review URL."""
        prefix = self.wpolicy["review_url_prefix"]
        number = review_url[len(prefix):] if review_url.startswith(prefix) else ""
        _check(number.isdigit(), "release review URL is outside the policy review prefix")
        parsed = urlsplit(prefix)
        parts = parsed.path.strip("/").split("/")
        _check(len(parts) == 3 and parts[2] == "pulls", "policy review prefix is not a proposal URL")
        body = self._json(f"https://{parsed.netloc}/api/v1/repos/{parts[0]}/{parts[1]}/pulls/{number}", "git")
        fields: dict[str, Any] = {"review_url": review_url}
        commit, merged_at = body.get("merge_commit_sha"), body.get("merged_at")
        if body.get("merged") is True and isinstance(commit, str):
            fields["commit"] = commit.lower()
        if isinstance(merged_at, str) and TIME.fullmatch(merged_at):
            fields["committed_at"] = merged_at
        return self._stamp(fields)

    def jenkins(self) -> dict[str, Any]:
        """Observe the last successful build of the policy job."""
        body = self._json(self.wpolicy["jenkins_job_url"] + "/lastSuccessfulBuild/api/json", "jenkins")
        fields: dict[str, Any] = {}
        url, result, number = body.get("url"), body.get("result"), body.get("number")
        if isinstance(url, str):
            fields["build_url"] = url.rstrip("/")
        if isinstance(result, str):
            fields["result"] = result
        if isinstance(number, int) and not isinstance(number, bool):
            fields["build_number"] = number
        for action in body.get("actions") or []:
            revision = action.get("lastBuiltRevision") if isinstance(action, dict) else None
            sha = revision.get("SHA1") if isinstance(revision, dict) else None
            if isinstance(sha, str) and COMMIT.fullmatch(sha.lower()):
                fields["revision"] = sha.lower()
                break
        advertised = SHA_ANY.search(str(body.get("description") or ""))
        if advertised:
            fields["image_digest"] = advertised.group(0)
        return self._stamp(fields)

    def harbor(self, tag: str) -> dict[str, Any]:
        """Observe the immutable Harbor artifact behind one release tag."""
        repository = self.wpolicy["image_repository"]
        host, _, path = repository.partition("/")
        project, _, name = path.partition("/")
        _check(host and project and name, "policy image repository is not host/project/name")
        body = self._json(f"https://{host}/api/v2.0/projects/{project}/repositories/{quote(name, safe='')}/artifacts/{quote(tag, safe='')}", "harbor")
        fields: dict[str, Any] = {"repository": repository}
        digest = body.get("digest")
        if isinstance(digest, str) and SHA.fullmatch(digest):
            fields["digest"] = digest
        names = [entry.get("name") for entry in body.get("tags") or [] if isinstance(entry, dict)]
        if tag in names:
            fields["tag"] = tag
        return self._stamp(fields)

    def _namespace(self) -> str:
        namespace = self.environ.get("HUX_PRODUCER_NAMESPACE", "")
        _check(NAMESPACE.fullmatch(namespace), "HUX_PRODUCER_NAMESPACE is missing or malformed")
        return namespace

    def _kube(self, path: str) -> dict[str, Any]:
        base = self.environ.get("HUX_PRODUCER_KUBE_API", DEFAULT_KUBE_API).rstrip("/")
        token_file = self.environ.get("HUX_PRODUCER_KUBE_TOKEN_FILE", DEFAULT_KUBE_TOKEN)
        try:
            token = Path(token_file).read_text(encoding="utf-8").strip()
        except (OSError, UnicodeDecodeError) as error:
            raise Invalid("kubernetes credentials are unavailable") from error
        _check(token, "kubernetes credentials are unavailable")
        context = _context(self.environ.get("HUX_PRODUCER_KUBE_CA_FILE", ""))
        return self._json(base + path, "kubernetes", {"Authorization": "Bearer " + token}, context)

    def _workload_image(self) -> str | None:
        kind = self.environ.get("HUX_PRODUCER_WORKLOAD_KIND", "statefulset").lower()
        plural = {"deployment": "deployments", "statefulset": "statefulsets"}.get(kind)
        _check(plural, "HUX_PRODUCER_WORKLOAD_KIND must be deployment or statefulset")
        name = self.environ.get("HUX_PRODUCER_WORKLOAD_NAME", self.workload)
        body = self._kube(f"/apis/apps/v1/namespaces/{self._namespace()}/{plural}/{name}")
        containers = (((body.get("spec") or {}).get("template") or {}).get("spec") or {}).get("containers") or []
        prefix = self.wpolicy["image_repository"] + ":"
        for container in containers:
            image = container.get("image") if isinstance(container, dict) else None
            if isinstance(image, str) and image.startswith(prefix):
                return image
        return None

    def workload_spec(self) -> dict[str, Any]:
        """Observe the live desired image of the policy workload."""
        image = self._workload_image()
        return self._stamp({"image": image} if image is not None else {})

    def flux(self, expected_image: str) -> dict[str, Any]:
        """Observe the policy Kustomization; pin containment is proven, not asserted."""
        name = self.wpolicy["flux_kustomization"]
        body = self._kube(f"/apis/kustomize.toolkit.fluxcd.io/v1/namespaces/{self._namespace()}/kustomizations/{name}")
        status = body.get("status") or {}
        applied = status.get("lastAppliedRevision")
        fields: dict[str, Any] = {"name": (body.get("metadata") or {}).get("name")}
        if isinstance(applied, str):
            fields["applied_revision"] = applied
        ready = any(isinstance(c, dict) and c.get("type") == "Ready" and c.get("status") == "True" for c in status.get("conditions") or [])
        settled = isinstance(applied, str) and status.get("lastAttemptedRevision") in (None, applied)
        pinned = bool(expected_image) and self._workload_image() == expected_image
        fields["pin_in_revision"] = bool(ready and settled and pinned)
        return self._stamp(fields)

    def pods(self) -> dict[str, Any]:
        """Observe readiness and running imageID for every selected pod."""
        selector = self.environ.get("HUX_PRODUCER_POD_SELECTOR", "")
        _check(selector, "HUX_PRODUCER_POD_SELECTOR is required to observe pods")
        body = self._kube(f"/api/v1/namespaces/{self._namespace()}/pods?labelSelector={quote(selector, safe='=,')}")
        prefix = self.wpolicy["image_repository"] + ":"
        entries = []
        for item in body.get("items") or []:
            if not isinstance(item, dict):
                continue
            status = item.get("status") or {}
            ready = any(isinstance(c, dict) and c.get("type") == "Ready" and c.get("status") == "True" for c in status.get("conditions") or [])
            image_id = None
            for container in status.get("containerStatuses") or []:
                if isinstance(container, dict) and str(container.get("image", "")).startswith(prefix):
                    image_id = container.get("imageID")
                    break
            entries.append({"name": (item.get("metadata") or {}).get("name"), "ready": ready, "image_id": image_id})
        return self._stamp({"pods": entries})

    def health(self) -> dict[str, Any]:
        """Observe the policy health URL; the receipt carries the real HTTP status."""
        url = self.wpolicy["health_url"]
        checked = self.clock().strftime("%Y-%m-%dT%H:%M:%SZ")
        status, _ = self._fetch(url, "health")
        return self._stamp({"url": url, "status": status, "checked_at": checked})

    def rollback_target(self) -> dict[str, Any]:
        """Observe the digest currently pinned in the live workload spec."""
        image = self._workload_image()
        fields: dict[str, Any] = {}
        digest = image.rpartition("@")[2] if isinstance(image, str) else ""
        if SHA.fullmatch(digest):
            fields["digest"] = digest
        return self._stamp(fields)

def collect_for(collector: Collector, target: str, release: Mapping[str, Any]) -> dict[str, Any]:
    """Gather exactly the observations the verifier needs for one target state."""
    evidence = release.get("evidence", {})
    if target == "merged":
        return {"git": collector.git(str(evidence.get("review_url", "")))}
    if target == "built":
        jenkins = collector.jenkins()
        observations = {"jenkins": jenkins}
        sha, number = jenkins.get("revision"), jenkins.get("build_number")
        if sha is not None and number is not None:
            observations["harbor"] = collector.harbor(f"git-{sha}-build-{number}-release")
        return observations
    if target == "verified":
        tag = str(evidence.get("image_ref", "")).rpartition("@")[0].rpartition(":")[2]
        return {"harbor": collector.harbor(tag)}
    if target == "deployed":
        return {"flux": collector.flux(str(evidence.get("image_ref", "")))}
    if target == "converged":
        return {"workload_spec": collector.workload_spec(), "pods": collector.pods()}
    if target == "live_verified":
        return {"health": collector.health()}
    _check(target == "rolled_back", "no collector plan exists for the target state")
    return {"rollback": collector.rollback_target()}

class ProducerClient:
    """Loopback HUX API client for the evidence identity; the key never leaves it."""

    def __init__(self, base_url: str, slot: str, subject: str, key: str, opener: Opener | None = None, timeout: float = 10.0) -> None:
        base = str(base_url or DEFAULT_BASE_URL).rstrip("/")
        parts = urlsplit(base)
        loopback = parts.scheme == "http" and parts.hostname in {"127.0.0.1", "::1", "localhost"}
        _check(parts.scheme == "https" or loopback, "HUX base URL must be loopback HTTP or HTTPS")
        self.base = base
        self.opener = opener or _default_opener
        self.timeout = timeout
        self._headers = {
            "X-Hermes-Tenant-Identity": slot, "X-Hux-Subject": subject, "X-Hux-Surface": "api",
            "X-Hux-Trust": "evidence", "X-Hux-Relay-Key": key, "Content-Type": "application/json",
        }

    def request(self, method: str, path: str, body: Any = None, headers: Mapping[str, str] | None = None) -> tuple[int, Any, bool]:
        data = None if body is None else json.dumps(body).encode()
        request = urllib.request.Request(  # noqa: S310
            self.base + path, data=data, headers={**self._headers, **(headers or {})}, method=method)
        try:
            response = self.opener(request, self.timeout, None)
        except urllib.error.HTTPError as error:
            response = error
        except OSError as error:
            raise Invalid("HUX service is unreachable") from error
        with response:
            status = int(response.status)
            raw = response.read(MAX_RESPONSE_BYTES)
            replayed = str(response.headers.get("HUX-Replayed") or "") == "true"
        try:
            parsed = json.loads(raw) if raw else None
        except ValueError:
            parsed = None
        return status, parsed, replayed

def advance(client: ProducerClient, collector: Collector, wpolicy: Mapping[str, Any], max_age: int, scope_path: str, release_id: str, now: datetime) -> dict[str, Any]:
    """Read the head, verify observations, and post at most one transition."""
    status, view, _ = client.request("GET", f"{scope_path}/{release_id}")
    _check(status == 200 and isinstance(view, dict), f"release read failed with HTTP {status}")
    release, revision = view.get("release") or {}, view.get("revision")
    _check(not isinstance(revision, bool) and isinstance(revision, int), "release view carries no revision")
    target = next_target(str(release.get("state", "")))
    if target is None:
        return {"changed": False, "release_id": release_id, "state": release.get("state"), "reason": "release is terminal"}
    observations = collect_for(collector, target, release)
    evidence = evidence_for(release, target, observations, wpolicy, max_age, now)
    status, view, replayed = client.request(
        "POST", f"{scope_path}/{release_id}/transitions", {"to": target, "evidence": evidence},
        {"If-Match": str(revision), "Idempotency-Key": f"producer-{release_id}-{target}-{revision}"[:120]})
    _check(status == 200 and isinstance(view, dict), f"release transition was rejected with HTTP {status}")
    head = view.get("release") or {}
    return {"changed": not replayed, "release_id": release_id, "state": head.get("state"),
            "revision": view.get("revision"), "replayed": replayed}

def _require(environ: Mapping[str, str], name: str) -> str:
    value = str(environ.get(name, "")).strip()
    _check(value, f"{name} is required")
    return value

def _pick_release(client: ProducerClient, scope_path: str, workload: str) -> str | None:
    status, listing, _ = client.request("GET", scope_path)
    _check(status == 200 and isinstance(listing, dict), f"release list failed with HTTP {status}")
    for item in listing.get("items") or []:
        release = (item.get("release") or {}) if isinstance(item, dict) else {}
        if release.get("workload") == workload and release.get("state") not in TERMINAL:
            return str(release.get("id"))
    return None

def run_once(environ: Mapping[str, str], opener: Opener | None = None, clock: Callable[[], datetime] | None = None) -> dict[str, Any]:
    """One idempotent producer pass wired entirely from the environment."""
    environ = dict(environ)
    workload = _require(environ, "HUX_PRODUCER_WORKLOAD")
    policy = release_security.load_policy(environ)
    wpolicy = policy["workloads"].get(workload)
    _check(wpolicy is not None, "workload is not enabled for release evidence")
    key = release_security.evidence_key(environ)
    client = ProducerClient(environ.get("HUX_BASE_URL", DEFAULT_BASE_URL), _require(environ, "HUX_TENANT_SLOT"),
                            _require(environ, "HUX_PRODUCER_SUBJECT"), key, opener, _bounded_timeout(environ))
    project = _require(environ, "HUX_PRODUCER_PROJECT_ID")
    conversation = _require(environ, "HUX_PRODUCER_CONVERSATION_ID")
    scope_path = f"/hux/v1/projects/{project}/conversations/{conversation}/releases"
    collector = Collector(environ, workload, wpolicy, opener, clock)
    release_id = str(environ.get("HUX_PRODUCER_RELEASE_ID", "")).strip() or _pick_release(client, scope_path, workload)
    if release_id is None:
        return {"changed": False, "reason": "no advanceable release"}
    return advance(client, collector, wpolicy, policy["max_evidence_age_seconds"], scope_path, release_id, collector.clock())