"""HUX-12 release follow-through evidence producer (companion process). This package lives OUTSIDE the network-free ``hux`` service package (SO-29): it is the only component allowed to open outbound connections. It may import ``hux.errors`` and ``hux.release_security``; ``hux`` never imports it. A pure observation -> verdict core (``evidence_for``) binds every observation to the release being advanced and the workload policy, returning the exact evidence payload for the next legal transition or raising ``Invalid``. A thin ``Collector`` gathers observations from Jenkins, Harbor, the Kubernetes API and the health URL over HTTPS with an injectable transport. The driver (``advance``/``run_once``) posts at most one verified transition per pass with If-Match and a deterministic Idempotency-Key; verdicts derive only from passed observations and the injected clock, and nothing here logs the evidence key. """ from __future__ import annotations import json import re import ssl import urllib.error import urllib.request from collections.abc import Callable, Mapping from datetime import datetime, timezone from pathlib import Path from typing import Any from urllib.parse import quote, urlsplit from hux import release_security from hux.errors import Invalid ORDER = ("reviewed", "merged", "built", "verified", "deployed", "converged", "live_verified") TERMINAL = frozenset({"live_verified", "rolled_back"}) ROLLBACK_STATES = frozenset({"built", "verified", "deployed", "converged", "live_verified"}) COMMIT = re.compile(r"^[0-9a-f]{40}$") SHA = re.compile(r"^sha256:[0-9a-f]{64}$") SHA_ANY = re.compile(r"sha256:[0-9a-f]{64}") TIME = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$") FLUX_REV = re.compile(r"^main@sha1:[0-9a-f]{40}$") NAMESPACE = re.compile(r"^[a-z0-9-]{1,63}$") DEFAULT_BASE_URL = "http://127.0.0.1:8790" DEFAULT_KUBE_API = "https://kubernetes.default.svc" DEFAULT_KUBE_TOKEN = "/var/run/secrets/kubernetes.io/serviceaccount/token" # noqa: S105 MAX_RESPONSE_BYTES = 1024 * 1024 Opener = Callable[[urllib.request.Request, float, ssl.SSLContext | None], Any] def _check(condition: Any, message: str) -> None: """Fail closed with a precise reason unless ``condition`` holds.""" if not condition: raise Invalid(message) def parse_time(value: Any, label: str) -> datetime: """Parse one strict RFC 3339 UTC second-precision timestamp.""" _check(isinstance(value, str) and TIME.fullmatch(value), f"{label} must be an RFC 3339 UTC timestamp") return datetime.strptime(value, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc) def _fresh(value: Any, now: datetime, max_age: int, label: str) -> str: age = (now - parse_time(value, label)).total_seconds() _check(0 <= age <= max_age, f"stale evidence: {label} is outside the freshness window") return str(value) def _observation(observations: Mapping[str, Any], name: str, workload: str, now: datetime, max_age: int) -> dict[str, Any]: obs = observations.get(name) _check(isinstance(obs, dict), f"incomplete observations: {name} is missing") _check(obs.get("workload") == workload, f"{name} observation is bound to a different workload") _fresh(obs.get("observed_at"), now, max_age, f"{name}.observed_at") return obs def _need(obs: Mapping[str, Any], name: str, field: str, kind: type = str) -> Any: value = obs.get(field) _check(not isinstance(value, bool) and isinstance(value, kind), f"incomplete observations: {name}.{field} is missing or malformed") return value def _verify_merged(release, observations, wpolicy, max_age, now): git = _observation(observations, "git", release["workload"], now, max_age) commit = _need(git, "git", "commit") _check(COMMIT.fullmatch(commit), "git commit must be an exact 40-hex SHA") _check(parse_time(_need(git, "git", "committed_at"), "git.committed_at") <= now, "git commit timestamp is in the future") review_url = _need(git, "git", "review_url") _check(review_url == release.get("evidence", {}).get("review_url"), "git observation is for a different review than this release") _check(review_url.startswith(wpolicy["review_url_prefix"]), "review URL is outside the policy review prefix") return {"merge_commit": commit} def _verify_built(release, observations, wpolicy, max_age, now): jenkins = _observation(observations, "jenkins", release["workload"], now, max_age) harbor = _observation(observations, "harbor", release["workload"], now, max_age) sha = str(release.get("evidence", {}).get("merge_commit", "")) build_url, job = _need(jenkins, "jenkins", "build_url"), wpolicy["jenkins_job_url"] _check(build_url == job or build_url.startswith(job + "/"), "Jenkins build URL is outside the policy job") _check(_need(jenkins, "jenkins", "result") == "SUCCESS", "Jenkins build result is not SUCCESS") _check(COMMIT.fullmatch(sha) and _need(jenkins, "jenkins", "revision") == sha, "Jenkins build revision does not equal the merged commit") number = _need(jenkins, "jenkins", "build_number", int) _check(number > 0, "Jenkins build number is invalid") tag = f"git-{sha}-build-{number}-release" _check(_need(harbor, "harbor", "tag") == tag, "Harbor tag does not bind the merged commit and Jenkins build") _check(_need(harbor, "harbor", "repository") == wpolicy["image_repository"], "Harbor repository does not match the policy image repository") digest = _need(harbor, "harbor", "digest") _check(SHA.fullmatch(digest), "Harbor digest is malformed") _check(_need(jenkins, "jenkins", "image_digest") == digest, "Harbor digest does not equal the advertised image digest") ref = f"{wpolicy['image_repository']}:{tag}@{digest}" return {"ci_build_url": build_url, "image_ref": ref, "image_digest": digest, "harbor_digest": digest} def _verify_verified(release, observations, wpolicy, max_age, now): harbor = _observation(observations, "harbor", release["workload"], now, max_age) evidence = release.get("evidence", {}) tag = str(evidence.get("image_ref", "")).rpartition("@")[0].rpartition(":")[2] same = (_need(harbor, "harbor", "repository") == wpolicy["image_repository"] and tag and _need(harbor, "harbor", "tag") == tag and _need(harbor, "harbor", "digest") == evidence.get("image_digest")) _check(same, "Harbor no longer serves the built tag at the recorded digest") return {} def _verify_deployed(release, observations, wpolicy, max_age, now): flux = _observation(observations, "flux", release["workload"], now, max_age) _check(_need(flux, "flux", "name") == wpolicy["flux_kustomization"], "Flux kustomization name does not match policy") revision = _need(flux, "flux", "applied_revision") _check(FLUX_REV.fullmatch(revision), "Flux applied revision must look like main@sha1:<40-hex>") _check(flux.get("pin_in_revision") is True, "manifest pin is not proven to be contained in the applied revision") return {"flux_revision": revision} def _verify_converged(release, observations, wpolicy, max_age, now): spec = _observation(observations, "workload_spec", release["workload"], now, max_age) pods = _observation(observations, "pods", release["workload"], now, max_age) evidence = release.get("evidence", {}) ref, digest = str(evidence.get("image_ref", "")), str(evidence.get("image_digest", "")) _check(ref and _need(spec, "workload_spec", "image") == ref, "desired workload image does not equal the built release reference") entries = _need(pods, "pods", "pods", list) _check(entries, "no Ready pods were observed for the workload") for pod in entries: _check(isinstance(pod, dict) and pod.get("ready") is True, "a workload pod is not Ready") image_id = pod.get("image_id") _check(isinstance(image_id, str) and image_id.endswith("@" + digest), "a Ready pod is not running the release digest") return {"pod_digest": digest} def _verify_live(release, observations, wpolicy, max_age, now): health = _observation(observations, "health", release["workload"], now, max_age) _check(_need(health, "health", "url") == wpolicy["health_url"], "health receipt is not for the policy health URL") status = _need(health, "health", "status", int) _check(status == 200, f"health check returned HTTP {status}, not 200") checked = _fresh(health.get("checked_at"), now, max_age, "health.checked_at") return {"health_check": {"url": health["url"], "status": "pass", "at": checked}} def _verify_rollback(release, observations, wpolicy, max_age, now): rollback = _observation(observations, "rollback", release["workload"], now, max_age) digest = _need(rollback, "rollback", "digest") _check(SHA.fullmatch(digest), "rollback target digest is malformed") return {"rollback_target": digest} _VERIFIERS = { "merged": _verify_merged, "built": _verify_built, "verified": _verify_verified, "deployed": _verify_deployed, "converged": _verify_converged, "live_verified": _verify_live, "rolled_back": _verify_rollback, } def next_target(state: str) -> str | None: """The only legal forward transition from ``state``; None when terminal.""" if state in TERMINAL: return None _check(state in ORDER, "release is in an unknown state") return ORDER[ORDER.index(state) + 1] def evidence_for(release: Mapping[str, Any], target: str, observations: Mapping[str, Any], wpolicy: Mapping[str, Any], max_age: int, now: datetime) -> dict[str, Any]: """Verify observations for exactly one legal transition or raise Invalid.""" _check(now.tzinfo is not None, "now must be a timezone-aware UTC datetime") state = str(release.get("state", "")) if target == "rolled_back": _check(state in ROLLBACK_STATES, "there is no deployed image to roll back") else: _check(target == next_target(state), f"transition to {target} would skip, repeat or downgrade from {state}") return _VERIFIERS[target](dict(release), observations, wpolicy, int(max_age), now) def _default_opener(request: urllib.request.Request, timeout: float, context: ssl.SSLContext | None): return urllib.request.urlopen(request, timeout=timeout, context=context) # noqa: S310 def _context(ca_file: str) -> ssl.SSLContext: """A verifying TLS context, trusting the given CA bundle when one is mounted.""" try: return ssl.create_default_context(cafile=ca_file or None) except (OSError, ssl.SSLError) as error: raise Invalid("collector CA bundle is unavailable") from error def _bounded_timeout(environ: Mapping[str, str]) -> float: try: value = float(environ.get("HUX_PRODUCER_TIMEOUT_SECONDS", 10.0)) except (TypeError, ValueError): return 10.0 return value if 0.1 <= value <= 60.0 else 10.0 class Collector: """Observation gatherer; every field is observed via the transport, never asserted.""" def __init__(self, environ: Mapping[str, str], workload: str, wpolicy: Mapping[str, Any], opener: Opener | None = None, clock: Callable[[], datetime] | None = None) -> None: self.environ = dict(environ) self.workload = workload self.wpolicy = dict(wpolicy) self.opener = opener or _default_opener self.clock = clock or (lambda: datetime.now(timezone.utc)) self.timeout = _bounded_timeout(environ) def _stamp(self, fields: dict[str, Any]) -> dict[str, Any]: observed = self.clock().strftime("%Y-%m-%dT%H:%M:%SZ") return {"workload": self.workload, "observed_at": observed, **fields} def _fetch(self, url: str, label: str, headers: Mapping[str, str] | None = None, context: ssl.SSLContext | None = None) -> tuple[int, bytes]: _check(urlsplit(url).scheme == "https", f"{label} observation URL must be HTTPS") request = urllib.request.Request(url, headers=dict(headers or {}), method="GET") # noqa: S310 try: with self.opener(request, self.timeout, context or _context("")) as response: status, data = int(response.status), response.read(MAX_RESPONSE_BYTES + 1) except urllib.error.HTTPError as error: return int(error.code), b"" except (OSError, ValueError) as error: raise Invalid(f"{label} observation fetch failed") from error _check(len(data) <= MAX_RESPONSE_BYTES, f"{label} observation response is too large") return status, data def _json(self, url: str, label: str, headers: Mapping[str, str] | None = None, context: ssl.SSLContext | None = None) -> dict[str, Any]: status, data = self._fetch(url, label, headers, context) _check(status == 200, f"{label} observation returned HTTP {status}") try: parsed = json.loads(data) except ValueError as error: raise Invalid(f"{label} observation is not JSON") from error _check(isinstance(parsed, dict), f"{label} observation is not a JSON object") return parsed def git(self, review_url: str) -> dict[str, Any]: """Observe the merged proposal named by the release's recorded review URL.""" prefix = self.wpolicy["review_url_prefix"] number = review_url[len(prefix):] if review_url.startswith(prefix) else "" _check(number.isdigit(), "release review URL is outside the policy review prefix") parsed = urlsplit(prefix) parts = parsed.path.strip("/").split("/") _check(len(parts) == 3 and parts[2] == "pulls", "policy review prefix is not a proposal URL") body = self._json(f"https://{parsed.netloc}/api/v1/repos/{parts[0]}/{parts[1]}/pulls/{number}", "git") fields: dict[str, Any] = {"review_url": review_url} commit, merged_at = body.get("merge_commit_sha"), body.get("merged_at") if body.get("merged") is True and isinstance(commit, str): fields["commit"] = commit.lower() if isinstance(merged_at, str) and TIME.fullmatch(merged_at): fields["committed_at"] = merged_at return self._stamp(fields) def jenkins(self) -> dict[str, Any]: """Observe the last successful build of the policy job.""" body = self._json(self.wpolicy["jenkins_job_url"] + "/lastSuccessfulBuild/api/json", "jenkins") fields: dict[str, Any] = {} url, result, number = body.get("url"), body.get("result"), body.get("number") if isinstance(url, str): fields["build_url"] = url.rstrip("/") if isinstance(result, str): fields["result"] = result if isinstance(number, int) and not isinstance(number, bool): fields["build_number"] = number for action in body.get("actions") or []: revision = action.get("lastBuiltRevision") if isinstance(action, dict) else None sha = revision.get("SHA1") if isinstance(revision, dict) else None if isinstance(sha, str) and COMMIT.fullmatch(sha.lower()): fields["revision"] = sha.lower() break advertised = SHA_ANY.search(str(body.get("description") or "")) if advertised: fields["image_digest"] = advertised.group(0) return self._stamp(fields) def harbor(self, tag: str) -> dict[str, Any]: """Observe the immutable Harbor artifact behind one release tag.""" repository = self.wpolicy["image_repository"] host, _, path = repository.partition("/") project, _, name = path.partition("/") _check(host and project and name, "policy image repository is not host/project/name") body = self._json(f"https://{host}/api/v2.0/projects/{project}/repositories/{quote(name, safe='')}/artifacts/{quote(tag, safe='')}", "harbor") fields: dict[str, Any] = {"repository": repository} digest = body.get("digest") if isinstance(digest, str) and SHA.fullmatch(digest): fields["digest"] = digest names = [entry.get("name") for entry in body.get("tags") or [] if isinstance(entry, dict)] if tag in names: fields["tag"] = tag return self._stamp(fields) def _namespace(self) -> str: namespace = self.environ.get("HUX_PRODUCER_NAMESPACE", "") _check(NAMESPACE.fullmatch(namespace), "HUX_PRODUCER_NAMESPACE is missing or malformed") return namespace def _kube(self, path: str) -> dict[str, Any]: base = self.environ.get("HUX_PRODUCER_KUBE_API", DEFAULT_KUBE_API).rstrip("/") token_file = self.environ.get("HUX_PRODUCER_KUBE_TOKEN_FILE", DEFAULT_KUBE_TOKEN) try: token = Path(token_file).read_text(encoding="utf-8").strip() except (OSError, UnicodeDecodeError) as error: raise Invalid("kubernetes credentials are unavailable") from error _check(token, "kubernetes credentials are unavailable") context = _context(self.environ.get("HUX_PRODUCER_KUBE_CA_FILE", "")) return self._json(base + path, "kubernetes", {"Authorization": "Bearer " + token}, context) def _workload_image(self) -> str | None: kind = self.environ.get("HUX_PRODUCER_WORKLOAD_KIND", "statefulset").lower() plural = {"deployment": "deployments", "statefulset": "statefulsets"}.get(kind) _check(plural, "HUX_PRODUCER_WORKLOAD_KIND must be deployment or statefulset") name = self.environ.get("HUX_PRODUCER_WORKLOAD_NAME", self.workload) body = self._kube(f"/apis/apps/v1/namespaces/{self._namespace()}/{plural}/{name}") containers = (((body.get("spec") or {}).get("template") or {}).get("spec") or {}).get("containers") or [] prefix = self.wpolicy["image_repository"] + ":" for container in containers: image = container.get("image") if isinstance(container, dict) else None if isinstance(image, str) and image.startswith(prefix): return image return None def workload_spec(self) -> dict[str, Any]: """Observe the live desired image of the policy workload.""" image = self._workload_image() return self._stamp({"image": image} if image is not None else {}) def flux(self, expected_image: str) -> dict[str, Any]: """Observe the policy Kustomization; pin containment is proven, not asserted.""" name = self.wpolicy["flux_kustomization"] body = self._kube(f"/apis/kustomize.toolkit.fluxcd.io/v1/namespaces/{self._namespace()}/kustomizations/{name}") status = body.get("status") or {} applied = status.get("lastAppliedRevision") fields: dict[str, Any] = {"name": (body.get("metadata") or {}).get("name")} if isinstance(applied, str): fields["applied_revision"] = applied ready = any(isinstance(c, dict) and c.get("type") == "Ready" and c.get("status") == "True" for c in status.get("conditions") or []) settled = isinstance(applied, str) and status.get("lastAttemptedRevision") in (None, applied) pinned = bool(expected_image) and self._workload_image() == expected_image fields["pin_in_revision"] = bool(ready and settled and pinned) return self._stamp(fields) def pods(self) -> dict[str, Any]: """Observe readiness and running imageID for every selected pod.""" selector = self.environ.get("HUX_PRODUCER_POD_SELECTOR", "") _check(selector, "HUX_PRODUCER_POD_SELECTOR is required to observe pods") body = self._kube(f"/api/v1/namespaces/{self._namespace()}/pods?labelSelector={quote(selector, safe='=,')}") prefix = self.wpolicy["image_repository"] + ":" entries = [] for item in body.get("items") or []: if not isinstance(item, dict): continue status = item.get("status") or {} ready = any(isinstance(c, dict) and c.get("type") == "Ready" and c.get("status") == "True" for c in status.get("conditions") or []) image_id = None for container in status.get("containerStatuses") or []: if isinstance(container, dict) and str(container.get("image", "")).startswith(prefix): image_id = container.get("imageID") break entries.append({"name": (item.get("metadata") or {}).get("name"), "ready": ready, "image_id": image_id}) return self._stamp({"pods": entries}) def health(self) -> dict[str, Any]: """Observe the policy health URL; the receipt carries the real HTTP status.""" url = self.wpolicy["health_url"] checked = self.clock().strftime("%Y-%m-%dT%H:%M:%SZ") status, _ = self._fetch(url, "health") return self._stamp({"url": url, "status": status, "checked_at": checked}) def rollback_target(self) -> dict[str, Any]: """Observe the digest currently pinned in the live workload spec.""" image = self._workload_image() fields: dict[str, Any] = {} digest = image.rpartition("@")[2] if isinstance(image, str) else "" if SHA.fullmatch(digest): fields["digest"] = digest return self._stamp(fields) def collect_for(collector: Collector, target: str, release: Mapping[str, Any]) -> dict[str, Any]: """Gather exactly the observations the verifier needs for one target state.""" evidence = release.get("evidence", {}) if target == "merged": return {"git": collector.git(str(evidence.get("review_url", "")))} if target == "built": jenkins = collector.jenkins() observations = {"jenkins": jenkins} sha, number = jenkins.get("revision"), jenkins.get("build_number") if sha is not None and number is not None: observations["harbor"] = collector.harbor(f"git-{sha}-build-{number}-release") return observations if target == "verified": tag = str(evidence.get("image_ref", "")).rpartition("@")[0].rpartition(":")[2] return {"harbor": collector.harbor(tag)} if target == "deployed": return {"flux": collector.flux(str(evidence.get("image_ref", "")))} if target == "converged": return {"workload_spec": collector.workload_spec(), "pods": collector.pods()} if target == "live_verified": return {"health": collector.health()} _check(target == "rolled_back", "no collector plan exists for the target state") return {"rollback": collector.rollback_target()} class ProducerClient: """Loopback HUX API client for the evidence identity; the key never leaves it.""" def __init__(self, base_url: str, slot: str, subject: str, key: str, opener: Opener | None = None, timeout: float = 10.0) -> None: base = str(base_url or DEFAULT_BASE_URL).rstrip("/") parts = urlsplit(base) loopback = parts.scheme == "http" and parts.hostname in {"127.0.0.1", "::1", "localhost"} _check(parts.scheme == "https" or loopback, "HUX base URL must be loopback HTTP or HTTPS") self.base = base self.opener = opener or _default_opener self.timeout = timeout self._headers = { "X-Hermes-Tenant-Identity": slot, "X-Hux-Subject": subject, "X-Hux-Surface": "api", "X-Hux-Trust": "evidence", "X-Hux-Relay-Key": key, "Content-Type": "application/json", } def request(self, method: str, path: str, body: Any = None, headers: Mapping[str, str] | None = None) -> tuple[int, Any, bool]: data = None if body is None else json.dumps(body).encode() request = urllib.request.Request( # noqa: S310 self.base + path, data=data, headers={**self._headers, **(headers or {})}, method=method) try: response = self.opener(request, self.timeout, None) except urllib.error.HTTPError as error: response = error except OSError as error: raise Invalid("HUX service is unreachable") from error with response: status = int(response.status) raw = response.read(MAX_RESPONSE_BYTES) replayed = str(response.headers.get("HUX-Replayed") or "") == "true" try: parsed = json.loads(raw) if raw else None except ValueError: parsed = None return status, parsed, replayed def advance(client: ProducerClient, collector: Collector, wpolicy: Mapping[str, Any], max_age: int, scope_path: str, release_id: str, now: datetime) -> dict[str, Any]: """Read the head, verify observations, and post at most one transition.""" status, view, _ = client.request("GET", f"{scope_path}/{release_id}") _check(status == 200 and isinstance(view, dict), f"release read failed with HTTP {status}") release, revision = view.get("release") or {}, view.get("revision") _check(not isinstance(revision, bool) and isinstance(revision, int), "release view carries no revision") target = next_target(str(release.get("state", ""))) if target is None: return {"changed": False, "release_id": release_id, "state": release.get("state"), "reason": "release is terminal"} observations = collect_for(collector, target, release) evidence = evidence_for(release, target, observations, wpolicy, max_age, now) status, view, replayed = client.request( "POST", f"{scope_path}/{release_id}/transitions", {"to": target, "evidence": evidence}, {"If-Match": str(revision), "Idempotency-Key": f"producer-{release_id}-{target}-{revision}"[:120]}) _check(status == 200 and isinstance(view, dict), f"release transition was rejected with HTTP {status}") head = view.get("release") or {} return {"changed": not replayed, "release_id": release_id, "state": head.get("state"), "revision": view.get("revision"), "replayed": replayed} def _require(environ: Mapping[str, str], name: str) -> str: value = str(environ.get(name, "")).strip() _check(value, f"{name} is required") return value def _pick_release(client: ProducerClient, scope_path: str, workload: str) -> str | None: status, listing, _ = client.request("GET", scope_path) _check(status == 200 and isinstance(listing, dict), f"release list failed with HTTP {status}") for item in listing.get("items") or []: release = (item.get("release") or {}) if isinstance(item, dict) else {} if release.get("workload") == workload and release.get("state") not in TERMINAL: return str(release.get("id")) return None def run_once(environ: Mapping[str, str], opener: Opener | None = None, clock: Callable[[], datetime] | None = None) -> dict[str, Any]: """One idempotent producer pass wired entirely from the environment.""" environ = dict(environ) workload = _require(environ, "HUX_PRODUCER_WORKLOAD") policy = release_security.load_policy(environ) wpolicy = policy["workloads"].get(workload) _check(wpolicy is not None, "workload is not enabled for release evidence") key = release_security.evidence_key(environ) client = ProducerClient(environ.get("HUX_BASE_URL", DEFAULT_BASE_URL), _require(environ, "HUX_TENANT_SLOT"), _require(environ, "HUX_PRODUCER_SUBJECT"), key, opener, _bounded_timeout(environ)) project = _require(environ, "HUX_PRODUCER_PROJECT_ID") conversation = _require(environ, "HUX_PRODUCER_CONVERSATION_ID") scope_path = f"/hux/v1/projects/{project}/conversations/{conversation}/releases" collector = Collector(environ, workload, wpolicy, opener, clock) release_id = str(environ.get("HUX_PRODUCER_RELEASE_ID", "")).strip() or _pick_release(client, scope_path, workload) if release_id is None: return {"changed": False, "reason": "no advanceable release"} return advance(client, collector, wpolicy, policy["max_evidence_age_seconds"], scope_path, release_id, collector.clock())