"""Contracts for Hermes' read-only image release follow-through helper.""" from __future__ import annotations import importlib.util import json import subprocess import sys from pathlib import Path import pytest ROOT = Path(__file__).resolve().parents[2] STATUS = ROOT / "services/hermes/scripts/hermes_image_release_status.py" TRIGGER = ROOT / "services/hermes/scripts/jenkins_image_build_trigger.py" def _load(path: Path, name: str): spec = importlib.util.spec_from_file_location(name, path) assert spec is not None and spec.loader is not None module = importlib.util.module_from_spec(spec) sys.modules[name] = module spec.loader.exec_module(module) return module def _condition(ready: bool = True, generation: int = 3) -> list[dict]: return [ { "type": "Ready", "status": "True" if ready else "False", "observedGeneration": generation, } ] def _policy(module, revision: str, digest: str, *, ready: bool = True) -> dict: return { "metadata": {"generation": 3}, "status": { "conditions": _condition(ready), "latestRef": { "tag": f"git-{revision}-build-17-release", "digest": digest, }, }, } def _controller(kind: str, repository: str, digest: str, *, ready: bool = True) -> dict: status = { "observedGeneration": 4, "readyReplicas": 1 if ready else 0, "updatedReplicas": 1, "availableReplicas": 1, "currentReplicas": 1, "currentRevision": "revision-2", "updateRevision": "revision-2", } if kind == "statefulset" and not ready: status["currentRevision"] = "revision-1" return { "metadata": {"generation": 4}, "spec": { "replicas": 1, "template": { "spec": { "initContainers": [{"image": "busybox:1"}], "containers": [{"image": f"{repository}@{digest}"}], } }, }, "status": status, } def _pods(repository: str, digest: str, *, ready: bool = True) -> dict: return { "items": [ { "status": { "conditions": [ {"type": "Ready", "status": "True" if ready else "False"} ], "containerStatuses": [ { # K3s/containerd can expose only the local image digest # here while retaining the registry identity in imageID. "image": "sha256:" + "9" * 64, "imageID": f"{repository}@{digest}", } ], } } ] } def _healthy_objects(module, component: str, revision: str, digest: str) -> dict: config = module.COMPONENTS[component] objects = { ("hermes", "imagepolicy", config["policy"]): _policy( module, revision, digest ), ("hermes", "imageupdateautomation", "hermes"): { "metadata": {"generation": 3}, "status": {"conditions": _condition()}, }, ("flux-system", "kustomization", "hermes"): { "metadata": {"generation": 3}, "status": { "conditions": _condition(), "lastAppliedRevision": "main@sha1:" + "f" * 40, }, }, } for kind, name, label in config["workloads"]: objects[("hermes", kind, name)] = _controller( kind, config["repository"], digest ) objects[("hermes", "pods", f"app={label}")] = _pods( config["repository"], digest ) return objects def _install_fake(module, monkeypatch: pytest.MonkeyPatch, objects: dict) -> None: def fake(*command): namespace = command[1] kind = command[3] if kind == "pods": key = (namespace, kind, command[5]) else: key = (namespace, kind, command[4]) return objects[key] monkeypatch.setattr(module, "_kubectl_json", fake) @pytest.mark.parametrize("component", ["agent", "router", "webui", "stt", "tts"]) def test_exact_release_convergence_covers_every_component( monkeypatch: pytest.MonkeyPatch, component: str ) -> None: """Every lane binds policy source/digest to Ready desired and running state.""" module = _load(STATUS, f"release_status_{component}") revision = "a" * 40 digest = "sha256:" + "b" * 64 _install_fake(module, monkeypatch, _healthy_objects(module, component, revision, digest)) result = module.inspect_release(component, revision) assert result["converged"] is True assert result["stage"] == "converged" assert result["selected_revision"] == revision assert result["digest"] == digest assert result["build"] == 17 assert all(item["converged"] for item in result["workloads"]) def test_pending_and_different_policy_never_claim_deployment( monkeypatch: pytest.MonkeyPatch, ) -> None: """No or different immutable source remains visibly incomplete.""" module = _load(STATUS, "release_status_policy_pending") revision = "a" * 40 digest = "sha256:" + "b" * 64 config = module.COMPONENTS["tts"] objects = _healthy_objects(module, "tts", revision, digest) policy_key = ("hermes", "imagepolicy", config["policy"]) objects[policy_key]["status"]["latestRef"] = {} _install_fake(module, monkeypatch, objects) assert module.inspect_release("tts", revision)["stage"] == "image_policy_pending" objects[policy_key] = _policy(module, "c" * 40, digest) result = module.inspect_release("tts", revision) assert result["stage"] == "different_revision_selected" assert result["selected_revision"] == "c" * 40 assert result["converged"] is False @pytest.mark.parametrize( ("mutation", "expected"), [ ("automation", "image_automation_pending"), ("flux", "flux_apply_pending"), ("desired", "flux_apply_pending"), ("pod", "rollout_pending"), ], ) def test_followthrough_reports_the_first_incomplete_delivery_stage( monkeypatch: pytest.MonkeyPatch, mutation: str, expected: str ) -> None: """The helper separates promotion, GitOps apply, and live rollout gaps.""" module = _load(STATUS, f"release_status_stage_{mutation}") revision = "d" * 40 digest = "sha256:" + "e" * 64 objects = _healthy_objects(module, "stt", revision, digest) if mutation == "automation": objects[("hermes", "imageupdateautomation", "hermes")]["status"][ "conditions" ] = _condition(False) elif mutation == "flux": objects[("flux-system", "kustomization", "hermes")]["status"][ "conditions" ] = _condition(False) elif mutation == "desired": objects[("hermes", "deployment", "hermes-stt")]["spec"]["template"][ "spec" ]["containers"][0]["image"] = "registry.bstein.dev/bstein/hermes-jetson-stt@sha256:" + "1" * 64 else: objects[("hermes", "pods", "app=hermes-stt")] = _pods( module.COMPONENTS["stt"]["repository"], digest, ready=False ) _install_fake(module, monkeypatch, objects) result = module.inspect_release("stt", revision) assert result["stage"] == expected assert result["converged"] is False def test_helpers_fail_closed_on_stale_generation_and_malformed_references() -> None: """Malformed release evidence and stale Ready conditions are not accepted.""" module = _load(STATUS, "release_status_helpers") assert module._condition_ready( {"metadata": {"generation": 4}, "status": {"conditions": _condition(True, 3)}} ) is False assert module._condition_ready({}) is False assert module._release_ref({"status": {"latestRef": {"tag": "latest"}}}) == ( "latest", None, None, None, ) assert module._rollout_ready("deployment", _controller("deployment", "r", "d"))[2] stale = _controller("deployment", "r", "d") stale["status"]["observedGeneration"] = 1 assert module._rollout_ready("deployment", stale)[2] is False with pytest.raises(ValueError): module.inspect_release("invalid", "a" * 40) with pytest.raises(ValueError): module.inspect_release("agent", "main") def test_kubectl_reader_is_bounded_and_requires_an_object( monkeypatch: pytest.MonkeyPatch, ) -> None: """Status uses argument arrays, a timeout, and object-shaped JSON only.""" module = _load(STATUS, "release_status_kubectl") seen = {} def run(command, **kwargs): seen["command"] = command seen["kwargs"] = kwargs return subprocess.CompletedProcess(command, 0, '{"ok": true}', "") monkeypatch.setattr(module.subprocess, "run", run) assert module._kubectl_json("get", "pod", "demo") == {"ok": True} assert seen["command"] == ["kubectl", "get", "pod", "demo", "-o", "json"] assert seen["kwargs"]["timeout"] == 30 monkeypatch.setattr( module.subprocess, "run", lambda *_args, **_kwargs: subprocess.CompletedProcess([], 0, "[]", ""), ) with pytest.raises(ValueError, match="non-object"): module._kubectl_json("get", "pods") def test_cli_waits_for_convergence_and_returns_safe_json( monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] ) -> None: """Wait mode polls bounded status and terminates only at convergence.""" module = _load(STATUS, "release_status_main") revision = "7" * 40 states = [ {"stage": "image_policy_pending", "converged": False}, {"stage": "converged", "converged": True, "digest": "sha256:" + "8" * 64}, ] monkeypatch.setattr(module, "inspect_release", lambda *_args: states.pop(0)) monkeypatch.setattr(module.time, "sleep", lambda _seconds: None) assert module.main( ["--component", "agent", "--revision", revision, "--wait", "--poll", "1"] ) == 0 assert json.loads(capsys.readouterr().out)["converged"] is True assert module.main( ["--component", "agent", "--revision", revision, "--timeout", "0"] ) == 2 assert "must be positive" in capsys.readouterr().err def test_trigger_returns_a_token_free_exact_follow_command(tmp_path: Path) -> None: """A successful trigger tells Hermes how to prove the approved release landed.""" module = _load(TRIGGER, "release_status_trigger") revision = "9" * 40 token = tmp_path / "token" token.write_text("never-print-me\n", encoding="utf-8") class Response: status = 201 headers = {"Location": "/queue/item/91/"} def __enter__(self): return self def __exit__(self, *_args): return None result = module.trigger_build( revision, component="webui", token_file=token, opener=lambda *_args, **_kwargs: Response(), ) assert result["follow_command"] == ( "/opt/coordinator/hermes_image_release_status.py --component webui " f"--revision {revision} --wait" ) assert "never-print-me" not in json.dumps(result) def test_trigger_cli_verifies_the_exact_revision_before_queueing( monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] ) -> None: """The CLI never spends a Jenkins build slot on a nonexistent commit.""" module = _load(TRIGGER, "release_status_trigger_revision") revision = "8" * 40 seen = [] def trigger(value, *, component, revision_verifier): revision_verifier(value) seen.append((value, component)) return {"source_revision": value, "component": component} monkeypatch.setattr(module, "trigger_build", trigger) monkeypatch.setattr(module, "_verify_reviewed_revision_exists", seen.append) monkeypatch.setattr(sys, "argv", ["trigger", "--component", "stt", revision]) assert module.main() == 0 assert seen == [revision, (revision, "stt")] assert json.loads(capsys.readouterr().out)["source_revision"] == revision def test_runtime_bundle_and_guidance_require_live_convergence() -> None: """The helper ships to Worker and merged code alone is explicitly insufficient.""" kustomization = (ROOT / "services/hermes/kustomization.yaml").read_text() guidance = (ROOT / "services/hermes/agent-configmap.yaml").read_text() assert "hermes_image_release_status.py=scripts/hermes_image_release_status.py" in kustomization assert "safe `follow_command`" in guidance assert "verifies that the requested commit is contained by main" in guidance assert "checks out and builds that exact commit" in guidance assert "builds the newest main containing that commit" not in guidance assert "Merged code is not a completed release" in guidance.replace( "merged code", "Merged code" )