"""Contracts for Hermes' read-only image release follow-through helper.""" from __future__ import annotations import importlib.util import json import subprocess import sys from pathlib import Path import pytest ROOT = Path(__file__).resolve().parents[2] STATUS = ROOT / "services/hermes/scripts/hermes_image_release_status.py" TRIGGER = ROOT / "services/hermes/scripts/jenkins_image_build_trigger.py" def _load(path: Path, name: str): spec = importlib.util.spec_from_file_location(name, path) assert spec is not None and spec.loader is not None module = importlib.util.module_from_spec(spec) sys.modules[name] = module spec.loader.exec_module(module) return module def _condition(ready: bool = True, generation: int = 3) -> list[dict]: return [ { "type": "Ready", "status": "True" if ready else "False", "observedGeneration": generation, } ] def _policy(module, revision: str, digest: str, *, ready: bool = True) -> dict: return { "metadata": {"generation": 3}, "status": { "conditions": _condition(ready), "latestRef": { "tag": f"git-{revision}-build-17-release", "digest": digest, }, }, } def _controller(kind: str, repository: str, digest: str, *, ready: bool = True) -> dict: status = { "observedGeneration": 4, "readyReplicas": 1 if ready else 0, "updatedReplicas": 1, "availableReplicas": 1, "currentReplicas": 1, "currentRevision": "revision-2", "updateRevision": "revision-2", } if kind == "statefulset" and not ready: status["currentRevision"] = "revision-1" return { "metadata": {"generation": 4}, "spec": { "replicas": 1, "template": { "spec": { "initContainers": [{"image": "busybox:1"}], "containers": [{"image": f"{repository}@{digest}"}], } }, }, "status": status, } def _pods(repository: str, digest: str, *, ready: bool = True) -> dict: return { "items": [ { "status": { "conditions": [ {"type": "Ready", "status": "True" if ready else "False"} ], "containerStatuses": [ { # K3s/containerd can expose only the local image digest # here while retaining the registry identity in imageID. "image": "sha256:" + "9" * 64, "imageID": f"{repository}@{digest}", } ], } } ] } def _healthy_objects(module, component: str, revision: str, digest: str) -> dict: config = module.COMPONENTS[component] objects = { ("hermes", "imagepolicy", config["policy"]): _policy( module, revision, digest ), ("hermes", "imageupdateautomation", "hermes"): { "metadata": {"generation": 3}, "status": {"conditions": _condition()}, }, ("flux-system", "kustomization", "hermes"): { "metadata": {"generation": 3}, "status": { "conditions": _condition(), "lastAppliedRevision": "main@sha1:" + "f" * 40, }, }, } for kind, name, label in config["workloads"]: objects[("hermes", kind, name)] = _controller( kind, config["repository"], digest ) objects[("hermes", "pods", f"app={label}")] = _pods( config["repository"], digest ) return objects def _install_fake(module, monkeypatch: pytest.MonkeyPatch, objects: dict) -> None: def fake(*command): namespace = command[1] kind = command[3] if kind == "pods": key = (namespace, kind, command[5]) else: key = (namespace, kind, command[4]) return objects[key] monkeypatch.setattr(module, "_kubectl_json", fake) @pytest.mark.parametrize("component", ["agent", "router", "webui", "stt", "tts"]) def test_exact_release_convergence_covers_every_component( monkeypatch: pytest.MonkeyPatch, component: str ) -> None: """Every lane binds policy source/digest to Ready desired and running state.""" module = _load(STATUS, f"release_status_{component}") revision = "a" * 40 digest = "sha256:" + "b" * 64 _install_fake(module, monkeypatch, _healthy_objects(module, component, revision, digest)) result = module.inspect_release(component, revision) assert result["converged"] is True assert result["stage"] == "converged" assert result["selected_revision"] == revision assert result["digest"] == digest assert result["build"] == 17 assert all(item["converged"] for item in result["workloads"]) def test_pending_and_different_policy_never_claim_deployment( monkeypatch: pytest.MonkeyPatch, ) -> None: """No or different immutable source remains visibly incomplete.""" module = _load(STATUS, "release_status_policy_pending") revision = "a" * 40 digest = "sha256:" + "b" * 64 config = module.COMPONENTS["tts"] objects = _healthy_objects(module, "tts", revision, digest) policy_key = ("hermes", "imagepolicy", config["policy"]) objects[policy_key]["status"]["latestRef"] = {} _install_fake(module, monkeypatch, objects) assert module.inspect_release("tts", revision)["stage"] == "image_policy_pending" objects[policy_key] = _policy(module, "c" * 40, digest) result = module.inspect_release("tts", revision) assert result["stage"] == "different_revision_selected" assert result["selected_revision"] == "c" * 40 assert result["converged"] is False @pytest.mark.parametrize( ("mutation", "expected"), [ ("automation", "image_automation_pending"), ("flux", "flux_apply_pending"), ("desired", "flux_apply_pending"), ("pod", "rollout_pending"), ], ) def test_followthrough_reports_the_first_incomplete_delivery_stage( monkeypatch: pytest.MonkeyPatch, mutation: str, expected: str ) -> None: """The helper separates promotion, GitOps apply, and live rollout gaps.""" module = _load(STATUS, f"release_status_stage_{mutation}") revision = "d" * 40 digest = "sha256:" + "e" * 64 objects = _healthy_objects(module, "stt", revision, digest) if mutation == "automation": objects[("hermes", "imageupdateautomation", "hermes")]["status"][ "conditions" ] = _condition(False) elif mutation == "flux": objects[("flux-system", "kustomization", "hermes")]["status"][ "conditions" ] = _condition(False) elif mutation == "desired": objects[("hermes", "deployment", "hermes-stt")]["spec"]["template"][ "spec" ]["containers"][0]["image"] = "registry.bstein.dev/bstein/hermes-jetson-stt@sha256:" + "1" * 64 else: objects[("hermes", "pods", "app=hermes-stt")] = _pods( module.COMPONENTS["stt"]["repository"], digest, ready=False ) _install_fake(module, monkeypatch, objects) result = module.inspect_release("stt", revision) assert result["stage"] == expected assert result["converged"] is False def test_helpers_fail_closed_on_stale_generation_and_malformed_references() -> None: """Malformed release evidence and stale Ready conditions are not accepted.""" module = _load(STATUS, "release_status_helpers") assert module._condition_ready( {"metadata": {"generation": 4}, "status": {"conditions": _condition(True, 3)}} ) is False assert module._condition_ready({}) is False assert module._release_ref({"status": {"latestRef": {"tag": "latest"}}}) == ( "latest", None, None, None, ) assert module._rollout_ready("deployment", _controller("deployment", "r", "d"))[2] stale = _controller("deployment", "r", "d") stale["status"]["observedGeneration"] = 1 assert module._rollout_ready("deployment", stale)[2] is False with pytest.raises(ValueError): module.inspect_release("invalid", "a" * 40) with pytest.raises(ValueError): module.inspect_release("agent", "main") def test_kubectl_reader_is_bounded_and_requires_an_object( monkeypatch: pytest.MonkeyPatch, ) -> None: """Status uses argument arrays, a timeout, and object-shaped JSON only.""" module = _load(STATUS, "release_status_kubectl") seen = {} def run(command, **kwargs): seen["command"] = command seen["kwargs"] = kwargs return subprocess.CompletedProcess(command, 0, '{"ok": true}', "") monkeypatch.setattr(module.subprocess, "run", run) assert module._kubectl_json("get", "pod", "demo") == {"ok": True} assert seen["command"] == ["kubectl", "get", "pod", "demo", "-o", "json"] assert seen["kwargs"]["timeout"] == 30 monkeypatch.setattr( module.subprocess, "run", lambda *_args, **_kwargs: subprocess.CompletedProcess([], 0, "[]", ""), ) with pytest.raises(ValueError, match="non-object"): module._kubectl_json("get", "pods") def test_cli_waits_for_convergence_and_returns_safe_json( monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] ) -> None: """Wait mode polls bounded status and terminates only at convergence.""" module = _load(STATUS, "release_status_main") revision = "7" * 40 states = [ {"stage": "image_policy_pending", "converged": False}, {"stage": "converged", "converged": True, "digest": "sha256:" + "8" * 64}, ] monkeypatch.setattr(module, "inspect_release", lambda *_args: states.pop(0)) monkeypatch.setattr(module.time, "sleep", lambda _seconds: None) assert module.main( ["--component", "agent", "--revision", revision, "--wait", "--poll", "1"] ) == 0 assert json.loads(capsys.readouterr().out)["converged"] is True assert module.main( ["--component", "agent", "--revision", revision, "--timeout", "0"] ) == 2 assert "must be positive" in capsys.readouterr().err def test_trigger_returns_a_token_free_exact_follow_command(tmp_path: Path) -> None: """A successful trigger tells Hermes how to prove the approved release landed.""" module = _load(TRIGGER, "release_status_trigger") revision = "9" * 40 token = tmp_path / "token" token.write_text("never-print-me\n", encoding="utf-8") class Response: status = 201 headers = {"Location": "/queue/item/91/"} def __enter__(self): return self def __exit__(self, *_args): return None result = module.trigger_build( revision, component="webui", token_file=token, opener=lambda *_args, **_kwargs: Response(), ) assert result["follow_command"] == ( "/opt/coordinator/hermes_image_release_status.py --component webui " f"--revision {revision} --wait" ) assert "never-print-me" not in json.dumps(result) def test_trigger_cli_verifies_the_exact_revision_before_queueing( monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] ) -> None: """The CLI never spends a Jenkins build slot on a nonexistent commit.""" module = _load(TRIGGER, "release_status_trigger_revision") revision = "8" * 40 seen = [] def trigger(value, *, component, revision_verifier): revision_verifier(value) seen.append((value, component)) return {"source_revision": value, "component": component} monkeypatch.setattr(module, "trigger_build", trigger) monkeypatch.setattr(module, "_verify_reviewed_revision_exists", seen.append) monkeypatch.setattr(sys, "argv", ["trigger", "--component", "stt", revision]) assert module.main() == 0 assert seen == [revision, (revision, "stt")] assert json.loads(capsys.readouterr().out)["source_revision"] == revision def test_runtime_bundle_and_guidance_require_live_convergence() -> None: """The helper ships to Worker and merged code alone is explicitly insufficient.""" kustomization = (ROOT / "services/hermes/kustomization.yaml").read_text() guidance = (ROOT / "services/hermes/agent-configmap.yaml").read_text() assert "hermes_image_release_status.py=scripts/hermes_image_release_status.py" in kustomization assert "safe `follow_command`" in guidance assert "Merged code is not a completed release" in guidance.replace( "merged code", "Merged code" )