359 lines
13 KiB
Python
359 lines
13 KiB
Python
"""Contracts for Hermes' read-only image release follow-through helper."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
STATUS = ROOT / "services/hermes/scripts/hermes_image_release_status.py"
|
|
TRIGGER = ROOT / "services/hermes/scripts/jenkins_image_build_trigger.py"
|
|
|
|
|
|
def _load(path: Path, name: str):
|
|
spec = importlib.util.spec_from_file_location(name, path)
|
|
assert spec is not None and spec.loader is not None
|
|
module = importlib.util.module_from_spec(spec)
|
|
sys.modules[name] = module
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
def _condition(ready: bool = True, generation: int = 3) -> list[dict]:
|
|
return [
|
|
{
|
|
"type": "Ready",
|
|
"status": "True" if ready else "False",
|
|
"observedGeneration": generation,
|
|
}
|
|
]
|
|
|
|
|
|
def _policy(module, revision: str, digest: str, *, ready: bool = True) -> dict:
|
|
return {
|
|
"metadata": {"generation": 3},
|
|
"status": {
|
|
"conditions": _condition(ready),
|
|
"latestRef": {
|
|
"tag": f"git-{revision}-build-17-release",
|
|
"digest": digest,
|
|
},
|
|
},
|
|
}
|
|
|
|
|
|
def _controller(kind: str, repository: str, digest: str, *, ready: bool = True) -> dict:
|
|
status = {
|
|
"observedGeneration": 4,
|
|
"readyReplicas": 1 if ready else 0,
|
|
"updatedReplicas": 1,
|
|
"availableReplicas": 1,
|
|
"currentReplicas": 1,
|
|
"currentRevision": "revision-2",
|
|
"updateRevision": "revision-2",
|
|
}
|
|
if kind == "statefulset" and not ready:
|
|
status["currentRevision"] = "revision-1"
|
|
return {
|
|
"metadata": {"generation": 4},
|
|
"spec": {
|
|
"replicas": 1,
|
|
"template": {
|
|
"spec": {
|
|
"initContainers": [{"image": "busybox:1"}],
|
|
"containers": [{"image": f"{repository}@{digest}"}],
|
|
}
|
|
},
|
|
},
|
|
"status": status,
|
|
}
|
|
|
|
|
|
def _pods(repository: str, digest: str, *, ready: bool = True) -> dict:
|
|
return {
|
|
"items": [
|
|
{
|
|
"status": {
|
|
"conditions": [
|
|
{"type": "Ready", "status": "True" if ready else "False"}
|
|
],
|
|
"containerStatuses": [
|
|
{
|
|
# K3s/containerd can expose only the local image digest
|
|
# here while retaining the registry identity in imageID.
|
|
"image": "sha256:" + "9" * 64,
|
|
"imageID": f"{repository}@{digest}",
|
|
}
|
|
],
|
|
}
|
|
}
|
|
]
|
|
}
|
|
|
|
|
|
def _healthy_objects(module, component: str, revision: str, digest: str) -> dict:
|
|
config = module.COMPONENTS[component]
|
|
objects = {
|
|
("hermes", "imagepolicy", config["policy"]): _policy(
|
|
module, revision, digest
|
|
),
|
|
("hermes", "imageupdateautomation", "hermes"): {
|
|
"metadata": {"generation": 3},
|
|
"status": {"conditions": _condition()},
|
|
},
|
|
("flux-system", "kustomization", "hermes"): {
|
|
"metadata": {"generation": 3},
|
|
"status": {
|
|
"conditions": _condition(),
|
|
"lastAppliedRevision": "main@sha1:" + "f" * 40,
|
|
},
|
|
},
|
|
}
|
|
for kind, name, label in config["workloads"]:
|
|
objects[("hermes", kind, name)] = _controller(
|
|
kind, config["repository"], digest
|
|
)
|
|
objects[("hermes", "pods", f"app={label}")] = _pods(
|
|
config["repository"], digest
|
|
)
|
|
return objects
|
|
|
|
|
|
def _install_fake(module, monkeypatch: pytest.MonkeyPatch, objects: dict) -> None:
|
|
def fake(*command):
|
|
namespace = command[1]
|
|
kind = command[3]
|
|
if kind == "pods":
|
|
key = (namespace, kind, command[5])
|
|
else:
|
|
key = (namespace, kind, command[4])
|
|
return objects[key]
|
|
|
|
monkeypatch.setattr(module, "_kubectl_json", fake)
|
|
|
|
|
|
@pytest.mark.parametrize("component", ["agent", "router", "webui", "stt", "tts"])
|
|
def test_exact_release_convergence_covers_every_component(
|
|
monkeypatch: pytest.MonkeyPatch, component: str
|
|
) -> None:
|
|
"""Every lane binds policy source/digest to Ready desired and running state."""
|
|
module = _load(STATUS, f"release_status_{component}")
|
|
revision = "a" * 40
|
|
digest = "sha256:" + "b" * 64
|
|
_install_fake(module, monkeypatch, _healthy_objects(module, component, revision, digest))
|
|
|
|
result = module.inspect_release(component, revision)
|
|
|
|
assert result["converged"] is True
|
|
assert result["stage"] == "converged"
|
|
assert result["selected_revision"] == revision
|
|
assert result["digest"] == digest
|
|
assert result["build"] == 17
|
|
assert all(item["converged"] for item in result["workloads"])
|
|
|
|
|
|
def test_pending_and_different_policy_never_claim_deployment(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
"""No or different immutable source remains visibly incomplete."""
|
|
module = _load(STATUS, "release_status_policy_pending")
|
|
revision = "a" * 40
|
|
digest = "sha256:" + "b" * 64
|
|
config = module.COMPONENTS["tts"]
|
|
objects = _healthy_objects(module, "tts", revision, digest)
|
|
policy_key = ("hermes", "imagepolicy", config["policy"])
|
|
objects[policy_key]["status"]["latestRef"] = {}
|
|
_install_fake(module, monkeypatch, objects)
|
|
assert module.inspect_release("tts", revision)["stage"] == "image_policy_pending"
|
|
|
|
objects[policy_key] = _policy(module, "c" * 40, digest)
|
|
result = module.inspect_release("tts", revision)
|
|
assert result["stage"] == "different_revision_selected"
|
|
assert result["selected_revision"] == "c" * 40
|
|
assert result["converged"] is False
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("mutation", "expected"),
|
|
[
|
|
("automation", "image_automation_pending"),
|
|
("flux", "flux_apply_pending"),
|
|
("desired", "flux_apply_pending"),
|
|
("pod", "rollout_pending"),
|
|
],
|
|
)
|
|
def test_followthrough_reports_the_first_incomplete_delivery_stage(
|
|
monkeypatch: pytest.MonkeyPatch, mutation: str, expected: str
|
|
) -> None:
|
|
"""The helper separates promotion, GitOps apply, and live rollout gaps."""
|
|
module = _load(STATUS, f"release_status_stage_{mutation}")
|
|
revision = "d" * 40
|
|
digest = "sha256:" + "e" * 64
|
|
objects = _healthy_objects(module, "stt", revision, digest)
|
|
if mutation == "automation":
|
|
objects[("hermes", "imageupdateautomation", "hermes")]["status"][
|
|
"conditions"
|
|
] = _condition(False)
|
|
elif mutation == "flux":
|
|
objects[("flux-system", "kustomization", "hermes")]["status"][
|
|
"conditions"
|
|
] = _condition(False)
|
|
elif mutation == "desired":
|
|
objects[("hermes", "deployment", "hermes-stt")]["spec"]["template"][
|
|
"spec"
|
|
]["containers"][0]["image"] = "registry.bstein.dev/bstein/hermes-jetson-stt@sha256:" + "1" * 64
|
|
else:
|
|
objects[("hermes", "pods", "app=hermes-stt")] = _pods(
|
|
module.COMPONENTS["stt"]["repository"], digest, ready=False
|
|
)
|
|
_install_fake(module, monkeypatch, objects)
|
|
|
|
result = module.inspect_release("stt", revision)
|
|
|
|
assert result["stage"] == expected
|
|
assert result["converged"] is False
|
|
|
|
|
|
def test_helpers_fail_closed_on_stale_generation_and_malformed_references() -> None:
|
|
"""Malformed release evidence and stale Ready conditions are not accepted."""
|
|
module = _load(STATUS, "release_status_helpers")
|
|
assert module._condition_ready(
|
|
{"metadata": {"generation": 4}, "status": {"conditions": _condition(True, 3)}}
|
|
) is False
|
|
assert module._condition_ready({}) is False
|
|
assert module._release_ref({"status": {"latestRef": {"tag": "latest"}}}) == (
|
|
"latest",
|
|
None,
|
|
None,
|
|
None,
|
|
)
|
|
assert module._rollout_ready("deployment", _controller("deployment", "r", "d"))[2]
|
|
stale = _controller("deployment", "r", "d")
|
|
stale["status"]["observedGeneration"] = 1
|
|
assert module._rollout_ready("deployment", stale)[2] is False
|
|
with pytest.raises(ValueError):
|
|
module.inspect_release("invalid", "a" * 40)
|
|
with pytest.raises(ValueError):
|
|
module.inspect_release("agent", "main")
|
|
|
|
|
|
def test_kubectl_reader_is_bounded_and_requires_an_object(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
"""Status uses argument arrays, a timeout, and object-shaped JSON only."""
|
|
module = _load(STATUS, "release_status_kubectl")
|
|
seen = {}
|
|
|
|
def run(command, **kwargs):
|
|
seen["command"] = command
|
|
seen["kwargs"] = kwargs
|
|
return subprocess.CompletedProcess(command, 0, '{"ok": true}', "")
|
|
|
|
monkeypatch.setattr(module.subprocess, "run", run)
|
|
assert module._kubectl_json("get", "pod", "demo") == {"ok": True}
|
|
assert seen["command"] == ["kubectl", "get", "pod", "demo", "-o", "json"]
|
|
assert seen["kwargs"]["timeout"] == 30
|
|
monkeypatch.setattr(
|
|
module.subprocess,
|
|
"run",
|
|
lambda *_args, **_kwargs: subprocess.CompletedProcess([], 0, "[]", ""),
|
|
)
|
|
with pytest.raises(ValueError, match="non-object"):
|
|
module._kubectl_json("get", "pods")
|
|
|
|
|
|
def test_cli_waits_for_convergence_and_returns_safe_json(
|
|
monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
|
|
) -> None:
|
|
"""Wait mode polls bounded status and terminates only at convergence."""
|
|
module = _load(STATUS, "release_status_main")
|
|
revision = "7" * 40
|
|
states = [
|
|
{"stage": "image_policy_pending", "converged": False},
|
|
{"stage": "converged", "converged": True, "digest": "sha256:" + "8" * 64},
|
|
]
|
|
monkeypatch.setattr(module, "inspect_release", lambda *_args: states.pop(0))
|
|
monkeypatch.setattr(module.time, "sleep", lambda _seconds: None)
|
|
assert module.main(
|
|
["--component", "agent", "--revision", revision, "--wait", "--poll", "1"]
|
|
) == 0
|
|
assert json.loads(capsys.readouterr().out)["converged"] is True
|
|
|
|
assert module.main(
|
|
["--component", "agent", "--revision", revision, "--timeout", "0"]
|
|
) == 2
|
|
assert "must be positive" in capsys.readouterr().err
|
|
|
|
|
|
def test_trigger_returns_a_token_free_exact_follow_command(tmp_path: Path) -> None:
|
|
"""A successful trigger tells Hermes how to prove the approved release landed."""
|
|
module = _load(TRIGGER, "release_status_trigger")
|
|
revision = "9" * 40
|
|
token = tmp_path / "token"
|
|
token.write_text("never-print-me\n", encoding="utf-8")
|
|
|
|
class Response:
|
|
status = 201
|
|
headers = {"Location": "/queue/item/91/"}
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *_args):
|
|
return None
|
|
|
|
result = module.trigger_build(
|
|
revision,
|
|
component="webui",
|
|
token_file=token,
|
|
opener=lambda *_args, **_kwargs: Response(),
|
|
)
|
|
|
|
assert result["follow_command"] == (
|
|
"/opt/coordinator/hermes_image_release_status.py --component webui "
|
|
f"--revision {revision} --wait"
|
|
)
|
|
assert "never-print-me" not in json.dumps(result)
|
|
|
|
|
|
def test_trigger_cli_verifies_the_exact_revision_before_queueing(
|
|
monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
|
|
) -> None:
|
|
"""The CLI never spends a Jenkins build slot on a nonexistent commit."""
|
|
module = _load(TRIGGER, "release_status_trigger_revision")
|
|
revision = "8" * 40
|
|
seen = []
|
|
|
|
def trigger(value, *, component, revision_verifier):
|
|
revision_verifier(value)
|
|
seen.append((value, component))
|
|
return {"source_revision": value, "component": component}
|
|
|
|
monkeypatch.setattr(module, "trigger_build", trigger)
|
|
monkeypatch.setattr(module, "_verify_reviewed_revision_exists", seen.append)
|
|
monkeypatch.setattr(sys, "argv", ["trigger", "--component", "stt", revision])
|
|
|
|
assert module.main() == 0
|
|
assert seen == [revision, (revision, "stt")]
|
|
assert json.loads(capsys.readouterr().out)["source_revision"] == revision
|
|
|
|
|
|
def test_runtime_bundle_and_guidance_require_live_convergence() -> None:
|
|
"""The helper ships to Worker and merged code alone is explicitly insufficient."""
|
|
kustomization = (ROOT / "services/hermes/kustomization.yaml").read_text()
|
|
guidance = (ROOT / "services/hermes/agent-configmap.yaml").read_text()
|
|
assert "hermes_image_release_status.py=scripts/hermes_image_release_status.py" in kustomization
|
|
assert "safe `follow_command`" in guidance
|
|
assert "verifies that the requested commit is contained by main" in guidance
|
|
assert "checks out and builds that exact commit" in guidance
|
|
assert "builds the newest main containing that commit" not in guidance
|
|
assert "Merged code is not a completed release" in guidance.replace(
|
|
"merged code", "Merged code"
|
|
)
|