33c6f187b3
Some checks failed
Tests / Declarative: Post Actions failed: 49, skipped: 19, passed: 2816
281 lines
10 KiB
Python
Executable File
281 lines
10 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Report whether a validated Hermes image has converged through Flux."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from dataclasses import asdict, dataclass
|
|
|
|
|
|
NAMESPACE = "hermes"
|
|
FLUX_NAMESPACE = "flux-system"
|
|
AUTOMATION = "hermes"
|
|
FLUX_KUSTOMIZATION = "hermes"
|
|
REVISION_PATTERN = re.compile(r"^[0-9a-f]{40}$")
|
|
RELEASE_TAG_PATTERN = re.compile(
|
|
r"^git-(?P<revision>[0-9a-f]{40})-build-(?P<build>[1-9][0-9]*)-release$"
|
|
)
|
|
COMPONENTS = {
|
|
"agent": {
|
|
"policy": "hermes-agent-release",
|
|
"repository": "registry.bstein.dev/bstein/hermes-agent",
|
|
"workloads": (("deployment", "hermes-agent", "hermes-agent"),),
|
|
},
|
|
"webui": {
|
|
"policy": "hermes-webui-release",
|
|
"repository": "registry.bstein.dev/bstein/hermes-webui",
|
|
"workloads": (
|
|
("deployment", "hermes", "hermes"),
|
|
("statefulset", "hermes-chat-tenant", "hermes-chat-tenant"),
|
|
),
|
|
},
|
|
"stt": {
|
|
"policy": "hermes-stt-release",
|
|
"repository": "registry.bstein.dev/bstein/hermes-jetson-stt",
|
|
"workloads": (("deployment", "hermes-stt", "hermes-stt"),),
|
|
},
|
|
"tts": {
|
|
"policy": "hermes-tts-release",
|
|
"repository": "registry.bstein.dev/bstein/hermes-jetson-tts",
|
|
"workloads": (("deployment", "hermes-tts", "hermes-tts"),),
|
|
},
|
|
}
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class WorkloadStatus:
|
|
"""Desired and observed state for one image-consuming workload."""
|
|
|
|
kind: str
|
|
name: str
|
|
desired_replicas: int
|
|
ready_replicas: int
|
|
desired_image_matches: bool
|
|
ready_pods: int
|
|
matching_pods: int
|
|
converged: bool
|
|
|
|
|
|
def _kubectl_json(*command: str) -> dict:
|
|
"""Read one Kubernetes object as JSON without shell or credential output."""
|
|
result = subprocess.run(
|
|
["kubectl", *command, "-o", "json"],
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=30,
|
|
)
|
|
value = json.loads(result.stdout)
|
|
if not isinstance(value, dict):
|
|
raise ValueError("kubectl returned a non-object JSON value")
|
|
return value
|
|
|
|
|
|
def _condition_ready(resource: dict) -> bool:
|
|
"""Return true only for an explicit current Ready condition."""
|
|
generation = resource.get("metadata", {}).get("generation")
|
|
for condition in resource.get("status", {}).get("conditions", []):
|
|
if condition.get("type") != "Ready" or condition.get("status") != "True":
|
|
continue
|
|
observed = condition.get("observedGeneration")
|
|
return generation is None or observed is None or int(observed) >= int(generation)
|
|
return False
|
|
|
|
|
|
def _release_ref(policy: dict) -> tuple[str | None, str | None, str | None, int | None]:
|
|
"""Extract the immutable release tag, digest, source revision, and build."""
|
|
ref = policy.get("status", {}).get("latestRef", {})
|
|
tag = ref.get("tag")
|
|
digest = ref.get("digest")
|
|
match = RELEASE_TAG_PATTERN.fullmatch(tag or "")
|
|
if not match or not re.fullmatch(r"sha256:[0-9a-f]{64}", digest or ""):
|
|
return tag, digest, None, None
|
|
return tag, digest, match.group("revision"), int(match.group("build"))
|
|
|
|
|
|
def _desired_images(workload: dict) -> list[str]:
|
|
"""Return pod-template container and init-container image references."""
|
|
spec = workload.get("spec", {}).get("template", {}).get("spec", {})
|
|
containers = list(spec.get("initContainers", [])) + list(spec.get("containers", []))
|
|
return [item.get("image", "") for item in containers]
|
|
|
|
|
|
def _rollout_ready(kind: str, workload: dict) -> tuple[int, int, bool]:
|
|
"""Evaluate generation and replica convergence for a workload controller."""
|
|
metadata = workload.get("metadata", {})
|
|
spec = workload.get("spec", {})
|
|
status = workload.get("status", {})
|
|
desired = int(spec.get("replicas", 1))
|
|
ready = int(status.get("readyReplicas", 0))
|
|
generation_ready = int(status.get("observedGeneration", 0)) >= int(
|
|
metadata.get("generation", 1)
|
|
)
|
|
if kind == "deployment":
|
|
controller_ready = (
|
|
int(status.get("updatedReplicas", 0)) >= desired
|
|
and int(status.get("availableReplicas", 0)) >= desired
|
|
)
|
|
else:
|
|
controller_ready = (
|
|
int(status.get("currentReplicas", 0)) >= desired
|
|
and int(status.get("updatedReplicas", 0)) >= desired
|
|
and status.get("currentRevision") == status.get("updateRevision")
|
|
)
|
|
return desired, ready, generation_ready and controller_ready and ready >= desired
|
|
|
|
|
|
def _pod_image_status(label: str, repository: str, digest: str) -> tuple[int, int]:
|
|
"""Count ready pods whose relevant running containers use the exact digest."""
|
|
pod_list = _kubectl_json(
|
|
"-n", NAMESPACE, "get", "pods", "-l", f"app={label}"
|
|
)
|
|
matching = 0
|
|
ready = 0
|
|
for pod in pod_list.get("items", []):
|
|
statuses = list(pod.get("status", {}).get("initContainerStatuses", []))
|
|
statuses.extend(pod.get("status", {}).get("containerStatuses", []))
|
|
relevant = [
|
|
item
|
|
for item in statuses
|
|
if item.get("image", "").startswith(repository)
|
|
or f"{repository}@" in (item.get("imageID") or "")
|
|
]
|
|
if not relevant:
|
|
continue
|
|
if all((item.get("imageID") or "").endswith(f"@{digest}") for item in relevant):
|
|
matching += 1
|
|
conditions = pod.get("status", {}).get("conditions", [])
|
|
if any(
|
|
item.get("type") == "Ready" and item.get("status") == "True"
|
|
for item in conditions
|
|
):
|
|
ready += 1
|
|
return matching, ready
|
|
|
|
|
|
def _workload_status(
|
|
kind: str, name: str, label: str, repository: str, digest: str
|
|
) -> WorkloadStatus:
|
|
"""Read desired image, controller rollout, and running-pod digest evidence."""
|
|
workload = _kubectl_json("-n", NAMESPACE, "get", kind, name)
|
|
desired, ready_replicas, controller_ready = _rollout_ready(kind, workload)
|
|
relevant_images = [
|
|
image for image in _desired_images(workload) if image.startswith(repository)
|
|
]
|
|
desired_matches = bool(relevant_images) and all(
|
|
image.endswith(f"@{digest}") for image in relevant_images
|
|
)
|
|
matching_pods, ready_pods = _pod_image_status(label, repository, digest)
|
|
converged = (
|
|
desired_matches
|
|
and controller_ready
|
|
and matching_pods >= desired
|
|
and ready_pods >= desired
|
|
)
|
|
return WorkloadStatus(
|
|
kind=kind,
|
|
name=name,
|
|
desired_replicas=desired,
|
|
ready_replicas=ready_replicas,
|
|
desired_image_matches=desired_matches,
|
|
ready_pods=ready_pods,
|
|
matching_pods=matching_pods,
|
|
converged=converged,
|
|
)
|
|
|
|
|
|
def inspect_release(component: str, revision: str) -> dict:
|
|
"""Return safe end-to-end release state for one exact reviewed revision."""
|
|
if component not in COMPONENTS:
|
|
raise ValueError("component must be agent, webui, stt, or tts")
|
|
if not REVISION_PATTERN.fullmatch(revision):
|
|
raise ValueError("revision must be a lowercase full 40-character commit")
|
|
config = COMPONENTS[component]
|
|
policy = _kubectl_json(
|
|
"-n", NAMESPACE, "get", "imagepolicy", config["policy"]
|
|
)
|
|
tag, digest, selected_revision, build = _release_ref(policy)
|
|
result = {
|
|
"component": component,
|
|
"requested_revision": revision,
|
|
"selected_revision": selected_revision,
|
|
"release_tag": tag,
|
|
"digest": digest,
|
|
"build": build,
|
|
"stage": "image_policy_pending",
|
|
"converged": False,
|
|
"workloads": [],
|
|
}
|
|
if selected_revision != revision or digest is None:
|
|
if selected_revision is not None:
|
|
result["stage"] = "different_revision_selected"
|
|
return result
|
|
automation = _kubectl_json(
|
|
"-n", NAMESPACE, "get", "imageupdateautomation", AUTOMATION
|
|
)
|
|
flux = _kubectl_json(
|
|
"-n", FLUX_NAMESPACE, "get", "kustomization", FLUX_KUSTOMIZATION
|
|
)
|
|
result["image_policy_ready"] = _condition_ready(policy)
|
|
result["image_automation_ready"] = _condition_ready(automation)
|
|
result["flux_ready"] = _condition_ready(flux)
|
|
result["flux_revision"] = flux.get("status", {}).get("lastAppliedRevision")
|
|
workloads = [
|
|
_workload_status(kind, name, label, config["repository"], digest)
|
|
for kind, name, label in config["workloads"]
|
|
]
|
|
result["workloads"] = [asdict(item) for item in workloads]
|
|
if not all((result["image_policy_ready"], result["image_automation_ready"])):
|
|
result["stage"] = "image_automation_pending"
|
|
elif not result["flux_ready"] or not all(item.desired_image_matches for item in workloads):
|
|
result["stage"] = "flux_apply_pending"
|
|
elif not all(item.converged for item in workloads):
|
|
result["stage"] = "rollout_pending"
|
|
else:
|
|
result["stage"] = "converged"
|
|
result["converged"] = True
|
|
return result
|
|
|
|
|
|
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
|
"""Parse one bounded release-follow request."""
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--component", required=True, choices=sorted(COMPONENTS))
|
|
parser.add_argument("--revision", required=True)
|
|
parser.add_argument("--wait", action="store_true")
|
|
parser.add_argument("--timeout", type=int, default=1800)
|
|
parser.add_argument("--poll", type=int, default=15)
|
|
return parser.parse_args(argv)
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
"""Print safe JSON immediately or wait for exact live convergence."""
|
|
args = parse_args(argv)
|
|
if args.timeout < 1 or args.poll < 1:
|
|
print("timeout and poll must be positive", file=sys.stderr)
|
|
return 2
|
|
deadline = time.monotonic() + args.timeout
|
|
try:
|
|
while True:
|
|
status = inspect_release(args.component, args.revision)
|
|
if status["converged"]:
|
|
print(json.dumps(status, indent=2, sort_keys=True))
|
|
return 0
|
|
if not args.wait or time.monotonic() >= deadline:
|
|
status["timed_out"] = bool(args.wait)
|
|
print(json.dumps(status, indent=2, sort_keys=True))
|
|
return 2 if args.wait else 1
|
|
time.sleep(args.poll)
|
|
except (OSError, ValueError, subprocess.SubprocessError, json.JSONDecodeError) as exc:
|
|
print(f"Hermes release status failed: {exc}", file=sys.stderr)
|
|
return 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|