titan/atlas-iac
3
0
Fork 0
Code Issues Pull Requests 9 Actions Packages Projects Releases Wiki Activity
atlas-iac/testing/tests/test_hermes_webui_release.py
jenkins 9c516b9808 build(hermes-webui): multi-arch image (arm64 + amd64) 
Make registry.bstein.dev/bstein/hermes-webui a linux/amd64 + linux/arm64
manifest list so the agent pod's `hux` sidecar (which runs the webui image)
can schedule onto the amd64 node titan-22. Reuses the hermes-agent multi-arch
pattern already on main.

- Dockerfile.hermes-webui: repoint both FROMs to multi-arch, internal sources.
  The upstream WebUI base (ghcr sha256:a83a3893..., already a multi-arch OCI
  index) is now pulled from the in-cluster Harbor mirror; the agent base moves
  from the retired arm64-only leaf (81970563) to the multi-arch agent index
  (a68d1c4d). Kaniko selects the matching arch leaf per build node.
- services/harbor/hermes-webui-base-mirror-job.yaml: new suspended, operator-run
  skopeo `copy --all` Job mirroring the upstream WebUI base index into Harbor's
  `mirror` project (modeled on hermes-agent-base-mirror-job.yaml; reuses the
  generic ensure-project helper). Wired into the harbor kustomization.
- Jenkinsfile.hermes-webui-image: arm64 leg (titan-20) + amd64 leg (titan-24,
  hostname+arch pin, toleration Exists, resource-capped, own checkout scm) +
  Combine multi-arch index stage; per-arch evidence archived alongside the index.
- hermes_multiarch_combine.py: generalize the destination pattern/component to
  serve both hermes-agent and hermes-webui (fail-closed to just those two).
- Tests updated to the two-arch topology (two legs, combine, both FROM bases,
  the mirror Job, twelve archived evidence files).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BvMSXH8VH2tMWXanb8SJdf
2026-08-25 18:17:35 -03:00

752 lines
30 KiB
Python
Raw Blame History

"""Independent build, Harbor evidence, and Flux handoff for Hermes WebUI."""
from __future__ import annotations
import importlib.util
import io
import json
from pathlib import Path
import sys
import pytest
import yaml
ROOT = Path(__file__).resolve().parents[2]
PIPELINE = ROOT / "ci/Jenkinsfile.hermes-webui-image"
RELEASE = ROOT / "ci/scripts/hermes_webui_release.py"
DOCKERFILE = ROOT / "dockerfiles/Dockerfile.hermes-webui"
CHAT = ROOT / "services/hermes/chat-statefulset.yaml"
DASHBOARD = ROOT / "services/hermes/deployment.yaml"
POLICY = ROOT / "services/harbor/scripts/harbor_hermes_webui_immutability_ensure.py"
sys.path.insert(0, str(RELEASE.parent))
def _load(path: Path, name: str):
spec = importlib.util.spec_from_file_location(name, path)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def _pod_spec() -> dict:
source = PIPELINE.read_text(encoding="utf-8")
pod_yaml = source.split('yaml """', 1)[1].split('"""', 1)[0]
return yaml.safe_load(pod_yaml)["spec"]
def _release_fixture(tmp_path: Path):
module = _load(RELEASE, f"hermes_webui_release_{tmp_path.name}")
digest = "sha256:" + "7" * 64
revision = "8" * 40
build = "23"
destination = f"{module.DEFAULT_IMAGE}:git-{revision}-build-{build}"
digest_file = tmp_path / "hermes-webui.digest"
image_file = tmp_path / "hermes-webui.image"
digest_file.write_text(digest + "\n", encoding="utf-8")
image_file.write_text(f"{destination}@{digest}\n", encoding="utf-8")
output = tmp_path / "release"
kwargs = {
"digest_file": digest_file,
"image_file": image_file,
"source_revision": revision,
"build_number": build,
"destination": destination,
"chat_manifest": CHAT,
"dashboard_manifest": DASHBOARD,
"output_dir": output,
}
module.write_release_artifacts(
digest=digest,
source_revision=revision,
build_number=build,
destination=destination,
chat_manifest=CHAT,
dashboard_manifest=DASHBOARD,
output_dir=output,
)
return module, digest, kwargs
def test_webui_job_is_independent_bounded_and_main_only() -> None:
"""WebUI has its own bounded job using the same Vault-injected release token."""
config = yaml.safe_load(
(ROOT / "services/jenkins/configmap-jcasc.yaml").read_text(encoding="utf-8")
)
jobs = config["data"]["jobs.yaml"]
assert jobs.count("pipelineJob('hermes-agent-image')") == 1
assert jobs.count("pipelineJob('hermes-webui-image')") == 1
block = jobs.split("pipelineJob('hermes-webui-image')", 1)[1].split(
"multibranchPipelineJob(", 1
)[0]
assert "branches('*/main')" in block
assert "scriptPath('ci/Jenkinsfile.hermes-webui-image')" in block
assert "pipelineTriggers" not in block
assert (
"authenticationToken(System.getenv('HERMES_AGENT_IMAGE_BUILD_TOKEN'))" in block
)
assert "booleanParam('PUBLISH_IMAGE', false" in block
assert "stringParam('EXPECTED_SOURCE_REVISION', ''" in block
assert "stringParam('CONFIRM_PUBLISH', ''" in block
def test_pipeline_builds_exact_reviewed_anchor_from_main() -> None:
"""Publish uses one reviewed source revision even when main later advances."""
source = PIPELINE.read_text(encoding="utf-8")
assert 'test "${CONFIRM_PUBLISH}" = "PUBLISH HERMES WEBUI"' in source
assert 'test "${main_revision}" = "$(git rev-parse origin/main)"' in source
assert "git fetch --no-tags origin main" not in source
assert 'git merge-base --is-ancestor "${EXPECTED_SOURCE_REVISION}"' in source
assert 'git checkout --detach "${EXPECTED_SOURCE_REVISION}"' in source
assert 'test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}"' in source
assert "dockerfiles/Dockerfile.hermes-webui" in source
assert "ci/scripts/hermes_webui_release.py" in source
assert "registry.bstein.dev/bstein/hermes-webui" in source
assert "Dockerfile.hermes-agent" not in source
assert "hermes_image_release.py" not in source
assert "HERMES_KANIKO_HEREDOC_COMPAT" not in source
assert "--digest-file=" in source
assert "--image-name-tag-with-digest-file=" in source
assert "org.opencontainers.image.revision=${source_revision}" in source
assert "assert-absent" in source and "verify-evidence" in source
assert "ci/scripts/hermes_oci_promote.py" in source
assert "test_hermes_webui_brand.py" in source
assert "test_hermes_webui_release.py" in source
for voice_suite in (
"test_hermes_chat_quality.py",
"test_hermes_handsfree_stt.py",
"test_hermes_voice_full_duplex.py",
"test_hermes_thinking_voice_cues.py",
"test_hermes_voice_instrument.py",
"test_hermes_voice_language_routing.py",
):
assert voice_suite in source
assert "apt-get install -y --no-install-recommends ffmpeg nodejs" in source
assert "command -v ffmpeg >/dev/null" in source
assert "command -v node >/dev/null" in source
assert source.index("apt-get install -y --no-install-recommends ffmpeg nodejs") < (
source.index("python3 -m pytest -q")
)
for forbidden in ("kubectl ", "flux reconcile", "git push", "git commit"):
assert forbidden not in source
spec = _pod_spec()
assert spec["serviceAccountName"] == "hermes-image-builder"
assert spec["automountServiceAccountToken"] is False
assert spec["enableServiceLinks"] is False
containers = {item["name"]: item for item in spec["containers"]}
assert "kaniko" in containers
python_security = containers["python"]["securityContext"]
assert python_security["runAsUser"] == 0
assert python_security["runAsNonRoot"] is False
assert set(python_security["capabilities"]["add"]) == {
"CHOWN",
"FOWNER",
"DAC_OVERRIDE",
"SETGID",
"SETUID",
}
assert containers["kaniko"]["resources"]["requests"]["ephemeral-storage"] == "10Gi"
assert containers["kaniko"]["resources"]["limits"]["ephemeral-storage"] == "20Gi"
assert spec["nodeSelector"] == {"kubernetes.io/arch": "arm64"}
build_nodes = spec["affinity"]["nodeAffinity"][
"requiredDuringSchedulingIgnoredDuringExecution"
]["nodeSelectorTerms"][0]["matchExpressions"][0]["values"]
assert build_nodes == ["titan-20"]
storage_nodes = {"titan-13", "titan-15", "titan-17", "titan-19"}
assert storage_nodes.isdisjoint(build_nodes)
assert "docker.sock" not in source and "hostPath" not in source
for container in containers.values():
assert container["securityContext"]["allowPrivilegeEscalation"] is False
assert container["securityContext"]["capabilities"]["drop"] == ["ALL"]
def test_webui_dockerfile_is_kaniko_safe_and_uses_reviewed_repo_source() -> None:
"""The dedicated build consumes tracked patches/assets without RUN heredocs."""
source = DOCKERFILE.read_text(encoding="utf-8")
assert "<<" not in source
assert "hermes-webui-base-patch.py" in source
assert "hermes-webui-brand-patch.py" in source
assert "hermes-webui-manifest-patch.py" in source
assert "hermes-webui-release-patch.py" in source
assert "hermes-webui-smoke.py" in source
assert "hermes-webui-stt-patch.py" in source
assert "hermes-webui-atlas-voice.js" in source
assert "hermes-webui-manifest.json" in source
assert (
'--build-arg="HERMES_WEBUI_RELEASE_ID='
'git-${source_revision}-build-${BUILD_NUMBER}"'
in PIPELINE.read_text(encoding="utf-8")
)
assert source.count("RUN /opt/hermes/.venv/bin/python /tmp/hermes-webui-") == 1
assert "urljoin(manifest_url, source)" in (
ROOT / "dockerfiles/hermes-webui-smoke.py"
).read_text(encoding="utf-8")
def test_renderer_updates_exact_chat_and_dashboard_webui_only(tmp_path: Path) -> None:
"""One digest patch spans the two same-policy WebUI consumers and nothing else."""
module, digest, kwargs = _release_fixture(tmp_path)
output = kwargs["output_dir"]
patch = (output / "hermes-webui-image-update.patch").read_text(encoding="utf-8")
chat_consumers = CHAT.read_text(encoding="utf-8").count(
f"image: {module.DEFAULT_IMAGE}:"
)
assert patch.count(f"+ image: {module.DEFAULT_IMAGE}@{digest}") == chat_consumers + 1
if "HUX_IMAGE_TAG" in CHAT.read_text(encoding="utf-8"):
tag = f"git-{kwargs['source_revision']}-build-{kwargs['build_number']}-release"
assert any(
line.startswith("+") and line.lstrip("+ ").startswith(f"value: {tag}")
for line in patch.splitlines()
), patch
assert any(
line.startswith("+") and line.lstrip("+ ").startswith(f"value: {digest}")
for line in patch.splitlines()
), patch
assert "services/hermes/chat-statefulset.yaml" in patch
assert "services/hermes/deployment.yaml" in patch
assert "hermes-agent@sha256" not in "\n".join(
line for line in patch.splitlines() if line.startswith("+")
)
assert CHAT.read_text(encoding="utf-8") != (
output / "hermes-chat-statefulset.yaml"
).read_text(encoding="utf-8")
assert DASHBOARD.read_text(encoding="utf-8") != (
output / "hermes-dashboard-deployment.yaml"
).read_text(encoding="utf-8")
metadata = json.loads((output / "hermes-webui-image.json").read_text())
assert metadata["digest"] == digest
assert metadata["flux_image"] == f"{module.DEFAULT_IMAGE}@{digest}"
assert metadata["flux_targets"] == [
"apps/StatefulSet/hermes/hermes-chat-tenant",
"apps/Deployment/hermes/hermes",
]
module.validate_release_artifacts(**kwargs)
def test_renderer_accepts_flux_tagged_digest_reference() -> None:
"""Flux's whole-image setter may retain the selected release tag."""
module = _load(RELEASE, "hermes_webui_release_tagged_digest")
old_digest = "sha256:" + "1" * 64
new_digest = "sha256:" + "2" * 64
source = (
"apiVersion: apps/v1\n"
"kind: Deployment\n"
"metadata:\n"
" name: hermes\n"
"spec:\n"
" image: registry.bstein.dev/bstein/hermes-webui:"
f"git-{'a' * 40}-build-7-release@{old_digest} "
'# {"$imagepolicy": "hermes:hermes-webui-release"}\n'
)
rendered = module.render_workload(
source, new_digest, kind="Deployment", name="hermes"
)
assert f"image: {module.DEFAULT_IMAGE}@{new_digest}" in rendered
assert '"$imagepolicy": "hermes:hermes-webui-release"' in rendered
def test_renderer_supports_pre_activation_single_consumer_without_hux_metadata() -> None:
"""The image can land safely before the HUX sidecar is enabled by Flux."""
module = _load(RELEASE, "hermes_webui_release_staged_activation")
old_digest = "sha256:" + "1" * 64
new_digest = "sha256:" + "2" * 64
revision = "3" * 40
source = (
"apiVersion: apps/v1\nkind: StatefulSet\nmetadata:\n"
" name: hermes-chat-tenant\nspec:\n template:\n spec:\n"
" containers:\n - name: webui\n"
f" image: {module.DEFAULT_IMAGE}@{old_digest}\n"
)
rendered = module.render_workload(
source,
new_digest,
kind="StatefulSet",
name="hermes-chat-tenant",
expected_images=(1, 2),
)
assert f"image: {module.DEFAULT_IMAGE}@{new_digest}" in rendered
assert module.render_hux_build_metadata(rendered, new_digest, revision, "23") == rendered
@pytest.mark.parametrize(
("source", "kind", "name", "match"),
[
(
"apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: wrong\n",
"Deployment",
"hermes",
"identity changed",
),
(
"apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: hermes\n",
"Deployment",
"hermes",
"found 0",
),
(
"apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: hermes\n"
"spec:\n image: registry.bstein.dev/bstein/hermes-webui:latest\n",
"Deployment",
"hermes",
"found 0",
),
],
)
def test_renderer_fails_closed_on_flux_target_drift(
source: str, kind: str, name: str, match: str
) -> None:
module = _load(RELEASE, f"webui_renderer_{abs(hash(source))}")
with pytest.raises(ValueError, match=match):
module.render_workload(source, "sha256:" + "a" * 64, kind=kind, name=name)
def test_evidence_revalidation_rejects_extra_or_changed_files(tmp_path: Path) -> None:
"""Archived output is an exact deterministic set, not a best-effort bundle."""
module, _digest, kwargs = _release_fixture(tmp_path)
module.validate_release_artifacts(**kwargs)
extra = kwargs["output_dir"] / "unexpected"
extra.write_text("surprise\n", encoding="utf-8")
with pytest.raises(ValueError, match="exactly four"):
module.validate_release_artifacts(**kwargs)
extra.unlink()
metadata = kwargs["output_dir"] / "hermes-webui-image.json"
metadata.write_text("{}\n", encoding="utf-8")
with pytest.raises(ValueError, match="incomplete or mismatched"):
module.validate_release_artifacts(**kwargs)
class _Response(io.BytesIO):
status = 200
def __init__(self, body: bytes, headers: dict[str, str] | None = None):
super().__init__(body)
self.headers = headers or {}
def __enter__(self):
return self
def __exit__(self, *_args):
self.close()
def test_release_verifies_exact_webui_harbor_artifact_and_policy() -> None:
"""Independent evidence resolves the tag and exact WebUI immutability rule."""
module = _load(RELEASE, "webui_registry_contract")
revision = "a" * 40
destination = f"{module.DEFAULT_IMAGE}:git-{revision}-build-9"
digest = "sha256:" + "b" * 64
seen = []
def artifact_open(request, timeout):
seen.append((request, timeout))
return _Response(
json.dumps(
{
"digest": digest,
"tags": [{"name": f"git-{revision}-build-9", "immutable": True}],
"extra_attrs": {
"config": {
"Labels": {
"org.opencontainers.image.revision": revision,
}
}
},
}
).encode()
)
module.verify_registry_digest(
destination,
digest,
revision,
username="robot",
password="private",
opener=artifact_open,
)
request = seen[0][0]
assert "/repositories/hermes-webui/artifacts/" in request.full_url
assert request.full_url.startswith("https://registry.bstein.dev/api/v2.0/")
assert request.get_header("Authorization").startswith("Basic ")
assert seen[0][1] == 20
def policy_open(request, timeout):
assert request.full_url.endswith(
"/projects/bstein/immutabletagrules?page=1&page_size=100"
)
return _Response(
json.dumps(
[
{
"disabled": False,
"action": "immutable",
"template": "immutable_template",
"tag_selectors": [
{
"kind": "doublestar",
"decoration": "matches",
"pattern": "git-*-build-*",
}
],
"scope_selectors": {
"repository": [
{
"kind": "doublestar",
"decoration": "repoMatches",
"pattern": "hermes-webui",
}
]
},
}
]
).encode(),
{"X-Total-Count": "1"},
)
module.verify_immutable_policy(
username="robot", password="private", opener=policy_open
)
@pytest.mark.parametrize(
"labels", [None, {}, {"org.opencontainers.image.revision": "c" * 40}]
)
def test_release_rejects_missing_or_wrong_harbor_source_revision(labels) -> None:
"""A tag derived from Git cannot substitute for the persisted OCI label."""
module = _load(RELEASE, f"webui_registry_revision_{labels!r}")
revision = "a" * 40
destination = f"{module.DEFAULT_IMAGE}:git-{revision}-build-9"
digest = "sha256:" + "b" * 64
def artifact_open(_request, _timeout):
config = {} if labels is None else {"Labels": labels}
return _Response(
json.dumps(
{
"digest": digest,
"tags": [{"name": f"git-{revision}-build-9", "immutable": True}],
"extra_attrs": {"config": config},
}
).encode()
)
with pytest.raises(RuntimeError, match="OCI (image labels|source-revision label)"):
module.verify_registry_digest(
destination,
digest,
revision,
username="robot",
password="private",
opener=artifact_open,
)
def test_flux_tracks_webui_policy_before_jenkins() -> None:
"""The immutable Harbor rule is reviewed desired state, not a pipeline wish."""
harbor = yaml.safe_load(
(
ROOT / "clusters/atlas/flux-system/applications/harbor/kustomization.yaml"
).read_text(encoding="utf-8")
)
checks = harbor["spec"]["healthChecks"]
assert {
"apiVersion": "batch/v1",
"kind": "Job",
"name": "harbor-hermes-webui-immutability-ensure-1",
"namespace": "harbor",
} in checks
jenkins = yaml.safe_load(
(
ROOT / "clusters/atlas/flux-system/applications/jenkins/kustomization.yaml"
).read_text(encoding="utf-8")
)
assert "harbor" in {item["name"] for item in jenkins["spec"]["dependsOn"]}
policy = _load(POLICY, "webui_policy_contract")
assert policy.REPOSITORY_PATTERN == "hermes-webui"
assert policy.TAG_PATTERN == "git-*-build-*"
assert policy.EXPECTED_RULE["disabled"] is False
assert "robot" not in POLICY.read_text(encoding="utf-8").lower()
job = yaml.safe_load(
(ROOT / "services/harbor/hermes-webui-immutability-job.yaml").read_text(
encoding="utf-8"
)
)
template = job["spec"]["template"]
annotations = template["metadata"]["annotations"]
assert annotations["vault.hashicorp.com/role"] == "harbor-policy-bootstrap"
assert (
annotations["vault.hashicorp.com/agent-inject-secret-harbor-admin-password"]
== "kv/data/atlas/harbor/harbor-core"
)
pod = template["spec"]
assert pod["serviceAccountName"] == "harbor-policy-bootstrap"
assert pod["enableServiceLinks"] is False
container = pod["containers"][0]
assert "@sha256:" in container["image"]
assert container["securityContext"]["readOnlyRootFilesystem"] is True
assert container["securityContext"]["runAsNonRoot"] is True
assert container["securityContext"]["capabilities"]["drop"] == ["ALL"]
release_docs = (ROOT / "docs/hermes_webui_release.md").read_text(encoding="utf-8")
assert "immutable-tag:list" in release_docs
assert "harbor-hermes-agent-immutability-ensure-1" in release_docs
class _FakePolicyClient:
origin = "https://registry.bstein.dev/api/v2.0"
def __init__(self, responses):
self.responses = list(responses)
self.calls = []
def request(self, method, path, payload=None):
self.calls.append((method, path, payload))
return self.responses.pop(0)
def test_webui_policy_is_idempotent_and_create_is_reread() -> None:
"""The desired Harbor rule validates in place or verifies its exact new ID."""
policy = _load(POLICY, "webui_policy_idempotency")
existing = {"id": 17, **policy.EXPECTED_RULE}
client = _FakePolicyClient(
[(200, json.dumps([existing]).encode(), {"X-Total-Count": "1"})]
)
assert policy.ensure_rule(client) == 17
assert [call[0] for call in client.calls] == ["GET"]
created = {"id": 23, **policy.EXPECTED_RULE}
client = _FakePolicyClient(
[
(200, b"[]", {"X-Total-Count": "0"}),
(
201,
b"",
{"Location": ("/api/v2.0/projects/bstein/immutabletagrules/23")},
),
(200, json.dumps([created]).encode(), {"X-Total-Count": "1"}),
]
)
assert policy.ensure_rule(client) == 23
assert client.calls[1] == (
"POST",
"/projects/bstein/immutabletagrules",
policy.EXPECTED_RULE,
)
def test_webui_policy_rejects_disabled_duplicate_or_truncated_rules() -> None:
"""Ambiguous or incomplete Harbor evidence can never unblock publication."""
policy = _load(POLICY, "webui_policy_rejections")
disabled = {"id": 17, **policy.EXPECTED_RULE, "disabled": True}
with pytest.raises(RuntimeError, match="not enabled and exact"):
policy.ensure_rule(
_FakePolicyClient(
[
(
200,
json.dumps([disabled]).encode(),
{"X-Total-Count": "1"},
)
]
)
)
duplicate = [
{"id": 17, **policy.EXPECTED_RULE},
{"id": 18, **policy.EXPECTED_RULE},
]
with pytest.raises(RuntimeError, match="multiple Hermes WebUI"):
policy.ensure_rule(
_FakePolicyClient(
[
(
200,
json.dumps(duplicate).encode(),
{"X-Total-Count": "2"},
)
]
)
)
with pytest.raises(RuntimeError, match="truncated"):
policy.list_rules(_FakePolicyClient([(200, b"[]", {"X-Total-Count": "1"})]))
def test_pipeline_archives_exact_twelve_files_before_release() -> None:
"""Release cannot pass with missing digest, workload, or metadata evidence.
The multi-arch topology archives the four arch-less index files, the four
per-arch leg leaves (arm64 + amd64 digest/image), and the four rendered Flux
handoff files -- twelve exact paths, no globs.
"""
source = PIPELINE.read_text(encoding="utf-8")
evidence = source.split("stage('Verify and archive release evidence')", 1)[1]
evidence = evidence.split("stage('Publish Flux release tag')", 1)[0]
archive = evidence.split("artifacts: '", 1)[1].split("'", 1)[0].split(",")
assert len(archive) == len(set(archive)) == 12
assert all("*" not in path for path in archive)
assert "find build -type f" in evidence
assert "allowEmptyArchive: false" in evidence
assert "build/hermes-webui.source-revision" in archive
for arch in ("arm64", "amd64"):
assert f"build/hermes-webui-{arch}.digest" in archive
assert f"build/hermes-webui-{arch}.image" in archive
assert "hermes-chat-statefulset.yaml" in evidence
assert "hermes-dashboard-deployment.yaml" in evidence
assert " post {" not in source
# --- Multi-arch (linux/amd64 + linux/arm64) topology -----------------------
#
# The WebUI image must be multi-arch so the agent pod's `hux` sidecar (which runs
# this image) can schedule onto the amd64 node titan-22. These tests lock the two
# legs + combine, the two internal multi-arch FROM bases, and the Harbor mirror
# Job that feeds the upstream WebUI base internally.
MIRROR_JOB = ROOT / "services/harbor/hermes-webui-base-mirror-job.yaml"
HARBOR_KUSTOMIZATION = ROOT / "services/harbor/kustomization.yaml"
WEBUI_BASE_DIGEST = (
"sha256:a83a3893111dcb250e7aa7aa657d3d6f4570b0e2fd00d9b7569246fc5e7339b2"
)
AGENT_MULTIARCH_INDEX_DIGEST = (
"sha256:a68d1c4d5517cc5e6719661e77be4f18d4964b07a85dcebaf6f62368646e4e6c"
)
AGENT_SINGLE_ARCH_DIGEST = (
"sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107"
)
def _amd64_pod_spec() -> dict:
"""Parse the nested amd64 build pod (the second inline pod template)."""
source = PIPELINE.read_text(encoding="utf-8")
blocks = source.split('yaml """')[1:]
for block in blocks:
body = block.split('"""', 1)[0]
doc = yaml.safe_load(body)
labels = doc.get("metadata", {}).get("labels", {})
if str(labels.get("atlas.bstein.dev/workload", "")).endswith("-amd64"):
return doc["spec"]
raise AssertionError("no amd64 build pod template found in the pipeline")
def test_pipeline_builds_two_arch_legs_and_combines_one_index() -> None:
"""The pipeline builds arm64 + amd64 legs and binds them into one index."""
source = PIPELINE.read_text(encoding="utf-8")
assert "stage('Build arm64 leg without a daemon')" in source
assert "stage('Build amd64 leg without a daemon')" in source
assert "stage('Combine multi-arch index')" in source
# Each leg publishes an arch-suffixed candidate tag and its own evidence.
assert 'destination="$(cat build/hermes-webui.destination)-arm64"' in source
assert 'destination="$(cat build/hermes-webui.destination)-amd64"' in source
for arch in ("arm64", "amd64"):
assert f"build/hermes-webui-{arch}.digest" in source
assert f"build/hermes-webui-{arch}.image" in source
# Both legs must still stamp the WebUI release id (no arch drift in args).
assert (
'--build-arg="HERMES_WEBUI_RELEASE_ID='
'git-${source_revision}-build-${BUILD_NUMBER}"'
) in source
# The combiner reuses the shared, fail-closed manifest-list assembler.
assert "ci/scripts/hermes_multiarch_combine.py" in source
assert "--arm64-digest-file build/hermes-webui-arm64.digest" in source
assert "--amd64-digest-file build/hermes-webui-amd64.digest" in source
assert "--digest-file build/hermes-webui.digest" in source
# The amd64 leg runs on its own fresh pod, so it re-checks out and re-runs the
# exact release boundary before building.
assert source.count("checkout scm") >= 2
assert source.count('test "${CONFIRM_PUBLISH}" = "PUBLISH HERMES WEBUI"') >= 2
# The final archived evidence covers the index AND both per-arch leaves.
for evidence in (
"build/hermes-webui-arm64.digest",
"build/hermes-webui-arm64.image",
"build/hermes-webui-amd64.digest",
"build/hermes-webui-amd64.image",
):
assert source.count(evidence) >= 3 # build, expected list, archive list
def test_amd64_leg_is_pinned_to_titan_24_without_worker_role() -> None:
"""The disposable amd64 leg targets the accelerator node by hostname only."""
spec = _amd64_pod_spec()
assert spec["serviceAccountName"] == "hermes-image-builder"
assert spec["automountServiceAccountToken"] is False
assert spec["nodeSelector"] == {
"kubernetes.io/arch": "amd64",
"kubernetes.io/hostname": "titan-24",
}
# titan-24 is an accelerator, not a general worker: never require the worker
# role label, and tolerate its guard taint so the pinned build lands.
assert "node-role.kubernetes.io/worker" not in spec["nodeSelector"]
assert spec["tolerations"] == [{"operator": "Exists"}]
containers = {item["name"]: item for item in spec["containers"]}
assert "kaniko" in containers
for container in containers.values():
sc = container["securityContext"]
assert sc["allowPrivilegeEscalation"] is False
assert sc["capabilities"]["drop"] == ["ALL"]
# Tight caps keep the disposable build off the co-hosted validator's back.
assert containers["kaniko"]["resources"]["limits"]["memory"] == "3Gi"
def test_dockerfile_bases_are_multiarch_and_pulled_internally() -> None:
"""Both FROM bases are multi-arch indexes sourced from inside the cluster."""
source = DOCKERFILE.read_text(encoding="utf-8")
# Upstream WebUI base now comes from the in-cluster Harbor mirror, not ghcr.
assert (
"FROM harbor-core.harbor.svc.cluster.local/mirror/hermes-webui@"
f"{WEBUI_BASE_DIGEST} AS webui"
) in source
assert "ghcr.io/nesquena" not in source
# The agent base is the multi-arch index, not the retired single-arch leaf.
assert (
f"FROM registry.bstein.dev/bstein/hermes-agent@{AGENT_MULTIARCH_INDEX_DIGEST}"
in source
)
assert AGENT_SINGLE_ARCH_DIGEST not in source
def test_webui_base_mirror_job_is_suspended_and_digest_pinned() -> None:
"""A suspended, digest-pinned skopeo Job mirrors the WebUI base into Harbor."""
job = yaml.safe_load(MIRROR_JOB.read_text(encoding="utf-8"))
assert job["kind"] == "Job"
assert job["metadata"]["name"] == "harbor-hermes-webui-base-mirror-1"
assert job["metadata"]["namespace"] == "harbor"
# Operator-run only: it needs ghcr egress and must never fire automatically.
assert job["spec"]["suspend"] is True
pod = job["spec"]["template"]["spec"]
mirror = next(c for c in pod["containers"] if c["name"] == "mirror")
args = "\n".join(mirror["args"])
assert "skopeo copy --all" in args
assert f"docker://ghcr.io/nesquena/hermes-webui@{WEBUI_BASE_DIGEST}" in args
assert (
f"docker://registry.bstein.dev/mirror/hermes-webui@{WEBUI_BASE_DIGEST}" in args
)
# Vault must init before the ensure-project init container reads the secret.
annotations = job["spec"]["template"]["metadata"]["annotations"]
assert annotations["vault.hashicorp.com/agent-init-first"] == "true"
init = next(c for c in pod["initContainers"] if c["name"] == "ensure-project")
assert init["command"] == ["python3", "/scripts/harbor_mirror_project_ensure.py"]
script_volume = next(v for v in pod["volumes"] if v["name"] == "scripts")
assert (
script_volume["configMap"]["name"] == "harbor-hermes-webui-base-mirror-script"
)
def test_webui_base_mirror_job_is_wired_into_harbor_kustomization() -> None:
"""Flux applies the mirror Job and mounts its ensure-project script."""
kustomization = yaml.safe_load(HARBOR_KUSTOMIZATION.read_text(encoding="utf-8"))
assert "hermes-webui-base-mirror-job.yaml" in kustomization["resources"]
generators = {g["name"]: g for g in kustomization["configMapGenerator"]}
assert "harbor-hermes-webui-base-mirror-script" in generators
assert (
"harbor_mirror_project_ensure.py=scripts/harbor_mirror_project_ensure.py"
in generators["harbor-hermes-webui-base-mirror-script"]["files"]
)
Reference in New Issue View Git Blame Copy Permalink