"""Independent build, Harbor evidence, and Flux handoff for Hermes WebUI.""" from __future__ import annotations import importlib.util import io import json from pathlib import Path import sys import pytest import yaml ROOT = Path(__file__).resolve().parents[2] PIPELINE = ROOT / "ci/Jenkinsfile.hermes-webui-image" RELEASE = ROOT / "ci/scripts/hermes_webui_release.py" DOCKERFILE = ROOT / "dockerfiles/Dockerfile.hermes-webui" CHAT = ROOT / "services/hermes/chat-statefulset.yaml" DASHBOARD = ROOT / "services/hermes/deployment.yaml" POLICY = ROOT / "services/harbor/scripts/harbor_hermes_webui_immutability_ensure.py" sys.path.insert(0, str(RELEASE.parent)) def _load(path: Path, name: str): spec = importlib.util.spec_from_file_location(name, path) assert spec is not None and spec.loader is not None module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module def _pod_spec() -> dict: source = PIPELINE.read_text(encoding="utf-8") pod_yaml = source.split('yaml """', 1)[1].split('"""', 1)[0] return yaml.safe_load(pod_yaml)["spec"] def _release_fixture(tmp_path: Path): module = _load(RELEASE, f"hermes_webui_release_{tmp_path.name}") digest = "sha256:" + "7" * 64 revision = "8" * 40 build = "23" destination = f"{module.DEFAULT_IMAGE}:git-{revision}-build-{build}" digest_file = tmp_path / "hermes-webui.digest" image_file = tmp_path / "hermes-webui.image" digest_file.write_text(digest + "\n", encoding="utf-8") image_file.write_text(f"{destination}@{digest}\n", encoding="utf-8") output = tmp_path / "release" kwargs = { "digest_file": digest_file, "image_file": image_file, "source_revision": revision, "build_number": build, "destination": destination, "chat_manifest": CHAT, "dashboard_manifest": DASHBOARD, "output_dir": output, } module.write_release_artifacts( digest=digest, source_revision=revision, build_number=build, destination=destination, chat_manifest=CHAT, dashboard_manifest=DASHBOARD, output_dir=output, ) return module, digest, kwargs def test_webui_job_is_independent_bounded_and_main_only() -> None: """WebUI has its own bounded job using the same Vault-injected release token.""" config = yaml.safe_load( (ROOT / "services/jenkins/configmap-jcasc.yaml").read_text(encoding="utf-8") ) jobs = config["data"]["jobs.yaml"] assert jobs.count("pipelineJob('hermes-agent-image')") == 1 assert jobs.count("pipelineJob('hermes-webui-image')") == 1 block = jobs.split("pipelineJob('hermes-webui-image')", 1)[1].split( "multibranchPipelineJob(", 1 )[0] assert "branches('*/main')" in block assert "scriptPath('ci/Jenkinsfile.hermes-webui-image')" in block assert "pipelineTriggers" not in block assert ( "authenticationToken(System.getenv('HERMES_AGENT_IMAGE_BUILD_TOKEN'))" in block ) assert "booleanParam('PUBLISH_IMAGE', false" in block assert "stringParam('EXPECTED_SOURCE_REVISION', ''" in block assert "stringParam('CONFIRM_PUBLISH', ''" in block def test_pipeline_builds_exact_reviewed_anchor_from_main() -> None: """Publish uses one reviewed source revision even when main later advances.""" source = PIPELINE.read_text(encoding="utf-8") assert 'test "${CONFIRM_PUBLISH}" = "PUBLISH HERMES WEBUI"' in source assert 'test "${main_revision}" = "$(git rev-parse origin/main)"' in source assert "git fetch --no-tags origin main" not in source assert 'git merge-base --is-ancestor "${EXPECTED_SOURCE_REVISION}"' in source assert 'git checkout --detach "${EXPECTED_SOURCE_REVISION}"' in source assert 'test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}"' in source assert "dockerfiles/Dockerfile.hermes-webui" in source assert "ci/scripts/hermes_webui_release.py" in source assert "registry.bstein.dev/bstein/hermes-webui" in source assert "Dockerfile.hermes-agent" not in source assert "hermes_image_release.py" not in source assert "HERMES_KANIKO_HEREDOC_COMPAT" not in source assert "--digest-file=" in source assert "--image-name-tag-with-digest-file=" in source assert "org.opencontainers.image.revision=${source_revision}" in source assert "assert-absent" in source and "verify-evidence" in source assert "ci/scripts/hermes_oci_promote.py" in source assert "test_hermes_webui_brand.py" in source assert "test_hermes_webui_release.py" in source for voice_suite in ( "test_hermes_chat_quality.py", "test_hermes_handsfree_stt.py", "test_hermes_voice_full_duplex.py", "test_hermes_thinking_voice_cues.py", "test_hermes_voice_instrument.py", "test_hermes_voice_language_routing.py", ): assert voice_suite in source assert "apt-get install -y --no-install-recommends ffmpeg nodejs" in source assert "command -v ffmpeg >/dev/null" in source assert "command -v node >/dev/null" in source assert source.index("apt-get install -y --no-install-recommends ffmpeg nodejs") < ( source.index("python3 -m pytest -q") ) for forbidden in ("kubectl ", "flux reconcile", "git push", "git commit"): assert forbidden not in source spec = _pod_spec() assert spec["serviceAccountName"] == "hermes-image-builder" assert spec["automountServiceAccountToken"] is False assert spec["enableServiceLinks"] is False containers = {item["name"]: item for item in spec["containers"]} assert "kaniko" in containers python_security = containers["python"]["securityContext"] assert python_security["runAsUser"] == 0 assert python_security["runAsNonRoot"] is False assert set(python_security["capabilities"]["add"]) == { "CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID", } assert containers["kaniko"]["resources"]["requests"]["ephemeral-storage"] == "10Gi" assert containers["kaniko"]["resources"]["limits"]["ephemeral-storage"] == "20Gi" assert spec["nodeSelector"] == {"kubernetes.io/arch": "arm64"} build_nodes = spec["affinity"]["nodeAffinity"][ "requiredDuringSchedulingIgnoredDuringExecution" ]["nodeSelectorTerms"][0]["matchExpressions"][0]["values"] assert build_nodes == ["titan-20"] storage_nodes = {"titan-13", "titan-15", "titan-17", "titan-19"} assert storage_nodes.isdisjoint(build_nodes) assert "docker.sock" not in source and "hostPath" not in source for container in containers.values(): assert container["securityContext"]["allowPrivilegeEscalation"] is False assert container["securityContext"]["capabilities"]["drop"] == ["ALL"] def test_webui_dockerfile_is_kaniko_safe_and_uses_reviewed_repo_source() -> None: """The dedicated build consumes tracked patches/assets without RUN heredocs.""" source = DOCKERFILE.read_text(encoding="utf-8") assert "<<" not in source assert "hermes-webui-base-patch.py" in source assert "hermes-webui-brand-patch.py" in source assert "hermes-webui-manifest-patch.py" in source assert "hermes-webui-release-patch.py" in source assert "hermes-webui-smoke.py" in source assert "hermes-webui-stt-patch.py" in source assert "hermes-webui-atlas-voice.js" in source assert "hermes-webui-manifest.json" in source assert ( '--build-arg="HERMES_WEBUI_RELEASE_ID=' 'git-${source_revision}-build-${BUILD_NUMBER}"' in PIPELINE.read_text(encoding="utf-8") ) assert source.count("RUN /opt/hermes/.venv/bin/python /tmp/hermes-webui-") == 1 assert "urljoin(manifest_url, source)" in ( ROOT / "dockerfiles/hermes-webui-smoke.py" ).read_text(encoding="utf-8") def test_renderer_updates_exact_chat_and_dashboard_webui_only(tmp_path: Path) -> None: """One digest patch spans the two same-policy WebUI consumers and nothing else.""" module, digest, kwargs = _release_fixture(tmp_path) output = kwargs["output_dir"] patch = (output / "hermes-webui-image-update.patch").read_text(encoding="utf-8") chat_consumers = CHAT.read_text(encoding="utf-8").count( f"image: {module.DEFAULT_IMAGE}:" ) assert patch.count(f"+ image: {module.DEFAULT_IMAGE}@{digest}") == chat_consumers + 1 if "HUX_IMAGE_TAG" in CHAT.read_text(encoding="utf-8"): tag = f"git-{kwargs['source_revision']}-build-{kwargs['build_number']}-release" assert any( line.startswith("+") and line.lstrip("+ ").startswith(f"value: {tag}") for line in patch.splitlines() ), patch assert any( line.startswith("+") and line.lstrip("+ ").startswith(f"value: {digest}") for line in patch.splitlines() ), patch assert "services/hermes/chat-statefulset.yaml" in patch assert "services/hermes/deployment.yaml" in patch assert "hermes-agent@sha256" not in "\n".join( line for line in patch.splitlines() if line.startswith("+") ) assert CHAT.read_text(encoding="utf-8") != ( output / "hermes-chat-statefulset.yaml" ).read_text(encoding="utf-8") assert DASHBOARD.read_text(encoding="utf-8") != ( output / "hermes-dashboard-deployment.yaml" ).read_text(encoding="utf-8") metadata = json.loads((output / "hermes-webui-image.json").read_text()) assert metadata["digest"] == digest assert metadata["flux_image"] == f"{module.DEFAULT_IMAGE}@{digest}" assert metadata["flux_targets"] == [ "apps/StatefulSet/hermes/hermes-chat-tenant", "apps/Deployment/hermes/hermes", ] module.validate_release_artifacts(**kwargs) def test_renderer_accepts_flux_tagged_digest_reference() -> None: """Flux's whole-image setter may retain the selected release tag.""" module = _load(RELEASE, "hermes_webui_release_tagged_digest") old_digest = "sha256:" + "1" * 64 new_digest = "sha256:" + "2" * 64 source = ( "apiVersion: apps/v1\n" "kind: Deployment\n" "metadata:\n" " name: hermes\n" "spec:\n" " image: registry.bstein.dev/bstein/hermes-webui:" f"git-{'a' * 40}-build-7-release@{old_digest} " '# {"$imagepolicy": "hermes:hermes-webui-release"}\n' ) rendered = module.render_workload( source, new_digest, kind="Deployment", name="hermes" ) assert f"image: {module.DEFAULT_IMAGE}@{new_digest}" in rendered assert '"$imagepolicy": "hermes:hermes-webui-release"' in rendered def test_renderer_supports_pre_activation_single_consumer_without_hux_metadata() -> None: """The image can land safely before the HUX sidecar is enabled by Flux.""" module = _load(RELEASE, "hermes_webui_release_staged_activation") old_digest = "sha256:" + "1" * 64 new_digest = "sha256:" + "2" * 64 revision = "3" * 40 source = ( "apiVersion: apps/v1\nkind: StatefulSet\nmetadata:\n" " name: hermes-chat-tenant\nspec:\n template:\n spec:\n" " containers:\n - name: webui\n" f" image: {module.DEFAULT_IMAGE}@{old_digest}\n" ) rendered = module.render_workload( source, new_digest, kind="StatefulSet", name="hermes-chat-tenant", expected_images=(1, 2), ) assert f"image: {module.DEFAULT_IMAGE}@{new_digest}" in rendered assert module.render_hux_build_metadata(rendered, new_digest, revision, "23") == rendered @pytest.mark.parametrize( ("source", "kind", "name", "match"), [ ( "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: wrong\n", "Deployment", "hermes", "identity changed", ), ( "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: hermes\n", "Deployment", "hermes", "found 0", ), ( "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: hermes\n" "spec:\n image: registry.bstein.dev/bstein/hermes-webui:latest\n", "Deployment", "hermes", "found 0", ), ], ) def test_renderer_fails_closed_on_flux_target_drift( source: str, kind: str, name: str, match: str ) -> None: module = _load(RELEASE, f"webui_renderer_{abs(hash(source))}") with pytest.raises(ValueError, match=match): module.render_workload(source, "sha256:" + "a" * 64, kind=kind, name=name) def test_evidence_revalidation_rejects_extra_or_changed_files(tmp_path: Path) -> None: """Archived output is an exact deterministic set, not a best-effort bundle.""" module, _digest, kwargs = _release_fixture(tmp_path) module.validate_release_artifacts(**kwargs) extra = kwargs["output_dir"] / "unexpected" extra.write_text("surprise\n", encoding="utf-8") with pytest.raises(ValueError, match="exactly four"): module.validate_release_artifacts(**kwargs) extra.unlink() metadata = kwargs["output_dir"] / "hermes-webui-image.json" metadata.write_text("{}\n", encoding="utf-8") with pytest.raises(ValueError, match="incomplete or mismatched"): module.validate_release_artifacts(**kwargs) class _Response(io.BytesIO): status = 200 def __init__(self, body: bytes, headers: dict[str, str] | None = None): super().__init__(body) self.headers = headers or {} def __enter__(self): return self def __exit__(self, *_args): self.close() def test_release_verifies_exact_webui_harbor_artifact_and_policy() -> None: """Independent evidence resolves the tag and exact WebUI immutability rule.""" module = _load(RELEASE, "webui_registry_contract") revision = "a" * 40 destination = f"{module.DEFAULT_IMAGE}:git-{revision}-build-9" digest = "sha256:" + "b" * 64 seen = [] def artifact_open(request, timeout): seen.append((request, timeout)) return _Response( json.dumps( { "digest": digest, "tags": [{"name": f"git-{revision}-build-9", "immutable": True}], "extra_attrs": { "config": { "Labels": { "org.opencontainers.image.revision": revision, } } }, } ).encode() ) module.verify_registry_digest( destination, digest, revision, username="robot", password="private", opener=artifact_open, ) request = seen[0][0] assert "/repositories/hermes-webui/artifacts/" in request.full_url assert request.full_url.startswith("https://registry.bstein.dev/api/v2.0/") assert request.get_header("Authorization").startswith("Basic ") assert seen[0][1] == 20 def policy_open(request, timeout): assert request.full_url.endswith( "/projects/bstein/immutabletagrules?page=1&page_size=100" ) return _Response( json.dumps( [ { "disabled": False, "action": "immutable", "template": "immutable_template", "tag_selectors": [ { "kind": "doublestar", "decoration": "matches", "pattern": "git-*-build-*", } ], "scope_selectors": { "repository": [ { "kind": "doublestar", "decoration": "repoMatches", "pattern": "hermes-webui", } ] }, } ] ).encode(), {"X-Total-Count": "1"}, ) module.verify_immutable_policy( username="robot", password="private", opener=policy_open ) @pytest.mark.parametrize( "labels", [None, {}, {"org.opencontainers.image.revision": "c" * 40}] ) def test_release_rejects_missing_or_wrong_harbor_source_revision(labels) -> None: """A tag derived from Git cannot substitute for the persisted OCI label.""" module = _load(RELEASE, f"webui_registry_revision_{labels!r}") revision = "a" * 40 destination = f"{module.DEFAULT_IMAGE}:git-{revision}-build-9" digest = "sha256:" + "b" * 64 def artifact_open(_request, _timeout): config = {} if labels is None else {"Labels": labels} return _Response( json.dumps( { "digest": digest, "tags": [{"name": f"git-{revision}-build-9", "immutable": True}], "extra_attrs": {"config": config}, } ).encode() ) with pytest.raises(RuntimeError, match="OCI (image labels|source-revision label)"): module.verify_registry_digest( destination, digest, revision, username="robot", password="private", opener=artifact_open, ) def test_flux_tracks_webui_policy_before_jenkins() -> None: """The immutable Harbor rule is reviewed desired state, not a pipeline wish.""" harbor = yaml.safe_load( ( ROOT / "clusters/atlas/flux-system/applications/harbor/kustomization.yaml" ).read_text(encoding="utf-8") ) checks = harbor["spec"]["healthChecks"] assert { "apiVersion": "batch/v1", "kind": "Job", "name": "harbor-hermes-webui-immutability-ensure-1", "namespace": "harbor", } in checks jenkins = yaml.safe_load( ( ROOT / "clusters/atlas/flux-system/applications/jenkins/kustomization.yaml" ).read_text(encoding="utf-8") ) assert "harbor" in {item["name"] for item in jenkins["spec"]["dependsOn"]} policy = _load(POLICY, "webui_policy_contract") assert policy.REPOSITORY_PATTERN == "hermes-webui" assert policy.TAG_PATTERN == "git-*-build-*" assert policy.EXPECTED_RULE["disabled"] is False assert "robot" not in POLICY.read_text(encoding="utf-8").lower() job = yaml.safe_load( (ROOT / "services/harbor/hermes-webui-immutability-job.yaml").read_text( encoding="utf-8" ) ) template = job["spec"]["template"] annotations = template["metadata"]["annotations"] assert annotations["vault.hashicorp.com/role"] == "harbor-policy-bootstrap" assert ( annotations["vault.hashicorp.com/agent-inject-secret-harbor-admin-password"] == "kv/data/atlas/harbor/harbor-core" ) pod = template["spec"] assert pod["serviceAccountName"] == "harbor-policy-bootstrap" assert pod["enableServiceLinks"] is False container = pod["containers"][0] assert "@sha256:" in container["image"] assert container["securityContext"]["readOnlyRootFilesystem"] is True assert container["securityContext"]["runAsNonRoot"] is True assert container["securityContext"]["capabilities"]["drop"] == ["ALL"] release_docs = (ROOT / "docs/hermes_webui_release.md").read_text(encoding="utf-8") assert "immutable-tag:list" in release_docs assert "harbor-hermes-agent-immutability-ensure-1" in release_docs class _FakePolicyClient: origin = "https://registry.bstein.dev/api/v2.0" def __init__(self, responses): self.responses = list(responses) self.calls = [] def request(self, method, path, payload=None): self.calls.append((method, path, payload)) return self.responses.pop(0) def test_webui_policy_is_idempotent_and_create_is_reread() -> None: """The desired Harbor rule validates in place or verifies its exact new ID.""" policy = _load(POLICY, "webui_policy_idempotency") existing = {"id": 17, **policy.EXPECTED_RULE} client = _FakePolicyClient( [(200, json.dumps([existing]).encode(), {"X-Total-Count": "1"})] ) assert policy.ensure_rule(client) == 17 assert [call[0] for call in client.calls] == ["GET"] created = {"id": 23, **policy.EXPECTED_RULE} client = _FakePolicyClient( [ (200, b"[]", {"X-Total-Count": "0"}), ( 201, b"", {"Location": ("/api/v2.0/projects/bstein/immutabletagrules/23")}, ), (200, json.dumps([created]).encode(), {"X-Total-Count": "1"}), ] ) assert policy.ensure_rule(client) == 23 assert client.calls[1] == ( "POST", "/projects/bstein/immutabletagrules", policy.EXPECTED_RULE, ) def test_webui_policy_rejects_disabled_duplicate_or_truncated_rules() -> None: """Ambiguous or incomplete Harbor evidence can never unblock publication.""" policy = _load(POLICY, "webui_policy_rejections") disabled = {"id": 17, **policy.EXPECTED_RULE, "disabled": True} with pytest.raises(RuntimeError, match="not enabled and exact"): policy.ensure_rule( _FakePolicyClient( [ ( 200, json.dumps([disabled]).encode(), {"X-Total-Count": "1"}, ) ] ) ) duplicate = [ {"id": 17, **policy.EXPECTED_RULE}, {"id": 18, **policy.EXPECTED_RULE}, ] with pytest.raises(RuntimeError, match="multiple Hermes WebUI"): policy.ensure_rule( _FakePolicyClient( [ ( 200, json.dumps(duplicate).encode(), {"X-Total-Count": "2"}, ) ] ) ) with pytest.raises(RuntimeError, match="truncated"): policy.list_rules(_FakePolicyClient([(200, b"[]", {"X-Total-Count": "1"})])) def test_pipeline_archives_exact_eight_files_before_release() -> None: """Release cannot pass with missing digest, workload, or metadata evidence.""" source = PIPELINE.read_text(encoding="utf-8") evidence = source.split("stage('Verify and archive release evidence')", 1)[1] evidence = evidence.split("stage('Publish Flux release tag')", 1)[0] archive = evidence.split("artifacts: '", 1)[1].split("'", 1)[0].split(",") assert len(archive) == len(set(archive)) == 8 assert all("*" not in path for path in archive) assert "find build -type f" in evidence assert "allowEmptyArchive: false" in evidence assert "build/hermes-webui.source-revision" in archive assert "hermes-chat-statefulset.yaml" in evidence assert "hermes-dashboard-deployment.yaml" in evidence assert " post {" not in source