"""Fresh fail-closed review cases for the Hermes image builder.""" from __future__ import annotations import importlib.util import sys from pathlib import Path import pytest import yaml REPO_ROOT = Path(__file__).resolve().parents[2] PIPELINE = REPO_ROOT / "ci/Jenkinsfile.hermes-agent-image" DOCKERFILE = REPO_ROOT / "dockerfiles/Dockerfile.hermes-agent" RUNNER = REPO_ROOT / "dockerfiles/hermes-kaniko-heredoc-runner.py" RELEASE = REPO_ROOT / "ci/scripts/hermes_image_release.py" BUILDER_SA = REPO_ROOT / "services/jenkins/hermes-image-builder-serviceaccount.yaml" def _load(path: Path, name: str): spec = importlib.util.spec_from_file_location(name, path) assert spec is not None and spec.loader is not None module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module def _pod_spec() -> dict: source = PIPELINE.read_text(encoding="utf-8") pod_yaml = source.split('yaml """', 1)[1].split('"""', 1)[0] return yaml.safe_load(pod_yaml)["spec"] def _amd64_pod_spec() -> dict: """Parse the second inline pod (the native amd64 build leg on titan-24).""" source = PIPELINE.read_text(encoding="utf-8") blocks = source.split('yaml """') assert len(blocks) == 3, "expected exactly the arm64 and amd64 build pods" pod_yaml = blocks[2].split('"""', 1)[0] return yaml.safe_load(pod_yaml)["spec"] def test_builder_prefers_rpi5_with_healthy_arm64_worker_fallback() -> None: """Disposable builds prefer rpi5 without excluding schedulable rpi4 workers.""" spec = _pod_spec() assert spec["nodeSelector"] == { "kubernetes.io/arch": "arm64", "node-role.kubernetes.io/worker": "true", } affinity = spec["affinity"]["nodeAffinity"] assert affinity["preferredDuringSchedulingIgnoredDuringExecution"] == [ { "weight": 100, "preference": { "matchExpressions": [ {"key": "hardware", "operator": "In", "values": ["rpi5"]} ] }, } ] expressions = affinity["requiredDuringSchedulingIgnoredDuringExecution"][ "nodeSelectorTerms" ][0]["matchExpressions"] host_rule = next( rule for rule in expressions if rule["key"] == "kubernetes.io/hostname" ) assert host_rule["operator"] == "NotIn" assert set(host_rule["values"]) >= { "titan-04", "titan-14", "titan-18", "titan-19", "titan-22", "titan-24", } @pytest.mark.parametrize( "unsupported", [ "RUN cat < None: """Any RUN heredoc outside the reviewed nine is rejected before replay.""" module = _load(RUNNER, f"heredoc_reject_{abs(hash(unsupported))}") dockerfile = tmp_path / "Dockerfile" dockerfile.write_text( DOCKERFILE.read_text(encoding="utf-8") + "\n" + unsupported + "\n", encoding="utf-8", ) calls = [] monkeypatch.setattr( module.subprocess, "run", lambda *_args, **_kwargs: calls.append(1) ) with pytest.raises(ValueError, match="unsupported RUN heredoc"): module.replay(dockerfile, 1) assert calls == [] def test_exact_reviewed_heredocs_remain_buildkit_native_by_default() -> None: """Current source inventories cleanly and enables replay only for Kaniko.""" module = _load(RUNNER, "heredoc_positive_contract") source = DOCKERFILE.read_text(encoding="utf-8") assert len(module.extract_blocks(source)) == 9 assert source.count("ARG HERMES_KANIKO_HEREDOC_COMPAT=0") == 1 assert "Kaniko v1.23.2" in RUNNER.read_text(encoding="utf-8") pipeline = PIPELINE.read_text(encoding="utf-8") # The reviewed heredoc replay is enabled on both native Kaniko legs. assert pipeline.count("--build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1") == 2 def test_appended_tenth_reviewed_form_rejects_before_execution( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: """Even an otherwise supported marker cannot expand the nine-block set.""" module = _load(RUNNER, "heredoc_tenth_block") dockerfile = tmp_path / "Dockerfile" dockerfile.write_text( DOCKERFILE.read_text(encoding="utf-8") + "\nRUN node <<'NODE'\nconsole.log('tenth');\nNODE\n", encoding="utf-8", ) calls = [] monkeypatch.setattr( module.subprocess, "run", lambda *_args, **_kwargs: calls.append(1) ) with pytest.raises(ValueError, match="contract changed"): module.replay(dockerfile, 1) assert calls == [] @pytest.mark.parametrize( ("prefix", "unsupported"), [ ("", "R\\\nUN cat < None: """Split RUN/opcode operators and backtick escapes cannot evade inventory.""" module = _load(RUNNER, f"heredoc_logical_{abs(hash((prefix, unsupported)))}") dockerfile = tmp_path / "Dockerfile" dockerfile.write_text( prefix + DOCKERFILE.read_text(encoding="utf-8") + "\n" + unsupported + "\n", encoding="utf-8", ) calls = [] monkeypatch.setattr( module.subprocess, "run", lambda *_args, **_kwargs: calls.append(1) ) with pytest.raises(ValueError, match="unsupported RUN heredoc"): module.replay(dockerfile, 1) assert calls == [] def test_merged_main_replays_each_block_before_dependent_work() -> None: """PR13's blocked-task regression runs only after its source patch replay.""" source = DOCKERFILE.read_text(encoding="utf-8") replay_positions = [ source.index(f"--block-index {index}") for index in range(1, 10) ] # Locate repeated markers rather than trusting one synthetic occurrence. block_positions = [] for marker in ("RUN node <<'NODE'", "RUN python - <<'PY'"): start = 0 while True: position = source.find(marker, start) if position < 0: break block_positions.append(position) start = position + len(marker) block_positions.sort() assert len(block_positions) == len(replay_positions) == 9 for index, (block, replay) in enumerate(zip(block_positions, replay_positions)): assert block < replay if index + 1 < len(block_positions): assert replay < block_positions[index + 1] regression = source.index( "RUN /opt/hermes/.venv/bin/python /tmp/hermes-kanban-blocked-regression.py" ) assert replay_positions[4] < regression < replay_positions[5] def test_builder_service_account_is_explicit_tokenless_and_unbound() -> None: """The build Pod selects one tokenless SA that no tracked RBAC grants bind.""" account = yaml.safe_load(BUILDER_SA.read_text(encoding="utf-8")) assert account == { "apiVersion": "v1", "kind": "ServiceAccount", "metadata": {"name": "hermes-image-builder", "namespace": "jenkins"}, "automountServiceAccountToken": False, } kustomization = yaml.safe_load( (REPO_ROOT / "services/jenkins/kustomization.yaml").read_text(encoding="utf-8") ) assert "hermes-image-builder-serviceaccount.yaml" in kustomization["resources"] spec = _pod_spec() assert spec["serviceAccountName"] == "hermes-image-builder" assert spec["automountServiceAccountToken"] is False for volume in spec.get("volumes", []): projected = volume.get("projected", {}) assert all( "serviceAccountToken" not in item for item in projected.get("sources", []) ) for manifest in (REPO_ROOT / "services/jenkins").glob("*.yaml"): for document in yaml.safe_load_all(manifest.read_text(encoding="utf-8")): if not isinstance(document, dict) or document.get("kind") not in { "RoleBinding", "ClusterRoleBinding", }: continue subjects = document.get("subjects") or [] assert not any( item.get("kind") == "ServiceAccount" and item.get("name") == "hermes-image-builder" for item in subjects ) def _release_fixture(tmp_path: Path): module = _load(RELEASE, f"release_evidence_{tmp_path.name}") digest = "sha256:" + "7" * 64 revision = "8" * 40 build = "23" destination = f"{module.DEFAULT_IMAGE}:git-{revision}-build-{build}" digest_file = tmp_path / "hermes-agent.digest" image_file = tmp_path / "hermes-agent.image" manifest = tmp_path / "kustomization.yaml" output = tmp_path / "release" digest_file.write_text(digest + "\n", encoding="utf-8") image_file.write_text(f"{destination}@{digest}\n", encoding="utf-8") manifest.write_text( "images:\n" f" - name: {module.DEFAULT_IMAGE}\n" " digest: sha256:" + "0" * 64 + "\n", encoding="utf-8", ) module.write_release_artifacts( digest=digest, source_revision=revision, build_number=build, destination=destination, kustomization=manifest, output_dir=output, ) kwargs = { "digest_file": digest_file, "image_file": image_file, "source_revision": revision, "build_number": build, "destination": destination, "kustomization": manifest, "output_dir": output, } return module, kwargs def test_success_evidence_revalidation_accepts_only_exact_complete_set( tmp_path: Path, ) -> None: """Exact evidence passes; an extra, missing, or altered file fails closed.""" module, kwargs = _release_fixture(tmp_path) module.validate_release_artifacts(**kwargs) extra = kwargs["output_dir"] / "unexpected" extra.write_text("surprise\n", encoding="utf-8") with pytest.raises(ValueError, match="exactly three"): module.validate_release_artifacts(**kwargs) extra.unlink() metadata = kwargs["output_dir"] / "hermes-agent-image.json" original = metadata.read_text(encoding="utf-8") metadata.write_text( original.replace('"build_number": "23"', '"build_number": "24"') ) with pytest.raises(ValueError, match="incomplete or mismatched"): module.validate_release_artifacts(**kwargs) metadata.write_text(original, encoding="utf-8") (kwargs["output_dir"] / "hermes-image-update.patch").unlink() with pytest.raises(ValueError, match="exactly three"): module.validate_release_artifacts(**kwargs) def test_verify_evidence_cli_needs_no_runtime_registry_credential( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """Post-success validation is deterministic and cannot mask missing creds.""" module, kwargs = _release_fixture(tmp_path) monkeypatch.delenv("HARBOR_USER", raising=False) monkeypatch.delenv("HARBOR_PASSWORD", raising=False) monkeypatch.setattr( sys, "argv", [ "hermes_image_release.py", "verify-evidence", "--digest-file", str(kwargs["digest_file"]), "--image-file", str(kwargs["image_file"]), "--source-revision", kwargs["source_revision"], "--build-number", kwargs["build_number"], "--destination", kwargs["destination"], "--kustomization", str(kwargs["kustomization"]), "--output-dir", str(kwargs["output_dir"]), ], ) assert module.main() == 0 def test_pipeline_requires_and_archives_exact_release_evidence() -> None: """Missing evidence must fail before the candidate becomes a release.""" source = PIPELINE.read_text(encoding="utf-8") evidence = source.split("stage('Verify and archive release evidence')", 1)[1] evidence = evidence.split("stage('Publish Flux release tag')", 1)[0] assert "verify-evidence" in evidence assert "allowEmptyArchive: false" in evidence archive = evidence.split("artifacts: '", 1)[1].split("'", 1)[0] paths = archive.split(",") # Multi-arch adds both per-arch leaf digests/images plus the index digest. assert len(paths) == 11 assert len(set(paths)) == 11 assert all("*" not in path for path in paths) assert "find build -type f" in evidence assert "build/hermes-agent.source-revision" in paths assert "build/hermes-agent.digest" in paths for arch in ("arm64", "amd64"): assert f"build/hermes-agent-{arch}.digest" in paths assert f"build/hermes-agent-{arch}.image" in paths promotion = source.split("stage('Publish Flux release tag')", 1)[1] assert "ci/scripts/hermes_oci_promote.py" in promotion assert " post {" not in source def test_amd64_leg_is_pinned_to_titan24_and_resource_capped() -> None: """The amd64 build leg lands on titan-24, tolerates its guard taint, and is capped.""" spec = _amd64_pod_spec() assert spec["nodeSelector"] == { "kubernetes.io/arch": "amd64", "kubernetes.io/hostname": "titan-24", "node-role.kubernetes.io/worker": "true", } # titan-24 co-hosts the Sui validator; the disposable build must tolerate # whatever guard taint the node carries so the pinned pod still schedules. assert {"operator": "Exists"} in spec["tolerations"] assert spec["serviceAccountName"] == "hermes-image-builder" assert spec["automountServiceAccountToken"] is False assert spec["enableServiceLinks"] is False containers = {item["name"]: item for item in spec["containers"]} kaniko = containers["kaniko"] # Identical pinned Kaniko across both legs -- no second, unreviewed builder. assert kaniko["image"] == ( "gcr.io/kaniko-project/executor@sha256:" "c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e" ) assert kaniko["securityContext"]["capabilities"]["add"] == [ "CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID", ] for container in containers.values(): security = container["securityContext"] assert security["allowPrivilegeEscalation"] is False assert security["capabilities"]["drop"] == ["ALL"] assert security["seccompProfile"]["type"] == "RuntimeDefault" assert security.get("privileged", False) is False # Tight caps keep the disposable build from starving the co-hosted validator: # the amd64 kaniko ceiling is strictly below the arm64 leg's 2 CPU / 4Gi. limits = kaniko["resources"]["limits"] assert limits["cpu"] == "1500m" assert limits["memory"] == "3Gi" def test_amd64_leg_source_is_independently_boundary_checked() -> None: """The amd64 pod re-derives and re-verifies the reviewed revision itself.""" source = PIPELINE.read_text(encoding="utf-8") amd64_stage = source.split("stage('Build amd64 leg without a daemon')", 1)[1] amd64_stage = amd64_stage.split("stage('Combine multi-arch index')", 1)[0] # Same fail-closed boundary as the coordinating pod, re-run in the amd64 pod. assert 'merge-base --is-ancestor "${EXPECTED_SOURCE_REVISION}"' in amd64_stage assert 'checkout --detach "${EXPECTED_SOURCE_REVISION}"' in amd64_stage assert 'test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}"' in amd64_stage assert 'status --porcelain' in amd64_stage assert '$(cat build/hermes-agent.destination)-amd64' in amd64_stage # The amd64 leaf evidence crosses pods only through an explicit stash. assert "stash" in amd64_stage assert "hermes-agent-amd64.digest" in amd64_stage def test_combine_stage_publishes_and_reverifies_the_index() -> None: """Kaniko cannot combine; the reviewed python combiner assembles the index.""" source = PIPELINE.read_text(encoding="utf-8") combine = source.split("stage('Combine multi-arch index')", 1)[1] combine = combine.split("stage('Render reviewed Flux handoff')", 1)[0] assert "unstash 'hermes-agent-amd64-evidence'" in combine assert "ci/scripts/hermes_multiarch_combine.py" in combine assert "--arm64-digest-file build/hermes-agent-arm64.digest" in combine assert "--amd64-digest-file build/hermes-agent-amd64.digest" in combine # The combiner emits the arch-less index evidence the existing chain promotes. assert "--digest-file build/hermes-agent.digest" in combine assert "--image-file build/hermes-agent.image" in combine # Downstream render/verify/promote still consume the single index digest file. render = source.split("stage('Render reviewed Flux handoff')", 1)[1] assert "--digest-file build/hermes-agent.digest" in render