462 lines
18 KiB
Python
462 lines
18 KiB
Python
"""Fresh fail-closed review cases for the Hermes image builder."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
import yaml
|
|
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
PIPELINE = REPO_ROOT / "ci/Jenkinsfile.hermes-agent-image"
|
|
DOCKERFILE = REPO_ROOT / "dockerfiles/Dockerfile.hermes-agent"
|
|
RUNNER = REPO_ROOT / "dockerfiles/hermes-kaniko-heredoc-runner.py"
|
|
RELEASE = REPO_ROOT / "ci/scripts/hermes_image_release.py"
|
|
BUILDER_SA = REPO_ROOT / "services/jenkins/hermes-image-builder-serviceaccount.yaml"
|
|
|
|
|
|
def _load(path: Path, name: str):
|
|
spec = importlib.util.spec_from_file_location(name, path)
|
|
assert spec is not None and spec.loader is not None
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
def _pod_spec() -> dict:
|
|
source = PIPELINE.read_text(encoding="utf-8")
|
|
pod_yaml = source.split('yaml """', 1)[1].split('"""', 1)[0]
|
|
return yaml.safe_load(pod_yaml)["spec"]
|
|
|
|
|
|
def _amd64_pod_spec() -> dict:
|
|
"""Parse the second inline pod (the native amd64 build leg on titan-24)."""
|
|
source = PIPELINE.read_text(encoding="utf-8")
|
|
blocks = source.split('yaml """')
|
|
assert len(blocks) == 3, "expected exactly the arm64 and amd64 build pods"
|
|
pod_yaml = blocks[2].split('"""', 1)[0]
|
|
return yaml.safe_load(pod_yaml)["spec"]
|
|
|
|
|
|
def test_builder_prefers_rpi5_with_healthy_arm64_worker_fallback() -> None:
|
|
"""Disposable builds prefer rpi5 without excluding schedulable rpi4 workers."""
|
|
spec = _pod_spec()
|
|
assert spec["nodeSelector"] == {
|
|
"kubernetes.io/arch": "arm64",
|
|
"node-role.kubernetes.io/worker": "true",
|
|
}
|
|
affinity = spec["affinity"]["nodeAffinity"]
|
|
assert affinity["preferredDuringSchedulingIgnoredDuringExecution"] == [
|
|
{
|
|
"weight": 100,
|
|
"preference": {
|
|
"matchExpressions": [
|
|
{"key": "hardware", "operator": "In", "values": ["rpi5"]}
|
|
]
|
|
},
|
|
}
|
|
]
|
|
expressions = affinity["requiredDuringSchedulingIgnoredDuringExecution"][
|
|
"nodeSelectorTerms"
|
|
][0]["matchExpressions"]
|
|
host_rule = next(
|
|
rule for rule in expressions if rule["key"] == "kubernetes.io/hostname"
|
|
)
|
|
assert host_rule["operator"] == "NotIn"
|
|
assert set(host_rule["values"]) >= {
|
|
"titan-04",
|
|
"titan-08",
|
|
"titan-14",
|
|
"titan-18",
|
|
"titan-19",
|
|
"titan-22",
|
|
"titan-24",
|
|
}
|
|
|
|
|
|
def test_builder_reserves_storage_before_kaniko_expands_the_workspace() -> None:
|
|
"""The ARM build cannot be scheduled onto the 25 GiB worker disks."""
|
|
containers = {item["name"]: item for item in _pod_spec()["containers"]}
|
|
resources = containers["kaniko"]["resources"]
|
|
assert resources["requests"]["ephemeral-storage"] == "32Gi"
|
|
assert resources["limits"]["ephemeral-storage"] == "40Gi"
|
|
source = PIPELINE.read_text(encoding="utf-8")
|
|
arm_build = source.split("stage('Build arm64 leg without a daemon')", 1)[1].split(
|
|
"stage('Build amd64 leg without a daemon')", 1
|
|
)[0]
|
|
assert 'minimum_available_kib=16777216' in arm_build
|
|
assert '/busybox/df -Pk /' in arm_build
|
|
assert 'cannot determine Kaniko ephemeral-storage availability' in arm_build
|
|
assert 'Kaniko requires at least 16 GiB of free ephemeral storage' in arm_build
|
|
assert arm_build.index("minimum_available_kib=16777216") < arm_build.index(
|
|
"withCredentials([usernamePassword("
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"unsupported",
|
|
[
|
|
"RUN cat <<EOF\nbody\nEOF",
|
|
"run cat <<'EOF'\nbody\nEOF",
|
|
' RUN cat <<"EOF"\nbody\nEOF',
|
|
"RUN <<-EOF\n\tbody\n\tEOF",
|
|
'RUN ["/bin/sh", "-c"] <<EOF\nbody\nEOF',
|
|
"RUN echo before <<EOF after\nbody\nEOF",
|
|
"RUN set -eu \\\n && cat <<'EOF'\nbody\nEOF",
|
|
],
|
|
)
|
|
def test_all_unsupported_run_heredoc_forms_reject_before_execution(
|
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, unsupported: str
|
|
) -> None:
|
|
"""Any RUN heredoc outside the reviewed nine is rejected before replay."""
|
|
module = _load(RUNNER, f"heredoc_reject_{abs(hash(unsupported))}")
|
|
dockerfile = tmp_path / "Dockerfile"
|
|
dockerfile.write_text(
|
|
DOCKERFILE.read_text(encoding="utf-8") + "\n" + unsupported + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
calls = []
|
|
monkeypatch.setattr(
|
|
module.subprocess, "run", lambda *_args, **_kwargs: calls.append(1)
|
|
)
|
|
with pytest.raises(ValueError, match="unsupported RUN heredoc"):
|
|
module.replay(dockerfile, 1)
|
|
assert calls == []
|
|
|
|
|
|
def test_exact_reviewed_heredocs_remain_buildkit_native_by_default() -> None:
|
|
"""Current source inventories cleanly and enables replay only for Kaniko."""
|
|
module = _load(RUNNER, "heredoc_positive_contract")
|
|
source = DOCKERFILE.read_text(encoding="utf-8")
|
|
assert len(module.extract_blocks(source)) == 9
|
|
assert source.count("ARG HERMES_KANIKO_HEREDOC_COMPAT=0") == 1
|
|
assert "Kaniko v1.23.2" in RUNNER.read_text(encoding="utf-8")
|
|
pipeline = PIPELINE.read_text(encoding="utf-8")
|
|
# The reviewed heredoc replay is enabled on both native Kaniko legs.
|
|
assert pipeline.count("--build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1") == 2
|
|
|
|
|
|
def test_appended_tenth_reviewed_form_rejects_before_execution(
|
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
|
) -> None:
|
|
"""Even an otherwise supported marker cannot expand the nine-block set."""
|
|
module = _load(RUNNER, "heredoc_tenth_block")
|
|
dockerfile = tmp_path / "Dockerfile"
|
|
dockerfile.write_text(
|
|
DOCKERFILE.read_text(encoding="utf-8")
|
|
+ "\nRUN node <<'NODE'\nconsole.log('tenth');\nNODE\n",
|
|
encoding="utf-8",
|
|
)
|
|
calls = []
|
|
monkeypatch.setattr(
|
|
module.subprocess, "run", lambda *_args, **_kwargs: calls.append(1)
|
|
)
|
|
with pytest.raises(ValueError, match="contract changed"):
|
|
module.replay(dockerfile, 1)
|
|
assert calls == []
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("prefix", "unsupported"),
|
|
[
|
|
("", "R\\\nUN cat <<EOF\nbody\nEOF"),
|
|
("", "RUN cat <\\\n<EOF\nbody\nEOF"),
|
|
("", "R\\\nUN cat <\\\n<EOF\nbody\nEOF"),
|
|
("# escape=`\n", "R`\nUN cat <<EOF\nbody\nEOF"),
|
|
("# escape=`\n", "RUN cat <`\n<EOF\nbody\nEOF"),
|
|
("# escape=`\n", "R`\nUN cat <`\n<EOF\nbody\nEOF"),
|
|
("", "R\\\n# removed comment\nUN cat <<EOF\nbody\nEOF"),
|
|
(
|
|
"# ordinary comment\n# escape=`\n",
|
|
"R\\\nUN cat <<EOF\nbody\nEOF",
|
|
),
|
|
],
|
|
)
|
|
def test_logical_instruction_normalization_rejects_split_heredocs_before_execution(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
tmp_path: Path,
|
|
prefix: str,
|
|
unsupported: str,
|
|
) -> None:
|
|
"""Split RUN/opcode operators and backtick escapes cannot evade inventory."""
|
|
module = _load(RUNNER, f"heredoc_logical_{abs(hash((prefix, unsupported)))}")
|
|
dockerfile = tmp_path / "Dockerfile"
|
|
dockerfile.write_text(
|
|
prefix + DOCKERFILE.read_text(encoding="utf-8") + "\n" + unsupported + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
calls = []
|
|
monkeypatch.setattr(
|
|
module.subprocess, "run", lambda *_args, **_kwargs: calls.append(1)
|
|
)
|
|
with pytest.raises(ValueError, match="unsupported RUN heredoc"):
|
|
module.replay(dockerfile, 1)
|
|
assert calls == []
|
|
|
|
|
|
def test_merged_main_replays_each_block_before_dependent_work() -> None:
|
|
"""PR13's blocked-task regression runs only after its source patch replay."""
|
|
source = DOCKERFILE.read_text(encoding="utf-8")
|
|
replay_positions = [
|
|
source.index(f"--block-index {index}") for index in range(1, 10)
|
|
]
|
|
# Locate repeated markers rather than trusting one synthetic occurrence.
|
|
block_positions = []
|
|
for marker in ("RUN node <<'NODE'", "RUN python - <<'PY'"):
|
|
start = 0
|
|
while True:
|
|
position = source.find(marker, start)
|
|
if position < 0:
|
|
break
|
|
block_positions.append(position)
|
|
start = position + len(marker)
|
|
block_positions.sort()
|
|
assert len(block_positions) == len(replay_positions) == 9
|
|
for index, (block, replay) in enumerate(zip(block_positions, replay_positions)):
|
|
assert block < replay
|
|
if index + 1 < len(block_positions):
|
|
assert replay < block_positions[index + 1]
|
|
regression = source.index(
|
|
"RUN /opt/hermes/.venv/bin/python /tmp/hermes-kanban-blocked-regression.py"
|
|
)
|
|
assert replay_positions[4] < regression < replay_positions[5]
|
|
|
|
|
|
def test_builder_service_account_is_explicit_tokenless_and_unbound() -> None:
|
|
"""The build Pod selects one tokenless SA that no tracked RBAC grants bind."""
|
|
account = yaml.safe_load(BUILDER_SA.read_text(encoding="utf-8"))
|
|
assert account == {
|
|
"apiVersion": "v1",
|
|
"kind": "ServiceAccount",
|
|
"metadata": {"name": "hermes-image-builder", "namespace": "jenkins"},
|
|
"automountServiceAccountToken": False,
|
|
}
|
|
kustomization = yaml.safe_load(
|
|
(REPO_ROOT / "services/jenkins/kustomization.yaml").read_text(encoding="utf-8")
|
|
)
|
|
assert "hermes-image-builder-serviceaccount.yaml" in kustomization["resources"]
|
|
|
|
spec = _pod_spec()
|
|
assert spec["serviceAccountName"] == "hermes-image-builder"
|
|
assert spec["automountServiceAccountToken"] is False
|
|
for volume in spec.get("volumes", []):
|
|
projected = volume.get("projected", {})
|
|
assert all(
|
|
"serviceAccountToken" not in item for item in projected.get("sources", [])
|
|
)
|
|
|
|
for manifest in (REPO_ROOT / "services/jenkins").glob("*.yaml"):
|
|
for document in yaml.safe_load_all(manifest.read_text(encoding="utf-8")):
|
|
if not isinstance(document, dict) or document.get("kind") not in {
|
|
"RoleBinding",
|
|
"ClusterRoleBinding",
|
|
}:
|
|
continue
|
|
subjects = document.get("subjects") or []
|
|
assert not any(
|
|
item.get("kind") == "ServiceAccount"
|
|
and item.get("name") == "hermes-image-builder"
|
|
for item in subjects
|
|
)
|
|
|
|
|
|
def _release_fixture(tmp_path: Path):
|
|
module = _load(RELEASE, f"release_evidence_{tmp_path.name}")
|
|
digest = "sha256:" + "7" * 64
|
|
revision = "8" * 40
|
|
build = "23"
|
|
destination = f"{module.DEFAULT_IMAGE}:git-{revision}-build-{build}"
|
|
digest_file = tmp_path / "hermes-agent.digest"
|
|
image_file = tmp_path / "hermes-agent.image"
|
|
manifest = tmp_path / "kustomization.yaml"
|
|
output = tmp_path / "release"
|
|
digest_file.write_text(digest + "\n", encoding="utf-8")
|
|
image_file.write_text(f"{destination}@{digest}\n", encoding="utf-8")
|
|
manifest.write_text(
|
|
"images:\n"
|
|
f" - name: {module.DEFAULT_IMAGE}\n"
|
|
" digest: sha256:" + "0" * 64 + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
module.write_release_artifacts(
|
|
digest=digest,
|
|
source_revision=revision,
|
|
build_number=build,
|
|
destination=destination,
|
|
kustomization=manifest,
|
|
output_dir=output,
|
|
)
|
|
kwargs = {
|
|
"digest_file": digest_file,
|
|
"image_file": image_file,
|
|
"source_revision": revision,
|
|
"build_number": build,
|
|
"destination": destination,
|
|
"kustomization": manifest,
|
|
"output_dir": output,
|
|
}
|
|
return module, kwargs
|
|
|
|
|
|
def test_success_evidence_revalidation_accepts_only_exact_complete_set(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
"""Exact evidence passes; an extra, missing, or altered file fails closed."""
|
|
module, kwargs = _release_fixture(tmp_path)
|
|
module.validate_release_artifacts(**kwargs)
|
|
|
|
extra = kwargs["output_dir"] / "unexpected"
|
|
extra.write_text("surprise\n", encoding="utf-8")
|
|
with pytest.raises(ValueError, match="exactly three"):
|
|
module.validate_release_artifacts(**kwargs)
|
|
extra.unlink()
|
|
|
|
metadata = kwargs["output_dir"] / "hermes-agent-image.json"
|
|
original = metadata.read_text(encoding="utf-8")
|
|
metadata.write_text(
|
|
original.replace('"build_number": "23"', '"build_number": "24"')
|
|
)
|
|
with pytest.raises(ValueError, match="incomplete or mismatched"):
|
|
module.validate_release_artifacts(**kwargs)
|
|
metadata.write_text(original, encoding="utf-8")
|
|
(kwargs["output_dir"] / "hermes-image-update.patch").unlink()
|
|
with pytest.raises(ValueError, match="exactly three"):
|
|
module.validate_release_artifacts(**kwargs)
|
|
|
|
|
|
def test_verify_evidence_cli_needs_no_runtime_registry_credential(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
"""Post-success validation is deterministic and cannot mask missing creds."""
|
|
module, kwargs = _release_fixture(tmp_path)
|
|
monkeypatch.delenv("HARBOR_USER", raising=False)
|
|
monkeypatch.delenv("HARBOR_PASSWORD", raising=False)
|
|
monkeypatch.setattr(
|
|
sys,
|
|
"argv",
|
|
[
|
|
"hermes_image_release.py",
|
|
"verify-evidence",
|
|
"--digest-file",
|
|
str(kwargs["digest_file"]),
|
|
"--image-file",
|
|
str(kwargs["image_file"]),
|
|
"--source-revision",
|
|
kwargs["source_revision"],
|
|
"--build-number",
|
|
kwargs["build_number"],
|
|
"--destination",
|
|
kwargs["destination"],
|
|
"--kustomization",
|
|
str(kwargs["kustomization"]),
|
|
"--output-dir",
|
|
str(kwargs["output_dir"]),
|
|
],
|
|
)
|
|
assert module.main() == 0
|
|
|
|
|
|
def test_pipeline_requires_and_archives_exact_release_evidence() -> None:
|
|
"""Missing evidence must fail before the candidate becomes a release."""
|
|
source = PIPELINE.read_text(encoding="utf-8")
|
|
evidence = source.split("stage('Verify and archive release evidence')", 1)[1]
|
|
evidence = evidence.split("stage('Publish Flux release tag')", 1)[0]
|
|
assert "verify-evidence" in evidence
|
|
assert "allowEmptyArchive: false" in evidence
|
|
archive = evidence.split("artifacts: '", 1)[1].split("'", 1)[0]
|
|
paths = archive.split(",")
|
|
# Multi-arch adds both per-arch leaf digests/images plus the index digest.
|
|
assert len(paths) == 11
|
|
assert len(set(paths)) == 11
|
|
assert all("*" not in path for path in paths)
|
|
assert "find build -type f" in evidence
|
|
assert "build/hermes-agent.source-revision" in paths
|
|
assert "build/hermes-agent.digest" in paths
|
|
for arch in ("arm64", "amd64"):
|
|
assert f"build/hermes-agent-{arch}.digest" in paths
|
|
assert f"build/hermes-agent-{arch}.image" in paths
|
|
promotion = source.split("stage('Publish Flux release tag')", 1)[1]
|
|
assert "ci/scripts/hermes_oci_promote.py" in promotion
|
|
assert " post {" not in source
|
|
|
|
|
|
def test_amd64_leg_is_pinned_to_titan24_and_resource_capped() -> None:
|
|
"""The amd64 build leg lands on titan-24 by hostname ONLY (titan-24 is an
|
|
accelerator, not a general worker), tolerates its taint, and is capped."""
|
|
spec = _amd64_pod_spec()
|
|
# Pin by arch + hostname only. Requiring node-role worker would force titan-24
|
|
# to be labeled a general worker, opening it to unrelated cluster scheduling.
|
|
assert spec["nodeSelector"] == {
|
|
"kubernetes.io/arch": "amd64",
|
|
"kubernetes.io/hostname": "titan-24",
|
|
}
|
|
assert "node-role.kubernetes.io/worker" not in spec["nodeSelector"]
|
|
# titan-24 co-hosts the Sui validator; the disposable build must tolerate
|
|
# whatever guard taint the node carries so the pinned pod still schedules.
|
|
assert {"operator": "Exists"} in spec["tolerations"]
|
|
assert spec["serviceAccountName"] == "hermes-image-builder"
|
|
assert spec["automountServiceAccountToken"] is False
|
|
assert spec["enableServiceLinks"] is False
|
|
|
|
containers = {item["name"]: item for item in spec["containers"]}
|
|
kaniko = containers["kaniko"]
|
|
# Identical pinned Kaniko across both legs -- no second, unreviewed builder.
|
|
assert kaniko["image"] == (
|
|
"gcr.io/kaniko-project/executor@sha256:"
|
|
"c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e"
|
|
)
|
|
assert kaniko["securityContext"]["capabilities"]["add"] == [
|
|
"CHOWN",
|
|
"FOWNER",
|
|
"DAC_OVERRIDE",
|
|
"SETGID",
|
|
"SETUID",
|
|
]
|
|
for container in containers.values():
|
|
security = container["securityContext"]
|
|
assert security["allowPrivilegeEscalation"] is False
|
|
assert security["capabilities"]["drop"] == ["ALL"]
|
|
assert security["seccompProfile"]["type"] == "RuntimeDefault"
|
|
assert security.get("privileged", False) is False
|
|
# Tight caps keep the disposable build from starving the co-hosted validator:
|
|
# the amd64 kaniko ceiling is strictly below the arm64 leg's 2 CPU / 4Gi.
|
|
limits = kaniko["resources"]["limits"]
|
|
assert limits["cpu"] == "1500m"
|
|
assert limits["memory"] == "3Gi"
|
|
|
|
|
|
def test_amd64_leg_source_is_independently_boundary_checked() -> None:
|
|
"""The amd64 pod re-derives and re-verifies the reviewed revision itself."""
|
|
source = PIPELINE.read_text(encoding="utf-8")
|
|
amd64_stage = source.split("stage('Build amd64 leg without a daemon')", 1)[1]
|
|
amd64_stage = amd64_stage.split("stage('Combine multi-arch index')", 1)[0]
|
|
# Same fail-closed boundary as the coordinating pod, re-run in the amd64 pod.
|
|
assert 'merge-base --is-ancestor "${EXPECTED_SOURCE_REVISION}"' in amd64_stage
|
|
assert 'checkout --detach "${EXPECTED_SOURCE_REVISION}"' in amd64_stage
|
|
assert 'test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}"' in amd64_stage
|
|
assert 'status --porcelain' in amd64_stage
|
|
assert '$(cat build/hermes-agent.destination)-amd64' in amd64_stage
|
|
# The amd64 leaf evidence crosses pods only through an explicit stash.
|
|
assert "stash" in amd64_stage
|
|
assert "hermes-agent-amd64.digest" in amd64_stage
|
|
|
|
|
|
def test_combine_stage_publishes_and_reverifies_the_index() -> None:
|
|
"""Kaniko cannot combine; the reviewed python combiner assembles the index."""
|
|
source = PIPELINE.read_text(encoding="utf-8")
|
|
combine = source.split("stage('Combine multi-arch index')", 1)[1]
|
|
combine = combine.split("stage('Render reviewed Flux handoff')", 1)[0]
|
|
assert "unstash 'hermes-agent-amd64-evidence'" in combine
|
|
assert "ci/scripts/hermes_multiarch_combine.py" in combine
|
|
assert "--arm64-digest-file build/hermes-agent-arm64.digest" in combine
|
|
assert "--amd64-digest-file build/hermes-agent-amd64.digest" in combine
|
|
# The combiner emits the arch-less index evidence the existing chain promotes.
|
|
assert "--digest-file build/hermes-agent.digest" in combine
|
|
assert "--image-file build/hermes-agent.image" in combine
|
|
# Downstream render/verify/promote still consume the single index digest file.
|
|
render = source.split("stage('Render reviewed Flux handoff')", 1)[1]
|
|
assert "--digest-file build/hermes-agent.digest" in render
|