336 lines
12 KiB
Python
336 lines
12 KiB
Python
"""Fresh fail-closed review cases for the Hermes image builder."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
import yaml
|
|
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
PIPELINE = REPO_ROOT / "ci/Jenkinsfile.hermes-agent-image"
|
|
DOCKERFILE = REPO_ROOT / "dockerfiles/Dockerfile.hermes-agent"
|
|
RUNNER = REPO_ROOT / "dockerfiles/hermes-kaniko-heredoc-runner.py"
|
|
RELEASE = REPO_ROOT / "ci/scripts/hermes_image_release.py"
|
|
BUILDER_SA = REPO_ROOT / "services/jenkins/hermes-image-builder-serviceaccount.yaml"
|
|
|
|
|
|
def _load(path: Path, name: str):
|
|
spec = importlib.util.spec_from_file_location(name, path)
|
|
assert spec is not None and spec.loader is not None
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
def _pod_spec() -> dict:
|
|
source = PIPELINE.read_text(encoding="utf-8")
|
|
pod_yaml = source.split('yaml """', 1)[1].split('"""', 1)[0]
|
|
return yaml.safe_load(pod_yaml)["spec"]
|
|
|
|
|
|
def test_builder_prefers_rpi5_with_healthy_arm64_worker_fallback() -> None:
|
|
"""Disposable builds prefer rpi5 without excluding schedulable rpi4 workers."""
|
|
spec = _pod_spec()
|
|
assert spec["nodeSelector"] == {
|
|
"kubernetes.io/arch": "arm64",
|
|
"node-role.kubernetes.io/worker": "true",
|
|
}
|
|
affinity = spec["affinity"]["nodeAffinity"]
|
|
assert affinity["preferredDuringSchedulingIgnoredDuringExecution"] == [
|
|
{
|
|
"weight": 100,
|
|
"preference": {
|
|
"matchExpressions": [
|
|
{"key": "hardware", "operator": "In", "values": ["rpi5"]}
|
|
]
|
|
},
|
|
}
|
|
]
|
|
expressions = affinity["requiredDuringSchedulingIgnoredDuringExecution"][
|
|
"nodeSelectorTerms"
|
|
][0]["matchExpressions"]
|
|
host_rule = next(
|
|
rule for rule in expressions if rule["key"] == "kubernetes.io/hostname"
|
|
)
|
|
assert host_rule["operator"] == "NotIn"
|
|
assert set(host_rule["values"]) >= {
|
|
"titan-04",
|
|
"titan-14",
|
|
"titan-18",
|
|
"titan-19",
|
|
"titan-22",
|
|
"titan-24",
|
|
}
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"unsupported",
|
|
[
|
|
"RUN cat <<EOF\nbody\nEOF",
|
|
"run cat <<'EOF'\nbody\nEOF",
|
|
' RUN cat <<"EOF"\nbody\nEOF',
|
|
"RUN <<-EOF\n\tbody\n\tEOF",
|
|
'RUN ["/bin/sh", "-c"] <<EOF\nbody\nEOF',
|
|
"RUN echo before <<EOF after\nbody\nEOF",
|
|
"RUN set -eu \\\n && cat <<'EOF'\nbody\nEOF",
|
|
],
|
|
)
|
|
def test_all_unsupported_run_heredoc_forms_reject_before_execution(
|
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, unsupported: str
|
|
) -> None:
|
|
"""Any RUN heredoc outside the reviewed nine is rejected before replay."""
|
|
module = _load(RUNNER, f"heredoc_reject_{abs(hash(unsupported))}")
|
|
dockerfile = tmp_path / "Dockerfile"
|
|
dockerfile.write_text(
|
|
DOCKERFILE.read_text(encoding="utf-8") + "\n" + unsupported + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
calls = []
|
|
monkeypatch.setattr(module.subprocess, "run", lambda *_args, **_kwargs: calls.append(1))
|
|
with pytest.raises(ValueError, match="unsupported RUN heredoc"):
|
|
module.replay(dockerfile, 1)
|
|
assert calls == []
|
|
|
|
|
|
def test_exact_reviewed_heredocs_remain_buildkit_native_by_default() -> None:
|
|
"""Current source inventories cleanly and enables replay only for Kaniko."""
|
|
module = _load(RUNNER, "heredoc_positive_contract")
|
|
source = DOCKERFILE.read_text(encoding="utf-8")
|
|
assert len(module.extract_blocks(source)) == 9
|
|
assert source.count("ARG HERMES_KANIKO_HEREDOC_COMPAT=0") == 1
|
|
assert "Kaniko v1.23.2" in RUNNER.read_text(encoding="utf-8")
|
|
pipeline = PIPELINE.read_text(encoding="utf-8")
|
|
assert pipeline.count("--build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1") == 1
|
|
|
|
|
|
def test_appended_tenth_reviewed_form_rejects_before_execution(
|
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
|
) -> None:
|
|
"""Even an otherwise supported marker cannot expand the nine-block set."""
|
|
module = _load(RUNNER, "heredoc_tenth_block")
|
|
dockerfile = tmp_path / "Dockerfile"
|
|
dockerfile.write_text(
|
|
DOCKERFILE.read_text(encoding="utf-8")
|
|
+ "\nRUN node <<'NODE'\nconsole.log('tenth');\nNODE\n",
|
|
encoding="utf-8",
|
|
)
|
|
calls = []
|
|
monkeypatch.setattr(module.subprocess, "run", lambda *_args, **_kwargs: calls.append(1))
|
|
with pytest.raises(ValueError, match="contract changed"):
|
|
module.replay(dockerfile, 1)
|
|
assert calls == []
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("prefix", "unsupported"),
|
|
[
|
|
("", "R\\\nUN cat <<EOF\nbody\nEOF"),
|
|
("", "RUN cat <\\\n<EOF\nbody\nEOF"),
|
|
("", "R\\\nUN cat <\\\n<EOF\nbody\nEOF"),
|
|
("# escape=`\n", "R`\nUN cat <<EOF\nbody\nEOF"),
|
|
("# escape=`\n", "RUN cat <`\n<EOF\nbody\nEOF"),
|
|
("# escape=`\n", "R`\nUN cat <`\n<EOF\nbody\nEOF"),
|
|
("", "R\\\n# removed comment\nUN cat <<EOF\nbody\nEOF"),
|
|
(
|
|
"# ordinary comment\n# escape=`\n",
|
|
"R\\\nUN cat <<EOF\nbody\nEOF",
|
|
),
|
|
],
|
|
)
|
|
def test_logical_instruction_normalization_rejects_split_heredocs_before_execution(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
tmp_path: Path,
|
|
prefix: str,
|
|
unsupported: str,
|
|
) -> None:
|
|
"""Split RUN/opcode operators and backtick escapes cannot evade inventory."""
|
|
module = _load(RUNNER, f"heredoc_logical_{abs(hash((prefix, unsupported)))}")
|
|
dockerfile = tmp_path / "Dockerfile"
|
|
dockerfile.write_text(
|
|
prefix + DOCKERFILE.read_text(encoding="utf-8") + "\n" + unsupported + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
calls = []
|
|
monkeypatch.setattr(module.subprocess, "run", lambda *_args, **_kwargs: calls.append(1))
|
|
with pytest.raises(ValueError, match="unsupported RUN heredoc"):
|
|
module.replay(dockerfile, 1)
|
|
assert calls == []
|
|
|
|
|
|
def test_merged_main_replays_each_block_before_dependent_work() -> None:
|
|
"""PR13's blocked-task regression runs only after its source patch replay."""
|
|
source = DOCKERFILE.read_text(encoding="utf-8")
|
|
replay_positions = [
|
|
source.index(f"--block-index {index}") for index in range(1, 10)
|
|
]
|
|
# Locate repeated markers rather than trusting one synthetic occurrence.
|
|
block_positions = []
|
|
for marker in ("RUN node <<'NODE'", "RUN python - <<'PY'"):
|
|
start = 0
|
|
while True:
|
|
position = source.find(marker, start)
|
|
if position < 0:
|
|
break
|
|
block_positions.append(position)
|
|
start = position + len(marker)
|
|
block_positions.sort()
|
|
assert len(block_positions) == len(replay_positions) == 9
|
|
for index, (block, replay) in enumerate(zip(block_positions, replay_positions)):
|
|
assert block < replay
|
|
if index + 1 < len(block_positions):
|
|
assert replay < block_positions[index + 1]
|
|
regression = source.index(
|
|
"RUN /opt/hermes/.venv/bin/python /tmp/hermes-kanban-blocked-regression.py"
|
|
)
|
|
assert replay_positions[4] < regression < replay_positions[5]
|
|
|
|
|
|
def test_builder_service_account_is_explicit_tokenless_and_unbound() -> None:
|
|
"""The build Pod selects one tokenless SA that no tracked RBAC grants bind."""
|
|
account = yaml.safe_load(BUILDER_SA.read_text(encoding="utf-8"))
|
|
assert account == {
|
|
"apiVersion": "v1",
|
|
"kind": "ServiceAccount",
|
|
"metadata": {"name": "hermes-image-builder", "namespace": "jenkins"},
|
|
"automountServiceAccountToken": False,
|
|
}
|
|
kustomization = yaml.safe_load(
|
|
(REPO_ROOT / "services/jenkins/kustomization.yaml").read_text(encoding="utf-8")
|
|
)
|
|
assert "hermes-image-builder-serviceaccount.yaml" in kustomization["resources"]
|
|
|
|
spec = _pod_spec()
|
|
assert spec["serviceAccountName"] == "hermes-image-builder"
|
|
assert spec["automountServiceAccountToken"] is False
|
|
for volume in spec.get("volumes", []):
|
|
projected = volume.get("projected", {})
|
|
assert all("serviceAccountToken" not in item for item in projected.get("sources", []))
|
|
|
|
for manifest in (REPO_ROOT / "services/jenkins").glob("*.yaml"):
|
|
for document in yaml.safe_load_all(manifest.read_text(encoding="utf-8")):
|
|
if not isinstance(document, dict) or document.get("kind") not in {
|
|
"RoleBinding",
|
|
"ClusterRoleBinding",
|
|
}:
|
|
continue
|
|
subjects = document.get("subjects") or []
|
|
assert not any(
|
|
item.get("kind") == "ServiceAccount"
|
|
and item.get("name") == "hermes-image-builder"
|
|
for item in subjects
|
|
)
|
|
|
|
|
|
def _release_fixture(tmp_path: Path):
|
|
module = _load(RELEASE, f"release_evidence_{tmp_path.name}")
|
|
digest = "sha256:" + "7" * 64
|
|
revision = "8" * 40
|
|
build = "23"
|
|
destination = f"{module.DEFAULT_IMAGE}:git-{revision}-build-{build}"
|
|
digest_file = tmp_path / "hermes-agent.digest"
|
|
image_file = tmp_path / "hermes-agent.image"
|
|
manifest = tmp_path / "kustomization.yaml"
|
|
output = tmp_path / "release"
|
|
digest_file.write_text(digest + "\n", encoding="utf-8")
|
|
image_file.write_text(f"{destination}@{digest}\n", encoding="utf-8")
|
|
manifest.write_text(
|
|
"images:\n"
|
|
f" - name: {module.DEFAULT_IMAGE}\n"
|
|
" digest: sha256:" + "0" * 64 + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
module.write_release_artifacts(
|
|
digest=digest,
|
|
source_revision=revision,
|
|
build_number=build,
|
|
destination=destination,
|
|
kustomization=manifest,
|
|
output_dir=output,
|
|
)
|
|
kwargs = {
|
|
"digest_file": digest_file,
|
|
"image_file": image_file,
|
|
"source_revision": revision,
|
|
"build_number": build,
|
|
"destination": destination,
|
|
"kustomization": manifest,
|
|
"output_dir": output,
|
|
}
|
|
return module, kwargs
|
|
|
|
|
|
def test_success_evidence_revalidation_accepts_only_exact_complete_set(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
"""Exact evidence passes; an extra, missing, or altered file fails closed."""
|
|
module, kwargs = _release_fixture(tmp_path)
|
|
module.validate_release_artifacts(**kwargs)
|
|
|
|
extra = kwargs["output_dir"] / "unexpected"
|
|
extra.write_text("surprise\n", encoding="utf-8")
|
|
with pytest.raises(ValueError, match="exactly three"):
|
|
module.validate_release_artifacts(**kwargs)
|
|
extra.unlink()
|
|
|
|
metadata = kwargs["output_dir"] / "hermes-agent-image.json"
|
|
original = metadata.read_text(encoding="utf-8")
|
|
metadata.write_text(original.replace('"build_number": "23"', '"build_number": "24"'))
|
|
with pytest.raises(ValueError, match="incomplete or mismatched"):
|
|
module.validate_release_artifacts(**kwargs)
|
|
metadata.write_text(original, encoding="utf-8")
|
|
(kwargs["output_dir"] / "hermes-image-update.patch").unlink()
|
|
with pytest.raises(ValueError, match="exactly three"):
|
|
module.validate_release_artifacts(**kwargs)
|
|
|
|
|
|
def test_verify_evidence_cli_needs_no_runtime_registry_credential(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
"""Post-success validation is deterministic and cannot mask missing creds."""
|
|
module, kwargs = _release_fixture(tmp_path)
|
|
monkeypatch.delenv("HARBOR_USER", raising=False)
|
|
monkeypatch.delenv("HARBOR_PASSWORD", raising=False)
|
|
monkeypatch.setattr(
|
|
sys,
|
|
"argv",
|
|
[
|
|
"hermes_image_release.py",
|
|
"verify-evidence",
|
|
"--digest-file",
|
|
str(kwargs["digest_file"]),
|
|
"--image-file",
|
|
str(kwargs["image_file"]),
|
|
"--source-revision",
|
|
kwargs["source_revision"],
|
|
"--build-number",
|
|
kwargs["build_number"],
|
|
"--destination",
|
|
kwargs["destination"],
|
|
"--kustomization",
|
|
str(kwargs["kustomization"]),
|
|
"--output-dir",
|
|
str(kwargs["output_dir"]),
|
|
],
|
|
)
|
|
assert module.main() == 0
|
|
|
|
|
|
def test_pipeline_success_post_requires_and_archives_exact_six_files() -> None:
|
|
"""Missing or partial post-success evidence must change the build to failed."""
|
|
source = PIPELINE.read_text(encoding="utf-8")
|
|
post = source.split(" post {", 1)[1]
|
|
assert "success {" in post
|
|
assert "always {" not in post
|
|
assert "verify-evidence" in post
|
|
assert 'allowEmptyArchive: false' in post
|
|
archive = post.split("artifacts: '", 1)[1].split("'", 1)[0]
|
|
paths = archive.split(",")
|
|
assert len(paths) == 6
|
|
assert len(set(paths)) == 6
|
|
assert all("*" not in path for path in paths)
|
|
assert "find build -type f" in post
|