atlas-iac/testing/tests/test_hermes_image_builder_fresh_review.py

"""Fresh fail-closed review cases for the Hermes image builder."""

from __future__ import annotations

import importlib.util
import sys
from pathlib import Path

import pytest
import yaml


REPO_ROOT = Path(__file__).resolve().parents[2]
PIPELINE = REPO_ROOT / "ci/Jenkinsfile.hermes-agent-image"
DOCKERFILE = REPO_ROOT / "dockerfiles/Dockerfile.hermes-agent"
RUNNER = REPO_ROOT / "dockerfiles/hermes-kaniko-heredoc-runner.py"
RELEASE = REPO_ROOT / "ci/scripts/hermes_image_release.py"
BUILDER_SA = REPO_ROOT / "services/jenkins/hermes-image-builder-serviceaccount.yaml"


def _load(path: Path, name: str):
    spec = importlib.util.spec_from_file_location(name, path)
    assert spec is not None and spec.loader is not None
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)
    return module


def _pod_spec() -> dict:
    source = PIPELINE.read_text(encoding="utf-8")
    pod_yaml = source.split('yaml """', 1)[1].split('"""', 1)[0]
    return yaml.safe_load(pod_yaml)["spec"]


def _amd64_pod_spec() -> dict:
    """Parse the second inline pod (the native amd64 build leg on titan-24)."""
    source = PIPELINE.read_text(encoding="utf-8")
    blocks = source.split('yaml """')
    assert len(blocks) == 3, "expected exactly the arm64 and amd64 build pods"
    pod_yaml = blocks[2].split('"""', 1)[0]
    return yaml.safe_load(pod_yaml)["spec"]


def test_builder_prefers_rpi5_with_healthy_arm64_worker_fallback() -> None:
    """Disposable builds prefer rpi5 without excluding schedulable rpi4 workers."""
    spec = _pod_spec()
    assert spec["nodeSelector"] == {
        "kubernetes.io/arch": "arm64",
        "node-role.kubernetes.io/worker": "true",
    }
    affinity = spec["affinity"]["nodeAffinity"]
    assert affinity["preferredDuringSchedulingIgnoredDuringExecution"] == [
        {
            "weight": 100,
            "preference": {
                "matchExpressions": [
                    {"key": "hardware", "operator": "In", "values": ["rpi5"]}
                ]
            },
        }
    ]
    expressions = affinity["requiredDuringSchedulingIgnoredDuringExecution"][
        "nodeSelectorTerms"
    ][0]["matchExpressions"]
    host_rule = next(
        rule for rule in expressions if rule["key"] == "kubernetes.io/hostname"
    )
    assert host_rule["operator"] == "NotIn"
    assert set(host_rule["values"]) >= {
        "titan-04",
        "titan-14",
        "titan-18",
        "titan-19",
        "titan-22",
        "titan-24",
    }


@pytest.mark.parametrize(
    "unsupported",
    [
        "RUN cat <<EOF\nbody\nEOF",
        "run cat <<'EOF'\nbody\nEOF",
        '  RUN cat <<"EOF"\nbody\nEOF',
        "RUN <<-EOF\n\tbody\n\tEOF",
        'RUN ["/bin/sh", "-c"] <<EOF\nbody\nEOF',
        "RUN echo before <<EOF after\nbody\nEOF",
        "RUN set -eu \\\n    && cat <<'EOF'\nbody\nEOF",
    ],
)
def test_all_unsupported_run_heredoc_forms_reject_before_execution(
    monkeypatch: pytest.MonkeyPatch, tmp_path: Path, unsupported: str
) -> None:
    """Any RUN heredoc outside the reviewed nine is rejected before replay."""
    module = _load(RUNNER, f"heredoc_reject_{abs(hash(unsupported))}")
    dockerfile = tmp_path / "Dockerfile"
    dockerfile.write_text(
        DOCKERFILE.read_text(encoding="utf-8") + "\n" + unsupported + "\n",
        encoding="utf-8",
    )
    calls = []
    monkeypatch.setattr(
        module.subprocess, "run", lambda *_args, **_kwargs: calls.append(1)
    )
    with pytest.raises(ValueError, match="unsupported RUN heredoc"):
        module.replay(dockerfile, 1)
    assert calls == []


def test_exact_reviewed_heredocs_remain_buildkit_native_by_default() -> None:
    """Current source inventories cleanly and enables replay only for Kaniko."""
    module = _load(RUNNER, "heredoc_positive_contract")
    source = DOCKERFILE.read_text(encoding="utf-8")
    assert len(module.extract_blocks(source)) == 9
    assert source.count("ARG HERMES_KANIKO_HEREDOC_COMPAT=0") == 1
    assert "Kaniko v1.23.2" in RUNNER.read_text(encoding="utf-8")
    pipeline = PIPELINE.read_text(encoding="utf-8")
    # The reviewed heredoc replay is enabled on both native Kaniko legs.
    assert pipeline.count("--build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1") == 2


def test_appended_tenth_reviewed_form_rejects_before_execution(
    monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
    """Even an otherwise supported marker cannot expand the nine-block set."""
    module = _load(RUNNER, "heredoc_tenth_block")
    dockerfile = tmp_path / "Dockerfile"
    dockerfile.write_text(
        DOCKERFILE.read_text(encoding="utf-8")
        + "\nRUN node <<'NODE'\nconsole.log('tenth');\nNODE\n",
        encoding="utf-8",
    )
    calls = []
    monkeypatch.setattr(
        module.subprocess, "run", lambda *_args, **_kwargs: calls.append(1)
    )
    with pytest.raises(ValueError, match="contract changed"):
        module.replay(dockerfile, 1)
    assert calls == []


@pytest.mark.parametrize(
    ("prefix", "unsupported"),
    [
        ("", "R\\\nUN cat <<EOF\nbody\nEOF"),
        ("", "RUN cat <\\\n<EOF\nbody\nEOF"),
        ("", "R\\\nUN cat <\\\n<EOF\nbody\nEOF"),
        ("# escape=`\n", "R`\nUN cat <<EOF\nbody\nEOF"),
        ("# escape=`\n", "RUN cat <`\n<EOF\nbody\nEOF"),
        ("# escape=`\n", "R`\nUN cat <`\n<EOF\nbody\nEOF"),
        ("", "R\\\n# removed comment\nUN cat <<EOF\nbody\nEOF"),
        (
            "# ordinary comment\n# escape=`\n",
            "R\\\nUN cat <<EOF\nbody\nEOF",
        ),
    ],
)
def test_logical_instruction_normalization_rejects_split_heredocs_before_execution(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
    prefix: str,
    unsupported: str,
) -> None:
    """Split RUN/opcode operators and backtick escapes cannot evade inventory."""
    module = _load(RUNNER, f"heredoc_logical_{abs(hash((prefix, unsupported)))}")
    dockerfile = tmp_path / "Dockerfile"
    dockerfile.write_text(
        prefix + DOCKERFILE.read_text(encoding="utf-8") + "\n" + unsupported + "\n",
        encoding="utf-8",
    )
    calls = []
    monkeypatch.setattr(
        module.subprocess, "run", lambda *_args, **_kwargs: calls.append(1)
    )
    with pytest.raises(ValueError, match="unsupported RUN heredoc"):
        module.replay(dockerfile, 1)
    assert calls == []


def test_merged_main_replays_each_block_before_dependent_work() -> None:
    """PR13's blocked-task regression runs only after its source patch replay."""
    source = DOCKERFILE.read_text(encoding="utf-8")
    replay_positions = [
        source.index(f"--block-index {index}") for index in range(1, 10)
    ]
    # Locate repeated markers rather than trusting one synthetic occurrence.
    block_positions = []
    for marker in ("RUN node <<'NODE'", "RUN python - <<'PY'"):
        start = 0
        while True:
            position = source.find(marker, start)
            if position < 0:
                break
            block_positions.append(position)
            start = position + len(marker)
    block_positions.sort()
    assert len(block_positions) == len(replay_positions) == 9
    for index, (block, replay) in enumerate(zip(block_positions, replay_positions)):
        assert block < replay
        if index + 1 < len(block_positions):
            assert replay < block_positions[index + 1]
    regression = source.index(
        "RUN /opt/hermes/.venv/bin/python /tmp/hermes-kanban-blocked-regression.py"
    )
    assert replay_positions[4] < regression < replay_positions[5]


def test_builder_service_account_is_explicit_tokenless_and_unbound() -> None:
    """The build Pod selects one tokenless SA that no tracked RBAC grants bind."""
    account = yaml.safe_load(BUILDER_SA.read_text(encoding="utf-8"))
    assert account == {
        "apiVersion": "v1",
        "kind": "ServiceAccount",
        "metadata": {"name": "hermes-image-builder", "namespace": "jenkins"},
        "automountServiceAccountToken": False,
    }
    kustomization = yaml.safe_load(
        (REPO_ROOT / "services/jenkins/kustomization.yaml").read_text(encoding="utf-8")
    )
    assert "hermes-image-builder-serviceaccount.yaml" in kustomization["resources"]

    spec = _pod_spec()
    assert spec["serviceAccountName"] == "hermes-image-builder"
    assert spec["automountServiceAccountToken"] is False
    for volume in spec.get("volumes", []):
        projected = volume.get("projected", {})
        assert all(
            "serviceAccountToken" not in item for item in projected.get("sources", [])
        )

    for manifest in (REPO_ROOT / "services/jenkins").glob("*.yaml"):
        for document in yaml.safe_load_all(manifest.read_text(encoding="utf-8")):
            if not isinstance(document, dict) or document.get("kind") not in {
                "RoleBinding",
                "ClusterRoleBinding",
            }:
                continue
            subjects = document.get("subjects") or []
            assert not any(
                item.get("kind") == "ServiceAccount"
                and item.get("name") == "hermes-image-builder"
                for item in subjects
            )


def _release_fixture(tmp_path: Path):
    module = _load(RELEASE, f"release_evidence_{tmp_path.name}")
    digest = "sha256:" + "7" * 64
    revision = "8" * 40
    build = "23"
    destination = f"{module.DEFAULT_IMAGE}:git-{revision}-build-{build}"
    digest_file = tmp_path / "hermes-agent.digest"
    image_file = tmp_path / "hermes-agent.image"
    manifest = tmp_path / "kustomization.yaml"
    output = tmp_path / "release"
    digest_file.write_text(digest + "\n", encoding="utf-8")
    image_file.write_text(f"{destination}@{digest}\n", encoding="utf-8")
    manifest.write_text(
        "images:\n"
        f"  - name: {module.DEFAULT_IMAGE}\n"
        "    digest: sha256:" + "0" * 64 + "\n",
        encoding="utf-8",
    )
    module.write_release_artifacts(
        digest=digest,
        source_revision=revision,
        build_number=build,
        destination=destination,
        kustomization=manifest,
        output_dir=output,
    )
    kwargs = {
        "digest_file": digest_file,
        "image_file": image_file,
        "source_revision": revision,
        "build_number": build,
        "destination": destination,
        "kustomization": manifest,
        "output_dir": output,
    }
    return module, kwargs


def test_success_evidence_revalidation_accepts_only_exact_complete_set(
    tmp_path: Path,
) -> None:
    """Exact evidence passes; an extra, missing, or altered file fails closed."""
    module, kwargs = _release_fixture(tmp_path)
    module.validate_release_artifacts(**kwargs)

    extra = kwargs["output_dir"] / "unexpected"
    extra.write_text("surprise\n", encoding="utf-8")
    with pytest.raises(ValueError, match="exactly three"):
        module.validate_release_artifacts(**kwargs)
    extra.unlink()

    metadata = kwargs["output_dir"] / "hermes-agent-image.json"
    original = metadata.read_text(encoding="utf-8")
    metadata.write_text(
        original.replace('"build_number": "23"', '"build_number": "24"')
    )
    with pytest.raises(ValueError, match="incomplete or mismatched"):
        module.validate_release_artifacts(**kwargs)
    metadata.write_text(original, encoding="utf-8")
    (kwargs["output_dir"] / "hermes-image-update.patch").unlink()
    with pytest.raises(ValueError, match="exactly three"):
        module.validate_release_artifacts(**kwargs)


def test_verify_evidence_cli_needs_no_runtime_registry_credential(
    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
    """Post-success validation is deterministic and cannot mask missing creds."""
    module, kwargs = _release_fixture(tmp_path)
    monkeypatch.delenv("HARBOR_USER", raising=False)
    monkeypatch.delenv("HARBOR_PASSWORD", raising=False)
    monkeypatch.setattr(
        sys,
        "argv",
        [
            "hermes_image_release.py",
            "verify-evidence",
            "--digest-file",
            str(kwargs["digest_file"]),
            "--image-file",
            str(kwargs["image_file"]),
            "--source-revision",
            kwargs["source_revision"],
            "--build-number",
            kwargs["build_number"],
            "--destination",
            kwargs["destination"],
            "--kustomization",
            str(kwargs["kustomization"]),
            "--output-dir",
            str(kwargs["output_dir"]),
        ],
    )
    assert module.main() == 0


def test_pipeline_requires_and_archives_exact_release_evidence() -> None:
    """Missing evidence must fail before the candidate becomes a release."""
    source = PIPELINE.read_text(encoding="utf-8")
    evidence = source.split("stage('Verify and archive release evidence')", 1)[1]
    evidence = evidence.split("stage('Publish Flux release tag')", 1)[0]
    assert "verify-evidence" in evidence
    assert "allowEmptyArchive: false" in evidence
    archive = evidence.split("artifacts: '", 1)[1].split("'", 1)[0]
    paths = archive.split(",")
    # Multi-arch adds both per-arch leaf digests/images plus the index digest.
    assert len(paths) == 11
    assert len(set(paths)) == 11
    assert all("*" not in path for path in paths)
    assert "find build -type f" in evidence
    assert "build/hermes-agent.source-revision" in paths
    assert "build/hermes-agent.digest" in paths
    for arch in ("arm64", "amd64"):
        assert f"build/hermes-agent-{arch}.digest" in paths
        assert f"build/hermes-agent-{arch}.image" in paths
    promotion = source.split("stage('Publish Flux release tag')", 1)[1]
    assert "ci/scripts/hermes_oci_promote.py" in promotion
    assert "  post {" not in source


def test_amd64_leg_is_pinned_to_titan24_and_resource_capped() -> None:
    """The amd64 build leg lands on titan-24, tolerates its guard taint, and is capped."""
    spec = _amd64_pod_spec()
    assert spec["nodeSelector"] == {
        "kubernetes.io/arch": "amd64",
        "kubernetes.io/hostname": "titan-24",
        "node-role.kubernetes.io/worker": "true",
    }
    # titan-24 co-hosts the Sui validator; the disposable build must tolerate
    # whatever guard taint the node carries so the pinned pod still schedules.
    assert {"operator": "Exists"} in spec["tolerations"]
    assert spec["serviceAccountName"] == "hermes-image-builder"
    assert spec["automountServiceAccountToken"] is False
    assert spec["enableServiceLinks"] is False

    containers = {item["name"]: item for item in spec["containers"]}
    kaniko = containers["kaniko"]
    # Identical pinned Kaniko across both legs -- no second, unreviewed builder.
    assert kaniko["image"] == (
        "gcr.io/kaniko-project/executor@sha256:"
        "c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e"
    )
    assert kaniko["securityContext"]["capabilities"]["add"] == [
        "CHOWN",
        "FOWNER",
        "DAC_OVERRIDE",
        "SETGID",
        "SETUID",
    ]
    for container in containers.values():
        security = container["securityContext"]
        assert security["allowPrivilegeEscalation"] is False
        assert security["capabilities"]["drop"] == ["ALL"]
        assert security["seccompProfile"]["type"] == "RuntimeDefault"
        assert security.get("privileged", False) is False
    # Tight caps keep the disposable build from starving the co-hosted validator:
    # the amd64 kaniko ceiling is strictly below the arm64 leg's 2 CPU / 4Gi.
    limits = kaniko["resources"]["limits"]
    assert limits["cpu"] == "1500m"
    assert limits["memory"] == "3Gi"


def test_amd64_leg_source_is_independently_boundary_checked() -> None:
    """The amd64 pod re-derives and re-verifies the reviewed revision itself."""
    source = PIPELINE.read_text(encoding="utf-8")
    amd64_stage = source.split("stage('Build amd64 leg without a daemon')", 1)[1]
    amd64_stage = amd64_stage.split("stage('Combine multi-arch index')", 1)[0]
    # Same fail-closed boundary as the coordinating pod, re-run in the amd64 pod.
    assert 'merge-base --is-ancestor "${EXPECTED_SOURCE_REVISION}"' in amd64_stage
    assert 'checkout --detach "${EXPECTED_SOURCE_REVISION}"' in amd64_stage
    assert 'test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}"' in amd64_stage
    assert 'status --porcelain' in amd64_stage
    assert '$(cat build/hermes-agent.destination)-amd64' in amd64_stage
    # The amd64 leaf evidence crosses pods only through an explicit stash.
    assert "stash" in amd64_stage
    assert "hermes-agent-amd64.digest" in amd64_stage


def test_combine_stage_publishes_and_reverifies_the_index() -> None:
    """Kaniko cannot combine; the reviewed python combiner assembles the index."""
    source = PIPELINE.read_text(encoding="utf-8")
    combine = source.split("stage('Combine multi-arch index')", 1)[1]
    combine = combine.split("stage('Render reviewed Flux handoff')", 1)[0]
    assert "unstash 'hermes-agent-amd64-evidence'" in combine
    assert "ci/scripts/hermes_multiarch_combine.py" in combine
    assert "--arm64-digest-file build/hermes-agent-arm64.digest" in combine
    assert "--amd64-digest-file build/hermes-agent-amd64.digest" in combine
    # The combiner emits the arch-less index evidence the existing chain promotes.
    assert "--digest-file build/hermes-agent.digest" in combine
    assert "--image-file build/hermes-agent.image" in combine
    # Downstream render/verify/promote still consume the single index digest file.
    render = source.split("stage('Render reviewed Flux handoff')", 1)[1]
    assert "--digest-file build/hermes-agent.digest" in render