atlas-iac/testing/tests/test_hermes_webui_release.py

"""Independent build, Harbor evidence, and Flux handoff for Hermes WebUI."""

from __future__ import annotations

import importlib.util
import io
import json
from pathlib import Path
import sys

import pytest
import yaml


ROOT = Path(__file__).resolve().parents[2]
PIPELINE = ROOT / "ci/Jenkinsfile.hermes-webui-image"
RELEASE = ROOT / "ci/scripts/hermes_webui_release.py"
DOCKERFILE = ROOT / "dockerfiles/Dockerfile.hermes-webui"
CHAT = ROOT / "services/hermes/chat-statefulset.yaml"
DASHBOARD = ROOT / "services/hermes/deployment.yaml"
POLICY = ROOT / "services/harbor/scripts/harbor_hermes_webui_immutability_ensure.py"
sys.path.insert(0, str(RELEASE.parent))


def _load(path: Path, name: str):
    spec = importlib.util.spec_from_file_location(name, path)
    assert spec is not None and spec.loader is not None
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)
    return module


def _pod_spec() -> dict:
    source = PIPELINE.read_text(encoding="utf-8")
    pod_yaml = source.split('yaml """', 1)[1].split('"""', 1)[0]
    return yaml.safe_load(pod_yaml)["spec"]


def _release_fixture(tmp_path: Path):
    module = _load(RELEASE, f"hermes_webui_release_{tmp_path.name}")
    digest = "sha256:" + "7" * 64
    revision = "8" * 40
    build = "23"
    destination = f"{module.DEFAULT_IMAGE}:git-{revision}-build-{build}"
    digest_file = tmp_path / "hermes-webui.digest"
    image_file = tmp_path / "hermes-webui.image"
    digest_file.write_text(digest + "\n", encoding="utf-8")
    image_file.write_text(f"{destination}@{digest}\n", encoding="utf-8")
    output = tmp_path / "release"
    kwargs = {
        "digest_file": digest_file,
        "image_file": image_file,
        "source_revision": revision,
        "build_number": build,
        "destination": destination,
        "chat_manifest": CHAT,
        "dashboard_manifest": DASHBOARD,
        "output_dir": output,
    }
    module.write_release_artifacts(
        digest=digest,
        source_revision=revision,
        build_number=build,
        destination=destination,
        chat_manifest=CHAT,
        dashboard_manifest=DASHBOARD,
        output_dir=output,
    )
    return module, digest, kwargs


def test_webui_job_is_independent_bounded_and_main_only() -> None:
    """WebUI has its own bounded job using the same Vault-injected release token."""
    config = yaml.safe_load(
        (ROOT / "services/jenkins/configmap-jcasc.yaml").read_text(encoding="utf-8")
    )
    jobs = config["data"]["jobs.yaml"]
    assert jobs.count("pipelineJob('hermes-agent-image')") == 1
    assert jobs.count("pipelineJob('hermes-webui-image')") == 1
    block = jobs.split("pipelineJob('hermes-webui-image')", 1)[1].split(
        "multibranchPipelineJob(", 1
    )[0]
    assert "branches('*/main')" in block
    assert "scriptPath('ci/Jenkinsfile.hermes-webui-image')" in block
    assert "pipelineTriggers" not in block
    assert (
        "authenticationToken(System.getenv('HERMES_AGENT_IMAGE_BUILD_TOKEN'))" in block
    )
    assert "booleanParam('PUBLISH_IMAGE', false" in block
    assert "stringParam('EXPECTED_SOURCE_REVISION', ''" in block
    assert "stringParam('CONFIRM_PUBLISH', ''" in block


def test_pipeline_builds_exact_reviewed_anchor_from_main() -> None:
    """Publish uses one reviewed source revision even when main later advances."""
    source = PIPELINE.read_text(encoding="utf-8")
    assert 'test "${CONFIRM_PUBLISH}" = "PUBLISH HERMES WEBUI"' in source
    assert 'test "${main_revision}" = "$(git rev-parse origin/main)"' in source
    assert "git fetch --no-tags origin main" not in source
    assert 'git merge-base --is-ancestor "${EXPECTED_SOURCE_REVISION}"' in source
    assert 'git checkout --detach "${EXPECTED_SOURCE_REVISION}"' in source
    assert 'test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}"' in source
    assert "dockerfiles/Dockerfile.hermes-webui" in source
    assert "ci/scripts/hermes_webui_release.py" in source
    assert "registry.bstein.dev/bstein/hermes-webui" in source
    assert "Dockerfile.hermes-agent" not in source
    assert "hermes_image_release.py" not in source
    assert "HERMES_KANIKO_HEREDOC_COMPAT" not in source
    assert "--digest-file=" in source
    assert "--image-name-tag-with-digest-file=" in source
    assert "org.opencontainers.image.revision=${source_revision}" in source
    assert "assert-absent" in source and "verify-evidence" in source
    assert "ci/scripts/hermes_oci_promote.py" in source
    assert "test_hermes_webui_brand.py" in source
    assert "test_hermes_webui_release.py" in source
    for voice_suite in (
        "test_hermes_chat_quality.py",
        "test_hermes_handsfree_stt.py",
        "test_hermes_voice_full_duplex.py",
        "test_hermes_thinking_voice_cues.py",
        "test_hermes_voice_instrument.py",
        "test_hermes_voice_language_routing.py",
    ):
        assert voice_suite in source
    assert "apt-get install -y --no-install-recommends ffmpeg nodejs" in source
    assert "command -v ffmpeg >/dev/null" in source
    assert "command -v node >/dev/null" in source
    assert source.index("apt-get install -y --no-install-recommends ffmpeg nodejs") < (
        source.index("python3 -m pytest -q")
    )
    for forbidden in ("kubectl ", "flux reconcile", "git push", "git commit"):
        assert forbidden not in source

    spec = _pod_spec()
    assert spec["serviceAccountName"] == "hermes-image-builder"
    assert spec["automountServiceAccountToken"] is False
    assert spec["enableServiceLinks"] is False
    containers = {item["name"]: item for item in spec["containers"]}
    assert "kaniko" in containers
    python_security = containers["python"]["securityContext"]
    assert python_security["runAsUser"] == 0
    assert python_security["runAsNonRoot"] is False
    assert set(python_security["capabilities"]["add"]) == {
        "CHOWN",
        "FOWNER",
        "DAC_OVERRIDE",
        "SETGID",
        "SETUID",
    }
    assert containers["kaniko"]["resources"]["requests"]["ephemeral-storage"] == "10Gi"
    assert containers["kaniko"]["resources"]["limits"]["ephemeral-storage"] == "20Gi"
    assert spec["nodeSelector"] == {"kubernetes.io/arch": "arm64"}
    build_nodes = spec["affinity"]["nodeAffinity"][
        "requiredDuringSchedulingIgnoredDuringExecution"
    ]["nodeSelectorTerms"][0]["matchExpressions"][0]["values"]
    assert build_nodes == ["titan-20"]
    storage_nodes = {"titan-13", "titan-15", "titan-17", "titan-19"}
    assert storage_nodes.isdisjoint(build_nodes)
    assert "docker.sock" not in source and "hostPath" not in source
    for container in containers.values():
        assert container["securityContext"]["allowPrivilegeEscalation"] is False
        assert container["securityContext"]["capabilities"]["drop"] == ["ALL"]


def test_webui_dockerfile_is_kaniko_safe_and_uses_reviewed_repo_source() -> None:
    """The dedicated build consumes tracked patches/assets without RUN heredocs."""
    source = DOCKERFILE.read_text(encoding="utf-8")
    assert "<<" not in source
    assert "hermes-webui-base-patch.py" in source
    assert "hermes-webui-brand-patch.py" in source
    assert "hermes-webui-manifest-patch.py" in source
    assert "hermes-webui-release-patch.py" in source
    assert "hermes-webui-smoke.py" in source
    assert "hermes-webui-stt-patch.py" in source
    assert "hermes-webui-atlas-voice.js" in source
    assert "hermes-webui-manifest.json" in source
    assert (
        '--build-arg="HERMES_WEBUI_RELEASE_ID='
        'git-${source_revision}-build-${BUILD_NUMBER}"'
        in PIPELINE.read_text(encoding="utf-8")
    )
    assert source.count("RUN /opt/hermes/.venv/bin/python /tmp/hermes-webui-") == 1
    assert "urljoin(manifest_url, source)" in (
        ROOT / "dockerfiles/hermes-webui-smoke.py"
    ).read_text(encoding="utf-8")


def test_renderer_updates_exact_chat_and_dashboard_webui_only(tmp_path: Path) -> None:
    """One digest patch spans the two same-policy WebUI consumers and nothing else."""
    module, digest, kwargs = _release_fixture(tmp_path)
    output = kwargs["output_dir"]
    patch = (output / "hermes-webui-image-update.patch").read_text(encoding="utf-8")
    chat_consumers = CHAT.read_text(encoding="utf-8").count(
        f"image: {module.DEFAULT_IMAGE}:"
    )
    assert patch.count(f"+          image: {module.DEFAULT_IMAGE}@{digest}") == chat_consumers + 1
    if "HUX_IMAGE_TAG" in CHAT.read_text(encoding="utf-8"):
        tag = f"git-{kwargs['source_revision']}-build-{kwargs['build_number']}-release"
        assert any(
            line.startswith("+") and line.lstrip("+ ").startswith(f"value: {tag}")
            for line in patch.splitlines()
        ), patch
        assert any(
            line.startswith("+") and line.lstrip("+ ").startswith(f"value: {digest}")
            for line in patch.splitlines()
        ), patch
    assert "services/hermes/chat-statefulset.yaml" in patch
    assert "services/hermes/deployment.yaml" in patch
    assert "hermes-agent@sha256" not in "\n".join(
        line for line in patch.splitlines() if line.startswith("+")
    )
    assert CHAT.read_text(encoding="utf-8") != (
        output / "hermes-chat-statefulset.yaml"
    ).read_text(encoding="utf-8")
    assert DASHBOARD.read_text(encoding="utf-8") != (
        output / "hermes-dashboard-deployment.yaml"
    ).read_text(encoding="utf-8")
    metadata = json.loads((output / "hermes-webui-image.json").read_text())
    assert metadata["digest"] == digest
    assert metadata["flux_image"] == f"{module.DEFAULT_IMAGE}@{digest}"
    assert metadata["flux_targets"] == [
        "apps/StatefulSet/hermes/hermes-chat-tenant",
        "apps/Deployment/hermes/hermes",
    ]
    module.validate_release_artifacts(**kwargs)


def test_renderer_accepts_flux_tagged_digest_reference() -> None:
    """Flux's whole-image setter may retain the selected release tag."""
    module = _load(RELEASE, "hermes_webui_release_tagged_digest")
    old_digest = "sha256:" + "1" * 64
    new_digest = "sha256:" + "2" * 64
    source = (
        "apiVersion: apps/v1\n"
        "kind: Deployment\n"
        "metadata:\n"
        "  name: hermes\n"
        "spec:\n"
        "  image: registry.bstein.dev/bstein/hermes-webui:"
        f"git-{'a' * 40}-build-7-release@{old_digest} "
        '# {"$imagepolicy": "hermes:hermes-webui-release"}\n'
    )

    rendered = module.render_workload(
        source, new_digest, kind="Deployment", name="hermes"
    )

    assert f"image: {module.DEFAULT_IMAGE}@{new_digest}" in rendered
    assert '"$imagepolicy": "hermes:hermes-webui-release"' in rendered


def test_renderer_supports_pre_activation_single_consumer_without_hux_metadata() -> None:
    """The image can land safely before the HUX sidecar is enabled by Flux."""
    module = _load(RELEASE, "hermes_webui_release_staged_activation")
    old_digest = "sha256:" + "1" * 64
    new_digest = "sha256:" + "2" * 64
    revision = "3" * 40
    source = (
        "apiVersion: apps/v1\nkind: StatefulSet\nmetadata:\n"
        "  name: hermes-chat-tenant\nspec:\n  template:\n    spec:\n"
        "      containers:\n        - name: webui\n"
        f"          image: {module.DEFAULT_IMAGE}@{old_digest}\n"
    )

    rendered = module.render_workload(
        source,
        new_digest,
        kind="StatefulSet",
        name="hermes-chat-tenant",
        expected_images=(1, 2),
    )

    assert f"image: {module.DEFAULT_IMAGE}@{new_digest}" in rendered
    assert module.render_hux_build_metadata(rendered, new_digest, revision, "23") == rendered


@pytest.mark.parametrize(
    ("source", "kind", "name", "match"),
    [
        (
            "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n  name: wrong\n",
            "Deployment",
            "hermes",
            "identity changed",
        ),
        (
            "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n  name: hermes\n",
            "Deployment",
            "hermes",
            "found 0",
        ),
        (
            "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n  name: hermes\n"
            "spec:\n  image: registry.bstein.dev/bstein/hermes-webui:latest\n",
            "Deployment",
            "hermes",
            "found 0",
        ),
    ],
)
def test_renderer_fails_closed_on_flux_target_drift(
    source: str, kind: str, name: str, match: str
) -> None:
    module = _load(RELEASE, f"webui_renderer_{abs(hash(source))}")
    with pytest.raises(ValueError, match=match):
        module.render_workload(source, "sha256:" + "a" * 64, kind=kind, name=name)


def test_evidence_revalidation_rejects_extra_or_changed_files(tmp_path: Path) -> None:
    """Archived output is an exact deterministic set, not a best-effort bundle."""
    module, _digest, kwargs = _release_fixture(tmp_path)
    module.validate_release_artifacts(**kwargs)
    extra = kwargs["output_dir"] / "unexpected"
    extra.write_text("surprise\n", encoding="utf-8")
    with pytest.raises(ValueError, match="exactly four"):
        module.validate_release_artifacts(**kwargs)
    extra.unlink()
    metadata = kwargs["output_dir"] / "hermes-webui-image.json"
    metadata.write_text("{}\n", encoding="utf-8")
    with pytest.raises(ValueError, match="incomplete or mismatched"):
        module.validate_release_artifacts(**kwargs)


class _Response(io.BytesIO):
    status = 200

    def __init__(self, body: bytes, headers: dict[str, str] | None = None):
        super().__init__(body)
        self.headers = headers or {}

    def __enter__(self):
        return self

    def __exit__(self, *_args):
        self.close()


def test_release_verifies_exact_webui_harbor_artifact_and_policy() -> None:
    """Independent evidence resolves the tag and exact WebUI immutability rule."""
    module = _load(RELEASE, "webui_registry_contract")
    revision = "a" * 40
    destination = f"{module.DEFAULT_IMAGE}:git-{revision}-build-9"
    digest = "sha256:" + "b" * 64
    seen = []

    def artifact_open(request, timeout):
        seen.append((request, timeout))
        return _Response(
            json.dumps(
                {
                    "digest": digest,
                    "tags": [{"name": f"git-{revision}-build-9", "immutable": True}],
                    "extra_attrs": {
                        "config": {
                            "Labels": {
                                "org.opencontainers.image.revision": revision,
                            }
                        }
                    },
                }
            ).encode()
        )

    module.verify_registry_digest(
        destination,
        digest,
        revision,
        username="robot",
        password="private",
        opener=artifact_open,
    )
    request = seen[0][0]
    assert "/repositories/hermes-webui/artifacts/" in request.full_url
    assert request.full_url.startswith("https://registry.bstein.dev/api/v2.0/")
    assert request.get_header("Authorization").startswith("Basic ")
    assert seen[0][1] == 20

    def policy_open(request, timeout):
        assert request.full_url.endswith(
            "/projects/bstein/immutabletagrules?page=1&page_size=100"
        )
        return _Response(
            json.dumps(
                [
                    {
                        "disabled": False,
                        "action": "immutable",
                        "template": "immutable_template",
                        "tag_selectors": [
                            {
                                "kind": "doublestar",
                                "decoration": "matches",
                                "pattern": "git-*-build-*",
                            }
                        ],
                        "scope_selectors": {
                            "repository": [
                                {
                                    "kind": "doublestar",
                                    "decoration": "repoMatches",
                                    "pattern": "hermes-webui",
                                }
                            ]
                        },
                    }
                ]
            ).encode(),
            {"X-Total-Count": "1"},
        )

    module.verify_immutable_policy(
        username="robot", password="private", opener=policy_open
    )


@pytest.mark.parametrize(
    "labels", [None, {}, {"org.opencontainers.image.revision": "c" * 40}]
)
def test_release_rejects_missing_or_wrong_harbor_source_revision(labels) -> None:
    """A tag derived from Git cannot substitute for the persisted OCI label."""
    module = _load(RELEASE, f"webui_registry_revision_{labels!r}")
    revision = "a" * 40
    destination = f"{module.DEFAULT_IMAGE}:git-{revision}-build-9"
    digest = "sha256:" + "b" * 64

    def artifact_open(_request, _timeout):
        config = {} if labels is None else {"Labels": labels}
        return _Response(
            json.dumps(
                {
                    "digest": digest,
                    "tags": [{"name": f"git-{revision}-build-9", "immutable": True}],
                    "extra_attrs": {"config": config},
                }
            ).encode()
        )

    with pytest.raises(RuntimeError, match="OCI (image labels|source-revision label)"):
        module.verify_registry_digest(
            destination,
            digest,
            revision,
            username="robot",
            password="private",
            opener=artifact_open,
        )


def test_flux_tracks_webui_policy_before_jenkins() -> None:
    """The immutable Harbor rule is reviewed desired state, not a pipeline wish."""
    harbor = yaml.safe_load(
        (
            ROOT / "clusters/atlas/flux-system/applications/harbor/kustomization.yaml"
        ).read_text(encoding="utf-8")
    )
    checks = harbor["spec"]["healthChecks"]
    assert {
        "apiVersion": "batch/v1",
        "kind": "Job",
        "name": "harbor-hermes-webui-immutability-ensure-1",
        "namespace": "harbor",
    } in checks
    jenkins = yaml.safe_load(
        (
            ROOT / "clusters/atlas/flux-system/applications/jenkins/kustomization.yaml"
        ).read_text(encoding="utf-8")
    )
    assert "harbor" in {item["name"] for item in jenkins["spec"]["dependsOn"]}

    policy = _load(POLICY, "webui_policy_contract")
    assert policy.REPOSITORY_PATTERN == "hermes-webui"
    assert policy.TAG_PATTERN == "git-*-build-*"
    assert policy.EXPECTED_RULE["disabled"] is False
    assert "robot" not in POLICY.read_text(encoding="utf-8").lower()

    job = yaml.safe_load(
        (ROOT / "services/harbor/hermes-webui-immutability-job.yaml").read_text(
            encoding="utf-8"
        )
    )
    template = job["spec"]["template"]
    annotations = template["metadata"]["annotations"]
    assert annotations["vault.hashicorp.com/role"] == "harbor-policy-bootstrap"
    assert (
        annotations["vault.hashicorp.com/agent-inject-secret-harbor-admin-password"]
        == "kv/data/atlas/harbor/harbor-core"
    )
    pod = template["spec"]
    assert pod["serviceAccountName"] == "harbor-policy-bootstrap"
    assert pod["enableServiceLinks"] is False
    container = pod["containers"][0]
    assert "@sha256:" in container["image"]
    assert container["securityContext"]["readOnlyRootFilesystem"] is True
    assert container["securityContext"]["runAsNonRoot"] is True
    assert container["securityContext"]["capabilities"]["drop"] == ["ALL"]

    release_docs = (ROOT / "docs/hermes_webui_release.md").read_text(encoding="utf-8")
    assert "immutable-tag:list" in release_docs
    assert "harbor-hermes-agent-immutability-ensure-1" in release_docs


class _FakePolicyClient:
    origin = "https://registry.bstein.dev/api/v2.0"

    def __init__(self, responses):
        self.responses = list(responses)
        self.calls = []

    def request(self, method, path, payload=None):
        self.calls.append((method, path, payload))
        return self.responses.pop(0)


def test_webui_policy_is_idempotent_and_create_is_reread() -> None:
    """The desired Harbor rule validates in place or verifies its exact new ID."""
    policy = _load(POLICY, "webui_policy_idempotency")
    existing = {"id": 17, **policy.EXPECTED_RULE}
    client = _FakePolicyClient(
        [(200, json.dumps([existing]).encode(), {"X-Total-Count": "1"})]
    )
    assert policy.ensure_rule(client) == 17
    assert [call[0] for call in client.calls] == ["GET"]

    created = {"id": 23, **policy.EXPECTED_RULE}
    client = _FakePolicyClient(
        [
            (200, b"[]", {"X-Total-Count": "0"}),
            (
                201,
                b"",
                {"Location": ("/api/v2.0/projects/bstein/immutabletagrules/23")},
            ),
            (200, json.dumps([created]).encode(), {"X-Total-Count": "1"}),
        ]
    )
    assert policy.ensure_rule(client) == 23
    assert client.calls[1] == (
        "POST",
        "/projects/bstein/immutabletagrules",
        policy.EXPECTED_RULE,
    )


def test_webui_policy_rejects_disabled_duplicate_or_truncated_rules() -> None:
    """Ambiguous or incomplete Harbor evidence can never unblock publication."""
    policy = _load(POLICY, "webui_policy_rejections")
    disabled = {"id": 17, **policy.EXPECTED_RULE, "disabled": True}
    with pytest.raises(RuntimeError, match="not enabled and exact"):
        policy.ensure_rule(
            _FakePolicyClient(
                [
                    (
                        200,
                        json.dumps([disabled]).encode(),
                        {"X-Total-Count": "1"},
                    )
                ]
            )
        )

    duplicate = [
        {"id": 17, **policy.EXPECTED_RULE},
        {"id": 18, **policy.EXPECTED_RULE},
    ]
    with pytest.raises(RuntimeError, match="multiple Hermes WebUI"):
        policy.ensure_rule(
            _FakePolicyClient(
                [
                    (
                        200,
                        json.dumps(duplicate).encode(),
                        {"X-Total-Count": "2"},
                    )
                ]
            )
        )

    with pytest.raises(RuntimeError, match="truncated"):
        policy.list_rules(_FakePolicyClient([(200, b"[]", {"X-Total-Count": "1"})]))


def test_pipeline_archives_exact_eight_files_before_release() -> None:
    """Release cannot pass with missing digest, workload, or metadata evidence."""
    source = PIPELINE.read_text(encoding="utf-8")
    evidence = source.split("stage('Verify and archive release evidence')", 1)[1]
    evidence = evidence.split("stage('Publish Flux release tag')", 1)[0]
    archive = evidence.split("artifacts: '", 1)[1].split("'", 1)[0].split(",")
    assert len(archive) == len(set(archive)) == 8
    assert all("*" not in path for path in archive)
    assert "find build -type f" in evidence
    assert "allowEmptyArchive: false" in evidence
    assert "build/hermes-webui.source-revision" in archive
    assert "hermes-chat-statefulset.yaml" in evidence
    assert "hermes-dashboard-deployment.yaml" in evidence
    assert "  post {" not in source