atlas-iac/testing/tests/test_hermes_worker_hux_delivery.py

"""Flux delivery gates for the Worker (hermes-agent) HUX service boundary.

Topology-adaptive like the chat gates in ``test_hermes_hux_delivery.py``:
while the agent Deployment carries no ``hux`` sidecar the manifests must
contain zero partial Worker HUX wiring; once the sidecar exists, the full
Worker boundary below is enforced strictly. The Worker instance is the
``hermes-agent`` Deployment behind worker.bstein.dev (via the
``oauth2-proxy-hermes-agent`` Service), NOT the dashboard ``hermes``
Deployment. See ``docs/hux/WORKER-PLAN.md``.
"""

import re
from pathlib import Path

import pytest
import yaml


ROOT = Path(__file__).resolve().parents[2]
SERVICE = ROOT / "services/hermes"
AGENT = SERVICE / "agent-deployment.yaml"
AGENT_CONFIG = SERVICE / "agent-configmap.yaml"
CHAT = SERVICE / "chat-statefulset.yaml"
KUSTOMIZATION = SERVICE / "kustomization.yaml"
WEBUI_IMAGE = "registry.bstein.dev/bstein/hermes-webui"
WEBUI_MARKER = '"$imagepolicy": "hermes:hermes-webui-release"'
TAG_MARKER = '"$imagepolicy": "hermes:hermes-webui-release:tag"'
DIGEST_MARKER = '"$imagepolicy": "hermes:hermes-webui-release:digest"'
WORKER_SLOT = "slot-100"


def _deployment() -> dict:
    return yaml.safe_load(AGENT.read_text(encoding="utf-8"))


def _pod() -> dict:
    return _deployment()["spec"]["template"]["spec"]


def _hux_active() -> bool:
    return any(item["name"] == "hux" for item in _pod()["containers"])


def _require_activation() -> None:
    if not _hux_active():
        pytest.skip("Worker HUX activation topology is not staged in this tree")


def _named(items: list[dict], name: str) -> dict:
    return next(item for item in items if item["name"] == name)


def _env(container: dict) -> dict[str, str]:
    return {item["name"]: item.get("value", "") for item in container.get("env", [])}


def _mounts(container: dict) -> list[dict]:
    return container.get("volumeMounts", [])


def test_worker_hux_topology_is_all_or_nothing() -> None:
    """A partial Worker HUX rollout (wiring without the sidecar) never ships."""
    pod = _pod()
    if _hux_active():
        return
    assert not any(item["name"].startswith("hux") for item in pod["containers"])
    assert not any(
        item["name"] == "init-hux-runtime" for item in pod.get("initContainers", [])
    )
    assert not any(item["name"].startswith("hux-") for item in pod["volumes"])
    for container in pod["containers"]:
        assert not any(name.startswith("HUX_") for name in _env(container))
    assert WEBUI_IMAGE not in AGENT.read_text(encoding="utf-8")


def test_worker_hux_sidecar_is_loopback_only_on_the_reviewed_webui_image() -> None:
    """One reviewed WebUI artifact serves chat and Worker; loopback is the only path."""
    _require_activation()
    pod = _pod()
    hux = _named(pod["containers"], "hux")
    values = _env(hux)
    raw = AGENT.read_text(encoding="utf-8")

    assert hux["image"].startswith(WEBUI_IMAGE + ":")
    assert "@sha256:" in hux["image"]
    # Exactly one WebUI consumer in this Deployment, on the Flux-managed line.
    assert raw.count(WEBUI_MARKER) == 1
    marked_line = next(line for line in raw.splitlines() if WEBUI_MARKER in line)
    assert hux["image"] in marked_line
    # The same reviewed image line as the chat tenants: the repo-wide Flux
    # Setters run (update.path services/hermes) rewrites both files together.
    chat_pod = yaml.safe_load(CHAT.read_text(encoding="utf-8"))["spec"]["template"][
        "spec"
    ]
    assert _named(chat_pod["containers"], "webui")["image"] == hux["image"]

    assert hux["command"] == ["/opt/hermes/.venv/bin/python", "-m", "hux.server"]
    assert values["PYTHONPATH"] == "/opt/hermes-hux"
    assert values["PYTHONDONTWRITEBYTECODE"] == "1"
    assert values["HUX_BIND"] == "127.0.0.1"
    assert values["HUX_PORT"] == "8790"
    assert values["HUX_TENANT_SLOT"] == WORKER_SLOT
    assert values["HUX_DATA_ROOT"] == "/var/lib/hux/store"
    # Foundation-only start: widening the Worker flag set is a deliberate,
    # reviewed change gated on the WORKER-PLAN lifecycle checks, so this pin
    # is updated in the same commit that turns each card on.
    assert {flag for flag in values["HUX_FLAGS"].split(",") if flag} == {
        "hux.foundation"
    }
    assert hux["securityContext"]["readOnlyRootFilesystem"] is True
    assert hux["securityContext"]["capabilities"]["drop"] == ["ALL"]
    assert hux["securityContext"]["runAsUser"] == 10000
    for probe in ("readinessProbe", "livenessProbe"):
        assert "tcpSocket" not in hux[probe]
        command = " ".join(hux[probe]["exec"]["command"])
        assert "127.0.0.1" in command
        assert "8790" in command
        assert "/healthz" in command
        # The chat canary's transient exit-137s were probe kills under a 2s
        # budget; the Worker ships with the corrected 5s budget from day one.
        assert hux[probe]["timeoutSeconds"] >= 5
    # Loopback only: no Service, proxy upstream, or NetworkPolicy exposes 8790.
    assert "8790" not in (SERVICE / "service.yaml").read_text(encoding="utf-8")
    assert "8790" not in (SERVICE / "oauth2-proxy.yaml").read_text(encoding="utf-8")
    assert "port: 8790" not in (SERVICE / "networkpolicy.yaml").read_text(
        encoding="utf-8"
    )


def test_worker_hux_store_is_a_private_durable_subtree() -> None:
    """The Worker owns its own store subtree and never shares chat's claim."""
    _require_activation()
    pod = _pod()
    hux = _named(pod["containers"], "hux")
    hermes = _named(pod["containers"], "hermes")

    # The store lives on the durable hermes-agent-home claim under the fixed
    # "hux" subtree (Deployment pod names churn, so no $(POD_NAME) scoping).
    assert {
        "name": "home",
        "mountPath": "/var/lib/hux",
        "subPath": "hux",
    } in _mounts(hux)
    assert not any("subPathExpr" in item for item in _mounts(hux))
    # The agent gets only its read-only key/context views of that subtree;
    # ledger records are reachable from the agent only through loopback HTTP.
    assert {
        "name": "home",
        "mountPath": "/run/hermes-hux-context",
        "subPath": "hux/context",
        "readOnly": True,
    } in _mounts(hermes)
    assert {
        "name": "home",
        "mountPath": "/run/hermes-hux-subject",
        "subPath": "hux/binding",
        "readOnly": True,
    } in _mounts(hermes)
    for container in pod["containers"]:
        if container["name"] == "hux":
            continue
        assert not any(
            item.get("mountPath") == "/var/lib/hux" for item in _mounts(container)
        )
        if container["name"] != "hermes":
            assert not any(
                item["name"].startswith("hux-") for item in _mounts(container)
            )
    volumes = {item["name"]: item for item in pod["volumes"]}
    assert volumes["hux-worker-key"]["emptyDir"]["medium"] == "Memory"
    # Single surface, single operator: no relay key and no evidence trust on
    # the Worker instance — those trusts fail closed by absence.
    assert "hux-relay-key" not in volumes
    assert "hux-evidence-key" not in volumes
    raw = AGENT.read_text(encoding="utf-8")
    assert "hermes-chat-hux-data" not in raw


def test_worker_hux_init_provisions_durable_identity_and_rotating_key() -> None:
    """Identity survives pod replacement; the transport key rotates with it."""
    _require_activation()
    pod = _pod()
    init = _named(pod["initContainers"], "init-hux-runtime")
    script = init["args"][0]

    assert 'worker_root="/hux-data/hux"' in script
    assert "${HOSTNAME}" not in script
    assert "$(POD_NAME)" not in script
    assert 'HUX_INIT_SLOT="slot-100"' in script
    assert "chown -R" not in script
    assert 'if [ ! -e "${worker_root}/context/context-key" ]' in script
    assert "bs=32 count=1" in script
    assert 'chmod 0600 "${worker_root}/context/context-key"' in script
    assert 'b"hux.subject.id.v1\\0" + slot.encode("ascii")' in script
    assert 'target = root / "binding/subject"' in script
    assert "os.O_EXCL" in script
    assert "or stat.S_IMODE(info.st_mode) != 0o440" in script
    assert "target=/hux-worker/worker-key" in script
    assert "chmod 0400" in script
    assert "relay" not in script
    assert init["securityContext"]["capabilities"] == {
        "drop": ["ALL"],
        "add": ["CHOWN", "DAC_OVERRIDE", "FOWNER"],
    }
    assert {"name": "home", "mountPath": "/hux-data"} in _mounts(init)
    assert {"name": "hux-worker-key", "mountPath": "/hux-worker"} in _mounts(init)


def test_worker_hux_hook_env_is_file_backed_and_observe_only() -> None:
    """The agent hook is wired for observation: file-backed keys, enforcement 0."""
    _require_activation()
    pod = _pod()
    hermes = _env(_named(pod["containers"], "hermes"))

    assert hermes["HUX_BASE_URL"] == "http://127.0.0.1:8790"
    assert hermes["HUX_RUNTIME_ENABLED"] == "1"
    # Observe-only pin: flipping enforcement on the Worker is its own gated
    # change (approvals need a human surface this instance does not have yet).
    assert hermes["HUX_TOOL_ENFORCEMENT"] == "0"
    assert hermes["HUX_TENANT_SLOT"] == WORKER_SLOT
    assert hermes["HUX_WORKER_KEY_FILE"] == "/run/hermes-hux-worker/worker-key"
    assert hermes["HUX_SUBJECT_FILE"] == "/run/hermes-hux-subject/subject"
    assert hermes["HUX_CONTEXT_KEY_FILE"] == "/run/hermes-hux-context/context-key"
    assert hermes["HUX_TIMEOUT_SECONDS"] == "3"
    forbidden = {"HUX_WORKER_KEY", "HUX_RELAY_KEY", "HUX_ROUTER_KEY", "HUX_SUBJECT"}
    for container in pod["containers"]:
        assert not forbidden & set(_env(container))


def test_worker_hux_runtime_plugin_is_staged_but_inert() -> None:
    """The vendored hook package is mounted, yet nothing activates it."""
    _require_activation()
    pod = _pod()
    hermes = _named(pod["containers"], "hermes")
    plugin = _named(pod["volumes"], "hux-runtime-plugin")["configMap"]
    items = {item["key"]: item["path"] for item in plugin["items"]}

    assert plugin["name"] == "hermes-hux-runtime-plugin"
    assert {
        "hux-hook-init.py": "hux_hook/__init__.py",
        "hux-hook-client.py": "hux_hook/client.py",
        "hux-hook-hooks.py": "hux_hook/hooks.py",
    }.items() <= items.items()
    assert {
        "name": "hux-runtime-plugin",
        "mountPath": "/opt/data/plugins/hux-runtime",
        "readOnly": True,
    } in _mounts(hermes)
    rendered = KUSTOMIZATION.read_text(encoding="utf-8")
    assert "hux-hook-client.py=plugins/hux-runtime/hux_hook/client.py" in rendered
    # Staged, not active: the activation commit adds hux-runtime to
    # plugins.enabled (docs/hux/WORKER-PLAN.md); until then the wiring is inert.
    configmap = yaml.safe_load(AGENT_CONFIG.read_text(encoding="utf-8"))
    config = yaml.safe_load(configmap["data"]["config.yaml"])
    assert "hux-runtime" not in config["plugins"]["enabled"]


def test_worker_hux_build_bindings_use_flux_setters() -> None:
    """Sidecar build provenance is bound by the same Flux setters as chat."""
    _require_activation()
    raw = AGENT.read_text(encoding="utf-8")
    hux = _env(_named(_pod()["containers"], "hux"))

    assert raw.count(TAG_MARKER) == 1
    assert raw.count(DIGEST_MARKER) == 1
    release = re.fullmatch(
        r"git-([0-9a-f]{40})-build-[1-9][0-9]*-release", hux["HUX_IMAGE_TAG"]
    )
    assert release is not None
    assert hux["HUX_IMAGE_DIGEST"].startswith("sha256:")
    image = _named(_pod()["containers"], "hux")["image"]
    assert image.split(":git-", 1)[1].split("@", 1)[0] == hux[
        "HUX_IMAGE_TAG"
    ].removeprefix("git-")
    assert image.endswith("@" + hux["HUX_IMAGE_DIGEST"])