atlas-iac/testing/tests/test_hermes_scm_broker.py

"""Executable contracts for the credential-isolated Hermes SCM broker."""

from __future__ import annotations

import importlib.util
import io
import json
import sys
from email.message import Message
from pathlib import Path

import pytest
import yaml

ROOT = Path(__file__).parents[2]
SCRIPTS = ROOT / "services/hermes/scm-common/scripts"
sys.path.insert(0, str(SCRIPTS))


def _load(name: str):
    path = SCRIPTS / f"{name}.py"
    spec = importlib.util.spec_from_file_location(f"test_{name}", path)
    assert spec and spec.loader
    module = importlib.util.module_from_spec(spec)
    sys.modules[spec.name] = module
    spec.loader.exec_module(module)
    return module


def _load_path(name: str, path: Path):
    spec = importlib.util.spec_from_file_location(name, path)
    assert spec and spec.loader
    module = importlib.util.module_from_spec(spec)
    sys.modules[spec.name] = module
    spec.loader.exec_module(module)
    return module


class Response:
    def __init__(self, body: bytes, *, status: int = 200, content_type: str):
        self.body = body
        self.stream = io.BytesIO(body)
        self.status = status
        self.headers = Message()
        self.headers["Content-Type"] = content_type

    def __enter__(self):
        return self

    def __exit__(self, *_args):
        return False

    def read(self, limit=-1):
        return self.stream.read(limit)


def _receive_command(old: bytes, new: bytes, ref: bytes) -> bytes:
    command = old + b" " + new + b" " + ref + b"\x00report-status\n"
    return f"{len(command) + 4:04x}".encode() + command + b"0000PACK"


@pytest.mark.parametrize(
    "target",
    [
        "/api/v1/repos/atlas/cassandra/pulls/1/merge",
        "/api/v1/repos/atlas/cassandra/pulls/1/reviews",
        "/api/v1/admin/users",
        "/git/evil/cassandra.git/info/refs?service=git-upload-pack",
        "/git/atlas/cassandra.git/hooks",
        "/git/atlas/cassandra.git/git-receive-pack?force=true",
        "/git/atlas/cassandra.git/info/refs?service=git-upload-pack\nignored",
        "/git/atlas/cassandra.git/info/refs?service=git-upload-pack\tignored",
        "/git/atlas/cassandra.git/info/refs?service=git-upload-pack#fragment",
    ],
)
def test_broker_rejects_privileged_or_non_atlas_routes(target: str):
    broker = _load("scm_broker")

    with pytest.raises(broker.PolicyError):
        broker._git_target(target)


def test_broker_metadata_boundary_reuses_explicit_read_allowlist():
    api = _load("gitea_api")

    assert (
        api.authorize_request("GET", "/api/v1/repos/atlas/cassandra/pulls/1", None)
        == "pull"
    )
    for path in (
        "/api/v1/repos/atlas/cassandra/pulls/1/merge",
        "/api/v1/repos/atlas/cassandra/pulls/1/reviews",
        "/api/v1/repos/atlas/cassandra/hooks",
        "/api/v1/admin/users",
    ):
        with pytest.raises(api.PolicyError):
            api.authorize_request("GET", path, None)


def test_receive_pack_allows_only_new_namespaced_feature_branch():
    broker = _load("scm_broker")
    zero = b"0" * 40
    commit = b"1" * 40

    broker._validate_receive_pack(
        _receive_command(zero, commit, b"refs/heads/hermes/focused-fix"),
        "runtime-sentinel",
    )
    for old, new, ref in (
        (commit, b"2" * 40, b"refs/heads/hermes/focused-fix"),
        (commit, zero, b"refs/heads/hermes/focused-fix"),
        (zero, commit, b"refs/heads/main"),
        (zero, commit, b"refs/heads/master"),
        (zero, commit, b"refs/tags/release"),
    ):
        with pytest.raises(broker.PolicyError):
            broker._validate_receive_pack(
                _receive_command(old, new, ref), "runtime-sentinel"
            )


def test_git_proxy_uses_fixed_origin_and_never_reflects_credential():
    broker = _load("scm_broker")
    seen = []

    def opener(request, timeout):
        seen.append((request, timeout))
        return Response(
            b"git-result",
            content_type="application/x-git-upload-pack-result",
        )

    result = broker._upstream_git_request(
        "/atlas/cassandra.git/git-upload-pack",
        method="POST",
        body=b"request",
        content_type="application/x-git-upload-pack-request",
        expected_type="application/x-git-upload-pack-result",
        token="runtime-sentinel",
        opener=opener,
    )

    assert result == b"git-result"
    request = seen[0][0]
    assert request.full_url == (
        "https://scm.bstein.dev/atlas/cassandra.git/git-upload-pack"
    )
    assert b"runtime-sentinel" not in request.data
    assert "runtime-sentinel" not in request.full_url

    with pytest.raises(broker.PolicyError, match="credential material"):
        broker._upstream_git_request(
            "/atlas/cassandra.git/git-upload-pack",
            method="POST",
            body=b"request",
            content_type="application/x-git-upload-pack-request",
            expected_type="application/x-git-upload-pack-result",
            token="runtime-sentinel",
            opener=lambda *_a, **_k: Response(
                b"runtime-sentinel",
                content_type="application/x-git-upload-pack-result",
            ),
        )

    encoded = broker._credential_forms("runtime-sentinel")[1]
    with pytest.raises(broker.PolicyError, match="credential material"):
        broker._upstream_git_request(
            "/atlas/cassandra.git/git-upload-pack",
            method="POST",
            body=b"request",
            content_type="application/x-git-upload-pack-request",
            expected_type="application/x-git-upload-pack-result",
            token="runtime-sentinel",
            opener=lambda *_a, **_k: Response(
                b"Authorization: Basic " + encoded,
                content_type="application/x-git-upload-pack-result",
            ),
        )


@pytest.mark.parametrize("status", [201, 202, 204, 206, 301, 302, 307, 308])
def test_git_proxy_requires_exact_http_200(status: int):
    broker = _load("scm_broker")

    with pytest.raises(broker.PolicyError, match="unexpected HTTP status"):
        broker._upstream_git_request(
            "/atlas/cassandra.git/git-upload-pack",
            method="POST",
            body=b"request",
            content_type="application/x-git-upload-pack-request",
            expected_type="application/x-git-upload-pack-result",
            token="runtime-sentinel",
            opener=lambda *_a, **_k: Response(
                b"result",
                status=status,
                content_type="application/x-git-upload-pack-result",
            ),
        )


def test_agent_broker_client_sends_no_credential_or_authorization_header():
    client = _load("scm_broker_client")
    seen = []

    def opener(request, timeout):
        seen.append((request, timeout))
        return Response(b"{}", content_type="application/json")

    assert client.read(
        "/api/v1/repos/atlas/cassandra", opener=opener
    ) == b"{}"
    request = seen[0][0]
    assert request.full_url == client.BROKER_ORIGIN + "/v1/metadata"
    assert request.get_header("Authorization") is None
    assert json.loads(request.data) == {
        "path": "/api/v1/repos/atlas/cassandra"
    }


@pytest.mark.parametrize("raw", ["", "01", "9" * 4000, "134217729", "１２"])
def test_broker_content_length_is_canonical_and_bounded(raw: str):
    broker = _load("scm_broker")
    headers = Message()
    headers["Content-Length"] = raw

    with pytest.raises(broker.PolicyError, match="request length"):
        broker._content_length(headers, broker.MAX_GIT_REQUEST)


def test_large_git_exchange_rolls_to_disk_and_scans_chunk_boundaries():
    broker = _load("scm_broker")
    token = "runtime-sentinel"
    safe = b"x" * (broker.SPOOL_MEMORY_LIMIT + 1)
    inbound_timeouts = []
    spool, length = broker._spool_bounded(
        io.BytesIO(safe),
        broker.MAX_GIT_REQUEST,
        len(safe),
        token=token,
        context="Git request",
        deadline=broker.time.monotonic() + 30,
        set_timeout=inbound_timeouts.append,
    )
    try:
        assert length == len(safe)
        assert spool._rolled is True
        assert spool.read(4) == b"xxxx"
        assert inbound_timeouts
        assert all(0 < value <= 30 for value in inbound_timeouts)
    finally:
        spool.close()

    crossing = b"x" * (broker.STREAM_CHUNK - 5) + token.encode() + b"tail"
    with pytest.raises(broker.PolicyError, match="credential material"):
        broker._spool_bounded(
            io.BytesIO(crossing),
            broker.MAX_GIT_REQUEST,
            len(crossing),
            token=token,
            context="Git request",
            deadline=broker.time.monotonic() + 30,
        )


def test_large_upstream_response_is_streamed_through_bounded_spool():
    broker = _load("scm_broker")
    body = b"z" * (broker.SPOOL_MEMORY_LIMIT + 1)
    seen = []

    def opener(request, **_kwargs):
        seen.append(request)
        assert request.get_header("Content-length") == "7"
        assert request.data.read() == b"request"
        return Response(body, content_type="application/x-git-upload-pack-result")

    streamed = broker._upstream_git_request(
        "/atlas/cassandra.git/git-upload-pack",
        method="POST",
        body=io.BytesIO(b"request"),
        body_length=7,
        content_type="application/x-git-upload-pack-request",
        expected_type="application/x-git-upload-pack-result",
        token="runtime-sentinel",
        stream_result=True,
        opener=opener,
    )
    spool, length = streamed
    try:
        assert length == len(body)
        assert spool._rolled is True
        assert spool.read(3) == b"zzz"
        assert len(seen) == 1
    finally:
        spool.close()


def test_body_deadline_fails_closed_before_reading():
    broker = _load("scm_broker")
    with pytest.raises(broker.PolicyError, match="deadline"):
        broker._spool_bounded(
            io.BytesIO(b"body"),
            broker.MAX_GIT_REQUEST,
            4,
            token="runtime-sentinel",
            context="Git request",
            deadline=broker.time.monotonic() - 1,
        )


def test_upstream_stream_applies_absolute_socket_deadline():
    broker = _load("scm_broker")

    class DeadlineSocket:
        def __init__(self):
            self.values = []

        def settimeout(self, value):
            self.values.append(value)

    response = Response(b"safe", content_type="application/x-git-upload-pack-result")
    sock = DeadlineSocket()
    response.fp = type("FP", (), {"raw": type("Raw", (), {"_sock": sock})()})()
    spool, length = broker._spool_response(response, broker.MAX_GIT_RESPONSE, "sentinel")
    try:
        assert length == 4
        assert sock.values
        assert all(0 < value <= broker.INBOUND_BODY_TIMEOUT for value in sock.values)
    finally:
        spool.close()


def test_broker_rejects_work_when_concurrency_slots_are_exhausted():
    broker = _load("scm_broker")

    class FakeSocket:
        def __init__(self):
            self.value = b""

        def sendall(self, value):
            self.value += value

        def shutdown(self, _how):
            return None

        def close(self):
            return None

    server = broker.BoundedThreadingHTTPServer(("127.0.0.1", 0), broker.BrokerHandler)
    try:
        assert server._slots.acquire(blocking=False)
        assert server._slots.acquire(blocking=False)
        request = FakeSocket()
        server.process_request(request, ("127.0.0.1", 1))
        assert b"503 Service Unavailable" in request.value
    finally:
        server._slots.release()
        server._slots.release()
        server.server_close()


def test_broker_bounds_header_count_size_and_duplicate_lengths():
    broker = _load("scm_broker")

    def validate(headers):
        handler = object.__new__(broker.BrokerHandler)
        handler.headers = headers
        handler._validate_headers()

    too_many = Message()
    for index in range(broker.MAX_HEADERS + 1):
        too_many[f"X-Test-{index}"] = "safe"
    with pytest.raises(broker.PolicyError, match="too many headers"):
        validate(too_many)

    too_large = Message()
    too_large["X-Test"] = "x" * broker.MAX_HEADER_BYTES
    with pytest.raises(broker.PolicyError, match="safe limit"):
        validate(too_large)

    duplicate = Message()
    duplicate["Content-Length"] = "1"
    duplicate["Content-Length"] = "1"
    with pytest.raises(broker.PolicyError, match="duplicate Content-Length"):
        validate(duplicate)


def _can_i(role: dict, api_group: str, resource: str, verb: str) -> bool:
    return any(
        (api_group in rule["apiGroups"] or "*" in rule["apiGroups"])
        and (resource in rule["resources"] or "*" in rule["resources"])
        and (verb in rule["verbs"] or "*" in rule["verbs"])
        for rule in role["rules"]
    )


def test_agent_rbac_preserves_read_diagnostics_without_administrator_authority():
    documents = list(
        yaml.safe_load_all(
            (ROOT / "services/hermes-observer-rbac/rbac.yaml").read_text(
                encoding="utf-8"
            )
        )
    )
    cluster_role, namespaced_role, binding = documents
    assert binding["roleRef"]["name"] == "hermes-agent-cluster-observer-v2"
    assert _can_i(cluster_role, "", "nodes", "get")
    assert _can_i(namespaced_role, "", "pods", "get")
    assert _can_i(namespaced_role, "", "pods/log", "get")
    assert _can_i(
        namespaced_role,
        "kustomize.toolkit.fluxcd.io",
        "kustomizations",
        "list",
    )

    denied = (
        ("", "secrets", "get"),
        ("", "serviceaccounts/token", "create"),
        ("", "pods", "create"),
        ("", "pods/exec", "create"),
        ("", "pods/attach", "create"),
        ("", "pods/portforward", "create"),
        ("rbac.authorization.k8s.io", "clusterrolebindings", "create"),
        ("authorization.k8s.io", "selfsubjectaccessreviews", "create"),
        ("apps", "deployments", "patch"),
    )
    assert all(
        not _can_i(role, *request)
        for request in denied
        for role in (cluster_role, namespaced_role)
    )

    bindings = yaml.safe_load(
        (ROOT / "services/hermes-observer-rbac/rolebindings.yaml").read_text(
            encoding="utf-8"
        )
    )["items"]
    namespaces = {item["metadata"]["namespace"] for item in bindings}
    assert "hermes-scm" not in namespaces
    assert {"cassandra", "flux-system", "hermes", "kube-system"} <= namespaces
    assert all(item["roleRef"]["name"].endswith("namespaced-observer-v2") for item in bindings)


def test_flux_boundary_keeps_broker_secret_and_network_separate_from_agent():
    agent = yaml.safe_load(
        (ROOT / "services/hermes/agent-deployment.yaml").read_text(encoding="utf-8")
    )
    annotations = agent["spec"]["template"]["metadata"]["annotations"]
    assert not any("gitea" in key.lower() for key in annotations)
    assert "GIT_ASKPASS" not in json.dumps(agent)

    broker = yaml.safe_load(
        (ROOT / "services/hermes-scm-broker/deployment.yaml").read_text(
            encoding="utf-8"
        )
    )
    assert broker["metadata"]["namespace"] == "hermes-scm"
    assert broker["spec"]["template"]["spec"]["serviceAccountName"] == (
        "hermes-scm-broker"
    )
    assert "gitea-token" in json.dumps(broker)
    container = broker["spec"]["template"]["spec"]["containers"][0]
    assert container["resources"]["limits"]["memory"] == "768Mi"
    assert container["livenessProbe"]["failureThreshold"] == 10
    tmp = next(
        item for item in broker["spec"]["template"]["spec"]["volumes"] if item["name"] == "tmp"
    )
    assert tmp["emptyDir"]["sizeLimit"] == "1Gi"

    policy = yaml.safe_load(
        (ROOT / "services/hermes-scm-broker/networkpolicy.yaml").read_text(
            encoding="utf-8"
        )
    )
    ingress = policy["spec"]["ingress"]
    assert ingress[0]["from"][0]["namespaceSelector"]["matchLabels"] == {
        "kubernetes.io/metadata.name": "hermes"
    }
    assert ingress[0]["from"][0]["podSelector"]["matchLabels"] == {
        "app": "hermes-agent"
    }

    vault = (
        ROOT / "services/vault/scripts/vault_k8s_auth_configure.sh"
    ).read_text(encoding="utf-8")
    assert 'write_policy_and_role "hermes-scm-broker" "hermes-scm"' in vault
    assert '"hermes/developer-gitea" ""' in vault
    agent_start = vault.index('write_policy_and_role "hermes-agent"')
    agent_end = vault.index("write_policy_and_role", agent_start + 1)
    assert "developer-gitea" not in vault[agent_start:agent_end]


def test_flux_bootstrap_has_no_agent_namespace_dependency_cycle():
    hermes_resources = yaml.safe_load(
        (ROOT / "services/hermes/kustomization.yaml").read_text(encoding="utf-8")
    )["resources"]
    assert "namespace.yaml" in hermes_resources
    assert "scm-common" in hermes_resources

    applications = (
        ROOT / "clusters/atlas/flux-system/applications/kustomization.yaml"
    ).read_text(encoding="utf-8")
    assert "hermes-scm-agent-code" not in applications

    broker_code = yaml.safe_load(
        (
            ROOT
            / "clusters/atlas/flux-system/applications/hermes-scm-broker-code/kustomization.yaml"
        ).read_text(encoding="utf-8")
    )
    assert broker_code["spec"]["path"] == "./services/hermes/scm-common"
    assert broker_code["spec"]["targetNamespace"] == "hermes-scm"
    assert broker_code["spec"]["dependsOn"] == [{"name": "hermes-scm-namespace"}]


def test_gitea_bootstrap_enforces_human_review_without_overwriting_drift():
    script = (
        ROOT / "services/gitea/scripts/gitea_atlas_identity_ensure.sh"
    ).read_text(encoding="utf-8")

    assert 'required_approvals\\\":1' in script
    assert 'enable_push_whitelist\\\":true' in script
    assert 'push_whitelist_usernames\\\":[\\\"${protected_reviewer}\\\"]' in script
    assert 'enable_merge_whitelist\\\":true' in script
    assert 'merge_whitelist_usernames\\\":[\\\"${protected_reviewer}\\\"]' in script
    assert "protection differs from the human-review policy" in script
    assert "api_request PATCH" not in script
    job = yaml.safe_load(
        (ROOT / "services/gitea/atlas-identity-bootstrap-job.yaml").read_text()
    )
    init = job["spec"]["template"]["spec"]["initContainers"][0]
    assert init["image"] == (
        "python@sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df"
    )
    assert init["securityContext"]["runAsNonRoot"] is True
    assert init["securityContext"]["capabilities"]["drop"] == ["ALL"]
    assert "/opt/python/bin/python3" in script


def _protection(rule_name: str, priority: int, helper, **overrides):
    value = {"rule_name": rule_name, "priority": priority, **helper._required("bstein")}
    value.update(overrides)
    return value


def test_branch_protection_uses_first_effective_glob_by_priority():
    helper = _load_path(
        "branch_protection_test",
        ROOT / "services/gitea/scripts/gitea_branch_protection_check.py",
    )
    earlier_drift = _protection("m*", 1, helper, required_approvals=0)
    later_exact = _protection("main", 2, helper)

    with pytest.raises(helper.PolicyError, match="effective main protection differs"):
        helper.evaluate(json.dumps([later_exact, earlier_drift]).encode(), "main", "bstein")

    earlier_drift["required_approvals"] = 1
    assert (
        helper.evaluate(json.dumps([later_exact, earlier_drift]).encode(), "main", "bstein")
        == "PRESENT"
    )
    brace_drift = _protection("m{ain,aster}", 1, helper, required_approvals=0)
    with pytest.raises(helper.PolicyError, match="effective master protection differs"):
        helper.evaluate(json.dumps([brace_drift]).encode(), "master", "bstein")


@pytest.mark.parametrize(
    "rules",
    [
        [{"rule_name": "main", "priority": 1}, {"rule_name": "m*", "priority": 1}],
        [{"rule_name": "[main", "priority": 1}],
        [{"rule_name": "main", "priority": 0}],
        [{"rule_name": "main"}],
    ],
)
def test_branch_protection_rejects_ambiguous_rules_before_creation(rules):
    helper = _load_path(
        "branch_protection_adversarial_test",
        ROOT / "services/gitea/scripts/gitea_branch_protection_check.py",
    )
    with pytest.raises(helper.PolicyError):
        helper.evaluate(json.dumps(rules).encode(), "main", "bstein")


def test_branch_protection_reports_absent_only_when_no_rule_matches():
    helper = _load_path(
        "branch_protection_absent_test",
        ROOT / "services/gitea/scripts/gitea_branch_protection_check.py",
    )
    rules = [_protection("release/*", 1, helper)]
    assert helper.evaluate(json.dumps(rules).encode(), "master", "bstein") == "ABSENT"