atlas-iac/testing/tests/test_hermes_hux_releases_backend.py

"""Focused HUX-12 immutable release-ledger tests."""

from __future__ import annotations

import json
import sys
from pathlib import Path
from types import SimpleNamespace

import pytest

ROOT = Path(__file__).resolve().parents[2]
FOUNDATION = ROOT / "dockerfiles" / "hermes-hux-foundation"
if str(FOUNDATION) not in sys.path:
    sys.path.insert(0, str(FOUNDATION))

from hux import contracts, identity, releases, store  # noqa: E402
from hux.errors import Invalid  # noqa: E402
from hux.server import build_router  # noqa: E402

HEADERS = {"X-Hermes-Tenant-Identity": "slot-3", "X-Hux-Subject": "usr_0123456789abcdef", "X-Hux-Surface": "chat", "X-Hux-Relay-Key": "rk"}
EVIDENCE = {**HEADERS, "X-Hux-Surface": "api", "X-Hux-Trust": "evidence", "X-Hux-Relay-Key": "evidence-key-value-that-is-at-least-32-bytes"}
OTHER = {**HEADERS, "X-Hux-Subject": "usr_fedcba9876543210"}
ALL_ON = ",".join(card["flag"] for card in contracts.load_flags()["cards"])
DIGEST = "sha256:" + "a" * 64
OTHER_DIGEST = "sha256:" + "b" * 64
SCHEMAS = contracts.load_all()
SCHEMAS["release-ledger.schema.json"] = contracts.load_schema("release-ledger.schema.json")


def call(router, method, path, body=None, headers=None):
    raw = b"" if body is None else json.dumps(body).encode()
    response = router.dispatch(method, path, {**HEADERS, **(headers or {})}, raw)
    return response.status, response.body, response.headers


def tenant(router):
    return store.TenantStore(router.data_root, identity.resolve(HEADERS, {"HUX_ROUTER_KEY": "rk"}))


@pytest.fixture
def setup(tmp_path):
    key_file = tmp_path / "evidence.key"
    key_file.write_text("evidence-key-value-that-is-at-least-32-bytes\n")
    key_file.chmod(0o400)
    policy_file = tmp_path / "evidence-policy.json"
    policy_file.write_text(json.dumps({
        "schema": "hux.release_evidence_policy.v1", "max_evidence_age_seconds": 900,
        "workloads": {"hermes-webui": {
            "review_url_prefix": "https://git.bstein.dev/atlas/titan-iac/pulls/",
            "jenkins_job_url": "https://jenkins.bstein.dev/job/hermes",
            "image_repository": "registry.bstein.dev/bstein/hermes",
            "flux_kustomization": "hermes", "health_url": "https://chat.bstein.dev/healthz",
        }},
    }))
    policy_file.chmod(0o444)
    router = build_router(tmp_path, {
        "HUX_FLAGS": ALL_ON, "HUX_ROUTER_KEY": "rk",
        "HUX_RELEASE_EVIDENCE_KEY_FILE": str(key_file),
        "HUX_RELEASE_EVIDENCE_POLICY_FILE": str(policy_file),
    })
    _, project, _ = call(router, "POST", "/hux/v1/projects", {"name": "P"})
    _, conversation, _ = call(router, "POST", "/hux/v1/conversations", {"title": "C", "project_id": project["id"]})
    base = f"/hux/v1/projects/{project['id']}/conversations/{conversation['id']}/releases"
    return router, project, conversation, base


def create(setup, key="release-key-001", commit="1" * 40, **changes):
    router, _, _, base = setup
    body = {
        "workload": "hermes-webui", "commit": commit, "feature_flags": ["hux.foundation"],
        "evidence": {"review_url": "https://git.bstein.dev/atlas/titan-iac/pulls/55"}, **changes,
    }
    return call(router, "POST", base, body, {"If-Match": "0", "Idempotency-Key": key})


def move(router, path, revision, key, target, evidence):
    return call(router, "POST", path + "/transitions", {"to": target, "evidence": evidence}, {
        **EVIDENCE, "If-Match": str(revision), "Idempotency-Key": key,
    })


def build_evidence(digest=DIGEST):
    return {
        "ci_build_url": "https://jenkins.bstein.dev/job/hermes/20/",
        "image_ref": f"registry.bstein.dev/bstein/hermes:build-20@{digest}",
        "image_digest": digest,
        "harbor_digest": digest,
    }


def through_built(setup, key_suffix="a", commit="2" * 40):
    router, _, _, base = setup
    created = create(setup, f"create-built-{key_suffix}", commit)[1]
    path = base + "/" + created["release"]["id"]
    move(router, path, 1, f"merge-built-{key_suffix}", "merged", {"merge_commit": "3" * 40})
    built = move(router, path, 2, f"build-built-{key_suffix}", "built", build_evidence())[1]
    return path, built


def test_full_release_chain_is_exact_immutable_and_live_only_after_convergence(setup):
    router, _, _, base = setup
    status, created, headers = create(setup)
    assert (status, created["revision"], headers["ETag"]) == (201, 1, "1")
    assert contracts.validate("release.schema.json", created["release"]) == []
    release_id = created["release"]["id"]
    path = base + "/" + release_id
    steps = [
        ("merged", {"merge_commit": "2" * 40}),
        ("built", build_evidence()),
        ("verified", {}),
        ("deployed", {"flux_revision": "main@sha1:" + "3" * 40}),
        ("converged", {"pod_digest": DIGEST}),
        ("live_verified", {"health_check": {"url": "/healthz", "status": "pass", "at": "2026-08-24T12:00:00Z"}}),
    ]
    view = created
    for revision, (target, evidence) in enumerate(steps, 1):
        status, view, headers = move(router, path, revision, f"transition-{target}", target, evidence)
        assert status == 200 and view["release"]["state"] == target
        assert view["revision"] == revision + 1 and headers["ETag"] == str(revision + 1)
        assert contracts.validate("release.schema.json", view["release"]) == []
    assert view["release"]["evidence"]["image_digest"] == view["release"]["evidence"]["harbor_digest"] == view["release"]["evidence"]["pod_digest"]
    rows = tenant(router).read(releases.FAMILY, release_id)
    assert len(rows) == 7 and [row["sequence"] for row in rows] == list(range(1, 8))
    assert all(contracts.validate("release-ledger.schema.json", row, SCHEMAS) == [] for row in rows)
    assert all(row["previous_hash"] == (releases.ZERO_HASH if index == 0 else rows[index - 1]["entry_hash"]) for index, row in enumerate(rows))
    assert call(router, "GET", path)[1] == view
    listed = call(router, "GET", base)[1]
    assert listed["items"] == [view]


def test_create_replay_uniqueness_and_scope(setup):
    router, project, _, base = setup
    status, created, _ = create(setup)
    status, replay, headers = create(setup)
    assert status == 200 and replay == created and headers["HUX-Replayed"] == "true"
    assert create(setup, "release-key-001", "2" * 40)[0] == 409
    assert create(setup, "release-key-002")[0] == 409
    assert create(setup, "release-key-003", "2" * 40)[0] == 201
    assert call(router, "GET", base + "/" + created["release"]["id"], headers=OTHER)[0] == 404
    wrong = base.replace(project["id"], "prj_missing0000")
    assert call(router, "GET", wrong + "/" + created["release"]["id"])[0] == 404
    assert call(router, "GET", base + "/rel_missing0000")[0] == 404
    assert call(router, "GET", base + "/BAD")[0] == 404


@pytest.mark.parametrize("changes", [
    {"workload": "unknown"}, {"commit": "short"}, {"feature_flags": "hux.foundation"},
    {"feature_flags": ["bad flag"]}, {"evidence": {}},
    {"evidence": {"review_url": "http://git.bstein.dev/pr/1"}},
    {"evidence": {"review_url": "https://user:pass@git.bstein.dev/pr/1"}},
    {"evidence": {"review_url": "https://git.bstein.dev/pr/1?token=secret"}},
])
def test_create_validation_is_fail_closed(setup, changes):
    commit = changes.get("commit", "4" * 40)
    rest = {key: value for key, value in changes.items() if key != "commit"}
    status, error, _ = create(setup, f"bad-release-{abs(hash(repr(changes)))}", commit, **rest)
    assert status == 400 and error["code"] == "invalid"


def test_create_requires_headers_body_and_capacity(setup, monkeypatch):
    router, _, _, base = setup
    body = {"workload": "hermes-webui", "commit": "1" * 40, "evidence": {"review_url": "https://git.bstein.dev/pr/1"}}
    assert call(router, "POST", base, body, {"Idempotency-Key": "no-if-match"})[0] == 400
    assert call(router, "POST", base, body, {"If-Match": "0"})[0] == 400
    assert router.dispatch("POST", base, HEADERS, b"[]").status == 400
    assert call(router, "POST", base, {**body, "extra": True}, {"If-Match": "0", "Idempotency-Key": "extra-field-key"})[0] == 400
    monkeypatch.setattr(releases, "MAX_RELEASES", 0)
    assert call(router, "POST", base, body, {"If-Match": "0", "Idempotency-Key": "over-release-limit"})[0] == 413


def test_transition_requires_exact_order_revision_and_evidence(setup):
    router, _, _, base = setup
    created = create(setup)[1]
    path = base + "/" + created["release"]["id"]
    assert move(router, path, 1, "skip-transition", "built", build_evidence())[0] == 409
    assert move(router, path, 1, "rollback-too-soon", "rolled_back", {"rollback_target": OTHER_DIGEST})[0] == 409
    assert move(router, path, 2, "stale-transition", "merged", {"merge_commit": "2" * 40})[0] == 409
    for index, evidence in enumerate(({}, {"merge_commit": "short"}, {"merge_commit": "2" * 40, "extra": "x"})):
        assert move(router, path, 1, f"bad-merge-{index}", "merged", evidence)[0] == 400
    status, merged, _ = move(router, path, 1, "good-merge-key", "merged", {"merge_commit": "2" * 40})
    assert status == 200 and merged["revision"] == 2
    status, replay, headers = move(router, path, 1, "good-merge-key", "merged", {"merge_commit": "2" * 40})
    assert status == 200 and replay == merged and headers["HUX-Replayed"] == "true"
    assert move(router, path, 2, "good-merge-key", "built", build_evidence())[0] == 409
    assert call(router, "POST", path + "/transitions", {"to": "built", "evidence": build_evidence()}, {
        **EVIDENCE, "Idempotency-Key": "missing-transition-if",
    })[0] == 400


def test_only_router_reviews_and_only_evidence_trust_transitions(setup):
    router, _, _, base = setup
    created = create(setup, "trust-bound-create")[1]
    path = base + "/" + created["release"]["id"]
    transition = {"to": "merged", "evidence": {"merge_commit": "2" * 40}}
    headers = {"If-Match": "1", "Idempotency-Key": "trust-bound-transition"}
    assert call(router, "POST", path + "/transitions", transition, headers)[0] == 403
    worker = {**HEADERS, "X-Hux-Surface": "worker", "X-Hux-Trust": "worker", "X-Hux-Relay-Key": "wk"}
    assert call(router, "POST", path + "/transitions", transition, {**worker, **headers})[0] == 401
    assert call(router, "POST", base, {
        "workload": "hermes-webui", "commit": "8" * 40,
        "evidence": {"review_url": "https://git.bstein.dev/atlas/titan-iac/pulls/8"},
    }, {**EVIDENCE, "If-Match": "0", "Idempotency-Key": "evidence-cannot-review"})[0] == 403


@pytest.mark.parametrize("evidence", [
    {**build_evidence(), "harbor_digest": OTHER_DIGEST},
    {**build_evidence(), "image_digest": OTHER_DIGEST},
    {**build_evidence(), "image_ref": "registry.bstein.dev/bstein/hermes:tag@" + OTHER_DIGEST},
    {**build_evidence(), "ci_build_url": "http://jenkins.bstein.dev/job/1"},
    {**build_evidence(), "ci_build_url": "https://jenkins.bstein.dev/job/1?token=x"},
    {"ci_build_url": "https://jenkins.bstein.dev/job/1", "image_ref": "bad", "image_digest": DIGEST, "harbor_digest": DIGEST},
])
def test_build_requires_matching_image_and_harbor_evidence(setup, evidence):
    router, _, _, base = setup
    suffix = hashlib_key(evidence)
    created = create(setup, f"create-{suffix}", (suffix * 4)[:40])[1]
    path = base + "/" + created["release"]["id"]
    assert move(router, path, 1, f"merge-{suffix}", "merged", {"merge_commit": "5" * 40})[0] == 200
    assert move(router, path, 2, f"build-{suffix}", "built", evidence)[0] == 400


def hashlib_key(value):
    import hashlib

    return hashlib.sha256(repr(value).encode()).hexdigest()[:12]


def test_deploy_convergence_health_and_rollback_checks(setup):
    router, _, _, _ = setup
    path, built = through_built(setup)
    assert built["release"]["state"] == "built"
    assert move(router, path, 3, "deploy-before-verify", "deployed", {"flux_revision": "main@sha1:" + "3" * 40})[0] == 409
    assert move(router, path, 3, "verify-with-data", "verified", {"harbor_digest": DIGEST})[0] == 400
    assert move(router, path, 3, "verify-ok", "verified", {})[0] == 200
    assert move(router, path, 4, "deploy-bad", "deployed", {"flux_revision": "feature@sha1:" + "3" * 40})[0] == 400
    assert move(router, path, 4, "deploy-ok", "deployed", {"flux_revision": "main@sha1:" + "3" * 40})[0] == 200
    assert move(router, path, 5, "converge-bad", "converged", {"pod_digest": OTHER_DIGEST})[0] == 400
    assert move(router, path, 5, "converge-ok", "converged", {"pod_digest": DIGEST})[0] == 200
    bad_health = [
        {"health_check": {"url": "/healthz", "status": "fail", "at": "2026-08-24T12:00:00Z"}},
        {"health_check": {"url": "http://public.example/health", "status": "pass", "at": "2026-08-24T12:00:00Z"}},
        {"health_check": {"url": "/healthz?token=x", "status": "pass", "at": "2026-08-24T12:00:00Z"}},
    ]
    for index, evidence in enumerate(bad_health):
        assert move(router, path, 6, f"health-bad-{index}", "live_verified", evidence)[0] == 400
    assert move(router, path, 6, "health-ok", "live_verified", {"health_check": {"url": "https://chat.bstein.dev/healthz", "status": "pass", "at": "2026-08-24T12:00:00Z"}})[0] == 200
    assert move(router, path, 7, "after-live", "live_verified", {})[0] == 409


def test_rollback_is_terminal_and_digest_bound(setup):
    router, _, _, _ = setup
    path, _ = through_built(setup, "rollback", "6" * 40)
    assert move(router, path, 3, "rollback-bad-digest", "rolled_back", {"rollback_target": "bad"})[0] == 400
    status, rolled, _ = move(router, path, 3, "rollback-ok", "rolled_back", {"rollback_target": OTHER_DIGEST})
    assert status == 200 and rolled["release"]["state"] == "rolled_back"
    assert move(router, path, 4, "rollback-terminal", "verified", {})[0] == 409


def test_tamper_breaks_reads_and_idempotency_fails_closed(setup):
    router, _, _, base = setup
    created = create(setup)[1]
    release_id = created["release"]["id"]
    ledger = tenant(router).root / releases.FAMILY / f"{release_id}.jsonl"
    rows = ledger.read_text().splitlines()
    row = json.loads(rows[0])
    row["snapshot"]["state"] = "live_verified"
    ledger.write_text(json.dumps(row) + "\n")
    status, error, _ = call(router, "GET", base + "/" + release_id)
    assert status == 400 and "integrity" in error["message"]
    status, error, _ = create(setup)
    assert status == 400 and "integrity" in error["message"]


def test_inconsistent_idempotency_and_cross_conversation_views_fail_closed(setup):
    router, project, _, base = setup
    created = create(setup)[1]
    idem = tenant(router).read(releases.FAMILY, "idempotency")
    idem[0]["sequence"] = 99
    tenant(router).rewrite(releases.FAMILY, "idempotency", idem)
    status, error, _ = create(setup)
    assert status == 400 and "idempotency ledger" in error["message"]
    _, conversation2, _ = call(router, "POST", "/hux/v1/conversations", {"title": "C2", "project_id": project["id"]})
    base2 = f"/hux/v1/projects/{project['id']}/conversations/{conversation2['id']}/releases"
    body = {"workload": "hermes-agent", "commit": "9" * 40, "evidence": {"review_url": "https://git.bstein.dev/pr/99"}}
    second = call(router, "POST", base2, body, {"If-Match": "0", "Idempotency-Key": "second-scope-key"})[1]
    first_id = created["release"]["id"]
    second_id = second["release"]["id"]
    assert call(router, "GET", base2 + "/" + first_id)[0] == 404
    assert call(router, "GET", base + "/" + second_id)[0] == 404
    assert move(router, base2 + "/" + first_id, 1, "cross-transition", "merged", {"merge_commit": "8" * 40})[0] == 404
    listed = call(router, "GET", base)[1]
    assert [item["release"]["id"] for item in listed["items"]] == [first_id]


def test_internal_helpers_are_bounded_and_worker_actor_has_no_subject():
    worker = SimpleNamespace(identity=identity.Identity("slot-3", "usr_0123456789abcdef", "worker", "worker"))
    assert releases._actor(worker) == {"type": "system", "id": "release-worker"}
    producer = SimpleNamespace(identity=identity.Identity("slot-3", "usr_0123456789abcdef", "api", "evidence"))
    assert releases._actor(producer) == {"type": "system", "id": "release-evidence-producer"}
    with pytest.raises(Invalid, match="missing or too long"):
        releases._safe_url(None)
    with pytest.raises(Invalid, match="secret"):
        releases._safe_url("https://git.bstein.dev/ghp_ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")
    with pytest.raises(Invalid):
        releases._safe_url("https://git.bstein.dev/pr/1#fragment")
    with pytest.raises(Invalid, match="ledger entry"):
        releases._validate_entry({"schema": "broken"})
    with pytest.raises(Invalid, match="does not match Harbor"):
        releases._transition_evidence({"evidence": {"image_digest": DIGEST, "harbor_digest": OTHER_DIGEST}}, "verified", {})
    for path in (FOUNDATION / "hux" / "releases.py", ROOT / "services" / "hermes" / "contracts" / "hux" / "release-ledger.schema.json", Path(__file__)):
        assert len(path.read_text().splitlines()) <= 500