atlas-iac/testing/tests/test_hermes_hux_memory_ledger.py

"""HUX-02 memory ledger: proposals, consent transitions, edits, If-Match and no-store.

Security obligations exercised: SO-10 (server-set owner and provenance),
SO-15 (server-assigned ids), SO-18 (foreign ids are 404), SO-21 (policy
violations suppressed with an event), SO-22 (no_store and forget write a
content-free line plus tombstone), SO-24 (forget re-redacts events), SO-25
(supersedes of a forgotten id needs a user approval), SO-26 (export scope and
audit), SO-28 (private mode refuses writes), SO-35-style human-only decisions,
SO-44 (If-Match conflicts, unconditional writes audited).
"""

from __future__ import annotations

import json
import sys
from pathlib import Path

ROOT = Path(__file__).resolve().parents[2]
FOUNDATION = ROOT / "dockerfiles" / "hermes-hux-foundation"
if str(FOUNDATION) not in sys.path:
    sys.path.insert(0, str(FOUNDATION))

from hux import audit, contracts, events, identity, memory, store  # noqa: E402
from hux.server import build_router  # noqa: E402

SCHEMAS = contracts.load_all()
HEADERS = {"X-Hermes-Tenant-Identity": "slot-3", "X-Hux-Subject": "usr_0123456789abcdef", "X-Hux-Surface": "chat", "X-Hux-Relay-Key": "rk"}
OTHER = {**HEADERS, "X-Hux-Subject": "usr_fedcba9876543210"}
ALL_ON = ",".join(card["flag"] for card in contracts.load_flags()["cards"])
CONV = "conv_0001abcd"
USER = {"proposed_by": "user"}


def ident(**overrides) -> identity.Identity:
    base = {"tenant_slot": "slot-3", "subject": "usr_0123456789abcdef", "surface": "chat", "trust": "router"}
    return identity.Identity(**{**base, **overrides})


def router_for(tmp_path):
    return build_router(tmp_path, {"HUX_FLAGS": ALL_ON, "HUX_ROUTER_KEY": "rk"})


def call(router, method, path, headers=HEADERS, body=None):
    raw = json.dumps(body).encode() if body is not None else b""
    response = router.dispatch(method, path, headers, raw)
    return response.status, response.body, response


def tenant(tmp_path, who=None) -> store.TenantStore:
    return store.TenantStore(tmp_path, who or ident())


def valid(record) -> None:
    assert contracts.validate_record(record, SCHEMAS) == [], record


def propose(router, headers=HEADERS, **fields):
    body = {"kind": "preference", "content": "Prefers terse answers with code first.", "reason": "asked twice", "conversation_id": CONV, **fields}
    return call(router, "POST", "/hux/v1/memory", headers, body)


def event_kinds(tmp_path, conversation=CONV):
    return [row["kind"] for row in tenant(tmp_path).read(events.FAMILY, conversation)]


# --- proposals ------------------------------------------------------------------

def test_user_proposal_with_personal_content_is_active_immediately(tmp_path):
    router = router_for(tmp_path)
    status, body, _ = propose(router, **USER, scope={"level": "project", "scope_id": "prj_0001aaaa"}, source={"kind": "message", "id": "msg-42"}, run_id="run_9f")
    assert status == 201 and body["status"] == "active" and body["approval_mode"] == "automatic" and body["retrievable"] is True
    assert body["owner"] == "usr_0123456789abcdef" and body["provenance"]["actor"]["type"] == "user" and body["provenance"]["run_id"] == "run_9f"
    assert body["ttl"] == {"policy": "decay", "decay_days": 180} and body["topic"] == "general" and body["revision"] == 1
    assert [row["action"] for row in body["audit"]] == ["proposed", "approved"]
    valid(body)
    assert event_kinds(tmp_path) == ["memory.committed"]
    ledger = tenant(tmp_path).read(memory.FAMILY, memory.LEDGER)
    assert [row["id"] for row in ledger] == [body["id"]]


def test_assistant_proposal_is_suggest_only(tmp_path):
    router = router_for(tmp_path)
    status, body, _ = propose(router)
    assert status == 201 and body["status"] == "proposed" and body["approval_mode"] == "ask" and body["retrievable"] is False
    assert body["provenance"]["actor"] == {"type": "assistant", "id": "hermes"}
    valid(body)
    assert event_kinds(tmp_path) == ["memory.proposed"]
    status, explicit, _ = propose(router, **USER, approval_mode="ask")
    assert explicit["status"] == "proposed"


def test_body_supplied_identity_and_ids_are_rejected_or_ignored(tmp_path):
    router = router_for(tmp_path)
    assert propose(router, id="mem_evil0001")[0] == 400
    assert propose(router, revision=7)[0] == 400
    assert propose(router, status="active")[0] == 400
    assert propose(router, content="")[0] == 400
    assert propose(router, content="x" * 2001)[0] == 400
    assert propose(router, sensitivity="ultra")[0] == 400
    assert call(router, "POST", "/hux/v1/memory", HEADERS, [1])[0] == 400
    status, body, _ = propose(router, **USER, owner="usr_fedcba9876543210", kind="weird", topic="not-a-topic", ttl="never", scope="bad", source="bad")
    assert status == 201 and body["owner"] == "usr_0123456789abcdef" and body["kind"] == "fact" and body["scope"] == {"level": "global"}
    assert body["source"] == {"kind": "message", "id": "unspecified"}


def test_sensitive_content_always_asks_and_decays(tmp_path):
    router = router_for(tmp_path)
    status, body, _ = propose(router, **USER, content="My therapist suggested a new medication for anxiety.", ttl={"policy": "never"})
    assert status == 201 and body["status"] == "proposed" and body["approval_mode"] == "ask"
    assert body["sensitivity"] == "sensitive" and body["topic"] == "health" and body["ttl"] == {"policy": "decay", "decay_days": 30}
    valid(body)
    status, body, _ = propose(router, **USER, content="plain", sensitivity="sensitive")
    assert body["approval_mode"] == "ask" and body["topic"] == "general"


def test_no_store_writes_tombstone_and_content_free_line(tmp_path):
    router = router_for(tmp_path)
    status, body, _ = propose(router, **USER, content="my password is hunter2hunter2 for the bank")
    assert status == 202 and body["status"] == "no_store" and body["content"] == "" and body["retrievable"] is False and "topic" not in body
    assert body["audit"][0]["action"] == "no_store" and "credentials" in body["audit"][0]["note"]
    valid(body)
    s = tenant(tmp_path)
    assert "hunter2" not in json.dumps(s.read(memory.FAMILY, memory.LEDGER))
    assert body["id"] in memory.tombstoned(s)
    assert event_kinds(tmp_path) == ["memory.suppressed"]
    status, declined, _ = propose(router, **USER, approval_mode="no_store", content="plain thing")
    assert status == 202 and declined["audit"][0]["note"].startswith("declined")
    status, restricted, _ = propose(router, **USER, sensitivity="restricted", content="plain")
    assert status == 202 and restricted["audit"][0]["note"].startswith("restricted")
    status, sensitive_declined, _ = propose(router, **USER, approval_mode="no_store", content="my mortgage rate")
    assert status == 202 and sensitive_declined["approval_mode"] == "ask"
    assert memory.retrieve(s, ["password", "mortgage", "plain"]) == []


def test_private_mode_conversation_refuses_memory(tmp_path):
    router = router_for(tmp_path)
    tenant(tmp_path).put("conversations", {"id": "conv_priv0001", "mode": "private"})
    status, body, _ = propose(router, **USER, conversation_id="conv_priv0001")
    assert (status, body["code"]) == (403, "forbidden")
    assert tenant(tmp_path).count(memory.FAMILY) == 0


def test_policy_violation_is_suppressed_with_event(tmp_path):
    router = router_for(tmp_path)
    status, body, _ = propose(router, **USER, ttl={"policy": "expires_at"})
    assert status == 400 and any("expires_at" in d for d in body["details"])
    assert event_kinds(tmp_path) == ["memory.suppressed"]
    assert tenant(tmp_path).count(memory.FAMILY) == 0


def test_proposal_idempotency_key_replays(tmp_path):
    router = router_for(tmp_path)
    propose(router, {**HEADERS, "Idempotency-Key": "conv:mem:0000"}, **USER)
    headers = {**HEADERS, "Idempotency-Key": "conv:mem:0001"}
    status, first, _ = propose(router, headers, **USER)
    status, again, response = propose(router, headers, **USER)
    assert status == 200 and again == first and response.headers["HUX-Replayed"] == "true"
    assert tenant(tmp_path).count(memory.FAMILY) == 2


# --- transitions -------------------------------------------------------------------

def test_approve_reject_and_illegal_transitions(tmp_path):
    router = router_for(tmp_path)
    _, proposed, _ = propose(router)
    path = f"/hux/v1/memory/{proposed['id']}"
    status, approved, response = call(router, "POST", f"{path}/approve", {**HEADERS, "If-Match": "1"})
    assert status == 200 and approved["status"] == "active" and approved["retrievable"] is True and approved["revision"] == 2
    assert response.headers["ETag"] == "2" and approved["audit"][-1]["actor"] == {"type": "user", "id": "usr_0123456789abcdef"}
    valid(approved)
    status, body, _ = call(router, "POST", f"{path}/approve")
    assert (status, body["code"]) == (409, "conflict")
    _, other, _ = propose(router)
    status, rejected, _ = call(router, "POST", f"/hux/v1/memory/{other['id']}/reject")
    assert status == 200 and rejected["status"] == "rejected" and rejected["content"] == "" and rejected["retrievable"] is False
    valid(rejected)
    assert call(router, "POST", f"/hux/v1/memory/{other['id']}/forget")[0] == 409
    assert event_kinds(tmp_path) == ["memory.proposed", "memory.committed", "memory.proposed", "memory.suppressed"]


def test_if_match_conflicts_and_unconditional_writes_are_audited(tmp_path):
    router = router_for(tmp_path)
    _, proposed, _ = propose(router)
    path = f"/hux/v1/memory/{proposed['id']}"
    status, body, _ = call(router, "POST", f"{path}/approve", {**HEADERS, "If-Match": "5"})
    assert (status, body["code"]) == (409, "conflict") and body["details"] == ["1"]
    assert call(router, "POST", f"{path}/approve", {**HEADERS, "If-Match": "x"})[0] == 400
    assert call(router, "POST", f"{path}/approve")[0] == 200
    rows = [(r["action"], r["outcome"], r.get("reason", "")) for r in audit.recent(tenant(tmp_path)) if r["action"] in {"memory.act", "memory.approve"}]
    assert rows[-3:] == [("memory.act", "conflict", "revision 5 does not match current revision 1"), ("memory.act", "deny", "If-Match must be a revision integer"), ("memory.approve", "allow", "unconditional_write")]


def test_forget_drops_content_tombstones_and_redacts_events(tmp_path):
    router = router_for(tmp_path)
    _, active, _ = propose(router, **USER)
    s = tenant(tmp_path)
    events.emit(s, ident(), CONV, "message.assistant", "used memory", evidence=[{"kind": "memory", "id": active["id"]}])
    status, forgotten, _ = call(router, "POST", f"/hux/v1/memory/{active['id']}/forget", {**HEADERS, "If-Match": "1"})
    assert status == 200 and forgotten["status"] == "forgotten" and forgotten["content"] == "" and forgotten["retrievable"] is False
    valid(forgotten)
    assert active["id"] in memory.tombstoned(s)
    rows = s.read(events.FAMILY, CONV)
    assert [r["kind"] for r in rows] == ["memory.committed", "message.assistant", "memory.forgotten"]
    assert rows[0]["redaction"]["level"] == "full" and rows[1]["redaction"]["level"] == "full"
    assert call(router, "GET", f"/hux/v1/memory/{active['id']}")[1]["status"] == "forgotten"
    assert call(router, "POST", f"/hux/v1/memory/{active['id']}/forget")[0] == 409


def test_edit_supersedes_and_forgets_the_old_entry(tmp_path):
    router = router_for(tmp_path)
    _, active, _ = propose(router, **USER)
    status, fresh, _ = call(router, "POST", f"/hux/v1/memory/{active['id']}/edit", {**HEADERS, "If-Match": "1"}, {"content": "Prefers long answers now."})
    assert status == 200 and fresh["id"] != active["id"] and fresh["supersedes"] == active["id"] and fresh["status"] == "active"
    assert fresh["content"] == "Prefers long answers now." and [a["action"] for a in fresh["audit"]] == ["edited", "approved"]
    valid(fresh)
    old = tenant(tmp_path).get(memory.FAMILY, active["id"])
    assert old["status"] == "forgotten" and old["content"] == "" and old["audit"][-1]["action"] == "superseded"
    assert call(router, "POST", f"/hux/v1/memory/{fresh['id']}/edit", HEADERS, {"content": ""})[0] == 400
    assert call(router, "POST", f"/hux/v1/memory/{fresh['id']}/edit", HEADERS)[0] == 400
    assert event_kinds(tmp_path)[-1] == "memory.committed"


def test_edit_of_sensitive_entry_goes_back_to_ask(tmp_path):
    router = router_for(tmp_path)
    _, proposed, _ = propose(router, **USER, content="my mortgage is with the bank")
    _, active, _ = call(router, "POST", f"/hux/v1/memory/{proposed['id']}/approve")
    status, fresh, _ = call(router, "POST", f"/hux/v1/memory/{active['id']}/edit", HEADERS, {"content": "refinanced the mortgage"})
    assert status == 200 and fresh["status"] == "proposed" and fresh["approval_mode"] == "ask" and fresh["retrievable"] is False
    valid(fresh)


def test_supersedes_of_forgotten_id_requires_ask(tmp_path):
    router = router_for(tmp_path)
    _, active, _ = propose(router, **USER)
    call(router, "POST", f"/hux/v1/memory/{active['id']}/forget")
    status, body, _ = propose(router, **USER, supersedes=active["id"])
    assert status == 201 and body["status"] == "proposed" and body["approval_mode"] == "ask" and body["supersedes"] == active["id"]
    status, body, _ = propose(router, **USER, source={"kind": "memory", "id": active["id"]})
    assert body["approval_mode"] == "ask"


def test_retrieval_removal_and_restore(tmp_path):
    router = router_for(tmp_path)
    _, active, _ = propose(router, **USER)
    path = f"/hux/v1/memory/{active['id']}"
    status, removed, _ = call(router, "POST", f"{path}/remove_retrieval")
    assert status == 200 and removed["status"] == "active" and removed["retrievable"] is False and removed["audit"][-1]["action"] == "retrieval_removed"
    valid(removed)
    assert memory.retrieve(tenant(tmp_path), []) == []
    status, restored, _ = call(router, "POST", f"{path}/restore_retrieval", {**HEADERS, "If-Match": "2"})
    assert status == 200 and restored["retrievable"] is True and restored["audit"][-1]["note"] == "retrieval restored"
    assert [m["id"] for m in memory.retrieve(tenant(tmp_path), [])] == [active["id"]]
    _, proposed, _ = propose(router)
    assert call(router, "POST", f"/hux/v1/memory/{proposed['id']}/remove_retrieval")[0] == 409
    assert event_kinds(tmp_path)[:3] == ["memory.committed", "memory.retrieval_removed", "memory.committed"]


def test_decisions_need_a_human_surface(tmp_path):
    router = build_router(tmp_path, {"HUX_FLAGS": ALL_ON, "HUX_ROUTER_KEY": "rk", "HUX_WORKER_KEY": "wk"})
    _, proposed, _ = propose(router)
    worker = {**HEADERS, "X-Hux-Surface": "worker", "X-Hux-Trust": "worker", "X-Hux-Relay-Key": "wk"}
    status, body, _ = call(router, "POST", f"/hux/v1/memory/{proposed['id']}/approve", worker)
    assert (status, body["code"]) == (403, "forbidden")
    api = {**HEADERS, "X-Hux-Surface": "api"}
    assert call(router, "POST", f"/hux/v1/memory/{proposed['id']}/approve", api)[0] == 403
    status, body, _ = propose(router, worker, proposed_by="user")
    assert body["provenance"]["actor"]["type"] == "assistant"


# --- reads, ownership, export ------------------------------------------------------

def test_list_get_and_cross_tenant_denial(tmp_path):
    router = router_for(tmp_path)
    _, active, _ = propose(router, **USER, scope={"level": "project", "scope_id": "prj_0001aaaa"})
    _, proposed, _ = propose(router)
    status, body, _ = call(router, "GET", "/hux/v1/memory")
    assert status == 200 and {m["id"] for m in body["items"]} == {active["id"], proposed["id"]}
    for item in body["items"]:
        valid(item)
    assert [m["id"] for m in call(router, "GET", "/hux/v1/memory?status=proposed")[1]["items"]] == [proposed["id"]]
    assert [m["id"] for m in call(router, "GET", "/hux/v1/memory?scope=project:prj_0001aaaa")[1]["items"]] == [active["id"]]
    assert call(router, "GET", "/hux/v1/memory?scope=project:prj_other000")[1]["items"] == []
    assert call(router, "GET", "/hux/v1/memory?scope=global")[1]["items"][0]["id"] == proposed["id"]
    status, got, response = call(router, "GET", f"/hux/v1/memory/{active['id']}")
    assert status == 200 and got == active and response.headers["ETag"] == "1"
    assert call(router, "GET", f"/hux/v1/memory/{active['id']}", OTHER)[0] == 404
    assert call(router, "POST", f"/hux/v1/memory/{active['id']}/forget", OTHER)[0] == 404
    assert call(router, "GET", "/hux/v1/memory", OTHER)[1]["items"] == []
    assert call(router, "GET", "/hux/v1/memory/not-an-id")[0] == 404
    assert call(router, "GET", "/hux/v1/memory/mem_missing00")[0] == 404
    assert call(router, "POST", f"/hux/v1/memory/{active['id']}/explode")[0] == 404
    assert call(router, "POST", "/hux/v1/memory/mem_missing00/approve")[0] == 404


def test_export_is_active_only_and_audited(tmp_path):
    router = router_for(tmp_path)
    _, active, _ = propose(router, **USER)
    _, hidden, _ = propose(router, **USER)
    call(router, "POST", f"/hux/v1/memory/{hidden['id']}/remove_retrieval")
    propose(router)
    _, gone, _ = propose(router, **USER)
    call(router, "POST", f"/hux/v1/memory/{gone['id']}/forget")
    status, body, response = call(router, "GET", "/hux/v1/memory/export")
    assert status == 200 and [m["id"] for m in body["items"]] == [active["id"]]
    assert body["items"][0]["audit"][-1]["action"] == "exported" and body["items"][0]["revision"] == 2
    assert response.headers["Content-Disposition"].startswith("attachment")
    valid(body["items"][0])
    assert tenant(tmp_path).read(memory.FAMILY, "exports")[0]["ids"] == [active["id"]]
    assert call(router, "GET", "/hux/v1/memory/export", OTHER)[1]["items"] == []


def test_memory_module_stays_under_500_lines():
    assert len((FOUNDATION / "hux" / "memory.py").read_text().splitlines()) <= 500