atlas-iac/testing/tests/test_hermes_hux_privacy_retention.py

"""HUX-10 retention job: expiry, topic decay, purge of forgotten content, audit record.

Security obligations exercised: SO-24 (forgotten content is purged from older
ledger lines), SO-27 (expiry enforced eagerly and reported), SO-47 (purge touches
only tombstoned ids and never the audit ledgers), SO-52 (retention keeps running
whatever the flags say).
"""

from __future__ import annotations

import json
import sys
from datetime import datetime, timedelta, timezone
from pathlib import Path

import pytest

ROOT = Path(__file__).resolve().parents[2]
FOUNDATION = ROOT / "dockerfiles" / "hermes-hux-foundation"
if str(FOUNDATION) not in sys.path:
    sys.path.insert(0, str(FOUNDATION))

from hux import audit, contracts, errors, events, identity, memory, privacy, store  # noqa: E402
from hux.server import build_router  # noqa: E402

SCHEMAS = contracts.load_all()
HEADERS = {"X-Hermes-Tenant-Identity": "slot-3", "X-Hux-Subject": "usr_0123456789abcdef", "X-Hux-Surface": "chat", "X-Hux-Relay-Key": "rk"}
OTHER = {**HEADERS, "X-Hux-Subject": "usr_fedcba9876543210"}
ALL_ON = ",".join(card["flag"] for card in contracts.load_flags()["cards"])
CONV = "conv_0001abcd"


def ident() -> identity.Identity:
    return identity.Identity("slot-3", "usr_0123456789abcdef", "chat", "router")


def router_for(tmp_path):
    return build_router(tmp_path, {"HUX_FLAGS": ALL_ON, "HUX_ROUTER_KEY": "rk"})


def call(router, method, path, headers=HEADERS, body=None):
    raw = json.dumps(body).encode() if body is not None else b""
    response = router.dispatch(method, path, headers, raw)
    return response.status, response.body


def tenant(tmp_path) -> store.TenantStore:
    return store.TenantStore(tmp_path, ident())


def remember(router, content, **fields):
    body = {"kind": "fact", "content": content, "reason": "r", "proposed_by": "user", "conversation_id": CONV, **fields}
    status, record = call(router, "POST", "/hux/v1/memory", HEADERS, body)
    assert status in (201, 202), record
    return record


def counts(record):
    return {row["action"]: row["count"] for row in record["results"]}


def test_retention_expires_decays_purges_and_reports(tmp_path):
    router = router_for(tmp_path)
    s = tenant(tmp_path)
    stale = remember(router, "expires soon", ttl={"policy": "expires_at", "expires_at": "2020-01-01T00:00:00Z"})
    keep = remember(router, "kept", ttl={"policy": "never"})
    gone = remember(router, "to be forgotten")
    call(router, "POST", f"/hux/v1/memory/{gone['id']}/forget")
    assert "to be forgotten" in json.dumps(s.read(memory.FAMILY, memory.LEDGER))
    events.emit(s, ident(), CONV, "message.user", "about my diagnosis", sensitivity="sensitive")
    events.emit(s, ident(), CONV, "message.assistant", "plain reply")
    privacy.mark_topic(s, CONV, "health", "2026-01-01T00:00:00Z")
    audit_rows_before = audit.recent(s)
    record = privacy.run_retention(s, datetime(2026, 8, 24, 3, 0, tzinfo=timezone.utc))
    assert record["schema"] == "hux.retention_audit.v1" and contracts.validate_record(record, SCHEMAS) == []
    assert counts(record) == {"expire_memory": 1, "decay_topic_context": 1, "purge_forgotten_content": 1, "report": 1}
    assert s.get(memory.FAMILY, stale["id"])["status"] == "expired" and s.get(memory.FAMILY, keep["id"])["status"] == "active"
    ledger = json.dumps(s.read(memory.FAMILY, memory.LEDGER))
    assert "to be forgotten" not in ledger and "kept" in ledger
    assert all(row["purged"] for row in s.read(memory.FAMILY, memory.TOMBSTONES))
    rows = s.read(events.FAMILY, CONV)
    sensitive = [r for r in rows if r["sensitivity"] == "sensitive"]
    assert sensitive and all(r["redaction"]["level"] == "full" and r["summary"].startswith("This looks like a health topic") for r in sensitive)
    assert [r["seq"] for r in rows] == list(range(1, len(rows) + 1))
    assert any(r["kind"] == "message.assistant" and r["redaction"]["level"] == "none" for r in rows)
    assert audit.recent(s)[: len(audit_rows_before)] == audit_rows_before
    assert privacy.conversation_state(s, CONV)["decayed"] is True
    again = privacy.run_retention(s, datetime(2026, 8, 25, 3, 0, tzinfo=timezone.utc))
    assert counts(again) == {"expire_memory": 0, "decay_topic_context": 0, "purge_forgotten_content": 0, "report": 1}


def test_retention_skips_topics_not_yet_due(tmp_path):
    s = tenant(tmp_path)
    events.emit(s, ident(), CONV, "message.user", "money talk", sensitivity="sensitive")
    privacy.mark_topic(s, CONV, "finance")
    privacy.set_flag(s, "conv_0002abcd", "memory_disabled", True)
    record = privacy.run_retention(s)
    assert counts(record)["decay_topic_context"] == 0
    assert s.read(events.FAMILY, CONV)[0]["redaction"]["level"] == "partial"
    assert not privacy.audit_stale(s)
    assert privacy.audit_stale(s, datetime.now(timezone.utc) + timedelta(days=3))


def test_audit_route_lists_newest_first_and_is_tenant_scoped(tmp_path):
    router = router_for(tmp_path)
    s = tenant(tmp_path)
    assert privacy.latest_audit(s) is None
    first = privacy.run_retention(s, datetime(2026, 8, 22, 3, 0, tzinfo=timezone.utc))
    second = privacy.run_retention(s, datetime(2026, 8, 23, 3, 0, tzinfo=timezone.utc))
    status, body = call(router, "GET", "/hux/v1/privacy/audit")
    assert status == 200 and [r["id"] for r in body["items"]] == [second["id"], first["id"]]
    for item in body["items"]:
        assert contracts.validate_record(item, SCHEMAS) == [] and "revision" not in item
    assert privacy.latest_audit(s)["id"] == second["id"]
    assert call(router, "GET", "/hux/v1/privacy/audit", OTHER)[1]["items"] == []
    assert [r["action"] for r in audit.recent(s)][-1] == "privacy.audit"


def test_retention_runs_without_flags(tmp_path):
    router = build_router(tmp_path, {"HUX_FLAGS": "", "HUX_ROUTER_KEY": "rk"})
    s = tenant(tmp_path)
    assert call(router, "GET", "/hux/v1/privacy/audit")[0] == 404
    assert counts(privacy.run_retention(s))["report"] == 1


def test_retention_record_is_validated_before_it_is_written(tmp_path, monkeypatch):
    s = tenant(tmp_path)
    monkeypatch.setattr(privacy, "new_id", lambda prefix: "bad id")
    with pytest.raises(errors.Invalid):
        privacy.run_retention(s)
    assert privacy.latest_audit(s) is None


def test_purge_leaves_untombstoned_content_alone(tmp_path):
    router = router_for(tmp_path)
    s = tenant(tmp_path)
    keep = remember(router, "keep me")
    assert memory.purge_forgotten(s) == 0
    s.append(memory.FAMILY, memory.TOMBSTONES, {"memory_id": "mem_ghost0000", "at": "2026-08-23T00:00:00Z", "reason": "test", "purged": False})
    assert memory.purge_forgotten(s) == 1
    assert s.get(memory.FAMILY, keep["id"])["content"] == "keep me"
    assert "keep me" in json.dumps(s.read(memory.FAMILY, memory.LEDGER))