"""HUX-02 memory control: an append-only ledger with explicit user consent. Every entry is a ``hux.memory.v1`` snapshot. The newest snapshot per id is kept as a revisioned document (so If-Match works) and every snapshot is also appended to ``ledger.jsonl``. Status moves only along ``rules.MEMORY_TRANSITIONS``; ``no_store`` and ``forgotten`` write a content-free line plus a tombstone before anything is returned (SO-22), and ``retrieve`` consults the tombstones before the documents (SO-23). """ from __future__ import annotations from datetime import datetime, timedelta, timezone from typing import Any from hux import contracts, redaction, rules from hux.errors import Conflict, Forbidden, Invalid, NotFound from hux.http import Request, Response, Router, page from hux.identity import Identity from hux.store import TenantStore, new_id, now_iso FAMILY = "memory" IDEM_FAMILY = "memory_idem" LEDGER = "ledger" TOMBSTONES = "tombstones" DEFAULT_CONVERSATION = "conv_memory" HUMAN_SURFACES = frozenset({"chat", "telegram", "voice"}) ACTIONS = ("approve", "reject", "forget", "edit", "remove_retrieval", "restore_retrieval") KINDS = ("preference", "fact", "instruction", "context") DEFAULT_DECAY_DAYS = 180 SCHEMAS = contracts.load_all() def _parse(stamp: str) -> datetime: return datetime.fromisoformat(stamp.replace("Z", "+00:00")) def _emit(store: TenantStore, identity: Identity, record: dict[str, Any], kind: str, summary: str) -> None: from hux import events detail = {"memory_id": record["id"], "kind": record["kind"], "sensitivity": record["sensitivity"], "topic": record.get("topic", "general")} conversation_id = record["provenance"].get("conversation_id", DEFAULT_CONVERSATION) events.emit(store, identity, conversation_id, kind, summary, detail, [{"kind": "memory", "id": record["id"]}], sensitivity=record["sensitivity"]) def tombstoned(store: TenantStore) -> set[str]: """Ids that must never surface again, whatever the documents say.""" return {row["memory_id"] for row in store.read(FAMILY, TOMBSTONES)} def _tombstone(store: TenantStore, memory_id: str, reason: str) -> None: store.append(FAMILY, TOMBSTONES, {"memory_id": memory_id, "at": now_iso(), "reason": reason, "purged": False}) def effective_expiry(record: dict[str, Any]) -> datetime | None: """When the entry stops being active: ``expires_at``, or decay from the last approval (or creation).""" ttl = record["ttl"] if ttl["policy"] == "expires_at": return _parse(ttl["expires_at"]) if ttl["policy"] == "decay": approvals = [row["at"] for row in record["audit"] if row["action"] == "approved"] return _parse(approvals[-1] if approvals else record["created_at"]) + timedelta(days=ttl["decay_days"]) return None def _persist(store: TenantStore, record: dict[str, Any], expected: int | None) -> dict[str, Any]: """Validate against contract and rules, then write document + ledger line under the family lock.""" problems = rules.memory_policy_violations(record) + contracts.validate_record({**record, "revision": record.get("revision") or 1}, SCHEMAS) if problems: raise Invalid("memory entry violates policy", problems) with store.lock(FAMILY): stored = store.put(FAMILY, record, expected) store.append(FAMILY, LEDGER, stored) return stored def _transition(store: TenantStore, identity: Identity, record: dict[str, Any], target: str, action: str, actor: dict[str, str], note: str = "", expected: int | None = None, **changes: Any) -> dict[str, Any]: if not rules.transition_allowed(rules.MEMORY_TRANSITIONS, record["status"], target): raise Conflict(f"{record['status']} -> {target} is not a legal memory transition") stamp = now_iso() updated = {**record, **changes, "status": target, "updated_at": stamp, "audit": [*record["audit"], {"at": stamp, "action": action, "actor": actor, **({"note": note[:200]} if note else {})}]} if target in {"forgotten", "rejected", "expired"}: updated["content"], updated["retrievable"] = "", False if target == "active": updated["retrievable"] = changes.get("retrievable", True) stored = _persist(store, updated, expected) if target == "forgotten": _tombstone(store, stored["id"], action) return stored def load(store: TenantStore, memory_id: str, now: datetime | None = None) -> dict[str, Any]: """Newest snapshot with lazy expiry (SO-27): an entry past its expiry is written as expired before it is served.""" record = store.get(FAMILY, memory_id) if record["status"] == "active": expiry = effective_expiry(record) if expiry is not None and expiry <= (now or datetime.now(timezone.utc)): record = _transition(store, None, record, "expired", "expired", {"type": "system", "id": "retention"}, expected=record["revision"]) return record def expire_due(store: TenantStore, now: datetime) -> int: """Retention pass: move every active entry past its expiry to expired. Returns the count.""" before = {row["id"]: row["status"] for row in store.scan(FAMILY)} return sum(1 for memory_id, status in before.items() if status == "active" and load(store, memory_id, now)["status"] == "expired") def purge_forgotten(store: TenantStore) -> int: """Retention pass (SO-47): blank ``content`` on every ledger snapshot of tombstoned ids, then mark them purged.""" with store.lock(FAMILY): stones = store.read(FAMILY, TOMBSTONES) pending = {row["memory_id"] for row in stones if not row.get("purged")} if not pending: return 0 rows = store.read(FAMILY, LEDGER) store.rewrite(FAMILY, LEDGER, [{**row, "content": ""} if row.get("id") in pending else row for row in rows]) store.rewrite(FAMILY, TOMBSTONES, [{**row, "purged": True} for row in stones]) return len(pending) def forget_from_conversation(store: TenantStore, identity: Identity, conversation_id: str) -> list[str]: """Forget every entry sourced from ``conversation_id`` (used by the privacy forget); returns the ids.""" from hux import events forgotten: list[str] = [] for row in list(store.scan(FAMILY)): if row["provenance"].get("conversation_id") != conversation_id: continue if row["status"] in {"proposed"}: row = _transition(store, identity, row, "rejected", "rejected", {"type": "system", "id": "forget"}, "conversation forgotten") if row["status"] in {"active", "expired"}: row = _transition(store, identity, row, "forgotten", "forgotten", {"type": "system", "id": "forget"}, "conversation forgotten") if row["status"] not in {"forgotten", "no_store"}: _tombstone(store, row["id"], "conversation forgotten") forgotten.append(row["id"]) events.redact_memory_references(store, row["id"]) return forgotten def retrieve(store: TenantStore, query_terms: list[str], scope: dict[str, Any] | None = None) -> list[dict[str, Any]]: """Agent read hook (SO-23): tombstones first, then only active + retrievable entries in scope; never no_store, forgotten or expired.""" from hux import privacy stones = tombstoned(store) blocked = privacy.blocked_conversations(store) if scope and scope.get("level") == "conversation" and scope.get("scope_id") in blocked: return [] terms = [term.lower() for term in query_terms if term] hits: list[dict[str, Any]] = [] for row in store.scan(FAMILY): if row["id"] in stones or row["provenance"].get("conversation_id") in blocked: continue record = load(store, row["id"]) if record["status"] != "active" or not record["retrievable"]: continue if scope and record["scope"]["level"] != "global" and record["scope"] != scope: continue text = record["content"].lower() if terms and not any(term in text for term in terms): continue hits.append(record) return sorted(hits, key=lambda r: r["updated_at"], reverse=True) # -- proposals ------------------------------------------------------------------ def _human(identity: Identity) -> bool: return identity.surface in HUMAN_SURFACES and identity.trust != "worker" def _actor(request: Request) -> dict[str, str]: if _human(request.identity) and request.body.get("proposed_by", "assistant") == "user": return {"type": "user", "id": request.identity.subject} return {"type": "assistant", "id": "hermes"} def _shape(request: Request, body: dict[str, Any]) -> tuple[dict[str, Any], str]: """Build the entry from a proposal body and decide its approval mode from policy; returns (entry, gate_reason).""" from hux import privacy if "id" in body or "revision" in body or "status" in body: raise Invalid("id, revision and status are server-assigned") content = body.get("content") if not isinstance(content, str) or not content.strip() or len(content) > 2000: raise Invalid("content must be a non-empty string of at most 2000 chars") hits: list[str] = [] content = redaction.scrub_value(content.strip(), hits) detected = privacy.detect_topics(content) + (["credentials"] if hits else []) topic = body.get("topic") if body.get("topic") in rules.PRIVACY_TOPICS or body.get("topic") == "general" else None topic = topic if topic and topic != "general" else (detected[0] if detected else "general") sensitivity = body.get("sensitivity", "personal") if sensitivity not in privacy.SENSITIVITY_RANK: raise Invalid("unknown sensitivity") floor = privacy.topic_sensitivity(detected + ([topic] if topic != "general" else [])) if privacy.SENSITIVITY_RANK[floor] > privacy.SENSITIVITY_RANK[sensitivity]: sensitivity = floor scope = body.get("scope") if isinstance(body.get("scope"), dict) else {"level": "global"} source = body.get("source") if isinstance(body.get("source"), dict) else {"kind": "message", "id": "unspecified"} ttl = body.get("ttl") if isinstance(body.get("ttl"), dict) else None if ttl is None or (sensitivity == "sensitive" and ttl.get("policy") == "never"): days = rules.PRIVACY_TOPICS[topic]["decay_days"] if topic in rules.PRIVACY_TOPICS else DEFAULT_DECAY_DAYS ttl = {"policy": "decay", "decay_days": days} actor = _actor(request) stamp = now_iso() conversation_id = body.get("conversation_id") if isinstance(body.get("conversation_id"), str) else None provenance = {"surface": request.identity.surface, "actor": actor, "recorded_at": stamp} if conversation_id: provenance["conversation_id"] = conversation_id if isinstance(body.get("run_id"), str): provenance["run_id"] = body["run_id"][:120] entry: dict[str, Any] = { "schema": "hux.memory.v1", "id": new_id("mem"), "owner": request.identity.subject, "scope": scope, "kind": body.get("kind") if body.get("kind") in KINDS else "fact", "content": content, "status": "proposed", "approval_mode": "ask", "sensitivity": sensitivity, "topic": topic, "ttl": ttl, "source": source, "provenance": provenance, "created_at": stamp, "updated_at": stamp, "audit": [{"at": stamp, "action": "proposed", "actor": actor}], "reason": str(body.get("reason") or "Proposed to be remembered.")[:280], "retrievable": False, "identity": request.identity.record(), } if isinstance(body.get("supersedes"), str): entry["supersedes"] = body["supersedes"] allowed, why = privacy.memory_write_allowed(request.store, conversation_id, entry) if why == "private_mode": raise Forbidden("memory writes are refused in private mode") references_forgotten = entry.get("supersedes") in tombstoned(request.store) or source.get("id") in tombstoned(request.store) wants = body.get("approval_mode", "ask" if actor["type"] == "assistant" else "automatic") if not allowed or wants == "no_store": # Content-free decision record. The topic moves into the note because # rules.memory_policy_violations reads a deny topic on any non-rejected # status as a write, and a sensitive entry must say approval_mode=ask. why = why if not allowed else "declined" entry.pop("topic") entry.update(status="no_store", approval_mode="ask" if sensitivity == "sensitive" else "no_store", content="", retrievable=False, audit=[{"at": stamp, "action": "no_store", "actor": actor, "note": f"{why}; topic={topic}"[:200]}]) elif sensitivity == "sensitive" or wants == "ask" or references_forgotten or actor["type"] != "user": entry["approval_mode"] = "ask" else: entry.update(status="active", approval_mode="automatic", retrievable=True) entry["audit"].append({"at": stamp, "action": "approved", "actor": actor, "note": "automatic"}) return entry, why def post_memory(request: Request) -> Response: """``POST /hux/v1/memory``: propose an entry; policy decides active, proposed (ask) or no_store (202).""" body = request.body if isinstance(request.body, dict) else None if body is None: raise Invalid("body must be an object") key = request.idempotency_key() if key: for row in request.store.read(IDEM_FAMILY, "keys"): if row["idempotency_key"] == key: request.audit("memory.propose", row["memory_id"], "allow", "replayed") return Response(200, load(request.store, row["memory_id"]), {"HUX-Replayed": "true"}) entry, why = _shape(request, body) try: stored = _persist(request.store, entry, None) except Invalid: request.store.append(FAMILY, TOMBSTONES, {"memory_id": entry["id"], "at": now_iso(), "reason": "policy_violation", "purged": True}) _emit(request.store, request.identity, entry, "memory.suppressed", "Memory proposal suppressed by policy") raise if key: request.store.append(IDEM_FAMILY, "keys", {"idempotency_key": key, "memory_id": stored["id"], "at": stored["created_at"]}) if stored["status"] == "no_store": _tombstone(request.store, stored["id"], why) _emit(request.store, request.identity, stored, "memory.suppressed", f"Not remembered ({why})") request.audit("memory.propose", stored["id"], "allow", "no_store") return Response(202, stored) kind = "memory.committed" if stored["status"] == "active" else "memory.proposed" _emit(request.store, request.identity, stored, kind, "Remembered" if stored["status"] == "active" else "Proposed to remember") request.audit("memory.propose", stored["id"]) return Response(201, stored) def list_memory(request: Request) -> Response: """``GET /hux/v1/memory?status=&scope=``: owner-scoped list; ``scope`` is ``level`` or ``level:scope_id``.""" status, scope = request.query.get("status"), request.query.get("scope") level, _, scope_id = (scope or "").partition(":") items = [] for row in list(request.store.scan(FAMILY)): record = load(request.store, row["id"]) if status and record["status"] != status: continue if level and (record["scope"]["level"] != level or (scope_id and record["scope"].get("scope_id") != scope_id)): continue items.append(record) request.audit("memory.list", "memory") return page(sorted(items, key=lambda r: r["updated_at"], reverse=True), None) def get_memory(request: Request) -> Response: """``GET /hux/v1/memory/{id}``.""" memory_id = request.params["id"] try: record = load(request.store, memory_id) except (NotFound, Invalid) as error: raise NotFound("memory entry not found") from error request.audit("memory.get", memory_id) return Response(200, record, {"ETag": str(record["revision"])}) def _edit(request: Request, record: dict[str, Any], actor: dict[str, str], expected: int | None) -> dict[str, Any]: body = request.body if isinstance(request.body, dict) else {} content = body.get("content") if not isinstance(content, str) or not content.strip(): raise Invalid("edit needs new content") old = _transition(request.store, request.identity, record, "forgotten", "superseded", actor, "edited", expected) from hux import events events.redact_memory_references(request.store, old["id"]) stamp = now_iso() fresh = { **{k: v for k, v in record.items() if k not in {"revision", "supersedes"}}, "id": new_id("mem"), "content": redaction.scrub_value(content.strip(), [])[:2000], "status": "active", "approval_mode": "automatic", "retrievable": True, "supersedes": old["id"], "created_at": stamp, "updated_at": stamp, "audit": [{"at": stamp, "action": "edited", "actor": actor, "note": f"supersedes {old['id']}"}, {"at": stamp, "action": "approved", "actor": actor}], } if fresh["sensitivity"] == "sensitive": fresh.update(status="proposed", approval_mode="ask", retrievable=False) fresh["audit"] = fresh["audit"][:1] return _persist(request.store, fresh, None) def act_memory(request: Request) -> Response: """``POST /hux/v1/memory/{id}/{action}``: approve, reject, forget, edit, remove_retrieval, restore_retrieval; every move emits a memory.* event.""" memory_id, action = request.params["id"], request.params["action"] if action not in ACTIONS: raise NotFound("unknown memory action") if not _human(request.identity): raise Forbidden("memory decisions come from a human surface") try: record = load(request.store, memory_id) except (NotFound, Invalid) as error: raise NotFound("memory entry not found") from error expected = request.if_match() if expected is not None and expected != record["revision"]: raise Conflict(f"revision {expected} does not match current revision {record['revision']}", [str(record["revision"])]) actor = {"type": "user", "id": request.identity.subject} store, identity = request.store, request.identity if action == "approve": stored = _transition(store, identity, record, "active", "approved", actor, expected=expected) _emit(store, identity, stored, "memory.committed", "Memory approved") elif action == "reject": stored = _transition(store, identity, record, "rejected", "rejected", actor, expected=expected) _emit(store, identity, stored, "memory.suppressed", "Memory rejected") elif action == "forget": stored = _transition(store, identity, record, "forgotten", "forgotten", actor, expected=expected) from hux import events events.redact_memory_references(store, stored["id"]) _emit(store, identity, stored, "memory.forgotten", "Memory forgotten") elif action == "edit": stored = _edit(request, record, actor, expected) _emit(store, identity, stored, "memory.committed" if stored["status"] == "active" else "memory.proposed", f"Memory edited, supersedes {record['id']}") else: if record["status"] != "active": raise Conflict("retrieval can only change on active entries") retrievable = action == "restore_retrieval" stamp = now_iso() stored = _persist(store, {**record, "retrievable": retrievable, "updated_at": stamp, "audit": [*record["audit"], {"at": stamp, "action": "retrieval_removed" if not retrievable else "approved", "actor": actor, "note": "retrieval restored" if retrievable else ""}]}, expected) _emit(store, identity, stored, "memory.retrieval_removed" if not retrievable else "memory.committed", "Retrieval removed" if not retrievable else "Retrieval restored") request.audit(f"memory.{action}", memory_id, "allow", "" if expected is not None else "unconditional_write") return Response(200, stored, {"ETag": str(stored["revision"])}) def export_memory(request: Request) -> Response: """``GET /hux/v1/memory/export``: active, retrievable entries only, each stamped ``exported`` (SO-26).""" stones = tombstoned(request.store) items = [] for row in list(request.store.scan(FAMILY)): record = load(request.store, row["id"]) if record["id"] in stones or record["status"] != "active" or not record["retrievable"]: continue stamp = now_iso() items.append(_persist(request.store, {**record, "audit": [*record["audit"], {"at": stamp, "action": "exported", "actor": {"type": "user", "id": request.identity.subject}}]}, record["revision"])) request.store.append(FAMILY, "exports", {"at": now_iso(), "ids": [item["id"] for item in items]}) request.audit("memory.export", "memory") response = page(items, None) response.headers["Content-Disposition"] = 'attachment; filename="hux-memory-export.json"' return response def register(router: Router) -> None: """Attach HUX-02 routes.""" router.add("GET", "/hux/v1/memory/export", "HUX-02", "memory.export", export_memory) router.add("POST", "/hux/v1/memory", "HUX-02", "memory.propose", post_memory) router.add("GET", "/hux/v1/memory", "HUX-02", "memory.list", list_memory) router.add("GET", "/hux/v1/memory/{id}", "HUX-02", "memory.get", get_memory) router.add("POST", "/hux/v1/memory/{id}/{action}", "HUX-02", "memory.act", act_memory)