1cb6f07c78
Per-conversation monotonic event ledger with idempotent emit, SSE replay from Last-Event-ID, per-kind detail allowlists and secret scrubbing, surface-aware serve-time redaction; memory as an append-only ledger with no-store, supersede, forget and retrieval tombstones so 'do not remember' blocks both persistence and retrieval; sensitive-topic scoping, notices, conversation forget and a retention job that never touches audit. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01RNPhwu2bsaRNg3DETSAZoM
388 lines
21 KiB
Python
388 lines
21 KiB
Python
"""HUX-02 memory control: an append-only ledger with explicit user consent.
|
|
|
|
Every entry is a ``hux.memory.v1`` snapshot. The newest snapshot per id is
|
|
kept as a revisioned document (so If-Match works) and every snapshot is also
|
|
appended to ``ledger.jsonl``. Status moves only along
|
|
``rules.MEMORY_TRANSITIONS``; ``no_store`` and ``forgotten`` write a
|
|
content-free line plus a tombstone before anything is returned (SO-22), and
|
|
``retrieve`` consults the tombstones before the documents (SO-23).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from datetime import datetime, timedelta, timezone
|
|
from typing import Any
|
|
|
|
from hux import contracts, redaction, rules
|
|
from hux.errors import Conflict, Forbidden, Invalid, NotFound
|
|
from hux.http import Request, Response, Router, page
|
|
from hux.identity import Identity
|
|
from hux.store import TenantStore, new_id, now_iso
|
|
|
|
FAMILY = "memory"
|
|
IDEM_FAMILY = "memory_idem"
|
|
LEDGER = "ledger"
|
|
TOMBSTONES = "tombstones"
|
|
DEFAULT_CONVERSATION = "conv_memory"
|
|
HUMAN_SURFACES = frozenset({"chat", "telegram", "voice"})
|
|
ACTIONS = ("approve", "reject", "forget", "edit", "remove_retrieval", "restore_retrieval")
|
|
KINDS = ("preference", "fact", "instruction", "context")
|
|
DEFAULT_DECAY_DAYS = 180
|
|
SCHEMAS = contracts.load_all()
|
|
|
|
|
|
def _parse(stamp: str) -> datetime:
|
|
return datetime.fromisoformat(stamp.replace("Z", "+00:00"))
|
|
|
|
|
|
def _emit(store: TenantStore, identity: Identity, record: dict[str, Any], kind: str, summary: str) -> None:
|
|
from hux import events
|
|
|
|
detail = {"memory_id": record["id"], "kind": record["kind"], "sensitivity": record["sensitivity"], "topic": record.get("topic", "general")}
|
|
conversation_id = record["provenance"].get("conversation_id", DEFAULT_CONVERSATION)
|
|
events.emit(store, identity, conversation_id, kind, summary, detail, [{"kind": "memory", "id": record["id"]}], sensitivity=record["sensitivity"])
|
|
|
|
|
|
def tombstoned(store: TenantStore) -> set[str]:
|
|
"""Ids that must never surface again, whatever the documents say."""
|
|
return {row["memory_id"] for row in store.read(FAMILY, TOMBSTONES)}
|
|
|
|
|
|
def _tombstone(store: TenantStore, memory_id: str, reason: str) -> None:
|
|
store.append(FAMILY, TOMBSTONES, {"memory_id": memory_id, "at": now_iso(), "reason": reason, "purged": False})
|
|
|
|
|
|
def effective_expiry(record: dict[str, Any]) -> datetime | None:
|
|
"""When the entry stops being active: ``expires_at``, or decay from the last approval (or creation)."""
|
|
ttl = record["ttl"]
|
|
if ttl["policy"] == "expires_at":
|
|
return _parse(ttl["expires_at"])
|
|
if ttl["policy"] == "decay":
|
|
approvals = [row["at"] for row in record["audit"] if row["action"] == "approved"]
|
|
return _parse(approvals[-1] if approvals else record["created_at"]) + timedelta(days=ttl["decay_days"])
|
|
return None
|
|
|
|
|
|
def _persist(store: TenantStore, record: dict[str, Any], expected: int | None) -> dict[str, Any]:
|
|
"""Validate against contract and rules, then write document + ledger line under the family lock."""
|
|
problems = rules.memory_policy_violations(record) + contracts.validate_record({**record, "revision": record.get("revision") or 1}, SCHEMAS)
|
|
if problems:
|
|
raise Invalid("memory entry violates policy", problems)
|
|
with store.lock(FAMILY):
|
|
stored = store.put(FAMILY, record, expected)
|
|
store.append(FAMILY, LEDGER, stored)
|
|
return stored
|
|
|
|
|
|
def _transition(store: TenantStore, identity: Identity, record: dict[str, Any], target: str, action: str, actor: dict[str, str],
|
|
note: str = "", expected: int | None = None, **changes: Any) -> dict[str, Any]:
|
|
if not rules.transition_allowed(rules.MEMORY_TRANSITIONS, record["status"], target):
|
|
raise Conflict(f"{record['status']} -> {target} is not a legal memory transition")
|
|
stamp = now_iso()
|
|
updated = {**record, **changes, "status": target, "updated_at": stamp, "audit": [*record["audit"], {"at": stamp, "action": action, "actor": actor, **({"note": note[:200]} if note else {})}]}
|
|
if target in {"forgotten", "rejected", "expired"}:
|
|
updated["content"], updated["retrievable"] = "", False
|
|
if target == "active":
|
|
updated["retrievable"] = changes.get("retrievable", True)
|
|
stored = _persist(store, updated, expected)
|
|
if target == "forgotten":
|
|
_tombstone(store, stored["id"], action)
|
|
return stored
|
|
|
|
|
|
def load(store: TenantStore, memory_id: str, now: datetime | None = None) -> dict[str, Any]:
|
|
"""Newest snapshot with lazy expiry (SO-27): an entry past its expiry is written as expired before it is served."""
|
|
record = store.get(FAMILY, memory_id)
|
|
if record["status"] == "active":
|
|
expiry = effective_expiry(record)
|
|
if expiry is not None and expiry <= (now or datetime.now(timezone.utc)):
|
|
record = _transition(store, None, record, "expired", "expired", {"type": "system", "id": "retention"}, expected=record["revision"])
|
|
return record
|
|
|
|
|
|
def expire_due(store: TenantStore, now: datetime) -> int:
|
|
"""Retention pass: move every active entry past its expiry to expired. Returns the count."""
|
|
before = {row["id"]: row["status"] for row in store.scan(FAMILY)}
|
|
return sum(1 for memory_id, status in before.items() if status == "active" and load(store, memory_id, now)["status"] == "expired")
|
|
|
|
|
|
def purge_forgotten(store: TenantStore) -> int:
|
|
"""Retention pass (SO-47): blank ``content`` on every ledger snapshot of tombstoned ids, then mark them purged."""
|
|
with store.lock(FAMILY):
|
|
stones = store.read(FAMILY, TOMBSTONES)
|
|
pending = {row["memory_id"] for row in stones if not row.get("purged")}
|
|
if not pending:
|
|
return 0
|
|
rows = store.read(FAMILY, LEDGER)
|
|
store.rewrite(FAMILY, LEDGER, [{**row, "content": ""} if row.get("id") in pending else row for row in rows])
|
|
store.rewrite(FAMILY, TOMBSTONES, [{**row, "purged": True} for row in stones])
|
|
return len(pending)
|
|
|
|
|
|
def forget_from_conversation(store: TenantStore, identity: Identity, conversation_id: str) -> list[str]:
|
|
"""Forget every entry sourced from ``conversation_id`` (used by the privacy forget); returns the ids."""
|
|
from hux import events
|
|
|
|
forgotten: list[str] = []
|
|
for row in list(store.scan(FAMILY)):
|
|
if row["provenance"].get("conversation_id") != conversation_id:
|
|
continue
|
|
if row["status"] in {"proposed"}:
|
|
row = _transition(store, identity, row, "rejected", "rejected", {"type": "system", "id": "forget"}, "conversation forgotten")
|
|
if row["status"] in {"active", "expired"}:
|
|
row = _transition(store, identity, row, "forgotten", "forgotten", {"type": "system", "id": "forget"}, "conversation forgotten")
|
|
if row["status"] not in {"forgotten", "no_store"}:
|
|
_tombstone(store, row["id"], "conversation forgotten")
|
|
forgotten.append(row["id"])
|
|
events.redact_memory_references(store, row["id"])
|
|
return forgotten
|
|
|
|
|
|
def retrieve(store: TenantStore, query_terms: list[str], scope: dict[str, Any] | None = None) -> list[dict[str, Any]]:
|
|
"""Agent read hook (SO-23): tombstones first, then only active + retrievable entries in scope; never no_store, forgotten or expired."""
|
|
from hux import privacy
|
|
|
|
stones = tombstoned(store)
|
|
blocked = privacy.blocked_conversations(store)
|
|
if scope and scope.get("level") == "conversation" and scope.get("scope_id") in blocked:
|
|
return []
|
|
terms = [term.lower() for term in query_terms if term]
|
|
hits: list[dict[str, Any]] = []
|
|
for row in store.scan(FAMILY):
|
|
if row["id"] in stones or row["provenance"].get("conversation_id") in blocked:
|
|
continue
|
|
record = load(store, row["id"])
|
|
if record["status"] != "active" or not record["retrievable"]:
|
|
continue
|
|
if scope and record["scope"]["level"] != "global" and record["scope"] != scope:
|
|
continue
|
|
text = record["content"].lower()
|
|
if terms and not any(term in text for term in terms):
|
|
continue
|
|
hits.append(record)
|
|
return sorted(hits, key=lambda r: r["updated_at"], reverse=True)
|
|
|
|
|
|
# -- proposals ------------------------------------------------------------------
|
|
|
|
def _human(identity: Identity) -> bool:
|
|
return identity.surface in HUMAN_SURFACES and identity.trust != "worker"
|
|
|
|
|
|
def _actor(request: Request) -> dict[str, str]:
|
|
if _human(request.identity) and request.body.get("proposed_by", "assistant") == "user":
|
|
return {"type": "user", "id": request.identity.subject}
|
|
return {"type": "assistant", "id": "hermes"}
|
|
|
|
|
|
def _shape(request: Request, body: dict[str, Any]) -> tuple[dict[str, Any], str]:
|
|
"""Build the entry from a proposal body and decide its approval mode from policy; returns (entry, gate_reason)."""
|
|
from hux import privacy
|
|
|
|
if "id" in body or "revision" in body or "status" in body:
|
|
raise Invalid("id, revision and status are server-assigned")
|
|
content = body.get("content")
|
|
if not isinstance(content, str) or not content.strip() or len(content) > 2000:
|
|
raise Invalid("content must be a non-empty string of at most 2000 chars")
|
|
hits: list[str] = []
|
|
content = redaction.scrub_value(content.strip(), hits)
|
|
detected = privacy.detect_topics(content) + (["credentials"] if hits else [])
|
|
topic = body.get("topic") if body.get("topic") in rules.PRIVACY_TOPICS or body.get("topic") == "general" else None
|
|
topic = topic if topic and topic != "general" else (detected[0] if detected else "general")
|
|
sensitivity = body.get("sensitivity", "personal")
|
|
if sensitivity not in privacy.SENSITIVITY_RANK:
|
|
raise Invalid("unknown sensitivity")
|
|
floor = privacy.topic_sensitivity(detected + ([topic] if topic != "general" else []))
|
|
if privacy.SENSITIVITY_RANK[floor] > privacy.SENSITIVITY_RANK[sensitivity]:
|
|
sensitivity = floor
|
|
scope = body.get("scope") if isinstance(body.get("scope"), dict) else {"level": "global"}
|
|
source = body.get("source") if isinstance(body.get("source"), dict) else {"kind": "message", "id": "unspecified"}
|
|
ttl = body.get("ttl") if isinstance(body.get("ttl"), dict) else None
|
|
if ttl is None or (sensitivity == "sensitive" and ttl.get("policy") == "never"):
|
|
days = rules.PRIVACY_TOPICS[topic]["decay_days"] if topic in rules.PRIVACY_TOPICS else DEFAULT_DECAY_DAYS
|
|
ttl = {"policy": "decay", "decay_days": days}
|
|
actor = _actor(request)
|
|
stamp = now_iso()
|
|
conversation_id = body.get("conversation_id") if isinstance(body.get("conversation_id"), str) else None
|
|
provenance = {"surface": request.identity.surface, "actor": actor, "recorded_at": stamp}
|
|
if conversation_id:
|
|
provenance["conversation_id"] = conversation_id
|
|
if isinstance(body.get("run_id"), str):
|
|
provenance["run_id"] = body["run_id"][:120]
|
|
entry: dict[str, Any] = {
|
|
"schema": "hux.memory.v1", "id": new_id("mem"), "owner": request.identity.subject, "scope": scope,
|
|
"kind": body.get("kind") if body.get("kind") in KINDS else "fact", "content": content, "status": "proposed",
|
|
"approval_mode": "ask", "sensitivity": sensitivity, "topic": topic, "ttl": ttl, "source": source,
|
|
"provenance": provenance, "created_at": stamp, "updated_at": stamp,
|
|
"audit": [{"at": stamp, "action": "proposed", "actor": actor}], "reason": str(body.get("reason") or "Proposed to be remembered.")[:280],
|
|
"retrievable": False, "identity": request.identity.record(),
|
|
}
|
|
if isinstance(body.get("supersedes"), str):
|
|
entry["supersedes"] = body["supersedes"]
|
|
allowed, why = privacy.memory_write_allowed(request.store, conversation_id, entry)
|
|
if why == "private_mode":
|
|
raise Forbidden("memory writes are refused in private mode")
|
|
references_forgotten = entry.get("supersedes") in tombstoned(request.store) or source.get("id") in tombstoned(request.store)
|
|
wants = body.get("approval_mode", "ask" if actor["type"] == "assistant" else "automatic")
|
|
if not allowed or wants == "no_store":
|
|
# Content-free decision record. The topic moves into the note because
|
|
# rules.memory_policy_violations reads a deny topic on any non-rejected
|
|
# status as a write, and a sensitive entry must say approval_mode=ask.
|
|
why = why if not allowed else "declined"
|
|
entry.pop("topic")
|
|
entry.update(status="no_store", approval_mode="ask" if sensitivity == "sensitive" else "no_store", content="", retrievable=False,
|
|
audit=[{"at": stamp, "action": "no_store", "actor": actor, "note": f"{why}; topic={topic}"[:200]}])
|
|
elif sensitivity == "sensitive" or wants == "ask" or references_forgotten or actor["type"] != "user":
|
|
entry["approval_mode"] = "ask"
|
|
else:
|
|
entry.update(status="active", approval_mode="automatic", retrievable=True)
|
|
entry["audit"].append({"at": stamp, "action": "approved", "actor": actor, "note": "automatic"})
|
|
return entry, why
|
|
|
|
|
|
def post_memory(request: Request) -> Response:
|
|
"""``POST /hux/v1/memory``: propose an entry; policy decides active, proposed (ask) or no_store (202)."""
|
|
body = request.body if isinstance(request.body, dict) else None
|
|
if body is None:
|
|
raise Invalid("body must be an object")
|
|
key = request.idempotency_key()
|
|
if key:
|
|
for row in request.store.read(IDEM_FAMILY, "keys"):
|
|
if row["idempotency_key"] == key:
|
|
request.audit("memory.propose", row["memory_id"], "allow", "replayed")
|
|
return Response(200, load(request.store, row["memory_id"]), {"HUX-Replayed": "true"})
|
|
entry, why = _shape(request, body)
|
|
try:
|
|
stored = _persist(request.store, entry, None)
|
|
except Invalid:
|
|
request.store.append(FAMILY, TOMBSTONES, {"memory_id": entry["id"], "at": now_iso(), "reason": "policy_violation", "purged": True})
|
|
_emit(request.store, request.identity, entry, "memory.suppressed", "Memory proposal suppressed by policy")
|
|
raise
|
|
if key:
|
|
request.store.append(IDEM_FAMILY, "keys", {"idempotency_key": key, "memory_id": stored["id"], "at": stored["created_at"]})
|
|
if stored["status"] == "no_store":
|
|
_tombstone(request.store, stored["id"], why)
|
|
_emit(request.store, request.identity, stored, "memory.suppressed", f"Not remembered ({why})")
|
|
request.audit("memory.propose", stored["id"], "allow", "no_store")
|
|
return Response(202, stored)
|
|
kind = "memory.committed" if stored["status"] == "active" else "memory.proposed"
|
|
_emit(request.store, request.identity, stored, kind, "Remembered" if stored["status"] == "active" else "Proposed to remember")
|
|
request.audit("memory.propose", stored["id"])
|
|
return Response(201, stored)
|
|
|
|
|
|
def list_memory(request: Request) -> Response:
|
|
"""``GET /hux/v1/memory?status=&scope=``: owner-scoped list; ``scope`` is ``level`` or ``level:scope_id``."""
|
|
status, scope = request.query.get("status"), request.query.get("scope")
|
|
level, _, scope_id = (scope or "").partition(":")
|
|
items = []
|
|
for row in list(request.store.scan(FAMILY)):
|
|
record = load(request.store, row["id"])
|
|
if status and record["status"] != status:
|
|
continue
|
|
if level and (record["scope"]["level"] != level or (scope_id and record["scope"].get("scope_id") != scope_id)):
|
|
continue
|
|
items.append(record)
|
|
request.audit("memory.list", "memory")
|
|
return page(sorted(items, key=lambda r: r["updated_at"], reverse=True), None)
|
|
|
|
|
|
def get_memory(request: Request) -> Response:
|
|
"""``GET /hux/v1/memory/{id}``."""
|
|
memory_id = request.params["id"]
|
|
try:
|
|
record = load(request.store, memory_id)
|
|
except (NotFound, Invalid) as error:
|
|
raise NotFound("memory entry not found") from error
|
|
request.audit("memory.get", memory_id)
|
|
return Response(200, record, {"ETag": str(record["revision"])})
|
|
|
|
|
|
def _edit(request: Request, record: dict[str, Any], actor: dict[str, str], expected: int | None) -> dict[str, Any]:
|
|
body = request.body if isinstance(request.body, dict) else {}
|
|
content = body.get("content")
|
|
if not isinstance(content, str) or not content.strip():
|
|
raise Invalid("edit needs new content")
|
|
old = _transition(request.store, request.identity, record, "forgotten", "superseded", actor, "edited", expected)
|
|
from hux import events
|
|
|
|
events.redact_memory_references(request.store, old["id"])
|
|
stamp = now_iso()
|
|
fresh = {
|
|
**{k: v for k, v in record.items() if k not in {"revision", "supersedes"}}, "id": new_id("mem"), "content": redaction.scrub_value(content.strip(), [])[:2000],
|
|
"status": "active", "approval_mode": "automatic", "retrievable": True, "supersedes": old["id"], "created_at": stamp, "updated_at": stamp,
|
|
"audit": [{"at": stamp, "action": "edited", "actor": actor, "note": f"supersedes {old['id']}"}, {"at": stamp, "action": "approved", "actor": actor}],
|
|
}
|
|
if fresh["sensitivity"] == "sensitive":
|
|
fresh.update(status="proposed", approval_mode="ask", retrievable=False)
|
|
fresh["audit"] = fresh["audit"][:1]
|
|
return _persist(request.store, fresh, None)
|
|
|
|
|
|
def act_memory(request: Request) -> Response:
|
|
"""``POST /hux/v1/memory/{id}/{action}``: approve, reject, forget, edit, remove_retrieval, restore_retrieval; every move emits a memory.* event."""
|
|
memory_id, action = request.params["id"], request.params["action"]
|
|
if action not in ACTIONS:
|
|
raise NotFound("unknown memory action")
|
|
if not _human(request.identity):
|
|
raise Forbidden("memory decisions come from a human surface")
|
|
try:
|
|
record = load(request.store, memory_id)
|
|
except (NotFound, Invalid) as error:
|
|
raise NotFound("memory entry not found") from error
|
|
expected = request.if_match()
|
|
if expected is not None and expected != record["revision"]:
|
|
raise Conflict(f"revision {expected} does not match current revision {record['revision']}", [str(record["revision"])])
|
|
actor = {"type": "user", "id": request.identity.subject}
|
|
store, identity = request.store, request.identity
|
|
if action == "approve":
|
|
stored = _transition(store, identity, record, "active", "approved", actor, expected=expected)
|
|
_emit(store, identity, stored, "memory.committed", "Memory approved")
|
|
elif action == "reject":
|
|
stored = _transition(store, identity, record, "rejected", "rejected", actor, expected=expected)
|
|
_emit(store, identity, stored, "memory.suppressed", "Memory rejected")
|
|
elif action == "forget":
|
|
stored = _transition(store, identity, record, "forgotten", "forgotten", actor, expected=expected)
|
|
from hux import events
|
|
|
|
events.redact_memory_references(store, stored["id"])
|
|
_emit(store, identity, stored, "memory.forgotten", "Memory forgotten")
|
|
elif action == "edit":
|
|
stored = _edit(request, record, actor, expected)
|
|
_emit(store, identity, stored, "memory.committed" if stored["status"] == "active" else "memory.proposed", f"Memory edited, supersedes {record['id']}")
|
|
else:
|
|
if record["status"] != "active":
|
|
raise Conflict("retrieval can only change on active entries")
|
|
retrievable = action == "restore_retrieval"
|
|
stamp = now_iso()
|
|
stored = _persist(store, {**record, "retrievable": retrievable, "updated_at": stamp, "audit": [*record["audit"], {"at": stamp, "action": "retrieval_removed" if not retrievable else "approved", "actor": actor, "note": "retrieval restored" if retrievable else ""}]}, expected)
|
|
_emit(store, identity, stored, "memory.retrieval_removed" if not retrievable else "memory.committed", "Retrieval removed" if not retrievable else "Retrieval restored")
|
|
request.audit(f"memory.{action}", memory_id, "allow", "" if expected is not None else "unconditional_write")
|
|
return Response(200, stored, {"ETag": str(stored["revision"])})
|
|
|
|
|
|
def export_memory(request: Request) -> Response:
|
|
"""``GET /hux/v1/memory/export``: active, retrievable entries only, each stamped ``exported`` (SO-26)."""
|
|
stones = tombstoned(request.store)
|
|
items = []
|
|
for row in list(request.store.scan(FAMILY)):
|
|
record = load(request.store, row["id"])
|
|
if record["id"] in stones or record["status"] != "active" or not record["retrievable"]:
|
|
continue
|
|
stamp = now_iso()
|
|
items.append(_persist(request.store, {**record, "audit": [*record["audit"], {"at": stamp, "action": "exported", "actor": {"type": "user", "id": request.identity.subject}}]}, record["revision"]))
|
|
request.store.append(FAMILY, "exports", {"at": now_iso(), "ids": [item["id"] for item in items]})
|
|
request.audit("memory.export", "memory")
|
|
response = page(items, None)
|
|
response.headers["Content-Disposition"] = 'attachment; filename="hux-memory-export.json"'
|
|
return response
|
|
|
|
|
|
def register(router: Router) -> None:
|
|
"""Attach HUX-02 routes."""
|
|
router.add("GET", "/hux/v1/memory/export", "HUX-02", "memory.export", export_memory)
|
|
router.add("POST", "/hux/v1/memory", "HUX-02", "memory.propose", post_memory)
|
|
router.add("GET", "/hux/v1/memory", "HUX-02", "memory.list", list_memory)
|
|
router.add("GET", "/hux/v1/memory/{id}", "HUX-02", "memory.get", get_memory)
|
|
router.add("POST", "/hux/v1/memory/{id}/{action}", "HUX-02", "memory.act", act_memory)
|