jenkins 681b040885 hermes(hux): close Wave A review findings in events, memory, privacy and organization
F3 memory edits go through the same privacy shaping as proposals; F5 seq is
derived from the ledger tail so a crash between append and checkpoint never
duplicates; F7 transitions re-read under the lock and always write with the
loaded revision; F9 secret scrub on titles, passages, claims and notebooks
and forget blanks the conversation document; F13 no ghost conversations
from notices, idempotency under the lock, artifact titles searchable,
normalised paths.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01RNPhwu2bsaRNg3DETSAZoM
2026-08-24 00:48:20 -03:00

460 lines
25 KiB
Python

"""HUX-02 memory control: an append-only ledger with explicit user consent.
Every entry is a ``hux.memory.v1`` snapshot. The newest snapshot per id is
kept as a revisioned document (so If-Match works) and every snapshot is also
appended to ``ledger.jsonl``. Status moves only along
``rules.MEMORY_TRANSITIONS``; ``no_store`` and ``forgotten`` write a
content-free line plus a tombstone before anything is returned (SO-22), and
``retrieve`` consults the tombstones before the documents (SO-23).
"""
from __future__ import annotations
from datetime import datetime, timedelta, timezone
from typing import Any
from hux import contracts, redaction, rules
from hux.errors import Conflict, Forbidden, Invalid, NotFound
from hux.http import Request, Response, Router, page
from hux.identity import Identity
from hux.store import TenantStore, new_id, now_iso
FAMILY = "memory"
IDEM_FAMILY = "memory_idem"
LEDGER = "ledger"
TOMBSTONES = "tombstones"
DEFAULT_CONVERSATION = "conv_memory"
HUMAN_SURFACES = frozenset({"chat", "telegram", "voice"})
ACTIONS = ("approve", "reject", "forget", "edit", "remove_retrieval", "restore_retrieval")
KINDS = ("preference", "fact", "instruction", "context")
DEFAULT_DECAY_DAYS = 180
SCHEMAS = contracts.load_all()
def _parse(stamp: str) -> datetime:
return datetime.fromisoformat(stamp.replace("Z", "+00:00"))
def _emit(store: TenantStore, identity: Identity, record: dict[str, Any], kind: str, summary: str) -> None:
from hux import events
detail = {"memory_id": record["id"], "kind": record["kind"], "sensitivity": record["sensitivity"], "topic": record.get("topic", "general")}
conversation_id = record["provenance"].get("conversation_id", DEFAULT_CONVERSATION)
events.emit(store, identity, conversation_id, kind, summary, detail, [{"kind": "memory", "id": record["id"]}], sensitivity=record["sensitivity"])
def tombstoned(store: TenantStore) -> set[str]:
"""Ids that must never surface again, whatever the documents say."""
return {row["memory_id"] for row in store.read(FAMILY, TOMBSTONES)}
def _tombstone(store: TenantStore, memory_id: str, reason: str) -> None:
store.append(FAMILY, TOMBSTONES, {"memory_id": memory_id, "at": now_iso(), "reason": reason, "purged": False})
def effective_expiry(record: dict[str, Any]) -> datetime | None:
"""When the entry stops being active: ``expires_at``, or decay from the last approval (or creation)."""
ttl = record["ttl"]
if ttl["policy"] == "expires_at":
return _parse(ttl["expires_at"])
if ttl["policy"] == "decay":
approvals = [row["at"] for row in record["audit"] if row["action"] == "approved"]
return _parse(approvals[-1] if approvals else record["created_at"]) + timedelta(days=ttl["decay_days"])
return None
def _persist(store: TenantStore, record: dict[str, Any], expected: int | None) -> dict[str, Any]:
"""Validate against contract and rules, then write document + ledger line under the family lock."""
problems = rules.memory_policy_violations(record) + contracts.validate_record({**record, "revision": record.get("revision") or 1}, SCHEMAS)
if problems:
raise Invalid("memory entry violates policy", problems)
with store.lock(FAMILY):
stored = store.put(FAMILY, record, expected)
store.append(FAMILY, LEDGER, stored)
return stored
def _transition(store: TenantStore, identity: Identity, record: dict[str, Any], target: str, action: str, actor: dict[str, str],
note: str = "", expected: int | None = None, **changes: Any) -> dict[str, Any]:
"""Move an entry to ``target`` from its *current* stored state (F7).
The caller's ``record`` may be stale: the entry is re-read under the family
lock, ``expected`` (If-Match) is checked against that copy, and the write is
always conditional on the loaded revision so a stale unconditional write can
never resurrect a forgotten entry (SO-22, SO-44).
"""
with store.lock(FAMILY):
record = _current(store, record, expected)
if not rules.transition_allowed(rules.MEMORY_TRANSITIONS, record["status"], target):
raise Conflict(f"{record['status']} -> {target} is not a legal memory transition")
stamp = now_iso()
updated = {**record, **changes, "status": target, "updated_at": stamp, "audit": [*record["audit"], {"at": stamp, "action": action, "actor": actor, **({"note": note[:200]} if note else {})}]}
if target in {"forgotten", "rejected", "expired"}:
updated["content"], updated["retrievable"] = "", False
if target == "active":
updated["retrievable"] = changes.get("retrievable", True)
stored = _persist(store, updated, record["revision"])
if target == "forgotten":
_tombstone(store, stored["id"], action)
return stored
def _current(store: TenantStore, record: dict[str, Any], expected: int | None) -> dict[str, Any]:
"""Fresh copy of ``record`` from the store; Conflict when If-Match no longer matches it."""
current = store.get(FAMILY, record["id"])
if expected is not None and expected != current["revision"]:
raise Conflict(f"revision {expected} does not match current revision {current['revision']}", [str(current["revision"])])
return current
def load(store: TenantStore, memory_id: str, now: datetime | None = None) -> dict[str, Any]:
"""Newest snapshot with lazy expiry (SO-27): an entry past its expiry is written as expired before it is served."""
record = store.get(FAMILY, memory_id)
if record["status"] == "active":
expiry = effective_expiry(record)
if expiry is not None and expiry <= (now or datetime.now(timezone.utc)):
record = _transition(store, None, record, "expired", "expired", {"type": "system", "id": "retention"}, expected=record["revision"])
return record
def expire_due(store: TenantStore, now: datetime) -> int:
"""Retention pass: move every active entry past its expiry to expired. Returns the count."""
before = {row["id"]: row["status"] for row in store.scan(FAMILY)}
return sum(1 for memory_id, status in before.items() if status == "active" and load(store, memory_id, now)["status"] == "expired")
def purge_forgotten(store: TenantStore) -> int:
"""Retention pass (SO-47): blank ``content`` on every ledger snapshot of tombstoned ids, then mark them purged."""
with store.lock(FAMILY):
stones = store.read(FAMILY, TOMBSTONES)
pending = {row["memory_id"] for row in stones if not row.get("purged")}
if not pending:
return 0
rows = store.read(FAMILY, LEDGER)
store.rewrite(FAMILY, LEDGER, [{**row, "content": ""} if row.get("id") in pending else row for row in rows])
store.rewrite(FAMILY, TOMBSTONES, [{**row, "purged": True} for row in stones])
return len(pending)
def forget_from_conversation(store: TenantStore, identity: Identity, conversation_id: str) -> list[str]:
"""Forget every entry sourced from ``conversation_id`` (used by the privacy forget); returns the ids."""
from hux import events
forgotten: list[str] = []
for row in list(store.scan(FAMILY)):
if row["provenance"].get("conversation_id") != conversation_id:
continue
if row["status"] in {"proposed"}:
row = _transition(store, identity, row, "rejected", "rejected", {"type": "system", "id": "forget"}, "conversation forgotten")
if row["status"] in {"active", "expired"}:
row = _transition(store, identity, row, "forgotten", "forgotten", {"type": "system", "id": "forget"}, "conversation forgotten")
if row["status"] not in {"forgotten", "no_store"}:
_tombstone(store, row["id"], "conversation forgotten")
forgotten.append(row["id"])
events.redact_memory_references(store, row["id"])
return forgotten
def retrieve(store: TenantStore, query_terms: list[str], scope: dict[str, Any] | None = None) -> list[dict[str, Any]]:
"""Agent read hook (SO-23): tombstones first, then only active + retrievable entries in scope; never no_store, forgotten or expired."""
from hux import privacy
stones = tombstoned(store)
blocked = privacy.blocked_conversations(store)
if scope and scope.get("level") == "conversation" and scope.get("scope_id") in blocked:
return []
terms = [term.lower() for term in query_terms if term]
hits: list[dict[str, Any]] = []
for row in store.scan(FAMILY):
if row["id"] in stones or row["provenance"].get("conversation_id") in blocked:
continue
record = load(store, row["id"])
if record["status"] != "active" or not record["retrievable"]:
continue
if scope and record["scope"]["level"] != "global" and record["scope"] != scope:
continue
text = record["content"].lower()
if terms and not any(term in text for term in terms):
continue
hits.append(record)
return sorted(hits, key=lambda r: r["updated_at"], reverse=True)
# -- proposals ------------------------------------------------------------------
def _human(identity: Identity) -> bool:
return identity.surface in HUMAN_SURFACES and identity.trust != "worker"
def _actor(request: Request) -> dict[str, str]:
if _human(request.identity) and request.body.get("proposed_by", "assistant") == "user":
return {"type": "user", "id": request.identity.subject}
return {"type": "assistant", "id": "hermes"}
def _classify(store: TenantStore, body: dict[str, Any], content: str, conversation_id: str | None) -> dict[str, Any]:
"""Scrub ``content`` and decide topic, sensitivity floor and the privacy gate; shared by create and edit (F3)."""
from hux import privacy
hits: list[str] = []
content = redaction.scrub_value(content.strip(), hits)[:2000]
detected = privacy.detect_topics(content) + (["credentials"] if hits else [])
topic = body.get("topic") if body.get("topic") in rules.PRIVACY_TOPICS or body.get("topic") == "general" else None
topic = topic if topic and topic != "general" else (detected[0] if detected else "general")
sensitivity = body.get("sensitivity", "personal")
if sensitivity not in privacy.SENSITIVITY_RANK:
raise Invalid("unknown sensitivity")
floor = privacy.topic_sensitivity(detected + ([topic] if topic != "general" else []))
if privacy.SENSITIVITY_RANK[floor] > privacy.SENSITIVITY_RANK[sensitivity]:
sensitivity = floor
allowed, why = privacy.memory_write_allowed(store, conversation_id, {"sensitivity": sensitivity, "topic": topic})
if why == "private_mode":
raise Forbidden("memory writes are refused in private mode")
return {"content": content, "topic": topic, "sensitivity": sensitivity, "allowed": allowed, "why": why}
def _decline(entry: dict[str, Any], why: str, actor: dict[str, str], stamp: str) -> dict[str, Any]:
"""Content-free ``no_store`` decision record.
The topic moves into the note because rules.memory_policy_violations reads a
deny topic on any non-rejected status as a write, and a sensitive entry must
say approval_mode=ask.
"""
topic = entry.pop("topic")
entry.update(status="no_store", approval_mode="ask" if entry["sensitivity"] == "sensitive" else "no_store", content="", retrievable=False,
audit=[{"at": stamp, "action": "no_store", "actor": actor, "note": f"{why}; topic={topic}"[:200]}])
return entry
def _shape(request: Request, body: dict[str, Any]) -> tuple[dict[str, Any], str]:
"""Build the entry from a proposal body and decide its approval mode from policy; returns (entry, gate_reason)."""
if "id" in body or "revision" in body or "status" in body:
raise Invalid("id, revision and status are server-assigned")
content = body.get("content")
if not isinstance(content, str) or not content.strip() or len(content) > 2000:
raise Invalid("content must be a non-empty string of at most 2000 chars")
conversation_id = body.get("conversation_id") if isinstance(body.get("conversation_id"), str) else None
verdict = _classify(request.store, body, content, conversation_id)
topic, sensitivity = verdict["topic"], verdict["sensitivity"]
scope = body.get("scope") if isinstance(body.get("scope"), dict) else {"level": "global"}
source = body.get("source") if isinstance(body.get("source"), dict) else {"kind": "message", "id": "unspecified"}
ttl = body.get("ttl") if isinstance(body.get("ttl"), dict) else None
if ttl is None or (sensitivity == "sensitive" and ttl.get("policy") == "never"):
days = rules.PRIVACY_TOPICS[topic]["decay_days"] if topic in rules.PRIVACY_TOPICS else DEFAULT_DECAY_DAYS
ttl = {"policy": "decay", "decay_days": days}
actor = _actor(request)
stamp = now_iso()
provenance = {"surface": request.identity.surface, "actor": actor, "recorded_at": stamp}
if conversation_id:
provenance["conversation_id"] = conversation_id
if isinstance(body.get("run_id"), str):
provenance["run_id"] = body["run_id"][:120]
entry: dict[str, Any] = {
"schema": "hux.memory.v1", "id": new_id("mem"), "owner": request.identity.subject, "scope": scope,
"kind": body.get("kind") if body.get("kind") in KINDS else "fact", "content": verdict["content"], "status": "proposed",
"approval_mode": "ask", "sensitivity": sensitivity, "topic": topic, "ttl": ttl, "source": source,
"provenance": provenance, "created_at": stamp, "updated_at": stamp,
"audit": [{"at": stamp, "action": "proposed", "actor": actor}], "reason": str(body.get("reason") or "Proposed to be remembered.")[:280],
"retrievable": False, "identity": request.identity.record(),
}
if isinstance(body.get("supersedes"), str):
entry["supersedes"] = body["supersedes"]
references_forgotten = entry.get("supersedes") in tombstoned(request.store) or source.get("id") in tombstoned(request.store)
wants = body.get("approval_mode", "ask" if actor["type"] == "assistant" else "automatic")
why = verdict["why"]
if not verdict["allowed"] or wants == "no_store":
why = why if not verdict["allowed"] else "declined"
_decline(entry, why, actor, stamp)
elif sensitivity == "sensitive" or wants == "ask" or references_forgotten or actor["type"] != "user":
entry["approval_mode"] = "ask"
else:
entry.update(status="active", approval_mode="automatic", retrievable=True)
entry["audit"].append({"at": stamp, "action": "approved", "actor": actor, "note": "automatic"})
return entry, why
def _replay(request: Request, key: str) -> Response | None:
for row in request.store.read(IDEM_FAMILY, "keys"):
if row["idempotency_key"] == key:
request.audit("memory.propose", row["memory_id"], "allow", "replayed")
return Response(200, load(request.store, row["memory_id"]), {"HUX-Replayed": "true"})
return None
def post_memory(request: Request) -> Response:
"""``POST /hux/v1/memory``: propose an entry; policy decides active, proposed (ask) or no_store (202).
The Idempotency-Key lookup, the write and the key mapping all happen under
the family lock so concurrent retries with one key yield one record (F13b).
"""
body = request.body if isinstance(request.body, dict) else None
if body is None:
raise Invalid("body must be an object")
key = request.idempotency_key()
with request.store.lock(FAMILY):
replayed = _replay(request, key) if key else None
if replayed is not None:
return replayed
entry, why = _shape(request, body)
try:
stored = _persist(request.store, entry, None)
except Invalid:
request.store.append(FAMILY, TOMBSTONES, {"memory_id": entry["id"], "at": now_iso(), "reason": "policy_violation", "purged": True})
_emit(request.store, request.identity, entry, "memory.suppressed", "Memory proposal suppressed by policy")
raise
if key:
request.store.append(IDEM_FAMILY, "keys", {"idempotency_key": key, "memory_id": stored["id"], "at": stored["created_at"]})
if stored["status"] == "no_store":
_tombstone(request.store, stored["id"], why)
_emit(request.store, request.identity, stored, "memory.suppressed", f"Not remembered ({why})")
request.audit("memory.propose", stored["id"], "allow", "no_store")
return Response(202, stored)
kind = "memory.committed" if stored["status"] == "active" else "memory.proposed"
_emit(request.store, request.identity, stored, kind, "Remembered" if stored["status"] == "active" else "Proposed to remember")
request.audit("memory.propose", stored["id"])
return Response(201, stored)
def list_memory(request: Request) -> Response:
"""``GET /hux/v1/memory?status=&scope=``: owner-scoped list; ``scope`` is ``level`` or ``level:scope_id``."""
status, scope = request.query.get("status"), request.query.get("scope")
level, _, scope_id = (scope or "").partition(":")
items = []
for row in list(request.store.scan(FAMILY)):
record = load(request.store, row["id"])
if status and record["status"] != status:
continue
if level and (record["scope"]["level"] != level or (scope_id and record["scope"].get("scope_id") != scope_id)):
continue
items.append(record)
request.audit("memory.list", "memory")
return page(sorted(items, key=lambda r: r["updated_at"], reverse=True), None)
def get_memory(request: Request) -> Response:
"""``GET /hux/v1/memory/{id}``."""
memory_id = request.params["id"]
try:
record = load(request.store, memory_id)
except (NotFound, Invalid) as error:
raise NotFound("memory entry not found") from error
request.audit("memory.get", memory_id)
return Response(200, record, {"ETag": str(record["revision"])})
def _edit(request: Request, record: dict[str, Any], actor: dict[str, str], expected: int | None) -> dict[str, Any]:
"""Supersede ``record`` with edited content under the same privacy shaping as a proposal (F3).
A deny verdict (deny topic, restricted, memory disabled, conversation
forgotten) yields a content-free ``no_store`` record and leaves the old
entry untouched; a sensitive floor makes the replacement ``proposed`` with
``approval_mode: ask``. Private mode is refused outright.
"""
body = request.body if isinstance(request.body, dict) else {}
content = body.get("content")
if not isinstance(content, str) or not content.strip() or len(content) > 2000:
raise Invalid("edit needs new content of at most 2000 chars")
conversation_id = record["provenance"].get("conversation_id")
verdict = _classify(request.store, {**body, "sensitivity": record["sensitivity"]}, content, conversation_id)
stamp = now_iso()
fresh = {
**{k: v for k, v in record.items() if k not in {"revision", "supersedes"}}, "id": new_id("mem"), "content": verdict["content"],
"topic": verdict["topic"], "sensitivity": verdict["sensitivity"], "status": "active", "approval_mode": "automatic", "retrievable": True,
"created_at": stamp, "updated_at": stamp, "audit": [{"at": stamp, "action": "edited", "actor": actor, "note": f"edit of {record['id']}"}],
}
if not verdict["allowed"]:
stored = _persist(request.store, _decline(fresh, verdict["why"], actor, stamp), None)
_tombstone(request.store, stored["id"], verdict["why"])
return stored
old = _transition(request.store, request.identity, record, "forgotten", "superseded", actor, "edited", expected)
from hux import events
events.redact_memory_references(request.store, old["id"])
fresh["supersedes"] = old["id"]
fresh["audit"][0]["note"] = f"supersedes {old['id']}"
if fresh["sensitivity"] == "sensitive":
fresh.update(status="proposed", approval_mode="ask", retrievable=False)
else:
fresh["audit"].append({"at": stamp, "action": "approved", "actor": actor})
return _persist(request.store, fresh, None)
def _set_retrievable(store: TenantStore, record: dict[str, Any], retrievable: bool, actor: dict[str, str], expected: int | None) -> dict[str, Any]:
"""Flip ``retrievable`` on an active entry, re-reading it under the lock so a stale copy cannot overwrite a forget (F7)."""
with store.lock(FAMILY):
record = _current(store, record, expected)
if record["status"] != "active":
raise Conflict("retrieval can only change on active entries")
stamp = now_iso()
entry = {"at": stamp, "action": "approved" if retrievable else "retrieval_removed", "actor": actor, "note": "retrieval restored" if retrievable else ""}
return _persist(store, {**record, "retrievable": retrievable, "updated_at": stamp, "audit": [*record["audit"], entry]}, record["revision"])
def act_memory(request: Request) -> Response:
"""``POST /hux/v1/memory/{id}/{action}``: approve, reject, forget, edit, remove_retrieval, restore_retrieval; every move emits a memory.* event."""
memory_id, action = request.params["id"], request.params["action"]
if action not in ACTIONS:
raise NotFound("unknown memory action")
if not _human(request.identity):
raise Forbidden("memory decisions come from a human surface")
try:
record = load(request.store, memory_id)
except (NotFound, Invalid) as error:
raise NotFound("memory entry not found") from error
expected = request.if_match()
if expected is not None and expected != record["revision"]:
raise Conflict(f"revision {expected} does not match current revision {record['revision']}", [str(record["revision"])])
actor = {"type": "user", "id": request.identity.subject}
store, identity = request.store, request.identity
if action == "approve":
stored = _transition(store, identity, record, "active", "approved", actor, expected=expected)
_emit(store, identity, stored, "memory.committed", "Memory approved")
elif action == "reject":
stored = _transition(store, identity, record, "rejected", "rejected", actor, expected=expected)
_emit(store, identity, stored, "memory.suppressed", "Memory rejected")
elif action == "forget":
stored = _transition(store, identity, record, "forgotten", "forgotten", actor, expected=expected)
from hux import events
events.redact_memory_references(store, stored["id"])
_emit(store, identity, stored, "memory.forgotten", "Memory forgotten")
elif action == "edit":
stored = _edit(request, record, actor, expected)
if stored["status"] == "no_store":
_emit(store, identity, stored, "memory.suppressed", f"Edit not remembered ({stored['audit'][0]['note'].split(';')[0]})")
request.audit("memory.edit", memory_id, "allow", "no_store")
return Response(202, stored)
_emit(store, identity, stored, "memory.committed" if stored["status"] == "active" else "memory.proposed", f"Memory edited, supersedes {record['id']}")
else:
retrievable = action == "restore_retrieval"
stored = _set_retrievable(store, record, retrievable, actor, expected)
_emit(store, identity, stored, "memory.retrieval_removed" if not retrievable else "memory.committed", "Retrieval removed" if not retrievable else "Retrieval restored")
request.audit(f"memory.{action}", memory_id, "allow", "" if expected is not None else "unconditional_write")
return Response(200, stored, {"ETag": str(stored["revision"])})
def export_memory(request: Request) -> Response:
"""``GET /hux/v1/memory/export``: active, retrievable entries only, each stamped ``exported`` (SO-26)."""
stones = tombstoned(request.store)
items = []
for row in list(request.store.scan(FAMILY)):
record = load(request.store, row["id"])
if record["id"] in stones or record["status"] != "active" or not record["retrievable"]:
continue
stamp = now_iso()
items.append(_persist(request.store, {**record, "audit": [*record["audit"], {"at": stamp, "action": "exported", "actor": {"type": "user", "id": request.identity.subject}}]}, record["revision"]))
request.store.append(FAMILY, "exports", {"at": now_iso(), "ids": [item["id"] for item in items]})
request.audit("memory.export", "memory")
response = page(items, None)
response.headers["Content-Disposition"] = 'attachment; filename="hux-memory-export.json"'
return response
def register(router: Router) -> None:
"""Attach HUX-02 routes."""
router.add("GET", "/hux/v1/memory/export", "HUX-02", "memory.export", export_memory)
router.add("POST", "/hux/v1/memory", "HUX-02", "memory.propose", post_memory)
router.add("GET", "/hux/v1/memory", "HUX-02", "memory.list", list_memory)
router.add("GET", "/hux/v1/memory/{id}", "HUX-02", "memory.get", get_memory)
router.add("POST", "/hux/v1/memory/{id}/{action}", "HUX-02", "memory.act", act_memory)