dc034cb738
HUX-03: GET /hux/v1/search now accepts include=message_text, an explicit opt-in that scans the stored message events of the 100 most recently active candidate conversations. Forgotten (tombstoned) conversations, private-mode conversations, restricted events and fully redacted events never match; the default indexed-fields search and its response contract are unchanged (the shipped UI keeps requiring message_text in not_indexed). Paginated mode uses one deterministic total order (score desc, updated_at desc, id) with an offset cursor. organization.py at 98% branch coverage. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01BvMSXH8VH2tMWXanb8SJdf
385 lines
18 KiB
Python
385 lines
18 KiB
Python
"""HUX-03 organisation: projects, conversations, branch lineage and search.
|
|
|
|
Projects own conversations; conversations carry tags, pins, a mode and an
|
|
optional branch pointer to the conversation they forked from. Search covers
|
|
only the ``project.schema.json#/$defs/search_index`` fields this increment
|
|
indexes (title, tags, project_name, and artifact titles when the artifacts
|
|
lane is present); message text is not indexed here and the response says so.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from typing import Any
|
|
|
|
from hux import contracts, redaction
|
|
from hux.errors import Conflict, Invalid, NotFound
|
|
from hux.http import Request, Response, Router, page
|
|
from hux.store import TenantStore, check_id, new_id, now_iso
|
|
|
|
CARD = "HUX-03"
|
|
PROJECTS = "projects"
|
|
CONVERSATIONS = "conversations"
|
|
MAX_PROJECTS = 200
|
|
MAX_CONVERSATIONS = 2000
|
|
PROJECT_FIELDS = ("name", "description", "tags", "pinned", "archived", "default_mode")
|
|
CONVERSATION_FIELDS = ("title", "tags", "pinned", "archived", "mode", "project_id")
|
|
INDEXED = ("title", "tags", "project_name", "artifact_titles")
|
|
NOT_INDEXED = ("message_text",)
|
|
MESSAGE_KINDS = ("message.user", "message.assistant")
|
|
MESSAGE_SCAN_CONVERSATIONS = 100
|
|
MESSAGE_SCAN_EVENTS = 300
|
|
SEARCH_PAGE = 50
|
|
SCHEMAS = contracts.load_all()
|
|
TOKEN_RE = re.compile(r"[a-z0-9]+")
|
|
|
|
|
|
def project_exists(store: TenantStore, project_id: str) -> bool:
|
|
"""True when this tenant owns a project with that id (helper for other lanes)."""
|
|
return isinstance(project_id, str) and bool(re.match(r"^[a-z]{2,6}_[A-Za-z0-9._-]{4,80}$", project_id)) and store.exists(PROJECTS, project_id)
|
|
|
|
|
|
def conversation_exists(store: TenantStore, conversation_id: str) -> bool:
|
|
"""True when this tenant owns a conversation with that id (helper for other lanes)."""
|
|
return isinstance(conversation_id, str) and bool(re.match(r"^[a-z]{2,6}_[A-Za-z0-9._-]{4,80}$", conversation_id)) and store.exists(CONVERSATIONS, conversation_id)
|
|
|
|
|
|
def project_of(store: TenantStore, conversation_id: str | None) -> str | None:
|
|
"""The project a conversation belongs to, or None when unknown or unfiled."""
|
|
if not conversation_id or not conversation_exists(store, conversation_id):
|
|
return None
|
|
return store.get(CONVERSATIONS, conversation_id).get("project_id")
|
|
|
|
|
|
def checked(record: dict[str, Any]) -> dict[str, Any]:
|
|
"""Raise Invalid unless ``record`` satisfies its contract."""
|
|
problems = contracts.validate_record(record, SCHEMAS)
|
|
if problems:
|
|
raise Invalid("record fails contract", problems)
|
|
return record
|
|
|
|
|
|
def body_dict(request: Request) -> dict[str, Any]:
|
|
"""The JSON object body or Invalid."""
|
|
if not isinstance(request.body, dict):
|
|
raise Invalid("body must be a JSON object")
|
|
return request.body
|
|
|
|
|
|
def pick(body: dict[str, Any], fields: tuple[str, ...]) -> dict[str, Any]:
|
|
"""Only the client-settable fields, secret-scrubbed (F9); ids, owner, timestamps and revision are server-set."""
|
|
return redaction.scrub_value({k: body[k] for k in fields if k in body}, [])
|
|
|
|
|
|
def replay(store: TenantStore, family: str, key: str) -> dict[str, Any] | None:
|
|
"""The record an Idempotency-Key already created in this family, if any."""
|
|
for row in store.read(family, "idempotency"):
|
|
if row["key"] == key:
|
|
return store.get(family, row["id"])
|
|
return None
|
|
|
|
|
|
def create(request: Request, family: str, record: dict[str, Any], key: str, cap: int) -> tuple[dict[str, Any], int]:
|
|
"""Create under the family lock honouring Idempotency-Key and the family count cap."""
|
|
with request.store.lock(family):
|
|
if key:
|
|
existing = replay(request.store, family, key)
|
|
if existing is not None:
|
|
return existing, 200
|
|
if request.store.count(family) >= cap:
|
|
raise Conflict(f"{family} cap of {cap} reached")
|
|
stored = request.store.put(family, checked({**record, "revision": 1}))
|
|
if key:
|
|
request.store.append(family, "idempotency", {"key": key, "id": stored["id"]})
|
|
return stored, 201
|
|
|
|
|
|
def update(request: Request, family: str, fields: tuple[str, ...]) -> dict[str, Any]:
|
|
"""PATCH under If-Match; a missing If-Match is accepted but audited as unconditional (SO-44)."""
|
|
changes = pick(body_dict(request), fields)
|
|
expected = request.if_match()
|
|
with request.store.lock(family):
|
|
current = request.store.get(family, check_id(request.params["id"]))
|
|
if "project_id" in changes and changes["project_id"] is not None and not project_exists(request.store, changes["project_id"]):
|
|
raise NotFound("project not found")
|
|
record = {**current, **{k: v for k, v in changes.items() if v is not None}, "updated_at": now_iso()}
|
|
if changes.get("project_id", "") is None:
|
|
record.pop("project_id", None)
|
|
stored = request.store.put(family, checked(record), expected_revision=expected)
|
|
request.audit(f"{family}.update", stored["id"], reason="" if expected is not None else "unconditional_write")
|
|
return stored
|
|
|
|
|
|
def etag(record: dict[str, Any]) -> dict[str, str]:
|
|
"""Revision as ETag so clients can send it back in If-Match."""
|
|
return {"ETag": str(record["revision"])}
|
|
|
|
|
|
# -- projects ------------------------------------------------------------------
|
|
|
|
def create_project(request: Request) -> Response:
|
|
"""``POST /hux/v1/projects``."""
|
|
body = pick(body_dict(request), PROJECT_FIELDS)
|
|
stamp = now_iso()
|
|
record = {
|
|
"schema": "hux.project.v1", "id": new_id("prj"), "owner": request.identity.subject,
|
|
"tags": [], "pinned": False, "archived": False, **body, "created_at": stamp, "updated_at": stamp,
|
|
}
|
|
stored, status = create(request, PROJECTS, record, request.idempotency_key(), MAX_PROJECTS)
|
|
request.audit("projects.create", stored["id"], reason="replayed" if status == 200 else "")
|
|
return Response(status, stored, etag(stored))
|
|
|
|
|
|
def list_projects(request: Request) -> Response:
|
|
"""``GET /hux/v1/projects?archived=``: pinned first, then most recently updated."""
|
|
archived = request.query.get("archived")
|
|
items = [p for p in request.store.scan(PROJECTS) if archived is None or p["archived"] == (archived == "true")]
|
|
items.sort(key=lambda p: p["updated_at"], reverse=True)
|
|
items.sort(key=lambda p: not p["pinned"])
|
|
request.audit("projects.list", "projects")
|
|
return page(items)
|
|
|
|
|
|
def get_project(request: Request) -> Response:
|
|
"""``GET /hux/v1/projects/{id}``."""
|
|
record = request.store.get(PROJECTS, check_id(request.params["id"]))
|
|
request.audit("projects.read", record["id"])
|
|
return Response(200, record, etag(record))
|
|
|
|
|
|
def patch_project(request: Request) -> Response:
|
|
"""``PATCH /hux/v1/projects/{id}`` with If-Match."""
|
|
stored = update(request, PROJECTS, PROJECT_FIELDS)
|
|
return Response(200, stored, etag(stored))
|
|
|
|
|
|
# -- conversations -------------------------------------------------------------
|
|
|
|
def new_conversation(request: Request, body: dict[str, Any], extra: dict[str, Any] | None = None) -> Response:
|
|
"""Build, validate and store a conversation from client fields plus server-set extras."""
|
|
if body.get("project_id") is not None and not project_exists(request.store, body["project_id"]):
|
|
raise NotFound("project not found")
|
|
stamp = now_iso()
|
|
record = {
|
|
"schema": "hux.conversation.v1", "id": new_id("conv"), "owner": request.identity.subject,
|
|
"tags": [], "pinned": False, "archived": False, **{k: v for k, v in body.items() if v is not None},
|
|
**(extra or {}), "artifact_ids": [], "created_at": stamp, "updated_at": stamp,
|
|
}
|
|
stored, status = create(request, CONVERSATIONS, record, request.idempotency_key(), MAX_CONVERSATIONS)
|
|
request.audit("conversations.create", stored["id"], reason="replayed" if status == 200 else "")
|
|
return Response(status, stored, etag(stored))
|
|
|
|
|
|
def create_conversation(request: Request) -> Response:
|
|
"""``POST /hux/v1/conversations``."""
|
|
return new_conversation(request, pick(body_dict(request), CONVERSATION_FIELDS))
|
|
|
|
|
|
def list_conversations(request: Request) -> Response:
|
|
"""``GET /hux/v1/conversations?project_id=&tag=&pinned=&archived=``: newest activity first."""
|
|
q = request.query
|
|
flags = {k: q[k] == "true" for k in ("pinned", "archived") if k in q}
|
|
items = []
|
|
for index, record in enumerate(request.store.scan(CONVERSATIONS)):
|
|
if "project_id" in q and record.get("project_id") != q["project_id"]:
|
|
continue
|
|
if "tag" in q and q["tag"] not in record["tags"]:
|
|
continue
|
|
if any(record[k] != v for k, v in flags.items()):
|
|
continue
|
|
items.append((record.get("last_message_at", record["updated_at"]), index, record))
|
|
items.sort(key=lambda item: item[:2], reverse=True)
|
|
request.audit("conversations.list", "conversations")
|
|
return page([record for _, _, record in items])
|
|
|
|
|
|
def get_conversation(request: Request) -> Response:
|
|
"""``GET /hux/v1/conversations/{id}``."""
|
|
record = request.store.get(CONVERSATIONS, check_id(request.params["id"]))
|
|
request.audit("conversations.read", record["id"])
|
|
return Response(200, record, etag(record))
|
|
|
|
|
|
def patch_conversation(request: Request) -> Response:
|
|
"""``PATCH /hux/v1/conversations/{id}`` with If-Match."""
|
|
stored = update(request, CONVERSATIONS, CONVERSATION_FIELDS)
|
|
return Response(200, stored, etag(stored))
|
|
|
|
|
|
def branch_conversation(request: Request) -> Response:
|
|
"""``POST /hux/v1/conversations/{id}/branch``: fork at a message, keeping project, tags and mode."""
|
|
body = body_dict(request)
|
|
point = body.get("branch_point_message_id")
|
|
if not isinstance(point, str) or not 1 <= len(point) <= 120:
|
|
raise Invalid("branch_point_message_id required")
|
|
parent = request.store.get(CONVERSATIONS, check_id(request.params["id"]))
|
|
fields = {"title": body.get("title") or f"{parent['title']} (branch)"[:200], "tags": list(parent["tags"]),
|
|
"project_id": parent.get("project_id"), "mode": parent.get("mode")}
|
|
return new_conversation(request, fields, {"branch": {"parent_conversation_id": parent["id"], "branch_point_message_id": point}})
|
|
|
|
|
|
def lineage(request: Request) -> Response:
|
|
"""``GET /hux/v1/conversations/{id}/lineage``: ancestors root-first plus direct children."""
|
|
record = request.store.get(CONVERSATIONS, check_id(request.params["id"]))
|
|
ancestors: list[dict[str, Any]] = []
|
|
cursor, seen = record, {record["id"]}
|
|
while "branch" in cursor and len(ancestors) < 64:
|
|
parent_id = cursor["branch"]["parent_conversation_id"]
|
|
if parent_id in seen or not request.store.exists(CONVERSATIONS, parent_id):
|
|
break
|
|
cursor = request.store.get(CONVERSATIONS, parent_id)
|
|
seen.add(parent_id)
|
|
ancestors.insert(0, cursor)
|
|
children = [c for c in request.store.scan(CONVERSATIONS) if c.get("branch", {}).get("parent_conversation_id") == record["id"]]
|
|
request.audit("conversations.lineage", record["id"])
|
|
return Response(200, {"conversation": record, "ancestors": ancestors, "children": children})
|
|
|
|
|
|
# -- search --------------------------------------------------------------------
|
|
|
|
def artifact_index(store: TenantStore) -> dict[str, list[str]]:
|
|
"""Conversation id -> titles of the artifacts filed under it (F13c): both the conversation's ``artifact_ids`` and artifacts that name the conversation."""
|
|
try:
|
|
from hux import artifacts
|
|
except ModuleNotFoundError:
|
|
return {}
|
|
index: dict[str, list[str]] = {}
|
|
for artifact in store.scan(artifacts.FAMILY):
|
|
if isinstance(artifact.get("conversation_id"), str) and isinstance(artifact.get("title"), str):
|
|
index.setdefault(artifact["conversation_id"], []).append(artifact["title"])
|
|
for conversation in store.scan(CONVERSATIONS):
|
|
ids = [i for i in conversation.get("artifact_ids", []) if isinstance(i, str)]
|
|
titles = artifacts.artifact_titles(store, ids).values() if ids else []
|
|
for title in titles:
|
|
index.setdefault(conversation["id"], []).append(title)
|
|
return index
|
|
|
|
|
|
def artifact_titles(store: TenantStore, conversation: dict[str, Any], index: dict[str, list[str]] | None = None) -> list[str]:
|
|
"""Artifact titles for a conversation via the artifacts lane, or nothing when it is absent."""
|
|
index = artifact_index(store) if index is None else index
|
|
return list(dict.fromkeys(index.get(conversation["id"], [])))
|
|
|
|
|
|
def mark_forgotten(store: TenantStore, conversation_id: str) -> bool:
|
|
"""Blank a forgotten conversation's title and tags and archive it (F9); False when there is no document."""
|
|
with store.lock(CONVERSATIONS):
|
|
if not conversation_exists(store, conversation_id):
|
|
return False
|
|
current = store.get(CONVERSATIONS, conversation_id)
|
|
record = {**current, "title": "[forgotten]", "tags": [], "archived": True, "updated_at": now_iso()}
|
|
store.put(CONVERSATIONS, checked(record), expected_revision=current["revision"])
|
|
return True
|
|
|
|
|
|
def tokens(text: str) -> list[str]:
|
|
"""Lowercased alphanumeric terms."""
|
|
return TOKEN_RE.findall(text.lower())
|
|
|
|
|
|
def score(record: dict[str, Any], terms: list[str], project_name: str, titles: list[str]) -> int:
|
|
"""Title hit beats tag hit beats project/artifact hit; every term must match somewhere."""
|
|
fields = {"title": tokens(record["title"]), "tags": [t for tag in record["tags"] for t in tokens(tag)],
|
|
"project_name": tokens(project_name), "artifact_titles": [t for title in titles for t in tokens(title)]}
|
|
weight = {"title": 4, "tags": 3, "project_name": 2, "artifact_titles": 1}
|
|
total = 0
|
|
for term in terms:
|
|
hit = sum(weight[f] * words.count(term) for f, words in fields.items())
|
|
if not hit:
|
|
return 0
|
|
total += hit
|
|
return total
|
|
|
|
|
|
def message_text_score(store: TenantStore, conversation: dict[str, Any], terms: list[str]) -> int:
|
|
"""Bounded match over this conversation's stored message events.
|
|
|
|
Privacy wins: forgotten conversations were already excluded, private-mode
|
|
conversations are never scanned, and restricted or fully redacted events
|
|
stay invisible to search exactly as they are on the timeline.
|
|
"""
|
|
from hux import events
|
|
|
|
if events.is_private(store, conversation["id"]):
|
|
return 0
|
|
rows = store.read(events.FAMILY, conversation["id"])[-MESSAGE_SCAN_EVENTS:]
|
|
total = 0
|
|
for row in rows:
|
|
if row.get("kind") not in MESSAGE_KINDS or row.get("sensitivity") == "restricted":
|
|
continue
|
|
if (row.get("redaction") or {}).get("level") == "full":
|
|
continue
|
|
words = tokens(str(row.get("summary", "")))
|
|
total += sum(words.count(term) for term in terms)
|
|
return total
|
|
|
|
|
|
def search(request: Request) -> Response:
|
|
"""``GET /hux/v1/search?q=&project_id=&include=&cursor=``: best match first.
|
|
|
|
The default scope is the indexed fields only. ``include=message_text``
|
|
additionally scans the stored message events of the most recently active
|
|
``MESSAGE_SCAN_CONVERSATIONS`` candidates (bounded, privacy-enforced) and
|
|
pages deterministically: rank order is (score desc, updated_at desc, id),
|
|
the cursor is the offset into that total order.
|
|
"""
|
|
terms = tokens(request.query.get("q", "")[:200])
|
|
if not terms:
|
|
raise Invalid("q is required")
|
|
include = request.query.get("include")
|
|
if include not in (None, "message_text"):
|
|
raise Invalid("include supports only message_text")
|
|
cursor = request.query.get("cursor", "0")
|
|
if not cursor.isdigit():
|
|
raise Invalid("cursor must be a non-negative integer")
|
|
project_filter = request.query.get("project_id")
|
|
names = {p["id"]: p["name"] for p in request.store.scan(PROJECTS)}
|
|
index = artifact_index(request.store)
|
|
candidates = [
|
|
record for record in request.store.scan(CONVERSATIONS)
|
|
if not project_filter or record.get("project_id") == project_filter
|
|
]
|
|
scanned = set()
|
|
if include:
|
|
recent = sorted(candidates, key=lambda r: (r["updated_at"], r["id"]), reverse=True)
|
|
scanned = {record["id"] for record in recent[:MESSAGE_SCAN_CONVERSATIONS]}
|
|
ranked = []
|
|
for record in candidates:
|
|
points = score(record, terms, names.get(record.get("project_id", ""), ""), artifact_titles(request.store, record, index))
|
|
if record["id"] in scanned and not record.get("archived"):
|
|
points += message_text_score(request.store, record, terms)
|
|
if points:
|
|
ranked.append((points, record))
|
|
if include:
|
|
# Paginated mode needs one total order; ids break updated_at ties.
|
|
ranked.sort(key=lambda pair: (-pair[0], pair[1]["updated_at"], pair[1]["id"]))
|
|
else:
|
|
ranked.sort(key=lambda pair: (-pair[0], pair[1]["updated_at"]))
|
|
request.audit("search.query", "conversations")
|
|
if not include:
|
|
return Response(200, {"items": [r for _, r in ranked], "next": None, "scores": {r["id"]: s for s, r in ranked},
|
|
"indexed": list(INDEXED), "not_indexed": list(NOT_INDEXED)})
|
|
start = int(cursor)
|
|
window = ranked[start : start + SEARCH_PAGE]
|
|
return Response(200, {
|
|
"items": [r for _, r in window], "scores": {r["id"]: s for s, r in window},
|
|
"next": str(start + SEARCH_PAGE) if len(ranked) > start + SEARCH_PAGE else None,
|
|
"indexed": list(INDEXED) + ["message_text"], "not_indexed": [],
|
|
"message_scan_limit": MESSAGE_SCAN_CONVERSATIONS,
|
|
})
|
|
|
|
|
|
def register(router: Router) -> None:
|
|
"""Attach HUX-03 routes."""
|
|
router.add("POST", "/hux/v1/projects", CARD, "projects.create", create_project)
|
|
router.add("GET", "/hux/v1/projects", CARD, "projects.list", list_projects)
|
|
router.add("GET", "/hux/v1/projects/{id}", CARD, "projects.read", get_project)
|
|
router.add("PATCH", "/hux/v1/projects/{id}", CARD, "projects.update", patch_project)
|
|
router.add("POST", "/hux/v1/conversations", CARD, "conversations.create", create_conversation)
|
|
router.add("GET", "/hux/v1/conversations", CARD, "conversations.list", list_conversations)
|
|
router.add("GET", "/hux/v1/conversations/{id}", CARD, "conversations.read", get_conversation)
|
|
router.add("PATCH", "/hux/v1/conversations/{id}", CARD, "conversations.update", patch_conversation)
|
|
router.add("POST", "/hux/v1/conversations/{id}/branch", CARD, "conversations.branch", branch_conversation)
|
|
router.add("GET", "/hux/v1/conversations/{id}/lineage", CARD, "conversations.lineage", lineage)
|
|
router.add("GET", "/hux/v1/search", CARD, "search.query", search)
|