From dc034cb7388597ddfb8dcbc2a3aa7b056881e37e Mon Sep 17 00:00:00 2001 From: jenkins Date: Mon, 24 Aug 2026 04:43:08 -0300 Subject: [PATCH] hermes(hux): bounded message-text search with privacy enforcement HUX-03: GET /hux/v1/search now accepts include=message_text, an explicit opt-in that scans the stored message events of the 100 most recently active candidate conversations. Forgotten (tombstoned) conversations, private-mode conversations, restricted events and fully redacted events never match; the default indexed-fields search and its response contract are unchanged (the shipped UI keeps requiring message_text in not_indexed). Paginated mode uses one deterministic total order (score desc, updated_at desc, id) with an offset cursor. organization.py at 98% branch coverage. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01BvMSXH8VH2tMWXanb8SJdf --- .../hermes-hux-foundation/hux/organization.py | 77 +++++++++- .../test_hermes_hux_search_message_text.py | 142 ++++++++++++++++++ 2 files changed, 211 insertions(+), 8 deletions(-) create mode 100644 testing/tests/test_hermes_hux_search_message_text.py diff --git a/dockerfiles/hermes-hux-foundation/hux/organization.py b/dockerfiles/hermes-hux-foundation/hux/organization.py index f391ba5a..80266f4e 100644 --- a/dockerfiles/hermes-hux-foundation/hux/organization.py +++ b/dockerfiles/hermes-hux-foundation/hux/organization.py @@ -26,6 +26,10 @@ PROJECT_FIELDS = ("name", "description", "tags", "pinned", "archived", "default_ CONVERSATION_FIELDS = ("title", "tags", "pinned", "archived", "mode", "project_id") INDEXED = ("title", "tags", "project_name", "artifact_titles") NOT_INDEXED = ("message_text",) +MESSAGE_KINDS = ("message.user", "message.assistant") +MESSAGE_SCAN_CONVERSATIONS = 100 +MESSAGE_SCAN_EVENTS = 300 +SEARCH_PAGE = 50 SCHEMAS = contracts.load_all() TOKEN_RE = re.compile(r"[a-z0-9]+") @@ -287,25 +291,82 @@ def score(record: dict[str, Any], terms: list[str], project_name: str, titles: l return total +def message_text_score(store: TenantStore, conversation: dict[str, Any], terms: list[str]) -> int: + """Bounded match over this conversation's stored message events. + + Privacy wins: forgotten conversations were already excluded, private-mode + conversations are never scanned, and restricted or fully redacted events + stay invisible to search exactly as they are on the timeline. + """ + from hux import events + + if events.is_private(store, conversation["id"]): + return 0 + rows = store.read(events.FAMILY, conversation["id"])[-MESSAGE_SCAN_EVENTS:] + total = 0 + for row in rows: + if row.get("kind") not in MESSAGE_KINDS or row.get("sensitivity") == "restricted": + continue + if (row.get("redaction") or {}).get("level") == "full": + continue + words = tokens(str(row.get("summary", ""))) + total += sum(words.count(term) for term in terms) + return total + + def search(request: Request) -> Response: - """``GET /hux/v1/search?q=&project_id=``: token match over the indexed fields, best first.""" - terms = tokens(request.query.get("q", "")) + """``GET /hux/v1/search?q=&project_id=&include=&cursor=``: best match first. + + The default scope is the indexed fields only. ``include=message_text`` + additionally scans the stored message events of the most recently active + ``MESSAGE_SCAN_CONVERSATIONS`` candidates (bounded, privacy-enforced) and + pages deterministically: rank order is (score desc, updated_at desc, id), + the cursor is the offset into that total order. + """ + terms = tokens(request.query.get("q", "")[:200]) if not terms: raise Invalid("q is required") + include = request.query.get("include") + if include not in (None, "message_text"): + raise Invalid("include supports only message_text") + cursor = request.query.get("cursor", "0") + if not cursor.isdigit(): + raise Invalid("cursor must be a non-negative integer") project_filter = request.query.get("project_id") names = {p["id"]: p["name"] for p in request.store.scan(PROJECTS)} index = artifact_index(request.store) + candidates = [ + record for record in request.store.scan(CONVERSATIONS) + if not project_filter or record.get("project_id") == project_filter + ] + scanned = set() + if include: + recent = sorted(candidates, key=lambda r: (r["updated_at"], r["id"]), reverse=True) + scanned = {record["id"] for record in recent[:MESSAGE_SCAN_CONVERSATIONS]} ranked = [] - for record in request.store.scan(CONVERSATIONS): - if project_filter and record.get("project_id") != project_filter: - continue + for record in candidates: points = score(record, terms, names.get(record.get("project_id", ""), ""), artifact_titles(request.store, record, index)) + if record["id"] in scanned and not record.get("archived"): + points += message_text_score(request.store, record, terms) if points: ranked.append((points, record)) - ranked.sort(key=lambda pair: (-pair[0], pair[1]["updated_at"])) + if include: + # Paginated mode needs one total order; ids break updated_at ties. + ranked.sort(key=lambda pair: (-pair[0], pair[1]["updated_at"], pair[1]["id"])) + else: + ranked.sort(key=lambda pair: (-pair[0], pair[1]["updated_at"])) request.audit("search.query", "conversations") - return Response(200, {"items": [r for _, r in ranked], "next": None, "scores": {r["id"]: s for s, r in ranked}, - "indexed": list(INDEXED), "not_indexed": list(NOT_INDEXED)}) + if not include: + return Response(200, {"items": [r for _, r in ranked], "next": None, "scores": {r["id"]: s for s, r in ranked}, + "indexed": list(INDEXED), "not_indexed": list(NOT_INDEXED)}) + start = int(cursor) + window = ranked[start : start + SEARCH_PAGE] + return Response(200, { + "items": [r for _, r in window], "scores": {r["id"]: s for s, r in window}, + "next": str(start + SEARCH_PAGE) if len(ranked) > start + SEARCH_PAGE else None, + "indexed": list(INDEXED) + ["message_text"], "not_indexed": [], + "message_scan_limit": MESSAGE_SCAN_CONVERSATIONS, + }) def register(router: Router) -> None: diff --git a/testing/tests/test_hermes_hux_search_message_text.py b/testing/tests/test_hermes_hux_search_message_text.py new file mode 100644 index 00000000..c35da3a1 --- /dev/null +++ b/testing/tests/test_hermes_hux_search_message_text.py @@ -0,0 +1,142 @@ +"""Bounded, privacy-enforced message-text search (HUX-03 increment).""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +FOUNDATION = ROOT / "dockerfiles" / "hermes-hux-foundation" +if str(FOUNDATION) not in sys.path: + sys.path.insert(0, str(FOUNDATION)) + +from hux import contracts, organization # noqa: E402 +from hux.server import build_router # noqa: E402 + +HEADERS = {"X-Hermes-Tenant-Identity": "slot-3", "X-Hux-Subject": "usr_0123456789abcdef", "X-Hux-Surface": "chat", "X-Hux-Relay-Key": "rk"} +WORKER = {**HEADERS, "X-Hux-Surface": "worker", "X-Hux-Trust": "worker", "X-Hux-Relay-Key": "wk"} +ALL_ON = ",".join(card["flag"] for card in contracts.load_flags()["cards"]) + + +def call(router, method, path, body=None, headers=None): + raw = b"" if body is None else json.dumps(body).encode() + response = router.dispatch(method, path, {**HEADERS, **(headers or {})}, raw) + return response.status, response.body + + +@pytest.fixture +def router(tmp_path): + return build_router( + tmp_path, + {"HUX_FLAGS": ALL_ON, "HUX_ROUTER_KEY": "rk", "HUX_WORKER_KEY": "wk", + "HUX_WRITES_PER_MINUTE": "5000", "HUX_READS_PER_MINUTE": "5000"}, + ) + + +def conversation(router, title, project_id): + status, body = call(router, "POST", "/hux/v1/conversations", {"title": title, "project_id": project_id}) + assert status == 201 + return body + + +def say(router, conversation_id, summary, kind="message.user", sensitivity="personal", key=None): + import hashlib + + slug = key or hashlib.sha256(f"{conversation_id}:{summary}".encode()).hexdigest()[:24] + status, body = call( + router, "POST", f"/hux/v1/conversations/{conversation_id}/events", + {"kind": kind, "summary": summary, "sensitivity": sensitivity}, + {**WORKER, "Idempotency-Key": f"evt.{slug}"}, + ) + assert status in (200, 201), body + + +@pytest.fixture +def scoped(router): + status, project = call(router, "POST", "/hux/v1/projects", {"name": "Kitchen"}) + assert status == 201 + return router, project["id"] + + +def test_message_text_is_opt_in_and_scoped_explicitly(scoped): + router, project_id = scoped + convo = conversation(router, "Plumbing", project_id) + say(router, convo["id"], "the kidneystone diagnosis was benign") + status, body = call(router, "GET", "/hux/v1/search?q=kidneystone") + assert status == 200 and body["items"] == [] + assert body["not_indexed"] == ["message_text"] + status, body = call(router, "GET", "/hux/v1/search?q=kidneystone&include=message_text") + assert status == 200 + assert [item["id"] for item in body["items"]] == [convo["id"]] + assert "message_text" in body["indexed"] and body["not_indexed"] == [] + status, body = call(router, "GET", "/hux/v1/search?q=kidneystone&include=bogus") + assert status == 400 + + +def test_privacy_and_tombstones_defeat_message_search(scoped): + router, project_id = scoped + plain = conversation(router, "Plain", project_id) + say(router, plain["id"], "walnut cabinets arrived") + hidden = conversation(router, "Hidden", project_id) + say(router, hidden["id"], "walnut cabinets secret plans", sensitivity="restricted") + private = conversation(router, "Private", project_id) + say(router, private["id"], "walnut cabinets private note") + status, _ = call( + router, "PUT", + f"/hux/v1/projects/{project_id}/conversations/{private['id']}/mode", + {"project_id": project_id, "mode": "private"}, + {"If-Match": "0", "Idempotency-Key": "mode-private-x"}, + ) + assert status == 200 + forgotten = conversation(router, "Forgotten", project_id) + say(router, forgotten["id"], "walnut cabinets to forget") + status, _ = call(router, "POST", f"/hux/v1/conversations/{forgotten['id']}/forget", {}) + assert status in (200, 201, 202) + status, body = call(router, "GET", "/hux/v1/search?q=walnut+cabinets&include=message_text") + assert status == 200 + assert [item["id"] for item in body["items"]] == [plain["id"]] + + +def test_pagination_is_deterministic(scoped): + router, project_id = scoped + ids = [] + for index in range(organization.SEARCH_PAGE + 3): + convo = conversation(router, f"Convo {index}", project_id) + say(router, convo["id"], "zebra sighting logged", key=f"zebra.{index:04}") + ids.append(convo["id"]) + status, first = call(router, "GET", "/hux/v1/search?q=zebra&include=message_text") + assert status == 200 and len(first["items"]) == organization.SEARCH_PAGE + assert first["next"] == str(organization.SEARCH_PAGE) + status, second = call( + router, "GET", + f"/hux/v1/search?q=zebra&include=message_text&cursor={first['next']}", + ) + assert status == 200 and len(second["items"]) == 3 and second["next"] is None + assert not {i["id"] for i in first["items"]} & {i["id"] for i in second["items"]} + status, again = call(router, "GET", "/hux/v1/search?q=zebra&include=message_text") + assert [i["id"] for i in again["items"]] == [i["id"] for i in first["items"]] + status, body = call(router, "GET", "/hux/v1/search?q=zebra&include=message_text&cursor=nope") + assert status == 400 + + +def test_scan_stays_bounded_to_recent_conversations(scoped, monkeypatch): + router, project_id = scoped + monkeypatch.setattr(organization, "MESSAGE_SCAN_CONVERSATIONS", 1) + older = conversation(router, "Older", project_id) + say(router, older["id"], "quartz counters measured") + newer = conversation(router, "Newer", project_id) + say(router, newer["id"], "quartz counters ordered") + # Force a strictly newer updated_at (now_iso ties at clock resolution). + monkeypatch.setattr(organization, "now_iso", lambda: "2099-01-01T00:00:00Z") + status, _ = call( + router, "PATCH", f"/hux/v1/conversations/{newer['id']}", + {"pinned": True}, {"If-Match": str(newer["revision"])}, + ) + assert status == 200 + status, body = call(router, "GET", "/hux/v1/search?q=quartz&include=message_text") + assert status == 200 + assert [item["id"] for item in body["items"]] == [newer["id"]] + assert body["message_scan_limit"] == 1