hermes(hux): bounded message-text search with privacy enforcement

HUX-03: GET /hux/v1/search now accepts include=message_text, an
explicit opt-in that scans the stored message events of the 100 most
recently active candidate conversations. Forgotten (tombstoned)
conversations, private-mode conversations, restricted events and fully
redacted events never match; the default indexed-fields search and its
response contract are unchanged (the shipped UI keeps requiring
message_text in not_indexed). Paginated mode uses one deterministic
total order (score desc, updated_at desc, id) with an offset cursor.
organization.py at 98% branch coverage.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BvMSXH8VH2tMWXanb8SJdf

dockerfiles/hermes-hux-foundation/hux/organization.py

@@ -26,6 +26,10 @@ PROJECT_FIELDS = ("name", "description", "tags", "pinned", "archived", "default_
 CONVERSATION_FIELDS = ("title", "tags", "pinned", "archived", "mode", "project_id")
 INDEXED = ("title", "tags", "project_name", "artifact_titles")
 NOT_INDEXED = ("message_text",)
+MESSAGE_KINDS = ("message.user", "message.assistant")
+MESSAGE_SCAN_CONVERSATIONS = 100
+MESSAGE_SCAN_EVENTS = 300
+SEARCH_PAGE = 50
 SCHEMAS = contracts.load_all()
 TOKEN_RE = re.compile(r"[a-z0-9]+")
 
@@ -287,25 +291,82 @@ def score(record: dict[str, Any], terms: list[str], project_name: str, titles: l
     return total
 
 
+def message_text_score(store: TenantStore, conversation: dict[str, Any], terms: list[str]) -> int:
+    """Bounded match over this conversation's stored message events.
+
+    Privacy wins: forgotten conversations were already excluded, private-mode
+    conversations are never scanned, and restricted or fully redacted events
+    stay invisible to search exactly as they are on the timeline.
+    """
+    from hux import events
+
+    if events.is_private(store, conversation["id"]):
+        return 0
+    rows = store.read(events.FAMILY, conversation["id"])[-MESSAGE_SCAN_EVENTS:]
+    total = 0
+    for row in rows:
+        if row.get("kind") not in MESSAGE_KINDS or row.get("sensitivity") == "restricted":
+            continue
+        if (row.get("redaction") or {}).get("level") == "full":
+            continue
+        words = tokens(str(row.get("summary", "")))
+        total += sum(words.count(term) for term in terms)
+    return total
+
+
 def search(request: Request) -> Response:
-    """``GET /hux/v1/search?q=&project_id=``: token match over the indexed fields, best first."""
+    """``GET /hux/v1/search?q=&project_id=&include=&cursor=``: best match first.
-    terms = tokens(request.query.get("q", ""))
+
+    The default scope is the indexed fields only. ``include=message_text``
+    additionally scans the stored message events of the most recently active
+    ``MESSAGE_SCAN_CONVERSATIONS`` candidates (bounded, privacy-enforced) and
+    pages deterministically: rank order is (score desc, updated_at desc, id),
+    the cursor is the offset into that total order.
+    """
+    terms = tokens(request.query.get("q", "")[:200])
     if not terms:
         raise Invalid("q is required")
+    include = request.query.get("include")
+    if include not in (None, "message_text"):
+        raise Invalid("include supports only message_text")
+    cursor = request.query.get("cursor", "0")
+    if not cursor.isdigit():
+        raise Invalid("cursor must be a non-negative integer")
     project_filter = request.query.get("project_id")
     names = {p["id"]: p["name"] for p in request.store.scan(PROJECTS)}
     index = artifact_index(request.store)
+    candidates = [
+        record for record in request.store.scan(CONVERSATIONS)
+        if not project_filter or record.get("project_id") == project_filter
+    ]
+    scanned = set()
+    if include:
+        recent = sorted(candidates, key=lambda r: (r["updated_at"], r["id"]), reverse=True)
+        scanned = {record["id"] for record in recent[:MESSAGE_SCAN_CONVERSATIONS]}
     ranked = []
-    for record in request.store.scan(CONVERSATIONS):
+    for record in candidates:
-        if project_filter and record.get("project_id") != project_filter:
-            continue
         points = score(record, terms, names.get(record.get("project_id", ""), ""), artifact_titles(request.store, record, index))
+        if record["id"] in scanned and not record.get("archived"):
+            points += message_text_score(request.store, record, terms)
         if points:
             ranked.append((points, record))
-    ranked.sort(key=lambda pair: (-pair[0], pair[1]["updated_at"]))
+    if include:
+        # Paginated mode needs one total order; ids break updated_at ties.
+        ranked.sort(key=lambda pair: (-pair[0], pair[1]["updated_at"], pair[1]["id"]))
+    else:
+        ranked.sort(key=lambda pair: (-pair[0], pair[1]["updated_at"]))
     request.audit("search.query", "conversations")
-    return Response(200, {"items": [r for _, r in ranked], "next": None, "scores": {r["id"]: s for s, r in ranked},
+    if not include:
-                          "indexed": list(INDEXED), "not_indexed": list(NOT_INDEXED)})
+        return Response(200, {"items": [r for _, r in ranked], "next": None, "scores": {r["id"]: s for s, r in ranked},
+                              "indexed": list(INDEXED), "not_indexed": list(NOT_INDEXED)})
+    start = int(cursor)
+    window = ranked[start : start + SEARCH_PAGE]
+    return Response(200, {
+        "items": [r for _, r in window], "scores": {r["id"]: s for s, r in window},
+        "next": str(start + SEARCH_PAGE) if len(ranked) > start + SEARCH_PAGE else None,
+        "indexed": list(INDEXED) + ["message_text"], "not_indexed": [],
+        "message_scan_limit": MESSAGE_SCAN_CONVERSATIONS,
+    })
 
 
 def register(router: Router) -> None:

testing/tests/test_hermes_hux_search_message_text.py

@@ -0,0 +1,142 @@
+"""Bounded, privacy-enforced message-text search (HUX-03 increment)."""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[2]
+FOUNDATION = ROOT / "dockerfiles" / "hermes-hux-foundation"
+if str(FOUNDATION) not in sys.path:
+    sys.path.insert(0, str(FOUNDATION))
+
+from hux import contracts, organization  # noqa: E402
+from hux.server import build_router  # noqa: E402
+
+HEADERS = {"X-Hermes-Tenant-Identity": "slot-3", "X-Hux-Subject": "usr_0123456789abcdef", "X-Hux-Surface": "chat", "X-Hux-Relay-Key": "rk"}
+WORKER = {**HEADERS, "X-Hux-Surface": "worker", "X-Hux-Trust": "worker", "X-Hux-Relay-Key": "wk"}
+ALL_ON = ",".join(card["flag"] for card in contracts.load_flags()["cards"])
+
+
+def call(router, method, path, body=None, headers=None):
+    raw = b"" if body is None else json.dumps(body).encode()
+    response = router.dispatch(method, path, {**HEADERS, **(headers or {})}, raw)
+    return response.status, response.body
+
+
+@pytest.fixture
+def router(tmp_path):
+    return build_router(
+        tmp_path,
+        {"HUX_FLAGS": ALL_ON, "HUX_ROUTER_KEY": "rk", "HUX_WORKER_KEY": "wk",
+         "HUX_WRITES_PER_MINUTE": "5000", "HUX_READS_PER_MINUTE": "5000"},
+    )
+
+
+def conversation(router, title, project_id):
+    status, body = call(router, "POST", "/hux/v1/conversations", {"title": title, "project_id": project_id})
+    assert status == 201
+    return body
+
+
+def say(router, conversation_id, summary, kind="message.user", sensitivity="personal", key=None):
+    import hashlib
+
+    slug = key or hashlib.sha256(f"{conversation_id}:{summary}".encode()).hexdigest()[:24]
+    status, body = call(
+        router, "POST", f"/hux/v1/conversations/{conversation_id}/events",
+        {"kind": kind, "summary": summary, "sensitivity": sensitivity},
+        {**WORKER, "Idempotency-Key": f"evt.{slug}"},
+    )
+    assert status in (200, 201), body
+
+
+@pytest.fixture
+def scoped(router):
+    status, project = call(router, "POST", "/hux/v1/projects", {"name": "Kitchen"})
+    assert status == 201
+    return router, project["id"]
+
+
+def test_message_text_is_opt_in_and_scoped_explicitly(scoped):
+    router, project_id = scoped
+    convo = conversation(router, "Plumbing", project_id)
+    say(router, convo["id"], "the kidneystone diagnosis was benign")
+    status, body = call(router, "GET", "/hux/v1/search?q=kidneystone")
+    assert status == 200 and body["items"] == []
+    assert body["not_indexed"] == ["message_text"]
+    status, body = call(router, "GET", "/hux/v1/search?q=kidneystone&include=message_text")
+    assert status == 200
+    assert [item["id"] for item in body["items"]] == [convo["id"]]
+    assert "message_text" in body["indexed"] and body["not_indexed"] == []
+    status, body = call(router, "GET", "/hux/v1/search?q=kidneystone&include=bogus")
+    assert status == 400
+
+
+def test_privacy_and_tombstones_defeat_message_search(scoped):
+    router, project_id = scoped
+    plain = conversation(router, "Plain", project_id)
+    say(router, plain["id"], "walnut cabinets arrived")
+    hidden = conversation(router, "Hidden", project_id)
+    say(router, hidden["id"], "walnut cabinets secret plans", sensitivity="restricted")
+    private = conversation(router, "Private", project_id)
+    say(router, private["id"], "walnut cabinets private note")
+    status, _ = call(
+        router, "PUT",
+        f"/hux/v1/projects/{project_id}/conversations/{private['id']}/mode",
+        {"project_id": project_id, "mode": "private"},
+        {"If-Match": "0", "Idempotency-Key": "mode-private-x"},
+    )
+    assert status == 200
+    forgotten = conversation(router, "Forgotten", project_id)
+    say(router, forgotten["id"], "walnut cabinets to forget")
+    status, _ = call(router, "POST", f"/hux/v1/conversations/{forgotten['id']}/forget", {})
+    assert status in (200, 201, 202)
+    status, body = call(router, "GET", "/hux/v1/search?q=walnut+cabinets&include=message_text")
+    assert status == 200
+    assert [item["id"] for item in body["items"]] == [plain["id"]]
+
+
+def test_pagination_is_deterministic(scoped):
+    router, project_id = scoped
+    ids = []
+    for index in range(organization.SEARCH_PAGE + 3):
+        convo = conversation(router, f"Convo {index}", project_id)
+        say(router, convo["id"], "zebra sighting logged", key=f"zebra.{index:04}")
+        ids.append(convo["id"])
+    status, first = call(router, "GET", "/hux/v1/search?q=zebra&include=message_text")
+    assert status == 200 and len(first["items"]) == organization.SEARCH_PAGE
+    assert first["next"] == str(organization.SEARCH_PAGE)
+    status, second = call(
+        router, "GET",
+        f"/hux/v1/search?q=zebra&include=message_text&cursor={first['next']}",
+    )
+    assert status == 200 and len(second["items"]) == 3 and second["next"] is None
+    assert not {i["id"] for i in first["items"]} & {i["id"] for i in second["items"]}
+    status, again = call(router, "GET", "/hux/v1/search?q=zebra&include=message_text")
+    assert [i["id"] for i in again["items"]] == [i["id"] for i in first["items"]]
+    status, body = call(router, "GET", "/hux/v1/search?q=zebra&include=message_text&cursor=nope")
+    assert status == 400
+
+
+def test_scan_stays_bounded_to_recent_conversations(scoped, monkeypatch):
+    router, project_id = scoped
+    monkeypatch.setattr(organization, "MESSAGE_SCAN_CONVERSATIONS", 1)
+    older = conversation(router, "Older", project_id)
+    say(router, older["id"], "quartz counters measured")
+    newer = conversation(router, "Newer", project_id)
+    say(router, newer["id"], "quartz counters ordered")
+    # Force a strictly newer updated_at (now_iso ties at clock resolution).
+    monkeypatch.setattr(organization, "now_iso", lambda: "2099-01-01T00:00:00Z")
+    status, _ = call(
+        router, "PATCH", f"/hux/v1/conversations/{newer['id']}",
+        {"pinned": True}, {"If-Match": str(newer["revision"])},
+    )
+    assert status == 200
+    status, body = call(router, "GET", "/hux/v1/search?q=quartz&include=message_text")
+    assert status == 200
+    assert [item["id"] for item in body["items"]] == [newer["id"]]
+    assert body["message_scan_limit"] == 1