hermes(hux): bounded message-text search with privacy enforcement
HUX-03: GET /hux/v1/search now accepts include=message_text, an explicit opt-in that scans the stored message events of the 100 most recently active candidate conversations. Forgotten (tombstoned) conversations, private-mode conversations, restricted events and fully redacted events never match; the default indexed-fields search and its response contract are unchanged (the shipped UI keeps requiring message_text in not_indexed). Paginated mode uses one deterministic total order (score desc, updated_at desc, id) with an offset cursor. organization.py at 98% branch coverage. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01BvMSXH8VH2tMWXanb8SJdf
This commit is contained in:
jenkins
parent
e40fc5ec5d
commit
dc034cb738
@ -26,6 +26,10 @@ PROJECT_FIELDS = ("name", "description", "tags", "pinned", "archived", "default_
|
||||
CONVERSATION_FIELDS = ("title", "tags", "pinned", "archived", "mode", "project_id")
|
||||
INDEXED = ("title", "tags", "project_name", "artifact_titles")
|
||||
NOT_INDEXED = ("message_text",)
|
||||
MESSAGE_KINDS = ("message.user", "message.assistant")
|
||||
MESSAGE_SCAN_CONVERSATIONS = 100
|
||||
MESSAGE_SCAN_EVENTS = 300
|
||||
SEARCH_PAGE = 50
|
||||
SCHEMAS = contracts.load_all()
|
||||
TOKEN_RE = re.compile(r"[a-z0-9]+")
|
||||
|
||||
@ -287,25 +291,82 @@ def score(record: dict[str, Any], terms: list[str], project_name: str, titles: l
|
||||
return total
|
||||
|
||||
|
||||
def message_text_score(store: TenantStore, conversation: dict[str, Any], terms: list[str]) -> int:
|
||||
"""Bounded match over this conversation's stored message events.
|
||||
|
||||
Privacy wins: forgotten conversations were already excluded, private-mode
|
||||
conversations are never scanned, and restricted or fully redacted events
|
||||
stay invisible to search exactly as they are on the timeline.
|
||||
"""
|
||||
from hux import events
|
||||
|
||||
if events.is_private(store, conversation["id"]):
|
||||
return 0
|
||||
rows = store.read(events.FAMILY, conversation["id"])[-MESSAGE_SCAN_EVENTS:]
|
||||
total = 0
|
||||
for row in rows:
|
||||
if row.get("kind") not in MESSAGE_KINDS or row.get("sensitivity") == "restricted":
|
||||
continue
|
||||
if (row.get("redaction") or {}).get("level") == "full":
|
||||
continue
|
||||
words = tokens(str(row.get("summary", "")))
|
||||
total += sum(words.count(term) for term in terms)
|
||||
return total
|
||||
|
||||
|
||||
def search(request: Request) -> Response:
|
||||
"""``GET /hux/v1/search?q=&project_id=``: token match over the indexed fields, best first."""
|
||||
terms = tokens(request.query.get("q", ""))
|
||||
"""``GET /hux/v1/search?q=&project_id=&include=&cursor=``: best match first.
|
||||
|
||||
The default scope is the indexed fields only. ``include=message_text``
|
||||
additionally scans the stored message events of the most recently active
|
||||
``MESSAGE_SCAN_CONVERSATIONS`` candidates (bounded, privacy-enforced) and
|
||||
pages deterministically: rank order is (score desc, updated_at desc, id),
|
||||
the cursor is the offset into that total order.
|
||||
"""
|
||||
terms = tokens(request.query.get("q", "")[:200])
|
||||
if not terms:
|
||||
raise Invalid("q is required")
|
||||
include = request.query.get("include")
|
||||
if include not in (None, "message_text"):
|
||||
raise Invalid("include supports only message_text")
|
||||
cursor = request.query.get("cursor", "0")
|
||||
if not cursor.isdigit():
|
||||
raise Invalid("cursor must be a non-negative integer")
|
||||
project_filter = request.query.get("project_id")
|
||||
names = {p["id"]: p["name"] for p in request.store.scan(PROJECTS)}
|
||||
index = artifact_index(request.store)
|
||||
candidates = [
|
||||
record for record in request.store.scan(CONVERSATIONS)
|
||||
if not project_filter or record.get("project_id") == project_filter
|
||||
]
|
||||
scanned = set()
|
||||
if include:
|
||||
recent = sorted(candidates, key=lambda r: (r["updated_at"], r["id"]), reverse=True)
|
||||
scanned = {record["id"] for record in recent[:MESSAGE_SCAN_CONVERSATIONS]}
|
||||
ranked = []
|
||||
for record in request.store.scan(CONVERSATIONS):
|
||||
if project_filter and record.get("project_id") != project_filter:
|
||||
continue
|
||||
for record in candidates:
|
||||
points = score(record, terms, names.get(record.get("project_id", ""), ""), artifact_titles(request.store, record, index))
|
||||
if record["id"] in scanned and not record.get("archived"):
|
||||
points += message_text_score(request.store, record, terms)
|
||||
if points:
|
||||
ranked.append((points, record))
|
||||
ranked.sort(key=lambda pair: (-pair[0], pair[1]["updated_at"]))
|
||||
if include:
|
||||
# Paginated mode needs one total order; ids break updated_at ties.
|
||||
ranked.sort(key=lambda pair: (-pair[0], pair[1]["updated_at"], pair[1]["id"]))
|
||||
else:
|
||||
ranked.sort(key=lambda pair: (-pair[0], pair[1]["updated_at"]))
|
||||
request.audit("search.query", "conversations")
|
||||
return Response(200, {"items": [r for _, r in ranked], "next": None, "scores": {r["id"]: s for s, r in ranked},
|
||||
"indexed": list(INDEXED), "not_indexed": list(NOT_INDEXED)})
|
||||
if not include:
|
||||
return Response(200, {"items": [r for _, r in ranked], "next": None, "scores": {r["id"]: s for s, r in ranked},
|
||||
"indexed": list(INDEXED), "not_indexed": list(NOT_INDEXED)})
|
||||
start = int(cursor)
|
||||
window = ranked[start : start + SEARCH_PAGE]
|
||||
return Response(200, {
|
||||
"items": [r for _, r in window], "scores": {r["id"]: s for s, r in window},
|
||||
"next": str(start + SEARCH_PAGE) if len(ranked) > start + SEARCH_PAGE else None,
|
||||
"indexed": list(INDEXED) + ["message_text"], "not_indexed": [],
|
||||
"message_scan_limit": MESSAGE_SCAN_CONVERSATIONS,
|
||||
})
|
||||
|
||||
|
||||
def register(router: Router) -> None:
|
||||
|
||||
142
testing/tests/test_hermes_hux_search_message_text.py
Normal file
142
testing/tests/test_hermes_hux_search_message_text.py
Normal file
@ -0,0 +1,142 @@
|
||||
"""Bounded, privacy-enforced message-text search (HUX-03 increment)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
FOUNDATION = ROOT / "dockerfiles" / "hermes-hux-foundation"
|
||||
if str(FOUNDATION) not in sys.path:
|
||||
sys.path.insert(0, str(FOUNDATION))
|
||||
|
||||
from hux import contracts, organization # noqa: E402
|
||||
from hux.server import build_router # noqa: E402
|
||||
|
||||
HEADERS = {"X-Hermes-Tenant-Identity": "slot-3", "X-Hux-Subject": "usr_0123456789abcdef", "X-Hux-Surface": "chat", "X-Hux-Relay-Key": "rk"}
|
||||
WORKER = {**HEADERS, "X-Hux-Surface": "worker", "X-Hux-Trust": "worker", "X-Hux-Relay-Key": "wk"}
|
||||
ALL_ON = ",".join(card["flag"] for card in contracts.load_flags()["cards"])
|
||||
|
||||
|
||||
def call(router, method, path, body=None, headers=None):
|
||||
raw = b"" if body is None else json.dumps(body).encode()
|
||||
response = router.dispatch(method, path, {**HEADERS, **(headers or {})}, raw)
|
||||
return response.status, response.body
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def router(tmp_path):
|
||||
return build_router(
|
||||
tmp_path,
|
||||
{"HUX_FLAGS": ALL_ON, "HUX_ROUTER_KEY": "rk", "HUX_WORKER_KEY": "wk",
|
||||
"HUX_WRITES_PER_MINUTE": "5000", "HUX_READS_PER_MINUTE": "5000"},
|
||||
)
|
||||
|
||||
|
||||
def conversation(router, title, project_id):
|
||||
status, body = call(router, "POST", "/hux/v1/conversations", {"title": title, "project_id": project_id})
|
||||
assert status == 201
|
||||
return body
|
||||
|
||||
|
||||
def say(router, conversation_id, summary, kind="message.user", sensitivity="personal", key=None):
|
||||
import hashlib
|
||||
|
||||
slug = key or hashlib.sha256(f"{conversation_id}:{summary}".encode()).hexdigest()[:24]
|
||||
status, body = call(
|
||||
router, "POST", f"/hux/v1/conversations/{conversation_id}/events",
|
||||
{"kind": kind, "summary": summary, "sensitivity": sensitivity},
|
||||
{**WORKER, "Idempotency-Key": f"evt.{slug}"},
|
||||
)
|
||||
assert status in (200, 201), body
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def scoped(router):
|
||||
status, project = call(router, "POST", "/hux/v1/projects", {"name": "Kitchen"})
|
||||
assert status == 201
|
||||
return router, project["id"]
|
||||
|
||||
|
||||
def test_message_text_is_opt_in_and_scoped_explicitly(scoped):
|
||||
router, project_id = scoped
|
||||
convo = conversation(router, "Plumbing", project_id)
|
||||
say(router, convo["id"], "the kidneystone diagnosis was benign")
|
||||
status, body = call(router, "GET", "/hux/v1/search?q=kidneystone")
|
||||
assert status == 200 and body["items"] == []
|
||||
assert body["not_indexed"] == ["message_text"]
|
||||
status, body = call(router, "GET", "/hux/v1/search?q=kidneystone&include=message_text")
|
||||
assert status == 200
|
||||
assert [item["id"] for item in body["items"]] == [convo["id"]]
|
||||
assert "message_text" in body["indexed"] and body["not_indexed"] == []
|
||||
status, body = call(router, "GET", "/hux/v1/search?q=kidneystone&include=bogus")
|
||||
assert status == 400
|
||||
|
||||
|
||||
def test_privacy_and_tombstones_defeat_message_search(scoped):
|
||||
router, project_id = scoped
|
||||
plain = conversation(router, "Plain", project_id)
|
||||
say(router, plain["id"], "walnut cabinets arrived")
|
||||
hidden = conversation(router, "Hidden", project_id)
|
||||
say(router, hidden["id"], "walnut cabinets secret plans", sensitivity="restricted")
|
||||
private = conversation(router, "Private", project_id)
|
||||
say(router, private["id"], "walnut cabinets private note")
|
||||
status, _ = call(
|
||||
router, "PUT",
|
||||
f"/hux/v1/projects/{project_id}/conversations/{private['id']}/mode",
|
||||
{"project_id": project_id, "mode": "private"},
|
||||
{"If-Match": "0", "Idempotency-Key": "mode-private-x"},
|
||||
)
|
||||
assert status == 200
|
||||
forgotten = conversation(router, "Forgotten", project_id)
|
||||
say(router, forgotten["id"], "walnut cabinets to forget")
|
||||
status, _ = call(router, "POST", f"/hux/v1/conversations/{forgotten['id']}/forget", {})
|
||||
assert status in (200, 201, 202)
|
||||
status, body = call(router, "GET", "/hux/v1/search?q=walnut+cabinets&include=message_text")
|
||||
assert status == 200
|
||||
assert [item["id"] for item in body["items"]] == [plain["id"]]
|
||||
|
||||
|
||||
def test_pagination_is_deterministic(scoped):
|
||||
router, project_id = scoped
|
||||
ids = []
|
||||
for index in range(organization.SEARCH_PAGE + 3):
|
||||
convo = conversation(router, f"Convo {index}", project_id)
|
||||
say(router, convo["id"], "zebra sighting logged", key=f"zebra.{index:04}")
|
||||
ids.append(convo["id"])
|
||||
status, first = call(router, "GET", "/hux/v1/search?q=zebra&include=message_text")
|
||||
assert status == 200 and len(first["items"]) == organization.SEARCH_PAGE
|
||||
assert first["next"] == str(organization.SEARCH_PAGE)
|
||||
status, second = call(
|
||||
router, "GET",
|
||||
f"/hux/v1/search?q=zebra&include=message_text&cursor={first['next']}",
|
||||
)
|
||||
assert status == 200 and len(second["items"]) == 3 and second["next"] is None
|
||||
assert not {i["id"] for i in first["items"]} & {i["id"] for i in second["items"]}
|
||||
status, again = call(router, "GET", "/hux/v1/search?q=zebra&include=message_text")
|
||||
assert [i["id"] for i in again["items"]] == [i["id"] for i in first["items"]]
|
||||
status, body = call(router, "GET", "/hux/v1/search?q=zebra&include=message_text&cursor=nope")
|
||||
assert status == 400
|
||||
|
||||
|
||||
def test_scan_stays_bounded_to_recent_conversations(scoped, monkeypatch):
|
||||
router, project_id = scoped
|
||||
monkeypatch.setattr(organization, "MESSAGE_SCAN_CONVERSATIONS", 1)
|
||||
older = conversation(router, "Older", project_id)
|
||||
say(router, older["id"], "quartz counters measured")
|
||||
newer = conversation(router, "Newer", project_id)
|
||||
say(router, newer["id"], "quartz counters ordered")
|
||||
# Force a strictly newer updated_at (now_iso ties at clock resolution).
|
||||
monkeypatch.setattr(organization, "now_iso", lambda: "2099-01-01T00:00:00Z")
|
||||
status, _ = call(
|
||||
router, "PATCH", f"/hux/v1/conversations/{newer['id']}",
|
||||
{"pinned": True}, {"If-Match": str(newer["revision"])},
|
||||
)
|
||||
assert status == 200
|
||||
status, body = call(router, "GET", "/hux/v1/search?q=quartz&include=message_text")
|
||||
assert status == 200
|
||||
assert [item["id"] for item in body["items"]] == [newer["id"]]
|
||||
assert body["message_scan_limit"] == 1
|
||||
Loading…
x
Reference in New Issue
Block a user