"""HUX-08 research: sources, passages and citations. Security obligations exercised: the service stores but never dereferences a source ``uri`` and only allowlisted schemes are accepted (SO-19); passage and citation references resolve under the caller's subtree and foreign ids are 404 (SO-33); hashes and dedupe keys are server-computed (SO-30 by analogy); every served record satisfies citation.schema.json. """ from __future__ import annotations import hashlib import json import sys import types from pathlib import Path import pytest ROOT = Path(__file__).resolve().parents[2] FOUNDATION = ROOT / "dockerfiles" / "hermes-hux-foundation" if str(FOUNDATION) not in sys.path: sys.path.insert(0, str(FOUNDATION)) from hux import audit, contracts, identity, research, store # noqa: E402 from hux.server import build_router # noqa: E402 SCHEMAS = contracts.load_all() OWNER = {"X-Hermes-Tenant-Identity": "slot-3", "X-Hux-Subject": "usr_0123456789abcdef", "X-Hux-Surface": "chat", "X-Hux-Relay-Key": "rk"} OTHER = {**OWNER, "X-Hux-Subject": "usr_fedcba9876543210"} ALL_ON = ",".join(card["flag"] for card in contracts.load_flags()["cards"]) def tenant(router, subject="usr_0123456789abcdef") -> store.TenantStore: return store.TenantStore(router.data_root, identity.Identity("slot-3", subject, "chat", "router")) @pytest.fixture def router(tmp_path): return build_router(tmp_path, {"HUX_FLAGS": ALL_ON, "HUX_ROUTER_KEY": "rk"}) def call(router, method, path, body=None, headers=None, raw=None): payload = raw if raw is not None else (json.dumps(body).encode() if body is not None else b"") response = router.dispatch(method, path, {**OWNER, **(headers or {})}, payload) return response.status, response.body, response.headers def valid(record) -> bool: return contracts.validate_record(record, SCHEMAS) == [] def source(router, uri="https://Example.com/cabinets/#lead", **extra): status, record, _ = call(router, "POST", "/hux/v1/sources", {"kind": "web", "uri": uri, "title": "Cabinet pricing", "classification": "primary", **extra}) assert status == 201, record return record def passage(router, source_id, text="Standard base cabinets ship in 3-4 weeks."): status, record, _ = call(router, "POST", "/hux/v1/passages", {"source_id": source_id, "text": text}) assert status == 201, record return record def cite(router, passage_ids, message="msg-44", claim="Base cabinets take three to four weeks.", **extra): status, record, _ = call(router, "POST", f"/hux/v1/messages/{message}/citations", {"claim": claim, "passage_ids": passage_ids, "support": "supports", **extra}) assert status == 201, record return record @pytest.fixture def events(monkeypatch): calls = [] module = types.ModuleType("hux.events") module.emit = lambda store, identity, conversation_id, kind, summary, detail=None, evidence=None, sensitivity="personal", run_id=None, turn=None, correlation_id=None: calls.append((conversation_id, kind, detail, evidence)) monkeypatch.setitem(sys.modules, "hux.events", module) return calls # --- sources ----------------------------------------------------------------------- def test_source_is_valid_and_deduped_on_normalised_uri(router): first = source(router, publisher="Example Co", published_at="2026-01-01T00:00:00Z", content_hash="sha256:" + "a" * 64, conversation_id="conv_0001abcd") assert valid(first) and first["provenance"]["actor"] == {"type": "user", "id": "usr_0123456789abcdef"} assert first["provenance"]["conversation_id"] == "conv_0001abcd" and first["publisher"] == "Example Co" expected = "sha256:" + hashlib.sha256(b"uri\nhttps://example.com/cabinets").hexdigest() assert first["dedupe_key"] == expected status, again, headers = call(router, "POST", "/hux/v1/sources", {"kind": "web", "uri": "HTTPS://example.com/cabinets", "title": "Different title"}) assert (status, again["id"], headers["HUX-Replayed"]) == (200, first["id"], "true") assert tenant(router).count("sources") == 1 status, record, _ = call(router, "GET", f"/hux/v1/sources/{first['id']}") assert status == 200 and record == first assert call(router, "GET", "/hux/v1/sources/src_missing00001")[0] == 404 assert call(router, "GET", "/hux/v1/sources/..")[0] == 404 def test_source_without_uri_dedupes_on_kind_and_title(router): status, memory, _ = call(router, "POST", "/hux/v1/sources", {"kind": "memory", "title": " Remembered fact "}) status, again, _ = call(router, "POST", "/hux/v1/sources", {"kind": "memory", "title": "remembered fact"}) assert status == 200 and again["id"] == memory["id"] and "uri" not in memory and memory["classification"] == "unknown" assert research.normalise_uri("artifact://art_0001aaaa@2/") == "artifact://art_0001aaaa@2/" assert research.normalise_uri("file:///opt/data/x/") == "file:///opt/data/x" assert research.normalise_uri("https://h.example/") == "https://h.example/" @pytest.mark.parametrize("bad", [ {"uri": "javascript:alert(1)"}, {"uri": "ftp://x"}, {"uri": "/relative"}, {"kind": "rumour"}, {"kind": None}, {"classification": "gospel"}, {"title": ""}, {"conversation_id": "x"}, {"content_hash": "md5:abc"}, {"published_at": "yesterday"}, ]) def test_source_rejects_bad_bodies(router, bad): status, error, _ = call(router, "POST", "/hux/v1/sources", {"kind": "web", "uri": "https://ok.example/", "title": "t", **bad}) assert status == 400 and valid(error), bad assert call(router, "POST", "/hux/v1/sources", raw=b'"str"')[0] == 400 def test_source_idempotency_and_cap(router, monkeypatch): status, one, _ = call(router, "POST", "/hux/v1/sources", {"kind": "web", "uri": "https://a.example/1", "title": "t"}, {"Idempotency-Key": "src-key-0001"}) status, two, _ = call(router, "POST", "/hux/v1/sources", {"kind": "web", "uri": "https://a.example/2", "title": "t"}, {"Idempotency-Key": "src-key-0001"}) assert status == 200 and two["id"] == one["id"] monkeypatch.setattr(research, "MAX_SOURCES", 1) status, error, _ = call(router, "POST", "/hux/v1/sources", {"kind": "web", "uri": "https://a.example/3", "title": "t"}) assert (status, error["code"]) == (413, "too_large") def test_service_has_no_outbound_client(): import hux.artifacts import hux.research for name in ("urllib.request", "http.client", "socket"): assert name not in sys.modules or not any(name in getattr(m, "__dict__", {}) for m in (hux.research, hux.artifacts)) text = (FOUNDATION / "hux" / "research.py").read_text() assert "urlopen" not in text and "http.client" not in text # --- passages ---------------------------------------------------------------------- def test_passage_hash_is_server_computed_and_deduped(router, monkeypatch): src = source(router) first = passage(router, src["id"]) assert valid(first) and first["hash"] == "sha256:" + hashlib.sha256(b"Standard base cabinets ship in 3-4 weeks.").hexdigest() status, again, headers = call(router, "POST", "/hux/v1/passages", {"source_id": src["id"], "text": "Standard base cabinets ship in 3-4 weeks.", "hash": "sha256:" + "0" * 64}) assert (status, again["id"], headers["HUX-Replayed"]) == (200, first["id"], "true") and again["hash"] == first["hash"] status, located, _ = call(router, "POST", "/hux/v1/passages", {"source_id": src["id"], "text": "Other", "locator": {"page": 2, "selector": "#x"}}, {"Idempotency-Key": "psg-key-00001"}) assert status == 201 and valid(located) and located["locator"] == {"page": 2, "selector": "#x"} status, replay, _ = call(router, "POST", "/hux/v1/passages", {"source_id": src["id"], "text": "Ignored"}, {"Idempotency-Key": "psg-key-00001"}) assert status == 200 and replay["id"] == located["id"] status, error, _ = call(router, "POST", "/hux/v1/passages", {"source_id": src["id"], "text": "Bad", "locator": {"page": 0}}) assert status == 400 and error["details"] assert call(router, "POST", "/hux/v1/passages", {"source_id": src["id"], "text": "x" * 4001})[0] == 400 assert call(router, "POST", "/hux/v1/passages", {"source_id": "src_missing00001", "text": "x"})[0] == 404 assert call(router, "POST", "/hux/v1/passages", {"text": "x"})[0] == 404 assert call(router, "POST", "/hux/v1/passages", raw=b"[]")[0] == 400 monkeypatch.setattr(research, "MAX_PASSAGES", 2) assert call(router, "POST", "/hux/v1/passages", {"source_id": src["id"], "text": "third"})[0] == 413 def test_passage_cannot_name_another_subjects_source(router): src = source(router) status, error, _ = call(router, "POST", "/hux/v1/passages", {"source_id": src["id"], "text": "steal"}, OTHER) assert (status, error["code"]) == (404, "not_found") assert call(router, "GET", f"/hux/v1/sources/{src['id']}", headers=OTHER)[0] == 404 rows = audit.recent(tenant(router, "usr_fedcba9876543210")) assert [r["outcome"] for r in rows] == ["not_found", "not_found"] # --- citations --------------------------------------------------------------------- def test_citation_attach_dedupes_and_lists_with_passages_and_sources(router, events): src = source(router) one = passage(router, src["id"]) two = passage(router, src["id"], "Delivery excludes installation.") cit = cite(router, [two["id"], one["id"]], conversation_id="conv_0001abcd", note="checked") assert valid(cit) and cit["passage_ids"] == [two["id"], one["id"]] and cit["note"] == "checked" assert cit["dedupe_key"] == "sha256:" + hashlib.sha256(("\n".join(["citation", "msg-44", "Base cabinets take three to four weeks.", *sorted([one["id"], two["id"]])])).encode()).hexdigest() assert events == [("conv_0001abcd", "citation.attached", {"message_id": "msg-44", "citation_id": cit["id"], "support": "supports"}, [{"kind": "passage", "id": two["id"], "hash": two["hash"]}, {"kind": "passage", "id": one["id"], "hash": one["hash"]}])] status, again, headers = call(router, "POST", "/hux/v1/messages/msg-44/citations", {"claim": "Base cabinets take three to four weeks.", "passage_ids": [one["id"], two["id"], one["id"]], "support": "contradicts"}) assert (status, again["id"], headers["HUX-Replayed"]) == (200, cit["id"], "true") and again["support"] == "supports" second = cite(router, [one["id"]], claim="Second claim") status, body, _ = call(router, "GET", "/hux/v1/messages/msg-44/citations") assert status == 200 and [i["citation"]["id"] for i in body["items"]] == [cit["id"], second["id"]] strip = body["items"][0] assert [p["id"] for p in strip["passages"]] == [two["id"], one["id"]] and strip["sources"] == [src] assert all(valid(r) for item in body["items"] for r in (item["citation"], *item["passages"], *item["sources"])) assert call(router, "GET", "/hux/v1/messages/msg-none/citations")[1]["items"] == [] assert len(events) == 1 # the second citation carried no conversation_id, so nothing to emit against @pytest.mark.parametrize("bad", [ {"claim": ""}, {"support": "maybe"}, {"passage_ids": "psg_x"}, {"passage_ids": []}, {"note": "n" * 501}, {"conversation_id": "nope"}, ]) def test_citation_rejects_bad_bodies(router, bad): src = source(router) psg = passage(router, src["id"]) status, error, _ = call(router, "POST", "/hux/v1/messages/msg-1/citations", {"claim": "c", "passage_ids": [psg["id"]], "support": "supports", **bad}) assert status == 400 and valid(error), bad def test_citation_reference_rules(router): src = source(router) psg = passage(router, src["id"]) assert call(router, "POST", "/hux/v1/messages/msg-1/citations", {"claim": "c", "passage_ids": ["psg_missing00001"]})[0] == 404 assert call(router, "POST", "/hux/v1/messages/msg-1/citations", {"claim": "c", "passage_ids": [psg["id"]]}, OTHER)[0] == 404 status, error, _ = call(router, "POST", "/hux/v1/messages/msg-1/citations", {"claim": "c", "passage_ids": [f"psg_{i:012d}" for i in range(33)]}) assert status == 413 assert call(router, "POST", f"/hux/v1/messages/{'m' * 121}/citations", {"claim": "c", "passage_ids": [psg["id"]]})[0] == 400 assert call(router, "POST", "/hux/v1/messages/msg-1/citations", raw=b"null")[0] == 400 status, cit, _ = call(router, "POST", "/hux/v1/messages/msg-1/citations", {"claim": "c", "passage_ids": [psg["id"]]}, {"Idempotency-Key": "cit-key-00001"}) status, replay, _ = call(router, "POST", "/hux/v1/messages/msg-1/citations", {"claim": "other", "passage_ids": [psg["id"]]}, {"Idempotency-Key": "cit-key-00001"}) assert status == 200 and replay["id"] == cit["id"] and replay["support"] == "unverified" assert call(router, "GET", "/hux/v1/messages/msg-1/citations", headers=OTHER)[1]["items"] == [] def test_validate_citations_reports_integrity_problems(router, monkeypatch): src = source(router) psg = passage(router, src["id"]) good = cite(router, [psg["id"]]) contradiction = cite(router, [psg["id"]], claim="Never", support="contradicts") noted = cite(router, [psg["id"]], claim="Never again", support="contradicts", note="supplier changed policy") tenant_store = tenant(router) assert research.validate_citations(tenant_store, [good["id"], noted["id"]]) == [] assert research.validate_citations(tenant_store, [contradiction["id"]]) == [f"{contradiction['id']}: contradicts without a note"] tenant_store.delete("sources", src["id"]) assert research.validate_citations(tenant_store, [good["id"]]) == [f"{good['id']}: passage {psg['id']} names missing source {src['id']}"] tenant_store.delete("passages", psg["id"]) problems = research.validate_citations(tenant_store, [good["id"], "cit_missing00001", "bad id"]) assert problems == [f"{good['id']}: passage {psg['id']} does not resolve", "cit_missing00001: citation does not resolve", "bad id: citation does not resolve"] monkeypatch.setitem(sys.modules, "hux.events", None) assert cite(router, [passage(router, source(router, "https://b.example/")["id"])["id"]], claim="silent", conversation_id="conv_0001abcd")["support"] == "supports" def test_f9_source_title_passage_text_and_citation_claim_are_scrubbed(router): """F9 / SO-07: secrets in free-text research fields never persist; the passage hash covers the scrubbed text.""" token = "ghp_ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" src = source(router, title=f"notes {token}") assert src["title"] == "notes [redacted:github_token]" psg = passage(router, src["id"], f"password: hunter2 and {token}") assert psg["text"] == "[redacted:password] and [redacted:github_token]" assert psg["hash"] == "sha256:" + hashlib.sha256(psg["text"].encode()).hexdigest() cit = cite(router, [psg["id"]], claim=f"claim {token}", note=f"note {token}") assert token not in cit["claim"] and token not in cit["note"] leaked = [path for path in Path(router.data_root).rglob("*.json*") if token in path.read_text()] assert leaked == [] def test_research_modules_stay_under_500_lines(): assert len((FOUNDATION / "hux" / "research.py").read_text().splitlines()) <= 500 def test_worker_trust_records_a_system_actor(tmp_path): router = build_router(tmp_path, {"HUX_FLAGS": ALL_ON, "HUX_ROUTER_KEY": "rk", "HUX_WORKER_KEY": "wk"}) headers = {"X-Hux-Trust": "worker", "X-Hux-Surface": "worker", "X-Hux-Relay-Key": "wk"} status, record, _ = call(router, "POST", "/hux/v1/sources", {"kind": "tool_output", "title": "grep"}, headers) assert status == 201 and record["provenance"] == {"surface": "worker", "actor": {"type": "system", "id": "worker"}, "recorded_at": record["retrieved_at"]}