/** Schema-aligned, fail-closed HUX-08 research normalization. */ import { asRecord, boundedText, isHash, isId, isUtc, normalizeIdentity, normalizeScope, safeSourceUri, sameIdentity, sameScope, } from "./security.ts"; import type { CitationView, Classification, HuxIdentity, NotebookStatus, NotebookView, PassageView, ProvenanceView, RawResearchEnvelope, ResearchPage, ResearchScope, SourceKind, SourceView, Support, } from "./types.ts"; export const FOUNDATION_FLAG = "hux.foundation"; export const FRIENDLY_MODES_FLAG = "hux.friendly_modes"; export const RESEARCH_FLAG = "hux.research"; const KINDS = new Set(["web", "document", "artifact", "memory", "tool_output", "dataset"]); const CLASSIFICATIONS = new Set(["primary", "secondary", "unknown"]); const SUPPORT = new Set(["supports", "partially_supports", "contradicts", "unverified"]); const STATUSES = new Set(["open", "answered", "abandoned"]); const MAX_RECORDS = 500; export function researchEnabled(flags?: Iterable): boolean { const active = new Set(flags || []); return [FOUNDATION_FLAG, FRIENDLY_MODES_FLAG, RESEARCH_FLAG].every((flag) => active.has(flag)); } function provenance(raw: unknown, conversationId: string): ProvenanceView | null | false { if (raw === undefined) return null; const value = asRecord(raw); const actor = asRecord(value?.actor); if (!value || !actor || !["chat", "worker", "telegram", "voice", "api"].includes(String(value.surface)) || !["user", "assistant", "tool", "system", "operator"].includes(String(actor.type)) || !boundedText(actor.id, 1, 120) || !isUtc(value.recorded_at) || (value.conversation_id !== undefined && value.conversation_id !== conversationId) || (value.run_id !== undefined && !boundedText(value.run_id, 1, 120))) return false; return {surface: value.surface as ProvenanceView["surface"], actorType: actor.type as ProvenanceView["actorType"], recordedAt: value.recorded_at, runId: value.run_id as string | undefined || null}; } function source(raw: unknown, conversationId: string): SourceView | null { const value = asRecord(raw); if (!value || value.schema !== "hux.source.v1" || !isId(value.id, "src") || !KINDS.has(value.kind as SourceKind) || !boundedText(value.title, 1, 300) || !boundedText(value.publisher ?? "", 0, 200) || !isUtc(value.retrieved_at) || !CLASSIFICATIONS.has(value.classification as Classification) || (value.published_at !== undefined && !isUtc(value.published_at)) || (isUtc(value.published_at) && value.published_at > value.retrieved_at) || (value.content_hash !== undefined && !isHash(value.content_hash))) return null; const uri = safeSourceUri(value.uri); if (value.uri !== undefined && value.uri !== "" && !uri) return null; const origin = provenance(value.provenance, conversationId); if (origin === false) return null; return {id: value.id, kind: value.kind as SourceKind, title: value.title, publisher: value.publisher as string | undefined || "", publishedAt: value.published_at as string | undefined || null, retrievedAt: value.retrieved_at, classification: value.classification as Classification, uri, provenance: origin}; } function locator(raw: unknown): string | null { if (raw === undefined) return ""; const value = asRecord(raw); if (!value || Object.keys(value).some((key) => !["page", "selector", "line_start", "line_end", "char_start", "char_end"].includes(key))) return null; const positive = (item: unknown) => Number.isSafeInteger(item) && (item as number) >= 1; const natural = (item: unknown) => Number.isSafeInteger(item) && (item as number) >= 0; if ((value.page !== undefined && !positive(value.page)) || (value.selector !== undefined && !boundedText(value.selector, 0, 500)) || (value.line_start !== undefined && !positive(value.line_start)) || (value.line_end !== undefined && !positive(value.line_end)) || (value.char_start !== undefined && !natural(value.char_start)) || (value.char_end !== undefined && !natural(value.char_end)) || (positive(value.line_start) && positive(value.line_end) && (value.line_end as number) < (value.line_start as number)) || (natural(value.char_start) && natural(value.char_end) && (value.char_end as number) < (value.char_start as number))) return null; const parts = [value.page && `page ${value.page}`, value.selector && String(value.selector), value.line_start && `lines ${value.line_start}${value.line_end ? `–${value.line_end}` : ""}`, value.char_start !== undefined && `characters ${value.char_start}${value.char_end !== undefined ? `–${value.char_end}` : ""}`]; return parts.filter(Boolean).join(" · "); } function passage(raw: unknown, sources: ReadonlySet): PassageView | null { const value = asRecord(raw); if (!value || value.schema !== "hux.passage.v1" || !isId(value.id, "psg") || !isId(value.source_id, "src") || !sources.has(value.source_id) || !boundedText(value.text, 1, 4_000) || !isHash(value.hash)) return null; const position = locator(value.locator); return position === null ? null : {id: value.id, sourceId: value.source_id, text: value.text, locator: position}; } function citation(raw: unknown, passages: ReadonlySet): CitationView | null { const value = asRecord(raw); if (!value || value.schema !== "hux.citation.v1" || !isId(value.id, "cit") || !boundedText(value.message_id, 1, 120) || !boundedText(value.claim, 1, 1_000) || !Array.isArray(value.passage_ids) || value.passage_ids.length < 1 || value.passage_ids.length > 32 || new Set(value.passage_ids).size !== value.passage_ids.length || value.passage_ids.some((id) => !isId(id, "psg") || !passages.has(id)) || !SUPPORT.has(value.support as Support) || !boundedText(value.note ?? "", 0, 500)) return null; return {id: value.id, messageId: value.message_id, claim: value.claim, passageIds: [...value.passage_ids] as string[], support: value.support as Support, note: value.note as string | undefined || ""}; } function idList(raw: unknown, prefix: string, allowed: ReadonlySet): string[] | null { return Array.isArray(raw) && raw.length <= MAX_RECORDS && new Set(raw).size === raw.length && raw.every((id) => isId(id, prefix) && allowed.has(id)) ? [...raw] as string[] : null; } function textList(raw: unknown): string[] | null { return Array.isArray(raw) && raw.length <= 64 && raw.every((item) => boundedText(item, 1, 500)) ? [...raw] : null; } function notebook(raw: unknown, scope: ResearchScope, sources: ReadonlySet, passages: ReadonlySet, citations: ReadonlyMap): NotebookView | null { const value = asRecord(raw); if (!value || value.schema !== "hux.research_notebook.v1" || !isId(value.id, "nb") || value.conversation_id !== scope.conversationId || !boundedText(value.question, 1, 1_000) || !STATUSES.has(value.status as NotebookStatus) || !isUtc(value.updated_at)) return null; const sourceIds = idList(value.source_ids, "src", sources); const passageIds = idList(value.passage_ids, "psg", passages); const citationIds = idList(value.citation_ids, "cit", new Set(citations.keys())); const assumptions = textList(value.assumptions); const unresolved = textList(value.unresolved_questions); if (!sourceIds || !passageIds || !citationIds || !assumptions || !unresolved) return null; const notes = citationIds.map((id) => citations.get(id)?.note || "").filter(Boolean); return {id: value.id, question: value.question, status: value.status as NotebookStatus, sourceIds, passageIds, citationIds, assumptions, unresolvedQuestions: unresolved, notes, updatedAt: value.updated_at}; } function unique(values: Array): {items: T[]; rejected: number} { const seen = new Set(); const items = values.filter((item): item is T => Boolean(item && !seen.has(item.id) && seen.add(item.id))); return {items, rejected: values.length - items.length}; } export function normalizeResearchPage(raw: RawResearchEnvelope, expected: HuxIdentity, scope: ResearchScope): ResearchPage | null { if (!raw || typeof raw !== "object" || raw.schema !== "hux.research_page.v1" || raw.api_version !== "hux.v1" || !sameIdentity(normalizeIdentity(raw.identity), expected) || !sameScope(normalizeScope(raw.binding), scope) || !Array.isArray(raw.sources) || raw.sources.length > MAX_RECORDS || !Array.isArray(raw.passages) || raw.passages.length > MAX_RECORDS || !Array.isArray(raw.citations) || raw.citations.length > MAX_RECORDS || !Array.isArray(raw.notebooks) || raw.notebooks.length > 100) return null; const sourceResult = unique(raw.sources.map((item) => source(item, scope.conversationId))); const sourceIds = new Set(sourceResult.items.map((item) => item.id)); const passageResult = unique(raw.passages.map((item) => passage(item, sourceIds))); const passageIds = new Set(passageResult.items.map((item) => item.id)); const citationResult = unique(raw.citations.map((item) => citation(item, passageIds))); const citationMap = new Map(citationResult.items.map((item) => [item.id, item])); const notebookResult = unique(raw.notebooks.map((item) => notebook(item, scope, sourceIds, passageIds, citationMap))); return {sources: sourceResult.items, passages: passageResult.items, citations: citationResult.items, notebooks: notebookResult.items, rejected: sourceResult.rejected + passageResult.rejected + citationResult.rejected + notebookResult.rejected}; }