atlas-iac/dockerfiles/hermes-webui-hux/multimodal/model.ts

/** Fail-closed multimodal normalization and intent builders for HUX-07. */

import { actionAuthorization } from "../autonomy/model.ts";
import type { ArtifactScope, HuxIdentity } from "../artifacts/types.ts";
import type { ApprovalView, PolicyView } from "../autonomy/types.ts";
import {
  isOpaqueId, isUtc, normalizeIdentity, normalizeScope, safeBlobUrl, safePreviewText, safeText,
  sameIdentity, sameScope,
} from "./security.ts";
import { validateUpload } from "./security.ts";
import type {
  AnnotationIntent, CaptureIntent, CaptureKind, GenerationThread, GenerationVariant,
  ImageEditIntent, MediaKind, MediaView, MultimodalPage, RawMediaEnvelope,
  TranscriptCorrectionIntent, TranscriptView,
} from "./types.ts";

export const FOUNDATION_FLAG = "hux.foundation";
export const PROJECTS_FLAG = "hux.projects";
export const ARTIFACTS_FLAG = "hux.artifacts";
export const AUTONOMY_FLAG = "hux.autonomy";
export const MULTIMODAL_FLAG = "hux.multimodal";
export const REQUIRED_FLAGS = Object.freeze([
  FOUNDATION_FLAG, PROJECTS_FLAG, ARTIFACTS_FLAG, AUTONOMY_FLAG, MULTIMODAL_FLAG,
]);

export function multimodalEnabled(flags?: Iterable<string>): boolean {
  const active = new Set(flags || []);
  return REQUIRED_FLAGS.every((flag) => active.has(flag));
}

function normalizeAnnotation(raw: unknown): MediaView["annotations"][number] | null {
  if (!raw || typeof raw !== "object" || Array.isArray(raw)) return null;
  const value = raw as Record<string, unknown>;
  const page = value.page === null ? null : value.page;
  const note = safeText(value.note, "", 1_000);
  return isOpaqueId(value.id, "ann") && (page === null ||
    (Number.isSafeInteger(page) && (page as number) > 0 && (page as number) <= 100_000)) && note ?
    {id: value.id, page: page as number | null, note} : null;
}

function normalizeMedia(raw: unknown, owner: string, scope: ArtifactScope): MediaView | null {
  if (!raw || typeof raw !== "object" || Array.isArray(raw)) return null;
  const value = raw as Record<string, unknown>;
  const kind = value.kind as MediaKind;
  const fileName = safeText(value.file_name, "", 180);
  const altText = safeText(value.alt_text, "Preview", 500);
  const upload = validateUpload({name: fileName, type: String(value.mime), size: Number(value.bytes)}, "authorized");
  if (value.schema !== "hux.media.v1" || !isOpaqueId(value.id, "med") || value.owner !== owner ||
      value.project_id !== scope.projectId || value.conversation_id !== scope.conversationId ||
      !isOpaqueId(value.artifact_id, "art") || !Number.isSafeInteger(value.artifact_version) ||
      (value.artifact_version as number) < 1 || !["image", "document", "audio"].includes(kind) ||
      !fileName || !Number.isSafeInteger(value.bytes) || !upload.ok || upload.kind !== kind ||
      typeof value.mime !== "string" || !Array.isArray(value.annotations) || value.annotations.length > 500) return null;
  const annotations = value.annotations.map(normalizeAnnotation);
  if (annotations.some((item) => item === null)) return null;
  const previewUrl = value.preview_url === null || value.preview_url === undefined ? null : safeBlobUrl(value.preview_url);
  const previewText = value.preview_text === null || value.preview_text === undefined ? null : safePreviewText(value.preview_text);
  if ((value.preview_url && (!previewUrl || value.preview_authorized !== true)) ||
      (value.preview_text !== null && value.preview_text !== undefined && (!previewText || kind !== "document")) ||
      (previewUrl && kind !== "image") || (previewText && value.preview_authorized !== true)) return null;
  let lineage: MediaView["lineage"] = null;
  if (value.lineage !== null && value.lineage !== undefined) {
    const parent = value.lineage as Record<string, unknown> | null;
    if (!parent || !isOpaqueId(parent.media_id, "med") || !isOpaqueId(parent.artifact_id, "art") ||
        !Number.isSafeInteger(parent.artifact_version) || (parent.artifact_version as number) < 1 ||
        parent.media_id === value.id) return null;
    lineage = {mediaId: parent.media_id, artifactId: parent.artifact_id,
      artifactVersion: parent.artifact_version as number};
  }
  return {id: value.id, owner, artifactId: value.artifact_id,
    artifactVersion: value.artifact_version as number, kind, fileName, mime: value.mime,
    bytes: value.bytes as number, previewUrl, previewText, altText,
    annotations: annotations as MediaView["annotations"], lineage};
}

function normalizeVariant(raw: unknown, allowed: ReadonlyMap<string, MediaView>): GenerationVariant | null {
  if (!raw || typeof raw !== "object" || Array.isArray(raw)) return null;
  const value = raw as Record<string, unknown>;
  const media = allowed.get(String(value.media_id));
  const parent = value.parent_variant_id;
  if (!isOpaqueId(value.id, "var") || !media || value.artifact_id !== media.artifactId ||
      value.artifact_version !== media.artifactVersion ||
      !(parent === null || isOpaqueId(parent, "var")) || !isUtc(value.created_at)) return null;
  return {id: value.id, mediaId: media.id, artifactId: media.artifactId,
    artifactVersion: media.artifactVersion, parentVariantId: parent as string | null,
    createdAt: value.created_at};
}

function normalizeThread(raw: unknown, media: ReadonlyMap<string, MediaView>, owner: string,
  scope: ArtifactScope): GenerationThread | null {
  if (!raw || typeof raw !== "object" || Array.isArray(raw)) return null;
  const value = raw as Record<string, unknown>;
  const prompt = safeText(value.prompt, "", 2_000);
  if (!isOpaqueId(value.id, "thr") || value.owner !== owner || value.project_id !== scope.projectId ||
      value.conversation_id !== scope.conversationId || !prompt || !Array.isArray(value.variants) ||
      value.variants.length < 1 || value.variants.length > 200) return null;
  const variants = value.variants.map((item) => normalizeVariant(item, media));
  if (variants.some((item) => item === null)) return null;
  const ordered = variants as GenerationVariant[];
  const ids = new Set<string>();
  for (const variant of ordered) {
    if (ids.has(variant.id) || (variant.parentVariantId && !ids.has(variant.parentVariantId))) return null;
    ids.add(variant.id);
  }
  return {id: value.id, prompt, variants: ordered};
}

function normalizeTranscript(raw: unknown, owner: string, scope: ArtifactScope): TranscriptView | null {
  if (raw === null || raw === undefined) return null;
  if (typeof raw !== "object" || Array.isArray(raw)) return null;
  const value = raw as Record<string, unknown>;
  const text = safeText(value.text, "", 20_000);
  const language = safeText(value.language, "", 40);
  return value.schema === "hux.voice_transcript.v1" && value.owner === owner &&
    value.project_id === scope.projectId && value.conversation_id === scope.conversationId &&
    isOpaqueId(value.id, "trn") &&
    isOpaqueId(value.turn_id, "turn") && Number.isSafeInteger(value.revision) &&
    (value.revision as number) >= 1 && Boolean(language) && Boolean(text) &&
    typeof value.finalized === "boolean" ? {id: value.id, turnId: value.turn_id,
      revision: value.revision as number, language, text, finalized: value.finalized} : null;
}

export function normalizeMultimodalPage(raw: RawMediaEnvelope, expected: HuxIdentity,
  scope: ArtifactScope): MultimodalPage | null {
  if (!raw || typeof raw !== "object" || raw.schema !== "hux.multimodal_page.v1" ||
      raw.api_version !== "hux.v1" || !sameIdentity(normalizeIdentity(raw.identity), expected) ||
      !sameScope(normalizeScope(raw.binding), scope) || !Array.isArray(raw.media) ||
      raw.media.length > 200 || !Array.isArray(raw.threads) || raw.threads.length > 100) return null;
  if (!["authorized", "requires_approval", "denied"].includes(String(
    (raw as Record<string, unknown>).upload_authorization))) return null;
  const normalized = raw.media.map((item) => normalizeMedia(item, expected.userRef, scope));
  const ids = new Set<string>();
  const unique = normalized.filter((item): item is MediaView => Boolean(item && !ids.has(item.id) && ids.add(item.id)));
  const allMedia = new Map(unique.map((item) => [item.id, item]));
  const media = unique.filter((item) => !item.lineage || (() => {
    const parent = allMedia.get(item.lineage!.mediaId);
    return Boolean(parent && parent.artifactId === item.lineage!.artifactId &&
      parent.artifactVersion === item.lineage!.artifactVersion);
  })());
  const mediaMap = new Map(media.map((item) => [item.id, item]));
  const threads = raw.threads.map((item) => normalizeThread(item, mediaMap, expected.userRef, scope));
  const transcript = normalizeTranscript(raw.transcript, expected.userRef, scope);
  return {media, threads: threads.filter((item): item is GenerationThread => item !== null), transcript,
    uploadAuthorization: (raw as Record<string, unknown>).upload_authorization as MultimodalPage["uploadAuthorization"],
    rejected: normalized.length - media.length + threads.filter((item) => item === null).length +
      (raw.transcript && !transcript ? 1 : 0)};
}

function baseIntent(media: MediaView, scope: ArtifactScope) {
  return {artifactId: media.artifactId, expectedVersion: media.artifactVersion, scope: {...scope}};
}

export function buildImageEdit(media: MediaView, scope: ArtifactScope,
  instruction: string): ImageEditIntent {
  const clean = safeText(instruction, "", 2_000);
  if (media.kind !== "image" || !clean) throw new TypeError("A bounded image edit instruction is required");
  return {...baseIntent(media, scope), mediaId: media.id, instruction: clean,
    lineage: {artifactId: media.artifactId, version: media.artifactVersion}};
}

export function buildAnnotation(media: MediaView, scope: ArtifactScope,
  note: string, page: number | null): AnnotationIntent {
  const clean = safeText(note, "", 1_000);
  if (media.kind !== "document" || !clean || !(page === null ||
      (Number.isSafeInteger(page) && page > 0 && page <= 100_000))) {
    throw new TypeError("A bounded document annotation is required");
  }
  return {...baseIntent(media, scope), mediaId: media.id, page, note: clean};
}

export function buildTranscriptCorrection(transcript: TranscriptView, correctedText: string,
  identity: HuxIdentity, scope: ArtifactScope): TranscriptCorrectionIntent {
  const clean = safeText(correctedText, "", 20_000);
  if (!transcript.finalized || !clean) throw new TypeError("Only a finalized transcript can be corrected");
  return {transcriptId: transcript.id, turnId: transcript.turnId,
    expectedRevision: transcript.revision, correctedText: clean, identity: {...identity}, scope: {...scope}};
}

/** Camera and screen input always require a fresh Ask First decision. */
export function captureAuthorization(policy: PolicyView | null, approval: ApprovalView | null,
  identity: HuxIdentity): "approval_required" | "deny" {
  return actionAuthorization(policy, "network", approval, identity) === "deny" ? "deny" : "approval_required";
}

export function buildCaptureIntent(kind: CaptureKind, policy: PolicyView | null,
  identity: HuxIdentity, scope: ArtifactScope): CaptureIntent {
  if (!['camera', 'screen'].includes(kind) || captureAuthorization(policy, null, identity) === "deny") {
    throw new TypeError("Capture is denied by the autonomy policy");
  }
  return {kind, autonomyCapability: "network", requiredDecision: "ask",
    identity: {...identity}, scope: {...scope}};
}