atlas-iac/dockerfiles/hermes-hux-foundation/hux/organization.py

"""HUX-03 organisation: projects, conversations, branch lineage and search.

Projects own conversations; conversations carry tags, pins, a mode and an
optional branch pointer to the conversation they forked from. Search covers
only the ``project.schema.json#/$defs/search_index`` fields this increment
indexes (title, tags, project_name, and artifact titles when the artifacts
lane is present); message text is not indexed here and the response says so.
"""

from __future__ import annotations

import re
from typing import Any

from hux import contracts, redaction
from hux.errors import Conflict, Invalid, NotFound
from hux.http import Request, Response, Router, page
from hux.store import TenantStore, check_id, new_id, now_iso

CARD = "HUX-03"
PROJECTS = "projects"
CONVERSATIONS = "conversations"
MAX_PROJECTS = 200
MAX_CONVERSATIONS = 2000
PROJECT_FIELDS = ("name", "description", "tags", "pinned", "archived", "default_mode")
CONVERSATION_FIELDS = ("title", "tags", "pinned", "archived", "mode", "project_id")
INDEXED = ("title", "tags", "project_name", "artifact_titles")
NOT_INDEXED = ("message_text",)
MESSAGE_KINDS = ("message.user", "message.assistant")
MESSAGE_SCAN_CONVERSATIONS = 100
MESSAGE_SCAN_EVENTS = 300
SEARCH_PAGE = 50
SCHEMAS = contracts.load_all()
TOKEN_RE = re.compile(r"[a-z0-9]+")


def project_exists(store: TenantStore, project_id: str) -> bool:
    """True when this tenant owns a project with that id (helper for other lanes)."""
    return isinstance(project_id, str) and bool(re.match(r"^[a-z]{2,6}_[A-Za-z0-9._-]{4,80}$", project_id)) and store.exists(PROJECTS, project_id)


def conversation_exists(store: TenantStore, conversation_id: str) -> bool:
    """True when this tenant owns a conversation with that id (helper for other lanes)."""
    return isinstance(conversation_id, str) and bool(re.match(r"^[a-z]{2,6}_[A-Za-z0-9._-]{4,80}$", conversation_id)) and store.exists(CONVERSATIONS, conversation_id)


def project_of(store: TenantStore, conversation_id: str | None) -> str | None:
    """The project a conversation belongs to, or None when unknown or unfiled."""
    if not conversation_id or not conversation_exists(store, conversation_id):
        return None
    return store.get(CONVERSATIONS, conversation_id).get("project_id")


def checked(record: dict[str, Any]) -> dict[str, Any]:
    """Raise Invalid unless ``record`` satisfies its contract."""
    problems = contracts.validate_record(record, SCHEMAS)
    if problems:
        raise Invalid("record fails contract", problems)
    return record


def body_dict(request: Request) -> dict[str, Any]:
    """The JSON object body or Invalid."""
    if not isinstance(request.body, dict):
        raise Invalid("body must be a JSON object")
    return request.body


def pick(body: dict[str, Any], fields: tuple[str, ...]) -> dict[str, Any]:
    """Only the client-settable fields, secret-scrubbed (F9); ids, owner, timestamps and revision are server-set."""
    return redaction.scrub_value({k: body[k] for k in fields if k in body}, [])


def replay(store: TenantStore, family: str, key: str) -> dict[str, Any] | None:
    """The record an Idempotency-Key already created in this family, if any."""
    for row in store.read(family, "idempotency"):
        if row["key"] == key:
            return store.get(family, row["id"])
    return None


def create(request: Request, family: str, record: dict[str, Any], key: str, cap: int) -> tuple[dict[str, Any], int]:
    """Create under the family lock honouring Idempotency-Key and the family count cap."""
    with request.store.lock(family):
        if key:
            existing = replay(request.store, family, key)
            if existing is not None:
                return existing, 200
        if request.store.count(family) >= cap:
            raise Conflict(f"{family} cap of {cap} reached")
        stored = request.store.put(family, checked({**record, "revision": 1}))
        if key:
            request.store.append(family, "idempotency", {"key": key, "id": stored["id"]})
    return stored, 201


def update(request: Request, family: str, fields: tuple[str, ...]) -> dict[str, Any]:
    """PATCH under If-Match; a missing If-Match is accepted but audited as unconditional (SO-44)."""
    changes = pick(body_dict(request), fields)
    expected = request.if_match()
    with request.store.lock(family):
        current = request.store.get(family, check_id(request.params["id"]))
        if "project_id" in changes and changes["project_id"] is not None and not project_exists(request.store, changes["project_id"]):
            raise NotFound("project not found")
        record = {**current, **{k: v for k, v in changes.items() if v is not None}, "updated_at": now_iso()}
        if changes.get("project_id", "") is None:
            record.pop("project_id", None)
        stored = request.store.put(family, checked(record), expected_revision=expected)
    request.audit(f"{family}.update", stored["id"], reason="" if expected is not None else "unconditional_write")
    return stored


def etag(record: dict[str, Any]) -> dict[str, str]:
    """Revision as ETag so clients can send it back in If-Match."""
    return {"ETag": str(record["revision"])}


# -- projects ------------------------------------------------------------------

def create_project(request: Request) -> Response:
    """``POST /hux/v1/projects``."""
    body = pick(body_dict(request), PROJECT_FIELDS)
    stamp = now_iso()
    record = {
        "schema": "hux.project.v1", "id": new_id("prj"), "owner": request.identity.subject,
        "tags": [], "pinned": False, "archived": False, **body, "created_at": stamp, "updated_at": stamp,
    }
    stored, status = create(request, PROJECTS, record, request.idempotency_key(), MAX_PROJECTS)
    request.audit("projects.create", stored["id"], reason="replayed" if status == 200 else "")
    return Response(status, stored, etag(stored))


def list_projects(request: Request) -> Response:
    """``GET /hux/v1/projects?archived=``: pinned first, then most recently updated."""
    archived = request.query.get("archived")
    items = [p for p in request.store.scan(PROJECTS) if archived is None or p["archived"] == (archived == "true")]
    items.sort(key=lambda p: p["updated_at"], reverse=True)
    items.sort(key=lambda p: not p["pinned"])
    request.audit("projects.list", "projects")
    return page(items)


def get_project(request: Request) -> Response:
    """``GET /hux/v1/projects/{id}``."""
    record = request.store.get(PROJECTS, check_id(request.params["id"]))
    request.audit("projects.read", record["id"])
    return Response(200, record, etag(record))


def patch_project(request: Request) -> Response:
    """``PATCH /hux/v1/projects/{id}`` with If-Match."""
    stored = update(request, PROJECTS, PROJECT_FIELDS)
    return Response(200, stored, etag(stored))


# -- conversations -------------------------------------------------------------

def new_conversation(request: Request, body: dict[str, Any], extra: dict[str, Any] | None = None) -> Response:
    """Build, validate and store a conversation from client fields plus server-set extras."""
    if body.get("project_id") is not None and not project_exists(request.store, body["project_id"]):
        raise NotFound("project not found")
    stamp = now_iso()
    record = {
        "schema": "hux.conversation.v1", "id": new_id("conv"), "owner": request.identity.subject,
        "tags": [], "pinned": False, "archived": False, **{k: v for k, v in body.items() if v is not None},
        **(extra or {}), "artifact_ids": [], "created_at": stamp, "updated_at": stamp,
    }
    stored, status = create(request, CONVERSATIONS, record, request.idempotency_key(), MAX_CONVERSATIONS)
    request.audit("conversations.create", stored["id"], reason="replayed" if status == 200 else "")
    return Response(status, stored, etag(stored))


def create_conversation(request: Request) -> Response:
    """``POST /hux/v1/conversations``."""
    return new_conversation(request, pick(body_dict(request), CONVERSATION_FIELDS))


def list_conversations(request: Request) -> Response:
    """``GET /hux/v1/conversations?project_id=&tag=&pinned=&archived=``: newest activity first."""
    q = request.query
    flags = {k: q[k] == "true" for k in ("pinned", "archived") if k in q}
    items = []
    for index, record in enumerate(request.store.scan(CONVERSATIONS)):
        if "project_id" in q and record.get("project_id") != q["project_id"]:
            continue
        if "tag" in q and q["tag"] not in record["tags"]:
            continue
        if any(record[k] != v for k, v in flags.items()):
            continue
        items.append((record.get("last_message_at", record["updated_at"]), index, record))
    items.sort(key=lambda item: item[:2], reverse=True)
    request.audit("conversations.list", "conversations")
    return page([record for _, _, record in items])


def get_conversation(request: Request) -> Response:
    """``GET /hux/v1/conversations/{id}``."""
    record = request.store.get(CONVERSATIONS, check_id(request.params["id"]))
    request.audit("conversations.read", record["id"])
    return Response(200, record, etag(record))


def patch_conversation(request: Request) -> Response:
    """``PATCH /hux/v1/conversations/{id}`` with If-Match."""
    stored = update(request, CONVERSATIONS, CONVERSATION_FIELDS)
    return Response(200, stored, etag(stored))


def branch_conversation(request: Request) -> Response:
    """``POST /hux/v1/conversations/{id}/branch``: fork at a message, keeping project, tags and mode."""
    body = body_dict(request)
    point = body.get("branch_point_message_id")
    if not isinstance(point, str) or not 1 <= len(point) <= 120:
        raise Invalid("branch_point_message_id required")
    parent = request.store.get(CONVERSATIONS, check_id(request.params["id"]))
    fields = {"title": body.get("title") or f"{parent['title']} (branch)"[:200], "tags": list(parent["tags"]),
              "project_id": parent.get("project_id"), "mode": parent.get("mode")}
    return new_conversation(request, fields, {"branch": {"parent_conversation_id": parent["id"], "branch_point_message_id": point}})


def lineage(request: Request) -> Response:
    """``GET /hux/v1/conversations/{id}/lineage``: ancestors root-first plus direct children."""
    record = request.store.get(CONVERSATIONS, check_id(request.params["id"]))
    ancestors: list[dict[str, Any]] = []
    cursor, seen = record, {record["id"]}
    while "branch" in cursor and len(ancestors) < 64:
        parent_id = cursor["branch"]["parent_conversation_id"]
        if parent_id in seen or not request.store.exists(CONVERSATIONS, parent_id):
            break
        cursor = request.store.get(CONVERSATIONS, parent_id)
        seen.add(parent_id)
        ancestors.insert(0, cursor)
    children = [c for c in request.store.scan(CONVERSATIONS) if c.get("branch", {}).get("parent_conversation_id") == record["id"]]
    request.audit("conversations.lineage", record["id"])
    return Response(200, {"conversation": record, "ancestors": ancestors, "children": children})


# -- search --------------------------------------------------------------------

def artifact_index(store: TenantStore) -> dict[str, list[str]]:
    """Conversation id -> titles of the artifacts filed under it (F13c): both the conversation's ``artifact_ids`` and artifacts that name the conversation."""
    try:
        from hux import artifacts
    except ModuleNotFoundError:
        return {}
    index: dict[str, list[str]] = {}
    for artifact in store.scan(artifacts.FAMILY):
        if isinstance(artifact.get("conversation_id"), str) and isinstance(artifact.get("title"), str):
            index.setdefault(artifact["conversation_id"], []).append(artifact["title"])
    for conversation in store.scan(CONVERSATIONS):
        ids = [i for i in conversation.get("artifact_ids", []) if isinstance(i, str)]
        titles = artifacts.artifact_titles(store, ids).values() if ids else []
        for title in titles:
            index.setdefault(conversation["id"], []).append(title)
    return index


def artifact_titles(store: TenantStore, conversation: dict[str, Any], index: dict[str, list[str]] | None = None) -> list[str]:
    """Artifact titles for a conversation via the artifacts lane, or nothing when it is absent."""
    index = artifact_index(store) if index is None else index
    return list(dict.fromkeys(index.get(conversation["id"], [])))


def mark_forgotten(store: TenantStore, conversation_id: str) -> bool:
    """Blank a forgotten conversation's title and tags and archive it (F9); False when there is no document."""
    with store.lock(CONVERSATIONS):
        if not conversation_exists(store, conversation_id):
            return False
        current = store.get(CONVERSATIONS, conversation_id)
        record = {**current, "title": "[forgotten]", "tags": [], "archived": True, "updated_at": now_iso()}
        store.put(CONVERSATIONS, checked(record), expected_revision=current["revision"])
    return True


def tokens(text: str) -> list[str]:
    """Lowercased alphanumeric terms."""
    return TOKEN_RE.findall(text.lower())


def score(record: dict[str, Any], terms: list[str], project_name: str, titles: list[str]) -> int:
    """Title hit beats tag hit beats project/artifact hit; every term must match somewhere."""
    fields = {"title": tokens(record["title"]), "tags": [t for tag in record["tags"] for t in tokens(tag)],
              "project_name": tokens(project_name), "artifact_titles": [t for title in titles for t in tokens(title)]}
    weight = {"title": 4, "tags": 3, "project_name": 2, "artifact_titles": 1}
    total = 0
    for term in terms:
        hit = sum(weight[f] * words.count(term) for f, words in fields.items())
        if not hit:
            return 0
        total += hit
    return total


def message_text_score(store: TenantStore, conversation: dict[str, Any], terms: list[str]) -> int:
    """Bounded match over this conversation's stored message events.

    Privacy wins: forgotten conversations were already excluded, private-mode
    conversations are never scanned, and restricted or fully redacted events
    stay invisible to search exactly as they are on the timeline.
    """
    from hux import events

    if events.is_private(store, conversation["id"]):
        return 0
    rows = store.read(events.FAMILY, conversation["id"])[-MESSAGE_SCAN_EVENTS:]
    total = 0
    for row in rows:
        if row.get("kind") not in MESSAGE_KINDS or row.get("sensitivity") == "restricted":
            continue
        if (row.get("redaction") or {}).get("level") == "full":
            continue
        words = tokens(str(row.get("summary", "")))
        total += sum(words.count(term) for term in terms)
    return total


def search(request: Request) -> Response:
    """``GET /hux/v1/search?q=&project_id=&include=&cursor=``: best match first.

    The default scope is the indexed fields only. ``include=message_text``
    additionally scans the stored message events of the most recently active
    ``MESSAGE_SCAN_CONVERSATIONS`` candidates (bounded, privacy-enforced) and
    pages deterministically: rank order is (score desc, updated_at desc, id),
    the cursor is the offset into that total order.
    """
    terms = tokens(request.query.get("q", "")[:200])
    if not terms:
        raise Invalid("q is required")
    include = request.query.get("include")
    if include not in (None, "message_text"):
        raise Invalid("include supports only message_text")
    cursor = request.query.get("cursor", "0")
    if not cursor.isdigit():
        raise Invalid("cursor must be a non-negative integer")
    project_filter = request.query.get("project_id")
    names = {p["id"]: p["name"] for p in request.store.scan(PROJECTS)}
    index = artifact_index(request.store)
    candidates = [
        record for record in request.store.scan(CONVERSATIONS)
        if not project_filter or record.get("project_id") == project_filter
    ]
    scanned = set()
    if include:
        recent = sorted(candidates, key=lambda r: (r["updated_at"], r["id"]), reverse=True)
        scanned = {record["id"] for record in recent[:MESSAGE_SCAN_CONVERSATIONS]}
    ranked = []
    for record in candidates:
        points = score(record, terms, names.get(record.get("project_id", ""), ""), artifact_titles(request.store, record, index))
        if record["id"] in scanned and not record.get("archived"):
            points += message_text_score(request.store, record, terms)
        if points:
            ranked.append((points, record))
    if include:
        # Paginated mode needs one total order; ids break updated_at ties.
        ranked.sort(key=lambda pair: (-pair[0], pair[1]["updated_at"], pair[1]["id"]))
    else:
        ranked.sort(key=lambda pair: (-pair[0], pair[1]["updated_at"]))
    request.audit("search.query", "conversations")
    if not include:
        return Response(200, {"items": [r for _, r in ranked], "next": None, "scores": {r["id"]: s for s, r in ranked},
                              "indexed": list(INDEXED), "not_indexed": list(NOT_INDEXED)})
    start = int(cursor)
    window = ranked[start : start + SEARCH_PAGE]
    return Response(200, {
        "items": [r for _, r in window], "scores": {r["id"]: s for s, r in window},
        "next": str(start + SEARCH_PAGE) if len(ranked) > start + SEARCH_PAGE else None,
        "indexed": list(INDEXED) + ["message_text"], "not_indexed": [],
        "message_scan_limit": MESSAGE_SCAN_CONVERSATIONS,
    })


def register(router: Router) -> None:
    """Attach HUX-03 routes."""
    router.add("POST", "/hux/v1/projects", CARD, "projects.create", create_project)
    router.add("GET", "/hux/v1/projects", CARD, "projects.list", list_projects)
    router.add("GET", "/hux/v1/projects/{id}", CARD, "projects.read", get_project)
    router.add("PATCH", "/hux/v1/projects/{id}", CARD, "projects.update", patch_project)
    router.add("POST", "/hux/v1/conversations", CARD, "conversations.create", create_conversation)
    router.add("GET", "/hux/v1/conversations", CARD, "conversations.list", list_conversations)
    router.add("GET", "/hux/v1/conversations/{id}", CARD, "conversations.read", get_conversation)
    router.add("PATCH", "/hux/v1/conversations/{id}", CARD, "conversations.update", patch_conversation)
    router.add("POST", "/hux/v1/conversations/{id}/branch", CARD, "conversations.branch", branch_conversation)
    router.add("GET", "/hux/v1/conversations/{id}/lineage", CARD, "conversations.lineage", lineage)
    router.add("GET", "/hux/v1/search", CARD, "search.query", search)