236 lines
12 KiB
Python
236 lines
12 KiB
Python
"""Redaction pipeline shared by every HUX family (SO-11..SO-14, SO-19).
|
|
|
|
Order matters and is fixed here: drop detail keys outside the per-kind
|
|
allowlist, scrub secret patterns from every remaining string, cap sizes,
|
|
then derive ``redaction.level``. Serve-time redaction (``redact_record``) is
|
|
applied again on the way out so a reader that ignores the level still never
|
|
sees more than the surface is allowed to.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import re
|
|
from typing import Any
|
|
|
|
DETAIL_CAP_BYTES = 32 * 1024
|
|
LINE_CAP_BYTES = 64 * 1024
|
|
SUMMARY_MAX = 280
|
|
URI_SCHEMES = ("https://", "hux://", "artifact://")
|
|
WORKSPACE_PREFIX = "/opt/data/workspace"
|
|
HASH_KEYS = frozenset({"hash", "argument_hash"})
|
|
HUMAN_SURFACES = frozenset({"telegram", "voice"})
|
|
FULL_KEEP = frozenset({"schema", "id", "seq", "ts", "conversation_id", "kind", "provenance", "sensitivity", "redaction", "turn", "identity"})
|
|
|
|
# Each pattern is (class, regex). Classes end up in the placeholder so a
|
|
# reviewer can tell what kind of thing was removed without seeing it.
|
|
SECRET_PATTERNS: tuple[tuple[str, re.Pattern[str]], ...] = (
|
|
("private_key", re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?(?:-----END [A-Z ]*PRIVATE KEY-----|$)")),
|
|
("bearer", re.compile(r"(?i)bearer\s+[A-Za-z0-9._-]{16,}")),
|
|
("jwt", re.compile(r"eyJ[A-Za-z0-9_-]{20,}\.[A-Za-z0-9_-]{10,}(?:\.[A-Za-z0-9_-]+)?")),
|
|
("openai_key", re.compile(r"sk-[A-Za-z0-9_-]{20,}")),
|
|
("github_token", re.compile(r"(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{20,}")),
|
|
("gitlab_token", re.compile(r"glpat-[A-Za-z0-9_-]{20,}")),
|
|
("aws_key", re.compile(r"AKIA[0-9A-Z]{16}")),
|
|
("slack_token", re.compile(r"xox[abp]-[A-Za-z0-9-]{10,}")),
|
|
("vault_token", re.compile(r"\b(?:hvs|hvb|hvr)\.[A-Za-z0-9_-]{20,}")),
|
|
("password", re.compile(r"(?i)(?:password|passwd|pwd|secret|token)\s*[=:]\s*\S+")),
|
|
("hex", re.compile(r"\b[0-9a-fA-F]{32,}\b")),
|
|
("base64", re.compile(r"\b(?=[A-Za-z0-9+/]*[0-9])(?=[A-Za-z0-9+/]*[a-z])(?=[A-Za-z0-9+/]*[A-Z])[A-Za-z0-9+/]{40,}={0,2}")),
|
|
)
|
|
|
|
# Detail keys each event kind may persist. Everything else is dropped before
|
|
# a byte reaches disk, never filtered on read only.
|
|
DETAIL_ALLOWLIST: dict[str, frozenset[str]] = {
|
|
"message.user": frozenset({"message_id", "chars", "has_attachments"}),
|
|
"message.assistant": frozenset({"message_id", "chars", "has_attachments"}),
|
|
"decision.route": frozenset({"requested", "resolved_target", "provider", "effort", "reason"}),
|
|
"decision.plan": frozenset({"steps"}),
|
|
"tool.call": frozenset({"tool", "capability", "argument_names", "argument_hash", "argument_bytes", "target_path"}),
|
|
"tool.result": frozenset({"tool", "ok", "duration_ms", "bytes", "exit_code"}),
|
|
"approval.requested": frozenset({"approval_id", "capability", "choice", "external"}),
|
|
"approval.resolved": frozenset({"approval_id", "capability", "choice", "external"}),
|
|
"side_effect.blocked": frozenset({"approval_id", "capability", "choice", "external"}),
|
|
"side_effect.released": frozenset({"approval_id", "capability", "choice", "external"}),
|
|
"memory.proposed": frozenset({"memory_id", "kind", "sensitivity", "topic"}),
|
|
"memory.committed": frozenset({"memory_id", "kind", "sensitivity", "topic"}),
|
|
"memory.forgotten": frozenset({"memory_id", "kind", "sensitivity", "topic"}),
|
|
"memory.suppressed": frozenset({"memory_id", "kind", "sensitivity", "topic"}),
|
|
"memory.retrieval_removed": frozenset({"memory_id", "kind", "sensitivity", "topic"}),
|
|
"artifact.created": frozenset({"artifact_id", "version", "type", "bytes", "hash"}),
|
|
"artifact.version": frozenset({"artifact_id", "version", "type", "bytes", "hash"}),
|
|
"artifact.promoted": frozenset({"artifact_id", "version", "type", "bytes", "hash"}),
|
|
"run.started": frozenset({"run_id", "outcome", "receipt_id", "spent", "limits", "child_run_id"}),
|
|
"run.cancelled": frozenset({"run_id", "outcome", "receipt_id", "spent", "limits", "child_run_id"}),
|
|
"run.completed": frozenset({"run_id", "outcome", "receipt_id", "spent", "limits", "child_run_id"}),
|
|
"run.failed": frozenset({"run_id", "outcome", "receipt_id", "spent", "limits", "child_run_id"}),
|
|
"delegation.started": frozenset({"run_id", "outcome", "receipt_id", "spent", "limits", "child_run_id"}),
|
|
"delegation.completed": frozenset({"run_id", "outcome", "receipt_id", "spent", "limits", "child_run_id"}),
|
|
"delegation.failed": frozenset({"run_id", "outcome", "receipt_id", "spent", "limits", "child_run_id"}),
|
|
"budget.exhausted": frozenset({"run_id", "outcome", "receipt_id", "spent", "limits", "child_run_id"}),
|
|
"privacy.notice": frozenset({"schema", "topic", "conversation_id", "text", "controls", "shown_at"}),
|
|
"citation.attached": frozenset({"citation_id", "message_id", "passage_ids", "verdict"}),
|
|
"mode.changed": frozenset({"mode", "previous_mode", "route_id"}),
|
|
"suggestion.shown": frozenset({"suggestion_id", "action"}),
|
|
"suggestion.dismissed": frozenset({"suggestion_id", "action"}),
|
|
"release.transition": frozenset({"release_id", "state", "previous_state"}),
|
|
}
|
|
|
|
|
|
def canaries(environ: dict[str, str] | None = None) -> list[str]:
|
|
"""Literal secrets the pod was started with (SO-07); every one is scrubbed wherever it appears."""
|
|
environ = os.environ if environ is None else environ
|
|
names = ("HUX_ROUTER_KEY", "HUX_RELAY_KEY", "HUX_WORKER_KEY")
|
|
values = [environ.get(name, "") for name in names]
|
|
for name in names:
|
|
key_file = environ.get(f"{name}_FILE", "")
|
|
if key_file and os.path.exists(key_file):
|
|
with open(key_file, encoding="utf-8", errors="replace") as handle:
|
|
values.append(handle.read(4097).strip())
|
|
path = environ.get("HUX_CANARY_FILE", "")
|
|
if path and os.path.exists(path):
|
|
with open(path, encoding="utf-8", errors="replace") as handle:
|
|
for line in handle:
|
|
_, _, value = line.strip().partition("=")
|
|
values.append(value.strip().strip("'\""))
|
|
return [v for v in values if len(v) >= 8]
|
|
|
|
|
|
def scrub_text(text: str, environ: dict[str, str] | None = None) -> tuple[str, list[str]]:
|
|
"""Replace secret-looking substrings with ``[redacted:<class>]``; return the text and the classes hit."""
|
|
hits: list[str] = []
|
|
for canary in canaries(environ):
|
|
if canary in text:
|
|
text = text.replace(canary, "[redacted:canary]")
|
|
hits.append("canary")
|
|
for name, pattern in SECRET_PATTERNS:
|
|
text, count = pattern.subn(f"[redacted:{name}]", text)
|
|
if count:
|
|
hits.append(name)
|
|
return text, hits
|
|
|
|
|
|
def scrub_value(value: Any, hits: list[str], environ: dict[str, str] | None = None) -> Any:
|
|
"""Scrub every string inside a JSON-like value, collecting classes into ``hits``.
|
|
|
|
Content hashes (``hash``, ``argument_hash``) are digests by construction and
|
|
are kept verbatim so the hex rule does not eat them.
|
|
"""
|
|
if isinstance(value, str):
|
|
text, found = scrub_text(value, environ)
|
|
hits.extend(found)
|
|
return text
|
|
if isinstance(value, list):
|
|
return [scrub_value(item, hits, environ) for item in value]
|
|
if isinstance(value, dict):
|
|
return {
|
|
str(key): item if key in HASH_KEYS and isinstance(item, str) else scrub_value(item, hits, environ)
|
|
for key, item in value.items()
|
|
}
|
|
return value
|
|
|
|
|
|
def filter_detail(kind: str, detail: dict[str, Any] | None) -> dict[str, Any]:
|
|
"""Keep only the allowlisted keys for ``kind`` (SO-11) and apply the kind-specific shape rules."""
|
|
if not detail:
|
|
return {}
|
|
allowed = DETAIL_ALLOWLIST.get(kind, frozenset())
|
|
kept = {key: value for key, value in detail.items() if key in allowed}
|
|
if kind == "decision.plan" and "steps" in kept:
|
|
steps = kept["steps"] if isinstance(kept["steps"], list) else []
|
|
kept["steps"] = [str(step)[:200] for step in steps[:20]]
|
|
if kind == "tool.call":
|
|
# Raw arguments never persist: only their names, a hash and a byte count.
|
|
# F13d: normalise before the prefix check so ``workspace/../.env`` cannot
|
|
# pass as a workspace path; the stored value is the normalised one.
|
|
path = os.path.normpath(kept["target_path"]) if isinstance(kept.get("target_path"), str) else ""
|
|
if path.startswith(WORKSPACE_PREFIX + "/"):
|
|
kept["target_path"] = path
|
|
else:
|
|
kept.pop("target_path", None)
|
|
if "argument_names" in kept:
|
|
names = kept["argument_names"] if isinstance(kept["argument_names"], list) else []
|
|
kept["argument_names"] = sorted({str(name)[:80] for name in names})[:64]
|
|
return kept
|
|
|
|
|
|
def cap_detail(detail: dict[str, Any]) -> tuple[dict[str, Any], bool]:
|
|
"""Replace detail over 32 KiB with a truncation marker (SO-13)."""
|
|
size = len(json.dumps(detail, sort_keys=True, separators=(",", ":")).encode())
|
|
if size > DETAIL_CAP_BYTES:
|
|
return {"truncated": True, "bytes": size}, True
|
|
return detail, False
|
|
|
|
|
|
def filter_evidence(evidence: list[dict[str, Any]] | None) -> list[dict[str, Any]]:
|
|
"""Keep well-formed refs, drop URIs outside the allowlisted schemes (SO-19), cap at 64."""
|
|
kept: list[dict[str, Any]] = []
|
|
for ref in evidence or []:
|
|
if not isinstance(ref, dict) or not isinstance(ref.get("kind"), str) or not isinstance(ref.get("id"), str):
|
|
continue
|
|
clean = {"kind": ref["kind"], "id": ref["id"][:200]}
|
|
uri = ref.get("uri")
|
|
if isinstance(uri, str) and uri.startswith(URI_SCHEMES):
|
|
clean["uri"] = uri[:2000]
|
|
if isinstance(ref.get("hash"), str):
|
|
clean["hash"] = ref["hash"]
|
|
kept.append(clean)
|
|
return kept[:64]
|
|
|
|
|
|
def derive_level(sensitivity: str, hits: list[str], truncated: bool) -> dict[str, str]:
|
|
"""Redaction block for a new event from its sensitivity and what the pipeline did."""
|
|
reasons: list[str] = []
|
|
level = "none"
|
|
if sensitivity == "sensitive":
|
|
level, reasons = "partial", ["sensitive topic"]
|
|
if hits:
|
|
level = "partial" if level == "none" else level
|
|
reasons.append("scrubbed " + ",".join(sorted(set(hits)))[:120])
|
|
if truncated:
|
|
level = "partial" if level == "none" else level
|
|
reasons.append("detail truncated")
|
|
if sensitivity == "restricted":
|
|
level, reasons = "full", ["restricted content"]
|
|
block = {"level": level}
|
|
if reasons:
|
|
block["reason"] = "; ".join(reasons)[:200]
|
|
return block
|
|
|
|
|
|
def redact_record(record: dict[str, Any], surface: str) -> dict[str, Any]:
|
|
"""Serve-time redaction (SO-14) for any record that carries ``redaction``; strips ``_meta`` (SO-51).
|
|
|
|
``none`` is served as stored except that telegram and voice are lifted to
|
|
``partial``; ``partial`` drops ``detail``; ``full`` keeps only the
|
|
contract-required envelope with the summary replaced by a placeholder.
|
|
"""
|
|
served = {key: value for key, value in record.items() if key != "_meta"}
|
|
block = served.get("redaction")
|
|
if not isinstance(block, dict):
|
|
return served
|
|
level = block.get("level", "none")
|
|
if level == "none" and surface in HUMAN_SURFACES:
|
|
level = "partial"
|
|
served["redaction"] = {"level": "partial", "reason": f"{surface} surface never receives unredacted detail"}
|
|
if level == "partial":
|
|
served.pop("detail", None)
|
|
if level == "full":
|
|
served = {key: value for key, value in served.items() if key in FULL_KEEP}
|
|
if "summary" in record:
|
|
served["summary"] = "[redacted]"
|
|
if "content" in record:
|
|
served["content"] = ""
|
|
return served
|
|
|
|
|
|
def full_redaction(record: dict[str, Any], summary: str, reason: str) -> dict[str, Any]:
|
|
"""Rewrite a stored event to ``redaction.level: full`` in place (forget and topic decay)."""
|
|
kept = {key: value for key, value in record.items() if key in FULL_KEEP}
|
|
kept["summary"] = summary[:SUMMARY_MAX] or "[redacted]"
|
|
kept["redaction"] = {"level": "full", "reason": reason[:200]}
|
|
if "idempotency_key" in record:
|
|
kept["idempotency_key"] = record["idempotency_key"]
|
|
return kept
|