139 lines
5.2 KiB
Python
139 lines
5.2 KiB
Python
"""Bounded in-process HUX-10 retention scheduling for one tenant pod."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import threading
|
|
from collections.abc import Callable, Mapping
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from hux import privacy
|
|
from hux.errors import Unauthorized
|
|
from hux.identity import SLOT_RE, Identity, _read_subject_binding
|
|
from hux.store import TenantStore
|
|
|
|
DAY_SECONDS = 86_400.0
|
|
MAX_JITTER_SECONDS = 3_600
|
|
MAX_BACKOFF_SECONDS = 900.0
|
|
|
|
|
|
class BindingUnavailable(ValueError):
|
|
"""The projected subject binding is absent or unsafe."""
|
|
|
|
|
|
def subject_binding(environ: Mapping[str, str]) -> Identity:
|
|
"""Read the canonical edge-published binding and combine it with the configured tenant slot."""
|
|
slot = environ.get("HUX_TENANT_SLOT", "")
|
|
raw_path = environ.get("HUX_SUBJECT_BINDING_FILE", "")
|
|
if not SLOT_RE.fullmatch(slot) or not raw_path:
|
|
raise BindingUnavailable("binding configuration unavailable")
|
|
try:
|
|
subject = _read_subject_binding(Path(raw_path))
|
|
except Unauthorized as error:
|
|
raise BindingUnavailable("binding file unavailable") from error
|
|
if subject is None:
|
|
raise BindingUnavailable("binding file unavailable")
|
|
return Identity(slot, subject, "worker", "worker")
|
|
|
|
|
|
class RetentionScheduler:
|
|
"""Run privacy retention without leaving the API process or tenant lock domain."""
|
|
|
|
def __init__(
|
|
self,
|
|
router: Any,
|
|
*,
|
|
clock: Callable[[], datetime] | None = None,
|
|
runner: Callable[..., dict[str, Any]] | None = None,
|
|
waiter: Callable[[float], bool] | None = None,
|
|
daily_seconds: float = DAY_SECONDS,
|
|
) -> None:
|
|
self.router = router
|
|
self.clock = clock or (lambda: datetime.now(timezone.utc))
|
|
self.runner = runner or privacy.run_retention
|
|
self.daily_seconds = max(1.0, daily_seconds)
|
|
self._stop = threading.Event()
|
|
self.waiter = waiter or self._stop.wait
|
|
self._thread: threading.Thread | None = None
|
|
self._lock = threading.Lock()
|
|
self._health: dict[str, Any] = {
|
|
"enabled": bool(router.flags.enabled("HUX-10")),
|
|
"status": "starting" if router.flags.enabled("HUX-10") else "disabled",
|
|
"last_run": None,
|
|
"last_success": None,
|
|
"errors": 0,
|
|
}
|
|
|
|
def health(self) -> dict[str, Any]:
|
|
"""Return bounded, credential-free scheduler evidence."""
|
|
with self._lock:
|
|
return dict(self._health)
|
|
|
|
def _set(self, **changes: Any) -> None:
|
|
with self._lock:
|
|
self._health.update(changes)
|
|
|
|
def _jitter(self, identity: Identity) -> float:
|
|
try:
|
|
cap = int(self.router.environ.get("HUX_RETENTION_JITTER_SECONDS", "900"))
|
|
except ValueError:
|
|
cap = 900
|
|
cap = min(MAX_JITTER_SECONDS, max(0, cap))
|
|
if cap == 0:
|
|
return 0.0
|
|
digest = hashlib.sha256(f"{identity.tenant_slot}\0{identity.subject}".encode()).digest()
|
|
return float(int.from_bytes(digest[:4], "big") % (cap + 1))
|
|
|
|
@staticmethod
|
|
def _backoff(failures: int) -> float:
|
|
return min(MAX_BACKOFF_SECONDS, float(5 * (2 ** min(max(failures - 1, 0), 7))))
|
|
|
|
def run(self) -> None:
|
|
"""Wait for a binding, run stale startup work, then run once per day."""
|
|
if not self.health()["enabled"]:
|
|
return
|
|
failures = 0
|
|
first = True
|
|
while not self._stop.is_set():
|
|
try:
|
|
identity = subject_binding(self.router.environ)
|
|
tenant = TenantStore(self.router.data_root, identity)
|
|
now = self.clock()
|
|
ran = not first or privacy.audit_stale(tenant, now)
|
|
if ran:
|
|
self.runner(tenant, now, identity)
|
|
stamp = now.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
self._set(last_run=stamp, last_success=stamp)
|
|
failures = 0
|
|
first = False
|
|
self._set(status="healthy", errors=0)
|
|
if self.waiter(self.daily_seconds + self._jitter(identity)):
|
|
break
|
|
except BindingUnavailable:
|
|
failures += 1
|
|
self._set(status="waiting_for_binding", errors=failures)
|
|
if self.waiter(self._backoff(failures)):
|
|
break
|
|
except Exception: # noqa: BLE001 - background failure cannot take down the API
|
|
failures += 1
|
|
self._set(status="error", errors=failures)
|
|
if self.waiter(self._backoff(failures)):
|
|
break
|
|
self._set(status="stopped")
|
|
|
|
def start(self) -> None:
|
|
"""Start one daemon thread when HUX-10 is enabled."""
|
|
if not self.health()["enabled"] or self._thread is not None:
|
|
return
|
|
self._thread = threading.Thread(target=self.run, name="hux-retention", daemon=True)
|
|
self._thread.start()
|
|
|
|
def stop(self) -> None:
|
|
"""Wake and join the scheduler during server shutdown."""
|
|
self._stop.set()
|
|
if self._thread is not None and self._thread is not threading.current_thread():
|
|
self._thread.join(timeout=2.0)
|
|
self._set(status="stopped")
|