atlas-iac/testing/tests/test_hermes_hux_retention_scheduler.py

"""HUX-10 same-process retention scheduler lifecycle and failure bounds."""

from __future__ import annotations

import sys
import threading
from datetime import datetime, timezone
from http.client import HTTPConnection
from pathlib import Path

import pytest

ROOT = Path(__file__).resolve().parents[2]
FOUNDATION = ROOT / "dockerfiles" / "hermes-hux-foundation"
if str(FOUNDATION) not in sys.path:
    sys.path.insert(0, str(FOUNDATION))

from hux import identity, privacy
from hux.errors import Unauthorized
from hux.http import serve
from hux.retention_scheduler import RetentionScheduler, subject_binding
from hux.server import build_router
from hux.store import TenantStore

SUBJECT = "usr_0123456789abcdef"
NOW = datetime(2026, 8, 24, 12, 0, tzinfo=timezone.utc)
FLAGS = "hux.foundation,hux.privacy"


def binding(tmp_path: Path, subject: str = SUBJECT, mode: int = 0o440) -> Path:
    path = tmp_path / "subject-binding"
    path.write_text(subject + "\n")
    path.chmod(mode)
    return path


def router_for(tmp_path: Path, path: Path | None, flags: str = FLAGS):
    environ = {"HUX_FLAGS": flags, "HUX_TENANT_SLOT": "slot-3"}
    if path is not None:
        environ["HUX_SUBJECT_BINDING_FILE"] = str(path)
    return build_router(tmp_path / "data", environ)


def test_scheduler_runs_stale_startup_then_stops_from_daily_wait(tmp_path):
    router = router_for(tmp_path, binding(tmp_path))
    calls = []
    waits = []

    def run(store, now, who):
        calls.append((store.identity, now, who))
        return privacy.run_retention(store, now, who)

    scheduler = RetentionScheduler(
        router, clock=lambda: NOW, runner=run, waiter=lambda delay: waits.append(delay) or True, daily_seconds=100
    )
    scheduler.run()
    assert len(calls) == 1 and calls[0][0] == calls[0][2]
    assert calls[0][0] == identity.Identity("slot-3", SUBJECT, "worker", "worker")
    assert waits == [100 + scheduler._jitter(calls[0][0])]
    assert scheduler.health() == {
        "enabled": True,
        "status": "stopped",
        "last_run": "2026-08-24T12:00:00Z",
        "last_success": "2026-08-24T12:00:00Z",
        "errors": 0,
    }


def test_fresh_startup_skips_work_but_next_daily_cycle_runs(tmp_path):
    path = binding(tmp_path)
    router = router_for(tmp_path, path)
    who = subject_binding(router.environ)
    privacy.run_retention(TenantStore(router.data_root, who), NOW, who)
    calls = []
    waits = []

    def wait(delay):
        waits.append(delay)
        return len(waits) == 2

    scheduler = RetentionScheduler(router, clock=lambda: NOW, runner=lambda *args: calls.append(args) or {}, waiter=wait, daily_seconds=20)
    scheduler.run()
    assert len(calls) == 1 and len(waits) == 2
    assert all(delay == 20 + scheduler._jitter(who) for delay in waits)


def test_binding_and_job_errors_back_off_and_recover_without_api_failure(tmp_path):
    path = tmp_path / "subject-binding"
    router = router_for(tmp_path, path)
    waits = []
    attempts = []

    def wait(delay):
        waits.append(delay)
        if len(waits) == 1:
            binding(tmp_path)
        return len(waits) == 3

    def run(*args):
        attempts.append(args)
        if len(attempts) == 1:
            raise RuntimeError("sensitive provider-looking detail")
        return {}

    scheduler = RetentionScheduler(router, clock=lambda: NOW, runner=run, waiter=wait, daily_seconds=30)
    scheduler.run()
    assert len(attempts) == 2
    assert waits[:2] == [5.0, 10.0]
    assert waits[2] == 30 + scheduler._jitter(subject_binding(router.environ))
    health = scheduler.health()
    assert health["status"] == "stopped" and health["errors"] == 0
    assert "sensitive" not in str(health)


def test_clock_error_is_isolated_and_retried(tmp_path):
    router = router_for(tmp_path, binding(tmp_path))
    clocks = [RuntimeError("clock detail"), NOW]
    waits = []

    def clock():
        value = clocks.pop(0)
        if isinstance(value, Exception):
            raise value
        return value

    scheduler = RetentionScheduler(router, clock=clock, runner=lambda *args: {}, waiter=lambda delay: waits.append(delay) or len(waits) > 1)
    scheduler.run()
    assert waits[0] == 5.0 and len(waits) == 2
    assert scheduler.health()["last_success"] == "2026-08-24T12:00:00Z"


def test_server_close_wakes_and_joins_scheduler_and_health_exposes_state(tmp_path):
    router = router_for(tmp_path, None)
    server = serve(router, "127.0.0.1", 0)
    serving = threading.Thread(target=server.serve_forever, daemon=True)
    serving.start()
    conn = HTTPConnection("127.0.0.1", server.server_address[1], timeout=5)
    conn.request("GET", "/healthz")
    body = __import__("json").loads(conn.getresponse().read())
    assert body["status"] == "ok" and body["retention"]["enabled"] is True
    assert body["retention"]["status"] in {"starting", "waiting_for_binding"}
    server.shutdown()
    server.server_close()
    assert router.retention_scheduler.health()["status"] == "stopped"
    assert not router.retention_scheduler._thread.is_alive()


def test_scheduler_is_off_unless_hux10_dependency_chain_is_enabled(tmp_path):
    router = router_for(tmp_path, binding(tmp_path), "hux.foundation")
    scheduler = RetentionScheduler(router)
    scheduler.start()
    assert scheduler.health()["status"] == "disabled" and scheduler._thread is None
    scheduler.run()


def test_binding_reader_and_jitter_fail_closed_at_edge_cases(tmp_path, monkeypatch):
    path = binding(tmp_path, mode=0o600)
    router = router_for(tmp_path, path)
    with pytest.raises(Exception, match="binding file unavailable"):
        subject_binding(router.environ)
    path.chmod(0o440)
    router.environ["HUX_RETENTION_JITTER_SECONDS"] = "bad"
    scheduler = RetentionScheduler(router)
    who = subject_binding(router.environ)
    assert 0 <= scheduler._jitter(who) <= 900
    router.environ["HUX_RETENTION_JITTER_SECONDS"] = "0"
    assert scheduler._jitter(who) == 0
    monkeypatch.setattr(
        "hux.retention_scheduler._read_subject_binding",
        lambda unused: (_ for _ in ()).throw(Unauthorized("unsafe internal detail")),
    )
    with pytest.raises(Exception, match="binding file unavailable") as denied:
        subject_binding(router.environ)
    assert "unsafe internal" not in str(denied.value)


def test_scheduler_stop_is_idempotent_and_backoff_is_bounded(tmp_path):
    scheduler = RetentionScheduler(router_for(tmp_path, binding(tmp_path)))
    scheduler._thread = threading.current_thread()
    scheduler.stop()
    scheduler.stop()
    assert scheduler.health()["status"] == "stopped"
    assert scheduler._backoff(1000) == 640.0