atlas-iac/services/hermes/scripts/cli_lane_retention.py

#!/usr/bin/env python3
"""Bounded age, count, and byte retention for non-pending lane artifacts."""

from __future__ import annotations

import os
import stat
import sys
import time
from pathlib import Path

from cli_lane_config import (
    ARTIFACT_GC_INTERVAL_SECONDS,
    ARTIFACT_RETENTION_AGE_SECONDS,
    ARTIFACT_RETENTION_BYTES,
    ARTIFACT_RETENTION_COUNT,
    STATE_ROOT,
)
from cli_lane_evidence import _retire_terminal_entry, _terminal_evidence_valid
from cli_lane_files import (
    _fsync_directory,
    _terminal_evidence_identity,
    _terminal_identity,
)
from cli_lane_quarantine import _quarantine_terminal
from cli_lane_recovery import _staged_terminal_authority
from cli_lane_records import _open_terminal_recovery_snapshot, _terminal_record_valid


def _artifact_gc_candidates(board_dir: Path) -> list[Path]:
    """List only bounded artifact classes; pending journals are excluded."""
    candidates: list[Path] = []
    patterns = (
        "*.provider-*.result.json",
        "*.candidate-*.json",
        "*.terminal.committed.json",
        "*.terminal.conflict-*.json",
        ".retire.*",
    )
    for pattern in patterns:
        candidates.extend(board_dir.glob(pattern))
    quarantine = board_dir / "quarantine"
    if quarantine.is_dir():
        candidates.extend(quarantine.glob("*.quarantine"))
        candidates.extend(quarantine.glob(".retire.*"))
    return candidates

def _unlink_artifact_if_same(path: Path, observed: os.stat_result) -> bool:
    """Remove only the exact artifact inode selected by retention."""
    directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
    directory_flags |= getattr(os, "O_NOFOLLOW", 0)
    descriptor = None
    source_descriptor = None
    try:
        descriptor = os.open(path.parent, directory_flags)
        file_flags = getattr(os, "O_PATH", os.O_RDONLY) | getattr(os, "O_NOFOLLOW", 0)
        source_descriptor = os.open(path.name, file_flags, dir_fd=descriptor)
        pinned = os.fstat(source_descriptor)
        if pinned.st_dev != observed.st_dev or pinned.st_ino != observed.st_ino:
            return False
        outcome = _retire_terminal_entry(
            path,
            observed,
            board_descriptor=descriptor,
            quarantine_descriptor=descriptor,
            source_descriptor=source_descriptor,
        )
        return outcome == "retired"
    except OSError:
        return False
    finally:
        if source_descriptor is not None:
            os.close(source_descriptor)
        if descriptor is not None:
            os.close(descriptor)


def _quarantine_invalid_retained_terminal(path: Path) -> tuple[bool, bool]:
    """Audit retained committed/conflict evidence on the bounded GC cadence."""
    committed = path.name.endswith(".terminal.committed.json")
    conflict = ".terminal.conflict-" in path.name and path.name.endswith(".json")
    if not committed and not conflict:
        return False, False
    identity = (
        _terminal_identity(path) if committed else _terminal_evidence_identity(path)
    )
    snapshot = _open_terminal_recovery_snapshot(path)
    if snapshot is None:
        return True, False
    document = snapshot.document
    valid = False
    if identity is not None and document is not None:
        valid = (
            _terminal_record_valid(document, identity)
            if committed
            else _terminal_evidence_valid(document, identity, "conflict")
        )
    if valid:
        snapshot.close()
        return False, False
    try:
        _quarantine_terminal(
            path,
            identity,
            snapshot.invalid_reason or "invalid-retained-terminal-evidence",
            snapshot=snapshot,
        )
    finally:
        snapshot.close()
    try:
        path.stat(follow_symlinks=False)
    except FileNotFoundError:
        return True, True
    except OSError:
        return True, False
    return True, False

def gc_lane_artifacts(
    *,
    now: float | None = None,
    max_age_seconds: int = ARTIFACT_RETENTION_AGE_SECONDS,
    max_count: int = ARTIFACT_RETENTION_COUNT,
    max_bytes: int = ARTIFACT_RETENTION_BYTES,
) -> int:
    """Bound non-pending lane artifacts by age, count, and bytes per board."""
    current = time.time() if now is None else now
    removed = 0
    for board_dir in STATE_ROOT.iterdir() if STATE_ROOT.is_dir() else ():
        if not board_dir.is_dir() or board_dir.is_symlink():
            continue
        board_removed = 0
        entries: list[tuple[Path, os.stat_result]] = []
        for path in _artifact_gc_candidates(board_dir):
            classified, quarantined = _quarantine_invalid_retained_terminal(path)
            if classified:
                if quarantined:
                    removed += 1
                    board_removed += 1
                continue
            if path.name.startswith(".retire."):
                snapshot = _open_terminal_recovery_snapshot(path)
                if snapshot is not None:
                    try:
                        if _staged_terminal_authority(path, snapshot) is not None:
                            # Accepted authority is first promoted/replayed by
                            # recovery; retention never deletes it directly.
                            continue
                    finally:
                        snapshot.close()
            try:
                file_stat = path.stat(follow_symlinks=False)
            except OSError:
                continue
            if not stat.S_ISREG(file_stat.st_mode):
                if _unlink_artifact_if_same(path, file_stat):
                    removed += 1
                    board_removed += 1
                continue
            if max_age_seconds >= 0 and current - file_stat.st_mtime > max_age_seconds:
                if _unlink_artifact_if_same(path, file_stat):
                    removed += 1
                    board_removed += 1
            else:
                entries.append((path, file_stat))
        retained_count = 0
        retained_bytes = 0
        for path, file_stat in sorted(
            entries, key=lambda item: item[1].st_mtime, reverse=True
        ):
            exceeds_count = max_count >= 0 and retained_count >= max_count
            exceeds_bytes = (
                max_bytes >= 0 and retained_bytes + file_stat.st_size > max_bytes
            )
            if exceeds_count or exceeds_bytes:
                if _unlink_artifact_if_same(path, file_stat):
                    removed += 1
                    board_removed += 1
                continue
            retained_count += 1
            retained_bytes += file_stat.st_size
        if board_removed:
            _fsync_directory(board_dir)
    return removed

LAST_ARTIFACT_GC = 0.0

def maybe_gc_lane_artifacts(*, now: float | None = None) -> int:
    """Run artifact retention on a bounded cadence, not every dispatcher tick."""
    global LAST_ARTIFACT_GC
    current = time.time() if now is None else now
    if current - LAST_ARTIFACT_GC < ARTIFACT_GC_INTERVAL_SECONDS:
        return 0
    LAST_ARTIFACT_GC = current
    try:
        return gc_lane_artifacts(now=current)
    except OSError as error:
        print(
            f"lane artifact retention deferred: {type(error).__name__}: {error}",
            file=sys.stderr,
            flush=True,
        )
        return 0