#!/usr/bin/env python3 """In-pod autonomous supervisor that sustains multi-hour unattended work. The deployed Hermes agent already decomposes objectives and iterates a within- card goal loop, but nothing in-pod spawns the *cross-card* review->repair-> re-review follow-ups after a card finishes; today that chain only exists as an external Claude Code "codex-shepherd" session on an operator's workstation, so unattended runs stall once the first implementation card completes. This supervisor closes that gap from inside the pod. It runs a bounded poll loop that reads board state through ``hermes_cli.kanban_db`` and, for each terminal card, applies exactly one follow-up: * a done implementation with a produced PR/commit and no existing review -> create a review card (routed to a review assignee via the normal cli lane); * a review card that returned SHIP -> mark the implementation ready for a human to merge (a comment/flag; the supervisor never merges, approves, or clears WIP); * a review card that returned BLOCK -> create a bounded repair card; * a repair card that produced a new commit -> create the re-review card. All decision logic lives in :mod:`supervisor_policy` (pure, unit tested). This module is only the I/O shell: config, board iteration, and turning a :class:`supervisor_policy.Decision` into ``create_task``/``block_task``/ ``add_comment`` calls. It imports no provider client and holds no metered path; the sole spawn is a Kanban card that routes through the existing subscription lanes. It is inert until ``kanban.auto_supervise`` is set true in the deployed config, re-read every tick like ``kanban.auto_decompose``. """ from __future__ import annotations import json import os import re import sys import time from dataclasses import dataclass from pathlib import Path from typing import Any, Callable import yaml import supervisor_policy as policy import scm_broker_client import supervisor_state DATA_ROOT = Path(os.environ.get("HERMES_HOME", "/opt/data")) CONFIG_PATH = DATA_ROOT / "config.yaml" # Persistent record of terminal (ship / escalate) emissions so each fires at # most once across the stateless poll loop, and survives a pod restart. LEDGER_PATH = DATA_ROOT / "supervisor" / "emitted.json" DEFAULT_INTERVAL_SECONDS = 30 DEFAULT_MAX_CYCLES = 5 DEFAULT_MAX_CHAINS = 20 DEFAULT_REVIEW_ASSIGNEE = "cli-auto" DEFAULT_REPAIR_ASSIGNEE = "cli-auto" PR_URL = re.compile( r"https://scm\.bstein\.dev/titan/(?P[A-Za-z0-9][A-Za-z0-9_.-]{0,99})/pulls/(?P[1-9][0-9]{0,9})\Z" ) @dataclass(frozen=True) class Settings: """Per-tick configuration, re-read from the deployed config each pass.""" enabled: bool interval: int limits: policy.Limits def _kanban_config() -> dict[str, Any]: try: document = yaml.safe_load(CONFIG_PATH.read_text(encoding="utf-8")) except (OSError, yaml.YAMLError): return {} kanban = document.get("kanban") if isinstance(document, dict) else None return kanban if isinstance(kanban, dict) else {} def _bool(value: Any, default: bool) -> bool: return value if isinstance(value, bool) else default def _positive_int(value: Any, default: int) -> int: if isinstance(value, bool) or not isinstance(value, (int, float)): return default parsed = int(value) return parsed if parsed > 0 else default def _text(value: Any, default: str) -> str: return value if isinstance(value, str) and value.strip() else default def load_settings() -> Settings: """Read the ``kanban`` config block; default to inert and conservative.""" cfg = _kanban_config() return Settings( enabled=_bool(cfg.get("auto_supervise"), False), interval=_positive_int( cfg.get("supervise_interval_seconds"), DEFAULT_INTERVAL_SECONDS ), limits=policy.Limits( max_cycles=_positive_int(cfg.get("supervise_max_cycles"), DEFAULT_MAX_CYCLES), max_chains=_positive_int(cfg.get("supervise_max_chains"), DEFAULT_MAX_CHAINS), review_assignee=_text( cfg.get("supervise_review_assignee"), DEFAULT_REVIEW_ASSIGNEE ), repair_assignee=_text( cfg.get("supervise_repair_assignee"), DEFAULT_REPAIR_ASSIGNEE ), ), ) def _board_slug(board: Any) -> str: if isinstance(board, dict): return str(board.get("slug") or board.get("id") or "") return str(getattr(board, "slug", None) or getattr(board, "id", None) or board or "") def _log(message: str) -> None: print(f"kanban-supervisor: {message}", file=sys.stderr, flush=True) class Ledger: """Durable set of emission keys the supervisor itself has already fired. The poll loop is stateless and re-plans every terminal card each tick, so a SHIP or a cycle-limit escalation would otherwise re-comment its target on every pass. This ledger is the authoritative read-before-write guard: an emission is performed only when its key is absent, then recorded, so each fires at most once regardless of whether the underlying ``kanban_db`` moves the source card out of its done state. """ def __init__(self, path: Path): self._path = path self._keys = self._load(path) @staticmethod def _load(path: Path) -> set[str]: try: data = json.loads(path.read_text(encoding="utf-8")) except (OSError, ValueError): return set() return {str(key) for key in data} if isinstance(data, list) else set() def has(self, key: str) -> bool: return key in self._keys def record(self, key: str) -> None: self._keys.add(key) try: self._path.parent.mkdir(parents=True, exist_ok=True) tmp = self._path.with_suffix(".tmp") tmp.write_text(json.dumps(sorted(self._keys)), encoding="utf-8") os.replace(tmp, self._path) except OSError as error: _log(f"could not persist emission ledger: {error}") def _iter_boards(kanban_db: Any) -> list[str]: try: boards = kanban_db.list_boards(include_archived=False) except Exception as error: # noqa: BLE001 - one bad registry never stops the tick _log(f"could not list boards: {error}") return [] slugs = [_board_slug(raw) for raw in boards] return [slug for slug in slugs if slug] def _comment(kanban_db: Any, conn: Any, task_id: str, body: str) -> None: try: kanban_db.add_comment(conn, task_id, policy.SUPERVISOR_AUTHOR, body) except Exception as error: # noqa: BLE001 - a comment is best-effort signalling _log(f"could not comment on {task_id}: {error}") def _with_supervisor_state(kanban_db: Any, conn: Any, board: str, task: Any) -> Any: """Overlay durable links and coordinator lineage on a native task row.""" task_id = str(getattr(task, "id", "") if not isinstance(task, dict) else task.get("id", "")) if not task_id: return task values = dict(task) if isinstance(task, dict) else dict(vars(task)) parent_ids = getattr(kanban_db, "parent_ids", None) if callable(parent_ids): try: raw_parents = parent_ids(conn, task_id) except Exception as error: # noqa: BLE001 - a missing link proof may duplicate work raise RuntimeError("native supervisor parent links are unavailable") from error if not isinstance(raw_parents, (list, tuple, set)): raise RuntimeError("native supervisor parent links are malformed") values["parents"] = [str(parent) for parent in raw_parents if isinstance(parent, (str, int))] try: root = supervisor_state.get_root(board, task_id) child = supervisor_state.get_child(board, task_id) except (OSError, ValueError) as error: # A missing row is represented as None. Any read failure or malformed # present row is authority loss, so stop this board rather than treating # a possible continuation as a new root task. raise RuntimeError("supervisor integrity state is unavailable") from error if root is None and child is None: return values metadata = dict(values.get("metadata") or {}) if root is not None: metadata["supervisor_lineage"] = root.stamp_fields() if child is not None: chain = child["lineage"] metadata["supervisor"] = { "kind": child["kind"], "root": child["root_task_id"], "parent": child["parent_task_id"], "head_commit": child["head_commit"], "cycle": child["cycle"], **chain.stamp_fields(), } values["metadata"] = metadata return values def _set_ready_metadata(kanban_db: Any, conn: Any, task_id: str, metadata: dict[str, Any]) -> None: """Persist an evidence-bound readiness state when the board API supports it.""" setter = getattr(kanban_db, "set_task_metadata", None) or getattr( kanban_db, "update_task_metadata", None ) if callable(setter): try: setter(conn, task_id, metadata) except Exception as error: # noqa: BLE001 - the comment is durable signalling _log(f"could not set ready flag on {task_id}: {error}") def _mark_ready_for_human( kanban_db: Any, conn: Any, task_id: str, body: str, evidence: dict[str, Any] | None = None ) -> None: """Flag the implementation as human-mergeable. Never merges or clears WIP.""" evidence = evidence or {} _set_ready_metadata( kanban_db, conn, task_id, {"supervisor_ready_for_human_merge": True, "supervisor_ready_commit": str(evidence.get("commit") or ""), "supervisor_ready_pull_request": str(evidence.get("pr") or "")}, ) _comment(kanban_db, conn, task_id, body) def _clear_ready(kanban_db: Any, conn: Any, task_id: str, current_commit: str, body: str) -> None: """Invalidate a readiness flag when a later verified revision supersedes it.""" _set_ready_metadata( kanban_db, conn, task_id, {"supervisor_ready_for_human_merge": False, "supervisor_ready_commit": current_commit}, ) _comment(kanban_db, conn, task_id, body) def _live_pr_matches(evidence: dict[str, Any]) -> tuple[bool | None, str]: """Read the canonical PR before SHIP; ``None`` defers on any unavailable proof.""" project = evidence.get("project") branch = evidence.get("branch") base = evidence.get("base_branch") commit = evidence.get("commit") pull = evidence.get("pr") if not all(isinstance(value, str) and value for value in (project, branch, base, commit, pull)): return None, "SHIP lacks complete trusted PR evidence" matched = PR_URL.fullmatch(pull) if matched is None or matched.group("project") != project: return None, "SHIP pull-request URL is not canonical trusted lineage" try: document = json.loads(scm_broker_client.read( f"/api/v1/repos/titan/{project}/pulls/{matched.group('number')}" )) except Exception as error: # noqa: BLE001 - read proof failure must defer readiness return None, f"current PR evidence is unavailable: {type(error).__name__}" if not isinstance(document, dict): return None, "current PR evidence is malformed" full_name = f"titan/{project}" head = document.get("head") pr_base = document.get("base") matches = ( document.get("state") == "open" and isinstance(head, dict) and isinstance(pr_base, dict) and head.get("ref") == branch and head.get("sha") == commit and isinstance(head.get("repo"), dict) and head["repo"].get("full_name") == full_name and pr_base.get("ref") == base and isinstance(pr_base.get("repo"), dict) and pr_base["repo"].get("full_name") == full_name ) return matches, "current PR does not match reviewed branch/base/head" if not matches else "" def _escalate(kanban_db: Any, conn: Any, task_id: str, reason: str) -> None: body = ( f"supervisor fail-closed escalation: {reason}. Human attention required; " "no follow-up card was auto-created." ) try: kanban_db.block_task(conn, task_id, reason=body, kind="needs_input") except Exception as error: # noqa: BLE001 - still record the human-visible comment _log(f"could not block {task_id}: {error}") _comment(kanban_db, conn, task_id, body) def _already_created(kanban_db: Any, conn: Any, payload: dict[str, Any], board: str = "") -> bool: """True if a card matching this spawn payload's stamp already exists.""" supervisor_stamp = (payload.get("metadata") or {}).get("supervisor") or {} kind = supervisor_stamp.get("kind") root = str(supervisor_stamp.get("root") or "") commit = str(supervisor_stamp.get("head_commit") or "") if not kind or not root: return False try: tasks = [ _with_supervisor_state(kanban_db, conn, board, task) for task in kanban_db.list_tasks(conn) ] except Exception as error: # noqa: BLE001 - never retry an unverified create _log(f"could not re-scan before spawn retry: {error}") raise RuntimeError("cannot verify existing follow-up") from error return policy.existing_followup(tasks, kind, root, commit) def _prevalidate_spawn(board: str, stamp: dict[str, Any]) -> None: """Refuse a native create until its root is in durable coordinator state.""" if not board or not stamp: return expected = policy.lineage.from_stamp(stamp) root_id = str(stamp.get("root") or "") actual = supervisor_state.get_root(board, root_id) if expected is None or actual != expected: raise RuntimeError("supervisor child has no matching trusted root lineage") def _spawn(kanban_db: Any, conn: Any, decision: policy.Decision, board: str = "") -> None: payload = dict(decision.payload or {}) stamp = (payload.get("metadata") or {}).get("supervisor") or {} _prevalidate_spawn(board, stamp) try: child_id = kanban_db.create_task(conn, **payload) except TypeError: # create_task may reject idempotency_key on older runtimes. That is # raised at call binding, before any insert, but a post-insert TypeError # is also possible, so re-run the dedup scan first: only retry the create # when no matching card exists, so a partial insert is never doubled. if _already_created(kanban_db, conn, decision.payload or {}, board): return # Native Hermes has no task metadata column. The state table records # this stamp after creation; old compatible test/runtime APIs may still # accept it on the first attempt. payload.pop("metadata", None) try: child_id = kanban_db.create_task(conn, **payload) except TypeError: if _already_created(kanban_db, conn, decision.payload or {}, board): return payload.pop("idempotency_key", None) child_id = kanban_db.create_task(conn, **payload) if board and isinstance(child_id, str) and stamp: try: supervisor_state.record_child( board, child_id, str(stamp["root"]), str(stamp["parent"]), str(stamp["kind"]), str(stamp["head_commit"]), str(payload.get("body") or ""), int(stamp["cycle"]), ) except (KeyError, TypeError, ValueError) as error: raise RuntimeError(f"could not persist supervisor child authority: {error}") from error _comment(kanban_db, conn, decision.target_id, f"supervisor: {decision.reason}") def apply_decision( kanban_db: Any, conn: Any, decision: policy.Decision, ledger: Ledger, board: str = "" ) -> bool: """Execute one decision. Returns True when an action was taken. Spawns are self-limiting across ticks (the created card fails the next dedup scan). SHIP and escalate targets stay in their done state, so they are gated on the persistent ledger and fire at most once per source card. """ if decision.action == "none": return False policy.assert_safe(decision) if decision.action == "spawn": _spawn(kanban_db, conn, decision, board) return True info = decision.payload or {} key = f"{decision.action}:{decision.target_id}:{info.get('commit', '')}" if ledger.has(key): return False # already emitted on a prior tick; never re-emit or re-block if decision.action == "ship": live, reason = _live_pr_matches(info) if live is None: _log(f"deferring SHIP for {decision.target_id}: {reason}") return False if not live: _clear_ready( kanban_db, conn, decision.target_id, "", f"supervisor: readiness cleared because {reason}.", ) ledger.record(f"clear_ready:{decision.target_id}:{info.get('commit', '')}") return True commit = info.get("commit", "") pr = info.get("pr", "") or "branch on record" body = ( f"supervisor: review returned SHIP for commit {commit} ({pr}). " "READY FOR HUMAN MERGE - a human must merge; the supervisor never " "merges, approves, closes, or clears WIP." ) _mark_ready_for_human(kanban_db, conn, decision.target_id, body, info) elif decision.action == "clear_ready": current = str(info.get("commit") or "") _clear_ready( kanban_db, conn, decision.target_id, current, f"supervisor: stale approval cleared; current verified head is {current}.", ) else: # escalate _escalate(kanban_db, conn, decision.target_id, decision.reason) ledger.record(key) return True def supervise_board( kanban_db: Any, board: str, limits: policy.Limits, ledger: Ledger ) -> int: actions = 0 with kanban_db.scoped_current_board(board): conn = kanban_db.connect(board=board) try: tasks = [ _with_supervisor_state(kanban_db, conn, board, task) for task in kanban_db.list_tasks(conn) ] for task in tasks: decision = policy.plan(task, tasks, limits) try: if apply_decision(kanban_db, conn, decision, ledger, board): actions += 1 except Exception as error: # noqa: BLE001 - one card never stops others _log(f"action failed on board {board!r}: {error}") finally: conn.close() return actions def supervise_once( kanban_db: Any, limits: policy.Limits, ledger: Ledger | None = None ) -> int: """One bounded pass over every non-archived board.""" if ledger is None: ledger = Ledger(LEDGER_PATH) total = 0 for board in _iter_boards(kanban_db): try: total += supervise_board(kanban_db, board, limits, ledger) except Exception as error: # noqa: BLE001 - isolate a faulty board _log(f"temporarily skipping board {board!r}: {error}") return total def run_forever( kanban_db: Any, *, sleep: Callable[[float], None] = time.sleep, load: Callable[[], Settings] = load_settings, max_ticks: int | None = None, ) -> int: """Poll loop. Inert while ``auto_supervise`` is false; re-reads config a tick.""" ticks = 0 while max_ticks is None or ticks < max_ticks: settings = load() if settings.enabled: try: supervise_once(kanban_db, settings.limits) except Exception as error: # noqa: BLE001 - never let one tick kill the loop _log(f"tick failed: {error}") sleep(settings.interval) ticks += 1 return ticks def main(argv: list[str] | None = None) -> int: from hermes_cli import kanban_db run_forever(kanban_db) return 0 if __name__ == "__main__": # pragma: no cover - process entry point sys.exit(main(sys.argv[1:]))