486 lines
20 KiB
Python
486 lines
20 KiB
Python
#!/usr/bin/env python3
|
|
"""In-pod autonomous supervisor that sustains multi-hour unattended work.
|
|
|
|
The deployed Hermes agent already decomposes objectives and iterates a within-
|
|
card goal loop, but nothing in-pod spawns the *cross-card* review->repair->
|
|
re-review follow-ups after a card finishes; today that chain only exists as an
|
|
external Claude Code "codex-shepherd" session on an operator's workstation, so
|
|
unattended runs stall once the first implementation card completes. This
|
|
supervisor closes that gap from inside the pod.
|
|
|
|
It runs a bounded poll loop that reads board state through ``hermes_cli.kanban_db``
|
|
and, for each terminal card, applies exactly one follow-up:
|
|
|
|
* a done implementation with a produced PR/commit and no existing review ->
|
|
create a review card (routed to a review assignee via the normal cli lane);
|
|
* a review card that returned SHIP -> mark the implementation ready for a human
|
|
to merge (a comment/flag; the supervisor never merges, approves, or clears WIP);
|
|
* a review card that returned BLOCK -> create a bounded repair card;
|
|
* a repair card that produced a new commit -> create the re-review card.
|
|
|
|
All decision logic lives in :mod:`supervisor_policy` (pure, unit tested). This
|
|
module is only the I/O shell: config, board iteration, and turning a
|
|
:class:`supervisor_policy.Decision` into ``create_task``/``block_task``/
|
|
``add_comment`` calls. It imports no provider client and holds no metered path;
|
|
the sole spawn is a Kanban card that routes through the existing subscription
|
|
lanes. It is inert until ``kanban.auto_supervise`` is set true in the deployed
|
|
config, re-read every tick like ``kanban.auto_decompose``.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
import time
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Any, Callable
|
|
|
|
import yaml
|
|
|
|
import supervisor_policy as policy
|
|
import scm_broker_client
|
|
import supervisor_state
|
|
|
|
DATA_ROOT = Path(os.environ.get("HERMES_HOME", "/opt/data"))
|
|
CONFIG_PATH = DATA_ROOT / "config.yaml"
|
|
# Persistent record of terminal (ship / escalate) emissions so each fires at
|
|
# most once across the stateless poll loop, and survives a pod restart.
|
|
LEDGER_PATH = DATA_ROOT / "supervisor" / "emitted.json"
|
|
|
|
DEFAULT_INTERVAL_SECONDS = 30
|
|
DEFAULT_MAX_CYCLES = 5
|
|
DEFAULT_MAX_CHAINS = 20
|
|
DEFAULT_REVIEW_ASSIGNEE = "cli-auto"
|
|
DEFAULT_REPAIR_ASSIGNEE = "cli-auto"
|
|
PR_URL = re.compile(
|
|
r"https://scm\.bstein\.dev/titan/(?P<project>[A-Za-z0-9][A-Za-z0-9_.-]{0,99})/pulls/(?P<number>[1-9][0-9]{0,9})\Z"
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Settings:
|
|
"""Per-tick configuration, re-read from the deployed config each pass."""
|
|
|
|
enabled: bool
|
|
interval: int
|
|
limits: policy.Limits
|
|
|
|
|
|
def _kanban_config() -> dict[str, Any]:
|
|
try:
|
|
document = yaml.safe_load(CONFIG_PATH.read_text(encoding="utf-8"))
|
|
except (OSError, yaml.YAMLError):
|
|
return {}
|
|
kanban = document.get("kanban") if isinstance(document, dict) else None
|
|
return kanban if isinstance(kanban, dict) else {}
|
|
|
|
|
|
def _bool(value: Any, default: bool) -> bool:
|
|
return value if isinstance(value, bool) else default
|
|
|
|
|
|
def _positive_int(value: Any, default: int) -> int:
|
|
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
|
return default
|
|
parsed = int(value)
|
|
return parsed if parsed > 0 else default
|
|
|
|
|
|
def _text(value: Any, default: str) -> str:
|
|
return value if isinstance(value, str) and value.strip() else default
|
|
|
|
|
|
def load_settings() -> Settings:
|
|
"""Read the ``kanban`` config block; default to inert and conservative."""
|
|
cfg = _kanban_config()
|
|
return Settings(
|
|
enabled=_bool(cfg.get("auto_supervise"), False),
|
|
interval=_positive_int(
|
|
cfg.get("supervise_interval_seconds"), DEFAULT_INTERVAL_SECONDS
|
|
),
|
|
limits=policy.Limits(
|
|
max_cycles=_positive_int(cfg.get("supervise_max_cycles"), DEFAULT_MAX_CYCLES),
|
|
max_chains=_positive_int(cfg.get("supervise_max_chains"), DEFAULT_MAX_CHAINS),
|
|
review_assignee=_text(
|
|
cfg.get("supervise_review_assignee"), DEFAULT_REVIEW_ASSIGNEE
|
|
),
|
|
repair_assignee=_text(
|
|
cfg.get("supervise_repair_assignee"), DEFAULT_REPAIR_ASSIGNEE
|
|
),
|
|
),
|
|
)
|
|
|
|
|
|
def _board_slug(board: Any) -> str:
|
|
if isinstance(board, dict):
|
|
return str(board.get("slug") or board.get("id") or "")
|
|
return str(getattr(board, "slug", None) or getattr(board, "id", None) or board or "")
|
|
|
|
|
|
def _log(message: str) -> None:
|
|
print(f"kanban-supervisor: {message}", file=sys.stderr, flush=True)
|
|
|
|
|
|
class Ledger:
|
|
"""Durable set of emission keys the supervisor itself has already fired.
|
|
|
|
The poll loop is stateless and re-plans every terminal card each tick, so a
|
|
SHIP or a cycle-limit escalation would otherwise re-comment its target on
|
|
every pass. This ledger is the authoritative read-before-write guard: an
|
|
emission is performed only when its key is absent, then recorded, so each
|
|
fires at most once regardless of whether the underlying ``kanban_db`` moves
|
|
the source card out of its done state.
|
|
"""
|
|
|
|
def __init__(self, path: Path):
|
|
self._path = path
|
|
self._keys = self._load(path)
|
|
|
|
@staticmethod
|
|
def _load(path: Path) -> set[str]:
|
|
try:
|
|
data = json.loads(path.read_text(encoding="utf-8"))
|
|
except (OSError, ValueError):
|
|
return set()
|
|
return {str(key) for key in data} if isinstance(data, list) else set()
|
|
|
|
def has(self, key: str) -> bool:
|
|
return key in self._keys
|
|
|
|
def record(self, key: str) -> None:
|
|
self._keys.add(key)
|
|
try:
|
|
self._path.parent.mkdir(parents=True, exist_ok=True)
|
|
tmp = self._path.with_suffix(".tmp")
|
|
tmp.write_text(json.dumps(sorted(self._keys)), encoding="utf-8")
|
|
os.replace(tmp, self._path)
|
|
except OSError as error:
|
|
_log(f"could not persist emission ledger: {error}")
|
|
|
|
|
|
def _iter_boards(kanban_db: Any) -> list[str]:
|
|
try:
|
|
boards = kanban_db.list_boards(include_archived=False)
|
|
except Exception as error: # noqa: BLE001 - one bad registry never stops the tick
|
|
_log(f"could not list boards: {error}")
|
|
return []
|
|
slugs = [_board_slug(raw) for raw in boards]
|
|
return [slug for slug in slugs if slug]
|
|
|
|
|
|
def _comment(kanban_db: Any, conn: Any, task_id: str, body: str) -> None:
|
|
try:
|
|
kanban_db.add_comment(conn, task_id, policy.SUPERVISOR_AUTHOR, body)
|
|
except Exception as error: # noqa: BLE001 - a comment is best-effort signalling
|
|
_log(f"could not comment on {task_id}: {error}")
|
|
|
|
|
|
def _with_supervisor_state(kanban_db: Any, conn: Any, board: str, task: Any) -> Any:
|
|
"""Overlay durable links and coordinator lineage on a native task row."""
|
|
task_id = str(getattr(task, "id", "") if not isinstance(task, dict) else task.get("id", ""))
|
|
if not task_id:
|
|
return task
|
|
values = dict(task) if isinstance(task, dict) else dict(vars(task))
|
|
parent_ids = getattr(kanban_db, "parent_ids", None)
|
|
if callable(parent_ids):
|
|
try:
|
|
raw_parents = parent_ids(conn, task_id)
|
|
except Exception as error: # noqa: BLE001 - a missing link proof may duplicate work
|
|
raise RuntimeError("native supervisor parent links are unavailable") from error
|
|
if not isinstance(raw_parents, (list, tuple, set)):
|
|
raise RuntimeError("native supervisor parent links are malformed")
|
|
values["parents"] = [str(parent) for parent in raw_parents if isinstance(parent, (str, int))]
|
|
try:
|
|
root = supervisor_state.get_root(board, task_id)
|
|
child = supervisor_state.get_child(board, task_id)
|
|
except (OSError, ValueError) as error:
|
|
# A missing row is represented as None. Any read failure or malformed
|
|
# present row is authority loss, so stop this board rather than treating
|
|
# a possible continuation as a new root task.
|
|
raise RuntimeError("supervisor integrity state is unavailable") from error
|
|
if root is None and child is None:
|
|
return values
|
|
metadata = dict(values.get("metadata") or {})
|
|
if root is not None:
|
|
metadata["supervisor_lineage"] = root.stamp_fields()
|
|
if child is not None:
|
|
chain = child["lineage"]
|
|
metadata["supervisor"] = {
|
|
"kind": child["kind"], "root": child["root_task_id"],
|
|
"parent": child["parent_task_id"], "head_commit": child["head_commit"],
|
|
"cycle": child["cycle"], **chain.stamp_fields(),
|
|
}
|
|
values["metadata"] = metadata
|
|
return values
|
|
|
|
|
|
def _set_ready_metadata(kanban_db: Any, conn: Any, task_id: str, metadata: dict[str, Any]) -> None:
|
|
"""Persist an evidence-bound readiness state when the board API supports it."""
|
|
setter = getattr(kanban_db, "set_task_metadata", None) or getattr(
|
|
kanban_db, "update_task_metadata", None
|
|
)
|
|
if callable(setter):
|
|
try:
|
|
setter(conn, task_id, metadata)
|
|
except Exception as error: # noqa: BLE001 - the comment is durable signalling
|
|
_log(f"could not set ready flag on {task_id}: {error}")
|
|
|
|
|
|
def _mark_ready_for_human(
|
|
kanban_db: Any, conn: Any, task_id: str, body: str, evidence: dict[str, Any] | None = None
|
|
) -> None:
|
|
"""Flag the implementation as human-mergeable. Never merges or clears WIP."""
|
|
evidence = evidence or {}
|
|
_set_ready_metadata(
|
|
kanban_db, conn, task_id,
|
|
{"supervisor_ready_for_human_merge": True,
|
|
"supervisor_ready_commit": str(evidence.get("commit") or ""),
|
|
"supervisor_ready_pull_request": str(evidence.get("pr") or "")},
|
|
)
|
|
_comment(kanban_db, conn, task_id, body)
|
|
|
|
|
|
def _clear_ready(kanban_db: Any, conn: Any, task_id: str, current_commit: str, body: str) -> None:
|
|
"""Invalidate a readiness flag when a later verified revision supersedes it."""
|
|
_set_ready_metadata(
|
|
kanban_db, conn, task_id,
|
|
{"supervisor_ready_for_human_merge": False, "supervisor_ready_commit": current_commit},
|
|
)
|
|
_comment(kanban_db, conn, task_id, body)
|
|
|
|
|
|
def _live_pr_matches(evidence: dict[str, Any]) -> tuple[bool | None, str]:
|
|
"""Read the canonical PR before SHIP; ``None`` defers on any unavailable proof."""
|
|
project = evidence.get("project")
|
|
branch = evidence.get("branch")
|
|
base = evidence.get("base_branch")
|
|
commit = evidence.get("commit")
|
|
pull = evidence.get("pr")
|
|
if not all(isinstance(value, str) and value for value in (project, branch, base, commit, pull)):
|
|
return None, "SHIP lacks complete trusted PR evidence"
|
|
matched = PR_URL.fullmatch(pull)
|
|
if matched is None or matched.group("project") != project:
|
|
return None, "SHIP pull-request URL is not canonical trusted lineage"
|
|
try:
|
|
document = json.loads(scm_broker_client.read(
|
|
f"/api/v1/repos/titan/{project}/pulls/{matched.group('number')}"
|
|
))
|
|
except Exception as error: # noqa: BLE001 - read proof failure must defer readiness
|
|
return None, f"current PR evidence is unavailable: {type(error).__name__}"
|
|
if not isinstance(document, dict):
|
|
return None, "current PR evidence is malformed"
|
|
full_name = f"titan/{project}"
|
|
head = document.get("head")
|
|
pr_base = document.get("base")
|
|
matches = (
|
|
document.get("state") == "open"
|
|
and isinstance(head, dict) and isinstance(pr_base, dict)
|
|
and head.get("ref") == branch and head.get("sha") == commit
|
|
and isinstance(head.get("repo"), dict) and head["repo"].get("full_name") == full_name
|
|
and pr_base.get("ref") == base
|
|
and isinstance(pr_base.get("repo"), dict) and pr_base["repo"].get("full_name") == full_name
|
|
)
|
|
return matches, "current PR does not match reviewed branch/base/head" if not matches else ""
|
|
|
|
|
|
def _escalate(kanban_db: Any, conn: Any, task_id: str, reason: str) -> None:
|
|
body = (
|
|
f"supervisor fail-closed escalation: {reason}. Human attention required; "
|
|
"no follow-up card was auto-created."
|
|
)
|
|
try:
|
|
kanban_db.block_task(conn, task_id, reason=body, kind="needs_input")
|
|
except Exception as error: # noqa: BLE001 - still record the human-visible comment
|
|
_log(f"could not block {task_id}: {error}")
|
|
_comment(kanban_db, conn, task_id, body)
|
|
|
|
|
|
def _already_created(kanban_db: Any, conn: Any, payload: dict[str, Any], board: str = "") -> bool:
|
|
"""True if a card matching this spawn payload's stamp already exists."""
|
|
supervisor_stamp = (payload.get("metadata") or {}).get("supervisor") or {}
|
|
kind = supervisor_stamp.get("kind")
|
|
root = str(supervisor_stamp.get("root") or "")
|
|
commit = str(supervisor_stamp.get("head_commit") or "")
|
|
if not kind or not root:
|
|
return False
|
|
try:
|
|
tasks = [
|
|
_with_supervisor_state(kanban_db, conn, board, task)
|
|
for task in kanban_db.list_tasks(conn)
|
|
]
|
|
except Exception as error: # noqa: BLE001 - never retry an unverified create
|
|
_log(f"could not re-scan before spawn retry: {error}")
|
|
raise RuntimeError("cannot verify existing follow-up") from error
|
|
return policy.existing_followup(tasks, kind, root, commit)
|
|
|
|
|
|
def _prevalidate_spawn(board: str, stamp: dict[str, Any]) -> None:
|
|
"""Refuse a native create until its root is in durable coordinator state."""
|
|
if not board or not stamp:
|
|
return
|
|
expected = policy.lineage.from_stamp(stamp)
|
|
root_id = str(stamp.get("root") or "")
|
|
actual = supervisor_state.get_root(board, root_id)
|
|
if expected is None or actual != expected:
|
|
raise RuntimeError("supervisor child has no matching trusted root lineage")
|
|
|
|
|
|
def _spawn(kanban_db: Any, conn: Any, decision: policy.Decision, board: str = "") -> None:
|
|
payload = dict(decision.payload or {})
|
|
stamp = (payload.get("metadata") or {}).get("supervisor") or {}
|
|
_prevalidate_spawn(board, stamp)
|
|
try:
|
|
child_id = kanban_db.create_task(conn, **payload)
|
|
except TypeError:
|
|
# create_task may reject idempotency_key on older runtimes. That is
|
|
# raised at call binding, before any insert, but a post-insert TypeError
|
|
# is also possible, so re-run the dedup scan first: only retry the create
|
|
# when no matching card exists, so a partial insert is never doubled.
|
|
if _already_created(kanban_db, conn, decision.payload or {}, board):
|
|
return
|
|
# Native Hermes has no task metadata column. The state table records
|
|
# this stamp after creation; old compatible test/runtime APIs may still
|
|
# accept it on the first attempt.
|
|
payload.pop("metadata", None)
|
|
try:
|
|
child_id = kanban_db.create_task(conn, **payload)
|
|
except TypeError:
|
|
if _already_created(kanban_db, conn, decision.payload or {}, board):
|
|
return
|
|
payload.pop("idempotency_key", None)
|
|
child_id = kanban_db.create_task(conn, **payload)
|
|
if board and isinstance(child_id, str) and stamp:
|
|
try:
|
|
supervisor_state.record_child(
|
|
board, child_id, str(stamp["root"]), str(stamp["parent"]),
|
|
str(stamp["kind"]), str(stamp["head_commit"]),
|
|
str(payload.get("body") or ""), int(stamp["cycle"]),
|
|
)
|
|
except (KeyError, TypeError, ValueError) as error:
|
|
raise RuntimeError(f"could not persist supervisor child authority: {error}") from error
|
|
_comment(kanban_db, conn, decision.target_id, f"supervisor: {decision.reason}")
|
|
|
|
|
|
def apply_decision(
|
|
kanban_db: Any, conn: Any, decision: policy.Decision, ledger: Ledger, board: str = ""
|
|
) -> bool:
|
|
"""Execute one decision. Returns True when an action was taken.
|
|
|
|
Spawns are self-limiting across ticks (the created card fails the next dedup
|
|
scan). SHIP and escalate targets stay in their done state, so they are gated
|
|
on the persistent ledger and fire at most once per source card.
|
|
"""
|
|
if decision.action == "none":
|
|
return False
|
|
policy.assert_safe(decision)
|
|
if decision.action == "spawn":
|
|
_spawn(kanban_db, conn, decision, board)
|
|
return True
|
|
info = decision.payload or {}
|
|
key = f"{decision.action}:{decision.target_id}:{info.get('commit', '')}"
|
|
if ledger.has(key):
|
|
return False # already emitted on a prior tick; never re-emit or re-block
|
|
if decision.action == "ship":
|
|
live, reason = _live_pr_matches(info)
|
|
if live is None:
|
|
_log(f"deferring SHIP for {decision.target_id}: {reason}")
|
|
return False
|
|
if not live:
|
|
_clear_ready(
|
|
kanban_db, conn, decision.target_id, "",
|
|
f"supervisor: readiness cleared because {reason}.",
|
|
)
|
|
ledger.record(f"clear_ready:{decision.target_id}:{info.get('commit', '')}")
|
|
return True
|
|
commit = info.get("commit", "")
|
|
pr = info.get("pr", "") or "branch on record"
|
|
body = (
|
|
f"supervisor: review returned SHIP for commit {commit} ({pr}). "
|
|
"READY FOR HUMAN MERGE - a human must merge; the supervisor never "
|
|
"merges, approves, closes, or clears WIP."
|
|
)
|
|
_mark_ready_for_human(kanban_db, conn, decision.target_id, body, info)
|
|
elif decision.action == "clear_ready":
|
|
current = str(info.get("commit") or "")
|
|
_clear_ready(
|
|
kanban_db, conn, decision.target_id, current,
|
|
f"supervisor: stale approval cleared; current verified head is {current}.",
|
|
)
|
|
else: # escalate
|
|
_escalate(kanban_db, conn, decision.target_id, decision.reason)
|
|
ledger.record(key)
|
|
return True
|
|
|
|
|
|
def supervise_board(
|
|
kanban_db: Any, board: str, limits: policy.Limits, ledger: Ledger
|
|
) -> int:
|
|
actions = 0
|
|
with kanban_db.scoped_current_board(board):
|
|
conn = kanban_db.connect(board=board)
|
|
try:
|
|
tasks = [
|
|
_with_supervisor_state(kanban_db, conn, board, task)
|
|
for task in kanban_db.list_tasks(conn)
|
|
]
|
|
for task in tasks:
|
|
decision = policy.plan(task, tasks, limits)
|
|
try:
|
|
if apply_decision(kanban_db, conn, decision, ledger, board):
|
|
actions += 1
|
|
except Exception as error: # noqa: BLE001 - one card never stops others
|
|
_log(f"action failed on board {board!r}: {error}")
|
|
finally:
|
|
conn.close()
|
|
return actions
|
|
|
|
|
|
def supervise_once(
|
|
kanban_db: Any, limits: policy.Limits, ledger: Ledger | None = None
|
|
) -> int:
|
|
"""One bounded pass over every non-archived board."""
|
|
if ledger is None:
|
|
ledger = Ledger(LEDGER_PATH)
|
|
total = 0
|
|
for board in _iter_boards(kanban_db):
|
|
try:
|
|
total += supervise_board(kanban_db, board, limits, ledger)
|
|
except Exception as error: # noqa: BLE001 - isolate a faulty board
|
|
_log(f"temporarily skipping board {board!r}: {error}")
|
|
return total
|
|
|
|
|
|
def run_forever(
|
|
kanban_db: Any,
|
|
*,
|
|
sleep: Callable[[float], None] = time.sleep,
|
|
load: Callable[[], Settings] = load_settings,
|
|
max_ticks: int | None = None,
|
|
) -> int:
|
|
"""Poll loop. Inert while ``auto_supervise`` is false; re-reads config a tick."""
|
|
ticks = 0
|
|
while max_ticks is None or ticks < max_ticks:
|
|
settings = load()
|
|
if settings.enabled:
|
|
try:
|
|
supervise_once(kanban_db, settings.limits)
|
|
except Exception as error: # noqa: BLE001 - never let one tick kill the loop
|
|
_log(f"tick failed: {error}")
|
|
sleep(settings.interval)
|
|
ticks += 1
|
|
return ticks
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
from hermes_cli import kanban_db
|
|
|
|
run_forever(kanban_db)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__": # pragma: no cover - process entry point
|
|
sys.exit(main(sys.argv[1:]))
|