diff --git a/clusters/atlas/flux-system/applications/hermes/kustomization.yaml b/clusters/atlas/flux-system/applications/hermes/kustomization.yaml index f29ee6c0a..57fed46e4 100644 --- a/clusters/atlas/flux-system/applications/hermes/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/hermes/kustomization.yaml @@ -26,6 +26,10 @@ spec: kind: Deployment name: hermes-switchyard namespace: hermes + - apiVersion: apps/v1 + kind: Deployment + name: hermes-agent + namespace: hermes - apiVersion: apps/v1 kind: Deployment name: hermes diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml index 5a6b167cd..b1215abc3 100644 --- a/services/hermes/agent-deployment.yaml +++ b/services/hermes/agent-deployment.yaml @@ -25,7 +25,7 @@ spec: ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available - ai.bstein.dev/config-rev: "20260813-kanban-degraded-startup" + ai.bstein.dev/config-rev: "20260813-kanban-corruption-isolation" vault.hashicorp.com/agent-inject: "true" vault.hashicorp.com/role: hermes-agent vault.hashicorp.com/agent-inject-secret-anthropic-token: kv/data/atlas/hermes/agent-tokens diff --git a/services/hermes/chat-router.yaml b/services/hermes/chat-router.yaml index b3e6c9fd2..0b244a9c8 100644 --- a/services/hermes/chat-router.yaml +++ b/services/hermes/chat-router.yaml @@ -20,7 +20,7 @@ spec: app: hermes-chat-router annotations: ai.bstein.dev/role: privacy-preserving-chat-tenant-router - ai.bstein.dev/config-rev: "20260813-telegram-operator-v2" + ai.bstein.dev/config-rev: "20260813-telegram-activated-v1" vault.hashicorp.com/agent-inject: "true" vault.hashicorp.com/agent-pre-populate-only: "true" vault.hashicorp.com/agent-init-first: "true" diff --git a/services/hermes/scripts/cli_lane_runner.py b/services/hermes/scripts/cli_lane_runner.py index 13a98b666..5faa23510 100644 --- a/services/hermes/scripts/cli_lane_runner.py +++ b/services/hermes/scripts/cli_lane_runner.py @@ -37,6 +37,7 @@ DEFAULT_CLAIM_TTL = 7 * 24 * 60 * 60 DEFAULT_MAX_RUNTIME = 12 * 60 * 60 HEARTBEAT_SECONDS = 20 WORKTREE_LOCK = threading.Lock() +BOARD_CORRUPTION_ERRORS: dict[str, str] = {} CAPACITY_PATTERN = re.compile( r"(?:rate.?limit|capacity|overload|usage.?limit|quota|credit|exhaust|429|529|authentication|oauth|token.*expired)", re.I, @@ -741,6 +742,24 @@ def _external(task: Any) -> bool: return str(_task_value(task, "assignee", "") or "").startswith(EXTERNAL_PREFIX) +def _connect_healthy_board(kanban_db: Any, board: str) -> Any | None: + """Open one board without letting localized corruption stop other lanes.""" + try: + conn = kanban_db.connect(board=board) + except getattr(kanban_db, "KanbanDbCorruptError", ()) as error: + detail = str(error) + if BOARD_CORRUPTION_ERRORS.get(board) != detail: + print( + f"quarantining corrupt Kanban board {board!r}: {detail}", + file=sys.stderr, + flush=True, + ) + BOARD_CORRUPTION_ERRORS[board] = detail + return None + BOARD_CORRUPTION_ERRORS.pop(board, None) + return conn + + def recover_orphans() -> None: """Return external running tasks to ready after a runner/pod restart.""" from hermes_cli import kanban_db @@ -750,7 +769,9 @@ def recover_orphans() -> None: if not board: continue with kanban_db.scoped_current_board(board): - conn = kanban_db.connect(board=board) + conn = _connect_healthy_board(kanban_db, board) + if conn is None: + continue try: for task in kanban_db.list_tasks(conn): if _external(task) and str(_task_value(task, "status", "")) == "running": @@ -775,7 +796,9 @@ def claim_ready(active: set[tuple[str, str]], limit: int) -> list[tuple[str, str if not board: continue with kanban_db.scoped_current_board(board): - conn = kanban_db.connect(board=board) + conn = _connect_healthy_board(kanban_db, board) + if conn is None: + continue try: kanban_db.recompute_ready(conn) for task in kanban_db.list_tasks(conn): diff --git a/testing/tests/test_hermes_cli_lanes.py b/testing/tests/test_hermes_cli_lanes.py index 02fe17d78..08c5742b5 100644 --- a/testing/tests/test_hermes_cli_lanes.py +++ b/testing/tests/test_hermes_cli_lanes.py @@ -276,6 +276,40 @@ def test_unassigned_ready_task_is_persistently_routed_to_auto_lane(monkeypatch): assert assigned == [("t_auto", "cli-auto")] +def test_corrupt_board_is_quarantined_without_stopping_healthy_lanes(monkeypatch, capsys): + class CorruptBoardError(Exception): + pass + + task = SimpleNamespace(id="t_healthy", assignee="cli-auto", status="ready") + + class Connection: + def close(self): + return None + + def connect(*, board): + if board == "cassandra": + raise CorruptBoardError("integrity_check failed") + return Connection() + + fake_db = SimpleNamespace( + KanbanDbCorruptError=CorruptBoardError, + list_boards=lambda include_archived=False: [ + {"slug": "cassandra"}, + {"slug": "healthy"}, + ], + scoped_current_board=lambda _board: nullcontext(), + connect=connect, + recompute_ready=lambda _conn: None, + list_tasks=lambda _conn: [task], + claim_task=lambda _conn, _task_id, **_kwargs: task, + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + lanes.BOARD_CORRUPTION_ERRORS.clear() + + assert lanes.claim_ready(set(), 1) == [("healthy", "t_healthy")] + assert "quarantining corrupt Kanban board 'cassandra'" in capsys.readouterr().err + + @pytest.mark.parametrize( ("result", "expected_action"), [