hermes: activate Telegram and isolate corrupt boards
All checks were successful
Tests / Declarative: Post Actions passed: 268

This commit is contained in:
jenkins 2026-08-13 12:24:15 -03:00
parent e755808dc1
commit fd3be220f3
5 changed files with 65 additions and 4 deletions

View File

@ -26,6 +26,10 @@ spec:
kind: Deployment kind: Deployment
name: hermes-switchyard name: hermes-switchyard
namespace: hermes namespace: hermes
- apiVersion: apps/v1
kind: Deployment
name: hermes-agent
namespace: hermes
- apiVersion: apps/v1 - apiVersion: apps/v1
kind: Deployment kind: Deployment
name: hermes name: hermes

View File

@ -25,7 +25,7 @@ spec:
ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers
ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback
ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available
ai.bstein.dev/config-rev: "20260813-kanban-degraded-startup" ai.bstein.dev/config-rev: "20260813-kanban-corruption-isolation"
vault.hashicorp.com/agent-inject: "true" vault.hashicorp.com/agent-inject: "true"
vault.hashicorp.com/role: hermes-agent vault.hashicorp.com/role: hermes-agent
vault.hashicorp.com/agent-inject-secret-anthropic-token: kv/data/atlas/hermes/agent-tokens vault.hashicorp.com/agent-inject-secret-anthropic-token: kv/data/atlas/hermes/agent-tokens

View File

@ -20,7 +20,7 @@ spec:
app: hermes-chat-router app: hermes-chat-router
annotations: annotations:
ai.bstein.dev/role: privacy-preserving-chat-tenant-router ai.bstein.dev/role: privacy-preserving-chat-tenant-router
ai.bstein.dev/config-rev: "20260813-telegram-operator-v2" ai.bstein.dev/config-rev: "20260813-telegram-activated-v1"
vault.hashicorp.com/agent-inject: "true" vault.hashicorp.com/agent-inject: "true"
vault.hashicorp.com/agent-pre-populate-only: "true" vault.hashicorp.com/agent-pre-populate-only: "true"
vault.hashicorp.com/agent-init-first: "true" vault.hashicorp.com/agent-init-first: "true"

View File

@ -37,6 +37,7 @@ DEFAULT_CLAIM_TTL = 7 * 24 * 60 * 60
DEFAULT_MAX_RUNTIME = 12 * 60 * 60 DEFAULT_MAX_RUNTIME = 12 * 60 * 60
HEARTBEAT_SECONDS = 20 HEARTBEAT_SECONDS = 20
WORKTREE_LOCK = threading.Lock() WORKTREE_LOCK = threading.Lock()
BOARD_CORRUPTION_ERRORS: dict[str, str] = {}
CAPACITY_PATTERN = re.compile( CAPACITY_PATTERN = re.compile(
r"(?:rate.?limit|capacity|overload|usage.?limit|quota|credit|exhaust|429|529|authentication|oauth|token.*expired)", r"(?:rate.?limit|capacity|overload|usage.?limit|quota|credit|exhaust|429|529|authentication|oauth|token.*expired)",
re.I, re.I,
@ -741,6 +742,24 @@ def _external(task: Any) -> bool:
return str(_task_value(task, "assignee", "") or "").startswith(EXTERNAL_PREFIX) return str(_task_value(task, "assignee", "") or "").startswith(EXTERNAL_PREFIX)
def _connect_healthy_board(kanban_db: Any, board: str) -> Any | None:
"""Open one board without letting localized corruption stop other lanes."""
try:
conn = kanban_db.connect(board=board)
except getattr(kanban_db, "KanbanDbCorruptError", ()) as error:
detail = str(error)
if BOARD_CORRUPTION_ERRORS.get(board) != detail:
print(
f"quarantining corrupt Kanban board {board!r}: {detail}",
file=sys.stderr,
flush=True,
)
BOARD_CORRUPTION_ERRORS[board] = detail
return None
BOARD_CORRUPTION_ERRORS.pop(board, None)
return conn
def recover_orphans() -> None: def recover_orphans() -> None:
"""Return external running tasks to ready after a runner/pod restart.""" """Return external running tasks to ready after a runner/pod restart."""
from hermes_cli import kanban_db from hermes_cli import kanban_db
@ -750,7 +769,9 @@ def recover_orphans() -> None:
if not board: if not board:
continue continue
with kanban_db.scoped_current_board(board): with kanban_db.scoped_current_board(board):
conn = kanban_db.connect(board=board) conn = _connect_healthy_board(kanban_db, board)
if conn is None:
continue
try: try:
for task in kanban_db.list_tasks(conn): for task in kanban_db.list_tasks(conn):
if _external(task) and str(_task_value(task, "status", "")) == "running": if _external(task) and str(_task_value(task, "status", "")) == "running":
@ -775,7 +796,9 @@ def claim_ready(active: set[tuple[str, str]], limit: int) -> list[tuple[str, str
if not board: if not board:
continue continue
with kanban_db.scoped_current_board(board): with kanban_db.scoped_current_board(board):
conn = kanban_db.connect(board=board) conn = _connect_healthy_board(kanban_db, board)
if conn is None:
continue
try: try:
kanban_db.recompute_ready(conn) kanban_db.recompute_ready(conn)
for task in kanban_db.list_tasks(conn): for task in kanban_db.list_tasks(conn):

View File

@ -276,6 +276,40 @@ def test_unassigned_ready_task_is_persistently_routed_to_auto_lane(monkeypatch):
assert assigned == [("t_auto", "cli-auto")] assert assigned == [("t_auto", "cli-auto")]
def test_corrupt_board_is_quarantined_without_stopping_healthy_lanes(monkeypatch, capsys):
class CorruptBoardError(Exception):
pass
task = SimpleNamespace(id="t_healthy", assignee="cli-auto", status="ready")
class Connection:
def close(self):
return None
def connect(*, board):
if board == "cassandra":
raise CorruptBoardError("integrity_check failed")
return Connection()
fake_db = SimpleNamespace(
KanbanDbCorruptError=CorruptBoardError,
list_boards=lambda include_archived=False: [
{"slug": "cassandra"},
{"slug": "healthy"},
],
scoped_current_board=lambda _board: nullcontext(),
connect=connect,
recompute_ready=lambda _conn: None,
list_tasks=lambda _conn: [task],
claim_task=lambda _conn, _task_id, **_kwargs: task,
)
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
lanes.BOARD_CORRUPTION_ERRORS.clear()
assert lanes.claim_ready(set(), 1) == [("healthy", "t_healthy")]
assert "quarantining corrupt Kanban board 'cassandra'" in capsys.readouterr().err
@pytest.mark.parametrize( @pytest.mark.parametrize(
("result", "expected_action"), ("result", "expected_action"),
[ [