hermes: activate Telegram and isolate corrupt boards
All checks were successful
Tests / Declarative: Post Actions passed: 268
All checks were successful
Tests / Declarative: Post Actions passed: 268
This commit is contained in:
parent
e755808dc1
commit
fd3be220f3
@ -26,6 +26,10 @@ spec:
|
||||
kind: Deployment
|
||||
name: hermes-switchyard
|
||||
namespace: hermes
|
||||
- apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
name: hermes-agent
|
||||
namespace: hermes
|
||||
- apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
name: hermes
|
||||
|
||||
@ -25,7 +25,7 @@ spec:
|
||||
ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers
|
||||
ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback
|
||||
ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available
|
||||
ai.bstein.dev/config-rev: "20260813-kanban-degraded-startup"
|
||||
ai.bstein.dev/config-rev: "20260813-kanban-corruption-isolation"
|
||||
vault.hashicorp.com/agent-inject: "true"
|
||||
vault.hashicorp.com/role: hermes-agent
|
||||
vault.hashicorp.com/agent-inject-secret-anthropic-token: kv/data/atlas/hermes/agent-tokens
|
||||
|
||||
@ -20,7 +20,7 @@ spec:
|
||||
app: hermes-chat-router
|
||||
annotations:
|
||||
ai.bstein.dev/role: privacy-preserving-chat-tenant-router
|
||||
ai.bstein.dev/config-rev: "20260813-telegram-operator-v2"
|
||||
ai.bstein.dev/config-rev: "20260813-telegram-activated-v1"
|
||||
vault.hashicorp.com/agent-inject: "true"
|
||||
vault.hashicorp.com/agent-pre-populate-only: "true"
|
||||
vault.hashicorp.com/agent-init-first: "true"
|
||||
|
||||
@ -37,6 +37,7 @@ DEFAULT_CLAIM_TTL = 7 * 24 * 60 * 60
|
||||
DEFAULT_MAX_RUNTIME = 12 * 60 * 60
|
||||
HEARTBEAT_SECONDS = 20
|
||||
WORKTREE_LOCK = threading.Lock()
|
||||
BOARD_CORRUPTION_ERRORS: dict[str, str] = {}
|
||||
CAPACITY_PATTERN = re.compile(
|
||||
r"(?:rate.?limit|capacity|overload|usage.?limit|quota|credit|exhaust|429|529|authentication|oauth|token.*expired)",
|
||||
re.I,
|
||||
@ -741,6 +742,24 @@ def _external(task: Any) -> bool:
|
||||
return str(_task_value(task, "assignee", "") or "").startswith(EXTERNAL_PREFIX)
|
||||
|
||||
|
||||
def _connect_healthy_board(kanban_db: Any, board: str) -> Any | None:
|
||||
"""Open one board without letting localized corruption stop other lanes."""
|
||||
try:
|
||||
conn = kanban_db.connect(board=board)
|
||||
except getattr(kanban_db, "KanbanDbCorruptError", ()) as error:
|
||||
detail = str(error)
|
||||
if BOARD_CORRUPTION_ERRORS.get(board) != detail:
|
||||
print(
|
||||
f"quarantining corrupt Kanban board {board!r}: {detail}",
|
||||
file=sys.stderr,
|
||||
flush=True,
|
||||
)
|
||||
BOARD_CORRUPTION_ERRORS[board] = detail
|
||||
return None
|
||||
BOARD_CORRUPTION_ERRORS.pop(board, None)
|
||||
return conn
|
||||
|
||||
|
||||
def recover_orphans() -> None:
|
||||
"""Return external running tasks to ready after a runner/pod restart."""
|
||||
from hermes_cli import kanban_db
|
||||
@ -750,7 +769,9 @@ def recover_orphans() -> None:
|
||||
if not board:
|
||||
continue
|
||||
with kanban_db.scoped_current_board(board):
|
||||
conn = kanban_db.connect(board=board)
|
||||
conn = _connect_healthy_board(kanban_db, board)
|
||||
if conn is None:
|
||||
continue
|
||||
try:
|
||||
for task in kanban_db.list_tasks(conn):
|
||||
if _external(task) and str(_task_value(task, "status", "")) == "running":
|
||||
@ -775,7 +796,9 @@ def claim_ready(active: set[tuple[str, str]], limit: int) -> list[tuple[str, str
|
||||
if not board:
|
||||
continue
|
||||
with kanban_db.scoped_current_board(board):
|
||||
conn = kanban_db.connect(board=board)
|
||||
conn = _connect_healthy_board(kanban_db, board)
|
||||
if conn is None:
|
||||
continue
|
||||
try:
|
||||
kanban_db.recompute_ready(conn)
|
||||
for task in kanban_db.list_tasks(conn):
|
||||
|
||||
@ -276,6 +276,40 @@ def test_unassigned_ready_task_is_persistently_routed_to_auto_lane(monkeypatch):
|
||||
assert assigned == [("t_auto", "cli-auto")]
|
||||
|
||||
|
||||
def test_corrupt_board_is_quarantined_without_stopping_healthy_lanes(monkeypatch, capsys):
|
||||
class CorruptBoardError(Exception):
|
||||
pass
|
||||
|
||||
task = SimpleNamespace(id="t_healthy", assignee="cli-auto", status="ready")
|
||||
|
||||
class Connection:
|
||||
def close(self):
|
||||
return None
|
||||
|
||||
def connect(*, board):
|
||||
if board == "cassandra":
|
||||
raise CorruptBoardError("integrity_check failed")
|
||||
return Connection()
|
||||
|
||||
fake_db = SimpleNamespace(
|
||||
KanbanDbCorruptError=CorruptBoardError,
|
||||
list_boards=lambda include_archived=False: [
|
||||
{"slug": "cassandra"},
|
||||
{"slug": "healthy"},
|
||||
],
|
||||
scoped_current_board=lambda _board: nullcontext(),
|
||||
connect=connect,
|
||||
recompute_ready=lambda _conn: None,
|
||||
list_tasks=lambda _conn: [task],
|
||||
claim_task=lambda _conn, _task_id, **_kwargs: task,
|
||||
)
|
||||
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
|
||||
lanes.BOARD_CORRUPTION_ERRORS.clear()
|
||||
|
||||
assert lanes.claim_ready(set(), 1) == [("healthy", "t_healthy")]
|
||||
assert "quarantining corrupt Kanban board 'cassandra'" in capsys.readouterr().err
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("result", "expected_action"),
|
||||
[
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user