hermes: activate Telegram and isolate corrupt boards
All checks were successful
Tests / Declarative: Post Actions passed: 268
All checks were successful
Tests / Declarative: Post Actions passed: 268
This commit is contained in:
parent
e755808dc1
commit
fd3be220f3
@ -26,6 +26,10 @@ spec:
|
|||||||
kind: Deployment
|
kind: Deployment
|
||||||
name: hermes-switchyard
|
name: hermes-switchyard
|
||||||
namespace: hermes
|
namespace: hermes
|
||||||
|
- apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
name: hermes-agent
|
||||||
|
namespace: hermes
|
||||||
- apiVersion: apps/v1
|
- apiVersion: apps/v1
|
||||||
kind: Deployment
|
kind: Deployment
|
||||||
name: hermes
|
name: hermes
|
||||||
|
|||||||
@ -25,7 +25,7 @@ spec:
|
|||||||
ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers
|
ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers
|
||||||
ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback
|
ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback
|
||||||
ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available
|
ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available
|
||||||
ai.bstein.dev/config-rev: "20260813-kanban-degraded-startup"
|
ai.bstein.dev/config-rev: "20260813-kanban-corruption-isolation"
|
||||||
vault.hashicorp.com/agent-inject: "true"
|
vault.hashicorp.com/agent-inject: "true"
|
||||||
vault.hashicorp.com/role: hermes-agent
|
vault.hashicorp.com/role: hermes-agent
|
||||||
vault.hashicorp.com/agent-inject-secret-anthropic-token: kv/data/atlas/hermes/agent-tokens
|
vault.hashicorp.com/agent-inject-secret-anthropic-token: kv/data/atlas/hermes/agent-tokens
|
||||||
|
|||||||
@ -20,7 +20,7 @@ spec:
|
|||||||
app: hermes-chat-router
|
app: hermes-chat-router
|
||||||
annotations:
|
annotations:
|
||||||
ai.bstein.dev/role: privacy-preserving-chat-tenant-router
|
ai.bstein.dev/role: privacy-preserving-chat-tenant-router
|
||||||
ai.bstein.dev/config-rev: "20260813-telegram-operator-v2"
|
ai.bstein.dev/config-rev: "20260813-telegram-activated-v1"
|
||||||
vault.hashicorp.com/agent-inject: "true"
|
vault.hashicorp.com/agent-inject: "true"
|
||||||
vault.hashicorp.com/agent-pre-populate-only: "true"
|
vault.hashicorp.com/agent-pre-populate-only: "true"
|
||||||
vault.hashicorp.com/agent-init-first: "true"
|
vault.hashicorp.com/agent-init-first: "true"
|
||||||
|
|||||||
@ -37,6 +37,7 @@ DEFAULT_CLAIM_TTL = 7 * 24 * 60 * 60
|
|||||||
DEFAULT_MAX_RUNTIME = 12 * 60 * 60
|
DEFAULT_MAX_RUNTIME = 12 * 60 * 60
|
||||||
HEARTBEAT_SECONDS = 20
|
HEARTBEAT_SECONDS = 20
|
||||||
WORKTREE_LOCK = threading.Lock()
|
WORKTREE_LOCK = threading.Lock()
|
||||||
|
BOARD_CORRUPTION_ERRORS: dict[str, str] = {}
|
||||||
CAPACITY_PATTERN = re.compile(
|
CAPACITY_PATTERN = re.compile(
|
||||||
r"(?:rate.?limit|capacity|overload|usage.?limit|quota|credit|exhaust|429|529|authentication|oauth|token.*expired)",
|
r"(?:rate.?limit|capacity|overload|usage.?limit|quota|credit|exhaust|429|529|authentication|oauth|token.*expired)",
|
||||||
re.I,
|
re.I,
|
||||||
@ -741,6 +742,24 @@ def _external(task: Any) -> bool:
|
|||||||
return str(_task_value(task, "assignee", "") or "").startswith(EXTERNAL_PREFIX)
|
return str(_task_value(task, "assignee", "") or "").startswith(EXTERNAL_PREFIX)
|
||||||
|
|
||||||
|
|
||||||
|
def _connect_healthy_board(kanban_db: Any, board: str) -> Any | None:
|
||||||
|
"""Open one board without letting localized corruption stop other lanes."""
|
||||||
|
try:
|
||||||
|
conn = kanban_db.connect(board=board)
|
||||||
|
except getattr(kanban_db, "KanbanDbCorruptError", ()) as error:
|
||||||
|
detail = str(error)
|
||||||
|
if BOARD_CORRUPTION_ERRORS.get(board) != detail:
|
||||||
|
print(
|
||||||
|
f"quarantining corrupt Kanban board {board!r}: {detail}",
|
||||||
|
file=sys.stderr,
|
||||||
|
flush=True,
|
||||||
|
)
|
||||||
|
BOARD_CORRUPTION_ERRORS[board] = detail
|
||||||
|
return None
|
||||||
|
BOARD_CORRUPTION_ERRORS.pop(board, None)
|
||||||
|
return conn
|
||||||
|
|
||||||
|
|
||||||
def recover_orphans() -> None:
|
def recover_orphans() -> None:
|
||||||
"""Return external running tasks to ready after a runner/pod restart."""
|
"""Return external running tasks to ready after a runner/pod restart."""
|
||||||
from hermes_cli import kanban_db
|
from hermes_cli import kanban_db
|
||||||
@ -750,7 +769,9 @@ def recover_orphans() -> None:
|
|||||||
if not board:
|
if not board:
|
||||||
continue
|
continue
|
||||||
with kanban_db.scoped_current_board(board):
|
with kanban_db.scoped_current_board(board):
|
||||||
conn = kanban_db.connect(board=board)
|
conn = _connect_healthy_board(kanban_db, board)
|
||||||
|
if conn is None:
|
||||||
|
continue
|
||||||
try:
|
try:
|
||||||
for task in kanban_db.list_tasks(conn):
|
for task in kanban_db.list_tasks(conn):
|
||||||
if _external(task) and str(_task_value(task, "status", "")) == "running":
|
if _external(task) and str(_task_value(task, "status", "")) == "running":
|
||||||
@ -775,7 +796,9 @@ def claim_ready(active: set[tuple[str, str]], limit: int) -> list[tuple[str, str
|
|||||||
if not board:
|
if not board:
|
||||||
continue
|
continue
|
||||||
with kanban_db.scoped_current_board(board):
|
with kanban_db.scoped_current_board(board):
|
||||||
conn = kanban_db.connect(board=board)
|
conn = _connect_healthy_board(kanban_db, board)
|
||||||
|
if conn is None:
|
||||||
|
continue
|
||||||
try:
|
try:
|
||||||
kanban_db.recompute_ready(conn)
|
kanban_db.recompute_ready(conn)
|
||||||
for task in kanban_db.list_tasks(conn):
|
for task in kanban_db.list_tasks(conn):
|
||||||
|
|||||||
@ -276,6 +276,40 @@ def test_unassigned_ready_task_is_persistently_routed_to_auto_lane(monkeypatch):
|
|||||||
assert assigned == [("t_auto", "cli-auto")]
|
assert assigned == [("t_auto", "cli-auto")]
|
||||||
|
|
||||||
|
|
||||||
|
def test_corrupt_board_is_quarantined_without_stopping_healthy_lanes(monkeypatch, capsys):
|
||||||
|
class CorruptBoardError(Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
task = SimpleNamespace(id="t_healthy", assignee="cli-auto", status="ready")
|
||||||
|
|
||||||
|
class Connection:
|
||||||
|
def close(self):
|
||||||
|
return None
|
||||||
|
|
||||||
|
def connect(*, board):
|
||||||
|
if board == "cassandra":
|
||||||
|
raise CorruptBoardError("integrity_check failed")
|
||||||
|
return Connection()
|
||||||
|
|
||||||
|
fake_db = SimpleNamespace(
|
||||||
|
KanbanDbCorruptError=CorruptBoardError,
|
||||||
|
list_boards=lambda include_archived=False: [
|
||||||
|
{"slug": "cassandra"},
|
||||||
|
{"slug": "healthy"},
|
||||||
|
],
|
||||||
|
scoped_current_board=lambda _board: nullcontext(),
|
||||||
|
connect=connect,
|
||||||
|
recompute_ready=lambda _conn: None,
|
||||||
|
list_tasks=lambda _conn: [task],
|
||||||
|
claim_task=lambda _conn, _task_id, **_kwargs: task,
|
||||||
|
)
|
||||||
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
|
||||||
|
lanes.BOARD_CORRUPTION_ERRORS.clear()
|
||||||
|
|
||||||
|
assert lanes.claim_ready(set(), 1) == [("healthy", "t_healthy")]
|
||||||
|
assert "quarantining corrupt Kanban board 'cassandra'" in capsys.readouterr().err
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
("result", "expected_action"),
|
("result", "expected_action"),
|
||||||
[
|
[
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user