From 0590cefa9b45a89c6d26be7e6e30909bb96408d5 Mon Sep 17 00:00:00 2001 From: jenkins Date: Thu, 13 Aug 2026 04:42:04 -0300 Subject: [PATCH] hermes: tolerate preserved Kanban corruption --- services/hermes/agent-deployment.yaml | 2 +- services/hermes/scripts/hermes_coordinator.py | 32 ++++++++++- testing/tests/test_hermes_coordinator.py | 57 ++++++++++++++++++- 3 files changed, 88 insertions(+), 3 deletions(-) diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml index 81c9ec2da..5a6b167cd 100644 --- a/services/hermes/agent-deployment.yaml +++ b/services/hermes/agent-deployment.yaml @@ -25,7 +25,7 @@ spec: ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available - ai.bstein.dev/config-rev: "20260813-terminal-lineage-oauth" + ai.bstein.dev/config-rev: "20260813-kanban-degraded-startup" vault.hashicorp.com/agent-inject: "true" vault.hashicorp.com/role: hermes-agent vault.hashicorp.com/agent-inject-secret-anthropic-token: kv/data/atlas/hermes/agent-tokens diff --git a/services/hermes/scripts/hermes_coordinator.py b/services/hermes/scripts/hermes_coordinator.py index 0619eb029..c357c9a32 100644 --- a/services/hermes/scripts/hermes_coordinator.py +++ b/services/hermes/scripts/hermes_coordinator.py @@ -109,6 +109,35 @@ def bootstrap_cassandra(root: Path) -> None: _migrate_open_cassandra_tasks(kb, workspace) +def _kanban_corruption_type() -> type[Exception]: + """Return Hermes's pinned corruption exception without importing it at startup.""" + from hermes_cli.kanban_db import KanbanDbCorruptError + + return KanbanDbCorruptError + + +def bootstrap_cassandra_state(root: Path) -> dict[str, str]: + """Bootstrap Cassandra while preserving a damaged board for explicit recovery.""" + try: + bootstrap_cassandra(root) + except Exception as error: + if not isinstance(error, _kanban_corruption_type()): + raise + status = { + "state": "corrupt-preserved", + "database": str(getattr(error, "db_path", "")), + "backup": str(getattr(error, "backup_path", "")), + "reason": str(getattr(error, "reason", "integrity check failed")), + } + print( + "Cassandra Kanban board needs recovery; the coordinator preserved " + f"the database and backup at {status['backup'] or 'the Hermes data volume'}.", + flush=True, + ) + return status + return {"state": "ready"} + + def sync_cassandra_repo(env_values: dict[str, str]) -> str: """Clone or fetch Cassandra when the optional Gitea token is available.""" token = env_values.get("GITEA_TOKEN", "").strip() @@ -166,7 +195,7 @@ def refresh_once(root: Path) -> dict[str, Any]: codex = discover_codex_models() claude = discover_claude_models() routes = configure_routes(root, codex, claude) - bootstrap_cassandra(root) + board_state = bootstrap_cassandra_state(root) repo_state = sync_cassandra_repo(env_values) status = { "updated_at": datetime.now(timezone.utc).isoformat(), @@ -179,6 +208,7 @@ def refresh_once(root: Path) -> dict[str, Any]: "projects": { "cassandra": { "board": "cassandra", + "board_state": board_state, "workspace": str(cassandra_workspace()), "repository": CASSANDRA_REMOTE, "state": repo_state, diff --git a/testing/tests/test_hermes_coordinator.py b/testing/tests/test_hermes_coordinator.py index 6dd5468e9..e23f8365f 100644 --- a/testing/tests/test_hermes_coordinator.py +++ b/testing/tests/test_hermes_coordinator.py @@ -294,7 +294,9 @@ def test_refresh_writes_non_secret_routing_status(tmp_path: Path, monkeypatch): claude = routing.Catalog("anthropic", ["claude-opus-5"], True, True, "connected") monkeypatch.setattr(coordinator, "discover_codex_models", lambda: codex) monkeypatch.setattr(coordinator, "discover_claude_models", lambda: claude) - monkeypatch.setattr(coordinator, "bootstrap_cassandra", lambda root: None) + monkeypatch.setattr( + coordinator, "bootstrap_cassandra_state", lambda root: {"state": "ready"} + ) monkeypatch.setattr(coordinator, "sync_cassandra_repo", lambda env: "ready") status = coordinator.refresh_once(tmp_path) @@ -302,9 +304,62 @@ def test_refresh_writes_non_secret_routing_status(tmp_path: Path, monkeypatch): status_path = tmp_path / "workspace/coordinator/model-routing.json" assert status_path.is_file() assert status["projects"]["cassandra"]["state"] == "ready" + assert status["projects"]["cassandra"]["board_state"] == {"state": "ready"} assert "do-not-report" not in status_path.read_text(encoding="utf-8") +def test_corrupt_cassandra_board_is_preserved_without_blocking_refresh( + tmp_path: Path, monkeypatch +): + """A damaged project board degrades Kanban instead of stopping all services.""" + class FakeKanbanDbCorruptError(Exception): + def __init__(self): + super().__init__("integrity check failed") + self.db_path = tmp_path / "kanban.db" + self.backup_path = tmp_path / "kanban.db.corrupt.backup" + self.reason = "row out of order" + + def raise_corruption(_root: Path) -> None: + raise FakeKanbanDbCorruptError() + + monkeypatch.setattr(coordinator, "bootstrap_cassandra", raise_corruption) + monkeypatch.setattr( + coordinator, "_kanban_corruption_type", lambda: FakeKanbanDbCorruptError + ) + + status = coordinator.bootstrap_cassandra_state(tmp_path) + + assert status == { + "state": "corrupt-preserved", + "database": str(tmp_path / "kanban.db"), + "backup": str(tmp_path / "kanban.db.corrupt.backup"), + "reason": "row out of order", + } + + +def test_unrelated_cassandra_bootstrap_failure_remains_fatal( + tmp_path: Path, monkeypatch +): + """Only the known fail-closed corruption state may be degraded.""" + class FakeKanbanDbCorruptError(Exception): + pass + + def raise_permission_error(_root: Path) -> None: + raise PermissionError("cannot access board") + + monkeypatch.setattr(coordinator, "bootstrap_cassandra", raise_permission_error) + monkeypatch.setattr( + coordinator, "_kanban_corruption_type", lambda: FakeKanbanDbCorruptError + ) + + try: + coordinator.bootstrap_cassandra_state(tmp_path) + except PermissionError as error: + assert str(error) == "cannot access board" + else: + raise AssertionError("unrelated board errors must fail the coordinator refresh") + + def test_cassandra_workspace_uses_valid_configured_worktree( tmp_path: Path, monkeypatch ):