hermes: tolerate preserved Kanban corruption

This commit is contained in:
jenkins 2026-08-13 04:42:04 -03:00
parent 4cb0faae7f
commit 0590cefa9b
3 changed files with 88 additions and 3 deletions

View File

@ -25,7 +25,7 @@ spec:
ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers
ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback
ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available
ai.bstein.dev/config-rev: "20260813-terminal-lineage-oauth" ai.bstein.dev/config-rev: "20260813-kanban-degraded-startup"
vault.hashicorp.com/agent-inject: "true" vault.hashicorp.com/agent-inject: "true"
vault.hashicorp.com/role: hermes-agent vault.hashicorp.com/role: hermes-agent
vault.hashicorp.com/agent-inject-secret-anthropic-token: kv/data/atlas/hermes/agent-tokens vault.hashicorp.com/agent-inject-secret-anthropic-token: kv/data/atlas/hermes/agent-tokens

View File

@ -109,6 +109,35 @@ def bootstrap_cassandra(root: Path) -> None:
_migrate_open_cassandra_tasks(kb, workspace) _migrate_open_cassandra_tasks(kb, workspace)
def _kanban_corruption_type() -> type[Exception]:
"""Return Hermes's pinned corruption exception without importing it at startup."""
from hermes_cli.kanban_db import KanbanDbCorruptError
return KanbanDbCorruptError
def bootstrap_cassandra_state(root: Path) -> dict[str, str]:
"""Bootstrap Cassandra while preserving a damaged board for explicit recovery."""
try:
bootstrap_cassandra(root)
except Exception as error:
if not isinstance(error, _kanban_corruption_type()):
raise
status = {
"state": "corrupt-preserved",
"database": str(getattr(error, "db_path", "")),
"backup": str(getattr(error, "backup_path", "")),
"reason": str(getattr(error, "reason", "integrity check failed")),
}
print(
"Cassandra Kanban board needs recovery; the coordinator preserved "
f"the database and backup at {status['backup'] or 'the Hermes data volume'}.",
flush=True,
)
return status
return {"state": "ready"}
def sync_cassandra_repo(env_values: dict[str, str]) -> str: def sync_cassandra_repo(env_values: dict[str, str]) -> str:
"""Clone or fetch Cassandra when the optional Gitea token is available.""" """Clone or fetch Cassandra when the optional Gitea token is available."""
token = env_values.get("GITEA_TOKEN", "").strip() token = env_values.get("GITEA_TOKEN", "").strip()
@ -166,7 +195,7 @@ def refresh_once(root: Path) -> dict[str, Any]:
codex = discover_codex_models() codex = discover_codex_models()
claude = discover_claude_models() claude = discover_claude_models()
routes = configure_routes(root, codex, claude) routes = configure_routes(root, codex, claude)
bootstrap_cassandra(root) board_state = bootstrap_cassandra_state(root)
repo_state = sync_cassandra_repo(env_values) repo_state = sync_cassandra_repo(env_values)
status = { status = {
"updated_at": datetime.now(timezone.utc).isoformat(), "updated_at": datetime.now(timezone.utc).isoformat(),
@ -179,6 +208,7 @@ def refresh_once(root: Path) -> dict[str, Any]:
"projects": { "projects": {
"cassandra": { "cassandra": {
"board": "cassandra", "board": "cassandra",
"board_state": board_state,
"workspace": str(cassandra_workspace()), "workspace": str(cassandra_workspace()),
"repository": CASSANDRA_REMOTE, "repository": CASSANDRA_REMOTE,
"state": repo_state, "state": repo_state,

View File

@ -294,7 +294,9 @@ def test_refresh_writes_non_secret_routing_status(tmp_path: Path, monkeypatch):
claude = routing.Catalog("anthropic", ["claude-opus-5"], True, True, "connected") claude = routing.Catalog("anthropic", ["claude-opus-5"], True, True, "connected")
monkeypatch.setattr(coordinator, "discover_codex_models", lambda: codex) monkeypatch.setattr(coordinator, "discover_codex_models", lambda: codex)
monkeypatch.setattr(coordinator, "discover_claude_models", lambda: claude) monkeypatch.setattr(coordinator, "discover_claude_models", lambda: claude)
monkeypatch.setattr(coordinator, "bootstrap_cassandra", lambda root: None) monkeypatch.setattr(
coordinator, "bootstrap_cassandra_state", lambda root: {"state": "ready"}
)
monkeypatch.setattr(coordinator, "sync_cassandra_repo", lambda env: "ready") monkeypatch.setattr(coordinator, "sync_cassandra_repo", lambda env: "ready")
status = coordinator.refresh_once(tmp_path) status = coordinator.refresh_once(tmp_path)
@ -302,9 +304,62 @@ def test_refresh_writes_non_secret_routing_status(tmp_path: Path, monkeypatch):
status_path = tmp_path / "workspace/coordinator/model-routing.json" status_path = tmp_path / "workspace/coordinator/model-routing.json"
assert status_path.is_file() assert status_path.is_file()
assert status["projects"]["cassandra"]["state"] == "ready" assert status["projects"]["cassandra"]["state"] == "ready"
assert status["projects"]["cassandra"]["board_state"] == {"state": "ready"}
assert "do-not-report" not in status_path.read_text(encoding="utf-8") assert "do-not-report" not in status_path.read_text(encoding="utf-8")
def test_corrupt_cassandra_board_is_preserved_without_blocking_refresh(
tmp_path: Path, monkeypatch
):
"""A damaged project board degrades Kanban instead of stopping all services."""
class FakeKanbanDbCorruptError(Exception):
def __init__(self):
super().__init__("integrity check failed")
self.db_path = tmp_path / "kanban.db"
self.backup_path = tmp_path / "kanban.db.corrupt.backup"
self.reason = "row out of order"
def raise_corruption(_root: Path) -> None:
raise FakeKanbanDbCorruptError()
monkeypatch.setattr(coordinator, "bootstrap_cassandra", raise_corruption)
monkeypatch.setattr(
coordinator, "_kanban_corruption_type", lambda: FakeKanbanDbCorruptError
)
status = coordinator.bootstrap_cassandra_state(tmp_path)
assert status == {
"state": "corrupt-preserved",
"database": str(tmp_path / "kanban.db"),
"backup": str(tmp_path / "kanban.db.corrupt.backup"),
"reason": "row out of order",
}
def test_unrelated_cassandra_bootstrap_failure_remains_fatal(
tmp_path: Path, monkeypatch
):
"""Only the known fail-closed corruption state may be degraded."""
class FakeKanbanDbCorruptError(Exception):
pass
def raise_permission_error(_root: Path) -> None:
raise PermissionError("cannot access board")
monkeypatch.setattr(coordinator, "bootstrap_cassandra", raise_permission_error)
monkeypatch.setattr(
coordinator, "_kanban_corruption_type", lambda: FakeKanbanDbCorruptError
)
try:
coordinator.bootstrap_cassandra_state(tmp_path)
except PermissionError as error:
assert str(error) == "cannot access board"
else:
raise AssertionError("unrelated board errors must fail the coordinator refresh")
def test_cassandra_workspace_uses_valid_configured_worktree( def test_cassandra_workspace_uses_valid_configured_worktree(
tmp_path: Path, monkeypatch tmp_path: Path, monkeypatch
): ):