hermes: tolerate preserved Kanban corruption
This commit is contained in:
parent
4cb0faae7f
commit
0590cefa9b
@ -25,7 +25,7 @@ spec:
|
||||
ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers
|
||||
ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback
|
||||
ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available
|
||||
ai.bstein.dev/config-rev: "20260813-terminal-lineage-oauth"
|
||||
ai.bstein.dev/config-rev: "20260813-kanban-degraded-startup"
|
||||
vault.hashicorp.com/agent-inject: "true"
|
||||
vault.hashicorp.com/role: hermes-agent
|
||||
vault.hashicorp.com/agent-inject-secret-anthropic-token: kv/data/atlas/hermes/agent-tokens
|
||||
|
||||
@ -109,6 +109,35 @@ def bootstrap_cassandra(root: Path) -> None:
|
||||
_migrate_open_cassandra_tasks(kb, workspace)
|
||||
|
||||
|
||||
def _kanban_corruption_type() -> type[Exception]:
|
||||
"""Return Hermes's pinned corruption exception without importing it at startup."""
|
||||
from hermes_cli.kanban_db import KanbanDbCorruptError
|
||||
|
||||
return KanbanDbCorruptError
|
||||
|
||||
|
||||
def bootstrap_cassandra_state(root: Path) -> dict[str, str]:
|
||||
"""Bootstrap Cassandra while preserving a damaged board for explicit recovery."""
|
||||
try:
|
||||
bootstrap_cassandra(root)
|
||||
except Exception as error:
|
||||
if not isinstance(error, _kanban_corruption_type()):
|
||||
raise
|
||||
status = {
|
||||
"state": "corrupt-preserved",
|
||||
"database": str(getattr(error, "db_path", "")),
|
||||
"backup": str(getattr(error, "backup_path", "")),
|
||||
"reason": str(getattr(error, "reason", "integrity check failed")),
|
||||
}
|
||||
print(
|
||||
"Cassandra Kanban board needs recovery; the coordinator preserved "
|
||||
f"the database and backup at {status['backup'] or 'the Hermes data volume'}.",
|
||||
flush=True,
|
||||
)
|
||||
return status
|
||||
return {"state": "ready"}
|
||||
|
||||
|
||||
def sync_cassandra_repo(env_values: dict[str, str]) -> str:
|
||||
"""Clone or fetch Cassandra when the optional Gitea token is available."""
|
||||
token = env_values.get("GITEA_TOKEN", "").strip()
|
||||
@ -166,7 +195,7 @@ def refresh_once(root: Path) -> dict[str, Any]:
|
||||
codex = discover_codex_models()
|
||||
claude = discover_claude_models()
|
||||
routes = configure_routes(root, codex, claude)
|
||||
bootstrap_cassandra(root)
|
||||
board_state = bootstrap_cassandra_state(root)
|
||||
repo_state = sync_cassandra_repo(env_values)
|
||||
status = {
|
||||
"updated_at": datetime.now(timezone.utc).isoformat(),
|
||||
@ -179,6 +208,7 @@ def refresh_once(root: Path) -> dict[str, Any]:
|
||||
"projects": {
|
||||
"cassandra": {
|
||||
"board": "cassandra",
|
||||
"board_state": board_state,
|
||||
"workspace": str(cassandra_workspace()),
|
||||
"repository": CASSANDRA_REMOTE,
|
||||
"state": repo_state,
|
||||
|
||||
@ -294,7 +294,9 @@ def test_refresh_writes_non_secret_routing_status(tmp_path: Path, monkeypatch):
|
||||
claude = routing.Catalog("anthropic", ["claude-opus-5"], True, True, "connected")
|
||||
monkeypatch.setattr(coordinator, "discover_codex_models", lambda: codex)
|
||||
monkeypatch.setattr(coordinator, "discover_claude_models", lambda: claude)
|
||||
monkeypatch.setattr(coordinator, "bootstrap_cassandra", lambda root: None)
|
||||
monkeypatch.setattr(
|
||||
coordinator, "bootstrap_cassandra_state", lambda root: {"state": "ready"}
|
||||
)
|
||||
monkeypatch.setattr(coordinator, "sync_cassandra_repo", lambda env: "ready")
|
||||
|
||||
status = coordinator.refresh_once(tmp_path)
|
||||
@ -302,9 +304,62 @@ def test_refresh_writes_non_secret_routing_status(tmp_path: Path, monkeypatch):
|
||||
status_path = tmp_path / "workspace/coordinator/model-routing.json"
|
||||
assert status_path.is_file()
|
||||
assert status["projects"]["cassandra"]["state"] == "ready"
|
||||
assert status["projects"]["cassandra"]["board_state"] == {"state": "ready"}
|
||||
assert "do-not-report" not in status_path.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_corrupt_cassandra_board_is_preserved_without_blocking_refresh(
|
||||
tmp_path: Path, monkeypatch
|
||||
):
|
||||
"""A damaged project board degrades Kanban instead of stopping all services."""
|
||||
class FakeKanbanDbCorruptError(Exception):
|
||||
def __init__(self):
|
||||
super().__init__("integrity check failed")
|
||||
self.db_path = tmp_path / "kanban.db"
|
||||
self.backup_path = tmp_path / "kanban.db.corrupt.backup"
|
||||
self.reason = "row out of order"
|
||||
|
||||
def raise_corruption(_root: Path) -> None:
|
||||
raise FakeKanbanDbCorruptError()
|
||||
|
||||
monkeypatch.setattr(coordinator, "bootstrap_cassandra", raise_corruption)
|
||||
monkeypatch.setattr(
|
||||
coordinator, "_kanban_corruption_type", lambda: FakeKanbanDbCorruptError
|
||||
)
|
||||
|
||||
status = coordinator.bootstrap_cassandra_state(tmp_path)
|
||||
|
||||
assert status == {
|
||||
"state": "corrupt-preserved",
|
||||
"database": str(tmp_path / "kanban.db"),
|
||||
"backup": str(tmp_path / "kanban.db.corrupt.backup"),
|
||||
"reason": "row out of order",
|
||||
}
|
||||
|
||||
|
||||
def test_unrelated_cassandra_bootstrap_failure_remains_fatal(
|
||||
tmp_path: Path, monkeypatch
|
||||
):
|
||||
"""Only the known fail-closed corruption state may be degraded."""
|
||||
class FakeKanbanDbCorruptError(Exception):
|
||||
pass
|
||||
|
||||
def raise_permission_error(_root: Path) -> None:
|
||||
raise PermissionError("cannot access board")
|
||||
|
||||
monkeypatch.setattr(coordinator, "bootstrap_cassandra", raise_permission_error)
|
||||
monkeypatch.setattr(
|
||||
coordinator, "_kanban_corruption_type", lambda: FakeKanbanDbCorruptError
|
||||
)
|
||||
|
||||
try:
|
||||
coordinator.bootstrap_cassandra_state(tmp_path)
|
||||
except PermissionError as error:
|
||||
assert str(error) == "cannot access board"
|
||||
else:
|
||||
raise AssertionError("unrelated board errors must fail the coordinator refresh")
|
||||
|
||||
|
||||
def test_cassandra_workspace_uses_valid_configured_worktree(
|
||||
tmp_path: Path, monkeypatch
|
||||
):
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user