hermes: tolerate preserved Kanban corruption
This commit is contained in:
parent
4cb0faae7f
commit
0590cefa9b
@ -25,7 +25,7 @@ spec:
|
|||||||
ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers
|
ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers
|
||||||
ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback
|
ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback
|
||||||
ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available
|
ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available
|
||||||
ai.bstein.dev/config-rev: "20260813-terminal-lineage-oauth"
|
ai.bstein.dev/config-rev: "20260813-kanban-degraded-startup"
|
||||||
vault.hashicorp.com/agent-inject: "true"
|
vault.hashicorp.com/agent-inject: "true"
|
||||||
vault.hashicorp.com/role: hermes-agent
|
vault.hashicorp.com/role: hermes-agent
|
||||||
vault.hashicorp.com/agent-inject-secret-anthropic-token: kv/data/atlas/hermes/agent-tokens
|
vault.hashicorp.com/agent-inject-secret-anthropic-token: kv/data/atlas/hermes/agent-tokens
|
||||||
|
|||||||
@ -109,6 +109,35 @@ def bootstrap_cassandra(root: Path) -> None:
|
|||||||
_migrate_open_cassandra_tasks(kb, workspace)
|
_migrate_open_cassandra_tasks(kb, workspace)
|
||||||
|
|
||||||
|
|
||||||
|
def _kanban_corruption_type() -> type[Exception]:
|
||||||
|
"""Return Hermes's pinned corruption exception without importing it at startup."""
|
||||||
|
from hermes_cli.kanban_db import KanbanDbCorruptError
|
||||||
|
|
||||||
|
return KanbanDbCorruptError
|
||||||
|
|
||||||
|
|
||||||
|
def bootstrap_cassandra_state(root: Path) -> dict[str, str]:
|
||||||
|
"""Bootstrap Cassandra while preserving a damaged board for explicit recovery."""
|
||||||
|
try:
|
||||||
|
bootstrap_cassandra(root)
|
||||||
|
except Exception as error:
|
||||||
|
if not isinstance(error, _kanban_corruption_type()):
|
||||||
|
raise
|
||||||
|
status = {
|
||||||
|
"state": "corrupt-preserved",
|
||||||
|
"database": str(getattr(error, "db_path", "")),
|
||||||
|
"backup": str(getattr(error, "backup_path", "")),
|
||||||
|
"reason": str(getattr(error, "reason", "integrity check failed")),
|
||||||
|
}
|
||||||
|
print(
|
||||||
|
"Cassandra Kanban board needs recovery; the coordinator preserved "
|
||||||
|
f"the database and backup at {status['backup'] or 'the Hermes data volume'}.",
|
||||||
|
flush=True,
|
||||||
|
)
|
||||||
|
return status
|
||||||
|
return {"state": "ready"}
|
||||||
|
|
||||||
|
|
||||||
def sync_cassandra_repo(env_values: dict[str, str]) -> str:
|
def sync_cassandra_repo(env_values: dict[str, str]) -> str:
|
||||||
"""Clone or fetch Cassandra when the optional Gitea token is available."""
|
"""Clone or fetch Cassandra when the optional Gitea token is available."""
|
||||||
token = env_values.get("GITEA_TOKEN", "").strip()
|
token = env_values.get("GITEA_TOKEN", "").strip()
|
||||||
@ -166,7 +195,7 @@ def refresh_once(root: Path) -> dict[str, Any]:
|
|||||||
codex = discover_codex_models()
|
codex = discover_codex_models()
|
||||||
claude = discover_claude_models()
|
claude = discover_claude_models()
|
||||||
routes = configure_routes(root, codex, claude)
|
routes = configure_routes(root, codex, claude)
|
||||||
bootstrap_cassandra(root)
|
board_state = bootstrap_cassandra_state(root)
|
||||||
repo_state = sync_cassandra_repo(env_values)
|
repo_state = sync_cassandra_repo(env_values)
|
||||||
status = {
|
status = {
|
||||||
"updated_at": datetime.now(timezone.utc).isoformat(),
|
"updated_at": datetime.now(timezone.utc).isoformat(),
|
||||||
@ -179,6 +208,7 @@ def refresh_once(root: Path) -> dict[str, Any]:
|
|||||||
"projects": {
|
"projects": {
|
||||||
"cassandra": {
|
"cassandra": {
|
||||||
"board": "cassandra",
|
"board": "cassandra",
|
||||||
|
"board_state": board_state,
|
||||||
"workspace": str(cassandra_workspace()),
|
"workspace": str(cassandra_workspace()),
|
||||||
"repository": CASSANDRA_REMOTE,
|
"repository": CASSANDRA_REMOTE,
|
||||||
"state": repo_state,
|
"state": repo_state,
|
||||||
|
|||||||
@ -294,7 +294,9 @@ def test_refresh_writes_non_secret_routing_status(tmp_path: Path, monkeypatch):
|
|||||||
claude = routing.Catalog("anthropic", ["claude-opus-5"], True, True, "connected")
|
claude = routing.Catalog("anthropic", ["claude-opus-5"], True, True, "connected")
|
||||||
monkeypatch.setattr(coordinator, "discover_codex_models", lambda: codex)
|
monkeypatch.setattr(coordinator, "discover_codex_models", lambda: codex)
|
||||||
monkeypatch.setattr(coordinator, "discover_claude_models", lambda: claude)
|
monkeypatch.setattr(coordinator, "discover_claude_models", lambda: claude)
|
||||||
monkeypatch.setattr(coordinator, "bootstrap_cassandra", lambda root: None)
|
monkeypatch.setattr(
|
||||||
|
coordinator, "bootstrap_cassandra_state", lambda root: {"state": "ready"}
|
||||||
|
)
|
||||||
monkeypatch.setattr(coordinator, "sync_cassandra_repo", lambda env: "ready")
|
monkeypatch.setattr(coordinator, "sync_cassandra_repo", lambda env: "ready")
|
||||||
|
|
||||||
status = coordinator.refresh_once(tmp_path)
|
status = coordinator.refresh_once(tmp_path)
|
||||||
@ -302,9 +304,62 @@ def test_refresh_writes_non_secret_routing_status(tmp_path: Path, monkeypatch):
|
|||||||
status_path = tmp_path / "workspace/coordinator/model-routing.json"
|
status_path = tmp_path / "workspace/coordinator/model-routing.json"
|
||||||
assert status_path.is_file()
|
assert status_path.is_file()
|
||||||
assert status["projects"]["cassandra"]["state"] == "ready"
|
assert status["projects"]["cassandra"]["state"] == "ready"
|
||||||
|
assert status["projects"]["cassandra"]["board_state"] == {"state": "ready"}
|
||||||
assert "do-not-report" not in status_path.read_text(encoding="utf-8")
|
assert "do-not-report" not in status_path.read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def test_corrupt_cassandra_board_is_preserved_without_blocking_refresh(
|
||||||
|
tmp_path: Path, monkeypatch
|
||||||
|
):
|
||||||
|
"""A damaged project board degrades Kanban instead of stopping all services."""
|
||||||
|
class FakeKanbanDbCorruptError(Exception):
|
||||||
|
def __init__(self):
|
||||||
|
super().__init__("integrity check failed")
|
||||||
|
self.db_path = tmp_path / "kanban.db"
|
||||||
|
self.backup_path = tmp_path / "kanban.db.corrupt.backup"
|
||||||
|
self.reason = "row out of order"
|
||||||
|
|
||||||
|
def raise_corruption(_root: Path) -> None:
|
||||||
|
raise FakeKanbanDbCorruptError()
|
||||||
|
|
||||||
|
monkeypatch.setattr(coordinator, "bootstrap_cassandra", raise_corruption)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
coordinator, "_kanban_corruption_type", lambda: FakeKanbanDbCorruptError
|
||||||
|
)
|
||||||
|
|
||||||
|
status = coordinator.bootstrap_cassandra_state(tmp_path)
|
||||||
|
|
||||||
|
assert status == {
|
||||||
|
"state": "corrupt-preserved",
|
||||||
|
"database": str(tmp_path / "kanban.db"),
|
||||||
|
"backup": str(tmp_path / "kanban.db.corrupt.backup"),
|
||||||
|
"reason": "row out of order",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_unrelated_cassandra_bootstrap_failure_remains_fatal(
|
||||||
|
tmp_path: Path, monkeypatch
|
||||||
|
):
|
||||||
|
"""Only the known fail-closed corruption state may be degraded."""
|
||||||
|
class FakeKanbanDbCorruptError(Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def raise_permission_error(_root: Path) -> None:
|
||||||
|
raise PermissionError("cannot access board")
|
||||||
|
|
||||||
|
monkeypatch.setattr(coordinator, "bootstrap_cassandra", raise_permission_error)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
coordinator, "_kanban_corruption_type", lambda: FakeKanbanDbCorruptError
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
coordinator.bootstrap_cassandra_state(tmp_path)
|
||||||
|
except PermissionError as error:
|
||||||
|
assert str(error) == "cannot access board"
|
||||||
|
else:
|
||||||
|
raise AssertionError("unrelated board errors must fail the coordinator refresh")
|
||||||
|
|
||||||
|
|
||||||
def test_cassandra_workspace_uses_valid_configured_worktree(
|
def test_cassandra_workspace_uses_valid_configured_worktree(
|
||||||
tmp_path: Path, monkeypatch
|
tmp_path: Path, monkeypatch
|
||||||
):
|
):
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user