122 lines
5.1 KiB
Python
122 lines
5.1 KiB
Python
"""Transactional publication-retry fence coverage."""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sqlite3
|
|
from contextlib import contextmanager
|
|
|
|
from testing.tests.test_hermes_cli_support import _load
|
|
|
|
|
|
fence = _load("publication_retry_fence")
|
|
|
|
|
|
class Native:
|
|
"""Minimal existing-native transaction surface used by the adapter."""
|
|
|
|
@staticmethod
|
|
@contextmanager
|
|
def write_txn(connection):
|
|
connection.execute("BEGIN IMMEDIATE")
|
|
try:
|
|
yield
|
|
except Exception:
|
|
connection.rollback()
|
|
raise
|
|
else:
|
|
connection.commit()
|
|
|
|
@staticmethod
|
|
def _append_event(connection, task_id, kind, payload):
|
|
connection.execute(
|
|
"INSERT INTO task_events(task_id,run_id,kind,payload) VALUES(?,?,?,?)",
|
|
(task_id, None, kind, json.dumps(payload, sort_keys=True)),
|
|
)
|
|
|
|
|
|
def _database():
|
|
connection = sqlite3.connect(":memory:")
|
|
connection.row_factory = sqlite3.Row
|
|
connection.executescript("""
|
|
CREATE TABLE tasks(id TEXT PRIMARY KEY,status TEXT,current_run_id INTEGER,
|
|
consecutive_failures INTEGER,last_failure_error TEXT);
|
|
CREATE TABLE task_links(child_id TEXT,parent_id TEXT);
|
|
CREATE TABLE task_events(id INTEGER PRIMARY KEY AUTOINCREMENT,task_id TEXT,
|
|
run_id INTEGER,kind TEXT,payload TEXT);
|
|
""")
|
|
connection.execute("INSERT INTO tasks VALUES('root','done',NULL,0,NULL)")
|
|
connection.execute("INSERT INTO tasks VALUES('child','blocked',NULL,1,'broker')")
|
|
connection.execute("INSERT INTO task_links VALUES('child','root')")
|
|
return connection
|
|
|
|
|
|
def _blocked(connection, marker):
|
|
event_id = connection.execute(
|
|
"INSERT INTO task_events(task_id,run_id,kind,payload) VALUES(?,?,?,?)",
|
|
("child", 8, "blocked", json.dumps({"reason": "transport\\n" + marker})),
|
|
).lastrowid
|
|
connection.commit()
|
|
return event_id
|
|
|
|
|
|
def test_adapter_reopens_only_the_exact_native_fence():
|
|
"""The existing native transaction protects the event check and state change."""
|
|
connection = _database()
|
|
marker = "[hermes-publication-retry-fence:8:" + "a" * 64 + "]"
|
|
event_id = _blocked(connection, marker)
|
|
|
|
assert fence.guarded_unblock(Native, connection, "child", "root", "8", (event_id, marker))
|
|
assert connection.execute("SELECT status FROM tasks WHERE id='child'").fetchone()[0] == "ready"
|
|
assert connection.execute("SELECT kind FROM task_events ORDER BY id DESC LIMIT 1").fetchone()[0] == "unblocked"
|
|
|
|
|
|
def test_adapter_keeps_a_later_human_transient_block_closed():
|
|
"""A later event, even with the same block kind, invalidates coordinator ownership."""
|
|
connection = _database()
|
|
marker = "[hermes-publication-retry-fence:8:" + "b" * 64 + "]"
|
|
event_id = _blocked(connection, marker)
|
|
connection.execute(
|
|
"INSERT INTO task_events(task_id,run_id,kind,payload) VALUES(?,?,?,?)",
|
|
("child", None, "blocked", json.dumps({"reason": "human decision"})),
|
|
)
|
|
connection.commit()
|
|
|
|
assert not fence.guarded_unblock(Native, connection, "child", "root", "8", (event_id, marker))
|
|
assert connection.execute("SELECT status FROM tasks WHERE id='child'").fetchone()[0] == "blocked"
|
|
|
|
|
|
def test_block_records_the_exact_native_event_as_coordinator_owned(tmp_path, monkeypatch):
|
|
"""Only the coordinator's successful fenced block creates a sidecar ownership row."""
|
|
monkeypatch.setattr(fence.supervisor_state, "KANBAN_ROOT", tmp_path / "boards")
|
|
connection = _database()
|
|
connection.execute("UPDATE tasks SET status='running',current_run_id=8 WHERE id='child'")
|
|
connection.commit()
|
|
|
|
class BlockingNative(Native):
|
|
@staticmethod
|
|
def block_task(conn, task_id, *, reason, kind, expected_run_id):
|
|
changed = conn.execute(
|
|
"UPDATE tasks SET status='blocked',current_run_id=NULL WHERE id=? AND status='running' AND current_run_id=?",
|
|
(task_id, expected_run_id),
|
|
).rowcount
|
|
if changed:
|
|
conn.execute(
|
|
"INSERT INTO task_events(task_id,run_id,kind,payload) VALUES(?,?,?,?)",
|
|
(task_id, expected_run_id, "blocked", json.dumps({"reason": reason, "kind": kind})),
|
|
)
|
|
return bool(changed)
|
|
|
|
receipt = {"result_digest": "c" * 64}
|
|
assert fence.block(BlockingNative, connection, "soteria", "child", "8", "transport", "transient", receipt)
|
|
event_id = connection.execute("SELECT id FROM task_events ORDER BY id DESC LIMIT 1").fetchone()[0]
|
|
marker = "[hermes-publication-retry-fence:8:" + "c" * 64 + "]"
|
|
assert fence.read("soteria", "child", "8") == (event_id, marker)
|
|
# Simulate a crash after native block but before its sidecar fence record.
|
|
with fence.supervisor_state._connect("soteria") as state:
|
|
state.execute("DELETE FROM publication_retry_fences")
|
|
state.execute(
|
|
"INSERT INTO publication_retries(board,child_task_id,source_run_id,source_ordinal,receipt_json) VALUES(?,?,?,?,?)",
|
|
("soteria", "child", "8", 0, json.dumps(receipt)),
|
|
)
|
|
assert fence.recover("soteria", "child", "8", connection, receipt) == (event_id, marker)
|