183 lines
8.8 KiB
Python
183 lines
8.8 KiB
Python
"""Scheduler coverage for coordinator-owned publication retry receipts."""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from contextlib import nullcontext
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
from testing.tests.test_hermes_cli_support import HERMES, _load
|
|
|
|
|
|
state = _load("supervisor_state")
|
|
retry = __import__("publication_retry")
|
|
scheduler = _load("publication_retry_scheduler")
|
|
|
|
|
|
class Native:
|
|
"""Small native surface that records only safe unblock transitions."""
|
|
|
|
def __init__(self, task, root, run):
|
|
self.task, self.root, self.run, self.unblocks, self.event_id = task, root, run, 0, 17
|
|
|
|
def scoped_current_board(self, _board):
|
|
return nullcontext()
|
|
|
|
def connect(self, *, board):
|
|
return SimpleNamespace(
|
|
execute=lambda _sql, _args: SimpleNamespace(fetchone=lambda: self.run),
|
|
close=lambda: None,
|
|
)
|
|
|
|
def list_tasks(self, _connection):
|
|
return [self.task]
|
|
|
|
def get_task(self, _connection, _task_id):
|
|
return self.task
|
|
|
|
def parent_ids(self, _connection, _task_id):
|
|
return [self.root]
|
|
|
|
def unblock_task(self, _connection, _task_id):
|
|
if self.task.status != "blocked":
|
|
return False
|
|
self.task.status = "ready"
|
|
self.unblocks += 1
|
|
return True
|
|
|
|
def unblock_task_if_event(self, connection, _task_id, *, expected_event_id, **_kwargs):
|
|
return self.unblock_task(connection, _task_id) if expected_event_id == self.event_id else False
|
|
|
|
|
|
class Store:
|
|
"""Pool records needed to prove the terminal retry belongs to Hermes."""
|
|
|
|
def __init__(self, records, active=()):
|
|
self.records, self.active = records, list(active)
|
|
|
|
def active_assignments(self):
|
|
return self.active
|
|
|
|
def available_ordinals(self):
|
|
return [0]
|
|
|
|
def terminal_record(self, board, task_id, run_id):
|
|
return self.records.get((board, task_id, run_id))
|
|
|
|
|
|
def _receipt(root, child, baseline, head):
|
|
structured = {
|
|
"status": "completed", "summary": "Replace cache literals.", "changed_files": ["a.go"],
|
|
"tests_run": ["go test ./..."], "artifacts": [], "findings": [], "blockers": [],
|
|
}
|
|
source = {
|
|
"board": "soteria", "task_id": child, "run_id": "8", "worker_ordinal": 0, "attempt": 1,
|
|
"root_task_id": root, "repo_url": "https://scm.bstein.dev/titan/soteria.git",
|
|
"branch": "hermes-repair/cache", "base_branch": "main",
|
|
}
|
|
title, body = structured["summary"], json.dumps(structured, sort_keys=True)
|
|
return {"source": source, "baseline_sha": baseline, "head": head, "title": title, "body": body,
|
|
"structured": structured, "result_digest": retry.receipt_digest(structured, title, body)}
|
|
|
|
|
|
def _setup(tmp_path, monkeypatch):
|
|
board, root, child, baseline, head = "soteria", "t_root", "t_child", "a" * 40, "b" * 40
|
|
monkeypatch.setattr(state, "KANBAN_ROOT", tmp_path / "boards")
|
|
lineage = state.Lineage(root, "hermes-repair/cache", "https://scm.bstein.dev/titan/soteria/pulls/3", "soteria", "main")
|
|
state.record_submission(board, root, lineage, baseline)
|
|
state.record_child(board, child, root, root, "repair", baseline, "replace cache literals")
|
|
receipt = _receipt(root, child, baseline, head)
|
|
binding = {name: receipt["source"][name] for name in ("board", "task_id", "run_id", "worker_ordinal", "attempt")}
|
|
state.record_publication_retry(board, child, binding, receipt)
|
|
task = SimpleNamespace(id=child, status="blocked", block_kind="transient", current_run_id=None)
|
|
native = Native(task, root, (8, "blocked", "blocked"))
|
|
source_record = {"state": "finalized", "worker_ordinal": 0, "payload": {}, "result": {
|
|
"scm_resume": receipt, "structured": {"status": "blocked"}, "capacity_failure": True,
|
|
"scm_submission": None,
|
|
}}
|
|
return board, root, child, baseline, receipt, task, native, source_record
|
|
|
|
|
|
def test_initial_retry_waits_then_reopens_once_with_exact_source_record(tmp_path, monkeypatch):
|
|
"""A clean initial refusal waits, then opens one source-ordinal retry."""
|
|
board, _root, child, _baseline, receipt, _task, native, source = _setup(tmp_path, monkeypatch)
|
|
pool = SimpleNamespace(store=Store({(board, child, "8"): source}))
|
|
monkeypatch.setattr(scheduler, "read_fence", lambda *_args: (17, "fence"))
|
|
monkeypatch.setattr(
|
|
scheduler, "guarded_unblock",
|
|
lambda _db, connection, _task, _root, _run, fence: native.unblock_task_if_event(
|
|
connection, _task, expected_event_id=fence[0]
|
|
),
|
|
)
|
|
|
|
assert scheduler.reopen_due(pool, native, [board], now=100) == 0
|
|
assert scheduler.reopen_due(pool, native, [board], now=401) == 1
|
|
assert native.unblocks == 1
|
|
assert scheduler.reopen_due(pool, native, [board], now=402) == 0
|
|
assert native.unblocks == 1
|
|
|
|
native.task.status, native.task.block_kind = "blocked", "capability"
|
|
assert scheduler.reopen_due(pool, native, [board], now=403) == 0
|
|
native.task.block_kind = "transient"
|
|
native.event_id = 18
|
|
assert scheduler.reopen_due(pool, native, [board], now=403) == 0
|
|
native.event_id = 17
|
|
lineage = state.get_root(board, "t_root")
|
|
state.record_submission(board, child, lineage, "c" * 40)
|
|
assert scheduler.reopen_due(pool, native, [board], now=404) == 0
|
|
|
|
|
|
def test_reissued_transient_receipt_reopens_only_its_last_terminal_run(tmp_path, monkeypatch):
|
|
"""A transient fresh-run failure uses the single release and same ordinal."""
|
|
board, _root, child, _baseline, receipt, task, native, source = _setup(tmp_path, monkeypatch)
|
|
monkeypatch.setattr(state.time, "time", lambda: 100)
|
|
state.issue_publication_retry(board, child, "9")
|
|
assert state.reissue_publication_retry(board, child, "9") is True
|
|
monkeypatch.setattr(state.time, "time", lambda: 401)
|
|
native.run = (9, "blocked", "blocked")
|
|
current = {"worker_ordinal": 0, "payload": {"scm_resume": receipt}, "result": {}}
|
|
pool = SimpleNamespace(store=Store({(board, child, "8"): source, (board, child, "9"): current}))
|
|
current["state"] = "finalized"
|
|
current["result"] = {"structured": {"status": "blocked"}, "capacity_failure": True, "scm_submission": None}
|
|
monkeypatch.setattr(scheduler, "read_fence", lambda *_args: (17, "fence"))
|
|
monkeypatch.setattr(
|
|
scheduler, "guarded_unblock",
|
|
lambda _db, connection, _task, _root, _run, fence: native.unblock_task_if_event(
|
|
connection, _task, expected_event_id=fence[0]
|
|
),
|
|
)
|
|
|
|
assert scheduler.reopen_due(pool, native, [board], now=401) == 1
|
|
assert task.status == "ready" and native.unblocks == 1
|
|
|
|
|
|
def test_reopen_rejects_unproven_or_human_changed_terminal_blocks(tmp_path, monkeypatch):
|
|
"""No stale pool row or later transient human action can reopen a card."""
|
|
board, _root, child, _baseline, _receipt, task, native, source = _setup(tmp_path, monkeypatch)
|
|
monkeypatch.setattr(scheduler, "read_fence", lambda *_args: (17, "fence"))
|
|
monkeypatch.setattr(scheduler, "guarded_unblock", lambda *_args: True)
|
|
# A non-terminal row has no final coordinator block evidence.
|
|
pending = dict(source, state="running")
|
|
assert scheduler.reopen_due(SimpleNamespace(store=Store({(board, child, "8"): pending})), native, [board], now=401) == 0
|
|
# A concurrent pool owner is an authoritative later-live assignment.
|
|
active = {"board": board, "task_id": child, "run_id": "10"}
|
|
assert scheduler.reopen_due(SimpleNamespace(store=Store({(board, child, "8"): source}, [active])), native, [board], now=401) == 0
|
|
# A terminal result without coordinator capacity evidence is not owned.
|
|
denied = dict(source, result={"structured": {"status": "blocked"}, "capacity_failure": False, "scm_submission": None})
|
|
assert scheduler.reopen_due(SimpleNamespace(store=Store({(board, child, "8"): denied})), native, [board], now=401) == 0
|
|
# A later human transient block changes the native event identity and remains blocked.
|
|
monkeypatch.setattr(scheduler, "guarded_unblock", lambda *_args: False)
|
|
assert scheduler.reopen_due(SimpleNamespace(store=Store({(board, child, "8"): source})), native, [board], now=401) == 0
|
|
assert task.status == "blocked" and native.unblocks == 0
|
|
|
|
|
|
def test_missing_receipt_never_turns_a_transient_block_into_retry_work(tmp_path, monkeypatch):
|
|
"""A generic coordinator or human transient block has no publication authority."""
|
|
board, _root, child, _baseline, _receipt, _task, native, source = _setup(tmp_path, monkeypatch)
|
|
with state._connect(board) as connection:
|
|
connection.execute("DELETE FROM publication_retries WHERE board=? AND child_task_id=?", (board, child))
|
|
monkeypatch.setattr(scheduler, "read_fence", lambda *_args: (17, "fence"))
|
|
monkeypatch.setattr(scheduler, "guarded_unblock", lambda *_args: True)
|
|
pool = SimpleNamespace(store=Store({(board, child, "8"): source}))
|
|
assert scheduler.reopen_due(pool, native, [board], now=401) == 0
|