titan/atlas-iac
3
0
Fork 0
Code Issues Pull Requests 9 Actions Packages Projects Releases Wiki Activity

hermes: resume preserved PR publications without model retries

Browse Source
This commit is contained in:
jenkins 2026-09-13 17:54:54 -05:00
parent 4a94977dbc
commit bf2a1ab528
26 changed files with 1647 additions and 208 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
dockerfiles/Dockerfile.hermes-agent
View File

@ -1628,10 +1628,7 @@ COPY dockerfiles/hermes_execution_regression_support.py /tmp/hermes_execution_re
COPY dockerfiles/hermes_run_safety_regression.py /tmp/hermes_run_safety_regression.py
COPY dockerfiles/hermes_decomposition_safety_regression.py /tmp/hermes_decomposition_safety_regression.py
COPY dockerfiles/hermes_lane_compatibility_regression.py /tmp/hermes_lane_compatibility_regression.py
COPY services/hermes/scripts/cli_lane_*.py /tmp/hermes-lane-regression/
COPY services/hermes/scripts/routing_catalog.py /tmp/hermes-lane-regression/
COPY services/hermes/scripts/supervisor_lineage.py /tmp/hermes-lane-regression/
COPY services/hermes/scripts/supervisor_state.py /tmp/hermes-lane-regression/
COPY services/hermes/scripts/*.py /tmp/hermes-lane-regression/
RUN HERMES_CLI_LANE_SOURCE=/tmp/hermes-lane-regression \
HERMES_COMPATIBILITY_MODE=legacy \
/opt/hermes/.venv/bin/python /tmp/hermes_lane_compatibility_regression.py \

5
dockerfiles/Dockerfile.hermes-agent.dockerignore
View File

@ -19,7 +19,4 @@
!services/
!services/hermes/
!services/hermes/scripts/
!services/hermes/scripts/cli_lane_*.py
!services/hermes/scripts/routing_catalog.py
!services/hermes/scripts/supervisor_lineage.py
!services/hermes/scripts/supervisor_state.py
!services/hermes/scripts/*.py

23
services/hermes/NOTES.md
View File

@ -63,6 +63,23 @@ the floor, change the failed plan, and raise capability or effort when needed.
The concrete selected model, capability, effort, and rationale are recorded on
durable worker receipts.
The hourly coordinator discovers the models visible to the Codex and Claude
accounts, writes non-secret catalog and health evidence, and evaluates catalog
candidates for capability fit before publishing Switchyard profiles. Claude
discovery uses its Agent SDK initialize metadata, including the account-visible
model list; it does not run a generation just to discover a model. A provider,
catalog, or evaluation outage preserves the last known-good routes and records
the refresh as deferred, so discovery does not silently become a routing bypass.
Existing pull-request continuation is available through coordinator-owned
lineage: `kanban_continue_pr.py` validates the root task, recorded PR, branch,
and current head before queuing a repair on that same PR. A signed continuation
worker may update that exact owned ref through the SCM broker. Publication-only
resume for a completed but unpublished SCM result is still being finalized;
until its exact-run ownership, evidence preservation, and bounded finite-retry
semantics are confirmed, keep the workspace commit and structured result
recoverable and do not claim that path is live.
## Private Jetson voice: multilingual TTS policy
`hermes-tts` on `titan-21` bakes three checksum-pinned Piper voices and
@ -501,9 +518,9 @@ A `voice` field from a browser is never honoured at any hop.
`healthChecks`: gating that 10m window on a best-effort pool would stall
`hermes-chat` and `hermes-observer-bindings`, which `dependsOn: hermes`.
- A hashed execution-pool ConfigMap, protocol-version readiness checks, and
versioned SCM boundary name make code/config changes controlled rollouts. A
rollout is still a human-reviewed operation; this repository change does not
reconcile or deploy it.
versioned SCM boundary name make code/config changes controlled rollouts.
Verify the config revision, protocol readiness, and worker/mediator health
after Flux applies a reviewed rollout.
## Your shortest path to fluency

6
services/hermes/kustomization.yaml
View File

@ -107,9 +107,13 @@ configMapGenerator:
- execution_pool_server.py=scripts/execution_pool_server.py
- execution_pool_client.py=scripts/execution_pool_client.py
- execution_pool_worker.py=scripts/execution_pool_worker.py
- execution_pool_resume.py=scripts/execution_pool_resume.py
- execution_pool_scm.py=scripts/execution_pool_scm.py
- supervisor_lineage.py=scripts/supervisor_lineage.py
- supervisor_state.py=scripts/supervisor_state.py
- publication_retry.py=scripts/publication_retry.py
- scm_resume_bootstrap.py=scripts/scm_resume_bootstrap.py
- bootstrap_soteria_publication_retry.py=scripts/bootstrap_soteria_publication_retry.py
- seed_legacy_scm_roots.py=scripts/seed_legacy_scm_roots.py
- deadline_http.py=scm-common/scripts/deadline_http.py
- gitea_api_policy.py=scm-common/scripts/gitea_api_policy.py
@ -178,6 +182,8 @@ configMapGenerator:
- supervisor_policy.py=scripts/supervisor_policy.py
- supervisor_lineage.py=scripts/supervisor_lineage.py
- supervisor_state.py=scripts/supervisor_state.py
- publication_retry.py=scripts/publication_retry.py
- scm_resume_bootstrap.py=scripts/scm_resume_bootstrap.py
- deadline_http.py=scm-common/scripts/deadline_http.py
- gitea_api_policy.py=scm-common/scripts/gitea_api_policy.py
- scm_broker_client.py=scm-common/scripts/scm_broker_client.py

38
services/hermes/scm-common/scripts/scm_broker.py
View File

@ -30,7 +30,7 @@ from scm_task_grants import TaskLedger, verify_grant
from scm_task_drafts import matches_pull, request_fields, update as update_draft
from scm_task_adoptions import seed as seed_adoptions
from scm_broker_io import RejectRedirect, response_status as _status, spool_response
from scm_broker_server import AbsoluteHeaderDeadlineMixin, BoundedThreadingHTTPServer
from scm_broker_server import AbsoluteHeaderDeadlineMixin, BoundedThreadingHTTPServer, log_rejection, validate_ascii_headers
BROKER_PORT = 9081
GIT_USER = "hermes-automation"
@ -264,8 +264,6 @@ def _upstream_git_request(
return result.read()
finally:
result.close()
class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler):
"""Expose only bounded metadata, draft creation, and smart-HTTP Git."""
@ -274,20 +272,8 @@ class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler):
header_deadline_seconds = INBOUND_HEADER_TIMEOUT
def _validate_headers(self) -> None:
items = list(self.headers.items())
if len(items) > MAX_HEADERS:
raise PolicyError("SCM request has too many headers")
total = 0
for name, value in items:
if not name.isascii() or not value.isascii():
raise PolicyError("SCM request headers must be ASCII")
total += len(name) + len(value) + 4
if any(ord(character) < 32 and character != "\t" for character in value):
raise PolicyError("SCM request header contains controls")
if total > MAX_HEADER_BYTES:
raise PolicyError("SCM request headers exceed the safe limit")
if len(self.headers.get_all("Content-Length", [])) > 1:
raise PolicyError("SCM request has duplicate Content-Length")
validate_ascii_headers(self.headers, maximum_count=MAX_HEADERS,
maximum_bytes=MAX_HEADER_BYTES, error=PolicyError)
def _stream(
self, status: int, content_type: str, body: BinaryIO, length: int
@ -318,7 +304,7 @@ class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler):
self.end_headers()
self.wfile.write(body)
def _reject(self, status: int = 400) -> None:
def _reject(self, status: int = 400, phase: str = "", category: str = "") -> None:
self._json(status, b'{"error":"request rejected"}')
def do_GET(self) -> None:
@ -358,9 +344,11 @@ class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler):
def do_POST(self) -> None:
try:
self._phase = "headers"
self._validate_headers()
self.connection.settimeout(INBOUND_BODY_TIMEOUT)
if self.path in {"/v1/metadata", "/v1/drafts", "/v1/tasks/register", "/v1/tasks/draft-update"}:
self._phase = "control"
self._control()
else:
self._git_rpc()
@ -371,7 +359,8 @@ class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler):
urllib.error.URLError,
ValueError,
json.JSONDecodeError,
):
) as error:
log_rejection(getattr(self, "_phase", "request"), error)
self._reject()
def _control(self) -> None:
@ -409,12 +398,14 @@ class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler):
self._json(200, result)
def _git_rpc(self) -> None:
self._phase = "target"
repo, operation, service = _git_target(self.path)
if (
operation not in {"git-upload-pack", "git-receive-pack"}
or service != operation
):
raise PolicyError("Git RPC operation is outside the allowlist")
self._phase = "headers"
expected_request = f"application/x-{service}-request"
if self.headers.get_content_type() != expected_request or self.headers.get(
"Transfer-Encoding"
@ -423,6 +414,7 @@ class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler):
length = _content_length(self.headers, MAX_GIT_REQUEST)
token = read_token()
_reject_forbidden(repo, "repository", (token,))
self._phase = "body"
body, body_length = _spool_bounded(
self.rfile,
MAX_GIT_REQUEST,
@ -437,10 +429,13 @@ class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler):
if service == "git-receive-pack":
raw_grant = self.headers.get("X-Hermes-Task-Grant", "")
if raw_grant:
self._phase = "grant"
claims = verify_grant(raw_grant)
if claims["repo"] != repo:
raise PolicyError("task grant repository does not match Git target")
self._phase = "ledger"
_task_ledger().authorize_update(claims)
self._phase = "pack"
validate_receive_pack(
body, token, _credential_forms(token),
expected=(claims["expected_old"], claims["new_head"], claims["ref"]),
@ -450,6 +445,7 @@ class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler):
# moved without an owned, signed task grant.
_validate_receive_pack(body, token)
expected = f"application/x-{service}-result"
self._phase = "upstream"
try:
streamed = _upstream_git_request(
f"/titan/{repo}.git/{service}",
@ -473,18 +469,18 @@ class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler):
# Smart HTTP uses HTTP 200 for both Git success and a rejected
# ref command. The authenticated branch read is the commit
# point; never advance the local ledger on packet status alone.
self._phase = "persist"
if _branch_head(repo, claims["ref"], token) != claims["new_head"]:
result.close()
raise PolicyError("Git upstream did not advance the granted branch")
_task_ledger().commit(claims)
try:
self._phase = "response"
self._stream(200, expected, result, result_length)
finally:
result.close()
finally:
body.close()
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--listen", default="0.0.0.0")

25
services/hermes/scm-common/scripts/scm_broker_server.py
View File

@ -5,11 +5,36 @@ from __future__ import annotations
import threading
import time
import logging
from http.server import ThreadingHTTPServer
MAX_CONCURRENT_REQUESTS = 3
def validate_ascii_headers(headers, *, maximum_count: int, maximum_bytes: int, error) -> None:
"""Reject oversized or ambiguous broker headers before a request body is read."""
items = list(headers.items())
if len(items) > maximum_count:
raise error("SCM request has too many headers")
total = 0
for name, value in items:
if not name.isascii() or not value.isascii():
raise error("SCM request headers must be ASCII")
total += len(name) + len(value) + 4
if any(ord(character) < 32 and character != "\t" for character in value):
raise error("SCM request header contains controls")
if total > maximum_bytes:
raise error("SCM request headers exceed the safe limit")
if len(headers.get_all("Content-Length", [])) > 1:
raise error("SCM request has duplicate Content-Length")
def log_rejection(phase: str, error: BaseException) -> None:
"""Log a fixed operational category without request material or error text."""
category = "policy" if error.__class__.__name__ == "PolicyError" else "upstream" if error.__class__.__name__ in {"TimeoutError", "URLError"} else "io"
logging.warning("scm_rejected phase=%s category=%s", phase, category)
class _AbsoluteDeadlineReader:
"""Read header lines against one wall-clock deadline, not idle timeouts."""

167
services/hermes/scripts/bootstrap_soteria_publication_retry.py Normal file
View File

@ -0,0 +1,167 @@
#!/usr/bin/env python3
"""One-time, fail-closed recovery of Soteria #3's retained publication evidence."""
from __future__ import annotations
import argparse
import hashlib
import json
import os
import sqlite3
import stat
from pathlib import Path
from typing import Any
import supervisor_state
from execution_pool_protocol import payload_digest, read_key, sign_envelope
BOARD = "soteria"
CHILD = "t_4095fe0d"
ROOT = "t_c7c42600"
RUN_ID = "8"
ORDINAL = 0
HEAD = "f21833e78768e7e4045a6304e311989196846ae4"
REQUIRED_RECEIPT = {
"source", "baseline_sha", "head", "title", "body", "structured", "result_digest"
}
def _value(task: Any, name: str) -> Any:
return task.get(name) if isinstance(task, dict) else getattr(task, name, None)
def _receipt(path: Path) -> dict[str, Any]:
"""Read the mediator-produced receipt without following an operator symlink."""
descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
try:
info = os.fstat(descriptor)
if not stat.S_ISREG(info.st_mode) or info.st_size > 48 * 1024:
raise ValueError("bootstrap receipt file is invalid")
raw = os.read(descriptor, 48 * 1024 + 1)
finally:
os.close(descriptor)
try:
value = json.loads(raw)
except (UnicodeDecodeError, json.JSONDecodeError) as error:
raise ValueError("bootstrap receipt is malformed") from error
if not isinstance(value, dict) or set(value) != REQUIRED_RECEIPT:
raise ValueError("bootstrap receipt shape is invalid")
return value
def _pool_record(database: Path) -> tuple[dict[str, Any], str, int]:
"""Read exactly run 8's finalized terminal evidence without writing pool state."""
source = sqlite3.connect(f"file:{database}?mode=ro", uri=True)
try:
row = source.execute(
"SELECT payload_json,result_json,result_digest,state,worker_ordinal,attempt FROM assignments "
"WHERE board=? AND task_id=? AND run_id=?", (BOARD, CHILD, RUN_ID)
).fetchone()
finally:
source.close()
if row is None or row[3] != "finalized" or row[4] != ORDINAL or not isinstance(row[5], int) or row[5] < 1:
raise ValueError("bootstrap source assignment is not the retained terminal run")
try:
assignment, result = json.loads(row[0]), json.loads(row[1])
except (TypeError, json.JSONDecodeError) as error:
raise ValueError("bootstrap terminal evidence is malformed") from error
structured = result.get("structured") if isinstance(result, dict) else None
if (
not isinstance(assignment, dict) or assignment.get("root_task_id") != ROOT
or assignment.get("continuation_kind") != "repair"
or not isinstance(structured, dict) or structured.get("status") != "blocked"
or result.get("returncode") != 0 or not isinstance(result.get("capacity_failure"), bool)
or result.get("scm_submission") is not None
or payload_digest(result) != row[2]
):
raise ValueError("bootstrap terminal evidence is not a clean publication refusal")
with sqlite3.connect(f"file:{database}?mode=ro", uri=True) as source:
later = source.execute(
"SELECT 1 FROM assignments WHERE board=? AND task_id=? AND run_id<>? "
"AND state IN ('assigned','running','result')", (BOARD, CHILD, RUN_ID)
).fetchone()
if later is not None:
raise ValueError("bootstrap has a newer live pool assignment")
return assignment, hashlib.sha256(row[1].encode()).hexdigest(), row[5]
def _native_guard() -> supervisor_state.Lineage:
"""Require the still-blocked exact native run and the private root/child chain."""
from hermes_cli import kanban_db
child = supervisor_state.get_child(BOARD, CHILD)
if child is None or child["root_task_id"] != ROOT or child["kind"] != "repair":
raise ValueError("bootstrap continuation lineage is unavailable")
with kanban_db.scoped_current_board(BOARD):
connection = kanban_db.connect(board=BOARD)
try:
task = kanban_db.get_task(connection, CHILD)
parents = kanban_db.parent_ids(connection, CHILD)
latest = connection.execute(
"SELECT id,status,outcome FROM task_runs WHERE task_id=? ORDER BY id DESC LIMIT 1", (CHILD,)
).fetchone()
finally:
connection.close()
if (
task is None or _value(task, "status") != "blocked"
or _value(task, "current_run_id") is not None
or latest is None or latest[0] != int(RUN_ID)
or latest[1] not in {"done", "failed"}
or not isinstance(parents, (list, tuple, set)) or ROOT not in {str(value) for value in parents}
):
raise ValueError("bootstrap native task is no longer the exact blocked run")
return child["lineage"]
def bootstrap(receipt_path: Path, pool_database: Path) -> str:
"""Persist one receipt only after the independent native and pool guards agree."""
receipt = _receipt(receipt_path)
assignment, raw_sha, attempt = _pool_record(pool_database)
lineage = _native_guard()
source = receipt["source"]
if (
receipt.get("head") != HEAD or not isinstance(source, dict)
or source.get("attempt") != attempt
or any(source.get(name) != value for name, value in {
"board": BOARD, "task_id": CHILD, "run_id": RUN_ID, "worker_ordinal": ORDINAL,
"root_task_id": ROOT, "repo_url": f"https://scm.bstein.dev/titan/{lineage.project}.git",
"branch": lineage.branch, "base_branch": lineage.base_branch,
}.items())
):
raise ValueError("bootstrap mediator receipt does not match retained authority")
binding = {name: source[name] for name in ("board", "task_id", "run_id", "worker_ordinal", "attempt")}
supervisor_state.record_publication_retry(BOARD, CHILD, binding, receipt)
supervisor_state.record_publication_retry_provenance(BOARD, CHILD, raw_sha)
return raw_sha
def signed_assignment(pool_database: Path, key_file: Path) -> bytes:
"""Attest the verified retained assignment with the current coordinator key."""
assignment, _raw_sha, attempt = _pool_record(pool_database)
_native_guard()
return json.dumps(sign_envelope(read_key(key_file), "assignment", {
"board": BOARD, "task_id": CHILD, "run_id": RUN_ID,
"worker_ordinal": ORDINAL, "attempt": attempt,
}, assignment), separators=(",", ":"), sort_keys=True).encode()
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--receipt-file", type=Path)
parser.add_argument("--pool-db", type=Path, default=Path(os.environ.get("HERMES_HOME", "/opt/data")) / "execution-pool/assignments.db")
parser.add_argument("--emit-assignment", action="store_true")
parser.add_argument("--key-file", type=Path, default=Path(os.environ.get("HERMES_EXECUTION_POOL_KEY_FILE", "/runtime-access/execution-pool-key")))
args = parser.parse_args()
if args.emit_assignment == (args.receipt_file is not None):
raise SystemExit("select exactly one bootstrap action")
if args.emit_assignment:
print(signed_assignment(args.pool_db, args.key_file).decode())
return 0
bootstrap(args.receipt_file, args.pool_db)
print(f"bootstrapped {BOARD}/{CHILD} run={RUN_ID} head={HEAD[:12]}")
return 0
if __name__ == "__main__":
raise SystemExit(main())

5
services/hermes/scripts/cli_lane_dispatch.py
View File

@ -141,6 +141,7 @@ def claim_ready(
limit: int,
eligible: Callable[[str, Any], bool] | None = None,
claimer: str = "direct-cli-lane",
priority: Callable[[str, Any], int] | None = None,
) -> list[tuple[str, str]]:
"""Atomically claim external ready tasks across all non-archived boards."""
from hermes_cli import kanban_db
@ -158,7 +159,9 @@ def claim_ready(
continue
try:
kanban_db.recompute_ready(conn)
tasks = kanban_db.list_tasks(conn)
tasks = list(kanban_db.list_tasks(conn))
if priority is not None:
tasks.sort(key=lambda task: priority(board, task))
BOARD_CORRUPTION_ERRORS.pop(board, None)
for task in tasks:
task_id = str(_task_value(task, "id", ""))

115
services/hermes/scripts/execution_pool_client.py
View File

@ -10,6 +10,7 @@ import urllib.error
import urllib.request
from http.server import BaseHTTPRequestHandler
from pathlib import Path
from collections.abc import Callable
from typing import Any
import cli_lane_goal
@ -72,6 +73,9 @@ class ClientBoundary:
self.key = key
self.scm = scm or SCMBoundary(key)
self.current: dict[str, Any] | None = None
self.resumed: dict[str, str] | None = None
self.resumed_evidence: dict[str, Any] | None = None
self.resume_transient = False
self.lock = threading.RLock()
def _post(self, path: str, envelope: dict[str, Any]) -> dict[str, Any]:
@ -105,8 +109,12 @@ class ClientBoundary:
return {"assignment": None}
if response["kind"] != "assignment" or response["worker_ordinal"] != ORDINAL:
raise ProtocolError("coordinator returned a foreign assignment")
checkout = self.scm.checkout(response)
publication_only = isinstance(response.get("payload", {}).get("scm_resume"), dict)
checkout = {"workspace": "", "baseline_sha": ""} if publication_only else self.scm.checkout(response)
self.current = response
self.resumed = None
self.resumed_evidence = None
self.resume_transient = False
assignment = {
**_binding(response),
"payload": response["payload"],
@ -123,6 +131,32 @@ class ClientBoundary:
raise ProtocolError("local request does not own the current assignment")
return self.current, dict(supplied)
def _publication_lease(self, binding: dict[str, Any]) -> tuple[Callable[[], None], Callable[[], None]]:
"""Keep a resume lease alive outside the SCM lock and fail before another step."""
stop, failures = threading.Event(), []
def renew() -> None:
try:
response = self._post("/v1/heartbeat", sign_envelope(self.key, "heartbeat", binding, {"note": "SCM publication in progress"}))
if response["kind"] != "ack" or _binding(response) != binding or not response["payload"].get("accepted"):
raise ProtocolError("coordinator rejected SCM publication lease")
except (OSError, ProtocolError, ValueError, urllib.error.URLError) as error:
failures.append(error)
renew()
def keepalive() -> None:
while not stop.wait(20):
renew()
thread = threading.Thread(target=keepalive, daemon=True)
thread.start()
def checkpoint() -> None:
if failures:
raise ProtocolError("SCM publication lease was lost") from failures[0]
return checkpoint, lambda: (stop.set(), thread.join(timeout=1))
def heartbeat(self, request: dict[str, Any]) -> dict[str, Any]:
payload = request.get("payload")
if not isinstance(payload, dict):
@ -150,8 +184,23 @@ class ClientBoundary:
prior work recoverable and reviewable by a human.
"""
try:
submission = self.scm.submit(assignment, request)
if isinstance(self.scm, SCMBoundary):
checkpoint, close = self._publication_lease(_binding(assignment))
try:
submission = self.scm.submit(assignment, request, checkpoint=checkpoint)
checkpoint()
finally:
close()
else:
submission = self.scm.submit(assignment, request)
except (OSError, ProtocolError, RuntimeError, ValueError) as error:
payload = assignment.get("payload")
resume = None
if isinstance(payload, dict) and payload.get("continuation_kind") == "repair":
try:
resume = self.scm.resume_artifact(assignment, request, structured)
except (OSError, ProtocolError, RuntimeError, ValueError):
resume = None
structured["status"] = "blocked"
reason = (
"Distributed SCM submission failed; the commits remain on this "
@ -159,7 +208,7 @@ class ClientBoundary:
)
if reason not in structured["blockers"]:
structured["blockers"].append(reason)
return None
return {"resume": resume} if resume is not None else None
pull = str(submission.get("pull_request") or "")
branch = str(submission.get("branch") or "")
for artifact in (pull, f"branch:{branch}" if branch else ""):
@ -168,17 +217,66 @@ class ClientBoundary:
head = str(submission.get("head") or "")
return {"branch": branch, "pull_request": pull, "head": head}
def resume(self, request: dict[str, Any]) -> dict[str, Any]:
"""Publish the coordinator-carried artifact without exposing it to the worker."""
with self.lock:
assignment, _binding = self._current_for(request.get("binding"))
artifact = assignment.get("payload", {}).get("scm_resume")
if not isinstance(artifact, dict):
raise ProtocolError("assignment has no SCM resume artifact")
evidence = _validate_result({"structured": artifact.get("structured")})["structured"]
if evidence["status"] != "completed":
raise ProtocolError("SCM resume evidence is not completed")
checkpoint, close = self._publication_lease(_binding)
try:
self.resume_transient = False
try:
self.resumed = self.scm.resume(assignment, artifact, checkpoint=checkpoint)
checkpoint()
except (OSError, RuntimeError, urllib.error.URLError) as error:
detail = str(error).lower()
if any(marker in detail for marker in ("timed out", "connection", "could not resolve", "http 502", "http 503", "http 504")):
self.resume_transient = True
return {"publication_retry_transient": True}
raise ProtocolError("SCM resume publication was rejected") from error
finally:
close()
self.resumed_evidence = json.loads(canonical_json(evidence))
return {"scm_submission": self.resumed}
def finish(self, request: dict[str, Any]) -> dict[str, Any]:
payload = _validate_result(request.get("payload"))
# The model-facing caller cannot classify a retry as transient. Only a
# preceding mediator resume may attach this coordinator control signal.
payload.pop("publication_retry_transient", None)
with self.lock:
assignment, binding = self._current_for(request.get("binding"))
structured = payload["structured"]
if self.resume_transient:
payload["publication_retry_transient"] = True
if structured["status"] == "completed" and int(payload.get("returncode", 1)) == 0:
submission = self._submit(assignment, request, structured)
if submission is not None:
# The mediator, not the model-facing request, records the
# broker-confirmed PR/branch in the signed terminal wire.
payload["scm_submission"] = submission
if self.resumed is not None:
if self.resumed_evidence is None:
raise ProtocolError("SCM resume evidence is unavailable")
structured = json.loads(canonical_json(self.resumed_evidence))
payload["structured"] = structured
payload["returncode"] = 0
payload["scm_submission"] = self.resumed
else:
submission = self._submit(assignment, request, structured)
if submission is not None and "resume" not in submission:
# The mediator, not the model-facing request, records the
# broker-confirmed PR/branch in the signed terminal wire.
payload["scm_submission"] = submission
elif submission is not None:
# The exact clean commit remains on the durable worker volume.
# A broker refusal is execution infrastructure, not a provider
# capability verdict, so let the coordinator retry it after the
# scoped SCM condition is corrected.
payload["capacity_failure"] = True
payload["scm_resume"] = submission["resume"]
else:
payload["capacity_failure"] = True
response = self._post(
"/v1/result", sign_envelope(self.key, "result", binding, payload)
)
@ -223,6 +321,7 @@ def handler_factory(boundary: ClientBoundary) -> type[BaseHTTPRequestHandler]:
routes = {
"poll": boundary.poll,
"heartbeat": lambda: boundary.heartbeat(request),
"resume": lambda: boundary.resume(request),
"finish": lambda: boundary.finish(request),
}
if operation not in routes:

49
services/hermes/scripts/execution_pool_coordinator.py
View File

@ -159,6 +159,17 @@ def assignment_payload(
else:
repo_url, branch, base_branch = resolve_scm(task, board)
root_task_id = lineage.root_task_id if lineage is not None else str(_task_value(task, "id"))
run_id = canonical_run_id(_task_value(task, "current_run_id", None))
resume = None
if child is not None and run_id is not None:
try:
resume = supervisor_state.publication_retry(board, str(_task_value(task, "id")), str(run_id))
except (OSError, ValueError) as error:
raise ProtocolError("supervisor publication retry state is unavailable") from error
if resume is not None:
# The mediator republishes private evidence; the worker never sees a
# coding objective and must not invoke a model for this fresh run.
context = "Republish the mediator-verified completed SCM handoff."
runtime = int(_task_value(task, "max_runtime_seconds", 0) or 12 * 60 * 60)
runtime = max(60, min(runtime, 12 * 60 * 60))
return {
@ -173,6 +184,7 @@ def assignment_payload(
"deadline_unix": int(time.time()) + runtime,
"goal_mode": bool(_task_value(task, "goal_mode", False)),
"goal_max_turns": max(1, min(int(_task_value(task, "goal_max_turns", 1) or 1), 12)),
"scm_resume": resume,
}
@ -349,9 +361,38 @@ class Coordinator:
"blockers": structured.get("blockers", []),
}
assignment = record.get("payload")
if isinstance(assignment, dict) and assignment.get("scm_resume") is not None:
# A coordinator can crash after durable add but before the
# maintenance pass fences this receipt. Finalization must
# repair that narrow gap before it advances the live PR head.
supervisor_state.issue_publication_retry(
binding["board"], binding["task_id"], binding["run_id"]
)
verified_submission = _submission_lineage(
binding, assignment, payload.get("scm_submission")
)
retry = payload.get("scm_resume")
retry_exhausted = False
if retry is not None:
if (
verified_submission is not None
or not payload.get("capacity_failure")
or int(payload.get("returncode", 1)) != 0
or structured.get("status") != "blocked"
):
raise ProtocolError("publication retry receipt has an invalid terminal state")
supervisor_state.record_publication_retry(
binding["board"], binding["task_id"], binding, retry
)
elif (
isinstance(assignment, dict) and assignment.get("scm_resume") is not None
and payload.get("publication_retry_transient") is True
and payload.get("capacity_failure") is True
and verified_submission is None and structured.get("status") == "blocked"
):
retry_exhausted = not supervisor_state.reissue_publication_retry(
binding["board"], binding["task_id"], binding["run_id"]
)
problem = cli_lane_goal.unfinished_result_reason(structured)
if (
structured.get("status") == "completed"
@ -372,6 +413,10 @@ class Coordinator:
supervisor_state.record_submission(
binding["board"], binding["task_id"], lineage, head
)
if isinstance(assignment, dict) and assignment.get("scm_resume") is not None:
supervisor_state.resolve_publication_retry(
binding["board"], binding["task_id"], binding["run_id"]
)
changed = kanban_db.complete_task(
connection, binding["task_id"],
result=json.dumps(structured, sort_keys=True),
@ -381,9 +426,11 @@ class Coordinator:
else:
reason = problem or "; ".join(map(str, structured.get("blockers", [])))
reason = reason or str(structured.get("summary") or "worker failed")
if retry_exhausted:
reason = "Publication retry budget exhausted; inspect the retained SCM evidence."
changed = kanban_db.block_task(
connection, binding["task_id"], reason=reason,
kind="transient" if payload.get("capacity_failure") else "capability",
kind="transient" if payload.get("capacity_failure") and not retry_exhausted else "capability",
expected_run_id=run_id,
)
self.store.finalize(binding, "finalized" if changed else "stale")

109
services/hermes/scripts/execution_pool_maintenance.py
View File

@ -24,6 +24,7 @@ import cli_lane_dispatch
from cli_lane_config import canonical_run_id
from execution_pool_project import ProjectPolicyError, distributed_workspace_eligible
from execution_pool_protocol import ProtocolError
import supervisor_state
RECOVERABLE_BOARD_ERRORS = (OSError, sqlite3.Error)
@ -59,6 +60,35 @@ def _context(record: dict[str, Any]) -> str:
return f"{record['board']}/{record['task_id']}#{record['run_id']}"
def _resume_ordinal(payload: Any) -> int | None:
"""Read the coordinator-issued source ordinal without accepting task text."""
receipt = payload.get("scm_resume") if isinstance(payload, dict) else None
if receipt is None:
return None
source = receipt.get("source") if isinstance(receipt, dict) else None
ordinal = source.get("worker_ordinal") if isinstance(source, dict) else None
if isinstance(ordinal, bool) or not isinstance(ordinal, int) or ordinal not in range(3):
raise ProtocolError("publication retry ordinal is invalid")
return ordinal
def _issue_retry(record: dict[str, Any]) -> None:
"""Fence a persisted receipt to its one fresh durable assignment."""
if _resume_ordinal(record.get("payload")) is not None:
supervisor_state.issue_publication_retry(
record["board"], record["task_id"], record["run_id"]
)
def _pending_retry_ordinal(board: str, task: Any) -> int | None:
"""Look up an unissued receipt before the ready card receives its fresh run."""
task_id = str(_task_value(task, "id", "") or "")
if not task_id:
return None
receipt = supervisor_state.publication_retry(board, task_id, "")
return _resume_ordinal({"scm_resume": receipt}) if receipt is not None else None
def _block_exact(
kanban_db: Any,
board: str,
@ -98,7 +128,7 @@ def recover_results(pool: Any) -> None:
def _lease_failure_state(
kanban_db: Any, connection: Any, binding: dict[str, Any], run_id: int
kanban_db: Any, connection: Any, binding: dict[str, Any], run_id: int, retry_exhausted: bool = False
) -> str:
"""Classify one exhausted lease from authoritative Kanban evidence only."""
task = kanban_db.get_task(connection, binding["task_id"])
@ -114,11 +144,12 @@ def _lease_failure_state(
if kanban_db.block_task(
connection,
binding["task_id"],
reason=(
reason=("Publication retry budget exhausted; inspect the retained SCM evidence." if retry_exhausted else
(
"Distributed worker lease expired after "
f"{binding['attempt']} fenced attempts"
),
kind="transient",
)),
kind="capability" if retry_exhausted else "transient",
expected_run_id=run_id,
):
return "finalized"
@ -143,10 +174,15 @@ def record_lease_failure(pool: Any, record: dict[str, Any]) -> None:
if run_id is None:
pool.store.finalize(binding, "stale")
return
retry_exhausted = False
if _resume_ordinal(record.get("payload")) is not None:
retry_exhausted = not supervisor_state.reissue_publication_retry(
binding["board"], binding["task_id"], binding["run_id"]
)
with pool._kanban_lock, kanban_db.scoped_current_board(binding["board"]):
connection = kanban_db.connect(board=binding["board"])
try:
state = _lease_failure_state(kanban_db, connection, binding, run_id)
state = _lease_failure_state(kanban_db, connection, binding, run_id, retry_exhausted)
finally:
connection.close()
pool.store.finalize(binding, state)
@ -222,9 +258,16 @@ def _adopt_task(
f"{type(error).__name__}: {error}",
)
return
ordinal = _resume_ordinal(payload)
if ordinal is not None:
if ordinal not in ordinals:
_defer(f"{board}/{task_id} recovery retry", ProtocolError("source ordinal is unavailable"))
return
else:
ordinal = ordinals[0]
binding = {
"board": board, "task_id": task_id, "run_id": run_id,
"worker_ordinal": ordinals[0], "attempt": 1,
"worker_ordinal": ordinal, "attempt": 1,
}
try:
pool.store.add(binding, payload)
@ -234,7 +277,8 @@ def _adopt_task(
_defer(f"{board}/{task_id} adoption", error)
return
known.add((board, task_id, run_id))
ordinals.pop(0)
_issue_retry({**binding, "payload": payload})
ordinals.remove(ordinal)
def _adopt_board(
@ -262,6 +306,7 @@ def reconcile(pool: Any) -> None:
for record in pool.store.active_assignments():
try:
_issue_retry(record)
_release_moved_run(pool, kanban_db, record)
except Exception as error: # noqa: BLE001 - other assignments still reconcile
_defer(f"{_context(record)} release", error)
@ -324,15 +369,22 @@ def _materialize(
finally:
connection.close()
run_id = str(database_run_id)
expected_ordinal = _resume_ordinal(payload)
if expected_ordinal is not None and expected_ordinal != ordinal:
raise ProtocolError("publication retry was not assigned to its source ordinal")
if (board, task_id, run_id) in known:
return
pool.store.add(
added = pool.store.add(
{
"board": board, "task_id": task_id, "run_id": run_id,
"worker_ordinal": ordinal, "attempt": 1,
},
payload,
)
if added:
_issue_retry({
"board": board, "task_id": task_id, "run_id": run_id, "payload": payload,
})
known.add((board, task_id, run_id))
_settled(f"{board}/{task_id} dispatch")
except (ProtocolError, *RECOVERABLE_BOARD_ERRORS) as error:
@ -354,19 +406,56 @@ def dispatch(pool: Any) -> None:
ordinals = pool.store.available_ordinals()
if not ordinals:
return
retry_ordinals: dict[tuple[str, str], int | None] = {}
def retry_ordinal(board: str, task: Any) -> int | None:
key = (board, str(_task_value(task, "id", "") or ""))
if key not in retry_ordinals:
retry_ordinals[key] = _pending_retry_ordinal(board, task)
return retry_ordinals[key]
def eligible(board: str, task: Any) -> bool:
if not distributed_workspace_eligible(task):
return False
try:
required = retry_ordinal(board, task)
except (OSError, ValueError) as error:
_defer(f"{board}/{_task_value(task, 'id', '')} retry lookup", error)
return False
return required is None or required in ordinals
def priority(board: str, task: Any) -> int:
try:
return 0 if retry_ordinal(board, task) is not None else 1
except (OSError, ValueError) as error:
_defer(f"{board}/{_task_value(task, 'id', '')} retry lookup", error)
return 2
try:
# Only storage faults are absorbed here. An incompatible claim API must
# still surface, because silently skipping the eligibility predicate is
# how the pool would start claiming the local lane's owned workspaces.
claimed = cli_lane_dispatch.claim_ready(
set(), len(ordinals),
lambda _board, task: distributed_workspace_eligible(task),
eligible,
claimer="execution-pool",
priority=priority,
)
except RECOVERABLE_BOARD_ERRORS as error:
_defer("claim-ready", error)
return
_settled("claim-ready")
known = pool.store.known_runs()
for ordinal, (board, task_id) in zip(ordinals, claimed, strict=False):
assigned: set[int] = set()
ordered = sorted(
claimed,
key=lambda item: 0 if retry_ordinals.get(item) is not None else 1,
)
for board, task_id in ordered:
ordinal = retry_ordinals.get((board, task_id))
if ordinal is None:
ordinal = next((value for value in ordinals if value not in assigned), None)
if ordinal is None or ordinal in assigned:
_defer(f"{board}/{task_id} dispatch", ProtocolError("no safe worker ordinal is available"))
continue
_materialize(pool, kanban_db, board, task_id, ordinal, known)
assigned.add(ordinal)

62
services/hermes/scripts/execution_pool_resume.py Normal file
View File

@ -0,0 +1,62 @@
#!/usr/bin/env python3
"""Build the fresh terminal result for a mediator-published SCM retry."""
from __future__ import annotations
import re
from typing import Any
from execution_pool_protocol import ProtocolError
FIELDS = ("changed_files", "tests_run", "artifacts", "findings", "blockers")
SHA = re.compile(r"[0-9a-f]{40,64}\Z")
def result(receipt: Any, submission: Any, node: str) -> tuple[dict[str, Any], str, str]:
"""Reuse completed evidence only after the mediator confirms its exact head."""
if not isinstance(receipt, dict) or not isinstance(submission, dict):
raise ProtocolError("publication retry response is malformed")
structured = receipt.get("structured")
title, body, head = receipt.get("title"), receipt.get("body"), receipt.get("head")
if (
not isinstance(structured, dict) or structured.get("status") != "completed"
or not isinstance(title, str) or not title or not isinstance(body, str)
or not isinstance(head, str) or not SHA.fullmatch(head)
or submission.get("head") != head
):
raise ProtocolError("publication retry evidence changed")
copied = {"status": "completed", "summary": str(structured.get("summary") or "")[:8000]}
if not copied["summary"]:
raise ProtocolError("publication retry summary is missing")
for name in FIELDS:
value = structured.get(name)
if not isinstance(value, list) or any(not isinstance(item, str) for item in value):
raise ProtocolError("publication retry result is malformed")
copied[name] = list(value[:100])
for artifact in (submission.get("pull_request"), f"branch:{submission.get('branch', '')}", f"preserved-head:{head}"):
if isinstance(artifact, str) and artifact and artifact not in copied["artifacts"]:
copied["artifacts"].append(artifact)
return ({"structured": copied, "returncode": 0, "capacity_failure": False, "node": node,
"route": {"provider": "mediator", "model": "publication-resume"},
"provider_sessions": {}, "final_activity": "Mediator republished preserved SCM commit."}, title[:240], body[:12000])
def transient_failure(node: str) -> tuple[dict[str, Any], str, str]:
"""Return the sole mediator-classified retryable publication failure."""
structured = {"status": "blocked", "summary": "SCM publication is temporarily unavailable.",
"changed_files": [], "tests_run": [], "artifacts": [], "findings": [],
"blockers": ["Mediator classified the preserved SCM publication as transient."]}
return ({"structured": structured, "returncode": 1, "capacity_failure": True,
"publication_retry_transient": True, "node": node, "route": {},
"provider_sessions": {}, "final_activity": "Mediator deferred preserved SCM publication."}, "SCM publication deferred", "")
def rejected_failure(node: str) -> tuple[dict[str, Any], str, str]:
"""Surface mediator policy rejection without treating it as capacity loss."""
structured = {"status": "blocked", "summary": "SCM publication retry was rejected.",
"changed_files": [], "tests_run": [], "artifacts": [], "findings": [],
"blockers": ["Mediator rejected the preserved SCM publication evidence or policy."]}
return ({"structured": structured, "returncode": 1, "capacity_failure": False, "node": node,
"route": {}, "provider_sessions": {}, "final_activity": "Mediator rejected publication retry."},
"SCM publication rejected", "")

193
services/hermes/scripts/execution_pool_scm.py
View File

@ -4,6 +4,7 @@
from __future__ import annotations
import json
import hashlib
import os
import re
import stat
@ -12,12 +13,13 @@ import threading
import time
import urllib.parse
from pathlib import Path
from collections.abc import Callable
from typing import Any
import scm_broker_client
from scm_task_grants import ZERO_SHA, sign_grant
from execution_pool_project import ATLAS_REPO, validate_branch
from execution_pool_protocol import ProtocolError, atomic_json, verify_envelope
from execution_pool_protocol import ProtocolError, atomic_json, canonical_json, verify_envelope
WORKSPACE_ROOT = Path(os.environ.get("HERMES_WORKER_ROOT", "/workspace"))
@ -25,6 +27,31 @@ SCM_ROOT = Path(os.environ.get("HERMES_SCM_STATE_ROOT", "/scm-state"))
ORDINAL = int(os.environ.get("HERMES_WORKER_ORDINAL", "-1"))
BROKER_ORIGIN = scm_broker_client.BROKER_ORIGIN.rstrip("/")
MAX_STATUS_BYTES = 4 * 1024 * 1024
MAX_PULL_PAGES = 20
def _push_failure(error: RuntimeError) -> str:
"""Return bounded remediation without reflecting Git headers or grant text."""
detail = str(error).lower()
if any(marker in detail for marker in (
"task branch head changed", "non-fast-forward", "fetch first",
)):
return "task branch changed; fetch and merge before retrying"
if any(marker in detail for marker in (
"signature", "not registered", "owned by another task",
"authentication", "unauthorized", "forbidden", "permission denied",
"http 401", "http 403",
)):
return (
"SCM broker authorization rejected the branch update; "
"inspect task ownership before retrying"
)
if any(marker in detail for marker in (
"timed out", "connection", "could not resolve", "http 502",
"http 503", "http 504",
)):
return "SCM broker transport failed; retry the preserved local commit"
return "SCM broker rejected the branch update; inspect broker or upstream state before retrying"
def _git_environment() -> dict[str, str]:
@ -46,16 +73,20 @@ def _run(*arguments: str, cwd: Path | None = None, timeout: int = 300) -> str:
"core.fsmonitor=false",
*arguments,
]
completed = subprocess.run(
command,
cwd=cwd,
env=_git_environment(),
stdin=subprocess.DEVNULL,
text=True,
capture_output=True,
timeout=timeout,
check=False,
)
try:
completed = subprocess.run(
command,
cwd=cwd,
env=_git_environment(),
stdin=subprocess.DEVNULL,
text=True,
capture_output=True,
timeout=timeout,
check=False,
)
except subprocess.TimeoutExpired as error:
# Do not serialize ``error``: it includes the full command and its grant header.
raise RuntimeError("SCM command timed out") from error
if completed.returncode:
message = (completed.stderr or completed.stdout or "SCM operation failed")[-2000:]
raise RuntimeError(message.strip())
@ -193,6 +224,20 @@ def _workspace_identity(destination: Path, repo: str, branch: str) -> str:
return head
def bootstrap_resume_artifact(key: bytes, assignment: Any, terminal: Any, *, title: str, body: str) -> dict[str, Any]:
"""Issue one receipt inside the mediator from two retained signed envelopes."""
boundary = Boundary(key)
source = boundary.verify(assignment)
result = verify_envelope(key, terminal, expected_kind="result")
names = ("board", "task_id", "run_id", "worker_ordinal", "attempt")
if any(source[name] != result[name] for name in names):
raise ProtocolError("retained terminal result does not match assignment")
payload = result.get("payload")
if not isinstance(payload, dict) or not isinstance(payload.get("structured"), dict):
raise ProtocolError("retained terminal result is malformed")
return boundary.resume_artifact(source, {"title": title, "body": body}, payload["structured"])
class Boundary:
"""The only process allowed to turn model output into an SCM/result handoff."""
@ -268,11 +313,13 @@ class Boundary:
@staticmethod
def _draft(repo: str, branch: str, base: str, head: str, title: str, body: str, grant: str = "", *, refresh: bool = True, existing_only: bool = False) -> str:
query = urllib.parse.urlencode({"state": "open", "limit": 50})
existing = json.loads(
scm_broker_client.read(f"/api/v1/repos/titan/{repo}/pulls?{query}")
)
if isinstance(existing, list):
for page in range(1, MAX_PULL_PAGES + 1):
query = urllib.parse.urlencode({"state": "open", "limit": 50, "page": page})
existing = json.loads(
scm_broker_client.read(f"/api/v1/repos/titan/{repo}/pulls?{query}")
)
if not isinstance(existing, list):
raise ProtocolError("open pull-request discovery is malformed")
for item in existing:
if not isinstance(item, dict):
continue
@ -287,6 +334,10 @@ class Boundary:
raise ProtocolError("existing task draft cannot be refreshed")
updated = json.loads(scm_broker_client.update_draft(grant, number, title, body))
return str(updated.get("html_url") or "")
if len(existing) < 50:
break
else:
raise ProtocolError("open pull-request discovery exceeds the safe page limit")
if existing_only:
raise ProtocolError("review continuation has no existing pull request")
created = json.loads(
@ -301,7 +352,7 @@ class Boundary:
)
return str(created.get("html_url") or "")
def submit(self, envelope: dict[str, Any], request: dict[str, Any]) -> dict[str, Any]:
def submit(self, envelope: dict[str, Any], request: dict[str, Any], *, checkpoint: Callable[[], None] | None = None) -> dict[str, Any]:
"""Enforce clean/committed state, broker push, and reviewed draft creation."""
_payload, repo, branch, base = _binding(envelope)
destination = workspace_path(envelope)
@ -310,6 +361,8 @@ class Boundary:
if not title or len(title.encode()) > 512 or len(body.encode()) > 32 * 1024:
raise ProtocolError("pull-request metadata exceeds the safe limit")
with self.lock:
if checkpoint:
checkpoint()
head = _workspace_identity(destination, repo, branch)
state = json.loads(_regular_text(_state_path(envelope), 16 * 1024))
baseline = state.get("baseline_sha") if isinstance(state, dict) else ""
@ -323,27 +376,125 @@ class Boundary:
if status:
raise ProtocolError("workspace has uncommitted or untracked files")
target = submission_refs(branch, int(envelope["attempt"]), head)[0]
review = _payload.get("continuation_kind") == "review"
kind = _payload.get("continuation_kind")
repair, review = kind == "repair", kind == "review"
if repair:
# Do not advance an owned branch after its human PR was closed.
self._draft(repo, target, base, head, title, body, refresh=False, existing_only=True)
heads = _remote_heads(destination, (target,))
if checkpoint:
checkpoint()
remote = heads.get(target)
ahead = int(_run("rev-list", "--count", f"{baseline}..{head}", cwd=destination))
if ahead <= 0 and remote is None:
return {"workspace": str(destination), "branch": branch, "pull_request": ""}
if remote == head:
if checkpoint:
checkpoint()
grant = self._grant(envelope, repo, target, base, remote, head)
pull = self._draft(repo, target, base, head, title, body, grant) if not review else self._draft(repo, target, base, head, title, body, grant, refresh=False, existing_only=True)
pull = self._draft(repo, target, base, head, title, body, grant) if not (repair or review) else self._draft(
repo, target, base, head, title, body, grant, refresh=not review, existing_only=True
)
return {"workspace": str(destination), "branch": target, "pull_request": pull, "head": head}
expected = remote or ZERO_SHA
grant = self._grant(envelope, repo, target, base, expected, head)
if remote is None:
scm_broker_client.register_task(grant)
try:
if checkpoint:
checkpoint()
_run(
"-c", f"http.extraHeader=X-Hermes-Task-Grant: {grant}",
"push", "--no-thin", "hermes-broker", f"HEAD:refs/heads/{target}",
cwd=destination, timeout=900,
)
except RuntimeError as error:
raise ProtocolError("task branch update was rejected; fetch and merge before retrying") from error
pull = self._draft(repo, target, base, head, title, body, self._grant(envelope, repo, target, base, head, head))
raise ProtocolError(_push_failure(error)) from error
if checkpoint:
checkpoint()
grant = self._grant(envelope, repo, target, base, head, head)
pull = self._draft(repo, target, base, head, title, body, grant) if not (repair or review) else self._draft(
repo, target, base, head, title, body, grant, refresh=not review, existing_only=True
)
return {"workspace": str(destination), "branch": target, "pull_request": pull, "head": head}
def resume_artifact(self, envelope: dict[str, Any], request: dict[str, Any], structured: dict[str, Any]) -> dict[str, Any]:
"""Bind one clean, locally durable commit for a later fresh-grant retry."""
payload, repo, branch, base = _binding(envelope)
destination = workspace_path(envelope)
head = _workspace_identity(destination, repo, branch)
state = json.loads(_regular_text(_state_path(envelope), 16 * 1024))
baseline = state.get("baseline_sha") if isinstance(state, dict) else ""
title, body = str(request.get("title") or "").strip(), str(request.get("body") or "")
if not isinstance(baseline, str) or not re.fullmatch(r"[0-9a-f]{40,64}", baseline):
raise ProtocolError("private SCM baseline is unavailable")
if _run("status", "--porcelain=v1", "--untracked-files=all", cwd=destination):
raise ProtocolError("workspace has uncommitted or untracked files")
if not title or len(title.encode()) > 512 or len(body.encode()) > 32 * 1024:
raise ProtocolError("pull-request metadata exceeds the safe limit")
root = payload.get("root_task_id")
if not isinstance(root, str) or not root:
raise ProtocolError("assignment continuation root is unavailable")
source = {name: envelope[name] for name in ("board", "task_id", "run_id", "worker_ordinal", "attempt")}
source.update({"repo_url": payload.get("repo_url"), "branch": branch, "base_branch": base, "root_task_id": root})
evidence = {
"structured": json.loads(canonical_json(structured)),
"title": title,
"body": body,
}
return {"source": source, "baseline_sha": baseline, "head": head, **evidence,
"result_digest": hashlib.sha256(canonical_json(evidence)).hexdigest()}
def resume(self, envelope: dict[str, Any], artifact: Any, *, checkpoint: Callable[[], None] | None = None) -> dict[str, Any]:
"""Publish a verified preserved head with only the new assignment grant."""
payload, repo, branch, base = _binding(envelope)
required = {"source", "baseline_sha", "head", "title", "body", "structured", "result_digest"}
if not isinstance(artifact, dict) or set(artifact) != required:
raise ProtocolError("SCM resume artifact is malformed")
source = artifact["source"]
source_keys = {"board", "task_id", "run_id", "worker_ordinal", "attempt", "repo_url", "branch", "base_branch", "root_task_id"}
if not isinstance(source, dict) or set(source) != source_keys:
raise ProtocolError("SCM resume source binding is malformed")
if (source["board"] != envelope["board"] or source["task_id"] != envelope["task_id"]
or source["run_id"] == envelope["run_id"] or not isinstance(source["attempt"], int) or source["attempt"] < 1
or not isinstance(source["worker_ordinal"], int) or source["worker_ordinal"] != envelope["worker_ordinal"]
or source["repo_url"] != payload.get("repo_url")
or source["branch"] != branch or source["base_branch"] != base or source["root_task_id"] != payload.get("root_task_id")):
raise ProtocolError("SCM resume source does not match assignment")
evidence = {name: artifact[name] for name in ("structured", "title", "body")}
if not isinstance(artifact["result_digest"], str) or artifact["result_digest"] != hashlib.sha256(canonical_json(evidence)).hexdigest():
raise ProtocolError("SCM resume evidence digest is invalid")
head, baseline = artifact["head"], artifact["baseline_sha"]
if not all(isinstance(value, str) and re.fullmatch(r"[0-9a-f]{40,64}", value) for value in (head, baseline)):
raise ProtocolError("SCM resume revisions are invalid")
if not isinstance(artifact["title"], str) or not artifact["title"] or len(artifact["title"].encode()) > 512 or not isinstance(artifact["body"], str) or len(artifact["body"].encode()) > 32 * 1024:
raise ProtocolError("SCM resume metadata is invalid")
source_envelope = {name: source[name] for name in ("board", "task_id", "run_id", "worker_ordinal", "attempt")}
destination = workspace_path(source_envelope)
if checkpoint:
checkpoint()
if _workspace_identity(destination, repo, branch) != head:
raise ProtocolError("preserved SCM workspace head changed")
state = json.loads(_regular_text(_state_path(source_envelope), 16 * 1024))
if not isinstance(state, dict) or state.get("baseline_sha") != baseline:
raise ProtocolError("preserved SCM baseline changed")
if _run("status", "--porcelain=v1", "--untracked-files=all", cwd=destination):
raise ProtocolError("preserved SCM workspace is not clean")
if checkpoint:
checkpoint()
target = submission_refs(branch, int(envelope["attempt"]), head)[0]
self._draft(repo, target, base, head, artifact["title"], artifact["body"], refresh=False, existing_only=True)
remote = _remote_heads(destination, (target,)).get(target)
if remote not in {baseline, head}:
raise ProtocolError("task branch changed; preserved commit needs reconciliation")
grant = self._grant(envelope, repo, target, base, head if remote == head else baseline, head)
if remote != head:
if checkpoint:
checkpoint()
_run("-c", f"http.extraHeader=X-Hermes-Task-Grant: {grant}", "push", "--no-thin", "hermes-broker", f"HEAD:refs/heads/{target}", cwd=destination, timeout=900)
if checkpoint:
checkpoint()
if checkpoint:
checkpoint()
pull = self._draft(repo, target, base, head, artifact["title"], artifact["body"], self._grant(envelope, repo, target, base, head, head), existing_only=True)
return {"workspace": str(destination), "branch": target, "pull_request": pull, "head": head}

86
services/hermes/scripts/execution_pool_worker.py
View File

@ -1,8 +1,6 @@
#!/usr/bin/env python3
"""Run one fenced Hermes assignment on an ordinal-scoped durable workspace."""
from __future__ import annotations
import json
import os
import re
@ -15,15 +13,13 @@ import urllib.request
from dataclasses import asdict
from pathlib import Path
from typing import Any
import cli_lane_runner
import execution_pool_resume
from execution_pool_protocol import (
ProtocolError,
atomic_json,
canonical_json,
)
ROOT = Path(os.environ.get("HERMES_WORKER_ROOT", "/workspace"))
ORDINAL = int(os.environ.get("HERMES_WORKER_ORDINAL", "-1"))
CLIENT = os.environ.get(
@ -34,8 +30,6 @@ NODE = os.environ.get("HERMES_WORKER_NODE", "unknown")[:128]
RUN_PART = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$")
RETENTION_SECONDS = int(os.environ.get("HERMES_WORKER_RETENTION_SECONDS", "1209600"))
DISABLED_PROVIDER = os.environ.get("HERMES_EXECUTION_DISABLED_PROVIDER", "").strip()
def _post(url: str, value: dict[str, Any], timeout: int = 60) -> dict[str, Any]:
request = urllib.request.Request(
url, data=canonical_json(value), method="POST",
@ -49,15 +43,11 @@ def _post(url: str, value: dict[str, Any], timeout: int = 60) -> dict[str, Any]:
if not isinstance(document, dict):
raise ProtocolError("server response is not an object")
return document
def _client(operation: str, **values: Any) -> dict[str, Any]:
response = _post(f"{CLIENT}/v1/client", {"operation": operation, **values})
if response.get("error"):
raise ProtocolError(str(response["error"]))
return response
def _poll() -> dict[str, Any] | None:
assignment = _client("poll").get("assignment")
if assignment is None:
@ -69,14 +59,10 @@ def _poll() -> dict[str, Any] | None:
):
raise ProtocolError("local boundary returned a foreign assignment")
return assignment
def _binding(assignment: dict[str, Any]) -> dict[str, Any]:
return {name: assignment[name] for name in (
"board", "task_id", "run_id", "worker_ordinal", "attempt"
)}
def _state_path(assignment: dict[str, Any]) -> Path:
parts = tuple(str(assignment[name]) for name in ("board", "task_id", "run_id"))
if any(not RUN_PART.fullmatch(part) for part in parts):
@ -91,8 +77,6 @@ def _state_path(assignment: dict[str, Any]) -> Path:
raise ProtocolError("assignment state path contains a symlink")
path.resolve(strict=False).relative_to(root)
return path
def _preserve_runtime_directory(runtime: Path) -> None:
"""Move one legacy provider directory aside before installing our symlink."""
parent = runtime.parent
@ -112,8 +96,6 @@ def _preserve_runtime_directory(runtime: Path) -> None:
raise ProtocolError(f"provider session migration failed: {runtime.name}") from error
if preserved.is_symlink() or not preserved.is_dir():
raise ProtocolError(f"provider session migration is unsafe: {runtime.name}")
def _bind_provider_sessions(assignment: dict[str, Any]) -> None:
"""Attach provider session directories to this exact durable task/run."""
parts = tuple(str(assignment[name]) for name in ("board", "task_id", "run_id"))
@ -169,18 +151,12 @@ def _bind_provider_sessions(assignment: dict[str, Any]) -> None:
elif runtime.exists():
raise ProtocolError(f"provider session path is not a symlink: {runtime.name}")
runtime.symlink_to(durable)
def _prompt(context: str, workspace: Path, binding: dict[str, Any]) -> str:
return f"""You are a durable coding worker managed by Hermes Kanban.
Work only on this objective and its acceptance criteria:
{context}
Workspace: {workspace}
Run binding: board={binding['board']} task={binding['task_id']} run={binding['run_id']} worker={ORDINAL}
Operate autonomously only inside this private assigned checkout. Inspect before editing,
preserve unrelated and untracked files, and fail closed rather than overwrite state you do
not understand. You have no Kubernetes identity and no SCM credential. Do not attempt to
@ -188,13 +164,10 @@ read Secrets, mutate workloads, use exec/attach/port-forward, reach node roots,
the reviewed SCM boundary. Commit intended changes locally on the assigned feature branch;
the worker boundary handles the bounded push and draft pull request after validation.
Switchyard owns provider/model/effort selection and cross-provider fallback.
Return a final JSON object matching the supplied schema. Use status=incomplete when work,
tests, commits, or verification remain. Use blocked only for a concrete task obstacle.
Completed must have no blockers. List changed files, tests, artifacts, findings, and blockers.
"""
def _read_activity(log_path: Path, offset: int) -> tuple[str, int]:
flags = os.O_RDONLY | os.O_NONBLOCK | getattr(os, "O_NOFOLLOW", 0)
try:
@ -213,8 +186,6 @@ def _read_activity(log_path: Path, offset: int) -> tuple[str, int]:
finally:
os.close(descriptor)
return value.decode("utf-8", "replace"), next_offset
def _git(workspace: Path, *arguments: str) -> str:
completed = subprocess.run(
["git", "-C", str(workspace), *arguments], stdin=subprocess.DEVNULL,
@ -224,8 +195,6 @@ def _git(workspace: Path, *arguments: str) -> str:
if completed.returncode:
raise RuntimeError((completed.stderr or "Git inspection failed")[-1000:])
return completed.stdout.strip()
def _bounded_result(value: dict[str, Any]) -> dict[str, Any]:
"""Keep terminal evidence useful while fitting the authenticated wire cap."""
bounded: dict[str, Any] = {}
@ -245,15 +214,11 @@ def _bounded_result(value: dict[str, Any]) -> dict[str, Any]:
else:
bounded["summary"] = bounded["summary"][: len(bounded["summary"]) // 2]
return bounded
def _refresh_assignment(binding: dict[str, Any]) -> dict[str, Any]:
fresh = _poll()
if fresh is None or _binding(fresh) != binding:
raise ProtocolError("assignment changed before SCM submission")
return fresh
def garbage_collect(now: float | None = None) -> int:
"""Remove only clean, terminal, assignment-derived workspaces after retention."""
current = time.time() if now is None else now
@ -274,17 +239,44 @@ def garbage_collect(now: float | None = None) -> int:
workspace, "status", "--porcelain=v1", "--untracked-files=all"
):
continue
try:
branch = _git(workspace, "symbolic-ref", "--short", "HEAD")
except RuntimeError:
continue
landed = False
for remote in ("origin", "hermes-broker"):
try:
_git(workspace, "merge-base", "--is-ancestor", "HEAD", f"{remote}/{branch}")
landed = True
break
except RuntimeError:
continue
if not landed:
# A clean terminal checkout can still contain an unpublished commit.
# Keep it until a trusted remote-tracking ref proves the handoff landed.
continue
shutil.rmtree(workspace)
state_file.unlink()
removed += 1
return removed
def execute(assignment: dict[str, Any]) -> None:
binding = _binding(assignment)
payload = assignment["payload"]
if not isinstance(payload, dict):
raise ProtocolError("assignment payload is invalid")
if payload.get("scm_resume") is not None:
try:
response = _client("resume", binding=binding)
terminal, title, body = (
execution_pool_resume.transient_failure(NODE)
if response.get("publication_retry_transient") is True else
execution_pool_resume.result(payload["scm_resume"], response.get("scm_submission"), NODE)
)
except ProtocolError:
terminal, title, body = execution_pool_resume.rejected_failure(NODE)
if not _client("finish", binding=binding, payload=terminal, title=title, body=body).get("ack", {}).get("accepted"):
raise ProtocolError("coordinator did not accept the publication retry")
return
_bind_provider_sessions(assignment)
workspace = Path(str(assignment.get("workspace") or "")).resolve(strict=True)
workspace.relative_to((ROOT / "runs").resolve())
@ -296,7 +288,6 @@ def execute(assignment: dict[str, Any]) -> None:
log_path = state_file.with_suffix(".log")
offset = 0
latest_route: dict[str, Any] = {}
def heartbeat(note: str) -> bool:
nonlocal offset
try:
@ -311,7 +302,6 @@ def execute(assignment: dict[str, Any]) -> None:
return bool(response.get("ack", {}).get("accepted"))
except (OSError, ValueError, urllib.error.URLError, json.JSONDecodeError):
return False
context = str(payload.get("context") or "")
assignee = str(payload.get("assignee") or "cli-auto")
excluded = None
@ -397,8 +387,6 @@ def execute(assignment: dict[str, Any]) -> None:
raise ProtocolError("coordinator did not accept the terminal result")
state["terminal_at"] = time.time()
atomic_json(state_file, state)
def report_exception(assignment: dict[str, Any], error: Exception) -> bool:
"""Surface one worker exception as an exact-run transient result."""
binding = _binding(assignment)
@ -431,8 +419,6 @@ def report_exception(assignment: dict[str, Any], error: Exception) -> bool:
)
except (OSError, ValueError, urllib.error.URLError, json.JSONDecodeError):
return False
def readiness() -> None:
if ORDINAL not in range(3):
raise ProtocolError("worker ordinal must be 0, 1, or 2")
@ -453,16 +439,6 @@ def readiness() -> None:
atomic_json(
cli_lane_runner.RESULT_SCHEMA_PATH, cli_lane_runner.RESULT_SCHEMA, 0o644
)
try:
_poll()
except urllib.error.HTTPError as error:
# A replacement pod can overlap the prior ordinal's coordinator lease.
# The main loop already defers this transient conflict until ownership
# transfers, so keep the container alive instead of CrashLooping.
if error.code != 409:
raise
def main() -> int:
readiness()
while True:
@ -482,7 +458,5 @@ def main() -> int:
flush=True,
)
time.sleep(10)
if __name__ == "__main__": # pragma: no cover - exercised by the container entrypoint
raise SystemExit(main())

72
services/hermes/scripts/publication_retry.py Normal file
View File

@ -0,0 +1,72 @@
#!/usr/bin/env python3
"""Validate mediator-issued, one-shot SCM publication retry receipts."""
from __future__ import annotations
import hashlib
import json
import re
from typing import Any
class PublicationRetryError(ValueError):
"""A receipt cannot safely bind a fresh continuation run to an old commit."""
SHA = re.compile(r"[0-9a-f]{40,64}\Z")
RESULT_FIELDS = frozenset(
{"status", "summary", "changed_files", "tests_run", "artifacts", "findings", "blockers"}
)
def receipt_digest(structured: dict[str, Any], title: str, body: str) -> str:
"""Return the exact digest the mediator must retain with completed evidence."""
value = {"structured": structured, "title": title, "body": body}
return hashlib.sha256(json.dumps(
value, ensure_ascii=False, separators=(",", ":"), sort_keys=True
).encode()).hexdigest()
def validate(
receipt: Any, binding: dict[str, Any], *, board: str, child_task_id: str,
root_task_id: str, project: str, branch: str, base_branch: str, live_head: str,
) -> tuple[dict[str, Any], int, str]:
"""Return a receipt only when its source and completed evidence are exact."""
source = receipt.get("source") if isinstance(receipt, dict) else None
if not isinstance(source, dict) or not all(
source.get(name) == binding.get(name)
for name in ("board", "task_id", "run_id", "worker_ordinal", "attempt")
):
raise PublicationRetryError("publication retry source binding is invalid")
expected = {
"board": board, "task_id": child_task_id, "root_task_id": root_task_id,
"repo_url": f"https://scm.bstein.dev/titan/{project}.git",
"branch": branch, "base_branch": base_branch,
}
if any(source.get(name) != value for name, value in expected.items()):
raise PublicationRetryError("publication retry source lineage is invalid")
run_id, ordinal, attempt = (source["run_id"], source["worker_ordinal"], source["attempt"])
if (
not isinstance(run_id, str) or not run_id.isdecimal() or int(run_id) < 1
or isinstance(ordinal, bool) or not isinstance(ordinal, int) or ordinal not in range(3)
or isinstance(attempt, bool) or not isinstance(attempt, int) or attempt < 1
):
raise PublicationRetryError("publication retry source run is invalid")
baseline, head, digest = (receipt.get(name) for name in ("baseline_sha", "head", "result_digest"))
title, body, structured = receipt.get("title"), receipt.get("body"), receipt.get("structured")
if (
not all(isinstance(value, str) and SHA.fullmatch(value) for value in (baseline, head))
or not isinstance(digest, str) or not re.fullmatch(r"[0-9a-f]{64}", digest)
or baseline != live_head or baseline == head
or not isinstance(title, str) or not title or len(title.encode()) > 512
or not isinstance(body, str) or len(body.encode()) > 32 * 1024
or not isinstance(structured, dict) or set(structured) != RESULT_FIELDS
or structured.get("status") != "completed"
):
raise PublicationRetryError("publication retry evidence is invalid")
if digest != receipt_digest(structured, title, body):
raise PublicationRetryError("publication retry result digest is invalid")
for name in RESULT_FIELDS - {"status", "summary"}:
if not isinstance(structured[name], list) or any(not isinstance(item, str) for item in structured[name]):
raise PublicationRetryError("publication retry result is invalid")
return receipt, ordinal, run_id

95
services/hermes/scripts/scm_resume_bootstrap.py Executable file
View File

@ -0,0 +1,95 @@
#!/usr/bin/env python3
"""Recover one completed SCM receipt from a mediator-owned worker result log."""
from __future__ import annotations
import argparse
import json
import os
import re
import stat
import sys
from pathlib import Path
from typing import Any
from execution_pool_protocol import ProtocolError, canonical_json, read_key
from execution_pool_scm import Boundary
ROOT = Path(os.environ.get("HERMES_WORKER_ROOT", "/workspace"))
KEY_PATH = Path(os.environ.get("HERMES_EXECUTION_POOL_KEY_FILE", "/pool-access/execution-pool-key"))
MAX_LOG_BYTES = 4 * 1024 * 1024
PART = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}\Z")
RESULT_FIELDS = frozenset({"status", "summary", "changed_files", "tests_run", "artifacts", "findings", "blockers"})
def _completed_result(assignment: dict[str, Any]) -> dict[str, Any]:
"""Return exactly one completed result JSON object from an ordinal's safe log."""
parts = tuple(str(assignment[name]) for name in ("board", "task_id", "run_id"))
if any(not PART.fullmatch(part) for part in parts):
raise ProtocolError("bootstrap assignment path binding is invalid")
if ROOT.is_symlink():
raise ProtocolError("bootstrap workspace root is unsafe")
root = ROOT.resolve()
state_root = ROOT / "session-state"
board_root = state_root / parts[0]
task_root = board_root / parts[1]
log_path = task_root / f"{parts[2]}.log"
if any(path.is_symlink() for path in (state_root, board_root, task_root, log_path)):
raise ProtocolError("bootstrap result log is unsafe")
try:
log_path.resolve(strict=False).relative_to(root)
except ValueError as error:
raise ProtocolError("bootstrap result log escapes workspace root") from error
descriptor = os.open(log_path, os.O_RDONLY | os.O_NONBLOCK | getattr(os, "O_NOFOLLOW", 0))
try:
info = os.fstat(descriptor)
if not stat.S_ISREG(info.st_mode) or info.st_size > MAX_LOG_BYTES:
raise ProtocolError("bootstrap result log is invalid")
raw = os.read(descriptor, MAX_LOG_BYTES + 1)
finally:
os.close(descriptor)
candidates: dict[bytes, dict[str, Any]] = {}
for line in raw.splitlines():
try:
value = json.loads(line)
except (UnicodeDecodeError, json.JSONDecodeError):
continue
if not isinstance(value, dict) or set(value) != RESULT_FIELDS or value.get("status") != "completed":
continue
if not isinstance(value.get("summary"), str) or not value["summary"].strip():
continue
if any(not isinstance(value.get(name), list) or any(not isinstance(item, str) for item in value[name]) for name in RESULT_FIELDS - {"status", "summary"}):
continue
candidates[canonical_json(value)] = value
if len(candidates) != 1:
raise ProtocolError("bootstrap result evidence is absent or ambiguous")
return next(iter(candidates.values()))
def bootstrap(key: bytes, raw_assignment: Any) -> dict[str, Any]:
"""Verify assignment HMAC, derive evidence locally, and emit only its receipt."""
boundary = Boundary(key)
assignment = boundary.verify(raw_assignment)
structured = _completed_result(assignment)
title = structured["summary"].strip()[:512]
body = json.dumps(structured, indent=2, sort_keys=True)
return boundary.resume_artifact(assignment, {"title": title, "body": body}, structured)
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--assignment-stdin", action="store_true", required=True)
args = parser.parse_args()
raw = sys.stdin.buffer.read(64 * 1024 + 1)
if not args.assignment_stdin or len(raw) > 64 * 1024:
raise SystemExit("signed assignment input is invalid")
try:
receipt = bootstrap(read_key(KEY_PATH), json.loads(raw))
except (OSError, ProtocolError, ValueError, json.JSONDecodeError) as error:
raise SystemExit(f"bootstrap rejected: {error}") from error
sys.stdout.buffer.write(canonical_json(receipt) + b"\n")
return 0
if __name__ == "__main__": # pragma: no cover - operator entrypoint
raise SystemExit(main())

212
services/hermes/scripts/supervisor_state.py
View File

@ -1,30 +1,24 @@
#!/usr/bin/env python3
"""Board-local integrity records for Hermes PR continuation cards."""
from __future__ import annotations
import hashlib
import fcntl
import json
import os
import sqlite3
import tempfile
import time
from contextlib import contextmanager
from pathlib import Path
from typing import Any, Iterator
from supervisor_lineage import Lineage
from publication_retry import PublicationRetryError, validate as validate_retry_receipt
KANBAN_ROOT = Path(
os.environ.get("HERMES_KANBAN_HOME", os.environ.get("HERMES_HOME", "/opt/data"))
) / "kanban/boards"
STATE_FILE = "supervisor-state.db"
class SupervisorStateError(ValueError):
"""A present integrity record cannot be decoded or read safely."""
def _path(board: str, path: Path | None) -> Path:
"""Keep integrity rows board-local without sharing Kanban's SQLite file."""
if path is not None:
@ -32,8 +26,6 @@ def _path(board: str, path: Path | None) -> Path:
if not board or "/" in board or "\\" in board:
raise ValueError("board name is invalid")
return KANBAN_ROOT / board / STATE_FILE
def _schema(connection: sqlite3.Connection) -> None:
"""Install the small coordinator schema in its isolated state database."""
connection.executescript(
@ -51,10 +43,21 @@ def _schema(connection: sqlite3.Connection) -> None:
PRIMARY KEY (board, child_task_id),
UNIQUE (board, root_task_id, head_commit, objective_digest)
);
CREATE TABLE IF NOT EXISTS publication_retries (
board TEXT NOT NULL, child_task_id TEXT NOT NULL,
source_run_id TEXT NOT NULL, source_ordinal INTEGER NOT NULL,
receipt_json TEXT NOT NULL, issued_run_id TEXT NOT NULL DEFAULT '',
resolved_run_id TEXT NOT NULL DEFAULT '',
reissue_count INTEGER NOT NULL DEFAULT 0, retry_after INTEGER NOT NULL DEFAULT 0,
last_reissued_run_id TEXT NOT NULL DEFAULT '',
PRIMARY KEY (board, child_task_id)
);
CREATE TABLE IF NOT EXISTS publication_retry_provenance (
board TEXT NOT NULL, child_task_id TEXT NOT NULL, raw_result_sha256 TEXT NOT NULL,
reconstruction TEXT NOT NULL, PRIMARY KEY (board, child_task_id)
);
"""
)
def _legacy_rows(board: str, database: Path) -> tuple[list[tuple[Any, ...]], list[tuple[Any, ...]]]:
"""Read a complete, healthy legacy state table set or refuse migration."""
if not database.exists():
@ -99,8 +102,6 @@ def _legacy_rows(board: str, database: Path) -> tuple[list[tuple[Any, ...]], lis
or any(str(row[2]) not in root_ids for row in children)):
raise SupervisorStateError("legacy supervisor state is malformed")
return roots, children
def _bootstrap_sidecar(board: str, state: Path) -> None:
"""Atomically import valid legacy rows once, never creating an empty fallback."""
lock = state.with_name(f".{STATE_FILE}.lock")
@ -138,8 +139,6 @@ def _bootstrap_sidecar(board: str, state: Path) -> None:
except Exception:
replacement.unlink(missing_ok=True)
raise
@contextmanager
def _connect(board: str, path: Path | None = None) -> Iterator[sqlite3.Connection]:
"""Open one sidecar transaction and always close its SQLite handle."""
@ -163,6 +162,15 @@ def _connect(board: str, path: Path | None = None) -> Iterator[sqlite3.Connectio
child_columns = {row[1] for row in connection.execute("PRAGMA table_info(supervisor_children)")}
if "cycle" not in child_columns:
connection.execute("ALTER TABLE supervisor_children ADD COLUMN cycle INTEGER NOT NULL DEFAULT 1")
retry_columns = {row[1] for row in connection.execute("PRAGMA table_info(publication_retries)")}
if "resolved_run_id" not in retry_columns:
connection.execute("ALTER TABLE publication_retries ADD COLUMN resolved_run_id TEXT NOT NULL DEFAULT ''")
if "reissue_count" not in retry_columns:
connection.execute("ALTER TABLE publication_retries ADD COLUMN reissue_count INTEGER NOT NULL DEFAULT 0")
if "retry_after" not in retry_columns:
connection.execute("ALTER TABLE publication_retries ADD COLUMN retry_after INTEGER NOT NULL DEFAULT 0")
if "last_reissued_run_id" not in retry_columns:
connection.execute("ALTER TABLE publication_retries ADD COLUMN last_reissued_run_id TEXT NOT NULL DEFAULT ''")
if path.name == STATE_FILE:
os.chmod(path, 0o600)
yield connection
@ -172,8 +180,6 @@ def _connect(board: str, path: Path | None = None) -> Iterator[sqlite3.Connectio
raise
finally:
connection.close()
def _lineage(row: tuple[Any, ...] | None) -> Lineage | None:
if row is None:
return None
@ -184,8 +190,6 @@ def _lineage(row: tuple[Any, ...] | None) -> Lineage | None:
if not all(values):
return None
return Lineage(*values)
def get_root(board: str, root_task_id: str, *, path: Path | None = None) -> Lineage | None:
"""Return one coordinator-issued root lineage, never task-body claims."""
try:
@ -202,8 +206,6 @@ def get_root(board: str, root_task_id: str, *, path: Path | None = None) -> Line
if lineage is None:
raise SupervisorStateError("supervisor root state is malformed")
return lineage
def get_live_head(board: str, root_task_id: str, *, path: Path | None = None) -> str:
"""Return the broker-confirmed head associated with a trusted root."""
try:
@ -219,8 +221,6 @@ def get_live_head(board: str, root_task_id: str, *, path: Path | None = None) ->
if not isinstance(row[0], str) or not row[0]:
raise SupervisorStateError("supervisor root state is malformed")
return row[0]
def get_child(board: str, child_task_id: str, *, path: Path | None = None) -> dict[str, Any] | None:
"""Return the only durable authority record for a continuation child."""
try:
@ -246,13 +246,10 @@ def get_child(board: str, child_task_id: str, *, path: Path | None = None) -> di
return {"lineage": lineage, "root_task_id": root, "parent_task_id": parent,
"kind": kind, "head_commit": head, "objective_digest": digest,
"cycle": parsed_cycle}
def record_submission(
board: str, task_id: str, lineage: Lineage, live_pr_head: str, *, path: Path | None = None
) -> None:
"""Record a coordinator-verified submission against its immutable root.
``task_id`` identifies the completing root or child. The coordinator has
already authenticated that task's signed assignment before calling this
function; the record deliberately remains keyed by ``root_task_id``.
@ -279,8 +276,6 @@ def record_submission(
"WHERE board=? AND root_task_id=?",
(live_pr_head, board, lineage.root_task_id),
)
def record_live_head(
board: str, root_task_id: str, live_pr_head: str, *, path: Path | None = None
) -> None:
@ -294,8 +289,6 @@ def record_live_head(
).rowcount
if changed != 1:
raise ValueError("unknown continuation root")
def set_ready(
board: str, root_task_id: str, head_commit: str, *, path: Path | None = None
) -> None:
@ -308,8 +301,6 @@ def set_ready(
).rowcount
if changed != 1:
raise ValueError("ready head is not the trusted live root head")
def clear_ready(board: str, root_task_id: str, *, path: Path | None = None) -> None:
"""Invalidate approval whenever a continuation is queued or head changes."""
with _connect(board, path) as connection:
@ -317,8 +308,6 @@ def clear_ready(board: str, root_task_id: str, *, path: Path | None = None) -> N
"UPDATE supervisor_roots SET ready_for_human_merge=0, ready_commit='' "
"WHERE board=? AND root_task_id=?", (board, root_task_id)
)
def record_child(
board: str, child_task_id: str, root_task_id: str, parent_task_id: str, kind: str,
head_commit: str, objective: str, cycle: int = 1, *, path: Path | None = None
@ -348,13 +337,9 @@ def record_child(
)
elif tuple(existing) != values:
raise ValueError("continuation child lineage conflicts with its existing record")
def objective_digest(objective: str) -> str:
"""Make duplicate user follow-ups idempotent without retaining extra prose."""
return hashlib.sha256(objective.strip().encode()).hexdigest()
def existing_child(
board: str, root_task_id: str, head_commit: str, objective: str, *, path: Path | None = None
) -> str:
@ -366,3 +351,150 @@ def existing_child(
(board, root_task_id, head_commit, objective_digest(objective)),
).fetchone()
return str(row[0]) if row else ""
def _retry_receipt(
board: str, child_task_id: str, binding: dict[str, Any], receipt: Any, *, path: Path | None
) -> tuple[dict[str, Any], int, str]:
"""Validate the mediator-signed retry receipt against private continuation state."""
child = get_child(board, child_task_id, path=path)
if child is None or child["kind"] != "repair" or not isinstance(binding, dict):
raise ValueError("publication retry lineage is invalid")
lineage = child["lineage"]
return validate_retry_receipt(
receipt, binding, board=board, child_task_id=child_task_id,
root_task_id=lineage.root_task_id, project=lineage.project, branch=lineage.branch,
base_branch=lineage.base_branch, live_head=get_live_head(board, lineage.root_task_id, path=path),
)
def record_publication_retry(
board: str, child_task_id: str, binding: dict[str, Any], receipt: Any, *, path: Path | None = None
) -> None:
"""Persist one mediator-signed failed-publication receipt for a fresh run."""
verified, ordinal, source_run_id = _retry_receipt(board, child_task_id, binding, receipt, path=path)
encoded = json.dumps(verified, separators=(",", ":"), sort_keys=True)
if len(encoded.encode()) > 48 * 1024:
raise ValueError("publication retry receipt exceeds its assignment limit")
with _connect(board, path) as connection:
existing = connection.execute(
"SELECT source_run_id,source_ordinal,receipt_json FROM publication_retries "
"WHERE board=? AND child_task_id=?", (board, child_task_id)
).fetchone()
values = (source_run_id, ordinal, encoded)
if existing is None:
connection.execute(
"INSERT INTO publication_retries(board,child_task_id,source_run_id,source_ordinal,receipt_json) "
"VALUES(?,?,?,?,?)", (board, child_task_id, *values)
)
elif tuple(existing) != values:
raise ValueError("publication retry receipt conflicts with existing evidence")
def publication_retry(
board: str, child_task_id: str, run_id: str, *, path: Path | None = None
) -> dict[str, Any] | None:
"""Return an unconsumed receipt, or its one exact fresh run after a crash."""
if not isinstance(run_id, str) or (run_id and (not run_id.isdecimal() or int(run_id) < 1)):
raise ValueError("publication retry run is invalid")
if path is None and not _path(board, None).exists():
return None
with _connect(board, path) as connection:
row = connection.execute(
"SELECT source_run_id,source_ordinal,receipt_json,issued_run_id,resolved_run_id,reissue_count,retry_after FROM publication_retries "
"WHERE board=? AND child_task_id=?", (board, child_task_id)
).fetchone()
if row is None or row[4]:
return None
if int(row[6]) > int(time.time()):
raise PublicationRetryError("publication retry backoff is active")
if row[3] and row[3] != run_id:
raise PublicationRetryError("publication retry was already issued")
try:
receipt = json.loads(row[2])
except (TypeError, json.JSONDecodeError) as error:
raise SupervisorStateError("publication retry receipt is malformed") from error
verified, ordinal, source_run_id = _retry_receipt(
board, child_task_id,
{"board": board, "task_id": child_task_id, "run_id": row[0], "worker_ordinal": row[1],
"attempt": receipt.get("source", {}).get("attempt") if isinstance(receipt, dict) else None},
receipt, path=path,
)
if source_run_id != row[0] or ordinal != row[1] or (run_id and run_id == source_run_id):
raise SupervisorStateError("publication retry receipt is inconsistent")
return verified
def issue_publication_retry(board: str, child_task_id: str, run_id: str, *, path: Path | None = None) -> None:
"""Fence a receipt to one fresh assignment after its durable pool row exists."""
with _connect(board, path) as connection:
row = connection.execute(
"SELECT issued_run_id,resolved_run_id,last_reissued_run_id FROM publication_retries "
"WHERE board=? AND child_task_id=?", (board, child_task_id)
).fetchone()
if row is not None and (
(row[0] == run_id and row[1] in ("", run_id))
or (not row[0] and not row[1] and row[2] == run_id)
):
return
if publication_retry(board, child_task_id, run_id, path=path) is None:
raise ValueError("publication retry is unavailable")
with _connect(board, path) as connection:
changed = connection.execute(
"UPDATE publication_retries SET issued_run_id=? WHERE board=? AND child_task_id=? "
"AND issued_run_id=''", (run_id, board, child_task_id)
).rowcount
if changed != 1:
with _connect(board, path) as connection:
row = connection.execute(
"SELECT issued_run_id FROM publication_retries WHERE board=? AND child_task_id=?",
(board, child_task_id),
).fetchone()
if row is None or row[0] != run_id:
raise SupervisorStateError("publication retry was already issued")
def resolve_publication_retry(board: str, child_task_id: str, run_id: str, *, path: Path | None = None) -> None:
"""Mark the one fresh broker-confirmed run as the receipt's trusted resolution."""
with _connect(board, path) as connection:
changed = connection.execute(
"UPDATE publication_retries SET resolved_run_id=? WHERE board=? AND child_task_id=? "
"AND issued_run_id=? AND resolved_run_id=''", (run_id, board, child_task_id, run_id)
).rowcount
if changed != 1:
with _connect(board, path) as connection:
row = connection.execute(
"SELECT issued_run_id,resolved_run_id FROM publication_retries WHERE board=? AND child_task_id=?",
(board, child_task_id),
).fetchone()
if row is None or tuple(row) != (run_id, run_id):
raise SupervisorStateError("publication retry cannot be resolved")
def reissue_publication_retry(board: str, child_task_id: str, run_id: str, *, path: Path | None = None) -> bool:
"""Release one interrupted transient retry once, with a five-minute backoff."""
with _connect(board, path) as connection:
changed = connection.execute(
"UPDATE publication_retries SET issued_run_id='',reissue_count=reissue_count+1,retry_after=?,last_reissued_run_id=? "
"WHERE board=? AND child_task_id=? AND issued_run_id=? AND resolved_run_id='' AND reissue_count<1",
(int(time.time()) + 300, run_id, board, child_task_id, run_id),
).rowcount
if changed:
return True
row = connection.execute(
"SELECT issued_run_id,resolved_run_id,reissue_count,last_reissued_run_id FROM publication_retries WHERE board=? AND child_task_id=?",
(board, child_task_id),
).fetchone()
if row is not None and not row[0] and not row[1] and row[3] == run_id and int(row[2]) == 1:
return True
if row is not None and row[0] == run_id and not row[1] and int(row[2]) >= 1:
return False
raise SupervisorStateError("publication retry cannot be reissued")
def record_publication_retry_provenance(board: str, child_task_id: str, raw_result_sha256: str, *, path: Path | None = None) -> None:
"""Keep a bootstrap's immutable blocked result hash beside its retry receipt."""
if not isinstance(raw_result_sha256, str) or len(raw_result_sha256) != 64:
raise ValueError("publication retry provenance is invalid")
with _connect(board, path) as connection:
if connection.execute(
"SELECT 1 FROM publication_retries WHERE board=? AND child_task_id=?", (board, child_task_id)
).fetchone() is None:
raise ValueError("publication retry receipt is unavailable")
existing = connection.execute(
"SELECT raw_result_sha256 FROM publication_retry_provenance WHERE board=? AND child_task_id=?",
(board, child_task_id),
).fetchone()
if existing is None:
connection.execute(
"INSERT INTO publication_retry_provenance VALUES(?,?,?,?)",
(board, child_task_id, raw_result_sha256, "operator-reconstructed-from-private-result-log"),
)
elif existing[0] != raw_result_sha256:
raise ValueError("publication retry provenance conflicts with existing evidence")

13
testing/tests/test_hermes_agent_layout.py
View File

@ -453,15 +453,6 @@ def test_agent_image_runs_execution_safety_patch_and_regressions():
assert "/opt/hermes/.venv/bin/python /tmp/hermes-execution-safety-regression.py" in (
dockerfile
)
assert "COPY services/hermes/scripts/cli_lane_*.py" in dockerfile
assert "!services/hermes/scripts/cli_lane_*.py" in dockerignore
for name in (
"routing_catalog.py",
"supervisor_lineage.py",
"supervisor_state.py",
):
assert f"COPY services/hermes/scripts/{name} /tmp/hermes-lane-regression/" in (
dockerfile
)
assert f"!services/hermes/scripts/{name}" in dockerignore
assert "COPY services/hermes/scripts/*.py /tmp/hermes-lane-regression/" in dockerfile
assert "!services/hermes/scripts/*.py" in dockerignore
assert "HERMES_CLI_LANE_SOURCE=/tmp/hermes-lane-regression" in dockerfile

5
testing/tests/test_hermes_execution_pool.py
View File

@ -475,6 +475,11 @@ def test_retention_gc_removes_only_clean_terminal_workspace(tmp_path, monkeypatc
(workspace / "tracked").write_text("safe\n")
subprocess.run(["git", "-C", str(workspace), "add", "tracked"], check=True)
subprocess.run(["git", "-C", str(workspace), "commit", "-qm", "initial"], check=True)
branch = subprocess.run(
["git", "-C", str(workspace), "symbolic-ref", "--short", "HEAD"],
text=True, capture_output=True, check=True,
).stdout.strip()
subprocess.run(["git", "-C", str(workspace), "update-ref", f"refs/remotes/origin/{branch}", "HEAD"], check=True)
state_file = tmp_path / "session-state/atlas/t_deadbeef/run-1234.json"
state_file.parent.mkdir(parents=True)
state_file.write_text(

45
testing/tests/test_hermes_execution_pool_coordinator_v2.py
View File

@ -192,6 +192,31 @@ def test_assignment_payload_requires_state_and_native_parents_for_continuations(
coordinator.assignment_payload(kanban, object(), rejected, "metis")
def test_continuation_resume_uses_only_private_receipt_and_never_model_context(monkeypatch):
lineage = coordinator.supervisor_lineage.Lineage(
"root", "wt/root", "https://scm.bstein.dev/titan/metis/pulls/7", "metis", "main"
)
child = {"lineage": lineage, "parent_task_id": "root", "kind": "repair"}
receipt = {"source": {"worker_ordinal": 0}, "head": "b" * 40}
monkeypatch.setattr(coordinator.supervisor_state, "get_child", lambda *_args: child)
monkeypatch.setattr(coordinator.supervisor_state, "get_root", lambda *_args: lineage)
monkeypatch.setattr(coordinator.supervisor_state, "publication_retry", lambda *_args: receipt)
kanban = SimpleNamespace(
build_worker_context=lambda *_args: "model objective must not be used",
get_task=lambda *_args: object(), parent_ids=lambda *_args: ["root"],
)
payload = coordinator.assignment_payload(kanban, object(), task(id="repair"), "metis")
assert payload["scm_resume"] is receipt
assert payload["context"] == "Republish the mediator-verified completed SCM handoff."
monkeypatch.setattr(
coordinator.supervisor_state, "publication_retry",
lambda *_args: (_ for _ in ()).throw(ValueError("already issued")),
)
with pytest.raises(protocol.ProtocolError, match="retry state"):
coordinator.assignment_payload(kanban, object(), task(id="repair"), "metis")
@pytest.mark.parametrize(
("parent_ids", "message"),
[
@ -293,6 +318,26 @@ def test_finalize_canonicalizes_integer_run_and_reconcile_does_not_reexecute(
assert store.active_assignments() == []
def test_finalize_fences_resume_receipt_before_any_live_head_transition(tmp_path, monkeypatch):
"""A result winning the add→maintenance race still issues its exact receipt first."""
live = task()
kanban = install_kanban(monkeypatch, [live], tmp_path)
payload = assignment_payload(scm_resume={"source": {"worker_ordinal": 0}})
store = protocol.PoolStore(tmp_path / "resume-finalize.db")
store.add(binding(), payload)
issued = []
monkeypatch.setattr(
coordinator.supervisor_state, "issue_publication_retry",
lambda *values: issued.append(values),
)
coordinator.Coordinator(MASTER, store).finalize({
**binding(), "payload": payload,
"result": {"structured": dict(STRUCTURED), "returncode": 0, "node": "node"},
})
assert issued == [("metis", "t_deadbeef", "23")]
assert kanban.completed[-1][1]["expected_run_id"] == 23
def test_finalize_records_a_verified_root_before_marking_the_task_complete(tmp_path, monkeypatch):
live = task()
kanban = install_kanban(monkeypatch, [live], tmp_path)

29
testing/tests/test_hermes_execution_pool_dispatch_v2.py
View File

@ -97,6 +97,35 @@ def test_dispatch_claims_only_pathless_task_with_safe_assignment_branch(
assert record["payload"]["branch"] == "wt/t_deadbeef"
def test_dispatch_reserves_a_pending_publication_retry_to_its_source_ordinal(
tmp_path, monkeypatch
):
"""A fresh retry cannot strand the retained commit on a different mediator."""
item = task(status="ready")
install_kanban(monkeypatch, [item], tmp_path)
store = protocol.PoolStore(tmp_path / "pool.db")
pool = coordinator.Coordinator(MASTER, store)
receipt = {"source": {"worker_ordinal": 1}}
monkeypatch.setattr(
maintenance.supervisor_state, "publication_retry",
lambda _board, _task_id, run_id: receipt if run_id == "" else receipt,
)
monkeypatch.setattr(maintenance.supervisor_state, "issue_publication_retry", lambda *_args: None)
monkeypatch.setattr(
coordinator, "assignment_payload", lambda *_args: {**assignment_payload(), "scm_resume": receipt},
)
observed = []
def claim(_active, _limit, eligible, **kwargs):
observed.append((eligible("metis", item), kwargs["priority"]("metis", item)))
return [("metis", item.id)]
monkeypatch.setattr(maintenance.cli_lane_dispatch, "claim_ready", claim)
pool.dispatch()
assert observed == [(True, 0)]
assert store.active_assignments()[0]["worker_ordinal"] == 1
def test_dispatch_incompatible_unversioned_claim_api_fails_closed(
tmp_path, monkeypatch
):

19
testing/tests/test_hermes_execution_pool_recovery.py
View File

@ -156,6 +156,25 @@ def test_noncanonical_run_lease_failure_is_released_without_a_board_call(
assert kanban.blocked == []
def test_lease_expiry_releases_only_a_trusted_publication_retry(tmp_path, monkeypatch):
"""An OOM-style lease expiry returns one receipt before parking the native run."""
kanban = install_kanban(monkeypatch, [task()], tmp_path)
store = pool_store.PoolStore(tmp_path / "resume-lease.db")
exact = binding(attempt=3)
store.add(exact, assignment_payload(scm_resume={"source": {"worker_ordinal": 0}}))
store.offer(0)
with store._connect() as connection:
connection.execute("UPDATE assignments SET lease_until=1")
released = []
monkeypatch.setattr(
coordinator.supervisor_state, "reissue_publication_retry",
lambda *values: released.append(values) or True,
)
coordinator.Coordinator(MASTER, store).expire_leases()
assert released and released[0][2] == "23"
assert kanban.blocked[-1][1]["kind"] == "transient"
def test_a_durable_row_in_any_state_blocks_conflicting_re_adoption(tmp_path, monkeypatch):
"""known_runs() covers terminal and retrying rows, not just live ones."""
live = task()

290
testing/tests/test_hermes_execution_pool_submission.py
View File

@ -3,6 +3,7 @@
from __future__ import annotations
import sys
import subprocess
from pathlib import Path
import pytest
@ -16,6 +17,7 @@ sys.path[:0] = [str(SCRIPTS), str(SCM_SCRIPTS)]
import execution_pool_client as client # noqa: E402
import execution_pool_protocol as protocol # noqa: E402
import execution_pool_scm as scm # noqa: E402
import scm_resume_bootstrap as resume_bootstrap # noqa: E402
import receive_pack_scan # noqa: E402
import scm_broker # noqa: E402
from testing.tests.test_hermes_scm_broker_support import _receive_command # noqa: E402
@ -213,6 +215,103 @@ def test_remote_head_parsing_ignores_anything_that_is_not_a_branch(
assert scm._remote_heads(tmp_path, ("feature/x",)) == {"feature/x": "bbb"}
def test_git_timeout_becomes_a_sanitized_retryable_error(monkeypatch):
"""A subprocess timeout must not skip terminal result submission or expose a grant."""
def timed_out(*_args, **_kwargs):
raise subprocess.TimeoutExpired(
["git", "-c", "http.extraHeader=X-Hermes-Task-Grant: secret", "push"], 900
)
monkeypatch.setattr(scm.subprocess, "run", timed_out)
with pytest.raises(RuntimeError, match="^SCM command timed out$"):
scm._run("push", "hermes-broker")
@pytest.mark.parametrize(
("detail", "expected"),
[
(
"remote: task branch head changed; fetch and merge before retrying",
"task branch changed; fetch and merge before retrying",
),
(
"RPC failed; HTTP 403",
"SCM broker authorization rejected the branch update; "
"inspect task ownership before retrying",
),
(
"fatal: unable to access: Connection timed out",
"SCM broker transport failed; retry the preserved local commit",
),
(
"RPC failed; HTTP 400",
"SCM broker rejected the branch update; "
"inspect broker or upstream state before retrying",
),
],
)
def test_push_failure_guidance_is_sanitized_and_specific(detail, expected):
assert scm._push_failure(RuntimeError(detail)) == expected
def test_draft_discovery_finds_a_same_branch_pr_after_the_first_page(monkeypatch):
first_page = [{"head": {"ref": f"other-{index}"}, "base": {"ref": "main"}} for index in range(50)]
found = "https://scm.bstein.dev/titan/metis/pulls/51"
calls = []
def read(path):
calls.append(path)
return __import__("json").dumps(
first_page if "page=1" in path else [{
"number": 51, "html_url": found,
"head": {"ref": "wt/t_deadbeef"}, "base": {"ref": "main"},
}]
).encode()
monkeypatch.setattr(scm.scm_broker_client, "read", read)
monkeypatch.setattr(
scm.scm_broker_client, "update_draft",
lambda grant, number, title, body: __import__("json").dumps({"html_url": found}).encode(),
)
assert scm.Boundary._draft("metis", "wt/t_deadbeef", "main", "b" * 40, "safe", "body", "grant") == found
assert len(calls) == 2
def test_repair_draft_refuses_to_create_a_replacement_when_its_pr_is_not_open(monkeypatch):
monkeypatch.setattr(scm.scm_broker_client, "read", lambda _path: b"[]")
monkeypatch.setattr(
scm.scm_broker_client, "create_draft",
lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("replacement PR")),
)
with pytest.raises(protocol.ProtocolError, match="no existing pull request"):
scm.Boundary._draft(
"metis", "wt/t_deadbeef", "main", "b" * 40, "repair", "evidence",
"grant", existing_only=True,
)
def test_repair_checks_its_open_pr_before_advancing_the_owned_branch(tmp_path, monkeypatch):
boundary, exact, _calls, _head = submit_harness(
tmp_path, monkeypatch, {"wt/t_deadbeef": "a" * 40}, continuation_kind="repair"
)
events = []
original_run = scm._run
def run(*arguments, **kwargs):
if "push" in arguments:
events.append("push")
return original_run(*arguments, **kwargs)
def draft(*_args, **kwargs):
events.append("pre" if kwargs.get("refresh") is False else "post")
return "https://scm/pulls/3"
monkeypatch.setattr(scm, "_run", run)
monkeypatch.setattr(boundary, "_draft", draft)
boundary.submit(exact, {"title": "repair", "body": "evidence"})
assert events == ["pre", "push", "post"]
class FailingSCM:
def __init__(self, error):
self.error = error
@ -222,6 +321,9 @@ class FailingSCM:
self.calls += 1
raise self.error
def resume_artifact(self, _assignment, _request, _structured):
return {"bounded": "resume-artifact"}
def test_a_refused_submission_downgrades_the_result_instead_of_discarding_it():
failing = FailingSCM(protocol.ProtocolError("every reviewed branch name is taken"))
@ -248,6 +350,194 @@ def test_a_refused_submission_downgrades_the_result_instead_of_discarding_it():
assert finished["ack"]["accepted"] is True
def test_a_refused_submission_is_transient_for_pool_recovery():
failing = FailingSCM(protocol.ProtocolError("SCM broker rejected the branch update"))
boundary = client.ClientBoundary(KEY, failing)
boundary.current = assignment(continuation_kind="repair", root_task_id="t_deadbeef")
posted = []
def post(_path, envelope):
posted.append(envelope)
return protocol.sign_envelope(
KEY, "ack", binding(), {"accepted": True, "duplicate": False}
)
boundary._post = post
finished = boundary.finish(
{
"binding": binding(),
"payload": {"structured": dict(RESULT), "returncode": 0, "capacity_failure": False},
"title": "safe", "body": "evidence",
}
)
assert finished["ack"]["accepted"] is True
assert finished["structured"]["status"] == "blocked"
terminal = protocol.verify_envelope(KEY, posted[0])["payload"]
assert terminal["capacity_failure"] is True
assert terminal["scm_resume"] == {"bounded": "resume-artifact"}
def test_ordinary_submission_failure_never_emits_a_continuation_receipt():
boundary = client.ClientBoundary(KEY, FailingSCM(protocol.ProtocolError("refused")))
boundary.current = assignment()
posted = []
boundary._post = lambda _path, envelope: posted.append(envelope) or protocol.sign_envelope(
KEY, "ack", binding(), {"accepted": True, "duplicate": False}
)
boundary.finish({"binding": binding(), "payload": {"structured": dict(RESULT), "returncode": 0}, "title": "safe", "body": "evidence"})
terminal = protocol.verify_envelope(KEY, posted[0])["payload"]
assert terminal["capacity_failure"] is True and "scm_resume" not in terminal
def test_real_resume_receipt_keeps_completed_evidence_after_submission_mutates_result(tmp_path, monkeypatch):
"""The signed receipt must not alias the blocked terminal result in memory."""
monkeypatch.setattr(scm, "WORKSPACE_ROOT", tmp_path / "workspace")
monkeypatch.setattr(scm, "SCM_ROOT", tmp_path / "state")
monkeypatch.setattr(scm, "ORDINAL", 0)
exact = protocol.sign_envelope(
KEY, "assignment", binding(), payload(root_task_id="t_deadbeef", continuation_kind="repair")
)
destination = scm.workspace_path(exact)
destination.mkdir(parents=True)
protocol.atomic_json(scm._state_path(exact), {"baseline_sha": "a" * 40})
monkeypatch.setattr(scm, "_workspace_identity", lambda *_args: "b" * 40)
monkeypatch.setattr(scm, "_run", lambda *args, **_kwargs: "")
broker = scm.Boundary(KEY)
monkeypatch.setattr(
broker, "submit", lambda *_args, **_kwargs: (_ for _ in ()).throw(protocol.ProtocolError("refused"))
)
boundary = client.ClientBoundary(KEY, broker)
boundary.current = exact
posted = []
boundary._post = lambda _path, envelope: posted.append(envelope) or protocol.sign_envelope(
KEY, "ack", binding(), {"accepted": True, "duplicate": False}
)
boundary.finish({"binding": binding(), "payload": {"structured": dict(RESULT), "returncode": 0}, "title": "safe", "body": "evidence"})
result_envelope = next(item for item in posted if item["kind"] == "result")
artifact = protocol.verify_envelope(KEY, result_envelope)["payload"]["scm_resume"]
assert artifact["structured"]["status"] == "completed"
assert artifact["result_digest"] == __import__("hashlib").sha256(
protocol.canonical_json({name: artifact[name] for name in ("structured", "title", "body")})
).hexdigest()
def test_mediator_bootstrap_recovers_one_unique_completed_log_result(tmp_path, monkeypatch):
exact = protocol.sign_envelope(
KEY, "assignment", binding(), payload(root_task_id="t_deadbeef", continuation_kind="repair")
)
log = tmp_path / "session-state" / "metis" / "t_deadbeef" / "42" / ".log"
# The live worker uses `<run>.log`; make the same task-bound parent safely.
log = log.parent.parent / "42.log"
log.parent.mkdir(parents=True)
completed = dict(RESULT)
log.write_bytes(protocol.canonical_json(completed) + b"\nnoise\n" + protocol.canonical_json(completed) + b"\n")
monkeypatch.setattr(resume_bootstrap, "ROOT", tmp_path)
assert resume_bootstrap._completed_result(exact) == completed
seen = {}
monkeypatch.setattr(
scm.Boundary, "resume_artifact",
lambda _self, assignment, request, structured: seen.update(
assignment=assignment, request=request, structured=structured
) or {"receipt": "only"},
)
assert resume_bootstrap.bootstrap(KEY, exact) == {"receipt": "only"}
assert seen["structured"] == completed and seen["request"]["title"] == completed["summary"]
def test_mediator_bootstrap_rejects_a_symlinked_log_ancestor(tmp_path, monkeypatch):
exact = protocol.sign_envelope(KEY, "assignment", binding(), payload())
outside = tmp_path / "outside"
outside.mkdir()
session_root = tmp_path / "session-state"
session_root.mkdir()
(session_root / "metis").symlink_to(outside, target_is_directory=True)
monkeypatch.setattr(resume_bootstrap, "ROOT", tmp_path)
with pytest.raises(protocol.ProtocolError, match="unsafe"):
resume_bootstrap._completed_result(exact)
def test_publication_lease_signs_the_exact_binding_and_stops_cleanly():
boundary = client.ClientBoundary(KEY, FailingSCM(protocol.ProtocolError("unused")))
calls = []
def post(path, envelope):
calls.append((path, envelope))
return protocol.sign_envelope(KEY, "ack", binding(), {"accepted": True})
boundary._post = post
checkpoint, close = boundary._publication_lease(binding())
checkpoint()
close()
assert len(calls) == 1 and calls[0][0] == "/v1/heartbeat"
verified = protocol.verify_envelope(KEY, calls[0][1], expected_kind="heartbeat")
assert {name: verified[name] for name in binding()} == binding()
def test_publication_lease_renews_during_a_slow_scm_step_without_sleeping(monkeypatch):
boundary = client.ClientBoundary(KEY, FailingSCM(protocol.ProtocolError("unused")))
calls = []
class Event:
def __init__(self):
self.waits = 0
def wait(self, _seconds):
self.waits += 1
return self.waits > 1
def set(self):
return None
class Thread:
def __init__(self, *, target, daemon):
self.target = target
def start(self):
self.target()
def join(self, timeout):
assert timeout == 1
monkeypatch.setattr(client.threading, "Event", Event)
monkeypatch.setattr(client.threading, "Thread", Thread)
boundary._post = lambda path, envelope: calls.append((path, envelope)) or protocol.sign_envelope(
KEY, "ack", binding(), {"accepted": True}
)
_checkpoint, close = boundary._publication_lease(binding())
close()
assert [path for path, _envelope in calls] == ["/v1/heartbeat", "/v1/heartbeat"]
def test_lost_publication_checkpoint_prevents_the_push(tmp_path, monkeypatch):
boundary, exact, calls, _head = submit_harness(tmp_path, monkeypatch, {"wt/t_deadbeef": "a" * 40})
count = 0
def checkpoint():
nonlocal count
count += 1
if count == 3:
raise protocol.ProtocolError("SCM publication lease was lost")
with pytest.raises(protocol.ProtocolError, match="lease was lost"):
boundary.submit(exact, {"title": "safe", "body": "evidence"}, checkpoint=checkpoint)
assert not any("push" in call for call in calls)
def test_worker_cannot_spoof_a_transient_publication_retry_result():
boundary = client.ClientBoundary(KEY, FailingSCM(protocol.ProtocolError("unused")))
boundary.current = assignment(payload={"scm_resume": {}})
posted = []
boundary._post = lambda _path, envelope: posted.append(envelope) or protocol.sign_envelope(
KEY, "ack", binding(), {"accepted": True, "duplicate": False}
)
blocked = {**RESULT, "status": "blocked", "blockers": ["policy refusal"]}
boundary.finish({"binding": binding(), "payload": {
"structured": blocked, "returncode": 1, "publication_retry_transient": True,
}})
terminal = protocol.verify_envelope(KEY, posted[0])["payload"]
assert "publication_retry_transient" not in terminal
def test_a_successful_submission_records_both_the_draft_and_the_exact_branch():
class Recording:
def submit(self, _assignment, _request):

125
testing/tests/test_hermes_execution_pool_worker_v2.py
View File

@ -248,6 +248,11 @@ def init_clean_repo(path):
(path / "tracked").write_text("safe")
subprocess.run(["git", "-C", str(path), "add", "tracked"], check=True)
subprocess.run(["git", "-C", str(path), "commit", "-qm", "initial"], check=True)
branch = subprocess.run(
["git", "-C", str(path), "symbolic-ref", "--short", "HEAD"], text=True,
capture_output=True, check=True,
).stdout.strip()
subprocess.run(["git", "-C", str(path), "update-ref", f"refs/remotes/origin/{branch}", "HEAD"], check=True)
def test_retention_skips_young_outside_dirty_and_symlink_workspaces(tmp_path, monkeypatch):
@ -278,6 +283,23 @@ def test_retention_skips_young_outside_dirty_and_symlink_workspaces(tmp_path, mo
assert link_state.exists()
def test_retention_keeps_a_clean_unpublished_commit(tmp_path, monkeypatch):
"""A terminal source checkout remains available for mediator publication retry."""
monkeypatch.setattr(worker, "ROOT", tmp_path)
monkeypatch.setattr(worker, "RETENTION_SECONDS", 3600)
workspace = tmp_path / "runs/metis/retry/23"
workspace.mkdir(parents=True)
init_clean_repo(workspace)
branch = subprocess.run(
["git", "-C", str(workspace), "symbolic-ref", "--short", "HEAD"], text=True,
capture_output=True, check=True,
).stdout.strip()
subprocess.run(["git", "-C", str(workspace), "update-ref", "-d", f"refs/remotes/origin/{branch}"], check=True)
state = write_gc_state(tmp_path, "retry", workspace, 1)
assert worker.garbage_collect(now=10_000) == 0
assert workspace.exists() and state.exists()
def test_readiness_checks_ordinal_paths_credentials_and_mediator(tmp_path, monkeypatch):
worker_root = tmp_path / "worker"
data_root = tmp_path / "data"
@ -292,10 +314,11 @@ def test_readiness_checks_ordinal_paths_credentials_and_mediator(tmp_path, monke
monkeypatch.setattr(worker.cli_lane_runner, "DATA_ROOT", data_root)
monkeypatch.setattr(worker.cli_lane_runner, "RESULT_SCHEMA_PATH", schema)
monkeypatch.setenv("CLAUDE_CODE_OAUTH_TOKEN_FILE", str(claude_token))
polled = []
monkeypatch.setattr(worker, "_poll", lambda: polled.append(True))
monkeypatch.setattr(
worker, "_poll", lambda: pytest.fail("readiness must not poll the mediator")
)
worker.readiness()
assert polled and json.loads(schema.read_text()) == worker.cli_lane_runner.RESULT_SCHEMA
assert json.loads(schema.read_text()) == worker.cli_lane_runner.RESULT_SCHEMA
monkeypatch.setattr(worker, "ORDINAL", 3)
with pytest.raises(protocol.ProtocolError, match="ordinal"):
@ -310,32 +333,78 @@ def test_readiness_checks_ordinal_paths_credentials_and_mediator(tmp_path, monke
worker.readiness()
def test_readiness_defers_transient_ordinal_lease_conflict(tmp_path, monkeypatch):
worker_root = tmp_path / "worker"
data_root = tmp_path / "data"
claude_token = tmp_path / "claude-oauth/token"
for path in (worker_root, worker_root / "provider-state", data_root):
path.mkdir(parents=True, exist_ok=True)
claude_token.parent.mkdir()
claude_token.write_text("setup-token")
monkeypatch.setattr(worker, "ORDINAL", 1)
monkeypatch.setattr(worker, "ROOT", worker_root)
monkeypatch.setattr(worker.cli_lane_runner, "DATA_ROOT", data_root)
monkeypatch.setattr(
worker.cli_lane_runner, "RESULT_SCHEMA_PATH", tmp_path / "schema/result.json"
)
monkeypatch.setenv("CLAUDE_CODE_OAUTH_TOKEN_FILE", str(claude_token))
def test_main_defers_rejected_or_malformed_poll_before_valid_assignment(monkeypatch):
monkeypatch.setattr(worker, "readiness", lambda: None)
monkeypatch.setattr(worker, "garbage_collect", lambda: 0)
expected = assignment()
polls = iter((
urllib.error.HTTPError("http://mediator", 409, "Conflict", {}, None),
protocol.ProtocolError("bad wire"),
expected,
))
executed = []
def conflict():
raise urllib.error.HTTPError("http://mediator", 409, "Conflict", {}, None)
def poll():
value = next(polls)
if isinstance(value, Exception):
raise value
return value
monkeypatch.setattr(worker, "_poll", conflict)
worker.readiness()
def execute(value):
executed.append(value)
raise StopIteration("stop")
def unauthorized():
raise urllib.error.HTTPError("http://mediator", 401, "Unauthorized", {}, None)
monkeypatch.setattr(worker, "_poll", poll)
monkeypatch.setattr(worker, "execute", execute)
monkeypatch.setattr(worker.time, "sleep", lambda _delay: None)
with pytest.raises(StopIteration, match="stop"):
worker.main()
assert executed == [expected]
monkeypatch.setattr(worker, "_poll", unauthorized)
with pytest.raises(urllib.error.HTTPError) as error:
worker.readiness()
assert error.value.code == 401
def test_publication_retry_calls_only_mediator_and_never_starts_a_model(monkeypatch):
"""A fresh run republishes retained evidence without a checkout or provider session."""
structured = {
"status": "completed", "summary": "Replace cache literals.",
"changed_files": ["internal/k8s/job_manifests.go"], "tests_run": ["go test ./..."],
"artifacts": [], "findings": [], "blockers": [],
}
resume = {"structured": structured, "title": "Repair", "body": "evidence", "head": "b" * 40}
item = assignment(payload={"scm_resume": resume})
calls = []
def client(operation, **values):
calls.append((operation, values))
if operation == "resume":
return {"scm_submission": {"branch": "wt/t_deadbeef", "pull_request": "https://scm/pulls/3", "head": "b" * 40}}
return {"ack": {"accepted": True}}
monkeypatch.setattr(worker, "_client", client)
monkeypatch.setattr(worker, "_bind_provider_sessions", lambda *_args: pytest.fail("must not bind provider state"))
monkeypatch.setattr(worker.cli_lane_runner, "run_provider", lambda *_args: pytest.fail("must not invoke a model"))
worker.execute(item)
assert [name for name, _values in calls] == ["resume", "finish"]
terminal = calls[1][1]["payload"]
assert terminal["structured"]["status"] == "completed"
assert "preserved-head:" + "b" * 40 in terminal["structured"]["artifacts"]
def test_publication_retry_policy_error_is_a_visible_nontransient_block(monkeypatch):
"""Only the mediator's explicit transient response can release a receipt."""
structured = {"status": "completed", "summary": "Repair", "changed_files": [], "tests_run": [], "artifacts": [], "findings": [], "blockers": []}
item = assignment(payload={"scm_resume": {"structured": structured, "title": "Repair", "body": "evidence", "head": "b" * 40}})
calls = []
def client(operation, **values):
calls.append((operation, values))
if operation == "resume":
raise protocol.ProtocolError("policy denied")
return {"ack": {"accepted": True}}
monkeypatch.setattr(worker, "_client", client)
monkeypatch.setattr(worker.cli_lane_runner, "run_provider", lambda *_args: pytest.fail("must not invoke a model"))
worker.execute(item)
terminal = calls[-1][1]["payload"]
assert [name for name, _ in calls] == ["resume", "finish"]
assert terminal["capacity_failure"] is False
assert terminal["structured"]["status"] == "blocked"

59
testing/tests/test_hermes_legacy_scm_roots.py
View File

@ -6,6 +6,7 @@ import json
from pathlib import Path
import sys
import pytest
import yaml
from testing.tests.test_hermes_cli_support import HERMES, _load
@ -15,6 +16,7 @@ ROOT = Path(__file__).parents[2]
sys.path.insert(0, str(HERMES / "scm-common/scripts"))
state = _load("supervisor_state")
seed = _load("seed_legacy_scm_roots")
retry = sys.modules["publication_retry"]
class NativeKanban:
@ -89,3 +91,60 @@ def test_seed_rerun_preserves_same_owned_newer_state_and_approval(tmp_path, monk
assert connection.execute(
"SELECT ready_for_human_merge,ready_commit FROM supervisor_roots"
).fetchone() == (1, newer)
def test_publication_retry_is_bound_once_and_never_falls_back_to_a_model(tmp_path, monkeypatch):
"""A mediator receipt can power one fresh ordinal-pinned publication only."""
board, root_id, child_id, baseline, head = (
"soteria", "t_root", "t_child", "a" * 40, "b" * 40
)
lineage = state.Lineage(
root_id, "hermes-repair/cache", "https://scm.bstein.dev/titan/soteria/pulls/3",
"soteria", "main",
)
monkeypatch.setattr(state, "KANBAN_ROOT", tmp_path / "boards")
state.record_submission(board, root_id, lineage, baseline)
state.record_child(board, child_id, root_id, root_id, "repair", baseline, "replace cache literals")
structured = {
"status": "completed", "summary": "Replace cache literals.",
"changed_files": ["internal/k8s/job_manifests.go"], "tests_run": ["go test ./..."],
"artifacts": [], "findings": [], "blockers": [],
}
title, body = "Replace cache literals", "verified completion evidence"
receipt = {
"source": {
"board": board, "task_id": child_id, "run_id": "8", "worker_ordinal": 0,
"attempt": 1, "root_task_id": root_id,
"repo_url": "https://scm.bstein.dev/titan/soteria.git",
"branch": lineage.branch, "base_branch": "main",
},
"baseline_sha": baseline, "head": head, "title": title, "body": body,
"structured": structured, "result_digest": retry.receipt_digest(structured, title, body),
}
binding = {name: receipt["source"][name] for name in (
"board", "task_id", "run_id", "worker_ordinal", "attempt"
)}
state.record_publication_retry(board, child_id, binding, receipt)
assert state.publication_retry(board, child_id, "")["head"] == head
state.issue_publication_retry(board, child_id, "9")
assert state.publication_retry(board, child_id, "9")["source"]["worker_ordinal"] == 0
with pytest.raises(retry.PublicationRetryError, match="already issued"):
state.publication_retry(board, child_id, "10")
monkeypatch.setattr(state.time, "time", lambda: 1_000)
assert state.reissue_publication_retry(board, child_id, "9") is True
# A failed native park can replay the same terminal result without spending
# another retry or extending the backoff.
assert state.reissue_publication_retry(board, child_id, "9") is True
state.issue_publication_retry(board, child_id, "9")
with pytest.raises(retry.PublicationRetryError, match="backoff"):
state.publication_retry(board, child_id, "")
monkeypatch.setattr(state.time, "time", lambda: 1_301)
assert state.publication_retry(board, child_id, "")["head"] == head
state.issue_publication_retry(board, child_id, "10")
# Completion advances the trusted root head before the terminal receipt is
# marked resolved; a finalize replay must not revalidate its old baseline.
state.record_submission(board, child_id, lineage, head)
state.issue_publication_retry(board, child_id, "10")
state.resolve_publication_retry(board, child_id, "10")
state.resolve_publication_retry(board, child_id, "10")
assert state.publication_retry(board, child_id, "") is None

3
testing/tests/test_hermes_scm_broker_handler_coverage.py
View File

@ -137,9 +137,12 @@ def test_post_dispatches_control_and_git_or_rejects(monkeypatch):
assert calls == ["control", "git"]
rejected = _handler(broker, path="/v1/metadata", body=b"{}")
events = []
monkeypatch.setattr(broker, "log_rejection", lambda phase, error: events.append((phase, type(error).__name__)))
rejected._control = lambda: (_ for _ in ()).throw(broker.PolicyError("no"))
rejected.do_POST()
assert json.loads(rejected.wfile.getvalue()) == {"error": "request rejected"}
assert events == [("control", "PolicyError")]
def test_control_metadata_and_draft_fields(monkeypatch):