hermes: resume preserved PR publications without model retries
This commit is contained in:
parent
4a94977dbc
commit
bf2a1ab528
@ -1628,10 +1628,7 @@ COPY dockerfiles/hermes_execution_regression_support.py /tmp/hermes_execution_re
|
|||||||
COPY dockerfiles/hermes_run_safety_regression.py /tmp/hermes_run_safety_regression.py
|
COPY dockerfiles/hermes_run_safety_regression.py /tmp/hermes_run_safety_regression.py
|
||||||
COPY dockerfiles/hermes_decomposition_safety_regression.py /tmp/hermes_decomposition_safety_regression.py
|
COPY dockerfiles/hermes_decomposition_safety_regression.py /tmp/hermes_decomposition_safety_regression.py
|
||||||
COPY dockerfiles/hermes_lane_compatibility_regression.py /tmp/hermes_lane_compatibility_regression.py
|
COPY dockerfiles/hermes_lane_compatibility_regression.py /tmp/hermes_lane_compatibility_regression.py
|
||||||
COPY services/hermes/scripts/cli_lane_*.py /tmp/hermes-lane-regression/
|
COPY services/hermes/scripts/*.py /tmp/hermes-lane-regression/
|
||||||
COPY services/hermes/scripts/routing_catalog.py /tmp/hermes-lane-regression/
|
|
||||||
COPY services/hermes/scripts/supervisor_lineage.py /tmp/hermes-lane-regression/
|
|
||||||
COPY services/hermes/scripts/supervisor_state.py /tmp/hermes-lane-regression/
|
|
||||||
RUN HERMES_CLI_LANE_SOURCE=/tmp/hermes-lane-regression \
|
RUN HERMES_CLI_LANE_SOURCE=/tmp/hermes-lane-regression \
|
||||||
HERMES_COMPATIBILITY_MODE=legacy \
|
HERMES_COMPATIBILITY_MODE=legacy \
|
||||||
/opt/hermes/.venv/bin/python /tmp/hermes_lane_compatibility_regression.py \
|
/opt/hermes/.venv/bin/python /tmp/hermes_lane_compatibility_regression.py \
|
||||||
|
|||||||
@ -19,7 +19,4 @@
|
|||||||
!services/
|
!services/
|
||||||
!services/hermes/
|
!services/hermes/
|
||||||
!services/hermes/scripts/
|
!services/hermes/scripts/
|
||||||
!services/hermes/scripts/cli_lane_*.py
|
!services/hermes/scripts/*.py
|
||||||
!services/hermes/scripts/routing_catalog.py
|
|
||||||
!services/hermes/scripts/supervisor_lineage.py
|
|
||||||
!services/hermes/scripts/supervisor_state.py
|
|
||||||
|
|||||||
@ -63,6 +63,23 @@ the floor, change the failed plan, and raise capability or effort when needed.
|
|||||||
The concrete selected model, capability, effort, and rationale are recorded on
|
The concrete selected model, capability, effort, and rationale are recorded on
|
||||||
durable worker receipts.
|
durable worker receipts.
|
||||||
|
|
||||||
|
The hourly coordinator discovers the models visible to the Codex and Claude
|
||||||
|
accounts, writes non-secret catalog and health evidence, and evaluates catalog
|
||||||
|
candidates for capability fit before publishing Switchyard profiles. Claude
|
||||||
|
discovery uses its Agent SDK initialize metadata, including the account-visible
|
||||||
|
model list; it does not run a generation just to discover a model. A provider,
|
||||||
|
catalog, or evaluation outage preserves the last known-good routes and records
|
||||||
|
the refresh as deferred, so discovery does not silently become a routing bypass.
|
||||||
|
|
||||||
|
Existing pull-request continuation is available through coordinator-owned
|
||||||
|
lineage: `kanban_continue_pr.py` validates the root task, recorded PR, branch,
|
||||||
|
and current head before queuing a repair on that same PR. A signed continuation
|
||||||
|
worker may update that exact owned ref through the SCM broker. Publication-only
|
||||||
|
resume for a completed but unpublished SCM result is still being finalized;
|
||||||
|
until its exact-run ownership, evidence preservation, and bounded finite-retry
|
||||||
|
semantics are confirmed, keep the workspace commit and structured result
|
||||||
|
recoverable and do not claim that path is live.
|
||||||
|
|
||||||
## Private Jetson voice: multilingual TTS policy
|
## Private Jetson voice: multilingual TTS policy
|
||||||
|
|
||||||
`hermes-tts` on `titan-21` bakes three checksum-pinned Piper voices and
|
`hermes-tts` on `titan-21` bakes three checksum-pinned Piper voices and
|
||||||
@ -501,9 +518,9 @@ A `voice` field from a browser is never honoured at any hop.
|
|||||||
`healthChecks`: gating that 10m window on a best-effort pool would stall
|
`healthChecks`: gating that 10m window on a best-effort pool would stall
|
||||||
`hermes-chat` and `hermes-observer-bindings`, which `dependsOn: hermes`.
|
`hermes-chat` and `hermes-observer-bindings`, which `dependsOn: hermes`.
|
||||||
- A hashed execution-pool ConfigMap, protocol-version readiness checks, and
|
- A hashed execution-pool ConfigMap, protocol-version readiness checks, and
|
||||||
versioned SCM boundary name make code/config changes controlled rollouts. A
|
versioned SCM boundary name make code/config changes controlled rollouts.
|
||||||
rollout is still a human-reviewed operation; this repository change does not
|
Verify the config revision, protocol readiness, and worker/mediator health
|
||||||
reconcile or deploy it.
|
after Flux applies a reviewed rollout.
|
||||||
|
|
||||||
## Your shortest path to fluency
|
## Your shortest path to fluency
|
||||||
|
|
||||||
|
|||||||
@ -107,9 +107,13 @@ configMapGenerator:
|
|||||||
- execution_pool_server.py=scripts/execution_pool_server.py
|
- execution_pool_server.py=scripts/execution_pool_server.py
|
||||||
- execution_pool_client.py=scripts/execution_pool_client.py
|
- execution_pool_client.py=scripts/execution_pool_client.py
|
||||||
- execution_pool_worker.py=scripts/execution_pool_worker.py
|
- execution_pool_worker.py=scripts/execution_pool_worker.py
|
||||||
|
- execution_pool_resume.py=scripts/execution_pool_resume.py
|
||||||
- execution_pool_scm.py=scripts/execution_pool_scm.py
|
- execution_pool_scm.py=scripts/execution_pool_scm.py
|
||||||
- supervisor_lineage.py=scripts/supervisor_lineage.py
|
- supervisor_lineage.py=scripts/supervisor_lineage.py
|
||||||
- supervisor_state.py=scripts/supervisor_state.py
|
- supervisor_state.py=scripts/supervisor_state.py
|
||||||
|
- publication_retry.py=scripts/publication_retry.py
|
||||||
|
- scm_resume_bootstrap.py=scripts/scm_resume_bootstrap.py
|
||||||
|
- bootstrap_soteria_publication_retry.py=scripts/bootstrap_soteria_publication_retry.py
|
||||||
- seed_legacy_scm_roots.py=scripts/seed_legacy_scm_roots.py
|
- seed_legacy_scm_roots.py=scripts/seed_legacy_scm_roots.py
|
||||||
- deadline_http.py=scm-common/scripts/deadline_http.py
|
- deadline_http.py=scm-common/scripts/deadline_http.py
|
||||||
- gitea_api_policy.py=scm-common/scripts/gitea_api_policy.py
|
- gitea_api_policy.py=scm-common/scripts/gitea_api_policy.py
|
||||||
@ -178,6 +182,8 @@ configMapGenerator:
|
|||||||
- supervisor_policy.py=scripts/supervisor_policy.py
|
- supervisor_policy.py=scripts/supervisor_policy.py
|
||||||
- supervisor_lineage.py=scripts/supervisor_lineage.py
|
- supervisor_lineage.py=scripts/supervisor_lineage.py
|
||||||
- supervisor_state.py=scripts/supervisor_state.py
|
- supervisor_state.py=scripts/supervisor_state.py
|
||||||
|
- publication_retry.py=scripts/publication_retry.py
|
||||||
|
- scm_resume_bootstrap.py=scripts/scm_resume_bootstrap.py
|
||||||
- deadline_http.py=scm-common/scripts/deadline_http.py
|
- deadline_http.py=scm-common/scripts/deadline_http.py
|
||||||
- gitea_api_policy.py=scm-common/scripts/gitea_api_policy.py
|
- gitea_api_policy.py=scm-common/scripts/gitea_api_policy.py
|
||||||
- scm_broker_client.py=scm-common/scripts/scm_broker_client.py
|
- scm_broker_client.py=scm-common/scripts/scm_broker_client.py
|
||||||
|
|||||||
@ -30,7 +30,7 @@ from scm_task_grants import TaskLedger, verify_grant
|
|||||||
from scm_task_drafts import matches_pull, request_fields, update as update_draft
|
from scm_task_drafts import matches_pull, request_fields, update as update_draft
|
||||||
from scm_task_adoptions import seed as seed_adoptions
|
from scm_task_adoptions import seed as seed_adoptions
|
||||||
from scm_broker_io import RejectRedirect, response_status as _status, spool_response
|
from scm_broker_io import RejectRedirect, response_status as _status, spool_response
|
||||||
from scm_broker_server import AbsoluteHeaderDeadlineMixin, BoundedThreadingHTTPServer
|
from scm_broker_server import AbsoluteHeaderDeadlineMixin, BoundedThreadingHTTPServer, log_rejection, validate_ascii_headers
|
||||||
|
|
||||||
BROKER_PORT = 9081
|
BROKER_PORT = 9081
|
||||||
GIT_USER = "hermes-automation"
|
GIT_USER = "hermes-automation"
|
||||||
@ -264,8 +264,6 @@ def _upstream_git_request(
|
|||||||
return result.read()
|
return result.read()
|
||||||
finally:
|
finally:
|
||||||
result.close()
|
result.close()
|
||||||
|
|
||||||
|
|
||||||
class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler):
|
class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler):
|
||||||
"""Expose only bounded metadata, draft creation, and smart-HTTP Git."""
|
"""Expose only bounded metadata, draft creation, and smart-HTTP Git."""
|
||||||
|
|
||||||
@ -274,20 +272,8 @@ class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler):
|
|||||||
header_deadline_seconds = INBOUND_HEADER_TIMEOUT
|
header_deadline_seconds = INBOUND_HEADER_TIMEOUT
|
||||||
|
|
||||||
def _validate_headers(self) -> None:
|
def _validate_headers(self) -> None:
|
||||||
items = list(self.headers.items())
|
validate_ascii_headers(self.headers, maximum_count=MAX_HEADERS,
|
||||||
if len(items) > MAX_HEADERS:
|
maximum_bytes=MAX_HEADER_BYTES, error=PolicyError)
|
||||||
raise PolicyError("SCM request has too many headers")
|
|
||||||
total = 0
|
|
||||||
for name, value in items:
|
|
||||||
if not name.isascii() or not value.isascii():
|
|
||||||
raise PolicyError("SCM request headers must be ASCII")
|
|
||||||
total += len(name) + len(value) + 4
|
|
||||||
if any(ord(character) < 32 and character != "\t" for character in value):
|
|
||||||
raise PolicyError("SCM request header contains controls")
|
|
||||||
if total > MAX_HEADER_BYTES:
|
|
||||||
raise PolicyError("SCM request headers exceed the safe limit")
|
|
||||||
if len(self.headers.get_all("Content-Length", [])) > 1:
|
|
||||||
raise PolicyError("SCM request has duplicate Content-Length")
|
|
||||||
|
|
||||||
def _stream(
|
def _stream(
|
||||||
self, status: int, content_type: str, body: BinaryIO, length: int
|
self, status: int, content_type: str, body: BinaryIO, length: int
|
||||||
@ -318,7 +304,7 @@ class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler):
|
|||||||
self.end_headers()
|
self.end_headers()
|
||||||
self.wfile.write(body)
|
self.wfile.write(body)
|
||||||
|
|
||||||
def _reject(self, status: int = 400) -> None:
|
def _reject(self, status: int = 400, phase: str = "", category: str = "") -> None:
|
||||||
self._json(status, b'{"error":"request rejected"}')
|
self._json(status, b'{"error":"request rejected"}')
|
||||||
|
|
||||||
def do_GET(self) -> None:
|
def do_GET(self) -> None:
|
||||||
@ -358,9 +344,11 @@ class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler):
|
|||||||
|
|
||||||
def do_POST(self) -> None:
|
def do_POST(self) -> None:
|
||||||
try:
|
try:
|
||||||
|
self._phase = "headers"
|
||||||
self._validate_headers()
|
self._validate_headers()
|
||||||
self.connection.settimeout(INBOUND_BODY_TIMEOUT)
|
self.connection.settimeout(INBOUND_BODY_TIMEOUT)
|
||||||
if self.path in {"/v1/metadata", "/v1/drafts", "/v1/tasks/register", "/v1/tasks/draft-update"}:
|
if self.path in {"/v1/metadata", "/v1/drafts", "/v1/tasks/register", "/v1/tasks/draft-update"}:
|
||||||
|
self._phase = "control"
|
||||||
self._control()
|
self._control()
|
||||||
else:
|
else:
|
||||||
self._git_rpc()
|
self._git_rpc()
|
||||||
@ -371,7 +359,8 @@ class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler):
|
|||||||
urllib.error.URLError,
|
urllib.error.URLError,
|
||||||
ValueError,
|
ValueError,
|
||||||
json.JSONDecodeError,
|
json.JSONDecodeError,
|
||||||
):
|
) as error:
|
||||||
|
log_rejection(getattr(self, "_phase", "request"), error)
|
||||||
self._reject()
|
self._reject()
|
||||||
|
|
||||||
def _control(self) -> None:
|
def _control(self) -> None:
|
||||||
@ -409,12 +398,14 @@ class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler):
|
|||||||
self._json(200, result)
|
self._json(200, result)
|
||||||
|
|
||||||
def _git_rpc(self) -> None:
|
def _git_rpc(self) -> None:
|
||||||
|
self._phase = "target"
|
||||||
repo, operation, service = _git_target(self.path)
|
repo, operation, service = _git_target(self.path)
|
||||||
if (
|
if (
|
||||||
operation not in {"git-upload-pack", "git-receive-pack"}
|
operation not in {"git-upload-pack", "git-receive-pack"}
|
||||||
or service != operation
|
or service != operation
|
||||||
):
|
):
|
||||||
raise PolicyError("Git RPC operation is outside the allowlist")
|
raise PolicyError("Git RPC operation is outside the allowlist")
|
||||||
|
self._phase = "headers"
|
||||||
expected_request = f"application/x-{service}-request"
|
expected_request = f"application/x-{service}-request"
|
||||||
if self.headers.get_content_type() != expected_request or self.headers.get(
|
if self.headers.get_content_type() != expected_request or self.headers.get(
|
||||||
"Transfer-Encoding"
|
"Transfer-Encoding"
|
||||||
@ -423,6 +414,7 @@ class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler):
|
|||||||
length = _content_length(self.headers, MAX_GIT_REQUEST)
|
length = _content_length(self.headers, MAX_GIT_REQUEST)
|
||||||
token = read_token()
|
token = read_token()
|
||||||
_reject_forbidden(repo, "repository", (token,))
|
_reject_forbidden(repo, "repository", (token,))
|
||||||
|
self._phase = "body"
|
||||||
body, body_length = _spool_bounded(
|
body, body_length = _spool_bounded(
|
||||||
self.rfile,
|
self.rfile,
|
||||||
MAX_GIT_REQUEST,
|
MAX_GIT_REQUEST,
|
||||||
@ -437,10 +429,13 @@ class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler):
|
|||||||
if service == "git-receive-pack":
|
if service == "git-receive-pack":
|
||||||
raw_grant = self.headers.get("X-Hermes-Task-Grant", "")
|
raw_grant = self.headers.get("X-Hermes-Task-Grant", "")
|
||||||
if raw_grant:
|
if raw_grant:
|
||||||
|
self._phase = "grant"
|
||||||
claims = verify_grant(raw_grant)
|
claims = verify_grant(raw_grant)
|
||||||
if claims["repo"] != repo:
|
if claims["repo"] != repo:
|
||||||
raise PolicyError("task grant repository does not match Git target")
|
raise PolicyError("task grant repository does not match Git target")
|
||||||
|
self._phase = "ledger"
|
||||||
_task_ledger().authorize_update(claims)
|
_task_ledger().authorize_update(claims)
|
||||||
|
self._phase = "pack"
|
||||||
validate_receive_pack(
|
validate_receive_pack(
|
||||||
body, token, _credential_forms(token),
|
body, token, _credential_forms(token),
|
||||||
expected=(claims["expected_old"], claims["new_head"], claims["ref"]),
|
expected=(claims["expected_old"], claims["new_head"], claims["ref"]),
|
||||||
@ -450,6 +445,7 @@ class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler):
|
|||||||
# moved without an owned, signed task grant.
|
# moved without an owned, signed task grant.
|
||||||
_validate_receive_pack(body, token)
|
_validate_receive_pack(body, token)
|
||||||
expected = f"application/x-{service}-result"
|
expected = f"application/x-{service}-result"
|
||||||
|
self._phase = "upstream"
|
||||||
try:
|
try:
|
||||||
streamed = _upstream_git_request(
|
streamed = _upstream_git_request(
|
||||||
f"/titan/{repo}.git/{service}",
|
f"/titan/{repo}.git/{service}",
|
||||||
@ -473,18 +469,18 @@ class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler):
|
|||||||
# Smart HTTP uses HTTP 200 for both Git success and a rejected
|
# Smart HTTP uses HTTP 200 for both Git success and a rejected
|
||||||
# ref command. The authenticated branch read is the commit
|
# ref command. The authenticated branch read is the commit
|
||||||
# point; never advance the local ledger on packet status alone.
|
# point; never advance the local ledger on packet status alone.
|
||||||
|
self._phase = "persist"
|
||||||
if _branch_head(repo, claims["ref"], token) != claims["new_head"]:
|
if _branch_head(repo, claims["ref"], token) != claims["new_head"]:
|
||||||
result.close()
|
result.close()
|
||||||
raise PolicyError("Git upstream did not advance the granted branch")
|
raise PolicyError("Git upstream did not advance the granted branch")
|
||||||
_task_ledger().commit(claims)
|
_task_ledger().commit(claims)
|
||||||
try:
|
try:
|
||||||
|
self._phase = "response"
|
||||||
self._stream(200, expected, result, result_length)
|
self._stream(200, expected, result, result_length)
|
||||||
finally:
|
finally:
|
||||||
result.close()
|
result.close()
|
||||||
finally:
|
finally:
|
||||||
body.close()
|
body.close()
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
def main() -> int:
|
||||||
parser = argparse.ArgumentParser(description=__doc__)
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
parser.add_argument("--listen", default="0.0.0.0")
|
parser.add_argument("--listen", default="0.0.0.0")
|
||||||
|
|||||||
@ -5,11 +5,36 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
|
import logging
|
||||||
from http.server import ThreadingHTTPServer
|
from http.server import ThreadingHTTPServer
|
||||||
|
|
||||||
MAX_CONCURRENT_REQUESTS = 3
|
MAX_CONCURRENT_REQUESTS = 3
|
||||||
|
|
||||||
|
|
||||||
|
def validate_ascii_headers(headers, *, maximum_count: int, maximum_bytes: int, error) -> None:
|
||||||
|
"""Reject oversized or ambiguous broker headers before a request body is read."""
|
||||||
|
items = list(headers.items())
|
||||||
|
if len(items) > maximum_count:
|
||||||
|
raise error("SCM request has too many headers")
|
||||||
|
total = 0
|
||||||
|
for name, value in items:
|
||||||
|
if not name.isascii() or not value.isascii():
|
||||||
|
raise error("SCM request headers must be ASCII")
|
||||||
|
total += len(name) + len(value) + 4
|
||||||
|
if any(ord(character) < 32 and character != "\t" for character in value):
|
||||||
|
raise error("SCM request header contains controls")
|
||||||
|
if total > maximum_bytes:
|
||||||
|
raise error("SCM request headers exceed the safe limit")
|
||||||
|
if len(headers.get_all("Content-Length", [])) > 1:
|
||||||
|
raise error("SCM request has duplicate Content-Length")
|
||||||
|
|
||||||
|
|
||||||
|
def log_rejection(phase: str, error: BaseException) -> None:
|
||||||
|
"""Log a fixed operational category without request material or error text."""
|
||||||
|
category = "policy" if error.__class__.__name__ == "PolicyError" else "upstream" if error.__class__.__name__ in {"TimeoutError", "URLError"} else "io"
|
||||||
|
logging.warning("scm_rejected phase=%s category=%s", phase, category)
|
||||||
|
|
||||||
|
|
||||||
class _AbsoluteDeadlineReader:
|
class _AbsoluteDeadlineReader:
|
||||||
"""Read header lines against one wall-clock deadline, not idle timeouts."""
|
"""Read header lines against one wall-clock deadline, not idle timeouts."""
|
||||||
|
|
||||||
|
|||||||
167
services/hermes/scripts/bootstrap_soteria_publication_retry.py
Normal file
167
services/hermes/scripts/bootstrap_soteria_publication_retry.py
Normal file
@ -0,0 +1,167 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""One-time, fail-closed recovery of Soteria #3's retained publication evidence."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sqlite3
|
||||||
|
import stat
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import supervisor_state
|
||||||
|
from execution_pool_protocol import payload_digest, read_key, sign_envelope
|
||||||
|
|
||||||
|
|
||||||
|
BOARD = "soteria"
|
||||||
|
CHILD = "t_4095fe0d"
|
||||||
|
ROOT = "t_c7c42600"
|
||||||
|
RUN_ID = "8"
|
||||||
|
ORDINAL = 0
|
||||||
|
HEAD = "f21833e78768e7e4045a6304e311989196846ae4"
|
||||||
|
REQUIRED_RECEIPT = {
|
||||||
|
"source", "baseline_sha", "head", "title", "body", "structured", "result_digest"
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _value(task: Any, name: str) -> Any:
|
||||||
|
return task.get(name) if isinstance(task, dict) else getattr(task, name, None)
|
||||||
|
|
||||||
|
|
||||||
|
def _receipt(path: Path) -> dict[str, Any]:
|
||||||
|
"""Read the mediator-produced receipt without following an operator symlink."""
|
||||||
|
descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
|
||||||
|
try:
|
||||||
|
info = os.fstat(descriptor)
|
||||||
|
if not stat.S_ISREG(info.st_mode) or info.st_size > 48 * 1024:
|
||||||
|
raise ValueError("bootstrap receipt file is invalid")
|
||||||
|
raw = os.read(descriptor, 48 * 1024 + 1)
|
||||||
|
finally:
|
||||||
|
os.close(descriptor)
|
||||||
|
try:
|
||||||
|
value = json.loads(raw)
|
||||||
|
except (UnicodeDecodeError, json.JSONDecodeError) as error:
|
||||||
|
raise ValueError("bootstrap receipt is malformed") from error
|
||||||
|
if not isinstance(value, dict) or set(value) != REQUIRED_RECEIPT:
|
||||||
|
raise ValueError("bootstrap receipt shape is invalid")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _pool_record(database: Path) -> tuple[dict[str, Any], str, int]:
|
||||||
|
"""Read exactly run 8's finalized terminal evidence without writing pool state."""
|
||||||
|
source = sqlite3.connect(f"file:{database}?mode=ro", uri=True)
|
||||||
|
try:
|
||||||
|
row = source.execute(
|
||||||
|
"SELECT payload_json,result_json,result_digest,state,worker_ordinal,attempt FROM assignments "
|
||||||
|
"WHERE board=? AND task_id=? AND run_id=?", (BOARD, CHILD, RUN_ID)
|
||||||
|
).fetchone()
|
||||||
|
finally:
|
||||||
|
source.close()
|
||||||
|
if row is None or row[3] != "finalized" or row[4] != ORDINAL or not isinstance(row[5], int) or row[5] < 1:
|
||||||
|
raise ValueError("bootstrap source assignment is not the retained terminal run")
|
||||||
|
try:
|
||||||
|
assignment, result = json.loads(row[0]), json.loads(row[1])
|
||||||
|
except (TypeError, json.JSONDecodeError) as error:
|
||||||
|
raise ValueError("bootstrap terminal evidence is malformed") from error
|
||||||
|
structured = result.get("structured") if isinstance(result, dict) else None
|
||||||
|
if (
|
||||||
|
not isinstance(assignment, dict) or assignment.get("root_task_id") != ROOT
|
||||||
|
or assignment.get("continuation_kind") != "repair"
|
||||||
|
or not isinstance(structured, dict) or structured.get("status") != "blocked"
|
||||||
|
or result.get("returncode") != 0 or not isinstance(result.get("capacity_failure"), bool)
|
||||||
|
or result.get("scm_submission") is not None
|
||||||
|
or payload_digest(result) != row[2]
|
||||||
|
):
|
||||||
|
raise ValueError("bootstrap terminal evidence is not a clean publication refusal")
|
||||||
|
with sqlite3.connect(f"file:{database}?mode=ro", uri=True) as source:
|
||||||
|
later = source.execute(
|
||||||
|
"SELECT 1 FROM assignments WHERE board=? AND task_id=? AND run_id<>? "
|
||||||
|
"AND state IN ('assigned','running','result')", (BOARD, CHILD, RUN_ID)
|
||||||
|
).fetchone()
|
||||||
|
if later is not None:
|
||||||
|
raise ValueError("bootstrap has a newer live pool assignment")
|
||||||
|
return assignment, hashlib.sha256(row[1].encode()).hexdigest(), row[5]
|
||||||
|
|
||||||
|
|
||||||
|
def _native_guard() -> supervisor_state.Lineage:
|
||||||
|
"""Require the still-blocked exact native run and the private root/child chain."""
|
||||||
|
from hermes_cli import kanban_db
|
||||||
|
|
||||||
|
child = supervisor_state.get_child(BOARD, CHILD)
|
||||||
|
if child is None or child["root_task_id"] != ROOT or child["kind"] != "repair":
|
||||||
|
raise ValueError("bootstrap continuation lineage is unavailable")
|
||||||
|
with kanban_db.scoped_current_board(BOARD):
|
||||||
|
connection = kanban_db.connect(board=BOARD)
|
||||||
|
try:
|
||||||
|
task = kanban_db.get_task(connection, CHILD)
|
||||||
|
parents = kanban_db.parent_ids(connection, CHILD)
|
||||||
|
latest = connection.execute(
|
||||||
|
"SELECT id,status,outcome FROM task_runs WHERE task_id=? ORDER BY id DESC LIMIT 1", (CHILD,)
|
||||||
|
).fetchone()
|
||||||
|
finally:
|
||||||
|
connection.close()
|
||||||
|
if (
|
||||||
|
task is None or _value(task, "status") != "blocked"
|
||||||
|
or _value(task, "current_run_id") is not None
|
||||||
|
or latest is None or latest[0] != int(RUN_ID)
|
||||||
|
or latest[1] not in {"done", "failed"}
|
||||||
|
or not isinstance(parents, (list, tuple, set)) or ROOT not in {str(value) for value in parents}
|
||||||
|
):
|
||||||
|
raise ValueError("bootstrap native task is no longer the exact blocked run")
|
||||||
|
return child["lineage"]
|
||||||
|
|
||||||
|
|
||||||
|
def bootstrap(receipt_path: Path, pool_database: Path) -> str:
|
||||||
|
"""Persist one receipt only after the independent native and pool guards agree."""
|
||||||
|
receipt = _receipt(receipt_path)
|
||||||
|
assignment, raw_sha, attempt = _pool_record(pool_database)
|
||||||
|
lineage = _native_guard()
|
||||||
|
source = receipt["source"]
|
||||||
|
if (
|
||||||
|
receipt.get("head") != HEAD or not isinstance(source, dict)
|
||||||
|
or source.get("attempt") != attempt
|
||||||
|
or any(source.get(name) != value for name, value in {
|
||||||
|
"board": BOARD, "task_id": CHILD, "run_id": RUN_ID, "worker_ordinal": ORDINAL,
|
||||||
|
"root_task_id": ROOT, "repo_url": f"https://scm.bstein.dev/titan/{lineage.project}.git",
|
||||||
|
"branch": lineage.branch, "base_branch": lineage.base_branch,
|
||||||
|
}.items())
|
||||||
|
):
|
||||||
|
raise ValueError("bootstrap mediator receipt does not match retained authority")
|
||||||
|
binding = {name: source[name] for name in ("board", "task_id", "run_id", "worker_ordinal", "attempt")}
|
||||||
|
supervisor_state.record_publication_retry(BOARD, CHILD, binding, receipt)
|
||||||
|
supervisor_state.record_publication_retry_provenance(BOARD, CHILD, raw_sha)
|
||||||
|
return raw_sha
|
||||||
|
|
||||||
|
|
||||||
|
def signed_assignment(pool_database: Path, key_file: Path) -> bytes:
|
||||||
|
"""Attest the verified retained assignment with the current coordinator key."""
|
||||||
|
assignment, _raw_sha, attempt = _pool_record(pool_database)
|
||||||
|
_native_guard()
|
||||||
|
return json.dumps(sign_envelope(read_key(key_file), "assignment", {
|
||||||
|
"board": BOARD, "task_id": CHILD, "run_id": RUN_ID,
|
||||||
|
"worker_ordinal": ORDINAL, "attempt": attempt,
|
||||||
|
}, assignment), separators=(",", ":"), sort_keys=True).encode()
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument("--receipt-file", type=Path)
|
||||||
|
parser.add_argument("--pool-db", type=Path, default=Path(os.environ.get("HERMES_HOME", "/opt/data")) / "execution-pool/assignments.db")
|
||||||
|
parser.add_argument("--emit-assignment", action="store_true")
|
||||||
|
parser.add_argument("--key-file", type=Path, default=Path(os.environ.get("HERMES_EXECUTION_POOL_KEY_FILE", "/runtime-access/execution-pool-key")))
|
||||||
|
args = parser.parse_args()
|
||||||
|
if args.emit_assignment == (args.receipt_file is not None):
|
||||||
|
raise SystemExit("select exactly one bootstrap action")
|
||||||
|
if args.emit_assignment:
|
||||||
|
print(signed_assignment(args.pool_db, args.key_file).decode())
|
||||||
|
return 0
|
||||||
|
bootstrap(args.receipt_file, args.pool_db)
|
||||||
|
print(f"bootstrapped {BOARD}/{CHILD} run={RUN_ID} head={HEAD[:12]}")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@ -141,6 +141,7 @@ def claim_ready(
|
|||||||
limit: int,
|
limit: int,
|
||||||
eligible: Callable[[str, Any], bool] | None = None,
|
eligible: Callable[[str, Any], bool] | None = None,
|
||||||
claimer: str = "direct-cli-lane",
|
claimer: str = "direct-cli-lane",
|
||||||
|
priority: Callable[[str, Any], int] | None = None,
|
||||||
) -> list[tuple[str, str]]:
|
) -> list[tuple[str, str]]:
|
||||||
"""Atomically claim external ready tasks across all non-archived boards."""
|
"""Atomically claim external ready tasks across all non-archived boards."""
|
||||||
from hermes_cli import kanban_db
|
from hermes_cli import kanban_db
|
||||||
@ -158,7 +159,9 @@ def claim_ready(
|
|||||||
continue
|
continue
|
||||||
try:
|
try:
|
||||||
kanban_db.recompute_ready(conn)
|
kanban_db.recompute_ready(conn)
|
||||||
tasks = kanban_db.list_tasks(conn)
|
tasks = list(kanban_db.list_tasks(conn))
|
||||||
|
if priority is not None:
|
||||||
|
tasks.sort(key=lambda task: priority(board, task))
|
||||||
BOARD_CORRUPTION_ERRORS.pop(board, None)
|
BOARD_CORRUPTION_ERRORS.pop(board, None)
|
||||||
for task in tasks:
|
for task in tasks:
|
||||||
task_id = str(_task_value(task, "id", ""))
|
task_id = str(_task_value(task, "id", ""))
|
||||||
|
|||||||
@ -10,6 +10,7 @@ import urllib.error
|
|||||||
import urllib.request
|
import urllib.request
|
||||||
from http.server import BaseHTTPRequestHandler
|
from http.server import BaseHTTPRequestHandler
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from collections.abc import Callable
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
import cli_lane_goal
|
import cli_lane_goal
|
||||||
@ -72,6 +73,9 @@ class ClientBoundary:
|
|||||||
self.key = key
|
self.key = key
|
||||||
self.scm = scm or SCMBoundary(key)
|
self.scm = scm or SCMBoundary(key)
|
||||||
self.current: dict[str, Any] | None = None
|
self.current: dict[str, Any] | None = None
|
||||||
|
self.resumed: dict[str, str] | None = None
|
||||||
|
self.resumed_evidence: dict[str, Any] | None = None
|
||||||
|
self.resume_transient = False
|
||||||
self.lock = threading.RLock()
|
self.lock = threading.RLock()
|
||||||
|
|
||||||
def _post(self, path: str, envelope: dict[str, Any]) -> dict[str, Any]:
|
def _post(self, path: str, envelope: dict[str, Any]) -> dict[str, Any]:
|
||||||
@ -105,8 +109,12 @@ class ClientBoundary:
|
|||||||
return {"assignment": None}
|
return {"assignment": None}
|
||||||
if response["kind"] != "assignment" or response["worker_ordinal"] != ORDINAL:
|
if response["kind"] != "assignment" or response["worker_ordinal"] != ORDINAL:
|
||||||
raise ProtocolError("coordinator returned a foreign assignment")
|
raise ProtocolError("coordinator returned a foreign assignment")
|
||||||
checkout = self.scm.checkout(response)
|
publication_only = isinstance(response.get("payload", {}).get("scm_resume"), dict)
|
||||||
|
checkout = {"workspace": "", "baseline_sha": ""} if publication_only else self.scm.checkout(response)
|
||||||
self.current = response
|
self.current = response
|
||||||
|
self.resumed = None
|
||||||
|
self.resumed_evidence = None
|
||||||
|
self.resume_transient = False
|
||||||
assignment = {
|
assignment = {
|
||||||
**_binding(response),
|
**_binding(response),
|
||||||
"payload": response["payload"],
|
"payload": response["payload"],
|
||||||
@ -123,6 +131,32 @@ class ClientBoundary:
|
|||||||
raise ProtocolError("local request does not own the current assignment")
|
raise ProtocolError("local request does not own the current assignment")
|
||||||
return self.current, dict(supplied)
|
return self.current, dict(supplied)
|
||||||
|
|
||||||
|
def _publication_lease(self, binding: dict[str, Any]) -> tuple[Callable[[], None], Callable[[], None]]:
|
||||||
|
"""Keep a resume lease alive outside the SCM lock and fail before another step."""
|
||||||
|
stop, failures = threading.Event(), []
|
||||||
|
|
||||||
|
def renew() -> None:
|
||||||
|
try:
|
||||||
|
response = self._post("/v1/heartbeat", sign_envelope(self.key, "heartbeat", binding, {"note": "SCM publication in progress"}))
|
||||||
|
if response["kind"] != "ack" or _binding(response) != binding or not response["payload"].get("accepted"):
|
||||||
|
raise ProtocolError("coordinator rejected SCM publication lease")
|
||||||
|
except (OSError, ProtocolError, ValueError, urllib.error.URLError) as error:
|
||||||
|
failures.append(error)
|
||||||
|
|
||||||
|
renew()
|
||||||
|
def keepalive() -> None:
|
||||||
|
while not stop.wait(20):
|
||||||
|
renew()
|
||||||
|
|
||||||
|
thread = threading.Thread(target=keepalive, daemon=True)
|
||||||
|
thread.start()
|
||||||
|
|
||||||
|
def checkpoint() -> None:
|
||||||
|
if failures:
|
||||||
|
raise ProtocolError("SCM publication lease was lost") from failures[0]
|
||||||
|
|
||||||
|
return checkpoint, lambda: (stop.set(), thread.join(timeout=1))
|
||||||
|
|
||||||
def heartbeat(self, request: dict[str, Any]) -> dict[str, Any]:
|
def heartbeat(self, request: dict[str, Any]) -> dict[str, Any]:
|
||||||
payload = request.get("payload")
|
payload = request.get("payload")
|
||||||
if not isinstance(payload, dict):
|
if not isinstance(payload, dict):
|
||||||
@ -150,8 +184,23 @@ class ClientBoundary:
|
|||||||
prior work recoverable and reviewable by a human.
|
prior work recoverable and reviewable by a human.
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
submission = self.scm.submit(assignment, request)
|
if isinstance(self.scm, SCMBoundary):
|
||||||
|
checkpoint, close = self._publication_lease(_binding(assignment))
|
||||||
|
try:
|
||||||
|
submission = self.scm.submit(assignment, request, checkpoint=checkpoint)
|
||||||
|
checkpoint()
|
||||||
|
finally:
|
||||||
|
close()
|
||||||
|
else:
|
||||||
|
submission = self.scm.submit(assignment, request)
|
||||||
except (OSError, ProtocolError, RuntimeError, ValueError) as error:
|
except (OSError, ProtocolError, RuntimeError, ValueError) as error:
|
||||||
|
payload = assignment.get("payload")
|
||||||
|
resume = None
|
||||||
|
if isinstance(payload, dict) and payload.get("continuation_kind") == "repair":
|
||||||
|
try:
|
||||||
|
resume = self.scm.resume_artifact(assignment, request, structured)
|
||||||
|
except (OSError, ProtocolError, RuntimeError, ValueError):
|
||||||
|
resume = None
|
||||||
structured["status"] = "blocked"
|
structured["status"] = "blocked"
|
||||||
reason = (
|
reason = (
|
||||||
"Distributed SCM submission failed; the commits remain on this "
|
"Distributed SCM submission failed; the commits remain on this "
|
||||||
@ -159,7 +208,7 @@ class ClientBoundary:
|
|||||||
)
|
)
|
||||||
if reason not in structured["blockers"]:
|
if reason not in structured["blockers"]:
|
||||||
structured["blockers"].append(reason)
|
structured["blockers"].append(reason)
|
||||||
return None
|
return {"resume": resume} if resume is not None else None
|
||||||
pull = str(submission.get("pull_request") or "")
|
pull = str(submission.get("pull_request") or "")
|
||||||
branch = str(submission.get("branch") or "")
|
branch = str(submission.get("branch") or "")
|
||||||
for artifact in (pull, f"branch:{branch}" if branch else ""):
|
for artifact in (pull, f"branch:{branch}" if branch else ""):
|
||||||
@ -168,17 +217,66 @@ class ClientBoundary:
|
|||||||
head = str(submission.get("head") or "")
|
head = str(submission.get("head") or "")
|
||||||
return {"branch": branch, "pull_request": pull, "head": head}
|
return {"branch": branch, "pull_request": pull, "head": head}
|
||||||
|
|
||||||
|
def resume(self, request: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
"""Publish the coordinator-carried artifact without exposing it to the worker."""
|
||||||
|
with self.lock:
|
||||||
|
assignment, _binding = self._current_for(request.get("binding"))
|
||||||
|
artifact = assignment.get("payload", {}).get("scm_resume")
|
||||||
|
if not isinstance(artifact, dict):
|
||||||
|
raise ProtocolError("assignment has no SCM resume artifact")
|
||||||
|
evidence = _validate_result({"structured": artifact.get("structured")})["structured"]
|
||||||
|
if evidence["status"] != "completed":
|
||||||
|
raise ProtocolError("SCM resume evidence is not completed")
|
||||||
|
checkpoint, close = self._publication_lease(_binding)
|
||||||
|
try:
|
||||||
|
self.resume_transient = False
|
||||||
|
try:
|
||||||
|
self.resumed = self.scm.resume(assignment, artifact, checkpoint=checkpoint)
|
||||||
|
checkpoint()
|
||||||
|
except (OSError, RuntimeError, urllib.error.URLError) as error:
|
||||||
|
detail = str(error).lower()
|
||||||
|
if any(marker in detail for marker in ("timed out", "connection", "could not resolve", "http 502", "http 503", "http 504")):
|
||||||
|
self.resume_transient = True
|
||||||
|
return {"publication_retry_transient": True}
|
||||||
|
raise ProtocolError("SCM resume publication was rejected") from error
|
||||||
|
finally:
|
||||||
|
close()
|
||||||
|
self.resumed_evidence = json.loads(canonical_json(evidence))
|
||||||
|
return {"scm_submission": self.resumed}
|
||||||
|
|
||||||
def finish(self, request: dict[str, Any]) -> dict[str, Any]:
|
def finish(self, request: dict[str, Any]) -> dict[str, Any]:
|
||||||
payload = _validate_result(request.get("payload"))
|
payload = _validate_result(request.get("payload"))
|
||||||
|
# The model-facing caller cannot classify a retry as transient. Only a
|
||||||
|
# preceding mediator resume may attach this coordinator control signal.
|
||||||
|
payload.pop("publication_retry_transient", None)
|
||||||
with self.lock:
|
with self.lock:
|
||||||
assignment, binding = self._current_for(request.get("binding"))
|
assignment, binding = self._current_for(request.get("binding"))
|
||||||
structured = payload["structured"]
|
structured = payload["structured"]
|
||||||
|
if self.resume_transient:
|
||||||
|
payload["publication_retry_transient"] = True
|
||||||
if structured["status"] == "completed" and int(payload.get("returncode", 1)) == 0:
|
if structured["status"] == "completed" and int(payload.get("returncode", 1)) == 0:
|
||||||
submission = self._submit(assignment, request, structured)
|
if self.resumed is not None:
|
||||||
if submission is not None:
|
if self.resumed_evidence is None:
|
||||||
# The mediator, not the model-facing request, records the
|
raise ProtocolError("SCM resume evidence is unavailable")
|
||||||
# broker-confirmed PR/branch in the signed terminal wire.
|
structured = json.loads(canonical_json(self.resumed_evidence))
|
||||||
payload["scm_submission"] = submission
|
payload["structured"] = structured
|
||||||
|
payload["returncode"] = 0
|
||||||
|
payload["scm_submission"] = self.resumed
|
||||||
|
else:
|
||||||
|
submission = self._submit(assignment, request, structured)
|
||||||
|
if submission is not None and "resume" not in submission:
|
||||||
|
# The mediator, not the model-facing request, records the
|
||||||
|
# broker-confirmed PR/branch in the signed terminal wire.
|
||||||
|
payload["scm_submission"] = submission
|
||||||
|
elif submission is not None:
|
||||||
|
# The exact clean commit remains on the durable worker volume.
|
||||||
|
# A broker refusal is execution infrastructure, not a provider
|
||||||
|
# capability verdict, so let the coordinator retry it after the
|
||||||
|
# scoped SCM condition is corrected.
|
||||||
|
payload["capacity_failure"] = True
|
||||||
|
payload["scm_resume"] = submission["resume"]
|
||||||
|
else:
|
||||||
|
payload["capacity_failure"] = True
|
||||||
response = self._post(
|
response = self._post(
|
||||||
"/v1/result", sign_envelope(self.key, "result", binding, payload)
|
"/v1/result", sign_envelope(self.key, "result", binding, payload)
|
||||||
)
|
)
|
||||||
@ -223,6 +321,7 @@ def handler_factory(boundary: ClientBoundary) -> type[BaseHTTPRequestHandler]:
|
|||||||
routes = {
|
routes = {
|
||||||
"poll": boundary.poll,
|
"poll": boundary.poll,
|
||||||
"heartbeat": lambda: boundary.heartbeat(request),
|
"heartbeat": lambda: boundary.heartbeat(request),
|
||||||
|
"resume": lambda: boundary.resume(request),
|
||||||
"finish": lambda: boundary.finish(request),
|
"finish": lambda: boundary.finish(request),
|
||||||
}
|
}
|
||||||
if operation not in routes:
|
if operation not in routes:
|
||||||
|
|||||||
@ -159,6 +159,17 @@ def assignment_payload(
|
|||||||
else:
|
else:
|
||||||
repo_url, branch, base_branch = resolve_scm(task, board)
|
repo_url, branch, base_branch = resolve_scm(task, board)
|
||||||
root_task_id = lineage.root_task_id if lineage is not None else str(_task_value(task, "id"))
|
root_task_id = lineage.root_task_id if lineage is not None else str(_task_value(task, "id"))
|
||||||
|
run_id = canonical_run_id(_task_value(task, "current_run_id", None))
|
||||||
|
resume = None
|
||||||
|
if child is not None and run_id is not None:
|
||||||
|
try:
|
||||||
|
resume = supervisor_state.publication_retry(board, str(_task_value(task, "id")), str(run_id))
|
||||||
|
except (OSError, ValueError) as error:
|
||||||
|
raise ProtocolError("supervisor publication retry state is unavailable") from error
|
||||||
|
if resume is not None:
|
||||||
|
# The mediator republishes private evidence; the worker never sees a
|
||||||
|
# coding objective and must not invoke a model for this fresh run.
|
||||||
|
context = "Republish the mediator-verified completed SCM handoff."
|
||||||
runtime = int(_task_value(task, "max_runtime_seconds", 0) or 12 * 60 * 60)
|
runtime = int(_task_value(task, "max_runtime_seconds", 0) or 12 * 60 * 60)
|
||||||
runtime = max(60, min(runtime, 12 * 60 * 60))
|
runtime = max(60, min(runtime, 12 * 60 * 60))
|
||||||
return {
|
return {
|
||||||
@ -173,6 +184,7 @@ def assignment_payload(
|
|||||||
"deadline_unix": int(time.time()) + runtime,
|
"deadline_unix": int(time.time()) + runtime,
|
||||||
"goal_mode": bool(_task_value(task, "goal_mode", False)),
|
"goal_mode": bool(_task_value(task, "goal_mode", False)),
|
||||||
"goal_max_turns": max(1, min(int(_task_value(task, "goal_max_turns", 1) or 1), 12)),
|
"goal_max_turns": max(1, min(int(_task_value(task, "goal_max_turns", 1) or 1), 12)),
|
||||||
|
"scm_resume": resume,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -349,9 +361,38 @@ class Coordinator:
|
|||||||
"blockers": structured.get("blockers", []),
|
"blockers": structured.get("blockers", []),
|
||||||
}
|
}
|
||||||
assignment = record.get("payload")
|
assignment = record.get("payload")
|
||||||
|
if isinstance(assignment, dict) and assignment.get("scm_resume") is not None:
|
||||||
|
# A coordinator can crash after durable add but before the
|
||||||
|
# maintenance pass fences this receipt. Finalization must
|
||||||
|
# repair that narrow gap before it advances the live PR head.
|
||||||
|
supervisor_state.issue_publication_retry(
|
||||||
|
binding["board"], binding["task_id"], binding["run_id"]
|
||||||
|
)
|
||||||
verified_submission = _submission_lineage(
|
verified_submission = _submission_lineage(
|
||||||
binding, assignment, payload.get("scm_submission")
|
binding, assignment, payload.get("scm_submission")
|
||||||
)
|
)
|
||||||
|
retry = payload.get("scm_resume")
|
||||||
|
retry_exhausted = False
|
||||||
|
if retry is not None:
|
||||||
|
if (
|
||||||
|
verified_submission is not None
|
||||||
|
or not payload.get("capacity_failure")
|
||||||
|
or int(payload.get("returncode", 1)) != 0
|
||||||
|
or structured.get("status") != "blocked"
|
||||||
|
):
|
||||||
|
raise ProtocolError("publication retry receipt has an invalid terminal state")
|
||||||
|
supervisor_state.record_publication_retry(
|
||||||
|
binding["board"], binding["task_id"], binding, retry
|
||||||
|
)
|
||||||
|
elif (
|
||||||
|
isinstance(assignment, dict) and assignment.get("scm_resume") is not None
|
||||||
|
and payload.get("publication_retry_transient") is True
|
||||||
|
and payload.get("capacity_failure") is True
|
||||||
|
and verified_submission is None and structured.get("status") == "blocked"
|
||||||
|
):
|
||||||
|
retry_exhausted = not supervisor_state.reissue_publication_retry(
|
||||||
|
binding["board"], binding["task_id"], binding["run_id"]
|
||||||
|
)
|
||||||
problem = cli_lane_goal.unfinished_result_reason(structured)
|
problem = cli_lane_goal.unfinished_result_reason(structured)
|
||||||
if (
|
if (
|
||||||
structured.get("status") == "completed"
|
structured.get("status") == "completed"
|
||||||
@ -372,6 +413,10 @@ class Coordinator:
|
|||||||
supervisor_state.record_submission(
|
supervisor_state.record_submission(
|
||||||
binding["board"], binding["task_id"], lineage, head
|
binding["board"], binding["task_id"], lineage, head
|
||||||
)
|
)
|
||||||
|
if isinstance(assignment, dict) and assignment.get("scm_resume") is not None:
|
||||||
|
supervisor_state.resolve_publication_retry(
|
||||||
|
binding["board"], binding["task_id"], binding["run_id"]
|
||||||
|
)
|
||||||
changed = kanban_db.complete_task(
|
changed = kanban_db.complete_task(
|
||||||
connection, binding["task_id"],
|
connection, binding["task_id"],
|
||||||
result=json.dumps(structured, sort_keys=True),
|
result=json.dumps(structured, sort_keys=True),
|
||||||
@ -381,9 +426,11 @@ class Coordinator:
|
|||||||
else:
|
else:
|
||||||
reason = problem or "; ".join(map(str, structured.get("blockers", [])))
|
reason = problem or "; ".join(map(str, structured.get("blockers", [])))
|
||||||
reason = reason or str(structured.get("summary") or "worker failed")
|
reason = reason or str(structured.get("summary") or "worker failed")
|
||||||
|
if retry_exhausted:
|
||||||
|
reason = "Publication retry budget exhausted; inspect the retained SCM evidence."
|
||||||
changed = kanban_db.block_task(
|
changed = kanban_db.block_task(
|
||||||
connection, binding["task_id"], reason=reason,
|
connection, binding["task_id"], reason=reason,
|
||||||
kind="transient" if payload.get("capacity_failure") else "capability",
|
kind="transient" if payload.get("capacity_failure") and not retry_exhausted else "capability",
|
||||||
expected_run_id=run_id,
|
expected_run_id=run_id,
|
||||||
)
|
)
|
||||||
self.store.finalize(binding, "finalized" if changed else "stale")
|
self.store.finalize(binding, "finalized" if changed else "stale")
|
||||||
|
|||||||
@ -24,6 +24,7 @@ import cli_lane_dispatch
|
|||||||
from cli_lane_config import canonical_run_id
|
from cli_lane_config import canonical_run_id
|
||||||
from execution_pool_project import ProjectPolicyError, distributed_workspace_eligible
|
from execution_pool_project import ProjectPolicyError, distributed_workspace_eligible
|
||||||
from execution_pool_protocol import ProtocolError
|
from execution_pool_protocol import ProtocolError
|
||||||
|
import supervisor_state
|
||||||
|
|
||||||
|
|
||||||
RECOVERABLE_BOARD_ERRORS = (OSError, sqlite3.Error)
|
RECOVERABLE_BOARD_ERRORS = (OSError, sqlite3.Error)
|
||||||
@ -59,6 +60,35 @@ def _context(record: dict[str, Any]) -> str:
|
|||||||
return f"{record['board']}/{record['task_id']}#{record['run_id']}"
|
return f"{record['board']}/{record['task_id']}#{record['run_id']}"
|
||||||
|
|
||||||
|
|
||||||
|
def _resume_ordinal(payload: Any) -> int | None:
|
||||||
|
"""Read the coordinator-issued source ordinal without accepting task text."""
|
||||||
|
receipt = payload.get("scm_resume") if isinstance(payload, dict) else None
|
||||||
|
if receipt is None:
|
||||||
|
return None
|
||||||
|
source = receipt.get("source") if isinstance(receipt, dict) else None
|
||||||
|
ordinal = source.get("worker_ordinal") if isinstance(source, dict) else None
|
||||||
|
if isinstance(ordinal, bool) or not isinstance(ordinal, int) or ordinal not in range(3):
|
||||||
|
raise ProtocolError("publication retry ordinal is invalid")
|
||||||
|
return ordinal
|
||||||
|
|
||||||
|
|
||||||
|
def _issue_retry(record: dict[str, Any]) -> None:
|
||||||
|
"""Fence a persisted receipt to its one fresh durable assignment."""
|
||||||
|
if _resume_ordinal(record.get("payload")) is not None:
|
||||||
|
supervisor_state.issue_publication_retry(
|
||||||
|
record["board"], record["task_id"], record["run_id"]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _pending_retry_ordinal(board: str, task: Any) -> int | None:
|
||||||
|
"""Look up an unissued receipt before the ready card receives its fresh run."""
|
||||||
|
task_id = str(_task_value(task, "id", "") or "")
|
||||||
|
if not task_id:
|
||||||
|
return None
|
||||||
|
receipt = supervisor_state.publication_retry(board, task_id, "")
|
||||||
|
return _resume_ordinal({"scm_resume": receipt}) if receipt is not None else None
|
||||||
|
|
||||||
|
|
||||||
def _block_exact(
|
def _block_exact(
|
||||||
kanban_db: Any,
|
kanban_db: Any,
|
||||||
board: str,
|
board: str,
|
||||||
@ -98,7 +128,7 @@ def recover_results(pool: Any) -> None:
|
|||||||
|
|
||||||
|
|
||||||
def _lease_failure_state(
|
def _lease_failure_state(
|
||||||
kanban_db: Any, connection: Any, binding: dict[str, Any], run_id: int
|
kanban_db: Any, connection: Any, binding: dict[str, Any], run_id: int, retry_exhausted: bool = False
|
||||||
) -> str:
|
) -> str:
|
||||||
"""Classify one exhausted lease from authoritative Kanban evidence only."""
|
"""Classify one exhausted lease from authoritative Kanban evidence only."""
|
||||||
task = kanban_db.get_task(connection, binding["task_id"])
|
task = kanban_db.get_task(connection, binding["task_id"])
|
||||||
@ -114,11 +144,12 @@ def _lease_failure_state(
|
|||||||
if kanban_db.block_task(
|
if kanban_db.block_task(
|
||||||
connection,
|
connection,
|
||||||
binding["task_id"],
|
binding["task_id"],
|
||||||
reason=(
|
reason=("Publication retry budget exhausted; inspect the retained SCM evidence." if retry_exhausted else
|
||||||
|
(
|
||||||
"Distributed worker lease expired after "
|
"Distributed worker lease expired after "
|
||||||
f"{binding['attempt']} fenced attempts"
|
f"{binding['attempt']} fenced attempts"
|
||||||
),
|
)),
|
||||||
kind="transient",
|
kind="capability" if retry_exhausted else "transient",
|
||||||
expected_run_id=run_id,
|
expected_run_id=run_id,
|
||||||
):
|
):
|
||||||
return "finalized"
|
return "finalized"
|
||||||
@ -143,10 +174,15 @@ def record_lease_failure(pool: Any, record: dict[str, Any]) -> None:
|
|||||||
if run_id is None:
|
if run_id is None:
|
||||||
pool.store.finalize(binding, "stale")
|
pool.store.finalize(binding, "stale")
|
||||||
return
|
return
|
||||||
|
retry_exhausted = False
|
||||||
|
if _resume_ordinal(record.get("payload")) is not None:
|
||||||
|
retry_exhausted = not supervisor_state.reissue_publication_retry(
|
||||||
|
binding["board"], binding["task_id"], binding["run_id"]
|
||||||
|
)
|
||||||
with pool._kanban_lock, kanban_db.scoped_current_board(binding["board"]):
|
with pool._kanban_lock, kanban_db.scoped_current_board(binding["board"]):
|
||||||
connection = kanban_db.connect(board=binding["board"])
|
connection = kanban_db.connect(board=binding["board"])
|
||||||
try:
|
try:
|
||||||
state = _lease_failure_state(kanban_db, connection, binding, run_id)
|
state = _lease_failure_state(kanban_db, connection, binding, run_id, retry_exhausted)
|
||||||
finally:
|
finally:
|
||||||
connection.close()
|
connection.close()
|
||||||
pool.store.finalize(binding, state)
|
pool.store.finalize(binding, state)
|
||||||
@ -222,9 +258,16 @@ def _adopt_task(
|
|||||||
f"{type(error).__name__}: {error}",
|
f"{type(error).__name__}: {error}",
|
||||||
)
|
)
|
||||||
return
|
return
|
||||||
|
ordinal = _resume_ordinal(payload)
|
||||||
|
if ordinal is not None:
|
||||||
|
if ordinal not in ordinals:
|
||||||
|
_defer(f"{board}/{task_id} recovery retry", ProtocolError("source ordinal is unavailable"))
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
ordinal = ordinals[0]
|
||||||
binding = {
|
binding = {
|
||||||
"board": board, "task_id": task_id, "run_id": run_id,
|
"board": board, "task_id": task_id, "run_id": run_id,
|
||||||
"worker_ordinal": ordinals[0], "attempt": 1,
|
"worker_ordinal": ordinal, "attempt": 1,
|
||||||
}
|
}
|
||||||
try:
|
try:
|
||||||
pool.store.add(binding, payload)
|
pool.store.add(binding, payload)
|
||||||
@ -234,7 +277,8 @@ def _adopt_task(
|
|||||||
_defer(f"{board}/{task_id} adoption", error)
|
_defer(f"{board}/{task_id} adoption", error)
|
||||||
return
|
return
|
||||||
known.add((board, task_id, run_id))
|
known.add((board, task_id, run_id))
|
||||||
ordinals.pop(0)
|
_issue_retry({**binding, "payload": payload})
|
||||||
|
ordinals.remove(ordinal)
|
||||||
|
|
||||||
|
|
||||||
def _adopt_board(
|
def _adopt_board(
|
||||||
@ -262,6 +306,7 @@ def reconcile(pool: Any) -> None:
|
|||||||
|
|
||||||
for record in pool.store.active_assignments():
|
for record in pool.store.active_assignments():
|
||||||
try:
|
try:
|
||||||
|
_issue_retry(record)
|
||||||
_release_moved_run(pool, kanban_db, record)
|
_release_moved_run(pool, kanban_db, record)
|
||||||
except Exception as error: # noqa: BLE001 - other assignments still reconcile
|
except Exception as error: # noqa: BLE001 - other assignments still reconcile
|
||||||
_defer(f"{_context(record)} release", error)
|
_defer(f"{_context(record)} release", error)
|
||||||
@ -324,15 +369,22 @@ def _materialize(
|
|||||||
finally:
|
finally:
|
||||||
connection.close()
|
connection.close()
|
||||||
run_id = str(database_run_id)
|
run_id = str(database_run_id)
|
||||||
|
expected_ordinal = _resume_ordinal(payload)
|
||||||
|
if expected_ordinal is not None and expected_ordinal != ordinal:
|
||||||
|
raise ProtocolError("publication retry was not assigned to its source ordinal")
|
||||||
if (board, task_id, run_id) in known:
|
if (board, task_id, run_id) in known:
|
||||||
return
|
return
|
||||||
pool.store.add(
|
added = pool.store.add(
|
||||||
{
|
{
|
||||||
"board": board, "task_id": task_id, "run_id": run_id,
|
"board": board, "task_id": task_id, "run_id": run_id,
|
||||||
"worker_ordinal": ordinal, "attempt": 1,
|
"worker_ordinal": ordinal, "attempt": 1,
|
||||||
},
|
},
|
||||||
payload,
|
payload,
|
||||||
)
|
)
|
||||||
|
if added:
|
||||||
|
_issue_retry({
|
||||||
|
"board": board, "task_id": task_id, "run_id": run_id, "payload": payload,
|
||||||
|
})
|
||||||
known.add((board, task_id, run_id))
|
known.add((board, task_id, run_id))
|
||||||
_settled(f"{board}/{task_id} dispatch")
|
_settled(f"{board}/{task_id} dispatch")
|
||||||
except (ProtocolError, *RECOVERABLE_BOARD_ERRORS) as error:
|
except (ProtocolError, *RECOVERABLE_BOARD_ERRORS) as error:
|
||||||
@ -354,19 +406,56 @@ def dispatch(pool: Any) -> None:
|
|||||||
ordinals = pool.store.available_ordinals()
|
ordinals = pool.store.available_ordinals()
|
||||||
if not ordinals:
|
if not ordinals:
|
||||||
return
|
return
|
||||||
|
retry_ordinals: dict[tuple[str, str], int | None] = {}
|
||||||
|
|
||||||
|
def retry_ordinal(board: str, task: Any) -> int | None:
|
||||||
|
key = (board, str(_task_value(task, "id", "") or ""))
|
||||||
|
if key not in retry_ordinals:
|
||||||
|
retry_ordinals[key] = _pending_retry_ordinal(board, task)
|
||||||
|
return retry_ordinals[key]
|
||||||
|
|
||||||
|
def eligible(board: str, task: Any) -> bool:
|
||||||
|
if not distributed_workspace_eligible(task):
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
required = retry_ordinal(board, task)
|
||||||
|
except (OSError, ValueError) as error:
|
||||||
|
_defer(f"{board}/{_task_value(task, 'id', '')} retry lookup", error)
|
||||||
|
return False
|
||||||
|
return required is None or required in ordinals
|
||||||
|
|
||||||
|
def priority(board: str, task: Any) -> int:
|
||||||
|
try:
|
||||||
|
return 0 if retry_ordinal(board, task) is not None else 1
|
||||||
|
except (OSError, ValueError) as error:
|
||||||
|
_defer(f"{board}/{_task_value(task, 'id', '')} retry lookup", error)
|
||||||
|
return 2
|
||||||
try:
|
try:
|
||||||
# Only storage faults are absorbed here. An incompatible claim API must
|
# Only storage faults are absorbed here. An incompatible claim API must
|
||||||
# still surface, because silently skipping the eligibility predicate is
|
# still surface, because silently skipping the eligibility predicate is
|
||||||
# how the pool would start claiming the local lane's owned workspaces.
|
# how the pool would start claiming the local lane's owned workspaces.
|
||||||
claimed = cli_lane_dispatch.claim_ready(
|
claimed = cli_lane_dispatch.claim_ready(
|
||||||
set(), len(ordinals),
|
set(), len(ordinals),
|
||||||
lambda _board, task: distributed_workspace_eligible(task),
|
eligible,
|
||||||
claimer="execution-pool",
|
claimer="execution-pool",
|
||||||
|
priority=priority,
|
||||||
)
|
)
|
||||||
except RECOVERABLE_BOARD_ERRORS as error:
|
except RECOVERABLE_BOARD_ERRORS as error:
|
||||||
_defer("claim-ready", error)
|
_defer("claim-ready", error)
|
||||||
return
|
return
|
||||||
_settled("claim-ready")
|
_settled("claim-ready")
|
||||||
known = pool.store.known_runs()
|
known = pool.store.known_runs()
|
||||||
for ordinal, (board, task_id) in zip(ordinals, claimed, strict=False):
|
assigned: set[int] = set()
|
||||||
|
ordered = sorted(
|
||||||
|
claimed,
|
||||||
|
key=lambda item: 0 if retry_ordinals.get(item) is not None else 1,
|
||||||
|
)
|
||||||
|
for board, task_id in ordered:
|
||||||
|
ordinal = retry_ordinals.get((board, task_id))
|
||||||
|
if ordinal is None:
|
||||||
|
ordinal = next((value for value in ordinals if value not in assigned), None)
|
||||||
|
if ordinal is None or ordinal in assigned:
|
||||||
|
_defer(f"{board}/{task_id} dispatch", ProtocolError("no safe worker ordinal is available"))
|
||||||
|
continue
|
||||||
_materialize(pool, kanban_db, board, task_id, ordinal, known)
|
_materialize(pool, kanban_db, board, task_id, ordinal, known)
|
||||||
|
assigned.add(ordinal)
|
||||||
|
|||||||
62
services/hermes/scripts/execution_pool_resume.py
Normal file
62
services/hermes/scripts/execution_pool_resume.py
Normal file
@ -0,0 +1,62 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Build the fresh terminal result for a mediator-published SCM retry."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import re
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from execution_pool_protocol import ProtocolError
|
||||||
|
|
||||||
|
|
||||||
|
FIELDS = ("changed_files", "tests_run", "artifacts", "findings", "blockers")
|
||||||
|
SHA = re.compile(r"[0-9a-f]{40,64}\Z")
|
||||||
|
|
||||||
|
|
||||||
|
def result(receipt: Any, submission: Any, node: str) -> tuple[dict[str, Any], str, str]:
|
||||||
|
"""Reuse completed evidence only after the mediator confirms its exact head."""
|
||||||
|
if not isinstance(receipt, dict) or not isinstance(submission, dict):
|
||||||
|
raise ProtocolError("publication retry response is malformed")
|
||||||
|
structured = receipt.get("structured")
|
||||||
|
title, body, head = receipt.get("title"), receipt.get("body"), receipt.get("head")
|
||||||
|
if (
|
||||||
|
not isinstance(structured, dict) or structured.get("status") != "completed"
|
||||||
|
or not isinstance(title, str) or not title or not isinstance(body, str)
|
||||||
|
or not isinstance(head, str) or not SHA.fullmatch(head)
|
||||||
|
or submission.get("head") != head
|
||||||
|
):
|
||||||
|
raise ProtocolError("publication retry evidence changed")
|
||||||
|
copied = {"status": "completed", "summary": str(structured.get("summary") or "")[:8000]}
|
||||||
|
if not copied["summary"]:
|
||||||
|
raise ProtocolError("publication retry summary is missing")
|
||||||
|
for name in FIELDS:
|
||||||
|
value = structured.get(name)
|
||||||
|
if not isinstance(value, list) or any(not isinstance(item, str) for item in value):
|
||||||
|
raise ProtocolError("publication retry result is malformed")
|
||||||
|
copied[name] = list(value[:100])
|
||||||
|
for artifact in (submission.get("pull_request"), f"branch:{submission.get('branch', '')}", f"preserved-head:{head}"):
|
||||||
|
if isinstance(artifact, str) and artifact and artifact not in copied["artifacts"]:
|
||||||
|
copied["artifacts"].append(artifact)
|
||||||
|
return ({"structured": copied, "returncode": 0, "capacity_failure": False, "node": node,
|
||||||
|
"route": {"provider": "mediator", "model": "publication-resume"},
|
||||||
|
"provider_sessions": {}, "final_activity": "Mediator republished preserved SCM commit."}, title[:240], body[:12000])
|
||||||
|
|
||||||
|
|
||||||
|
def transient_failure(node: str) -> tuple[dict[str, Any], str, str]:
|
||||||
|
"""Return the sole mediator-classified retryable publication failure."""
|
||||||
|
structured = {"status": "blocked", "summary": "SCM publication is temporarily unavailable.",
|
||||||
|
"changed_files": [], "tests_run": [], "artifacts": [], "findings": [],
|
||||||
|
"blockers": ["Mediator classified the preserved SCM publication as transient."]}
|
||||||
|
return ({"structured": structured, "returncode": 1, "capacity_failure": True,
|
||||||
|
"publication_retry_transient": True, "node": node, "route": {},
|
||||||
|
"provider_sessions": {}, "final_activity": "Mediator deferred preserved SCM publication."}, "SCM publication deferred", "")
|
||||||
|
|
||||||
|
|
||||||
|
def rejected_failure(node: str) -> tuple[dict[str, Any], str, str]:
|
||||||
|
"""Surface mediator policy rejection without treating it as capacity loss."""
|
||||||
|
structured = {"status": "blocked", "summary": "SCM publication retry was rejected.",
|
||||||
|
"changed_files": [], "tests_run": [], "artifacts": [], "findings": [],
|
||||||
|
"blockers": ["Mediator rejected the preserved SCM publication evidence or policy."]}
|
||||||
|
return ({"structured": structured, "returncode": 1, "capacity_failure": False, "node": node,
|
||||||
|
"route": {}, "provider_sessions": {}, "final_activity": "Mediator rejected publication retry."},
|
||||||
|
"SCM publication rejected", "")
|
||||||
@ -4,6 +4,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
import hashlib
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
import stat
|
import stat
|
||||||
@ -12,12 +13,13 @@ import threading
|
|||||||
import time
|
import time
|
||||||
import urllib.parse
|
import urllib.parse
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from collections.abc import Callable
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
import scm_broker_client
|
import scm_broker_client
|
||||||
from scm_task_grants import ZERO_SHA, sign_grant
|
from scm_task_grants import ZERO_SHA, sign_grant
|
||||||
from execution_pool_project import ATLAS_REPO, validate_branch
|
from execution_pool_project import ATLAS_REPO, validate_branch
|
||||||
from execution_pool_protocol import ProtocolError, atomic_json, verify_envelope
|
from execution_pool_protocol import ProtocolError, atomic_json, canonical_json, verify_envelope
|
||||||
|
|
||||||
|
|
||||||
WORKSPACE_ROOT = Path(os.environ.get("HERMES_WORKER_ROOT", "/workspace"))
|
WORKSPACE_ROOT = Path(os.environ.get("HERMES_WORKER_ROOT", "/workspace"))
|
||||||
@ -25,6 +27,31 @@ SCM_ROOT = Path(os.environ.get("HERMES_SCM_STATE_ROOT", "/scm-state"))
|
|||||||
ORDINAL = int(os.environ.get("HERMES_WORKER_ORDINAL", "-1"))
|
ORDINAL = int(os.environ.get("HERMES_WORKER_ORDINAL", "-1"))
|
||||||
BROKER_ORIGIN = scm_broker_client.BROKER_ORIGIN.rstrip("/")
|
BROKER_ORIGIN = scm_broker_client.BROKER_ORIGIN.rstrip("/")
|
||||||
MAX_STATUS_BYTES = 4 * 1024 * 1024
|
MAX_STATUS_BYTES = 4 * 1024 * 1024
|
||||||
|
MAX_PULL_PAGES = 20
|
||||||
|
|
||||||
|
|
||||||
|
def _push_failure(error: RuntimeError) -> str:
|
||||||
|
"""Return bounded remediation without reflecting Git headers or grant text."""
|
||||||
|
detail = str(error).lower()
|
||||||
|
if any(marker in detail for marker in (
|
||||||
|
"task branch head changed", "non-fast-forward", "fetch first",
|
||||||
|
)):
|
||||||
|
return "task branch changed; fetch and merge before retrying"
|
||||||
|
if any(marker in detail for marker in (
|
||||||
|
"signature", "not registered", "owned by another task",
|
||||||
|
"authentication", "unauthorized", "forbidden", "permission denied",
|
||||||
|
"http 401", "http 403",
|
||||||
|
)):
|
||||||
|
return (
|
||||||
|
"SCM broker authorization rejected the branch update; "
|
||||||
|
"inspect task ownership before retrying"
|
||||||
|
)
|
||||||
|
if any(marker in detail for marker in (
|
||||||
|
"timed out", "connection", "could not resolve", "http 502",
|
||||||
|
"http 503", "http 504",
|
||||||
|
)):
|
||||||
|
return "SCM broker transport failed; retry the preserved local commit"
|
||||||
|
return "SCM broker rejected the branch update; inspect broker or upstream state before retrying"
|
||||||
|
|
||||||
|
|
||||||
def _git_environment() -> dict[str, str]:
|
def _git_environment() -> dict[str, str]:
|
||||||
@ -46,16 +73,20 @@ def _run(*arguments: str, cwd: Path | None = None, timeout: int = 300) -> str:
|
|||||||
"core.fsmonitor=false",
|
"core.fsmonitor=false",
|
||||||
*arguments,
|
*arguments,
|
||||||
]
|
]
|
||||||
completed = subprocess.run(
|
try:
|
||||||
command,
|
completed = subprocess.run(
|
||||||
cwd=cwd,
|
command,
|
||||||
env=_git_environment(),
|
cwd=cwd,
|
||||||
stdin=subprocess.DEVNULL,
|
env=_git_environment(),
|
||||||
text=True,
|
stdin=subprocess.DEVNULL,
|
||||||
capture_output=True,
|
text=True,
|
||||||
timeout=timeout,
|
capture_output=True,
|
||||||
check=False,
|
timeout=timeout,
|
||||||
)
|
check=False,
|
||||||
|
)
|
||||||
|
except subprocess.TimeoutExpired as error:
|
||||||
|
# Do not serialize ``error``: it includes the full command and its grant header.
|
||||||
|
raise RuntimeError("SCM command timed out") from error
|
||||||
if completed.returncode:
|
if completed.returncode:
|
||||||
message = (completed.stderr or completed.stdout or "SCM operation failed")[-2000:]
|
message = (completed.stderr or completed.stdout or "SCM operation failed")[-2000:]
|
||||||
raise RuntimeError(message.strip())
|
raise RuntimeError(message.strip())
|
||||||
@ -193,6 +224,20 @@ def _workspace_identity(destination: Path, repo: str, branch: str) -> str:
|
|||||||
return head
|
return head
|
||||||
|
|
||||||
|
|
||||||
|
def bootstrap_resume_artifact(key: bytes, assignment: Any, terminal: Any, *, title: str, body: str) -> dict[str, Any]:
|
||||||
|
"""Issue one receipt inside the mediator from two retained signed envelopes."""
|
||||||
|
boundary = Boundary(key)
|
||||||
|
source = boundary.verify(assignment)
|
||||||
|
result = verify_envelope(key, terminal, expected_kind="result")
|
||||||
|
names = ("board", "task_id", "run_id", "worker_ordinal", "attempt")
|
||||||
|
if any(source[name] != result[name] for name in names):
|
||||||
|
raise ProtocolError("retained terminal result does not match assignment")
|
||||||
|
payload = result.get("payload")
|
||||||
|
if not isinstance(payload, dict) or not isinstance(payload.get("structured"), dict):
|
||||||
|
raise ProtocolError("retained terminal result is malformed")
|
||||||
|
return boundary.resume_artifact(source, {"title": title, "body": body}, payload["structured"])
|
||||||
|
|
||||||
|
|
||||||
class Boundary:
|
class Boundary:
|
||||||
"""The only process allowed to turn model output into an SCM/result handoff."""
|
"""The only process allowed to turn model output into an SCM/result handoff."""
|
||||||
|
|
||||||
@ -268,11 +313,13 @@ class Boundary:
|
|||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _draft(repo: str, branch: str, base: str, head: str, title: str, body: str, grant: str = "", *, refresh: bool = True, existing_only: bool = False) -> str:
|
def _draft(repo: str, branch: str, base: str, head: str, title: str, body: str, grant: str = "", *, refresh: bool = True, existing_only: bool = False) -> str:
|
||||||
query = urllib.parse.urlencode({"state": "open", "limit": 50})
|
for page in range(1, MAX_PULL_PAGES + 1):
|
||||||
existing = json.loads(
|
query = urllib.parse.urlencode({"state": "open", "limit": 50, "page": page})
|
||||||
scm_broker_client.read(f"/api/v1/repos/titan/{repo}/pulls?{query}")
|
existing = json.loads(
|
||||||
)
|
scm_broker_client.read(f"/api/v1/repos/titan/{repo}/pulls?{query}")
|
||||||
if isinstance(existing, list):
|
)
|
||||||
|
if not isinstance(existing, list):
|
||||||
|
raise ProtocolError("open pull-request discovery is malformed")
|
||||||
for item in existing:
|
for item in existing:
|
||||||
if not isinstance(item, dict):
|
if not isinstance(item, dict):
|
||||||
continue
|
continue
|
||||||
@ -287,6 +334,10 @@ class Boundary:
|
|||||||
raise ProtocolError("existing task draft cannot be refreshed")
|
raise ProtocolError("existing task draft cannot be refreshed")
|
||||||
updated = json.loads(scm_broker_client.update_draft(grant, number, title, body))
|
updated = json.loads(scm_broker_client.update_draft(grant, number, title, body))
|
||||||
return str(updated.get("html_url") or "")
|
return str(updated.get("html_url") or "")
|
||||||
|
if len(existing) < 50:
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
raise ProtocolError("open pull-request discovery exceeds the safe page limit")
|
||||||
if existing_only:
|
if existing_only:
|
||||||
raise ProtocolError("review continuation has no existing pull request")
|
raise ProtocolError("review continuation has no existing pull request")
|
||||||
created = json.loads(
|
created = json.loads(
|
||||||
@ -301,7 +352,7 @@ class Boundary:
|
|||||||
)
|
)
|
||||||
return str(created.get("html_url") or "")
|
return str(created.get("html_url") or "")
|
||||||
|
|
||||||
def submit(self, envelope: dict[str, Any], request: dict[str, Any]) -> dict[str, Any]:
|
def submit(self, envelope: dict[str, Any], request: dict[str, Any], *, checkpoint: Callable[[], None] | None = None) -> dict[str, Any]:
|
||||||
"""Enforce clean/committed state, broker push, and reviewed draft creation."""
|
"""Enforce clean/committed state, broker push, and reviewed draft creation."""
|
||||||
_payload, repo, branch, base = _binding(envelope)
|
_payload, repo, branch, base = _binding(envelope)
|
||||||
destination = workspace_path(envelope)
|
destination = workspace_path(envelope)
|
||||||
@ -310,6 +361,8 @@ class Boundary:
|
|||||||
if not title or len(title.encode()) > 512 or len(body.encode()) > 32 * 1024:
|
if not title or len(title.encode()) > 512 or len(body.encode()) > 32 * 1024:
|
||||||
raise ProtocolError("pull-request metadata exceeds the safe limit")
|
raise ProtocolError("pull-request metadata exceeds the safe limit")
|
||||||
with self.lock:
|
with self.lock:
|
||||||
|
if checkpoint:
|
||||||
|
checkpoint()
|
||||||
head = _workspace_identity(destination, repo, branch)
|
head = _workspace_identity(destination, repo, branch)
|
||||||
state = json.loads(_regular_text(_state_path(envelope), 16 * 1024))
|
state = json.loads(_regular_text(_state_path(envelope), 16 * 1024))
|
||||||
baseline = state.get("baseline_sha") if isinstance(state, dict) else ""
|
baseline = state.get("baseline_sha") if isinstance(state, dict) else ""
|
||||||
@ -323,27 +376,125 @@ class Boundary:
|
|||||||
if status:
|
if status:
|
||||||
raise ProtocolError("workspace has uncommitted or untracked files")
|
raise ProtocolError("workspace has uncommitted or untracked files")
|
||||||
target = submission_refs(branch, int(envelope["attempt"]), head)[0]
|
target = submission_refs(branch, int(envelope["attempt"]), head)[0]
|
||||||
review = _payload.get("continuation_kind") == "review"
|
kind = _payload.get("continuation_kind")
|
||||||
|
repair, review = kind == "repair", kind == "review"
|
||||||
|
if repair:
|
||||||
|
# Do not advance an owned branch after its human PR was closed.
|
||||||
|
self._draft(repo, target, base, head, title, body, refresh=False, existing_only=True)
|
||||||
heads = _remote_heads(destination, (target,))
|
heads = _remote_heads(destination, (target,))
|
||||||
|
if checkpoint:
|
||||||
|
checkpoint()
|
||||||
remote = heads.get(target)
|
remote = heads.get(target)
|
||||||
ahead = int(_run("rev-list", "--count", f"{baseline}..{head}", cwd=destination))
|
ahead = int(_run("rev-list", "--count", f"{baseline}..{head}", cwd=destination))
|
||||||
if ahead <= 0 and remote is None:
|
if ahead <= 0 and remote is None:
|
||||||
return {"workspace": str(destination), "branch": branch, "pull_request": ""}
|
return {"workspace": str(destination), "branch": branch, "pull_request": ""}
|
||||||
if remote == head:
|
if remote == head:
|
||||||
|
if checkpoint:
|
||||||
|
checkpoint()
|
||||||
grant = self._grant(envelope, repo, target, base, remote, head)
|
grant = self._grant(envelope, repo, target, base, remote, head)
|
||||||
pull = self._draft(repo, target, base, head, title, body, grant) if not review else self._draft(repo, target, base, head, title, body, grant, refresh=False, existing_only=True)
|
pull = self._draft(repo, target, base, head, title, body, grant) if not (repair or review) else self._draft(
|
||||||
|
repo, target, base, head, title, body, grant, refresh=not review, existing_only=True
|
||||||
|
)
|
||||||
return {"workspace": str(destination), "branch": target, "pull_request": pull, "head": head}
|
return {"workspace": str(destination), "branch": target, "pull_request": pull, "head": head}
|
||||||
expected = remote or ZERO_SHA
|
expected = remote or ZERO_SHA
|
||||||
grant = self._grant(envelope, repo, target, base, expected, head)
|
grant = self._grant(envelope, repo, target, base, expected, head)
|
||||||
if remote is None:
|
if remote is None:
|
||||||
scm_broker_client.register_task(grant)
|
scm_broker_client.register_task(grant)
|
||||||
try:
|
try:
|
||||||
|
if checkpoint:
|
||||||
|
checkpoint()
|
||||||
_run(
|
_run(
|
||||||
"-c", f"http.extraHeader=X-Hermes-Task-Grant: {grant}",
|
"-c", f"http.extraHeader=X-Hermes-Task-Grant: {grant}",
|
||||||
"push", "--no-thin", "hermes-broker", f"HEAD:refs/heads/{target}",
|
"push", "--no-thin", "hermes-broker", f"HEAD:refs/heads/{target}",
|
||||||
cwd=destination, timeout=900,
|
cwd=destination, timeout=900,
|
||||||
)
|
)
|
||||||
except RuntimeError as error:
|
except RuntimeError as error:
|
||||||
raise ProtocolError("task branch update was rejected; fetch and merge before retrying") from error
|
raise ProtocolError(_push_failure(error)) from error
|
||||||
pull = self._draft(repo, target, base, head, title, body, self._grant(envelope, repo, target, base, head, head))
|
if checkpoint:
|
||||||
|
checkpoint()
|
||||||
|
grant = self._grant(envelope, repo, target, base, head, head)
|
||||||
|
pull = self._draft(repo, target, base, head, title, body, grant) if not (repair or review) else self._draft(
|
||||||
|
repo, target, base, head, title, body, grant, refresh=not review, existing_only=True
|
||||||
|
)
|
||||||
|
return {"workspace": str(destination), "branch": target, "pull_request": pull, "head": head}
|
||||||
|
|
||||||
|
def resume_artifact(self, envelope: dict[str, Any], request: dict[str, Any], structured: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
"""Bind one clean, locally durable commit for a later fresh-grant retry."""
|
||||||
|
payload, repo, branch, base = _binding(envelope)
|
||||||
|
destination = workspace_path(envelope)
|
||||||
|
head = _workspace_identity(destination, repo, branch)
|
||||||
|
state = json.loads(_regular_text(_state_path(envelope), 16 * 1024))
|
||||||
|
baseline = state.get("baseline_sha") if isinstance(state, dict) else ""
|
||||||
|
title, body = str(request.get("title") or "").strip(), str(request.get("body") or "")
|
||||||
|
if not isinstance(baseline, str) or not re.fullmatch(r"[0-9a-f]{40,64}", baseline):
|
||||||
|
raise ProtocolError("private SCM baseline is unavailable")
|
||||||
|
if _run("status", "--porcelain=v1", "--untracked-files=all", cwd=destination):
|
||||||
|
raise ProtocolError("workspace has uncommitted or untracked files")
|
||||||
|
if not title or len(title.encode()) > 512 or len(body.encode()) > 32 * 1024:
|
||||||
|
raise ProtocolError("pull-request metadata exceeds the safe limit")
|
||||||
|
root = payload.get("root_task_id")
|
||||||
|
if not isinstance(root, str) or not root:
|
||||||
|
raise ProtocolError("assignment continuation root is unavailable")
|
||||||
|
source = {name: envelope[name] for name in ("board", "task_id", "run_id", "worker_ordinal", "attempt")}
|
||||||
|
source.update({"repo_url": payload.get("repo_url"), "branch": branch, "base_branch": base, "root_task_id": root})
|
||||||
|
evidence = {
|
||||||
|
"structured": json.loads(canonical_json(structured)),
|
||||||
|
"title": title,
|
||||||
|
"body": body,
|
||||||
|
}
|
||||||
|
return {"source": source, "baseline_sha": baseline, "head": head, **evidence,
|
||||||
|
"result_digest": hashlib.sha256(canonical_json(evidence)).hexdigest()}
|
||||||
|
|
||||||
|
def resume(self, envelope: dict[str, Any], artifact: Any, *, checkpoint: Callable[[], None] | None = None) -> dict[str, Any]:
|
||||||
|
"""Publish a verified preserved head with only the new assignment grant."""
|
||||||
|
payload, repo, branch, base = _binding(envelope)
|
||||||
|
required = {"source", "baseline_sha", "head", "title", "body", "structured", "result_digest"}
|
||||||
|
if not isinstance(artifact, dict) or set(artifact) != required:
|
||||||
|
raise ProtocolError("SCM resume artifact is malformed")
|
||||||
|
source = artifact["source"]
|
||||||
|
source_keys = {"board", "task_id", "run_id", "worker_ordinal", "attempt", "repo_url", "branch", "base_branch", "root_task_id"}
|
||||||
|
if not isinstance(source, dict) or set(source) != source_keys:
|
||||||
|
raise ProtocolError("SCM resume source binding is malformed")
|
||||||
|
if (source["board"] != envelope["board"] or source["task_id"] != envelope["task_id"]
|
||||||
|
or source["run_id"] == envelope["run_id"] or not isinstance(source["attempt"], int) or source["attempt"] < 1
|
||||||
|
or not isinstance(source["worker_ordinal"], int) or source["worker_ordinal"] != envelope["worker_ordinal"]
|
||||||
|
or source["repo_url"] != payload.get("repo_url")
|
||||||
|
or source["branch"] != branch or source["base_branch"] != base or source["root_task_id"] != payload.get("root_task_id")):
|
||||||
|
raise ProtocolError("SCM resume source does not match assignment")
|
||||||
|
evidence = {name: artifact[name] for name in ("structured", "title", "body")}
|
||||||
|
if not isinstance(artifact["result_digest"], str) or artifact["result_digest"] != hashlib.sha256(canonical_json(evidence)).hexdigest():
|
||||||
|
raise ProtocolError("SCM resume evidence digest is invalid")
|
||||||
|
head, baseline = artifact["head"], artifact["baseline_sha"]
|
||||||
|
if not all(isinstance(value, str) and re.fullmatch(r"[0-9a-f]{40,64}", value) for value in (head, baseline)):
|
||||||
|
raise ProtocolError("SCM resume revisions are invalid")
|
||||||
|
if not isinstance(artifact["title"], str) or not artifact["title"] or len(artifact["title"].encode()) > 512 or not isinstance(artifact["body"], str) or len(artifact["body"].encode()) > 32 * 1024:
|
||||||
|
raise ProtocolError("SCM resume metadata is invalid")
|
||||||
|
source_envelope = {name: source[name] for name in ("board", "task_id", "run_id", "worker_ordinal", "attempt")}
|
||||||
|
destination = workspace_path(source_envelope)
|
||||||
|
if checkpoint:
|
||||||
|
checkpoint()
|
||||||
|
if _workspace_identity(destination, repo, branch) != head:
|
||||||
|
raise ProtocolError("preserved SCM workspace head changed")
|
||||||
|
state = json.loads(_regular_text(_state_path(source_envelope), 16 * 1024))
|
||||||
|
if not isinstance(state, dict) or state.get("baseline_sha") != baseline:
|
||||||
|
raise ProtocolError("preserved SCM baseline changed")
|
||||||
|
if _run("status", "--porcelain=v1", "--untracked-files=all", cwd=destination):
|
||||||
|
raise ProtocolError("preserved SCM workspace is not clean")
|
||||||
|
if checkpoint:
|
||||||
|
checkpoint()
|
||||||
|
target = submission_refs(branch, int(envelope["attempt"]), head)[0]
|
||||||
|
self._draft(repo, target, base, head, artifact["title"], artifact["body"], refresh=False, existing_only=True)
|
||||||
|
remote = _remote_heads(destination, (target,)).get(target)
|
||||||
|
if remote not in {baseline, head}:
|
||||||
|
raise ProtocolError("task branch changed; preserved commit needs reconciliation")
|
||||||
|
grant = self._grant(envelope, repo, target, base, head if remote == head else baseline, head)
|
||||||
|
if remote != head:
|
||||||
|
if checkpoint:
|
||||||
|
checkpoint()
|
||||||
|
_run("-c", f"http.extraHeader=X-Hermes-Task-Grant: {grant}", "push", "--no-thin", "hermes-broker", f"HEAD:refs/heads/{target}", cwd=destination, timeout=900)
|
||||||
|
if checkpoint:
|
||||||
|
checkpoint()
|
||||||
|
if checkpoint:
|
||||||
|
checkpoint()
|
||||||
|
pull = self._draft(repo, target, base, head, artifact["title"], artifact["body"], self._grant(envelope, repo, target, base, head, head), existing_only=True)
|
||||||
return {"workspace": str(destination), "branch": target, "pull_request": pull, "head": head}
|
return {"workspace": str(destination), "branch": target, "pull_request": pull, "head": head}
|
||||||
|
|||||||
@ -1,8 +1,6 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
"""Run one fenced Hermes assignment on an ordinal-scoped durable workspace."""
|
"""Run one fenced Hermes assignment on an ordinal-scoped durable workspace."""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
@ -15,15 +13,13 @@ import urllib.request
|
|||||||
from dataclasses import asdict
|
from dataclasses import asdict
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
import cli_lane_runner
|
import cli_lane_runner
|
||||||
|
import execution_pool_resume
|
||||||
from execution_pool_protocol import (
|
from execution_pool_protocol import (
|
||||||
ProtocolError,
|
ProtocolError,
|
||||||
atomic_json,
|
atomic_json,
|
||||||
canonical_json,
|
canonical_json,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
ROOT = Path(os.environ.get("HERMES_WORKER_ROOT", "/workspace"))
|
ROOT = Path(os.environ.get("HERMES_WORKER_ROOT", "/workspace"))
|
||||||
ORDINAL = int(os.environ.get("HERMES_WORKER_ORDINAL", "-1"))
|
ORDINAL = int(os.environ.get("HERMES_WORKER_ORDINAL", "-1"))
|
||||||
CLIENT = os.environ.get(
|
CLIENT = os.environ.get(
|
||||||
@ -34,8 +30,6 @@ NODE = os.environ.get("HERMES_WORKER_NODE", "unknown")[:128]
|
|||||||
RUN_PART = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$")
|
RUN_PART = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$")
|
||||||
RETENTION_SECONDS = int(os.environ.get("HERMES_WORKER_RETENTION_SECONDS", "1209600"))
|
RETENTION_SECONDS = int(os.environ.get("HERMES_WORKER_RETENTION_SECONDS", "1209600"))
|
||||||
DISABLED_PROVIDER = os.environ.get("HERMES_EXECUTION_DISABLED_PROVIDER", "").strip()
|
DISABLED_PROVIDER = os.environ.get("HERMES_EXECUTION_DISABLED_PROVIDER", "").strip()
|
||||||
|
|
||||||
|
|
||||||
def _post(url: str, value: dict[str, Any], timeout: int = 60) -> dict[str, Any]:
|
def _post(url: str, value: dict[str, Any], timeout: int = 60) -> dict[str, Any]:
|
||||||
request = urllib.request.Request(
|
request = urllib.request.Request(
|
||||||
url, data=canonical_json(value), method="POST",
|
url, data=canonical_json(value), method="POST",
|
||||||
@ -49,15 +43,11 @@ def _post(url: str, value: dict[str, Any], timeout: int = 60) -> dict[str, Any]:
|
|||||||
if not isinstance(document, dict):
|
if not isinstance(document, dict):
|
||||||
raise ProtocolError("server response is not an object")
|
raise ProtocolError("server response is not an object")
|
||||||
return document
|
return document
|
||||||
|
|
||||||
|
|
||||||
def _client(operation: str, **values: Any) -> dict[str, Any]:
|
def _client(operation: str, **values: Any) -> dict[str, Any]:
|
||||||
response = _post(f"{CLIENT}/v1/client", {"operation": operation, **values})
|
response = _post(f"{CLIENT}/v1/client", {"operation": operation, **values})
|
||||||
if response.get("error"):
|
if response.get("error"):
|
||||||
raise ProtocolError(str(response["error"]))
|
raise ProtocolError(str(response["error"]))
|
||||||
return response
|
return response
|
||||||
|
|
||||||
|
|
||||||
def _poll() -> dict[str, Any] | None:
|
def _poll() -> dict[str, Any] | None:
|
||||||
assignment = _client("poll").get("assignment")
|
assignment = _client("poll").get("assignment")
|
||||||
if assignment is None:
|
if assignment is None:
|
||||||
@ -69,14 +59,10 @@ def _poll() -> dict[str, Any] | None:
|
|||||||
):
|
):
|
||||||
raise ProtocolError("local boundary returned a foreign assignment")
|
raise ProtocolError("local boundary returned a foreign assignment")
|
||||||
return assignment
|
return assignment
|
||||||
|
|
||||||
|
|
||||||
def _binding(assignment: dict[str, Any]) -> dict[str, Any]:
|
def _binding(assignment: dict[str, Any]) -> dict[str, Any]:
|
||||||
return {name: assignment[name] for name in (
|
return {name: assignment[name] for name in (
|
||||||
"board", "task_id", "run_id", "worker_ordinal", "attempt"
|
"board", "task_id", "run_id", "worker_ordinal", "attempt"
|
||||||
)}
|
)}
|
||||||
|
|
||||||
|
|
||||||
def _state_path(assignment: dict[str, Any]) -> Path:
|
def _state_path(assignment: dict[str, Any]) -> Path:
|
||||||
parts = tuple(str(assignment[name]) for name in ("board", "task_id", "run_id"))
|
parts = tuple(str(assignment[name]) for name in ("board", "task_id", "run_id"))
|
||||||
if any(not RUN_PART.fullmatch(part) for part in parts):
|
if any(not RUN_PART.fullmatch(part) for part in parts):
|
||||||
@ -91,8 +77,6 @@ def _state_path(assignment: dict[str, Any]) -> Path:
|
|||||||
raise ProtocolError("assignment state path contains a symlink")
|
raise ProtocolError("assignment state path contains a symlink")
|
||||||
path.resolve(strict=False).relative_to(root)
|
path.resolve(strict=False).relative_to(root)
|
||||||
return path
|
return path
|
||||||
|
|
||||||
|
|
||||||
def _preserve_runtime_directory(runtime: Path) -> None:
|
def _preserve_runtime_directory(runtime: Path) -> None:
|
||||||
"""Move one legacy provider directory aside before installing our symlink."""
|
"""Move one legacy provider directory aside before installing our symlink."""
|
||||||
parent = runtime.parent
|
parent = runtime.parent
|
||||||
@ -112,8 +96,6 @@ def _preserve_runtime_directory(runtime: Path) -> None:
|
|||||||
raise ProtocolError(f"provider session migration failed: {runtime.name}") from error
|
raise ProtocolError(f"provider session migration failed: {runtime.name}") from error
|
||||||
if preserved.is_symlink() or not preserved.is_dir():
|
if preserved.is_symlink() or not preserved.is_dir():
|
||||||
raise ProtocolError(f"provider session migration is unsafe: {runtime.name}")
|
raise ProtocolError(f"provider session migration is unsafe: {runtime.name}")
|
||||||
|
|
||||||
|
|
||||||
def _bind_provider_sessions(assignment: dict[str, Any]) -> None:
|
def _bind_provider_sessions(assignment: dict[str, Any]) -> None:
|
||||||
"""Attach provider session directories to this exact durable task/run."""
|
"""Attach provider session directories to this exact durable task/run."""
|
||||||
parts = tuple(str(assignment[name]) for name in ("board", "task_id", "run_id"))
|
parts = tuple(str(assignment[name]) for name in ("board", "task_id", "run_id"))
|
||||||
@ -169,18 +151,12 @@ def _bind_provider_sessions(assignment: dict[str, Any]) -> None:
|
|||||||
elif runtime.exists():
|
elif runtime.exists():
|
||||||
raise ProtocolError(f"provider session path is not a symlink: {runtime.name}")
|
raise ProtocolError(f"provider session path is not a symlink: {runtime.name}")
|
||||||
runtime.symlink_to(durable)
|
runtime.symlink_to(durable)
|
||||||
|
|
||||||
|
|
||||||
def _prompt(context: str, workspace: Path, binding: dict[str, Any]) -> str:
|
def _prompt(context: str, workspace: Path, binding: dict[str, Any]) -> str:
|
||||||
return f"""You are a durable coding worker managed by Hermes Kanban.
|
return f"""You are a durable coding worker managed by Hermes Kanban.
|
||||||
|
|
||||||
Work only on this objective and its acceptance criteria:
|
Work only on this objective and its acceptance criteria:
|
||||||
|
|
||||||
{context}
|
{context}
|
||||||
|
|
||||||
Workspace: {workspace}
|
Workspace: {workspace}
|
||||||
Run binding: board={binding['board']} task={binding['task_id']} run={binding['run_id']} worker={ORDINAL}
|
Run binding: board={binding['board']} task={binding['task_id']} run={binding['run_id']} worker={ORDINAL}
|
||||||
|
|
||||||
Operate autonomously only inside this private assigned checkout. Inspect before editing,
|
Operate autonomously only inside this private assigned checkout. Inspect before editing,
|
||||||
preserve unrelated and untracked files, and fail closed rather than overwrite state you do
|
preserve unrelated and untracked files, and fail closed rather than overwrite state you do
|
||||||
not understand. You have no Kubernetes identity and no SCM credential. Do not attempt to
|
not understand. You have no Kubernetes identity and no SCM credential. Do not attempt to
|
||||||
@ -188,13 +164,10 @@ read Secrets, mutate workloads, use exec/attach/port-forward, reach node roots,
|
|||||||
the reviewed SCM boundary. Commit intended changes locally on the assigned feature branch;
|
the reviewed SCM boundary. Commit intended changes locally on the assigned feature branch;
|
||||||
the worker boundary handles the bounded push and draft pull request after validation.
|
the worker boundary handles the bounded push and draft pull request after validation.
|
||||||
Switchyard owns provider/model/effort selection and cross-provider fallback.
|
Switchyard owns provider/model/effort selection and cross-provider fallback.
|
||||||
|
|
||||||
Return a final JSON object matching the supplied schema. Use status=incomplete when work,
|
Return a final JSON object matching the supplied schema. Use status=incomplete when work,
|
||||||
tests, commits, or verification remain. Use blocked only for a concrete task obstacle.
|
tests, commits, or verification remain. Use blocked only for a concrete task obstacle.
|
||||||
Completed must have no blockers. List changed files, tests, artifacts, findings, and blockers.
|
Completed must have no blockers. List changed files, tests, artifacts, findings, and blockers.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
|
||||||
def _read_activity(log_path: Path, offset: int) -> tuple[str, int]:
|
def _read_activity(log_path: Path, offset: int) -> tuple[str, int]:
|
||||||
flags = os.O_RDONLY | os.O_NONBLOCK | getattr(os, "O_NOFOLLOW", 0)
|
flags = os.O_RDONLY | os.O_NONBLOCK | getattr(os, "O_NOFOLLOW", 0)
|
||||||
try:
|
try:
|
||||||
@ -213,8 +186,6 @@ def _read_activity(log_path: Path, offset: int) -> tuple[str, int]:
|
|||||||
finally:
|
finally:
|
||||||
os.close(descriptor)
|
os.close(descriptor)
|
||||||
return value.decode("utf-8", "replace"), next_offset
|
return value.decode("utf-8", "replace"), next_offset
|
||||||
|
|
||||||
|
|
||||||
def _git(workspace: Path, *arguments: str) -> str:
|
def _git(workspace: Path, *arguments: str) -> str:
|
||||||
completed = subprocess.run(
|
completed = subprocess.run(
|
||||||
["git", "-C", str(workspace), *arguments], stdin=subprocess.DEVNULL,
|
["git", "-C", str(workspace), *arguments], stdin=subprocess.DEVNULL,
|
||||||
@ -224,8 +195,6 @@ def _git(workspace: Path, *arguments: str) -> str:
|
|||||||
if completed.returncode:
|
if completed.returncode:
|
||||||
raise RuntimeError((completed.stderr or "Git inspection failed")[-1000:])
|
raise RuntimeError((completed.stderr or "Git inspection failed")[-1000:])
|
||||||
return completed.stdout.strip()
|
return completed.stdout.strip()
|
||||||
|
|
||||||
|
|
||||||
def _bounded_result(value: dict[str, Any]) -> dict[str, Any]:
|
def _bounded_result(value: dict[str, Any]) -> dict[str, Any]:
|
||||||
"""Keep terminal evidence useful while fitting the authenticated wire cap."""
|
"""Keep terminal evidence useful while fitting the authenticated wire cap."""
|
||||||
bounded: dict[str, Any] = {}
|
bounded: dict[str, Any] = {}
|
||||||
@ -245,15 +214,11 @@ def _bounded_result(value: dict[str, Any]) -> dict[str, Any]:
|
|||||||
else:
|
else:
|
||||||
bounded["summary"] = bounded["summary"][: len(bounded["summary"]) // 2]
|
bounded["summary"] = bounded["summary"][: len(bounded["summary"]) // 2]
|
||||||
return bounded
|
return bounded
|
||||||
|
|
||||||
|
|
||||||
def _refresh_assignment(binding: dict[str, Any]) -> dict[str, Any]:
|
def _refresh_assignment(binding: dict[str, Any]) -> dict[str, Any]:
|
||||||
fresh = _poll()
|
fresh = _poll()
|
||||||
if fresh is None or _binding(fresh) != binding:
|
if fresh is None or _binding(fresh) != binding:
|
||||||
raise ProtocolError("assignment changed before SCM submission")
|
raise ProtocolError("assignment changed before SCM submission")
|
||||||
return fresh
|
return fresh
|
||||||
|
|
||||||
|
|
||||||
def garbage_collect(now: float | None = None) -> int:
|
def garbage_collect(now: float | None = None) -> int:
|
||||||
"""Remove only clean, terminal, assignment-derived workspaces after retention."""
|
"""Remove only clean, terminal, assignment-derived workspaces after retention."""
|
||||||
current = time.time() if now is None else now
|
current = time.time() if now is None else now
|
||||||
@ -274,17 +239,44 @@ def garbage_collect(now: float | None = None) -> int:
|
|||||||
workspace, "status", "--porcelain=v1", "--untracked-files=all"
|
workspace, "status", "--porcelain=v1", "--untracked-files=all"
|
||||||
):
|
):
|
||||||
continue
|
continue
|
||||||
|
try:
|
||||||
|
branch = _git(workspace, "symbolic-ref", "--short", "HEAD")
|
||||||
|
except RuntimeError:
|
||||||
|
continue
|
||||||
|
landed = False
|
||||||
|
for remote in ("origin", "hermes-broker"):
|
||||||
|
try:
|
||||||
|
_git(workspace, "merge-base", "--is-ancestor", "HEAD", f"{remote}/{branch}")
|
||||||
|
landed = True
|
||||||
|
break
|
||||||
|
except RuntimeError:
|
||||||
|
continue
|
||||||
|
if not landed:
|
||||||
|
# A clean terminal checkout can still contain an unpublished commit.
|
||||||
|
# Keep it until a trusted remote-tracking ref proves the handoff landed.
|
||||||
|
continue
|
||||||
shutil.rmtree(workspace)
|
shutil.rmtree(workspace)
|
||||||
state_file.unlink()
|
state_file.unlink()
|
||||||
removed += 1
|
removed += 1
|
||||||
return removed
|
return removed
|
||||||
|
|
||||||
|
|
||||||
def execute(assignment: dict[str, Any]) -> None:
|
def execute(assignment: dict[str, Any]) -> None:
|
||||||
binding = _binding(assignment)
|
binding = _binding(assignment)
|
||||||
payload = assignment["payload"]
|
payload = assignment["payload"]
|
||||||
if not isinstance(payload, dict):
|
if not isinstance(payload, dict):
|
||||||
raise ProtocolError("assignment payload is invalid")
|
raise ProtocolError("assignment payload is invalid")
|
||||||
|
if payload.get("scm_resume") is not None:
|
||||||
|
try:
|
||||||
|
response = _client("resume", binding=binding)
|
||||||
|
terminal, title, body = (
|
||||||
|
execution_pool_resume.transient_failure(NODE)
|
||||||
|
if response.get("publication_retry_transient") is True else
|
||||||
|
execution_pool_resume.result(payload["scm_resume"], response.get("scm_submission"), NODE)
|
||||||
|
)
|
||||||
|
except ProtocolError:
|
||||||
|
terminal, title, body = execution_pool_resume.rejected_failure(NODE)
|
||||||
|
if not _client("finish", binding=binding, payload=terminal, title=title, body=body).get("ack", {}).get("accepted"):
|
||||||
|
raise ProtocolError("coordinator did not accept the publication retry")
|
||||||
|
return
|
||||||
_bind_provider_sessions(assignment)
|
_bind_provider_sessions(assignment)
|
||||||
workspace = Path(str(assignment.get("workspace") or "")).resolve(strict=True)
|
workspace = Path(str(assignment.get("workspace") or "")).resolve(strict=True)
|
||||||
workspace.relative_to((ROOT / "runs").resolve())
|
workspace.relative_to((ROOT / "runs").resolve())
|
||||||
@ -296,7 +288,6 @@ def execute(assignment: dict[str, Any]) -> None:
|
|||||||
log_path = state_file.with_suffix(".log")
|
log_path = state_file.with_suffix(".log")
|
||||||
offset = 0
|
offset = 0
|
||||||
latest_route: dict[str, Any] = {}
|
latest_route: dict[str, Any] = {}
|
||||||
|
|
||||||
def heartbeat(note: str) -> bool:
|
def heartbeat(note: str) -> bool:
|
||||||
nonlocal offset
|
nonlocal offset
|
||||||
try:
|
try:
|
||||||
@ -311,7 +302,6 @@ def execute(assignment: dict[str, Any]) -> None:
|
|||||||
return bool(response.get("ack", {}).get("accepted"))
|
return bool(response.get("ack", {}).get("accepted"))
|
||||||
except (OSError, ValueError, urllib.error.URLError, json.JSONDecodeError):
|
except (OSError, ValueError, urllib.error.URLError, json.JSONDecodeError):
|
||||||
return False
|
return False
|
||||||
|
|
||||||
context = str(payload.get("context") or "")
|
context = str(payload.get("context") or "")
|
||||||
assignee = str(payload.get("assignee") or "cli-auto")
|
assignee = str(payload.get("assignee") or "cli-auto")
|
||||||
excluded = None
|
excluded = None
|
||||||
@ -397,8 +387,6 @@ def execute(assignment: dict[str, Any]) -> None:
|
|||||||
raise ProtocolError("coordinator did not accept the terminal result")
|
raise ProtocolError("coordinator did not accept the terminal result")
|
||||||
state["terminal_at"] = time.time()
|
state["terminal_at"] = time.time()
|
||||||
atomic_json(state_file, state)
|
atomic_json(state_file, state)
|
||||||
|
|
||||||
|
|
||||||
def report_exception(assignment: dict[str, Any], error: Exception) -> bool:
|
def report_exception(assignment: dict[str, Any], error: Exception) -> bool:
|
||||||
"""Surface one worker exception as an exact-run transient result."""
|
"""Surface one worker exception as an exact-run transient result."""
|
||||||
binding = _binding(assignment)
|
binding = _binding(assignment)
|
||||||
@ -431,8 +419,6 @@ def report_exception(assignment: dict[str, Any], error: Exception) -> bool:
|
|||||||
)
|
)
|
||||||
except (OSError, ValueError, urllib.error.URLError, json.JSONDecodeError):
|
except (OSError, ValueError, urllib.error.URLError, json.JSONDecodeError):
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
def readiness() -> None:
|
def readiness() -> None:
|
||||||
if ORDINAL not in range(3):
|
if ORDINAL not in range(3):
|
||||||
raise ProtocolError("worker ordinal must be 0, 1, or 2")
|
raise ProtocolError("worker ordinal must be 0, 1, or 2")
|
||||||
@ -453,16 +439,6 @@ def readiness() -> None:
|
|||||||
atomic_json(
|
atomic_json(
|
||||||
cli_lane_runner.RESULT_SCHEMA_PATH, cli_lane_runner.RESULT_SCHEMA, 0o644
|
cli_lane_runner.RESULT_SCHEMA_PATH, cli_lane_runner.RESULT_SCHEMA, 0o644
|
||||||
)
|
)
|
||||||
try:
|
|
||||||
_poll()
|
|
||||||
except urllib.error.HTTPError as error:
|
|
||||||
# A replacement pod can overlap the prior ordinal's coordinator lease.
|
|
||||||
# The main loop already defers this transient conflict until ownership
|
|
||||||
# transfers, so keep the container alive instead of CrashLooping.
|
|
||||||
if error.code != 409:
|
|
||||||
raise
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
def main() -> int:
|
||||||
readiness()
|
readiness()
|
||||||
while True:
|
while True:
|
||||||
@ -482,7 +458,5 @@ def main() -> int:
|
|||||||
flush=True,
|
flush=True,
|
||||||
)
|
)
|
||||||
time.sleep(10)
|
time.sleep(10)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__": # pragma: no cover - exercised by the container entrypoint
|
if __name__ == "__main__": # pragma: no cover - exercised by the container entrypoint
|
||||||
raise SystemExit(main())
|
raise SystemExit(main())
|
||||||
|
|||||||
72
services/hermes/scripts/publication_retry.py
Normal file
72
services/hermes/scripts/publication_retry.py
Normal file
@ -0,0 +1,72 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Validate mediator-issued, one-shot SCM publication retry receipts."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
|
||||||
|
class PublicationRetryError(ValueError):
|
||||||
|
"""A receipt cannot safely bind a fresh continuation run to an old commit."""
|
||||||
|
|
||||||
|
|
||||||
|
SHA = re.compile(r"[0-9a-f]{40,64}\Z")
|
||||||
|
RESULT_FIELDS = frozenset(
|
||||||
|
{"status", "summary", "changed_files", "tests_run", "artifacts", "findings", "blockers"}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def receipt_digest(structured: dict[str, Any], title: str, body: str) -> str:
|
||||||
|
"""Return the exact digest the mediator must retain with completed evidence."""
|
||||||
|
value = {"structured": structured, "title": title, "body": body}
|
||||||
|
return hashlib.sha256(json.dumps(
|
||||||
|
value, ensure_ascii=False, separators=(",", ":"), sort_keys=True
|
||||||
|
).encode()).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def validate(
|
||||||
|
receipt: Any, binding: dict[str, Any], *, board: str, child_task_id: str,
|
||||||
|
root_task_id: str, project: str, branch: str, base_branch: str, live_head: str,
|
||||||
|
) -> tuple[dict[str, Any], int, str]:
|
||||||
|
"""Return a receipt only when its source and completed evidence are exact."""
|
||||||
|
source = receipt.get("source") if isinstance(receipt, dict) else None
|
||||||
|
if not isinstance(source, dict) or not all(
|
||||||
|
source.get(name) == binding.get(name)
|
||||||
|
for name in ("board", "task_id", "run_id", "worker_ordinal", "attempt")
|
||||||
|
):
|
||||||
|
raise PublicationRetryError("publication retry source binding is invalid")
|
||||||
|
expected = {
|
||||||
|
"board": board, "task_id": child_task_id, "root_task_id": root_task_id,
|
||||||
|
"repo_url": f"https://scm.bstein.dev/titan/{project}.git",
|
||||||
|
"branch": branch, "base_branch": base_branch,
|
||||||
|
}
|
||||||
|
if any(source.get(name) != value for name, value in expected.items()):
|
||||||
|
raise PublicationRetryError("publication retry source lineage is invalid")
|
||||||
|
run_id, ordinal, attempt = (source["run_id"], source["worker_ordinal"], source["attempt"])
|
||||||
|
if (
|
||||||
|
not isinstance(run_id, str) or not run_id.isdecimal() or int(run_id) < 1
|
||||||
|
or isinstance(ordinal, bool) or not isinstance(ordinal, int) or ordinal not in range(3)
|
||||||
|
or isinstance(attempt, bool) or not isinstance(attempt, int) or attempt < 1
|
||||||
|
):
|
||||||
|
raise PublicationRetryError("publication retry source run is invalid")
|
||||||
|
baseline, head, digest = (receipt.get(name) for name in ("baseline_sha", "head", "result_digest"))
|
||||||
|
title, body, structured = receipt.get("title"), receipt.get("body"), receipt.get("structured")
|
||||||
|
if (
|
||||||
|
not all(isinstance(value, str) and SHA.fullmatch(value) for value in (baseline, head))
|
||||||
|
or not isinstance(digest, str) or not re.fullmatch(r"[0-9a-f]{64}", digest)
|
||||||
|
or baseline != live_head or baseline == head
|
||||||
|
or not isinstance(title, str) or not title or len(title.encode()) > 512
|
||||||
|
or not isinstance(body, str) or len(body.encode()) > 32 * 1024
|
||||||
|
or not isinstance(structured, dict) or set(structured) != RESULT_FIELDS
|
||||||
|
or structured.get("status") != "completed"
|
||||||
|
):
|
||||||
|
raise PublicationRetryError("publication retry evidence is invalid")
|
||||||
|
if digest != receipt_digest(structured, title, body):
|
||||||
|
raise PublicationRetryError("publication retry result digest is invalid")
|
||||||
|
for name in RESULT_FIELDS - {"status", "summary"}:
|
||||||
|
if not isinstance(structured[name], list) or any(not isinstance(item, str) for item in structured[name]):
|
||||||
|
raise PublicationRetryError("publication retry result is invalid")
|
||||||
|
return receipt, ordinal, run_id
|
||||||
95
services/hermes/scripts/scm_resume_bootstrap.py
Executable file
95
services/hermes/scripts/scm_resume_bootstrap.py
Executable file
@ -0,0 +1,95 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Recover one completed SCM receipt from a mediator-owned worker result log."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import stat
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from execution_pool_protocol import ProtocolError, canonical_json, read_key
|
||||||
|
from execution_pool_scm import Boundary
|
||||||
|
|
||||||
|
ROOT = Path(os.environ.get("HERMES_WORKER_ROOT", "/workspace"))
|
||||||
|
KEY_PATH = Path(os.environ.get("HERMES_EXECUTION_POOL_KEY_FILE", "/pool-access/execution-pool-key"))
|
||||||
|
MAX_LOG_BYTES = 4 * 1024 * 1024
|
||||||
|
PART = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}\Z")
|
||||||
|
RESULT_FIELDS = frozenset({"status", "summary", "changed_files", "tests_run", "artifacts", "findings", "blockers"})
|
||||||
|
|
||||||
|
|
||||||
|
def _completed_result(assignment: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
"""Return exactly one completed result JSON object from an ordinal's safe log."""
|
||||||
|
parts = tuple(str(assignment[name]) for name in ("board", "task_id", "run_id"))
|
||||||
|
if any(not PART.fullmatch(part) for part in parts):
|
||||||
|
raise ProtocolError("bootstrap assignment path binding is invalid")
|
||||||
|
if ROOT.is_symlink():
|
||||||
|
raise ProtocolError("bootstrap workspace root is unsafe")
|
||||||
|
root = ROOT.resolve()
|
||||||
|
state_root = ROOT / "session-state"
|
||||||
|
board_root = state_root / parts[0]
|
||||||
|
task_root = board_root / parts[1]
|
||||||
|
log_path = task_root / f"{parts[2]}.log"
|
||||||
|
if any(path.is_symlink() for path in (state_root, board_root, task_root, log_path)):
|
||||||
|
raise ProtocolError("bootstrap result log is unsafe")
|
||||||
|
try:
|
||||||
|
log_path.resolve(strict=False).relative_to(root)
|
||||||
|
except ValueError as error:
|
||||||
|
raise ProtocolError("bootstrap result log escapes workspace root") from error
|
||||||
|
descriptor = os.open(log_path, os.O_RDONLY | os.O_NONBLOCK | getattr(os, "O_NOFOLLOW", 0))
|
||||||
|
try:
|
||||||
|
info = os.fstat(descriptor)
|
||||||
|
if not stat.S_ISREG(info.st_mode) or info.st_size > MAX_LOG_BYTES:
|
||||||
|
raise ProtocolError("bootstrap result log is invalid")
|
||||||
|
raw = os.read(descriptor, MAX_LOG_BYTES + 1)
|
||||||
|
finally:
|
||||||
|
os.close(descriptor)
|
||||||
|
candidates: dict[bytes, dict[str, Any]] = {}
|
||||||
|
for line in raw.splitlines():
|
||||||
|
try:
|
||||||
|
value = json.loads(line)
|
||||||
|
except (UnicodeDecodeError, json.JSONDecodeError):
|
||||||
|
continue
|
||||||
|
if not isinstance(value, dict) or set(value) != RESULT_FIELDS or value.get("status") != "completed":
|
||||||
|
continue
|
||||||
|
if not isinstance(value.get("summary"), str) or not value["summary"].strip():
|
||||||
|
continue
|
||||||
|
if any(not isinstance(value.get(name), list) or any(not isinstance(item, str) for item in value[name]) for name in RESULT_FIELDS - {"status", "summary"}):
|
||||||
|
continue
|
||||||
|
candidates[canonical_json(value)] = value
|
||||||
|
if len(candidates) != 1:
|
||||||
|
raise ProtocolError("bootstrap result evidence is absent or ambiguous")
|
||||||
|
return next(iter(candidates.values()))
|
||||||
|
|
||||||
|
|
||||||
|
def bootstrap(key: bytes, raw_assignment: Any) -> dict[str, Any]:
|
||||||
|
"""Verify assignment HMAC, derive evidence locally, and emit only its receipt."""
|
||||||
|
boundary = Boundary(key)
|
||||||
|
assignment = boundary.verify(raw_assignment)
|
||||||
|
structured = _completed_result(assignment)
|
||||||
|
title = structured["summary"].strip()[:512]
|
||||||
|
body = json.dumps(structured, indent=2, sort_keys=True)
|
||||||
|
return boundary.resume_artifact(assignment, {"title": title, "body": body}, structured)
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument("--assignment-stdin", action="store_true", required=True)
|
||||||
|
args = parser.parse_args()
|
||||||
|
raw = sys.stdin.buffer.read(64 * 1024 + 1)
|
||||||
|
if not args.assignment_stdin or len(raw) > 64 * 1024:
|
||||||
|
raise SystemExit("signed assignment input is invalid")
|
||||||
|
try:
|
||||||
|
receipt = bootstrap(read_key(KEY_PATH), json.loads(raw))
|
||||||
|
except (OSError, ProtocolError, ValueError, json.JSONDecodeError) as error:
|
||||||
|
raise SystemExit(f"bootstrap rejected: {error}") from error
|
||||||
|
sys.stdout.buffer.write(canonical_json(receipt) + b"\n")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__": # pragma: no cover - operator entrypoint
|
||||||
|
raise SystemExit(main())
|
||||||
@ -1,30 +1,24 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
"""Board-local integrity records for Hermes PR continuation cards."""
|
"""Board-local integrity records for Hermes PR continuation cards."""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import hashlib
|
import hashlib
|
||||||
import fcntl
|
import fcntl
|
||||||
|
import json
|
||||||
import os
|
import os
|
||||||
import sqlite3
|
import sqlite3
|
||||||
import tempfile
|
import tempfile
|
||||||
|
import time
|
||||||
from contextlib import contextmanager
|
from contextlib import contextmanager
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Iterator
|
from typing import Any, Iterator
|
||||||
|
|
||||||
from supervisor_lineage import Lineage
|
from supervisor_lineage import Lineage
|
||||||
|
from publication_retry import PublicationRetryError, validate as validate_retry_receipt
|
||||||
|
|
||||||
KANBAN_ROOT = Path(
|
KANBAN_ROOT = Path(
|
||||||
os.environ.get("HERMES_KANBAN_HOME", os.environ.get("HERMES_HOME", "/opt/data"))
|
os.environ.get("HERMES_KANBAN_HOME", os.environ.get("HERMES_HOME", "/opt/data"))
|
||||||
) / "kanban/boards"
|
) / "kanban/boards"
|
||||||
STATE_FILE = "supervisor-state.db"
|
STATE_FILE = "supervisor-state.db"
|
||||||
|
|
||||||
|
|
||||||
class SupervisorStateError(ValueError):
|
class SupervisorStateError(ValueError):
|
||||||
"""A present integrity record cannot be decoded or read safely."""
|
"""A present integrity record cannot be decoded or read safely."""
|
||||||
|
|
||||||
|
|
||||||
def _path(board: str, path: Path | None) -> Path:
|
def _path(board: str, path: Path | None) -> Path:
|
||||||
"""Keep integrity rows board-local without sharing Kanban's SQLite file."""
|
"""Keep integrity rows board-local without sharing Kanban's SQLite file."""
|
||||||
if path is not None:
|
if path is not None:
|
||||||
@ -32,8 +26,6 @@ def _path(board: str, path: Path | None) -> Path:
|
|||||||
if not board or "/" in board or "\\" in board:
|
if not board or "/" in board or "\\" in board:
|
||||||
raise ValueError("board name is invalid")
|
raise ValueError("board name is invalid")
|
||||||
return KANBAN_ROOT / board / STATE_FILE
|
return KANBAN_ROOT / board / STATE_FILE
|
||||||
|
|
||||||
|
|
||||||
def _schema(connection: sqlite3.Connection) -> None:
|
def _schema(connection: sqlite3.Connection) -> None:
|
||||||
"""Install the small coordinator schema in its isolated state database."""
|
"""Install the small coordinator schema in its isolated state database."""
|
||||||
connection.executescript(
|
connection.executescript(
|
||||||
@ -51,10 +43,21 @@ def _schema(connection: sqlite3.Connection) -> None:
|
|||||||
PRIMARY KEY (board, child_task_id),
|
PRIMARY KEY (board, child_task_id),
|
||||||
UNIQUE (board, root_task_id, head_commit, objective_digest)
|
UNIQUE (board, root_task_id, head_commit, objective_digest)
|
||||||
);
|
);
|
||||||
|
CREATE TABLE IF NOT EXISTS publication_retries (
|
||||||
|
board TEXT NOT NULL, child_task_id TEXT NOT NULL,
|
||||||
|
source_run_id TEXT NOT NULL, source_ordinal INTEGER NOT NULL,
|
||||||
|
receipt_json TEXT NOT NULL, issued_run_id TEXT NOT NULL DEFAULT '',
|
||||||
|
resolved_run_id TEXT NOT NULL DEFAULT '',
|
||||||
|
reissue_count INTEGER NOT NULL DEFAULT 0, retry_after INTEGER NOT NULL DEFAULT 0,
|
||||||
|
last_reissued_run_id TEXT NOT NULL DEFAULT '',
|
||||||
|
PRIMARY KEY (board, child_task_id)
|
||||||
|
);
|
||||||
|
CREATE TABLE IF NOT EXISTS publication_retry_provenance (
|
||||||
|
board TEXT NOT NULL, child_task_id TEXT NOT NULL, raw_result_sha256 TEXT NOT NULL,
|
||||||
|
reconstruction TEXT NOT NULL, PRIMARY KEY (board, child_task_id)
|
||||||
|
);
|
||||||
"""
|
"""
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _legacy_rows(board: str, database: Path) -> tuple[list[tuple[Any, ...]], list[tuple[Any, ...]]]:
|
def _legacy_rows(board: str, database: Path) -> tuple[list[tuple[Any, ...]], list[tuple[Any, ...]]]:
|
||||||
"""Read a complete, healthy legacy state table set or refuse migration."""
|
"""Read a complete, healthy legacy state table set or refuse migration."""
|
||||||
if not database.exists():
|
if not database.exists():
|
||||||
@ -99,8 +102,6 @@ def _legacy_rows(board: str, database: Path) -> tuple[list[tuple[Any, ...]], lis
|
|||||||
or any(str(row[2]) not in root_ids for row in children)):
|
or any(str(row[2]) not in root_ids for row in children)):
|
||||||
raise SupervisorStateError("legacy supervisor state is malformed")
|
raise SupervisorStateError("legacy supervisor state is malformed")
|
||||||
return roots, children
|
return roots, children
|
||||||
|
|
||||||
|
|
||||||
def _bootstrap_sidecar(board: str, state: Path) -> None:
|
def _bootstrap_sidecar(board: str, state: Path) -> None:
|
||||||
"""Atomically import valid legacy rows once, never creating an empty fallback."""
|
"""Atomically import valid legacy rows once, never creating an empty fallback."""
|
||||||
lock = state.with_name(f".{STATE_FILE}.lock")
|
lock = state.with_name(f".{STATE_FILE}.lock")
|
||||||
@ -138,8 +139,6 @@ def _bootstrap_sidecar(board: str, state: Path) -> None:
|
|||||||
except Exception:
|
except Exception:
|
||||||
replacement.unlink(missing_ok=True)
|
replacement.unlink(missing_ok=True)
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
|
||||||
@contextmanager
|
@contextmanager
|
||||||
def _connect(board: str, path: Path | None = None) -> Iterator[sqlite3.Connection]:
|
def _connect(board: str, path: Path | None = None) -> Iterator[sqlite3.Connection]:
|
||||||
"""Open one sidecar transaction and always close its SQLite handle."""
|
"""Open one sidecar transaction and always close its SQLite handle."""
|
||||||
@ -163,6 +162,15 @@ def _connect(board: str, path: Path | None = None) -> Iterator[sqlite3.Connectio
|
|||||||
child_columns = {row[1] for row in connection.execute("PRAGMA table_info(supervisor_children)")}
|
child_columns = {row[1] for row in connection.execute("PRAGMA table_info(supervisor_children)")}
|
||||||
if "cycle" not in child_columns:
|
if "cycle" not in child_columns:
|
||||||
connection.execute("ALTER TABLE supervisor_children ADD COLUMN cycle INTEGER NOT NULL DEFAULT 1")
|
connection.execute("ALTER TABLE supervisor_children ADD COLUMN cycle INTEGER NOT NULL DEFAULT 1")
|
||||||
|
retry_columns = {row[1] for row in connection.execute("PRAGMA table_info(publication_retries)")}
|
||||||
|
if "resolved_run_id" not in retry_columns:
|
||||||
|
connection.execute("ALTER TABLE publication_retries ADD COLUMN resolved_run_id TEXT NOT NULL DEFAULT ''")
|
||||||
|
if "reissue_count" not in retry_columns:
|
||||||
|
connection.execute("ALTER TABLE publication_retries ADD COLUMN reissue_count INTEGER NOT NULL DEFAULT 0")
|
||||||
|
if "retry_after" not in retry_columns:
|
||||||
|
connection.execute("ALTER TABLE publication_retries ADD COLUMN retry_after INTEGER NOT NULL DEFAULT 0")
|
||||||
|
if "last_reissued_run_id" not in retry_columns:
|
||||||
|
connection.execute("ALTER TABLE publication_retries ADD COLUMN last_reissued_run_id TEXT NOT NULL DEFAULT ''")
|
||||||
if path.name == STATE_FILE:
|
if path.name == STATE_FILE:
|
||||||
os.chmod(path, 0o600)
|
os.chmod(path, 0o600)
|
||||||
yield connection
|
yield connection
|
||||||
@ -172,8 +180,6 @@ def _connect(board: str, path: Path | None = None) -> Iterator[sqlite3.Connectio
|
|||||||
raise
|
raise
|
||||||
finally:
|
finally:
|
||||||
connection.close()
|
connection.close()
|
||||||
|
|
||||||
|
|
||||||
def _lineage(row: tuple[Any, ...] | None) -> Lineage | None:
|
def _lineage(row: tuple[Any, ...] | None) -> Lineage | None:
|
||||||
if row is None:
|
if row is None:
|
||||||
return None
|
return None
|
||||||
@ -184,8 +190,6 @@ def _lineage(row: tuple[Any, ...] | None) -> Lineage | None:
|
|||||||
if not all(values):
|
if not all(values):
|
||||||
return None
|
return None
|
||||||
return Lineage(*values)
|
return Lineage(*values)
|
||||||
|
|
||||||
|
|
||||||
def get_root(board: str, root_task_id: str, *, path: Path | None = None) -> Lineage | None:
|
def get_root(board: str, root_task_id: str, *, path: Path | None = None) -> Lineage | None:
|
||||||
"""Return one coordinator-issued root lineage, never task-body claims."""
|
"""Return one coordinator-issued root lineage, never task-body claims."""
|
||||||
try:
|
try:
|
||||||
@ -202,8 +206,6 @@ def get_root(board: str, root_task_id: str, *, path: Path | None = None) -> Line
|
|||||||
if lineage is None:
|
if lineage is None:
|
||||||
raise SupervisorStateError("supervisor root state is malformed")
|
raise SupervisorStateError("supervisor root state is malformed")
|
||||||
return lineage
|
return lineage
|
||||||
|
|
||||||
|
|
||||||
def get_live_head(board: str, root_task_id: str, *, path: Path | None = None) -> str:
|
def get_live_head(board: str, root_task_id: str, *, path: Path | None = None) -> str:
|
||||||
"""Return the broker-confirmed head associated with a trusted root."""
|
"""Return the broker-confirmed head associated with a trusted root."""
|
||||||
try:
|
try:
|
||||||
@ -219,8 +221,6 @@ def get_live_head(board: str, root_task_id: str, *, path: Path | None = None) ->
|
|||||||
if not isinstance(row[0], str) or not row[0]:
|
if not isinstance(row[0], str) or not row[0]:
|
||||||
raise SupervisorStateError("supervisor root state is malformed")
|
raise SupervisorStateError("supervisor root state is malformed")
|
||||||
return row[0]
|
return row[0]
|
||||||
|
|
||||||
|
|
||||||
def get_child(board: str, child_task_id: str, *, path: Path | None = None) -> dict[str, Any] | None:
|
def get_child(board: str, child_task_id: str, *, path: Path | None = None) -> dict[str, Any] | None:
|
||||||
"""Return the only durable authority record for a continuation child."""
|
"""Return the only durable authority record for a continuation child."""
|
||||||
try:
|
try:
|
||||||
@ -246,13 +246,10 @@ def get_child(board: str, child_task_id: str, *, path: Path | None = None) -> di
|
|||||||
return {"lineage": lineage, "root_task_id": root, "parent_task_id": parent,
|
return {"lineage": lineage, "root_task_id": root, "parent_task_id": parent,
|
||||||
"kind": kind, "head_commit": head, "objective_digest": digest,
|
"kind": kind, "head_commit": head, "objective_digest": digest,
|
||||||
"cycle": parsed_cycle}
|
"cycle": parsed_cycle}
|
||||||
|
|
||||||
|
|
||||||
def record_submission(
|
def record_submission(
|
||||||
board: str, task_id: str, lineage: Lineage, live_pr_head: str, *, path: Path | None = None
|
board: str, task_id: str, lineage: Lineage, live_pr_head: str, *, path: Path | None = None
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Record a coordinator-verified submission against its immutable root.
|
"""Record a coordinator-verified submission against its immutable root.
|
||||||
|
|
||||||
``task_id`` identifies the completing root or child. The coordinator has
|
``task_id`` identifies the completing root or child. The coordinator has
|
||||||
already authenticated that task's signed assignment before calling this
|
already authenticated that task's signed assignment before calling this
|
||||||
function; the record deliberately remains keyed by ``root_task_id``.
|
function; the record deliberately remains keyed by ``root_task_id``.
|
||||||
@ -279,8 +276,6 @@ def record_submission(
|
|||||||
"WHERE board=? AND root_task_id=?",
|
"WHERE board=? AND root_task_id=?",
|
||||||
(live_pr_head, board, lineage.root_task_id),
|
(live_pr_head, board, lineage.root_task_id),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def record_live_head(
|
def record_live_head(
|
||||||
board: str, root_task_id: str, live_pr_head: str, *, path: Path | None = None
|
board: str, root_task_id: str, live_pr_head: str, *, path: Path | None = None
|
||||||
) -> None:
|
) -> None:
|
||||||
@ -294,8 +289,6 @@ def record_live_head(
|
|||||||
).rowcount
|
).rowcount
|
||||||
if changed != 1:
|
if changed != 1:
|
||||||
raise ValueError("unknown continuation root")
|
raise ValueError("unknown continuation root")
|
||||||
|
|
||||||
|
|
||||||
def set_ready(
|
def set_ready(
|
||||||
board: str, root_task_id: str, head_commit: str, *, path: Path | None = None
|
board: str, root_task_id: str, head_commit: str, *, path: Path | None = None
|
||||||
) -> None:
|
) -> None:
|
||||||
@ -308,8 +301,6 @@ def set_ready(
|
|||||||
).rowcount
|
).rowcount
|
||||||
if changed != 1:
|
if changed != 1:
|
||||||
raise ValueError("ready head is not the trusted live root head")
|
raise ValueError("ready head is not the trusted live root head")
|
||||||
|
|
||||||
|
|
||||||
def clear_ready(board: str, root_task_id: str, *, path: Path | None = None) -> None:
|
def clear_ready(board: str, root_task_id: str, *, path: Path | None = None) -> None:
|
||||||
"""Invalidate approval whenever a continuation is queued or head changes."""
|
"""Invalidate approval whenever a continuation is queued or head changes."""
|
||||||
with _connect(board, path) as connection:
|
with _connect(board, path) as connection:
|
||||||
@ -317,8 +308,6 @@ def clear_ready(board: str, root_task_id: str, *, path: Path | None = None) -> N
|
|||||||
"UPDATE supervisor_roots SET ready_for_human_merge=0, ready_commit='' "
|
"UPDATE supervisor_roots SET ready_for_human_merge=0, ready_commit='' "
|
||||||
"WHERE board=? AND root_task_id=?", (board, root_task_id)
|
"WHERE board=? AND root_task_id=?", (board, root_task_id)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def record_child(
|
def record_child(
|
||||||
board: str, child_task_id: str, root_task_id: str, parent_task_id: str, kind: str,
|
board: str, child_task_id: str, root_task_id: str, parent_task_id: str, kind: str,
|
||||||
head_commit: str, objective: str, cycle: int = 1, *, path: Path | None = None
|
head_commit: str, objective: str, cycle: int = 1, *, path: Path | None = None
|
||||||
@ -348,13 +337,9 @@ def record_child(
|
|||||||
)
|
)
|
||||||
elif tuple(existing) != values:
|
elif tuple(existing) != values:
|
||||||
raise ValueError("continuation child lineage conflicts with its existing record")
|
raise ValueError("continuation child lineage conflicts with its existing record")
|
||||||
|
|
||||||
|
|
||||||
def objective_digest(objective: str) -> str:
|
def objective_digest(objective: str) -> str:
|
||||||
"""Make duplicate user follow-ups idempotent without retaining extra prose."""
|
"""Make duplicate user follow-ups idempotent without retaining extra prose."""
|
||||||
return hashlib.sha256(objective.strip().encode()).hexdigest()
|
return hashlib.sha256(objective.strip().encode()).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
def existing_child(
|
def existing_child(
|
||||||
board: str, root_task_id: str, head_commit: str, objective: str, *, path: Path | None = None
|
board: str, root_task_id: str, head_commit: str, objective: str, *, path: Path | None = None
|
||||||
) -> str:
|
) -> str:
|
||||||
@ -366,3 +351,150 @@ def existing_child(
|
|||||||
(board, root_task_id, head_commit, objective_digest(objective)),
|
(board, root_task_id, head_commit, objective_digest(objective)),
|
||||||
).fetchone()
|
).fetchone()
|
||||||
return str(row[0]) if row else ""
|
return str(row[0]) if row else ""
|
||||||
|
def _retry_receipt(
|
||||||
|
board: str, child_task_id: str, binding: dict[str, Any], receipt: Any, *, path: Path | None
|
||||||
|
) -> tuple[dict[str, Any], int, str]:
|
||||||
|
"""Validate the mediator-signed retry receipt against private continuation state."""
|
||||||
|
child = get_child(board, child_task_id, path=path)
|
||||||
|
if child is None or child["kind"] != "repair" or not isinstance(binding, dict):
|
||||||
|
raise ValueError("publication retry lineage is invalid")
|
||||||
|
lineage = child["lineage"]
|
||||||
|
return validate_retry_receipt(
|
||||||
|
receipt, binding, board=board, child_task_id=child_task_id,
|
||||||
|
root_task_id=lineage.root_task_id, project=lineage.project, branch=lineage.branch,
|
||||||
|
base_branch=lineage.base_branch, live_head=get_live_head(board, lineage.root_task_id, path=path),
|
||||||
|
)
|
||||||
|
def record_publication_retry(
|
||||||
|
board: str, child_task_id: str, binding: dict[str, Any], receipt: Any, *, path: Path | None = None
|
||||||
|
) -> None:
|
||||||
|
"""Persist one mediator-signed failed-publication receipt for a fresh run."""
|
||||||
|
verified, ordinal, source_run_id = _retry_receipt(board, child_task_id, binding, receipt, path=path)
|
||||||
|
encoded = json.dumps(verified, separators=(",", ":"), sort_keys=True)
|
||||||
|
if len(encoded.encode()) > 48 * 1024:
|
||||||
|
raise ValueError("publication retry receipt exceeds its assignment limit")
|
||||||
|
with _connect(board, path) as connection:
|
||||||
|
existing = connection.execute(
|
||||||
|
"SELECT source_run_id,source_ordinal,receipt_json FROM publication_retries "
|
||||||
|
"WHERE board=? AND child_task_id=?", (board, child_task_id)
|
||||||
|
).fetchone()
|
||||||
|
values = (source_run_id, ordinal, encoded)
|
||||||
|
if existing is None:
|
||||||
|
connection.execute(
|
||||||
|
"INSERT INTO publication_retries(board,child_task_id,source_run_id,source_ordinal,receipt_json) "
|
||||||
|
"VALUES(?,?,?,?,?)", (board, child_task_id, *values)
|
||||||
|
)
|
||||||
|
elif tuple(existing) != values:
|
||||||
|
raise ValueError("publication retry receipt conflicts with existing evidence")
|
||||||
|
def publication_retry(
|
||||||
|
board: str, child_task_id: str, run_id: str, *, path: Path | None = None
|
||||||
|
) -> dict[str, Any] | None:
|
||||||
|
"""Return an unconsumed receipt, or its one exact fresh run after a crash."""
|
||||||
|
if not isinstance(run_id, str) or (run_id and (not run_id.isdecimal() or int(run_id) < 1)):
|
||||||
|
raise ValueError("publication retry run is invalid")
|
||||||
|
if path is None and not _path(board, None).exists():
|
||||||
|
return None
|
||||||
|
with _connect(board, path) as connection:
|
||||||
|
row = connection.execute(
|
||||||
|
"SELECT source_run_id,source_ordinal,receipt_json,issued_run_id,resolved_run_id,reissue_count,retry_after FROM publication_retries "
|
||||||
|
"WHERE board=? AND child_task_id=?", (board, child_task_id)
|
||||||
|
).fetchone()
|
||||||
|
if row is None or row[4]:
|
||||||
|
return None
|
||||||
|
if int(row[6]) > int(time.time()):
|
||||||
|
raise PublicationRetryError("publication retry backoff is active")
|
||||||
|
if row[3] and row[3] != run_id:
|
||||||
|
raise PublicationRetryError("publication retry was already issued")
|
||||||
|
try:
|
||||||
|
receipt = json.loads(row[2])
|
||||||
|
except (TypeError, json.JSONDecodeError) as error:
|
||||||
|
raise SupervisorStateError("publication retry receipt is malformed") from error
|
||||||
|
verified, ordinal, source_run_id = _retry_receipt(
|
||||||
|
board, child_task_id,
|
||||||
|
{"board": board, "task_id": child_task_id, "run_id": row[0], "worker_ordinal": row[1],
|
||||||
|
"attempt": receipt.get("source", {}).get("attempt") if isinstance(receipt, dict) else None},
|
||||||
|
receipt, path=path,
|
||||||
|
)
|
||||||
|
if source_run_id != row[0] or ordinal != row[1] or (run_id and run_id == source_run_id):
|
||||||
|
raise SupervisorStateError("publication retry receipt is inconsistent")
|
||||||
|
return verified
|
||||||
|
def issue_publication_retry(board: str, child_task_id: str, run_id: str, *, path: Path | None = None) -> None:
|
||||||
|
"""Fence a receipt to one fresh assignment after its durable pool row exists."""
|
||||||
|
with _connect(board, path) as connection:
|
||||||
|
row = connection.execute(
|
||||||
|
"SELECT issued_run_id,resolved_run_id,last_reissued_run_id FROM publication_retries "
|
||||||
|
"WHERE board=? AND child_task_id=?", (board, child_task_id)
|
||||||
|
).fetchone()
|
||||||
|
if row is not None and (
|
||||||
|
(row[0] == run_id and row[1] in ("", run_id))
|
||||||
|
or (not row[0] and not row[1] and row[2] == run_id)
|
||||||
|
):
|
||||||
|
return
|
||||||
|
if publication_retry(board, child_task_id, run_id, path=path) is None:
|
||||||
|
raise ValueError("publication retry is unavailable")
|
||||||
|
with _connect(board, path) as connection:
|
||||||
|
changed = connection.execute(
|
||||||
|
"UPDATE publication_retries SET issued_run_id=? WHERE board=? AND child_task_id=? "
|
||||||
|
"AND issued_run_id=''", (run_id, board, child_task_id)
|
||||||
|
).rowcount
|
||||||
|
if changed != 1:
|
||||||
|
with _connect(board, path) as connection:
|
||||||
|
row = connection.execute(
|
||||||
|
"SELECT issued_run_id FROM publication_retries WHERE board=? AND child_task_id=?",
|
||||||
|
(board, child_task_id),
|
||||||
|
).fetchone()
|
||||||
|
if row is None or row[0] != run_id:
|
||||||
|
raise SupervisorStateError("publication retry was already issued")
|
||||||
|
def resolve_publication_retry(board: str, child_task_id: str, run_id: str, *, path: Path | None = None) -> None:
|
||||||
|
"""Mark the one fresh broker-confirmed run as the receipt's trusted resolution."""
|
||||||
|
with _connect(board, path) as connection:
|
||||||
|
changed = connection.execute(
|
||||||
|
"UPDATE publication_retries SET resolved_run_id=? WHERE board=? AND child_task_id=? "
|
||||||
|
"AND issued_run_id=? AND resolved_run_id=''", (run_id, board, child_task_id, run_id)
|
||||||
|
).rowcount
|
||||||
|
if changed != 1:
|
||||||
|
with _connect(board, path) as connection:
|
||||||
|
row = connection.execute(
|
||||||
|
"SELECT issued_run_id,resolved_run_id FROM publication_retries WHERE board=? AND child_task_id=?",
|
||||||
|
(board, child_task_id),
|
||||||
|
).fetchone()
|
||||||
|
if row is None or tuple(row) != (run_id, run_id):
|
||||||
|
raise SupervisorStateError("publication retry cannot be resolved")
|
||||||
|
def reissue_publication_retry(board: str, child_task_id: str, run_id: str, *, path: Path | None = None) -> bool:
|
||||||
|
"""Release one interrupted transient retry once, with a five-minute backoff."""
|
||||||
|
with _connect(board, path) as connection:
|
||||||
|
changed = connection.execute(
|
||||||
|
"UPDATE publication_retries SET issued_run_id='',reissue_count=reissue_count+1,retry_after=?,last_reissued_run_id=? "
|
||||||
|
"WHERE board=? AND child_task_id=? AND issued_run_id=? AND resolved_run_id='' AND reissue_count<1",
|
||||||
|
(int(time.time()) + 300, run_id, board, child_task_id, run_id),
|
||||||
|
).rowcount
|
||||||
|
if changed:
|
||||||
|
return True
|
||||||
|
row = connection.execute(
|
||||||
|
"SELECT issued_run_id,resolved_run_id,reissue_count,last_reissued_run_id FROM publication_retries WHERE board=? AND child_task_id=?",
|
||||||
|
(board, child_task_id),
|
||||||
|
).fetchone()
|
||||||
|
if row is not None and not row[0] and not row[1] and row[3] == run_id and int(row[2]) == 1:
|
||||||
|
return True
|
||||||
|
if row is not None and row[0] == run_id and not row[1] and int(row[2]) >= 1:
|
||||||
|
return False
|
||||||
|
raise SupervisorStateError("publication retry cannot be reissued")
|
||||||
|
def record_publication_retry_provenance(board: str, child_task_id: str, raw_result_sha256: str, *, path: Path | None = None) -> None:
|
||||||
|
"""Keep a bootstrap's immutable blocked result hash beside its retry receipt."""
|
||||||
|
if not isinstance(raw_result_sha256, str) or len(raw_result_sha256) != 64:
|
||||||
|
raise ValueError("publication retry provenance is invalid")
|
||||||
|
with _connect(board, path) as connection:
|
||||||
|
if connection.execute(
|
||||||
|
"SELECT 1 FROM publication_retries WHERE board=? AND child_task_id=?", (board, child_task_id)
|
||||||
|
).fetchone() is None:
|
||||||
|
raise ValueError("publication retry receipt is unavailable")
|
||||||
|
existing = connection.execute(
|
||||||
|
"SELECT raw_result_sha256 FROM publication_retry_provenance WHERE board=? AND child_task_id=?",
|
||||||
|
(board, child_task_id),
|
||||||
|
).fetchone()
|
||||||
|
if existing is None:
|
||||||
|
connection.execute(
|
||||||
|
"INSERT INTO publication_retry_provenance VALUES(?,?,?,?)",
|
||||||
|
(board, child_task_id, raw_result_sha256, "operator-reconstructed-from-private-result-log"),
|
||||||
|
)
|
||||||
|
elif existing[0] != raw_result_sha256:
|
||||||
|
raise ValueError("publication retry provenance conflicts with existing evidence")
|
||||||
|
|||||||
@ -453,15 +453,6 @@ def test_agent_image_runs_execution_safety_patch_and_regressions():
|
|||||||
assert "/opt/hermes/.venv/bin/python /tmp/hermes-execution-safety-regression.py" in (
|
assert "/opt/hermes/.venv/bin/python /tmp/hermes-execution-safety-regression.py" in (
|
||||||
dockerfile
|
dockerfile
|
||||||
)
|
)
|
||||||
assert "COPY services/hermes/scripts/cli_lane_*.py" in dockerfile
|
assert "COPY services/hermes/scripts/*.py /tmp/hermes-lane-regression/" in dockerfile
|
||||||
assert "!services/hermes/scripts/cli_lane_*.py" in dockerignore
|
assert "!services/hermes/scripts/*.py" in dockerignore
|
||||||
for name in (
|
|
||||||
"routing_catalog.py",
|
|
||||||
"supervisor_lineage.py",
|
|
||||||
"supervisor_state.py",
|
|
||||||
):
|
|
||||||
assert f"COPY services/hermes/scripts/{name} /tmp/hermes-lane-regression/" in (
|
|
||||||
dockerfile
|
|
||||||
)
|
|
||||||
assert f"!services/hermes/scripts/{name}" in dockerignore
|
|
||||||
assert "HERMES_CLI_LANE_SOURCE=/tmp/hermes-lane-regression" in dockerfile
|
assert "HERMES_CLI_LANE_SOURCE=/tmp/hermes-lane-regression" in dockerfile
|
||||||
|
|||||||
@ -475,6 +475,11 @@ def test_retention_gc_removes_only_clean_terminal_workspace(tmp_path, monkeypatc
|
|||||||
(workspace / "tracked").write_text("safe\n")
|
(workspace / "tracked").write_text("safe\n")
|
||||||
subprocess.run(["git", "-C", str(workspace), "add", "tracked"], check=True)
|
subprocess.run(["git", "-C", str(workspace), "add", "tracked"], check=True)
|
||||||
subprocess.run(["git", "-C", str(workspace), "commit", "-qm", "initial"], check=True)
|
subprocess.run(["git", "-C", str(workspace), "commit", "-qm", "initial"], check=True)
|
||||||
|
branch = subprocess.run(
|
||||||
|
["git", "-C", str(workspace), "symbolic-ref", "--short", "HEAD"],
|
||||||
|
text=True, capture_output=True, check=True,
|
||||||
|
).stdout.strip()
|
||||||
|
subprocess.run(["git", "-C", str(workspace), "update-ref", f"refs/remotes/origin/{branch}", "HEAD"], check=True)
|
||||||
state_file = tmp_path / "session-state/atlas/t_deadbeef/run-1234.json"
|
state_file = tmp_path / "session-state/atlas/t_deadbeef/run-1234.json"
|
||||||
state_file.parent.mkdir(parents=True)
|
state_file.parent.mkdir(parents=True)
|
||||||
state_file.write_text(
|
state_file.write_text(
|
||||||
|
|||||||
@ -192,6 +192,31 @@ def test_assignment_payload_requires_state_and_native_parents_for_continuations(
|
|||||||
coordinator.assignment_payload(kanban, object(), rejected, "metis")
|
coordinator.assignment_payload(kanban, object(), rejected, "metis")
|
||||||
|
|
||||||
|
|
||||||
|
def test_continuation_resume_uses_only_private_receipt_and_never_model_context(monkeypatch):
|
||||||
|
lineage = coordinator.supervisor_lineage.Lineage(
|
||||||
|
"root", "wt/root", "https://scm.bstein.dev/titan/metis/pulls/7", "metis", "main"
|
||||||
|
)
|
||||||
|
child = {"lineage": lineage, "parent_task_id": "root", "kind": "repair"}
|
||||||
|
receipt = {"source": {"worker_ordinal": 0}, "head": "b" * 40}
|
||||||
|
monkeypatch.setattr(coordinator.supervisor_state, "get_child", lambda *_args: child)
|
||||||
|
monkeypatch.setattr(coordinator.supervisor_state, "get_root", lambda *_args: lineage)
|
||||||
|
monkeypatch.setattr(coordinator.supervisor_state, "publication_retry", lambda *_args: receipt)
|
||||||
|
kanban = SimpleNamespace(
|
||||||
|
build_worker_context=lambda *_args: "model objective must not be used",
|
||||||
|
get_task=lambda *_args: object(), parent_ids=lambda *_args: ["root"],
|
||||||
|
)
|
||||||
|
payload = coordinator.assignment_payload(kanban, object(), task(id="repair"), "metis")
|
||||||
|
assert payload["scm_resume"] is receipt
|
||||||
|
assert payload["context"] == "Republish the mediator-verified completed SCM handoff."
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
coordinator.supervisor_state, "publication_retry",
|
||||||
|
lambda *_args: (_ for _ in ()).throw(ValueError("already issued")),
|
||||||
|
)
|
||||||
|
with pytest.raises(protocol.ProtocolError, match="retry state"):
|
||||||
|
coordinator.assignment_payload(kanban, object(), task(id="repair"), "metis")
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
("parent_ids", "message"),
|
("parent_ids", "message"),
|
||||||
[
|
[
|
||||||
@ -293,6 +318,26 @@ def test_finalize_canonicalizes_integer_run_and_reconcile_does_not_reexecute(
|
|||||||
assert store.active_assignments() == []
|
assert store.active_assignments() == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_finalize_fences_resume_receipt_before_any_live_head_transition(tmp_path, monkeypatch):
|
||||||
|
"""A result winning the add→maintenance race still issues its exact receipt first."""
|
||||||
|
live = task()
|
||||||
|
kanban = install_kanban(monkeypatch, [live], tmp_path)
|
||||||
|
payload = assignment_payload(scm_resume={"source": {"worker_ordinal": 0}})
|
||||||
|
store = protocol.PoolStore(tmp_path / "resume-finalize.db")
|
||||||
|
store.add(binding(), payload)
|
||||||
|
issued = []
|
||||||
|
monkeypatch.setattr(
|
||||||
|
coordinator.supervisor_state, "issue_publication_retry",
|
||||||
|
lambda *values: issued.append(values),
|
||||||
|
)
|
||||||
|
coordinator.Coordinator(MASTER, store).finalize({
|
||||||
|
**binding(), "payload": payload,
|
||||||
|
"result": {"structured": dict(STRUCTURED), "returncode": 0, "node": "node"},
|
||||||
|
})
|
||||||
|
assert issued == [("metis", "t_deadbeef", "23")]
|
||||||
|
assert kanban.completed[-1][1]["expected_run_id"] == 23
|
||||||
|
|
||||||
|
|
||||||
def test_finalize_records_a_verified_root_before_marking_the_task_complete(tmp_path, monkeypatch):
|
def test_finalize_records_a_verified_root_before_marking_the_task_complete(tmp_path, monkeypatch):
|
||||||
live = task()
|
live = task()
|
||||||
kanban = install_kanban(monkeypatch, [live], tmp_path)
|
kanban = install_kanban(monkeypatch, [live], tmp_path)
|
||||||
|
|||||||
@ -97,6 +97,35 @@ def test_dispatch_claims_only_pathless_task_with_safe_assignment_branch(
|
|||||||
assert record["payload"]["branch"] == "wt/t_deadbeef"
|
assert record["payload"]["branch"] == "wt/t_deadbeef"
|
||||||
|
|
||||||
|
|
||||||
|
def test_dispatch_reserves_a_pending_publication_retry_to_its_source_ordinal(
|
||||||
|
tmp_path, monkeypatch
|
||||||
|
):
|
||||||
|
"""A fresh retry cannot strand the retained commit on a different mediator."""
|
||||||
|
item = task(status="ready")
|
||||||
|
install_kanban(monkeypatch, [item], tmp_path)
|
||||||
|
store = protocol.PoolStore(tmp_path / "pool.db")
|
||||||
|
pool = coordinator.Coordinator(MASTER, store)
|
||||||
|
receipt = {"source": {"worker_ordinal": 1}}
|
||||||
|
monkeypatch.setattr(
|
||||||
|
maintenance.supervisor_state, "publication_retry",
|
||||||
|
lambda _board, _task_id, run_id: receipt if run_id == "" else receipt,
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(maintenance.supervisor_state, "issue_publication_retry", lambda *_args: None)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
coordinator, "assignment_payload", lambda *_args: {**assignment_payload(), "scm_resume": receipt},
|
||||||
|
)
|
||||||
|
observed = []
|
||||||
|
|
||||||
|
def claim(_active, _limit, eligible, **kwargs):
|
||||||
|
observed.append((eligible("metis", item), kwargs["priority"]("metis", item)))
|
||||||
|
return [("metis", item.id)]
|
||||||
|
|
||||||
|
monkeypatch.setattr(maintenance.cli_lane_dispatch, "claim_ready", claim)
|
||||||
|
pool.dispatch()
|
||||||
|
assert observed == [(True, 0)]
|
||||||
|
assert store.active_assignments()[0]["worker_ordinal"] == 1
|
||||||
|
|
||||||
|
|
||||||
def test_dispatch_incompatible_unversioned_claim_api_fails_closed(
|
def test_dispatch_incompatible_unversioned_claim_api_fails_closed(
|
||||||
tmp_path, monkeypatch
|
tmp_path, monkeypatch
|
||||||
):
|
):
|
||||||
|
|||||||
@ -156,6 +156,25 @@ def test_noncanonical_run_lease_failure_is_released_without_a_board_call(
|
|||||||
assert kanban.blocked == []
|
assert kanban.blocked == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_lease_expiry_releases_only_a_trusted_publication_retry(tmp_path, monkeypatch):
|
||||||
|
"""An OOM-style lease expiry returns one receipt before parking the native run."""
|
||||||
|
kanban = install_kanban(monkeypatch, [task()], tmp_path)
|
||||||
|
store = pool_store.PoolStore(tmp_path / "resume-lease.db")
|
||||||
|
exact = binding(attempt=3)
|
||||||
|
store.add(exact, assignment_payload(scm_resume={"source": {"worker_ordinal": 0}}))
|
||||||
|
store.offer(0)
|
||||||
|
with store._connect() as connection:
|
||||||
|
connection.execute("UPDATE assignments SET lease_until=1")
|
||||||
|
released = []
|
||||||
|
monkeypatch.setattr(
|
||||||
|
coordinator.supervisor_state, "reissue_publication_retry",
|
||||||
|
lambda *values: released.append(values) or True,
|
||||||
|
)
|
||||||
|
coordinator.Coordinator(MASTER, store).expire_leases()
|
||||||
|
assert released and released[0][2] == "23"
|
||||||
|
assert kanban.blocked[-1][1]["kind"] == "transient"
|
||||||
|
|
||||||
|
|
||||||
def test_a_durable_row_in_any_state_blocks_conflicting_re_adoption(tmp_path, monkeypatch):
|
def test_a_durable_row_in_any_state_blocks_conflicting_re_adoption(tmp_path, monkeypatch):
|
||||||
"""known_runs() covers terminal and retrying rows, not just live ones."""
|
"""known_runs() covers terminal and retrying rows, not just live ones."""
|
||||||
live = task()
|
live = task()
|
||||||
|
|||||||
@ -3,6 +3,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import sys
|
import sys
|
||||||
|
import subprocess
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
@ -16,6 +17,7 @@ sys.path[:0] = [str(SCRIPTS), str(SCM_SCRIPTS)]
|
|||||||
import execution_pool_client as client # noqa: E402
|
import execution_pool_client as client # noqa: E402
|
||||||
import execution_pool_protocol as protocol # noqa: E402
|
import execution_pool_protocol as protocol # noqa: E402
|
||||||
import execution_pool_scm as scm # noqa: E402
|
import execution_pool_scm as scm # noqa: E402
|
||||||
|
import scm_resume_bootstrap as resume_bootstrap # noqa: E402
|
||||||
import receive_pack_scan # noqa: E402
|
import receive_pack_scan # noqa: E402
|
||||||
import scm_broker # noqa: E402
|
import scm_broker # noqa: E402
|
||||||
from testing.tests.test_hermes_scm_broker_support import _receive_command # noqa: E402
|
from testing.tests.test_hermes_scm_broker_support import _receive_command # noqa: E402
|
||||||
@ -213,6 +215,103 @@ def test_remote_head_parsing_ignores_anything_that_is_not_a_branch(
|
|||||||
assert scm._remote_heads(tmp_path, ("feature/x",)) == {"feature/x": "bbb"}
|
assert scm._remote_heads(tmp_path, ("feature/x",)) == {"feature/x": "bbb"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_git_timeout_becomes_a_sanitized_retryable_error(monkeypatch):
|
||||||
|
"""A subprocess timeout must not skip terminal result submission or expose a grant."""
|
||||||
|
def timed_out(*_args, **_kwargs):
|
||||||
|
raise subprocess.TimeoutExpired(
|
||||||
|
["git", "-c", "http.extraHeader=X-Hermes-Task-Grant: secret", "push"], 900
|
||||||
|
)
|
||||||
|
|
||||||
|
monkeypatch.setattr(scm.subprocess, "run", timed_out)
|
||||||
|
with pytest.raises(RuntimeError, match="^SCM command timed out$"):
|
||||||
|
scm._run("push", "hermes-broker")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("detail", "expected"),
|
||||||
|
[
|
||||||
|
(
|
||||||
|
"remote: task branch head changed; fetch and merge before retrying",
|
||||||
|
"task branch changed; fetch and merge before retrying",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"RPC failed; HTTP 403",
|
||||||
|
"SCM broker authorization rejected the branch update; "
|
||||||
|
"inspect task ownership before retrying",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"fatal: unable to access: Connection timed out",
|
||||||
|
"SCM broker transport failed; retry the preserved local commit",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"RPC failed; HTTP 400",
|
||||||
|
"SCM broker rejected the branch update; "
|
||||||
|
"inspect broker or upstream state before retrying",
|
||||||
|
),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_push_failure_guidance_is_sanitized_and_specific(detail, expected):
|
||||||
|
assert scm._push_failure(RuntimeError(detail)) == expected
|
||||||
|
|
||||||
|
|
||||||
|
def test_draft_discovery_finds_a_same_branch_pr_after_the_first_page(monkeypatch):
|
||||||
|
first_page = [{"head": {"ref": f"other-{index}"}, "base": {"ref": "main"}} for index in range(50)]
|
||||||
|
found = "https://scm.bstein.dev/titan/metis/pulls/51"
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
def read(path):
|
||||||
|
calls.append(path)
|
||||||
|
return __import__("json").dumps(
|
||||||
|
first_page if "page=1" in path else [{
|
||||||
|
"number": 51, "html_url": found,
|
||||||
|
"head": {"ref": "wt/t_deadbeef"}, "base": {"ref": "main"},
|
||||||
|
}]
|
||||||
|
).encode()
|
||||||
|
|
||||||
|
monkeypatch.setattr(scm.scm_broker_client, "read", read)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
scm.scm_broker_client, "update_draft",
|
||||||
|
lambda grant, number, title, body: __import__("json").dumps({"html_url": found}).encode(),
|
||||||
|
)
|
||||||
|
assert scm.Boundary._draft("metis", "wt/t_deadbeef", "main", "b" * 40, "safe", "body", "grant") == found
|
||||||
|
assert len(calls) == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_repair_draft_refuses_to_create_a_replacement_when_its_pr_is_not_open(monkeypatch):
|
||||||
|
monkeypatch.setattr(scm.scm_broker_client, "read", lambda _path: b"[]")
|
||||||
|
monkeypatch.setattr(
|
||||||
|
scm.scm_broker_client, "create_draft",
|
||||||
|
lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("replacement PR")),
|
||||||
|
)
|
||||||
|
with pytest.raises(protocol.ProtocolError, match="no existing pull request"):
|
||||||
|
scm.Boundary._draft(
|
||||||
|
"metis", "wt/t_deadbeef", "main", "b" * 40, "repair", "evidence",
|
||||||
|
"grant", existing_only=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_repair_checks_its_open_pr_before_advancing_the_owned_branch(tmp_path, monkeypatch):
|
||||||
|
boundary, exact, _calls, _head = submit_harness(
|
||||||
|
tmp_path, monkeypatch, {"wt/t_deadbeef": "a" * 40}, continuation_kind="repair"
|
||||||
|
)
|
||||||
|
events = []
|
||||||
|
original_run = scm._run
|
||||||
|
|
||||||
|
def run(*arguments, **kwargs):
|
||||||
|
if "push" in arguments:
|
||||||
|
events.append("push")
|
||||||
|
return original_run(*arguments, **kwargs)
|
||||||
|
|
||||||
|
def draft(*_args, **kwargs):
|
||||||
|
events.append("pre" if kwargs.get("refresh") is False else "post")
|
||||||
|
return "https://scm/pulls/3"
|
||||||
|
|
||||||
|
monkeypatch.setattr(scm, "_run", run)
|
||||||
|
monkeypatch.setattr(boundary, "_draft", draft)
|
||||||
|
boundary.submit(exact, {"title": "repair", "body": "evidence"})
|
||||||
|
assert events == ["pre", "push", "post"]
|
||||||
|
|
||||||
|
|
||||||
class FailingSCM:
|
class FailingSCM:
|
||||||
def __init__(self, error):
|
def __init__(self, error):
|
||||||
self.error = error
|
self.error = error
|
||||||
@ -222,6 +321,9 @@ class FailingSCM:
|
|||||||
self.calls += 1
|
self.calls += 1
|
||||||
raise self.error
|
raise self.error
|
||||||
|
|
||||||
|
def resume_artifact(self, _assignment, _request, _structured):
|
||||||
|
return {"bounded": "resume-artifact"}
|
||||||
|
|
||||||
|
|
||||||
def test_a_refused_submission_downgrades_the_result_instead_of_discarding_it():
|
def test_a_refused_submission_downgrades_the_result_instead_of_discarding_it():
|
||||||
failing = FailingSCM(protocol.ProtocolError("every reviewed branch name is taken"))
|
failing = FailingSCM(protocol.ProtocolError("every reviewed branch name is taken"))
|
||||||
@ -248,6 +350,194 @@ def test_a_refused_submission_downgrades_the_result_instead_of_discarding_it():
|
|||||||
assert finished["ack"]["accepted"] is True
|
assert finished["ack"]["accepted"] is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_refused_submission_is_transient_for_pool_recovery():
|
||||||
|
failing = FailingSCM(protocol.ProtocolError("SCM broker rejected the branch update"))
|
||||||
|
boundary = client.ClientBoundary(KEY, failing)
|
||||||
|
boundary.current = assignment(continuation_kind="repair", root_task_id="t_deadbeef")
|
||||||
|
posted = []
|
||||||
|
|
||||||
|
def post(_path, envelope):
|
||||||
|
posted.append(envelope)
|
||||||
|
return protocol.sign_envelope(
|
||||||
|
KEY, "ack", binding(), {"accepted": True, "duplicate": False}
|
||||||
|
)
|
||||||
|
|
||||||
|
boundary._post = post
|
||||||
|
finished = boundary.finish(
|
||||||
|
{
|
||||||
|
"binding": binding(),
|
||||||
|
"payload": {"structured": dict(RESULT), "returncode": 0, "capacity_failure": False},
|
||||||
|
"title": "safe", "body": "evidence",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
assert finished["ack"]["accepted"] is True
|
||||||
|
assert finished["structured"]["status"] == "blocked"
|
||||||
|
terminal = protocol.verify_envelope(KEY, posted[0])["payload"]
|
||||||
|
assert terminal["capacity_failure"] is True
|
||||||
|
assert terminal["scm_resume"] == {"bounded": "resume-artifact"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_ordinary_submission_failure_never_emits_a_continuation_receipt():
|
||||||
|
boundary = client.ClientBoundary(KEY, FailingSCM(protocol.ProtocolError("refused")))
|
||||||
|
boundary.current = assignment()
|
||||||
|
posted = []
|
||||||
|
boundary._post = lambda _path, envelope: posted.append(envelope) or protocol.sign_envelope(
|
||||||
|
KEY, "ack", binding(), {"accepted": True, "duplicate": False}
|
||||||
|
)
|
||||||
|
boundary.finish({"binding": binding(), "payload": {"structured": dict(RESULT), "returncode": 0}, "title": "safe", "body": "evidence"})
|
||||||
|
terminal = protocol.verify_envelope(KEY, posted[0])["payload"]
|
||||||
|
assert terminal["capacity_failure"] is True and "scm_resume" not in terminal
|
||||||
|
|
||||||
|
|
||||||
|
def test_real_resume_receipt_keeps_completed_evidence_after_submission_mutates_result(tmp_path, monkeypatch):
|
||||||
|
"""The signed receipt must not alias the blocked terminal result in memory."""
|
||||||
|
monkeypatch.setattr(scm, "WORKSPACE_ROOT", tmp_path / "workspace")
|
||||||
|
monkeypatch.setattr(scm, "SCM_ROOT", tmp_path / "state")
|
||||||
|
monkeypatch.setattr(scm, "ORDINAL", 0)
|
||||||
|
exact = protocol.sign_envelope(
|
||||||
|
KEY, "assignment", binding(), payload(root_task_id="t_deadbeef", continuation_kind="repair")
|
||||||
|
)
|
||||||
|
destination = scm.workspace_path(exact)
|
||||||
|
destination.mkdir(parents=True)
|
||||||
|
protocol.atomic_json(scm._state_path(exact), {"baseline_sha": "a" * 40})
|
||||||
|
monkeypatch.setattr(scm, "_workspace_identity", lambda *_args: "b" * 40)
|
||||||
|
monkeypatch.setattr(scm, "_run", lambda *args, **_kwargs: "")
|
||||||
|
broker = scm.Boundary(KEY)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
broker, "submit", lambda *_args, **_kwargs: (_ for _ in ()).throw(protocol.ProtocolError("refused"))
|
||||||
|
)
|
||||||
|
boundary = client.ClientBoundary(KEY, broker)
|
||||||
|
boundary.current = exact
|
||||||
|
posted = []
|
||||||
|
boundary._post = lambda _path, envelope: posted.append(envelope) or protocol.sign_envelope(
|
||||||
|
KEY, "ack", binding(), {"accepted": True, "duplicate": False}
|
||||||
|
)
|
||||||
|
boundary.finish({"binding": binding(), "payload": {"structured": dict(RESULT), "returncode": 0}, "title": "safe", "body": "evidence"})
|
||||||
|
result_envelope = next(item for item in posted if item["kind"] == "result")
|
||||||
|
artifact = protocol.verify_envelope(KEY, result_envelope)["payload"]["scm_resume"]
|
||||||
|
assert artifact["structured"]["status"] == "completed"
|
||||||
|
assert artifact["result_digest"] == __import__("hashlib").sha256(
|
||||||
|
protocol.canonical_json({name: artifact[name] for name in ("structured", "title", "body")})
|
||||||
|
).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def test_mediator_bootstrap_recovers_one_unique_completed_log_result(tmp_path, monkeypatch):
|
||||||
|
exact = protocol.sign_envelope(
|
||||||
|
KEY, "assignment", binding(), payload(root_task_id="t_deadbeef", continuation_kind="repair")
|
||||||
|
)
|
||||||
|
log = tmp_path / "session-state" / "metis" / "t_deadbeef" / "42" / ".log"
|
||||||
|
# The live worker uses `<run>.log`; make the same task-bound parent safely.
|
||||||
|
log = log.parent.parent / "42.log"
|
||||||
|
log.parent.mkdir(parents=True)
|
||||||
|
completed = dict(RESULT)
|
||||||
|
log.write_bytes(protocol.canonical_json(completed) + b"\nnoise\n" + protocol.canonical_json(completed) + b"\n")
|
||||||
|
monkeypatch.setattr(resume_bootstrap, "ROOT", tmp_path)
|
||||||
|
assert resume_bootstrap._completed_result(exact) == completed
|
||||||
|
seen = {}
|
||||||
|
monkeypatch.setattr(
|
||||||
|
scm.Boundary, "resume_artifact",
|
||||||
|
lambda _self, assignment, request, structured: seen.update(
|
||||||
|
assignment=assignment, request=request, structured=structured
|
||||||
|
) or {"receipt": "only"},
|
||||||
|
)
|
||||||
|
assert resume_bootstrap.bootstrap(KEY, exact) == {"receipt": "only"}
|
||||||
|
assert seen["structured"] == completed and seen["request"]["title"] == completed["summary"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_mediator_bootstrap_rejects_a_symlinked_log_ancestor(tmp_path, monkeypatch):
|
||||||
|
exact = protocol.sign_envelope(KEY, "assignment", binding(), payload())
|
||||||
|
outside = tmp_path / "outside"
|
||||||
|
outside.mkdir()
|
||||||
|
session_root = tmp_path / "session-state"
|
||||||
|
session_root.mkdir()
|
||||||
|
(session_root / "metis").symlink_to(outside, target_is_directory=True)
|
||||||
|
monkeypatch.setattr(resume_bootstrap, "ROOT", tmp_path)
|
||||||
|
with pytest.raises(protocol.ProtocolError, match="unsafe"):
|
||||||
|
resume_bootstrap._completed_result(exact)
|
||||||
|
|
||||||
|
|
||||||
|
def test_publication_lease_signs_the_exact_binding_and_stops_cleanly():
|
||||||
|
boundary = client.ClientBoundary(KEY, FailingSCM(protocol.ProtocolError("unused")))
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
def post(path, envelope):
|
||||||
|
calls.append((path, envelope))
|
||||||
|
return protocol.sign_envelope(KEY, "ack", binding(), {"accepted": True})
|
||||||
|
|
||||||
|
boundary._post = post
|
||||||
|
checkpoint, close = boundary._publication_lease(binding())
|
||||||
|
checkpoint()
|
||||||
|
close()
|
||||||
|
assert len(calls) == 1 and calls[0][0] == "/v1/heartbeat"
|
||||||
|
verified = protocol.verify_envelope(KEY, calls[0][1], expected_kind="heartbeat")
|
||||||
|
assert {name: verified[name] for name in binding()} == binding()
|
||||||
|
|
||||||
|
|
||||||
|
def test_publication_lease_renews_during_a_slow_scm_step_without_sleeping(monkeypatch):
|
||||||
|
boundary = client.ClientBoundary(KEY, FailingSCM(protocol.ProtocolError("unused")))
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
class Event:
|
||||||
|
def __init__(self):
|
||||||
|
self.waits = 0
|
||||||
|
|
||||||
|
def wait(self, _seconds):
|
||||||
|
self.waits += 1
|
||||||
|
return self.waits > 1
|
||||||
|
|
||||||
|
def set(self):
|
||||||
|
return None
|
||||||
|
|
||||||
|
class Thread:
|
||||||
|
def __init__(self, *, target, daemon):
|
||||||
|
self.target = target
|
||||||
|
|
||||||
|
def start(self):
|
||||||
|
self.target()
|
||||||
|
|
||||||
|
def join(self, timeout):
|
||||||
|
assert timeout == 1
|
||||||
|
|
||||||
|
monkeypatch.setattr(client.threading, "Event", Event)
|
||||||
|
monkeypatch.setattr(client.threading, "Thread", Thread)
|
||||||
|
boundary._post = lambda path, envelope: calls.append((path, envelope)) or protocol.sign_envelope(
|
||||||
|
KEY, "ack", binding(), {"accepted": True}
|
||||||
|
)
|
||||||
|
_checkpoint, close = boundary._publication_lease(binding())
|
||||||
|
close()
|
||||||
|
assert [path for path, _envelope in calls] == ["/v1/heartbeat", "/v1/heartbeat"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_lost_publication_checkpoint_prevents_the_push(tmp_path, monkeypatch):
|
||||||
|
boundary, exact, calls, _head = submit_harness(tmp_path, monkeypatch, {"wt/t_deadbeef": "a" * 40})
|
||||||
|
count = 0
|
||||||
|
|
||||||
|
def checkpoint():
|
||||||
|
nonlocal count
|
||||||
|
count += 1
|
||||||
|
if count == 3:
|
||||||
|
raise protocol.ProtocolError("SCM publication lease was lost")
|
||||||
|
|
||||||
|
with pytest.raises(protocol.ProtocolError, match="lease was lost"):
|
||||||
|
boundary.submit(exact, {"title": "safe", "body": "evidence"}, checkpoint=checkpoint)
|
||||||
|
assert not any("push" in call for call in calls)
|
||||||
|
|
||||||
|
|
||||||
|
def test_worker_cannot_spoof_a_transient_publication_retry_result():
|
||||||
|
boundary = client.ClientBoundary(KEY, FailingSCM(protocol.ProtocolError("unused")))
|
||||||
|
boundary.current = assignment(payload={"scm_resume": {}})
|
||||||
|
posted = []
|
||||||
|
boundary._post = lambda _path, envelope: posted.append(envelope) or protocol.sign_envelope(
|
||||||
|
KEY, "ack", binding(), {"accepted": True, "duplicate": False}
|
||||||
|
)
|
||||||
|
blocked = {**RESULT, "status": "blocked", "blockers": ["policy refusal"]}
|
||||||
|
boundary.finish({"binding": binding(), "payload": {
|
||||||
|
"structured": blocked, "returncode": 1, "publication_retry_transient": True,
|
||||||
|
}})
|
||||||
|
terminal = protocol.verify_envelope(KEY, posted[0])["payload"]
|
||||||
|
assert "publication_retry_transient" not in terminal
|
||||||
|
|
||||||
|
|
||||||
def test_a_successful_submission_records_both_the_draft_and_the_exact_branch():
|
def test_a_successful_submission_records_both_the_draft_and_the_exact_branch():
|
||||||
class Recording:
|
class Recording:
|
||||||
def submit(self, _assignment, _request):
|
def submit(self, _assignment, _request):
|
||||||
|
|||||||
@ -248,6 +248,11 @@ def init_clean_repo(path):
|
|||||||
(path / "tracked").write_text("safe")
|
(path / "tracked").write_text("safe")
|
||||||
subprocess.run(["git", "-C", str(path), "add", "tracked"], check=True)
|
subprocess.run(["git", "-C", str(path), "add", "tracked"], check=True)
|
||||||
subprocess.run(["git", "-C", str(path), "commit", "-qm", "initial"], check=True)
|
subprocess.run(["git", "-C", str(path), "commit", "-qm", "initial"], check=True)
|
||||||
|
branch = subprocess.run(
|
||||||
|
["git", "-C", str(path), "symbolic-ref", "--short", "HEAD"], text=True,
|
||||||
|
capture_output=True, check=True,
|
||||||
|
).stdout.strip()
|
||||||
|
subprocess.run(["git", "-C", str(path), "update-ref", f"refs/remotes/origin/{branch}", "HEAD"], check=True)
|
||||||
|
|
||||||
|
|
||||||
def test_retention_skips_young_outside_dirty_and_symlink_workspaces(tmp_path, monkeypatch):
|
def test_retention_skips_young_outside_dirty_and_symlink_workspaces(tmp_path, monkeypatch):
|
||||||
@ -278,6 +283,23 @@ def test_retention_skips_young_outside_dirty_and_symlink_workspaces(tmp_path, mo
|
|||||||
assert link_state.exists()
|
assert link_state.exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_retention_keeps_a_clean_unpublished_commit(tmp_path, monkeypatch):
|
||||||
|
"""A terminal source checkout remains available for mediator publication retry."""
|
||||||
|
monkeypatch.setattr(worker, "ROOT", tmp_path)
|
||||||
|
monkeypatch.setattr(worker, "RETENTION_SECONDS", 3600)
|
||||||
|
workspace = tmp_path / "runs/metis/retry/23"
|
||||||
|
workspace.mkdir(parents=True)
|
||||||
|
init_clean_repo(workspace)
|
||||||
|
branch = subprocess.run(
|
||||||
|
["git", "-C", str(workspace), "symbolic-ref", "--short", "HEAD"], text=True,
|
||||||
|
capture_output=True, check=True,
|
||||||
|
).stdout.strip()
|
||||||
|
subprocess.run(["git", "-C", str(workspace), "update-ref", "-d", f"refs/remotes/origin/{branch}"], check=True)
|
||||||
|
state = write_gc_state(tmp_path, "retry", workspace, 1)
|
||||||
|
assert worker.garbage_collect(now=10_000) == 0
|
||||||
|
assert workspace.exists() and state.exists()
|
||||||
|
|
||||||
|
|
||||||
def test_readiness_checks_ordinal_paths_credentials_and_mediator(tmp_path, monkeypatch):
|
def test_readiness_checks_ordinal_paths_credentials_and_mediator(tmp_path, monkeypatch):
|
||||||
worker_root = tmp_path / "worker"
|
worker_root = tmp_path / "worker"
|
||||||
data_root = tmp_path / "data"
|
data_root = tmp_path / "data"
|
||||||
@ -292,10 +314,11 @@ def test_readiness_checks_ordinal_paths_credentials_and_mediator(tmp_path, monke
|
|||||||
monkeypatch.setattr(worker.cli_lane_runner, "DATA_ROOT", data_root)
|
monkeypatch.setattr(worker.cli_lane_runner, "DATA_ROOT", data_root)
|
||||||
monkeypatch.setattr(worker.cli_lane_runner, "RESULT_SCHEMA_PATH", schema)
|
monkeypatch.setattr(worker.cli_lane_runner, "RESULT_SCHEMA_PATH", schema)
|
||||||
monkeypatch.setenv("CLAUDE_CODE_OAUTH_TOKEN_FILE", str(claude_token))
|
monkeypatch.setenv("CLAUDE_CODE_OAUTH_TOKEN_FILE", str(claude_token))
|
||||||
polled = []
|
monkeypatch.setattr(
|
||||||
monkeypatch.setattr(worker, "_poll", lambda: polled.append(True))
|
worker, "_poll", lambda: pytest.fail("readiness must not poll the mediator")
|
||||||
|
)
|
||||||
worker.readiness()
|
worker.readiness()
|
||||||
assert polled and json.loads(schema.read_text()) == worker.cli_lane_runner.RESULT_SCHEMA
|
assert json.loads(schema.read_text()) == worker.cli_lane_runner.RESULT_SCHEMA
|
||||||
|
|
||||||
monkeypatch.setattr(worker, "ORDINAL", 3)
|
monkeypatch.setattr(worker, "ORDINAL", 3)
|
||||||
with pytest.raises(protocol.ProtocolError, match="ordinal"):
|
with pytest.raises(protocol.ProtocolError, match="ordinal"):
|
||||||
@ -310,32 +333,78 @@ def test_readiness_checks_ordinal_paths_credentials_and_mediator(tmp_path, monke
|
|||||||
worker.readiness()
|
worker.readiness()
|
||||||
|
|
||||||
|
|
||||||
def test_readiness_defers_transient_ordinal_lease_conflict(tmp_path, monkeypatch):
|
def test_main_defers_rejected_or_malformed_poll_before_valid_assignment(monkeypatch):
|
||||||
worker_root = tmp_path / "worker"
|
monkeypatch.setattr(worker, "readiness", lambda: None)
|
||||||
data_root = tmp_path / "data"
|
monkeypatch.setattr(worker, "garbage_collect", lambda: 0)
|
||||||
claude_token = tmp_path / "claude-oauth/token"
|
expected = assignment()
|
||||||
for path in (worker_root, worker_root / "provider-state", data_root):
|
polls = iter((
|
||||||
path.mkdir(parents=True, exist_ok=True)
|
urllib.error.HTTPError("http://mediator", 409, "Conflict", {}, None),
|
||||||
claude_token.parent.mkdir()
|
protocol.ProtocolError("bad wire"),
|
||||||
claude_token.write_text("setup-token")
|
expected,
|
||||||
monkeypatch.setattr(worker, "ORDINAL", 1)
|
))
|
||||||
monkeypatch.setattr(worker, "ROOT", worker_root)
|
executed = []
|
||||||
monkeypatch.setattr(worker.cli_lane_runner, "DATA_ROOT", data_root)
|
|
||||||
monkeypatch.setattr(
|
|
||||||
worker.cli_lane_runner, "RESULT_SCHEMA_PATH", tmp_path / "schema/result.json"
|
|
||||||
)
|
|
||||||
monkeypatch.setenv("CLAUDE_CODE_OAUTH_TOKEN_FILE", str(claude_token))
|
|
||||||
|
|
||||||
def conflict():
|
def poll():
|
||||||
raise urllib.error.HTTPError("http://mediator", 409, "Conflict", {}, None)
|
value = next(polls)
|
||||||
|
if isinstance(value, Exception):
|
||||||
|
raise value
|
||||||
|
return value
|
||||||
|
|
||||||
monkeypatch.setattr(worker, "_poll", conflict)
|
def execute(value):
|
||||||
worker.readiness()
|
executed.append(value)
|
||||||
|
raise StopIteration("stop")
|
||||||
|
|
||||||
def unauthorized():
|
monkeypatch.setattr(worker, "_poll", poll)
|
||||||
raise urllib.error.HTTPError("http://mediator", 401, "Unauthorized", {}, None)
|
monkeypatch.setattr(worker, "execute", execute)
|
||||||
|
monkeypatch.setattr(worker.time, "sleep", lambda _delay: None)
|
||||||
|
with pytest.raises(StopIteration, match="stop"):
|
||||||
|
worker.main()
|
||||||
|
assert executed == [expected]
|
||||||
|
|
||||||
monkeypatch.setattr(worker, "_poll", unauthorized)
|
|
||||||
with pytest.raises(urllib.error.HTTPError) as error:
|
def test_publication_retry_calls_only_mediator_and_never_starts_a_model(monkeypatch):
|
||||||
worker.readiness()
|
"""A fresh run republishes retained evidence without a checkout or provider session."""
|
||||||
assert error.value.code == 401
|
structured = {
|
||||||
|
"status": "completed", "summary": "Replace cache literals.",
|
||||||
|
"changed_files": ["internal/k8s/job_manifests.go"], "tests_run": ["go test ./..."],
|
||||||
|
"artifacts": [], "findings": [], "blockers": [],
|
||||||
|
}
|
||||||
|
resume = {"structured": structured, "title": "Repair", "body": "evidence", "head": "b" * 40}
|
||||||
|
item = assignment(payload={"scm_resume": resume})
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
def client(operation, **values):
|
||||||
|
calls.append((operation, values))
|
||||||
|
if operation == "resume":
|
||||||
|
return {"scm_submission": {"branch": "wt/t_deadbeef", "pull_request": "https://scm/pulls/3", "head": "b" * 40}}
|
||||||
|
return {"ack": {"accepted": True}}
|
||||||
|
|
||||||
|
monkeypatch.setattr(worker, "_client", client)
|
||||||
|
monkeypatch.setattr(worker, "_bind_provider_sessions", lambda *_args: pytest.fail("must not bind provider state"))
|
||||||
|
monkeypatch.setattr(worker.cli_lane_runner, "run_provider", lambda *_args: pytest.fail("must not invoke a model"))
|
||||||
|
worker.execute(item)
|
||||||
|
assert [name for name, _values in calls] == ["resume", "finish"]
|
||||||
|
terminal = calls[1][1]["payload"]
|
||||||
|
assert terminal["structured"]["status"] == "completed"
|
||||||
|
assert "preserved-head:" + "b" * 40 in terminal["structured"]["artifacts"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_publication_retry_policy_error_is_a_visible_nontransient_block(monkeypatch):
|
||||||
|
"""Only the mediator's explicit transient response can release a receipt."""
|
||||||
|
structured = {"status": "completed", "summary": "Repair", "changed_files": [], "tests_run": [], "artifacts": [], "findings": [], "blockers": []}
|
||||||
|
item = assignment(payload={"scm_resume": {"structured": structured, "title": "Repair", "body": "evidence", "head": "b" * 40}})
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
def client(operation, **values):
|
||||||
|
calls.append((operation, values))
|
||||||
|
if operation == "resume":
|
||||||
|
raise protocol.ProtocolError("policy denied")
|
||||||
|
return {"ack": {"accepted": True}}
|
||||||
|
|
||||||
|
monkeypatch.setattr(worker, "_client", client)
|
||||||
|
monkeypatch.setattr(worker.cli_lane_runner, "run_provider", lambda *_args: pytest.fail("must not invoke a model"))
|
||||||
|
worker.execute(item)
|
||||||
|
terminal = calls[-1][1]["payload"]
|
||||||
|
assert [name for name, _ in calls] == ["resume", "finish"]
|
||||||
|
assert terminal["capacity_failure"] is False
|
||||||
|
assert terminal["structured"]["status"] == "blocked"
|
||||||
|
|||||||
@ -6,6 +6,7 @@ import json
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
|
import pytest
|
||||||
import yaml
|
import yaml
|
||||||
|
|
||||||
from testing.tests.test_hermes_cli_support import HERMES, _load
|
from testing.tests.test_hermes_cli_support import HERMES, _load
|
||||||
@ -15,6 +16,7 @@ ROOT = Path(__file__).parents[2]
|
|||||||
sys.path.insert(0, str(HERMES / "scm-common/scripts"))
|
sys.path.insert(0, str(HERMES / "scm-common/scripts"))
|
||||||
state = _load("supervisor_state")
|
state = _load("supervisor_state")
|
||||||
seed = _load("seed_legacy_scm_roots")
|
seed = _load("seed_legacy_scm_roots")
|
||||||
|
retry = sys.modules["publication_retry"]
|
||||||
|
|
||||||
|
|
||||||
class NativeKanban:
|
class NativeKanban:
|
||||||
@ -89,3 +91,60 @@ def test_seed_rerun_preserves_same_owned_newer_state_and_approval(tmp_path, monk
|
|||||||
assert connection.execute(
|
assert connection.execute(
|
||||||
"SELECT ready_for_human_merge,ready_commit FROM supervisor_roots"
|
"SELECT ready_for_human_merge,ready_commit FROM supervisor_roots"
|
||||||
).fetchone() == (1, newer)
|
).fetchone() == (1, newer)
|
||||||
|
|
||||||
|
|
||||||
|
def test_publication_retry_is_bound_once_and_never_falls_back_to_a_model(tmp_path, monkeypatch):
|
||||||
|
"""A mediator receipt can power one fresh ordinal-pinned publication only."""
|
||||||
|
board, root_id, child_id, baseline, head = (
|
||||||
|
"soteria", "t_root", "t_child", "a" * 40, "b" * 40
|
||||||
|
)
|
||||||
|
lineage = state.Lineage(
|
||||||
|
root_id, "hermes-repair/cache", "https://scm.bstein.dev/titan/soteria/pulls/3",
|
||||||
|
"soteria", "main",
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(state, "KANBAN_ROOT", tmp_path / "boards")
|
||||||
|
state.record_submission(board, root_id, lineage, baseline)
|
||||||
|
state.record_child(board, child_id, root_id, root_id, "repair", baseline, "replace cache literals")
|
||||||
|
structured = {
|
||||||
|
"status": "completed", "summary": "Replace cache literals.",
|
||||||
|
"changed_files": ["internal/k8s/job_manifests.go"], "tests_run": ["go test ./..."],
|
||||||
|
"artifacts": [], "findings": [], "blockers": [],
|
||||||
|
}
|
||||||
|
title, body = "Replace cache literals", "verified completion evidence"
|
||||||
|
receipt = {
|
||||||
|
"source": {
|
||||||
|
"board": board, "task_id": child_id, "run_id": "8", "worker_ordinal": 0,
|
||||||
|
"attempt": 1, "root_task_id": root_id,
|
||||||
|
"repo_url": "https://scm.bstein.dev/titan/soteria.git",
|
||||||
|
"branch": lineage.branch, "base_branch": "main",
|
||||||
|
},
|
||||||
|
"baseline_sha": baseline, "head": head, "title": title, "body": body,
|
||||||
|
"structured": structured, "result_digest": retry.receipt_digest(structured, title, body),
|
||||||
|
}
|
||||||
|
binding = {name: receipt["source"][name] for name in (
|
||||||
|
"board", "task_id", "run_id", "worker_ordinal", "attempt"
|
||||||
|
)}
|
||||||
|
state.record_publication_retry(board, child_id, binding, receipt)
|
||||||
|
assert state.publication_retry(board, child_id, "")["head"] == head
|
||||||
|
state.issue_publication_retry(board, child_id, "9")
|
||||||
|
assert state.publication_retry(board, child_id, "9")["source"]["worker_ordinal"] == 0
|
||||||
|
with pytest.raises(retry.PublicationRetryError, match="already issued"):
|
||||||
|
state.publication_retry(board, child_id, "10")
|
||||||
|
monkeypatch.setattr(state.time, "time", lambda: 1_000)
|
||||||
|
assert state.reissue_publication_retry(board, child_id, "9") is True
|
||||||
|
# A failed native park can replay the same terminal result without spending
|
||||||
|
# another retry or extending the backoff.
|
||||||
|
assert state.reissue_publication_retry(board, child_id, "9") is True
|
||||||
|
state.issue_publication_retry(board, child_id, "9")
|
||||||
|
with pytest.raises(retry.PublicationRetryError, match="backoff"):
|
||||||
|
state.publication_retry(board, child_id, "")
|
||||||
|
monkeypatch.setattr(state.time, "time", lambda: 1_301)
|
||||||
|
assert state.publication_retry(board, child_id, "")["head"] == head
|
||||||
|
state.issue_publication_retry(board, child_id, "10")
|
||||||
|
# Completion advances the trusted root head before the terminal receipt is
|
||||||
|
# marked resolved; a finalize replay must not revalidate its old baseline.
|
||||||
|
state.record_submission(board, child_id, lineage, head)
|
||||||
|
state.issue_publication_retry(board, child_id, "10")
|
||||||
|
state.resolve_publication_retry(board, child_id, "10")
|
||||||
|
state.resolve_publication_retry(board, child_id, "10")
|
||||||
|
assert state.publication_retry(board, child_id, "") is None
|
||||||
|
|||||||
@ -137,9 +137,12 @@ def test_post_dispatches_control_and_git_or_rejects(monkeypatch):
|
|||||||
assert calls == ["control", "git"]
|
assert calls == ["control", "git"]
|
||||||
|
|
||||||
rejected = _handler(broker, path="/v1/metadata", body=b"{}")
|
rejected = _handler(broker, path="/v1/metadata", body=b"{}")
|
||||||
|
events = []
|
||||||
|
monkeypatch.setattr(broker, "log_rejection", lambda phase, error: events.append((phase, type(error).__name__)))
|
||||||
rejected._control = lambda: (_ for _ in ()).throw(broker.PolicyError("no"))
|
rejected._control = lambda: (_ for _ in ()).throw(broker.PolicyError("no"))
|
||||||
rejected.do_POST()
|
rejected.do_POST()
|
||||||
assert json.loads(rejected.wfile.getvalue()) == {"error": "request rejected"}
|
assert json.loads(rejected.wfile.getvalue()) == {"error": "request rejected"}
|
||||||
|
assert events == [("control", "PolicyError")]
|
||||||
|
|
||||||
|
|
||||||
def test_control_metadata_and_draft_fields(monkeypatch):
|
def test_control_metadata_and_draft_fields(monkeypatch):
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user