2026-09-13 15:04:01 -05:00
|
|
|
"""Continuing-task submission contracts against the signed Atlas SCM broker."""
|
hermes: make pool lease recovery and release isolation safe
Independent review t_5975c06a blocked this branch on a P1: a Kanban write that
failed while a lease expired left a `lease_failed` row that was invisible to
every pass, immortal to garbage collection, and fatal to the coordinator. It
poisoned `reconcile()` forever with a conflicting-duplicate primary key,
produced a spurious capability `block_task` from `dispatch()`, and -- because
startup maintenance ran unguarded before the port bound, against a store on a
PVC -- crash-looped the coordinator with no automatic recovery.
`lease_failed` is now a retryable state that every maintenance pass drains, and
a row only reaches a terminal state on authoritative evidence about its exact
Kanban run, so nothing is collected before its outcome is known and nothing is
silently dropped. Each row, task, and board is processed in isolation, and a
coordinator-side fault is never converted into a Kanban mutation. Startup runs
through the same guarded cycle as the steady-state loop.
The wire protocol and the durable store are now separate modules, and the
maintenance passes moved out of the coordinator, so each file stays under the
managed line ceiling with room for the recovery logic.
Also closes three consequential handoff risks the same review raised:
* mediator-N pinned itself hard to worker-N while sharing a ReadWriteOnce
claim, so a drain or preemption that moved only the lower-priority worker
deadlocked the ordinal on Multi-Attach until an operator deleted a Pod. The
shared workspace is now ReadWriteMany (as the hermes-chat tenant workspaces
already are on the same class), colocation is a preference, and the mediator
shares the worker's preemption priority, so each Pod reschedules on its own.
* the broker permits only branch creation, so a retry that added commits could
never submit and the run's work was discarded with the failure. Submission
now targets a fresh attempt- or content-scoped ref in the same reviewed
namespace -- never an update -- and is idempotent under replay. A refused
submission downgrades the result and says why instead of unwinding the run.
* the provider CLIs were reinstalled into an emptyDir on every Pod start inside
the 10m Flux health window for the whole hermes app. They now install once
per pinned version onto a durable volume, re-verified against the real
binaries and time-bounded, and the best-effort pool no longer gates the
health of the app its dependents wait on.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 16:31:15 +00:00
|
|
|
|
|
|
|
|
from __future__ import annotations
|
|
|
|
|
|
|
|
|
|
import sys
|
2026-09-13 17:54:54 -05:00
|
|
|
import subprocess
|
hermes: make pool lease recovery and release isolation safe
Independent review t_5975c06a blocked this branch on a P1: a Kanban write that
failed while a lease expired left a `lease_failed` row that was invisible to
every pass, immortal to garbage collection, and fatal to the coordinator. It
poisoned `reconcile()` forever with a conflicting-duplicate primary key,
produced a spurious capability `block_task` from `dispatch()`, and -- because
startup maintenance ran unguarded before the port bound, against a store on a
PVC -- crash-looped the coordinator with no automatic recovery.
`lease_failed` is now a retryable state that every maintenance pass drains, and
a row only reaches a terminal state on authoritative evidence about its exact
Kanban run, so nothing is collected before its outcome is known and nothing is
silently dropped. Each row, task, and board is processed in isolation, and a
coordinator-side fault is never converted into a Kanban mutation. Startup runs
through the same guarded cycle as the steady-state loop.
The wire protocol and the durable store are now separate modules, and the
maintenance passes moved out of the coordinator, so each file stays under the
managed line ceiling with room for the recovery logic.
Also closes three consequential handoff risks the same review raised:
* mediator-N pinned itself hard to worker-N while sharing a ReadWriteOnce
claim, so a drain or preemption that moved only the lower-priority worker
deadlocked the ordinal on Multi-Attach until an operator deleted a Pod. The
shared workspace is now ReadWriteMany (as the hermes-chat tenant workspaces
already are on the same class), colocation is a preference, and the mediator
shares the worker's preemption priority, so each Pod reschedules on its own.
* the broker permits only branch creation, so a retry that added commits could
never submit and the run's work was discarded with the failure. Submission
now targets a fresh attempt- or content-scoped ref in the same reviewed
namespace -- never an update -- and is idempotent under replay. A refused
submission downgrades the result and says why instead of unwinding the run.
* the provider CLIs were reinstalled into an emptyDir on every Pod start inside
the 10m Flux health window for the whole hermes app. They now install once
per pinned version onto a durable volume, re-verified against the real
binaries and time-bounded, and the best-effort pool no longer gates the
health of the app its dependents wait on.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 16:31:15 +00:00
|
|
|
from pathlib import Path
|
|
|
|
|
|
|
|
|
|
import pytest
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
ROOT = Path(__file__).parents[2]
|
|
|
|
|
SCRIPTS = ROOT / "services/hermes/scripts"
|
|
|
|
|
SCM_SCRIPTS = ROOT / "services/hermes/scm-common/scripts"
|
|
|
|
|
sys.path[:0] = [str(SCRIPTS), str(SCM_SCRIPTS)]
|
|
|
|
|
|
|
|
|
|
import execution_pool_client as client # noqa: E402
|
2026-09-13 18:30:05 -05:00
|
|
|
import gitea_api_policy as gitea_policy # noqa: E402
|
hermes: make pool lease recovery and release isolation safe
Independent review t_5975c06a blocked this branch on a P1: a Kanban write that
failed while a lease expired left a `lease_failed` row that was invisible to
every pass, immortal to garbage collection, and fatal to the coordinator. It
poisoned `reconcile()` forever with a conflicting-duplicate primary key,
produced a spurious capability `block_task` from `dispatch()`, and -- because
startup maintenance ran unguarded before the port bound, against a store on a
PVC -- crash-looped the coordinator with no automatic recovery.
`lease_failed` is now a retryable state that every maintenance pass drains, and
a row only reaches a terminal state on authoritative evidence about its exact
Kanban run, so nothing is collected before its outcome is known and nothing is
silently dropped. Each row, task, and board is processed in isolation, and a
coordinator-side fault is never converted into a Kanban mutation. Startup runs
through the same guarded cycle as the steady-state loop.
The wire protocol and the durable store are now separate modules, and the
maintenance passes moved out of the coordinator, so each file stays under the
managed line ceiling with room for the recovery logic.
Also closes three consequential handoff risks the same review raised:
* mediator-N pinned itself hard to worker-N while sharing a ReadWriteOnce
claim, so a drain or preemption that moved only the lower-priority worker
deadlocked the ordinal on Multi-Attach until an operator deleted a Pod. The
shared workspace is now ReadWriteMany (as the hermes-chat tenant workspaces
already are on the same class), colocation is a preference, and the mediator
shares the worker's preemption priority, so each Pod reschedules on its own.
* the broker permits only branch creation, so a retry that added commits could
never submit and the run's work was discarded with the failure. Submission
now targets a fresh attempt- or content-scoped ref in the same reviewed
namespace -- never an update -- and is idempotent under replay. A refused
submission downgrades the result and says why instead of unwinding the run.
* the provider CLIs were reinstalled into an emptyDir on every Pod start inside
the 10m Flux health window for the whole hermes app. They now install once
per pinned version onto a durable volume, re-verified against the real
binaries and time-bounded, and the best-effort pool no longer gates the
health of the app its dependents wait on.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 16:31:15 +00:00
|
|
|
import execution_pool_protocol as protocol # noqa: E402
|
|
|
|
|
import execution_pool_scm as scm # noqa: E402
|
2026-09-13 17:54:54 -05:00
|
|
|
import scm_resume_bootstrap as resume_bootstrap # noqa: E402
|
2026-08-18 02:00:19 -03:00
|
|
|
import receive_pack_scan # noqa: E402
|
hermes: make pool lease recovery and release isolation safe
Independent review t_5975c06a blocked this branch on a P1: a Kanban write that
failed while a lease expired left a `lease_failed` row that was invisible to
every pass, immortal to garbage collection, and fatal to the coordinator. It
poisoned `reconcile()` forever with a conflicting-duplicate primary key,
produced a spurious capability `block_task` from `dispatch()`, and -- because
startup maintenance ran unguarded before the port bound, against a store on a
PVC -- crash-looped the coordinator with no automatic recovery.
`lease_failed` is now a retryable state that every maintenance pass drains, and
a row only reaches a terminal state on authoritative evidence about its exact
Kanban run, so nothing is collected before its outcome is known and nothing is
silently dropped. Each row, task, and board is processed in isolation, and a
coordinator-side fault is never converted into a Kanban mutation. Startup runs
through the same guarded cycle as the steady-state loop.
The wire protocol and the durable store are now separate modules, and the
maintenance passes moved out of the coordinator, so each file stays under the
managed line ceiling with room for the recovery logic.
Also closes three consequential handoff risks the same review raised:
* mediator-N pinned itself hard to worker-N while sharing a ReadWriteOnce
claim, so a drain or preemption that moved only the lower-priority worker
deadlocked the ordinal on Multi-Attach until an operator deleted a Pod. The
shared workspace is now ReadWriteMany (as the hermes-chat tenant workspaces
already are on the same class), colocation is a preference, and the mediator
shares the worker's preemption priority, so each Pod reschedules on its own.
* the broker permits only branch creation, so a retry that added commits could
never submit and the run's work was discarded with the failure. Submission
now targets a fresh attempt- or content-scoped ref in the same reviewed
namespace -- never an update -- and is idempotent under replay. A refused
submission downgrades the result and says why instead of unwinding the run.
* the provider CLIs were reinstalled into an emptyDir on every Pod start inside
the 10m Flux health window for the whole hermes app. They now install once
per pinned version onto a durable volume, re-verified against the real
binaries and time-bounded, and the best-effort pool no longer gates the
health of the app its dependents wait on.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 16:31:15 +00:00
|
|
|
import scm_broker # noqa: E402
|
2026-08-18 02:00:19 -03:00
|
|
|
from testing.tests.test_hermes_scm_broker_support import _receive_command # noqa: E402
|
hermes: make pool lease recovery and release isolation safe
Independent review t_5975c06a blocked this branch on a P1: a Kanban write that
failed while a lease expired left a `lease_failed` row that was invisible to
every pass, immortal to garbage collection, and fatal to the coordinator. It
poisoned `reconcile()` forever with a conflicting-duplicate primary key,
produced a spurious capability `block_task` from `dispatch()`, and -- because
startup maintenance ran unguarded before the port bound, against a store on a
PVC -- crash-looped the coordinator with no automatic recovery.
`lease_failed` is now a retryable state that every maintenance pass drains, and
a row only reaches a terminal state on authoritative evidence about its exact
Kanban run, so nothing is collected before its outcome is known and nothing is
silently dropped. Each row, task, and board is processed in isolation, and a
coordinator-side fault is never converted into a Kanban mutation. Startup runs
through the same guarded cycle as the steady-state loop.
The wire protocol and the durable store are now separate modules, and the
maintenance passes moved out of the coordinator, so each file stays under the
managed line ceiling with room for the recovery logic.
Also closes three consequential handoff risks the same review raised:
* mediator-N pinned itself hard to worker-N while sharing a ReadWriteOnce
claim, so a drain or preemption that moved only the lower-priority worker
deadlocked the ordinal on Multi-Attach until an operator deleted a Pod. The
shared workspace is now ReadWriteMany (as the hermes-chat tenant workspaces
already are on the same class), colocation is a preference, and the mediator
shares the worker's preemption priority, so each Pod reschedules on its own.
* the broker permits only branch creation, so a retry that added commits could
never submit and the run's work was discarded with the failure. Submission
now targets a fresh attempt- or content-scoped ref in the same reviewed
namespace -- never an update -- and is idempotent under replay. A refused
submission downgrades the result and says why instead of unwinding the run.
* the provider CLIs were reinstalled into an emptyDir on every Pod start inside
the 10m Flux health window for the whole hermes app. They now install once
per pinned version onto a durable volume, re-verified against the real
binaries and time-bounded, and the best-effort pool no longer gates the
health of the app its dependents wait on.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 16:31:15 +00:00
|
|
|
from testing.tests.test_hermes_execution_pool_mediator import ( # noqa: E402
|
|
|
|
|
KEY,
|
|
|
|
|
RESULT,
|
|
|
|
|
assignment,
|
|
|
|
|
binding,
|
|
|
|
|
payload,
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
2026-09-13 19:11:35 -05:00
|
|
|
MEDIATOR_FIELDS = {"scm_submission", "scm_resume", "publication_retry_transient", "publication_retry_lease_fence", "lease_fence"}
|
|
|
|
|
|
|
|
|
|
|
2026-09-13 15:04:01 -05:00
|
|
|
def attempt_assignment(attempt, continuation_kind=""):
|
hermes: make pool lease recovery and release isolation safe
Independent review t_5975c06a blocked this branch on a P1: a Kanban write that
failed while a lease expired left a `lease_failed` row that was invisible to
every pass, immortal to garbage collection, and fatal to the coordinator. It
poisoned `reconcile()` forever with a conflicting-duplicate primary key,
produced a spurious capability `block_task` from `dispatch()`, and -- because
startup maintenance ran unguarded before the port bound, against a store on a
PVC -- crash-looped the coordinator with no automatic recovery.
`lease_failed` is now a retryable state that every maintenance pass drains, and
a row only reaches a terminal state on authoritative evidence about its exact
Kanban run, so nothing is collected before its outcome is known and nothing is
silently dropped. Each row, task, and board is processed in isolation, and a
coordinator-side fault is never converted into a Kanban mutation. Startup runs
through the same guarded cycle as the steady-state loop.
The wire protocol and the durable store are now separate modules, and the
maintenance passes moved out of the coordinator, so each file stays under the
managed line ceiling with room for the recovery logic.
Also closes three consequential handoff risks the same review raised:
* mediator-N pinned itself hard to worker-N while sharing a ReadWriteOnce
claim, so a drain or preemption that moved only the lower-priority worker
deadlocked the ordinal on Multi-Attach until an operator deleted a Pod. The
shared workspace is now ReadWriteMany (as the hermes-chat tenant workspaces
already are on the same class), colocation is a preference, and the mediator
shares the worker's preemption priority, so each Pod reschedules on its own.
* the broker permits only branch creation, so a retry that added commits could
never submit and the run's work was discarded with the failure. Submission
now targets a fresh attempt- or content-scoped ref in the same reviewed
namespace -- never an update -- and is idempotent under replay. A refused
submission downgrades the result and says why instead of unwinding the run.
* the provider CLIs were reinstalled into an emptyDir on every Pod start inside
the 10m Flux health window for the whole hermes app. They now install once
per pinned version onto a durable volume, re-verified against the real
binaries and time-bounded, and the best-effort pool no longer gates the
health of the app its dependents wait on.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 16:31:15 +00:00
|
|
|
"""One signed assignment envelope bound to an exact retry attempt."""
|
|
|
|
|
return protocol.sign_envelope(
|
2026-09-13 15:04:01 -05:00
|
|
|
KEY, "assignment", binding(attempt=attempt), payload(continuation_kind=continuation_kind)
|
hermes: make pool lease recovery and release isolation safe
Independent review t_5975c06a blocked this branch on a P1: a Kanban write that
failed while a lease expired left a `lease_failed` row that was invisible to
every pass, immortal to garbage collection, and fatal to the coordinator. It
poisoned `reconcile()` forever with a conflicting-duplicate primary key,
produced a spurious capability `block_task` from `dispatch()`, and -- because
startup maintenance ran unguarded before the port bound, against a store on a
PVC -- crash-looped the coordinator with no automatic recovery.
`lease_failed` is now a retryable state that every maintenance pass drains, and
a row only reaches a terminal state on authoritative evidence about its exact
Kanban run, so nothing is collected before its outcome is known and nothing is
silently dropped. Each row, task, and board is processed in isolation, and a
coordinator-side fault is never converted into a Kanban mutation. Startup runs
through the same guarded cycle as the steady-state loop.
The wire protocol and the durable store are now separate modules, and the
maintenance passes moved out of the coordinator, so each file stays under the
managed line ceiling with room for the recovery logic.
Also closes three consequential handoff risks the same review raised:
* mediator-N pinned itself hard to worker-N while sharing a ReadWriteOnce
claim, so a drain or preemption that moved only the lower-priority worker
deadlocked the ordinal on Multi-Attach until an operator deleted a Pod. The
shared workspace is now ReadWriteMany (as the hermes-chat tenant workspaces
already are on the same class), colocation is a preference, and the mediator
shares the worker's preemption priority, so each Pod reschedules on its own.
* the broker permits only branch creation, so a retry that added commits could
never submit and the run's work was discarded with the failure. Submission
now targets a fresh attempt- or content-scoped ref in the same reviewed
namespace -- never an update -- and is idempotent under replay. A refused
submission downgrades the result and says why instead of unwinding the run.
* the provider CLIs were reinstalled into an emptyDir on every Pod start inside
the 10m Flux health window for the whole hermes app. They now install once
per pinned version onto a durable volume, re-verified against the real
binaries and time-bounded, and the best-effort pool no longer gates the
health of the app its dependents wait on.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 16:31:15 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
2026-09-13 15:04:01 -05:00
|
|
|
def test_submission_refs_keep_one_task_branch_across_attempts():
|
hermes: make pool lease recovery and release isolation safe
Independent review t_5975c06a blocked this branch on a P1: a Kanban write that
failed while a lease expired left a `lease_failed` row that was invisible to
every pass, immortal to garbage collection, and fatal to the coordinator. It
poisoned `reconcile()` forever with a conflicting-duplicate primary key,
produced a spurious capability `block_task` from `dispatch()`, and -- because
startup maintenance ran unguarded before the port bound, against a store on a
PVC -- crash-looped the coordinator with no automatic recovery.
`lease_failed` is now a retryable state that every maintenance pass drains, and
a row only reaches a terminal state on authoritative evidence about its exact
Kanban run, so nothing is collected before its outcome is known and nothing is
silently dropped. Each row, task, and board is processed in isolation, and a
coordinator-side fault is never converted into a Kanban mutation. Startup runs
through the same guarded cycle as the steady-state loop.
The wire protocol and the durable store are now separate modules, and the
maintenance passes moved out of the coordinator, so each file stays under the
managed line ceiling with room for the recovery logic.
Also closes three consequential handoff risks the same review raised:
* mediator-N pinned itself hard to worker-N while sharing a ReadWriteOnce
claim, so a drain or preemption that moved only the lower-priority worker
deadlocked the ordinal on Multi-Attach until an operator deleted a Pod. The
shared workspace is now ReadWriteMany (as the hermes-chat tenant workspaces
already are on the same class), colocation is a preference, and the mediator
shares the worker's preemption priority, so each Pod reschedules on its own.
* the broker permits only branch creation, so a retry that added commits could
never submit and the run's work was discarded with the failure. Submission
now targets a fresh attempt- or content-scoped ref in the same reviewed
namespace -- never an update -- and is idempotent under replay. A refused
submission downgrades the result and says why instead of unwinding the run.
* the provider CLIs were reinstalled into an emptyDir on every Pod start inside
the 10m Flux health window for the whole hermes app. They now install once
per pinned version onto a durable volume, re-verified against the real
binaries and time-bounded, and the best-effort pool no longer gates the
health of the app its dependents wait on.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 16:31:15 +00:00
|
|
|
first = scm.submission_refs("feature/pool", 1, "a" * 40)
|
|
|
|
|
assert first[0] == "feature/pool"
|
|
|
|
|
retry = scm.submission_refs("feature/pool", 3, "b" * 40)
|
2026-09-13 15:04:01 -05:00
|
|
|
assert retry == first
|
hermes: make pool lease recovery and release isolation safe
Independent review t_5975c06a blocked this branch on a P1: a Kanban write that
failed while a lease expired left a `lease_failed` row that was invisible to
every pass, immortal to garbage collection, and fatal to the coordinator. It
poisoned `reconcile()` forever with a conflicting-duplicate primary key,
produced a spurious capability `block_task` from `dispatch()`, and -- because
startup maintenance ran unguarded before the port bound, against a store on a
PVC -- crash-looped the coordinator with no automatic recovery.
`lease_failed` is now a retryable state that every maintenance pass drains, and
a row only reaches a terminal state on authoritative evidence about its exact
Kanban run, so nothing is collected before its outcome is known and nothing is
silently dropped. Each row, task, and board is processed in isolation, and a
coordinator-side fault is never converted into a Kanban mutation. Startup runs
through the same guarded cycle as the steady-state loop.
The wire protocol and the durable store are now separate modules, and the
maintenance passes moved out of the coordinator, so each file stays under the
managed line ceiling with room for the recovery logic.
Also closes three consequential handoff risks the same review raised:
* mediator-N pinned itself hard to worker-N while sharing a ReadWriteOnce
claim, so a drain or preemption that moved only the lower-priority worker
deadlocked the ordinal on Multi-Attach until an operator deleted a Pod. The
shared workspace is now ReadWriteMany (as the hermes-chat tenant workspaces
already are on the same class), colocation is a preference, and the mediator
shares the worker's preemption priority, so each Pod reschedules on its own.
* the broker permits only branch creation, so a retry that added commits could
never submit and the run's work was discarded with the failure. Submission
now targets a fresh attempt- or content-scoped ref in the same reviewed
namespace -- never an update -- and is idempotent under replay. A refused
submission downgrades the result and says why instead of unwinding the run.
* the provider CLIs were reinstalled into an emptyDir on every Pod start inside
the 10m Flux health window for the whole hermes app. They now install once
per pinned version onto a durable volume, re-verified against the real
binaries and time-bounded, and the best-effort pool no longer gates the
health of the app its dependents wait on.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 16:31:15 +00:00
|
|
|
# Every candidate stays inside a reviewed namespace the broker accepts, and
|
|
|
|
|
# none of them is a protected or base ref.
|
|
|
|
|
for candidate in first + retry:
|
2026-08-18 02:00:19 -03:00
|
|
|
assert receive_pack_scan.FEATURE_REF_RE.fullmatch(f"refs/heads/{candidate}")
|
hermes: make pool lease recovery and release isolation safe
Independent review t_5975c06a blocked this branch on a P1: a Kanban write that
failed while a lease expired left a `lease_failed` row that was invisible to
every pass, immortal to garbage collection, and fatal to the coordinator. It
poisoned `reconcile()` forever with a conflicting-duplicate primary key,
produced a spurious capability `block_task` from `dispatch()`, and -- because
startup maintenance ran unguarded before the port bound, against a store on a
PVC -- crash-looped the coordinator with no automatic recovery.
`lease_failed` is now a retryable state that every maintenance pass drains, and
a row only reaches a terminal state on authoritative evidence about its exact
Kanban run, so nothing is collected before its outcome is known and nothing is
silently dropped. Each row, task, and board is processed in isolation, and a
coordinator-side fault is never converted into a Kanban mutation. Startup runs
through the same guarded cycle as the steady-state loop.
The wire protocol and the durable store are now separate modules, and the
maintenance passes moved out of the coordinator, so each file stays under the
managed line ceiling with room for the recovery logic.
Also closes three consequential handoff risks the same review raised:
* mediator-N pinned itself hard to worker-N while sharing a ReadWriteOnce
claim, so a drain or preemption that moved only the lower-priority worker
deadlocked the ordinal on Multi-Attach until an operator deleted a Pod. The
shared workspace is now ReadWriteMany (as the hermes-chat tenant workspaces
already are on the same class), colocation is a preference, and the mediator
shares the worker's preemption priority, so each Pod reschedules on its own.
* the broker permits only branch creation, so a retry that added commits could
never submit and the run's work was discarded with the failure. Submission
now targets a fresh attempt- or content-scoped ref in the same reviewed
namespace -- never an update -- and is idempotent under replay. A refused
submission downgrades the result and says why instead of unwinding the run.
* the provider CLIs were reinstalled into an emptyDir on every Pod start inside
the 10m Flux health window for the whole hermes app. They now install once
per pinned version onto a durable volume, re-verified against the real
binaries and time-bounded, and the best-effort pool no longer gates the
health of the app its dependents wait on.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 16:31:15 +00:00
|
|
|
assert candidate not in {"main", "master"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_submission_refs_drop_oversized_names_and_fail_closed_when_empty():
|
|
|
|
|
long_branch = "feature/" + "a" * 185
|
|
|
|
|
candidates = scm.submission_refs(long_branch, 2, "c" * 40)
|
|
|
|
|
assert candidates == (long_branch,)
|
|
|
|
|
with pytest.raises(protocol.ProtocolError, match="reviewed branch name"):
|
|
|
|
|
scm.submission_refs("main", 1, "d" * 40)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_the_broker_accepts_the_attempt_ref_as_a_creation_and_still_refuses_updates():
|
|
|
|
|
zero, head = b"0" * 40, b"1" * 40
|
|
|
|
|
ref = b"refs/heads/feature/pool-attempt-2"
|
2026-08-18 02:00:19 -03:00
|
|
|
scm_broker._validate_receive_pack(_receive_command(zero, head, ref), "token")
|
|
|
|
|
update = _receive_command(b"2" * 40, head, ref)
|
|
|
|
|
with pytest.raises(scm_broker.PolicyError, match="feature-branch creation"):
|
|
|
|
|
scm_broker._validate_receive_pack(update, "token")
|
hermes: make pool lease recovery and release isolation safe
Independent review t_5975c06a blocked this branch on a P1: a Kanban write that
failed while a lease expired left a `lease_failed` row that was invisible to
every pass, immortal to garbage collection, and fatal to the coordinator. It
poisoned `reconcile()` forever with a conflicting-duplicate primary key,
produced a spurious capability `block_task` from `dispatch()`, and -- because
startup maintenance ran unguarded before the port bound, against a store on a
PVC -- crash-looped the coordinator with no automatic recovery.
`lease_failed` is now a retryable state that every maintenance pass drains, and
a row only reaches a terminal state on authoritative evidence about its exact
Kanban run, so nothing is collected before its outcome is known and nothing is
silently dropped. Each row, task, and board is processed in isolation, and a
coordinator-side fault is never converted into a Kanban mutation. Startup runs
through the same guarded cycle as the steady-state loop.
The wire protocol and the durable store are now separate modules, and the
maintenance passes moved out of the coordinator, so each file stays under the
managed line ceiling with room for the recovery logic.
Also closes three consequential handoff risks the same review raised:
* mediator-N pinned itself hard to worker-N while sharing a ReadWriteOnce
claim, so a drain or preemption that moved only the lower-priority worker
deadlocked the ordinal on Multi-Attach until an operator deleted a Pod. The
shared workspace is now ReadWriteMany (as the hermes-chat tenant workspaces
already are on the same class), colocation is a preference, and the mediator
shares the worker's preemption priority, so each Pod reschedules on its own.
* the broker permits only branch creation, so a retry that added commits could
never submit and the run's work was discarded with the failure. Submission
now targets a fresh attempt- or content-scoped ref in the same reviewed
namespace -- never an update -- and is idempotent under replay. A refused
submission downgrades the result and says why instead of unwinding the run.
* the provider CLIs were reinstalled into an emptyDir on every Pod start inside
the 10m Flux health window for the whole hermes app. They now install once
per pinned version onto a durable volume, re-verified against the real
binaries and time-bounded, and the best-effort pool no longer gates the
health of the app its dependents wait on.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 16:31:15 +00:00
|
|
|
|
|
|
|
|
|
2026-09-13 15:04:01 -05:00
|
|
|
def submit_harness(tmp_path, monkeypatch, remote_refs, *, attempt=1, continuation_kind=""):
|
hermes: make pool lease recovery and release isolation safe
Independent review t_5975c06a blocked this branch on a P1: a Kanban write that
failed while a lease expired left a `lease_failed` row that was invisible to
every pass, immortal to garbage collection, and fatal to the coordinator. It
poisoned `reconcile()` forever with a conflicting-duplicate primary key,
produced a spurious capability `block_task` from `dispatch()`, and -- because
startup maintenance ran unguarded before the port bound, against a store on a
PVC -- crash-looped the coordinator with no automatic recovery.
`lease_failed` is now a retryable state that every maintenance pass drains, and
a row only reaches a terminal state on authoritative evidence about its exact
Kanban run, so nothing is collected before its outcome is known and nothing is
silently dropped. Each row, task, and board is processed in isolation, and a
coordinator-side fault is never converted into a Kanban mutation. Startup runs
through the same guarded cycle as the steady-state loop.
The wire protocol and the durable store are now separate modules, and the
maintenance passes moved out of the coordinator, so each file stays under the
managed line ceiling with room for the recovery logic.
Also closes three consequential handoff risks the same review raised:
* mediator-N pinned itself hard to worker-N while sharing a ReadWriteOnce
claim, so a drain or preemption that moved only the lower-priority worker
deadlocked the ordinal on Multi-Attach until an operator deleted a Pod. The
shared workspace is now ReadWriteMany (as the hermes-chat tenant workspaces
already are on the same class), colocation is a preference, and the mediator
shares the worker's preemption priority, so each Pod reschedules on its own.
* the broker permits only branch creation, so a retry that added commits could
never submit and the run's work was discarded with the failure. Submission
now targets a fresh attempt- or content-scoped ref in the same reviewed
namespace -- never an update -- and is idempotent under replay. A refused
submission downgrades the result and says why instead of unwinding the run.
* the provider CLIs were reinstalled into an emptyDir on every Pod start inside
the 10m Flux health window for the whole hermes app. They now install once
per pinned version onto a durable volume, re-verified against the real
binaries and time-bounded, and the best-effort pool no longer gates the
health of the app its dependents wait on.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 16:31:15 +00:00
|
|
|
"""A Boundary whose broker advertises exactly ``remote_refs``."""
|
|
|
|
|
monkeypatch.setattr(scm, "WORKSPACE_ROOT", tmp_path / "workspace")
|
|
|
|
|
monkeypatch.setattr(scm, "SCM_ROOT", tmp_path / "state")
|
|
|
|
|
monkeypatch.setattr(scm, "ORDINAL", 0)
|
2026-09-13 15:04:01 -05:00
|
|
|
exact = attempt_assignment(attempt, continuation_kind)
|
hermes: make pool lease recovery and release isolation safe
Independent review t_5975c06a blocked this branch on a P1: a Kanban write that
failed while a lease expired left a `lease_failed` row that was invisible to
every pass, immortal to garbage collection, and fatal to the coordinator. It
poisoned `reconcile()` forever with a conflicting-duplicate primary key,
produced a spurious capability `block_task` from `dispatch()`, and -- because
startup maintenance ran unguarded before the port bound, against a store on a
PVC -- crash-looped the coordinator with no automatic recovery.
`lease_failed` is now a retryable state that every maintenance pass drains, and
a row only reaches a terminal state on authoritative evidence about its exact
Kanban run, so nothing is collected before its outcome is known and nothing is
silently dropped. Each row, task, and board is processed in isolation, and a
coordinator-side fault is never converted into a Kanban mutation. Startup runs
through the same guarded cycle as the steady-state loop.
The wire protocol and the durable store are now separate modules, and the
maintenance passes moved out of the coordinator, so each file stays under the
managed line ceiling with room for the recovery logic.
Also closes three consequential handoff risks the same review raised:
* mediator-N pinned itself hard to worker-N while sharing a ReadWriteOnce
claim, so a drain or preemption that moved only the lower-priority worker
deadlocked the ordinal on Multi-Attach until an operator deleted a Pod. The
shared workspace is now ReadWriteMany (as the hermes-chat tenant workspaces
already are on the same class), colocation is a preference, and the mediator
shares the worker's preemption priority, so each Pod reschedules on its own.
* the broker permits only branch creation, so a retry that added commits could
never submit and the run's work was discarded with the failure. Submission
now targets a fresh attempt- or content-scoped ref in the same reviewed
namespace -- never an update -- and is idempotent under replay. A refused
submission downgrades the result and says why instead of unwinding the run.
* the provider CLIs were reinstalled into an emptyDir on every Pod start inside
the 10m Flux health window for the whole hermes app. They now install once
per pinned version onto a durable volume, re-verified against the real
binaries and time-bounded, and the best-effort pool no longer gates the
health of the app its dependents wait on.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 16:31:15 +00:00
|
|
|
destination = scm.workspace_path(exact)
|
|
|
|
|
destination.mkdir(parents=True)
|
|
|
|
|
protocol.atomic_json(scm._state_path(exact), {"baseline_sha": "a" * 40})
|
|
|
|
|
head = "b" * 40
|
|
|
|
|
monkeypatch.setattr(scm, "_workspace_identity", lambda *_a: head)
|
|
|
|
|
calls: list[tuple[str, ...]] = []
|
|
|
|
|
|
|
|
|
|
def run(*arguments, **_kwargs):
|
|
|
|
|
calls.append(arguments)
|
|
|
|
|
if arguments[0] == "ls-remote":
|
|
|
|
|
return "\n".join(
|
|
|
|
|
f"{sha}\trefs/heads/{ref}" for ref, sha in remote_refs.items()
|
|
|
|
|
)
|
|
|
|
|
if arguments[0] == "rev-list":
|
|
|
|
|
return "2"
|
|
|
|
|
return ""
|
|
|
|
|
|
|
|
|
|
monkeypatch.setattr(scm, "_run", run)
|
|
|
|
|
boundary = scm.Boundary(KEY)
|
2026-09-13 15:04:01 -05:00
|
|
|
monkeypatch.setattr(boundary, "_grant", lambda *_args: "signed-grant")
|
|
|
|
|
monkeypatch.setattr(scm.scm_broker_client, "register_task", lambda _grant: b'{"registered":true}')
|
|
|
|
|
monkeypatch.setattr(boundary, "_draft", lambda *args, **_kwargs: f"https://scm/pulls/{args[1]}")
|
hermes: make pool lease recovery and release isolation safe
Independent review t_5975c06a blocked this branch on a P1: a Kanban write that
failed while a lease expired left a `lease_failed` row that was invisible to
every pass, immortal to garbage collection, and fatal to the coordinator. It
poisoned `reconcile()` forever with a conflicting-duplicate primary key,
produced a spurious capability `block_task` from `dispatch()`, and -- because
startup maintenance ran unguarded before the port bound, against a store on a
PVC -- crash-looped the coordinator with no automatic recovery.
`lease_failed` is now a retryable state that every maintenance pass drains, and
a row only reaches a terminal state on authoritative evidence about its exact
Kanban run, so nothing is collected before its outcome is known and nothing is
silently dropped. Each row, task, and board is processed in isolation, and a
coordinator-side fault is never converted into a Kanban mutation. Startup runs
through the same guarded cycle as the steady-state loop.
The wire protocol and the durable store are now separate modules, and the
maintenance passes moved out of the coordinator, so each file stays under the
managed line ceiling with room for the recovery logic.
Also closes three consequential handoff risks the same review raised:
* mediator-N pinned itself hard to worker-N while sharing a ReadWriteOnce
claim, so a drain or preemption that moved only the lower-priority worker
deadlocked the ordinal on Multi-Attach until an operator deleted a Pod. The
shared workspace is now ReadWriteMany (as the hermes-chat tenant workspaces
already are on the same class), colocation is a preference, and the mediator
shares the worker's preemption priority, so each Pod reschedules on its own.
* the broker permits only branch creation, so a retry that added commits could
never submit and the run's work was discarded with the failure. Submission
now targets a fresh attempt- or content-scoped ref in the same reviewed
namespace -- never an update -- and is idempotent under replay. A refused
submission downgrades the result and says why instead of unwinding the run.
* the provider CLIs were reinstalled into an emptyDir on every Pod start inside
the 10m Flux health window for the whole hermes app. They now install once
per pinned version onto a durable volume, re-verified against the real
binaries and time-bounded, and the best-effort pool no longer gates the
health of the app its dependents wait on.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 16:31:15 +00:00
|
|
|
return boundary, exact, calls, head
|
|
|
|
|
|
|
|
|
|
|
2026-09-13 15:04:01 -05:00
|
|
|
def test_a_retry_updates_the_same_task_ref(
|
hermes: make pool lease recovery and release isolation safe
Independent review t_5975c06a blocked this branch on a P1: a Kanban write that
failed while a lease expired left a `lease_failed` row that was invisible to
every pass, immortal to garbage collection, and fatal to the coordinator. It
poisoned `reconcile()` forever with a conflicting-duplicate primary key,
produced a spurious capability `block_task` from `dispatch()`, and -- because
startup maintenance ran unguarded before the port bound, against a store on a
PVC -- crash-looped the coordinator with no automatic recovery.
`lease_failed` is now a retryable state that every maintenance pass drains, and
a row only reaches a terminal state on authoritative evidence about its exact
Kanban run, so nothing is collected before its outcome is known and nothing is
silently dropped. Each row, task, and board is processed in isolation, and a
coordinator-side fault is never converted into a Kanban mutation. Startup runs
through the same guarded cycle as the steady-state loop.
The wire protocol and the durable store are now separate modules, and the
maintenance passes moved out of the coordinator, so each file stays under the
managed line ceiling with room for the recovery logic.
Also closes three consequential handoff risks the same review raised:
* mediator-N pinned itself hard to worker-N while sharing a ReadWriteOnce
claim, so a drain or preemption that moved only the lower-priority worker
deadlocked the ordinal on Multi-Attach until an operator deleted a Pod. The
shared workspace is now ReadWriteMany (as the hermes-chat tenant workspaces
already are on the same class), colocation is a preference, and the mediator
shares the worker's preemption priority, so each Pod reschedules on its own.
* the broker permits only branch creation, so a retry that added commits could
never submit and the run's work was discarded with the failure. Submission
now targets a fresh attempt- or content-scoped ref in the same reviewed
namespace -- never an update -- and is idempotent under replay. A refused
submission downgrades the result and says why instead of unwinding the run.
* the provider CLIs were reinstalled into an emptyDir on every Pod start inside
the 10m Flux health window for the whole hermes app. They now install once
per pinned version onto a durable volume, re-verified against the real
binaries and time-bounded, and the best-effort pool no longer gates the
health of the app its dependents wait on.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 16:31:15 +00:00
|
|
|
tmp_path, monkeypatch
|
|
|
|
|
):
|
|
|
|
|
boundary, exact, calls, _head = submit_harness(
|
|
|
|
|
tmp_path, monkeypatch, {"wt/t_deadbeef": "c" * 40}, attempt=2
|
|
|
|
|
)
|
|
|
|
|
result = boundary.submit(exact, {"title": "retry", "body": "evidence"})
|
2026-09-13 15:04:01 -05:00
|
|
|
pushes = [item for item in calls if item[-4:] == ("push", "--no-thin", "hermes-broker", "HEAD:refs/heads/wt/t_deadbeef")]
|
|
|
|
|
assert len(pushes) == 1
|
|
|
|
|
assert result["branch"] == "wt/t_deadbeef"
|
|
|
|
|
assert result["pull_request"] == "https://scm/pulls/wt/t_deadbeef"
|
hermes: make pool lease recovery and release isolation safe
Independent review t_5975c06a blocked this branch on a P1: a Kanban write that
failed while a lease expired left a `lease_failed` row that was invisible to
every pass, immortal to garbage collection, and fatal to the coordinator. It
poisoned `reconcile()` forever with a conflicting-duplicate primary key,
produced a spurious capability `block_task` from `dispatch()`, and -- because
startup maintenance ran unguarded before the port bound, against a store on a
PVC -- crash-looped the coordinator with no automatic recovery.
`lease_failed` is now a retryable state that every maintenance pass drains, and
a row only reaches a terminal state on authoritative evidence about its exact
Kanban run, so nothing is collected before its outcome is known and nothing is
silently dropped. Each row, task, and board is processed in isolation, and a
coordinator-side fault is never converted into a Kanban mutation. Startup runs
through the same guarded cycle as the steady-state loop.
The wire protocol and the durable store are now separate modules, and the
maintenance passes moved out of the coordinator, so each file stays under the
managed line ceiling with room for the recovery logic.
Also closes three consequential handoff risks the same review raised:
* mediator-N pinned itself hard to worker-N while sharing a ReadWriteOnce
claim, so a drain or preemption that moved only the lower-priority worker
deadlocked the ordinal on Multi-Attach until an operator deleted a Pod. The
shared workspace is now ReadWriteMany (as the hermes-chat tenant workspaces
already are on the same class), colocation is a preference, and the mediator
shares the worker's preemption priority, so each Pod reschedules on its own.
* the broker permits only branch creation, so a retry that added commits could
never submit and the run's work was discarded with the failure. Submission
now targets a fresh attempt- or content-scoped ref in the same reviewed
namespace -- never an update -- and is idempotent under replay. A refused
submission downgrades the result and says why instead of unwinding the run.
* the provider CLIs were reinstalled into an emptyDir on every Pod start inside
the 10m Flux health window for the whole hermes app. They now install once
per pinned version onto a durable volume, re-verified against the real
binaries and time-bounded, and the best-effort pool no longer gates the
health of the app its dependents wait on.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 16:31:15 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_an_already_published_head_is_adopted_without_a_second_push(
|
|
|
|
|
tmp_path, monkeypatch
|
|
|
|
|
):
|
|
|
|
|
boundary, exact, calls, head = submit_harness(
|
2026-09-13 15:04:01 -05:00
|
|
|
tmp_path, monkeypatch, {"wt/t_deadbeef": "b" * 40}, attempt=2
|
hermes: make pool lease recovery and release isolation safe
Independent review t_5975c06a blocked this branch on a P1: a Kanban write that
failed while a lease expired left a `lease_failed` row that was invisible to
every pass, immortal to garbage collection, and fatal to the coordinator. It
poisoned `reconcile()` forever with a conflicting-duplicate primary key,
produced a spurious capability `block_task` from `dispatch()`, and -- because
startup maintenance ran unguarded before the port bound, against a store on a
PVC -- crash-looped the coordinator with no automatic recovery.
`lease_failed` is now a retryable state that every maintenance pass drains, and
a row only reaches a terminal state on authoritative evidence about its exact
Kanban run, so nothing is collected before its outcome is known and nothing is
silently dropped. Each row, task, and board is processed in isolation, and a
coordinator-side fault is never converted into a Kanban mutation. Startup runs
through the same guarded cycle as the steady-state loop.
The wire protocol and the durable store are now separate modules, and the
maintenance passes moved out of the coordinator, so each file stays under the
managed line ceiling with room for the recovery logic.
Also closes three consequential handoff risks the same review raised:
* mediator-N pinned itself hard to worker-N while sharing a ReadWriteOnce
claim, so a drain or preemption that moved only the lower-priority worker
deadlocked the ordinal on Multi-Attach until an operator deleted a Pod. The
shared workspace is now ReadWriteMany (as the hermes-chat tenant workspaces
already are on the same class), colocation is a preference, and the mediator
shares the worker's preemption priority, so each Pod reschedules on its own.
* the broker permits only branch creation, so a retry that added commits could
never submit and the run's work was discarded with the failure. Submission
now targets a fresh attempt- or content-scoped ref in the same reviewed
namespace -- never an update -- and is idempotent under replay. A refused
submission downgrades the result and says why instead of unwinding the run.
* the provider CLIs were reinstalled into an emptyDir on every Pod start inside
the 10m Flux health window for the whole hermes app. They now install once
per pinned version onto a durable volume, re-verified against the real
binaries and time-bounded, and the best-effort pool no longer gates the
health of the app its dependents wait on.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 16:31:15 +00:00
|
|
|
)
|
|
|
|
|
assert head == "b" * 40
|
|
|
|
|
result = boundary.submit(exact, {"title": "replay", "body": "evidence"})
|
2026-09-13 15:04:01 -05:00
|
|
|
assert [item for item in calls if "push" in item] == []
|
|
|
|
|
assert result["branch"] == "wt/t_deadbeef"
|
|
|
|
|
assert result["pull_request"].endswith("wt/t_deadbeef")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_no_change_review_returns_existing_pr_without_rewriting_its_handoff(tmp_path, monkeypatch):
|
|
|
|
|
"""A review at the already-published head must not mutate implementation prose."""
|
|
|
|
|
boundary, exact, calls, head = submit_harness(
|
|
|
|
|
tmp_path, monkeypatch, {"wt/t_deadbeef": "b" * 40}, continuation_kind="review"
|
|
|
|
|
)
|
|
|
|
|
existing = "https://scm.bstein.dev/titan/metis/pulls/42"
|
|
|
|
|
monkeypatch.setattr(
|
|
|
|
|
scm.scm_broker_client, "read",
|
|
|
|
|
lambda _path: __import__("json").dumps([{
|
|
|
|
|
"number": 42, "html_url": existing,
|
|
|
|
|
"head": {"ref": "wt/t_deadbeef"}, "base": {"ref": "main"},
|
|
|
|
|
}]).encode(),
|
|
|
|
|
)
|
|
|
|
|
monkeypatch.setattr(
|
|
|
|
|
scm.scm_broker_client, "update_draft",
|
|
|
|
|
lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("review rewrote PR metadata")),
|
|
|
|
|
)
|
|
|
|
|
monkeypatch.setattr(
|
|
|
|
|
scm.scm_broker_client, "create_draft",
|
|
|
|
|
lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("review created a PR")),
|
|
|
|
|
)
|
|
|
|
|
# The helper installs a simple draft stub for other cases; restore the real
|
|
|
|
|
# bounded discovery path for this no-change review assertion.
|
|
|
|
|
monkeypatch.setattr(boundary, "_draft", scm.Boundary._draft)
|
|
|
|
|
|
|
|
|
|
result = boundary.submit(exact, {"title": "Review", "body": "No changes"})
|
|
|
|
|
|
|
|
|
|
assert result["head"] == head and result["pull_request"] == existing
|
|
|
|
|
assert [item for item in calls if "push" in item] == []
|
hermes: make pool lease recovery and release isolation safe
Independent review t_5975c06a blocked this branch on a P1: a Kanban write that
failed while a lease expired left a `lease_failed` row that was invisible to
every pass, immortal to garbage collection, and fatal to the coordinator. It
poisoned `reconcile()` forever with a conflicting-duplicate primary key,
produced a spurious capability `block_task` from `dispatch()`, and -- because
startup maintenance ran unguarded before the port bound, against a store on a
PVC -- crash-looped the coordinator with no automatic recovery.
`lease_failed` is now a retryable state that every maintenance pass drains, and
a row only reaches a terminal state on authoritative evidence about its exact
Kanban run, so nothing is collected before its outcome is known and nothing is
silently dropped. Each row, task, and board is processed in isolation, and a
coordinator-side fault is never converted into a Kanban mutation. Startup runs
through the same guarded cycle as the steady-state loop.
The wire protocol and the durable store are now separate modules, and the
maintenance passes moved out of the coordinator, so each file stays under the
managed line ceiling with room for the recovery logic.
Also closes three consequential handoff risks the same review raised:
* mediator-N pinned itself hard to worker-N while sharing a ReadWriteOnce
claim, so a drain or preemption that moved only the lower-priority worker
deadlocked the ordinal on Multi-Attach until an operator deleted a Pod. The
shared workspace is now ReadWriteMany (as the hermes-chat tenant workspaces
already are on the same class), colocation is a preference, and the mediator
shares the worker's preemption priority, so each Pod reschedules on its own.
* the broker permits only branch creation, so a retry that added commits could
never submit and the run's work was discarded with the failure. Submission
now targets a fresh attempt- or content-scoped ref in the same reviewed
namespace -- never an update -- and is idempotent under replay. A refused
submission downgrades the result and says why instead of unwinding the run.
* the provider CLIs were reinstalled into an emptyDir on every Pod start inside
the 10m Flux health window for the whole hermes app. They now install once
per pinned version onto a durable volume, re-verified against the real
binaries and time-bounded, and the best-effort pool no longer gates the
health of the app its dependents wait on.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 16:31:15 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_a_run_with_no_new_commits_still_reports_work_a_prior_attempt_published(
|
|
|
|
|
tmp_path, monkeypatch
|
|
|
|
|
):
|
|
|
|
|
boundary, exact, calls, _head = submit_harness(
|
|
|
|
|
tmp_path, monkeypatch, {"wt/t_deadbeef": "b" * 40}
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
real_run = scm._run
|
|
|
|
|
|
|
|
|
|
def run(*arguments, **kwargs):
|
|
|
|
|
if arguments[0] == "rev-list":
|
|
|
|
|
return "0"
|
|
|
|
|
return real_run(*arguments, **kwargs)
|
|
|
|
|
|
|
|
|
|
monkeypatch.setattr(scm, "_run", run)
|
|
|
|
|
result = boundary.submit(exact, {"title": "nothing new", "body": "evidence"})
|
2026-09-13 15:04:01 -05:00
|
|
|
assert [item for item in calls if "push" in item] == []
|
hermes: make pool lease recovery and release isolation safe
Independent review t_5975c06a blocked this branch on a P1: a Kanban write that
failed while a lease expired left a `lease_failed` row that was invisible to
every pass, immortal to garbage collection, and fatal to the coordinator. It
poisoned `reconcile()` forever with a conflicting-duplicate primary key,
produced a spurious capability `block_task` from `dispatch()`, and -- because
startup maintenance ran unguarded before the port bound, against a store on a
PVC -- crash-looped the coordinator with no automatic recovery.
`lease_failed` is now a retryable state that every maintenance pass drains, and
a row only reaches a terminal state on authoritative evidence about its exact
Kanban run, so nothing is collected before its outcome is known and nothing is
silently dropped. Each row, task, and board is processed in isolation, and a
coordinator-side fault is never converted into a Kanban mutation. Startup runs
through the same guarded cycle as the steady-state loop.
The wire protocol and the durable store are now separate modules, and the
maintenance passes moved out of the coordinator, so each file stays under the
managed line ceiling with room for the recovery logic.
Also closes three consequential handoff risks the same review raised:
* mediator-N pinned itself hard to worker-N while sharing a ReadWriteOnce
claim, so a drain or preemption that moved only the lower-priority worker
deadlocked the ordinal on Multi-Attach until an operator deleted a Pod. The
shared workspace is now ReadWriteMany (as the hermes-chat tenant workspaces
already are on the same class), colocation is a preference, and the mediator
shares the worker's preemption priority, so each Pod reschedules on its own.
* the broker permits only branch creation, so a retry that added commits could
never submit and the run's work was discarded with the failure. Submission
now targets a fresh attempt- or content-scoped ref in the same reviewed
namespace -- never an update -- and is idempotent under replay. A refused
submission downgrades the result and says why instead of unwinding the run.
* the provider CLIs were reinstalled into an emptyDir on every Pod start inside
the 10m Flux health window for the whole hermes app. They now install once
per pinned version onto a durable volume, re-verified against the real
binaries and time-bounded, and the best-effort pool no longer gates the
health of the app its dependents wait on.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 16:31:15 +00:00
|
|
|
assert result["pull_request"].endswith("wt/t_deadbeef")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_a_run_that_produced_nothing_at_all_submits_nothing(tmp_path, monkeypatch):
|
|
|
|
|
boundary, exact, calls, _head = submit_harness(tmp_path, monkeypatch, {})
|
|
|
|
|
|
|
|
|
|
real_run = scm._run
|
|
|
|
|
|
|
|
|
|
def run(*arguments, **kwargs):
|
|
|
|
|
if arguments[0] == "rev-list":
|
|
|
|
|
return "0"
|
|
|
|
|
return real_run(*arguments, **kwargs)
|
|
|
|
|
|
|
|
|
|
monkeypatch.setattr(scm, "_run", run)
|
|
|
|
|
result = boundary.submit(exact, {})
|
2026-09-13 15:04:01 -05:00
|
|
|
assert [item for item in calls if "push" in item] == []
|
hermes: make pool lease recovery and release isolation safe
Independent review t_5975c06a blocked this branch on a P1: a Kanban write that
failed while a lease expired left a `lease_failed` row that was invisible to
every pass, immortal to garbage collection, and fatal to the coordinator. It
poisoned `reconcile()` forever with a conflicting-duplicate primary key,
produced a spurious capability `block_task` from `dispatch()`, and -- because
startup maintenance ran unguarded before the port bound, against a store on a
PVC -- crash-looped the coordinator with no automatic recovery.
`lease_failed` is now a retryable state that every maintenance pass drains, and
a row only reaches a terminal state on authoritative evidence about its exact
Kanban run, so nothing is collected before its outcome is known and nothing is
silently dropped. Each row, task, and board is processed in isolation, and a
coordinator-side fault is never converted into a Kanban mutation. Startup runs
through the same guarded cycle as the steady-state loop.
The wire protocol and the durable store are now separate modules, and the
maintenance passes moved out of the coordinator, so each file stays under the
managed line ceiling with room for the recovery logic.
Also closes three consequential handoff risks the same review raised:
* mediator-N pinned itself hard to worker-N while sharing a ReadWriteOnce
claim, so a drain or preemption that moved only the lower-priority worker
deadlocked the ordinal on Multi-Attach until an operator deleted a Pod. The
shared workspace is now ReadWriteMany (as the hermes-chat tenant workspaces
already are on the same class), colocation is a preference, and the mediator
shares the worker's preemption priority, so each Pod reschedules on its own.
* the broker permits only branch creation, so a retry that added commits could
never submit and the run's work was discarded with the failure. Submission
now targets a fresh attempt- or content-scoped ref in the same reviewed
namespace -- never an update -- and is idempotent under replay. A refused
submission downgrades the result and says why instead of unwinding the run.
* the provider CLIs were reinstalled into an emptyDir on every Pod start inside
the 10m Flux health window for the whole hermes app. They now install once
per pinned version onto a durable volume, re-verified against the real
binaries and time-bounded, and the best-effort pool no longer gates the
health of the app its dependents wait on.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 16:31:15 +00:00
|
|
|
assert result["pull_request"] == "" and result["branch"] == "wt/t_deadbeef"
|
|
|
|
|
|
|
|
|
|
|
2026-09-13 15:04:01 -05:00
|
|
|
def test_submission_updates_an_existing_task_ref_without_creating_a_retry_ref(
|
hermes: make pool lease recovery and release isolation safe
Independent review t_5975c06a blocked this branch on a P1: a Kanban write that
failed while a lease expired left a `lease_failed` row that was invisible to
every pass, immortal to garbage collection, and fatal to the coordinator. It
poisoned `reconcile()` forever with a conflicting-duplicate primary key,
produced a spurious capability `block_task` from `dispatch()`, and -- because
startup maintenance ran unguarded before the port bound, against a store on a
PVC -- crash-looped the coordinator with no automatic recovery.
`lease_failed` is now a retryable state that every maintenance pass drains, and
a row only reaches a terminal state on authoritative evidence about its exact
Kanban run, so nothing is collected before its outcome is known and nothing is
silently dropped. Each row, task, and board is processed in isolation, and a
coordinator-side fault is never converted into a Kanban mutation. Startup runs
through the same guarded cycle as the steady-state loop.
The wire protocol and the durable store are now separate modules, and the
maintenance passes moved out of the coordinator, so each file stays under the
managed line ceiling with room for the recovery logic.
Also closes three consequential handoff risks the same review raised:
* mediator-N pinned itself hard to worker-N while sharing a ReadWriteOnce
claim, so a drain or preemption that moved only the lower-priority worker
deadlocked the ordinal on Multi-Attach until an operator deleted a Pod. The
shared workspace is now ReadWriteMany (as the hermes-chat tenant workspaces
already are on the same class), colocation is a preference, and the mediator
shares the worker's preemption priority, so each Pod reschedules on its own.
* the broker permits only branch creation, so a retry that added commits could
never submit and the run's work was discarded with the failure. Submission
now targets a fresh attempt- or content-scoped ref in the same reviewed
namespace -- never an update -- and is idempotent under replay. A refused
submission downgrades the result and says why instead of unwinding the run.
* the provider CLIs were reinstalled into an emptyDir on every Pod start inside
the 10m Flux health window for the whole hermes app. They now install once
per pinned version onto a durable volume, re-verified against the real
binaries and time-bounded, and the best-effort pool no longer gates the
health of the app its dependents wait on.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 16:31:15 +00:00
|
|
|
tmp_path, monkeypatch
|
|
|
|
|
):
|
|
|
|
|
taken = {
|
|
|
|
|
"wt/t_deadbeef": "c" * 40,
|
|
|
|
|
"wt/t_deadbeef-attempt-1": "d" * 40,
|
|
|
|
|
f"wt/t_deadbeef-{'b' * 12}": "e" * 40,
|
|
|
|
|
}
|
|
|
|
|
boundary, exact, calls, _head = submit_harness(tmp_path, monkeypatch, taken)
|
2026-09-13 15:04:01 -05:00
|
|
|
result = boundary.submit(exact, {"title": "continued", "body": "evidence"})
|
|
|
|
|
pushes = [item for item in calls if item[-4:] == ("push", "--no-thin", "hermes-broker", "HEAD:refs/heads/wt/t_deadbeef")]
|
|
|
|
|
assert len(pushes) == 1
|
|
|
|
|
assert result["branch"] == "wt/t_deadbeef"
|
hermes: make pool lease recovery and release isolation safe
Independent review t_5975c06a blocked this branch on a P1: a Kanban write that
failed while a lease expired left a `lease_failed` row that was invisible to
every pass, immortal to garbage collection, and fatal to the coordinator. It
poisoned `reconcile()` forever with a conflicting-duplicate primary key,
produced a spurious capability `block_task` from `dispatch()`, and -- because
startup maintenance ran unguarded before the port bound, against a store on a
PVC -- crash-looped the coordinator with no automatic recovery.
`lease_failed` is now a retryable state that every maintenance pass drains, and
a row only reaches a terminal state on authoritative evidence about its exact
Kanban run, so nothing is collected before its outcome is known and nothing is
silently dropped. Each row, task, and board is processed in isolation, and a
coordinator-side fault is never converted into a Kanban mutation. Startup runs
through the same guarded cycle as the steady-state loop.
The wire protocol and the durable store are now separate modules, and the
maintenance passes moved out of the coordinator, so each file stays under the
managed line ceiling with room for the recovery logic.
Also closes three consequential handoff risks the same review raised:
* mediator-N pinned itself hard to worker-N while sharing a ReadWriteOnce
claim, so a drain or preemption that moved only the lower-priority worker
deadlocked the ordinal on Multi-Attach until an operator deleted a Pod. The
shared workspace is now ReadWriteMany (as the hermes-chat tenant workspaces
already are on the same class), colocation is a preference, and the mediator
shares the worker's preemption priority, so each Pod reschedules on its own.
* the broker permits only branch creation, so a retry that added commits could
never submit and the run's work was discarded with the failure. Submission
now targets a fresh attempt- or content-scoped ref in the same reviewed
namespace -- never an update -- and is idempotent under replay. A refused
submission downgrades the result and says why instead of unwinding the run.
* the provider CLIs were reinstalled into an emptyDir on every Pod start inside
the 10m Flux health window for the whole hermes app. They now install once
per pinned version onto a durable volume, re-verified against the real
binaries and time-bounded, and the best-effort pool no longer gates the
health of the app its dependents wait on.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 16:31:15 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_remote_head_parsing_ignores_anything_that_is_not_a_branch(
|
|
|
|
|
tmp_path, monkeypatch
|
|
|
|
|
):
|
|
|
|
|
monkeypatch.setattr(
|
|
|
|
|
scm, "_run",
|
|
|
|
|
lambda *_a, **_k: "aaa\trefs/tags/v1\nbbb\trefs/heads/feature/x\ngarbage\n",
|
|
|
|
|
)
|
|
|
|
|
assert scm._remote_heads(tmp_path, ("feature/x",)) == {"feature/x": "bbb"}
|
|
|
|
|
|
|
|
|
|
|
2026-09-13 17:54:54 -05:00
|
|
|
def test_git_timeout_becomes_a_sanitized_retryable_error(monkeypatch):
|
|
|
|
|
"""A subprocess timeout must not skip terminal result submission or expose a grant."""
|
|
|
|
|
def timed_out(*_args, **_kwargs):
|
|
|
|
|
raise subprocess.TimeoutExpired(
|
|
|
|
|
["git", "-c", "http.extraHeader=X-Hermes-Task-Grant: secret", "push"], 900
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
monkeypatch.setattr(scm.subprocess, "run", timed_out)
|
|
|
|
|
with pytest.raises(RuntimeError, match="^SCM command timed out$"):
|
|
|
|
|
scm._run("push", "hermes-broker")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
|
|
|
("detail", "expected"),
|
|
|
|
|
[
|
|
|
|
|
(
|
|
|
|
|
"remote: task branch head changed; fetch and merge before retrying",
|
|
|
|
|
"task branch changed; fetch and merge before retrying",
|
|
|
|
|
),
|
|
|
|
|
(
|
|
|
|
|
"RPC failed; HTTP 403",
|
|
|
|
|
"SCM broker authorization rejected the branch update; "
|
|
|
|
|
"inspect task ownership before retrying",
|
|
|
|
|
),
|
|
|
|
|
(
|
|
|
|
|
"fatal: unable to access: Connection timed out",
|
|
|
|
|
"SCM broker transport failed; retry the preserved local commit",
|
|
|
|
|
),
|
|
|
|
|
(
|
|
|
|
|
"RPC failed; HTTP 400",
|
|
|
|
|
"SCM broker rejected the branch update; "
|
|
|
|
|
"inspect broker or upstream state before retrying",
|
|
|
|
|
),
|
|
|
|
|
],
|
|
|
|
|
)
|
|
|
|
|
def test_push_failure_guidance_is_sanitized_and_specific(detail, expected):
|
|
|
|
|
assert scm._push_failure(RuntimeError(detail)) == expected
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_draft_discovery_finds_a_same_branch_pr_after_the_first_page(monkeypatch):
|
|
|
|
|
first_page = [{"head": {"ref": f"other-{index}"}, "base": {"ref": "main"}} for index in range(50)]
|
|
|
|
|
found = "https://scm.bstein.dev/titan/metis/pulls/51"
|
|
|
|
|
calls = []
|
|
|
|
|
|
|
|
|
|
def read(path):
|
|
|
|
|
calls.append(path)
|
|
|
|
|
return __import__("json").dumps(
|
|
|
|
|
first_page if "page=1" in path else [{
|
|
|
|
|
"number": 51, "html_url": found,
|
|
|
|
|
"head": {"ref": "wt/t_deadbeef"}, "base": {"ref": "main"},
|
|
|
|
|
}]
|
|
|
|
|
).encode()
|
|
|
|
|
|
|
|
|
|
monkeypatch.setattr(scm.scm_broker_client, "read", read)
|
|
|
|
|
monkeypatch.setattr(
|
|
|
|
|
scm.scm_broker_client, "update_draft",
|
|
|
|
|
lambda grant, number, title, body: __import__("json").dumps({"html_url": found}).encode(),
|
|
|
|
|
)
|
|
|
|
|
assert scm.Boundary._draft("metis", "wt/t_deadbeef", "main", "b" * 40, "safe", "body", "grant") == found
|
|
|
|
|
assert len(calls) == 2
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_repair_draft_refuses_to_create_a_replacement_when_its_pr_is_not_open(monkeypatch):
|
|
|
|
|
monkeypatch.setattr(scm.scm_broker_client, "read", lambda _path: b"[]")
|
|
|
|
|
monkeypatch.setattr(
|
|
|
|
|
scm.scm_broker_client, "create_draft",
|
|
|
|
|
lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("replacement PR")),
|
|
|
|
|
)
|
|
|
|
|
with pytest.raises(protocol.ProtocolError, match="no existing pull request"):
|
|
|
|
|
scm.Boundary._draft(
|
|
|
|
|
"metis", "wt/t_deadbeef", "main", "b" * 40, "repair", "evidence",
|
|
|
|
|
"grant", existing_only=True,
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_repair_checks_its_open_pr_before_advancing_the_owned_branch(tmp_path, monkeypatch):
|
|
|
|
|
boundary, exact, _calls, _head = submit_harness(
|
|
|
|
|
tmp_path, monkeypatch, {"wt/t_deadbeef": "a" * 40}, continuation_kind="repair"
|
|
|
|
|
)
|
|
|
|
|
events = []
|
|
|
|
|
original_run = scm._run
|
|
|
|
|
|
|
|
|
|
def run(*arguments, **kwargs):
|
|
|
|
|
if "push" in arguments:
|
|
|
|
|
events.append("push")
|
|
|
|
|
return original_run(*arguments, **kwargs)
|
|
|
|
|
|
|
|
|
|
def draft(*_args, **kwargs):
|
|
|
|
|
events.append("pre" if kwargs.get("refresh") is False else "post")
|
|
|
|
|
return "https://scm/pulls/3"
|
|
|
|
|
|
|
|
|
|
monkeypatch.setattr(scm, "_run", run)
|
|
|
|
|
monkeypatch.setattr(boundary, "_draft", draft)
|
|
|
|
|
boundary.submit(exact, {"title": "repair", "body": "evidence"})
|
|
|
|
|
assert events == ["pre", "push", "post"]
|
|
|
|
|
|
|
|
|
|
|
hermes: make pool lease recovery and release isolation safe
Independent review t_5975c06a blocked this branch on a P1: a Kanban write that
failed while a lease expired left a `lease_failed` row that was invisible to
every pass, immortal to garbage collection, and fatal to the coordinator. It
poisoned `reconcile()` forever with a conflicting-duplicate primary key,
produced a spurious capability `block_task` from `dispatch()`, and -- because
startup maintenance ran unguarded before the port bound, against a store on a
PVC -- crash-looped the coordinator with no automatic recovery.
`lease_failed` is now a retryable state that every maintenance pass drains, and
a row only reaches a terminal state on authoritative evidence about its exact
Kanban run, so nothing is collected before its outcome is known and nothing is
silently dropped. Each row, task, and board is processed in isolation, and a
coordinator-side fault is never converted into a Kanban mutation. Startup runs
through the same guarded cycle as the steady-state loop.
The wire protocol and the durable store are now separate modules, and the
maintenance passes moved out of the coordinator, so each file stays under the
managed line ceiling with room for the recovery logic.
Also closes three consequential handoff risks the same review raised:
* mediator-N pinned itself hard to worker-N while sharing a ReadWriteOnce
claim, so a drain or preemption that moved only the lower-priority worker
deadlocked the ordinal on Multi-Attach until an operator deleted a Pod. The
shared workspace is now ReadWriteMany (as the hermes-chat tenant workspaces
already are on the same class), colocation is a preference, and the mediator
shares the worker's preemption priority, so each Pod reschedules on its own.
* the broker permits only branch creation, so a retry that added commits could
never submit and the run's work was discarded with the failure. Submission
now targets a fresh attempt- or content-scoped ref in the same reviewed
namespace -- never an update -- and is idempotent under replay. A refused
submission downgrades the result and says why instead of unwinding the run.
* the provider CLIs were reinstalled into an emptyDir on every Pod start inside
the 10m Flux health window for the whole hermes app. They now install once
per pinned version onto a durable volume, re-verified against the real
binaries and time-bounded, and the best-effort pool no longer gates the
health of the app its dependents wait on.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 16:31:15 +00:00
|
|
|
class FailingSCM:
|
|
|
|
|
def __init__(self, error):
|
|
|
|
|
self.error = error
|
|
|
|
|
self.calls = 0
|
|
|
|
|
|
|
|
|
|
def submit(self, _assignment, _request):
|
|
|
|
|
self.calls += 1
|
|
|
|
|
raise self.error
|
|
|
|
|
|
2026-09-13 17:54:54 -05:00
|
|
|
def resume_artifact(self, _assignment, _request, _structured):
|
|
|
|
|
return {"bounded": "resume-artifact"}
|
|
|
|
|
|
hermes: make pool lease recovery and release isolation safe
Independent review t_5975c06a blocked this branch on a P1: a Kanban write that
failed while a lease expired left a `lease_failed` row that was invisible to
every pass, immortal to garbage collection, and fatal to the coordinator. It
poisoned `reconcile()` forever with a conflicting-duplicate primary key,
produced a spurious capability `block_task` from `dispatch()`, and -- because
startup maintenance ran unguarded before the port bound, against a store on a
PVC -- crash-looped the coordinator with no automatic recovery.
`lease_failed` is now a retryable state that every maintenance pass drains, and
a row only reaches a terminal state on authoritative evidence about its exact
Kanban run, so nothing is collected before its outcome is known and nothing is
silently dropped. Each row, task, and board is processed in isolation, and a
coordinator-side fault is never converted into a Kanban mutation. Startup runs
through the same guarded cycle as the steady-state loop.
The wire protocol and the durable store are now separate modules, and the
maintenance passes moved out of the coordinator, so each file stays under the
managed line ceiling with room for the recovery logic.
Also closes three consequential handoff risks the same review raised:
* mediator-N pinned itself hard to worker-N while sharing a ReadWriteOnce
claim, so a drain or preemption that moved only the lower-priority worker
deadlocked the ordinal on Multi-Attach until an operator deleted a Pod. The
shared workspace is now ReadWriteMany (as the hermes-chat tenant workspaces
already are on the same class), colocation is a preference, and the mediator
shares the worker's preemption priority, so each Pod reschedules on its own.
* the broker permits only branch creation, so a retry that added commits could
never submit and the run's work was discarded with the failure. Submission
now targets a fresh attempt- or content-scoped ref in the same reviewed
namespace -- never an update -- and is idempotent under replay. A refused
submission downgrades the result and says why instead of unwinding the run.
* the provider CLIs were reinstalled into an emptyDir on every Pod start inside
the 10m Flux health window for the whole hermes app. They now install once
per pinned version onto a durable volume, re-verified against the real
binaries and time-bounded, and the best-effort pool no longer gates the
health of the app its dependents wait on.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 16:31:15 +00:00
|
|
|
|
|
|
|
|
def test_a_refused_submission_downgrades_the_result_instead_of_discarding_it():
|
|
|
|
|
failing = FailingSCM(protocol.ProtocolError("every reviewed branch name is taken"))
|
|
|
|
|
boundary = client.ClientBoundary(KEY, failing)
|
|
|
|
|
boundary.current = assignment()
|
|
|
|
|
boundary._post = lambda _path, _envelope: protocol.sign_envelope(
|
|
|
|
|
KEY, "ack", binding(), {"accepted": True, "duplicate": False}
|
|
|
|
|
)
|
|
|
|
|
finished = boundary.finish(
|
|
|
|
|
{
|
|
|
|
|
"binding": binding(),
|
|
|
|
|
"payload": {"structured": dict(RESULT), "returncode": 0},
|
|
|
|
|
"title": "safe", "body": "evidence",
|
|
|
|
|
}
|
|
|
|
|
)
|
|
|
|
|
assert failing.calls == 1
|
|
|
|
|
structured = finished["structured"]
|
|
|
|
|
# The run is recorded rather than unwound, and it says exactly why.
|
|
|
|
|
assert structured["status"] == "blocked"
|
|
|
|
|
assert structured["changed_files"] == ["safe.py"]
|
|
|
|
|
assert len(structured["blockers"]) == 1
|
|
|
|
|
assert "commits remain on this ordinal's workspace" in structured["blockers"][0]
|
|
|
|
|
assert "every reviewed branch name is taken" in structured["blockers"][0]
|
|
|
|
|
assert finished["ack"]["accepted"] is True
|
|
|
|
|
|
|
|
|
|
|
2026-09-13 17:54:54 -05:00
|
|
|
def test_a_refused_submission_is_transient_for_pool_recovery():
|
|
|
|
|
failing = FailingSCM(protocol.ProtocolError("SCM broker rejected the branch update"))
|
|
|
|
|
boundary = client.ClientBoundary(KEY, failing)
|
|
|
|
|
boundary.current = assignment(continuation_kind="repair", root_task_id="t_deadbeef")
|
|
|
|
|
posted = []
|
|
|
|
|
|
|
|
|
|
def post(_path, envelope):
|
|
|
|
|
posted.append(envelope)
|
|
|
|
|
return protocol.sign_envelope(
|
|
|
|
|
KEY, "ack", binding(), {"accepted": True, "duplicate": False}
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
boundary._post = post
|
|
|
|
|
finished = boundary.finish(
|
|
|
|
|
{
|
|
|
|
|
"binding": binding(),
|
|
|
|
|
"payload": {"structured": dict(RESULT), "returncode": 0, "capacity_failure": False},
|
|
|
|
|
"title": "safe", "body": "evidence",
|
|
|
|
|
}
|
|
|
|
|
)
|
|
|
|
|
assert finished["ack"]["accepted"] is True
|
|
|
|
|
assert finished["structured"]["status"] == "blocked"
|
|
|
|
|
terminal = protocol.verify_envelope(KEY, posted[0])["payload"]
|
|
|
|
|
assert terminal["capacity_failure"] is True
|
|
|
|
|
assert terminal["scm_resume"] == {"bounded": "resume-artifact"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_ordinary_submission_failure_never_emits_a_continuation_receipt():
|
|
|
|
|
boundary = client.ClientBoundary(KEY, FailingSCM(protocol.ProtocolError("refused")))
|
|
|
|
|
boundary.current = assignment()
|
|
|
|
|
posted = []
|
|
|
|
|
boundary._post = lambda _path, envelope: posted.append(envelope) or protocol.sign_envelope(
|
|
|
|
|
KEY, "ack", binding(), {"accepted": True, "duplicate": False}
|
|
|
|
|
)
|
|
|
|
|
boundary.finish({"binding": binding(), "payload": {"structured": dict(RESULT), "returncode": 0}, "title": "safe", "body": "evidence"})
|
|
|
|
|
terminal = protocol.verify_envelope(KEY, posted[0])["payload"]
|
|
|
|
|
assert terminal["capacity_failure"] is True and "scm_resume" not in terminal
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_real_resume_receipt_keeps_completed_evidence_after_submission_mutates_result(tmp_path, monkeypatch):
|
|
|
|
|
"""The signed receipt must not alias the blocked terminal result in memory."""
|
|
|
|
|
monkeypatch.setattr(scm, "WORKSPACE_ROOT", tmp_path / "workspace")
|
|
|
|
|
monkeypatch.setattr(scm, "SCM_ROOT", tmp_path / "state")
|
|
|
|
|
monkeypatch.setattr(scm, "ORDINAL", 0)
|
|
|
|
|
exact = protocol.sign_envelope(
|
|
|
|
|
KEY, "assignment", binding(), payload(root_task_id="t_deadbeef", continuation_kind="repair")
|
|
|
|
|
)
|
|
|
|
|
destination = scm.workspace_path(exact)
|
|
|
|
|
destination.mkdir(parents=True)
|
|
|
|
|
protocol.atomic_json(scm._state_path(exact), {"baseline_sha": "a" * 40})
|
|
|
|
|
monkeypatch.setattr(scm, "_workspace_identity", lambda *_args: "b" * 40)
|
|
|
|
|
monkeypatch.setattr(scm, "_run", lambda *args, **_kwargs: "")
|
|
|
|
|
broker = scm.Boundary(KEY)
|
|
|
|
|
monkeypatch.setattr(
|
|
|
|
|
broker, "submit", lambda *_args, **_kwargs: (_ for _ in ()).throw(protocol.ProtocolError("refused"))
|
|
|
|
|
)
|
|
|
|
|
boundary = client.ClientBoundary(KEY, broker)
|
|
|
|
|
boundary.current = exact
|
|
|
|
|
posted = []
|
|
|
|
|
boundary._post = lambda _path, envelope: posted.append(envelope) or protocol.sign_envelope(
|
|
|
|
|
KEY, "ack", binding(), {"accepted": True, "duplicate": False}
|
|
|
|
|
)
|
|
|
|
|
boundary.finish({"binding": binding(), "payload": {"structured": dict(RESULT), "returncode": 0}, "title": "safe", "body": "evidence"})
|
|
|
|
|
result_envelope = next(item for item in posted if item["kind"] == "result")
|
|
|
|
|
artifact = protocol.verify_envelope(KEY, result_envelope)["payload"]["scm_resume"]
|
|
|
|
|
assert artifact["structured"]["status"] == "completed"
|
|
|
|
|
assert artifact["result_digest"] == __import__("hashlib").sha256(
|
|
|
|
|
protocol.canonical_json({name: artifact[name] for name in ("structured", "title", "body")})
|
|
|
|
|
).hexdigest()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_mediator_bootstrap_recovers_one_unique_completed_log_result(tmp_path, monkeypatch):
|
|
|
|
|
exact = protocol.sign_envelope(
|
|
|
|
|
KEY, "assignment", binding(), payload(root_task_id="t_deadbeef", continuation_kind="repair")
|
|
|
|
|
)
|
|
|
|
|
log = tmp_path / "session-state" / "metis" / "t_deadbeef" / "42" / ".log"
|
|
|
|
|
# The live worker uses `<run>.log`; make the same task-bound parent safely.
|
|
|
|
|
log = log.parent.parent / "42.log"
|
|
|
|
|
log.parent.mkdir(parents=True)
|
|
|
|
|
completed = dict(RESULT)
|
|
|
|
|
log.write_bytes(protocol.canonical_json(completed) + b"\nnoise\n" + protocol.canonical_json(completed) + b"\n")
|
|
|
|
|
monkeypatch.setattr(resume_bootstrap, "ROOT", tmp_path)
|
|
|
|
|
assert resume_bootstrap._completed_result(exact) == completed
|
|
|
|
|
seen = {}
|
|
|
|
|
monkeypatch.setattr(
|
|
|
|
|
scm.Boundary, "resume_artifact",
|
|
|
|
|
lambda _self, assignment, request, structured: seen.update(
|
|
|
|
|
assignment=assignment, request=request, structured=structured
|
|
|
|
|
) or {"receipt": "only"},
|
|
|
|
|
)
|
|
|
|
|
assert resume_bootstrap.bootstrap(KEY, exact) == {"receipt": "only"}
|
|
|
|
|
assert seen["structured"] == completed and seen["request"]["title"] == completed["summary"]
|
|
|
|
|
|
|
|
|
|
|
2026-09-13 18:09:47 -05:00
|
|
|
def test_mediator_bootstrap_recovers_one_unique_nested_session_result(tmp_path, monkeypatch):
|
|
|
|
|
"""A Claude stream carries the completed result under structured_output."""
|
|
|
|
|
exact = protocol.sign_envelope(
|
|
|
|
|
KEY, "assignment", binding(), payload(root_task_id="t_deadbeef", continuation_kind="repair")
|
|
|
|
|
)
|
|
|
|
|
log = tmp_path / "session-state" / "metis" / "t_deadbeef" / "42.log"
|
|
|
|
|
log.parent.mkdir(parents=True)
|
|
|
|
|
completed = dict(RESULT)
|
|
|
|
|
event = {"event": "result", "structured_output": completed}
|
|
|
|
|
log.write_bytes(protocol.canonical_json(event) + b"\n" + protocol.canonical_json(event) + b"\n")
|
|
|
|
|
monkeypatch.setattr(resume_bootstrap, "ROOT", tmp_path)
|
|
|
|
|
|
|
|
|
|
assert resume_bootstrap._completed_result(exact) == completed
|
|
|
|
|
|
|
|
|
|
|
2026-09-13 18:17:39 -05:00
|
|
|
def test_mediator_bootstrap_truncates_multibyte_title_at_the_byte_limit(tmp_path, monkeypatch):
|
|
|
|
|
"""Receipt title limits count UTF-8 bytes while preserving full evidence."""
|
|
|
|
|
exact = protocol.sign_envelope(
|
|
|
|
|
KEY, "assignment", binding(), payload(root_task_id="t_deadbeef", continuation_kind="repair")
|
|
|
|
|
)
|
|
|
|
|
log = tmp_path / "session-state" / "metis" / "t_deadbeef" / "42.log"
|
|
|
|
|
log.parent.mkdir(parents=True)
|
|
|
|
|
completed = {**RESULT, "summary": "界" * 200}
|
|
|
|
|
log.write_bytes(protocol.canonical_json({"structured_output": completed}) + b"\n")
|
|
|
|
|
monkeypatch.setattr(resume_bootstrap, "ROOT", tmp_path)
|
|
|
|
|
seen = {}
|
|
|
|
|
monkeypatch.setattr(
|
|
|
|
|
scm.Boundary, "resume_artifact",
|
|
|
|
|
lambda _self, _assignment, request, structured: seen.update(
|
|
|
|
|
request=request, structured=structured
|
|
|
|
|
) or {},
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
resume_bootstrap.bootstrap(KEY, exact)
|
|
|
|
|
|
2026-09-13 18:30:05 -05:00
|
|
|
assert len(seen["request"]["title"]) <= 240
|
|
|
|
|
assert len(seen["request"]["title"].encode()) <= 507
|
|
|
|
|
assert gitea_policy._draft_title(seen["request"]["title"]) == "WIP: " + seen["request"]["title"]
|
2026-09-13 18:17:39 -05:00
|
|
|
assert seen["structured"]["summary"] == completed["summary"]
|
|
|
|
|
|
|
|
|
|
|
2026-09-13 18:30:05 -05:00
|
|
|
@pytest.mark.parametrize("summary", ["x" * 300, "界" * 300, "Ж" * 300])
|
|
|
|
|
def test_mediator_bootstrap_titles_are_accepted_by_gitea_draft_policy(tmp_path, monkeypatch, summary):
|
|
|
|
|
"""Resume titles leave room for Gitea's draft prefix in both limits."""
|
|
|
|
|
exact = protocol.sign_envelope(
|
|
|
|
|
KEY, "assignment", binding(), payload(root_task_id="t_deadbeef", continuation_kind="repair")
|
|
|
|
|
)
|
|
|
|
|
log = tmp_path / "session-state" / "metis" / "t_deadbeef" / "42.log"
|
|
|
|
|
log.parent.mkdir(parents=True)
|
|
|
|
|
completed = {**RESULT, "summary": summary}
|
|
|
|
|
log.write_bytes(protocol.canonical_json({"structured_output": completed}) + b"\n")
|
|
|
|
|
monkeypatch.setattr(resume_bootstrap, "ROOT", tmp_path)
|
|
|
|
|
seen = {}
|
|
|
|
|
monkeypatch.setattr(
|
|
|
|
|
scm.Boundary, "resume_artifact",
|
|
|
|
|
lambda _self, _assignment, request, _structured: seen.update(request=request) or {},
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
resume_bootstrap.bootstrap(KEY, exact)
|
|
|
|
|
|
|
|
|
|
title = seen["request"]["title"]
|
|
|
|
|
assert len(title) <= 240 and len(title.encode()) <= 507
|
|
|
|
|
assert gitea_policy._draft_title(title) == f"WIP: {title}"
|
|
|
|
|
|
|
|
|
|
|
2026-09-13 17:54:54 -05:00
|
|
|
def test_mediator_bootstrap_rejects_a_symlinked_log_ancestor(tmp_path, monkeypatch):
|
|
|
|
|
exact = protocol.sign_envelope(KEY, "assignment", binding(), payload())
|
|
|
|
|
outside = tmp_path / "outside"
|
|
|
|
|
outside.mkdir()
|
|
|
|
|
session_root = tmp_path / "session-state"
|
|
|
|
|
session_root.mkdir()
|
|
|
|
|
(session_root / "metis").symlink_to(outside, target_is_directory=True)
|
|
|
|
|
monkeypatch.setattr(resume_bootstrap, "ROOT", tmp_path)
|
|
|
|
|
with pytest.raises(protocol.ProtocolError, match="unsafe"):
|
|
|
|
|
resume_bootstrap._completed_result(exact)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_publication_lease_signs_the_exact_binding_and_stops_cleanly():
|
|
|
|
|
boundary = client.ClientBoundary(KEY, FailingSCM(protocol.ProtocolError("unused")))
|
|
|
|
|
calls = []
|
|
|
|
|
|
|
|
|
|
def post(path, envelope):
|
|
|
|
|
calls.append((path, envelope))
|
|
|
|
|
return protocol.sign_envelope(KEY, "ack", binding(), {"accepted": True})
|
|
|
|
|
|
|
|
|
|
boundary._post = post
|
|
|
|
|
checkpoint, close = boundary._publication_lease(binding())
|
|
|
|
|
checkpoint()
|
|
|
|
|
close()
|
|
|
|
|
assert len(calls) == 1 and calls[0][0] == "/v1/heartbeat"
|
|
|
|
|
verified = protocol.verify_envelope(KEY, calls[0][1], expected_kind="heartbeat")
|
|
|
|
|
assert {name: verified[name] for name in binding()} == binding()
|
|
|
|
|
|
|
|
|
|
|
2026-09-13 18:30:05 -05:00
|
|
|
def test_publication_lease_preserves_a_fixed_coordinator_conflict_category():
|
|
|
|
|
"""An initial coordinator conflict remains observable at the mediator boundary."""
|
|
|
|
|
boundary = client.ClientBoundary(KEY, FailingSCM(protocol.ProtocolError("unused")))
|
|
|
|
|
boundary._post = lambda *_args: (_ for _ in ()).throw(client.CoordinatorRejected("lease"))
|
|
|
|
|
|
|
|
|
|
checkpoint, close = boundary._publication_lease(binding())
|
|
|
|
|
try:
|
|
|
|
|
with pytest.raises(client.CoordinatorRejected, match="lease") as rejected:
|
|
|
|
|
checkpoint()
|
|
|
|
|
assert rejected.value.category == "lease"
|
|
|
|
|
finally:
|
|
|
|
|
close()
|
|
|
|
|
|
|
|
|
|
|
2026-09-13 17:54:54 -05:00
|
|
|
def test_publication_lease_renews_during_a_slow_scm_step_without_sleeping(monkeypatch):
|
|
|
|
|
boundary = client.ClientBoundary(KEY, FailingSCM(protocol.ProtocolError("unused")))
|
|
|
|
|
calls = []
|
|
|
|
|
|
|
|
|
|
class Event:
|
|
|
|
|
def __init__(self):
|
|
|
|
|
self.waits = 0
|
|
|
|
|
|
|
|
|
|
def wait(self, _seconds):
|
|
|
|
|
self.waits += 1
|
|
|
|
|
return self.waits > 1
|
|
|
|
|
|
|
|
|
|
def set(self):
|
|
|
|
|
return None
|
|
|
|
|
|
|
|
|
|
class Thread:
|
|
|
|
|
def __init__(self, *, target, daemon):
|
|
|
|
|
self.target = target
|
|
|
|
|
|
|
|
|
|
def start(self):
|
|
|
|
|
self.target()
|
|
|
|
|
|
|
|
|
|
def join(self, timeout):
|
|
|
|
|
assert timeout == 1
|
|
|
|
|
|
|
|
|
|
monkeypatch.setattr(client.threading, "Event", Event)
|
|
|
|
|
monkeypatch.setattr(client.threading, "Thread", Thread)
|
|
|
|
|
boundary._post = lambda path, envelope: calls.append((path, envelope)) or protocol.sign_envelope(
|
|
|
|
|
KEY, "ack", binding(), {"accepted": True}
|
|
|
|
|
)
|
|
|
|
|
_checkpoint, close = boundary._publication_lease(binding())
|
|
|
|
|
close()
|
|
|
|
|
assert [path for path, _envelope in calls] == ["/v1/heartbeat", "/v1/heartbeat"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_lost_publication_checkpoint_prevents_the_push(tmp_path, monkeypatch):
|
|
|
|
|
boundary, exact, calls, _head = submit_harness(tmp_path, monkeypatch, {"wt/t_deadbeef": "a" * 40})
|
|
|
|
|
count = 0
|
|
|
|
|
|
|
|
|
|
def checkpoint():
|
|
|
|
|
nonlocal count
|
|
|
|
|
count += 1
|
|
|
|
|
if count == 3:
|
|
|
|
|
raise protocol.ProtocolError("SCM publication lease was lost")
|
|
|
|
|
|
|
|
|
|
with pytest.raises(protocol.ProtocolError, match="lease was lost"):
|
|
|
|
|
boundary.submit(exact, {"title": "safe", "body": "evidence"}, checkpoint=checkpoint)
|
|
|
|
|
assert not any("push" in call for call in calls)
|
|
|
|
|
|
|
|
|
|
|
2026-09-13 19:11:35 -05:00
|
|
|
def test_worker_cannot_sign_forged_scm_control_on_blocked_or_failed_finish():
|
|
|
|
|
"""Only private mediator SCM work may add terminal publication evidence."""
|
|
|
|
|
forged = {
|
|
|
|
|
"scm_submission": {"branch": "attacker/ref", "pull_request": "https://scm/pulls/99", "head": "f" * 40},
|
|
|
|
|
"scm_resume": {"result_digest": "e" * 64},
|
|
|
|
|
"publication_retry_transient": True,
|
|
|
|
|
"publication_retry_lease_fence": "forged-fence", "lease_fence": "forged-lease",
|
|
|
|
|
}
|
|
|
|
|
boundary = client.ClientBoundary(KEY, FailingSCM(protocol.ProtocolError("refused")))
|
|
|
|
|
boundary.current = assignment()
|
|
|
|
|
posted = []
|
|
|
|
|
boundary._post = lambda _path, envelope: posted.append(envelope) or protocol.sign_envelope(
|
|
|
|
|
KEY, "ack", binding(), {"accepted": True, "duplicate": False}
|
|
|
|
|
)
|
|
|
|
|
blocked = {**RESULT, "status": "blocked", "blockers": ["worker policy"]}
|
|
|
|
|
boundary.finish({"binding": binding(), "payload": {"structured": blocked, "returncode": 1, **forged}})
|
|
|
|
|
terminal = protocol.verify_envelope(KEY, posted.pop())["payload"]
|
|
|
|
|
assert not MEDIATOR_FIELDS.intersection(terminal)
|
|
|
|
|
|
|
|
|
|
boundary.current = assignment()
|
|
|
|
|
boundary.finish({"binding": binding(), "payload": {"structured": dict(RESULT), "returncode": 0, **forged}})
|
|
|
|
|
terminal = protocol.verify_envelope(KEY, posted.pop())["payload"]
|
|
|
|
|
assert terminal["capacity_failure"] is True
|
|
|
|
|
assert not MEDIATOR_FIELDS.intersection(terminal)
|
|
|
|
|
|
|
|
|
|
|
2026-09-13 17:54:54 -05:00
|
|
|
def test_worker_cannot_spoof_a_transient_publication_retry_result():
|
|
|
|
|
boundary = client.ClientBoundary(KEY, FailingSCM(protocol.ProtocolError("unused")))
|
|
|
|
|
boundary.current = assignment(payload={"scm_resume": {}})
|
|
|
|
|
posted = []
|
|
|
|
|
boundary._post = lambda _path, envelope: posted.append(envelope) or protocol.sign_envelope(
|
|
|
|
|
KEY, "ack", binding(), {"accepted": True, "duplicate": False}
|
|
|
|
|
)
|
|
|
|
|
blocked = {**RESULT, "status": "blocked", "blockers": ["policy refusal"]}
|
|
|
|
|
boundary.finish({"binding": binding(), "payload": {
|
|
|
|
|
"structured": blocked, "returncode": 1, "publication_retry_transient": True,
|
|
|
|
|
}})
|
|
|
|
|
terminal = protocol.verify_envelope(KEY, posted[0])["payload"]
|
|
|
|
|
assert "publication_retry_transient" not in terminal
|
|
|
|
|
|
|
|
|
|
|
hermes: make pool lease recovery and release isolation safe
Independent review t_5975c06a blocked this branch on a P1: a Kanban write that
failed while a lease expired left a `lease_failed` row that was invisible to
every pass, immortal to garbage collection, and fatal to the coordinator. It
poisoned `reconcile()` forever with a conflicting-duplicate primary key,
produced a spurious capability `block_task` from `dispatch()`, and -- because
startup maintenance ran unguarded before the port bound, against a store on a
PVC -- crash-looped the coordinator with no automatic recovery.
`lease_failed` is now a retryable state that every maintenance pass drains, and
a row only reaches a terminal state on authoritative evidence about its exact
Kanban run, so nothing is collected before its outcome is known and nothing is
silently dropped. Each row, task, and board is processed in isolation, and a
coordinator-side fault is never converted into a Kanban mutation. Startup runs
through the same guarded cycle as the steady-state loop.
The wire protocol and the durable store are now separate modules, and the
maintenance passes moved out of the coordinator, so each file stays under the
managed line ceiling with room for the recovery logic.
Also closes three consequential handoff risks the same review raised:
* mediator-N pinned itself hard to worker-N while sharing a ReadWriteOnce
claim, so a drain or preemption that moved only the lower-priority worker
deadlocked the ordinal on Multi-Attach until an operator deleted a Pod. The
shared workspace is now ReadWriteMany (as the hermes-chat tenant workspaces
already are on the same class), colocation is a preference, and the mediator
shares the worker's preemption priority, so each Pod reschedules on its own.
* the broker permits only branch creation, so a retry that added commits could
never submit and the run's work was discarded with the failure. Submission
now targets a fresh attempt- or content-scoped ref in the same reviewed
namespace -- never an update -- and is idempotent under replay. A refused
submission downgrades the result and says why instead of unwinding the run.
* the provider CLIs were reinstalled into an emptyDir on every Pod start inside
the 10m Flux health window for the whole hermes app. They now install once
per pinned version onto a durable volume, re-verified against the real
binaries and time-bounded, and the best-effort pool no longer gates the
health of the app its dependents wait on.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 16:31:15 +00:00
|
|
|
def test_a_successful_submission_records_both_the_draft_and_the_exact_branch():
|
|
|
|
|
class Recording:
|
|
|
|
|
def submit(self, _assignment, _request):
|
|
|
|
|
return {
|
|
|
|
|
"workspace": "/workspace/runs/metis/t_deadbeef/42",
|
|
|
|
|
"branch": "wt/t_deadbeef-attempt-2",
|
2026-09-13 19:11:35 -05:00
|
|
|
"pull_request": "https://scm/pulls/9", "head": "a" * 40,
|
hermes: make pool lease recovery and release isolation safe
Independent review t_5975c06a blocked this branch on a P1: a Kanban write that
failed while a lease expired left a `lease_failed` row that was invisible to
every pass, immortal to garbage collection, and fatal to the coordinator. It
poisoned `reconcile()` forever with a conflicting-duplicate primary key,
produced a spurious capability `block_task` from `dispatch()`, and -- because
startup maintenance ran unguarded before the port bound, against a store on a
PVC -- crash-looped the coordinator with no automatic recovery.
`lease_failed` is now a retryable state that every maintenance pass drains, and
a row only reaches a terminal state on authoritative evidence about its exact
Kanban run, so nothing is collected before its outcome is known and nothing is
silently dropped. Each row, task, and board is processed in isolation, and a
coordinator-side fault is never converted into a Kanban mutation. Startup runs
through the same guarded cycle as the steady-state loop.
The wire protocol and the durable store are now separate modules, and the
maintenance passes moved out of the coordinator, so each file stays under the
managed line ceiling with room for the recovery logic.
Also closes three consequential handoff risks the same review raised:
* mediator-N pinned itself hard to worker-N while sharing a ReadWriteOnce
claim, so a drain or preemption that moved only the lower-priority worker
deadlocked the ordinal on Multi-Attach until an operator deleted a Pod. The
shared workspace is now ReadWriteMany (as the hermes-chat tenant workspaces
already are on the same class), colocation is a preference, and the mediator
shares the worker's preemption priority, so each Pod reschedules on its own.
* the broker permits only branch creation, so a retry that added commits could
never submit and the run's work was discarded with the failure. Submission
now targets a fresh attempt- or content-scoped ref in the same reviewed
namespace -- never an update -- and is idempotent under replay. A refused
submission downgrades the result and says why instead of unwinding the run.
* the provider CLIs were reinstalled into an emptyDir on every Pod start inside
the 10m Flux health window for the whole hermes app. They now install once
per pinned version onto a durable volume, re-verified against the real
binaries and time-bounded, and the best-effort pool no longer gates the
health of the app its dependents wait on.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 16:31:15 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
boundary = client.ClientBoundary(KEY, Recording())
|
|
|
|
|
boundary.current = assignment()
|
2026-09-13 19:11:35 -05:00
|
|
|
posted = []
|
|
|
|
|
boundary._post = lambda _path, envelope: posted.append(envelope) or protocol.sign_envelope(
|
hermes: make pool lease recovery and release isolation safe
Independent review t_5975c06a blocked this branch on a P1: a Kanban write that
failed while a lease expired left a `lease_failed` row that was invisible to
every pass, immortal to garbage collection, and fatal to the coordinator. It
poisoned `reconcile()` forever with a conflicting-duplicate primary key,
produced a spurious capability `block_task` from `dispatch()`, and -- because
startup maintenance ran unguarded before the port bound, against a store on a
PVC -- crash-looped the coordinator with no automatic recovery.
`lease_failed` is now a retryable state that every maintenance pass drains, and
a row only reaches a terminal state on authoritative evidence about its exact
Kanban run, so nothing is collected before its outcome is known and nothing is
silently dropped. Each row, task, and board is processed in isolation, and a
coordinator-side fault is never converted into a Kanban mutation. Startup runs
through the same guarded cycle as the steady-state loop.
The wire protocol and the durable store are now separate modules, and the
maintenance passes moved out of the coordinator, so each file stays under the
managed line ceiling with room for the recovery logic.
Also closes three consequential handoff risks the same review raised:
* mediator-N pinned itself hard to worker-N while sharing a ReadWriteOnce
claim, so a drain or preemption that moved only the lower-priority worker
deadlocked the ordinal on Multi-Attach until an operator deleted a Pod. The
shared workspace is now ReadWriteMany (as the hermes-chat tenant workspaces
already are on the same class), colocation is a preference, and the mediator
shares the worker's preemption priority, so each Pod reschedules on its own.
* the broker permits only branch creation, so a retry that added commits could
never submit and the run's work was discarded with the failure. Submission
now targets a fresh attempt- or content-scoped ref in the same reviewed
namespace -- never an update -- and is idempotent under replay. A refused
submission downgrades the result and says why instead of unwinding the run.
* the provider CLIs were reinstalled into an emptyDir on every Pod start inside
the 10m Flux health window for the whole hermes app. They now install once
per pinned version onto a durable volume, re-verified against the real
binaries and time-bounded, and the best-effort pool no longer gates the
health of the app its dependents wait on.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 16:31:15 +00:00
|
|
|
KEY, "ack", binding(), {"accepted": True, "duplicate": False}
|
|
|
|
|
)
|
|
|
|
|
finished = boundary.finish(
|
|
|
|
|
{
|
|
|
|
|
"binding": binding(),
|
|
|
|
|
"payload": {"structured": dict(RESULT), "returncode": 0},
|
|
|
|
|
"title": "safe", "body": "evidence",
|
|
|
|
|
}
|
|
|
|
|
)
|
|
|
|
|
assert finished["structured"]["status"] == "completed"
|
|
|
|
|
assert finished["structured"]["artifacts"] == [
|
|
|
|
|
"https://scm/pulls/9", "branch:wt/t_deadbeef-attempt-2",
|
|
|
|
|
]
|
2026-09-13 19:11:35 -05:00
|
|
|
terminal = protocol.verify_envelope(KEY, posted[0])["payload"]
|
|
|
|
|
assert terminal["scm_submission"] == {
|
|
|
|
|
"branch": "wt/t_deadbeef-attempt-2", "pull_request": "https://scm/pulls/9", "head": "a" * 40,
|
|
|
|
|
}
|