hermes: reserve PR follow-ups for mediated workers

This commit is contained in:
jenkins 2026-09-13 16:15:50 -05:00
parent 74d8485f6a
commit 13a7dbab46
12 changed files with 230 additions and 47 deletions

View File

@ -28,6 +28,7 @@ spec:
operator: NotIn
values:
- titan-04
- titan-08
- titan-14
- titan-18
- titan-19

View File

@ -415,16 +415,18 @@ data:
Use `--component agent` for backend/runtime changes, `--component webui`
for chat UI changes, and `--component stt` or `--component tts` for the
private voice services, passing a full commit already contained by `main`.
Jenkins builds the newest main containing that commit, publishes a final
immutable release tag only after its evidence passes, and Flux applies the
resulting digest. Candidate tags and failed builds never deploy. Run the
Jenkins verifies that the requested commit is contained by main, then
checks out and builds that exact commit. It publishes a final immutable
release tag only after its evidence passes, and Flux applies the resulting
digest. Candidate tags and failed builds never deploy. Run the
safe `follow_command` returned by the trigger and do not report completion
until it says `converged`: that proves the exact reviewed source is the
selected immutable tag/digest, Flux is Ready, every consumer has the digest
in desired state, and all corresponding pods are Ready on that digest. If
main advanced before Jenkins checked it out, the status command reports
`different_revision_selected`; independently prove the requested commit is
an ancestor of that exact source, then follow the exact selected revision.
the status command reports `different_revision_selected`, treat it as a
release mismatch: stop, investigate the selected revision, and do not
follow or deploy that different SHA. Re-establish convergence for the
requested exact commit instead.
Use `jenkins_build_evidence.py` for a terminal failure cause when its
bounded controller evidence path is available. Candidate-only, queued, or
merged code is not a completed release. Finally verify the public SSO

View File

@ -25,7 +25,7 @@ spec:
ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers
ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback
ai.bstein.dev/placement: primary amd64 accelerator titan-22; arm64 rpi5 fleet fallback; storage-backbone nodes excluded
ai.bstein.dev/config-rev: "20260913-soteria-kanban-recovery-v1"
ai.bstein.dev/config-rev: "20260913-soteria-kanban-recovery-v2"
prometheus.io/scrape: "true"
prometheus.io/path: /metrics
prometheus.io/port: "9010"
@ -287,26 +287,6 @@ spec:
resources:
requests: {cpu: 25m, memory: 32Mi}
limits: {cpu: 100m, memory: 64Mi}
- name: recover-soteria-kanban
image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
imagePullPolicy: IfNotPresent
command:
- /opt/hermes/.venv/bin/python
- /opt/coordinator/recover_soteria_kanban.py
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: [ALL]
runAsUser: 10000
runAsGroup: 10000
seccompProfile:
type: RuntimeDefault
volumeMounts:
- {name: home, mountPath: /opt/data}
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
resources:
requests: {cpu: 25m, memory: 64Mi}
limits: {cpu: 250m, memory: 256Mi}
# Worker HUX identity provisioning. Mirrors the chat tenants'
# init-hux-runtime, adapted to the single-replica Deployment: the
# subtree name is the fixed literal "hux" (Deployment pod names churn,

View File

@ -2,6 +2,12 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: hermes
metadata:
annotations:
hermes.bstein.dev/agent-image-release-hold: "true"
hermes.bstein.dev/agent-image-release-hold-digest: "sha256:bebfe9c24cdd877ed2c16335027ccf393892161642756f54e8d12984eefdd903"
hermes.bstein.dev/agent-image-release-hold-minimum-source: "106ee8ce14aa67c3825fca6fb537a557b44ecd86"
hermes.bstein.dev/agent-image-release-hold-reason: "Hold the verified SQLite 3.51.3 runtime until the normal Hermes release includes it."
images:
- name: registry.bstein.dev/bstein/hermes-agent
# Hold this verified SQLite fix until the normal image release includes it.

View File

@ -5,6 +5,7 @@ from __future__ import annotations
import concurrent.futures
import os
import sqlite3
import sys
import time
from collections.abc import Callable
@ -29,6 +30,9 @@ from cli_lane_files import atomic_json
from cli_lane_metrics import start_metrics_server
from cli_lane_recovery import _has_pending_finalization, recover_pending_finalizations
from cli_lane_retention import maybe_gc_lane_artifacts
import supervisor_state
OWNED_WORKSPACES_ONLY = os.environ.get("HERMES_CLI_LANE_OWNED_WORKSPACES_ONLY", "").lower() == "true"
def _board_slug(board: Any) -> str:
@ -46,6 +50,47 @@ def _connect_healthy_board(kanban_db: Any, board: str) -> Any | None:
return None
def _direct_lane_eligible(board: str, task: Any) -> bool:
"""Reserve signed PR continuations for the mediated execution pool."""
task_id = str(_task_value(task, "id", "") or "")
if not task_id or (OWNED_WORKSPACES_ONLY and not str(_task_value(task, "workspace_path", "") or "").strip()):
return False
try:
return supervisor_state.get_child(board, task_id) is None
except (OSError, ValueError) as error:
_record_board_access_error(board, error)
return False
def _listed_boards(kanban_db: Any) -> list[Any]:
"""Fall back to board-directory discovery when one metadata read is unreadable."""
try:
return list(kanban_db.list_boards(include_archived=False))
except (OSError, sqlite3.Error) as error:
_record_board_access_error("board-registry", error)
root = getattr(kanban_db, "boards_root", None)
metadata = getattr(kanban_db, "read_board_metadata", None)
if not callable(root) or not callable(metadata):
return []
try:
candidates = [path for path in root().iterdir() if not path.is_symlink() and path.is_dir()]
except (OSError, sqlite3.Error) as error:
_record_board_access_error("board-registry", error)
return []
boards: list[Any] = []
for path in sorted(candidates, key=lambda item: item.name):
try:
if not ((path / "kanban.db").is_file() or (path / "board.json").is_file()):
continue
board = metadata(path.name)
except (OSError, sqlite3.Error, ValueError) as error:
_record_board_access_error(path.name, error)
continue
if isinstance(board, dict) and not board.get("archived") and _board_slug(board) == path.name:
boards.append(board)
return boards
def recover_orphans() -> None:
"""Return external running tasks to ready after a runner/pod restart."""
from hermes_cli import kanban_db
@ -53,10 +98,8 @@ def recover_orphans() -> None:
recover_pending_finalizations()
if not kanban_capabilities(kanban_db).exact_run_reclaim:
return
try:
boards = kanban_db.list_boards(include_archived=False)
except Exception as error:
_record_board_access_error("board-registry", error)
boards = _listed_boards(kanban_db)
if not boards:
return
for raw_board in boards:
board = _board_slug(raw_board)
@ -71,6 +114,8 @@ def recover_orphans() -> None:
if (
_external(task)
and str(_task_value(task, "status", "")) == "running"
and str(_task_value(task, "claim_lock", "") or "") == "direct-cli-lane"
and _direct_lane_eligible(board, task)
):
task_id = str(_task_value(task, "id"))
run_id = _task_value(task, "current_run_id", None)
@ -95,6 +140,7 @@ def claim_ready(
active: set[tuple[str, str]],
limit: int,
eligible: Callable[[str, Any], bool] | None = None,
claimer: str = "direct-cli-lane",
) -> list[tuple[str, str]]:
"""Atomically claim external ready tasks across all non-archived boards."""
from hermes_cli import kanban_db
@ -102,7 +148,7 @@ def claim_ready(
claimed: list[tuple[str, str]] = []
if limit <= 0:
return claimed
for raw_board in kanban_db.list_boards(include_archived=False):
for raw_board in _listed_boards(kanban_db):
board = _board_slug(raw_board)
if not board:
continue
@ -117,20 +163,23 @@ def claim_ready(
for task in tasks:
task_id = str(_task_value(task, "id", ""))
assignee = str(_task_value(task, "assignee", "") or "")
if (
not task_id
or (board, task_id) in active
or str(_task_value(task, "status", "")) != "ready"
or (eligible is not None and not eligible(board, task))
):
continue
if (
task_id
and not assignee
and str(_task_value(task, "status", "")) == "ready"
and kanban_db.assign_task(conn, task_id, "cli-auto")
):
task = kanban_db.get_task(conn, task_id)
assignee = "cli-auto"
if (
not task_id
or (board, task_id) in active
or not assignee.startswith(EXTERNAL_PREFIX)
or str(_task_value(task, "status", "")) != "ready"
or (eligible is not None and not eligible(board, task))
):
continue
try:
@ -138,7 +187,7 @@ def claim_ready(
conn,
task_id,
ttl_seconds=DEFAULT_CLAIM_TTL,
claimer="direct-cli-lane",
claimer=claimer,
)
except Exception:
continue
@ -190,7 +239,7 @@ def main() -> int:
active = set(futures.values())
try:
newly_claimed = (
claim_ready(active, workers - len(futures))
claim_ready(active, workers - len(futures), _direct_lane_eligible)
if capabilities.ready
else []
)

View File

@ -361,6 +361,7 @@ def dispatch(pool: Any) -> None:
claimed = cli_lane_dispatch.claim_ready(
set(), len(ordinals),
lambda _board, task: distributed_workspace_eligible(task),
claimer="execution-pool",
)
except RECOVERABLE_BOARD_ERRORS as error:
_defer("claim-ready", error)

View File

@ -305,6 +305,7 @@ def test_orphan_recovery_passes_the_scanned_run_as_atomic_reclaim_guard(
status="running",
assignee="cli-auto",
current_run_id=91,
claim_lock="direct-cli-lane",
),
SimpleNamespace(
id="t_no_run",
@ -330,6 +331,7 @@ def test_orphan_recovery_passes_the_scanned_run_as_atomic_reclaim_guard(
)
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0)
monkeypatch.setattr(lanes.supervisor_state, "get_child", lambda *_args: None)
lanes.recover_orphans()

View File

@ -101,6 +101,68 @@ def test_claim_ready_handles_zero_limit_empty_boards_and_claim_errors(monkeypatc
assert lanes.claim_ready(set(), 1) == []
def test_claim_ready_skips_an_unreadable_board_and_claims_a_healthy_fallback(tmp_path, monkeypatch):
"""A metadata failure on one board cannot starve healthy ready cards."""
root = tmp_path / "boards"
(root / "broken").mkdir(parents=True)
(root / "soteria").mkdir()
(root / "archived").mkdir()
for board in ("broken", "soteria", "archived"):
(root / board / "kanban.db").touch()
failures = []
task = SimpleNamespace(id="t_soteria", status="ready", assignee="cli-auto")
class Connection:
def __init__(self, board): self.board = board
def close(self): return None
def connect(*, board):
if board == "broken":
raise PermissionError("board database unreadable")
return Connection(board)
db = SimpleNamespace(
list_boards=lambda **_kwargs: (_ for _ in ()).throw(PermissionError("probe metadata")),
boards_root=lambda: root, scoped_current_board=lambda _board: nullcontext(),
read_board_metadata=lambda board: {"slug": board, "archived": board == "archived"},
connect=connect, recompute_ready=lambda _conn: None,
list_tasks=lambda conn: [task] if conn.board == "soteria" else [],
claim_task=lambda _conn, task_id, **_kwargs: task_id,
)
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db))
monkeypatch.setattr(lanes, "_record_board_access_error", lambda board, error: failures.append(board))
assert lanes.claim_ready(set(), 1) == [("soteria", "t_soteria")]
assert failures.count("board-registry") == 1 and failures.count("broken") == 1
def test_direct_lane_reserves_signed_continuations_for_the_mediator(monkeypatch):
task = SimpleNamespace(id="t_continue")
monkeypatch.setattr(lanes.supervisor_state, "get_child", lambda *_args: {"kind": "repair"})
assert lanes._direct_lane_eligible("soteria", task) is False
monkeypatch.setattr(lanes.supervisor_state, "get_child", lambda *_args: None)
assert lanes._direct_lane_eligible("soteria", task) is True
monkeypatch.setattr(lanes.supervisor_state, "get_child", lambda *_args: (_ for _ in ()).throw(OSError("state")))
assert lanes._direct_lane_eligible("soteria", task) is False
def test_direct_lane_owned_workspace_policy_precedes_unassigned_mutation(monkeypatch):
task = SimpleNamespace(id="t_pathless", workspace_path="", status="ready", assignee="")
monkeypatch.setattr(lanes, "OWNED_WORKSPACES_ONLY", True)
assert lanes._direct_lane_eligible("soteria", task) is False
assigned = []
db = SimpleNamespace(
list_boards=lambda **_kwargs: [{"slug": "soteria"}],
scoped_current_board=lambda _board: nullcontext(),
connect=lambda **_kwargs: SimpleNamespace(close=lambda: None),
recompute_ready=lambda _conn: None, list_tasks=lambda _conn: [task],
assign_task=lambda *_args: assigned.append(True),
)
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db))
assert lanes.claim_ready(set(), 1, lanes._direct_lane_eligible) == []
assert assigned == []
class _Future:
def __init__(self):
self.polls = 0
@ -258,7 +320,7 @@ def test_two_loop_api_transition_refreshes_dispatch_and_health(
monkeypatch.setattr(lanes, "maybe_gc_lane_artifacts", lambda: 0)
claim_states = []
def claim(_active, _limit):
def claim(_active, _limit, _eligible=None):
ready = lanes.kanban_capabilities(db).ready
claim_states.append(ready)
return [("cassandra", "t_loop")] if ready and not _active else []
@ -304,6 +366,11 @@ def test_orphan_recovery_skips_unreachable_boards_and_settled_tasks(
status="running",
current_run_id=4,
assignee="cli-auto",
claim_lock="direct-cli-lane",
)
pool_running = SimpleNamespace(
id="t_pool", status="running", current_run_id=5, assignee="cli-auto",
claim_lock="execution-pool",
)
settled = SimpleNamespace(
id="t_done",
@ -325,7 +392,7 @@ def test_orphan_recovery_skips_unreachable_boards_and_settled_tasks(
],
scoped_current_board=lambda _board: nullcontext(),
connect=connect,
list_tasks=lambda _conn: [settled, running],
list_tasks=lambda _conn: [settled, running, pool_running],
reclaim_task=lambda _conn, task_id, **kwargs: reclaims.append(
(task_id, kwargs["expected_run_id"])
),
@ -333,6 +400,7 @@ def test_orphan_recovery_skips_unreachable_boards_and_settled_tasks(
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db))
monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0)
monkeypatch.setattr(lanes, "_has_pending_finalization", lambda *_args: False)
monkeypatch.setattr(lanes.supervisor_state, "get_child", lambda *_args: None)
lanes.recover_orphans()

View File

@ -81,8 +81,8 @@ def test_dispatch_claims_only_pathless_task_with_safe_assignment_branch(
pool = coordinator.Coordinator(MASTER, store)
observed = []
def claim(active, limit, eligible):
observed.append((active, limit, eligible("metis", item)))
def claim(active, limit, eligible, **kwargs):
observed.append((active, limit, eligible("metis", item), kwargs.get("claimer")))
return [("metis", item.id)]
monkeypatch.setattr(maintenance.cli_lane_dispatch, "claim_ready", claim)
@ -91,7 +91,7 @@ def test_dispatch_claims_only_pathless_task_with_safe_assignment_branch(
lambda *_a: assignment_payload(branch="wt/t_deadbeef"),
)
pool.dispatch()
assert observed == [(set(), 3, True)]
assert observed == [(set(), 3, True, "execution-pool")]
record = store.active_assignments()[0]
assert record["run_id"] == "23"
assert record["payload"]["branch"] == "wt/t_deadbeef"
@ -128,7 +128,7 @@ def test_dispatch_workspace_ownership_race_is_exactly_fenced(
monkeypatch.setattr(
maintenance.cli_lane_dispatch,
"claim_ready",
lambda *_a: [("metis", owned.id)],
lambda *_a, **_kwargs: [("metis", owned.id)],
)
pool.dispatch()
assert store.active_assignments() == []
@ -157,7 +157,7 @@ def test_dispatch_preparation_failure_is_surfaced_and_full_pool_does_not_claim(
monkeypatch.setattr(
maintenance.cli_lane_dispatch,
"claim_ready",
lambda *_a: [("metis", broken.id)],
lambda *_a, **_kwargs: [("metis", broken.id)],
)
pool.dispatch()
assert kanban.blocked[0][1]["expected_run_id"] is None
@ -286,7 +286,7 @@ def test_reconcile_and_dispatch_cover_empty_error_and_missing_task_paths(
empty = protocol.PoolStore(tmp_path / "missing.db")
monkeypatch.setattr(
maintenance.cli_lane_dispatch, "claim_ready", lambda *_a: [("metis", "missing")]
maintenance.cli_lane_dispatch, "claim_ready", lambda *_a, **_kwargs: [("metis", "missing")]
)
coordinator.Coordinator(MASTER, empty).dispatch()

View File

@ -5,6 +5,7 @@ from __future__ import annotations
from pathlib import Path
import re
import pytest
import yaml
@ -12,6 +13,41 @@ ROOT = Path(__file__).resolve().parents[2]
SERVICE = ROOT / "services/hermes"
APPLICATIONS = ROOT / "clusters/atlas/flux-system/applications"
AGENT_MARKER = '"$imagepolicy": "hermes:hermes-agent-release:digest"'
HOLD_KEYS = {
"hermes.bstein.dev/agent-image-release-hold",
"hermes.bstein.dev/agent-image-release-hold-digest",
"hermes.bstein.dev/agent-image-release-hold-minimum-source",
"hermes.bstein.dev/agent-image-release-hold-reason",
}
def _agent_release_binding(agent_text: str) -> str:
"""Validate exactly one normal setter or one explicit temporary hold."""
document = yaml.safe_load(agent_text)
annotations = document.get("metadata", {}).get("annotations", {})
setter_count = agent_text.count(AGENT_MARKER)
hold_present = bool(HOLD_KEYS & set(annotations))
if setter_count == 1:
assert not hold_present, "agent image cannot have both setter and release hold"
return "setter"
assert setter_count == 0, "agent image has duplicate release setters"
assert annotations.get("hermes.bstein.dev/agent-image-release-hold") == "true"
image = next(
item for item in document.get("images", [])
if item.get("name") == "registry.bstein.dev/bstein/hermes-agent"
)
digest = annotations.get("hermes.bstein.dev/agent-image-release-hold-digest", "")
assert re.fullmatch(r"sha256:[0-9a-f]{64}", digest), "hold digest is invalid"
assert image.get("digest") == digest, "hold digest does not match the pinned image"
source = annotations.get("hermes.bstein.dev/agent-image-release-hold-minimum-source", "")
assert re.fullmatch(r"[0-9a-f]{40}", source), "hold source must be a full commit"
assert annotations.get("hermes.bstein.dev/agent-image-release-hold-reason", "").strip(), (
"hold reason is required"
)
assert HOLD_KEYS <= set(annotations), "agent image hold metadata is incomplete"
return "hold"
def test_image_policies_observe_only_validated_release_tags() -> None:
"""Candidates remain invisible until Jenkins publishes the release suffix."""
@ -70,7 +106,7 @@ def test_flux_updates_only_the_reviewed_hermes_image_digests() -> None:
"strategy": "Setters",
"path": "services/hermes",
}
assert agent.count('"$imagepolicy": "hermes:hermes-agent-release:digest"') == 1
assert _agent_release_binding(agent) in {"setter", "hold"}
webui_marker = '"$imagepolicy": "hermes:hermes-webui-release"'
chat_object = yaml.safe_load(chat)
containers = chat_object["spec"]["template"]["spec"]["containers"]
@ -127,3 +163,37 @@ def test_flux_updates_only_the_reviewed_hermes_image_digests() -> None:
marked_line = next(line for line in router.splitlines() if router_marker in line)
assert "registry.bstein.dev/bstein/hermes-chat-router" in marked_line
assert "@sha256:" in marked_line
def test_agent_release_hold_rejects_mismatched_digest_and_blank_reason() -> None:
"""A temporary hold must bind the exact image and explain its purpose."""
image_digest = "sha256:" + "a" * 64
def hold_text(*, digest: str = image_digest, reason: str = "verified hold") -> str:
return yaml.safe_dump(
{
"apiVersion": "kustomize.config.k8s.io/v1beta1",
"kind": "Kustomization",
"metadata": {
"annotations": {
"hermes.bstein.dev/agent-image-release-hold": "true",
"hermes.bstein.dev/agent-image-release-hold-digest": digest,
"hermes.bstein.dev/agent-image-release-hold-minimum-source": "b" * 40,
"hermes.bstein.dev/agent-image-release-hold-reason": reason,
}
},
"images": [
{
"name": "registry.bstein.dev/bstein/hermes-agent",
"digest": image_digest,
}
],
}
)
bad_digest = hold_text(digest="sha256:" + "0" * 64)
with pytest.raises(AssertionError, match="digest"):
_agent_release_binding(bad_digest)
blank_reason = hold_text(reason="")
with pytest.raises(AssertionError, match="reason"):
_agent_release_binding(blank_reason)

View File

@ -68,6 +68,7 @@ def test_builder_prefers_rpi5_with_healthy_arm64_worker_fallback() -> None:
assert host_rule["operator"] == "NotIn"
assert set(host_rule["values"]) >= {
"titan-04",
"titan-08",
"titan-14",
"titan-18",
"titan-19",

View File

@ -350,6 +350,9 @@ def test_runtime_bundle_and_guidance_require_live_convergence() -> None:
guidance = (ROOT / "services/hermes/agent-configmap.yaml").read_text()
assert "hermes_image_release_status.py=scripts/hermes_image_release_status.py" in kustomization
assert "safe `follow_command`" in guidance
assert "verifies that the requested commit is contained by main" in guidance
assert "checks out and builds that exact commit" in guidance
assert "builds the newest main containing that commit" not in guidance
assert "Merged code is not a completed release" in guidance.replace(
"merged code", "Merged code"
)