hermes: reserve PR follow-ups for mediated workers
This commit is contained in:
parent
74d8485f6a
commit
13a7dbab46
@ -28,6 +28,7 @@ spec:
|
|||||||
operator: NotIn
|
operator: NotIn
|
||||||
values:
|
values:
|
||||||
- titan-04
|
- titan-04
|
||||||
|
- titan-08
|
||||||
- titan-14
|
- titan-14
|
||||||
- titan-18
|
- titan-18
|
||||||
- titan-19
|
- titan-19
|
||||||
|
|||||||
@ -415,16 +415,18 @@ data:
|
|||||||
Use `--component agent` for backend/runtime changes, `--component webui`
|
Use `--component agent` for backend/runtime changes, `--component webui`
|
||||||
for chat UI changes, and `--component stt` or `--component tts` for the
|
for chat UI changes, and `--component stt` or `--component tts` for the
|
||||||
private voice services, passing a full commit already contained by `main`.
|
private voice services, passing a full commit already contained by `main`.
|
||||||
Jenkins builds the newest main containing that commit, publishes a final
|
Jenkins verifies that the requested commit is contained by main, then
|
||||||
immutable release tag only after its evidence passes, and Flux applies the
|
checks out and builds that exact commit. It publishes a final immutable
|
||||||
resulting digest. Candidate tags and failed builds never deploy. Run the
|
release tag only after its evidence passes, and Flux applies the resulting
|
||||||
|
digest. Candidate tags and failed builds never deploy. Run the
|
||||||
safe `follow_command` returned by the trigger and do not report completion
|
safe `follow_command` returned by the trigger and do not report completion
|
||||||
until it says `converged`: that proves the exact reviewed source is the
|
until it says `converged`: that proves the exact reviewed source is the
|
||||||
selected immutable tag/digest, Flux is Ready, every consumer has the digest
|
selected immutable tag/digest, Flux is Ready, every consumer has the digest
|
||||||
in desired state, and all corresponding pods are Ready on that digest. If
|
in desired state, and all corresponding pods are Ready on that digest. If
|
||||||
main advanced before Jenkins checked it out, the status command reports
|
the status command reports `different_revision_selected`, treat it as a
|
||||||
`different_revision_selected`; independently prove the requested commit is
|
release mismatch: stop, investigate the selected revision, and do not
|
||||||
an ancestor of that exact source, then follow the exact selected revision.
|
follow or deploy that different SHA. Re-establish convergence for the
|
||||||
|
requested exact commit instead.
|
||||||
Use `jenkins_build_evidence.py` for a terminal failure cause when its
|
Use `jenkins_build_evidence.py` for a terminal failure cause when its
|
||||||
bounded controller evidence path is available. Candidate-only, queued, or
|
bounded controller evidence path is available. Candidate-only, queued, or
|
||||||
merged code is not a completed release. Finally verify the public SSO
|
merged code is not a completed release. Finally verify the public SSO
|
||||||
|
|||||||
@ -25,7 +25,7 @@ spec:
|
|||||||
ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers
|
ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers
|
||||||
ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback
|
ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback
|
||||||
ai.bstein.dev/placement: primary amd64 accelerator titan-22; arm64 rpi5 fleet fallback; storage-backbone nodes excluded
|
ai.bstein.dev/placement: primary amd64 accelerator titan-22; arm64 rpi5 fleet fallback; storage-backbone nodes excluded
|
||||||
ai.bstein.dev/config-rev: "20260913-soteria-kanban-recovery-v1"
|
ai.bstein.dev/config-rev: "20260913-soteria-kanban-recovery-v2"
|
||||||
prometheus.io/scrape: "true"
|
prometheus.io/scrape: "true"
|
||||||
prometheus.io/path: /metrics
|
prometheus.io/path: /metrics
|
||||||
prometheus.io/port: "9010"
|
prometheus.io/port: "9010"
|
||||||
@ -287,26 +287,6 @@ spec:
|
|||||||
resources:
|
resources:
|
||||||
requests: {cpu: 25m, memory: 32Mi}
|
requests: {cpu: 25m, memory: 32Mi}
|
||||||
limits: {cpu: 100m, memory: 64Mi}
|
limits: {cpu: 100m, memory: 64Mi}
|
||||||
- name: recover-soteria-kanban
|
|
||||||
image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
|
|
||||||
imagePullPolicy: IfNotPresent
|
|
||||||
command:
|
|
||||||
- /opt/hermes/.venv/bin/python
|
|
||||||
- /opt/coordinator/recover_soteria_kanban.py
|
|
||||||
securityContext:
|
|
||||||
allowPrivilegeEscalation: false
|
|
||||||
capabilities:
|
|
||||||
drop: [ALL]
|
|
||||||
runAsUser: 10000
|
|
||||||
runAsGroup: 10000
|
|
||||||
seccompProfile:
|
|
||||||
type: RuntimeDefault
|
|
||||||
volumeMounts:
|
|
||||||
- {name: home, mountPath: /opt/data}
|
|
||||||
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
|
|
||||||
resources:
|
|
||||||
requests: {cpu: 25m, memory: 64Mi}
|
|
||||||
limits: {cpu: 250m, memory: 256Mi}
|
|
||||||
# Worker HUX identity provisioning. Mirrors the chat tenants'
|
# Worker HUX identity provisioning. Mirrors the chat tenants'
|
||||||
# init-hux-runtime, adapted to the single-replica Deployment: the
|
# init-hux-runtime, adapted to the single-replica Deployment: the
|
||||||
# subtree name is the fixed literal "hux" (Deployment pod names churn,
|
# subtree name is the fixed literal "hux" (Deployment pod names churn,
|
||||||
|
|||||||
@ -2,6 +2,12 @@
|
|||||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||||
kind: Kustomization
|
kind: Kustomization
|
||||||
namespace: hermes
|
namespace: hermes
|
||||||
|
metadata:
|
||||||
|
annotations:
|
||||||
|
hermes.bstein.dev/agent-image-release-hold: "true"
|
||||||
|
hermes.bstein.dev/agent-image-release-hold-digest: "sha256:bebfe9c24cdd877ed2c16335027ccf393892161642756f54e8d12984eefdd903"
|
||||||
|
hermes.bstein.dev/agent-image-release-hold-minimum-source: "106ee8ce14aa67c3825fca6fb537a557b44ecd86"
|
||||||
|
hermes.bstein.dev/agent-image-release-hold-reason: "Hold the verified SQLite 3.51.3 runtime until the normal Hermes release includes it."
|
||||||
images:
|
images:
|
||||||
- name: registry.bstein.dev/bstein/hermes-agent
|
- name: registry.bstein.dev/bstein/hermes-agent
|
||||||
# Hold this verified SQLite fix until the normal image release includes it.
|
# Hold this verified SQLite fix until the normal image release includes it.
|
||||||
|
|||||||
@ -5,6 +5,7 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import concurrent.futures
|
import concurrent.futures
|
||||||
import os
|
import os
|
||||||
|
import sqlite3
|
||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
from collections.abc import Callable
|
from collections.abc import Callable
|
||||||
@ -29,6 +30,9 @@ from cli_lane_files import atomic_json
|
|||||||
from cli_lane_metrics import start_metrics_server
|
from cli_lane_metrics import start_metrics_server
|
||||||
from cli_lane_recovery import _has_pending_finalization, recover_pending_finalizations
|
from cli_lane_recovery import _has_pending_finalization, recover_pending_finalizations
|
||||||
from cli_lane_retention import maybe_gc_lane_artifacts
|
from cli_lane_retention import maybe_gc_lane_artifacts
|
||||||
|
import supervisor_state
|
||||||
|
|
||||||
|
OWNED_WORKSPACES_ONLY = os.environ.get("HERMES_CLI_LANE_OWNED_WORKSPACES_ONLY", "").lower() == "true"
|
||||||
|
|
||||||
|
|
||||||
def _board_slug(board: Any) -> str:
|
def _board_slug(board: Any) -> str:
|
||||||
@ -46,6 +50,47 @@ def _connect_healthy_board(kanban_db: Any, board: str) -> Any | None:
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _direct_lane_eligible(board: str, task: Any) -> bool:
|
||||||
|
"""Reserve signed PR continuations for the mediated execution pool."""
|
||||||
|
task_id = str(_task_value(task, "id", "") or "")
|
||||||
|
if not task_id or (OWNED_WORKSPACES_ONLY and not str(_task_value(task, "workspace_path", "") or "").strip()):
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
return supervisor_state.get_child(board, task_id) is None
|
||||||
|
except (OSError, ValueError) as error:
|
||||||
|
_record_board_access_error(board, error)
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _listed_boards(kanban_db: Any) -> list[Any]:
|
||||||
|
"""Fall back to board-directory discovery when one metadata read is unreadable."""
|
||||||
|
try:
|
||||||
|
return list(kanban_db.list_boards(include_archived=False))
|
||||||
|
except (OSError, sqlite3.Error) as error:
|
||||||
|
_record_board_access_error("board-registry", error)
|
||||||
|
root = getattr(kanban_db, "boards_root", None)
|
||||||
|
metadata = getattr(kanban_db, "read_board_metadata", None)
|
||||||
|
if not callable(root) or not callable(metadata):
|
||||||
|
return []
|
||||||
|
try:
|
||||||
|
candidates = [path for path in root().iterdir() if not path.is_symlink() and path.is_dir()]
|
||||||
|
except (OSError, sqlite3.Error) as error:
|
||||||
|
_record_board_access_error("board-registry", error)
|
||||||
|
return []
|
||||||
|
boards: list[Any] = []
|
||||||
|
for path in sorted(candidates, key=lambda item: item.name):
|
||||||
|
try:
|
||||||
|
if not ((path / "kanban.db").is_file() or (path / "board.json").is_file()):
|
||||||
|
continue
|
||||||
|
board = metadata(path.name)
|
||||||
|
except (OSError, sqlite3.Error, ValueError) as error:
|
||||||
|
_record_board_access_error(path.name, error)
|
||||||
|
continue
|
||||||
|
if isinstance(board, dict) and not board.get("archived") and _board_slug(board) == path.name:
|
||||||
|
boards.append(board)
|
||||||
|
return boards
|
||||||
|
|
||||||
|
|
||||||
def recover_orphans() -> None:
|
def recover_orphans() -> None:
|
||||||
"""Return external running tasks to ready after a runner/pod restart."""
|
"""Return external running tasks to ready after a runner/pod restart."""
|
||||||
from hermes_cli import kanban_db
|
from hermes_cli import kanban_db
|
||||||
@ -53,10 +98,8 @@ def recover_orphans() -> None:
|
|||||||
recover_pending_finalizations()
|
recover_pending_finalizations()
|
||||||
if not kanban_capabilities(kanban_db).exact_run_reclaim:
|
if not kanban_capabilities(kanban_db).exact_run_reclaim:
|
||||||
return
|
return
|
||||||
try:
|
boards = _listed_boards(kanban_db)
|
||||||
boards = kanban_db.list_boards(include_archived=False)
|
if not boards:
|
||||||
except Exception as error:
|
|
||||||
_record_board_access_error("board-registry", error)
|
|
||||||
return
|
return
|
||||||
for raw_board in boards:
|
for raw_board in boards:
|
||||||
board = _board_slug(raw_board)
|
board = _board_slug(raw_board)
|
||||||
@ -71,6 +114,8 @@ def recover_orphans() -> None:
|
|||||||
if (
|
if (
|
||||||
_external(task)
|
_external(task)
|
||||||
and str(_task_value(task, "status", "")) == "running"
|
and str(_task_value(task, "status", "")) == "running"
|
||||||
|
and str(_task_value(task, "claim_lock", "") or "") == "direct-cli-lane"
|
||||||
|
and _direct_lane_eligible(board, task)
|
||||||
):
|
):
|
||||||
task_id = str(_task_value(task, "id"))
|
task_id = str(_task_value(task, "id"))
|
||||||
run_id = _task_value(task, "current_run_id", None)
|
run_id = _task_value(task, "current_run_id", None)
|
||||||
@ -95,6 +140,7 @@ def claim_ready(
|
|||||||
active: set[tuple[str, str]],
|
active: set[tuple[str, str]],
|
||||||
limit: int,
|
limit: int,
|
||||||
eligible: Callable[[str, Any], bool] | None = None,
|
eligible: Callable[[str, Any], bool] | None = None,
|
||||||
|
claimer: str = "direct-cli-lane",
|
||||||
) -> list[tuple[str, str]]:
|
) -> list[tuple[str, str]]:
|
||||||
"""Atomically claim external ready tasks across all non-archived boards."""
|
"""Atomically claim external ready tasks across all non-archived boards."""
|
||||||
from hermes_cli import kanban_db
|
from hermes_cli import kanban_db
|
||||||
@ -102,7 +148,7 @@ def claim_ready(
|
|||||||
claimed: list[tuple[str, str]] = []
|
claimed: list[tuple[str, str]] = []
|
||||||
if limit <= 0:
|
if limit <= 0:
|
||||||
return claimed
|
return claimed
|
||||||
for raw_board in kanban_db.list_boards(include_archived=False):
|
for raw_board in _listed_boards(kanban_db):
|
||||||
board = _board_slug(raw_board)
|
board = _board_slug(raw_board)
|
||||||
if not board:
|
if not board:
|
||||||
continue
|
continue
|
||||||
@ -117,20 +163,23 @@ def claim_ready(
|
|||||||
for task in tasks:
|
for task in tasks:
|
||||||
task_id = str(_task_value(task, "id", ""))
|
task_id = str(_task_value(task, "id", ""))
|
||||||
assignee = str(_task_value(task, "assignee", "") or "")
|
assignee = str(_task_value(task, "assignee", "") or "")
|
||||||
|
if (
|
||||||
|
not task_id
|
||||||
|
or (board, task_id) in active
|
||||||
|
or str(_task_value(task, "status", "")) != "ready"
|
||||||
|
or (eligible is not None and not eligible(board, task))
|
||||||
|
):
|
||||||
|
continue
|
||||||
if (
|
if (
|
||||||
task_id
|
task_id
|
||||||
and not assignee
|
and not assignee
|
||||||
and str(_task_value(task, "status", "")) == "ready"
|
|
||||||
and kanban_db.assign_task(conn, task_id, "cli-auto")
|
and kanban_db.assign_task(conn, task_id, "cli-auto")
|
||||||
):
|
):
|
||||||
task = kanban_db.get_task(conn, task_id)
|
task = kanban_db.get_task(conn, task_id)
|
||||||
assignee = "cli-auto"
|
assignee = "cli-auto"
|
||||||
if (
|
if (
|
||||||
not task_id
|
not task_id
|
||||||
or (board, task_id) in active
|
|
||||||
or not assignee.startswith(EXTERNAL_PREFIX)
|
or not assignee.startswith(EXTERNAL_PREFIX)
|
||||||
or str(_task_value(task, "status", "")) != "ready"
|
|
||||||
or (eligible is not None and not eligible(board, task))
|
|
||||||
):
|
):
|
||||||
continue
|
continue
|
||||||
try:
|
try:
|
||||||
@ -138,7 +187,7 @@ def claim_ready(
|
|||||||
conn,
|
conn,
|
||||||
task_id,
|
task_id,
|
||||||
ttl_seconds=DEFAULT_CLAIM_TTL,
|
ttl_seconds=DEFAULT_CLAIM_TTL,
|
||||||
claimer="direct-cli-lane",
|
claimer=claimer,
|
||||||
)
|
)
|
||||||
except Exception:
|
except Exception:
|
||||||
continue
|
continue
|
||||||
@ -190,7 +239,7 @@ def main() -> int:
|
|||||||
active = set(futures.values())
|
active = set(futures.values())
|
||||||
try:
|
try:
|
||||||
newly_claimed = (
|
newly_claimed = (
|
||||||
claim_ready(active, workers - len(futures))
|
claim_ready(active, workers - len(futures), _direct_lane_eligible)
|
||||||
if capabilities.ready
|
if capabilities.ready
|
||||||
else []
|
else []
|
||||||
)
|
)
|
||||||
|
|||||||
@ -361,6 +361,7 @@ def dispatch(pool: Any) -> None:
|
|||||||
claimed = cli_lane_dispatch.claim_ready(
|
claimed = cli_lane_dispatch.claim_ready(
|
||||||
set(), len(ordinals),
|
set(), len(ordinals),
|
||||||
lambda _board, task: distributed_workspace_eligible(task),
|
lambda _board, task: distributed_workspace_eligible(task),
|
||||||
|
claimer="execution-pool",
|
||||||
)
|
)
|
||||||
except RECOVERABLE_BOARD_ERRORS as error:
|
except RECOVERABLE_BOARD_ERRORS as error:
|
||||||
_defer("claim-ready", error)
|
_defer("claim-ready", error)
|
||||||
|
|||||||
@ -305,6 +305,7 @@ def test_orphan_recovery_passes_the_scanned_run_as_atomic_reclaim_guard(
|
|||||||
status="running",
|
status="running",
|
||||||
assignee="cli-auto",
|
assignee="cli-auto",
|
||||||
current_run_id=91,
|
current_run_id=91,
|
||||||
|
claim_lock="direct-cli-lane",
|
||||||
),
|
),
|
||||||
SimpleNamespace(
|
SimpleNamespace(
|
||||||
id="t_no_run",
|
id="t_no_run",
|
||||||
@ -330,6 +331,7 @@ def test_orphan_recovery_passes_the_scanned_run_as_atomic_reclaim_guard(
|
|||||||
)
|
)
|
||||||
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
|
||||||
monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0)
|
monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0)
|
||||||
|
monkeypatch.setattr(lanes.supervisor_state, "get_child", lambda *_args: None)
|
||||||
|
|
||||||
lanes.recover_orphans()
|
lanes.recover_orphans()
|
||||||
|
|
||||||
|
|||||||
@ -101,6 +101,68 @@ def test_claim_ready_handles_zero_limit_empty_boards_and_claim_errors(monkeypatc
|
|||||||
assert lanes.claim_ready(set(), 1) == []
|
assert lanes.claim_ready(set(), 1) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_claim_ready_skips_an_unreadable_board_and_claims_a_healthy_fallback(tmp_path, monkeypatch):
|
||||||
|
"""A metadata failure on one board cannot starve healthy ready cards."""
|
||||||
|
root = tmp_path / "boards"
|
||||||
|
(root / "broken").mkdir(parents=True)
|
||||||
|
(root / "soteria").mkdir()
|
||||||
|
(root / "archived").mkdir()
|
||||||
|
for board in ("broken", "soteria", "archived"):
|
||||||
|
(root / board / "kanban.db").touch()
|
||||||
|
failures = []
|
||||||
|
task = SimpleNamespace(id="t_soteria", status="ready", assignee="cli-auto")
|
||||||
|
|
||||||
|
class Connection:
|
||||||
|
def __init__(self, board): self.board = board
|
||||||
|
def close(self): return None
|
||||||
|
|
||||||
|
def connect(*, board):
|
||||||
|
if board == "broken":
|
||||||
|
raise PermissionError("board database unreadable")
|
||||||
|
return Connection(board)
|
||||||
|
|
||||||
|
db = SimpleNamespace(
|
||||||
|
list_boards=lambda **_kwargs: (_ for _ in ()).throw(PermissionError("probe metadata")),
|
||||||
|
boards_root=lambda: root, scoped_current_board=lambda _board: nullcontext(),
|
||||||
|
read_board_metadata=lambda board: {"slug": board, "archived": board == "archived"},
|
||||||
|
connect=connect, recompute_ready=lambda _conn: None,
|
||||||
|
list_tasks=lambda conn: [task] if conn.board == "soteria" else [],
|
||||||
|
claim_task=lambda _conn, task_id, **_kwargs: task_id,
|
||||||
|
)
|
||||||
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db))
|
||||||
|
monkeypatch.setattr(lanes, "_record_board_access_error", lambda board, error: failures.append(board))
|
||||||
|
|
||||||
|
assert lanes.claim_ready(set(), 1) == [("soteria", "t_soteria")]
|
||||||
|
assert failures.count("board-registry") == 1 and failures.count("broken") == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_direct_lane_reserves_signed_continuations_for_the_mediator(monkeypatch):
|
||||||
|
task = SimpleNamespace(id="t_continue")
|
||||||
|
monkeypatch.setattr(lanes.supervisor_state, "get_child", lambda *_args: {"kind": "repair"})
|
||||||
|
assert lanes._direct_lane_eligible("soteria", task) is False
|
||||||
|
monkeypatch.setattr(lanes.supervisor_state, "get_child", lambda *_args: None)
|
||||||
|
assert lanes._direct_lane_eligible("soteria", task) is True
|
||||||
|
monkeypatch.setattr(lanes.supervisor_state, "get_child", lambda *_args: (_ for _ in ()).throw(OSError("state")))
|
||||||
|
assert lanes._direct_lane_eligible("soteria", task) is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_direct_lane_owned_workspace_policy_precedes_unassigned_mutation(monkeypatch):
|
||||||
|
task = SimpleNamespace(id="t_pathless", workspace_path="", status="ready", assignee="")
|
||||||
|
monkeypatch.setattr(lanes, "OWNED_WORKSPACES_ONLY", True)
|
||||||
|
assert lanes._direct_lane_eligible("soteria", task) is False
|
||||||
|
assigned = []
|
||||||
|
db = SimpleNamespace(
|
||||||
|
list_boards=lambda **_kwargs: [{"slug": "soteria"}],
|
||||||
|
scoped_current_board=lambda _board: nullcontext(),
|
||||||
|
connect=lambda **_kwargs: SimpleNamespace(close=lambda: None),
|
||||||
|
recompute_ready=lambda _conn: None, list_tasks=lambda _conn: [task],
|
||||||
|
assign_task=lambda *_args: assigned.append(True),
|
||||||
|
)
|
||||||
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db))
|
||||||
|
assert lanes.claim_ready(set(), 1, lanes._direct_lane_eligible) == []
|
||||||
|
assert assigned == []
|
||||||
|
|
||||||
|
|
||||||
class _Future:
|
class _Future:
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.polls = 0
|
self.polls = 0
|
||||||
@ -258,7 +320,7 @@ def test_two_loop_api_transition_refreshes_dispatch_and_health(
|
|||||||
monkeypatch.setattr(lanes, "maybe_gc_lane_artifacts", lambda: 0)
|
monkeypatch.setattr(lanes, "maybe_gc_lane_artifacts", lambda: 0)
|
||||||
claim_states = []
|
claim_states = []
|
||||||
|
|
||||||
def claim(_active, _limit):
|
def claim(_active, _limit, _eligible=None):
|
||||||
ready = lanes.kanban_capabilities(db).ready
|
ready = lanes.kanban_capabilities(db).ready
|
||||||
claim_states.append(ready)
|
claim_states.append(ready)
|
||||||
return [("cassandra", "t_loop")] if ready and not _active else []
|
return [("cassandra", "t_loop")] if ready and not _active else []
|
||||||
@ -304,6 +366,11 @@ def test_orphan_recovery_skips_unreachable_boards_and_settled_tasks(
|
|||||||
status="running",
|
status="running",
|
||||||
current_run_id=4,
|
current_run_id=4,
|
||||||
assignee="cli-auto",
|
assignee="cli-auto",
|
||||||
|
claim_lock="direct-cli-lane",
|
||||||
|
)
|
||||||
|
pool_running = SimpleNamespace(
|
||||||
|
id="t_pool", status="running", current_run_id=5, assignee="cli-auto",
|
||||||
|
claim_lock="execution-pool",
|
||||||
)
|
)
|
||||||
settled = SimpleNamespace(
|
settled = SimpleNamespace(
|
||||||
id="t_done",
|
id="t_done",
|
||||||
@ -325,7 +392,7 @@ def test_orphan_recovery_skips_unreachable_boards_and_settled_tasks(
|
|||||||
],
|
],
|
||||||
scoped_current_board=lambda _board: nullcontext(),
|
scoped_current_board=lambda _board: nullcontext(),
|
||||||
connect=connect,
|
connect=connect,
|
||||||
list_tasks=lambda _conn: [settled, running],
|
list_tasks=lambda _conn: [settled, running, pool_running],
|
||||||
reclaim_task=lambda _conn, task_id, **kwargs: reclaims.append(
|
reclaim_task=lambda _conn, task_id, **kwargs: reclaims.append(
|
||||||
(task_id, kwargs["expected_run_id"])
|
(task_id, kwargs["expected_run_id"])
|
||||||
),
|
),
|
||||||
@ -333,6 +400,7 @@ def test_orphan_recovery_skips_unreachable_boards_and_settled_tasks(
|
|||||||
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db))
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db))
|
||||||
monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0)
|
monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0)
|
||||||
monkeypatch.setattr(lanes, "_has_pending_finalization", lambda *_args: False)
|
monkeypatch.setattr(lanes, "_has_pending_finalization", lambda *_args: False)
|
||||||
|
monkeypatch.setattr(lanes.supervisor_state, "get_child", lambda *_args: None)
|
||||||
|
|
||||||
lanes.recover_orphans()
|
lanes.recover_orphans()
|
||||||
|
|
||||||
|
|||||||
@ -81,8 +81,8 @@ def test_dispatch_claims_only_pathless_task_with_safe_assignment_branch(
|
|||||||
pool = coordinator.Coordinator(MASTER, store)
|
pool = coordinator.Coordinator(MASTER, store)
|
||||||
observed = []
|
observed = []
|
||||||
|
|
||||||
def claim(active, limit, eligible):
|
def claim(active, limit, eligible, **kwargs):
|
||||||
observed.append((active, limit, eligible("metis", item)))
|
observed.append((active, limit, eligible("metis", item), kwargs.get("claimer")))
|
||||||
return [("metis", item.id)]
|
return [("metis", item.id)]
|
||||||
|
|
||||||
monkeypatch.setattr(maintenance.cli_lane_dispatch, "claim_ready", claim)
|
monkeypatch.setattr(maintenance.cli_lane_dispatch, "claim_ready", claim)
|
||||||
@ -91,7 +91,7 @@ def test_dispatch_claims_only_pathless_task_with_safe_assignment_branch(
|
|||||||
lambda *_a: assignment_payload(branch="wt/t_deadbeef"),
|
lambda *_a: assignment_payload(branch="wt/t_deadbeef"),
|
||||||
)
|
)
|
||||||
pool.dispatch()
|
pool.dispatch()
|
||||||
assert observed == [(set(), 3, True)]
|
assert observed == [(set(), 3, True, "execution-pool")]
|
||||||
record = store.active_assignments()[0]
|
record = store.active_assignments()[0]
|
||||||
assert record["run_id"] == "23"
|
assert record["run_id"] == "23"
|
||||||
assert record["payload"]["branch"] == "wt/t_deadbeef"
|
assert record["payload"]["branch"] == "wt/t_deadbeef"
|
||||||
@ -128,7 +128,7 @@ def test_dispatch_workspace_ownership_race_is_exactly_fenced(
|
|||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
maintenance.cli_lane_dispatch,
|
maintenance.cli_lane_dispatch,
|
||||||
"claim_ready",
|
"claim_ready",
|
||||||
lambda *_a: [("metis", owned.id)],
|
lambda *_a, **_kwargs: [("metis", owned.id)],
|
||||||
)
|
)
|
||||||
pool.dispatch()
|
pool.dispatch()
|
||||||
assert store.active_assignments() == []
|
assert store.active_assignments() == []
|
||||||
@ -157,7 +157,7 @@ def test_dispatch_preparation_failure_is_surfaced_and_full_pool_does_not_claim(
|
|||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
maintenance.cli_lane_dispatch,
|
maintenance.cli_lane_dispatch,
|
||||||
"claim_ready",
|
"claim_ready",
|
||||||
lambda *_a: [("metis", broken.id)],
|
lambda *_a, **_kwargs: [("metis", broken.id)],
|
||||||
)
|
)
|
||||||
pool.dispatch()
|
pool.dispatch()
|
||||||
assert kanban.blocked[0][1]["expected_run_id"] is None
|
assert kanban.blocked[0][1]["expected_run_id"] is None
|
||||||
@ -286,7 +286,7 @@ def test_reconcile_and_dispatch_cover_empty_error_and_missing_task_paths(
|
|||||||
|
|
||||||
empty = protocol.PoolStore(tmp_path / "missing.db")
|
empty = protocol.PoolStore(tmp_path / "missing.db")
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
maintenance.cli_lane_dispatch, "claim_ready", lambda *_a: [("metis", "missing")]
|
maintenance.cli_lane_dispatch, "claim_ready", lambda *_a, **_kwargs: [("metis", "missing")]
|
||||||
)
|
)
|
||||||
coordinator.Coordinator(MASTER, empty).dispatch()
|
coordinator.Coordinator(MASTER, empty).dispatch()
|
||||||
|
|
||||||
|
|||||||
@ -5,6 +5,7 @@ from __future__ import annotations
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import re
|
import re
|
||||||
|
|
||||||
|
import pytest
|
||||||
import yaml
|
import yaml
|
||||||
|
|
||||||
|
|
||||||
@ -12,6 +13,41 @@ ROOT = Path(__file__).resolve().parents[2]
|
|||||||
SERVICE = ROOT / "services/hermes"
|
SERVICE = ROOT / "services/hermes"
|
||||||
APPLICATIONS = ROOT / "clusters/atlas/flux-system/applications"
|
APPLICATIONS = ROOT / "clusters/atlas/flux-system/applications"
|
||||||
|
|
||||||
|
AGENT_MARKER = '"$imagepolicy": "hermes:hermes-agent-release:digest"'
|
||||||
|
HOLD_KEYS = {
|
||||||
|
"hermes.bstein.dev/agent-image-release-hold",
|
||||||
|
"hermes.bstein.dev/agent-image-release-hold-digest",
|
||||||
|
"hermes.bstein.dev/agent-image-release-hold-minimum-source",
|
||||||
|
"hermes.bstein.dev/agent-image-release-hold-reason",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _agent_release_binding(agent_text: str) -> str:
|
||||||
|
"""Validate exactly one normal setter or one explicit temporary hold."""
|
||||||
|
document = yaml.safe_load(agent_text)
|
||||||
|
annotations = document.get("metadata", {}).get("annotations", {})
|
||||||
|
setter_count = agent_text.count(AGENT_MARKER)
|
||||||
|
hold_present = bool(HOLD_KEYS & set(annotations))
|
||||||
|
if setter_count == 1:
|
||||||
|
assert not hold_present, "agent image cannot have both setter and release hold"
|
||||||
|
return "setter"
|
||||||
|
assert setter_count == 0, "agent image has duplicate release setters"
|
||||||
|
assert annotations.get("hermes.bstein.dev/agent-image-release-hold") == "true"
|
||||||
|
image = next(
|
||||||
|
item for item in document.get("images", [])
|
||||||
|
if item.get("name") == "registry.bstein.dev/bstein/hermes-agent"
|
||||||
|
)
|
||||||
|
digest = annotations.get("hermes.bstein.dev/agent-image-release-hold-digest", "")
|
||||||
|
assert re.fullmatch(r"sha256:[0-9a-f]{64}", digest), "hold digest is invalid"
|
||||||
|
assert image.get("digest") == digest, "hold digest does not match the pinned image"
|
||||||
|
source = annotations.get("hermes.bstein.dev/agent-image-release-hold-minimum-source", "")
|
||||||
|
assert re.fullmatch(r"[0-9a-f]{40}", source), "hold source must be a full commit"
|
||||||
|
assert annotations.get("hermes.bstein.dev/agent-image-release-hold-reason", "").strip(), (
|
||||||
|
"hold reason is required"
|
||||||
|
)
|
||||||
|
assert HOLD_KEYS <= set(annotations), "agent image hold metadata is incomplete"
|
||||||
|
return "hold"
|
||||||
|
|
||||||
|
|
||||||
def test_image_policies_observe_only_validated_release_tags() -> None:
|
def test_image_policies_observe_only_validated_release_tags() -> None:
|
||||||
"""Candidates remain invisible until Jenkins publishes the release suffix."""
|
"""Candidates remain invisible until Jenkins publishes the release suffix."""
|
||||||
@ -70,7 +106,7 @@ def test_flux_updates_only_the_reviewed_hermes_image_digests() -> None:
|
|||||||
"strategy": "Setters",
|
"strategy": "Setters",
|
||||||
"path": "services/hermes",
|
"path": "services/hermes",
|
||||||
}
|
}
|
||||||
assert agent.count('"$imagepolicy": "hermes:hermes-agent-release:digest"') == 1
|
assert _agent_release_binding(agent) in {"setter", "hold"}
|
||||||
webui_marker = '"$imagepolicy": "hermes:hermes-webui-release"'
|
webui_marker = '"$imagepolicy": "hermes:hermes-webui-release"'
|
||||||
chat_object = yaml.safe_load(chat)
|
chat_object = yaml.safe_load(chat)
|
||||||
containers = chat_object["spec"]["template"]["spec"]["containers"]
|
containers = chat_object["spec"]["template"]["spec"]["containers"]
|
||||||
@ -127,3 +163,37 @@ def test_flux_updates_only_the_reviewed_hermes_image_digests() -> None:
|
|||||||
marked_line = next(line for line in router.splitlines() if router_marker in line)
|
marked_line = next(line for line in router.splitlines() if router_marker in line)
|
||||||
assert "registry.bstein.dev/bstein/hermes-chat-router" in marked_line
|
assert "registry.bstein.dev/bstein/hermes-chat-router" in marked_line
|
||||||
assert "@sha256:" in marked_line
|
assert "@sha256:" in marked_line
|
||||||
|
|
||||||
|
|
||||||
|
def test_agent_release_hold_rejects_mismatched_digest_and_blank_reason() -> None:
|
||||||
|
"""A temporary hold must bind the exact image and explain its purpose."""
|
||||||
|
image_digest = "sha256:" + "a" * 64
|
||||||
|
|
||||||
|
def hold_text(*, digest: str = image_digest, reason: str = "verified hold") -> str:
|
||||||
|
return yaml.safe_dump(
|
||||||
|
{
|
||||||
|
"apiVersion": "kustomize.config.k8s.io/v1beta1",
|
||||||
|
"kind": "Kustomization",
|
||||||
|
"metadata": {
|
||||||
|
"annotations": {
|
||||||
|
"hermes.bstein.dev/agent-image-release-hold": "true",
|
||||||
|
"hermes.bstein.dev/agent-image-release-hold-digest": digest,
|
||||||
|
"hermes.bstein.dev/agent-image-release-hold-minimum-source": "b" * 40,
|
||||||
|
"hermes.bstein.dev/agent-image-release-hold-reason": reason,
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"images": [
|
||||||
|
{
|
||||||
|
"name": "registry.bstein.dev/bstein/hermes-agent",
|
||||||
|
"digest": image_digest,
|
||||||
|
}
|
||||||
|
],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
bad_digest = hold_text(digest="sha256:" + "0" * 64)
|
||||||
|
with pytest.raises(AssertionError, match="digest"):
|
||||||
|
_agent_release_binding(bad_digest)
|
||||||
|
blank_reason = hold_text(reason="")
|
||||||
|
with pytest.raises(AssertionError, match="reason"):
|
||||||
|
_agent_release_binding(blank_reason)
|
||||||
|
|||||||
@ -68,6 +68,7 @@ def test_builder_prefers_rpi5_with_healthy_arm64_worker_fallback() -> None:
|
|||||||
assert host_rule["operator"] == "NotIn"
|
assert host_rule["operator"] == "NotIn"
|
||||||
assert set(host_rule["values"]) >= {
|
assert set(host_rule["values"]) >= {
|
||||||
"titan-04",
|
"titan-04",
|
||||||
|
"titan-08",
|
||||||
"titan-14",
|
"titan-14",
|
||||||
"titan-18",
|
"titan-18",
|
||||||
"titan-19",
|
"titan-19",
|
||||||
|
|||||||
@ -350,6 +350,9 @@ def test_runtime_bundle_and_guidance_require_live_convergence() -> None:
|
|||||||
guidance = (ROOT / "services/hermes/agent-configmap.yaml").read_text()
|
guidance = (ROOT / "services/hermes/agent-configmap.yaml").read_text()
|
||||||
assert "hermes_image_release_status.py=scripts/hermes_image_release_status.py" in kustomization
|
assert "hermes_image_release_status.py=scripts/hermes_image_release_status.py" in kustomization
|
||||||
assert "safe `follow_command`" in guidance
|
assert "safe `follow_command`" in guidance
|
||||||
|
assert "verifies that the requested commit is contained by main" in guidance
|
||||||
|
assert "checks out and builds that exact commit" in guidance
|
||||||
|
assert "builds the newest main containing that commit" not in guidance
|
||||||
assert "Merged code is not a completed release" in guidance.replace(
|
assert "Merged code is not a completed release" in guidance.replace(
|
||||||
"merged code", "Merged code"
|
"merged code", "Merged code"
|
||||||
)
|
)
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user