From 82cc28b383a61fa1b7baba32c90e22921acc5364 Mon Sep 17 00:00:00 2001 From: jenkins Date: Sun, 13 Sep 2026 19:26:44 -0500 Subject: [PATCH] hermes: enforce native provider availability for manual assignments --- services/hermes/NOTES.md | 107 ++++++++---------- services/hermes/agent-deployment.yaml | 2 +- .../hermes/scripts/execution_pool_worker.py | 8 +- ...st_hermes_execution_pool_worker_execute.py | 40 +++++++ 4 files changed, 96 insertions(+), 61 deletions(-) diff --git a/services/hermes/NOTES.md b/services/hermes/NOTES.md index 2feb7789..1044a950 100644 --- a/services/hermes/NOTES.md +++ b/services/hermes/NOTES.md @@ -71,14 +71,21 @@ model list; it does not run a generation just to discover a model. A provider, catalog, or evaluation outage preserves the last known-good routes and records the refresh as deferred, so discovery does not silently become a routing bypass. -Existing pull-request continuation is available through coordinator-owned -lineage: `kanban_continue_pr.py` validates the root task, recorded PR, branch, -and current head before queuing a repair on that same PR. A signed continuation -worker may update that exact owned ref through the SCM broker. Publication-only -resume for a completed but unpublished SCM result is still being finalized; -until its exact-run ownership, evidence preservation, and bounded finite-retry -semantics are confirmed, keep the workspace commit and structured result -recoverable and do not claim that path is live. +Existing pull-request continuation stays on one recorded PR and branch throughout +repair and review. `kanban_continue_pr.py` validates the root, PR, branch, and +current head before delegation; inspect the current head diff and wait for the +delegated result before creating a review or repair follow-up. A signed +continuation updates only that owned ref through the SCM broker, never creates +a replacement PR or adopts a local checkout. + +A clean, broker-refused publication retains its exact completed evidence and +may receive one automatic retry after a five-minute backoff. The retry opens the +same child only when the current native blocked event matches the coordinator's +receipt fence, root link, and terminal pool evidence. A later human block, +cancellation, head change, missing receipt, or failed fence check leaves the +card blocked for operator review. Publication retries use a fresh signed run and +must republish the preserved head without a model call; current-head review +still follows a broker-confirmed publication. ## Private Jetson voice: multilingual TTS policy @@ -471,56 +478,40 @@ A `voice` field from a browser is never honoured at any hop. ## Distributed execution-pool rollout boundary -- Do not reconcile the pool until the approved credential owner has provisioned - six distinct Vault fields: `execution_worker_{0,1,2}_claude_credentials_json` - and `execution_worker_{0,1,2}_codex_auth_json`. Each ordinal needs an - independent account or refresh-token lineage; copying one rotating refresh - token into multiple fields recreates the lost-update failure this design - prevents. -- Worker credential files live on separate ordinal-owned RWO `provider-access` - claims. Provider refresh updates these durable private copies across Pod - restarts; they are never synchronized back to Vault. Rotate one bootstrap - credential at a time through the reviewed Vault workflow and reinitialize - only that ordinal after human approval. -- Existing task worktrees remain on the single local owner lane. Only tasks - without `workspace_path` enter the distributed pool, where each ordinal owns - one RWO checkout. Repository and base-branch identity comes from the canonical - board registry rather than task-supplied metadata. -- Model Pods have no pool key, broker mount, broker egress, or Kubernetes token. - Ordinal mediator Deployments are separate network identities that co-mount the - matching workspace PVC; they alone authenticate exact-run results and reach the - PR14 SCM broker. -- The ordinal workspace claim is `ReadWriteMany` because the worker and its - mediator both mount it. With `ReadWriteOnce` a drain or preemption that moved - only one of them left the other holding the attachment on the old node, and - the ordinal deadlocked on Multi-Attach until an operator deleted a Pod. - Colocation is now a scheduling preference and both Pods run at `scavenger` - priority, so either can be evicted and rescheduled on its own. There is - deliberately no PodDisruptionBudget: this is a best-effort pool and a budget - would impede node drains. Claims only one Pod mounts stay `ReadWriteOnce`. -- `lease_failed` is a *retryable* durable state, not a terminal one. It means the - ordinal was released but Kanban has not confirmed the outcome, so every - maintenance pass retries it. Only `finalized` and `stale` are terminal, and - only terminal rows are garbage-collected, so no run is dropped before the pool - knows what happened to it. If the log keeps reporting a deferred lease park for - the same run, Kanban is refusing an exact-run park on a task it still reports - as running: inspect that task rather than deleting the row. -- Ordinary work publishes a newly created broker branch. A trusted - coordinator-issued continuation is the sole exception: it private-clones the - latest verified head of the recorded PR branch and receives a signed grant to - update that exact ref. It never adopts a caller-provided ref, force-pushes, or - creates a replacement PR. A submission that cannot land blocks the run with - the reason and leaves the commits on the ordinal's workspace, so prior work is - recoverable. -- Provider CLIs install once per pinned version onto a durable per-ordinal - `tools` claim, re-verified against the real binaries so a pruned cache - reinstalls. The pool is intentionally absent from the `hermes` Kustomization's - `healthChecks`: gating that 10m window on a best-effort pool would stall - `hermes-chat` and `hermes-observer-bindings`, which `dependsOn: hermes`. -- A hashed execution-pool ConfigMap, protocol-version readiness checks, and - versioned SCM boundary name make code/config changes controlled rollouts. - Verify the config revision, protocol readiness, and worker/mediator health - after Flux applies a reviewed rollout. +- Each execution ordinal keeps its own durable task and provider-session state. + Do not copy, delete, or reconstruct provider credentials or session paths to + recover a worker; bind the trusted task session and let the worker defer when + that state is unavailable. +- Each worker and its mediator share an RWX workspace so either can reschedule + during a drain. Provider-session and tools claims remain per-ordinal RWO, + keeping private state isolated while retaining pinned CLI installations. +- The durable tools claim records a pinned CLI version only after the expected + binaries are executable; a missing or pruned binary causes a bounded reinstall. +- The native execution pool uses its authenticated Claude provider lane for + execution. Sol and Astra are models selected for advanced/frontier routes in + Hermes operator/chat routing; they do not create a separate credential prerequisite + for an execution ordinal. +- The direct CLI lane may continue only an existing owned workspace. Pathless + work belongs to the pool, and a trusted continuation always derives its + repository and branch from the recorded PR lineage rather than a legacy local + checkout. +- Model pods have no pool key, broker mount, broker egress, or Kubernetes token. + Per-ordinal mediator deployments co-mount the matching workspace state; they + alone authenticate exact-run results and reach the SCM broker. +- `lease_failed` is durable retryable state. It is retained until native Kanban + confirms the exact run is finalized or stale; do not delete a deferred row to + force new work. +- Ordinary work creates an allowed broker branch. A trusted continuation instead + clones the latest verified head of its recorded PR branch and receives a + signed grant for that same ref. It never force-pushes or opens a replacement + PR. +- SCM broker availability is a runtime condition for mediated SCM operations, + not a Flux rollout gate for Hermes. Normal Hermes image, chat, model, and + tool rollouts continue while the broker is unavailable; publication work + defers until it returns. +- The pool remains outside the `hermes` Kustomization health checks because it + is best-effort capacity. Verify the mounted config revision, protocol + readiness, and worker/mediator health after a reviewed rollout. ## Your shortest path to fluency diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml index 22197ac7..fdc64a5d 100644 --- a/services/hermes/agent-deployment.yaml +++ b/services/hermes/agent-deployment.yaml @@ -22,7 +22,7 @@ spec: annotations: ai.bstein.dev/role: project-coordinator ai.bstein.dev/router-wire-contract: ollama-numeric-keepalive - ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers + ai.bstein.dev/execution: Hermes Kanban with isolated Claude workers and credential-owning Switchyard lanes ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback ai.bstein.dev/placement: primary amd64 accelerator titan-22; arm64 rpi5 fleet fallback; storage-backbone nodes excluded ai.bstein.dev/config-rev: "20260913-soteria-kanban-recovery-v2" diff --git a/services/hermes/scripts/execution_pool_worker.py b/services/hermes/scripts/execution_pool_worker.py index 65396bcb..7d3c3709 100644 --- a/services/hermes/scripts/execution_pool_worker.py +++ b/services/hermes/scripts/execution_pool_worker.py @@ -15,6 +15,7 @@ from pathlib import Path from typing import Any import cli_lane_runner import execution_pool_resume +from cli_lane_routing import parse_assignee_capability from execution_pool_protocol import ( ProtocolError, atomic_json, @@ -288,6 +289,10 @@ def execute(assignment: dict[str, Any]) -> None: if not _client("finish", binding=binding, payload=terminal, title=title, body=body).get("ack", {}).get("accepted"): raise ProtocolError("coordinator did not accept the publication retry") return + assignee = str(payload.get("assignee") or "cli-auto") + requested_provider, _effort, _capability = parse_assignee_capability(assignee) + if requested_provider == DISABLED_PROVIDER: + raise ProtocolError("assignment selects a disabled provider") _bind_provider_sessions(assignment) workspace = Path(str(assignment.get("workspace") or "")).resolve(strict=True) workspace.relative_to((ROOT / "runs").resolve()) @@ -314,9 +319,8 @@ def execute(assignment: dict[str, Any]) -> None: except (OSError, ValueError, urllib.error.URLError, json.JSONDecodeError): return False context = str(payload.get("context") or "") - assignee = str(payload.get("assignee") or "cli-auto") excluded = None - if assignee == "cli-auto": + if requested_provider is None: excluded = DISABLED_PROVIDER or cli_lane_runner.fresh_unavailable_provider() route = cli_lane_runner.select_route( context, diff --git a/testing/tests/test_hermes_execution_pool_worker_execute.py b/testing/tests/test_hermes_execution_pool_worker_execute.py index ac79f94f..dfef1519 100644 --- a/testing/tests/test_hermes_execution_pool_worker_execute.py +++ b/testing/tests/test_hermes_execution_pool_worker_execute.py @@ -122,6 +122,46 @@ def test_execute_completed_clean_result_refreshes_and_finishes_exact_run( assert state["terminal_at"] > 0 and state["baseline_sha"] == "a" * 40 +@pytest.mark.parametrize("assignee", ["cli-codex-high", "CLI-CODEX-FRONTIER-XHIGH"]) +def test_execute_rejects_an_explicitly_disabled_provider_before_any_worker_action( + tmp_path, monkeypatch, assignee +): + """An unavailable native lane cannot be selected by a manual assignee.""" + exact = assignment(payload={ + "context": "safe objective", "assignee": assignee, + "deadline_unix": 10_000_000_000, "max_runtime_seconds": 3600, + }) + exact, _workspace, calls, providers, routes, _refreshed = prepare_execute( + tmp_path, monkeypatch, item=exact + ) + monkeypatch.setattr(worker, "DISABLED_PROVIDER", "codex") + monkeypatch.setattr( + worker, "_bind_provider_sessions", lambda _a: pytest.fail("staged provider state") + ) + + with pytest.raises(protocol.ProtocolError, match="disabled provider"): + worker.execute(exact) + + assert calls == [] and providers == [] and routes == [] + + +def test_execute_normalizes_auto_before_excluding_disabled_provider(tmp_path, monkeypatch): + """Whitespace and case cannot bypass the automatic disabled-provider guard.""" + exact = assignment(payload={ + "context": "safe objective", "assignee": " CLI-AUTO ", + "deadline_unix": 10_000_000_000, "max_runtime_seconds": 3600, + }) + exact, workspace, _calls, _providers, routes, _refreshed = prepare_execute( + tmp_path, monkeypatch, item=exact + ) + exact["workspace"] = str(workspace) + monkeypatch.setattr(worker, "DISABLED_PROVIDER", "codex") + + worker.execute(exact) + + assert routes[0][1]["exclude_provider"] == "codex" + + @pytest.mark.parametrize("summary", ["x" * 300, "界" * 300, "Ж" * 300]) def test_execute_titles_are_accepted_by_gitea_draft_policy(tmp_path, monkeypatch, summary): """Ordinary worker publication preserves Gitea's character and byte limits."""