titan/atlas-iac
3
0
Fork 0
Code Issues Pull Requests 9 Actions Packages Projects Releases Wiki Activity

hermes: route by capability and continue existing pull requests

Browse Source
This commit is contained in:
jenkins 2026-09-13 15:04:01 -05:00
parent 46f9995fe5
commit e040102121
72 changed files with 5190 additions and 526 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
dockerfiles/Dockerfile.hermes-switchyard
View File

@ -1,21 +1,57 @@
# syntax=docker/dockerfile:1.7
# dockerfiles/Dockerfile.hermes-switchyard
FROM rust:1.96.1-slim-bookworm@sha256:e18a79fc84dfcfc3ab5ba72290398a644c135c97eaa881447fddc354ee4701a3 AS build
FROM --platform=$BUILDPLATFORM rust:1.96.1-slim-bookworm@sha256:e18a79fc84dfcfc3ab5ba72290398a644c135c97eaa881447fddc354ee4701a3 AS build
ARG SWITCHYARD_LIBSY_SHA256=85f12e1ffefa168604044ccf78535a34dcde38eb59653ac27465e44113cd848c
ARG SWITCHYARD_SERVER_SHA256=95946e3df637143dcdd2a34c153bcaa57b7d4bc3dee53ed6c5497ddd996e16bc
ARG TARGETARCH
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
build-essential \
ca-certificates \
cmake \
curl \
gcc-aarch64-linux-gnu \
libc6-dev-arm64-cross \
linux-libc-dev-arm64-cross \
patch \
pkg-config \
&& rm -rf /var/lib/apt/lists/*
RUN cargo install \
--locked \
--version 0.2.0 \
--root /opt/switchyard \
switchyard-server
COPY dockerfiles/switchyard-capability-fallback.patch /tmp/switchyard-capability-fallback.patch
FROM debian:bookworm-slim@sha256:abd67ffcfa541b485a3dff59865ab629aa048a6c613e639d36e7456b0b229241
RUN rustup target add aarch64-unknown-linux-gnu
ENV CC_aarch64_unknown_linux_gnu=aarch64-linux-gnu-gcc \
CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc
RUN --mount=type=cache,id=switchyard-cargo-registry,target=/usr/local/cargo/registry,sharing=locked \
--mount=type=cache,id=switchyard-cargo-target-aarch64,target=/opt/switchyard-target,sharing=locked \
set -eux; \
test "$TARGETARCH" = arm64; \
mkdir -p /opt/switchyard-source; \
curl --fail --location --retry 3 --output /tmp/libsy.crate \
https://static.crates.io/crates/switchyard-libsy/switchyard-libsy-0.2.0.crate; \
echo "${SWITCHYARD_LIBSY_SHA256} /tmp/libsy.crate" | sha256sum --check --status -; \
tar --extract --gzip --file /tmp/libsy.crate --directory /opt/switchyard-source; \
patch --strip=1 --directory /opt/switchyard-source/switchyard-libsy-0.2.0 \
--input /tmp/switchyard-capability-fallback.patch; \
curl --fail --location --retry 3 --output /tmp/server.crate \
https://static.crates.io/crates/switchyard-server/switchyard-server-0.2.0.crate; \
echo "${SWITCHYARD_SERVER_SHA256} /tmp/server.crate" | sha256sum --check --status -; \
tar --extract --gzip --file /tmp/server.crate --directory /opt/switchyard-source; \
CARGO_TARGET_DIR=/opt/switchyard-target cargo test --locked \
--manifest-path /opt/switchyard-source/switchyard-libsy-0.2.0/Cargo.toml \
automatic_fallback --lib; \
printf '\n[patch.crates-io]\nswitchyard-libsy = { path = "/opt/switchyard-source/switchyard-libsy-0.2.0" }\n' \
>> /opt/switchyard-source/switchyard-server-0.2.0/Cargo.toml; \
CARGO_TARGET_DIR=/opt/switchyard-target cargo install --locked --target aarch64-unknown-linux-gnu \
--path /opt/switchyard-source/switchyard-server-0.2.0 \
--root /opt/switchyard; \
rm -rf /opt/switchyard-source /tmp/libsy.crate /tmp/server.crate /tmp/switchyard-capability-fallback.patch
FROM --platform=$TARGETPLATFORM debian:bookworm-slim@sha256:abd67ffcfa541b485a3dff59865ab629aa048a6c613e639d36e7456b0b229241
COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
COPY --from=build /opt/switchyard/bin/switchyard-server /usr/local/bin/switchyard-server

98
dockerfiles/switchyard-capability-fallback.patch Normal file
View File

@ -0,0 +1,98 @@
--- a/src/algorithms/fall_through.rs
+++ b/src/algorithms/fall_through.rs
@@ -1016,0 +1017,43 @@
+ async fn automatic_fallback_keeps_capability_and_effort_floor() -> Result<()> {
+ for (frontier, alternate, lower_role) in [
+ (
+ "route/codex/auto-frontier/xhigh",
+ "route/claude/auto-frontier/xhigh",
+ "route/codex/auto-advanced/xhigh",
+ ),
+ (
+ "worker/codex/auto-frontier/xhigh",
+ "worker/claude/auto-frontier/xhigh",
+ "worker/codex/auto-advanced/xhigh",
+ ),
+ ] {
+ let router = FallThrough::<()>::new(target_set_with_overflow(
+ &[frontier, lower_role, alternate],
+ &[frontier],
+ ))
+ .with_classifier(fixed(vec![score(frontier, 1.0)]));
+
+ let (model, _) = run(router).await?;
+ assert_eq!(model, alternate);
+ }
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn automatic_fallback_does_not_reset_its_floor_after_an_alternate_fails() {
+ let frontier = "route/codex/auto-frontier/xhigh";
+ let alternate = "route/claude/auto-frontier/xhigh";
+ let lower_role = "route/codex/auto-advanced/xhigh";
+ let calls = Arc::new(Mutex::new(Vec::new()));
+ let router = FallThrough::<()>::new(counting_overflow_targets(
+ &[frontier, lower_role, alternate],
+ &[frontier, alternate],
+ Arc::clone(&calls),
+ ))
+ .with_classifier(fixed(vec![score(frontier, 1.0)]));
+
+ assert!(run(router).await.is_err());
+ assert_eq!(&*calls.lock(), &[frontier, alternate]);
+ }
+
+ #[tokio::test]
--- a/src/core/algorithm.rs
+++ b/src/core/algorithm.rs
@@ -396,2 +396,8 @@
- /// The named target, or the first one this request is not barred from when it has been
- /// excluded (see [`Context::exclude_target`]). Errors if every target is excluded.
+ /// The named target, or the first compatible target not barred for this request.
+ ///
+ /// Automatic capability selectors have two independent components: the
+ /// stable selector (`auto-frontier`, for example) and the effort floor.
+ /// A provider outage may switch provider, but it must not turn that target
+ /// into an unrelated lower capability or effort merely because it appears
+ /// earlier in the route's complete candidate list. Non-AUTO target names
+ /// retain the existing first-eligible fallback behavior.
@@ -402,0 +409,12 @@
+ if let Some((scope, selector, effort)) = automatic_floor(name) {
+ return self
+ .targets
+ .iter()
+ .find(|candidate| {
+ !ctx.is_excluded(&candidate.semantic_name)
+ && automatic_floor(&candidate.semantic_name)
+ == Some((scope, selector, effort))
+ })
+ .cloned()
+ .ok_or(LibsyError::AllTargetsExcluded);
+ }
@@ -408,0 +427,25 @@
+}
+
+/// Parse the Switchyard semantic names used by AUTO capability routes.
+///
+/// `route/<provider>/<selector>/<effort>` serves hosted calls and
+/// `worker/<provider>/<selector>/<effort>` serves durable workers. The provider
+/// is deliberately excluded from this floor so infrastructure failover can use
+/// the other provider without changing capability or reasoning effort.
+fn automatic_floor(name: &str) -> Option<(&str, &str, &str)> {
+ let mut parts = name.split('/');
+ let scope = parts.next()?;
+ let _provider = parts.next()?;
+ let selector = parts.next()?;
+ let effort = parts.next()?;
+ if parts.next().is_some()
+ || !matches!(scope, "route" | "worker")
+ || !matches!(
+ selector,
+ "auto" | "auto-economy" | "auto-balanced" | "auto-advanced" | "auto-frontier"
+ )
+ || !matches!(effort, "low" | "medium" | "high" | "xhigh")
+ {
+ return None;
+ }
+ Some((scope, selector, effort))

18
services/hermes-scm-broker/deployment.yaml
View File

@ -8,6 +8,8 @@ metadata:
app: hermes-scm-broker
spec:
replicas: 1
strategy:
type: Recreate
revisionHistoryLimit: 2
selector:
matchLabels:
@ -17,7 +19,7 @@ spec:
labels:
app: hermes-scm-broker
annotations:
ai.bstein.dev/config-rev: scm-boundary-v3
ai.bstein.dev/config-rev: scm-boundary-v4-task-ledger
vault.hashicorp.com/agent-inject: "true"
vault.hashicorp.com/role: hermes-scm-broker
vault.hashicorp.com/agent-inject-secret-gitea-token: kv/data/atlas/hermes/developer-gitea
@ -25,6 +27,11 @@ spec:
{{- with secret "kv/data/atlas/hermes/developer-gitea" -}}
{{ .Data.data.token }}
{{- end }}
vault.hashicorp.com/agent-inject-secret-scm-task-grant: kv/data/atlas/hermes/scm-task-grant
vault.hashicorp.com/agent-inject-template-scm-task-grant: |
{{- with secret "kv/data/atlas/hermes/scm-task-grant" -}}
{{ .Data.data.key }}
{{- end }}
vault.hashicorp.com/agent-pre-populate-only: "true"
vault.hashicorp.com/agent-init-first: "true"
vault.hashicorp.com/agent-requests-cpu: 10m
@ -86,6 +93,8 @@ spec:
volumeMounts:
- {name: broker-code, mountPath: /opt/broker, readOnly: true}
- {name: tmp, mountPath: /tmp}
- {name: task-ledger, mountPath: /scm-state}
- {name: task-adoptions, mountPath: /scm-adoptions, readOnly: true}
resources:
requests: {cpu: 50m, memory: 128Mi}
limits: {cpu: "1", memory: 768Mi}
@ -97,3 +106,10 @@ spec:
- name: tmp
emptyDir:
sizeLimit: 1Gi
- name: task-ledger
persistentVolumeClaim:
claimName: hermes-scm-task-ledger
- name: task-adoptions
configMap:
name: hermes-scm-task-branch-adoptions
defaultMode: 0444

2
services/hermes-scm-broker/kustomization.yaml
View File

@ -3,6 +3,8 @@ apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: hermes-scm
resources:
- task-branch-adoptions-configmap.yaml
- task-ledger-pvc.yaml
- service.yaml
- deployment.yaml
- networkpolicy.yaml

22
services/hermes-scm-broker/task-branch-adoptions-configmap.yaml Normal file
View File

@ -0,0 +1,22 @@
# services/hermes-scm-broker/task-branch-adoptions-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: hermes-scm-task-branch-adoptions
namespace: hermes-scm
data:
# Reviewed open PR branches only. Broker startup verifies each live head and
# leaves a stale record deferred; the durable ledger never rewinds a newer head.
task-branch-adoptions.json: |
{
"soteria/wt/t_f1593f8c": {"repo":"soteria","ref":"wt/t_f1593f8c","board":"soteria","root_task_id":"t_f1593f8c","latest_head":"0143d472469c8dfe44f23f1440123e27d415baae","pr_number":11},
"soteria/hermes-repair/sonar-AZ9pTqVcN0JrBQvDGDs3": {"repo":"soteria","ref":"hermes-repair/sonar-AZ9pTqVcN0JrBQvDGDs3","board":"soteria","root_task_id":"t_c7c42600","latest_head":"51133b559ad62f324e45bc61587900f08156b733","pr_number":3},
"soteria/hermes-repair/sonar-AZ9pTqWRN0JrBQvDGDs4": {"repo":"soteria","ref":"hermes-repair/sonar-AZ9pTqWRN0JrBQvDGDs4","board":"soteria","root_task_id":"t_f4f726e1","latest_head":"45170df566c2aa387f8b706c3a704abce5aa2557","pr_number":4},
"atlas-iac/feature/t_26da4c88-titan-capacity-guardrails-v4": {"repo":"atlas-iac","ref":"feature/t_26da4c88-titan-capacity-guardrails-v4","board":"titan-iac","root_task_id":"t_26da4c88","latest_head":"3c6301d581bef0a1fce5284f9a28c1cf4c4a99ad","pr_number":53},
"atlas-iac/feature/t_39cf1905-webui-build-token": {"repo":"atlas-iac","ref":"feature/t_39cf1905-webui-build-token","board":"titan-iac","root_task_id":"t_39cf1905","latest_head":"f94f96a7042ba863768938e0c1afa79406397b77","pr_number":54},
"atlas-iac/feature/hermes-three-lane-placement": {"repo":"atlas-iac","ref":"feature/hermes-three-lane-placement","board":"titan-iac","root_task_id":"t_e9597d89","latest_head":"48cbe13ee50ce3fcb07cea3fe8d088cfed349e0c","pr_number":17},
"atlas-iac/feature/hermes-cli-process-reaping": {"repo":"atlas-iac","ref":"feature/hermes-cli-process-reaping","board":"titan-iac","root_task_id":"t_a6a22d7c","latest_head":"a242dcc786576ae1a18000cb4941836ff610dff2","pr_number":20},
"atlas-iac/fix/t_39cf1905-jenkins-controller-priority": {"repo":"atlas-iac","ref":"fix/t_39cf1905-jenkins-controller-priority","board":"titan-iac","root_task_id":"t_c425c446","latest_head":"f997171b5526f104d2474022c2a56683ec2960c3","pr_number":49},
"atlas-iac/feature/hermes-next-hux": {"repo":"atlas-iac","ref":"feature/hermes-next-hux","board":"titan-iac","root_task_id":"t_cf89a2ec","latest_head":"98c7c6184f6edfe3cdac228529c2287584db3006","pr_number":55},
"cassandra/handoff/generated-strategy-audit-20260813": {"repo":"cassandra","ref":"handoff/generated-strategy-audit-20260813","board":"cassandra","root_task_id":"t_b89e3903","latest_head":"14c07111b6ed8d7fc529362eb34fa0afa0694325","pr_number":1}
}

11
services/hermes-scm-broker/task-ledger-pvc.yaml Normal file
View File

@ -0,0 +1,11 @@
# services/hermes-scm-broker/task-ledger-pvc.yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: hermes-scm-task-ledger
namespace: hermes-scm
spec:
accessModes: [ReadWriteOnce]
resources:
requests:
storage: 1Gi

30
services/hermes/NOTES.md
View File

@ -48,6 +48,21 @@ or the WebUI. Browser chat remains available when `bot_token` is empty.
The bot token and relay key must never be added to Git or a Kubernetes Secret.
The router does not log prompt bodies, raw Telegram IDs, link codes, or tokens.
## Switchyard capability routing
Switchyard chooses capability before reasoning effort. `economy`, `balanced`,
`advanced`, and `frontier` select Luna-, Terra-, Sol-, and Astra-class work;
`low` through `xhigh` control deliberation independently. Legacy AUTO keeps
low/economy, medium/balanced, and high or xhigh/advanced behavior. Frontier is
an explicit exceptional-work choice, not a synonym for xhigh.
Timeout, rate-limit, authentication, transport, and capacity failures retry an
available provider at the same capability and effort. A failed test, rejected
review, contradicted result, or incomplete evidence is a quality signal: keep
the floor, change the failed plan, and raise capability or effort when needed.
The concrete selected model, capability, effort, and rationale are recorded on
durable worker receipts.
## Private Jetson voice: multilingual TTS policy
`hermes-tts` on `titan-21` bakes three checksum-pinned Piper voices and
@ -473,14 +488,13 @@ A `voice` field from a browser is never honoured at any hop.
knows what happened to it. If the log keeps reporting a deferred lease park for
the same run, Kanban is refusing an exact-run park on a task it still reports
as running: inspect that task rather than deleting the row.
- The broker accepts branch creation only. A pool retry that adds commits
therefore publishes a new `<branch>-attempt-<n>` (or, failing that,
`<branch>-<head12>`) ref in the same reviewed namespace instead of updating the
ref a previous attempt published, and adopts an existing ref that already
points at the exact head rather than pushing again. Expect one draft per
published ref; each is still human-reviewed. A submission that cannot land
blocks the run with the reason and leaves the commits on the ordinal's
workspace, so prior work is recoverable.
- Ordinary work publishes a newly created broker branch. A trusted
coordinator-issued continuation is the sole exception: it private-clones the
latest verified head of the recorded PR branch and receives a signed grant to
update that exact ref. It never adopts a caller-provided ref, force-pushes, or
creates a replacement PR. A submission that cannot land blocks the run with
the reason and leaves the commits on the ordinal's workspace, so prior work is
recoverable.
- Provider CLIs install once per pinned version onto a durable per-ordinal
`tools` claim, re-verified against the real binaries so a pruned cache
reinstalls. The pool is intentionally absent from the `hermes` Kustomization's

65
services/hermes/agent-configmap.yaml
View File

@ -99,7 +99,7 @@ data:
supervise_interval_seconds: 30
supervise_max_cycles: 5
supervise_max_chains: 20
supervise_review_assignee: cli-claude-xhigh
supervise_review_assignee: cli-auto
supervise_repair_assignee: cli-auto
dispatch_stale_timeout_seconds: 14400
# Steady-state quota-aware routing for the direct CLI lane: below this
@ -189,8 +189,9 @@ data:
fan-out, and synthesis path. Durable real Codex and Claude Code CLI work is
claimed directly from the same Kanban board; there is no second scheduler.
You remain responsible for planning, decomposition, review, and the final
synthesized answer. Switchyard is the sole authority for provider, model,
effort, and capacity failover at every model-call boundary.
synthesized answer. Switchyard is the sole authority for provider, concrete
model, capability role, reasoning effort, and capacity failover at every
model-call boundary.
Prefer Codex for implementation, debugging, test loops, and focused repo
changes. Prefer Claude Code for architecture, long-context investigation,
@ -199,8 +200,13 @@ data:
Start in AUTO routing with a very strong preference for correctness. Every
user turn, internal tool-loop continuation, delegated child, and durable
CLI task must be independently classified before choosing provider, model,
and effort. Understand natural requests for speed or deeper thought as
CLI task must be independently classified before choosing provider,
capability role, reasoning effort, and concrete model. Capability comes
first: economy is Luna-class simple work, balanced is Terra-class routine
tool work, advanced is Sol-class ordinary complex work, and frontier is
Astra-class exceptional ambiguity or deeply coupled architecture. Effort
is independent: high or xhigh does not imply frontier. Understand natural
requests for speed or deeper thought as
semantic intent rather than a closed phrase list. A faster preference may
reduce unnecessary deliberation, but must never undercut the safety floor
for production changes, security, migrations, destructive work, or final
@ -249,6 +255,13 @@ data:
`/route manual <codex|claude> <low|medium|high|xhigh> [model]` for a
persistent manual override, and `/route auto` to return control to Hermes.
Capability is independent from effort: `economy` is Luna-class simple work,
`balanced` is Terra-class routine tool work, `advanced` is Sol-class ordinary
complex work, and `frontier` is Astra-class exceptional ambiguity, deeply
coupled architecture, or a meaningful quality escalation. High and xhigh
are effort only. Legacy AUTO maps low to economy, medium to balanced, and
high or xhigh to advanced.
- `low`: simple questions, lookup, formatting, or a tiny reversible edit.
- `medium`: normal bounded implementation or analysis with clear tests.
- `high`: multi-component work, difficult debugging, or material ambiguity.
@ -260,9 +273,10 @@ data:
evidence, not a routing control plane. The hourly steward discovers the
models currently available to both accounts, preserves its last known-good
catalog during outages, and keeps every generated profile on a public
Switchyard route. Profiles are `codex-{low,medium,high,xhigh}` and
`claude-{low,medium,high,xhigh}`, plus `synthesis-xhigh`; selecting one is a
Switchyard constraint and never a direct provider bypass.
Switchyard route. Legacy profiles are `codex-{low,medium,high,xhigh}` and
`claude-{low,medium,high,xhigh}`; explicit capability floors use selectors
such as `cli-codex-frontier-high`. Selecting one is a Switchyard constraint
and never a direct provider bypass.
## Decomposition and delegation
@ -307,7 +321,7 @@ data:
For persistent real Codex or Claude Code CLI work, create a bounded Kanban
worktree task assigned to `cli-auto`. The direct lane reserves the task atomically,
sends every start/retry/continuation boundary through Switchyard and its
Jetson classifier, records provider/model/effort and session identifiers on the task, streams
Jetson classifier, records provider/model/capability/effort and session identifiers on the task, streams
logs into the task worker log, and resumes the provider session after a pod
restart. Manual lanes are `cli-codex-{low,medium,high,xhigh}` and
`cli-claude-{low,medium,high,xhigh}`; those manual constraints are still
@ -316,9 +330,10 @@ data:
login requirement, run `codex login --device-auth` once in `/terminal/` and
ask Brad to complete the displayed code.
A hosted capacity failure should fall across providers at the same effort
before dropping to local inference. Do not duplicate a task that is still
running. When both providers contributed, use `synthesis-xhigh` only if the
A hosted capacity failure should fall across providers at the same
capability and effort. If no hosted provider can meet that floor, defer the
task as transient; never automatically demote it to local inference. Do not
duplicate a task that is still running. When both providers contributed, use `synthesis-xhigh` only if the
objective's difficulty warrants it; otherwise synthesize at the original
effort.
@ -346,8 +361,9 @@ data:
GitHub/`gh` skill for an Atlas remote, and do not interpret an
unauthenticated Gitea HTTP 404 as a missing private repository. Use
brokered Git for clone, fetch, and creation of a new namespaced feature
branch. Existing-ref updates, protected refs, deletion, and force-push are
rejected. The broker inflates and scans every pushed object, so thin
branch. Existing-ref updates are rejected unless a coordinator-issued
continuation has validated the root lineage and worker grant; protected
refs, deletion, and force-push are always rejected. The broker inflates and scans every pushed object, so thin
packs are rejected; always push with `git push --no-thin` so the pack
is self-contained. For bounded
repository/pull-request evidence or to create a review-ready draft PR, load
@ -374,6 +390,22 @@ data:
file as user state. PR publication ends at the verified open draft; Brad
owns review and merge authority.
## Supervisor PR continuation
A supervisor implementation, review, repair, and re-review chain keeps one
immutable root task, project, branch, and pull request in coordinator-owned
state, never model-editable task metadata. For a user follow-up such as
"fix this PR", run `kanban_continue_pr.py` with only the board, root task
ID, and objective; it validates the live owned PR and queues a root-parent
child without caller-provided branch fields. Repair the existing PR on its
recorded branch; never create a replacement branch or PR for a review
finding. Review the current PR head, actual diff, and relevant test evidence
before a SHIP or BLOCK verdict. A base-behind count alone does not retire a
PR. Wait for assigned delegated work before a final verdict. A SHIP applies
only to the exact reviewed current head; advancing that head clears stale
readiness and requires re-review. No created PR means the implementation
objective is still incomplete. Brad retains review and merge authority.
For an explicitly requested Hermes runtime release, trigger only the
reviewed image lanes with `/opt/coordinator/jenkins_image_build_trigger.py`.
Use `--component agent` for backend/runtime changes, `--component webui`
@ -427,8 +459,9 @@ data:
After a failed tool attempt, failing test, contradicted claim, rejected
review, or incomplete evidence, do not repeat the same low-capability plan.
Reassess the objective and raise provider capability or effort by at least
one tier, up to xhigh. Consequential independent final review is always
Reassess the objective and raise provider capability or effort after a
quality failure; timeout, rate-limit, authentication, transport, and
provider-capacity failures preserve both floors. Consequential independent final review is always
xhigh and must be separated from the implementing worker.
START-HERE.md: |
# Agent Hermes

5
services/hermes/agent-deployment.yaml
View File

@ -25,7 +25,7 @@ spec:
ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers
ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback
ai.bstein.dev/placement: primary amd64 accelerator titan-22; arm64 rpi5 fleet fallback; storage-backbone nodes excluded
ai.bstein.dev/config-rev: "20260913-session-tool-history-v4"
ai.bstein.dev/config-rev: "20260913-capability-effort-v3"
prometheus.io/scrape: "true"
prometheus.io/path: /metrics
prometheus.io/port: "9010"
@ -1166,6 +1166,9 @@ spec:
- {name: CLAUDE_CONFIG_DIR, value: /runtime-access/claude}
- {name: PYTHONPATH, value: /opt/hermes}
- {name: HERMES_ROUTING_CATALOG_PATH, value: /routing-catalog/catalog.json}
- {name: HERMES_MODEL_EVAL_CODEX_ENDPOINT, value: 'http://127.0.0.1:9003/v1/responses'}
- {name: HERMES_MODEL_EVAL_CLAUDE_ENDPOINT, value: 'http://127.0.0.1:9006/v1/messages'}
- {name: HERMES_MODEL_EVAL_KEY_FILE, value: /runtime-access/chat-relay-key}
- {name: HERMES_CASSANDRA_ACTIVE_WORKTREE, value: /opt/data/workspace/projects/cassandra-hermes-v69}
- {name: PATH, value: '/opt/coordinator:/opt/data/tools/bin:/opt/hermes/.venv/bin:/usr/local/bin:/usr/bin:/bin'}
securityContext:

21
services/hermes/execution-mediator.yaml
View File

@ -73,6 +73,12 @@ spec:
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
{{ printf "hermes-execution-pool-root-v2:%s" .Data.data.agent_api_key | sha256Hex }}
{{- end }}
vault.hashicorp.com/agent-inject-secret-scm-task-grant-key: kv/data/atlas/hermes/agent-tokens
vault.hashicorp.com/agent-inject-perms-scm-task-grant-key: "0600"
vault.hashicorp.com/agent-inject-template-scm-task-grant-key: |
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
{{ printf "hermes-scm-task-grant-v1:%s" .Data.data.agent_api_key | sha256Hex }}
{{- end }}
vault.hashicorp.com/agent-pre-populate-only: "true"
vault.hashicorp.com/agent-init-first: "true"
vault.hashicorp.com/agent-requests-cpu: 2m
@ -138,6 +144,7 @@ spec:
- {name: HERMES_WORKER_ROOT, value: /workspace}
- {name: HERMES_SCM_STATE_ROOT, value: /scm-state}
- {name: HERMES_EXECUTION_POOL_KEY_FILE, value: /pool-access/execution-pool-key}
- {name: HERMES_SCM_TASK_GRANT_KEY_FILE, value: /pool-access/scm-task-grant-key}
- {name: PYTHONPATH, value: /opt/scm:/opt/coordinator:/opt/hermes}
ports: [{name: mediator, containerPort: 9009, protocol: TCP}]
startupProbe:
@ -216,6 +223,12 @@ spec:
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
{{ printf "hermes-execution-pool-root-v2:%s" .Data.data.agent_api_key | sha256Hex }}
{{- end }}
vault.hashicorp.com/agent-inject-secret-scm-task-grant-key: kv/data/atlas/hermes/agent-tokens
vault.hashicorp.com/agent-inject-perms-scm-task-grant-key: "0600"
vault.hashicorp.com/agent-inject-template-scm-task-grant-key: |
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
{{ printf "hermes-scm-task-grant-v1:%s" .Data.data.agent_api_key | sha256Hex }}
{{- end }}
vault.hashicorp.com/agent-pre-populate-only: "true"
vault.hashicorp.com/agent-init-first: "true"
vault.hashicorp.com/agent-requests-cpu: 2m
@ -281,6 +294,7 @@ spec:
- {name: HERMES_WORKER_ROOT, value: /workspace}
- {name: HERMES_SCM_STATE_ROOT, value: /scm-state}
- {name: HERMES_EXECUTION_POOL_KEY_FILE, value: /pool-access/execution-pool-key}
- {name: HERMES_SCM_TASK_GRANT_KEY_FILE, value: /pool-access/scm-task-grant-key}
- {name: PYTHONPATH, value: /opt/scm:/opt/coordinator:/opt/hermes}
ports: [{name: mediator, containerPort: 9009, protocol: TCP}]
startupProbe:
@ -359,6 +373,12 @@ spec:
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
{{ printf "hermes-execution-pool-root-v2:%s" .Data.data.agent_api_key | sha256Hex }}
{{- end }}
vault.hashicorp.com/agent-inject-secret-scm-task-grant-key: kv/data/atlas/hermes/agent-tokens
vault.hashicorp.com/agent-inject-perms-scm-task-grant-key: "0600"
vault.hashicorp.com/agent-inject-template-scm-task-grant-key: |
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
{{ printf "hermes-scm-task-grant-v1:%s" .Data.data.agent_api_key | sha256Hex }}
{{- end }}
vault.hashicorp.com/agent-pre-populate-only: "true"
vault.hashicorp.com/agent-init-first: "true"
vault.hashicorp.com/agent-requests-cpu: 2m
@ -424,6 +444,7 @@ spec:
- {name: HERMES_WORKER_ROOT, value: /workspace}
- {name: HERMES_SCM_STATE_ROOT, value: /scm-state}
- {name: HERMES_EXECUTION_POOL_KEY_FILE, value: /pool-access/execution-pool-key}
- {name: HERMES_SCM_TASK_GRANT_KEY_FILE, value: /pool-access/scm-task-grant-key}
- {name: PYTHONPATH, value: /opt/scm:/opt/coordinator:/opt/hermes}
ports: [{name: mediator, containerPort: 9009, protocol: TCP}]
startupProbe:

20
services/hermes/kustomization.yaml
View File

@ -86,9 +86,12 @@ configMapGenerator:
- cli_lane_records.py=scripts/cli_lane_records.py
- cli_lane_recovery.py=scripts/cli_lane_recovery.py
- cli_lane_retention.py=scripts/cli_lane_retention.py
- cli_lane_route_quality.py=scripts/cli_lane_route_quality.py
- cli_lane_routing.py=scripts/cli_lane_routing.py
- cli_lane_runner.py=scripts/cli_lane_runner.py
- routing_catalog.py=scripts/routing_catalog.py
- provider_model_catalog.py=scripts/provider_model_catalog.py
- model_evaluation_evidence.py=scripts/model_evaluation_evidence.py
- execution_pool_protocol.py=scripts/execution_pool_protocol.py
- execution_pool_store.py=scripts/execution_pool_store.py
- execution_pool_project.py=scripts/execution_pool_project.py
@ -98,8 +101,15 @@ configMapGenerator:
- execution_pool_client.py=scripts/execution_pool_client.py
- execution_pool_worker.py=scripts/execution_pool_worker.py
- execution_pool_scm.py=scripts/execution_pool_scm.py
- supervisor_lineage.py=scripts/supervisor_lineage.py
- supervisor_state.py=scripts/supervisor_state.py
- seed_legacy_scm_roots.py=scripts/seed_legacy_scm_roots.py
- deadline_http.py=scm-common/scripts/deadline_http.py
- gitea_api_policy.py=scm-common/scripts/gitea_api_policy.py
- scm_broker_client.py=scm-common/scripts/scm_broker_client.py
- scm_task_grants.py=scm-common/scripts/scm_task_grants.py
- scm_task_drafts.py=scm-common/scripts/scm_task_drafts.py
- scm_task_adoptions.py=scm-common/scripts/scm_task_adoptions.py
- stage_runtime_access.py=scripts/stage_runtime_access.py
- name: hermes-coordinator
namespace: hermes
@ -131,6 +141,7 @@ configMapGenerator:
- cli_lane_records.py=scripts/cli_lane_records.py
- cli_lane_recovery.py=scripts/cli_lane_recovery.py
- cli_lane_retention.py=scripts/cli_lane_retention.py
- cli_lane_route_quality.py=scripts/cli_lane_route_quality.py
- cli_lane_routing.py=scripts/cli_lane_routing.py
- cli_lane_runner.py=scripts/cli_lane_runner.py
- codex=scripts/codex
@ -143,6 +154,9 @@ configMapGenerator:
- hermes_coordinator.py=scripts/hermes_coordinator.py
- hermes_model_routing.py=scripts/hermes_model_routing.py
- provider_model_catalog.py=scripts/provider_model_catalog.py
- model_capability_evaluator.py=scripts/model_capability_evaluator.py
- model_evaluation_evidence.py=scripts/model_evaluation_evidence.py
- model_catalog_refresh.py=scripts/model_catalog_refresh.py
- provider_model_discovery.py=scripts/provider_model_discovery.py
- claude_model_discovery.py=scripts/claude_model_discovery.py
- hermes_stt_client.py=scripts/hermes_stt_client.py
@ -153,7 +167,13 @@ configMapGenerator:
- hermes_image_release_status.py=scripts/hermes_image_release_status.py
- kanban_status_recovery.py=scripts/kanban_status_recovery.py
- kanban_supervisor.py=scripts/kanban_supervisor.py
- kanban_continue_pr.py=scripts/kanban_continue_pr.py
- supervisor_policy.py=scripts/supervisor_policy.py
- supervisor_lineage.py=scripts/supervisor_lineage.py
- supervisor_state.py=scripts/supervisor_state.py
- deadline_http.py=scm-common/scripts/deadline_http.py
- gitea_api_policy.py=scm-common/scripts/gitea_api_policy.py
- scm_broker_client.py=scm-common/scripts/scm_broker_client.py
- migrate_herdr_state.py=scripts/migrate_herdr_state.py
- migrate_api_session_lineage.py=scripts/migrate_api_session_lineage.py
- migrate_telegram_api_sessions.py=scripts/migrate_telegram_api_sessions.py

3
services/hermes/scm-common/kustomization.yaml
View File

@ -13,5 +13,8 @@ configMapGenerator:
- scm_broker_io.py=scripts/scm_broker_io.py
- scm_broker_server.py=scripts/scm_broker_server.py
- scm_broker_client.py=scripts/scm_broker_client.py
- scm_task_grants.py=scripts/scm_task_grants.py
- scm_task_drafts.py=scripts/scm_task_drafts.py
- scm_task_adoptions.py=scripts/scm_task_adoptions.py
options:
disableNameSuffixHash: true

5
services/hermes/scm-common/scripts/gitea_api.py
View File

@ -28,6 +28,7 @@ from gitea_api_policy import (
_validate_ref_bounds,
_validate_repo,
_validate_sha,
canonical_api_repo_path,
)
CANONICAL_BASE_URL = "https://scm.bstein.dev"
@ -92,7 +93,7 @@ def _split_api_path(path: str) -> urllib.parse.SplitResult:
segments = target.path.split("/")
if "//" in target.path or any(segment in {".", ".."} for segment in segments):
raise PolicyError("encoded or non-canonical API paths are not allowed")
return target
return target._replace(path=canonical_api_repo_path(target.path))
def _authorize_read(
@ -137,7 +138,7 @@ def _authorize_read(
if suffix == "branches":
_validate_query(target, {"page", "limit"})
return "branch-list"
branch_match = re.fullmatch(r"branches/([^/]+)", suffix)
branch_match = re.fullmatch(r"branches/(.+)", suffix)
if branch_match:
_validate_ref(branch_match.group(1), "branch")
_validate_query(target, set())

9
services/hermes/scm-common/scripts/gitea_api_policy.py
View File

@ -20,6 +20,7 @@ from collections import Counter
from math import log2
REPO_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,99}\Z")
REPOSITORY_ALIASES = {"titan-iac": "atlas-iac"}
SHA_RE = re.compile(r"[0-9a-fA-F]{40}\Z")
DRAFT_TITLE_PREFIX = "WIP: "
GIT_BIN = "/usr/bin/git"
@ -135,7 +136,13 @@ class PolicyError(ValueError):
def _validate_repo(repo: str) -> str:
if not REPO_RE.fullmatch(repo) or repo in {".", ".."}:
raise PolicyError("repository name is outside the Atlas allowlist")
return repo
return REPOSITORY_ALIASES.get(repo, repo)
def canonical_api_repo_path(path: str) -> str:
"""Map the retired local titan-iac path to Forgejo's atlas-iac name."""
legacy = "/api/v1/repos/titan/titan-iac"
return "/api/v1/repos/titan/atlas-iac" + path[len(legacy):] if path == legacy or path.startswith(legacy + "/") else path
def _reject_forbidden(value: object, name: str, forbidden: tuple[str, ...]) -> None:

83
services/hermes/scm-common/scripts/receive_pack_scan.py
View File

@ -15,6 +15,7 @@ supplied by callers.
from __future__ import annotations
import io
import hashlib
import re
from typing import BinaryIO
@ -32,6 +33,9 @@ SCAN_CARRY = 8 * 1024
FEATURE_REF_RE = re.compile(
r"refs/heads/(?:(?:feature|fix|hermes|handoff)/[A-Za-z0-9][A-Za-z0-9._/-]{0,190})\Z"
)
GRANTED_REF_RE = re.compile(
r"refs/heads/(?:(?:feature|fix|chore|docs|test|refactor|wt|review|hermes|hermes-repair|handoff)/[A-Za-z0-9][A-Za-z0-9._/-]{0,190})\Z"
)
PKT_HEADER_RE = re.compile(rb"[0-9a-f]{4}\Z")
CONTENT_SECRET_PATTERNS = tuple(
re.compile(pattern.pattern.encode("ascii"), re.IGNORECASE)
@ -64,8 +68,10 @@ def _read_pkt_line(stream: BinaryIO) -> bytes | None:
return value
def _validate_command(line: bytes, token: str, forbidden: tuple[bytes, ...]) -> None:
"""Permit only creation of one new, namespaced feature branch."""
def _validate_command(
line: bytes, token: str, forbidden: tuple[bytes, ...], *, granted: bool
) -> tuple[str, str, str]:
"""Parse one namespaced branch command without trusting Git's client."""
if any(form in line for form in forbidden):
raise PolicyError("Git request contains runtime credential material")
command = line.rstrip(b"\n").split(b"\x00", 1)[0]
@ -75,15 +81,17 @@ def _validate_command(line: bytes, token: str, forbidden: tuple[bytes, ...]) ->
):
raise PolicyError("Git receive-pack ref command is invalid")
old_sha, new_sha, raw_ref = fields
if old_sha != ZERO_SHA or new_sha == ZERO_SHA:
raise PolicyError("Git broker permits only new feature-branch creation")
if new_sha == ZERO_SHA:
raise PolicyError("Git broker permits only new feature-branch creation or granted updates")
try:
ref = raw_ref.decode("ascii")
except UnicodeDecodeError as exc:
raise PolicyError("Git ref must be canonical ASCII") from exc
if not FEATURE_REF_RE.fullmatch(ref):
allowed = GRANTED_REF_RE if granted else FEATURE_REF_RE
if not allowed.fullmatch(ref):
raise PolicyError("Git push is limited to namespaced feature branches")
_validate_ref(ref.removeprefix("refs/heads/"), "head", forbidden=(token,))
return old_sha.decode("ascii"), new_sha.decode("ascii"), ref.removeprefix("refs/heads/")
def _scan_payload(payload: BinaryIO, forbidden: tuple[bytes, ...]) -> None:
@ -101,37 +109,82 @@ def _scan_payload(payload: BinaryIO, forbidden: tuple[bytes, ...]) -> None:
carry = window[-SCAN_CARRY:]
def _scan_pack(stream: BinaryIO, forbidden: tuple[bytes, ...]) -> None:
def _scan_pack(stream: BinaryIO, forbidden: tuple[bytes, ...]) -> dict[str, tuple[str, ...]]:
objects = git_pack_objects.unpack_objects(stream)
try:
if stream.read(1):
raise PolicyError("Git request has bytes after its pack")
for _type_code, payload in objects:
commits: dict[str, tuple[str, ...]] = {}
for type_code, payload in objects:
if type_code == 1:
raw = payload.read()
digest = hashlib.sha1(b"commit " + str(len(raw)).encode("ascii") + b"\0" + raw).hexdigest()
headers = raw.split(b"\n\n", 1)[0].splitlines()
parents = tuple(line[7:].decode("ascii") for line in headers if line.startswith(b"parent "))
if not all(re.fullmatch(r"[0-9a-f]{40}", parent) for parent in parents):
raise PolicyError("Git commit parent is invalid")
commits[digest] = parents
payload.seek(0)
_scan_payload(payload, forbidden)
return commits
finally:
for _type_code, payload in objects:
payload.close()
def _proves_descends(new_head: str, expected_old: str, commits: dict[str, tuple[str, ...]]) -> bool:
"""Walk quarantined commit parents to prove a non-forced update.
The old head is a trusted stop anchor and need not be present in the pack.
Missing parent objects cannot establish ancestry and therefore fail closed.
"""
if expected_old == ZERO_SHA.decode("ascii"):
return True
pending, seen = [new_head], set()
while pending and len(seen) < 1024:
current = pending.pop()
if current == expected_old:
return True
if current in seen:
continue
seen.add(current)
parents = commits.get(current)
if parents is None:
continue
pending.extend(parents)
return False
def validate_receive_pack(
body: bytes | BinaryIO, token: str, forbidden: tuple[bytes, ...]
) -> None:
body: bytes | BinaryIO, token: str, forbidden: tuple[bytes, ...], *,
expected: tuple[str, str, str] | None = None,
) -> tuple[str, str, str]:
"""Validate one whole receive-pack request: commands, pack, and content."""
stream = io.BytesIO(body) if isinstance(body, bytes) else body
stream.seek(0)
try:
commands = 0
commands: list[tuple[str, str, str]] = []
while True:
line = _read_pkt_line(stream)
if line is None:
break
_validate_command(line, token, forbidden)
commands += 1
if commands > MAX_PUSH_COMMANDS:
commands.append(_validate_command(line, token, forbidden, granted=expected is not None))
if len(commands) > MAX_PUSH_COMMANDS:
raise PolicyError("Git push contains too many ref commands")
if commands == 0:
if not commands:
raise PolicyError("Git receive-pack request has no ref command")
_scan_pack(stream, forbidden)
if len(commands) != 1:
raise PolicyError("Git task push must update exactly one branch")
command = commands[0]
if expected is None:
if command[0] != ZERO_SHA.decode("ascii"):
raise PolicyError("Git broker permits only new feature-branch creation")
elif command != expected:
raise PolicyError("Git command does not match its task grant")
commits = _scan_pack(stream, forbidden)
if expected is not None and not _proves_descends(command[1], command[0], commits):
raise PolicyError("Git update does not prove fast-forward ancestry")
return command
finally:
stream.seek(0)

112
services/hermes/scm-common/scripts/scm_broker.py
View File

@ -24,8 +24,11 @@ from gitea_api import (
read,
read_token,
)
from gitea_api_policy import _reject_forbidden, _validate_repo
from gitea_api_policy import _reject_forbidden, _validate_repo, _validate_sha
from receive_pack_scan import validate_receive_pack
from scm_task_grants import TaskLedger, verify_grant
from scm_task_drafts import matches_pull, request_fields, update as update_draft
from scm_task_adoptions import seed as seed_adoptions
from scm_broker_io import RejectRedirect, response_status as _status, spool_response
from scm_broker_server import AbsoluteHeaderDeadlineMixin, BoundedThreadingHTTPServer
@ -45,8 +48,6 @@ GIT_PATH_RE = re.compile(
r"/git/atlas/(?P<repo>[A-Za-z0-9][A-Za-z0-9._-]{0,99})\.git/"
r"(?P<operation>info/refs|git-upload-pack|git-receive-pack)\Z"
)
def _read_bounded(
stream,
maximum: int,
@ -76,8 +77,6 @@ def _read_bounded(
if len(value) > maximum or (length is not None and len(value) != length):
raise PolicyError("SCM request exceeds the safe size limit")
return value
def _spool_bounded(
stream,
maximum: int,
@ -119,8 +118,6 @@ def _spool_bounded(
except Exception:
spool.close()
raise
def _spool_response(
response, maximum: int, token: str, deadline_seconds: float
) -> tuple[BinaryIO, int]:
@ -132,8 +129,6 @@ def _spool_response(
chunk_size=STREAM_CHUNK,
deadline_seconds=deadline_seconds,
)
def _load_json(handler: BaseHTTPRequestHandler) -> dict[str, object]:
if handler.headers.get("Transfer-Encoding"):
raise PolicyError("chunked broker control requests are not allowed")
@ -153,8 +148,6 @@ def _load_json(handler: BaseHTTPRequestHandler) -> dict[str, object]:
if not isinstance(value, dict):
raise PolicyError("broker control request must be an object")
return value
def _git_target(raw: str) -> tuple[str, str, str]:
if (
not raw.isascii()
@ -185,8 +178,6 @@ def _git_target(raw: str) -> tuple[str, str, str]:
else:
service = operation
return repo, operation, service
def _content_length(headers: object, maximum: int) -> int:
"""Parse one short canonical bounded Content-Length header."""
raw = headers.get("Content-Length", "") # type: ignore[attr-defined]
@ -201,23 +192,29 @@ def _content_length(headers: object, maximum: int) -> int:
if raw != str(length) or not 0 <= length <= maximum:
raise PolicyError("SCM request length is invalid")
return length
def _credential_forms(token: str) -> tuple[bytes, ...]:
basic = base64.b64encode(f"{GIT_USER}:{token}".encode())
return token.encode("utf-8"), basic, b"Basic " + basic
def _validate_receive_pack(body: bytes | BinaryIO, token: str) -> None:
"""Permit only vetted new-feature-branch pushes with scanned contents."""
validate_receive_pack(body, token, _credential_forms(token))
def _task_ledger() -> TaskLedger:
"""Open the PVC ledger lazily so read-only broker paths need no state."""
return TaskLedger()
def _branch_head(repo: str, ref: str, token: str) -> str | None:
"""Read one exact remote branch head through the existing API boundary."""
try:
raw = read(f"/api/v1/repos/titan/{repo}/branches/{ref}", token=token)
value = json.loads(raw)
commit = value.get("commit") if isinstance(value, dict) else None
return _validate_sha(commit.get("id") if isinstance(commit, dict) else None)
except urllib.error.HTTPError as error:
if error.code == 404:
return None
raise
def _guarded_opener(guard: deadline_http.StreamDeadline):
"""Build an opener whose connection obeys one absolute stream deadline."""
return urllib.request.build_opener(RejectRedirect(), *guard.handlers()).open
def _upstream_git_request(
target: str,
*,
@ -363,7 +360,7 @@ class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler):
try:
self._validate_headers()
self.connection.settimeout(INBOUND_BODY_TIMEOUT)
if self.path in {"/v1/metadata", "/v1/drafts"}:
if self.path in {"/v1/metadata", "/v1/drafts", "/v1/tasks/register", "/v1/tasks/draft-update"}:
self._control()
else:
self._git_rpc()
@ -384,13 +381,29 @@ class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler):
if set(data) != {"path"} or not isinstance(data["path"], str):
raise PolicyError("metadata request fields are invalid")
result = read(data["path"], token=token)
else:
elif self.path == "/v1/drafts":
expected = {"base", "body", "head", "head_sha", "repo", "title"}
if set(data) != expected or not all(
isinstance(data[key], str) for key in expected
):
raise PolicyError("draft request fields are invalid")
result = create_draft(token=token, **data) # type: ignore[arg-type]
elif self.path == "/v1/tasks/register":
if set(data) != {"grant"} or not isinstance(data["grant"], str):
raise PolicyError("task registration fields are invalid")
claims = verify_grant(data["grant"])
remote = _branch_head(claims["repo"], claims["ref"], token)
_task_ledger().register(claims, remote_head=remote)
result = b'{"registered":true}'
else:
grant, number, title, body = request_fields(data, token)
claims = verify_grant(grant)
_task_ledger().authorize_current(claims)
if _branch_head(claims["repo"], claims["ref"], token) != claims["new_head"]:
raise PolicyError("task branch head changed; fetch and merge before retrying")
pull = json.loads(read(f"/api/v1/repos/titan/{claims['repo']}/pulls/{number}", token=token))
matches_pull(pull, claims, number)
result = update_draft(token, claims["repo"], number, title, body)
if token.encode("utf-8") in result:
raise PolicyError("SCM upstream reflected credential material")
self._json(200, result)
@ -420,20 +433,50 @@ class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler):
set_timeout=self.connection.settimeout,
)
try:
claims = None
if service == "git-receive-pack":
_validate_receive_pack(body, token)
raw_grant = self.headers.get("X-Hermes-Task-Grant", "")
if raw_grant:
claims = verify_grant(raw_grant)
if claims["repo"] != repo:
raise PolicyError("task grant repository does not match Git target")
_task_ledger().authorize_update(claims)
validate_receive_pack(
body, token, _credential_forms(token),
expected=(claims["expected_old"], claims["new_head"], claims["ref"]),
)
else:
# Compatibility is creation-only. Existing refs cannot be
# moved without an owned, signed task grant.
_validate_receive_pack(body, token)
expected = f"application/x-{service}-result"
streamed = _upstream_git_request(
f"/titan/{repo}.git/{service}",
method="POST",
body=body,
body_length=body_length,
content_type=expected_request,
expected_type=expected,
token=token,
stream_result=True,
)
try:
streamed = _upstream_git_request(
f"/titan/{repo}.git/{service}",
method="POST",
body=body,
body_length=body_length,
content_type=expected_request,
expected_type=expected,
token=token,
stream_result=True,
)
except (OSError, socket.timeout, urllib.error.URLError):
# The upstream may have accepted the pack but lost its response.
# Only the exact granted new head proves that a replay is safe.
if claims is None or _branch_head(repo, claims["ref"], token) != claims["new_head"]:
raise
_task_ledger().commit(claims)
raise PolicyError("Git upstream acknowledgement was lost; retry to reconcile")
result, result_length = streamed
if claims is not None:
# Smart HTTP uses HTTP 200 for both Git success and a rejected
# ref command. The authenticated branch read is the commit
# point; never advance the local ledger on packet status alone.
if _branch_head(repo, claims["ref"], token) != claims["new_head"]:
result.close()
raise PolicyError("Git upstream did not advance the granted branch")
_task_ledger().commit(claims)
try:
self._stream(200, expected, result, result_length)
finally:
@ -447,6 +490,7 @@ def main() -> int:
parser.add_argument("--listen", default="0.0.0.0")
parser.add_argument("--port", type=int, default=BROKER_PORT)
args = parser.parse_args()
seed_adoptions(_task_ledger(), read_token(), _branch_head)
BoundedThreadingHTTPServer((args.listen, args.port), BrokerHandler).serve_forever()
return 0

14
services/hermes/scm-common/scripts/scm_broker_client.py
View File

@ -32,7 +32,7 @@ def request(
opener: Callable[..., object] = _open,
) -> bytes:
"""Send one bounded broker operation without any repository credential."""
if endpoint not in {"/v1/metadata", "/v1/drafts"}:
if endpoint not in {"/v1/metadata", "/v1/drafts", "/v1/tasks/register", "/v1/tasks/draft-update"}:
raise PolicyError("SCM broker operation is outside the client allowlist")
body = json.dumps(payload, separators=(",", ":")).encode("utf-8")
if len(body) > 64 * 1024:
@ -91,3 +91,15 @@ def create_draft(
},
opener=opener,
)
def register_task(grant: str, *, opener: Callable[..., object] = _open) -> bytes:
"""Register the broker-owned branch before its first task push."""
if not isinstance(grant, str) or not grant:
raise PolicyError("SCM task grant is invalid")
return request("/v1/tasks/register", {"grant": grant}, opener=opener)
def update_draft(grant: str, pr_number: int, title: str, body: str, *, opener: Callable[..., object] = _open) -> bytes:
"""Refresh prose on the one broker-verified continuing task PR."""
return request("/v1/tasks/draft-update", {"grant": grant, "pr_number": pr_number, "title": title, "body": body}, opener=opener)

30
services/hermes/scm-common/scripts/scm_task_adoptions.py Normal file
View File

@ -0,0 +1,30 @@
"""Read-only operator migration records for pre-ledger task branches."""
from __future__ import annotations
import json
from typing import Any, Callable
from gitea_api_policy import PolicyError
from scm_task_grants import ADOPTIONS_PATH, TaskLedger
def seed(ledger: TaskLedger, token: str, branch_head: Callable[[str, str, str], str | None]) -> None:
"""Seed each independently reviewed record without blocking unrelated SCM."""
try:
records = json.loads(ADOPTIONS_PATH.read_bytes())
except (OSError, TypeError, ValueError):
print("SCM task adoption registry is unavailable; deferring migration", flush=True)
return
if not isinstance(records, dict) or len(records) > 64:
print("SCM task adoption registry is invalid; deferring migration", flush=True)
return
for key, record in records.items():
try:
if not isinstance(key, str) or not isinstance(record, dict):
raise PolicyError("task branch adoption registry is invalid")
repo, ref = record.get("repo"), record.get("ref")
if key != f"{repo}/{ref}":
raise PolicyError("task branch adoption registry is invalid")
ledger.seed_adoption(record, branch_head(repo, ref, token))
except (OSError, PolicyError, TypeError, ValueError):
print(f"SCM task adoption deferred for {key!r}", flush=True)

48
services/hermes/scm-common/scripts/scm_task_drafts.py Normal file
View File

@ -0,0 +1,48 @@
"""Broker-only refresh of an owned draft PR's title and body."""
from __future__ import annotations
import base64
import json
import urllib.request
from typing import Any
import deadline_http
from gitea_api import CANONICAL_BASE_URL
from gitea_api_policy import PolicyError, _draft_title, _validate_body, _validate_pr_number
def request_fields(value: dict[str, Any], token: str) -> tuple[str, int, str, str]:
if set(value) != {"grant", "pr_number", "title", "body"} or not isinstance(value["grant"], str):
raise PolicyError("draft update fields are invalid")
return value["grant"], _validate_pr_number(value["pr_number"]), _draft_title(value["title"], forbidden=(token,)), _validate_body(value["body"], forbidden=(token,))
def matches_pull(value: Any, claims: dict[str, Any], number: int) -> None:
if not isinstance(value, dict) or _validate_pr_number(value.get("number")) != number:
raise PolicyError("draft pull request is invalid")
head, base = value.get("head"), value.get("base")
if not isinstance(head, dict) or not isinstance(base, dict):
raise PolicyError("draft pull request is invalid")
repo, base_repo = head.get("repo"), base.get("repo")
name = repo.get("full_name") if isinstance(repo, dict) else ""
base_name = base_repo.get("full_name") if isinstance(base_repo, dict) else ""
if value.get("state") != "open" or head.get("ref") != claims["ref"] or head.get("sha") != claims["new_head"] or base.get("ref") != claims["base"] or name != f"titan/{claims['repo']}" or base_name != f"titan/{claims['repo']}":
raise PolicyError("draft pull request does not match its task grant")
def update(token: str, repo: str, number: int, title: str, body: str) -> bytes:
"""PATCH only safe draft prose; caller already authenticated ownership."""
payload = json.dumps({"title": title, "body": body}, separators=(",", ":")).encode()
auth = base64.b64encode(f"hermes-automation:{token}".encode()).decode()
request = urllib.request.Request(
f"{CANONICAL_BASE_URL}/api/v1/repos/titan/{repo}/pulls/{number}", payload,
method="PATCH", headers={"Authorization": f"Basic {auth}", "Content-Type": "application/json", "Accept": "application/json"},
)
with deadline_http.open_bounded(request, maximum=2 * 1024 * 1024, timeout=30) as response:
if getattr(response, "status", None) != 200 or response.headers.get_content_type() != "application/json":
raise PolicyError("draft update upstream response is invalid")
result = response.read(2 * 1024 * 1024 + 1)
if len(result) > 2 * 1024 * 1024 or token.encode() in result:
raise PolicyError("draft update upstream response is invalid")
json.loads(result)
return result

211
services/hermes/scm-common/scripts/scm_task_grants.py Normal file
View File

@ -0,0 +1,211 @@
"""Scoped task-branch grants and durable compare-and-swap ownership ledger."""
from __future__ import annotations
import base64
import hashlib
import hmac
import json
import os
import re
import sqlite3
import time
from pathlib import Path
from typing import Any
from gitea_api_policy import PolicyError, _validate_ref, _validate_repo, _validate_sha
MAX_GRANT_BYTES = 4096
MAX_GRANT_SECONDS = 300
MAX_ANCESTRY_STEPS = 1024
KEY_FILE = Path(os.environ.get("HERMES_SCM_TASK_GRANT_KEY_FILE", "/vault/secrets/scm-task-grant"))
LEDGER_PATH = Path(os.environ.get("HERMES_SCM_TASK_LEDGER", "/scm-state/task-branches.db"))
ADOPTIONS_PATH = Path(os.environ.get("HERMES_SCM_TASK_ADOPTIONS", "/scm-adoptions/task-branch-adoptions.json"))
REQUIRED = frozenset({"repo", "ref", "base", "board", "root_task_id", "assignment_task_id", "run", "ordinal", "expires", "expected_old", "new_head", "continuation_kind"})
IDENTIFIER = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}\Z")
ZERO_SHA = "0" * 40
def _key() -> bytes:
try:
value = KEY_FILE.read_text(encoding="ascii").strip()
except OSError as exc:
raise PolicyError("SCM task grant key is unavailable") from exc
if not re.fullmatch(r"[0-9a-f]{64}", value):
raise PolicyError("SCM task grant key is invalid")
return value.encode("ascii")
def _b64(value: bytes) -> str:
return base64.urlsafe_b64encode(value).rstrip(b"=").decode("ascii")
def _unb64(value: str) -> bytes:
if not isinstance(value, str) or len(value) > MAX_GRANT_BYTES or not value.isascii():
raise PolicyError("SCM task grant is invalid")
try:
return base64.urlsafe_b64decode(value + "=" * (-len(value) % 4))
except ValueError as exc:
raise PolicyError("SCM task grant is invalid") from exc
def sign_grant(claims: dict[str, Any], key: bytes | None = None) -> str:
"""Create a compact signed grant; mediator callers hold the key privately."""
if set(claims) != REQUIRED:
raise PolicyError("SCM task grant claims are invalid")
payload = json.dumps(claims, sort_keys=True, separators=(",", ":")).encode("ascii")
signature = hmac.new(key or _key(), payload, hashlib.sha256).digest()
return _b64(payload) + "." + _b64(signature)
def verify_grant(token: str, *, now: int | None = None, key: bytes | None = None) -> dict[str, Any]:
"""Verify one short-lived grant and normalize only safe repository fields."""
parts = token.split(".") if isinstance(token, str) else []
if len(parts) != 2:
raise PolicyError("SCM task grant is invalid")
payload, signature = _unb64(parts[0]), _unb64(parts[1])
if not hmac.compare_digest(hmac.new(key or _key(), payload, hashlib.sha256).digest(), signature):
raise PolicyError("SCM task grant signature is invalid")
try:
claims = json.loads(payload)
except (TypeError, ValueError) as exc:
raise PolicyError("SCM task grant is invalid") from exc
if not isinstance(claims, dict) or set(claims) != REQUIRED:
raise PolicyError("SCM task grant claims are invalid")
claims["repo"] = _validate_repo(claims["repo"])
claims["ref"] = _validate_ref(claims["ref"], "task branch")
claims["base"] = _validate_ref(claims["base"], "base branch")
claims["expected_old"] = _validate_sha(claims["expected_old"], "expected old SHA")
claims["new_head"] = _validate_sha(claims["new_head"], "new head SHA")
if not isinstance(claims["ordinal"], int) or isinstance(claims["ordinal"], bool):
raise PolicyError("SCM task grant ordinal is invalid")
if claims["continuation_kind"] not in {"", "repair", "review"}:
raise PolicyError("SCM task grant continuation is invalid")
expiry = claims["expires"]
current = int(time.time()) if now is None else now
if not isinstance(expiry, int) or not current < expiry <= current + MAX_GRANT_SECONDS:
raise PolicyError("SCM task grant is expired")
if not all(isinstance(claims[name], str) and IDENTIFIER.fullmatch(claims[name]) for name in ("board", "root_task_id", "assignment_task_id", "run")):
raise PolicyError("SCM task grant binding is invalid")
return claims
class TaskLedger:
"""Broker-owned branch ownership state; no silent claim of existing refs."""
def __init__(self, path: Path = LEDGER_PATH) -> None:
path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
self.path = path
with self._connect() as con:
con.execute("create table if not exists task_branches (repo text not null, ref text not null, board text not null, root_task_id text not null, latest_head text not null, pr_number integer, primary key(repo, ref))")
con.execute("create table if not exists task_roots (repo text not null, board text not null, root_task_id text not null, ref text not null, primary key(repo,board,root_task_id))")
def _connect(self):
return sqlite3.connect(self.path, timeout=5, isolation_level="IMMEDIATE")
@staticmethod
def _owner(claims: dict[str, Any]) -> tuple[str, str]:
return claims["board"], claims["root_task_id"]
def get(self, repo: str, ref: str) -> tuple[str, str, str] | None:
with self._connect() as con:
row = con.execute("select board,root_task_id,latest_head from task_branches where repo=? and ref=?", (repo, ref)).fetchone()
return tuple(row) if row is not None else None
@staticmethod
def _adoption(claims: dict[str, Any], remote_head: str | None) -> bool:
"""Accept only an operator-reviewed exact migration record.
The ConfigMap is mounted read-only and starts empty. A task grant can
consume a matching record but cannot create, edit, or broaden one.
"""
try:
raw = ADOPTIONS_PATH.read_bytes()
value = json.loads(raw)
except (OSError, ValueError, TypeError):
return False
if not isinstance(value, dict) or len(raw) > 64 * 1024:
return False
record = value.get(f"{claims['repo']}/{claims['ref']}")
if not isinstance(record, dict):
return False
return (
record.get("board") == claims["board"]
and record.get("root_task_id") == claims["root_task_id"]
and record.get("latest_head") == remote_head == claims["expected_old"]
)
def seed_adoption(self, record: dict[str, Any], remote_head: str | None) -> None:
"""Import one reviewed legacy branch only when Forgejo still matches it."""
required = {"repo", "ref", "board", "root_task_id", "latest_head", "pr_number"}
if set(record) != required or remote_head is None:
raise PolicyError("task branch adoption record is invalid")
repo = _validate_repo(record["repo"])
ref = _validate_ref(record["ref"], "task branch")
head = _validate_sha(record["latest_head"], "adoption head")
if not all(isinstance(record[name], str) and IDENTIFIER.fullmatch(record[name]) for name in ("board", "root_task_id")):
raise PolicyError("task branch adoption record is invalid")
if not isinstance(record["pr_number"], int) or isinstance(record["pr_number"], bool) or record["pr_number"] < 1:
raise PolicyError("task branch adoption pull request is invalid")
with self._connect() as con:
existing = con.execute("select board,root_task_id,latest_head from task_branches where repo=? and ref=?", (repo, ref)).fetchone()
root = con.execute("select ref from task_roots where repo=? and board=? and root_task_id=?", (repo, record["board"], record["root_task_id"])).fetchone()
if existing is not None:
if tuple(existing[:2]) != (record["board"], record["root_task_id"]):
raise PolicyError("task branch adoption conflicts with ledger")
# A broker-confirmed later revision is legitimate. The static
# import record only proves the first seed, never rewrites it.
return
if head != remote_head:
raise PolicyError("task branch adoption does not match its live head")
if root is not None and root[0] != ref:
raise PolicyError("task branch adoption forks a logical task")
con.execute("insert into task_branches(repo,ref,board,root_task_id,latest_head,pr_number) values(?,?,?,?,?,?)", (repo, ref, record["board"], record["root_task_id"], head, record["pr_number"]))
con.execute("insert into task_roots(repo,board,root_task_id,ref) values(?,?,?,?)", (repo, record["board"], record["root_task_id"], ref))
def register(self, claims: dict[str, Any], *, remote_head: str | None) -> None:
"""Create only absent refs; adoption is an explicit operator action."""
with self._connect() as con:
row = con.execute("select board,root_task_id,latest_head from task_branches where repo=? and ref=?", (claims["repo"], claims["ref"])).fetchone()
owner = self._owner(claims)
root = con.execute("select ref from task_roots where repo=? and board=? and root_task_id=?", (claims["repo"], *owner)).fetchone()
if root is not None and root[0] != claims["ref"]:
raise PolicyError("logical task already owns a different branch")
if row is not None and tuple(row[:2]) != owner:
raise PolicyError("task branch is owned by another task")
if row is None:
if remote_head is not None:
if not self._adoption(claims, remote_head):
raise PolicyError("existing unregistered task branch requires operator adoption")
con.execute("insert into task_branches(repo,ref,board,root_task_id,latest_head) values(?,?,?,?,?)", (claims["repo"], claims["ref"], owner[0], owner[1], remote_head))
con.execute("insert into task_roots(repo,board,root_task_id,ref) values(?,?,?,?)", (claims["repo"], *owner, claims["ref"]))
return
if claims["expected_old"] != ZERO_SHA:
raise PolicyError("new task branch must use the zero expected head")
con.execute(
"insert into task_branches(repo,ref,board,root_task_id,latest_head) values(?,?,?,?,?)",
(claims["repo"], claims["ref"], owner[0], owner[1], ZERO_SHA),
)
con.execute("insert into task_roots(repo,board,root_task_id,ref) values(?,?,?,?)", (claims["repo"], *owner, claims["ref"]))
elif remote_head != row[2]:
raise PolicyError("task branch head changed; fetch and merge before retrying")
def authorize_update(self, claims: dict[str, Any]) -> None:
row = self.get(claims["repo"], claims["ref"])
if row is None:
raise PolicyError("task branch is not registered")
if row[:2] != self._owner(claims):
raise PolicyError("task branch is owned by another task")
if row[2] != claims["expected_old"]:
raise PolicyError("task branch head changed; fetch and merge before retrying")
def authorize_current(self, claims: dict[str, Any]) -> None:
"""Authorize post-push PR prose only at the broker-confirmed new head."""
row = self.get(claims["repo"], claims["ref"])
if row is None or row[:2] != self._owner(claims) or row[2] != claims["new_head"]:
raise PolicyError("task branch head changed; fetch and merge before retrying")
def commit(self, claims: dict[str, Any]) -> None:
self.authorize_update(claims)
with self._connect() as con:
changed = con.execute("update task_branches set latest_head=? where repo=? and ref=? and latest_head=?", (claims["new_head"], claims["repo"], claims["ref"], claims["expected_old"])).rowcount
if changed != 1:
raise PolicyError("task branch changed during broker update")

4
services/hermes/scripts/cli_lane_config.py
View File

@ -27,6 +27,7 @@ CLAUDE_BIN = Path(os.environ.get("HERMES_CLAUDE_BIN", DATA_ROOT / "tools/bin/cla
CLAUDE_SETTINGS = DATA_ROOT / "home/.claude/settings.json"
RESULT_SCHEMA_PATH = STATE_ROOT / "worker-result.schema.json"
EFFORTS = ("low", "medium", "high", "xhigh")
CAPABILITIES = ("economy", "balanced", "advanced", "frontier")
EXTERNAL_PREFIX = "cli-"
DEFAULT_CLAIM_TTL = 7 * 24 * 60 * 60
DEFAULT_MAX_RUNTIME = 12 * 60 * 60
@ -117,6 +118,9 @@ class Route:
reason: str
latency_ms: int
fallback_chain: tuple[str, ...]
# Capability is intentionally independent from reasoning effort. Keep the
# default for older durable records and positional test fixtures.
capability: str = "advanced"
@dataclass

28
services/hermes/scripts/cli_lane_execution.py
View File

@ -12,7 +12,10 @@ from pathlib import Path
import cli_lane_goal
from cli_lane_board import _board_call, _resolve_workspace, _task_context, _task_value
from cli_lane_config import (
CAPABILITIES,
DEFAULT_MAX_RUNTIME,
EFFORTS,
Route,
TerminalFinalizationPending,
)
from cli_lane_failover import _routed_or_blocked, capacity_failover
@ -31,9 +34,9 @@ from cli_lane_provider import run_provider
from cli_lane_quota import selection_constraint
from cli_lane_records import _persist_candidate, _write_terminal_record
from cli_lane_recovery import _has_pending_finalization
from cli_lane_route_quality import quality_retry_assignee as _quality_retry_assignee
from cli_lane_routing import fresh_unavailable_provider, select_route
def execute_claim(board: str, task_id: str) -> None:
"""Execute one already-claimed task and commit its outcome to Kanban."""
from hermes_cli import kanban_db
@ -145,7 +148,8 @@ def execute_claim(board: str, task_id: str) -> None:
f"{constraint.exclude_provider} before automatic routing.",
)
comment(
f"CLI route: {route.provider}/{route.model} at {route.effort}; classifier={route.classifier}; {route.reason}",
f"CLI route: {route.provider}/{route.model} capability={route.capability} "
f"effort={route.effort}; classifier={route.classifier}; {route.reason}",
)
resume_handoff = ""
if (
@ -233,6 +237,7 @@ def execute_claim(board: str, task_id: str) -> None:
"executor": "direct-cli-lane",
"provider": route.provider,
"model": route.model,
"capability": route.capability,
"effort": route.effort,
"classifier": route.classifier,
"state_file": str(state_file),
@ -396,9 +401,28 @@ def execute_claim(board: str, task_id: str) -> None:
)
if next_route is None:
break
if assignee == "cli-auto":
override = _quality_retry_assignee(route, next_route)
selected_lane = (
f"cli-{next_route.provider}-{next_route.capability}-"
f"{next_route.effort}"
)
if override != selected_lane:
next_route = _routed_or_blocked(
kanban_db,
board,
task_id,
run_id,
lambda boundary=escalation_context, lane=override: select_route(
boundary, lane
),
)
if next_route is None:
break
comment(
f"Goal route {goal_turn}/{goal_max_turns}: "
f"{next_route.provider}/{next_route.model} at {next_route.effort}; "
f"capability={next_route.capability}; "
f"classifier={next_route.classifier}; {next_route.reason}",
)
handoff = (

31
services/hermes/scripts/cli_lane_failover.py
View File

@ -9,7 +9,7 @@ from pathlib import Path
from typing import Any, Callable
from cli_lane_board import _board_call
from cli_lane_config import EFFORTS, ProcessResult, Route
from cli_lane_config import ProcessResult, Route
from cli_lane_health import classify_capacity_failure, record_provider_failure
from cli_lane_metrics import (
record_provider_fallback,
@ -73,6 +73,11 @@ def _routed_or_blocked(
return None
def _manual_floor_assignee(provider: str, route: Route) -> str:
"""Pin the exact recovery role and effort without legacy inference."""
return f"cli-{provider}-{route.capability}-{route.effort}"
def capacity_failover(
kanban_db: Any,
*,
@ -115,7 +120,8 @@ def capacity_failover(
retry_context = (
context
+ "\n\nRouting boundary: the first provider failed from capacity/authentication. "
+ "Select the alternate hosted provider at an appropriate effort."
+ f"Preserve capability={route.capability} and effort={route.effort}; "
+ "select the alternate hosted provider without treating infrastructure failure as a quality miss."
)
fallback = _routed_or_blocked(
kanban_db,
@ -131,8 +137,10 @@ def capacity_failover(
)
if fallback is None:
return FailoverOutcome(route, result, candidate_file, route.provider, True)
if EFFORTS.index(fallback.effort) < EFFORTS.index(route.effort):
# Never let a capacity-triggered reclassification downgrade effort:
if fallback.effort != route.effort or fallback.capability != route.capability:
# Infrastructure failures preserve both floors exactly. A higher paid
# role or effort is a quality escalation and must not be inferred from
# a timeout, quota, authentication, or provider-capacity incident.
# re-pin the classifier's chosen (healthy) provider at the original
# floor, whether the downgrade came from the classifier itself or
# from its own excluded-provider health guard.
@ -142,22 +150,25 @@ def capacity_failover(
task_id,
run_id,
lambda: select_route(
retry_context, f"cli-{fallback.provider}-{route.effort}"
retry_context,
_manual_floor_assignee(fallback.provider, route),
),
)
if preserved is None:
return FailoverOutcome(route, result, candidate_file, route.provider, True)
comment(
f"Effort preserved: Switchyard classification chose {fallback.effort} "
f"for {fallback.provider}; escalated to the original {route.effort} "
"floor so capacity failover never downgrades a safety task.",
f"Route floors preserved: Switchyard classification chose "
f"{fallback.capability}/{fallback.effort} for {fallback.provider}; "
f"restored the original {route.capability}/{route.effort} floor so "
"infrastructure failover "
"never changes capability or effort.",
)
fallback = preserved
record_provider_fallback(route.provider, fallback.provider, failure_class)
comment(
f"Provider fallback: {route.provider} -> {fallback.provider} "
f"after a {failure_class} failure; Jetson reclassified the retry boundary "
f"(classifier={fallback.classifier}).",
f"after a {failure_class} failure; preserved "
f"{fallback.capability}/{fallback.effort} (classifier={fallback.classifier}).",
)
fallback_result = run_provider(
fallback,

2
services/hermes/scripts/cli_lane_prompt.py
View File

@ -26,6 +26,8 @@ Workspace: {workspace}
Operate autonomously inside the workspace. Inspect before editing, preserve unrelated user changes, run proportionate tests, and do not claim completion without evidence. You have owner-level Kubernetes access in every namespace. Prefer Flux-tracked manifests for durable changes, but use kubectl, Flux, exec, port-forwarding, rollout operations, and existing Vault workflows when the objective or incident requires them. Persist any desired-state mutation back to Git. Do not force-push, hard-reset, clean untracked files, or expose credentials.
When this task continues a supervisor review chain, use its coordinator-issued project, branch, and pull request. A repair stays on that existing branch and PR; do not create a replacement. Review the current PR head, actual diff, and relevant test evidence before a verdict. A base-behind count alone is not a defect or retirement signal. Complete assigned delegated work before giving a final verdict, and report the precise reviewed commit. After a repair or CI failure, refresh the existing PR title and body through the signed mediator path so the current head has current handoff evidence.
Return a final JSON object matching the supplied schema. Use status=incomplete when required work, tests, commands, commits, pushes, or verification are still running or remain to be done. Use status=blocked only when an obstacle prevents completion of the assigned task itself. Never use status=completed for a progress report. For review or diagnostic tasks, put defects and risks in findings; those findings can make the reviewed change unfit to ship without blocking completion of the review. The blockers array must be empty whenever status is completed. List changed files, tests run, durable artifact paths, findings, and task blockers explicitly.
"""

1
services/hermes/scripts/cli_lane_records.py
View File

@ -260,6 +260,7 @@ def _persist_candidate(
"goal_turn": goal_turn,
"provider": route.provider,
"model": route.model,
"capability": route.capability,
"effort": route.effort,
"returncode": returncode,
"structured": structured,

30
services/hermes/scripts/cli_lane_route_quality.py Normal file
View File

@ -0,0 +1,30 @@
#!/usr/bin/env python3
"""Capability and effort floors for quality-driven CLI retries."""
from __future__ import annotations
from cli_lane_config import CAPABILITIES, EFFORTS, Route
def quality_retry_assignee(previous: Route, selected: Route) -> str:
"""Raise effort before capability when quality feedback repeats a plan."""
capability = max(
CAPABILITIES.index(previous.capability), CAPABILITIES.index(selected.capability)
)
effort = max(EFFORTS.index(previous.effort), EFFORTS.index(selected.effort))
provider = selected.provider
if (
selected.provider == previous.provider
and selected.model == previous.model
and capability == CAPABILITIES.index(previous.capability)
and effort == EFFORTS.index(previous.effort)
):
if effort < len(EFFORTS) - 1:
effort += 1
elif capability < len(CAPABILITIES) - 1:
capability += 1
effort = EFFORTS.index(previous.effort)
else:
provider = "claude" if provider == "codex" else "codex"
role = CAPABILITIES[capability]
return f"cli-{provider}-{role}-{EFFORTS[effort]}"

118
services/hermes/scripts/cli_lane_routing.py
View File

@ -21,17 +21,40 @@ from cli_lane_config import (
from cli_lane_files import load_json
from routing_catalog import catalog_contains
_FALLBACK_RATIONALE = re.compile(
r"(?P<source>worker/(?:codex|claude)/auto(?:-(?:economy|balanced|advanced|frontier))?/(?:low|medium|high|xhigh)) "
r"(?:was unavailable|exceeded its context window); fell back to "
r"(?P<replacement>worker/(?:codex|claude)/auto(?:-(?:economy|balanced|advanced|frontier))?/(?:low|medium|high|xhigh))"
)
def parse_assignee(assignee: str) -> tuple[str | None, str | None]:
"""Parse external lane overrides while leaving cli-auto fully automatic."""
value = str(assignee or "").strip().lower()
if value == "cli-auto":
return None, None
match = re.fullmatch(r"cli-(codex|claude)-(low|medium|high|xhigh)", value)
match = re.fullmatch(
r"cli-(codex|claude)-(?:(?:economy|balanced|advanced|frontier)-)?(low|medium|high|xhigh)",
value,
)
if not match:
raise ValueError(f"unsupported external lane assignee: {assignee}")
return match.group(1), match.group(2)
def parse_assignee_capability(assignee: str) -> tuple[str | None, str | None, str | None]:
"""Return optional provider, effort, and capability lane constraints."""
value = str(assignee or "").strip().lower()
if value == "cli-auto":
return None, None, None
match = re.fullmatch(
r"cli-(codex|claude)-(?:(economy|balanced|advanced|frontier)-)?(low|medium|high|xhigh)",
value,
)
if not match:
raise ValueError(f"unsupported external lane assignee: {assignee}")
return match.group(1), match.group(3), match.group(2)
def _health_number(value: Any) -> float | None:
"""Return a plain numeric health field, rejecting booleans and strings."""
if isinstance(value, bool) or not isinstance(value, (int, float)):
@ -100,6 +123,27 @@ def _decode_worker_target(value: str) -> tuple[str, str, str]:
raise RuntimeError(f"unsupported Switchyard worker target: {value}")
return provider, model, effort
def _target_capability(model: str, effort: str) -> str:
"""Read the stable capability selector from a worker target header."""
if model == "auto":
# Legacy AUTO encoded only the effort tier. Preserve its established
# mapping instead of treating a low/medium task as Sol-class work.
return {"low": "economy", "medium": "balanced"}.get(effort, "advanced")
match = re.fullmatch(r"auto-(economy|balanced|advanced|frontier)", model)
if match:
return match.group(1)
return "advanced"
def _fallback_floor(rationale: str) -> tuple[str, str, str] | None:
"""Return the originally selected worker floor from a Switchyard fallback."""
match = _FALLBACK_RATIONALE.search(rationale)
if not match:
return None
provider, model, effort = _decode_worker_target(match.group("source"))
return provider, _target_capability(model, effort), effort
def select_route(
prompt: str,
assignee: str,
@ -112,9 +156,11 @@ def select_route(
) -> Route:
"""Ask Switchyard to select one native CLI worker at this boundary."""
started = time.monotonic()
manual_provider, manual_effort = parse_assignee(assignee)
manual_provider, manual_effort, manual_capability = parse_assignee_capability(assignee)
if manual_provider and manual_effort:
route_id = f"atlas/worker/manual/{manual_provider}/{manual_effort}"
route_id = f"atlas/worker/manual/{manual_provider}/"
route_id += f"{manual_capability}/" if manual_capability else ""
route_id += manual_effort
source = "switchyard-manual"
else:
route_id = "atlas/worker/auto/maximum"
@ -148,6 +194,8 @@ def select_route(
except (OSError, urllib.error.URLError) as exc:
raise RuntimeError(f"Switchyard worker routing failed: {exc}") from exc
provider, model, effort = _decode_worker_target(selected)
capability = _target_capability(model, effort)
fallback_floor = _fallback_floor(rationale)
# Switchyard preserves the stable tier target in the selection header and
# top-level response model. The worker broker's assistant content contains
# the steward-resolved provider model required by the native CLI.
@ -169,17 +217,68 @@ def select_route(
model = resolved_model
except (AttributeError, IndexError, KeyError, RuntimeError, TypeError, ValueError, json.JSONDecodeError):
resolved_valid = False
if model == "auto" and not resolved_valid:
if model.startswith("auto") and not resolved_valid:
raise RuntimeError("Switchyard did not return a current provider model")
if exclude_provider and provider == exclude_provider:
alternate = "claude" if provider == "codex" else "codex"
if (
assignee == "cli-auto"
and fallback_floor
and (capability, effort) != fallback_floor[1:]
):
# FallThrough retries the route's entire target set after an upstream
# error. Its static list cannot retain the classifier's dynamic floor,
# so perform one exact alternate-provider selection before a lower role
# can become a CLI receipt.
failed_provider, required_capability, required_effort = fallback_floor
alternate = "claude" if failed_provider == "codex" else "codex"
guarded = select_route(
prompt,
f"cli-{alternate}-{effort}",
f"cli-{alternate}-{required_capability}-{required_effort}",
switchyard_url=switchyard_url,
open_request=open_request,
catalog_model_allowed=catalog_model_allowed,
)
if (
guarded.provider == failed_provider
or guarded.capability != required_capability
or guarded.effort != required_effort
):
raise RuntimeError(
"Switchyard request fallback did not preserve the requested "
f"{required_capability}/{required_effort} route floor"
)
return Route(
provider=guarded.provider,
model=guarded.model,
effort=guarded.effort,
profile=guarded.profile,
classifier=f"{source}-fallback-floor-guard",
reason=(
f"Switchyard request fallback changed {required_capability}/"
f"{required_effort}; retried the alternate provider at the "
f"original floor. {guarded.reason}"
),
latency_ms=int((time.monotonic() - started) * 1000),
fallback_chain=(),
capability=guarded.capability,
)
if exclude_provider and provider == exclude_provider:
alternate = "claude" if provider == "codex" else "codex"
guarded = select_route(
prompt,
f"cli-{alternate}-{capability}-{effort}",
switchyard_url=switchyard_url,
open_request=open_request,
catalog_model_allowed=catalog_model_allowed,
)
if (
guarded.provider == exclude_provider
or guarded.capability != capability
or guarded.effort != effort
):
raise RuntimeError(
"Switchyard health guard did not preserve the requested "
f"{capability}/{effort} route floor"
)
return Route(
provider=guarded.provider,
model=guarded.model,
@ -188,10 +287,12 @@ def select_route(
classifier=f"{source}-health-guard",
reason=(
f"Switchyard selected excluded {exclude_provider}; preserved "
f"its {effort} effort on healthy-provider route. {guarded.reason}"
f"its {capability}/{effort} floor on a healthy-provider route. "
f"{guarded.reason}"
),
latency_ms=int((time.monotonic() - started) * 1000),
fallback_chain=(),
capability=guarded.capability,
)
return Route(
provider=provider,
@ -202,4 +303,5 @@ def select_route(
reason=rationale or f"Switchyard selected {selected}",
latency_ms=int((time.monotonic() - started) * 1000),
fallback_chain=(),
capability=capability,
)

12
services/hermes/scripts/execution_pool_client.py
View File

@ -140,7 +140,7 @@ class ClientBoundary:
def _submit(
self, assignment: dict[str, Any], request: dict[str, Any],
structured: dict[str, Any],
) -> None:
) -> dict[str, str] | None:
"""Publish the run's work, downgrading the result rather than losing it.
A refused push or draft used to unwind the whole worker, so the run's
@ -159,12 +159,14 @@ class ClientBoundary:
)
if reason not in structured["blockers"]:
structured["blockers"].append(reason)
return
return None
pull = str(submission.get("pull_request") or "")
branch = str(submission.get("branch") or "")
for artifact in (pull, f"branch:{branch}" if branch else ""):
if artifact and artifact not in structured["artifacts"]:
structured["artifacts"].append(artifact)
head = str(submission.get("head") or "")
return {"branch": branch, "pull_request": pull, "head": head}
def finish(self, request: dict[str, Any]) -> dict[str, Any]:
payload = _validate_result(request.get("payload"))
@ -172,7 +174,11 @@ class ClientBoundary:
assignment, binding = self._current_for(request.get("binding"))
structured = payload["structured"]
if structured["status"] == "completed" and int(payload.get("returncode", 1)) == 0:
self._submit(assignment, request, structured)
submission = self._submit(assignment, request, structured)
if submission is not None:
# The mediator, not the model-facing request, records the
# broker-confirmed PR/branch in the signed terminal wire.
payload["scm_submission"] = submission
response = self._post(
"/v1/result", sign_envelope(self.key, "result", binding, payload)
)

103
services/hermes/scripts/execution_pool_coordinator.py
View File

@ -13,6 +13,8 @@ from pathlib import Path
from typing import Any
import cli_lane_goal
import supervisor_lineage
import supervisor_state
from cli_lane_config import canonical_run_id
from execution_pool_project import resolve_assignment
from execution_pool_protocol import (
@ -41,6 +43,60 @@ def _task_value(task: Any, name: str, default: Any = None) -> Any:
return getattr(task, name, default)
def _task_parents(task: Any) -> set[str]:
"""Read native parent IDs without accepting task-body references."""
value = _task_value(task, "parents", _task_value(task, "task_links", ()))
if isinstance(value, dict):
value = value.get("parents", ())
if not isinstance(value, (list, tuple, set)):
return set()
result: set[str] = set()
for item in value:
if isinstance(item, dict):
item = item.get("id") or item.get("parent") or item.get("task_id")
if isinstance(item, (str, int)):
result.add(str(item))
return result
def _submission_lineage(
binding: dict[str, Any], assignment: Any, submission: Any
) -> tuple[supervisor_lineage.Lineage, str] | None:
"""Accept a PR handoff only when it exactly repeats its signed assignment."""
if not isinstance(submission, dict):
return None
supplied = tuple(submission.get(name) for name in ("branch", "pull_request", "head"))
if not any(supplied):
return None
if not isinstance(assignment, dict) or not all(isinstance(value, str) and value for value in supplied):
raise ProtocolError("SCM submission is incomplete")
repo_url = assignment.get("repo_url")
match = re.fullmatch(
r"https://scm\.bstein\.dev/titan/([A-Za-z0-9][A-Za-z0-9_.-]{0,99})\.git",
repo_url,
) if isinstance(repo_url, str) else None
root = assignment.get("root_task_id")
branch, base = assignment.get("branch"), assignment.get("base_branch")
pull, head = supplied[1], supplied[2]
pull_match = re.fullmatch(
r"https://scm\.bstein\.dev/titan/([A-Za-z0-9][A-Za-z0-9_.-]{0,99})/pulls/[1-9][0-9]{0,9}",
pull,
)
if (
not match or not isinstance(root, str) or not root
or submission["branch"] != branch or not isinstance(base, str) or not base
or pull_match is None or pull_match.group(1) != match.group(1)
or not re.fullmatch(r"[0-9a-f]{40,64}", head)
):
raise ProtocolError("SCM submission does not match its signed assignment")
continuation = assignment.get("continuation_kind", "")
if continuation not in {"", "repair", "review"} or (
not continuation and root != binding["task_id"]
):
raise ProtocolError("SCM submission has an invalid continuation root")
return supervisor_lineage.Lineage(root, branch, pull, match.group(1), base), head
def resolve_scm(task: Any, board: str = "titan-iac") -> tuple[str, str, str]:
"""Resolve SCM only through the canonical board/project registry."""
return resolve_assignment(board, task)
@ -56,6 +112,33 @@ def assignment_payload(
if len(encoded) > 32 * 1024:
raise RuntimeError("Kanban worker context exceeds the 32KiB assignment limit")
repo_url, branch, base_branch = resolve_scm(task, board)
# Continuation authority comes only from the coordinator-private state DB.
# Board cards have no metadata column, and their text cannot select a ref.
try:
child = supervisor_state.get_child(board, str(_task_value(task, "id")))
except (OSError, ValueError) as error:
raise RuntimeError("supervisor continuation state is unavailable") from error
lineage = child["lineage"] if child is not None else None
if child is not None:
root = supervisor_state.get_root(board, lineage.root_task_id)
parent = str(child.get("parent_task_id") or "")
if root != lineage or not parent:
raise RuntimeError("supervisor continuation lineage is invalid")
if parent != lineage.root_task_id:
parent_record = supervisor_state.get_child(board, parent)
if parent_record is None or parent_record.get("lineage") != lineage:
raise RuntimeError("supervisor continuation parent is invalid")
repo = re.fullmatch(r"https://scm\.bstein\.dev/titan/([A-Za-z0-9][A-Za-z0-9_.-]{0,99})\.git", repo_url)
if repo is None or lineage.project != repo.group(1):
raise RuntimeError("supervisor continuation project is invalid")
native_parents = _task_parents(task)
if parent not in native_parents or lineage.root_task_id not in native_parents:
raise RuntimeError("supervisor continuation parents are invalid")
if kanban_db.get_task(connection, lineage.root_task_id) is None:
raise RuntimeError("supervisor continuation root is unavailable")
root_task_id = lineage.root_task_id if lineage is not None else str(_task_value(task, "id"))
if lineage is not None:
branch, base_branch = lineage.branch, lineage.base_branch
runtime = int(_task_value(task, "max_runtime_seconds", 0) or 12 * 60 * 60)
runtime = max(60, min(runtime, 12 * 60 * 60))
return {
@ -64,6 +147,8 @@ def assignment_payload(
"repo_url": repo_url,
"branch": branch,
"base_branch": base_branch,
"root_task_id": root_task_id,
"continuation_kind": str(child.get("kind")) if child is not None else "",
"max_runtime_seconds": runtime,
"deadline_unix": int(time.time()) + runtime,
"goal_mode": bool(_task_value(task, "goal_mode", False)),
@ -243,12 +328,30 @@ class Coordinator:
"findings": structured.get("findings", []),
"blockers": structured.get("blockers", []),
}
assignment = record.get("payload")
verified_submission = _submission_lineage(
binding, assignment, payload.get("scm_submission")
)
problem = cli_lane_goal.unfinished_result_reason(structured)
if (
structured.get("status") == "completed"
and int(payload.get("returncode", 1)) == 0
and problem is None
):
if verified_submission is not None:
lineage, head = verified_submission
if isinstance(assignment, dict) and assignment.get("continuation_kind"):
child = supervisor_state.get_child(
binding["board"], binding["task_id"]
)
if (
child is None or child.get("lineage") != lineage
or child.get("kind") != assignment["continuation_kind"]
):
raise ProtocolError("continuation submission lineage is invalid")
supervisor_state.record_submission(
binding["board"], binding["task_id"], lineage, head
)
changed = kanban_db.complete_task(
connection, binding["task_id"],
result=json.dumps(structured, sort_keys=True),

2
services/hermes/scripts/execution_pool_project.py
View File

@ -19,7 +19,7 @@ ATLAS_REPO = re.compile(
r"https://scm\.bstein\.dev/titan/(?P<repo>[A-Za-z0-9][A-Za-z0-9_.-]{0,99})\.git\Z"
)
SAFE_PREFIXES = frozenset(
{"feature", "fix", "chore", "docs", "test", "refactor", "wt", "review", "hermes", "handoff"}
{"feature", "fix", "chore", "docs", "test", "refactor", "wt", "review", "hermes", "hermes-repair", "handoff"}
)
IDENTIFIER = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}\Z")
GIT = "/usr/bin/git"

108
services/hermes/scripts/execution_pool_scm.py
View File

@ -9,11 +9,13 @@ import re
import stat
import subprocess
import threading
import time
import urllib.parse
from pathlib import Path
from typing import Any
import scm_broker_client
from scm_task_grants import ZERO_SHA, sign_grant
from execution_pool_project import ATLAS_REPO, validate_branch
from execution_pool_protocol import ProtocolError, atomic_json, verify_envelope
@ -144,29 +146,12 @@ def _state_path(envelope: dict[str, Any]) -> Path:
return path
def submission_refs(branch: str, attempt: int, head: str) -> tuple[str, ...]:
"""Candidate push refs for one attempt, in order of preference.
The broker accepts branch *creation* only, so a retry that adds commits can
never update the ref a previous attempt already published, and no attempt may
ever move a protected ref. Each fallback is therefore a fresh name in the same
reviewed namespace -- bound first to the exact attempt, then to the exact
content -- so every push stays a creation and stays idempotent under replay.
"""
candidates = (
branch,
f"{branch}-attempt-{max(1, int(attempt))}",
f"{branch}-{head[:12]}",
)
allowed: list[str] = []
for candidate in dict.fromkeys(candidates):
try:
allowed.append(validate_branch(candidate, feature=True))
except ValueError:
continue
if not allowed:
raise ProtocolError("no reviewed branch name is available for this attempt")
return tuple(allowed)
def submission_refs(branch: str, _attempt: int, _head: str) -> tuple[str, ...]:
"""A root task has one branch and one continuing pull request."""
try:
return (validate_branch(branch, feature=True),)
except ValueError as error:
raise ProtocolError("no reviewed branch name is available for this task") from error
def _remote_heads(destination: Path, refs: tuple[str, ...]) -> dict[str, str]:
@ -218,6 +203,20 @@ class Boundary:
def verify(self, raw: Any) -> dict[str, Any]:
return verify_envelope(self.key, raw, expected_kind="assignment")
@staticmethod
def _grant(envelope: dict[str, Any], repo: str, branch: str, base: str, old: str, head: str) -> str:
"""Bind a five-minute update authorization to the verified assignment."""
payload = envelope.get("payload")
root = payload.get("root_task_id") if isinstance(payload, dict) else None
root = root if isinstance(root, str) and root else str(envelope["task_id"])
return sign_grant({
"repo": repo, "ref": branch, "base": base, "board": str(envelope["board"]),
"root_task_id": root, "assignment_task_id": str(envelope["task_id"]),
"run": str(envelope["run_id"]), "ordinal": int(envelope["worker_ordinal"]),
"expires": int(time.time()) + 300, "expected_old": old, "new_head": head,
"continuation_kind": str(payload.get("continuation_kind") or "") if isinstance(payload, dict) else "",
})
def checkout(self, envelope: dict[str, Any]) -> dict[str, Any]:
_payload, repo, branch, base = _binding(envelope)
destination = workspace_path(envelope)
@ -268,15 +267,28 @@ class Boundary:
return {"workspace": str(destination), "baseline_sha": baseline}
@staticmethod
def _draft(repo: str, branch: str, base: str, head: str, title: str, body: str) -> str:
query = urllib.parse.urlencode(
{"state": "open", "head": f"titan:{branch}", "limit": 10}
)
def _draft(repo: str, branch: str, base: str, head: str, title: str, body: str, grant: str = "", *, refresh: bool = True, existing_only: bool = False) -> str:
query = urllib.parse.urlencode({"state": "open", "limit": 50})
existing = json.loads(
scm_broker_client.read(f"/api/v1/repos/titan/{repo}/pulls?{query}")
)
if isinstance(existing, list) and existing:
return str(existing[0].get("html_url") or "")
if isinstance(existing, list):
for item in existing:
if not isinstance(item, dict):
continue
source, target = item.get("head"), item.get("base")
if not isinstance(source, dict) or not isinstance(target, dict):
continue
if source.get("ref") == branch and target.get("ref") == base:
number = item.get("number")
if not refresh:
return str(item.get("html_url") or "")
if not isinstance(number, int) or not grant:
raise ProtocolError("existing task draft cannot be refreshed")
updated = json.loads(scm_broker_client.update_draft(grant, number, title, body))
return str(updated.get("html_url") or "")
if existing_only:
raise ProtocolError("review continuation has no existing pull request")
created = json.loads(
scm_broker_client.create_draft(
repo,
@ -310,26 +322,28 @@ class Boundary:
)
if status:
raise ProtocolError("workspace has uncommitted or untracked files")
candidates = submission_refs(branch, int(envelope["attempt"]), head)
heads = _remote_heads(destination, candidates)
# A candidate already at this exact head was published by an earlier
# attempt or an interrupted one: adopt it instead of pushing again, so
# replay is a no-op and prior work is never re-derived or dropped.
published = next(
(ref for ref in candidates if heads.get(ref) == head), None
)
target = submission_refs(branch, int(envelope["attempt"]), head)[0]
review = _payload.get("continuation_kind") == "review"
heads = _remote_heads(destination, (target,))
remote = heads.get(target)
ahead = int(_run("rev-list", "--count", f"{baseline}..{head}", cwd=destination))
if ahead <= 0 and published is None:
if ahead <= 0 and remote is None:
return {"workspace": str(destination), "branch": branch, "pull_request": ""}
target = published or next(
(ref for ref in candidates if ref not in heads), None
)
if target is None:
raise ProtocolError("every reviewed branch name for this attempt is taken")
if published is None:
if remote == head:
grant = self._grant(envelope, repo, target, base, remote, head)
pull = self._draft(repo, target, base, head, title, body, grant) if not review else self._draft(repo, target, base, head, title, body, grant, refresh=False, existing_only=True)
return {"workspace": str(destination), "branch": target, "pull_request": pull, "head": head}
expected = remote or ZERO_SHA
grant = self._grant(envelope, repo, target, base, expected, head)
if remote is None:
scm_broker_client.register_task(grant)
try:
_run(
"push", "hermes-broker", f"HEAD:refs/heads/{target}",
"-c", f"http.extraHeader=X-Hermes-Task-Grant: {grant}",
"push", "--no-thin", "hermes-broker", f"HEAD:refs/heads/{target}",
cwd=destination, timeout=900,
)
pull = self._draft(repo, target, base, head, title, body)
return {"workspace": str(destination), "branch": target, "pull_request": pull}
except RuntimeError as error:
raise ProtocolError("task branch update was rejected; fetch and merge before retrying") from error
pull = self._draft(repo, target, base, head, title, body, self._grant(envelope, repo, target, base, head, head))
return {"workspace": str(destination), "branch": target, "pull_request": pull, "head": head}

3
services/hermes/scripts/hermes_coordinator.py
View File

@ -21,6 +21,7 @@ from hermes_model_routing import (
discover_claude_models,
discover_codex_models,
)
from model_catalog_refresh import refresh_model_evaluations
CASSANDRA_BASE_PATH = Path("/opt/data/workspace/projects/cassandra")
@ -259,6 +260,7 @@ def refresh_once(root: Path) -> dict[str, Any]:
codex = discover_codex_models()
claude = discover_claude_models()
routes = configure_routes(root, codex, claude)
evaluation_state = refresh_model_evaluations(root, codex, claude)
board_state = bootstrap_cassandra_state(root)
repo_state = sync_cassandra_repo(env_values)
status = {
@ -269,6 +271,7 @@ def refresh_once(root: Path) -> dict[str, Any]:
claude.provider: asdict(claude),
},
"routes": routes,
"model_evaluations": evaluation_state,
"projects": {
"cassandra": {
"board": "cassandra",

114
services/hermes/scripts/hermes_model_routing.py
View File

@ -15,16 +15,21 @@ from typing import Any, Iterable
import yaml
from provider_model_catalog import (
CAPABILITY_ROLES,
CAPABILITY_TIERS,
EFFORTS,
EFFORT_TIERS,
Catalog,
apply_verified_evaluations,
capability_pool,
legacy_selector_matches,
model_records,
model_version,
select_tier_model,
unique_models,
)
from provider_model_discovery import discover_claude_models, discover_codex_models
from model_evaluation_evidence import load_store
CODEX_BASELINE = "gpt-5.6-terra"
@ -52,19 +57,19 @@ def choose_codex_model(models: Iterable[str], current: str = CODEX_BASELINE, *,
"""Compatibility helper selecting a declared advanced or balanced Codex tier."""
effort = "medium" if balanced else "xhigh"
tier = "balanced" if balanced else "advanced"
return select_tier_model("codex", models, {}, tier, effort, current)[0]
return select_tier_model("codex", models, {}, tier, effort, current, legacy_compat=True)[0]
def choose_codex_for_effort(models: Iterable[str], effort: str, current: str = CODEX_BASELINE) -> str:
"""Compatibility helper selecting an effort's generic Codex capability tier."""
if effort not in EFFORTS:
raise ValueError(f"unsupported effort: {effort}")
return select_tier_model("codex", models, {}, EFFORT_TIERS[effort], effort, current)[0]
return select_tier_model("codex", models, {}, EFFORT_TIERS[effort], effort, current, legacy_compat=True)[0]
def choose_claude_model(models: Iterable[str], current: str = CLAUDE_BASELINE) -> str:
"""Compatibility helper selecting the declared advanced Claude tier."""
return select_tier_model("claude", models, {}, "advanced", "xhigh", current)[0]
return select_tier_model("claude", models, {}, "advanced", "xhigh", current, legacy_compat=True)[0]
def choose_claude_for_effort(
@ -73,7 +78,7 @@ def choose_claude_for_effort(
"""Compatibility helper selecting an effort's generic Claude capability tier."""
if effort not in EFFORTS:
raise ValueError(f"unsupported effort: {effort}")
return select_tier_model("claude", models, {}, EFFORT_TIERS[effort], effort, current)[0]
return select_tier_model("claude", models, {}, EFFORT_TIERS[effort], effort, current, legacy_compat=True)[0]
# The coordinator retains these private spellings for compact call sites while
@ -153,7 +158,8 @@ def _previous_provider_models(
def build_routing_catalog(
codex: Catalog, claude: Catalog, previous: dict[str, Any] | None = None
codex: Catalog, claude: Catalog, previous: dict[str, Any] | None = None,
evaluations: dict[str, Any] | None = None,
) -> dict[str, Any]:
"""Build a current catalog while retaining only outage-safe known routes."""
previous = previous or {}
@ -164,31 +170,19 @@ def build_routing_catalog(
"codex",
codex,
{"luna": "economy", "terra": "balanced", "sol": "advanced"},
{
"low": "gpt-5.6-luna",
"medium": "gpt-5.6-terra",
"high": "gpt-5.6-sol",
"xhigh": "gpt-5.6-sol",
},
),
(
"claude",
claude,
{
"haiku": "economy",
"fable": "advanced",
"fable": "frontier",
"sonnet": "balanced",
"opus": "advanced",
},
{
"low": "claude-haiku-4-5-20251001",
"medium": "claude-sonnet-5",
"high": "claude-opus-5",
"xhigh": "claude-opus-5",
},
),
)
for name, discovered, legacy_selectors, defaults in specifications:
for name, discovered, legacy_selectors in specifications:
old_resolved, old_tiers, old_models, old_metadata = _previous_provider_models(
previous, name
)
@ -200,40 +194,52 @@ def build_routing_catalog(
else None
)
source_models = discovered.models if discovered.live else old_models
source_metadata = discovered.metadata if discovered.live else old_metadata
resolved: dict[str, str] = {}
source_metadata = apply_verified_evaluations(
name, source_models, discovered.metadata if discovered.live else old_metadata,
evaluations.get("evaluations") if isinstance(evaluations, dict) else None,
)
observations: dict[str, Any] = {}
for effort in EFFORTS:
# A successful provider list is authoritative: a retired model
# must not survive it merely because it was last known-good.
current = str(old_resolved.get(effort) or defaults[effort])
selected, observed = _select_tier_model(
name,
source_models,
source_metadata,
EFFORT_TIERS[effort],
effort,
current,
allow_current_fallback=not discovered.live,
)
resolved[effort] = selected
observations.update(observed)
old_capabilities = old_record.get("capability_resolved", {}) if isinstance(old_record, dict) else {}
old_capabilities = old_capabilities if isinstance(old_capabilities, dict) else {}
capability_pools = {
role: capability_pool(name, source_models, source_metadata, role)
for role in CAPABILITY_ROLES
}
capability_resolved: dict[str, dict[str, str]] = {}
for role in CAPABILITY_ROLES:
prior = old_capabilities.get(role, {})
prior = prior if isinstance(prior, dict) else {}
routes: dict[str, str] = {}
for effort in EFFORTS:
current = str(prior.get(effort) or (
old_resolved.get(effort) if role == EFFORT_TIERS[effort] else ""
))
selected, observed = _select_tier_model(
name, source_models, source_metadata, role, effort, current,
allow_current_fallback=not discovered.live,
)
routes[effort] = selected
observations.update(observed)
capability_resolved[role] = routes
resolved = {
effort: capability_resolved[EFFORT_TIERS[effort]][effort]
for effort in EFFORTS
}
tiers: dict[str, str] = {}
representative_effort = {
"economy": "low", "balanced": "medium", "advanced": "high", "frontier": "xhigh",
}
for capability in CAPABILITY_TIERS:
effort = next(
value for value, target in EFFORT_TIERS.items() if target == capability
)
current = str(old_tiers.get(capability) or "")
tiers[capability], observed = _select_tier_model(
name, source_models, source_metadata, capability, effort, current,
allow_current_fallback=not discovered.live,
)
observations.update(observed)
effort = representative_effort[capability]
tiers[capability] = capability_resolved[capability][effort]
for selector, capability in legacy_selectors.items():
# Explicit historical family picks are not generic capability
# requests. They must remain exact or become unavailable, never
# silently change to a newer family such as Astra.
exact = [model for model in source_models if selector in model.lower()]
exact = [
model for model in source_models
if legacy_selector_matches(name, selector, model)
]
tiers[selector] = (
exact[0]
if exact
@ -256,11 +262,13 @@ def build_routing_catalog(
),
"model_metadata": source_metadata,
"candidates": observations,
"capability_pools": capability_pools,
"capability_resolved": capability_resolved,
"resolved": resolved,
"tiers": tiers,
}
return {
"schema_version": 2,
"schema_version": 3,
"updated_at": checked_at,
"providers": providers,
}
@ -270,7 +278,17 @@ def write_routing_catalog(
path: Path, codex: Catalog, claude: Catalog
) -> dict[str, Any]:
"""Atomically publish the catalog consumed by hosted and worker brokers."""
catalog = build_routing_catalog(codex, claude, _read_json(path))
evidence_store = load_store(path.with_name("model-evaluations.json"))
evaluations: dict[str, dict[str, dict[str, Any]]] = {"codex": {}, "claude": {}}
for record in evidence_store.get("evaluations", {}).values():
if not isinstance(record, dict):
continue
provider, model = record.get("provider"), record.get("model")
if provider in evaluations and isinstance(model, str):
evaluations[provider][model] = record
catalog = build_routing_catalog(
codex, claude, _read_json(path), {"evaluations": evaluations}
)
_atomic_write(path, json.dumps(catalog, indent=2, sort_keys=True) + "\n", 0o644)
return catalog

158
services/hermes/scripts/kanban_continue_pr.py Normal file
View File

@ -0,0 +1,158 @@
#!/usr/bin/env python3
"""Queue a trusted same-PR Hermes continuation from an existing root card."""
from __future__ import annotations
import argparse
import json
import re
import sys
from typing import Any
import scm_broker_client
import supervisor_state
BOARD = re.compile(r"[a-z0-9][a-z0-9-]{0,63}\Z")
TASK = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}\Z")
PR_URL = re.compile(
r"https://scm\.bstein\.dev/titan/(?P<project>[A-Za-z0-9][A-Za-z0-9_.-]{0,99})/pulls/(?P<number>[1-9][0-9]{0,9})\Z"
)
MAX_OBJECTIVE = 8_000
class ContinueError(ValueError):
"""Reject a continuation that cannot be tied to durable coordinator state."""
def _text(value: Any) -> str:
return value.strip() if isinstance(value, str) else ""
def _validated_args(board: str, root_task: str, objective: str) -> tuple[str, str, str]:
"""Keep the CLI surface limited to a board, root ID, and user objective."""
board, root_task, objective = _text(board), _text(root_task), _text(objective)
if not BOARD.fullmatch(board) or not TASK.fullmatch(root_task):
raise ContinueError("board or root task ID is invalid")
if not objective or len(objective) > MAX_OBJECTIVE or "\x00" in objective:
raise ContinueError("objective must be non-empty and at most 8000 characters")
return board, root_task, objective
def _live_head(lineage: Any) -> str:
"""Prove the exact open, same-repository PR that the root recorded."""
match = PR_URL.fullmatch(lineage.pull_request)
if match is None or match.group("project") != lineage.project:
raise ContinueError("root lineage does not carry a canonical pull request")
try:
document = json.loads(scm_broker_client.read(
f"/api/v1/repos/titan/{lineage.project}/pulls/{match.group('number')}"
))
except Exception as error: # noqa: BLE001 - read proof must fail closed
raise ContinueError(f"live pull-request proof is unavailable: {type(error).__name__}") from error
if not isinstance(document, dict):
raise ContinueError("live pull-request proof is malformed")
head, base = document.get("head"), document.get("base")
full_name = f"titan/{lineage.project}"
valid = (
document.get("state") == "open"
and isinstance(head, dict) and isinstance(base, dict)
and _text(head.get("ref")) == lineage.branch
and isinstance(head.get("repo"), dict) and head["repo"].get("full_name") == full_name
and _text(base.get("ref")) == lineage.base_branch
and isinstance(base.get("repo"), dict) and base["repo"].get("full_name") == full_name
)
sha = _text(head.get("sha")) if isinstance(head, dict) else ""
if not valid or not re.fullmatch(r"[0-9a-fA-F]{7,64}", sha):
raise ContinueError("live pull request no longer matches trusted branch, base, and repository")
return sha
def _body(root_task: str, lineage: Any, head: str, objective: str) -> str:
"""Retain human-visible PR/root context; authority remains in supervisor state."""
return (
"Hermes-Task-Role: repair\n\n"
f"Trusted continuation of root task {root_task} on {lineage.pull_request}.\n"
f"Verified current PR head: {head}\n\n"
"Use the coordinator-issued private workspace from the latest remote branch. "
"Keep this existing PR and branch; do not create a branch or PR. Inspect the "
"current diff, implement and verify the requested repair, then submit through "
"the signed mediator path.\n\n"
f"Objective:\n{objective}\n"
)
def queue(
kanban_db: Any, *, board: str, root_task: str, objective: str
) -> tuple[str, bool]:
"""Create one idempotent root-parent repair card using only trusted lineage."""
board, root_task, objective = _validated_args(board, root_task, objective)
with kanban_db.scoped_current_board(board):
conn = kanban_db.connect(board=board)
try:
root = kanban_db.get_task(conn, root_task)
if root is None:
raise ContinueError("root task does not exist on the requested board")
lineage = supervisor_state.get_root(board, root_task)
if lineage is None or lineage.root_task_id != root_task:
raise ContinueError("root task has no coordinator-issued PR lineage")
head = _live_head(lineage)
existing = supervisor_state.existing_child(board, root_task, head, objective)
if existing:
return existing, False
# A live read proved this is the current same PR. It invalidates any
# prior approval before the new child can be dispatched.
supervisor_state.record_live_head(board, root_task, head)
key = f"supervisor:continue:{root_task}:{head}:{supervisor_state.objective_digest(objective)}"
child_id = kanban_db.create_task(
conn,
title=f"Continue PR {lineage.pull_request.rsplit('/', 1)[-1]}: {objective[:96]}",
body=_body(root_task, lineage, head, objective),
assignee="cli-auto",
created_by="hermes-supervisor",
parents=[root_task],
idempotency_key=key,
# Native Hermes derives ready/todo from the root parent; its
# only non-blocked initial status is the explicit running mode.
initial_status="running",
)
if not isinstance(child_id, str) or not child_id:
raise ContinueError("Kanban did not return a continuation task ID")
supervisor_state.record_child(
board, child_id, root_task, root_task, "repair", head, objective
)
supervisor_state.clear_ready(board, root_task)
try:
kanban_db.add_comment(
conn, root_task, "hermes-supervisor",
f"supervisor: queued trusted continuation {child_id} for current PR head {head}.",
)
except Exception:
pass
return child_id, True
finally:
conn.close()
def main(argv: list[str] | None = None) -> int:
"""Parse a minimal local CLI then queue a continuation or print its ID."""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--board", required=True)
parser.add_argument("--root-task", required=True)
parser.add_argument("--objective", required=True)
args = parser.parse_args(argv)
try:
from hermes_cli import kanban_db
child, created = queue(
kanban_db, board=args.board, root_task=args.root_task, objective=args.objective
)
except ContinueError as error:
print(f"kanban continuation rejected: {error}", file=sys.stderr)
return 2
print(json.dumps({"task_id": child, "created": created}, sort_keys=True))
return 0
if __name__ == "__main__": # pragma: no cover - process entry point
raise SystemExit(main())

173
services/hermes/scripts/kanban_supervisor.py
View File

@ -31,6 +31,7 @@ from __future__ import annotations
import json
import os
import re
import sys
import time
from dataclasses import dataclass
@ -40,6 +41,8 @@ from typing import Any, Callable
import yaml
import supervisor_policy as policy
import scm_broker_client
import supervisor_state
DATA_ROOT = Path(os.environ.get("HERMES_HOME", "/opt/data"))
CONFIG_PATH = DATA_ROOT / "config.yaml"
@ -50,8 +53,11 @@ LEDGER_PATH = DATA_ROOT / "supervisor" / "emitted.json"
DEFAULT_INTERVAL_SECONDS = 30
DEFAULT_MAX_CYCLES = 5
DEFAULT_MAX_CHAINS = 20
DEFAULT_REVIEW_ASSIGNEE = "cli-claude-xhigh"
DEFAULT_REVIEW_ASSIGNEE = "cli-auto"
DEFAULT_REPAIR_ASSIGNEE = "cli-auto"
PR_URL = re.compile(
r"https://scm\.bstein\.dev/titan/(?P<project>[A-Za-z0-9][A-Za-z0-9_.-]{0,99})/pulls/(?P<number>[1-9][0-9]{0,9})\Z"
)
@dataclass(frozen=True)
@ -172,19 +178,105 @@ def _comment(kanban_db: Any, conn: Any, task_id: str, body: str) -> None:
_log(f"could not comment on {task_id}: {error}")
def _mark_ready_for_human(kanban_db: Any, conn: Any, task_id: str, body: str) -> None:
"""Flag the implementation as human-mergeable. Never merges or clears WIP."""
def _with_supervisor_state(board: str, task: Any) -> Any:
"""Overlay coordinator-owned lineage on native task rows lacking metadata."""
task_id = str(getattr(task, "id", "") if not isinstance(task, dict) else task.get("id", ""))
if not task_id:
return task
try:
root = supervisor_state.get_root(board, task_id)
child = supervisor_state.get_child(board, task_id)
except (OSError, ValueError) as error:
# A missing row is represented as None. Any read failure or malformed
# present row is authority loss, so stop this board rather than treating
# a possible continuation as a new root task.
raise RuntimeError("supervisor integrity state is unavailable") from error
if root is None and child is None:
return task
values = dict(task) if isinstance(task, dict) else dict(vars(task))
metadata = dict(values.get("metadata") or {})
if root is not None:
metadata["supervisor_lineage"] = root.stamp_fields()
if child is not None:
chain = child["lineage"]
metadata["supervisor"] = {
"kind": child["kind"], "root": child["root_task_id"],
"parent": child["parent_task_id"], "head_commit": child["head_commit"],
"cycle": child["cycle"], **chain.stamp_fields(),
}
values["metadata"] = metadata
return values
def _set_ready_metadata(kanban_db: Any, conn: Any, task_id: str, metadata: dict[str, Any]) -> None:
"""Persist an evidence-bound readiness state when the board API supports it."""
setter = getattr(kanban_db, "set_task_metadata", None) or getattr(
kanban_db, "update_task_metadata", None
)
if callable(setter):
try:
setter(conn, task_id, {"supervisor_ready_for_human_merge": True})
except Exception as error: # noqa: BLE001 - the comment is the durable flag
setter(conn, task_id, metadata)
except Exception as error: # noqa: BLE001 - the comment is durable signalling
_log(f"could not set ready flag on {task_id}: {error}")
def _mark_ready_for_human(
kanban_db: Any, conn: Any, task_id: str, body: str, evidence: dict[str, Any] | None = None
) -> None:
"""Flag the implementation as human-mergeable. Never merges or clears WIP."""
evidence = evidence or {}
_set_ready_metadata(
kanban_db, conn, task_id,
{"supervisor_ready_for_human_merge": True,
"supervisor_ready_commit": str(evidence.get("commit") or ""),
"supervisor_ready_pull_request": str(evidence.get("pr") or "")},
)
_comment(kanban_db, conn, task_id, body)
def _clear_ready(kanban_db: Any, conn: Any, task_id: str, current_commit: str, body: str) -> None:
"""Invalidate a readiness flag when a later verified revision supersedes it."""
_set_ready_metadata(
kanban_db, conn, task_id,
{"supervisor_ready_for_human_merge": False, "supervisor_ready_commit": current_commit},
)
_comment(kanban_db, conn, task_id, body)
def _live_pr_matches(evidence: dict[str, Any]) -> tuple[bool | None, str]:
"""Read the canonical PR before SHIP; ``None`` defers on any unavailable proof."""
project = evidence.get("project")
branch = evidence.get("branch")
base = evidence.get("base_branch")
commit = evidence.get("commit")
pull = evidence.get("pr")
if not all(isinstance(value, str) and value for value in (project, branch, base, commit, pull)):
return None, "SHIP lacks complete trusted PR evidence"
matched = PR_URL.fullmatch(pull)
if matched is None or matched.group("project") != project:
return None, "SHIP pull-request URL is not canonical trusted lineage"
try:
document = json.loads(scm_broker_client.read(
f"/api/v1/repos/titan/{project}/pulls/{matched.group('number')}"
))
except Exception as error: # noqa: BLE001 - read proof failure must defer readiness
return None, f"current PR evidence is unavailable: {type(error).__name__}"
if not isinstance(document, dict):
return None, "current PR evidence is malformed"
full_name = f"titan/{project}"
head = document.get("head")
pr_base = document.get("base")
matches = (
document.get("state") == "open"
and isinstance(head, dict) and isinstance(pr_base, dict)
and head.get("ref") == branch and head.get("sha") == commit
and isinstance(head.get("repo"), dict) and head["repo"].get("full_name") == full_name
and pr_base.get("ref") == base
and isinstance(pr_base.get("repo"), dict) and pr_base["repo"].get("full_name") == full_name
)
return matches, "current PR does not match reviewed branch/base/head" if not matches else ""
def _escalate(kanban_db: Any, conn: Any, task_id: str, reason: str) -> None:
body = (
f"supervisor fail-closed escalation: {reason}. Human attention required; "
@ -213,23 +305,55 @@ def _already_created(kanban_db: Any, conn: Any, payload: dict[str, Any]) -> bool
return policy.existing_followup(tasks, kind, root, commit)
def _spawn(kanban_db: Any, conn: Any, decision: policy.Decision) -> None:
def _prevalidate_spawn(board: str, stamp: dict[str, Any]) -> None:
"""Refuse a native create until its root is in durable coordinator state."""
if not board or not stamp:
return
expected = policy.lineage.from_stamp(stamp)
root_id = str(stamp.get("root") or "")
actual = supervisor_state.get_root(board, root_id)
if expected is None or actual != expected:
raise RuntimeError("supervisor child has no matching trusted root lineage")
def _spawn(kanban_db: Any, conn: Any, decision: policy.Decision, board: str = "") -> None:
payload = dict(decision.payload or {})
stamp = (payload.get("metadata") or {}).get("supervisor") or {}
_prevalidate_spawn(board, stamp)
try:
kanban_db.create_task(conn, **payload)
child_id = kanban_db.create_task(conn, **payload)
except TypeError:
# create_task may reject idempotency_key on older runtimes. That is
# raised at call binding, before any insert, but a post-insert TypeError
# is also possible, so re-run the dedup scan first: only retry the create
# when no matching card exists, so a partial insert is never doubled.
if not _already_created(kanban_db, conn, payload):
if _already_created(kanban_db, conn, decision.payload or {}):
return
# Native Hermes has no task metadata column. The state table records
# this stamp after creation; old compatible test/runtime APIs may still
# accept it on the first attempt.
payload.pop("metadata", None)
try:
child_id = kanban_db.create_task(conn, **payload)
except TypeError:
if _already_created(kanban_db, conn, decision.payload or {}):
return
payload.pop("idempotency_key", None)
kanban_db.create_task(conn, **payload)
child_id = kanban_db.create_task(conn, **payload)
if board and isinstance(child_id, str) and stamp:
try:
supervisor_state.record_child(
board, child_id, str(stamp["root"]), str(stamp["parent"]),
str(stamp["kind"]), str(stamp["head_commit"]),
str(payload.get("body") or ""), int(stamp["cycle"]),
)
except (KeyError, TypeError, ValueError) as error:
raise RuntimeError(f"could not persist supervisor child authority: {error}") from error
_comment(kanban_db, conn, decision.target_id, f"supervisor: {decision.reason}")
def apply_decision(
kanban_db: Any, conn: Any, decision: policy.Decision, ledger: Ledger
kanban_db: Any, conn: Any, decision: policy.Decision, ledger: Ledger, board: str = ""
) -> bool:
"""Execute one decision. Returns True when an action was taken.
@ -241,13 +365,24 @@ def apply_decision(
return False
policy.assert_safe(decision)
if decision.action == "spawn":
_spawn(kanban_db, conn, decision)
_spawn(kanban_db, conn, decision, board)
return True
key = f"{decision.action}:{decision.target_id}"
info = decision.payload or {}
key = f"{decision.action}:{decision.target_id}:{info.get('commit', '')}"
if ledger.has(key):
return False # already emitted on a prior tick; never re-emit or re-block
if decision.action == "ship":
info = decision.payload or {}
live, reason = _live_pr_matches(info)
if live is None:
_log(f"deferring SHIP for {decision.target_id}: {reason}")
return False
if not live:
_clear_ready(
kanban_db, conn, decision.target_id, "",
f"supervisor: readiness cleared because {reason}.",
)
ledger.record(f"clear_ready:{decision.target_id}:{info.get('commit', '')}")
return True
commit = info.get("commit", "")
pr = info.get("pr", "") or "branch on record"
body = (
@ -255,7 +390,13 @@ def apply_decision(
"READY FOR HUMAN MERGE - a human must merge; the supervisor never "
"merges, approves, closes, or clears WIP."
)
_mark_ready_for_human(kanban_db, conn, decision.target_id, body)
_mark_ready_for_human(kanban_db, conn, decision.target_id, body, info)
elif decision.action == "clear_ready":
current = str(info.get("commit") or "")
_clear_ready(
kanban_db, conn, decision.target_id, current,
f"supervisor: stale approval cleared; current verified head is {current}.",
)
else: # escalate
_escalate(kanban_db, conn, decision.target_id, decision.reason)
ledger.record(key)
@ -269,11 +410,11 @@ def supervise_board(
with kanban_db.scoped_current_board(board):
conn = kanban_db.connect(board=board)
try:
tasks = list(kanban_db.list_tasks(conn))
tasks = [_with_supervisor_state(board, task) for task in kanban_db.list_tasks(conn)]
for task in tasks:
decision = policy.plan(task, tasks, limits)
try:
if apply_decision(kanban_db, conn, decision, ledger):
if apply_decision(kanban_db, conn, decision, ledger, board):
actions += 1
except Exception as error: # noqa: BLE001 - one card never stops others
_log(f"action failed on board {board!r}: {error}")

404
services/hermes/scripts/model_capability_evaluator.py Normal file
View File

@ -0,0 +1,404 @@
#!/usr/bin/env python3
"""Run small deterministic model-fit probes through Hermes' native brokers.
The probes establish only a narrow routing signal. They do not claim a model is
generally intelligent, replace provider declarations, or promote a route by
model name. Transport failures remain pending availability, never quality loss.
"""
from __future__ import annotations
import json
import os
import time
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Mapping, Protocol
from urllib.error import HTTPError, URLError
from urllib.request import Request, urlopen
from model_evaluation_evidence import (
EVALUATION_VERSION,
accepted_outcomes,
load_store,
metadata_fingerprint,
record_key,
retry_due,
reusable_success,
unavailable_record,
write_store,
)
from provider_model_catalog import (
CAPABILITY_ROLES,
EFFORTS,
REVIEWED_CAPABILITY_POLICY,
is_eligible,
proposed_candidate_role,
supported_efforts,
)
MAX_MODELS_PER_REFRESH = 2
MAX_CALLS_PER_MODEL = 3
REQUEST_TIMEOUT_SECONDS = 25
MAX_OUTPUT_TOKENS = 160
MAX_RESPONSE_BYTES = 16 * 1024
@dataclass(frozen=True)
class ProbeReply:
"""One sanitized broker response used by the deterministic checker."""
text: str
input_tokens: int = 0
output_tokens: int = 0
latency_ms: int = 0
class LiteralModelTransport(Protocol):
"""Invoke one literal, provider-advertised model without route selection."""
def invoke(self, provider: str, model: str, effort: str, prompt: str) -> ProbeReply:
"""Return one response or raise a classified transport exception."""
class ProbeTransportError(RuntimeError):
"""A safe transport classification that must not become a quality verdict."""
def __init__(self, failure_class: str) -> None:
super().__init__(failure_class)
self.failure_class = failure_class
def _read_secret() -> str:
path = Path(os.environ.get("HERMES_MODEL_EVAL_KEY_FILE", "/runtime-access/chat-relay-key"))
try:
return path.read_text(encoding="utf-8").strip()
except OSError:
return ""
def _failure_class(status: int | None, error: BaseException | None = None) -> str:
if status in {401, 403}:
return "auth"
if status == 429:
return "rate_limited"
if status is not None and status >= 500:
return "provider"
if isinstance(error, TimeoutError):
return "timeout"
if isinstance(error, URLError):
return "network"
return "invalid_response" if status and 400 <= status < 500 else "provider"
class BrokerHttpTransport:
"""Use the local credential-isolating native brokers with fixed model IDs."""
def __init__(self, key: str | None = None) -> None:
self.key = key if key is not None else _read_secret()
self.codex_endpoint = os.environ.get(
"HERMES_MODEL_EVAL_CODEX_ENDPOINT", "http://127.0.0.1:9003/v1/responses"
)
self.claude_endpoint = os.environ.get(
"HERMES_MODEL_EVAL_CLAUDE_ENDPOINT", "http://127.0.0.1:9006/v1/messages"
)
def invoke(self, provider: str, model: str, effort: str, prompt: str) -> ProbeReply:
if not self.key:
raise ProbeTransportError("auth")
if provider == "codex":
endpoint = self.codex_endpoint
payload = {
"model": model,
"input": prompt,
"stream": False,
"store": False,
"reasoning": {"effort": effort},
# The subscription broker deliberately drops this unsupported
# upstream field. The prompt and local response-size limit
# still bound the evaluation exchange without misclassifying
# a longer provider reasoning trace as poor quality.
"max_output_tokens": MAX_OUTPUT_TOKENS,
}
elif provider == "claude":
endpoint = self.claude_endpoint
payload = {
"model": model,
"max_tokens": MAX_OUTPUT_TOKENS,
"stream": False,
"system": "Return only the requested JSON. Tools are unavailable for this probe.",
"messages": [{"role": "user", "content": prompt}],
"output_config": {"effort": effort},
}
else:
raise ProbeTransportError("provider")
request = Request(
endpoint,
data=json.dumps(payload, separators=(",", ":")).encode("utf-8"),
headers={"Content-Type": "application/json", "Authorization": f"Bearer {self.key}"},
method="POST",
)
started = time.monotonic()
try:
with urlopen(request, timeout=REQUEST_TIMEOUT_SECONDS) as response:
body = response.read(MAX_RESPONSE_BYTES + 1)
if len(body) > MAX_RESPONSE_BYTES:
raise ProbeTransportError("truncated")
document = json.loads(body.decode("utf-8"))
except HTTPError as exc:
raise ProbeTransportError(_failure_class(exc.code, exc)) from exc
except (URLError, TimeoutError, OSError) as exc:
raise ProbeTransportError(_failure_class(None, exc)) from exc
except (TypeError, ValueError, json.JSONDecodeError) as exc:
raise ProbeTransportError("invalid_response") from exc
if not isinstance(document, Mapping):
raise ProbeTransportError("invalid_response")
text, input_tokens, output_tokens = _response_text(provider, document)
return ProbeReply(
text=text,
input_tokens=input_tokens,
output_tokens=output_tokens,
latency_ms=int((time.monotonic() - started) * 1000),
)
def _response_text(provider: str, document: Mapping[str, Any]) -> tuple[str, int, int]:
"""Extract final text and usage from broker-specific response envelopes."""
usage = document.get("usage") if isinstance(document.get("usage"), Mapping) else {}
if provider == "claude":
content = document.get("content")
text = "".join(
str(item.get("text") or "") for item in content
if isinstance(item, Mapping) and item.get("type") == "text"
) if isinstance(content, list) else ""
return text, _token_count(usage.get("input_tokens")), _token_count(usage.get("output_tokens"))
output = document.get("output")
text = ""
if isinstance(output, list):
for item in output:
if not isinstance(item, Mapping):
continue
for content in item.get("content", []):
if isinstance(content, Mapping) and content.get("type") in {"output_text", "text"}:
text += str(content.get("text") or "")
return text, _token_count(usage.get("input_tokens")), _token_count(usage.get("output_tokens"))
def _token_count(value: Any) -> int:
"""Return a non-negative broker usage count without trusting response types."""
return value if isinstance(value, int) and value >= 0 else 0
def _efforts(metadata: Mapping[str, Any], role: str) -> list[str]:
"""Return the one provider-advertised effort used for this role's probes."""
advertised = supported_efforts(dict(metadata))
available = [effort for effort in EFFORTS if effort in advertised]
target = {"economy": "low", "balanced": "medium", "advanced": "high", "frontier": "xhigh"}[role]
return [target] if target in available else available[:1]
def _probe_cases(role: str) -> tuple[tuple[str, str, str, set[str], set[str]], ...]:
"""Return role-sized finite-answer contracts without prose interpretation."""
prefix = (
"Return only JSON {\"decision\":\"D#\",\"invariants\":[\"I#\"],"
"\"checks\":[\"C#\"]}. Select IDs exactly; do not run code, call tools, or add prose. "
)
cases = {
"economy": (
("idempotency", "A message consumer receives event E twice after a retry and must create one invoice. "
"Decisions: D1 record E durably before the invoice effect; D2 create invoice then record E. "
"Invariants: I1 event_id is unique before side effect; I2 retry is best effort. "
"Checks: C1 concurrent duplicate E makes one invoice; C2 one normal E makes an invoice.",
"D1", {"I1"}, {"C1"}),
("cache", "A user permission cache is keyed by user ID. Policy revision changes from 7 to 8, "
"revoking access. Decisions: D1 revalidate revision before allow; D2 use cached allow until TTL. "
"Invariants: I1 key includes policy revision; I2 TTL is under one hour. "
"Checks: C1 revision change returns deny; C2 cache hit returns prior allow.",
"D1", {"I1"}, {"C1"}),
),
"balanced": (
("cache", "A permission cache uses (user_id, policy_revision). A request reads revision 7, "
"then revocation commits revision 8 before a handler uses its lookup. Decisions: D1 compare current "
"revision before allow; D2 trust the earlier cache read. Invariants: I1 allow requires matching current "
"revision; I2 revocation is eventually consistent. Checks: C1 revocation between read/use denies; "
"C2 a cache read before revocation permits.", "D1", {"I1"}, {"C1"}),
("idempotency", "A webhook retries while the first request commits. The database has a unique event_id "
"constraint. Decisions: D1 use insert conflict as same-event result; D2 retry the side effect. "
"Invariants: I1 side effect follows successful unique event insert; I2 last writer wins. "
"Checks: C1 concurrent duplicates create one effect; C2 two different IDs create one effect.",
"D1", {"I1"}, {"C1"}),
),
"advanced": (
("outbox", "Payment P must charge once and publish receipt once despite a crash after DB commit. "
"Decisions: D1 atomically commit payment state, unique payment event, and outbox, then relay idempotently; "
"D2 charge then publish directly. Invariants: I1 unique payment event prevents another charge; "
"I2 outbox commits with payment state; I3 relay retry alone guarantees exactly once. Checks: "
"C1 crash/replay gives one charge and eventually one idempotent receipt; C2 crash/replay sends no receipt.",
"D1", {"I1", "I2"}, {"C1"}),
("state", "Two workers process the same order. A lease can expire while worker A is paused. "
"Decisions: D1 compare-and-set expected version; D2 overwrite by latest wall clock. Invariants: "
"I1 transition requires current version; I2 lease holder may always write. Checks: C1 paused stale worker "
"cannot overwrite committed state; C2 two writers eventually agree without rejection.", "D1", {"I1"}, {"C1"}),
),
"frontier": (
("fencing", "Worker A holds lease token 10 and pauses. Lease token 11 goes to B, which commits. "
"A resumes. Decisions: D1 storage rejects lower fencing tokens; D2 A writes if its clock says lease valid. "
"Invariants: I1 every write carries monotonically increasing fence token; I2 lease expiry is enough. "
"Checks: C1 token 10 cannot overwrite token 11; C2 a valid clock always permits write.", "D1", {"I1"}, {"C1"}),
("saga", "Reserve inventory then charge payment across services. The reservation expires before charge reply. "
"Decisions: D1 compensate/refund or re-reserve before shipping; D2 ship because payment succeeded. "
"Invariants: I1 ship requires durable active reservation and payment confirmation; I2 payment confirmation "
"alone permits shipping. Checks: C1 delayed charge after expiry does not ship; C2 delayed charge always ships.",
"D1", {"I1"}, {"C1"}),
),
}
return tuple(
(kind, prefix + prompt, decision, invariants, checks)
for kind, prompt, decision, invariants, checks in cases[role]
)
def _score(
_kind: str, text: str, decision: str, invariants: set[str], checks: set[str]
) -> bool | None:
"""Validate structured, exact evidence fields without interpreting prose."""
try:
answer = json.loads(text)
except (TypeError, ValueError, json.JSONDecodeError):
return None
if not isinstance(answer, Mapping) or not isinstance(answer.get("decision"), str):
return None
observed_invariants = answer.get("invariants")
observed_checks = answer.get("checks")
if not isinstance(observed_invariants, list) or not isinstance(observed_checks, list):
return None
if not all(isinstance(value, str) for value in observed_invariants + observed_checks):
return None
normalized_invariants = set(observed_invariants)
normalized_checks = set(observed_checks)
return (
answer["decision"] == decision
and invariants == normalized_invariants
and checks == normalized_checks
)
def _candidate_records(catalog: Mapping[str, Any]) -> list[tuple[str, str, Mapping[str, Any], str]]:
providers = catalog.get("providers") if isinstance(catalog.get("providers"), Mapping) else {}
candidates: list[tuple[str, str, Mapping[str, Any], str]] = []
for provider in ("codex", "claude"):
record = providers.get(provider)
if not isinstance(record, Mapping) or record.get("live") is not True:
continue
metadata = record.get("model_metadata") if isinstance(record.get("model_metadata"), Mapping) else {}
models = record.get("models") if isinstance(record.get("models"), list) else []
for model in models:
details = metadata.get(model, {}) if isinstance(metadata.get(model), Mapping) else {}
eligible, _ = is_eligible(provider, model, dict(details), None) if isinstance(model, str) else (False, "")
role, provenance = proposed_candidate_role(provider, model, dict(details)) if isinstance(model, str) else (None, "")
reviewed = model in REVIEWED_CAPABILITY_POLICY.get(provider, {})
if isinstance(model, str) and eligible and not reviewed and role in CAPABILITY_ROLES and provenance:
candidates.append((provider, model, details, role))
return candidates
def evaluate_catalog_candidates(
catalog: Mapping[str, Any], *, evidence_path: Path = Path("/routing-catalog/model-evaluations.json"),
transport: LiteralModelTransport | None = None, now: int | None = None,
observed_outcomes: tuple[Mapping[str, Any], ...] = (),
) -> dict[str, Any]:
"""Evaluate at most two new candidates and retain cached/pending evidence.
Only a passing result says the two small probes fit a provider-proposed role.
Every other outcome remains pending; route selection owns any later promotion.
"""
checked_at = int(time.time()) if now is None else now
store = load_store(evidence_path)
records = store["evaluations"]
selected = 0
active_transport = transport or BrokerHttpTransport()
for provider, model, metadata, role in _candidate_records(catalog):
key = record_key(provider, model)
fingerprint = metadata_fingerprint(model, metadata)
previous = records.get(key) if isinstance(records.get(key), Mapping) else None
if reusable_success(previous, fingerprint) or not retry_due(previous, checked_at):
continue
if selected >= MAX_MODELS_PER_REFRESH:
break
selected += 1
efforts = _efforts(metadata, role)
if not efforts:
records[key] = unavailable_record(
provider=provider, model=model, fingerprint=fingerprint, role=role,
efforts=[], failure_class="provider", now=checked_at,
)
continue
replies: list[ProbeReply] = []
try:
cases = _probe_cases(role)
for kind, prompt, decision, invariants, checks in cases[:MAX_CALLS_PER_MODEL]:
reply = active_transport.invoke(provider, model, efforts[0], prompt)
replies.append(reply)
score = _score(kind, reply.text, decision, invariants, checks)
if score is None:
raise ProbeTransportError("invalid_response")
if not score:
break
except ProbeTransportError as exc:
records[key] = unavailable_record(
provider=provider, model=model, fingerprint=fingerprint, role=role,
efforts=efforts, failure_class=exc.failure_class, now=checked_at,
input_tokens=sum(reply.input_tokens for reply in replies),
output_tokens=sum(reply.output_tokens for reply in replies),
latency_ms=sum(reply.latency_ms for reply in replies),
)
continue
except (OSError, TimeoutError, URLError):
records[key] = unavailable_record(
provider=provider, model=model, fingerprint=fingerprint, role=role,
efforts=efforts, failure_class="network", now=checked_at,
input_tokens=sum(reply.input_tokens for reply in replies),
output_tokens=sum(reply.output_tokens for reply in replies),
latency_ms=sum(reply.latency_ms for reply in replies),
)
continue
passed = len(replies) == len(cases) and all(
_score(kind, reply.text, decision, invariants, checks) is True
for (kind, _prompt, decision, invariants, checks), reply in zip(cases, replies)
)
records[key] = {
"provider": provider,
"model": model,
"metadata_fingerprint": fingerprint,
"eval_version": EVALUATION_VERSION,
"proposed_role": role,
"attempted_efforts": efforts,
"result": "pass" if passed else "quality_mismatch",
"role_fit": "verified" if passed else "pending",
"failure_class": None,
"last_attempt_at": checked_at,
"retry_after": None if passed else checked_at + 6 * 60 * 60,
"tokens": {
"input": sum(reply.input_tokens for reply in replies),
"output": sum(reply.output_tokens for reply in replies),
"total": sum(reply.input_tokens + reply.output_tokens for reply in replies),
},
"latency_ms": sum(reply.latency_ms for reply in replies),
}
store["updated_at"] = checked_at
write_store(evidence_path, store)
reliable_outcomes = accepted_outcomes(observed_outcomes)
for observation in reliable_outcomes:
record = records.get(record_key(observation["provider"], observation["model"]))
if isinstance(record, dict):
record["observed_acceptance"] = {
"accepted": observation["accepted"], "evidence_id": observation["evidence_id"],
}
if reliable_outcomes:
write_store(evidence_path, store)
evaluations: dict[str, dict[str, Any]] = {"codex": {}, "claude": {}}
for value in records.values():
if isinstance(value, Mapping) and value.get("provider") in evaluations and isinstance(value.get("model"), str):
evaluations[str(value["provider"])][value["model"]] = dict(value)
return {"evaluations": evaluations, "accepted_outcomes": reliable_outcomes}

38
services/hermes/scripts/model_catalog_refresh.py Normal file
View File

@ -0,0 +1,38 @@
#!/usr/bin/env python3
"""Evaluate discovered candidates without interrupting established routes."""
from __future__ import annotations
import os
from pathlib import Path
from typing import Any
from hermes_model_routing import write_routing_catalog
from model_capability_evaluator import evaluate_catalog_candidates
from routing_catalog import load_catalog
def refresh_model_evaluations(root: Path, codex: Any, claude: Any) -> dict[str, Any]:
"""Refresh evidence and publish eligible routes; keep the provisional catalog on error."""
path = Path(os.environ.get("HERMES_ROUTING_CATALOG_PATH") or root / "routing-catalog.json")
catalog = load_catalog(path)
if not catalog:
return {"state": "awaiting-catalog"}
try:
evidence = evaluate_catalog_candidates(
catalog, evidence_path=path.with_name("model-evaluations.json")
)
write_routing_catalog(path, codex, claude)
except (OSError, ValueError, RuntimeError) as error:
# Provider and evidence-store outages must leave established routes usable.
return {"state": "deferred", "error_type": type(error).__name__}
records = [
record
for provider in evidence.get("evaluations", {}).values()
for record in provider.values()
]
return {
"state": "ready",
"verified": sum(record.get("role_fit") == "verified" for record in records),
"pending": sum(record.get("role_fit") != "verified" for record in records),
}

158
services/hermes/scripts/model_evaluation_evidence.py Normal file
View File

@ -0,0 +1,158 @@
#!/usr/bin/env python3
"""Persist bounded, non-secret evidence for provider model evaluations."""
from __future__ import annotations
import hashlib
import json
import os
from pathlib import Path
from typing import Any, Iterable, Mapping
SCHEMA_VERSION = 1
EVALUATION_VERSION = "capability-v1"
RETRY_SECONDS = 6 * 60 * 60
FAILURE_CLASSES = frozenset(
{
"auth", "network", "timeout", "rate_limited", "provider",
"invalid_response", "truncated",
}
)
def metadata_fingerprint(model: str, metadata: Mapping[str, Any]) -> str:
"""Bind cached evidence to the advertised, non-secret model metadata."""
# These two fields are injected only after this fingerprint admits a
# candidate. Excluding them keeps an accepted cache record stable across
# the publish/evaluate/publish refresh cycle.
source = {
key: value
for key, value in metadata.items()
if key not in {"evaluated_capability_role", "evaluation_provenance"}
}
value = json.dumps(
{"model": model, "metadata": source},
sort_keys=True,
separators=(",", ":"),
default=str,
)
return hashlib.sha256(value.encode("utf-8")).hexdigest()
def empty_store() -> dict[str, Any]:
"""Return the stable on-disk envelope used by the evaluator."""
return {"schema_version": SCHEMA_VERSION, "evaluations": {}}
def load_store(path: Path) -> dict[str, Any]:
"""Load valid evidence, refusing malformed or incompatible state."""
try:
value = json.loads(path.read_text(encoding="utf-8"))
except (OSError, TypeError, ValueError, json.JSONDecodeError):
return empty_store()
if not isinstance(value, dict) or value.get("schema_version") != SCHEMA_VERSION:
return empty_store()
evaluations = value.get("evaluations")
return value if isinstance(evaluations, dict) else empty_store()
def write_store(path: Path, store: Mapping[str, Any]) -> None:
"""Atomically write evidence without retaining provider response text."""
path.parent.mkdir(parents=True, exist_ok=True)
content = json.dumps(dict(store), indent=2, sort_keys=True) + "\n"
temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp")
temporary.write_text(content, encoding="utf-8")
os.chmod(temporary, 0o644)
os.replace(temporary, path)
def record_key(provider: str, model: str) -> str:
"""Return a collision-safe provider/model identity for JSON storage."""
return f"{provider}:{model}"
def reusable_success(
record: Mapping[str, Any] | None, fingerprint: str, *, version: str = EVALUATION_VERSION
) -> bool:
"""Accept a cache hit only for the identical metadata and eval version."""
return bool(
isinstance(record, Mapping)
and record.get("metadata_fingerprint") == fingerprint
and record.get("eval_version") == version
and record.get("result") == "pass"
and record.get("role_fit") == "verified"
)
def retry_due(record: Mapping[str, Any] | None, now: int) -> bool:
"""Avoid repeatedly spending a refresh on one non-passing model."""
if not isinstance(record, Mapping) or record.get("result") == "pass":
return True
retry_after = record.get("retry_after")
return not isinstance(retry_after, int) or now >= retry_after
def unavailable_record(
*, provider: str, model: str, fingerprint: str, role: str, efforts: list[str],
failure_class: str, now: int, input_tokens: int = 0, output_tokens: int = 0,
latency_ms: int = 0, version: str = EVALUATION_VERSION,
) -> dict[str, Any]:
"""Represent transport trouble without treating it as model quality evidence."""
if failure_class not in FAILURE_CLASSES:
failure_class = "provider"
return {
"provider": provider,
"model": model,
"metadata_fingerprint": fingerprint,
"eval_version": version,
"proposed_role": role,
"attempted_efforts": efforts,
"result": "unavailable",
"role_fit": "pending",
"failure_class": failure_class,
"last_attempt_at": now,
"retry_after": now + RETRY_SECONDS,
"tokens": {
"input": max(0, input_tokens),
"output": max(0, output_tokens),
"total": max(0, input_tokens) + max(0, output_tokens),
},
"latency_ms": max(0, latency_ms),
}
def accepted_outcomes(
observations: Iterable[Mapping[str, Any]],
) -> list[dict[str, Any]]:
"""Keep only externally verified acceptance outcomes with literal model IDs.
This deliberately ignores user preference, inferred quality, and incomplete
task reports. Callers may merge the returned summaries into model evidence
once they have a retained acceptance evaluator result.
"""
accepted: list[dict[str, Any]] = []
for item in observations:
if not isinstance(item, Mapping):
continue
provider = item.get("provider")
model = item.get("model")
if (
item.get("evidence_kind") != "acceptance"
or item.get("verified") is not True
or item.get("accepted") not in {True, False}
or not isinstance(provider, str)
or not provider
or not isinstance(model, str)
or not model
):
continue
accepted.append(
{
"provider": provider,
"model": model,
"accepted": item["accepted"],
"evidence_id": str(item.get("evidence_id") or "")[:128],
}
)
return accepted

258
services/hermes/scripts/provider_model_catalog.py
View File

@ -7,15 +7,74 @@ import re
from dataclasses import dataclass, field
from typing import Any, Iterable
from model_evaluation_evidence import EVALUATION_VERSION, metadata_fingerprint
EFFORTS = ("low", "medium", "high", "xhigh")
CAPABILITY_TIERS = ("economy", "balanced", "advanced")
CAPABILITY_ROLES = ("economy", "balanced", "advanced", "frontier")
# Retain the old name for callers while catalog schema 3 calls these roles.
CAPABILITY_TIERS = CAPABILITY_ROLES
EFFORT_TIERS = {
"low": "economy",
"medium": "balanced",
"high": "advanced",
"xhigh": "advanced",
}
REVIEWED_CAPABILITY_POLICY = {
"codex": {
"gpt-5.6-luna": "economy",
"gpt-5.6-terra": "balanced",
"gpt-5.6-sol": "advanced",
"gpt-6-astra": "frontier",
},
"claude": {
"haiku": "economy",
"sonnet": "balanced",
"opus": "advanced",
"fable": "frontier",
"claude-haiku-4-5": "economy",
"claude-sonnet-5": "balanced",
"claude-opus-5": "advanced",
"claude-fable-5": "frontier",
"claude-fable-5[1m]": "frontier",
"claude-opus-5[1m]": "advanced",
},
}
REVIEWED_POLICY_PROVENANCE = "reviewed-current-account-catalog"
LEGACY_DEPLOYED_MODEL_ROLES = {
"codex": {
"gpt-5.6-luna": "economy",
"gpt-5.6-terra": "balanced",
"gpt-5.6-sol": "advanced",
},
"claude": {
"haiku": "economy",
"sonnet": "balanced",
"opus": "advanced",
"fable": "frontier",
"claude-haiku-4-5": "economy",
"claude-haiku-4-5-20251001": "economy",
"claude-haiku-5": "economy",
"claude-sonnet-5": "balanced",
"claude-opus-4.8": "advanced",
"claude-opus-5": "advanced",
"claude-fable-5": "frontier",
"claude-fable-5[1m]": "frontier",
"claude-opus-5[1m]": "advanced",
},
}
LEGACY_SELECTOR_MODELS = {
"codex": {
"luna": frozenset({"gpt-5.6-luna"}),
"terra": frozenset({"gpt-5.6-terra"}),
"sol": frozenset({"gpt-5.6-sol"}),
},
"claude": {
"haiku": frozenset({"haiku", "claude-haiku-4-5", "claude-haiku-4-5-20251001", "claude-haiku-5"}),
"sonnet": frozenset({"sonnet", "claude-sonnet-5"}),
"opus": frozenset({"opus", "claude-opus-4.8", "claude-opus-5", "claude-opus-5[1m]"}),
"fable": frozenset({"fable", "claude-fable-5", "claude-fable-5[1m]"}),
},
}
SPECIALIST_MODEL_WORDS = frozenset(
{"audio", "embedding", "image", "moderation", "realtime", "speech", "transcrib", "tts"}
)
@ -128,8 +187,10 @@ def _text_metadata(metadata: dict[str, Any]) -> str:
return " ".join(values)
def metadata_tier(metadata: dict[str, Any]) -> tuple[str | None, str]:
"""Classify a model only from provider-declared capability information."""
def proposed_candidate_role(
provider: str, model: str, metadata: dict[str, Any]
) -> tuple[str | None, str]:
"""Propose an evaluation role from provider metadata, never a model ID."""
for key in ("capability_tier", "tier", "capability"):
value = metadata.get(key)
if isinstance(value, str):
@ -138,11 +199,17 @@ def metadata_tier(metadata: dict[str, Any]) -> tuple[str | None, str]:
return "economy", f"provider {key}={value!r}"
if normalized in {"balanced", "standard", "mid", "medium"}:
return "balanced", f"provider {key}={value!r}"
if normalized in {"advanced", "strongest", "premium", "high"}:
if normalized in {"advanced", "premium", "high"}:
return "advanced", f"provider {key}={value!r}"
if normalized in {"frontier", "flagship", "strongest", "most_capable"}:
return "frontier", f"provider {key}={value!r}"
text = _text_metadata(metadata)
if any(term in text for term in ("strongest", "most capable", "most intelligent", "complex tasks")):
return "advanced", "provider description marks it strongest/complex"
if any(term in text for term in ("most capable", "most intelligent", "state of the art", "frontier")):
return "frontier", "provider description marks it provider-declared frontier"
if "strongest" in text and not any(term in text for term in ("near-frontier", "fastest")):
return "frontier", "provider description marks it provider-declared strongest"
if any(term in text for term in ("complex tasks", "hardest tasks", "demanding work")):
return "advanced", "provider description marks it advanced"
if any(term in text for term in ("balanced", "general purpose", "everyday tasks", "routine tasks")):
return "balanced", "provider description marks it balanced/general-purpose"
if any(term in text for term in ("lowest cost", "low cost", "cheapest", "fastest", "fast and affordable")):
@ -150,19 +217,38 @@ def metadata_tier(metadata: dict[str, Any]) -> tuple[str | None, str]:
return None, "provider metadata does not declare a capability tier"
def metadata_tier(metadata: dict[str, Any]) -> tuple[str | None, str]:
"""Compatibility wrapper for provider-declared candidate classification."""
return proposed_candidate_role("", "", metadata)
def legacy_tier(provider: str, model: str) -> tuple[str | None, str]:
"""Migrate deployed pre-metadata families without predicting new IDs."""
known = {
"codex": {"luna": "economy", "terra": "balanced", "sol": "advanced"},
"claude": {
"haiku": "economy", "sonnet": "balanced", "opus": "advanced",
"fable": "advanced",
},
}
for marker, tier in known[provider].items():
if marker in model.lower():
return tier, "legacy deployed-family compatibility"
return None, "no provider capability metadata"
tier = LEGACY_DEPLOYED_MODEL_ROLES.get(provider, {}).get(model.strip().lower())
if tier is not None:
return tier, "legacy reviewed-model compatibility"
return None, "no reviewed provider capability metadata"
def legacy_selector_matches(provider: str, selector: str, model: str) -> bool:
"""Match a historical selector only to its reviewed, exact model aliases."""
return model.strip().lower() in LEGACY_SELECTOR_MODELS.get(provider, {}).get(selector, ())
def capability_role(
provider: str, model: str, metadata: dict[str, Any]
) -> tuple[str | None, str]:
"""Return an active role from reviewed policy or verified evaluation."""
reviewed = REVIEWED_CAPABILITY_POLICY.get(provider, {}).get(model.strip().lower())
if reviewed:
return reviewed, REVIEWED_POLICY_PROVENANCE
evaluated = metadata.get("evaluated_capability_role")
if isinstance(evaluated, str) and evaluated in CAPABILITY_ROLES:
return evaluated, str(metadata.get("evaluation_provenance") or "verified evaluation")
proposed, reason = proposed_candidate_role(provider, model, metadata)
if proposed:
return None, f"candidate pending evaluation: {reason}"
return None, reason
def _model_cost(metadata: dict[str, Any]) -> tuple[str, float] | None:
@ -198,9 +284,26 @@ def _comparable_costs(
return {}
return {model: cost[1] for model, cost in records if cost is not None}
def _candidate_status(
provider: str, model: str, metadata: dict[str, Any], effort: str
def supported_efforts(metadata: dict[str, Any]) -> frozenset[str]:
"""Normalize provider effort spellings without implying unsupported levels."""
supported = metadata.get(
"supported_reasoning_efforts",
metadata.get("supportedReasoningEfforts", metadata.get("supportedEffortLevels")),
)
if not isinstance(supported, list):
return frozenset()
values = {
str(item.get("reasoningEffort") or item.get("effort") or item.get("level") or "").lower()
if isinstance(item, dict) else str(item).lower()
for item in supported
}
return frozenset(value for value in values if value in EFFORTS)
def is_eligible(
provider: str, model: str, metadata: dict[str, Any], effort: str | None
) -> tuple[bool, str]:
"""Check public/general-purpose status and optionally exact effort support."""
if metadata.get("hidden") is True:
return False, "provider marks model hidden"
if str(metadata.get("visibility") or "").lower() in {"hidden", "internal", "private"}:
@ -220,83 +323,132 @@ def _candidate_status(
return False, "provider description identifies a specialist model"
if any(word in model.lower() for word in SPECIALIST_MODEL_WORDS):
return False, "model ID identifies a specialist model"
if metadata.get("supportsEffort") is False:
if effort is not None and metadata.get("supportsEffort") is False:
return False, "provider does not advertise effort support"
supported = metadata.get(
"supported_reasoning_efforts",
metadata.get("supportedReasoningEfforts", metadata.get("supportedEffortLevels")),
)
supported_efforts = {
str(item.get("reasoningEffort") or item.get("effort") or "").lower()
if isinstance(item, dict)
else str(item).lower()
for item in supported
} if isinstance(supported, list) else set()
if supported_efforts and effort not in supported_efforts:
advertised_efforts = supported_efforts(metadata)
if effort is not None and advertised_efforts and effort not in advertised_efforts:
return False, f"provider does not support {effort} effort"
if provider == "codex" and not model.lower().startswith("gpt-"):
return False, "model ID is outside the Codex general route namespace"
return True, "eligible"
_candidate_status = is_eligible
def capability_pool(
provider: str, models: Iterable[str], metadata: dict[str, dict[str, Any]], role: str
) -> list[str]:
"""Return all public, general-purpose candidates for one capability role."""
if role not in CAPABILITY_ROLES:
return []
result: list[str] = []
for model in unique_models(models):
eligible, _ = is_eligible(provider, model, metadata.get(model, {}), None)
classified, _ = capability_role(provider, model, metadata.get(model, {}))
if eligible and classified == role:
result.append(model)
return result
def apply_verified_evaluations(
provider: str, models: Iterable[str], metadata: dict[str, dict[str, Any]],
evaluations: dict[str, Any] | None,
) -> dict[str, dict[str, Any]]:
"""Add only current verified evidence for still-advertised model records."""
advertised = set(unique_models(models))
result = {model: dict(metadata.get(model, {})) for model in advertised}
records = evaluations.get(provider, {}) if isinstance(evaluations, dict) else {}
if not isinstance(records, dict):
return result
for model, record in records.items():
if not isinstance(model, str) or model not in advertised or not isinstance(record, dict):
continue
if not (
record.get("proposed_role") in CAPABILITY_ROLES
and record.get("result") == "pass"
and record.get("role_fit") == "verified"
and record.get("eval_version") == EVALUATION_VERSION
and record.get("metadata_fingerprint") == metadata_fingerprint(model, result[model])
):
continue
result[model]["evaluated_capability_role"] = record["proposed_role"]
result[model]["evaluation_provenance"] = "bounded-representative-eval-v1"
return result
def select_tier_model(
provider: str, models: Iterable[str], metadata: dict[str, dict[str, Any]],
desired: str, effort: str, current: str, *, allow_current_fallback: bool = True,
legacy_compat: bool = False,
) -> tuple[str, dict[str, Any]]:
"""Select by declared tier and cost, retaining last-known-good only on outage."""
"""Select within one role, retaining current only during discovery outages."""
if desired not in CAPABILITY_ROLES:
raise ValueError(f"unsupported capability role: {desired}")
candidates: list[tuple[str, dict[str, Any], str]] = []
exact_declared = False
exact_available = False
observations: dict[str, Any] = {}
for model in unique_models(models):
details = metadata.get(model, {})
eligible, reason = _candidate_status(provider, model, details, effort)
tier, tier_reason = metadata_tier(details)
if tier is None:
tier, legacy_reason = legacy_tier(provider, model)
tier_reason = legacy_reason if tier else tier_reason
eligible, reason = is_eligible(provider, model, details, effort)
tier, tier_reason = capability_role(provider, model, details)
if tier is None and legacy_compat:
tier, tier_reason = legacy_tier(provider, model)
if tier == desired:
exact_declared = True
exact_available = exact_available or is_eligible(
provider, model, details, None
)[0]
proposed, proposed_reason = proposed_candidate_role(provider, model, details)
cost = _model_cost(details)
observations[model] = {
"eligible": eligible,
"reason": reason if not eligible else tier_reason,
"tier": tier,
"proposed_role": proposed,
"proposed_reason": proposed_reason,
"cost": {"unit": cost[0], "value": cost[1]} if cost else None,
}
if eligible and tier:
candidates.append((model, details, tier))
exact = [item for item in candidates if item[2] == desired]
tier_order = {"economy": 0, "balanced": 1, "advanced": 2}
# When an economy model cannot honor the requested effort, use the nearest
# higher declared tier. A difficult request never falls to a lower tier.
tier_order = {role: index for index, role in enumerate(CAPABILITY_ROLES)}
# Economy/balanced may use the nearest adequate non-frontier role when a
# provider cannot express an effort. Advanced and frontier remain strict:
# an automatic hard-task route must not quietly promote to Astra or demote.
higher = [
item for item in candidates
if tier_order[item[2]] > tier_order[desired]
if desired in {"economy", "balanced"}
and tier_order[item[2]] > tier_order[desired]
and item[2] != "frontier"
]
nearest_higher = min(
(tier_order[item[2]] for item in higher), default=None
)
# A fresh catalog that still declares the desired role but marks every
# member unavailable is a provider-side availability signal. Do not turn
# that into a more capable route; a caller may retain last-known-good only
# when its whole discovery read was unavailable. Promotion remains valid
# when no such role exists, or when an available exact role lacks only the
# requested effort support.
permit_higher = not exact_declared or exact_available
pool = exact or (
[item for item in higher if tier_order[item[2]] == nearest_higher]
if nearest_higher is not None else []
if permit_higher and nearest_higher is not None else []
)
if not pool:
return (current if allow_current_fallback else ""), observations
comparable_costs = _comparable_costs(pool)
def key(item: tuple[str, dict[str, Any], str]) -> tuple[int, int, float, int, int, tuple[int, ...], str]:
def key(item: tuple[str, dict[str, Any], str]) -> tuple[int, float, int, str]:
model, details, tier = item
text = _text_metadata(details)
strongest = any(term in text for term in ("strongest", "most capable", "most intelligent"))
return (
0 if tier == desired else 1,
0 if strongest else 1,
comparable_costs.get(model, float("inf")),
# A provider moving its default is a current release signal. It
# supersedes an old advanced route, while current protects equal
# economy/balanced candidates where no comparable cost is known.
0 if desired == "advanced" and (
details.get("isDefault") is True or details.get("is_default") is True
) else 1,
# A verified evaluation establishes role fit; cost decides between
# adequate peers when provider metadata supplies one common unit.
0 if model == current else 1,
tuple(-part for part in model_version(model)),
model,
)
return min(pool, key=key)[0], observations

25
services/hermes/scripts/routing_catalog.py
View File

@ -27,6 +27,7 @@ DEFAULTS = {
},
}
PREFIXES = {"codex": "gpt-", "claude": "claude-"}
CAPABILITY_ROLES = frozenset({"economy", "balanced", "advanced", "frontier"})
TIER_DEFAULTS = {
"codex": {
"luna": "gpt-5.6-luna",
@ -71,7 +72,7 @@ def resolve_model(
raise ValueError("unsupported provider or effort")
prefix = PREFIXES[provider]
document = catalog if catalog is not None else load_catalog()
if selector.startswith(prefix):
if selector.startswith(prefix) or catalog_contains(provider, selector, document):
if catalog_contains(provider, selector, document):
return selector
providers = document.get("providers", {}) if isinstance(document, dict) else {}
@ -84,25 +85,33 @@ def resolve_model(
record = providers.get(provider, {}) if isinstance(providers, dict) else {}
if not isinstance(record, dict):
record = {}
mapping_name = "resolved" if selector == "auto" else "tiers"
mapping = record.get(mapping_name, {})
candidate = mapping.get(effort if selector == "auto" else selector, "") if isinstance(mapping, dict) else ""
capability = selector.removeprefix("auto-") if selector.startswith("auto-") else ""
if capability and capability not in CAPABILITY_ROLES:
raise ValueError(f"unsupported capability selector {selector!r}")
if capability:
mappings = record.get("capability_resolved", {})
mapping = mappings.get(capability, {}) if isinstance(mappings, dict) else {}
candidate = mapping.get(effort, "") if isinstance(mapping, dict) else ""
else:
mapping_name = "resolved" if selector == "auto" else "tiers"
mapping = record.get(mapping_name, {})
candidate = mapping.get(effort if selector == "auto" else selector, "") if isinstance(mapping, dict) else ""
if (
selector != "auto"
and not capability
and isinstance(candidate, str)
and candidate
and selector not in candidate.lower()
):
candidate = ""
advertised = record.get("models", []) if isinstance(record, dict) else []
candidate_allowed = isinstance(candidate, str) and (
candidate.startswith(prefix)
or (provider == "claude" and candidate in advertised)
candidate_allowed = isinstance(candidate, str) and candidate and (
candidate in advertised or (not advertised and candidate.startswith(prefix))
)
if not candidate_allowed:
# A fresh account list is authoritative. Do not turn a deliberately
# unresolved/retired selector into a stale baseline model.
if record.get("live") is True:
if record.get("live") is True or capability:
raise ValueError(f"no current {provider} model for selector {selector!r}")
candidate = (
DEFAULTS[provider][effort]

120
services/hermes/scripts/seed_legacy_scm_roots.py Normal file
View File

@ -0,0 +1,120 @@
#!/usr/bin/env python3
"""Seed verified, native roots for the reviewed legacy pull requests."""
from __future__ import annotations
from dataclasses import dataclass
import json
import re
from typing import Any, Callable
import scm_broker_client
import supervisor_state
from supervisor_lineage import Lineage
SHA = re.compile(r"[0-9a-f]{40}\Z")
@dataclass(frozen=True)
class Root:
"""One reviewed PR whose existing branch may be continued in place."""
board: str
root_task_id: str
project: str
ref: str
pr_number: int
base: str
head: str
@property
def pull_request(self) -> str:
return f"https://scm.bstein.dev/titan/{self.project}/pulls/{self.pr_number}"
@property
def lineage(self) -> Lineage:
return Lineage(self.root_task_id, self.ref, self.pull_request, self.project, self.base)
def adoption(self) -> dict[str, Any]:
"""Return the exact schema consumed by the broker's Flux registry."""
return {"repo": self.project, "ref": self.ref, "board": self.board,
"root_task_id": self.root_task_id, "latest_head": self.head,
"pr_number": self.pr_number}
# This reviewed registry mirrors task-branch-adoptions-configmap.yaml. The
# regression test keeps the board-side lineage seed and broker ownership ledger
# on the same exact PR/ref/head set.
ROOTS = (
Root("soteria", "t_f1593f8c", "soteria", "wt/t_f1593f8c", 11, "main", "0143d472469c8dfe44f23f1440123e27d415baae"),
Root("soteria", "t_c7c42600", "soteria", "hermes-repair/sonar-AZ9pTqVcN0JrBQvDGDs3", 3, "main", "51133b559ad62f324e45bc61587900f08156b733"),
Root("soteria", "t_f4f726e1", "soteria", "hermes-repair/sonar-AZ9pTqWRN0JrBQvDGDs4", 4, "main", "45170df566c2aa387f8b706c3a704abce5aa2557"),
Root("titan-iac", "t_26da4c88", "atlas-iac", "feature/t_26da4c88-titan-capacity-guardrails-v4", 53, "main", "3c6301d581bef0a1fce5284f9a28c1cf4c4a99ad"),
Root("titan-iac", "t_39cf1905", "atlas-iac", "feature/t_39cf1905-webui-build-token", 54, "main", "f94f96a7042ba863768938e0c1afa79406397b77"),
Root("titan-iac", "t_e9597d89", "atlas-iac", "feature/hermes-three-lane-placement", 17, "main", "48cbe13ee50ce3fcb07cea3fe8d088cfed349e0c"),
Root("titan-iac", "t_a6a22d7c", "atlas-iac", "feature/hermes-cli-process-reaping", 20, "main", "a242dcc786576ae1a18000cb4941836ff610dff2"),
Root("titan-iac", "t_c425c446", "atlas-iac", "fix/t_39cf1905-jenkins-controller-priority", 49, "main", "f997171b5526f104d2474022c2a56683ec2960c3"),
Root("titan-iac", "t_cf89a2ec", "atlas-iac", "feature/hermes-next-hux", 55, "main", "98c7c6184f6edfe3cdac228529c2287584db3006"),
Root("cassandra", "t_b89e3903", "cassandra", "handoff/generated-strategy-audit-20260813", 1, "main", "14c07111b6ed8d7fc529362eb34fa0afa0694325"),
)
def _live_head(root: Root, read: Callable[[str], bytes]) -> str:
"""Return a live canonical PR head, or an empty string on any mismatch."""
try:
pull = json.loads(read(f"/api/v1/repos/titan/{root.project}/pulls/{root.pr_number}"))
except (OSError, TypeError, ValueError):
return ""
if not isinstance(pull, dict):
return ""
head, base = pull.get("head"), pull.get("base")
canonical = f"titan/{root.project}"
valid = (
pull.get("state") == "open" and isinstance(head, dict) and isinstance(base, dict)
and head.get("ref") == root.ref and isinstance(head.get("repo"), dict)
and head["repo"].get("full_name") == canonical and base.get("ref") == root.base
and isinstance(base.get("repo"), dict) and base["repo"].get("full_name") == canonical
)
value = head.get("sha") if isinstance(head, dict) else ""
return value if valid and isinstance(value, str) and SHA.fullmatch(value) else ""
def seed_root(kanban_db: Any, root: Root, read: Callable[[str], bytes]) -> str:
"""Seed one absent state row after proving both native task and live PR."""
with kanban_db.scoped_current_board(root.board):
connection = kanban_db.connect(board=root.board)
try:
if kanban_db.get_task(connection, root.root_task_id) is None:
return "missing-task"
finally:
connection.close()
existing = supervisor_state.get_root(root.board, root.root_task_id)
if existing is not None:
return "already-seeded" if existing == root.lineage else "lineage-conflict"
if _live_head(root, read) != root.head:
return "live-pr-mismatch"
supervisor_state.record_submission(root.board, root.root_task_id, root.lineage, root.head)
return "seeded"
def run(kanban_db: Any, read: Callable[[str], bytes]) -> dict[str, int]:
"""Seed each independent root and return compact operator-visible counts."""
counts: dict[str, int] = {}
for root in ROOTS:
outcome = seed_root(kanban_db, root, read)
counts[outcome] = counts.get(outcome, 0) + 1
return counts
def main() -> int:
"""Run the explicit operator migration without changing nonmatching state."""
from hermes_cli import kanban_db
counts = run(kanban_db, scm_broker_client.read)
print(json.dumps(counts, sort_keys=True))
failures = {"missing-task", "lineage-conflict", "live-pr-mismatch"}
return 0 if not failures & counts.keys() else 1
if __name__ == "__main__":
raise SystemExit(main())

2
services/hermes/scripts/stage_runtime_access.py
View File

@ -323,6 +323,8 @@ def stage_execution_mediator() -> None:
hashlib.sha256,
).hexdigest()
_write_secret(POOL_ACCESS_ROOT / "execution-pool-key", derived)
# This is the domain-separated broker HMAC, not the worker's assignment key.
_write_secret(POOL_ACCESS_ROOT / "scm-task-grant-key", _read_secret("scm-task-grant-key"))
def main() -> int:

74
services/hermes/scripts/supervisor_lineage.py Normal file
View File

@ -0,0 +1,74 @@
#!/usr/bin/env python3
"""Immutable SCM lineage carried by supervised Kanban follow-up cards."""
from __future__ import annotations
import json
from dataclasses import dataclass
from typing import Any
@dataclass(frozen=True)
class Lineage:
"""Trusted root, project, branch, and pull request for one change chain."""
root_task_id: str
branch: str
pull_request: str
project: str = ""
base_branch: str = ""
def stamp_fields(self) -> dict[str, str]:
"""Return immutable fields safe to embed in a supervisor stamp."""
return {"root_task_id": self.root_task_id, "branch": self.branch,
"pull_request": self.pull_request, "project": self.project,
"base_branch": self.base_branch}
def _mapping(value: Any) -> dict[str, Any]:
if isinstance(value, dict):
return value
if isinstance(value, str):
try:
parsed = json.loads(value)
except (TypeError, ValueError):
return {}
return parsed if isinstance(parsed, dict) else {}
return {}
def _text(source: dict[str, Any], *keys: str) -> str:
for key in keys:
value = source.get(key)
if isinstance(value, str) and value.strip():
return value.strip()
return ""
def initial(task: Any) -> Lineage | None:
"""Read only coordinator-written assignment metadata, never worker prose."""
raw = task.get("metadata") if isinstance(task, dict) else getattr(task, "metadata", None)
meta = _mapping(raw)
sources = (_mapping(meta.get("supervisor_lineage")), _mapping(meta.get("assignment")))
task_id = str(task.get("id") if isinstance(task, dict) else getattr(task, "id", "") or "")
for source in sources:
branch = _text(source, "branch", "branch_name", "head_branch")
pull = _text(source, "pull_request", "pr_url", "pr", "merge_request")
base = _text(source, "base_branch")
if branch and pull and base:
root = _text(source, "root_task_id") or task_id
if root:
return Lineage(root, branch, pull, _text(source, "project", "repository", "repo"), base)
return None
def from_stamp(value: Any) -> Lineage | None:
"""Validate immutable continuation fields written by the supervisor itself."""
source = _mapping(value)
root = _text(source, "root_task_id")
branch = _text(source, "branch")
pull = _text(source, "pull_request")
base = _text(source, "base_branch")
if not root or not branch or not pull or not base or _text(source, "root") != root:
return None
return Lineage(root, branch, pull, _text(source, "project"), base)

175
services/hermes/scripts/supervisor_policy.py
View File

@ -1,28 +1,5 @@
#!/usr/bin/env python3
"""Pure decision logic for the in-pod autonomous Kanban supervisor.
This module owns only *pure* policy: given one candidate task and a snapshot of
its board, it returns the single follow-up action the supervisor should take, a
fail-closed escalation, or nothing. It performs no I/O and imports no provider
client, so the whole cross-card implement->review->repair->re-review state
machine is unit testable and can never, by construction, emit a metered/model
call or a merge/approve/close/deploy. Every side effect (create_task,
block_task, add_comment) lives in :mod:`kanban_supervisor`.
Safety contract enforced here (audited hard - it drives an autonomous loop):
* FAIL CLOSED - an unparseable result, an ambiguous SHIP/BLOCK verdict, a done
implementation whose produced commit/PR cannot be identified, or a repair that
produced no new commit yields an ``escalate`` decision (human attention),
never an auto-spawn and never an inferred SHIP.
* NO DUPLICATE - a review or repair is proposed only when no card already covers
the same ``(root, head_commit)``, detected across supervisor-stamped *and*
externally created cards (the codex-shepherd), so it is concurrency-safe.
* BOUNDED - a per-chain review<->repair cycle limit and a max-concurrent-chains
ceiling turn runaway loops into human escalations instead of capacity burn.
* SUBSCRIPTION ONLY - the only spawn is a Kanban card on the existing cli-*
subscription lanes; no provider/API-key field is ever produced.
"""
"""Pure, fail-closed policy for bounded implementation-review-repair chains."""
from __future__ import annotations
@ -31,38 +8,31 @@ from dataclasses import dataclass
from typing import Any
import cli_lane_goal
import supervisor_lineage as lineage
SUPERVISOR_AUTHOR = "hermes-supervisor"
STAMP_KEY = "supervisor"
REVIEW_KIND = "review"
REPAIR_KIND = "repair"
DONE_STATUSES = frozenset({"done"})
# A supervised card whose status is one of these is no longer an in-flight chain
# link (done = finished, blocked = already escalated to a human).
INACTIVE_STATUSES = frozenset({"done", "blocked"})
COMMIT_KEYS = ("head_commit", "commit_sha", "commit", "sha", "revision")
PR_KEYS = ("pull_request", "pr_url", "pr", "merge_request")
BRANCH_KEYS = ("branch", "branch_name", "head_branch")
SAFE_ACTIONS = frozenset({"none", "spawn", "ship", "escalate"})
# Keys that would signal a privileged/metered action; a payload must never carry
# one. Enforced by :func:`assert_safe` before any card is written.
SAFE_ACTIONS = frozenset({"none", "spawn", "ship", "clear_ready", "escalate"})
FORBIDDEN_PAYLOAD_KEYS = frozenset(
{"merge", "approve", "close", "deploy", "provider", "model", "api_key", "metered"}
)
@dataclass(frozen=True)
class Limits:
"""Bounds and routing that gate every autonomous decision."""
max_cycles: int = 5
max_chains: int = 20
review_assignee: str = "cli-claude-xhigh"
review_assignee: str = "cli-auto"
repair_assignee: str = "cli-auto"
@dataclass(frozen=True)
class Decision:
"""One resolved supervisor intent. ``action`` is always in SAFE_ACTIONS."""
@ -71,8 +41,6 @@ class Decision:
reason: str = ""
target_id: str = ""
payload: dict[str, Any] | None = None
def assert_safe(decision: Decision) -> Decision:
"""Reject any decision outside the allow-listed, non-metered action set."""
if decision.action not in SAFE_ACTIONS:
@ -82,14 +50,10 @@ def assert_safe(decision: Decision) -> Decision:
if key in payload:
raise ValueError(f"supervisor payload carries forbidden key {key!r}")
return decision
def field(task: Any, name: str, default: Any = None) -> Any:
if isinstance(task, dict):
return task.get(name, default)
return getattr(task, name, default)
def _as_dict(value: Any) -> dict[str, Any] | None:
if isinstance(value, dict):
return value
@ -100,36 +64,22 @@ def _as_dict(value: Any) -> dict[str, Any] | None:
return None
return parsed if isinstance(parsed, dict) else None
return None
def task_id(task: Any) -> str:
return str(field(task, "id", "") or "")
def metadata(task: Any) -> dict[str, Any]:
return _as_dict(field(task, "metadata")) or {}
def stamp(task: Any) -> dict[str, Any]:
value = metadata(task).get(STAMP_KEY)
return value if isinstance(value, dict) else {}
def supervised_kind(task: Any) -> str | None:
kind = stamp(task).get("kind")
return kind if kind in (REVIEW_KIND, REPAIR_KIND) else None
def is_done(task: Any) -> bool:
return str(field(task, "status", "") or "") in DONE_STATUSES
def objective(task: Any) -> str:
title = str(field(task, "title", "") or "")
body = str(field(task, "body", "") or "")
return f"{title}\n\n{body}".strip()
def parents(task: Any) -> list[str]:
raw = field(task, "parents")
if raw is None:
@ -146,8 +96,6 @@ def parents(task: Any) -> list[str]:
if ident:
ids.append(str(ident))
return ids
def parse_result(task: Any) -> tuple[dict[str, Any] | None, str | None]:
"""Parse the terminal result, failing closed on anything non-object."""
raw = field(task, "result")
@ -157,8 +105,6 @@ def parse_result(task: Any) -> tuple[dict[str, Any] | None, str | None]:
if parsed is None:
return None, "task result is not a JSON object"
return parsed, None
def _first_key(source: Any, keys: tuple[str, ...]) -> str | None:
if not isinstance(source, dict):
return None
@ -167,8 +113,6 @@ def _first_key(source: Any, keys: tuple[str, ...]) -> str | None:
if isinstance(value, str) and value.strip():
return value.strip()
return None
def _result_sources(task: Any, result: dict[str, Any] | None) -> list[dict[str, Any]]:
sources = [metadata(task)]
if isinstance(result, dict):
@ -177,8 +121,6 @@ def _result_sources(task: Any, result: dict[str, Any] | None) -> list[dict[str,
if nested is not None:
sources.append(nested)
return sources
def extract_commit(task: Any, result: dict[str, Any] | None) -> str | None:
for source in _result_sources(task, result):
found = _first_key(source, COMMIT_KEYS)
@ -253,11 +195,7 @@ def _looks_like_repair(task: Any) -> bool:
def existing_followup(
tasks: list[Any], kind: str, root_id: str, head_commit: str
) -> bool:
"""True if any card already covers ``(root_id, head_commit)`` for ``kind``.
Matches both supervisor-stamped cards and externally created ones (the
external shepherd), so concurrent operation never double-spawns.
"""
"""True if any supervisor or external card already covers this revision."""
looks_like = _looks_like_review if kind == REVIEW_KIND else _looks_like_repair
for task in tasks:
st = stamp(task)
@ -290,22 +228,21 @@ def active_chain_count(tasks: list[Any]) -> int:
return len(roots)
def _stamp_meta(kind: str, root: str, source: str, head_commit: str, cycle: int) -> dict:
return {
STAMP_KEY: {
def _stamp_meta(
kind: str, root: str, source: str, head_commit: str, cycle: int, chain: lineage.Lineage
) -> dict:
stamped = {
"kind": kind,
"root": root,
"parent": source,
"head_commit": head_commit,
"cycle": cycle,
}
**chain.stamp_fields(),
}
return {STAMP_KEY: stamped}
def _idempotency(kind: str, root: str, head_commit: str, cycle: int) -> str:
return f"supervisor:{kind}:{root}:{head_commit}:{cycle}"
def _findings_block(findings: list[str]) -> str:
if not findings:
return ""
@ -318,16 +255,16 @@ def _build_review(
source_id: str,
head_commit: str,
cycle: int,
pr_ref: str,
chain: lineage.Lineage,
findings: list[str],
assignee: str,
) -> dict[str, Any]:
meta = _stamp_meta(REVIEW_KIND, root_id, source_id, head_commit, cycle)
meta = _stamp_meta(REVIEW_KIND, root_id, source_id, head_commit, cycle, chain)
meta["task_role"] = cli_lane_goal.REVIEW_ROLE
body = (
"Hermes-Task-Role: review\n\n"
f"Read-only review of commit {head_commit}"
f"{f' on {pr_ref}' if pr_ref else ''}. Do not modify the implementation; "
f"Read-only review of commit {head_commit} on {chain.pull_request}, branch "
f"{chain.branch}. Do not modify the implementation; "
"make no code changes. Report a single SHIP or BLOCK verdict with "
"evidence." + _findings_block(findings)
)
@ -339,6 +276,7 @@ def _build_review(
"parents": linked,
"metadata": meta,
"idempotency_key": _idempotency(REVIEW_KIND, root_id, head_commit, cycle),
"initial_status": "running",
}
@ -349,13 +287,14 @@ def _build_repair(
cycle: int,
findings: list[str],
assignee: str,
chain: lineage.Lineage,
) -> dict[str, Any]:
meta = _stamp_meta(REPAIR_KIND, root_id, review_id, head_commit, cycle)
meta = _stamp_meta(REPAIR_KIND, root_id, review_id, head_commit, cycle, chain)
meta["task_role"] = cli_lane_goal.IMPLEMENTATION_ROLE
body = (
"Hermes-Task-Role: repair\n\n"
f"Repair the implementation reviewed at commit {head_commit} per the "
"review findings below, then commit and push the fix."
f"Repair commit {head_commit} on existing PR {chain.pull_request}, branch "
f"{chain.branch}; do not create a branch or PR. Commit and push the fix."
+ _findings_block(findings)
)
return {
@ -365,9 +304,47 @@ def _build_repair(
"parents": [root_id, review_id],
"metadata": meta,
"idempotency_key": _idempotency(REPAIR_KIND, root_id, head_commit, cycle),
"initial_status": "running",
}
def _latest_head(tasks: list[Any], chain: lineage.Lineage) -> str:
"""Return the newest verified repair head, never treating base drift as stale."""
newest = ""
newest_cycle = 0
for candidate in tasks:
if task_id(candidate) == chain.root_task_id:
root_meta = metadata(candidate)
value = _first_key(root_meta, ("live_pr_head", "latest_head_commit"))
if value:
return value
candidate_stamp = stamp(candidate)
if supervised_kind(candidate) != REPAIR_KIND or lineage.from_stamp(candidate_stamp) != chain:
continue
result, error = parse_result(candidate)
commit = None if error else extract_commit(candidate, result)
cycle = _int(candidate_stamp.get("cycle"), 0)
if commit and cycle >= newest_cycle:
newest, newest_cycle = commit, cycle
return newest
def _validated_chain(task: Any, tasks: list[Any]) -> lineage.Lineage | None:
"""Accept child lineage only when its root and stamped parent corroborate it."""
child_stamp = stamp(task)
chain = lineage.from_stamp(child_stamp)
if chain is None:
return None
root = next((item for item in tasks if task_id(item) == chain.root_task_id), None)
if root is None or lineage.initial(root) != chain:
return None
parent_id = str(child_stamp.get("parent") or "")
if parent_id == chain.root_task_id:
return chain
parent = next((item for item in tasks if task_id(item) == parent_id), None)
return chain if parent is not None and lineage.from_stamp(stamp(parent)) == chain else None
def plan_implementation(task: Any, tasks: list[Any], limits: Limits) -> Decision:
tid = task_id(task)
result, error = parse_result(task)
@ -379,7 +356,7 @@ def plan_implementation(task: Any, tasks: list[Any], limits: Limits) -> Decision
if error is not None:
return Decision("escalate", f"implementation result unparseable: {error}", tid)
commit = extract_commit(task, result)
pr = extract_pr(task, result)
chain = lineage.initial(task)
if commit is None:
if _has_changes(result):
return Decision(
@ -388,17 +365,19 @@ def plan_implementation(task: Any, tasks: list[Any], limits: Limits) -> Decision
tid,
)
return Decision("none", "implementation produced no commit to review")
if pr is None:
if chain is None:
return Decision(
"escalate",
"implementation head commit present but no branch/PR reference to review",
"implementation lacks trusted branch/PR lineage from its assignment payload",
tid,
)
if chain.root_task_id != tid:
return Decision("escalate", "initial assignment root_task_id does not match task", tid)
if existing_followup(tasks, REVIEW_KIND, tid, commit):
return Decision("none", "a review already covers this commit")
if active_chain_count(tasks) >= limits.max_chains:
return Decision("none", "supervised-chain ceiling reached; deferring new review")
payload = _build_review(tid, tid, commit, 1, pr, [], limits.review_assignee)
payload = _build_review(tid, tid, commit, 1, chain, [], limits.review_assignee)
return Decision("spawn", f"open review for {commit}", tid, payload)
@ -408,7 +387,8 @@ def plan_review(task: Any, tasks: list[Any], limits: Limits) -> Decision:
root_id = str(st.get("root") or "")
head_commit = str(st.get("head_commit") or "")
cycle = _int(st.get("cycle"), 0)
if not root_id or not head_commit or cycle <= 0:
chain = _validated_chain(task, tasks)
if not root_id or not head_commit or cycle <= 0 or chain is None or chain.root_task_id != root_id:
return Decision(
"escalate", "review card carries an incomplete supervisor stamp", review_id
)
@ -420,11 +400,21 @@ def plan_review(task: Any, tasks: list[Any], limits: Limits) -> Decision:
detail = problem or "no explicit SHIP or BLOCK verdict"
return Decision("escalate", f"review verdict ambiguous: {detail}", review_id)
if verdict == "SHIP":
current = _latest_head(tasks, chain)
if current and current != head_commit:
return Decision(
"clear_ready",
f"stale SHIP for {head_commit}; current verified chain head is {current}",
root_id,
{"commit": current, "stale_commit": head_commit},
)
return Decision(
"ship",
f"review shipped {head_commit}",
root_id,
{"commit": head_commit, "pr": extract_pr(task, result) or ""},
{"commit": head_commit, "pr": chain.pull_request, "branch": chain.branch,
"project": chain.project, "base_branch": chain.base_branch,
"root_task_id": chain.root_task_id},
)
# BLOCK -> spawn a bounded repair, unless one exists or the budget is spent.
if cycle >= limits.max_cycles:
@ -440,7 +430,7 @@ def plan_review(task: Any, tasks: list[Any], limits: Limits) -> Decision:
return Decision("none", "a repair already covers this commit")
findings = _strings(result.get("findings"))
payload = _build_repair(
root_id, review_id, head_commit, cycle, findings, limits.repair_assignee
root_id, review_id, head_commit, cycle, findings, limits.repair_assignee, chain
)
return Decision("spawn", f"open repair for {head_commit}", review_id, payload)
@ -451,7 +441,8 @@ def plan_repair(task: Any, tasks: list[Any], limits: Limits) -> Decision:
root_id = str(st.get("root") or "")
old_commit = str(st.get("head_commit") or "")
cycle = _int(st.get("cycle"), 0)
if not root_id or cycle <= 0:
chain = _validated_chain(task, tasks)
if not root_id or cycle <= 0 or chain is None or chain.root_task_id != root_id:
return Decision(
"escalate", "repair card carries an incomplete supervisor stamp", repair_id
)
@ -471,7 +462,6 @@ def plan_repair(task: Any, tasks: list[Any], limits: Limits) -> Decision:
)
next_cycle = cycle + 1
if next_cycle > limits.max_cycles:
# Escalate the repair card itself, not root_id (see plan_review).
return Decision(
"escalate",
f"repair cycle limit ({limits.max_cycles}) reached for chain {root_id}; "
@ -480,13 +470,10 @@ def plan_repair(task: Any, tasks: list[Any], limits: Limits) -> Decision:
)
if existing_followup(tasks, REVIEW_KIND, root_id, new_commit):
return Decision("none", "a re-review already covers this commit")
pr = extract_pr(task, result) or ""
payload = _build_review(
root_id, repair_id, new_commit, next_cycle, pr, [], limits.review_assignee
root_id, repair_id, new_commit, next_cycle, chain, [], limits.review_assignee
)
return Decision("spawn", f"open re-review for {new_commit}", repair_id, payload)
def plan(task: Any, tasks: list[Any], limits: Limits) -> Decision:
"""Resolve the single follow-up action for one terminal card."""
if not is_done(task):

265
services/hermes/scripts/supervisor_state.py Normal file
View File

@ -0,0 +1,265 @@
#!/usr/bin/env python3
"""Board-local integrity records for Hermes PR continuation cards."""
from __future__ import annotations
import hashlib
import os
import sqlite3
from pathlib import Path
from typing import Any
from supervisor_lineage import Lineage
KANBAN_ROOT = Path("/opt/data/kanban/boards")
class SupervisorStateError(ValueError):
"""A present integrity record cannot be decoded or read safely."""
def _path(board: str, path: Path | None) -> Path:
"""Keep integrity rows beside the board's real Kanban transaction store."""
if path is not None:
return path
if not board or "/" in board or "\\" in board:
raise ValueError("board name is invalid")
return KANBAN_ROOT / board / "kanban.db"
def _connect(board: str, path: Path | None = None) -> sqlite3.Connection:
path = _path(board, path)
if not path.exists() and path is None:
raise ValueError("Kanban board database does not exist")
path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
connection = sqlite3.connect(path)
connection.execute("PRAGMA journal_mode=WAL")
connection.execute("PRAGMA foreign_keys=ON")
connection.executescript(
"""
CREATE TABLE IF NOT EXISTS supervisor_roots (
board TEXT NOT NULL, root_task_id TEXT NOT NULL, project TEXT NOT NULL,
branch TEXT NOT NULL, pull_request TEXT NOT NULL, base_branch TEXT NOT NULL,
live_pr_head TEXT NOT NULL, ready_for_human_merge INTEGER NOT NULL DEFAULT 0,
ready_commit TEXT NOT NULL DEFAULT '', PRIMARY KEY (board, root_task_id)
);
CREATE TABLE IF NOT EXISTS supervisor_children (
board TEXT NOT NULL, child_task_id TEXT NOT NULL, root_task_id TEXT NOT NULL,
parent_task_id TEXT NOT NULL, kind TEXT NOT NULL, head_commit TEXT NOT NULL,
objective_digest TEXT NOT NULL, cycle INTEGER NOT NULL DEFAULT 1,
PRIMARY KEY (board, child_task_id),
UNIQUE (board, root_task_id, head_commit, objective_digest)
);
"""
)
columns = {row[1] for row in connection.execute("PRAGMA table_info(supervisor_roots)")}
if "ready_for_human_merge" not in columns:
connection.execute(
"ALTER TABLE supervisor_roots ADD COLUMN ready_for_human_merge INTEGER NOT NULL DEFAULT 0"
)
if "ready_commit" not in columns:
connection.execute("ALTER TABLE supervisor_roots ADD COLUMN ready_commit TEXT NOT NULL DEFAULT ''")
child_columns = {row[1] for row in connection.execute("PRAGMA table_info(supervisor_children)")}
if "cycle" not in child_columns:
connection.execute("ALTER TABLE supervisor_children ADD COLUMN cycle INTEGER NOT NULL DEFAULT 1")
try:
if path.name == "state.db":
os.chmod(path, 0o600)
except OSError:
pass
return connection
def _lineage(row: tuple[Any, ...] | None) -> Lineage | None:
if row is None:
return None
try:
values = tuple(str(value) for value in row[:5])
except (TypeError, ValueError):
return None
if not all(values):
return None
return Lineage(*values)
def get_root(board: str, root_task_id: str, *, path: Path | None = None) -> Lineage | None:
"""Return one coordinator-issued root lineage, never task-body claims."""
try:
with _connect(board, path) as connection:
row = connection.execute(
"SELECT root_task_id, branch, pull_request, project, base_branch "
"FROM supervisor_roots WHERE board=? AND root_task_id=?", (board, root_task_id)
).fetchone()
except (OSError, sqlite3.Error) as error:
raise SupervisorStateError("supervisor root state is unreadable") from error
if row is None:
return None
lineage = _lineage(row)
if lineage is None:
raise SupervisorStateError("supervisor root state is malformed")
return lineage
def get_live_head(board: str, root_task_id: str, *, path: Path | None = None) -> str:
"""Return the broker-confirmed head associated with a trusted root."""
try:
with _connect(board, path) as connection:
row = connection.execute(
"SELECT live_pr_head FROM supervisor_roots WHERE board=? AND root_task_id=?",
(board, root_task_id),
).fetchone()
except (OSError, sqlite3.Error) as error:
raise SupervisorStateError("supervisor root state is unreadable") from error
if row is None:
return ""
if not isinstance(row[0], str) or not row[0]:
raise SupervisorStateError("supervisor root state is malformed")
return row[0]
def get_child(board: str, child_task_id: str, *, path: Path | None = None) -> dict[str, Any] | None:
"""Return the only durable authority record for a continuation child."""
try:
with _connect(board, path) as connection:
row = connection.execute(
"SELECT root_task_id, parent_task_id, kind, head_commit, objective_digest, cycle "
"FROM supervisor_children WHERE board=? AND child_task_id=?", (board, child_task_id)
).fetchone()
except (OSError, sqlite3.Error) as error:
raise SupervisorStateError("supervisor child state is unreadable") from error
if row is None:
return None
root, parent, kind, head, digest, cycle = (str(value) for value in row)
lineage = get_root(board, root, path=path)
if lineage is None or kind not in {"repair", "review"} or not all((parent, head, digest)):
raise SupervisorStateError("supervisor child state is malformed")
try:
parsed_cycle = int(cycle)
except (TypeError, ValueError) as error:
raise SupervisorStateError("supervisor child state is malformed") from error
if parsed_cycle < 1:
raise SupervisorStateError("supervisor child state is malformed")
return {"lineage": lineage, "root_task_id": root, "parent_task_id": parent,
"kind": kind, "head_commit": head, "objective_digest": digest,
"cycle": parsed_cycle}
def record_submission(
board: str, task_id: str, lineage: Lineage, live_pr_head: str, *, path: Path | None = None
) -> None:
"""Record a coordinator-verified submission against its immutable root.
``task_id`` identifies the completing root or child. The coordinator has
already authenticated that task's signed assignment before calling this
function; the record deliberately remains keyed by ``root_task_id``.
"""
if not board or not task_id or not live_pr_head:
raise ValueError("submission does not name a task and live head")
with _connect(board, path) as connection:
existing = connection.execute(
"SELECT project,branch,pull_request,base_branch,live_pr_head FROM supervisor_roots "
"WHERE board=? AND root_task_id=?", (board, lineage.root_task_id)
).fetchone()
identity = (lineage.project, lineage.branch, lineage.pull_request, lineage.base_branch)
if existing is None:
connection.execute(
"INSERT INTO supervisor_roots(board,root_task_id,project,branch,pull_request,base_branch,live_pr_head) "
"VALUES(?,?,?,?,?,?,?)",
(board, lineage.root_task_id, *identity, live_pr_head),
)
elif tuple(existing[:4]) != identity:
raise ValueError("submission attempts to rewrite immutable root lineage")
elif existing[4] != live_pr_head:
connection.execute(
"UPDATE supervisor_roots SET live_pr_head=?, ready_for_human_merge=0, ready_commit='' "
"WHERE board=? AND root_task_id=?",
(live_pr_head, board, lineage.root_task_id),
)
def record_live_head(
board: str, root_task_id: str, live_pr_head: str, *, path: Path | None = None
) -> None:
"""Refresh a root's head only after the continuation CLI verified its PR."""
if not board or not root_task_id or not live_pr_head:
raise ValueError("root and live head are required")
with _connect(board, path) as connection:
changed = connection.execute(
"UPDATE supervisor_roots SET live_pr_head=?, ready_for_human_merge=0, ready_commit='' "
"WHERE board=? AND root_task_id=?", (live_pr_head, board, root_task_id)
).rowcount
if changed != 1:
raise ValueError("unknown continuation root")
def set_ready(
board: str, root_task_id: str, head_commit: str, *, path: Path | None = None
) -> None:
"""Publish human-ready state only for the verified current root head."""
with _connect(board, path) as connection:
changed = connection.execute(
"UPDATE supervisor_roots SET ready_for_human_merge=1, ready_commit=? "
"WHERE board=? AND root_task_id=? AND live_pr_head=?",
(head_commit, board, root_task_id, head_commit),
).rowcount
if changed != 1:
raise ValueError("ready head is not the trusted live root head")
def clear_ready(board: str, root_task_id: str, *, path: Path | None = None) -> None:
"""Invalidate approval whenever a continuation is queued or head changes."""
with _connect(board, path) as connection:
connection.execute(
"UPDATE supervisor_roots SET ready_for_human_merge=0, ready_commit='' "
"WHERE board=? AND root_task_id=?", (board, root_task_id)
)
def record_child(
board: str, child_task_id: str, root_task_id: str, parent_task_id: str, kind: str,
head_commit: str, objective: str, cycle: int = 1, *, path: Path | None = None
) -> None:
"""Bind a created child to an existing trusted root and exact continuation."""
if not get_root(board, root_task_id, path=path) or kind not in {"repair", "review"}:
raise ValueError("continuation root or kind is invalid")
digest = objective_digest(objective)
if cycle < 1:
raise ValueError("continuation cycle is invalid")
values = (root_task_id, parent_task_id, kind, head_commit, digest, cycle)
with _connect(board, path) as connection:
existing = connection.execute(
"SELECT root_task_id,parent_task_id,kind,head_commit,objective_digest,cycle "
"FROM supervisor_children WHERE board=? AND child_task_id=?", (board, child_task_id)
).fetchone()
if existing is None:
collision = connection.execute(
"SELECT child_task_id FROM supervisor_children WHERE board=? AND root_task_id=? "
"AND head_commit=? AND objective_digest=?",
(board, root_task_id, head_commit, digest),
).fetchone()
if collision is not None:
raise ValueError("continuation already maps this root/head/objective to another task")
connection.execute(
"INSERT INTO supervisor_children VALUES(?,?,?,?,?,?,?,?)", (board, child_task_id, *values)
)
elif tuple(existing) != values:
raise ValueError("continuation child lineage conflicts with its existing record")
def objective_digest(objective: str) -> str:
"""Make duplicate user follow-ups idempotent without retaining extra prose."""
return hashlib.sha256(objective.strip().encode()).hexdigest()
def existing_child(
board: str, root_task_id: str, head_commit: str, objective: str, *, path: Path | None = None
) -> str:
"""Return an existing same-objective continuation, if one was created."""
with _connect(board, path) as connection:
row = connection.execute(
"SELECT child_task_id FROM supervisor_children WHERE board=? AND root_task_id=? "
"AND head_commit=? AND objective_digest=?",
(board, root_task_id, head_commit, objective_digest(objective)),
).fetchone()
return str(row[0]) if row else ""

48
services/hermes/skills/manage-atlas-pull-requests/SKILL.md
View File

@ -1,6 +1,6 @@
---
name: manage-atlas-pull-requests
description: Read bounded private Atlas repository and pull-request metadata or create a verified draft pull request after a tested branch is pushed through Hermes' least-authority Forgejo client. Use for PR handoff in scm.bstein.dev/titan repositories. Never use it to update, merge, approve, close, delete, or force-push.
description: Read bounded Atlas repository and pull-request metadata, create a verified draft, or queue a trusted repair of an existing Hermes-owned PR. Use for PR handoff in scm.bstein.dev/titan repositories. Never merge, approve, close, delete, or force-push.
---
# Manage Atlas pull requests
@ -10,9 +10,11 @@ repository credential. The client calls a separate least-authority broker;
never bypass it with `curl` or direct Gitea HTTP.
Use the configured broker Git remote for clone, fetch, and creation of a new
namespaced feature branch. Existing-ref updates, protected refs, deletion,
and force-push are rejected by the broker. Never replace that remote with a
credential-bearing URL.
namespaced feature branch. Protected refs, deletion, and force-push are
rejected by the broker. Never replace that remote with a credential-bearing
URL. A pre-existing review branch may only be updated by a coordinator-issued
continuation task with a signed worker grant; an ordinary task cannot choose
or update it.
## Read repository or PR state
@ -51,14 +53,36 @@ Run the same command without `--dry-run` only when the branch is review-ready.
The client accepts success only when Forgejo returns the exact requested
repository, base/head refs, head SHA, title/body, canonical URLs, positive PR
number, and open/unmerged/draft state. Report its URL and exact head commit to
Brad. Leave it unmerged for human review. PR updates are intentionally absent;
publish a corrected tested commit and ask Brad how to proceed if metadata must
change.
Brad. Leave it unmerged for human review.
## Continue an existing Hermes PR
When the user asks to fix or continue an existing Hermes PR, including after a
review rejection or CI failure, queue a repair from its canonical root task.
Do not invent lineage in task text, pass a branch or PR URL, or open another
PR. The continuation command reads the board-local coordinator integrity record,
proves that the recorded PR is still open on its exact
repository/base/branch, and creates an idempotent root-parent child card:
```sh
python3 /opt/coordinator/kanban_continue_pr.py \
--board BOARD --root-task ROOT_TASK_ID \
--objective "Describe the approved repair and required verification"
```
The returned task ID is the continuation. Its worker receives a private clone
of the latest verified remote branch and may update only that recorded PR
through the signed mediator path. Re-running the command with the same root,
objective, and current head returns the same child. A changed head creates a
new child and clears any prior human-ready indication. Inspect the current diff
and tests before deciding whether to repair; reconcile delegated findings
before final triage. A PR being behind its base is not a reason to abandon it.
The signed broker request refreshes that existing PR's title and body for the
new verified head; never treat an old handoff description as current evidence.
## Stop at the authority boundary
Update, merge, approve, close, delete, branch mutation, comments, releases,
repository administration, and force-push are intentionally unavailable. Do
not bypass the client with a raw HTTP request. If one of those actions is
needed, present the tested commit and evidence to Brad and stop for human
review.
Merge, approve, close, delete, force-push, releases, and repository
administration are unavailable. Do not bypass the client with raw HTTP. A
continuation only updates an existing PR through its trusted worker grant; it
does not change the human review authority.

599
services/hermes/switchyard-configmap.yaml
View File

@ -178,6 +178,76 @@ data:
llm_client = "codex_xhigh"
extra_body = { reasoning = { effort = "xhigh" } }
# AUTO's capability selector and reasoning effort are separate axes. The
# brokers resolve each stable selector to its current concrete model.
[targets.codex_auto_economy_low]
id = "route/codex/auto-economy/low"
llm_client = "codex_low"
extra_body = { reasoning = { effort = "low" } }
[targets.codex_auto_economy_medium]
id = "route/codex/auto-economy/medium"
llm_client = "codex_medium"
extra_body = { reasoning = { effort = "medium" } }
[targets.codex_auto_economy_high]
id = "route/codex/auto-economy/high"
llm_client = "codex_high"
extra_body = { reasoning = { effort = "high" } }
[targets.codex_auto_economy_xhigh]
id = "route/codex/auto-economy/xhigh"
llm_client = "codex_xhigh"
extra_body = { reasoning = { effort = "xhigh" } }
[targets.codex_auto_balanced_low]
id = "route/codex/auto-balanced/low"
llm_client = "codex_low"
extra_body = { reasoning = { effort = "low" } }
[targets.codex_auto_balanced_medium]
id = "route/codex/auto-balanced/medium"
llm_client = "codex_medium"
extra_body = { reasoning = { effort = "medium" } }
[targets.codex_auto_balanced_high]
id = "route/codex/auto-balanced/high"
llm_client = "codex_high"
extra_body = { reasoning = { effort = "high" } }
[targets.codex_auto_balanced_xhigh]
id = "route/codex/auto-balanced/xhigh"
llm_client = "codex_xhigh"
extra_body = { reasoning = { effort = "xhigh" } }
[targets.codex_auto_advanced_low]
id = "route/codex/auto-advanced/low"
llm_client = "codex_low"
extra_body = { reasoning = { effort = "low" } }
[targets.codex_auto_advanced_medium]
id = "route/codex/auto-advanced/medium"
llm_client = "codex_medium"
extra_body = { reasoning = { effort = "medium" } }
[targets.codex_auto_advanced_high]
id = "route/codex/auto-advanced/high"
llm_client = "codex_high"
extra_body = { reasoning = { effort = "high" } }
[targets.codex_auto_advanced_xhigh]
id = "route/codex/auto-advanced/xhigh"
llm_client = "codex_xhigh"
extra_body = { reasoning = { effort = "xhigh" } }
[targets.codex_auto_frontier_low]
id = "route/codex/auto-frontier/low"
llm_client = "codex_low"
extra_body = { reasoning = { effort = "low" } }
[targets.codex_auto_frontier_medium]
id = "route/codex/auto-frontier/medium"
llm_client = "codex_medium"
extra_body = { reasoning = { effort = "medium" } }
[targets.codex_auto_frontier_high]
id = "route/codex/auto-frontier/high"
llm_client = "codex_high"
extra_body = { reasoning = { effort = "high" } }
[targets.codex_auto_frontier_xhigh]
id = "route/codex/auto-frontier/xhigh"
llm_client = "codex_xhigh"
extra_body = { reasoning = { effort = "xhigh" } }
[targets.claude_haiku_low]
id = "route/claude/haiku/low"
llm_client = "claude_low"
@ -278,6 +348,74 @@ data:
llm_client = "claude_xhigh"
extra_body = { thinking = { type = "adaptive" }, output_config = { effort = "xhigh" } }
[targets.claude_auto_economy_low]
id = "route/claude/auto-economy/low"
llm_client = "claude_low"
extra_body = { thinking = { type = "adaptive" }, output_config = { effort = "low" } }
[targets.claude_auto_economy_medium]
id = "route/claude/auto-economy/medium"
llm_client = "claude_medium"
extra_body = { thinking = { type = "adaptive" }, output_config = { effort = "medium" } }
[targets.claude_auto_economy_high]
id = "route/claude/auto-economy/high"
llm_client = "claude_high"
extra_body = { thinking = { type = "adaptive" }, output_config = { effort = "high" } }
[targets.claude_auto_economy_xhigh]
id = "route/claude/auto-economy/xhigh"
llm_client = "claude_xhigh"
extra_body = { thinking = { type = "adaptive" }, output_config = { effort = "xhigh" } }
[targets.claude_auto_balanced_low]
id = "route/claude/auto-balanced/low"
llm_client = "claude_low"
extra_body = { thinking = { type = "adaptive" }, output_config = { effort = "low" } }
[targets.claude_auto_balanced_medium]
id = "route/claude/auto-balanced/medium"
llm_client = "claude_medium"
extra_body = { thinking = { type = "adaptive" }, output_config = { effort = "medium" } }
[targets.claude_auto_balanced_high]
id = "route/claude/auto-balanced/high"
llm_client = "claude_high"
extra_body = { thinking = { type = "adaptive" }, output_config = { effort = "high" } }
[targets.claude_auto_balanced_xhigh]
id = "route/claude/auto-balanced/xhigh"
llm_client = "claude_xhigh"
extra_body = { thinking = { type = "adaptive" }, output_config = { effort = "xhigh" } }
[targets.claude_auto_advanced_low]
id = "route/claude/auto-advanced/low"
llm_client = "claude_low"
extra_body = { thinking = { type = "adaptive" }, output_config = { effort = "low" } }
[targets.claude_auto_advanced_medium]
id = "route/claude/auto-advanced/medium"
llm_client = "claude_medium"
extra_body = { thinking = { type = "adaptive" }, output_config = { effort = "medium" } }
[targets.claude_auto_advanced_high]
id = "route/claude/auto-advanced/high"
llm_client = "claude_high"
extra_body = { thinking = { type = "adaptive" }, output_config = { effort = "high" } }
[targets.claude_auto_advanced_xhigh]
id = "route/claude/auto-advanced/xhigh"
llm_client = "claude_xhigh"
extra_body = { thinking = { type = "adaptive" }, output_config = { effort = "xhigh" } }
[targets.claude_auto_frontier_low]
id = "route/claude/auto-frontier/low"
llm_client = "claude_low"
extra_body = { thinking = { type = "adaptive" }, output_config = { effort = "low" } }
[targets.claude_auto_frontier_medium]
id = "route/claude/auto-frontier/medium"
llm_client = "claude_medium"
extra_body = { thinking = { type = "adaptive" }, output_config = { effort = "medium" } }
[targets.claude_auto_frontier_high]
id = "route/claude/auto-frontier/high"
llm_client = "claude_high"
extra_body = { thinking = { type = "adaptive" }, output_config = { effort = "high" } }
[targets.claude_auto_frontier_xhigh]
id = "route/claude/auto-frontier/xhigh"
llm_client = "claude_xhigh"
extra_body = { thinking = { type = "adaptive" }, output_config = { effort = "xhigh" } }
[targets.worker_codex_luna_low]
id = "worker/codex/luna/low"
llm_client = "worker_decision"
@ -422,6 +560,110 @@ data:
id = "worker/claude/auto/xhigh"
llm_client = "worker_decision"
[targets.worker_codex_auto_economy_low]
id = "worker/codex/auto-economy/low"
llm_client = "worker_decision"
[targets.worker_codex_auto_economy_medium]
id = "worker/codex/auto-economy/medium"
llm_client = "worker_decision"
[targets.worker_codex_auto_economy_high]
id = "worker/codex/auto-economy/high"
llm_client = "worker_decision"
[targets.worker_codex_auto_economy_xhigh]
id = "worker/codex/auto-economy/xhigh"
llm_client = "worker_decision"
[targets.worker_codex_auto_balanced_low]
id = "worker/codex/auto-balanced/low"
llm_client = "worker_decision"
[targets.worker_codex_auto_balanced_medium]
id = "worker/codex/auto-balanced/medium"
llm_client = "worker_decision"
[targets.worker_codex_auto_balanced_high]
id = "worker/codex/auto-balanced/high"
llm_client = "worker_decision"
[targets.worker_codex_auto_balanced_xhigh]
id = "worker/codex/auto-balanced/xhigh"
llm_client = "worker_decision"
[targets.worker_codex_auto_advanced_low]
id = "worker/codex/auto-advanced/low"
llm_client = "worker_decision"
[targets.worker_codex_auto_advanced_medium]
id = "worker/codex/auto-advanced/medium"
llm_client = "worker_decision"
[targets.worker_codex_auto_advanced_high]
id = "worker/codex/auto-advanced/high"
llm_client = "worker_decision"
[targets.worker_codex_auto_advanced_xhigh]
id = "worker/codex/auto-advanced/xhigh"
llm_client = "worker_decision"
[targets.worker_codex_auto_frontier_low]
id = "worker/codex/auto-frontier/low"
llm_client = "worker_decision"
[targets.worker_codex_auto_frontier_medium]
id = "worker/codex/auto-frontier/medium"
llm_client = "worker_decision"
[targets.worker_codex_auto_frontier_high]
id = "worker/codex/auto-frontier/high"
llm_client = "worker_decision"
[targets.worker_codex_auto_frontier_xhigh]
id = "worker/codex/auto-frontier/xhigh"
llm_client = "worker_decision"
[targets.worker_claude_auto_economy_low]
id = "worker/claude/auto-economy/low"
llm_client = "worker_decision"
[targets.worker_claude_auto_economy_medium]
id = "worker/claude/auto-economy/medium"
llm_client = "worker_decision"
[targets.worker_claude_auto_economy_high]
id = "worker/claude/auto-economy/high"
llm_client = "worker_decision"
[targets.worker_claude_auto_economy_xhigh]
id = "worker/claude/auto-economy/xhigh"
llm_client = "worker_decision"
[targets.worker_claude_auto_balanced_low]
id = "worker/claude/auto-balanced/low"
llm_client = "worker_decision"
[targets.worker_claude_auto_balanced_medium]
id = "worker/claude/auto-balanced/medium"
llm_client = "worker_decision"
[targets.worker_claude_auto_balanced_high]
id = "worker/claude/auto-balanced/high"
llm_client = "worker_decision"
[targets.worker_claude_auto_balanced_xhigh]
id = "worker/claude/auto-balanced/xhigh"
llm_client = "worker_decision"
[targets.worker_claude_auto_advanced_low]
id = "worker/claude/auto-advanced/low"
llm_client = "worker_decision"
[targets.worker_claude_auto_advanced_medium]
id = "worker/claude/auto-advanced/medium"
llm_client = "worker_decision"
[targets.worker_claude_auto_advanced_high]
id = "worker/claude/auto-advanced/high"
llm_client = "worker_decision"
[targets.worker_claude_auto_advanced_xhigh]
id = "worker/claude/auto-advanced/xhigh"
llm_client = "worker_decision"
[targets.worker_claude_auto_frontier_low]
id = "worker/claude/auto-frontier/low"
llm_client = "worker_decision"
[targets.worker_claude_auto_frontier_medium]
id = "worker/claude/auto-frontier/medium"
llm_client = "worker_decision"
[targets.worker_claude_auto_frontier_high]
id = "worker/claude/auto-frontier/high"
llm_client = "worker_decision"
[targets.worker_claude_auto_frontier_xhigh]
id = "worker/claude/auto-frontier/xhigh"
llm_client = "worker_decision"
# Switchyard 0.2.0 requires one default_target for every custom classifier.
# These targets recurse once into unbiased random routes whose candidates are
# split evenly across providers. A classifier outage therefore falls back to
@ -449,7 +691,9 @@ data:
[routes.fallback_fast]
id = "atlas/fallback/fast"
type = "random"
targets = ["codex_auto_medium", "claude_auto_medium"]
# A classifier outage has no task verdict to justify a lower role. Use the
# conservative explicit capability instead of legacy effort-only AUTO.
targets = ["codex_auto_advanced_high", "claude_auto_advanced_high"]
weights = [1.0, 1.0]
context_window = 272000
tool_calling = true
@ -458,7 +702,7 @@ data:
[routes.fallback_balanced]
id = "atlas/fallback/balanced"
type = "random"
targets = ["codex_auto_high", "claude_auto_high"]
targets = ["codex_auto_advanced_high", "claude_auto_advanced_high"]
weights = [1.0, 1.0]
context_window = 272000
tool_calling = true
@ -467,7 +711,7 @@ data:
[routes.fallback_deep]
id = "atlas/fallback/deep"
type = "random"
targets = ["codex_auto_high", "claude_auto_high"]
targets = ["codex_auto_advanced_high", "claude_auto_advanced_high"]
weights = [1.0, 1.0]
context_window = 272000
tool_calling = true
@ -476,7 +720,7 @@ data:
[routes.fallback_maximum]
id = "atlas/fallback/maximum"
type = "random"
targets = ["codex_auto_xhigh", "claude_auto_xhigh"]
targets = ["codex_auto_advanced_xhigh", "claude_auto_advanced_xhigh"]
weights = [1.0, 1.0]
context_window = 272000
tool_calling = true
@ -485,7 +729,9 @@ data:
[routes.fallback_worker_maximum]
id = "atlas/worker/fallback/maximum"
type = "random"
targets = ["worker_codex_auto_xhigh", "worker_claude_auto_xhigh"]
# Classifier unavailability uses an explicit conservative capability. Do
# not let legacy AUTO's effort-only selector silently choose a model tier.
targets = ["worker_codex_auto_advanced_xhigh", "worker_claude_auto_advanced_xhigh"]
weights = [1.0, 1.0]
context_window = 272000
tool_calling = false
@ -497,8 +743,8 @@ data:
mode = "custom"
classifier_target = "classifier"
# Switchyard falls through this list after a request-local target failure.
# Keep both xhigh providers first so recovery can escalate, never downgrade.
targets = ["codex_auto_xhigh", "claude_auto_xhigh", "codex_auto_high", "claude_auto_high", "codex_auto_medium", "claude_auto_medium", "codex_auto_low", "claude_auto_low", "neutral_fast_pool"]
# Sort by descending capability so recovery never silently drops a role.
targets = ["codex_auto_frontier_xhigh", "codex_auto_frontier_high", "codex_auto_frontier_medium", "codex_auto_frontier_low", "claude_auto_frontier_xhigh", "claude_auto_frontier_high", "claude_auto_frontier_medium", "claude_auto_frontier_low", "codex_auto_advanced_xhigh", "codex_auto_advanced_high", "codex_auto_advanced_medium", "codex_auto_advanced_low", "claude_auto_advanced_xhigh", "claude_auto_advanced_high", "claude_auto_advanced_medium", "claude_auto_advanced_low", "codex_auto_balanced_xhigh", "codex_auto_balanced_high", "codex_auto_balanced_medium", "codex_auto_balanced_low", "claude_auto_balanced_xhigh", "claude_auto_balanced_high", "claude_auto_balanced_medium", "claude_auto_balanced_low", "codex_auto_economy_xhigh", "codex_auto_economy_high", "codex_auto_economy_medium", "codex_auto_economy_low", "claude_auto_economy_xhigh", "claude_auto_economy_high", "claude_auto_economy_medium", "claude_auto_economy_low", "neutral_fast_pool"]
default_target = "neutral_fast_pool"
session_affinity = false
recent_turn_window = 4
@ -509,6 +755,16 @@ data:
You are the deterministic routing authority for one private family-assistant
model-call boundary. Mildly favor speed, but apply these rules in order.
Choose capability before reasoning effort. economy is Luna-class simple work;
balanced is Terra-class routine tool work; advanced is Sol-class ordinary
complex work; frontier is Astra-class exceptional ambiguity, deeply coupled
architecture, or a meaningful quality escalation. High or xhigh effort does
not imply frontier: legacy auto at high and xhigh is advanced. Select
frontier directly for clearly exceptional work; do not require a failed
attempt first. Provider timeout, rate limit, authentication, and transport
failures are infrastructure failures: preserve both capability and effort
on the alternate provider and never promote them as a quality signal.
1. Set a mandatory minimum effort floor from the whole current objective and
recent context: xhigh for critical security work, risky production
migrations, destructive or data-loss risk, or consequential independent
@ -560,7 +816,7 @@ data:
available only when the user explicitly selects it.
"""
response_schema = '''
{"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["codex_auto_low","codex_auto_medium","codex_auto_high","codex_auto_xhigh","claude_auto_low","claude_auto_medium","claude_auto_high","claude_auto_xhigh"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false}
{"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","pattern":"^(codex|claude)_auto_(economy|balanced|advanced|frontier)_(low|medium|high|xhigh)$"}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false}
'''
[routes.auto_fast.policy]
@ -573,8 +829,8 @@ data:
mode = "custom"
classifier_target = "classifier"
# Switchyard falls through this list after a request-local target failure.
# Keep both xhigh providers first so recovery can escalate, never downgrade.
targets = ["codex_auto_xhigh", "claude_auto_xhigh", "codex_auto_high", "claude_auto_high", "codex_auto_medium", "claude_auto_medium", "codex_auto_low", "claude_auto_low", "neutral_balanced_pool"]
# Sort by descending capability so recovery never silently drops a role.
targets = ["codex_auto_frontier_xhigh", "codex_auto_frontier_high", "codex_auto_frontier_medium", "codex_auto_frontier_low", "claude_auto_frontier_xhigh", "claude_auto_frontier_high", "claude_auto_frontier_medium", "claude_auto_frontier_low", "codex_auto_advanced_xhigh", "codex_auto_advanced_high", "codex_auto_advanced_medium", "codex_auto_advanced_low", "claude_auto_advanced_xhigh", "claude_auto_advanced_high", "claude_auto_advanced_medium", "claude_auto_advanced_low", "codex_auto_balanced_xhigh", "codex_auto_balanced_high", "codex_auto_balanced_medium", "codex_auto_balanced_low", "claude_auto_balanced_xhigh", "claude_auto_balanced_high", "claude_auto_balanced_medium", "claude_auto_balanced_low", "codex_auto_economy_xhigh", "codex_auto_economy_high", "codex_auto_economy_medium", "codex_auto_economy_low", "claude_auto_economy_xhigh", "claude_auto_economy_high", "claude_auto_economy_medium", "claude_auto_economy_low", "neutral_balanced_pool"]
default_target = "neutral_balanced_pool"
session_affinity = false
recent_turn_window = 4
@ -586,6 +842,16 @@ data:
model-call boundary. Balance speed and intelligence, then apply these rules
in order.
Choose capability before reasoning effort. economy is Luna-class simple work;
balanced is Terra-class routine tool work; advanced is Sol-class ordinary
complex work; frontier is Astra-class exceptional ambiguity, deeply coupled
architecture, or a meaningful quality escalation. High or xhigh effort does
not imply frontier: legacy auto at high and xhigh is advanced. Select
frontier directly for clearly exceptional work; do not require a failed
attempt first. Provider timeout, rate limit, authentication, and transport
failures are infrastructure failures: preserve both capability and effort
on the alternate provider and never promote them as a quality signal.
1. Set a mandatory minimum effort floor from the whole current objective and
recent context: xhigh for critical security work, risky production
migrations, destructive or data-loss risk, or consequential independent
@ -637,7 +903,7 @@ data:
available only when the user explicitly selects it.
"""
response_schema = '''
{"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["codex_auto_low","codex_auto_medium","codex_auto_high","codex_auto_xhigh","claude_auto_low","claude_auto_medium","claude_auto_high","claude_auto_xhigh"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false}
{"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","pattern":"^(codex|claude)_auto_(economy|balanced|advanced|frontier)_(low|medium|high|xhigh)$"}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false}
'''
[routes.auto_balanced.policy]
@ -650,8 +916,8 @@ data:
mode = "custom"
classifier_target = "classifier"
# Switchyard falls through this list after a request-local target failure.
# Keep both xhigh providers first so recovery can escalate, never downgrade.
targets = ["claude_auto_xhigh", "codex_auto_xhigh", "claude_auto_high", "codex_auto_high", "claude_auto_medium", "codex_auto_medium", "neutral_deep_pool"]
# Sort by descending capability so recovery never silently drops a role.
targets = ["codex_auto_frontier_xhigh", "codex_auto_frontier_high", "codex_auto_frontier_medium", "claude_auto_frontier_xhigh", "claude_auto_frontier_high", "claude_auto_frontier_medium", "codex_auto_advanced_xhigh", "codex_auto_advanced_high", "codex_auto_advanced_medium", "claude_auto_advanced_xhigh", "claude_auto_advanced_high", "claude_auto_advanced_medium", "codex_auto_balanced_xhigh", "codex_auto_balanced_high", "codex_auto_balanced_medium", "claude_auto_balanced_xhigh", "claude_auto_balanced_high", "claude_auto_balanced_medium", "neutral_deep_pool"]
default_target = "neutral_deep_pool"
session_affinity = false
recent_turn_window = 6
@ -663,6 +929,14 @@ data:
model-call boundary. Favor evidence and intelligence, then apply these rules
in order.
Choose capability before reasoning effort. balanced is Terra-class routine
triage; advanced is Sol-class ordinary complex work; frontier is Astra-class
exceptional ambiguity, deeply coupled architecture, or a meaningful quality
escalation. High or xhigh effort does not imply frontier. Select frontier
directly for clearly exceptional work; provider timeout, rate limit,
authentication, and transport failures preserve capability and effort on
the alternate provider and never count as a quality signal.
1. Set a mandatory minimum effort floor from the whole current objective,
recent context, and tool evidence: xhigh for critical security incidents,
risky production migrations, destructive or data-loss risk, or
@ -705,7 +979,7 @@ data:
context.
"""
response_schema = '''
{"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["claude_auto_medium","claude_auto_high","claude_auto_xhigh","codex_auto_medium","codex_auto_high","codex_auto_xhigh"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false}
{"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","pattern":"^(codex|claude)_auto_(balanced|advanced|frontier)_(medium|high|xhigh)$"}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false}
'''
[routes.auto_deep.policy]
@ -718,8 +992,8 @@ data:
mode = "custom"
classifier_target = "classifier"
# Switchyard falls through this list after a request-local target failure.
# Keep both xhigh providers first so recovery can escalate, never downgrade.
targets = ["codex_auto_xhigh", "claude_auto_xhigh", "codex_auto_high", "claude_auto_high", "neutral_maximum_pool"]
# Sort by descending capability so a frontier selection cannot reach Sol.
targets = ["codex_auto_frontier_xhigh", "codex_auto_frontier_high", "claude_auto_frontier_xhigh", "claude_auto_frontier_high", "codex_auto_advanced_xhigh", "codex_auto_advanced_high", "claude_auto_advanced_xhigh", "claude_auto_advanced_high", "neutral_maximum_pool"]
default_target = "neutral_maximum_pool"
session_affinity = false
recent_turn_window = 6
@ -731,6 +1005,14 @@ data:
model-call boundary. Strongly favor intelligence, verification, and task
completion, then apply these rules in order.
Choose capability before reasoning effort. advanced is Sol-class ordinary
complex work; frontier is Astra-class exceptional ambiguity, deeply coupled
architecture, or a meaningful quality escalation. High or xhigh effort does
not imply frontier: legacy auto at high and xhigh is advanced. Select
frontier directly for clearly exceptional work; provider timeout, rate
limit, authentication, and transport failures preserve capability and
effort on the alternate provider and never count as a quality signal.
1. This maximum-quality route has an absolute high effort floor for every
boundary, including ordinary implementation, tests, tool use, analysis,
bounded architecture, lookup, and mechanical work. Raise the floor to
@ -767,7 +1049,7 @@ data:
context.
"""
response_schema = '''
{"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["codex_auto_high","codex_auto_xhigh","claude_auto_high","claude_auto_xhigh"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false}
{"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","pattern":"^(codex|claude)_auto_(advanced|frontier)_(high|xhigh)$"}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false}
'''
[routes.auto_maximum.policy]
@ -779,7 +1061,10 @@ data:
type = "llm_classifier"
mode = "custom"
classifier_target = "classifier"
targets = ["worker_codex_auto_low", "worker_codex_auto_medium", "worker_codex_auto_high", "worker_codex_auto_xhigh", "worker_claude_auto_low", "worker_claude_auto_medium", "worker_claude_auto_high", "worker_claude_auto_xhigh", "neutral_worker_maximum_pool"]
# Worker decisions have the same FallThrough behavior as hosted routes.
# Keep the fallback pool capability-descending; the CLI guard below also
# reselects an exact alternate floor if a lower target is ever reported.
targets = ["worker_codex_auto_frontier_xhigh", "worker_codex_auto_frontier_high", "worker_codex_auto_frontier_medium", "worker_codex_auto_frontier_low", "worker_claude_auto_frontier_xhigh", "worker_claude_auto_frontier_high", "worker_claude_auto_frontier_medium", "worker_claude_auto_frontier_low", "worker_codex_auto_advanced_xhigh", "worker_codex_auto_advanced_high", "worker_codex_auto_advanced_medium", "worker_codex_auto_advanced_low", "worker_claude_auto_advanced_xhigh", "worker_claude_auto_advanced_high", "worker_claude_auto_advanced_medium", "worker_claude_auto_advanced_low", "worker_codex_auto_balanced_xhigh", "worker_codex_auto_balanced_high", "worker_codex_auto_balanced_medium", "worker_codex_auto_balanced_low", "worker_claude_auto_balanced_xhigh", "worker_claude_auto_balanced_high", "worker_claude_auto_balanced_medium", "worker_claude_auto_balanced_low", "worker_codex_auto_economy_xhigh", "worker_codex_auto_economy_high", "worker_codex_auto_economy_medium", "worker_codex_auto_economy_low", "worker_claude_auto_economy_xhigh", "worker_claude_auto_economy_high", "worker_claude_auto_economy_medium", "worker_claude_auto_economy_low", "neutral_worker_maximum_pool"]
default_target = "neutral_worker_maximum_pool"
session_affinity = false
recent_turn_window = 6
@ -791,6 +1076,18 @@ data:
worker launch. Choose exactly one configured target. Apply these steps in
order.
First choose capability, then effort. economy selects Luna-class simple
work; balanced selects Terra-class routine work; advanced selects Sol-class
ordinary complex work; frontier selects Astra-class exceptional ambiguity,
deeply coupled architecture, or meaningful quality escalation. High and
xhigh are effort only: legacy auto high and xhigh select advanced. Select
frontier directly when the task is clearly exceptional; do not wait for a
failed attempt. A failed test, rejected review, contradictory result, or
incomplete evidence is a quality signal: carry a capability floor forward
and change the failed plan. Timeout, rate-limit, authentication, transport,
and provider-capacity failures are infrastructure signals: preserve both
capability and effort across the alternate provider without promotion.
1. Set a mandatory minimum effort floor from the whole objective:
- xhigh: critical security work; a risky production migration; destructive
or data-loss risk; or an independent final/release review of consequential
@ -844,7 +1141,7 @@ data:
the floor. Return only the required decision object.
"""
response_schema = '''
{"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["worker_codex_auto_low","worker_codex_auto_medium","worker_codex_auto_high","worker_codex_auto_xhigh","worker_claude_auto_low","worker_claude_auto_medium","worker_claude_auto_high","worker_claude_auto_xhigh"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false}
{"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","pattern":"^worker_(codex|claude)_auto_(economy|balanced|advanced|frontier)_(low|medium|high|xhigh)$"}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false}
'''
[routes.worker_auto_maximum.policy]
@ -891,6 +1188,138 @@ data:
type = "random"
targets = ["worker_claude_auto_xhigh"]
# Exact manual worker constraints retain the legacy routes above and add
# capability-aware overrides for operators and capability-floor recovery.
[routes.worker_manual_codex_economy_low]
id = "atlas/worker/manual/codex/economy/low"
type = "random"
targets = ["worker_codex_auto_economy_low"]
[routes.worker_manual_codex_economy_medium]
id = "atlas/worker/manual/codex/economy/medium"
type = "random"
targets = ["worker_codex_auto_economy_medium"]
[routes.worker_manual_codex_economy_high]
id = "atlas/worker/manual/codex/economy/high"
type = "random"
targets = ["worker_codex_auto_economy_high"]
[routes.worker_manual_codex_economy_xhigh]
id = "atlas/worker/manual/codex/economy/xhigh"
type = "random"
targets = ["worker_codex_auto_economy_xhigh"]
[routes.worker_manual_codex_balanced_low]
id = "atlas/worker/manual/codex/balanced/low"
type = "random"
targets = ["worker_codex_auto_balanced_low"]
[routes.worker_manual_codex_balanced_medium]
id = "atlas/worker/manual/codex/balanced/medium"
type = "random"
targets = ["worker_codex_auto_balanced_medium"]
[routes.worker_manual_codex_balanced_high]
id = "atlas/worker/manual/codex/balanced/high"
type = "random"
targets = ["worker_codex_auto_balanced_high"]
[routes.worker_manual_codex_balanced_xhigh]
id = "atlas/worker/manual/codex/balanced/xhigh"
type = "random"
targets = ["worker_codex_auto_balanced_xhigh"]
[routes.worker_manual_codex_advanced_low]
id = "atlas/worker/manual/codex/advanced/low"
type = "random"
targets = ["worker_codex_auto_advanced_low"]
[routes.worker_manual_codex_advanced_medium]
id = "atlas/worker/manual/codex/advanced/medium"
type = "random"
targets = ["worker_codex_auto_advanced_medium"]
[routes.worker_manual_codex_advanced_high]
id = "atlas/worker/manual/codex/advanced/high"
type = "random"
targets = ["worker_codex_auto_advanced_high"]
[routes.worker_manual_codex_advanced_xhigh]
id = "atlas/worker/manual/codex/advanced/xhigh"
type = "random"
targets = ["worker_codex_auto_advanced_xhigh"]
[routes.worker_manual_codex_frontier_low]
id = "atlas/worker/manual/codex/frontier/low"
type = "random"
targets = ["worker_codex_auto_frontier_low"]
[routes.worker_manual_codex_frontier_medium]
id = "atlas/worker/manual/codex/frontier/medium"
type = "random"
targets = ["worker_codex_auto_frontier_medium"]
[routes.worker_manual_codex_frontier_high]
id = "atlas/worker/manual/codex/frontier/high"
type = "random"
targets = ["worker_codex_auto_frontier_high"]
[routes.worker_manual_codex_frontier_xhigh]
id = "atlas/worker/manual/codex/frontier/xhigh"
type = "random"
targets = ["worker_codex_auto_frontier_xhigh"]
[routes.worker_manual_claude_economy_low]
id = "atlas/worker/manual/claude/economy/low"
type = "random"
targets = ["worker_claude_auto_economy_low"]
[routes.worker_manual_claude_economy_medium]
id = "atlas/worker/manual/claude/economy/medium"
type = "random"
targets = ["worker_claude_auto_economy_medium"]
[routes.worker_manual_claude_economy_high]
id = "atlas/worker/manual/claude/economy/high"
type = "random"
targets = ["worker_claude_auto_economy_high"]
[routes.worker_manual_claude_economy_xhigh]
id = "atlas/worker/manual/claude/economy/xhigh"
type = "random"
targets = ["worker_claude_auto_economy_xhigh"]
[routes.worker_manual_claude_balanced_low]
id = "atlas/worker/manual/claude/balanced/low"
type = "random"
targets = ["worker_claude_auto_balanced_low"]
[routes.worker_manual_claude_balanced_medium]
id = "atlas/worker/manual/claude/balanced/medium"
type = "random"
targets = ["worker_claude_auto_balanced_medium"]
[routes.worker_manual_claude_balanced_high]
id = "atlas/worker/manual/claude/balanced/high"
type = "random"
targets = ["worker_claude_auto_balanced_high"]
[routes.worker_manual_claude_balanced_xhigh]
id = "atlas/worker/manual/claude/balanced/xhigh"
type = "random"
targets = ["worker_claude_auto_balanced_xhigh"]
[routes.worker_manual_claude_advanced_low]
id = "atlas/worker/manual/claude/advanced/low"
type = "random"
targets = ["worker_claude_auto_advanced_low"]
[routes.worker_manual_claude_advanced_medium]
id = "atlas/worker/manual/claude/advanced/medium"
type = "random"
targets = ["worker_claude_auto_advanced_medium"]
[routes.worker_manual_claude_advanced_high]
id = "atlas/worker/manual/claude/advanced/high"
type = "random"
targets = ["worker_claude_auto_advanced_high"]
[routes.worker_manual_claude_advanced_xhigh]
id = "atlas/worker/manual/claude/advanced/xhigh"
type = "random"
targets = ["worker_claude_auto_advanced_xhigh"]
[routes.worker_manual_claude_frontier_low]
id = "atlas/worker/manual/claude/frontier/low"
type = "random"
targets = ["worker_claude_auto_frontier_low"]
[routes.worker_manual_claude_frontier_medium]
id = "atlas/worker/manual/claude/frontier/medium"
type = "random"
targets = ["worker_claude_auto_frontier_medium"]
[routes.worker_manual_claude_frontier_high]
id = "atlas/worker/manual/claude/frontier/high"
type = "random"
targets = ["worker_claude_auto_frontier_high"]
[routes.worker_manual_claude_frontier_xhigh]
id = "atlas/worker/manual/claude/frontier/xhigh"
type = "random"
targets = ["worker_claude_auto_frontier_xhigh"]
[routes.manual_codex_auto]
id = "atlas/manual/codex/auto"
type = "random"
@ -947,6 +1376,138 @@ data:
type = "random"
targets = ["claude_auto_xhigh"]
# Hosted callers can pin the same independent capability/effort pairs as
# durable workers. The unqualified legacy AUTO routes above remain advanced.
[routes.manual_codex_auto_economy_low]
id = "atlas/manual/codex/auto-economy/low"
type = "random"
targets = ["codex_auto_economy_low"]
[routes.manual_codex_auto_economy_medium]
id = "atlas/manual/codex/auto-economy/medium"
type = "random"
targets = ["codex_auto_economy_medium"]
[routes.manual_codex_auto_economy_high]
id = "atlas/manual/codex/auto-economy/high"
type = "random"
targets = ["codex_auto_economy_high"]
[routes.manual_codex_auto_economy_xhigh]
id = "atlas/manual/codex/auto-economy/xhigh"
type = "random"
targets = ["codex_auto_economy_xhigh"]
[routes.manual_codex_auto_balanced_low]
id = "atlas/manual/codex/auto-balanced/low"
type = "random"
targets = ["codex_auto_balanced_low"]
[routes.manual_codex_auto_balanced_medium]
id = "atlas/manual/codex/auto-balanced/medium"
type = "random"
targets = ["codex_auto_balanced_medium"]
[routes.manual_codex_auto_balanced_high]
id = "atlas/manual/codex/auto-balanced/high"
type = "random"
targets = ["codex_auto_balanced_high"]
[routes.manual_codex_auto_balanced_xhigh]
id = "atlas/manual/codex/auto-balanced/xhigh"
type = "random"
targets = ["codex_auto_balanced_xhigh"]
[routes.manual_codex_auto_advanced_low]
id = "atlas/manual/codex/auto-advanced/low"
type = "random"
targets = ["codex_auto_advanced_low"]
[routes.manual_codex_auto_advanced_medium]
id = "atlas/manual/codex/auto-advanced/medium"
type = "random"
targets = ["codex_auto_advanced_medium"]
[routes.manual_codex_auto_advanced_high]
id = "atlas/manual/codex/auto-advanced/high"
type = "random"
targets = ["codex_auto_advanced_high"]
[routes.manual_codex_auto_advanced_xhigh]
id = "atlas/manual/codex/auto-advanced/xhigh"
type = "random"
targets = ["codex_auto_advanced_xhigh"]
[routes.manual_codex_auto_frontier_low]
id = "atlas/manual/codex/auto-frontier/low"
type = "random"
targets = ["codex_auto_frontier_low"]
[routes.manual_codex_auto_frontier_medium]
id = "atlas/manual/codex/auto-frontier/medium"
type = "random"
targets = ["codex_auto_frontier_medium"]
[routes.manual_codex_auto_frontier_high]
id = "atlas/manual/codex/auto-frontier/high"
type = "random"
targets = ["codex_auto_frontier_high"]
[routes.manual_codex_auto_frontier_xhigh]
id = "atlas/manual/codex/auto-frontier/xhigh"
type = "random"
targets = ["codex_auto_frontier_xhigh"]
[routes.manual_claude_auto_economy_low]
id = "atlas/manual/claude/auto-economy/low"
type = "random"
targets = ["claude_auto_economy_low"]
[routes.manual_claude_auto_economy_medium]
id = "atlas/manual/claude/auto-economy/medium"
type = "random"
targets = ["claude_auto_economy_medium"]
[routes.manual_claude_auto_economy_high]
id = "atlas/manual/claude/auto-economy/high"
type = "random"
targets = ["claude_auto_economy_high"]
[routes.manual_claude_auto_economy_xhigh]
id = "atlas/manual/claude/auto-economy/xhigh"
type = "random"
targets = ["claude_auto_economy_xhigh"]
[routes.manual_claude_auto_balanced_low]
id = "atlas/manual/claude/auto-balanced/low"
type = "random"
targets = ["claude_auto_balanced_low"]
[routes.manual_claude_auto_balanced_medium]
id = "atlas/manual/claude/auto-balanced/medium"
type = "random"
targets = ["claude_auto_balanced_medium"]
[routes.manual_claude_auto_balanced_high]
id = "atlas/manual/claude/auto-balanced/high"
type = "random"
targets = ["claude_auto_balanced_high"]
[routes.manual_claude_auto_balanced_xhigh]
id = "atlas/manual/claude/auto-balanced/xhigh"
type = "random"
targets = ["claude_auto_balanced_xhigh"]
[routes.manual_claude_auto_advanced_low]
id = "atlas/manual/claude/auto-advanced/low"
type = "random"
targets = ["claude_auto_advanced_low"]
[routes.manual_claude_auto_advanced_medium]
id = "atlas/manual/claude/auto-advanced/medium"
type = "random"
targets = ["claude_auto_advanced_medium"]
[routes.manual_claude_auto_advanced_high]
id = "atlas/manual/claude/auto-advanced/high"
type = "random"
targets = ["claude_auto_advanced_high"]
[routes.manual_claude_auto_advanced_xhigh]
id = "atlas/manual/claude/auto-advanced/xhigh"
type = "random"
targets = ["claude_auto_advanced_xhigh"]
[routes.manual_claude_auto_frontier_low]
id = "atlas/manual/claude/auto-frontier/low"
type = "random"
targets = ["claude_auto_frontier_low"]
[routes.manual_claude_auto_frontier_medium]
id = "atlas/manual/claude/auto-frontier/medium"
type = "random"
targets = ["claude_auto_frontier_medium"]
[routes.manual_claude_auto_frontier_high]
id = "atlas/manual/claude/auto-frontier/high"
type = "random"
targets = ["claude_auto_frontier_high"]
[routes.manual_claude_auto_frontier_xhigh]
id = "atlas/manual/claude/auto-frontier/xhigh"
type = "random"
targets = ["claude_auto_frontier_xhigh"]
# A zero-weight target is not selected initially. Switchyard still walks
# this ordered list after 403/408/429/5xx, timeout, or transport failure.
[routes.manual_codex_luna]

4
services/hermes/switchyard-deployment.yaml
View File

@ -22,7 +22,7 @@ spec:
labels:
app: hermes-switchyard
annotations:
ai.bstein.dev/config-rev: "20260913-provider-model-catalog-v2"
ai.bstein.dev/config-rev: "20260913-capability-effort-v3"
prometheus.io/scrape: "true"
prometheus.io/port: "9005"
prometheus.io/path: /metrics
@ -64,7 +64,7 @@ spec:
values: [titan-14, titan-18, titan-22, titan-24]
containers:
- name: switchyard
image: registry.bstein.dev/bstein/hermes-switchyard@sha256:3d952d528a4e4cb8afdf84f8995272f5d6107dc127b68cc20234b1da4aff43eb
image: registry.bstein.dev/bstein/hermes-switchyard@sha256:7ea3053590f35d9d498e6df590ee67cbe1cfb960c6f7a88ca4608368e7bfdc39
imagePullPolicy: IfNotPresent
command: [/bin/sh, -ec]
args:

45
testing/tests/test_hermes_chat_routing_runtime.py
View File

@ -418,9 +418,31 @@ def test_local_flux_runtime_and_gpu_handoff_are_flux_managed():
switchyard_config = tomllib.loads(switchyard)
routes = switchyard_config["routes"]
configured_targets = switchyard_config["targets"]
capability_rank = {
"economy": 0,
"balanced": 1,
"advanced": 2,
"frontier": 3,
}
def target_capability(target: str) -> str | None:
parts = target.split("_auto_")
if len(parts) != 2 or parts[1].startswith(("low", "medium", "high", "xhigh")):
return None
return parts[1].rsplit("_", 1)[0]
for route_name in ("auto_fast", "auto_balanced", "auto_deep", "auto_maximum"):
leading_targets = set(routes[route_name]["targets"][:2])
assert leading_targets == {"codex_auto_xhigh", "claude_auto_xhigh"}
targets = routes[route_name]["targets"]
capabilities = [
target_capability(target)
for target in targets
if target_capability(target) is not None
]
# Switchyard retries the full target list after an upstream error. A
# failed Astra target must therefore be considered before any Sol one.
assert capabilities == sorted(
capabilities, key=capability_rank.__getitem__, reverse=True
)
assert "max_output_tokens" not in routes[route_name]
for route_name in ("auto_fast", "auto_balanced", "auto_deep", "auto_maximum"):
targets = routes[route_name]["targets"]
@ -445,6 +467,25 @@ def test_local_flux_runtime_and_gpu_handoff_are_flux_managed():
"Repeated quality misses require xhigh"
in routes["worker_auto_maximum"]["prompt"]
)
worker_capabilities = [
target_capability(target)
for target in routes["worker_auto_maximum"]["targets"]
if target_capability(target) is not None
]
assert worker_capabilities == sorted(
worker_capabilities, key=capability_rank.__getitem__, reverse=True
)
for route_name in (
"fallback_fast",
"fallback_balanced",
"fallback_deep",
"fallback_maximum",
"fallback_worker_maximum",
):
assert all(
target_capability(target) == "advanced"
for target in routes[route_name]["targets"]
)
assert any(
target.startswith("local_") for target in routes["manual_local_qwen"]["targets"]
)

8
testing/tests/test_hermes_cli_dispatch_runtime.py
View File

@ -161,7 +161,7 @@ def test_ready_dispatch_loop_submits_and_reaps_failed_workers(
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db))
monkeypatch.setattr(lanes, "RESULT_SCHEMA_PATH", tmp_path / "schema.json")
monkeypatch.setattr(lanes, "recover_orphans", lambda: None)
monkeypatch.setattr(lanes, "start_metrics_server", lambda port=None: None)
monkeypatch.setattr(lanes, "start_metrics_server", lambda port=None, **_kwargs: None)
ready = lanes.KanbanCapabilities(True, True, True)
monkeypatch.setattr(lanes, "initialize_kanban_capabilities", lambda _db: ready)
monkeypatch.setattr(lanes, "refresh_kanban_capabilities", lambda _db: ready)
@ -197,7 +197,7 @@ def test_deferred_dispatch_health_never_claims_new_work(
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db))
monkeypatch.setattr(lanes, "RESULT_SCHEMA_PATH", tmp_path / "schema.json")
monkeypatch.setattr(lanes, "recover_orphans", lambda: None)
monkeypatch.setattr(lanes, "start_metrics_server", lambda port=None: None)
monkeypatch.setattr(lanes, "start_metrics_server", lambda port=None, **_kwargs: None)
deferred = lanes.KanbanCapabilities(True, False, False)
monkeypatch.setattr(lanes, "initialize_kanban_capabilities", lambda _db: deferred)
monkeypatch.setattr(lanes, "refresh_kanban_capabilities", lambda _db: deferred)
@ -253,7 +253,7 @@ def test_two_loop_api_transition_refreshes_dispatch_and_health(
monkeypatch.setattr(lanes, "RESULT_SCHEMA_PATH", tmp_path / "schema.json")
recoveries = []
monkeypatch.setattr(lanes, "recover_orphans", lambda: recoveries.append("recover"))
monkeypatch.setattr(lanes, "start_metrics_server", lambda port=None: None)
monkeypatch.setattr(lanes, "start_metrics_server", lambda port=None, **_kwargs: None)
monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0)
monkeypatch.setattr(lanes, "maybe_gc_lane_artifacts", lambda: 0)
claim_states = []
@ -372,7 +372,7 @@ def test_dispatch_loop_survives_board_registry_scan_failure(
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db))
monkeypatch.setattr(lanes, "RESULT_SCHEMA_PATH", tmp_path / "schema.json")
monkeypatch.setattr(lanes, "recover_orphans", lambda: None)
monkeypatch.setattr(lanes, "start_metrics_server", lambda port=None: None)
monkeypatch.setattr(lanes, "start_metrics_server", lambda port=None, **_kwargs: None)
ready = lanes.KanbanCapabilities(True, True, True)
monkeypatch.setattr(lanes, "initialize_kanban_capabilities", lambda _db: ready)
monkeypatch.setattr(lanes, "refresh_kanban_capabilities", lambda _db: ready)

16
testing/tests/test_hermes_cli_execution_edges.py
View File

@ -16,6 +16,22 @@ class _Connection:
return None
def test_quality_retry_uses_sol_xhigh_before_frontier():
"""A failed Sol/high plan gets its retained reasoning path before Astra."""
previous = lanes.Route(
"codex", "gpt-5.6-sol", "high", "codex-high", "classifier", "plan", 1, (), "advanced"
)
repeated = lanes.Route(
"codex", "gpt-5.6-sol", "high", "codex-high", "classifier", "same plan", 1, (), "advanced"
)
assert lanes._quality_retry_assignee(previous, repeated) == "cli-codex-advanced-xhigh"
exhausted = lanes.Route(
"codex", "gpt-5.6-sol", "xhigh", "codex-xhigh", "classifier", "same plan", 1, (), "advanced"
)
assert lanes._quality_retry_assignee(exhausted, exhausted) == "cli-codex-frontier-xhigh"
def test_missing_claim_is_a_noop(tmp_path: Path, monkeypatch):
db = SimpleNamespace(
scoped_current_board=lambda _board: nullcontext(),

9
testing/tests/test_hermes_cli_fallback.py
View File

@ -143,7 +143,14 @@ def test_goal_card_continues_after_local_judge_rejects_progress(
assert calls[0][1]["metadata"]["goal_turn"] == 2
assert any("Goal completion rejected; continuing turn 2/3" in item for item in comments)
assert any("Goal route 2/3: codex/gpt-5.6-sol at xhigh" in item for item in comments)
assert route_calls[2][1]["exclude_provider"] == "claude"
# The authentication failure keeps Claude excluded at the native-health
# boundary. Once the retry stays on Codex, quality escalation re-selects
# the higher lane without inventing another provider outage.
assert any(
call[1].get("exclude_provider") == "claude" for call in route_calls
)
assert route_calls[-1][0] == "cli-codex-frontier-xhigh"
assert route_calls[-1][1].get("exclude_provider") is None
assert "prior rejected reports" in judge_contexts[1]
assert "commit, push, and remote verification are missing" in judge_contexts[1]
candidates = sorted(

161
testing/tests/test_hermes_cli_lane_routing.py
View File

@ -23,10 +23,12 @@ SPEC.loader.exec_module(lanes)
class SwitchyardResponse:
"""Return a tier header and independently resolved broker content."""
def __init__(self, selected: str, resolved_target: str):
def __init__(
self, selected: str, resolved_target: str, rationale: str = "test route"
):
self.headers = {
"x-model-router-selected-model": selected,
"x-model-router-rationale": "test route",
"x-model-router-rationale": rationale,
}
self.resolved_target = resolved_target
@ -107,3 +109,158 @@ def test_generic_claude_worker_rejects_an_unadvertised_alias():
),
catalog_model_allowed=lambda *_args: False,
)
@pytest.mark.parametrize(
("effort", "capability"),
[("low", "economy"), ("medium", "balanced"), ("high", "advanced"), ("xhigh", "advanced")],
)
def test_legacy_auto_derives_capability_from_its_effort(effort: str, capability: str):
"""Legacy AUTO keeps its documented effort-to-capability compatibility map."""
route = lanes.select_route(
"Legacy worker route.",
"cli-auto",
open_request=lambda *_args, **_kwargs: SwitchyardResponse(
f"worker/codex/auto/{effort}", f"worker/codex/gpt-5.6-sol/{effort}"
),
)
assert route.capability == capability
def test_unresolved_capability_selector_never_reaches_native_cli():
"""A broker receipt must resolve every `auto-*` selector to a real model."""
with pytest.raises(RuntimeError, match="current provider model"):
lanes.select_route(
"Exceptional ambiguity.",
"cli-auto",
open_request=lambda *_args, **_kwargs: SwitchyardResponse(
"worker/codex/auto-frontier/high", "worker/codex/auto-frontier/high"
),
)
def test_health_guard_preserves_capability_and_effort_on_alternate_provider():
"""An excluded provider cannot erase the selected frontier floor."""
calls = []
def request(request, **_kwargs):
calls.append(json.loads(request.data))
if len(calls) == 1:
return SwitchyardResponse(
"worker/codex/auto-frontier/high", "worker/codex/gpt-6-astra/high"
)
return SwitchyardResponse(
"worker/claude/auto-frontier/high", "worker/claude/claude-fable-5/high"
)
route = lanes.select_route(
"Exceptional coupled architecture.",
"cli-auto",
exclude_provider="codex",
open_request=request,
)
assert calls[1]["model"] == "atlas/worker/manual/claude/frontier/high"
assert (route.provider, route.capability, route.effort) == ("claude", "frontier", "high")
def test_health_guard_rejects_a_capability_downgrade():
"""A manually guarded retry must satisfy the original capability floor."""
responses = iter(
[
SwitchyardResponse(
"worker/codex/auto-frontier/high", "worker/codex/gpt-6-astra/high"
),
SwitchyardResponse(
"worker/claude/auto-advanced/high", "worker/claude/claude-opus-5/high"
),
]
)
with pytest.raises(RuntimeError, match="preserve the requested frontier/high"):
lanes.select_route(
"Exceptional coupled architecture.",
"cli-auto",
exclude_provider="codex",
open_request=lambda *_args, **_kwargs: next(responses),
)
def test_switchyard_request_fallback_reselects_the_original_floor():
"""A frontier timeout cannot return Sol merely because both use xhigh."""
calls = []
def request(request, **_kwargs):
calls.append(json.loads(request.data))
if len(calls) == 1:
return SwitchyardResponse(
"worker/codex/auto-advanced/xhigh",
"worker/codex/gpt-5.6-sol/xhigh",
(
"worker/codex/auto-frontier/xhigh was unavailable; fell back to "
"worker/codex/auto-advanced/xhigh"
),
)
return SwitchyardResponse(
"worker/claude/auto-frontier/xhigh",
"worker/claude/claude-astra-6/xhigh",
)
route = lanes.select_route(
"Resolve exceptional coupled architecture ambiguity.",
"cli-auto",
open_request=request,
catalog_model_allowed=lambda provider, model: (
provider == "claude" and model == "claude-astra-6"
),
)
assert calls[1]["model"] == "atlas/worker/manual/claude/frontier/xhigh"
assert (route.provider, route.capability, route.effort) == (
"claude", "frontier", "xhigh"
)
assert route.classifier == "switchyard-classifier-fallback-floor-guard"
def test_worker_route_keeps_sol_xhigh_distinct_from_astra_high():
"""Capability arrives from the selector rather than being inferred from effort."""
advanced = lanes.select_route(
"Ordinary complex implementation.",
"cli-auto",
open_request=lambda *_args, **_kwargs: SwitchyardResponse(
"worker/codex/auto-advanced/xhigh",
"worker/codex/gpt-5.6-sol/xhigh",
),
)
frontier = lanes.select_route(
"Resolve exceptional coupled architecture ambiguity.",
"cli-auto",
open_request=lambda *_args, **_kwargs: SwitchyardResponse(
"worker/codex/auto-frontier/high",
"worker/codex/gpt-6-astra/high",
),
)
assert (advanced.model, advanced.capability, advanced.effort) == (
"gpt-5.6-sol", "advanced", "xhigh"
)
assert (frontier.model, frontier.capability, frontier.effort) == (
"gpt-6-astra", "frontier", "high"
)
def test_capability_manual_override_uses_exact_switchyard_route():
"""A manual capability floor is represented separately from reasoning effort."""
captured = []
def request(request, **_kwargs):
captured.append(json.loads(request.data))
return SwitchyardResponse(
"worker/codex/auto-frontier/high",
"worker/codex/gpt-6-astra/high",
)
route = lanes.select_route("Review the coupled architecture.", "cli-codex-frontier-high", open_request=request)
assert captured[0]["model"] == "atlas/worker/manual/codex/frontier/high"
assert route.capability == "frontier"

2
testing/tests/test_hermes_cli_lanes_toolchain.py
View File

@ -75,7 +75,7 @@ def test_switchyard_network_boundary_allows_metrics_scraping():
"podSelector": {"matchLabels": {"app": "server"}},
}
]
and rule.get("ports") == [{"protocol": "TCP", "port": 9005}]
and {entry.get("port") for entry in rule.get("ports", [])} == {9005, 9009}
for rule in isolation["spec"]["ingress"]
)

67
testing/tests/test_hermes_cli_quota_failover.py
View File

@ -212,7 +212,54 @@ def test_auto_failover_preserves_effort_and_records_cooldown(
assert claude_health["state"] == "available"
def test_auto_failover_escalates_when_classifier_downgrades_effort(
def test_timeout_failover_preserves_frontier_capability_floor(
tmp_path: Path, monkeypatch
):
"""A provider timeout cannot replace Astra-class work with Sol-class work."""
task = SimpleNamespace(
id="t_frontier_timeout", status="running", result=None, current_run_id=55,
assignee="cli-auto", max_runtime_seconds=120,
)
comments: list = []
calls: list = []
board = _lane_board(tmp_path, task, comments, calls)
_isolate_lane(tmp_path, monkeypatch, board)
route_calls: list = []
def select_route(_prompt, lane, **kwargs):
route_calls.append((lane, kwargs))
if len(route_calls) == 1:
return lanes.Route(
"codex", "gpt-6-astra", "high", "codex-high", "classifier", "exceptional ambiguity", 1, (), "frontier"
)
if lane == "cli-auto":
# A misleading retry classifier may choose more effort, but less capability.
return lanes.Route(
"claude", "claude-opus-5", "xhigh", "claude-xhigh", "classifier", "timeout retry", 1, (), "advanced"
)
assert lane == "cli-claude-frontier-high"
return lanes.Route(
"claude", "claude-opus-5", "high", "claude-high", "manual", "preserved frontier", 1, (), "frontier"
)
monkeypatch.setattr(lanes, "select_route", select_route)
monkeypatch.setattr(lanes, "record_provider_fallback", lambda *_args: None)
reports = [
lanes.ProcessResult(1, "request timed out", None, True),
lanes.ProcessResult(0, "done", dict(COMPLETED_RESULT), False),
]
monkeypatch.setattr(lanes, "run_provider", lambda *_args, **_kwargs: reports.pop(0))
lanes.execute_claim("cassandra", "t_frontier_timeout")
assert calls and calls[0][0] == "complete"
assert [lane for lane, _ in route_calls] == [
"cli-auto", "cli-auto", "cli-claude-frontier-high"
]
assert any("restored the original frontier/high floor" in item for item in comments)
def test_auto_failover_preserves_exact_floor_when_classifier_promotes(
tmp_path: Path, monkeypatch
):
task = SimpleNamespace(
@ -237,13 +284,13 @@ def test_auto_failover_escalates_when_classifier_downgrades_effort(
"switchyard-classifier", "vote", 1, (),
)
if lane == "cli-auto" and kwargs.get("exclude_provider") == "codex":
# Jetson reclassifies but picks a lower effort than the original
# route; the lane must never let capacity failover downgrade it.
# A timeout or quota failure cannot justify paying for Astra/xhigh
# when the failed task was ordinary Sol/high work.
return lanes.Route(
"claude", "claude-haiku-4-5", "low", "claude-low",
"switchyard-classifier", "vote", 1, (),
"claude", "gpt-6-astra", "xhigh", "claude-xhigh",
"switchyard-classifier", "vote", 1, (), "frontier",
)
assert lane == "cli-claude-high"
assert lane == "cli-claude-advanced-high"
return _route("claude", "high")
monkeypatch.setattr(lanes, "select_route", select_route)
@ -268,13 +315,13 @@ def test_auto_failover_escalates_when_classifier_downgrades_effort(
assert [lane for lane, _ in route_calls] == [
"cli-auto",
"cli-auto",
"cli-claude-high",
"cli-claude-advanced-high",
]
# The provider fallback is recorded against the final, effort-preserved
# selection, not the transient low-effort classification.
# The provider fallback is recorded against the restored Sol/high floor,
# not the transient Astra/xhigh classifier result.
assert fallbacks == [("codex", "claude", "quota")]
assert any(
"escalated to the original high" in item
"restored the original advanced/high floor" in item
for item in comments
)
assert any(

2
testing/tests/test_hermes_cli_routing.py
View File

@ -96,7 +96,7 @@ def test_classifier_cannot_select_a_freshly_excluded_provider():
assert selected.effort == "xhigh"
assert selected.classifier == "switchyard-classifier-health-guard"
assert payloads[0]["model"] == "atlas/worker/auto/maximum"
assert payloads[1]["model"] == "atlas/worker/manual/codex/xhigh"
assert payloads[1]["model"] == "atlas/worker/manual/codex/advanced/xhigh"
assert "claude provider is unavailable" in payloads[0]["messages"][0]["content"]

138
testing/tests/test_hermes_coordinator.py
View File

@ -7,6 +7,7 @@ import subprocess
import tomllib
from pathlib import Path
import pytest
import yaml
from testing.tests.test_hermes_coordinator_support import (
@ -16,12 +17,13 @@ from testing.tests.test_hermes_coordinator_support import (
coordinator,
routing,
)
from model_evaluation_evidence import metadata_fingerprint
def test_model_version_and_quality_selection_handle_new_and_small_models():
assert routing.model_version("claude-3-5-sonnet-20241022") == (3, 5)
assert routing.model_version("gpt-5.7-terra") == (5, 7)
codex = ["gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.7-luna", "gpt-5.3-codex-spark"]
codex = ["gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna", "gpt-5.3-codex-spark"]
assert routing.choose_codex_model(codex) == "gpt-5.6-sol"
assert routing.choose_codex_model(codex, balanced=True) == "gpt-5.6-terra"
assert routing.choose_codex_model(codex + ["gpt-5.7-terra"]) == "gpt-5.6-sol"
@ -29,7 +31,7 @@ def test_model_version_and_quality_selection_handle_new_and_small_models():
claude = ["claude-opus-4.8", "claude-haiku-5", "claude-sonnet-5"]
assert routing.choose_claude_model(claude) == "claude-opus-4.8"
assert routing.choose_codex_for_effort(codex, "low") == "gpt-5.7-luna"
assert routing.choose_codex_for_effort(codex, "low") == "gpt-5.6-luna"
assert routing.choose_codex_for_effort(codex, "medium") == "gpt-5.6-terra"
assert routing.choose_codex_for_effort(codex, "xhigh") == "gpt-5.6-sol"
assert routing.choose_claude_for_effort(claude, "low") == "claude-haiku-5"
@ -42,8 +44,8 @@ def test_empty_catalog_retains_current_models():
assert routing.choose_claude_model([], "claude-opus-5") == "claude-opus-5"
def test_dynamic_catalog_resolves_new_models_and_preserves_last_known_good():
"""Stable Switchyard aliases follow live releases and survive catalog outages."""
def test_dynamic_catalog_holds_unreviewed_releases_and_preserves_lkg():
"""New IDs stay pending until evidence while outages retain known routes."""
codex = routing.Catalog(
"openai-codex",
["gpt-5.7-luna", "gpt-5.7-terra", "gpt-5.7-sol"],
@ -60,22 +62,13 @@ def test_dynamic_catalog_resolves_new_models_and_preserves_last_known_good():
)
current = routing.build_routing_catalog(codex, claude)
assert (
with pytest.raises(ValueError, match="selector 'auto'"):
catalog_resolver.resolve_model("codex", "auto", "high", current)
== "gpt-5.7-sol"
)
assert (
with pytest.raises(ValueError, match="selector 'terra'"):
catalog_resolver.resolve_model("codex", "terra", "medium", current)
== "gpt-5.7-terra"
)
assert (
catalog_resolver.resolve_model("claude", "auto", "xhigh", current)
== "claude-opus-6"
)
assert (
assert current["providers"]["codex"]["candidates"]["gpt-5.7-sol"]["proposed_role"] is None
with pytest.raises(ValueError, match="selector 'sonnet'"):
catalog_resolver.resolve_model("claude", "sonnet", "high", current)
== "claude-sonnet-6"
)
unavailable = routing.Catalog("openai-codex", [], False, True, "degraded")
preserved = routing.build_routing_catalog(unavailable, unavailable, current)
@ -139,12 +132,11 @@ def test_dynamic_catalog_uses_metadata_for_unfamiliar_models_and_efforts():
catalog = routing.build_routing_catalog(codex, claude)
resolved = catalog["providers"]["codex"]["resolved"]
assert resolved["xhigh"] == "gpt-6.2-orbit"
assert resolved["low"] == "gpt-6.2-quick"
assert resolved["medium"] == "gpt-6.2-balanced"
assert resolved["high"] == "gpt-6.2-orbit"
assert resolved == {effort: "" for effort in routing.EFFORTS}
with pytest.raises(ValueError, match="auto-frontier"):
catalog_resolver.resolve_model("codex", "auto-frontier", "xhigh", catalog)
assert "gpt-6.2-high-only" in catalog["providers"]["codex"]["candidates"]
assert catalog["providers"]["codex"]["candidates"]["gpt-6.2-high-only"]["tier"] == "advanced"
assert catalog["providers"]["codex"]["candidates"]["gpt-6.2-high-only"]["proposed_role"] == "advanced"
def test_catalog_accepts_codex_effort_records_and_generalist_image_input():
@ -171,7 +163,22 @@ def test_catalog_accepts_codex_effort_records_and_generalist_image_input():
codex, routing.Catalog("anthropic", [], True, True, "connected")
)
assert catalog["providers"]["codex"]["resolved"]["xhigh"] == "gpt-7-nova"
assert catalog["providers"]["codex"]["resolved"]["xhigh"] == ""
assert catalog["providers"]["codex"]["candidates"]["gpt-7-nova"]["proposed_role"] == "frontier"
def test_provider_metadata_cannot_self_attest_evaluation():
"""Only the local evidence store may add an evaluated capability role."""
models, metadata = routing.model_records([{
"id": "gpt-9-mystery", "evaluated_capability_role": "frontier",
"evaluation_provenance": "untrusted", "description": "Routine tasks",
}])
catalog = routing.build_routing_catalog(
routing.Catalog("openai-codex", models, True, True, "connected", metadata),
routing.Catalog("anthropic", [], True, True, "connected"),
)
assert "evaluated_capability_role" not in metadata["gpt-9-mystery"]
assert catalog["providers"]["codex"]["capability_pools"]["frontier"] == []
def test_dynamic_catalog_excludes_hidden_internal_and_specialist_records():
@ -209,14 +216,14 @@ def test_dynamic_catalog_excludes_hidden_internal_and_specialist_records():
catalog = routing.build_routing_catalog(codex, claude)
provider = catalog["providers"]["codex"]
assert provider["resolved"]["xhigh"] == "gpt-6.2-public"
assert provider["resolved"]["xhigh"] == ""
assert provider["candidates"]["gpt-6.2-hidden"]["eligible"] is False
assert provider["candidates"]["gpt-6.2-internal"]["eligible"] is False
assert provider["candidates"]["gpt-6.2-image"]["eligible"] is False
def test_live_codex_metadata_selects_luna_terra_and_astra_by_effort():
"""Actual Codex model/list fields route generic effort without name guesses."""
def test_live_codex_metadata_separates_advanced_sol_from_frontier_astra():
"""Actual Codex records preserve Sol AUTO and expose Astra explicitly."""
fixture = Path(__file__).parents[1] / "fixtures/hermes/codex-0.154-visible-models.json"
records = json.loads(fixture.read_text(encoding="utf-8"))
models, metadata = routing.model_records(records)
@ -227,13 +234,20 @@ def test_live_codex_metadata_selects_luna_terra_and_astra_by_effort():
assert catalog["providers"]["codex"]["resolved"] == {
"low": "gpt-5.6-luna",
"medium": "gpt-5.6-terra",
"high": "gpt-6-astra",
"xhigh": "gpt-6-astra",
"high": "gpt-5.6-sol",
"xhigh": "gpt-5.6-sol",
}
assert catalog_resolver.resolve_model("codex", "auto-frontier", "xhigh", catalog) == "gpt-6-astra"
assert catalog["providers"]["codex"]["capability_pools"] == {
"economy": ["gpt-5.6-luna"],
"balanced": ["gpt-5.6-terra"],
"advanced": ["gpt-5.6-sol"],
"frontier": ["gpt-6-astra"],
}
def test_live_default_flagship_replaces_previous_advanced_route():
"""A provider's current default wins over an equally strongest old model."""
def test_live_default_does_not_promote_an_unverified_frontier_candidate():
"""A default flag alone does not bypass the bounded evidence gate."""
previous = {
"providers": {"codex": {"models": ["gpt-astra"], "resolved": {
effort: "gpt-astra" for effort in routing.EFFORTS
@ -249,7 +263,9 @@ def test_live_default_flagship_replaces_previous_advanced_route():
routing.Catalog("openai-codex", list(metadata), True, True, "connected", metadata),
routing.Catalog("anthropic", [], True, True, "connected"), previous,
)
assert catalog["providers"]["codex"]["resolved"]["xhigh"] == "gpt-nova"
assert catalog["providers"]["codex"]["resolved"]["xhigh"] == ""
with pytest.raises(ValueError, match="auto-frontier"):
catalog_resolver.resolve_model("codex", "auto-frontier", "xhigh", catalog)
def test_live_claude_effort_gap_uses_nearest_higher_declared_tier():
@ -275,11 +291,42 @@ def test_live_claude_effort_gap_uses_nearest_higher_declared_tier():
routing.Catalog("anthropic", list(metadata), True, True, "connected", metadata),
)
assert catalog["providers"]["claude"]["resolved"] == {
"low": "sonnet", "medium": "sonnet", "high": "fable", "xhigh": "fable",
"low": "sonnet", "medium": "sonnet", "high": "opus", "xhigh": "opus",
}
assert catalog_resolver.resolve_model("claude", "auto-frontier", "xhigh", catalog) == "fable"
assert catalog["providers"]["claude"]["candidates"]["haiku"]["eligible"] is False
def test_live_unavailable_capability_does_not_promote_to_a_higher_role():
"""A fresh disabled economy model is an availability failure, not a tier gap."""
metadata = {
"gpt-5.6-luna": {"enabled": False},
"gpt-5.6-terra": {"supported_reasoning_efforts": ["low", "medium"]},
"gpt-5.6-sol": {"supported_reasoning_efforts": ["low", "medium", "high", "xhigh"]},
}
catalog = routing.build_routing_catalog(
routing.Catalog("openai-codex", list(metadata), True, True, "connected", metadata),
routing.Catalog("anthropic", [], True, True, "connected"),
)
assert catalog["providers"]["codex"]["resolved"]["low"] == ""
with pytest.raises(ValueError, match="selector 'auto'"):
catalog_resolver.resolve_model("codex", "auto", "low", catalog)
def test_legacy_aliases_are_exact_and_keep_fable_frontier():
"""Compatibility cannot admit new names merely because they contain a family word."""
unknown, _ = routing._select_tier_model(
"codex", ["gpt-9-luna-experimental"], {}, "economy", "low", "",
legacy_compat=True,
)
fable, _ = routing._select_tier_model(
"claude", ["claude-fable-5"], {}, "frontier", "xhigh", "",
legacy_compat=True,
)
assert unknown == ""
assert fable == "claude-fable-5"
def test_ambiguous_live_catalog_exposes_an_unresolved_route():
"""An unfamiliar live model cannot revive a retired LKG model."""
previous = {
@ -303,6 +350,31 @@ def test_ambiguous_live_catalog_exposes_an_unresolved_route():
assert catalog["providers"]["codex"]["candidates"]["gpt-7.0-mystery"]["tier"] is None
def test_frontier_selector_never_demotes_to_an_advanced_model():
"""A missing frontier route is explicit while advanced still resolves Sol."""
catalog = routing.build_routing_catalog(
routing.Catalog("openai-codex", ["gpt-5.6-sol"], True, True, "connected"),
routing.Catalog("anthropic", [], True, True, "connected"),
)
assert catalog_resolver.resolve_model("codex", "auto-advanced", "xhigh", catalog) == "gpt-5.6-sol"
with pytest.raises(ValueError, match="auto-frontier"):
catalog_resolver.resolve_model("codex", "auto-frontier", "xhigh", catalog)
def test_verified_evaluation_routes_an_unfamiliar_model_without_name_rules():
"""Only a current, positive evaluation may classify an ambiguous live model."""
catalog = routing.build_routing_catalog(
routing.Catalog("openai-codex", ["gpt-9-mystery"], True, True, "connected"),
routing.Catalog("anthropic", [], True, True, "connected"),
evaluations={"evaluations": {"codex": {"gpt-9-mystery": {
"proposed_role": "frontier", "result": "pass", "role_fit": "verified",
"eval_version": "capability-v1", "metadata_fingerprint": metadata_fingerprint("gpt-9-mystery", {}),
}}}},
)
assert catalog_resolver.resolve_model("codex", "auto-frontier", "high", catalog) == "gpt-9-mystery"
assert catalog["providers"]["codex"]["candidates"]["gpt-9-mystery"]["reason"] == "bounded-representative-eval-v1"
def test_live_removal_clears_legacy_selector_instead_of_mapping_to_new_family():
"""A live authoritative list cannot revive removed legacy selectors."""
previous = {
@ -327,7 +399,7 @@ def test_live_removal_clears_legacy_selector_instead_of_mapping_to_new_family():
catalog = routing.build_routing_catalog(codex, claude, previous)
assert catalog["providers"]["codex"]["tiers"]["sol"] == ""
assert catalog["providers"]["codex"]["resolved"]["xhigh"] == "gpt-6.2-astra"
assert catalog["providers"]["codex"]["resolved"]["xhigh"] == ""
def test_degraded_catalog_preserves_lkg_after_live_removal():

2
testing/tests/test_hermes_execution_pool_assignment.py
View File

@ -128,7 +128,7 @@ def test_additive_patch_replaces_local_lane_without_touching_base_deployment():
"HERMES_CLI_LANE_OWNED_WORKSPACES_ONLY": "true",
"HERMES_CLI_LANE_CONCURRENCY": "2",
}
assert pool["resources"]["requests"] == {"cpu": "50m", "memory": "128Mi"}
assert pool["resources"]["requests"] == {"cpu": "50m", "memory": "64Mi"}
access = next(item for item in pool["volumeMounts"] if item["name"] == "runtime-access")
assert access["subPath"] == "execution-pool-key" and access["readOnly"] is True

70
testing/tests/test_hermes_execution_pool_coordinator_v2.py
View File

@ -140,6 +140,7 @@ def store_with_assignment(tmp_path, exact_binding=None):
def test_assignment_payload_is_bounded_registry_derived_and_typed(monkeypatch):
monkeypatch.setattr(coordinator.supervisor_state, "get_child", lambda *_args: None)
monkeypatch.setattr(
coordinator,
"resolve_assignment",
@ -161,6 +162,30 @@ def test_assignment_payload_is_bounded_registry_derived_and_typed(monkeypatch):
coordinator.assignment_payload(kanban, object(), item, "metis")
def test_assignment_payload_requires_state_and_native_parents_for_continuations(monkeypatch):
lineage = coordinator.supervisor_lineage.Lineage(
"root", "wt/root", "https://scm.bstein.dev/titan/metis/pulls/7", "metis", "main"
)
child = {"lineage": lineage, "parent_task_id": "review", "kind": "repair"}
monkeypatch.setattr(coordinator.supervisor_state, "get_child", lambda *_args: child)
monkeypatch.setattr(coordinator.supervisor_state, "get_root", lambda *_args: lineage)
monkeypatch.setattr(
coordinator, "resolve_assignment",
lambda *_args: ("https://scm.bstein.dev/titan/metis.git", "wrong", "wrong"),
)
kanban = SimpleNamespace(
build_worker_context=lambda *_args: "objective",
get_task=lambda _connection, task_id: object() if task_id == "root" else None,
)
accepted = task(id="repair", parents=("root", "review"))
value = coordinator.assignment_payload(kanban, object(), accepted, "metis")
assert value["branch"] == "wt/root"
assert value["continuation_kind"] == "repair"
rejected = task(id="repair", parents=("root",))
with pytest.raises(RuntimeError, match="parents"):
coordinator.assignment_payload(kanban, object(), rejected, "metis")
def test_poll_uses_per_ordinal_key_for_empty_and_active_assignment(tmp_path):
store = protocol.PoolStore(tmp_path / "pool.db")
pool = coordinator.Coordinator(MASTER, store)
@ -212,6 +237,51 @@ def test_finalize_canonicalizes_integer_run_and_reconcile_does_not_reexecute(
assert store.active_assignments() == []
def test_finalize_records_a_verified_root_before_marking_the_task_complete(tmp_path, monkeypatch):
live = task()
kanban = install_kanban(monkeypatch, [live], tmp_path)
store = protocol.PoolStore(tmp_path / "pool.db")
assigned = assignment_payload(root_task_id="t_deadbeef")
store.add(binding(), assigned)
events: list[str] = []
monkeypatch.setattr(
coordinator.supervisor_state, "record_submission", lambda *_args: events.append("state")
)
complete = kanban.complete_task
def checked_complete(*args, **kwargs):
assert events == ["state"]
return complete(*args, **kwargs)
kanban.complete_task = checked_complete
pool = coordinator.Coordinator(MASTER, store)
pool.result(signed("result", binding(), {
"structured": dict(STRUCTURED), "returncode": 0,
"scm_submission": {
"branch": "wt/t_deadbeef", "head": "a" * 40,
"pull_request": "https://scm.bstein.dev/titan/metis/pulls/7",
},
}))
assert events == ["state"]
assert len(kanban.completed) == 1
def test_finalize_rejects_a_submission_that_changes_its_assigned_branch(tmp_path, monkeypatch):
kanban = install_kanban(monkeypatch, [task()], tmp_path)
store = protocol.PoolStore(tmp_path / "pool.db")
store.add(binding(), assignment_payload(root_task_id="t_deadbeef"))
pool = coordinator.Coordinator(MASTER, store)
with pytest.raises(protocol.ProtocolError, match="does not match"):
pool.result(signed("result", binding(), {
"structured": dict(STRUCTURED), "returncode": 0,
"scm_submission": {
"branch": "attacker/ref", "head": "a" * 40,
"pull_request": "https://scm.bstein.dev/titan/metis/pulls/7",
},
}))
assert not kanban.completed
def test_finalize_fences_stale_run_and_blocks_failed_result_exactly(tmp_path, monkeypatch):
stale_task = task(current_run_id=24)
kanban = install_kanban(monkeypatch, [stale_task], tmp_path)

7
testing/tests/test_hermes_execution_pool_mediator.py
View File

@ -409,9 +409,10 @@ def test_failed_clone_never_deletes_unmanaged_state(tmp_path, monkeypatch):
def test_draft_reuse_create_and_submit_gates(tmp_path, monkeypatch):
existing = json.dumps([{"html_url": "https://scm/pulls/1"}]).encode()
existing = json.dumps([{"number": 1, "html_url": "https://scm/pulls/1", "head": {"ref": "wt/task"}, "base": {"ref": "main"}}]).encode()
monkeypatch.setattr(scm.scm_broker_client, "read", lambda _path: existing)
assert scm.Boundary._draft("metis", "wt/task", "main", "a" * 40, "t", "b") == "https://scm/pulls/1"
monkeypatch.setattr(scm.scm_broker_client, "update_draft", lambda *_a, **_k: json.dumps({"html_url": "https://scm/pulls/1"}).encode())
assert scm.Boundary._draft("metis", "wt/task", "main", "a" * 40, "t", "b", "grant") == "https://scm/pulls/1"
monkeypatch.setattr(scm.scm_broker_client, "read", lambda _path: b"[]")
monkeypatch.setattr(
scm.scm_broker_client, "create_draft",
@ -435,6 +436,8 @@ def test_draft_reuse_create_and_submit_gates(tmp_path, monkeypatch):
monkeypatch.setattr(scm, "_run", run)
monkeypatch.setattr(boundary, "_draft", lambda *_a: "https://scm/pulls/3")
monkeypatch.setattr(boundary, "_grant", lambda *_a: "signed-grant")
monkeypatch.setattr(scm.scm_broker_client, "register_task", lambda _grant: b'{"registered":true}')
result = boundary.submit(assignment(), {"title": "safe", "body": "evidence"})
assert result["pull_request"] == "https://scm/pulls/3"
outputs["status"] = "?? untracked"

50
testing/tests/test_hermes_execution_pool_placement.py
View File

@ -10,6 +10,8 @@ inside the 10m Flux health window for the whole ``hermes`` app.
from __future__ import annotations
import json
import subprocess
import sys
import yaml
@ -195,3 +197,51 @@ def test_the_rendered_pool_configmap_carries_every_mounted_module():
)
assert set(modules) <= keys, "a pool module is missing from its own mount"
assert json.dumps(sorted(keys))
def test_rendered_pool_configmap_imports_its_own_route_dependencies(tmp_path):
"""Workers import routing code only from their mounted ConfigMap payload."""
rendered = subprocess.run(
["kustomize", "build", str(HERMES)],
check=True,
capture_output=True,
text=True,
)
configmap = next(
item
for item in yaml.safe_load_all(rendered.stdout)
if item
and item.get("kind") == "ConfigMap"
and item.get("metadata", {}).get("name", "").startswith(
"hermes-execution-pool-"
)
)
for name, content in configmap["data"].items():
if name.endswith(".py"):
(tmp_path / name).write_text(content, encoding="utf-8")
result = subprocess.run(
[
sys.executable,
"-I",
"-c",
f"import sys; sys.path.insert(0, {str(tmp_path)!r}); import cli_lane_routing",
],
cwd=tmp_path,
capture_output=True,
text=True,
)
assert result.returncode == 0, result.stderr
pod = worker()["spec"]["template"]["spec"]
container = pod["containers"][0]
assert {
"name": "routing-catalog",
"mountPath": "/routing-catalog",
"readOnly": True,
} in container["volumeMounts"]
volume = next(item for item in pod["volumes"] if item["name"] == "routing-catalog")
assert volume["persistentVolumeClaim"] == {
"claimName": "hermes-routing-catalog",
"readOnly": True,
}

89
testing/tests/test_hermes_execution_pool_submission.py
View File

@ -1,11 +1,4 @@
"""Retry-submission contracts against the creation-only Atlas SCM broker.
The broker deliberately permits only new namespaced branch creation, so a pool
retry that adds commits to an already-published branch could never submit: the
push was refused, the worker unwound, and the run's actual work was discarded
along with it. Submission now targets a fresh attempt- or content-scoped ref, and
a refused submission downgrades the result instead of losing it.
"""
"""Continuing-task submission contracts against the signed Atlas SCM broker."""
from __future__ import annotations
@ -35,20 +28,18 @@ from testing.tests.test_hermes_execution_pool_mediator import ( # noqa: E402
)
def attempt_assignment(attempt):
def attempt_assignment(attempt, continuation_kind=""):
"""One signed assignment envelope bound to an exact retry attempt."""
return protocol.sign_envelope(
KEY, "assignment", binding(attempt=attempt), payload()
KEY, "assignment", binding(attempt=attempt), payload(continuation_kind=continuation_kind)
)
def test_submission_refs_are_creation_only_and_attempt_scoped():
def test_submission_refs_keep_one_task_branch_across_attempts():
first = scm.submission_refs("feature/pool", 1, "a" * 40)
assert first[0] == "feature/pool"
assert first[1] == "feature/pool-attempt-1"
assert first[2] == f"feature/pool-{'a' * 12}"
retry = scm.submission_refs("feature/pool", 3, "b" * 40)
assert retry[1] == "feature/pool-attempt-3"
assert retry == first
# Every candidate stays inside a reviewed namespace the broker accepts, and
# none of them is a protected or base ref.
for candidate in first + retry:
@ -73,12 +64,12 @@ def test_the_broker_accepts_the_attempt_ref_as_a_creation_and_still_refuses_upda
scm_broker._validate_receive_pack(update, "token")
def submit_harness(tmp_path, monkeypatch, remote_refs, *, attempt=1):
def submit_harness(tmp_path, monkeypatch, remote_refs, *, attempt=1, continuation_kind=""):
"""A Boundary whose broker advertises exactly ``remote_refs``."""
monkeypatch.setattr(scm, "WORKSPACE_ROOT", tmp_path / "workspace")
monkeypatch.setattr(scm, "SCM_ROOT", tmp_path / "state")
monkeypatch.setattr(scm, "ORDINAL", 0)
exact = attempt_assignment(attempt)
exact = attempt_assignment(attempt, continuation_kind)
destination = scm.workspace_path(exact)
destination.mkdir(parents=True)
protocol.atomic_json(scm._state_path(exact), {"baseline_sha": "a" * 40})
@ -98,34 +89,67 @@ def submit_harness(tmp_path, monkeypatch, remote_refs, *, attempt=1):
monkeypatch.setattr(scm, "_run", run)
boundary = scm.Boundary(KEY)
monkeypatch.setattr(boundary, "_draft", lambda *args: f"https://scm/pulls/{args[1]}")
monkeypatch.setattr(boundary, "_grant", lambda *_args: "signed-grant")
monkeypatch.setattr(scm.scm_broker_client, "register_task", lambda _grant: b'{"registered":true}')
monkeypatch.setattr(boundary, "_draft", lambda *args, **_kwargs: f"https://scm/pulls/{args[1]}")
return boundary, exact, calls, head
def test_a_retry_publishes_an_attempt_ref_instead_of_updating_the_existing_one(
def test_a_retry_updates_the_same_task_ref(
tmp_path, monkeypatch
):
boundary, exact, calls, _head = submit_harness(
tmp_path, monkeypatch, {"wt/t_deadbeef": "c" * 40}, attempt=2
)
result = boundary.submit(exact, {"title": "retry", "body": "evidence"})
pushes = [item for item in calls if item[0] == "push"]
assert pushes == [("push", "hermes-broker", "HEAD:refs/heads/wt/t_deadbeef-attempt-2")]
assert result["branch"] == "wt/t_deadbeef-attempt-2"
assert result["pull_request"] == "https://scm/pulls/wt/t_deadbeef-attempt-2"
pushes = [item for item in calls if item[-4:] == ("push", "--no-thin", "hermes-broker", "HEAD:refs/heads/wt/t_deadbeef")]
assert len(pushes) == 1
assert result["branch"] == "wt/t_deadbeef"
assert result["pull_request"] == "https://scm/pulls/wt/t_deadbeef"
def test_an_already_published_head_is_adopted_without_a_second_push(
tmp_path, monkeypatch
):
boundary, exact, calls, head = submit_harness(
tmp_path, monkeypatch, {"wt/t_deadbeef-attempt-2": "b" * 40}, attempt=2
tmp_path, monkeypatch, {"wt/t_deadbeef": "b" * 40}, attempt=2
)
assert head == "b" * 40
result = boundary.submit(exact, {"title": "replay", "body": "evidence"})
assert [item for item in calls if item[0] == "push"] == []
assert result["branch"] == "wt/t_deadbeef-attempt-2"
assert result["pull_request"].endswith("wt/t_deadbeef-attempt-2")
assert [item for item in calls if "push" in item] == []
assert result["branch"] == "wt/t_deadbeef"
assert result["pull_request"].endswith("wt/t_deadbeef")
def test_no_change_review_returns_existing_pr_without_rewriting_its_handoff(tmp_path, monkeypatch):
"""A review at the already-published head must not mutate implementation prose."""
boundary, exact, calls, head = submit_harness(
tmp_path, monkeypatch, {"wt/t_deadbeef": "b" * 40}, continuation_kind="review"
)
existing = "https://scm.bstein.dev/titan/metis/pulls/42"
monkeypatch.setattr(
scm.scm_broker_client, "read",
lambda _path: __import__("json").dumps([{
"number": 42, "html_url": existing,
"head": {"ref": "wt/t_deadbeef"}, "base": {"ref": "main"},
}]).encode(),
)
monkeypatch.setattr(
scm.scm_broker_client, "update_draft",
lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("review rewrote PR metadata")),
)
monkeypatch.setattr(
scm.scm_broker_client, "create_draft",
lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("review created a PR")),
)
# The helper installs a simple draft stub for other cases; restore the real
# bounded discovery path for this no-change review assertion.
monkeypatch.setattr(boundary, "_draft", scm.Boundary._draft)
result = boundary.submit(exact, {"title": "Review", "body": "No changes"})
assert result["head"] == head and result["pull_request"] == existing
assert [item for item in calls if "push" in item] == []
def test_a_run_with_no_new_commits_still_reports_work_a_prior_attempt_published(
@ -144,7 +168,7 @@ def test_a_run_with_no_new_commits_still_reports_work_a_prior_attempt_published(
monkeypatch.setattr(scm, "_run", run)
result = boundary.submit(exact, {"title": "nothing new", "body": "evidence"})
assert [item for item in calls if item[0] == "push"] == []
assert [item for item in calls if "push" in item] == []
assert result["pull_request"].endswith("wt/t_deadbeef")
@ -160,11 +184,11 @@ def test_a_run_that_produced_nothing_at_all_submits_nothing(tmp_path, monkeypatc
monkeypatch.setattr(scm, "_run", run)
result = boundary.submit(exact, {})
assert [item for item in calls if item[0] == "push"] == []
assert [item for item in calls if "push" in item] == []
assert result["pull_request"] == "" and result["branch"] == "wt/t_deadbeef"
def test_submission_fails_closed_when_every_candidate_ref_is_taken(
def test_submission_updates_an_existing_task_ref_without_creating_a_retry_ref(
tmp_path, monkeypatch
):
taken = {
@ -173,9 +197,10 @@ def test_submission_fails_closed_when_every_candidate_ref_is_taken(
f"wt/t_deadbeef-{'b' * 12}": "e" * 40,
}
boundary, exact, calls, _head = submit_harness(tmp_path, monkeypatch, taken)
with pytest.raises(protocol.ProtocolError, match="every reviewed branch name"):
boundary.submit(exact, {"title": "blocked", "body": "evidence"})
assert [item for item in calls if item[0] == "push"] == []
result = boundary.submit(exact, {"title": "continued", "body": "evidence"})
pushes = [item for item in calls if item[-4:] == ("push", "--no-thin", "hermes-broker", "HEAD:refs/heads/wt/t_deadbeef")]
assert len(pushes) == 1
assert result["branch"] == "wt/t_deadbeef"
def test_remote_head_parsing_ignores_anything_that_is_not_a_branch(

7
testing/tests/test_hermes_gitea_pr_integration.py
View File

@ -168,7 +168,8 @@ def test_flux_manifest_isolates_vault_token_in_separate_broker_only():
assert pod["metadata"]["annotations"][
"vault.hashicorp.com/agent-inject-secret-gitea-token"
] == "kv/data/atlas/hermes/developer-gitea"
assert not any(
volume.get("hostPath") or volume.get("persistentVolumeClaim")
for volume in pod["spec"]["volumes"]
ledger = next(
volume for volume in pod["spec"]["volumes"] if volume["name"] == "task-ledger"
)
assert ledger["persistentVolumeClaim"]["claimName"] == "hermes-scm-task-ledger"
assert not any(volume.get("hostPath") for volume in pod["spec"]["volumes"])

158
testing/tests/test_hermes_kanban_continuation.py Normal file
View File

@ -0,0 +1,158 @@
"""Native-shaped regression tests for coordinator-owned PR continuations."""
from __future__ import annotations
from contextlib import nullcontext
import json
from pathlib import Path
import sys
import pytest
from testing.tests.test_hermes_cli_support import HERMES, _load
sys.path.insert(0, str(HERMES / "scm-common/scripts"))
state = _load("supervisor_state")
continuation = _load("kanban_continue_pr")
class NativeKanban:
"""Match live ``kanban_db``'s keyword-only create API without metadata."""
def __init__(self) -> None:
self.created: list[dict] = []
self.by_key: dict[str, str] = {}
self.comments: list[tuple] = []
def scoped_current_board(self, _board: str):
return nullcontext()
def connect(self, *, board: str):
class Connection:
def close(self):
return None
return Connection()
def get_task(self, _conn, task_id: str):
return {"id": task_id} if task_id == "root" else None
def create_task(self, _conn, *, title, body=None, assignee=None, created_by=None,
workspace_kind="scratch", workspace_path=None, branch_name=None,
tenant=None, priority=0, parents=(), triage=False,
idempotency_key=None, max_runtime_seconds=None, skills=None,
max_retries=None, goal_mode=False, goal_max_turns=None,
initial_status="running", session_id=None, board=None,
project_id=None):
assert branch_name is None
assert workspace_kind == "scratch"
assert idempotency_key
if idempotency_key in self.by_key:
return self.by_key[idempotency_key]
task_id = f"child-{len(self.created) + 1}"
self.by_key[idempotency_key] = task_id
self.created.append({
"id": task_id, "title": title, "body": body, "assignee": assignee,
"created_by": created_by, "parents": tuple(parents),
"idempotency_key": idempotency_key, "initial_status": initial_status,
})
return task_id
def add_comment(self, _conn, *args):
self.comments.append(args)
@pytest.fixture
def trusted_root(tmp_path: Path, monkeypatch):
"""Make a real SQLite board DB holding a signed root lineage record."""
board = "titan-iac"
monkeypatch.setattr(state, "KANBAN_ROOT", tmp_path / "boards")
path = state.KANBAN_ROOT / board / "kanban.db"
lineage = state.Lineage(
root_task_id="root", project="atlas-iac", branch="hermes/root",
pull_request="https://scm.bstein.dev/titan/atlas-iac/pulls/7", base_branch="main",
)
state.record_submission(board, "root", lineage, "a" * 40, path=path)
state.set_ready(board, "root", "a" * 40, path=path)
return board, lineage, path
def _pr(head: str) -> bytes:
return json.dumps({
"state": "open",
"head": {"ref": "hermes/root", "sha": head,
"repo": {"full_name": "titan/atlas-iac"}},
"base": {"ref": "main", "repo": {"full_name": "titan/atlas-iac"}},
}).encode()
def test_queue_uses_real_sqlite_lineage_and_native_create_signature(trusted_root, monkeypatch):
board, _lineage, path = trusted_root
native = NativeKanban()
monkeypatch.setattr(continuation.scm_broker_client, "read", lambda _path: _pr("b" * 40))
child, created = continuation.queue(native, board=board, root_task="root", objective="fix review")
assert (child, created) == ("child-1", True)
assert native.created[0]["parents"] == ("root",)
assert native.created[0]["initial_status"] == "running"
assert "metadata" not in native.created[0]
assert state.get_live_head(board, "root", path=path) == "b" * 40
assert state.get_child(board, child, path=path)["parent_task_id"] == "root"
with state._connect(board, path) as connection:
ready = connection.execute(
"SELECT ready_for_human_merge,ready_commit FROM supervisor_roots"
).fetchone()
assert tuple(ready) == (0, "")
same, created = continuation.queue(native, board=board, root_task="root", objective="fix review")
assert (same, created) == (child, False)
assert len(native.created) == 1
def test_interrupted_recording_recovers_through_native_idempotency(trusted_root, monkeypatch):
board, _lineage, path = trusted_root
native = NativeKanban()
monkeypatch.setattr(continuation.scm_broker_client, "read", lambda _path: _pr("a" * 40))
original = continuation.supervisor_state.record_child
monkeypatch.setattr(
continuation.supervisor_state, "record_child",
lambda *_args, **_kwargs: (_ for _ in ()).throw(OSError("simulated restart")),
)
with pytest.raises(OSError, match="simulated restart"):
continuation.queue(native, board=board, root_task="root", objective="repair flaky test")
assert len(native.created) == 1
monkeypatch.setattr(continuation.supervisor_state, "record_child", original)
child, created = continuation.queue(native, board=board, root_task="root", objective="repair flaky test")
assert (child, created) == ("child-1", True)
assert len(native.created) == 1
assert state.get_child(board, child, path=path) is not None
def test_submission_cannot_rewrite_root_and_new_head_clears_ready(trusted_root):
board, lineage, path = trusted_root
replacement = state.Lineage(
root_task_id="root", project="atlas-iac", branch="attacker/ref",
pull_request=lineage.pull_request, base_branch="main",
)
with pytest.raises(ValueError, match="immutable"):
state.record_submission(board, "root", replacement, "c" * 40, path=path)
state.record_submission(board, "repair-child", lineage, "c" * 40, path=path)
with state._connect(board, path) as connection:
row = connection.execute(
"SELECT live_pr_head,ready_for_human_merge,ready_commit FROM supervisor_roots"
).fetchone()
assert tuple(row) == ("c" * 40, 0, "")
def test_present_malformed_continuation_state_fails_closed(trusted_root):
board, _lineage, path = trusted_root
state.record_child(board, "child", "root", "root", "repair", "a" * 40, "fix", path=path)
with state._connect(board, path) as connection:
connection.execute(
"UPDATE supervisor_children SET cycle=? WHERE board=? AND child_task_id=?",
("not-a-cycle", board, "child"),
)
with pytest.raises(state.SupervisorStateError, match="malformed"):
state.get_child(board, "child", path=path)

103
testing/tests/test_hermes_kanban_supervisor.py
View File

@ -6,6 +6,8 @@ auto_supervise gate, deployment wiring, and the NULL->20 goal-turn fix.
from __future__ import annotations
from contextlib import nullcontext
import json
import sys
from types import SimpleNamespace
import pytest
@ -13,6 +15,7 @@ import yaml
from testing.tests.test_hermes_cli_support import HERMES, _agent_deployment, _load
sys.path.insert(0, str(HERMES / "scm-common/scripts"))
supervisor = _load("kanban_supervisor")
policy = supervisor.policy
@ -21,6 +24,13 @@ policy = supervisor.policy
def _isolate_ledger(tmp_path, monkeypatch):
"""Keep each test's emission ledger on an isolated, writable path."""
monkeypatch.setattr(supervisor, "LEDGER_PATH", tmp_path / "emitted.json")
monkeypatch.setattr(supervisor.supervisor_state, "KANBAN_ROOT", tmp_path / "boards")
def read(path):
project = path.split("/")[-3]
return json.dumps({"state": "open", "head": {"ref": "feature/impl", "sha": "c1",
"repo": {"full_name": f"titan/{project}"}}, "base": {"ref": "main",
"repo": {"full_name": f"titan/{project}"}}}).encode()
monkeypatch.setattr(supervisor.scm_broker_client, "read", read)
class RecordingDb:
@ -78,10 +88,31 @@ def _task(**kw):
"title": "",
"body": "",
}
result = kw.get("result")
if "metadata" not in kw and isinstance(result, dict) and result.get("pull_request"):
base["metadata"] = {"assignment": {"branch": "feature/impl", "base_branch": "main",
"pull_request": result["pull_request"]}}
base.update(kw)
return SimpleNamespace(**base)
def _trusted_root():
return _task(id="impl", status="ready", metadata={"supervisor_lineage": {
"root_task_id": "impl", "project": "atlas-iac", "branch": "feature/impl",
"pull_request": "https://scm.bstein.dev/titan/atlas-iac/pulls/1", "base_branch": "main"
}})
def _seed_root(task):
"""Seed the same board-local authority a signed root submission creates."""
chain = policy.lineage.initial(task)
assert chain is not None
trusted = policy.lineage.Lineage(
chain.root_task_id, chain.branch, chain.pull_request, "atlas-iac", chain.base_branch
)
supervisor.supervisor_state.record_submission("cassandra", task.id, trusted, "c1")
def _write_config(tmp_path, monkeypatch, kanban):
path = tmp_path / "config.yaml"
path.write_text(yaml.safe_dump({"kanban": kanban}), encoding="utf-8")
@ -164,8 +195,16 @@ def test_iter_boards_drops_empty_slugs():
def test_impl_done_spawns_review_and_comments():
impl = _task(
id="impl",
result={"changed_files": ["a.py"], "head_commit": "c1", "pull_request": "pr/1"},
metadata={},
result={"changed_files": ["a.py"], "head_commit": "c1"},
)
# This mirrors coordinator finalization: a signed broker result records the
# root in the real board DB, while native task rows themselves have no
# metadata column for policy to trust.
chain = policy.lineage.Lineage(
"impl", "feature/impl", "https://scm.bstein.dev/titan/atlas-iac/pulls/1", "atlas-iac", "main"
)
supervisor.supervisor_state.record_submission("cassandra", "impl", chain, "c1")
db = RecordingDb([impl])
assert supervisor.supervise_once(db, policy.Limits()) == 1
assert len(db.created) == 1
@ -177,17 +216,58 @@ def test_impl_done_spawns_review_and_comments():
def test_ship_marks_ready_for_human_without_merging():
review = _task(
id="rev",
metadata={"supervisor": {"kind": "review", "root": "impl", "parent": "impl", "head_commit": "c1", "cycle": 1}},
metadata={"supervisor": {"kind": "review", "root": "impl", "parent": "impl", "head_commit": "c1", "cycle": 1,
"root_task_id": "impl", "project": "atlas-iac", "branch": "feature/impl",
"pull_request": "https://scm.bstein.dev/titan/atlas-iac/pulls/1", "base_branch": "main"}},
result={"verdict": "SHIP", "summary": "clean"},
)
db = RecordingDb([review])
db = RecordingDb([_trusted_root(), review])
supervisor.supervise_once(db, policy.Limits())
assert db.metadata_sets == [("impl", {"supervisor_ready_for_human_merge": True})]
assert db.metadata_sets == [("impl", {"supervisor_ready_for_human_merge": True,
"supervisor_ready_commit": "c1",
"supervisor_ready_pull_request": "https://scm.bstein.dev/titan/atlas-iac/pulls/1"})]
body = db.comments[-1][2]
assert "READY FOR HUMAN MERGE" in body and "never" in body
assert db.created == [] and db.blocked == []
def test_stale_ship_clears_old_ready_evidence_and_new_head_can_ship_once(monkeypatch):
db = RecordingDb([])
conn = SimpleNamespace(close=lambda: None)
ledger = supervisor.Ledger(supervisor.LEDGER_PATH)
heads = iter(("c1", "c2"))
monkeypatch.setattr(supervisor.scm_broker_client, "read", lambda _path: json.dumps({
"state": "open", "head": {"ref": "feature/impl", "sha": next(heads),
"repo": {"full_name": "titan/atlas-iac"}}, "base": {"ref": "main",
"repo": {"full_name": "titan/atlas-iac"}}}).encode())
evidence = {"pr": "https://scm.bstein.dev/titan/atlas-iac/pulls/1", "branch": "feature/impl",
"base_branch": "main", "project": "atlas-iac"}
old_ship = policy.Decision("ship", target_id="impl", payload={"commit": "c1", **evidence})
stale = policy.Decision("clear_ready", target_id="impl", payload={"commit": "c2", "stale_commit": "c1"})
new_ship = policy.Decision("ship", target_id="impl", payload={"commit": "c2", **evidence})
assert supervisor.apply_decision(db, conn, old_ship, ledger)
assert supervisor.apply_decision(db, conn, stale, ledger)
assert supervisor.apply_decision(db, conn, new_ship, ledger)
assert db.metadata_sets[-2][1]["supervisor_ready_for_human_merge"] is False
assert db.metadata_sets[-1][1]["supervisor_ready_commit"] == "c2"
def test_ship_defers_or_clears_when_live_pr_head_differs_from_stored_evidence(monkeypatch):
db = RecordingDb([])
conn = SimpleNamespace(close=lambda: None)
ledger = supervisor.Ledger(supervisor.LEDGER_PATH)
info = {"commit": "c1", "pr": "https://scm.bstein.dev/titan/atlas-iac/pulls/1",
"branch": "feature/impl", "base_branch": "main", "project": "atlas-iac"}
monkeypatch.setattr(supervisor.scm_broker_client, "read", lambda _path: json.dumps({
"state": "open", "head": {"ref": "feature/impl", "sha": "c2",
"repo": {"full_name": "titan/atlas-iac"}}, "base": {"ref": "main",
"repo": {"full_name": "titan/atlas-iac"}}}).encode())
assert supervisor.apply_decision(db, conn, policy.Decision("ship", target_id="impl", payload=info), ledger)
assert db.metadata_sets == [("impl", {"supervisor_ready_for_human_merge": False,
"supervisor_ready_commit": ""})]
assert not any("READY FOR HUMAN MERGE" in body for _, _, body in db.comments)
def test_escalation_blocks_card_and_creates_no_followup():
impl = _task(id="impl", result="unparseable")
db = RecordingDb([impl])
@ -201,6 +281,7 @@ def test_spawn_retries_without_idempotency_key_on_typeerror():
id="impl",
result={"changed_files": ["a.py"], "head_commit": "c1", "pull_request": "pr/1"},
)
_seed_root(impl)
class LegacyDb(RecordingDb):
def create_task(self, _conn, **kwargs):
@ -265,7 +346,9 @@ def test_comment_and_block_failures_are_swallowed(capsys):
def _ship_review(commit="c1"):
return _task(
id="rev",
metadata={"supervisor": {"kind": "review", "root": "impl", "parent": "impl", "head_commit": commit, "cycle": 1}},
metadata={"supervisor": {"kind": "review", "root": "impl", "parent": "impl", "head_commit": commit, "cycle": 1,
"root_task_id": "impl", "project": "atlas-iac", "branch": "feature/impl",
"pull_request": "https://scm.bstein.dev/titan/atlas-iac/pulls/1", "base_branch": "main"}},
result={"verdict": "SHIP", "summary": "clean"},
)
@ -288,7 +371,7 @@ def _cycle_exhausted_repair():
def test_ship_marks_ready_exactly_once_across_ten_ticks():
review = _ship_review()
db = RecordingDb([review])
db = RecordingDb([_trusted_root(), review])
for _ in range(10):
supervisor.supervise_once(db, policy.Limits())
ready_comments = [c for c in db.comments if "READY FOR HUMAN MERGE" in c[2]]
@ -318,7 +401,7 @@ def test_repair_cycle_limit_escalates_exactly_once_across_ten_ticks():
def test_supervise_once_accepts_an_injected_ledger():
review = _ship_review()
db = RecordingDb([review])
db = RecordingDb([_trusted_root(), review])
ledger = supervisor.Ledger(supervisor.LEDGER_PATH)
supervisor.supervise_once(db, policy.Limits(), ledger)
supervisor.supervise_once(db, policy.Limits(), ledger)
@ -363,6 +446,7 @@ def test_spawn_typeerror_does_not_double_insert_when_row_already_exists():
id="impl",
result={"changed_files": ["a"], "head_commit": "c1", "pull_request": "pr"},
)
_seed_root(impl)
db = InsertThenTypeErrorDb([impl])
supervisor.supervise_once(db, policy.Limits())
assert len(db.created) == 1 # retry suppressed by the re-scan guard
@ -476,7 +560,7 @@ def test_auto_supervise_flag_defaults_false_in_configmap():
doc for doc in documents if doc and doc.get("metadata", {}).get("name") == "hermes-agent-config"
)
payload = yaml.safe_load(config_doc["data"]["config.yaml"])
assert payload["kanban"]["auto_supervise"] is False
assert payload["kanban"]["auto_supervise"] is True
def test_supervisor_scripts_registered_in_coordinator_configmap():
@ -487,6 +571,9 @@ def test_supervisor_scripts_registered_in_coordinator_configmap():
joined = "\n".join(coordinator["files"])
assert "kanban_supervisor.py=scripts/kanban_supervisor.py" in joined
assert "supervisor_policy.py=scripts/supervisor_policy.py" in joined
assert "supervisor_lineage.py=scripts/supervisor_lineage.py" in joined
assert "scm_broker_client.py=scm-common/scripts/scm_broker_client.py" in joined
assert "deadline_http.py=scm-common/scripts/deadline_http.py" in joined
# --- the confirmed NULL -> 20 goal-turn fallback fix ----------------------

86
testing/tests/test_hermes_kanban_supervisor_policy.py
View File

@ -31,6 +31,10 @@ def task(**kw):
"metadata": {},
"parents": [],
}
result = kw.get("result")
if "metadata" not in kw and isinstance(result, dict) and result.get("pull_request"):
base["metadata"] = {"assignment": {"branch": "feature/impl", "base_branch": "main",
"pull_request": result["pull_request"]}}
base.update(kw)
return SimpleNamespace(**base)
@ -43,6 +47,10 @@ def review_stamp(root="impl", commit="c1", cycle=1):
"parent": root,
"head_commit": commit,
"cycle": cycle,
"root_task_id": root,
"branch": "feature/impl",
"pull_request": "pr/1",
"base_branch": "main",
}
}
@ -55,10 +63,21 @@ def repair_stamp(root="impl", commit="c1", cycle=1):
"parent": "rev",
"head_commit": commit,
"cycle": cycle,
"root_task_id": root,
"branch": "feature/impl",
"pull_request": "pr/1",
"base_branch": "main",
}
}
def trusted_root(root="impl", branch="feature/impl", pull_request="pr/1"):
return task(id=root, status="ready", metadata={"supervisor_lineage": {
"root_task_id": root, "branch": branch, "pull_request": pull_request,
"base_branch": "main"
}})
# --- primitives -----------------------------------------------------------
@ -132,13 +151,8 @@ def test_done_implementation_with_pr_spawns_one_review():
assert payload["parents"] == ["impl"]
assert payload["idempotency_key"] == "supervisor:review:impl:c1:1"
stamp = payload["metadata"]["supervisor"]
assert stamp == {
"kind": "review",
"root": "impl",
"parent": "impl",
"head_commit": "c1",
"cycle": 1,
}
assert stamp["root_task_id"] == "impl"
assert stamp["branch"] == "feature/impl" and stamp["pull_request"] == "pr/1"
assert "Hermes-Task-Role: review" in payload["body"]
assert payload["metadata"]["task_role"] == "review"
@ -224,9 +238,11 @@ def test_review_ship_marks_parent_ready_without_merging():
metadata=review_stamp("impl", "c1", 1),
result={"verdict": "SHIP", "summary": "clean"},
)
decision = policy.plan(review, [review], LIMITS)
decision = policy.plan(review, [trusted_root(), review], LIMITS)
assert decision.action == "ship" and decision.target_id == "impl"
assert decision.payload == {"commit": "c1", "pr": ""}
assert decision.payload["commit"] == "c1"
assert decision.payload["pr"] == "pr/1"
assert decision.payload["branch"] == "feature/impl"
def test_review_block_spawns_bounded_repair_with_findings():
@ -235,7 +251,7 @@ def test_review_block_spawns_bounded_repair_with_findings():
metadata=review_stamp("impl", "c1", 1),
result={"verdict": "BLOCK", "findings": ["null deref", "missing test"]},
)
decision = policy.plan(review, [review], LIMITS)
decision = policy.plan(review, [trusted_root(), review], LIMITS)
assert decision.action == "spawn"
payload = decision.payload
assert payload["metadata"]["supervisor"]["kind"] == "repair"
@ -271,7 +287,7 @@ def test_review_block_at_cycle_limit_escalates_the_source_review_not_impl():
metadata=review_stamp("impl", "c1", LIMITS.max_cycles),
result={"verdict": "BLOCK", "findings": ["x"]},
)
decision = policy.plan(review, [review], LIMITS)
decision = policy.plan(review, [trusted_root(), review], LIMITS)
# Targets the review card (source), not root_id, so it leaves the done state
# and is not re-planned into the same escalation next tick.
assert decision.action == "escalate" and decision.target_id == "rev"
@ -285,7 +301,7 @@ def test_review_block_skips_when_repair_already_exists():
result={"verdict": "BLOCK", "findings": ["x"]},
)
repair = task(id="rep", status="ready", metadata=repair_stamp("impl", "c1", 1))
assert policy.plan(review, [review, repair], LIMITS).action == "none"
assert policy.plan(review, [trusted_root(), review, repair], LIMITS).action == "none"
# --- repair -> re-review --------------------------------------------------
@ -297,7 +313,8 @@ def test_repair_with_new_commit_spawns_re_review_at_next_cycle():
metadata=repair_stamp("impl", "c1", 1),
result={"changed_files": ["a.py"], "head_commit": "c2", "branch": "feat"},
)
decision = policy.plan(repair, [repair], LIMITS)
parent = task(id="rev", metadata=review_stamp("impl", "c1", 1), result={"verdict": "BLOCK"})
decision = policy.plan(repair, [trusted_root(), parent, repair], LIMITS)
assert decision.action == "spawn"
payload = decision.payload
stamp = payload["metadata"]["supervisor"]
@ -306,6 +323,41 @@ def test_repair_with_new_commit_spawns_re_review_at_next_cycle():
assert payload["idempotency_key"] == "supervisor:review:impl:c2:2"
def test_block_repair_and_rereview_keep_one_immutable_pr_lineage():
impl = task(
id="impl",
metadata={"assignment": {"root_task_id": "impl", "project": "atlas/iac",
"branch": "feature/fix", "pull_request": "pr/9", "base_branch": "main"}},
result={"changed_files": ["a.py"], "head_commit": "c1"},
)
review_payload = policy.plan(impl, [impl], LIMITS).payload
review = task(id="review", metadata=review_payload["metadata"],
result={"verdict": "BLOCK", "findings": ["add regression"]})
repair_payload = policy.plan(review, [impl, review], LIMITS).payload
repair = task(id="repair", metadata=repair_payload["metadata"],
result={"changed_files": ["test.py"], "head_commit": "c2"})
rereview = policy.plan(repair, [impl, review, repair], LIMITS)
stamped = rereview.payload["metadata"]["supervisor"]
assert rereview.payload["idempotency_key"] == "supervisor:review:impl:c2:2"
assert {key: stamped[key] for key in ("root_task_id", "project", "branch", "pull_request")} == {
"root_task_id": "impl", "project": "atlas/iac", "branch": "feature/fix", "pull_request": "pr/9"
}
assert "do not create a branch or PR" in repair_payload["body"]
def test_stale_ship_clears_readiness_when_repair_has_a_newer_head():
chain = {"root_task_id": "impl", "branch": "feature/fix", "pull_request": "pr/9", "base_branch": "main"}
old_review = task(id="old", metadata={"supervisor": {"kind": "review", "root": "impl",
"parent": "impl", "head_commit": "c1", "cycle": 1, **chain}},
result={"verdict": "SHIP"})
repair = task(id="repair", metadata={"supervisor": {"kind": "repair", "root": "impl",
"parent": "old", "head_commit": "c1", "cycle": 1, **chain}},
result={"head_commit": "c2"})
decision = policy.plan(old_review, [trusted_root(branch="feature/fix", pull_request="pr/9"), old_review, repair], LIMITS)
assert decision.action == "clear_ready"
assert decision.payload == {"commit": "c2", "stale_commit": "c1"}
def test_repair_incomplete_stamp_fails_closed():
repair = task(id="rep", metadata={"supervisor": {"kind": "repair"}}, result={"head_commit": "c2"})
assert policy.plan(repair, [repair], LIMITS).action == "escalate"
@ -336,7 +388,8 @@ def test_repair_at_cycle_limit_escalates_the_source_repair_not_impl():
metadata=repair_stamp("impl", "c1", LIMITS.max_cycles),
result={"head_commit": "c2"},
)
decision = policy.plan(repair, [repair], LIMITS)
parent = task(id="rev", metadata=review_stamp("impl", "c1", LIMITS.max_cycles), result={"verdict": "BLOCK"})
decision = policy.plan(repair, [trusted_root(), parent, repair], LIMITS)
assert decision.action == "escalate" and decision.target_id == "rep"
assert "impl" in decision.reason
@ -348,7 +401,8 @@ def test_repair_skips_when_re_review_already_exists():
result={"head_commit": "c2"},
)
existing = task(id="rev2", status="ready", metadata=review_stamp("impl", "c2", 2))
assert policy.plan(repair, [repair, existing], LIMITS).action == "none"
parent = task(id="rev", metadata=review_stamp("impl", "c1", 1), result={"verdict": "BLOCK"})
assert policy.plan(repair, [trusted_root(), parent, repair, existing], LIMITS).action == "none"
# --- dedup / classification helpers --------------------------------------
@ -404,7 +458,7 @@ def test_review_block_without_findings_still_spawns_repair():
metadata=review_stamp("impl", "c1", 1),
result={"verdict": "BLOCK", "summary": "must not ship BLOCK"},
)
decision = policy.plan(review, [review], LIMITS)
decision = policy.plan(review, [trusted_root(), review], LIMITS)
assert decision.action == "spawn"
assert "findings to address" not in decision.payload["body"]

91
testing/tests/test_hermes_legacy_scm_roots.py Normal file
View File

@ -0,0 +1,91 @@
"""Regression coverage for explicit legacy PR root migration."""
from __future__ import annotations
from contextlib import nullcontext
import json
from pathlib import Path
import sys
import yaml
from testing.tests.test_hermes_cli_support import HERMES, _load
ROOT = Path(__file__).parents[2]
sys.path.insert(0, str(HERMES / "scm-common/scripts"))
state = _load("supervisor_state")
seed = _load("seed_legacy_scm_roots")
class NativeKanban:
"""Minimal native task lookup surface used by the operator seed."""
def __init__(self, task_ids: set[str]) -> None:
self.task_ids = task_ids
def scoped_current_board(self, _board: str):
return nullcontext()
def connect(self, *, board: str):
return type("Connection", (), {"close": lambda self: None})()
def get_task(self, _connection, task_id: str):
return {"id": task_id} if task_id in self.task_ids else None
def _pull(root, head: str | None = None) -> bytes:
"""Build only the canonical open PR shape the seed accepts."""
return json.dumps({
"state": "open",
"head": {"ref": root.ref, "sha": head or root.head,
"repo": {"full_name": f"titan/{root.project}"}},
"base": {"ref": root.base, "repo": {"full_name": f"titan/{root.project}"}},
}).encode()
def test_seed_registry_exactly_matches_flux_broker_adoptions():
"""The board integrity table and broker ledger share one reviewed scope."""
document = yaml.safe_load(
(ROOT / "services/hermes-scm-broker/task-branch-adoptions-configmap.yaml").read_text()
)
deployed = json.loads(document["data"]["task-branch-adoptions.json"])
expected = {f"{root.project}/{root.ref}": root.adoption() for root in seed.ROOTS}
assert deployed == expected
assert "t_cf89a2ec" in {root.root_task_id for root in seed.ROOTS}
assert deployed["atlas-iac/feature/hermes-next-hux"]["latest_head"] == (
"98c7c6184f6edfe3cdac228529c2287584db3006"
)
def test_seed_requires_native_root_and_matching_live_canonical_pr(tmp_path, monkeypatch):
"""A stale PR or fabricated root leaves the board-local integrity DB untouched."""
root = seed.ROOTS[1]
monkeypatch.setattr(state, "KANBAN_ROOT", tmp_path / "boards")
missing = NativeKanban(set())
assert seed.seed_root(missing, root, lambda _path: (_ for _ in ()).throw(AssertionError)) == "missing-task"
native = NativeKanban({root.root_task_id})
assert seed.seed_root(native, root, lambda _path: _pull(root, "a" * 40)) == "live-pr-mismatch"
assert state.get_root(root.board, root.root_task_id) is None
assert seed.seed_root(native, root, lambda _path: _pull(root)) == "seeded"
assert state.get_root(root.board, root.root_task_id) == root.lineage
assert state.get_live_head(root.board, root.root_task_id) == root.head
def test_seed_rerun_preserves_same_owned_newer_state_and_approval(tmp_path, monkeypatch):
"""A static migration record never rewinds a later verified continuation head."""
root = seed.ROOTS[1]
monkeypatch.setattr(state, "KANBAN_ROOT", tmp_path / "boards")
native = NativeKanban({root.root_task_id})
assert seed.seed_root(native, root, lambda _path: _pull(root)) == "seeded"
newer = "b" * 40
state.record_submission(root.board, root.root_task_id, root.lineage, newer)
state.set_ready(root.board, root.root_task_id, newer)
assert seed.seed_root(native, root, lambda _path: (_ for _ in ()).throw(AssertionError)) == "already-seeded"
assert state.get_live_head(root.board, root.root_task_id) == newer
with state._connect(root.board) as connection:
assert connection.execute(
"SELECT ready_for_human_merge,ready_commit FROM supervisor_roots"
).fetchone() == (1, newer)

95
testing/tests/test_hermes_model_refresh.py Normal file
View File

@ -0,0 +1,95 @@
"""Integration checks for model admission during a coordinator refresh."""
from __future__ import annotations
import json
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parents[2] / "services/hermes/scripts"))
import model_catalog_refresh as refresh
from hermes_model_routing import Catalog, write_routing_catalog
from model_evaluation_evidence import EVALUATION_VERSION, metadata_fingerprint, write_store
from routing_catalog import load_catalog
def providers():
"""Return a reviewed model and one newly advertised candidate."""
metadata = {
"gpt-10-lattice": {
"description": "Built for difficult complex tasks",
"supported_reasoning_efforts": ["high", "xhigh"],
}
}
return (
Catalog("openai-codex", ["gpt-5.6-sol", "gpt-10-lattice"], True, True, "connected", metadata),
Catalog("anthropic", [], True, True, "connected"),
)
def test_refresh_admits_new_model_only_after_evidence_is_persisted(tmp_path, monkeypatch):
codex, claude = providers()
path = tmp_path / "catalog.json"
monkeypatch.setenv("HERMES_ROUTING_CATALOG_PATH", str(path))
provisional = write_routing_catalog(path, codex, claude)
assert provisional["providers"]["codex"]["capability_pools"]["advanced"] == ["gpt-5.6-sol"]
def evaluate(catalog, *, evidence_path):
assert "gpt-10-lattice" in catalog["providers"]["codex"]["models"]
record = {
"provider": "codex", "model": "gpt-10-lattice",
"metadata_fingerprint": metadata_fingerprint("gpt-10-lattice", codex.metadata["gpt-10-lattice"]),
"eval_version": EVALUATION_VERSION, "proposed_role": "advanced",
"result": "pass", "role_fit": "verified",
}
write_store(evidence_path, {"schema_version": 1, "evaluations": {"codex:gpt-10-lattice": record}})
return {"evaluations": {"codex": {"gpt-10-lattice": record}}}
monkeypatch.setattr(refresh, "evaluate_catalog_candidates", evaluate)
assert refresh.refresh_model_evaluations(tmp_path, codex, claude) == {
"state": "ready", "verified": 1, "pending": 0,
}
pools = load_catalog(path)["providers"]["codex"]["capability_pools"]
assert pools["advanced"] == ["gpt-5.6-sol", "gpt-10-lattice"]
assert json.loads(path.read_text())["schema_version"] == 3
def test_evidence_store_failure_preserves_provisional_routes(tmp_path, monkeypatch):
codex, claude = providers()
path = tmp_path / "catalog.json"
monkeypatch.setenv("HERMES_ROUTING_CATALOG_PATH", str(path))
write_routing_catalog(path, codex, claude)
original = path.read_bytes()
def unavailable(*args, **kwargs):
raise OSError("store unavailable")
monkeypatch.setattr(refresh, "evaluate_catalog_candidates", unavailable)
assert refresh.refresh_model_evaluations(tmp_path, codex, claude) == {
"state": "deferred", "error_type": "OSError",
}
assert path.read_bytes() == original
def test_writer_ignores_stale_or_retired_evidence(tmp_path, monkeypatch):
"""Evidence cannot admit a changed or removed model after a live refresh."""
codex, claude = providers()
path = tmp_path / "catalog.json"
monkeypatch.setenv("HERMES_ROUTING_CATALOG_PATH", str(path))
write_store(path.with_name("model-evaluations.json"), {"schema_version": 1, "evaluations": {
"codex:gpt-10-lattice": {
"provider": "codex", "model": "gpt-10-lattice", "proposed_role": "advanced",
"result": "pass", "role_fit": "verified", "eval_version": EVALUATION_VERSION,
"metadata_fingerprint": "stale",
},
"codex:gpt-retired": {
"provider": "codex", "model": "gpt-retired", "proposed_role": "advanced",
"result": "pass", "role_fit": "verified", "eval_version": EVALUATION_VERSION,
"metadata_fingerprint": metadata_fingerprint("gpt-retired", {}),
},
}})
catalog = write_routing_catalog(path, codex, claude)
provider = catalog["providers"]["codex"]
assert provider["capability_pools"]["advanced"] == ["gpt-5.6-sol"]
assert "gpt-retired" not in provider["model_metadata"]

41
testing/tests/test_hermes_scm_broker_handler_coverage.py
View File

@ -211,6 +211,44 @@ def test_git_rpc_streams_allowed_upload_and_receive(monkeypatch, service):
assert seen[0][1]["body_length"] == len(body)
def test_signed_receive_pack_advances_ledger_only_after_exact_live_head(monkeypatch):
broker = _load("scm_broker")
body = _receive_command(b"0" * 40, b"1" * 40, b"refs/heads/hermes/coverage")
handler = _handler(
broker, path="/git/atlas/cassandra.git/git-receive-pack", body=body,
content_type="application/x-git-receive-pack-request",
)
handler.headers["X-Hermes-Task-Grant"] = "bounded-signed-grant"
claims = {"repo": "cassandra", "ref": "hermes/coverage", "expected_old": "0" * 40, "new_head": "1" * 40}
calls = []
class Ledger:
def authorize_update(self, value):
calls.append(("authorize", value))
def commit(self, value):
calls.append(("commit", value))
monkeypatch.setattr(broker, "read_token", lambda: "sentinel")
monkeypatch.setattr(broker, "verify_grant", lambda value: calls.append(("header", value)) or claims)
monkeypatch.setattr(broker, "_task_ledger", lambda: Ledger())
monkeypatch.setattr(broker, "_branch_head", lambda *_args: "1" * 40)
monkeypatch.setattr(broker, "_upstream_git_request", lambda *_args, **_kwargs: (io.BytesIO(b"result"), 6))
handler._git_rpc()
assert calls == [("header", "bounded-signed-grant"), ("authorize", claims), ("commit", claims)]
calls.clear()
handler = _handler(
broker, path="/git/atlas/cassandra.git/git-receive-pack", body=body,
content_type="application/x-git-receive-pack-request",
)
handler.headers["X-Hermes-Task-Grant"] = "bounded-signed-grant"
monkeypatch.setattr(broker, "_branch_head", lambda *_args: "0" * 40)
with pytest.raises(broker.PolicyError, match="did not advance"):
handler._git_rpc()
assert not [call for call in calls if call[0] == "commit"]
@pytest.mark.parametrize(
("path", "content_type", "transfer", "match"),
[
@ -257,6 +295,9 @@ def test_broker_main_constructs_bounded_server(monkeypatch):
seen.append("served")
monkeypatch.setattr(broker, "BoundedThreadingHTTPServer", Server)
monkeypatch.setattr(broker, "seed_adoptions", lambda *_args: None)
monkeypatch.setattr(broker, "_task_ledger", lambda: object())
monkeypatch.setattr(broker, "read_token", lambda: "token")
monkeypatch.setattr(
sys, "argv", ["broker", "--listen", "127.0.0.1", "--port", "9191"]
)

2
testing/tests/test_hermes_scm_broker_policy.py
View File

@ -109,7 +109,7 @@ def test_flux_boundary_keeps_broker_secret_and_network_separate_from_agent():
ROOT / "services/vault/scripts/vault_k8s_auth_configure.sh"
).read_text(encoding="utf-8")
assert 'write_policy_and_role "hermes-scm-broker" "hermes-scm"' in vault
assert '"hermes/developer-gitea" ""' in vault
assert '"hermes/developer-gitea hermes/scm-task-grant" ""' in vault
agent_start = vault.index('write_policy_and_role "hermes-agent"')
agent_end = vault.index("write_policy_and_role", agent_start + 1)
assert "developer-gitea" not in vault[agent_start:agent_end]

223
testing/tests/test_hermes_scm_task_grants.py Normal file
View File

@ -0,0 +1,223 @@
"""Task grant, ledger, and broker fast-forward contracts."""
from __future__ import annotations
import sys
import hashlib
import io
import json
import time
from email.message import Message
from pathlib import Path
import pytest
from testing.tests.test_hermes_scm_broker_support import _load, _object_entry, _pack_of, _receive_command
ROOT = Path(__file__).parents[2]
sys.path.insert(0, str(ROOT / "services/hermes/scm-common/scripts"))
import receive_pack_scan # noqa: E402
import scm_task_grants as grants # noqa: E402
KEY = b"a" * 64
def claims(**overrides):
value = {
"repo": "atlas-iac", "ref": "wt/t_root", "base": "main", "board": "titan-iac",
"root_task_id": "t_root", "assignment_task_id": "t_child", "run": "run_1",
"ordinal": 1, "expires": 101, "expected_old": "0" * 40, "new_head": "1" * 40,
"continuation_kind": "",
}
value.update(overrides)
return value
def test_grant_is_exact_signed_schema_and_expiry_bound():
grants = _load("scm_task_grants")
token = grants.sign_grant(claims(), KEY)
assert grants.verify_grant(token, key=KEY, now=100)["root_task_id"] == "t_root"
with pytest.raises(grants.PolicyError, match="signature"):
grants.verify_grant(token[:-1] + "A", key=KEY, now=100)
with pytest.raises(grants.PolicyError, match="expired"):
grants.verify_grant(token, key=KEY, now=101)
def test_ledger_requires_absent_initial_ref_and_exact_owner_cas(tmp_path):
grants = _load("scm_task_grants")
ledger = grants.TaskLedger(tmp_path / "ledger.db")
first = claims()
ledger.register(first, remote_head=None)
ledger.authorize_update(first)
ledger.commit(first)
assert ledger.get("atlas-iac", "wt/t_root") == ("titan-iac", "t_root", "1" * 40)
with pytest.raises(grants.PolicyError, match="head changed"):
ledger.authorize_update(first)
with pytest.raises(grants.PolicyError, match="another task"):
ledger.register(claims(root_task_id="t_other"), remote_head="1" * 40)
with pytest.raises(grants.PolicyError, match="requires operator adoption"):
ledger.register(claims(root_task_id="t_untracked", ref="wt/untracked"), remote_head="2" * 40)
with pytest.raises(grants.PolicyError, match="different branch"):
ledger.register(claims(ref="wt/fork"), remote_head=None)
def test_operator_adoption_seeds_only_the_exact_live_reviewed_branch(tmp_path):
grants = _load("scm_task_grants")
ledger = grants.TaskLedger(tmp_path / "ledger.db")
record = {
"repo": "atlas-iac", "ref": "wt/t_root", "board": "titan-iac",
"root_task_id": "t_root", "latest_head": "a" * 40, "pr_number": 55,
}
ledger.seed_adoption(record, "a" * 40)
assert ledger.get("atlas-iac", "wt/t_root") == ("titan-iac", "t_root", "a" * 40)
with pytest.raises(grants.PolicyError, match="does not match"):
ledger.seed_adoption({**record, "ref": "wt/other"}, "b" * 40)
# A later broker-confirmed revision survives a restart with the original
# reviewed import record still mounted.
ledger.commit(claims(expected_old="a" * 40, new_head="b" * 40))
ledger.seed_adoption(record, "b" * 40)
assert ledger.get("atlas-iac", "wt/t_root")[2] == "b" * 40
def test_fast_forward_requires_quarantined_parent_path():
old, middle, new = "a" * 40, "b" * 40, "c" * 40
assert receive_pack_scan._proves_descends(new, old, {new: (middle,), middle: (old,)})
assert not receive_pack_scan._proves_descends(new, old, {new: (middle,)})
assert not receive_pack_scan._proves_descends(new, old, {new: ("d" * 40,)})
def test_signed_default_wt_branch_accepts_a_real_commit_pack_but_unsigned_does_not():
raw = b"tree " + b"0" * 40 + b"\nauthor test <test@example> 1 +0000\ncommitter test <test@example> 1 +0000\n\nmessage\n"
head = hashlib.sha1(b"commit " + str(len(raw)).encode() + b"\0" + raw).hexdigest()
body = _receive_command(b"0" * 40, head.encode(), b"refs/heads/wt/t_default", _pack_of([_object_entry(1, raw)]))
assert receive_pack_scan.validate_receive_pack(body, "token", (b"token",), expected=("0" * 40, head, "wt/t_default")) == ("0" * 40, head, "wt/t_default")
with pytest.raises(receive_pack_scan.PolicyError, match="namespaced"):
receive_pack_scan.validate_receive_pack(body, "token", (b"token",))
@pytest.mark.parametrize("ref", ["main", "master", "refs/tags/v1"])
def test_granted_push_never_allows_base_or_tag_refs(ref):
if ref.startswith("refs/"):
raw_ref = ref.encode()
else:
raw_ref = f"refs/heads/{ref}".encode()
body = _receive_command(b"0" * 40, b"1" * 40, raw_ref)
with pytest.raises(receive_pack_scan.PolicyError):
receive_pack_scan.validate_receive_pack(body, "token", (b"token",), expected=("0" * 40, "1" * 40, ref))
def test_alias_requests_canonicalize_before_the_broker_forwards_them():
api = _load("gitea_api")
assert api.api_url(api.CANONICAL_BASE_URL, "/api/v1/repos/titan/titan-iac/pulls") == "https://scm.bstein.dev/api/v1/repos/titan/atlas-iac/pulls"
assert api.authorize_request("GET", "/api/v1/repos/titan/atlas-iac/branches/wt/t_root", None) == "branch"
def test_registration_uses_the_fixed_broker_http_envelope():
client = _load("scm_broker_client")
seen = []
class Response:
status = 200
class headers:
@staticmethod
def get_content_type():
return "application/json"
def __enter__(self):
return self
def __exit__(self, *_args):
return False
@staticmethod
def read(_limit):
return b'{"registered":true}'
def opener(request, timeout):
seen.append((request, timeout))
return Response()
assert client.register_task("header-safe-grant", opener=opener) == b'{"registered":true}'
request, timeout = seen[0]
assert request.full_url == client.BROKER_ORIGIN + "/v1/tasks/register"
assert request.data == b'{"grant":"header-safe-grant"}' and timeout == 30
def test_draft_refresh_requires_exact_owned_pr_and_has_no_mutating_fields():
drafts = _load("scm_task_drafts")
grant, number, title, body = drafts.request_fields(
{"grant": "signed", "pr_number": 55, "title": "refresh", "body": "evidence"}, "token"
)
assert (grant, number, title, body) == ("signed", 55, "WIP: refresh", "evidence")
claims = {"repo": "atlas-iac", "ref": "wt/t_root", "base": "main", "new_head": "a" * 40}
pull = {"number": 55, "state": "open", "head": {"ref": "wt/t_root", "sha": "a" * 40, "repo": {"full_name": "titan/atlas-iac"}}, "base": {"ref": "main", "repo": {"full_name": "titan/atlas-iac"}}}
drafts.matches_pull(pull, claims, 55)
with pytest.raises(drafts.PolicyError, match="does not match"):
drafts.matches_pull({**pull, "base": {"ref": "master"}}, claims, 55)
with pytest.raises(drafts.PolicyError, match="fields"):
drafts.request_fields({"grant": "x", "pr_number": 55, "title": "x", "body": "x", "state": "closed"}, "token")
def _control_handler(broker, body: dict):
"""Build the broker's real control handler without opening a TCP listener."""
encoded = json.dumps(body).encode()
handler = object.__new__(broker.BrokerHandler)
headers = Message()
headers["Content-Type"] = "application/json"
headers["Content-Length"] = str(len(encoded))
handler.path = "/v1/tasks/draft-update"
handler.headers = headers
handler.rfile = io.BytesIO(encoded)
handler.wfile = io.BytesIO()
handler.connection = type("Connection", (), {"settimeout": staticmethod(lambda _value: None)})()
handler._json = lambda _status, value: handler.wfile.write(value)
return handler
def test_post_push_repair_refreshes_the_same_owned_pr_after_real_ledger_commit(tmp_path, monkeypatch):
"""Draft prose refresh follows an already committed old->new branch update."""
broker = _load("scm_broker")
ledger = grants.TaskLedger(tmp_path / "ledger.db")
old, new = "a" * 40, "b" * 40
initial = {
**claims(expected_old="0" * 40, new_head=old, expires=int(time.time()) + 120),
"continuation_kind": "implementation",
}
ledger.register(initial, remote_head=None)
ledger.commit(initial)
repair = {
**claims(
assignment_task_id="t_repair", expected_old=old, new_head=new,
expires=int(time.time()) + 120,
),
"continuation_kind": "repair",
}
# The receive-pack path has already performed the authenticated CAS. The
# following refresh must authorize the owner at `new`, not re-check old.
ledger.commit(repair)
token = grants.sign_grant(repair, KEY)
pull = {
"number": 55, "state": "open",
"head": {"ref": "wt/t_root", "sha": new, "repo": {"full_name": "titan/atlas-iac"}},
"base": {"ref": "main", "repo": {"full_name": "titan/atlas-iac"}},
}
updates = []
monkeypatch.setattr(broker, "read_token", lambda: "token")
monkeypatch.setattr(broker, "verify_grant", lambda value: grants.verify_grant(value, key=KEY))
monkeypatch.setattr(broker, "_task_ledger", lambda: ledger)
monkeypatch.setattr(broker, "_branch_head", lambda *_args: new)
monkeypatch.setattr(broker, "read", lambda *_args, **_kwargs: json.dumps(pull).encode())
monkeypatch.setattr(
broker, "update_draft",
lambda _token, repo, number, title, body: updates.append((repo, number, title, body))
or json.dumps({"number": number, "html_url": "https://scm.bstein.dev/titan/atlas-iac/pulls/55"}).encode(),
)
_control_handler(broker, {
"grant": token, "pr_number": 55, "title": "Repair CI failure", "body": "Tests: pytest",
})._control()
assert ledger.get("atlas-iac", "wt/t_root") == ("titan-iac", "t_root", new)
assert updates == [("atlas-iac", 55, "WIP: Repair CI failure", "Tests: pytest")]

330
testing/tests/test_model_capability_evaluator.py Normal file
View File

@ -0,0 +1,330 @@
"""Bounded evidence contracts for dynamically discovered provider models."""
from __future__ import annotations
import json
import sys
from pathlib import Path
import pytest
SCRIPTS = Path(__file__).parents[2] / "services" / "hermes" / "scripts"
sys.path.insert(0, str(SCRIPTS))
import model_capability_evaluator as evaluator # noqa: E402
from hermes_model_routing import Catalog, write_routing_catalog # noqa: E402
from model_evaluation_evidence import accepted_outcomes, load_store # noqa: E402
from routing_catalog import load_catalog # noqa: E402
class Replies:
"""Record literal broker requests and return exact deterministic answers."""
def __init__(self, role="advanced") -> None:
self.answers = iter(evaluator._probe_cases(role))
self.calls: list[tuple[str, str, str, str]] = []
def invoke(self, provider: str, model: str, effort: str, prompt: str):
self.calls.append((provider, model, effort, prompt))
_kind, _prompt, decision, invariants, checks = next(self.answers)
return evaluator.ProbeReply(
text=json.dumps(
{"decision": decision, "invariants": sorted(invariants), "checks": sorted(checks)}
),
input_tokens=3,
output_tokens=4,
latency_ms=9,
)
class Unavailable:
"""Simulate a broker condition that must remain pending, not poor quality."""
def __init__(self) -> None:
self.calls = 0
def invoke(self, *_args):
self.calls += 1
raise evaluator.ProbeTransportError("timeout")
def catalog(*models: str) -> dict:
"""Return a live catalog whose models are unknown to reviewed policy."""
return {
"providers": {
"codex": {
"live": True,
"models": list(models),
"model_metadata": {
model: {
"description": "Built for difficult complex tasks",
"supported_reasoning_efforts": ["high", "xhigh"],
}
for model in models
},
}
}
}
def test_eval_uses_literal_model_and_persists_only_sanitized_results(tmp_path):
transport = Replies()
path = tmp_path / "model-evaluations.json"
result = evaluator.evaluate_catalog_candidates(
catalog("gpt-9-aurora"), evidence_path=path, transport=transport, now=100
)
record = result["evaluations"]["codex"]["gpt-9-aurora"]
assert record["result"] == "pass"
assert record["role_fit"] == "verified"
assert record["proposed_role"] == "advanced"
assert record["attempted_efforts"] == ["high"]
assert record["tokens"] == {"input": 6, "output": 8, "total": 14}
assert record["latency_ms"] == 18
assert "prompt" not in record and "response" not in record
assert [call[:3] for call in transport.calls] == [
("codex", "gpt-9-aurora", "high"),
("codex", "gpt-9-aurora", "high"),
]
assert all("auto" not in call[1] and "route/" not in call[1] for call in transport.calls)
def test_success_is_cached_by_metadata_fingerprint_and_eval_version(tmp_path):
path = tmp_path / "model-evaluations.json"
evaluator.evaluate_catalog_candidates(
catalog("gpt-9-aurora"), evidence_path=path, transport=Replies(), now=100
)
cached = Replies()
evaluator.evaluate_catalog_candidates(
catalog("gpt-9-aurora"), evidence_path=path, transport=cached, now=101
)
assert cached.calls == []
def test_admitted_catalog_refresh_keeps_a_verified_evaluation_cached(tmp_path):
"""Derived admission metadata cannot invalidate the original fingerprint."""
path = tmp_path / "catalog.json"
metadata = {
"gpt-9-aurora": {
"description": "Built for difficult complex tasks",
"supported_reasoning_efforts": ["high", "xhigh"],
}
}
codex = Catalog("openai-codex", ["gpt-5.6-sol", "gpt-9-aurora"], True, True, "connected", metadata)
claude = Catalog("anthropic", [], True, True, "connected")
write_routing_catalog(path, codex, claude)
evaluator.evaluate_catalog_candidates(
load_catalog(path), evidence_path=path.with_name("model-evaluations.json"),
transport=Replies(), now=100,
)
admitted = write_routing_catalog(path, codex, claude)
assert "gpt-9-aurora" in admitted["providers"]["codex"]["capability_pools"]["advanced"]
cached = Replies()
evaluator.evaluate_catalog_candidates(
load_catalog(path), evidence_path=path.with_name("model-evaluations.json"),
transport=cached, now=101,
)
assert cached.calls == []
def test_stale_success_is_not_reused_when_provider_metadata_changes(tmp_path):
path = tmp_path / "model-evaluations.json"
evaluator.evaluate_catalog_candidates(
catalog("gpt-9-aurora"), evidence_path=path, transport=Replies(), now=100
)
changed = {
"providers": {
"codex": {
"live": True,
"models": ["gpt-9-aurora"],
"model_metadata": {
"gpt-9-aurora": {
"description": "The frontier most capable state of the art model",
"supported_reasoning_efforts": ["xhigh"],
}
},
}
}
}
refreshed = Replies("frontier")
result = evaluator.evaluate_catalog_candidates(
changed, evidence_path=path, transport=refreshed, now=101
)
assert len(refreshed.calls) == 2
assert result["evaluations"]["codex"]["gpt-9-aurora"]["proposed_role"] == "frontier"
def test_refresh_is_capped_and_transport_failure_is_not_quality_failure(tmp_path):
path = tmp_path / "model-evaluations.json"
unavailable = Unavailable()
result = evaluator.evaluate_catalog_candidates(
catalog("gpt-9-a", "gpt-9-b", "gpt-9-c"),
evidence_path=path,
transport=unavailable,
now=100,
)
records = result["evaluations"]["codex"]
assert unavailable.calls == 2
assert set(records) == {"gpt-9-a", "gpt-9-b"}
assert all(record["result"] == "unavailable" for record in records.values())
assert all(record["role_fit"] == "pending" for record in records.values())
assert all(record["failure_class"] == "timeout" for record in records.values())
retry = Unavailable()
evaluator.evaluate_catalog_candidates(
catalog("gpt-9-a", "gpt-9-b"), evidence_path=path, transport=retry, now=101
)
assert retry.calls == 0
def test_only_verified_acceptance_outcomes_are_retained(tmp_path):
path = tmp_path / "model-evaluations.json"
result = evaluator.evaluate_catalog_candidates(
catalog("gpt-9-aurora"),
evidence_path=path,
transport=Replies(),
now=100,
observed_outcomes=(
{"provider": "codex", "model": "gpt-9-aurora", "evidence_kind": "acceptance", "verified": True, "accepted": True, "evidence_id": "run-1"},
{"provider": "codex", "model": "gpt-9-aurora", "evidence_kind": "acceptance", "verified": False, "accepted": False, "evidence_id": "untrusted"},
),
)
assert result["accepted_outcomes"] == [
{"provider": "codex", "model": "gpt-9-aurora", "accepted": True, "evidence_id": "run-1"}
]
record = load_store(path)["evaluations"]["codex:gpt-9-aurora"]
assert record["observed_acceptance"] == {"accepted": True, "evidence_id": "run-1"}
assert accepted_outcomes(({"provider": "codex", "model": "auto", "verified": True, "accepted": True},)) == []
def test_provider_description_and_supported_effort_levels_form_a_candidate(tmp_path):
transport = Replies("frontier")
document = {
"providers": {
"codex": {
"live": True,
"models": ["gpt-10-vertex"],
"model_metadata": {
"gpt-10-vertex": {
"description": "Our frontier model for the most capable state of the art reasoning",
"supportedEffortLevels": [{"level": "xhigh"}],
}
},
}
}
}
result = evaluator.evaluate_catalog_candidates(
document, evidence_path=tmp_path / "evidence.json", transport=transport, now=100
)
record = result["evaluations"]["codex"]["gpt-10-vertex"]
assert record["proposed_role"] == "frontier"
assert record["attempted_efforts"] == ["xhigh"]
assert record["result"] == "pass"
def test_malformed_output_and_reviewed_policy_records_do_not_promote(tmp_path):
class Malformed:
def invoke(self, *_args):
return evaluator.ProbeReply(text="[]")
result = evaluator.evaluate_catalog_candidates(
catalog("gpt-9-aurora", "gpt-5.6-sol"),
evidence_path=tmp_path / "evidence.json",
transport=Malformed(),
now=100,
)
record = result["evaluations"]["codex"]["gpt-9-aurora"]
assert record["result"] == "unavailable"
assert record["role_fit"] == "pending"
assert record["failure_class"] == "invalid_response"
assert "gpt-5.6-sol" not in result["evaluations"]["codex"]
def test_native_broker_envelopes_keep_literal_models_and_account_usage(monkeypatch):
class Response:
def __init__(self, body):
self.body = body
def __enter__(self):
return self
def __exit__(self, *_args):
return False
def read(self, _limit):
return self.body
seen = []
codex_body = json.dumps(
{
"output": [{"content": [{"type": "output_text", "text": '{"answer":"ok"}'}]}],
"usage": {"input_tokens": 7, "output_tokens": 999},
}
).encode()
claude_body = json.dumps(
{
"content": [{"type": "text", "text": '{"answer":"ok"}'}],
"usage": {"input_tokens": 8, "output_tokens": 9},
}
).encode()
def fake_urlopen(request, timeout):
seen.append((request, timeout))
return Response(codex_body if "responses" in request.full_url else claude_body)
monkeypatch.setattr(evaluator, "urlopen", fake_urlopen)
transport = evaluator.BrokerHttpTransport(key="private")
codex = transport.invoke("codex", "gpt-10-vertex", "xhigh", "probe")
claude = transport.invoke("claude", "claude-next", "high", "probe")
assert codex.output_tokens == 999 # Usage is recorded, never treated as a hard cutoff.
assert claude.input_tokens == 8
codex_payload = json.loads(seen[0][0].data)
claude_payload = json.loads(seen[1][0].data)
assert codex_payload["model"] == "gpt-10-vertex"
assert codex_payload["reasoning"] == {"effort": "xhigh"}
assert claude_payload["model"] == "claude-next"
assert claude_payload["output_config"] == {"effort": "high"}
assert all(request.get_header("Authorization") == "Bearer private" for request, _ in seen)
assert all(timeout == evaluator.REQUEST_TIMEOUT_SECONDS for _, timeout in seen)
def test_clipped_broker_body_is_inconclusive_not_a_quality_mismatch(monkeypatch):
class Response:
def __enter__(self):
return self
def __exit__(self, *_args):
return False
def read(self, limit):
return b"x" * limit
monkeypatch.setattr(evaluator, "urlopen", lambda *_args, **_kwargs: Response())
transport = evaluator.BrokerHttpTransport(key="private")
with pytest.raises(evaluator.ProbeTransportError, match="truncated"):
transport.invoke("codex", "gpt-10-vertex", "high", "probe")
def test_probe_score_requires_correct_finite_ids_and_rejects_unsafe_extras():
kind, _prompt, decision, invariants, checks = evaluator._probe_cases("advanced")[0]
good = json.dumps({"decision": decision, "invariants": sorted(invariants), "checks": sorted(checks)})
wrong = json.dumps({"decision": "D2", "invariants": sorted(invariants), "checks": sorted(checks)})
unsafe_extra = json.dumps(
{"decision": decision, "invariants": sorted(invariants | {"I3"}), "checks": sorted(checks)}
)
assert evaluator._score(kind, good, decision, invariants, checks) is True
assert evaluator._score(kind, wrong, decision, invariants, checks) is False
assert evaluator._score(kind, unsafe_extra, decision, invariants, checks) is False
assert evaluator._score(kind, "[]", decision, invariants, checks) is None