Compare commits
33 Commits
54156f839e
...
5f27e50c74
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5f27e50c74 | ||
|
|
8036101f0b | ||
|
|
7db9f61659 | ||
|
|
603524ba6a | ||
|
|
a3077224e0 | ||
| be1ff33d4b | |||
|
|
8c7074ae16 | ||
|
|
6b11331520 | ||
|
|
376cc14dd1 | ||
|
|
77df54e3ba | ||
|
|
9d42ff33f5 | ||
|
|
a5d0abb678 | ||
|
|
31439f096a | ||
|
|
6c1123201e | ||
|
|
6e9e8c6ab5 | ||
| 41d10ad3f7 | |||
|
|
6df688adcc | ||
|
|
034c8372c7 | ||
|
|
62d8cd984b | ||
|
|
6f993c049e | ||
|
|
5020e49057 | ||
|
|
b8f2163af0 | ||
|
|
340ec58b68 | ||
|
|
578239a496 | ||
|
|
2019b08276 | ||
|
|
f77119b238 | ||
|
|
64272f52d2 | ||
|
|
3a57371989 | ||
|
|
df75479069 | ||
|
|
e3ecc18d0e | ||
|
|
360d8aad8f | ||
|
|
b080b5f622 | ||
|
|
e8c26ecf85 |
288
ci/Jenkinsfile.hermes-agent-image
Normal file
288
ci/Jenkinsfile.hermes-agent-image
Normal file
@ -0,0 +1,288 @@
|
||||
pipeline {
|
||||
agent {
|
||||
kubernetes {
|
||||
defaultContainer 'python'
|
||||
yaml """
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
labels:
|
||||
atlas.bstein.dev/workload: hermes-agent-image-builder
|
||||
spec:
|
||||
serviceAccountName: hermes-image-builder
|
||||
automountServiceAccountToken: false
|
||||
enableServiceLinks: false
|
||||
restartPolicy: Never
|
||||
securityContext:
|
||||
fsGroup: 1000
|
||||
fsGroupChangePolicy: OnRootMismatch
|
||||
nodeSelector:
|
||||
kubernetes.io/arch: arm64
|
||||
node-role.kubernetes.io/worker: "true"
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- key: kubernetes.io/hostname
|
||||
operator: NotIn
|
||||
values:
|
||||
- titan-04
|
||||
- titan-14
|
||||
- titan-18
|
||||
- titan-19
|
||||
- titan-22
|
||||
- titan-24
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
- weight: 100
|
||||
preference:
|
||||
matchExpressions:
|
||||
- key: hardware
|
||||
operator: In
|
||||
values:
|
||||
- rpi5
|
||||
imagePullSecrets:
|
||||
- name: harbor-bstein-robot
|
||||
containers:
|
||||
- name: jnlp
|
||||
image: jenkins/inbound-agent@sha256:8eda4fe2a66bcf6a5e43436d9918fc14c306204dc8fcd75f4e15e0e6e5dc759a
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
resources:
|
||||
requests:
|
||||
cpu: 25m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
- name: python
|
||||
image: registry.bstein.dev/bstein/python@sha256:269541d3387baae008df4608ead893dba2b5cdaad1a5a380731a88992d34b808
|
||||
command: ["sleep"]
|
||||
args: ["99d"]
|
||||
tty: true
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
resources:
|
||||
requests:
|
||||
cpu: 25m
|
||||
memory: 64Mi
|
||||
limits:
|
||||
cpu: 250m
|
||||
memory: 256Mi
|
||||
- name: kaniko
|
||||
image: gcr.io/kaniko-project/executor@sha256:c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e
|
||||
command: ["/busybox/sh", "-c"]
|
||||
args: ["/busybox/sleep 99d"]
|
||||
tty: true
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"]
|
||||
privileged: false
|
||||
runAsUser: 0
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 1Gi
|
||||
limits:
|
||||
cpu: "2"
|
||||
memory: 4Gi
|
||||
"""
|
||||
}
|
||||
}
|
||||
parameters {
|
||||
booleanParam(
|
||||
name: 'PUBLISH_IMAGE',
|
||||
defaultValue: false,
|
||||
description: 'Publish the reviewed main revision to Harbor.'
|
||||
)
|
||||
string(
|
||||
name: 'EXPECTED_SOURCE_REVISION',
|
||||
defaultValue: '',
|
||||
description: 'Exact 40-character commit on atlas/titan-iac main.'
|
||||
)
|
||||
string(
|
||||
name: 'CONFIRM_PUBLISH',
|
||||
defaultValue: '',
|
||||
description: 'Enter PUBLISH HERMES AGENT to confirm the release.'
|
||||
)
|
||||
}
|
||||
environment {
|
||||
HERMES_IMAGE = 'registry.bstein.dev/bstein/hermes-agent'
|
||||
}
|
||||
options {
|
||||
disableConcurrentBuilds()
|
||||
buildDiscarder(logRotator(daysToKeepStr: '30', numToKeepStr: '100', artifactDaysToKeepStr: '30', artifactNumToKeepStr: '100'))
|
||||
skipDefaultCheckout(true)
|
||||
timeout(time: 90, unit: 'MINUTES')
|
||||
}
|
||||
stages {
|
||||
stage('Checkout reviewed source') {
|
||||
steps {
|
||||
checkout scm
|
||||
}
|
||||
}
|
||||
stage('Enforce release boundary') {
|
||||
steps {
|
||||
sh '''
|
||||
set -eu
|
||||
mkdir -p build
|
||||
test "${PUBLISH_IMAGE}" = "true"
|
||||
test "${CONFIRM_PUBLISH}" = "PUBLISH HERMES AGENT"
|
||||
case "${EXPECTED_SOURCE_REVISION}" in
|
||||
*[!0-9a-f]*|'')
|
||||
echo "EXPECTED_SOURCE_REVISION must be a lowercase full commit" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
test "${#EXPECTED_SOURCE_REVISION}" -eq 40
|
||||
actual_revision="$(git rev-parse HEAD)"
|
||||
test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}"
|
||||
test "${actual_revision}" = "$(git rev-parse origin/main)"
|
||||
test -z "$(git status --porcelain)"
|
||||
test -f dockerfiles/Dockerfile.hermes-agent
|
||||
case "${BUILD_NUMBER}" in
|
||||
''|0*|*[!0-9]*)
|
||||
echo "BUILD_NUMBER must be a positive decimal integer" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
printf '%s\n' \
|
||||
"${HERMES_IMAGE}:git-${actual_revision}-build-${BUILD_NUMBER}" \
|
||||
> build/hermes-agent.destination
|
||||
'''
|
||||
}
|
||||
}
|
||||
stage('Reject replay before publish') {
|
||||
steps {
|
||||
withCredentials([usernamePassword(
|
||||
credentialsId: 'harbor-robot',
|
||||
usernameVariable: 'HARBOR_USER',
|
||||
passwordVariable: 'HARBOR_PASSWORD'
|
||||
)]) {
|
||||
sh '''
|
||||
set -eu
|
||||
set +x
|
||||
destination="$(cat build/hermes-agent.destination)"
|
||||
python3 ci/scripts/hermes_image_release.py assert-absent \
|
||||
--source-revision "${EXPECTED_SOURCE_REVISION}" \
|
||||
--build-number "${BUILD_NUMBER}" \
|
||||
--destination "${destination}"
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Build and publish without a daemon') {
|
||||
steps {
|
||||
container('kaniko') {
|
||||
withCredentials([usernamePassword(
|
||||
credentialsId: 'harbor-robot',
|
||||
usernameVariable: 'HARBOR_USER',
|
||||
passwordVariable: 'HARBOR_PASSWORD'
|
||||
)]) {
|
||||
sh '''#!/busybox/sh
|
||||
set -eu
|
||||
set +x
|
||||
config_path=/kaniko/.docker/config.json
|
||||
destination="$(cat build/hermes-agent.destination)"
|
||||
umask 077
|
||||
auth="$(printf '%s:%s' "${HARBOR_USER}" "${HARBOR_PASSWORD}" | /busybox/base64 | /busybox/tr -d '\n')"
|
||||
/busybox/mkdir -p /kaniko/.docker
|
||||
/busybox/printf '{"auths":{"registry.bstein.dev":{"auth":"%s"}}}\n' "${auth}" > "${config_path}"
|
||||
unset HARBOR_USER HARBOR_PASSWORD auth
|
||||
trap '/busybox/rm -f "${config_path}"' EXIT HUP INT TERM
|
||||
/kaniko/executor \
|
||||
--context="dir://${WORKSPACE}" \
|
||||
--dockerfile="${WORKSPACE}/dockerfiles/Dockerfile.hermes-agent" \
|
||||
--destination="${destination}" \
|
||||
--digest-file="${WORKSPACE}/build/hermes-agent.digest" \
|
||||
--image-name-tag-with-digest-file="${WORKSPACE}/build/hermes-agent.image" \
|
||||
--build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1 \
|
||||
--label="org.opencontainers.image.revision=${EXPECTED_SOURCE_REVISION}" \
|
||||
--cleanup \
|
||||
--push-retry=3
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Render reviewed Flux handoff') {
|
||||
steps {
|
||||
withCredentials([usernamePassword(
|
||||
credentialsId: 'harbor-robot',
|
||||
usernameVariable: 'HARBOR_USER',
|
||||
passwordVariable: 'HARBOR_PASSWORD'
|
||||
)]) {
|
||||
sh '''
|
||||
set -eu
|
||||
set +x
|
||||
destination="$(cat build/hermes-agent.destination)"
|
||||
python3 ci/scripts/hermes_image_release.py render \
|
||||
--digest-file build/hermes-agent.digest \
|
||||
--image-file build/hermes-agent.image \
|
||||
--source-revision "${EXPECTED_SOURCE_REVISION}" \
|
||||
--build-number "${BUILD_NUMBER}" \
|
||||
--destination "${destination}" \
|
||||
--kustomization services/hermes/kustomization.yaml \
|
||||
--output-dir build/hermes-agent-release
|
||||
test -s build/hermes-agent-release/hermes-image-update.patch
|
||||
test -s build/hermes-agent-release/hermes-agent-image.json
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
post {
|
||||
success {
|
||||
sh '''
|
||||
set -eu
|
||||
expected_files="$(printf '%s\n' \
|
||||
build/hermes-agent.destination \
|
||||
build/hermes-agent.digest \
|
||||
build/hermes-agent.image \
|
||||
build/hermes-agent-release/hermes-agent-image.json \
|
||||
build/hermes-agent-release/hermes-image-update.patch \
|
||||
build/hermes-agent-release/hermes-kustomization.yaml \
|
||||
| LC_ALL=C sort)"
|
||||
actual_files="$(find build -type f -print | LC_ALL=C sort)"
|
||||
test "${actual_files}" = "${expected_files}"
|
||||
destination="$(cat build/hermes-agent.destination)"
|
||||
python3 ci/scripts/hermes_image_release.py verify-evidence \
|
||||
--digest-file build/hermes-agent.digest \
|
||||
--image-file build/hermes-agent.image \
|
||||
--source-revision "${EXPECTED_SOURCE_REVISION}" \
|
||||
--build-number "${BUILD_NUMBER}" \
|
||||
--destination "${destination}" \
|
||||
--kustomization services/hermes/kustomization.yaml \
|
||||
--output-dir build/hermes-agent-release
|
||||
'''
|
||||
archiveArtifacts(
|
||||
artifacts: 'build/hermes-agent.destination,build/hermes-agent.digest,build/hermes-agent.image,build/hermes-agent-release/hermes-agent-image.json,build/hermes-agent-release/hermes-image-update.patch,build/hermes-agent-release/hermes-kustomization.yaml',
|
||||
allowEmptyArchive: false,
|
||||
fingerprint: true
|
||||
)
|
||||
}
|
||||
cleanup {
|
||||
container('kaniko') {
|
||||
sh '''#!/busybox/sh
|
||||
/busybox/rm -f /kaniko/.docker/config.json
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
500
ci/scripts/hermes_image_release.py
Executable file
500
ci/scripts/hermes_image_release.py
Executable file
@ -0,0 +1,500 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify and render a reviewable Hermes agent image release."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import difflib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable
|
||||
|
||||
|
||||
DEFAULT_IMAGE = "registry.bstein.dev/bstein/hermes-agent"
|
||||
HARBOR_API_ORIGIN = "https://registry.bstein.dev/api/v2.0"
|
||||
HARBOR_PROJECT = "bstein"
|
||||
HARBOR_REPOSITORY = "hermes-agent"
|
||||
IMMUTABLE_REPOSITORY_PATTERN = "hermes-agent"
|
||||
IMMUTABLE_TAG_PATTERN = "git-*-build-*"
|
||||
DIGEST_PATTERN = re.compile(r"^sha256:[0-9a-f]{64}$")
|
||||
REVISION_PATTERN = re.compile(r"^[0-9a-f]{40}$")
|
||||
BUILD_PATTERN = re.compile(r"^[1-9][0-9]*$")
|
||||
DESTINATION_PATTERN = re.compile(
|
||||
r"^registry\.bstein\.dev/bstein/hermes-agent:"
|
||||
r"git-([0-9a-f]{40})-build-([1-9][0-9]*)$"
|
||||
)
|
||||
|
||||
|
||||
class _NoRedirect(urllib.request.HTTPRedirectHandler):
|
||||
"""Never send registry credentials to a redirect target."""
|
||||
|
||||
def redirect_request(self, _request, _file, _code, _message, _headers, _url):
|
||||
return None
|
||||
|
||||
|
||||
def _validated(value: str, pattern: re.Pattern[str], label: str) -> str:
|
||||
"""Return a normalized value when it matches the release contract."""
|
||||
normalized = value.strip()
|
||||
if not pattern.fullmatch(normalized):
|
||||
raise ValueError(f"invalid {label}: expected {pattern.pattern}")
|
||||
return normalized
|
||||
|
||||
|
||||
def validate_destination(
|
||||
destination: str, source_revision: str, build_number: str
|
||||
) -> tuple[str, str]:
|
||||
"""Bind one unique build tag to the reviewed revision and Jenkins build."""
|
||||
revision = _validated(source_revision, REVISION_PATTERN, "source revision")
|
||||
build = _validated(build_number, BUILD_PATTERN, "build number")
|
||||
match = DESTINATION_PATTERN.fullmatch(destination.strip())
|
||||
if not match or match.groups() != (revision, build):
|
||||
raise ValueError(
|
||||
"destination must bind the reviewed revision and unique Jenkins build"
|
||||
)
|
||||
return revision, build
|
||||
|
||||
|
||||
def validate_kaniko_evidence(
|
||||
*, digest_text: str, image_text: str, destination: str
|
||||
) -> str:
|
||||
"""Cross-check both independent Kaniko output files against the destination."""
|
||||
digest_lines = digest_text.splitlines()
|
||||
image_lines = image_text.splitlines()
|
||||
if len(digest_lines) != 1:
|
||||
raise ValueError("Kaniko digest evidence must contain exactly one line")
|
||||
if len(image_lines) != 1:
|
||||
raise ValueError("Kaniko image evidence must contain exactly one line")
|
||||
digest = _validated(digest_lines[0], DIGEST_PATTERN, "image digest")
|
||||
if image_lines[0].strip() != f"{destination}@{digest}":
|
||||
raise ValueError("Kaniko image evidence does not match destination and digest")
|
||||
return digest
|
||||
|
||||
|
||||
def _registry_request(request: urllib.request.Request, timeout: int) -> Any:
|
||||
"""Make a registry request without following redirects."""
|
||||
opener = urllib.request.build_opener(_NoRedirect())
|
||||
try:
|
||||
return opener.open(request, timeout=timeout)
|
||||
except urllib.error.HTTPError as exc:
|
||||
return exc
|
||||
|
||||
|
||||
def _artifact_response(
|
||||
destination: str,
|
||||
*,
|
||||
username: str,
|
||||
password: str,
|
||||
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
|
||||
) -> tuple[int, bytes]:
|
||||
"""Read one exact Harbor artifact by tag with bounded response size."""
|
||||
match = DESTINATION_PATTERN.fullmatch(destination)
|
||||
if not match:
|
||||
raise ValueError("invalid destination")
|
||||
if not username or not password:
|
||||
raise RuntimeError("Harbor credentials are empty")
|
||||
tag = destination.rsplit(":", 1)[1]
|
||||
encoded_tag = urllib.parse.quote(tag, safe="")
|
||||
auth = base64.b64encode(f"{username}:{password}".encode()).decode("ascii")
|
||||
request = urllib.request.Request(
|
||||
f"{HARBOR_API_ORIGIN}/projects/{HARBOR_PROJECT}/repositories/"
|
||||
f"{HARBOR_REPOSITORY}/artifacts/{encoded_tag}"
|
||||
"?with_immutable_status=true",
|
||||
headers={"Accept": "application/json", "Authorization": f"Basic {auth}"},
|
||||
method="GET",
|
||||
)
|
||||
with opener(request, 20) as response:
|
||||
body = response.read(1_048_577)
|
||||
if len(body) > 1_048_576:
|
||||
raise RuntimeError("Harbor artifact response exceeded the size limit")
|
||||
return int(response.status), body
|
||||
|
||||
|
||||
def _immutable_rules_response(
|
||||
*,
|
||||
username: str,
|
||||
password: str,
|
||||
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
|
||||
) -> tuple[int, bytes, dict[str, str]]:
|
||||
"""Read the project policy with the same least-privilege publish identity."""
|
||||
if not username or not password:
|
||||
raise RuntimeError("Harbor credentials are empty")
|
||||
auth = base64.b64encode(f"{username}:{password}".encode()).decode("ascii")
|
||||
request = urllib.request.Request(
|
||||
f"{HARBOR_API_ORIGIN}/projects/{HARBOR_PROJECT}/immutabletagrules"
|
||||
"?page=1&page_size=100",
|
||||
headers={"Accept": "application/json", "Authorization": f"Basic {auth}"},
|
||||
method="GET",
|
||||
)
|
||||
with opener(request, 20) as response:
|
||||
body = response.read(1_048_577)
|
||||
if len(body) > 1_048_576:
|
||||
raise RuntimeError("Harbor immutable rule response exceeded the size limit")
|
||||
return int(response.status), body, dict(response.headers)
|
||||
|
||||
|
||||
def _require_complete_rule_page(
|
||||
rules: list[dict[str, Any]], headers: dict[str, str]
|
||||
) -> None:
|
||||
"""Require proof that the bounded first page contains every rule."""
|
||||
raw_total = next(
|
||||
(value for key, value in headers.items() if key.lower() == "x-total-count"),
|
||||
None,
|
||||
)
|
||||
if raw_total is None or not str(raw_total).isdecimal():
|
||||
raise RuntimeError("Harbor immutable rule list omitted a valid total count")
|
||||
if int(raw_total) != len(rules):
|
||||
raise RuntimeError("Harbor immutable rule list was truncated")
|
||||
|
||||
|
||||
def _normalized_immutable_rule(rule: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Select only fields that bind the server-side build-tag policy."""
|
||||
return {
|
||||
"disabled": bool(rule.get("disabled", False)),
|
||||
"action": rule.get("action"),
|
||||
"template": rule.get("template"),
|
||||
"tag_selectors": [
|
||||
{
|
||||
"kind": item.get("kind"),
|
||||
"decoration": item.get("decoration"),
|
||||
"pattern": item.get("pattern"),
|
||||
}
|
||||
for item in rule.get("tag_selectors") or []
|
||||
if isinstance(item, dict)
|
||||
],
|
||||
"scope_selectors": {
|
||||
"repository": [
|
||||
{
|
||||
"kind": item.get("kind"),
|
||||
"decoration": item.get("decoration"),
|
||||
"pattern": item.get("pattern"),
|
||||
}
|
||||
for item in (rule.get("scope_selectors") or {}).get(
|
||||
"repository", []
|
||||
)
|
||||
if isinstance(item, dict)
|
||||
]
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def verify_immutable_policy(
|
||||
*,
|
||||
username: str,
|
||||
password: str,
|
||||
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
|
||||
) -> None:
|
||||
"""Fail closed before build unless the exact Harbor rule is active."""
|
||||
status, body, headers = _immutable_rules_response(
|
||||
username=username, password=password, opener=opener
|
||||
)
|
||||
if status != 200:
|
||||
raise RuntimeError(f"Harbor immutable policy preflight returned HTTP {status}")
|
||||
try:
|
||||
rules = json.loads(body.decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise RuntimeError("Harbor returned invalid immutable rule JSON") from exc
|
||||
if not isinstance(rules, list) or not all(isinstance(item, dict) for item in rules):
|
||||
raise RuntimeError("Harbor immutable rule list has an invalid shape")
|
||||
_require_complete_rule_page(rules, headers)
|
||||
expected = {
|
||||
"disabled": False,
|
||||
"action": "immutable",
|
||||
"template": "immutable_template",
|
||||
"tag_selectors": [
|
||||
{
|
||||
"kind": "doublestar",
|
||||
"decoration": "matches",
|
||||
"pattern": IMMUTABLE_TAG_PATTERN,
|
||||
}
|
||||
],
|
||||
"scope_selectors": {
|
||||
"repository": [
|
||||
{
|
||||
"kind": "doublestar",
|
||||
"decoration": "repoMatches",
|
||||
"pattern": IMMUTABLE_REPOSITORY_PATTERN,
|
||||
}
|
||||
]
|
||||
},
|
||||
}
|
||||
matches = [
|
||||
_normalized_immutable_rule(item)
|
||||
for item in rules
|
||||
if _normalized_immutable_rule(item)["tag_selectors"]
|
||||
== expected["tag_selectors"]
|
||||
and _normalized_immutable_rule(item)["scope_selectors"]
|
||||
== expected["scope_selectors"]
|
||||
]
|
||||
if matches != [expected]:
|
||||
raise RuntimeError("Harbor immutable build-tag policy is absent or not exact")
|
||||
|
||||
|
||||
def assert_tag_absent(
|
||||
destination: str,
|
||||
*,
|
||||
username: str,
|
||||
password: str,
|
||||
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
|
||||
) -> None:
|
||||
"""Reject replay before Kaniko can push an already-used immutable identity."""
|
||||
status, _body = _artifact_response(
|
||||
destination, username=username, password=password, opener=opener
|
||||
)
|
||||
if status == 404:
|
||||
return
|
||||
if status == 200:
|
||||
raise RuntimeError("Harbor destination tag already exists; refusing overwrite")
|
||||
raise RuntimeError(f"Harbor destination preflight returned HTTP {status}")
|
||||
|
||||
|
||||
def verify_registry_digest(
|
||||
destination: str,
|
||||
digest: str,
|
||||
*,
|
||||
username: str,
|
||||
password: str,
|
||||
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
|
||||
) -> None:
|
||||
"""Verify Harbor independently resolves the pushed tag to Kaniko's digest."""
|
||||
digest = _validated(digest, DIGEST_PATTERN, "image digest")
|
||||
status, body = _artifact_response(
|
||||
destination, username=username, password=password, opener=opener
|
||||
)
|
||||
if status != 200:
|
||||
raise RuntimeError(f"Harbor manifest verification returned HTTP {status}")
|
||||
try:
|
||||
artifact = json.loads(body.decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise RuntimeError("Harbor returned invalid artifact JSON") from exc
|
||||
harbor_digest = str(artifact.get("digest") or "").strip()
|
||||
if not DIGEST_PATTERN.fullmatch(harbor_digest):
|
||||
raise RuntimeError("Harbor response omitted a valid artifact digest")
|
||||
if harbor_digest != digest:
|
||||
raise RuntimeError("Harbor digest does not match Kaniko evidence")
|
||||
expected_tag = destination.rsplit(":", 1)[1]
|
||||
matching_tags = [
|
||||
item
|
||||
for item in artifact.get("tags") or []
|
||||
if isinstance(item, dict) and item.get("name") == expected_tag
|
||||
]
|
||||
if len(matching_tags) != 1:
|
||||
raise RuntimeError("Harbor artifact does not contain the expected tag")
|
||||
if matching_tags[0].get("immutable") is not True:
|
||||
raise RuntimeError("Harbor did not enforce the expected tag as immutable")
|
||||
|
||||
|
||||
def render_kustomization(source: str, digest: str, image: str = DEFAULT_IMAGE) -> str:
|
||||
"""Replace exactly one matching Kustomize image digest without reformatting."""
|
||||
digest = _validated(digest, DIGEST_PATTERN, "image digest")
|
||||
lines = source.splitlines(keepends=True)
|
||||
matches: list[int] = []
|
||||
|
||||
for index, line in enumerate(lines):
|
||||
if line.strip() != f"- name: {image}":
|
||||
continue
|
||||
name_indent = len(line) - len(line.lstrip())
|
||||
for candidate_index in range(index + 1, len(lines)):
|
||||
candidate = lines[candidate_index]
|
||||
stripped = candidate.strip()
|
||||
candidate_indent = len(candidate) - len(candidate.lstrip())
|
||||
if stripped.startswith("- name:") and candidate_indent == name_indent:
|
||||
break
|
||||
if stripped.startswith("digest:") and candidate_indent > name_indent:
|
||||
matches.append(candidate_index)
|
||||
break
|
||||
|
||||
if len(matches) != 1:
|
||||
raise ValueError(
|
||||
f"expected exactly one digest for image {image!r}; found {len(matches)}"
|
||||
)
|
||||
|
||||
index = matches[0]
|
||||
newline = "\n" if lines[index].endswith("\n") else ""
|
||||
prefix = lines[index][: len(lines[index]) - len(lines[index].lstrip())]
|
||||
lines[index] = f"{prefix}digest: {digest}{newline}"
|
||||
return "".join(lines)
|
||||
|
||||
|
||||
def write_release_artifacts(
|
||||
*,
|
||||
digest: str,
|
||||
source_revision: str,
|
||||
build_number: str,
|
||||
destination: str,
|
||||
kustomization: Path,
|
||||
output_dir: Path,
|
||||
) -> dict[str, str]:
|
||||
"""Write a rendered manifest, patch, and credential-free release metadata."""
|
||||
digest = _validated(digest, DIGEST_PATTERN, "image digest")
|
||||
source_revision, build_number = validate_destination(
|
||||
destination, source_revision, build_number
|
||||
)
|
||||
source = kustomization.read_text(encoding="utf-8")
|
||||
rendered = render_kustomization(source, digest)
|
||||
relative_name = kustomization.name
|
||||
patch = "".join(
|
||||
difflib.unified_diff(
|
||||
source.splitlines(keepends=True),
|
||||
rendered.splitlines(keepends=True),
|
||||
fromfile=f"a/services/hermes/{relative_name}",
|
||||
tofile=f"b/services/hermes/{relative_name}",
|
||||
)
|
||||
)
|
||||
if not patch:
|
||||
raise ValueError("published digest already matches the Flux manifest")
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
(output_dir / "hermes-kustomization.yaml").write_text(rendered, encoding="utf-8")
|
||||
(output_dir / "hermes-image-update.patch").write_text(patch, encoding="utf-8")
|
||||
metadata = {
|
||||
"build_number": build_number,
|
||||
"digest": digest,
|
||||
"flux_image": f"{DEFAULT_IMAGE}@{digest}",
|
||||
"image": DEFAULT_IMAGE,
|
||||
"published_tag": destination,
|
||||
"source_revision": source_revision,
|
||||
}
|
||||
(output_dir / "hermes-agent-image.json").write_text(
|
||||
json.dumps(metadata, indent=2, sort_keys=True) + "\n", encoding="utf-8"
|
||||
)
|
||||
return metadata
|
||||
|
||||
|
||||
def validate_release_artifacts(
|
||||
*,
|
||||
digest_file: Path,
|
||||
image_file: Path,
|
||||
source_revision: str,
|
||||
build_number: str,
|
||||
destination: str,
|
||||
kustomization: Path,
|
||||
output_dir: Path,
|
||||
) -> None:
|
||||
"""Revalidate the exact successful-build evidence without rewriting it."""
|
||||
digest = validate_kaniko_evidence(
|
||||
digest_text=digest_file.read_text(encoding="utf-8"),
|
||||
image_text=image_file.read_text(encoding="utf-8"),
|
||||
destination=destination,
|
||||
)
|
||||
source_revision, build_number = validate_destination(
|
||||
destination, source_revision, build_number
|
||||
)
|
||||
expected_names = {
|
||||
"hermes-agent-image.json",
|
||||
"hermes-image-update.patch",
|
||||
"hermes-kustomization.yaml",
|
||||
}
|
||||
entries = list(output_dir.iterdir())
|
||||
if {entry.name for entry in entries} != expected_names or not all(
|
||||
entry.is_file() and not entry.is_symlink() for entry in entries
|
||||
):
|
||||
raise ValueError("release output must contain exactly three evidence files")
|
||||
source = kustomization.read_text(encoding="utf-8")
|
||||
rendered = render_kustomization(source, digest)
|
||||
relative_name = kustomization.name
|
||||
patch = "".join(
|
||||
difflib.unified_diff(
|
||||
source.splitlines(keepends=True),
|
||||
rendered.splitlines(keepends=True),
|
||||
fromfile=f"a/services/hermes/{relative_name}",
|
||||
tofile=f"b/services/hermes/{relative_name}",
|
||||
)
|
||||
)
|
||||
metadata = {
|
||||
"build_number": build_number,
|
||||
"digest": digest,
|
||||
"flux_image": f"{DEFAULT_IMAGE}@{digest}",
|
||||
"image": DEFAULT_IMAGE,
|
||||
"published_tag": destination,
|
||||
"source_revision": source_revision,
|
||||
}
|
||||
expected = {
|
||||
"hermes-agent-image.json": json.dumps(metadata, indent=2, sort_keys=True) + "\n",
|
||||
"hermes-image-update.patch": patch,
|
||||
"hermes-kustomization.yaml": rendered,
|
||||
}
|
||||
for name, expected_text in expected.items():
|
||||
if (output_dir / name).read_text(encoding="utf-8") != expected_text:
|
||||
raise ValueError(f"release evidence is incomplete or mismatched: {name}")
|
||||
|
||||
|
||||
def _credentials() -> tuple[str, str]:
|
||||
"""Read the masked, runtime-only Jenkins credential environment."""
|
||||
username = os.environ.get("HARBOR_USER", "")
|
||||
password = os.environ.get("HARBOR_PASSWORD", "")
|
||||
if not username or not password:
|
||||
raise RuntimeError("Harbor credentials are unavailable")
|
||||
return username, password
|
||||
|
||||
|
||||
def _common_arguments(parser: argparse.ArgumentParser) -> None:
|
||||
parser.add_argument("--source-revision", required=True)
|
||||
parser.add_argument("--build-number", required=True)
|
||||
parser.add_argument("--destination", required=True)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Fail closed around the unique tag, then verify and render after push."""
|
||||
parser = argparse.ArgumentParser()
|
||||
commands = parser.add_subparsers(dest="command", required=True)
|
||||
absent = commands.add_parser("assert-absent")
|
||||
_common_arguments(absent)
|
||||
render = commands.add_parser("render")
|
||||
_common_arguments(render)
|
||||
render.add_argument("--digest-file", required=True, type=Path)
|
||||
render.add_argument("--image-file", required=True, type=Path)
|
||||
render.add_argument("--kustomization", required=True, type=Path)
|
||||
render.add_argument("--output-dir", required=True, type=Path)
|
||||
verify = commands.add_parser("verify-evidence")
|
||||
_common_arguments(verify)
|
||||
verify.add_argument("--digest-file", required=True, type=Path)
|
||||
verify.add_argument("--image-file", required=True, type=Path)
|
||||
verify.add_argument("--kustomization", required=True, type=Path)
|
||||
verify.add_argument("--output-dir", required=True, type=Path)
|
||||
args = parser.parse_args()
|
||||
|
||||
validate_destination(args.destination, args.source_revision, args.build_number)
|
||||
if args.command == "verify-evidence":
|
||||
validate_release_artifacts(
|
||||
digest_file=args.digest_file,
|
||||
image_file=args.image_file,
|
||||
source_revision=args.source_revision,
|
||||
build_number=args.build_number,
|
||||
destination=args.destination,
|
||||
kustomization=args.kustomization,
|
||||
output_dir=args.output_dir,
|
||||
)
|
||||
return 0
|
||||
|
||||
username, password = _credentials()
|
||||
if args.command == "assert-absent":
|
||||
verify_immutable_policy(username=username, password=password)
|
||||
assert_tag_absent(args.destination, username=username, password=password)
|
||||
return 0
|
||||
|
||||
digest = validate_kaniko_evidence(
|
||||
digest_text=args.digest_file.read_text(encoding="utf-8"),
|
||||
image_text=args.image_file.read_text(encoding="utf-8"),
|
||||
destination=args.destination,
|
||||
)
|
||||
verify_registry_digest(
|
||||
args.destination, digest, username=username, password=password
|
||||
)
|
||||
write_release_artifacts(
|
||||
digest=digest,
|
||||
source_revision=args.source_revision,
|
||||
build_number=args.build_number,
|
||||
destination=args.destination,
|
||||
kustomization=args.kustomization,
|
||||
output_dir=args.output_dir,
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover - exercised through main()
|
||||
raise SystemExit(main())
|
||||
@ -93,9 +93,8 @@ def _sonar_issue_from_finding(finding: dict[str, Any]) -> dict[str, Any] | None:
|
||||
start = finding.get("start") if isinstance(finding.get("start"), dict) else {}
|
||||
end = finding.get("end") if isinstance(finding.get("end"), dict) else {}
|
||||
start_line = _line_number(start.get("line") if isinstance(start, dict) else None)
|
||||
# _line_number floors the end line at start_line, so the range never inverts.
|
||||
end_line = _line_number(end.get("line") if isinstance(end, dict) else None, start_line)
|
||||
if end_line < start_line:
|
||||
end_line = start_line
|
||||
return {
|
||||
"engineId": "semgrep",
|
||||
"ruleId": str(finding.get("check_id") or "semgrep.unknown"),
|
||||
|
||||
@ -15,7 +15,13 @@ spec:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
namespace: flux-system
|
||||
wait: false
|
||||
wait: true
|
||||
timeout: 10m
|
||||
healthChecks:
|
||||
- apiVersion: batch/v1
|
||||
kind: Job
|
||||
name: harbor-hermes-agent-immutability-ensure-1
|
||||
namespace: harbor
|
||||
dependsOn:
|
||||
- name: core
|
||||
- name: longhorn
|
||||
|
||||
@ -66,5 +66,6 @@ spec:
|
||||
- name: keycloak
|
||||
- name: longhorn
|
||||
- name: vault
|
||||
- name: jenkins
|
||||
- name: hermes-scm-broker
|
||||
- name: hermes-observer-rbac
|
||||
|
||||
@ -6,10 +6,9 @@ metadata:
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
atlas.bstein.dev/suspend-reason: "CI controller changes are applied only during planned maintenance."
|
||||
spec:
|
||||
interval: 10m
|
||||
suspend: true
|
||||
suspend: false
|
||||
path: ./services/jenkins
|
||||
prune: true
|
||||
sourceRef:
|
||||
@ -18,6 +17,8 @@ spec:
|
||||
targetNamespace: jenkins
|
||||
dependsOn:
|
||||
- name: helm
|
||||
- name: harbor
|
||||
- name: vault-hermes-jenkins-token-seed
|
||||
healthChecks:
|
||||
- apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
|
||||
@ -4,6 +4,7 @@ kind: Kustomization
|
||||
resources:
|
||||
- gitea/kustomization.yaml
|
||||
- vault/kustomization.yaml
|
||||
- vault-hermes-jenkins-token-seed/kustomization.yaml
|
||||
- vaultwarden/kustomization.yaml
|
||||
- comms/kustomization.yaml
|
||||
- crypto/kustomization.yaml
|
||||
|
||||
@ -0,0 +1,26 @@
|
||||
# clusters/atlas/flux-system/applications/vault-hermes-jenkins-token-seed/kustomization.yaml
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: vault-hermes-jenkins-token-seed
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
spec:
|
||||
interval: 10m
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
namespace: flux-system
|
||||
path: ./services/vault-hermes-jenkins-token-seed
|
||||
targetNamespace: vault
|
||||
prune: true
|
||||
wait: true
|
||||
timeout: 5m
|
||||
dependsOn:
|
||||
- name: vault
|
||||
healthChecks:
|
||||
- apiVersion: batch/v1
|
||||
kind: Job
|
||||
name: vault-hermes-jenkins-build-token-seed-2
|
||||
namespace: vault
|
||||
@ -16,6 +16,11 @@ spec:
|
||||
targetNamespace: vault
|
||||
prune: true
|
||||
wait: true
|
||||
healthChecks:
|
||||
- apiVersion: batch/v1
|
||||
kind: Job
|
||||
name: vault-k8s-auth-hermes-10
|
||||
namespace: vault
|
||||
dependsOn:
|
||||
- name: longhorn
|
||||
- name: helm
|
||||
|
||||
@ -15,6 +15,10 @@ RUN apt-get update \
|
||||
# Paid/provider-backed search remains selectable through normal Hermes config.
|
||||
RUN uv pip install --python /opt/hermes/.venv/bin/python ddgs==9.14.4
|
||||
|
||||
ARG HERMES_KANIKO_HEREDOC_COMPAT=0
|
||||
COPY dockerfiles/Dockerfile.hermes-agent /tmp/hermes-agent.Dockerfile
|
||||
COPY dockerfiles/hermes-kaniko-heredoc-runner.py /tmp/hermes-kaniko-heredoc-runner.py
|
||||
|
||||
# Keep dashboard chat sockets tied to the intended React mount and conversation.
|
||||
# A resumed conversation needs a different PTY attachment key from a fresh chat;
|
||||
# reconnects to that same conversation must keep using the same key.
|
||||
@ -252,6 +256,7 @@ for (const [before, after, label] of [
|
||||
}
|
||||
fs.writeFileSync(sidebarPath, sidebar);
|
||||
NODE
|
||||
RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 1 ;; *) exit 2 ;; esac
|
||||
|
||||
# The upstream OIDC gate authenticates users but deliberately treats the
|
||||
# dashboard as one shared workstation. Allow a deployment to narrow that
|
||||
@ -324,6 +329,7 @@ for before, after, label in (
|
||||
source = source.replace(before, after, 1)
|
||||
path.write_text(source)
|
||||
PY
|
||||
RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 2 ;; *) exit 2 ;; esac
|
||||
|
||||
# Give trusted plugins a pre-turn routing hook. It runs after fallback runtime
|
||||
# restoration but before Hermes builds its provider-specific system prompt.
|
||||
@ -493,6 +499,7 @@ if oneshot.count(oneshot_before) != 1:
|
||||
)
|
||||
oneshot_path.write_text(oneshot.replace(oneshot_before, oneshot_after, 1))
|
||||
PY
|
||||
RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 3 ;; *) exit 2 ;; esac
|
||||
|
||||
# Hermes WebUI sends its model/provider/reasoning selection on /v1/runs.
|
||||
# Upstream currently applies only statically declared model_routes there, so
|
||||
@ -628,6 +635,7 @@ runs_source = runs_source.replace(runs_agent_before, runs_agent_after, 1)
|
||||
source = source[:runs_start] + runs_source
|
||||
path.write_text(source)
|
||||
PY
|
||||
RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 4 ;; *) exit 2 ;; esac
|
||||
|
||||
# ``create_task(initial_status="blocked")`` is an explicit operator park, but
|
||||
# upstream only treats later block_task() events as sticky. Recognize the
|
||||
@ -779,6 +787,7 @@ if cli_source.count(help_before) != 1:
|
||||
)
|
||||
cli_path.write_text(cli_source.replace(help_before, help_after, 1))
|
||||
PY
|
||||
RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 5 ;; *) exit 2 ;; esac
|
||||
|
||||
COPY dockerfiles/hermes-kanban-blocked-regression.py /tmp/hermes-kanban-blocked-regression.py
|
||||
RUN /opt/hermes/.venv/bin/python /tmp/hermes-kanban-blocked-regression.py \
|
||||
@ -816,6 +825,7 @@ if source.count(before) != 1:
|
||||
)
|
||||
path.write_text(source.replace(before, after, 1))
|
||||
PY
|
||||
RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 6 ;; *) exit 2 ;; esac
|
||||
|
||||
# Keep the dashboard's account cards aligned with the managed provider lanes.
|
||||
# Switchyard receives Claude Code subscription OAuth through the environment;
|
||||
@ -882,6 +892,7 @@ for before, after, label in replacements:
|
||||
|
||||
path.write_text(source)
|
||||
PY
|
||||
RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 7 ;; *) exit 2 ;; esac
|
||||
|
||||
# Make parent-linked API workers first-class live dashboard sessions. They stay
|
||||
# active until the API runner closes them, even during a long final model call;
|
||||
@ -975,6 +986,7 @@ source = source.replace(row_before, row_after)
|
||||
|
||||
path.write_text(source)
|
||||
PY
|
||||
RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 8 ;; *) exit 2 ;; esac
|
||||
|
||||
# Keep the browser terminal reliable across GPU context loss, make durable
|
||||
# worker lineage visible, expose an accessible API-worker transcript, and
|
||||
@ -1543,6 +1555,7 @@ function RootRedirect() {
|
||||
fs.writeFileSync(path, source);
|
||||
}
|
||||
NODE
|
||||
RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 9 ;; *) exit 2 ;; esac
|
||||
|
||||
COPY dockerfiles/patch-hermes-execution-safety.py /tmp/patch-hermes-execution-safety.py
|
||||
COPY dockerfiles/hermes-execution-safety-regression.py /tmp/hermes-execution-safety-regression.py
|
||||
@ -1576,6 +1589,8 @@ RUN HERMES_CLI_LANE_SOURCE=/tmp/hermes-lane-regression \
|
||||
|
||||
COPY dockerfiles/hermes-session-migrate.py /opt/hermes/bin/hermes-session-migrate
|
||||
|
||||
RUN rm -f /tmp/hermes-agent.Dockerfile /tmp/hermes-kaniko-heredoc-runner.py
|
||||
|
||||
RUN cd /opt/hermes/web \
|
||||
&& npm run build \
|
||||
&& grep -Fq 'await api.getSessions(1, 0' src/pages/ChatPage.tsx \
|
||||
|
||||
@ -5,6 +5,8 @@
|
||||
!dockerfiles/hermes-public-extract/**
|
||||
!dockerfiles/hermes-session-activity-panel.tsx
|
||||
!dockerfiles/hermes-session-migrate.py
|
||||
!dockerfiles/Dockerfile.hermes-agent
|
||||
!dockerfiles/hermes-kaniko-heredoc-runner.py
|
||||
!dockerfiles/patch-hermes-execution-safety.py
|
||||
!dockerfiles/hermes-execution-safety-regression.py
|
||||
!dockerfiles/hermes_execution_patch_support.py
|
||||
|
||||
146
dockerfiles/hermes-kaniko-heredoc-runner.py
Normal file
146
dockerfiles/hermes-kaniko-heredoc-runner.py
Normal file
@ -0,0 +1,146 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Replay the reviewed Hermes Dockerfile heredocs for pinned Kaniko v1.23.2.
|
||||
|
||||
Kaniko v1.23.2 parses shell heredoc bodies into ``RunCommand.Files`` but its RUN
|
||||
implementation executes only ``CmdLine``. The interpreters therefore receive
|
||||
empty stdin and exit successfully. Docker and BuildKit execute these blocks
|
||||
normally, so this compatibility runner is enabled only by the Kaniko pipeline.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
MAX_DOCKERFILE_BYTES = 2_000_000
|
||||
BLOCKS = (
|
||||
("RUN node <<'NODE'", "NODE", ("node",)),
|
||||
("RUN python - <<'PY'", "PY", ("python", "-")),
|
||||
)
|
||||
EXPECTED_COMMANDS = ("node", *("python",) * 7, "node")
|
||||
PARSER_DIRECTIVE = re.compile(
|
||||
r"^\s*#\s*([a-z]+)\s*=\s*(.*?)\s*$", re.IGNORECASE
|
||||
)
|
||||
|
||||
|
||||
def _escape_character(lines: list[str]) -> str:
|
||||
"""Return the one Dockerfile parser escape directive, or its default."""
|
||||
escape = "\\"
|
||||
directives: set[str] = set()
|
||||
for line in lines:
|
||||
if not line.strip():
|
||||
break
|
||||
match = PARSER_DIRECTIVE.fullmatch(line)
|
||||
if not match:
|
||||
break
|
||||
name, value = match.group(1).lower(), match.group(2)
|
||||
if name in directives:
|
||||
raise ValueError(f"multiple Dockerfile {name} directives")
|
||||
directives.add(name)
|
||||
if name == "escape":
|
||||
if value not in {"\\", "`"}:
|
||||
raise ValueError("unsupported Dockerfile escape directive")
|
||||
escape = value
|
||||
return escape
|
||||
|
||||
|
||||
def _continued(line: str, escape: str) -> bool:
|
||||
"""Match Docker's unescaped continuation marker at physical line end."""
|
||||
stripped = line.rstrip()
|
||||
count = len(stripped) - len(stripped.rstrip(escape))
|
||||
return count % 2 == 1
|
||||
|
||||
|
||||
def _logical_instruction(
|
||||
lines: list[str], index: int, escape: str
|
||||
) -> tuple[str, int]:
|
||||
"""Normalize escape-newline pairs, including split opcodes/operators."""
|
||||
logical = lines[index]
|
||||
while _continued(logical.split("\n")[-1], escape):
|
||||
logical = logical.rstrip()
|
||||
logical = logical[:-1]
|
||||
index += 1
|
||||
while index < len(lines) and lines[index].lstrip().startswith("#"):
|
||||
index += 1
|
||||
if index >= len(lines) or not lines[index].strip():
|
||||
raise ValueError("unterminated Dockerfile line continuation")
|
||||
logical += lines[index].lstrip()
|
||||
return logical, index
|
||||
|
||||
|
||||
def extract_blocks(source: str) -> list[tuple[tuple[str, ...], str]]:
|
||||
"""Inventory every RUN heredoc, then return only the reviewed contract."""
|
||||
markers = {start: (end, command) for start, end, command in BLOCKS}
|
||||
lines = source.splitlines()
|
||||
escape = _escape_character(lines)
|
||||
blocks: list[tuple[tuple[str, ...], str]] = []
|
||||
index = 0
|
||||
while index < len(lines):
|
||||
marker = markers.get(lines[index])
|
||||
if marker is None:
|
||||
if not lines[index].strip() or lines[index].lstrip().startswith("#"):
|
||||
index += 1
|
||||
continue
|
||||
logical, index = _logical_instruction(lines, index, escape)
|
||||
stripped = logical.lstrip()
|
||||
if (
|
||||
stripped[:3].upper() == "RUN"
|
||||
and (len(stripped) == 3 or stripped[3].isspace())
|
||||
and "<<" in stripped
|
||||
):
|
||||
raise ValueError(
|
||||
"unsupported RUN heredoc outside the exact reviewed contract"
|
||||
)
|
||||
index += 1
|
||||
continue
|
||||
end, command = marker
|
||||
body_start = index + 1
|
||||
index = body_start
|
||||
while index < len(lines) and lines[index] != end:
|
||||
index += 1
|
||||
if index == len(lines):
|
||||
raise ValueError(f"unterminated {command[0]} heredoc")
|
||||
body = "\n".join(lines[body_start:index]) + "\n"
|
||||
if not body.strip():
|
||||
raise ValueError(f"empty {command[0]} heredoc")
|
||||
blocks.append((command, body))
|
||||
index += 1
|
||||
|
||||
commands = tuple(command[0] for command, _body in blocks)
|
||||
if commands != EXPECTED_COMMANDS:
|
||||
raise ValueError(
|
||||
"Hermes Dockerfile heredoc contract changed: "
|
||||
f"expected {EXPECTED_COMMANDS!r}, received {commands!r}"
|
||||
)
|
||||
return blocks
|
||||
|
||||
|
||||
def replay(dockerfile: Path, block_index: int) -> None:
|
||||
"""Execute one exact heredoc after inventorying the complete Dockerfile."""
|
||||
size = dockerfile.stat().st_size
|
||||
if size < 1 or size > MAX_DOCKERFILE_BYTES:
|
||||
raise ValueError("Hermes Dockerfile size is outside the reviewed boundary")
|
||||
source = dockerfile.read_text(encoding="utf-8")
|
||||
blocks = extract_blocks(source)
|
||||
if block_index < 1 or block_index > len(blocks):
|
||||
raise ValueError("heredoc block index is outside the reviewed contract")
|
||||
command, body = blocks[block_index - 1]
|
||||
print(f"replaying reviewed Dockerfile heredoc {block_index}/{len(blocks)}")
|
||||
subprocess.run(command, input=body, text=True, check=True)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Parse the one explicit Dockerfile path and replay its reviewed patches."""
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--dockerfile", required=True, type=Path)
|
||||
parser.add_argument("--block-index", required=True, type=int)
|
||||
args = parser.parse_args()
|
||||
replay(args.dockerfile, args.block_index)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@ -232,6 +232,24 @@ def test_retry_db_connect_reraises_final_error(monkeypatch):
|
||||
sync.retry_db_connect(attempts=1)
|
||||
|
||||
|
||||
def test_retry_helpers_return_none_without_attempts(monkeypatch):
|
||||
sync = load_sync_module(monkeypatch)
|
||||
|
||||
assert sync.retry_request("request", lambda: "never", attempts=0) is None
|
||||
assert sync.retry_db_connect(attempts=0) is None
|
||||
|
||||
|
||||
def test_kc_update_attributes_replaces_non_dict_attributes(monkeypatch):
|
||||
sync = load_sync_module(monkeypatch)
|
||||
current_resp = _FakeResponse({"attributes": "not-a-dict"})
|
||||
ok_resp = _FakeResponse({"attributes": {"mailu_app_password": ["abc"]}})
|
||||
sync.SESSION = _FakeSession(_FakeResponse({}), [current_resp, ok_resp])
|
||||
|
||||
sync.kc_update_attributes("token", {"id": "u1", "username": "u1"}, {"mailu_app_password": "abc"})
|
||||
|
||||
assert sync.SESSION.put_called
|
||||
|
||||
|
||||
def test_ensure_mailu_user_skips_foreign_domain(monkeypatch):
|
||||
sync = load_sync_module(monkeypatch)
|
||||
executed = []
|
||||
@ -405,3 +423,62 @@ def test_main_generates_password_and_upserts(monkeypatch):
|
||||
# Only mail-enabled users (or legacy users with a mailbox) are synced and backfilled.
|
||||
assert len(updated) == 3
|
||||
assert conns and len(conns[0]._cursor.executions) == 3
|
||||
|
||||
|
||||
def test_main_skips_disabled_users_and_provisioned_users_without_update(monkeypatch):
|
||||
sync = load_sync_module(monkeypatch)
|
||||
monkeypatch.setattr(sync.bcrypt_sha256, "hash", lambda password: f"hash:{password}")
|
||||
users = [
|
||||
{
|
||||
"id": "u1",
|
||||
"username": "disabled",
|
||||
"email": "disabled@example.com",
|
||||
"enabled": False,
|
||||
"attributes": {"mailu_enabled": ["true"]},
|
||||
},
|
||||
{
|
||||
"id": "u2",
|
||||
"username": "settled",
|
||||
"email": "settled@example.com",
|
||||
"attributes": {
|
||||
"mailu_enabled": ["true"],
|
||||
"mailu_email": ["settled@example.com"],
|
||||
"mailu_app_password": ["existing"],
|
||||
},
|
||||
},
|
||||
]
|
||||
updated = []
|
||||
|
||||
class _Cursor:
|
||||
def __init__(self):
|
||||
self.executions = []
|
||||
|
||||
def execute(self, sql, params):
|
||||
self.executions.append(params)
|
||||
|
||||
def close(self):
|
||||
return None
|
||||
|
||||
class _Conn:
|
||||
def __init__(self):
|
||||
self.autocommit = False
|
||||
self._cursor = _Cursor()
|
||||
|
||||
def cursor(self, cursor_factory=None):
|
||||
return self._cursor
|
||||
|
||||
def close(self):
|
||||
return None
|
||||
|
||||
conn = _Conn()
|
||||
monkeypatch.setattr(sync, "get_kc_token", lambda: "tok")
|
||||
monkeypatch.setattr(sync, "kc_get_users", lambda token: users)
|
||||
monkeypatch.setattr(sync, "kc_update_attributes", lambda token, user, attrs: updated.append(user["id"]))
|
||||
monkeypatch.setattr(sync.psycopg2, "connect", lambda **kwargs: conn)
|
||||
|
||||
sync.main()
|
||||
|
||||
# The already-provisioned user is synced without a Keycloak write; the
|
||||
# disabled user never reaches the mailbox upsert.
|
||||
assert updated == []
|
||||
assert len(conn._cursor.executions) == 1
|
||||
|
||||
96
scripts/tests/test_mailu_sync_edges.py
Normal file
96
scripts/tests/test_mailu_sync_edges.py
Normal file
@ -0,0 +1,96 @@
|
||||
"""Retry, attribute-corruption, and skip-path edges for the Mailu sync job."""
|
||||
|
||||
from test_mailu_sync import _FakeResponse, _FakeSession, load_sync_module
|
||||
|
||||
def test_retry_helpers_with_zero_attempts_do_nothing(monkeypatch):
|
||||
sync = load_sync_module(monkeypatch)
|
||||
monkeypatch.setattr(
|
||||
sync.psycopg2,
|
||||
"connect",
|
||||
lambda **kwargs: (_ for _ in ()).throw(AssertionError("must not connect")),
|
||||
)
|
||||
|
||||
assert sync.retry_request(
|
||||
"request",
|
||||
lambda: (_ for _ in ()).throw(AssertionError("must not run")),
|
||||
attempts=0,
|
||||
) is None
|
||||
assert sync.retry_db_connect(attempts=0) is None
|
||||
|
||||
|
||||
def test_kc_update_attributes_replaces_a_corrupt_attribute_map(monkeypatch):
|
||||
sync = load_sync_module(monkeypatch)
|
||||
current_resp = _FakeResponse({"username": "u1", "attributes": "corrupt"})
|
||||
ok_resp = _FakeResponse({"attributes": {"mailu_app_password": ["abc"]}})
|
||||
sync.SESSION = _FakeSession(_FakeResponse({}), [current_resp, ok_resp])
|
||||
|
||||
sync.kc_update_attributes(
|
||||
"token",
|
||||
{"id": "u1", "username": "u1"},
|
||||
{"mailu_app_password": "abc"},
|
||||
)
|
||||
|
||||
assert sync.SESSION.put_called and sync.SESSION.get_calls == 2
|
||||
|
||||
|
||||
def test_main_skips_disabled_users_and_settled_attribute_sets(monkeypatch):
|
||||
sync = load_sync_module(monkeypatch)
|
||||
monkeypatch.setattr(sync.bcrypt_sha256, "hash", lambda password: f"hash:{password}")
|
||||
users = [
|
||||
{
|
||||
"id": "u_disabled",
|
||||
"username": "disabled",
|
||||
"email": "disabled@example.com",
|
||||
"enabled": False,
|
||||
"attributes": {"mailu_enabled": ["true"]},
|
||||
},
|
||||
{
|
||||
"id": "u_settled",
|
||||
"username": "settled",
|
||||
"email": "settled@example.com",
|
||||
"attributes": {
|
||||
"mailu_enabled": ["true"],
|
||||
"mailu_email": ["settled@example.com"],
|
||||
"mailu_app_password": ["keepme"],
|
||||
},
|
||||
},
|
||||
]
|
||||
updated = []
|
||||
|
||||
class _Cursor:
|
||||
def __init__(self):
|
||||
self.executions = []
|
||||
|
||||
def execute(self, sql, params):
|
||||
self.executions.append(params)
|
||||
|
||||
def close(self):
|
||||
return None
|
||||
|
||||
class _Conn:
|
||||
def __init__(self):
|
||||
self.autocommit = False
|
||||
self._cursor = _Cursor()
|
||||
|
||||
def cursor(self, cursor_factory=None):
|
||||
return self._cursor
|
||||
|
||||
def close(self):
|
||||
return None
|
||||
|
||||
conn = _Conn()
|
||||
monkeypatch.setattr(sync, "get_kc_token", lambda: "tok")
|
||||
monkeypatch.setattr(sync, "kc_get_users", lambda token: users)
|
||||
monkeypatch.setattr(
|
||||
sync,
|
||||
"kc_update_attributes",
|
||||
lambda token, user, attrs: updated.append(user["id"]),
|
||||
)
|
||||
monkeypatch.setattr(sync.psycopg2, "connect", lambda **kwargs: conn)
|
||||
|
||||
sync.main()
|
||||
|
||||
assert updated == []
|
||||
assert [params["email"] for params in conn._cursor.executions] == [
|
||||
"settled@example.com"
|
||||
]
|
||||
@ -127,8 +127,64 @@ def test_listener_post_wait_keeps_running_request_successful(monkeypatch):
|
||||
assert handler.responses == [200]
|
||||
|
||||
|
||||
def test_listener_post_treats_non_object_json_as_plain_trigger(monkeypatch):
|
||||
listener = load_listener_module(monkeypatch)
|
||||
called = []
|
||||
monkeypatch.setattr(listener, "_trigger_sync_async", lambda force=False: called.append(force) or True)
|
||||
handler = _handler_for(listener, "[1, 2]")
|
||||
|
||||
handler.do_POST()
|
||||
|
||||
assert called == [False]
|
||||
assert handler.responses == [202]
|
||||
|
||||
|
||||
def test_listener_log_message_is_quiet(monkeypatch):
|
||||
listener = load_listener_module(monkeypatch)
|
||||
handler = listener.Handler.__new__(listener.Handler)
|
||||
|
||||
assert handler.log_message("ignored %s", "value") is None
|
||||
|
||||
|
||||
def test_listener_post_treats_non_object_payloads_as_plain_triggers(monkeypatch):
|
||||
listener = load_listener_module(monkeypatch)
|
||||
called = []
|
||||
monkeypatch.setattr(
|
||||
listener,
|
||||
"_trigger_sync_async",
|
||||
lambda force=False: called.append(force) or True,
|
||||
)
|
||||
handler = _handler_for(listener, "[1, 2, 3]")
|
||||
|
||||
handler.do_POST()
|
||||
|
||||
assert called == [False]
|
||||
assert handler.responses == [202]
|
||||
|
||||
|
||||
def test_listener_module_serves_forever_when_run_directly(monkeypatch):
|
||||
import http.server
|
||||
import runpy
|
||||
|
||||
served = []
|
||||
|
||||
class _FakeServer:
|
||||
def __init__(self, address, handler):
|
||||
served.append((address, handler.__name__))
|
||||
|
||||
def serve_forever(self):
|
||||
served.append("serving")
|
||||
|
||||
monkeypatch.setenv("MAILU_SYNC_WAIT_TIMEOUT_SEC", "0")
|
||||
monkeypatch.setattr(http.server, "ThreadingHTTPServer", _FakeServer)
|
||||
module_path = (
|
||||
pathlib.Path(__file__).resolve().parents[2]
|
||||
/ "services"
|
||||
/ "mailu"
|
||||
/ "scripts"
|
||||
/ "mailu_sync_listener.py"
|
||||
)
|
||||
|
||||
runpy.run_path(str(module_path), run_name="__main__")
|
||||
|
||||
assert served == [(("", 8080), "Handler"), "serving"]
|
||||
|
||||
@ -7,7 +7,6 @@ import argparse
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import stat
|
||||
from pathlib import Path
|
||||
|
||||
@ -35,90 +34,25 @@ def _read_bounded(path: Path) -> bytes:
|
||||
return value
|
||||
|
||||
|
||||
def _glob_regex(pattern: str, position: int = 0, terminators: str = "") -> tuple[str, int]:
|
||||
"""Compile the gobwas/glob syntax Gitea v1.23.8 uses."""
|
||||
pieces: list[str] = []
|
||||
while position < len(pattern):
|
||||
character = pattern[position]
|
||||
if character in terminators:
|
||||
break
|
||||
if character == "\\":
|
||||
position += 1
|
||||
if position >= len(pattern):
|
||||
raise ValueError("trailing escape")
|
||||
pieces.append(re.escape(pattern[position]))
|
||||
elif character == "*":
|
||||
if position + 1 < len(pattern) and pattern[position + 1] == "*":
|
||||
pieces.append(".*")
|
||||
position += 1
|
||||
else:
|
||||
pieces.append("[^/]*")
|
||||
elif character == "?":
|
||||
pieces.append("[^/]")
|
||||
elif character == "[":
|
||||
end = pattern.find("]", position + 1)
|
||||
if end < 0:
|
||||
raise ValueError("unterminated range")
|
||||
value = pattern[position + 1 : end]
|
||||
if not value:
|
||||
raise ValueError("empty range")
|
||||
negate = value.startswith("!")
|
||||
value = value[1:] if negate else value
|
||||
if not value or "[" in value or "\\" in value:
|
||||
raise ValueError("invalid range")
|
||||
if "-" in value:
|
||||
if value.count("-") != 1 or value.startswith("-") or value.endswith("-"):
|
||||
raise ValueError("invalid range")
|
||||
low, high = value.split("-")
|
||||
if len(low) != 1 or len(high) != 1 or ord(high) < ord(low):
|
||||
raise ValueError("invalid range")
|
||||
content = re.escape(low) + "-" + re.escape(high)
|
||||
else:
|
||||
content = re.escape(value)
|
||||
pieces.append("[" + ("^" if negate else "") + content + "]")
|
||||
position = end
|
||||
elif character == "{":
|
||||
alternatives: list[str] = []
|
||||
position += 1
|
||||
while True:
|
||||
alternative, position = _glob_regex(pattern, position, ",}")
|
||||
alternatives.append(alternative)
|
||||
if position >= len(pattern):
|
||||
raise ValueError("unterminated alternatives")
|
||||
if pattern[position] == "}":
|
||||
break
|
||||
position += 1
|
||||
if len(alternatives) < 2 or any(not item for item in alternatives):
|
||||
raise ValueError("invalid alternatives")
|
||||
pieces.append("(?:" + "|".join(alternatives) + ")")
|
||||
elif character in "]}":
|
||||
# A closing delimiter outside its grammar context is literal in
|
||||
# gobwas/glob's lexer.
|
||||
pieces.append(re.escape(character))
|
||||
else:
|
||||
pieces.append(re.escape(character))
|
||||
position += 1
|
||||
return "".join(pieces), position
|
||||
|
||||
|
||||
def _is_plain(pattern: str) -> bool:
|
||||
return not any(character in SPECIAL for character in pattern)
|
||||
|
||||
|
||||
def _matches(pattern: str, branch: str) -> bool:
|
||||
def _literal_rule(pattern: str) -> str:
|
||||
"""Return one exact rule name or reject Gitea glob interpretation.
|
||||
|
||||
Gitea 1.23.8 delegates matching to gobwas/glob. Reimplementing that
|
||||
grammar and its error quoting rules here would create a second policy
|
||||
engine. The reconciler instead accepts only plain ASCII literals. This
|
||||
deliberately rejects braces, ranges, escapes, malformed globs, and even
|
||||
special rules that appear unrelated: an operator must remove ambiguity
|
||||
before this job may attest the primary-branch boundary.
|
||||
"""
|
||||
if not pattern.isascii() or not 1 <= len(pattern) <= MAX_RULE_NAME:
|
||||
raise PolicyError("branch protection rule name is invalid")
|
||||
if _is_plain(pattern):
|
||||
return pattern.casefold() == branch.casefold()
|
||||
try:
|
||||
expression, position = _glob_regex(pattern)
|
||||
if position != len(pattern):
|
||||
raise ValueError("incomplete glob")
|
||||
except (re.error, ValueError):
|
||||
# Gitea quotes an invalid special pattern and matches it literally.
|
||||
# A literal containing a special byte cannot equal main or master.
|
||||
return False
|
||||
return re.fullmatch(expression, branch) is not None
|
||||
if not _is_plain(pattern):
|
||||
raise PolicyError("glob branch protection rules require human review")
|
||||
return pattern
|
||||
|
||||
|
||||
def _created(value: object) -> datetime.datetime:
|
||||
@ -159,7 +93,7 @@ def evaluate(value: bytes, branch: str, reviewer: str) -> str:
|
||||
data = json.loads(value)
|
||||
if not isinstance(data, list) or len(data) > MAX_RULES:
|
||||
raise PolicyError("branch protection response is invalid")
|
||||
ordered: list[tuple[int, bool, datetime.datetime, int, dict[str, object]]] = []
|
||||
ordered: list[tuple[int, datetime.datetime, int, dict[str, object]]] = []
|
||||
for index, raw in enumerate(data):
|
||||
if not isinstance(raw, dict):
|
||||
raise PolicyError("branch protection entry is invalid")
|
||||
@ -168,18 +102,23 @@ def evaluate(value: bytes, branch: str, reviewer: str) -> str:
|
||||
if (
|
||||
not isinstance(priority, int)
|
||||
or isinstance(priority, bool)
|
||||
or not 1 <= priority <= 1_000_000
|
||||
or not 0 <= priority <= 1_000_000
|
||||
or not isinstance(rule_name, str)
|
||||
):
|
||||
raise PolicyError("branch protection priority is ambiguous")
|
||||
created = _created(raw.get("created_at"))
|
||||
if _matches(rule_name, branch):
|
||||
# Gitea v1.23.8 sorts by priority, then puts plain names before
|
||||
# globs, then uses creation time. The API exposes every component.
|
||||
ordered.append((priority, not _is_plain(rule_name), created, index, raw))
|
||||
literal = _literal_rule(rule_name)
|
||||
if literal == branch:
|
||||
# With every rule constrained to a plain literal, Gitea's glob
|
||||
# comparator cannot affect this result. Preserve API ordering:
|
||||
# priority first (including legacy zero), creation time, then the
|
||||
# stable response order for an otherwise exact tie.
|
||||
ordered.append((priority, created, index, raw))
|
||||
if not ordered:
|
||||
return "ABSENT"
|
||||
effective = min(ordered)[4]
|
||||
if len(ordered) != 1:
|
||||
raise PolicyError("duplicate primary-branch protections are ambiguous")
|
||||
effective = ordered[0][3]
|
||||
for key, expected in _required(reviewer).items():
|
||||
if effective.get(key) != expected:
|
||||
raise PolicyError(
|
||||
|
||||
79
services/harbor/hermes-agent-immutability-job.yaml
Normal file
79
services/harbor/hermes-agent-immutability-job.yaml
Normal file
@ -0,0 +1,79 @@
|
||||
# services/harbor/hermes-agent-immutability-job.yaml
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: harbor-hermes-agent-immutability-ensure-1
|
||||
namespace: harbor
|
||||
spec:
|
||||
backoffLimit: 2
|
||||
activeDeadlineSeconds: 600
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
vault.hashicorp.com/agent-inject: "true"
|
||||
vault.hashicorp.com/agent-pre-populate-only: "true"
|
||||
vault.hashicorp.com/agent-run-as-user: "65532"
|
||||
vault.hashicorp.com/agent-run-as-group: "65532"
|
||||
vault.hashicorp.com/role: harbor-policy-bootstrap
|
||||
vault.hashicorp.com/agent-inject-secret-harbor-admin-password: kv/data/atlas/harbor/harbor-core
|
||||
vault.hashicorp.com/agent-inject-template-harbor-admin-password: |
|
||||
{{- with secret "kv/data/atlas/harbor/harbor-core" -}}
|
||||
{{ .Data.data.harbor_admin_password }}
|
||||
{{- end -}}
|
||||
spec:
|
||||
serviceAccountName: harbor-policy-bootstrap
|
||||
enableServiceLinks: false
|
||||
restartPolicy: Never
|
||||
nodeSelector:
|
||||
hardware: rpi5
|
||||
kubernetes.io/arch: arm64
|
||||
node-role.kubernetes.io/worker: "true"
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- key: kubernetes.io/hostname
|
||||
operator: NotIn
|
||||
values: [titan-04, titan-14, titan-18, titan-19, titan-24]
|
||||
securityContext:
|
||||
fsGroup: 65532
|
||||
fsGroupChangePolicy: OnRootMismatch
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: ensure
|
||||
image: docker.io/library/python@sha256:efcdfa6a6b2fd2afb9c7dfa9a5b288a6f68338b5cfdebe6b637d986067d85757
|
||||
imagePullPolicy: IfNotPresent
|
||||
command: [python3, /scripts/harbor_hermes_agent_immutability_ensure.py]
|
||||
env:
|
||||
- name: HARBOR_API_ORIGIN
|
||||
value: https://registry.bstein.dev/api/v2.0
|
||||
- name: HARBOR_ADMIN_PASSWORD_FILE
|
||||
value: /vault/secrets/harbor-admin-password
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
readOnlyRootFilesystem: true
|
||||
runAsGroup: 65532
|
||||
runAsNonRoot: true
|
||||
runAsUser: 65532
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
volumeMounts:
|
||||
- name: scripts
|
||||
mountPath: /scripts
|
||||
readOnly: true
|
||||
- name: tmp
|
||||
mountPath: /tmp
|
||||
resources:
|
||||
requests: {cpu: 25m, memory: 32Mi}
|
||||
limits: {cpu: 250m, memory: 128Mi}
|
||||
volumes:
|
||||
- name: scripts
|
||||
configMap:
|
||||
name: harbor-hermes-agent-immutability-script
|
||||
defaultMode: 0555
|
||||
- name: tmp
|
||||
emptyDir: {}
|
||||
@ -12,9 +12,14 @@ resources:
|
||||
- certificate.yaml
|
||||
- helmrelease.yaml
|
||||
- vault-sync-deployment.yaml
|
||||
- policy-bootstrap-serviceaccount.yaml
|
||||
- hermes-agent-immutability-job.yaml
|
||||
- bootstrap-jobs/cassandra-registry-ensure-job.yaml
|
||||
- image.yaml
|
||||
configMapGenerator:
|
||||
- name: harbor-vault-entrypoint
|
||||
files:
|
||||
- scripts/vault-entrypoint.sh
|
||||
- name: harbor-hermes-agent-immutability-script
|
||||
files:
|
||||
- harbor_hermes_agent_immutability_ensure.py=scripts/harbor_hermes_agent_immutability_ensure.py
|
||||
|
||||
7
services/harbor/policy-bootstrap-serviceaccount.yaml
Normal file
7
services/harbor/policy-bootstrap-serviceaccount.yaml
Normal file
@ -0,0 +1,7 @@
|
||||
# services/harbor/policy-bootstrap-serviceaccount.yaml
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: harbor-policy-bootstrap
|
||||
namespace: harbor
|
||||
automountServiceAccountToken: true
|
||||
@ -0,0 +1,378 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create and verify the narrowly scoped Hermes agent immutable-tag rule."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
PROJECT = "bstein"
|
||||
REPOSITORY_PATTERN = "hermes-agent"
|
||||
TAG_PATTERN = "git-*-build-*"
|
||||
PUBLISH_ROBOT = "robot$jenkins-pipelines"
|
||||
EXPECTED_ORIGIN = "https://registry.bstein.dev/api/v2.0"
|
||||
MAX_RESPONSE = 1_048_576
|
||||
TRANSIENT_STATUSES = {429, 502, 503, 504}
|
||||
EXPECTED_RULE = {
|
||||
"disabled": False,
|
||||
"action": "immutable",
|
||||
"template": "immutable_template",
|
||||
"tag_selectors": [
|
||||
{
|
||||
"kind": "doublestar",
|
||||
"decoration": "matches",
|
||||
"pattern": TAG_PATTERN,
|
||||
}
|
||||
],
|
||||
"scope_selectors": {
|
||||
"repository": [
|
||||
{
|
||||
"kind": "doublestar",
|
||||
"decoration": "repoMatches",
|
||||
"pattern": REPOSITORY_PATTERN,
|
||||
}
|
||||
]
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class NoRedirect(urllib.request.HTTPRedirectHandler):
|
||||
"""Prevent Basic credentials from following an unexpected redirect."""
|
||||
|
||||
def redirect_request(self, _request, _file, _code, _message, _headers, _url):
|
||||
return None
|
||||
|
||||
|
||||
class HarborUnavailable(RuntimeError):
|
||||
"""Harbor is not ready yet, rather than returning a policy decision."""
|
||||
|
||||
|
||||
def normalized_rule(rule: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Return only the immutable contract fields Harbor must preserve."""
|
||||
return {
|
||||
"disabled": bool(rule.get("disabled", False)),
|
||||
"action": rule.get("action"),
|
||||
"template": rule.get("template"),
|
||||
"tag_selectors": [
|
||||
{
|
||||
"kind": selector.get("kind"),
|
||||
"decoration": selector.get("decoration"),
|
||||
"pattern": selector.get("pattern"),
|
||||
}
|
||||
for selector in rule.get("tag_selectors") or []
|
||||
if isinstance(selector, dict)
|
||||
],
|
||||
"scope_selectors": {
|
||||
"repository": [
|
||||
{
|
||||
"kind": selector.get("kind"),
|
||||
"decoration": selector.get("decoration"),
|
||||
"pattern": selector.get("pattern"),
|
||||
}
|
||||
for selector in (rule.get("scope_selectors") or {}).get(
|
||||
"repository", []
|
||||
)
|
||||
if isinstance(selector, dict)
|
||||
]
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def targets_hermes_builds(rule: dict[str, Any]) -> bool:
|
||||
"""Detect a rule that claims this exact repository and tag selector."""
|
||||
normalized = normalized_rule(rule)
|
||||
return (
|
||||
normalized["tag_selectors"] == EXPECTED_RULE["tag_selectors"]
|
||||
and normalized["scope_selectors"] == EXPECTED_RULE["scope_selectors"]
|
||||
)
|
||||
|
||||
|
||||
class HarborClient:
|
||||
"""Bounded same-origin client for Harbor's immutable-tag API."""
|
||||
|
||||
def __init__(self, origin: str, username: str, password: str) -> None:
|
||||
normalized_origin = origin.rstrip("/")
|
||||
if normalized_origin != EXPECTED_ORIGIN:
|
||||
raise ValueError("Harbor API origin is not the pinned production API")
|
||||
self.origin = normalized_origin
|
||||
token = base64.b64encode(f"{username}:{password}".encode()).decode("ascii")
|
||||
self.headers = {"Authorization": f"Basic {token}"}
|
||||
self.opener = urllib.request.build_opener(NoRedirect())
|
||||
|
||||
def request(
|
||||
self, method: str, path: str, payload: dict[str, Any] | None = None
|
||||
) -> tuple[int, bytes, dict[str, str]]:
|
||||
"""Issue one request, returning even non-2xx responses for strict checks."""
|
||||
data = None
|
||||
headers = dict(self.headers)
|
||||
if payload is not None:
|
||||
data = json.dumps(payload, separators=(",", ":")).encode()
|
||||
headers["Content-Type"] = "application/json"
|
||||
request = urllib.request.Request(
|
||||
f"{self.origin}{path}", data=data, headers=headers, method=method
|
||||
)
|
||||
try:
|
||||
response = self.opener.open(request, timeout=20)
|
||||
except urllib.error.HTTPError as exc:
|
||||
response = exc
|
||||
except (urllib.error.URLError, TimeoutError) as exc:
|
||||
raise HarborUnavailable("Harbor policy API is unavailable") from exc
|
||||
with response:
|
||||
body = response.read(MAX_RESPONSE + 1)
|
||||
if len(body) > MAX_RESPONSE:
|
||||
raise RuntimeError("Harbor response exceeded the size limit")
|
||||
return int(response.status), body, dict(response.headers)
|
||||
|
||||
|
||||
def list_rules(client: HarborClient) -> list[dict[str, Any]]:
|
||||
"""Read and validate the complete small rule set for the project."""
|
||||
path = f"/projects/{PROJECT}/immutabletagrules?page=1&page_size=100"
|
||||
status, body, headers = client.request("GET", path)
|
||||
if status in TRANSIENT_STATUSES:
|
||||
raise HarborUnavailable(f"Harbor immutable rule list returned HTTP {status}")
|
||||
if status != 200:
|
||||
raise RuntimeError(f"Harbor immutable rule list returned HTTP {status}")
|
||||
values = _json_list(body, "immutable rule")
|
||||
_require_complete_page(values, headers, "immutable rule")
|
||||
return values
|
||||
|
||||
|
||||
def _json_list(body: bytes, label: str) -> list[dict[str, Any]]:
|
||||
"""Decode one bounded Harbor list without accepting a partial shape."""
|
||||
try:
|
||||
value = json.loads(body.decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise RuntimeError(f"Harbor returned invalid {label} JSON") from exc
|
||||
if not isinstance(value, list) or not all(isinstance(item, dict) for item in value):
|
||||
raise RuntimeError(f"Harbor {label} list has an invalid shape")
|
||||
return value
|
||||
|
||||
|
||||
def _require_complete_page(
|
||||
values: list[dict[str, Any]], headers: dict[str, str], label: str
|
||||
) -> None:
|
||||
"""Reject a truncated first page or a proxy that strips count evidence."""
|
||||
raw_total = next(
|
||||
(value for key, value in headers.items() if key.lower() == "x-total-count"),
|
||||
None,
|
||||
)
|
||||
if raw_total is None or not str(raw_total).isdecimal():
|
||||
raise RuntimeError(f"Harbor {label} list omitted a valid total count")
|
||||
if int(raw_total) != len(values):
|
||||
raise RuntimeError(f"Harbor {label} list was truncated")
|
||||
|
||||
|
||||
def _allowed(access: dict[str, Any], resource: str, action: str) -> bool:
|
||||
"""Match one positive robot permission, accepting Harbor's default effect."""
|
||||
return (
|
||||
access.get("resource") == resource
|
||||
and access.get("action") == action
|
||||
and access.get("effect") in (None, "", "allow")
|
||||
)
|
||||
|
||||
|
||||
def _publisher_scope(robot: dict[str, Any]) -> tuple[dict[str, Any], list[dict[str, Any]]]:
|
||||
"""Return the existing bstein scope after validating its push boundary."""
|
||||
permissions = robot.get("permissions")
|
||||
if not isinstance(permissions, list) or not all(
|
||||
isinstance(item, dict) for item in permissions
|
||||
):
|
||||
raise RuntimeError("Jenkins Harbor robot permissions have an invalid shape")
|
||||
matches = [
|
||||
item
|
||||
for item in permissions
|
||||
if item.get("kind") == "project" and item.get("namespace") == PROJECT
|
||||
]
|
||||
if len(matches) != 1:
|
||||
raise RuntimeError("Jenkins Harbor robot must have one existing bstein scope")
|
||||
access = matches[0].get("access")
|
||||
if not isinstance(access, list) or not all(isinstance(item, dict) for item in access):
|
||||
raise RuntimeError("Jenkins Harbor robot bstein access has an invalid shape")
|
||||
for required in (("repository", "pull"), ("repository", "push")):
|
||||
if not any(_allowed(item, *required) for item in access):
|
||||
raise RuntimeError("Jenkins Harbor robot lacks its existing pull/push boundary")
|
||||
immutable_access = [
|
||||
item for item in access if item.get("resource") == "immutable-tag"
|
||||
]
|
||||
if immutable_access and not (
|
||||
len(immutable_access) == 1
|
||||
and _allowed(immutable_access[0], "immutable-tag", "list")
|
||||
):
|
||||
raise RuntimeError("Jenkins Harbor robot has broader immutable-tag access")
|
||||
return matches[0], access
|
||||
|
||||
|
||||
def _verified_publisher(robot: dict[str, Any]) -> bool:
|
||||
"""Check the exact identity and least-privilege policy after persistence."""
|
||||
if (
|
||||
robot.get("name") != PUBLISH_ROBOT
|
||||
or robot.get("level") != "system"
|
||||
or robot.get("editable") is not True
|
||||
or robot.get("disable") is not False
|
||||
):
|
||||
return False
|
||||
try:
|
||||
_scope, access = _publisher_scope(robot)
|
||||
except RuntimeError:
|
||||
return False
|
||||
return any(_allowed(item, "immutable-tag", "list") for item in access)
|
||||
|
||||
|
||||
def ensure_publisher_can_read_rule(client: HarborClient) -> int:
|
||||
"""Grant only immutable-tag:list to the existing Jenkins push robot."""
|
||||
status, body, headers = client.request("GET", "/robots?page=1&page_size=100")
|
||||
if status in TRANSIENT_STATUSES:
|
||||
raise HarborUnavailable(f"Harbor robot list returned HTTP {status}")
|
||||
if status != 200:
|
||||
raise RuntimeError(f"Harbor robot list returned HTTP {status}")
|
||||
robots = _json_list(body, "robot")
|
||||
_require_complete_page(robots, headers, "robot")
|
||||
matches = [
|
||||
item
|
||||
for item in robots
|
||||
if item.get("name") == PUBLISH_ROBOT
|
||||
]
|
||||
if len(matches) != 1:
|
||||
raise RuntimeError("expected exactly one Jenkins Harbor publisher robot")
|
||||
robot_id = matches[0].get("id")
|
||||
if not isinstance(robot_id, int) or robot_id < 1:
|
||||
raise RuntimeError("Jenkins Harbor robot omitted a valid ID")
|
||||
status, body, _headers = client.request("GET", f"/robots/{robot_id}")
|
||||
if status in TRANSIENT_STATUSES:
|
||||
raise HarborUnavailable(f"Harbor robot read returned HTTP {status}")
|
||||
if status != 200:
|
||||
raise RuntimeError(f"Harbor robot read returned HTTP {status}")
|
||||
try:
|
||||
robot = json.loads(body.decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise RuntimeError("Harbor returned invalid robot JSON") from exc
|
||||
if not isinstance(robot, dict):
|
||||
raise RuntimeError("Harbor robot response has an invalid shape")
|
||||
if (
|
||||
robot.get("name") != PUBLISH_ROBOT
|
||||
or robot.get("level") != "system"
|
||||
or robot.get("editable") is not True
|
||||
or robot.get("disable") is not False
|
||||
):
|
||||
raise RuntimeError("Jenkins Harbor publisher identity is not active and exact")
|
||||
scope, access = _publisher_scope(robot)
|
||||
if any(_allowed(item, "immutable-tag", "list") for item in access):
|
||||
return robot_id
|
||||
|
||||
# Harbor updates permissions through PUT /robots/{id}; its separate PATCH
|
||||
# endpoint is the only secret-rotation operation. Preserve every current
|
||||
# scope and field while adding the one read-only action.
|
||||
access.append({"resource": "immutable-tag", "action": "list"})
|
||||
payload = {
|
||||
"name": robot["name"],
|
||||
"description": robot.get("description") or "",
|
||||
"level": robot["level"],
|
||||
"disable": robot["disable"],
|
||||
"permissions": robot["permissions"],
|
||||
}
|
||||
duration = robot.get("duration")
|
||||
if duration is not None:
|
||||
if not isinstance(duration, int):
|
||||
raise RuntimeError("Jenkins Harbor robot duration has an invalid shape")
|
||||
payload["duration"] = duration
|
||||
# Keep the validated object reference live in the preserved permission list.
|
||||
if scope.get("access") is not access:
|
||||
raise RuntimeError("Jenkins Harbor robot permission normalization drifted")
|
||||
status, _body, _headers = client.request(
|
||||
"PUT", f"/robots/{robot_id}", payload
|
||||
)
|
||||
if status in TRANSIENT_STATUSES:
|
||||
raise HarborUnavailable(f"Harbor robot update returned HTTP {status}")
|
||||
if status != 200:
|
||||
raise RuntimeError(f"Harbor robot policy update returned HTTP {status}")
|
||||
|
||||
for attempt in range(1, 6):
|
||||
status, body, _headers = client.request("GET", f"/robots/{robot_id}")
|
||||
if status == 200:
|
||||
try:
|
||||
persisted = json.loads(body.decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError):
|
||||
persisted = None
|
||||
if isinstance(persisted, dict) and _verified_publisher(persisted):
|
||||
return robot_id
|
||||
if attempt < 5:
|
||||
time.sleep(attempt)
|
||||
raise RuntimeError("Jenkins Harbor robot policy did not verify exactly")
|
||||
|
||||
|
||||
def ensure_rule(client: HarborClient) -> int:
|
||||
"""Create once, or validate the one exact enabled rule already present."""
|
||||
rules = list_rules(client)
|
||||
matches = [rule for rule in rules if targets_hermes_builds(rule)]
|
||||
if len(matches) > 1:
|
||||
raise RuntimeError("multiple Hermes agent immutable rules exist")
|
||||
if matches:
|
||||
if normalized_rule(matches[0]) != EXPECTED_RULE:
|
||||
raise RuntimeError("Hermes agent immutable rule exists but is not enabled/exact")
|
||||
rule_id = matches[0].get("id")
|
||||
if not isinstance(rule_id, int) or rule_id < 1:
|
||||
raise RuntimeError("Harbor immutable rule omitted a valid ID")
|
||||
return rule_id
|
||||
|
||||
path = f"/projects/{PROJECT}/immutabletagrules"
|
||||
status, _body, headers = client.request("POST", path, EXPECTED_RULE)
|
||||
if status in TRANSIENT_STATUSES:
|
||||
raise HarborUnavailable(f"Harbor immutable rule create returned HTTP {status}")
|
||||
if status != 201:
|
||||
raise RuntimeError(f"Harbor immutable rule create returned HTTP {status}")
|
||||
location = headers.get("Location") or headers.get("location") or ""
|
||||
api_path = urllib.parse.urlsplit(client.origin).path.rstrip("/")
|
||||
expected_prefix = f"{api_path}{path}/"
|
||||
if not location.startswith(expected_prefix):
|
||||
raise RuntimeError("Harbor immutable rule create omitted the exact Location")
|
||||
suffix = location[len(expected_prefix) :]
|
||||
if not suffix.isdecimal() or int(suffix) < 1:
|
||||
raise RuntimeError("Harbor immutable rule Location has an invalid ID")
|
||||
|
||||
# Harbor persists synchronously, but bounded retries distinguish a stale
|
||||
# read from accepting an unverified policy.
|
||||
for attempt in range(1, 6):
|
||||
matches = [rule for rule in list_rules(client) if targets_hermes_builds(rule)]
|
||||
if len(matches) == 1 and normalized_rule(matches[0]) == EXPECTED_RULE:
|
||||
rule_id = matches[0].get("id")
|
||||
if rule_id == int(suffix):
|
||||
return rule_id
|
||||
if attempt < 5:
|
||||
time.sleep(attempt)
|
||||
raise RuntimeError("created Harbor immutable rule did not verify exactly")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Load the runtime-only admin credential and enforce the tracked policy."""
|
||||
origin = os.environ.get("HARBOR_API_ORIGIN", "")
|
||||
password_file = Path(os.environ.get("HARBOR_ADMIN_PASSWORD_FILE", ""))
|
||||
password = password_file.read_text(encoding="utf-8").strip()
|
||||
if not password:
|
||||
raise RuntimeError("Harbor admin password is empty")
|
||||
client = HarborClient(origin, "admin", password)
|
||||
for attempt in range(1, 13):
|
||||
try:
|
||||
rule_id = ensure_rule(client)
|
||||
robot_id = ensure_publisher_can_read_rule(client)
|
||||
break
|
||||
except HarborUnavailable:
|
||||
if attempt == 12:
|
||||
raise
|
||||
time.sleep(min(attempt * 2, 15))
|
||||
print(
|
||||
"Hermes agent immutable build-tag rule is active "
|
||||
f"(id={rule_id}); publisher preflight access is active (robot={robot_id})"
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover - exercised through main()
|
||||
raise SystemExit(main())
|
||||
@ -102,6 +102,110 @@ console tails and named artifact contents in its deterministic bundle. A report
|
||||
must say `retained Ariadne evidence` when that fallback is used; it must not
|
||||
pretend direct Jenkins access succeeded.
|
||||
|
||||
## Publishing an agent image after review
|
||||
|
||||
The `hermes-agent-image` Jenkins job is the only supported agent image builder.
|
||||
It runs daemonless Kaniko v1.23.2 under the unbound `hermes-image-builder`
|
||||
ServiceAccount without a service-account token, host socket,
|
||||
privileged container, or writable Git credential. It accepts only an exact
|
||||
40-character revision that is both the checked-out commit and current
|
||||
`atlas/titan-iac` `main`, so a human must merge the source PR first.
|
||||
|
||||
The build pod requires an ARM64 worker and gives `hardware=rpi5` the maximum
|
||||
scheduler preference. Healthy rpi4 workers remain valid fallback capacity when
|
||||
the rpi5 pool cannot fit the full request. Cordoned, unhealthy, amd64, and
|
||||
reserved last-resort nodes remain excluded by the required placement boundary.
|
||||
|
||||
Kaniko runs as UID 0 because it must unpack an image root filesystem and enter
|
||||
that filesystem for Dockerfile `RUN` instructions. Its capability set is the
|
||||
minimum proven by an exact ARM64 no-push build of this Dockerfile:
|
||||
`CHOWN`, `FOWNER`, `DAC_OVERRIDE`, `SETGID`, and `SETUID`. `SETGID` is required
|
||||
because pinned Kaniko applies the base image's supplementary group list before
|
||||
each Dockerfile `RUN`; `SETUID` lets apt drop privileges to its `_apt` account
|
||||
while downloading package indexes. It still has no privilege
|
||||
escalation, service-account token, host path, daemon socket, or Docker/BuildKit
|
||||
TCP endpoint and uses the runtime-default seccomp profile. This is residual
|
||||
root-in-the-build-pod risk, bounded to disposable `emptyDir` storage and a
|
||||
human-reviewed `main` revision; Dockerfile changes require the same scrutiny as
|
||||
executable cluster code.
|
||||
|
||||
Pinned Kaniko v1.23.2 parses Dockerfile `RUN` heredocs but does not materialize
|
||||
their inline files when executing a command. The Jenkins lane therefore enables
|
||||
a bounded compatibility replay after copying the reviewed Dockerfile and runner
|
||||
from the same checked-out commit. Before every replay, the runner inventories
|
||||
the entire Dockerfile as logical instructions, including split opcodes,
|
||||
operators, and either Dockerfile escape character. It rejects every heredoc
|
||||
form outside the exact nine Node/Python blocks before launching an interpreter.
|
||||
Each accepted body is then replayed immediately after its corresponding Kaniko
|
||||
`RUN`, preserving source order before any dependent regression. Docker and
|
||||
BuildKit keep their native behavior because the compatibility argument defaults
|
||||
off. The final TypeScript build, Python compile, and source assertions remain
|
||||
mandatory, so an omitted or drifted replay fails before any image can pass
|
||||
release verification.
|
||||
|
||||
From agent.hermes, trigger that one fixed job with:
|
||||
|
||||
```sh
|
||||
jenkins_image_build_trigger.py '<reviewed full main commit>'
|
||||
```
|
||||
|
||||
The helper has no general Jenkins credential or caller-selectable job name. Its
|
||||
Vault-projected token is bound by Jenkins only to `hermes-agent-image`. The job
|
||||
also requires its fixed publish confirmation. Each run claims the unique tag
|
||||
`git-<reviewed-sha>-build-<jenkins-build-number>` and refuses to overwrite an
|
||||
existing tag. It cross-checks Kaniko's digest and tagged-image evidence, then
|
||||
independently reads the pushed tag from Harbor before archiving a JSON record
|
||||
and Flux digest patch. A successful post-condition then revalidates the exact
|
||||
six-file evidence set and archives those six explicit paths with empty archives
|
||||
forbidden. Apply that patch on a new branch and submit it for human review; the
|
||||
build never changes Git, reconciles Flux, or deploys by itself.
|
||||
The helper posts only to the fixed HTTPS Build Token Root endpoint and accepts
|
||||
only its real queue responses: HTTP 201 or a non-followed HTTP 303 with an exact
|
||||
same-origin `/queue/item/<number>/` location. An unauthenticated request is
|
||||
denied, and the token cannot select, configure, read, or administer another
|
||||
Jenkins job. The pipeline independently rejects any commit that is not the
|
||||
current `origin/main`, preserving the human merge/review boundary.
|
||||
|
||||
Harbor independently enforces an enabled immutable-tag rule scoped to only the
|
||||
`bstein/hermes-agent` repository and `git-*-build-*` tags. A second manifest PUT,
|
||||
retag, or deletion is rejected by Harbor even if a caller bypasses the Jenkins
|
||||
preflight. A revisioned, Flux-tracked policy Job creates or verifies that exact
|
||||
rule with a runtime-only Vault credential; it refuses to alter a conflicting
|
||||
rule and retries only explicit transport/readiness failures while Harbor
|
||||
starts. The same Job grants the existing Jenkins publisher only
|
||||
`immutable-tag:list` on `bstein`, preserving its other project scopes and never
|
||||
calling Harbor's separate secret-rotation endpoint. The pipeline uses that
|
||||
read-only permission to require the exact enabled rule before Kaniko starts.
|
||||
That check also makes an already-running Jenkins controller fail closed during
|
||||
rollout; Jenkins cannot become Ready on the new revision before the policy Job
|
||||
succeeds.
|
||||
|
||||
The tracked rollout order is deliberate: the revisioned Vault role Job must
|
||||
complete before the Vault Kustomization becomes Ready. A separate Flux
|
||||
Kustomization then runs the seed Job under the dedicated
|
||||
`hermes-jenkins-token-seed` identity, which can only create/read the exact
|
||||
`kv/data/atlas/hermes/developer-jenkins` path and request random bytes. Jenkins
|
||||
depends on both that seed and the Harbor policy Job; Hermes depends on Jenkins.
|
||||
The seeder uses KV-v2 CAS create-only semantics. It never changes an existing
|
||||
token, never fills a missing field in an existing secret, and fails closed when
|
||||
it cannot distinguish absence from a read error. The recurring Vault
|
||||
configuration job reconciles the same narrow role and policy.
|
||||
|
||||
Rotate this fixed-job token only as a coordinated operator action:
|
||||
|
||||
1. Stop new `hermes-agent-image` triggers and wait for its queue and executor to
|
||||
drain.
|
||||
2. Read the current KV-v2 metadata version for
|
||||
`kv/atlas/hermes/developer-jenkins`.
|
||||
3. Generate a fresh value from Vault and patch only `build_token` with
|
||||
`vault kv patch -cas=<current-version>`. Keep the value in a mode-0600
|
||||
temporary file or standard input, never a command argument or log.
|
||||
4. Roll Jenkins first and wait until it is Ready, then roll `hermes-agent` and
|
||||
wait until it is Ready. Do not resume triggers between those two consumers.
|
||||
5. Run one reviewed-main canary. If rollback is required, repeat the CAS patch
|
||||
with the previous value as another coordinated rotation; never delete the
|
||||
secret to make the seeder recreate it.
|
||||
|
||||
## The actual supervised triage algorithm
|
||||
|
||||
1. Classify the request as test/build triage, service health, or alert tuning.
|
||||
|
||||
@ -88,6 +88,14 @@ data:
|
||||
auto_decompose: true
|
||||
auto_decompose_per_tick: 2
|
||||
dispatch_stale_timeout_seconds: 14400
|
||||
# Steady-state quota-aware routing for the direct CLI lane: below this
|
||||
# remaining-percent a provider stops receiving NEW cli-auto work while
|
||||
# active work finishes; lane-observed capacity/auth failures exclude a
|
||||
# provider until the cooldown elapses, its quota window resets, or a
|
||||
# fresh success is recorded.
|
||||
provider_quota_min_remaining_percent: 15
|
||||
provider_capacity_cooldown_seconds: 300
|
||||
provider_auth_cooldown_seconds: 3600
|
||||
|
||||
model_catalog:
|
||||
enabled: true
|
||||
@ -320,7 +328,9 @@ data:
|
||||
unauthenticated Gitea HTTP 404 as a missing private repository. Use
|
||||
brokered Git for clone, fetch, and creation of a new namespaced feature
|
||||
branch. Existing-ref updates, protected refs, deletion, and force-push are
|
||||
rejected. For bounded
|
||||
rejected. The broker inflates and scans every pushed object, so thin
|
||||
packs are rejected; always push with `git push --no-thin` so the pack
|
||||
is self-contained. For bounded
|
||||
repository/pull-request evidence or to create a review-ready draft PR, load
|
||||
`$manage-atlas-pull-requests` and use `/opt/scm/gitea_api.py`. The
|
||||
client carries no repository credential; a separate least-authority
|
||||
|
||||
@ -25,7 +25,7 @@ spec:
|
||||
ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers
|
||||
ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback
|
||||
ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available
|
||||
ai.bstein.dev/config-rev: "20260816-auth-health-quarantine-v6"
|
||||
ai.bstein.dev/config-rev: "20260817-cli-auto-quota-routing-v1"
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/path: /metrics
|
||||
prometheus.io/port: "9010"
|
||||
@ -46,6 +46,11 @@ spec:
|
||||
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
|
||||
{{ .Data.data.codex_auth_json }}
|
||||
{{- end }}
|
||||
vault.hashicorp.com/agent-inject-secret-jenkins-image-build-token: kv/data/atlas/hermes/developer-jenkins
|
||||
vault.hashicorp.com/agent-inject-template-jenkins-image-build-token: |
|
||||
{{- with secret "kv/data/atlas/hermes/developer-jenkins" -}}
|
||||
{{ .Data.data.build_token }}
|
||||
{{- end }}
|
||||
vault.hashicorp.com/agent-inject-secret-node-ssh-private-key: kv/data/atlas/hermes/developer-ssh
|
||||
vault.hashicorp.com/agent-inject-template-node-ssh-private-key: |
|
||||
{{- with secret "kv/data/atlas/hermes/developer-ssh" -}}
|
||||
@ -859,6 +864,8 @@ spec:
|
||||
. /opt/data/.env
|
||||
set +a
|
||||
exec /opt/hermes/.venv/bin/python /opt/coordinator/cli_lane_runner.py
|
||||
ports:
|
||||
- {name: lane-metrics, containerPort: 9011, protocol: TCP}
|
||||
env:
|
||||
- {name: HERMES_HOME, value: /opt/data}
|
||||
- {name: HERMES_AUTH_FILE, value: /runtime-access/hermes-auth.json}
|
||||
@ -868,6 +875,9 @@ spec:
|
||||
- {name: KUBECONFIG, value: /opt/data/home/.kube/config}
|
||||
- {name: PYTHONPATH, value: /opt/hermes}
|
||||
- {name: HERMES_CLI_LANE_CONCURRENCY, value: "2"}
|
||||
- {name: HERMES_CLI_HEALTH_MAX_AGE_SECONDS, value: "60"}
|
||||
- {name: HERMES_CLI_LANE_METRICS_PORT, value: "9011"}
|
||||
- {name: HERMES_CLI_QUOTA_METRICS_URL, value: "http://127.0.0.1:9010/metrics"}
|
||||
- {name: HERMES_AUTO_ROUTER_PROFILE, value: agent}
|
||||
- {name: PATH, value: /opt/coordinator:/opt/data/tools/bin:/opt/data/home/.local/bin:/opt/hermes/.venv/bin:/usr/local/bin:/usr/bin:/bin}
|
||||
securityContext:
|
||||
@ -890,6 +900,15 @@ spec:
|
||||
- {name: atlas-pr-skill, mountPath: /opt/data/workspace/skills/manage-atlas-pull-requests, readOnly: true}
|
||||
- {name: scm-boundary, mountPath: /opt/scm, readOnly: true}
|
||||
- {name: tmp, mountPath: /tmp}
|
||||
readinessProbe:
|
||||
exec:
|
||||
command:
|
||||
- /opt/hermes/.venv/bin/python
|
||||
- /opt/coordinator/cli_lane_capabilities.py
|
||||
initialDelaySeconds: 2
|
||||
periodSeconds: 5
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 3
|
||||
resources:
|
||||
requests: {cpu: 100m, memory: 256Mi}
|
||||
limits: {cpu: "2", memory: 6Gi}
|
||||
|
||||
@ -129,6 +129,7 @@ configMapGenerator:
|
||||
- image_broker.py=scripts/image_broker.py
|
||||
- install_agent_tools.sh=scripts/install_agent_tools.sh
|
||||
- jenkins_build_evidence.py=scripts/jenkins_build_evidence.py
|
||||
- jenkins_image_build_trigger.py=scripts/jenkins_image_build_trigger.py
|
||||
- kanban_status_recovery.py=scripts/kanban_status_recovery.py
|
||||
- migrate_herdr_state.py=scripts/migrate_herdr_state.py
|
||||
- migrate_api_session_lineage.py=scripts/migrate_api_session_lineage.py
|
||||
@ -162,6 +163,7 @@ configMapGenerator:
|
||||
- node_account_hardening.py=scripts/node_account_hardening.py
|
||||
- node_account_audit.py=scripts/node_account_audit.py
|
||||
- node_account_io.py=scripts/node_account_io.py
|
||||
- node_polkit_audit.py=scripts/node_polkit_audit.py
|
||||
options:
|
||||
disableNameSuffixHash: true
|
||||
- name: hermes-auto-router-plugin
|
||||
|
||||
@ -119,6 +119,7 @@ spec:
|
||||
app: server
|
||||
ports:
|
||||
- {protocol: TCP, port: 9010}
|
||||
- {protocol: TCP, port: 9011}
|
||||
egress:
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
|
||||
@ -16,6 +16,9 @@ metadata:
|
||||
spec:
|
||||
selector:
|
||||
app: hermes-agent
|
||||
# The agent dashboard and terminal must stay reachable while the
|
||||
# cli-lane-runner readiness probe holds the shared pod NotReady.
|
||||
publishNotReadyAddresses: true
|
||||
ports:
|
||||
- {name: http, port: 80, targetPort: auth-http}
|
||||
---
|
||||
|
||||
@ -4,8 +4,11 @@ kind: Kustomization
|
||||
configMapGenerator:
|
||||
- name: hermes-scm-boundary-v2
|
||||
files:
|
||||
- deadline_http.py=scripts/deadline_http.py
|
||||
- git_pack_objects.py=scripts/git_pack_objects.py
|
||||
- gitea_api.py=scripts/gitea_api.py
|
||||
- gitea_api_policy.py=scripts/gitea_api_policy.py
|
||||
- receive_pack_scan.py=scripts/receive_pack_scan.py
|
||||
- scm_broker.py=scripts/scm_broker.py
|
||||
- scm_broker_io.py=scripts/scm_broker_io.py
|
||||
- scm_broker_server.py=scripts/scm_broker_server.py
|
||||
|
||||
344
services/hermes/scm-common/scripts/deadline_http.py
Normal file
344
services/hermes/scm-common/scripts/deadline_http.py
Normal file
@ -0,0 +1,344 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Absolute wall-clock deadlines for every outbound Hermes HTTP exchange.
|
||||
|
||||
Small control exchanges run inside a killable helper process, so connect,
|
||||
send, read, and DNS all share one hard bound that no blocking socket state
|
||||
can outlive. Large streaming Git exchanges cannot cross a process boundary;
|
||||
they instead register their connection with a watchdog that force-closes the
|
||||
socket when the same kind of absolute deadline expires.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import contextlib
|
||||
import email.message
|
||||
import http.client
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from dataclasses import dataclass
|
||||
|
||||
from gitea_api_policy import PolicyError
|
||||
|
||||
MAX_CONTROL_BYTES = 128 * 1024
|
||||
MAX_PROTOCOL_BYTES = 4 * 1024 * 1024
|
||||
MAX_STREAM_SECONDS = 900.0
|
||||
READ_CHUNK = 64 * 1024
|
||||
|
||||
|
||||
class _RejectRedirect(urllib.request.HTTPRedirectHandler):
|
||||
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
||||
raise urllib.error.URLError("redirect rejected")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Result:
|
||||
"""Minimal HTTP evidence returned across the process boundary."""
|
||||
|
||||
status: int
|
||||
content_type: str
|
||||
body: bytes
|
||||
|
||||
|
||||
def _read_response(response, maximum: int, deadline: float) -> bytes:
|
||||
chunks: list[bytes] = []
|
||||
total = 0
|
||||
while True:
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
raise TimeoutError
|
||||
raw = getattr(getattr(response, "fp", None), "raw", None)
|
||||
sock = getattr(raw, "_sock", raw)
|
||||
if hasattr(sock, "settimeout"):
|
||||
sock.settimeout(remaining)
|
||||
chunk = response.read(min(READ_CHUNK, maximum + 1 - total))
|
||||
if not chunk:
|
||||
break
|
||||
chunks.append(chunk)
|
||||
total += len(chunk)
|
||||
if total > maximum:
|
||||
raise ValueError("response too large")
|
||||
return b"".join(chunks)
|
||||
|
||||
|
||||
def _child_exchange(control: dict[str, object]) -> Result:
|
||||
url = control.get("url")
|
||||
method = control.get("method")
|
||||
headers = control.get("headers")
|
||||
encoded = control.get("body")
|
||||
maximum = control.get("maximum")
|
||||
timeout = control.get("timeout")
|
||||
if (
|
||||
not isinstance(url, str)
|
||||
or not isinstance(method, str)
|
||||
or not isinstance(headers, dict)
|
||||
or not all(isinstance(key, str) and isinstance(value, str) for key, value in headers.items())
|
||||
or not isinstance(encoded, str)
|
||||
or not isinstance(maximum, int)
|
||||
or not 0 <= maximum <= 2 * 1024 * 1024
|
||||
or not isinstance(timeout, (int, float))
|
||||
or not 0 < timeout <= 120
|
||||
):
|
||||
raise ValueError("invalid control")
|
||||
body = base64.b64decode(encoded, validate=True) if encoded else None
|
||||
if body is not None and len(body) > MAX_CONTROL_BYTES:
|
||||
raise ValueError("request too large")
|
||||
deadline = time.monotonic() + float(timeout)
|
||||
request = urllib.request.Request(
|
||||
url, data=body, method=method, headers=headers # type: ignore[arg-type]
|
||||
)
|
||||
opener = urllib.request.build_opener(_RejectRedirect())
|
||||
try:
|
||||
response = opener.open(request, timeout=max(deadline - time.monotonic(), 0.001))
|
||||
except urllib.error.HTTPError as error:
|
||||
response = error
|
||||
with response:
|
||||
status = getattr(response, "status", response.getcode())
|
||||
content_type = response.headers.get_content_type()
|
||||
value = _read_response(response, maximum, deadline)
|
||||
return Result(int(status), str(content_type), value)
|
||||
|
||||
|
||||
def _child_main() -> int:
|
||||
try:
|
||||
raw = sys.stdin.buffer.read(MAX_CONTROL_BYTES + 1)
|
||||
if len(raw) > MAX_CONTROL_BYTES:
|
||||
raise ValueError("control too large")
|
||||
control = json.loads(raw)
|
||||
if not isinstance(control, dict):
|
||||
raise ValueError("invalid control")
|
||||
result = _child_exchange(control)
|
||||
output = {
|
||||
"ok": True,
|
||||
"status": result.status,
|
||||
"content_type": result.content_type,
|
||||
"body": base64.b64encode(result.body).decode("ascii"),
|
||||
}
|
||||
except Exception:
|
||||
output = {"ok": False}
|
||||
sys.stdout.write(json.dumps(output, separators=(",", ":")))
|
||||
return 0 if output["ok"] else 1
|
||||
|
||||
|
||||
def _terminate(process: subprocess.Popen[bytes]) -> None:
|
||||
with contextlib.suppress(ProcessLookupError):
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
process.communicate()
|
||||
|
||||
|
||||
def exchange(
|
||||
request: urllib.request.Request,
|
||||
*,
|
||||
maximum: int,
|
||||
timeout: float,
|
||||
popen=subprocess.Popen,
|
||||
) -> Result:
|
||||
"""Execute a whole request under one monotonic deadline and size limit."""
|
||||
if not 0 < timeout <= 120 or not 0 <= maximum <= 2 * 1024 * 1024:
|
||||
raise PolicyError("HTTP control request bounds are invalid")
|
||||
body = request.data or b""
|
||||
if not isinstance(body, bytes) or len(body) > MAX_CONTROL_BYTES:
|
||||
raise PolicyError("HTTP control request exceeds the safe size limit")
|
||||
control = json.dumps(
|
||||
{
|
||||
"url": request.full_url,
|
||||
"method": request.get_method(),
|
||||
"headers": dict(request.header_items()),
|
||||
"body": base64.b64encode(body).decode("ascii"),
|
||||
"maximum": maximum,
|
||||
"timeout": timeout,
|
||||
},
|
||||
separators=(",", ":"),
|
||||
).encode()
|
||||
deadline = time.monotonic() + timeout
|
||||
process = popen(
|
||||
[sys.executable, os.path.realpath(__file__), "--child"],
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
start_new_session=True,
|
||||
env={"LANG": "C.UTF-8", "PATH": "/usr/bin:/bin"},
|
||||
)
|
||||
try:
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
raise subprocess.TimeoutExpired(process.args, timeout)
|
||||
output, _error = process.communicate(control, timeout=remaining)
|
||||
except subprocess.TimeoutExpired as exc:
|
||||
_terminate(process)
|
||||
raise PolicyError("HTTP control request deadline exceeded") from exc
|
||||
if process.returncode != 0 or len(output) > MAX_PROTOCOL_BYTES:
|
||||
raise PolicyError("HTTP control request failed")
|
||||
try:
|
||||
result = json.loads(output)
|
||||
if not isinstance(result, dict) or result.get("ok") is not True:
|
||||
raise ValueError
|
||||
status = result["status"]
|
||||
content_type = result["content_type"]
|
||||
encoded = result["body"]
|
||||
if not isinstance(status, int) or not isinstance(content_type, str) or not isinstance(encoded, str):
|
||||
raise ValueError
|
||||
value = base64.b64decode(encoded, validate=True)
|
||||
except (KeyError, ValueError, TypeError, json.JSONDecodeError) as exc:
|
||||
raise PolicyError("HTTP control helper returned invalid evidence") from exc
|
||||
if len(value) > maximum:
|
||||
raise PolicyError("HTTP control response exceeds the safe size limit")
|
||||
return Result(status, content_type, value)
|
||||
|
||||
|
||||
class _AdaptedResponse:
|
||||
"""Present one finished exchange through the urlopen reader interface."""
|
||||
|
||||
def __init__(self, result: Result):
|
||||
self.status = result.status
|
||||
self.headers = email.message.Message()
|
||||
self.headers["Content-Type"] = result.content_type
|
||||
self._stream = io.BytesIO(result.body)
|
||||
|
||||
def read(self, limit: int = -1) -> bytes:
|
||||
return self._stream.read(limit)
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_excinfo: object) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def open_bounded(
|
||||
request: urllib.request.Request,
|
||||
*,
|
||||
maximum: int,
|
||||
timeout: float,
|
||||
popen=subprocess.Popen,
|
||||
) -> _AdaptedResponse:
|
||||
"""Open one control exchange whose whole lifetime shares one deadline."""
|
||||
return _AdaptedResponse(
|
||||
exchange(request, maximum=maximum, timeout=timeout, popen=popen)
|
||||
)
|
||||
|
||||
|
||||
def _resolve_within(host: str, port: int, timeout: float) -> None:
|
||||
"""Resolve one host under a hard timeout the socket layer never covers.
|
||||
|
||||
``getaddrinfo`` ignores socket timeouts and runs before any socket
|
||||
exists, so a slow or hostile resolver would otherwise outlive the whole
|
||||
stream deadline. Running it in a joinable worker bounds resolution by the
|
||||
same wall clock; a stuck lookup leaks only one daemon thread.
|
||||
"""
|
||||
outcome: dict[str, object] = {}
|
||||
|
||||
def run() -> None:
|
||||
try:
|
||||
outcome["value"] = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM)
|
||||
except OSError as exc:
|
||||
outcome["error"] = exc
|
||||
|
||||
worker = threading.Thread(target=run, daemon=True)
|
||||
worker.start()
|
||||
worker.join(timeout)
|
||||
if worker.is_alive():
|
||||
raise PolicyError("HTTP stream DNS resolution deadline exceeded")
|
||||
if "error" in outcome:
|
||||
raise outcome["error"] # type: ignore[misc]
|
||||
|
||||
|
||||
class StreamDeadline:
|
||||
"""Force-close tracked connections once an absolute deadline passes."""
|
||||
|
||||
def __init__(self, timeout: float, *, timer=threading.Timer):
|
||||
if not 0 < timeout <= MAX_STREAM_SECONDS:
|
||||
raise PolicyError("HTTP stream deadline bounds are invalid")
|
||||
self._deadline = time.monotonic() + timeout
|
||||
self._lock = threading.Lock()
|
||||
self._connections: list[http.client.HTTPConnection] = []
|
||||
self.expired = False
|
||||
self._timer = timer(timeout, self._expire)
|
||||
self._timer.daemon = True
|
||||
self._timer.start()
|
||||
|
||||
def remaining(self) -> float:
|
||||
value = self._deadline - time.monotonic()
|
||||
if value <= 0 or self.expired:
|
||||
raise PolicyError("HTTP stream deadline exceeded")
|
||||
return value
|
||||
|
||||
def _expire(self) -> None:
|
||||
with self._lock:
|
||||
self.expired = True
|
||||
connections = list(self._connections)
|
||||
for connection in connections:
|
||||
# Shut the raw socket down first so a blocked send or recv wakes
|
||||
# immediately; close alone leaves peers waiting on the old fd.
|
||||
sock = getattr(connection, "sock", None)
|
||||
if sock is not None:
|
||||
with contextlib.suppress(OSError):
|
||||
sock.shutdown(socket.SHUT_RDWR)
|
||||
connection.close()
|
||||
|
||||
def _track(self, connection: http.client.HTTPConnection):
|
||||
with self._lock:
|
||||
expired = self.expired
|
||||
if not expired:
|
||||
self._connections.append(connection)
|
||||
if expired:
|
||||
connection.close()
|
||||
raise PolicyError("HTTP stream deadline exceeded")
|
||||
return connection
|
||||
|
||||
def _bounded_connection(self, connection: http.client.HTTPConnection):
|
||||
"""Register one connection and bound its DNS phase by the deadline.
|
||||
|
||||
The watchdog can only close a live socket, but ``connect`` resolves
|
||||
the host before any socket exists. Wrapping ``connect`` runs that
|
||||
resolution under the same absolute deadline first, then hands off to
|
||||
the real connect for the socket phases the watchdog already covers.
|
||||
"""
|
||||
original_connect = connection.connect
|
||||
|
||||
def connect() -> None:
|
||||
_resolve_within(connection.host, connection.port, self.remaining())
|
||||
original_connect()
|
||||
|
||||
connection.connect = connect # type: ignore[method-assign]
|
||||
return self._track(connection)
|
||||
|
||||
def cancel(self) -> None:
|
||||
self._timer.cancel()
|
||||
|
||||
def handlers(self) -> tuple[urllib.request.BaseHandler, ...]:
|
||||
"""Build urllib handlers whose connections obey this deadline."""
|
||||
deadline = self
|
||||
|
||||
class GuardedHTTPHandler(urllib.request.HTTPHandler):
|
||||
def http_open(self, req):
|
||||
return self.do_open(
|
||||
lambda host, **kwargs: deadline._bounded_connection(
|
||||
http.client.HTTPConnection(host, **kwargs)
|
||||
),
|
||||
req,
|
||||
)
|
||||
|
||||
class GuardedHTTPSHandler(urllib.request.HTTPSHandler):
|
||||
def https_open(self, req):
|
||||
return self.do_open(
|
||||
lambda host, **kwargs: deadline._bounded_connection(
|
||||
http.client.HTTPSConnection(host, **kwargs)
|
||||
),
|
||||
req,
|
||||
)
|
||||
|
||||
return (GuardedHTTPHandler(), GuardedHTTPSHandler())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(_child_main() if sys.argv[1:] == ["--child"] else 2)
|
||||
308
services/hermes/scm-common/scripts/git_pack_objects.py
Normal file
308
services/hermes/scm-common/scripts/git_pack_objects.py
Normal file
@ -0,0 +1,308 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Bounded Git pack inflation and delta resolution for quarantine scanning.
|
||||
|
||||
Pushed objects arrive zlib-compressed inside a packfile, so any content
|
||||
screening must inflate the real payloads first. This module unpacks one
|
||||
self-contained pack under explicit object, size, and checksum bounds and
|
||||
resolves every delta against bases inside the same pack. Thin packs are
|
||||
rejected outright: a delta against a base the broker cannot see would leave
|
||||
part of the pushed content unscanned.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import tempfile
|
||||
import zlib
|
||||
from typing import BinaryIO
|
||||
|
||||
from gitea_api_policy import PolicyError
|
||||
|
||||
OBJECT_TYPE_NAMES = {1: b"commit", 2: b"tree", 3: b"blob", 4: b"tag"}
|
||||
OFS_DELTA = 6
|
||||
REF_DELTA = 7
|
||||
MAX_PACK_OBJECTS = 50_000
|
||||
MAX_OBJECT_BYTES = 32 * 1024 * 1024
|
||||
MAX_TOTAL_BYTES = 256 * 1024 * 1024
|
||||
MAX_RESOLVE_PASSES = 64
|
||||
SPOOL_MEMORY_LIMIT = 256 * 1024
|
||||
READ_CHUNK = 64 * 1024
|
||||
|
||||
|
||||
def _read_exact(stream: BinaryIO, size: int) -> bytes:
|
||||
value = stream.read(size)
|
||||
if len(value) != size:
|
||||
raise PolicyError("Git pack data ended early")
|
||||
return value
|
||||
|
||||
|
||||
def _object_header(stream: BinaryIO) -> tuple[int, int]:
|
||||
"""Parse one object's type and declared inflated size."""
|
||||
byte = _read_exact(stream, 1)[0]
|
||||
type_code = (byte >> 4) & 0x7
|
||||
size = byte & 0x0F
|
||||
shift = 4
|
||||
while byte & 0x80:
|
||||
if shift > 60:
|
||||
raise PolicyError("Git object header is invalid")
|
||||
byte = _read_exact(stream, 1)[0]
|
||||
size |= (byte & 0x7F) << shift
|
||||
shift += 7
|
||||
return type_code, size
|
||||
|
||||
|
||||
def _delta_distance(stream: BinaryIO) -> int:
|
||||
"""Parse one ofs-delta base distance."""
|
||||
byte = _read_exact(stream, 1)[0]
|
||||
value = byte & 0x7F
|
||||
while byte & 0x80:
|
||||
byte = _read_exact(stream, 1)[0]
|
||||
value = ((value + 1) << 7) | (byte & 0x7F)
|
||||
if value > 1 << 48:
|
||||
raise PolicyError("Git delta base distance is invalid")
|
||||
return value
|
||||
|
||||
|
||||
def _inflate(stream: BinaryIO, declared: int) -> BinaryIO:
|
||||
"""Inflate one object into a quarantine spool, bounded by its header."""
|
||||
spool = tempfile.SpooledTemporaryFile( # noqa: SIM115 - caller owns returned spool
|
||||
max_size=SPOOL_MEMORY_LIMIT, dir="/tmp"
|
||||
)
|
||||
decompressor = zlib.decompressobj()
|
||||
produced = 0
|
||||
try:
|
||||
while not decompressor.eof:
|
||||
data = stream.read(READ_CHUNK)
|
||||
if not data:
|
||||
raise PolicyError("Git pack object data ended early")
|
||||
while True:
|
||||
value = decompressor.decompress(data, READ_CHUNK)
|
||||
produced += len(value)
|
||||
if produced > declared:
|
||||
raise PolicyError("Git object size does not match its header")
|
||||
spool.write(value)
|
||||
data = decompressor.unconsumed_tail
|
||||
if decompressor.eof or not data:
|
||||
break
|
||||
if decompressor.unused_data:
|
||||
stream.seek(-len(decompressor.unused_data), 1)
|
||||
if produced != declared:
|
||||
raise PolicyError("Git object size does not match its header")
|
||||
spool.seek(0)
|
||||
return spool
|
||||
except zlib.error as exc:
|
||||
spool.close()
|
||||
raise PolicyError("Git pack object data is corrupt") from exc
|
||||
except Exception:
|
||||
spool.close()
|
||||
raise
|
||||
|
||||
|
||||
def _delta_size(delta: bytes, position: int) -> tuple[int, int]:
|
||||
size = 0
|
||||
shift = 0
|
||||
while True:
|
||||
if position >= len(delta) or shift > 60:
|
||||
raise PolicyError("Git delta header is invalid")
|
||||
byte = delta[position]
|
||||
position += 1
|
||||
size |= (byte & 0x7F) << shift
|
||||
shift += 7
|
||||
if not byte & 0x80:
|
||||
return size, position
|
||||
|
||||
|
||||
def _apply_delta(base: bytes, delta: bytes) -> bytes:
|
||||
"""Apply one Git delta and require exact declared sizes throughout."""
|
||||
source_size, position = _delta_size(delta, 0)
|
||||
target_size, position = _delta_size(delta, position)
|
||||
if source_size != len(base) or target_size > MAX_OBJECT_BYTES:
|
||||
raise PolicyError("Git delta sizes are invalid")
|
||||
parts: list[bytes] = []
|
||||
produced = 0
|
||||
while position < len(delta):
|
||||
opcode = delta[position]
|
||||
position += 1
|
||||
if opcode & 0x80:
|
||||
operands = [bit for bit in (1, 2, 4, 8, 16, 32, 64) if opcode & bit]
|
||||
if position + len(operands) > len(delta):
|
||||
raise PolicyError("Git delta copy is truncated")
|
||||
offset = 0
|
||||
size = 0
|
||||
for shift, bit in enumerate((1, 2, 4, 8)):
|
||||
if opcode & bit:
|
||||
offset |= delta[position] << (8 * shift)
|
||||
position += 1
|
||||
for shift, bit in enumerate((16, 32, 64)):
|
||||
if opcode & bit:
|
||||
size |= delta[position] << (8 * shift)
|
||||
position += 1
|
||||
size = size or 0x10000
|
||||
if offset + size > len(base):
|
||||
raise PolicyError("Git delta copy is out of bounds")
|
||||
parts.append(base[offset : offset + size])
|
||||
produced += size
|
||||
elif opcode:
|
||||
if position + opcode > len(delta):
|
||||
raise PolicyError("Git delta insert is truncated")
|
||||
parts.append(delta[position : position + opcode])
|
||||
position += opcode
|
||||
produced += opcode
|
||||
else:
|
||||
raise PolicyError("Git delta instruction is invalid")
|
||||
if produced > target_size:
|
||||
raise PolicyError("Git delta result exceeds its declared size")
|
||||
if produced != target_size:
|
||||
raise PolicyError("Git delta result size is invalid")
|
||||
return b"".join(parts)
|
||||
|
||||
|
||||
def _payload_sha(type_code: int, payload: BinaryIO, size: int) -> bytes:
|
||||
digest = hashlib.sha1(OBJECT_TYPE_NAMES[type_code] + b" %d\x00" % size)
|
||||
payload.seek(0)
|
||||
while True:
|
||||
chunk = payload.read(READ_CHUNK)
|
||||
if not chunk:
|
||||
break
|
||||
digest.update(chunk)
|
||||
payload.seek(0)
|
||||
return digest.digest()
|
||||
|
||||
|
||||
def _verify_checksum(stream: BinaryIO, start: int, end: int) -> None:
|
||||
trailer = _read_exact(stream, 20)
|
||||
stream.seek(start)
|
||||
digest = hashlib.sha1()
|
||||
remaining = end - start
|
||||
while remaining:
|
||||
chunk = stream.read(min(READ_CHUNK, remaining))
|
||||
digest.update(chunk)
|
||||
remaining -= len(chunk)
|
||||
if digest.digest() != trailer:
|
||||
raise PolicyError("Git pack checksum is invalid")
|
||||
stream.seek(end + 20)
|
||||
|
||||
|
||||
class _Entry:
|
||||
"""One pack entry: either resolved content or an unresolved delta."""
|
||||
|
||||
def __init__(self, type_code: int, payload: BinaryIO, size: int, base):
|
||||
self.type_code = type_code
|
||||
self.payload = payload
|
||||
self.size = size
|
||||
self.base = base
|
||||
|
||||
|
||||
def _resolve(entries: dict[int, _Entry], total: list[int]) -> None:
|
||||
"""Resolve every delta against in-pack bases only, to a fixed point."""
|
||||
by_sha: dict[bytes, int] = {}
|
||||
for offset, entry in entries.items():
|
||||
if entry.base is None:
|
||||
by_sha.setdefault(
|
||||
_payload_sha(entry.type_code, entry.payload, entry.size), offset
|
||||
)
|
||||
for _ in range(MAX_RESOLVE_PASSES):
|
||||
pending = [item for item in entries.items() if item[1].base is not None]
|
||||
if not pending:
|
||||
return
|
||||
progressed = False
|
||||
for offset, entry in pending:
|
||||
base_key = entry.base
|
||||
base_offset = base_key if isinstance(base_key, int) else by_sha.get(base_key)
|
||||
base = entries.get(base_offset) if base_offset is not None else None
|
||||
if base is None or base.base is not None:
|
||||
if isinstance(base_key, int) and base is None:
|
||||
raise PolicyError("Git delta base offset is invalid")
|
||||
continue
|
||||
base.payload.seek(0)
|
||||
entry.payload.seek(0)
|
||||
value = _apply_delta(base.payload.read(), entry.payload.read())
|
||||
total[0] += len(value)
|
||||
if total[0] > MAX_TOTAL_BYTES:
|
||||
raise PolicyError("Git pack contents exceed the safe size limit")
|
||||
entry.payload.close()
|
||||
spool = tempfile.SpooledTemporaryFile( # noqa: SIM115 - entry owns spool
|
||||
max_size=SPOOL_MEMORY_LIMIT, dir="/tmp"
|
||||
)
|
||||
spool.write(value)
|
||||
spool.seek(0)
|
||||
entry.type_code = base.type_code
|
||||
entry.payload = spool
|
||||
entry.size = len(value)
|
||||
entry.base = None
|
||||
by_sha.setdefault(_payload_sha(entry.type_code, spool, entry.size), offset)
|
||||
progressed = True
|
||||
if not progressed:
|
||||
raise PolicyError("Git delta base is outside the pack; push full packs")
|
||||
raise PolicyError("Git delta chains exceed the safe resolution limit")
|
||||
|
||||
|
||||
def unpack_objects(stream: BinaryIO) -> list[tuple[int, BinaryIO]]:
|
||||
"""Inflate one self-contained pack and return every resolved payload."""
|
||||
start = stream.tell()
|
||||
header = _read_exact(stream, 12)
|
||||
version = int.from_bytes(header[4:8], "big")
|
||||
count = int.from_bytes(header[8:12], "big")
|
||||
if header[:4] != b"PACK" or version not in {2, 3}:
|
||||
raise PolicyError("Git pack header is invalid")
|
||||
if count > MAX_PACK_OBJECTS:
|
||||
raise PolicyError("Git pack contains too many objects")
|
||||
entries: dict[int, _Entry] = {}
|
||||
total = [0]
|
||||
try:
|
||||
for _ in range(count):
|
||||
offset = stream.tell() - start
|
||||
type_code, declared = _object_header(stream)
|
||||
if declared > MAX_OBJECT_BYTES:
|
||||
raise PolicyError("Git object exceeds the safe size limit")
|
||||
total[0] += declared
|
||||
if total[0] > MAX_TOTAL_BYTES:
|
||||
raise PolicyError("Git pack contents exceed the safe size limit")
|
||||
base: int | bytes | None = None
|
||||
if type_code == OFS_DELTA:
|
||||
distance = _delta_distance(stream)
|
||||
base = offset - distance
|
||||
if base < 0:
|
||||
raise PolicyError("Git delta base offset is invalid")
|
||||
elif type_code == REF_DELTA:
|
||||
base = _read_exact(stream, 20)
|
||||
elif type_code not in OBJECT_TYPE_NAMES:
|
||||
raise PolicyError("Git pack object type is invalid")
|
||||
entries[offset] = _Entry(type_code, _inflate(stream, declared), declared, base)
|
||||
_verify_checksum(stream, start, stream.tell())
|
||||
_resolve(entries, total)
|
||||
except Exception:
|
||||
for entry in entries.values():
|
||||
entry.payload.close()
|
||||
raise
|
||||
for entry in entries.values():
|
||||
entry.payload.seek(0)
|
||||
return [
|
||||
(entries[offset].type_code, entries[offset].payload)
|
||||
for offset in sorted(entries)
|
||||
]
|
||||
|
||||
|
||||
def parse_loose_object(value: bytes) -> tuple[int, bytes]:
|
||||
"""Inflate one quarantined loose object and return its typed payload."""
|
||||
if len(value) > MAX_OBJECT_BYTES:
|
||||
raise PolicyError("Git object exceeds the safe size limit")
|
||||
decompressor = zlib.decompressobj()
|
||||
try:
|
||||
inflated = decompressor.decompress(value, MAX_OBJECT_BYTES + 1)
|
||||
except zlib.error as exc:
|
||||
raise PolicyError("Git loose object data is corrupt") from exc
|
||||
if not decompressor.eof or decompressor.unused_data or len(inflated) > MAX_OBJECT_BYTES:
|
||||
raise PolicyError("Git loose object framing is invalid")
|
||||
kind, separator, remainder = inflated.partition(b"\x00")
|
||||
name, space, size = kind.partition(b" ")
|
||||
types = {label: code for code, label in OBJECT_TYPE_NAMES.items()}
|
||||
if (
|
||||
not separator
|
||||
or not space
|
||||
or name not in types
|
||||
or not size.isdigit()
|
||||
or int(size) != len(remainder)
|
||||
):
|
||||
raise PolicyError("Git loose object header is invalid")
|
||||
return types[name], remainder
|
||||
@ -15,6 +15,7 @@ import urllib.request
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
|
||||
import deadline_http
|
||||
from gitea_api_policy import (
|
||||
PolicyError,
|
||||
_draft_title,
|
||||
@ -40,19 +41,16 @@ MAX_API_PATH_LENGTH = 512
|
||||
RAW_API_TARGET_RE = re.compile(r"[A-Za-z0-9/_.?&=-]+\Z")
|
||||
|
||||
|
||||
class RejectRedirectHandler(urllib.request.HTTPRedirectHandler):
|
||||
"""Reject every redirect before urllib can copy authentication headers."""
|
||||
|
||||
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
||||
raise PolicyError("Forgejo redirects are not allowed")
|
||||
|
||||
|
||||
_SAFE_OPENER = urllib.request.build_opener(RejectRedirectHandler())
|
||||
|
||||
|
||||
def _safe_urlopen(request: urllib.request.Request, timeout: int):
|
||||
"""Open one request using a handler that never follows redirects."""
|
||||
return _SAFE_OPENER.open(request, timeout=timeout)
|
||||
"""Open one exchange under a killable absolute wall-clock deadline.
|
||||
|
||||
The helper process rejects redirects before urllib can copy
|
||||
authentication headers, and its whole connect/send/read lifetime
|
||||
shares one hard deadline.
|
||||
"""
|
||||
return deadline_http.open_bounded(
|
||||
request, maximum=MAX_RESPONSE_BYTES, timeout=timeout
|
||||
)
|
||||
|
||||
|
||||
def read_token(path: Path = DEFAULT_TOKEN_FILE) -> str:
|
||||
|
||||
143
services/hermes/scm-common/scripts/receive_pack_scan.py
Normal file
143
services/hermes/scm-common/scripts/receive_pack_scan.py
Normal file
@ -0,0 +1,143 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate Git receive-pack requests and scan pushed object content.
|
||||
|
||||
Ref commands may only create new namespaced feature branches. Every pushed
|
||||
object is then inflated in quarantine and its decompressed payload scanned,
|
||||
so credentials cannot ride through the broker inside compressed pack data.
|
||||
The screening is a fail-closed accident barrier for credential-shaped
|
||||
content: exact runtime-token forms, private keys, SSH key material, and
|
||||
known provider token formats. Entropy heuristics are deliberately excluded
|
||||
so SOPS ciphertext, pinned digests, and lock files remain pushable; a short
|
||||
secret disguised as prose is not reliably detectable and must never be
|
||||
supplied by callers.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import re
|
||||
from typing import BinaryIO
|
||||
|
||||
import git_pack_objects
|
||||
from gitea_api_policy import (
|
||||
STANDALONE_SECRET_PATTERNS,
|
||||
PolicyError,
|
||||
_validate_ref,
|
||||
)
|
||||
|
||||
ZERO_SHA = b"0" * 40
|
||||
MAX_PUSH_COMMANDS = 16
|
||||
SCAN_CHUNK = 256 * 1024
|
||||
SCAN_CARRY = 8 * 1024
|
||||
FEATURE_REF_RE = re.compile(
|
||||
r"refs/heads/(?:(?:feature|fix|hermes|handoff)/[A-Za-z0-9][A-Za-z0-9._/-]{0,190})\Z"
|
||||
)
|
||||
PKT_HEADER_RE = re.compile(rb"[0-9a-f]{4}\Z")
|
||||
CONTENT_SECRET_PATTERNS = tuple(
|
||||
re.compile(pattern.pattern.encode("ascii"), re.IGNORECASE)
|
||||
for pattern in STANDALONE_SECRET_PATTERNS
|
||||
) + (
|
||||
re.compile(rb"(?i)(?<![A-Za-z0-9])AGE-SECRET-KEY-1[A-Z0-9]{20,}"),
|
||||
re.compile(rb"(?i)PuTTY-User-Key-File"),
|
||||
re.compile(
|
||||
rb"(?im)^[ \t]*authorization[ \t]*:[ \t]*(?:basic|bearer|token)[ \t]+"
|
||||
rb"(?=[A-Za-z0-9+/_=.:-]*[0-9+/=.-])[A-Za-z0-9+/_=.:-]{8,}"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _read_pkt_line(stream: BinaryIO) -> bytes | None:
|
||||
"""Read one pkt-line; None marks the flush packet before the pack."""
|
||||
header = stream.read(4)
|
||||
if not header:
|
||||
raise PolicyError("Git receive-pack request omitted its command terminator")
|
||||
if not PKT_HEADER_RE.fullmatch(header):
|
||||
raise PolicyError("Git receive-pack command framing is invalid")
|
||||
size = int(header, 16)
|
||||
if size == 0:
|
||||
return None
|
||||
if size < 4:
|
||||
raise PolicyError("Git receive-pack command framing is invalid")
|
||||
value = stream.read(size - 4)
|
||||
if len(value) != size - 4:
|
||||
raise PolicyError("Git receive-pack command framing is invalid")
|
||||
return value
|
||||
|
||||
|
||||
def _validate_command(line: bytes, token: str, forbidden: tuple[bytes, ...]) -> None:
|
||||
"""Permit only creation of one new, namespaced feature branch."""
|
||||
if any(form in line for form in forbidden):
|
||||
raise PolicyError("Git request contains runtime credential material")
|
||||
command = line.rstrip(b"\n").split(b"\x00", 1)[0]
|
||||
fields = command.split(b" ")
|
||||
if len(fields) != 3 or not all(
|
||||
re.fullmatch(rb"[0-9a-f]{40}", item) for item in fields[:2]
|
||||
):
|
||||
raise PolicyError("Git receive-pack ref command is invalid")
|
||||
old_sha, new_sha, raw_ref = fields
|
||||
if old_sha != ZERO_SHA or new_sha == ZERO_SHA:
|
||||
raise PolicyError("Git broker permits only new feature-branch creation")
|
||||
try:
|
||||
ref = raw_ref.decode("ascii")
|
||||
except UnicodeDecodeError as exc:
|
||||
raise PolicyError("Git ref must be canonical ASCII") from exc
|
||||
if not FEATURE_REF_RE.fullmatch(ref):
|
||||
raise PolicyError("Git push is limited to namespaced feature branches")
|
||||
_validate_ref(ref.removeprefix("refs/heads/"), "head", forbidden=(token,))
|
||||
|
||||
|
||||
def _scan_payload(payload: BinaryIO, forbidden: tuple[bytes, ...]) -> None:
|
||||
"""Scan one decompressed object payload across chunk boundaries."""
|
||||
carry = b""
|
||||
while True:
|
||||
chunk = payload.read(SCAN_CHUNK)
|
||||
if not chunk:
|
||||
return
|
||||
window = carry + chunk
|
||||
if any(form in window for form in forbidden):
|
||||
raise PolicyError("Git push contains runtime credential material")
|
||||
if any(pattern.search(window) for pattern in CONTENT_SECRET_PATTERNS):
|
||||
raise PolicyError("Git push contains credential-shaped content")
|
||||
carry = window[-SCAN_CARRY:]
|
||||
|
||||
|
||||
def _scan_pack(stream: BinaryIO, forbidden: tuple[bytes, ...]) -> None:
|
||||
objects = git_pack_objects.unpack_objects(stream)
|
||||
try:
|
||||
if stream.read(1):
|
||||
raise PolicyError("Git request has bytes after its pack")
|
||||
for _type_code, payload in objects:
|
||||
_scan_payload(payload, forbidden)
|
||||
finally:
|
||||
for _type_code, payload in objects:
|
||||
payload.close()
|
||||
|
||||
|
||||
def validate_receive_pack(
|
||||
body: bytes | BinaryIO, token: str, forbidden: tuple[bytes, ...]
|
||||
) -> None:
|
||||
"""Validate one whole receive-pack request: commands, pack, and content."""
|
||||
stream = io.BytesIO(body) if isinstance(body, bytes) else body
|
||||
stream.seek(0)
|
||||
try:
|
||||
commands = 0
|
||||
while True:
|
||||
line = _read_pkt_line(stream)
|
||||
if line is None:
|
||||
break
|
||||
_validate_command(line, token, forbidden)
|
||||
commands += 1
|
||||
if commands > MAX_PUSH_COMMANDS:
|
||||
raise PolicyError("Git push contains too many ref commands")
|
||||
if commands == 0:
|
||||
raise PolicyError("Git receive-pack request has no ref command")
|
||||
_scan_pack(stream, forbidden)
|
||||
finally:
|
||||
stream.seek(0)
|
||||
|
||||
|
||||
def scan_loose_object(value: bytes, forbidden: tuple[bytes, ...]) -> int:
|
||||
"""Scan one quarantined loose object and return its parsed type code."""
|
||||
type_code, payload = git_pack_objects.parse_loose_object(value)
|
||||
_scan_payload(io.BytesIO(payload), forbidden)
|
||||
return type_code
|
||||
@ -7,6 +7,7 @@ import argparse
|
||||
import base64
|
||||
import json
|
||||
import re
|
||||
import socket
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.error
|
||||
@ -15,6 +16,7 @@ import urllib.request
|
||||
from http.server import BaseHTTPRequestHandler
|
||||
from typing import BinaryIO
|
||||
|
||||
import deadline_http
|
||||
from gitea_api import (
|
||||
CANONICAL_BASE_URL,
|
||||
PolicyError,
|
||||
@ -22,7 +24,8 @@ from gitea_api import (
|
||||
read,
|
||||
read_token,
|
||||
)
|
||||
from gitea_api_policy import _reject_forbidden, _validate_ref, _validate_repo
|
||||
from gitea_api_policy import _reject_forbidden, _validate_repo
|
||||
from receive_pack_scan import validate_receive_pack
|
||||
from scm_broker_io import RejectRedirect, response_status as _status, spool_response
|
||||
from scm_broker_server import AbsoluteHeaderDeadlineMixin, BoundedThreadingHTTPServer
|
||||
|
||||
@ -31,25 +34,17 @@ GIT_USER = "hermes-automation"
|
||||
MAX_CONTROL_BODY = 64 * 1024
|
||||
MAX_GIT_REQUEST = 128 * 1024 * 1024
|
||||
MAX_GIT_RESPONSE = 128 * 1024 * 1024
|
||||
MAX_PUSH_COMMANDS = 16
|
||||
MAX_HEADERS = 32
|
||||
MAX_HEADER_BYTES = 16 * 1024
|
||||
SPOOL_MEMORY_LIMIT = 1024 * 1024
|
||||
STREAM_CHUNK = 64 * 1024
|
||||
INBOUND_HEADER_TIMEOUT = 10.0
|
||||
INBOUND_BODY_TIMEOUT = 120.0
|
||||
ZERO_SHA = b"0" * 40
|
||||
UPSTREAM_DEADLINE_SECONDS = 300.0
|
||||
GIT_PATH_RE = re.compile(
|
||||
r"/git/atlas/(?P<repo>[A-Za-z0-9][A-Za-z0-9._-]{0,99})\.git/"
|
||||
r"(?P<operation>info/refs|git-upload-pack|git-receive-pack)\Z"
|
||||
)
|
||||
FEATURE_REF_RE = re.compile(
|
||||
r"refs/heads/(?:(?:feature|fix|chore|docs|test|refactor|wt|review|hermes|handoff)/"
|
||||
r"[A-Za-z0-9][A-Za-z0-9._/-]{0,190})\Z"
|
||||
)
|
||||
|
||||
|
||||
UPSTREAM_OPENER = urllib.request.build_opener(RejectRedirect())
|
||||
|
||||
|
||||
def _read_bounded(
|
||||
@ -96,7 +91,7 @@ def _spool_bounded(
|
||||
"""Copy a fixed-length exchange through a bounded-memory disk spool."""
|
||||
if not 0 <= length <= maximum:
|
||||
raise PolicyError("SCM request exceeds the safe size limit")
|
||||
spool = tempfile.SpooledTemporaryFile( # noqa: SIM115 - caller owns lifetime
|
||||
spool = tempfile.SpooledTemporaryFile( # noqa: SIM115 - caller owns returned spool
|
||||
max_size=SPOOL_MEMORY_LIMIT, dir="/tmp"
|
||||
)
|
||||
remaining = length
|
||||
@ -126,14 +121,16 @@ def _spool_bounded(
|
||||
raise
|
||||
|
||||
|
||||
def _spool_response(response, maximum: int, token: str) -> tuple[BinaryIO, int]:
|
||||
def _spool_response(
|
||||
response, maximum: int, token: str, deadline_seconds: float
|
||||
) -> tuple[BinaryIO, int]:
|
||||
return spool_response(
|
||||
response,
|
||||
maximum,
|
||||
_credential_forms(token),
|
||||
memory_limit=SPOOL_MEMORY_LIMIT,
|
||||
chunk_size=STREAM_CHUNK,
|
||||
deadline_seconds=INBOUND_BODY_TIMEOUT,
|
||||
deadline_seconds=deadline_seconds,
|
||||
)
|
||||
|
||||
|
||||
@ -211,59 +208,14 @@ def _credential_forms(token: str) -> tuple[bytes, ...]:
|
||||
return token.encode("utf-8"), basic, b"Basic " + basic
|
||||
|
||||
|
||||
def _reject_credential_bytes(value: bytes, token: str, context: str) -> None:
|
||||
if any(form in value for form in _credential_forms(token)):
|
||||
raise PolicyError(f"{context} contains runtime credential material")
|
||||
|
||||
|
||||
def _receive_prefix(body: bytes | BinaryIO) -> bytes:
|
||||
if isinstance(body, bytes):
|
||||
return body
|
||||
position = body.tell()
|
||||
try:
|
||||
body.seek(0)
|
||||
return body.read(64 * 1024)
|
||||
finally:
|
||||
body.seek(position)
|
||||
|
||||
|
||||
def _validate_receive_pack(body: bytes | BinaryIO, token: str) -> None:
|
||||
"""Permit only creation of new branches in reviewed namespaces."""
|
||||
body = _receive_prefix(body)
|
||||
_reject_credential_bytes(body, token, "Git request")
|
||||
position = 0
|
||||
commands = 0
|
||||
while position + 4 <= len(body):
|
||||
header = body[position : position + 4]
|
||||
if not re.fullmatch(rb"[0-9a-f]{4}", header):
|
||||
raise PolicyError("Git receive-pack command framing is invalid")
|
||||
size = int(header, 16)
|
||||
if size == 0:
|
||||
if commands == 0:
|
||||
raise PolicyError("Git receive-pack request has no ref command")
|
||||
return
|
||||
if size < 4 or position + size > len(body):
|
||||
raise PolicyError("Git receive-pack command framing is invalid")
|
||||
command = body[position + 4 : position + size].rstrip(b"\n")
|
||||
command = command.split(b"\x00", 1)[0]
|
||||
fields = command.split(b" ")
|
||||
if len(fields) != 3 or not all(re.fullmatch(rb"[0-9a-f]{40}", item) for item in fields[:2]):
|
||||
raise PolicyError("Git receive-pack ref command is invalid")
|
||||
old_sha, new_sha, raw_ref = fields
|
||||
if old_sha != ZERO_SHA or new_sha == ZERO_SHA:
|
||||
raise PolicyError("Git broker permits only new namespaced branch creation")
|
||||
try:
|
||||
ref = raw_ref.decode("ascii")
|
||||
except UnicodeDecodeError as exc:
|
||||
raise PolicyError("Git ref must be canonical ASCII") from exc
|
||||
if not FEATURE_REF_RE.fullmatch(ref):
|
||||
raise PolicyError("Git push is limited to namespaced feature branches")
|
||||
_validate_ref(ref.removeprefix("refs/heads/"), "head", forbidden=(token,))
|
||||
commands += 1
|
||||
if commands > MAX_PUSH_COMMANDS:
|
||||
raise PolicyError("Git push contains too many ref commands")
|
||||
position += size
|
||||
raise PolicyError("Git receive-pack request omitted its command terminator")
|
||||
"""Permit only vetted new-feature-branch pushes with scanned contents."""
|
||||
validate_receive_pack(body, token, _credential_forms(token))
|
||||
|
||||
|
||||
def _guarded_opener(guard: deadline_http.StreamDeadline):
|
||||
"""Build an opener whose connection obeys one absolute stream deadline."""
|
||||
return urllib.request.build_opener(RejectRedirect(), *guard.handlers()).open
|
||||
|
||||
|
||||
def _upstream_git_request(
|
||||
@ -276,7 +228,7 @@ def _upstream_git_request(
|
||||
token: str,
|
||||
body_length: int | None = None,
|
||||
stream_result: bool = False,
|
||||
opener=UPSTREAM_OPENER.open,
|
||||
opener=None,
|
||||
) -> bytes | tuple[BinaryIO, int]:
|
||||
credentials = base64.b64encode(f"{GIT_USER}:{token}".encode()).decode("ascii")
|
||||
headers = {"Accept": expected_type, "Authorization": f"Basic {credentials}"}
|
||||
@ -294,12 +246,21 @@ def _upstream_git_request(
|
||||
method=method,
|
||||
headers=headers,
|
||||
)
|
||||
with opener(request, timeout=120) as response:
|
||||
if _status(response) != 200:
|
||||
raise PolicyError("Git upstream returned an unexpected HTTP status")
|
||||
if response.headers.get_content_type() != expected_type:
|
||||
raise PolicyError("Git upstream returned an unexpected response type")
|
||||
result, result_length = _spool_response(response, MAX_GIT_RESPONSE, token)
|
||||
# One wall-clock budget covers connect, send, headers, and body read; the
|
||||
# watchdog closes the upstream socket if any phase overstays it.
|
||||
guard = deadline_http.StreamDeadline(UPSTREAM_DEADLINE_SECONDS)
|
||||
try:
|
||||
open_upstream = opener if opener is not None else _guarded_opener(guard)
|
||||
with open_upstream(request, timeout=guard.remaining()) as response:
|
||||
if _status(response) != 200:
|
||||
raise PolicyError("Git upstream returned an unexpected HTTP status")
|
||||
if response.headers.get_content_type() != expected_type:
|
||||
raise PolicyError("Git upstream returned an unexpected response type")
|
||||
result, result_length = _spool_response(
|
||||
response, MAX_GIT_RESPONSE, token, guard.remaining()
|
||||
)
|
||||
finally:
|
||||
guard.cancel()
|
||||
if stream_result:
|
||||
return result, result_length
|
||||
try:
|
||||
@ -331,7 +292,9 @@ class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler):
|
||||
if len(self.headers.get_all("Content-Length", [])) > 1:
|
||||
raise PolicyError("SCM request has duplicate Content-Length")
|
||||
|
||||
def _stream(self, status: int, content_type: str, body: BinaryIO, length: int) -> None:
|
||||
def _stream(
|
||||
self, status: int, content_type: str, body: BinaryIO, length: int
|
||||
) -> None:
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", content_type)
|
||||
self.send_header("Content-Length", str(length))
|
||||
@ -387,7 +350,13 @@ class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler):
|
||||
self._stream(200, expected, body, length)
|
||||
finally:
|
||||
body.close()
|
||||
except (TimeoutError, OSError, PolicyError, urllib.error.URLError, ValueError):
|
||||
except (
|
||||
OSError,
|
||||
PolicyError,
|
||||
socket.timeout,
|
||||
urllib.error.URLError,
|
||||
ValueError,
|
||||
):
|
||||
self._reject()
|
||||
|
||||
def do_POST(self) -> None:
|
||||
@ -399,9 +368,9 @@ class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler):
|
||||
else:
|
||||
self._git_rpc()
|
||||
except (
|
||||
TimeoutError,
|
||||
OSError,
|
||||
PolicyError,
|
||||
socket.timeout,
|
||||
urllib.error.URLError,
|
||||
ValueError,
|
||||
json.JSONDecodeError,
|
||||
@ -417,7 +386,9 @@ class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler):
|
||||
result = read(data["path"], token=token)
|
||||
else:
|
||||
expected = {"base", "body", "head", "head_sha", "repo", "title"}
|
||||
if set(data) != expected or not all(isinstance(data[key], str) for key in expected):
|
||||
if set(data) != expected or not all(
|
||||
isinstance(data[key], str) for key in expected
|
||||
):
|
||||
raise PolicyError("draft request fields are invalid")
|
||||
result = create_draft(token=token, **data) # type: ignore[arg-type]
|
||||
if token.encode("utf-8") in result:
|
||||
@ -426,10 +397,15 @@ class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler):
|
||||
|
||||
def _git_rpc(self) -> None:
|
||||
repo, operation, service = _git_target(self.path)
|
||||
if operation not in {"git-upload-pack", "git-receive-pack"} or service != operation:
|
||||
if (
|
||||
operation not in {"git-upload-pack", "git-receive-pack"}
|
||||
or service != operation
|
||||
):
|
||||
raise PolicyError("Git RPC operation is outside the allowlist")
|
||||
expected_request = f"application/x-{service}-request"
|
||||
if self.headers.get_content_type() != expected_request or self.headers.get("Transfer-Encoding"):
|
||||
if self.headers.get_content_type() != expected_request or self.headers.get(
|
||||
"Transfer-Encoding"
|
||||
):
|
||||
raise PolicyError("Git RPC request type is invalid")
|
||||
length = _content_length(self.headers, MAX_GIT_REQUEST)
|
||||
token = read_token()
|
||||
|
||||
@ -7,24 +7,22 @@ import json
|
||||
import urllib.request
|
||||
from collections.abc import Callable
|
||||
|
||||
import deadline_http
|
||||
from gitea_api_policy import PolicyError
|
||||
|
||||
BROKER_ORIGIN = "http://hermes-scm-broker.hermes-scm.svc.cluster.local:9081"
|
||||
MAX_RESPONSE_BYTES = 2 * 1024 * 1024
|
||||
|
||||
|
||||
class RejectRedirectHandler(urllib.request.HTTPRedirectHandler):
|
||||
"""Keep every broker request on its fixed in-cluster origin."""
|
||||
|
||||
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
||||
raise PolicyError("SCM broker redirects are not allowed")
|
||||
|
||||
|
||||
_OPENER = urllib.request.build_opener(RejectRedirectHandler())
|
||||
|
||||
|
||||
def _open(request: urllib.request.Request, timeout: int):
|
||||
return _OPENER.open(request, timeout=timeout)
|
||||
"""Open one broker exchange under a killable absolute deadline.
|
||||
|
||||
The helper process keeps every request on its fixed in-cluster origin by
|
||||
rejecting redirects before they are followed.
|
||||
"""
|
||||
return deadline_http.open_bounded(
|
||||
request, maximum=MAX_RESPONSE_BYTES, timeout=timeout
|
||||
)
|
||||
|
||||
|
||||
def request(
|
||||
|
||||
@ -1,11 +1,38 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Detect and publish the Kanban API boundary required by CLI lanes."""
|
||||
"""Detect and publish the Kanban API boundary required by CLI lanes.
|
||||
|
||||
During a rollout the coordinator container and the cli-lane-runner worker can
|
||||
run different hermes-agent image generations against one shared Kanban DB.
|
||||
The image APIs that differ on the decomposition/finalization path:
|
||||
|
||||
* ``complete_task``: both generations accept ``expected_run_id`` (active
|
||||
exact-run completion); only the patched image accepts
|
||||
``replay_ended_run_id`` (ended-run replay).
|
||||
* ``reclaim_task``: only the patched image accepts ``expected_run_id``
|
||||
(guarded orphan reclaim after a runner restart).
|
||||
* ``specify_triage_task``/``decompose_triage_task``: only the patched image
|
||||
accepts ``require_no_runs`` (refuses redefining tasks with run history).
|
||||
|
||||
Worker-side handling per combination (proven in
|
||||
testing/tests/test_hermes_cli_capabilities.py):
|
||||
|
||||
* legacy worker: health/readiness stay deferred, no new claims, no unguarded
|
||||
reclaim, replay-needing results stay journaled for a patched successor.
|
||||
* patched worker: ready health, exact completion, guarded replay/reclaim; a
|
||||
run voided by a legacy coordinator's unguarded mid-run decomposition
|
||||
converges to stale/conflict evidence instead of completing.
|
||||
* patched coordinator: mid-run decomposition is refused entirely, so the
|
||||
active exact run completes on either worker generation.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from dataclasses import asdict, dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable
|
||||
|
||||
@ -85,6 +112,7 @@ def _health_document(capabilities: KanbanCapabilities) -> dict[str, Any]:
|
||||
"""Return a bounded operator-readable compatibility health document."""
|
||||
return {
|
||||
"component": "cli-lane-runner",
|
||||
"schema_version": 1,
|
||||
"state": "ready" if capabilities.ready else "deferred",
|
||||
"ready": capabilities.ready,
|
||||
"active_run_completion_safe": capabilities.exact_run_completion,
|
||||
@ -117,6 +145,21 @@ def initialize_kanban_capabilities(
|
||||
return capabilities
|
||||
|
||||
|
||||
def refresh_kanban_capabilities(
|
||||
kanban_db: Any,
|
||||
*,
|
||||
health_path: Path | None = None,
|
||||
) -> KanbanCapabilities:
|
||||
"""Reinspect the live callables and refresh bounded readiness evidence.
|
||||
|
||||
ConfigMap script updates and image changes are deliberately not imported or
|
||||
reloaded here. A running process observes only the already-loaded module,
|
||||
which prevents a half-old/half-new Python module graph. Tests may replace
|
||||
the two callables to model an image API transition at a loop boundary.
|
||||
"""
|
||||
return initialize_kanban_capabilities(kanban_db, health_path=health_path)
|
||||
|
||||
|
||||
def kanban_capabilities(kanban_db: Any) -> KanbanCapabilities:
|
||||
"""Return startup evidence, redetecting only when the loaded API changes."""
|
||||
if _CAPABILITIES is None or _source_identity(kanban_db) != _CAPABILITY_SOURCE:
|
||||
@ -128,3 +171,68 @@ def runtime_health(path: Path | None = None) -> dict[str, Any]:
|
||||
"""Read the bounded health document for probes and operator diagnostics."""
|
||||
document = load_json(path or STATE_ROOT / "runtime-health.json")
|
||||
return document if isinstance(document, dict) else {}
|
||||
|
||||
|
||||
def readiness_issue(
|
||||
path: Path | None = None,
|
||||
*,
|
||||
now: datetime | None = None,
|
||||
maximum_age_seconds: float = 60.0,
|
||||
) -> str | None:
|
||||
"""Return a bounded readiness reason without exposing health contents."""
|
||||
destination = path or STATE_ROOT / "runtime-health.json"
|
||||
try:
|
||||
document = json.loads(destination.read_text(encoding="utf-8"))
|
||||
except FileNotFoundError:
|
||||
return "runtime health is missing"
|
||||
except (OSError, UnicodeError, json.JSONDecodeError):
|
||||
return "runtime health is malformed"
|
||||
if not isinstance(document, dict) or document.get("schema_version") != 1:
|
||||
return "runtime health is malformed"
|
||||
if document.get("state") == "deferred" and document.get("ready") is False:
|
||||
return "runtime compatibility is deferred"
|
||||
capabilities = document.get("capabilities")
|
||||
if (
|
||||
document.get("state") != "ready"
|
||||
or document.get("ready") is not True
|
||||
or not isinstance(capabilities, dict)
|
||||
or any(
|
||||
capabilities.get(name) is not True
|
||||
for name in (
|
||||
"exact_run_completion",
|
||||
"ended_run_replay",
|
||||
"exact_run_reclaim",
|
||||
)
|
||||
)
|
||||
):
|
||||
return "runtime health is malformed"
|
||||
observed_at = document.get("observed_at")
|
||||
if not isinstance(observed_at, str):
|
||||
return "runtime health is malformed"
|
||||
try:
|
||||
observed = datetime.fromisoformat(observed_at.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return "runtime health is malformed"
|
||||
if observed.tzinfo is None:
|
||||
return "runtime health is malformed"
|
||||
current = now or datetime.now(timezone.utc)
|
||||
age = (
|
||||
current.astimezone(timezone.utc) - observed.astimezone(timezone.utc)
|
||||
).total_seconds()
|
||||
if age < -5.0 or age > maximum_age_seconds:
|
||||
return "runtime health is stale"
|
||||
return None
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Provide the non-mutating Kubernetes readiness command."""
|
||||
maximum_age = float(os.environ.get("HERMES_CLI_HEALTH_MAX_AGE_SECONDS", "60"))
|
||||
issue = readiness_issue(maximum_age_seconds=maximum_age)
|
||||
if issue:
|
||||
print(issue, file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
|
||||
@ -11,6 +11,8 @@ from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
import cli_lane_goal
|
||||
|
||||
|
||||
@ -31,6 +33,12 @@ DEFAULT_MAX_RUNTIME = 12 * 60 * 60
|
||||
HEARTBEAT_SECONDS = 20
|
||||
PROVIDER_HEALTH_MAX_AGE_SECONDS = 5 * 60
|
||||
PROVIDER_AUTH_FAILURE_MAX_AGE_SECONDS = 12 * 60 * 60
|
||||
QUOTA_MIN_REMAINING_PERCENT_DEFAULT = 15.0
|
||||
CAPACITY_COOLDOWN_SECONDS_DEFAULT = 300.0
|
||||
AUTH_COOLDOWN_SECONDS_DEFAULT = 3600.0
|
||||
QUOTA_METRICS_URL = os.environ.get(
|
||||
"HERMES_CLI_QUOTA_METRICS_URL", "http://127.0.0.1:9010/metrics"
|
||||
)
|
||||
KANBAN_STORAGE_ATTEMPTS = 5
|
||||
ARTIFACT_GC_INTERVAL_SECONDS = 5 * 60
|
||||
ARTIFACT_RETENTION_AGE_SECONDS = 30 * 24 * 60 * 60
|
||||
@ -47,7 +55,8 @@ WORKTREE_LOCK = threading.Lock()
|
||||
BOARD_CORRUPTION_ERRORS: dict[str, str] = {}
|
||||
LAST_ARTIFACT_GC = 0.0
|
||||
CAPACITY_PATTERN = re.compile(
|
||||
r"(?:rate.?limit|capacity|overload|usage.?limit|quota|credit|exhaust|429|529|authentication|oauth|token.*expired)",
|
||||
r"(?:rate.?limit|capacity|overload|usage.?limit|quota|credit|exhaust|429|529"
|
||||
r"|authentication|unauthorized|forbidden|oauth|token.*expired|401|403)",
|
||||
re.I,
|
||||
)
|
||||
NO_CLAUDE_SESSION = "No conversation found with session ID:"
|
||||
@ -194,3 +203,18 @@ class TerminalRecoverySnapshot:
|
||||
|
||||
def utc_now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def kanban_setting(name: str, default: float) -> float:
|
||||
"""Read one numeric routing setting from the deployed kanban config block."""
|
||||
try:
|
||||
document = yaml.safe_load(
|
||||
(DATA_ROOT / "config.yaml").read_text(encoding="utf-8")
|
||||
)
|
||||
except (OSError, yaml.YAMLError):
|
||||
return default
|
||||
kanban = document.get("kanban") if isinstance(document, dict) else None
|
||||
value = kanban.get(name) if isinstance(kanban, dict) else None
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
||||
return default
|
||||
return float(value)
|
||||
|
||||
@ -11,7 +11,11 @@ from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from cli_lane_board import _external, _record_board_access_error, _task_value
|
||||
from cli_lane_capabilities import initialize_kanban_capabilities, kanban_capabilities
|
||||
from cli_lane_capabilities import (
|
||||
initialize_kanban_capabilities,
|
||||
kanban_capabilities,
|
||||
refresh_kanban_capabilities,
|
||||
)
|
||||
from cli_lane_config import (
|
||||
BOARD_CORRUPTION_ERRORS,
|
||||
DEFAULT_CLAIM_TTL,
|
||||
@ -21,25 +25,17 @@ from cli_lane_config import (
|
||||
)
|
||||
from cli_lane_execution import execute_claim
|
||||
from cli_lane_files import atomic_json
|
||||
from cli_lane_metrics import start_metrics_server
|
||||
from cli_lane_recovery import _has_pending_finalization, recover_pending_finalizations
|
||||
from cli_lane_retention import maybe_gc_lane_artifacts
|
||||
|
||||
|
||||
OWNED_WORKSPACES_ONLY = os.environ.get(
|
||||
"HERMES_CLI_LANE_OWNED_WORKSPACES_ONLY", ""
|
||||
).strip().lower() in {"1", "true", "yes", "on"}
|
||||
|
||||
|
||||
def _owns_local_workspace(task: Any) -> bool:
|
||||
"""Keep legacy/dirty worktree tasks on their existing single-host owner."""
|
||||
return bool(str(_task_value(task, "workspace_path", "") or "").strip())
|
||||
|
||||
|
||||
def _board_slug(board: Any) -> str:
|
||||
if isinstance(board, dict):
|
||||
return str(board.get("slug") or board.get("id") or "")
|
||||
return str(getattr(board, "slug", None) or getattr(board, "id", None) or board)
|
||||
|
||||
|
||||
def _connect_healthy_board(kanban_db: Any, board: str) -> Any | None:
|
||||
"""Open one board without letting localized storage faults stop other lanes."""
|
||||
try:
|
||||
@ -48,6 +44,7 @@ def _connect_healthy_board(kanban_db: Any, board: str) -> Any | None:
|
||||
_record_board_access_error(board, error)
|
||||
return None
|
||||
|
||||
|
||||
def recover_orphans() -> None:
|
||||
"""Return external running tasks to ready after a runner/pod restart."""
|
||||
from hermes_cli import kanban_db
|
||||
@ -73,7 +70,6 @@ def recover_orphans() -> None:
|
||||
if (
|
||||
_external(task)
|
||||
and str(_task_value(task, "status", "")) == "running"
|
||||
and (not OWNED_WORKSPACES_ONLY or _owns_local_workspace(task))
|
||||
):
|
||||
task_id = str(_task_value(task, "id"))
|
||||
run_id = _task_value(task, "current_run_id", None)
|
||||
@ -93,6 +89,7 @@ def recover_orphans() -> None:
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def claim_ready(
|
||||
active: set[tuple[str, str]],
|
||||
limit: int,
|
||||
@ -132,7 +129,6 @@ def claim_ready(
|
||||
or (board, task_id) in active
|
||||
or not assignee.startswith(EXTERNAL_PREFIX)
|
||||
or str(_task_value(task, "status", "")) != "ready"
|
||||
or (OWNED_WORKSPACES_ONLY and not _owns_local_workspace(task))
|
||||
or (eligible is not None and not eligible(board, task))
|
||||
):
|
||||
continue
|
||||
@ -155,24 +151,38 @@ def claim_ready(
|
||||
conn.close()
|
||||
return claimed
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Continuously bridge external Kanban lanes to provider CLIs."""
|
||||
"""Continuously bridge external Kanban lanes to provider CLIs.
|
||||
|
||||
The loop is exited only by process signals or unrecoverable exceptions;
|
||||
every per-board and per-claim fault is bounded inside one pass.
|
||||
"""
|
||||
from hermes_cli import kanban_db
|
||||
|
||||
RESULT_SCHEMA_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
atomic_json(RESULT_SCHEMA_PATH, RESULT_SCHEMA, 0o644)
|
||||
start_metrics_server()
|
||||
capabilities = initialize_kanban_capabilities(kanban_db)
|
||||
recover_orphans()
|
||||
workers = max(1, min(int(os.environ.get("HERMES_CLI_LANE_CONCURRENCY", "4")), 8))
|
||||
futures: dict[concurrent.futures.Future[None], tuple[str, str]] = {}
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as pool:
|
||||
while True:
|
||||
previous_capabilities = capabilities
|
||||
capabilities = refresh_kanban_capabilities(kanban_db)
|
||||
if capabilities.ready and not previous_capabilities.ready:
|
||||
recover_orphans()
|
||||
for future in list(futures):
|
||||
if future.done():
|
||||
try:
|
||||
future.result()
|
||||
except Exception as error:
|
||||
print(f"worker future failed: {error}", file=sys.stderr, flush=True)
|
||||
print(
|
||||
f"worker future failed: {error}",
|
||||
file=sys.stderr,
|
||||
flush=True,
|
||||
)
|
||||
del futures[future]
|
||||
recover_pending_finalizations()
|
||||
maybe_gc_lane_artifacts()
|
||||
@ -191,4 +201,3 @@ def main() -> int:
|
||||
future = pool.submit(execute_claim, board, task_id)
|
||||
futures[future] = (board, task_id)
|
||||
time.sleep(5)
|
||||
return 0
|
||||
|
||||
@ -15,6 +15,7 @@ from cli_lane_config import (
|
||||
DEFAULT_MAX_RUNTIME,
|
||||
TerminalFinalizationPending,
|
||||
)
|
||||
from cli_lane_failover import _routed_or_blocked, capacity_failover
|
||||
from cli_lane_files import (
|
||||
_terminal_identity,
|
||||
_terminal_path,
|
||||
@ -23,19 +24,16 @@ from cli_lane_files import (
|
||||
state_path,
|
||||
)
|
||||
from cli_lane_finalization import _finalize_terminal_record, _recover_exact_run
|
||||
from cli_lane_health import record_provider_success
|
||||
from cli_lane_metrics import record_route_decision
|
||||
from cli_lane_prompt import build_prompt, git_handoff, workspace_artifacts
|
||||
from cli_lane_provider import run_provider
|
||||
from cli_lane_quota import selection_constraint
|
||||
from cli_lane_records import _persist_candidate, _write_terminal_record
|
||||
from cli_lane_recovery import _has_pending_finalization
|
||||
from cli_lane_routing import fresh_unavailable_provider, select_route
|
||||
|
||||
|
||||
def _goal_rejections(state: dict[str, object]) -> list[object]:
|
||||
"""Return only bounded list-shaped prior judge evidence."""
|
||||
history = state.get("goal_rejections", [])
|
||||
return history if isinstance(history, list) else []
|
||||
|
||||
|
||||
def execute_claim(board: str, task_id: str) -> None:
|
||||
"""Execute one already-claimed task and commit its outcome to Kanban."""
|
||||
from hermes_cli import kanban_db
|
||||
@ -121,21 +119,30 @@ def execute_claim(board: str, task_id: str) -> None:
|
||||
|
||||
try:
|
||||
previous_route = state.get("current_route")
|
||||
excluded_provider = (
|
||||
fresh_unavailable_provider() if assignee == "cli-auto" else None
|
||||
constraint = selection_constraint(assignee)
|
||||
unavailable_provider = (
|
||||
constraint.exclude_provider if constraint.source == "health" else None
|
||||
)
|
||||
unavailable_provider = excluded_provider
|
||||
route = select_route(
|
||||
context,
|
||||
assignee,
|
||||
exclude_provider=excluded_provider,
|
||||
exclude_reason="is unavailable according to fresh native health"
|
||||
if excluded_provider
|
||||
else None,
|
||||
route = _routed_or_blocked(
|
||||
kanban_db,
|
||||
board,
|
||||
task_id,
|
||||
run_id,
|
||||
lambda: select_route(
|
||||
context,
|
||||
assignee,
|
||||
exclude_provider=constraint.exclude_provider,
|
||||
exclude_reason=constraint.exclude_reason,
|
||||
),
|
||||
)
|
||||
if excluded_provider:
|
||||
if route is None:
|
||||
return
|
||||
for note in constraint.notes:
|
||||
comment(note)
|
||||
if constraint.exclude_provider:
|
||||
comment(
|
||||
f"Provider health guard excluded {excluded_provider} before automatic routing.",
|
||||
f"Provider routing guard ({constraint.source}) excluded "
|
||||
f"{constraint.exclude_provider} before automatic routing.",
|
||||
)
|
||||
comment(
|
||||
f"CLI route: {route.provider}/{route.model} at {route.effort}; classifier={route.classifier}; {route.reason}",
|
||||
@ -187,44 +194,34 @@ def execute_claim(board: str, task_id: str) -> None:
|
||||
goal_turn=goal_turn,
|
||||
)
|
||||
if result.capacity_failure:
|
||||
unavailable_provider = route.provider
|
||||
retry_context = (
|
||||
context
|
||||
+ "\n\nRouting boundary: the first provider failed from capacity/authentication. "
|
||||
+ "Select the alternate hosted provider at an appropriate effort."
|
||||
boundary = capacity_failover(
|
||||
kanban_db,
|
||||
board=board,
|
||||
task_id=task_id,
|
||||
run_id=run_id,
|
||||
assignee=assignee,
|
||||
comment=comment,
|
||||
context=context,
|
||||
workspace=workspace,
|
||||
state=state,
|
||||
state_file=state_file,
|
||||
log_path=log_path,
|
||||
heartbeat=heartbeat,
|
||||
deadline=deadline,
|
||||
route=route,
|
||||
result=result,
|
||||
candidate_file=candidate_file,
|
||||
goal_turn=goal_turn,
|
||||
)
|
||||
alternate = "claude" if route.provider == "codex" else "codex"
|
||||
fallback = select_route(
|
||||
retry_context,
|
||||
f"cli-{alternate}-{route.effort}",
|
||||
)
|
||||
comment(
|
||||
f"Provider fallback: {route.provider} -> {fallback.provider}; Jetson reclassified the retry boundary.",
|
||||
)
|
||||
result = run_provider(
|
||||
fallback,
|
||||
build_prompt(
|
||||
context,
|
||||
workspace,
|
||||
git_handoff(workspace, result.output),
|
||||
),
|
||||
workspace,
|
||||
state,
|
||||
state_file,
|
||||
log_path,
|
||||
heartbeat,
|
||||
max(1, int(deadline - time.monotonic())),
|
||||
)
|
||||
route = fallback
|
||||
if result.structured:
|
||||
candidate_file = _persist_candidate(
|
||||
state,
|
||||
state_file,
|
||||
dict(result.structured),
|
||||
route=route,
|
||||
returncode=result.returncode,
|
||||
goal_turn=goal_turn,
|
||||
)
|
||||
route = boundary.route
|
||||
result = boundary.result
|
||||
candidate_file = boundary.candidate_file
|
||||
if boundary.unavailable_provider is not None:
|
||||
unavailable_provider = boundary.unavailable_provider
|
||||
if boundary.blocked:
|
||||
break
|
||||
if result.returncode == 0:
|
||||
record_provider_success(route.provider)
|
||||
structured = result.structured
|
||||
if structured:
|
||||
structured = dict(structured)
|
||||
@ -259,19 +256,36 @@ def execute_claim(board: str, task_id: str) -> None:
|
||||
|
||||
completion_problem = None
|
||||
if structured and result.returncode == 0:
|
||||
completion_problem = cli_lane_goal.unfinished_result_reason(structured)
|
||||
# Resolve the role from the card itself, before any
|
||||
# controller evidence is appended, and hand it to the gate
|
||||
# so a review is judged on its verdict rather than on prose
|
||||
# about the artifact it reviewed.
|
||||
task_role, role_source = cli_lane_goal.task_role(
|
||||
context,
|
||||
structured,
|
||||
)
|
||||
metadata["task_role"] = task_role
|
||||
metadata["task_role_source"] = role_source
|
||||
completion_problem = cli_lane_goal.unfinished_result_reason(
|
||||
structured,
|
||||
role=task_role,
|
||||
)
|
||||
if (
|
||||
structured.get("status") == "completed"
|
||||
and completion_problem is None
|
||||
and goal_mode
|
||||
):
|
||||
heartbeat("local goal-completion judge active")
|
||||
rejection_history = _goal_rejections(state)
|
||||
judge_context = context + (
|
||||
"\n\nAuthoritative Hermes goal-controller evidence: "
|
||||
f"current turn {goal_turn}/{goal_max_turns}; prior rejected "
|
||||
f"reports: {json.dumps(rejection_history[-5:])}."
|
||||
)
|
||||
rejection_history = state.get("goal_rejections", [])
|
||||
if not isinstance(rejection_history, list):
|
||||
rejection_history = []
|
||||
judge_context = context
|
||||
if goal_turn > 1 or rejection_history:
|
||||
judge_context += (
|
||||
f"\n\n{cli_lane_goal.CONTROLLER_EVIDENCE_HEADING}\n"
|
||||
f"current turn {goal_turn}/{goal_max_turns}; prior rejected "
|
||||
f"reports: {json.dumps(rejection_history[-5:])}."
|
||||
)
|
||||
accepted, judge_reason = cli_lane_goal.judge_goal_completion(
|
||||
judge_context,
|
||||
structured,
|
||||
@ -331,6 +345,9 @@ def execute_claim(board: str, task_id: str) -> None:
|
||||
"accepted worker result is durably journaled but "
|
||||
f"Kanban finalization is {outcome}"
|
||||
)
|
||||
record_route_decision(
|
||||
route.provider, route.effort, route.classifier, "completed"
|
||||
)
|
||||
break
|
||||
|
||||
can_continue = (
|
||||
@ -340,7 +357,9 @@ def execute_claim(board: str, task_id: str) -> None:
|
||||
and deadline - time.monotonic() > 30
|
||||
)
|
||||
if can_continue:
|
||||
rejection_history = _goal_rejections(state)
|
||||
rejection_history = state.get("goal_rejections", [])
|
||||
if not isinstance(rejection_history, list):
|
||||
rejection_history = []
|
||||
state["goal_rejections"] = [
|
||||
*rejection_history[-4:],
|
||||
completion_problem,
|
||||
@ -358,14 +377,25 @@ def execute_claim(board: str, task_id: str) -> None:
|
||||
excluded = unavailable_provider
|
||||
if excluded is None and assignee == "cli-auto":
|
||||
excluded = fresh_unavailable_provider()
|
||||
next_route = select_route(
|
||||
escalation_context,
|
||||
assignee,
|
||||
exclude_provider=excluded,
|
||||
exclude_reason="is unavailable according to fresh native health"
|
||||
if excluded
|
||||
else None,
|
||||
record_route_decision(
|
||||
route.provider, route.effort, route.classifier, "goal-continued"
|
||||
)
|
||||
next_route = _routed_or_blocked(
|
||||
kanban_db,
|
||||
board,
|
||||
task_id,
|
||||
run_id,
|
||||
lambda boundary=escalation_context, blocked=excluded: select_route(
|
||||
boundary,
|
||||
assignee,
|
||||
exclude_provider=blocked,
|
||||
exclude_reason="is unavailable according to fresh native health"
|
||||
if blocked
|
||||
else None,
|
||||
),
|
||||
)
|
||||
if next_route is None:
|
||||
break
|
||||
comment(
|
||||
f"Goal route {goal_turn}/{goal_max_turns}: "
|
||||
f"{next_route.provider}/{next_route.model} at {next_route.effort}; "
|
||||
@ -400,6 +430,9 @@ def execute_claim(board: str, task_id: str) -> None:
|
||||
failure_kind = (
|
||||
"transient" if result.capacity_failure else "capability"
|
||||
)
|
||||
record_route_decision(
|
||||
route.provider, route.effort, route.classifier, f"blocked-{failure_kind}"
|
||||
)
|
||||
_board_call(
|
||||
kanban_db,
|
||||
board,
|
||||
|
||||
181
services/hermes/scripts/cli_lane_failover.py
Normal file
181
services/hermes/scripts/cli_lane_failover.py
Normal file
@ -0,0 +1,181 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Bounded, fail-closed cross-provider failover for one capacity boundary."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable
|
||||
|
||||
from cli_lane_board import _board_call
|
||||
from cli_lane_config import ProcessResult, Route
|
||||
from cli_lane_health import classify_capacity_failure, record_provider_failure
|
||||
from cli_lane_metrics import (
|
||||
record_provider_fallback,
|
||||
record_route_decision,
|
||||
record_router_selection_failure,
|
||||
)
|
||||
from cli_lane_prompt import build_prompt, git_handoff
|
||||
from cli_lane_provider import run_provider
|
||||
from cli_lane_records import _persist_candidate
|
||||
from cli_lane_routing import select_route
|
||||
|
||||
|
||||
@dataclass
|
||||
class FailoverOutcome:
|
||||
"""Where one capacity boundary left the claim's route and result."""
|
||||
|
||||
route: Route
|
||||
result: ProcessResult
|
||||
candidate_file: Path | None
|
||||
unavailable_provider: str | None
|
||||
blocked: bool
|
||||
|
||||
|
||||
def _block_transient(
|
||||
kanban_db: Any, board: str, task_id: str, run_id: Any, reason: str
|
||||
) -> None:
|
||||
"""Park one card as retryable so it re-enters when providers recover."""
|
||||
_board_call(
|
||||
kanban_db,
|
||||
board,
|
||||
lambda fresh: kanban_db.block_task(
|
||||
fresh,
|
||||
task_id,
|
||||
reason=reason,
|
||||
kind="transient",
|
||||
expected_run_id=run_id,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _routed_or_blocked(
|
||||
kanban_db: Any,
|
||||
board: str,
|
||||
task_id: str,
|
||||
run_id: Any,
|
||||
select: Callable[[], Route],
|
||||
) -> Route | None:
|
||||
"""Turn a Switchyard outage into a transient block, never a capability one."""
|
||||
try:
|
||||
return select()
|
||||
except RuntimeError as error:
|
||||
record_router_selection_failure()
|
||||
_block_transient(
|
||||
kanban_db,
|
||||
board,
|
||||
task_id,
|
||||
run_id,
|
||||
"Switchyard route selection is unavailable; the card retries "
|
||||
f"when the router recovers: {error}",
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def capacity_failover(
|
||||
kanban_db: Any,
|
||||
*,
|
||||
board: str,
|
||||
task_id: str,
|
||||
run_id: Any,
|
||||
assignee: str,
|
||||
comment: Callable[[str], None],
|
||||
context: str,
|
||||
workspace: Path,
|
||||
state: dict[str, Any],
|
||||
state_file: Path,
|
||||
log_path: Path,
|
||||
heartbeat: Callable[[str], bool],
|
||||
deadline: float,
|
||||
route: Route,
|
||||
result: ProcessResult,
|
||||
candidate_file: Path | None,
|
||||
goal_turn: int,
|
||||
) -> FailoverOutcome:
|
||||
"""Apply the capacity policy once: record, fail closed, or fail over."""
|
||||
failure_class = classify_capacity_failure(result.output)
|
||||
record_provider_failure(route.provider, failure_class)
|
||||
record_route_decision(
|
||||
route.provider, route.effort, route.classifier, "capacity-failure"
|
||||
)
|
||||
if assignee != "cli-auto":
|
||||
# Fail closed: an operator pinned this provider, so a capacity or
|
||||
# auth failure must wait for that provider, never switch away.
|
||||
_block_transient(
|
||||
kanban_db,
|
||||
board,
|
||||
task_id,
|
||||
run_id,
|
||||
f"Manually pinned provider {route.provider} hit a "
|
||||
f"{failure_class} failure; the explicit {assignee} lane never "
|
||||
"switches providers. Retry after recovery or reassign to cli-auto.",
|
||||
)
|
||||
return FailoverOutcome(route, result, candidate_file, None, True)
|
||||
retry_context = (
|
||||
context
|
||||
+ "\n\nRouting boundary: the first provider failed from capacity/authentication. "
|
||||
+ "Select the alternate hosted provider at an appropriate effort."
|
||||
)
|
||||
alternate = "claude" if route.provider == "codex" else "codex"
|
||||
fallback = _routed_or_blocked(
|
||||
kanban_db,
|
||||
board,
|
||||
task_id,
|
||||
run_id,
|
||||
lambda: select_route(retry_context, f"cli-{alternate}-{route.effort}"),
|
||||
)
|
||||
if fallback is None:
|
||||
return FailoverOutcome(route, result, candidate_file, route.provider, True)
|
||||
record_provider_fallback(route.provider, fallback.provider, failure_class)
|
||||
comment(
|
||||
f"Provider fallback: {route.provider} -> {fallback.provider} "
|
||||
f"after a {failure_class} failure; Jetson reclassified the retry boundary.",
|
||||
)
|
||||
fallback_result = run_provider(
|
||||
fallback,
|
||||
build_prompt(
|
||||
context,
|
||||
workspace,
|
||||
git_handoff(workspace, result.output),
|
||||
),
|
||||
workspace,
|
||||
state,
|
||||
state_file,
|
||||
log_path,
|
||||
heartbeat,
|
||||
max(1, int(deadline - time.monotonic())),
|
||||
)
|
||||
if fallback_result.capacity_failure:
|
||||
# Bounded failover depth: two hosted providers, never a third
|
||||
# metered or local path.
|
||||
fallback_class = classify_capacity_failure(fallback_result.output)
|
||||
record_provider_failure(fallback.provider, fallback_class)
|
||||
record_route_decision(
|
||||
fallback.provider, fallback.effort, fallback.classifier, "capacity-failure"
|
||||
)
|
||||
_block_transient(
|
||||
kanban_db,
|
||||
board,
|
||||
task_id,
|
||||
run_id,
|
||||
"Both hosted providers failed at this boundary "
|
||||
f"({route.provider}: {failure_class}; {fallback.provider}: "
|
||||
f"{fallback_class}); the lane never falls to a metered or "
|
||||
"local path. Waiting for provider recovery.",
|
||||
)
|
||||
return FailoverOutcome(
|
||||
fallback, fallback_result, candidate_file, route.provider, True
|
||||
)
|
||||
if fallback_result.structured:
|
||||
candidate_file = _persist_candidate(
|
||||
state,
|
||||
state_file,
|
||||
dict(fallback_result.structured),
|
||||
route=fallback,
|
||||
returncode=fallback_result.returncode,
|
||||
goal_turn=goal_turn,
|
||||
)
|
||||
return FailoverOutcome(
|
||||
fallback, fallback_result, candidate_file, route.provider, False
|
||||
)
|
||||
@ -1,5 +1,49 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fail-closed completion checks for durable external Kanban workers."""
|
||||
"""Role-aware completion checks for durable external Kanban workers.
|
||||
|
||||
Completion is judged against the action the card assigned, not against the
|
||||
health of whatever the card asked the worker to look at.
|
||||
|
||||
* Implementation cards keep the fail-closed local goal judge: a report is
|
||||
accepted only when the model agrees every explicit requirement finished.
|
||||
* Review, audit, and diagnostic cards deliver a *verdict*. A completed report
|
||||
carrying a truthful SHIP or BLOCK verdict with evidence finalizes the card
|
||||
even when the verdict says the reviewed artifact must not ship. Findings are
|
||||
the deliverable, not a task blocker, so the card is never resumed with an
|
||||
instruction to repair an implementation the reviewer may not touch.
|
||||
|
||||
Three deliberate boundaries keep that regime honest. First, ``role`` belongs to
|
||||
the caller that holds the card: ``unfinished_result_reason`` cannot classify a
|
||||
card it was never handed, so every caller holding the objective MUST pass
|
||||
``role``; the role-blind default is the replay contract below, never a weaker
|
||||
classifier that could short-circuit the role-aware gate.
|
||||
|
||||
Second, the verdict contract binds in single-shot mode exactly as in the goal
|
||||
loop: a review card's deliverable *is* its verdict, so the lane fails closed
|
||||
rather than record a verdictless review as done. Two single-shot outcomes change
|
||||
against main d8f2d818, deliberately - a completed review whose prose calls the
|
||||
*reviewed* artifact pending now completes instead of being discarded, and one
|
||||
carrying no usable verdict now blocks instead of completing.
|
||||
|
||||
Third, a journalled terminal record stays valid across an upgrade. PR15 re-runs
|
||||
this gate on replay without a card, so ``role=None`` keeps the unfinished-work
|
||||
heuristic - that integrity check still works - and skips it only for a report
|
||||
that satisfies the whole review contract and changed no files. That is precisely
|
||||
the set the role-aware lane accepts as a review, so a record accepted under one
|
||||
version re-validates under every other instead of being quarantined into a
|
||||
re-dispatch of an already-accepted task.
|
||||
|
||||
The task role comes from explicit card metadata (``Hermes-Task-Role: review``)
|
||||
whenever the card author supplies it, read from real newlines and from the
|
||||
literal ``\\n`` escapes the board stores in single-line bodies. Pre-contract
|
||||
cards fall back to a narrow inference needing a read-only scope, a requested
|
||||
SHIP/BLOCK verdict, no requested mutation deliverable, and a report that changed
|
||||
no files. A card whose deliverable is a findings list rather than a verdict
|
||||
deliberately stays on the implementation regime: the verdict *is* the review
|
||||
contract's gate, so inferring a review role for a card that never asked for one
|
||||
could only fail closed. A report that changed files never resolves to the review
|
||||
role, even when the card declares it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@ -10,6 +54,11 @@ import urllib.error
|
||||
import urllib.request
|
||||
from typing import Any, Callable
|
||||
|
||||
try: # The lane runs inside the agent image, which owns the canonical redactor.
|
||||
from agent.redact import redact_sensitive_text as _canonical_redact
|
||||
except Exception: # pragma: no cover - only when the agent runtime is absent
|
||||
_canonical_redact = None
|
||||
|
||||
|
||||
GOAL_JUDGE_URL = os.environ.get(
|
||||
"HERMES_GOAL_JUDGE_URL",
|
||||
@ -24,6 +73,46 @@ GOAL_JUDGE_TIMEOUT_SECONDS = max(
|
||||
min(180, int(os.environ.get("HERMES_GOAL_JUDGE_TIMEOUT_SECONDS", "120"))),
|
||||
)
|
||||
RESULT_STATUSES = frozenset({"completed", "blocked", "incomplete"})
|
||||
REVIEW_ROLE = "review"
|
||||
IMPLEMENTATION_ROLE = "implementation"
|
||||
TASK_ROLES = frozenset({REVIEW_ROLE, IMPLEMENTATION_ROLE})
|
||||
ROLE_ALIASES = {
|
||||
"audit": REVIEW_ROLE,
|
||||
"assessment": REVIEW_ROLE,
|
||||
"diagnosis": REVIEW_ROLE,
|
||||
"diagnostic": REVIEW_ROLE,
|
||||
"inspection": REVIEW_ROLE,
|
||||
"review": REVIEW_ROLE,
|
||||
"reviewer": REVIEW_ROLE,
|
||||
"verdict": REVIEW_ROLE,
|
||||
"build": IMPLEMENTATION_ROLE,
|
||||
"change": IMPLEMENTATION_ROLE,
|
||||
"implement": IMPLEMENTATION_ROLE,
|
||||
"implementation": IMPLEMENTATION_ROLE,
|
||||
"repair": IMPLEMENTATION_ROLE,
|
||||
}
|
||||
JUDGE_REASON_LIMIT = 600
|
||||
MIN_REVIEW_SUMMARY_CHARS = 80
|
||||
READ_ONLY_GUARD = (
|
||||
"the assigned action is a read-only review: report the verdict and evidence, "
|
||||
"and do not modify the reviewed implementation"
|
||||
)
|
||||
# The runner appends its own turn/rejection evidence to the judge objective.
|
||||
# It is emitted under this heading so ``card_scope`` drops it with every other
|
||||
# non-card section instead of letting a quoted worker sentence reassign a role.
|
||||
CONTROLLER_EVIDENCE_HEADING = "## Hermes goal-controller evidence"
|
||||
# ``hermes_cli.kanban_db.build_worker_context`` (Hermes runtime 0.18.2) renders
|
||||
# the worker context as H2 sections and emits exactly CARD_SECTIONS followed by
|
||||
# HISTORY_SECTIONS; ``test_hermes_cli_review_context.py`` pins that contract
|
||||
# against the installed runtime. The cut below allow-lists the two card
|
||||
# sections, so an upstream rename fails closed - the card just ends early.
|
||||
CARD_SECTIONS = ("Body", "Attachments")
|
||||
HISTORY_SECTIONS = (
|
||||
"Prior attempts on this task",
|
||||
"Parent task results",
|
||||
"Recent work by",
|
||||
"Comment thread",
|
||||
)
|
||||
GOAL_JUDGE_SCHEMA: dict[str, Any] = {
|
||||
"type": "object",
|
||||
"additionalProperties": False,
|
||||
@ -43,6 +132,73 @@ UNFINISHED_EVIDENCE = re.compile(
|
||||
r"\b(?:tests?|checks?|build|validation|verification|commit|push)\b)",
|
||||
re.IGNORECASE | re.DOTALL,
|
||||
)
|
||||
HISTORY_SECTION = re.compile(
|
||||
r"^##\s+(?!(?:Body|Attachments)\b)"
|
||||
r"|^Authoritative Hermes goal-controller evidence\b",
|
||||
re.IGNORECASE | re.MULTILINE,
|
||||
)
|
||||
# Explicit, machine-readable card metadata. This is the supported contract;
|
||||
# every pattern below it only keeps pre-contract cards working.
|
||||
ROLE_DIRECTIVE = re.compile(
|
||||
r"^[\s>#*\-]*(?:hermes[\s_-]*)?task[\s_-]*role\s*[:=]\s*([A-Za-z_-]+)\s*$",
|
||||
re.IGNORECASE | re.MULTILINE,
|
||||
)
|
||||
OUTPUT_DIRECTIVE = re.compile(
|
||||
r"^[\s>#*\-]*(?:hermes[\s_-]*)?expected[\s_-]*output\s*[:=]\s*([A-Za-z_-]+)\s*$",
|
||||
re.IGNORECASE | re.MULTILINE,
|
||||
)
|
||||
READ_ONLY_SCOPE = re.compile(
|
||||
r"(?:\b(?:do|does|must|should|shall|may|will)\s+not\s+(?:\w+\s+){0,3}?"
|
||||
r"(?:edit|modify|change|alter|patch|rewrite|implement|repair|author)\b"
|
||||
r"|\bdon'?t\s+(?:\w+\s+){0,3}?(?:edit|modify|change|alter|patch)\b"
|
||||
r"|\bwithout\s+(?:editing|modifying|changing|altering|patching)\b"
|
||||
r"|\bno\s+(?:\w+\s+){0,2}?(?:edits?|changes?|modifications?)\b"
|
||||
r"|\bmake\s+no\s+(?:code|file|source)?\s*changes\b"
|
||||
r"|\bpristine\b|\bread[\s-]?only\b)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
# A SHIP/BLOCK-shaped deliverable, not merely a mention of the word "verdict":
|
||||
# cards that audit some other component's verdict machinery must not be read as
|
||||
# owing one themselves.
|
||||
VERDICT_DELIVERABLE = re.compile(
|
||||
r"(?:\bship\b[^.\n]{0,120}?\bblock\b|\bblock\b[^.\n]{0,120}?\bship\b"
|
||||
r"|\bship\s*/\s*not-?ship\b|\bverdict\s+of\b"
|
||||
r"|\b(?:return|record|report|give|provide|deliver|state|produce|emit)\b"
|
||||
r"(?:\s+[-\w']+){0,6}?\s+verdict\b)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
MUTATION_DELIVERABLE = re.compile(
|
||||
r"(?:\bpushed\s+(?:sha|commit|head|branch|revision)\b"
|
||||
r"|\bpush\s+(?:\w+\s+){0,3}?(?:branch|commit|change\w*|fix\w*)\b"
|
||||
r"|\bcommit\s+(?:\w+\s+){0,4}?and\s+push\b|\bamend\b"
|
||||
r"|\bopen\s+(?:a|an|one|the)\s+(?:new\s+)?(?:draft\s+)?(?:pr\b|pull\s+request)"
|
||||
r"|\bdraft\s+pr\s+only\b)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
VERDICT_DIRECTIVE = re.compile(
|
||||
r"\b(?:final\s+|strict\s+|overall\s+)?(?:verdict|recommendation|decision)\s*"
|
||||
r"(?:is\s*)?[:=—-]\s*[*_`\s]*(ship|block)\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
VERDICT_TOKEN = re.compile(r"(?<![A-Za-z])(SHIP|BLOCK)(?![A-Za-z])")
|
||||
CONTROL_CHARACTERS = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]")
|
||||
# Fallback for environments without the agent runtime (repo-only checkouts).
|
||||
SECRET_LIKE = re.compile(
|
||||
r"(?:gh[pousr]_[A-Za-z0-9]{16,}|xox[abprs]-[A-Za-z0-9-]{10,}"
|
||||
r"|sk-[A-Za-z0-9_-]{16,}|AKIA[0-9A-Z]{16}"
|
||||
r"|eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,})"
|
||||
)
|
||||
# Two shapes the canonical redactor deliberately passes through but this lane
|
||||
# handles: ``scheme://user:secret@host`` clone URLs (upstream leaves web
|
||||
# userinfo alone so round-trip workflow URLs keep working) and an opaque
|
||||
# credential named in prose. Both are anchored on surrounding syntax so a 40-hex
|
||||
# git SHA - indistinguishable from a Gitea PAT alone - survives as evidence.
|
||||
URL_USERINFO = re.compile(r"(?<=://)([^/\s:@]{1,64}):([^/\s@]{1,256})@")
|
||||
CREDENTIAL_CONTEXT = re.compile(
|
||||
r"((?:pat|token|secret|password|passwd|credential|api[_-]?key|authorization)"
|
||||
r"[\"'\s:=]+(?:bearer\s+|token\s+)?)([A-Za-z0-9_.\-]{20,})",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _bounded(value: str, limit: int) -> str:
|
||||
@ -55,19 +211,155 @@ def _bounded(value: str, limit: int) -> str:
|
||||
return f"{value[:head]}{marker}{value[-(remaining - head):]}"
|
||||
|
||||
|
||||
def unfinished_result_reason(result: dict[str, Any]) -> str | None:
|
||||
"""Reject self-contradictory completion reports without model inference."""
|
||||
status = str(result.get("status") or "")
|
||||
def redact(value: str) -> str:
|
||||
"""Redact with the runtime's canonical helper plus this lane's own gaps."""
|
||||
text = SECRET_LIKE.sub("[redacted]", value)
|
||||
text = URL_USERINFO.sub(r"\1:[redacted]@", text)
|
||||
text = CREDENTIAL_CONTEXT.sub(r"\1[redacted]", text)
|
||||
return _canonical_redact(text, force=True) if _canonical_redact else text
|
||||
|
||||
|
||||
def sanitize_reason(value: Any, limit: int = JUDGE_REASON_LIMIT) -> str:
|
||||
"""Return one bounded, single-line, secret-free judge reason."""
|
||||
text = redact(str(value or ""))
|
||||
text = " ".join(CONTROL_CHARACTERS.sub(" ", text).split())
|
||||
if not text:
|
||||
return "local judge supplied no reason"
|
||||
return text if len(text) <= limit else f"{text[: limit - 3].rstrip()}..."
|
||||
|
||||
|
||||
def _strings(value: Any) -> list[str]:
|
||||
"""Return the non-empty string entries of a reported evidence list."""
|
||||
if not isinstance(value, list):
|
||||
return []
|
||||
return [str(item).strip() for item in value if str(item).strip()]
|
||||
|
||||
|
||||
def card_scope(objective: str) -> str:
|
||||
"""Return the card the worker was assigned, without appended run history.
|
||||
|
||||
Literal ``\\n`` escapes are expanded first: the board stores most bodies on
|
||||
a single physical line, and every pattern here is line-anchored or depends
|
||||
on word boundaries the escape sequence would otherwise destroy. Expanding
|
||||
before the cut is monotone - it can only move the first history heading
|
||||
earlier - so it cannot smuggle appended history into the card. The
|
||||
normalization is for analysis only; the judge objective is never rewritten.
|
||||
"""
|
||||
text = str(objective or "").replace("\\r\\n", "\n").replace("\\n", "\n")
|
||||
match = HISTORY_SECTION.search(text)
|
||||
return text[: match.start()] if match else text
|
||||
|
||||
|
||||
def _declared_role(objective: str) -> str | None:
|
||||
"""Return the role a card declared through explicit worker metadata."""
|
||||
for pattern in (ROLE_DIRECTIVE, OUTPUT_DIRECTIVE):
|
||||
roles = {
|
||||
ROLE_ALIASES[value]
|
||||
for value in (
|
||||
match.group(1).strip().lower() for match in pattern.finditer(objective)
|
||||
)
|
||||
if value in ROLE_ALIASES
|
||||
}
|
||||
if len(roles) == 1:
|
||||
return roles.pop()
|
||||
if roles:
|
||||
# A card that declares two different roles has no usable contract.
|
||||
# Hold it to the stricter implementation regime rather than letting
|
||||
# the more permissive review path win an ambiguous declaration.
|
||||
return IMPLEMENTATION_ROLE
|
||||
return None
|
||||
|
||||
|
||||
def task_role(objective: str, result: dict[str, Any] | None = None) -> tuple[str, str]:
|
||||
"""Resolve the assigned task role and how that resolution was reached."""
|
||||
text = card_scope(objective)
|
||||
changed = _strings((result or {}).get("changed_files"))
|
||||
declared = _declared_role(text)
|
||||
if declared == REVIEW_ROLE and changed:
|
||||
# Mutation evidence outranks a declaration: a card cannot label itself
|
||||
# read-only and then self-certify a report that edited the tree.
|
||||
return IMPLEMENTATION_ROLE, "conflict"
|
||||
if declared is not None:
|
||||
return declared, "directive"
|
||||
if (
|
||||
not changed
|
||||
and READ_ONLY_SCOPE.search(text)
|
||||
and VERDICT_DELIVERABLE.search(text)
|
||||
and not MUTATION_DELIVERABLE.search(text)
|
||||
):
|
||||
return REVIEW_ROLE, "inferred"
|
||||
return IMPLEMENTATION_ROLE, "default"
|
||||
|
||||
|
||||
def review_verdict(result: dict[str, Any]) -> tuple[str | None, str | None]:
|
||||
"""Return the declared SHIP/BLOCK verdict, or why it cannot be trusted."""
|
||||
declared: set[str] = set()
|
||||
field = result.get("verdict")
|
||||
if field is not None:
|
||||
value = str(field).strip().upper()
|
||||
if value not in {"SHIP", "BLOCK"}:
|
||||
return None, f"review verdict field is not SHIP or BLOCK: {value[:40]!r}"
|
||||
declared.add(value)
|
||||
summary = str(result.get("summary") or "")
|
||||
declared.update(match.upper() for match in VERDICT_DIRECTIVE.findall(summary))
|
||||
if len(declared) > 1:
|
||||
return None, "review report declares both SHIP and BLOCK verdicts"
|
||||
if declared:
|
||||
return declared.pop(), None
|
||||
tokens = {match.upper() for match in VERDICT_TOKEN.findall(summary)}
|
||||
if len(tokens) > 1:
|
||||
return None, "review summary names both SHIP and BLOCK without a verdict line"
|
||||
if tokens:
|
||||
return tokens.pop(), None
|
||||
return None, "review report declares no explicit SHIP or BLOCK verdict"
|
||||
|
||||
|
||||
def review_completion_problem(result: dict[str, Any]) -> str | None:
|
||||
"""Fail closed on a review report that does not carry a usable verdict."""
|
||||
verdict, problem = review_verdict(result)
|
||||
if problem or verdict is None:
|
||||
return problem or "review report declares no explicit SHIP or BLOCK verdict"
|
||||
summary = str(result.get("summary") or "").strip()
|
||||
blockers = result.get("blockers")
|
||||
if len(summary) < MIN_REVIEW_SUMMARY_CHARS:
|
||||
return f"{verdict} verdict carries no reviewable rationale in its summary"
|
||||
findings = _strings(result.get("findings"))
|
||||
if verdict == "BLOCK" and not findings:
|
||||
return "BLOCK verdict reports no findings to justify it"
|
||||
if not findings and not _strings(result.get("tests_run")) and not _strings(
|
||||
result.get("artifacts")
|
||||
):
|
||||
return f"{verdict} verdict reports no findings, tests, or artifacts as evidence"
|
||||
return None
|
||||
|
||||
|
||||
def _completion_problem(result: dict[str, Any], role: str | None) -> str | None:
|
||||
"""Return the unguarded reason this report is not a finished deliverable."""
|
||||
status = str(result.get("status") or "")
|
||||
if status == "incomplete":
|
||||
summary = str(result.get("summary") or "").strip()
|
||||
return summary or "worker explicitly reported incomplete work"
|
||||
if status != "completed":
|
||||
return None
|
||||
blockers = result.get("blockers")
|
||||
if isinstance(blockers, list) and any(str(item).strip() for item in blockers):
|
||||
return "worker reported blockers while claiming completion"
|
||||
if role == REVIEW_ROLE:
|
||||
# A review's prose describes the reviewed artifact, so the unfinished
|
||||
# work heuristic below would read the artifact's state as the review's
|
||||
# own. The verdict contract is the deterministic gate instead.
|
||||
return review_completion_problem(result)
|
||||
if (
|
||||
role is None
|
||||
and not _strings(result.get("changed_files"))
|
||||
and review_completion_problem(result) is None
|
||||
):
|
||||
# A replay caller holds no card. This is exactly the shape the
|
||||
# role-aware lane accepts as a review, so honouring it here keeps a
|
||||
# journalled record valid without weakening the heuristic for anything
|
||||
# that is not already a complete review deliverable.
|
||||
return None
|
||||
tests = result.get("tests_run")
|
||||
evidence = [summary]
|
||||
evidence = [str(result.get("summary") or "").strip()]
|
||||
if isinstance(tests, list):
|
||||
evidence.extend(str(item) for item in tests)
|
||||
match = UNFINISHED_EVIDENCE.search("\n".join(evidence))
|
||||
@ -76,18 +368,34 @@ def unfinished_result_reason(result: dict[str, Any]) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
def judge_goal_completion(
|
||||
objective: str,
|
||||
def unfinished_result_reason(
|
||||
result: dict[str, Any],
|
||||
*,
|
||||
url: str = GOAL_JUDGE_URL,
|
||||
open_request: Callable[..., Any] = urllib.request.urlopen,
|
||||
) -> tuple[bool, str]:
|
||||
"""Use the local model as a fail-closed judge for explicit goal cards."""
|
||||
deterministic_reason = unfinished_result_reason(result)
|
||||
if deterministic_reason:
|
||||
return False, deterministic_reason
|
||||
payload = {
|
||||
role: str | None = None,
|
||||
) -> str | None:
|
||||
"""Reject self-contradictory completion reports without model inference.
|
||||
|
||||
``role`` must be supplied by any caller holding the card; see the module
|
||||
docstring for the role-blind replay contract a caller without one gets.
|
||||
"""
|
||||
problem = _completion_problem(result, role)
|
||||
if problem is None:
|
||||
return None
|
||||
if role == REVIEW_ROLE:
|
||||
# Every reason a reviewer may ever read carries the guard, so a resumed
|
||||
# review is never told to repair the implementation it must not touch.
|
||||
problem = f"{READ_ONLY_GUARD}; {problem}"
|
||||
return sanitize_reason(problem)
|
||||
|
||||
|
||||
def _completion_claimed(result: dict[str, Any]) -> bool:
|
||||
"""Report whether the worker actually claimed a finished task."""
|
||||
return str(result.get("status") or "") == "completed"
|
||||
|
||||
|
||||
def _judge_payload(objective: str, result: dict[str, Any], role: str) -> dict[str, Any]:
|
||||
"""Build the fail-closed local judge request for an implementation card."""
|
||||
return {
|
||||
"model": GOAL_JUDGE_MODEL,
|
||||
"messages": [
|
||||
{
|
||||
@ -98,13 +406,16 @@ def judge_goal_completion(
|
||||
"Return complete only when the report gives concrete evidence that every "
|
||||
"explicit requirement finished. Return continue when any work, test, command, "
|
||||
"commit, push, review, or verification is pending, in progress, omitted, or "
|
||||
"inconclusive. Do not trust the report status field."
|
||||
"inconclusive. Defects the worker reports about some other artifact are "
|
||||
"findings, not unfinished work on this task. Do not trust the report status "
|
||||
"field."
|
||||
),
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": json.dumps(
|
||||
{
|
||||
"task_role": role,
|
||||
"objective": _bounded(objective, 6000),
|
||||
"worker_report": _bounded(
|
||||
json.dumps(result, default=str), 4000
|
||||
@ -126,9 +437,40 @@ def judge_goal_completion(
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def judge_goal_completion(
|
||||
objective: str,
|
||||
result: dict[str, Any],
|
||||
*,
|
||||
url: str = GOAL_JUDGE_URL,
|
||||
open_request: Callable[..., Any] = urllib.request.urlopen,
|
||||
) -> tuple[bool, str]:
|
||||
"""Judge one worker report against the action its card actually assigned."""
|
||||
role, source = task_role(objective, result)
|
||||
reason = unfinished_result_reason(result, role=role)
|
||||
if reason is None and not _completion_claimed(result):
|
||||
reason = (
|
||||
f"worker reported status {str(result.get('status') or 'unknown')!r} "
|
||||
"rather than a finished task"
|
||||
)
|
||||
if role == REVIEW_ROLE:
|
||||
reason = f"{READ_ONLY_GUARD}; {reason}"
|
||||
if reason:
|
||||
return False, sanitize_reason(reason)
|
||||
if role == REVIEW_ROLE:
|
||||
# The requested deliverable is present and self-consistent. A verdict
|
||||
# that the reviewed artifact must not ship is a finished review, so the
|
||||
# card finalizes instead of resuming a worker that may not repair it.
|
||||
verdict, _ = review_verdict(result)
|
||||
findings = len(_strings(result.get("findings")))
|
||||
return True, sanitize_reason(
|
||||
f"review deliverable complete: {verdict} verdict with {findings} finding(s); "
|
||||
f"task role {role} resolved by {source}"
|
||||
)
|
||||
request = urllib.request.Request(
|
||||
url,
|
||||
data=json.dumps(payload).encode("utf-8"),
|
||||
data=json.dumps(_judge_payload(objective, result, role)).encode("utf-8"),
|
||||
headers={"Content-Type": "application/json"},
|
||||
method="POST",
|
||||
)
|
||||
@ -139,7 +481,7 @@ def judge_goal_completion(
|
||||
verdict = json.loads(content)
|
||||
if verdict.get("verdict") not in {"complete", "continue"}:
|
||||
raise ValueError("local judge returned an invalid verdict")
|
||||
reason = str(verdict.get("reason") or "local judge supplied no reason")
|
||||
reason = sanitize_reason(verdict.get("reason"))
|
||||
return verdict.get("verdict") == "complete", reason
|
||||
except (
|
||||
AttributeError,
|
||||
@ -151,4 +493,6 @@ def judge_goal_completion(
|
||||
json.JSONDecodeError,
|
||||
urllib.error.URLError,
|
||||
) as error:
|
||||
return False, f"local completion judge unavailable: {type(error).__name__}: {error}"
|
||||
return False, sanitize_reason(
|
||||
f"local completion judge unavailable: {type(error).__name__}: {error}"
|
||||
)
|
||||
|
||||
102
services/hermes/scripts/cli_lane_health.py
Normal file
102
services/hermes/scripts/cli_lane_health.py
Normal file
@ -0,0 +1,102 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Lane-observed provider health with failure classification and cooldowns."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from cli_lane_config import (
|
||||
AUTH_COOLDOWN_SECONDS_DEFAULT,
|
||||
CAPACITY_COOLDOWN_SECONDS_DEFAULT,
|
||||
PROVIDER_HEALTH_PATHS,
|
||||
kanban_setting,
|
||||
utc_now,
|
||||
)
|
||||
from cli_lane_files import load_json
|
||||
|
||||
|
||||
# Ordered from most to least specific so one output maps to one actionable
|
||||
# reason instead of the single conflated capacity signal.
|
||||
FAILURE_CLASSIFICATIONS = (
|
||||
("auth", re.compile(r"authentication|unauthorized|forbidden|oauth|token.*expired|401|403", re.I)),
|
||||
("rate-limit", re.compile(r"rate.?limit|429|529|overload", re.I)),
|
||||
("quota", re.compile(r"usage.?limit|quota|credit|exhaust|capacity", re.I)),
|
||||
)
|
||||
|
||||
|
||||
def classify_capacity_failure(output: str) -> str:
|
||||
"""Split the broad capacity regex into quota/auth/rate-limit/transport."""
|
||||
for reason, pattern in FAILURE_CLASSIFICATIONS:
|
||||
if pattern.search(output):
|
||||
return reason
|
||||
return "transport"
|
||||
|
||||
|
||||
def _write_provider_health(
|
||||
provider: str, updates: dict[str, Any], clear: tuple[str, ...] = ()
|
||||
) -> None:
|
||||
"""Merge one lane observation into the shared provider health snapshot."""
|
||||
path = PROVIDER_HEALTH_PATHS[provider]
|
||||
value = load_json(path)
|
||||
for key in clear:
|
||||
value.pop(key, None)
|
||||
value.update(updates)
|
||||
try:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp")
|
||||
temporary.write_text(
|
||||
json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8"
|
||||
)
|
||||
os.replace(temporary, path)
|
||||
except OSError:
|
||||
# Health snapshots are advisory; never fail the lane over one write.
|
||||
pass
|
||||
|
||||
|
||||
def record_provider_success(provider: str) -> None:
|
||||
"""Publish fresh evidence of success so the provider re-enters routing."""
|
||||
_write_provider_health(
|
||||
provider,
|
||||
{
|
||||
"source": "cli-lane-runner",
|
||||
"state": "available",
|
||||
"authenticated": True,
|
||||
"checked_at": utc_now(),
|
||||
},
|
||||
clear=("failure_reason", "failed_at", "cooldown_until"),
|
||||
)
|
||||
|
||||
|
||||
def record_provider_failure(
|
||||
provider: str, reason: str, now: float | None = None
|
||||
) -> float:
|
||||
"""Persist one lane-observed failure with its re-admission cooldown."""
|
||||
current = time.time() if now is None else now
|
||||
if reason == "auth":
|
||||
cooldown = kanban_setting(
|
||||
"provider_auth_cooldown_seconds", AUTH_COOLDOWN_SECONDS_DEFAULT
|
||||
)
|
||||
state, authenticated = "unavailable", False
|
||||
else:
|
||||
cooldown = kanban_setting(
|
||||
"provider_capacity_cooldown_seconds", CAPACITY_COOLDOWN_SECONDS_DEFAULT
|
||||
)
|
||||
state, authenticated = "capacity-limited", True
|
||||
cooldown_until = current + max(0.0, cooldown)
|
||||
_write_provider_health(
|
||||
provider,
|
||||
{
|
||||
"source": "cli-lane-runner",
|
||||
"state": state,
|
||||
"authenticated": authenticated,
|
||||
"checked_at": utc_now(),
|
||||
"failure_reason": reason,
|
||||
"failed_at": current,
|
||||
"cooldown_until": cooldown_until,
|
||||
},
|
||||
)
|
||||
return cooldown_until
|
||||
188
services/hermes/scripts/cli_lane_metrics.py
Normal file
188
services/hermes/scripts/cli_lane_metrics.py
Normal file
@ -0,0 +1,188 @@
|
||||
#!/usr/bin/env python3
|
||||
"""In-process Prometheus metrics for the direct CLI lane's routing decisions."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
|
||||
|
||||
METRIC_HELP = {
|
||||
"hermes_cli_route_decisions_total": (
|
||||
"Terminal outcome of each executed CLI lane route decision."
|
||||
),
|
||||
"hermes_cli_provider_fallbacks_total": (
|
||||
"Cross-provider failovers taken by the CLI lane, by failure reason."
|
||||
),
|
||||
"hermes_cli_router_selection_failures_total": (
|
||||
"Switchyard route selections that failed and blocked a card as transient."
|
||||
),
|
||||
"hermes_cli_quota_fetch_failures_total": (
|
||||
"Quota snapshot reads that failed; routing then fails open."
|
||||
),
|
||||
"hermes_cli_quota_remaining_percent": (
|
||||
"Binding remaining-quota percent per provider as seen by the CLI lane."
|
||||
),
|
||||
"hermes_cli_quota_reset_timestamp_seconds": (
|
||||
"Unix reset time of the binding quota window as seen by the CLI lane."
|
||||
),
|
||||
"hermes_cli_provider_soft_excluded": (
|
||||
"Whether new automatic work is currently routed away from a provider."
|
||||
),
|
||||
}
|
||||
COUNTER_NAMES = frozenset(name for name in METRIC_HELP if name.endswith("_total"))
|
||||
|
||||
|
||||
def _escape_label(value: str) -> str:
|
||||
"""Escape one Prometheus label value."""
|
||||
return value.replace("\\", "\\\\").replace("\n", "\\n").replace('"', '\\"')
|
||||
|
||||
|
||||
class LaneMetricsRegistry:
|
||||
"""Thread-safe counter and gauge store rendered in Prometheus text format."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._lock = threading.Lock()
|
||||
self._values: dict[tuple[str, tuple[tuple[str, str], ...]], float] = {}
|
||||
|
||||
def _key(
|
||||
self, name: str, labels: dict[str, str] | None
|
||||
) -> tuple[str, tuple[tuple[str, str], ...]]:
|
||||
return name, tuple(sorted((labels or {}).items()))
|
||||
|
||||
def increment(
|
||||
self, name: str, labels: dict[str, str] | None = None, amount: float = 1.0
|
||||
) -> None:
|
||||
key = self._key(name, labels)
|
||||
with self._lock:
|
||||
self._values[key] = self._values.get(key, 0.0) + amount
|
||||
|
||||
def set_value(
|
||||
self, name: str, labels: dict[str, str] | None, value: float
|
||||
) -> None:
|
||||
with self._lock:
|
||||
self._values[self._key(name, labels)] = value
|
||||
|
||||
def render(self) -> bytes:
|
||||
"""Render every recorded sample with stable ordering."""
|
||||
with self._lock:
|
||||
values = dict(self._values)
|
||||
lines: list[str] = []
|
||||
for name in sorted({name for name, _ in values}):
|
||||
kind = "counter" if name in COUNTER_NAMES else "gauge"
|
||||
lines.extend((f"# HELP {name} {METRIC_HELP[name]}", f"# TYPE {name} {kind}"))
|
||||
for (sample_name, labels), value in sorted(values.items()):
|
||||
if sample_name != name:
|
||||
continue
|
||||
rendered = ",".join(
|
||||
f'{key}="{_escape_label(text)}"' for key, text in labels
|
||||
)
|
||||
body = f"{{{rendered}}}" if rendered else ""
|
||||
lines.append(f"{name}{body} {value:.12g}")
|
||||
return ("\n".join(lines) + "\n").encode("utf-8")
|
||||
|
||||
|
||||
METRICS = LaneMetricsRegistry()
|
||||
|
||||
|
||||
def record_route_decision(
|
||||
provider: str, effort: str, classifier: str, outcome: str
|
||||
) -> None:
|
||||
"""Count one executed route with its terminal outcome."""
|
||||
METRICS.increment(
|
||||
"hermes_cli_route_decisions_total",
|
||||
{
|
||||
"provider": provider,
|
||||
"effort": effort,
|
||||
"classifier": classifier,
|
||||
"outcome": outcome,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def record_provider_fallback(
|
||||
from_provider: str, to_provider: str, reason: str
|
||||
) -> None:
|
||||
"""Count one cross-provider failover with its classified reason."""
|
||||
METRICS.increment(
|
||||
"hermes_cli_provider_fallbacks_total",
|
||||
{
|
||||
"from_provider": from_provider,
|
||||
"to_provider": to_provider,
|
||||
"reason": reason,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def record_router_selection_failure() -> None:
|
||||
"""Count one Switchyard outage observed at a selection boundary."""
|
||||
METRICS.increment("hermes_cli_router_selection_failures_total")
|
||||
|
||||
|
||||
def record_quota_fetch_failure() -> None:
|
||||
"""Count one unavailable quota snapshot; routing continues fail-open."""
|
||||
METRICS.increment("hermes_cli_quota_fetch_failures_total")
|
||||
|
||||
|
||||
def record_provider_quota(
|
||||
provider: str, remaining_percent: float, reset_timestamp: float | None
|
||||
) -> None:
|
||||
"""Publish the binding quota window the lane based its routing on."""
|
||||
labels = {"provider": provider}
|
||||
METRICS.set_value(
|
||||
"hermes_cli_quota_remaining_percent", labels, remaining_percent
|
||||
)
|
||||
if reset_timestamp is not None:
|
||||
METRICS.set_value(
|
||||
"hermes_cli_quota_reset_timestamp_seconds", labels, reset_timestamp
|
||||
)
|
||||
|
||||
|
||||
def record_soft_exclusion(provider: str, excluded: bool) -> None:
|
||||
"""Publish whether new automatic work avoids this provider right now."""
|
||||
METRICS.set_value(
|
||||
"hermes_cli_provider_soft_excluded",
|
||||
{"provider": provider},
|
||||
1.0 if excluded else 0.0,
|
||||
)
|
||||
|
||||
|
||||
def start_metrics_server(port: int | None = None) -> ThreadingHTTPServer | None:
|
||||
"""Serve lane metrics on the pod-local scrape port without blocking dispatch."""
|
||||
selected_port = (
|
||||
int(os.environ.get("HERMES_CLI_LANE_METRICS_PORT", "9011"))
|
||||
if port is None
|
||||
else port
|
||||
)
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def do_GET(self) -> None: # noqa: N802
|
||||
if self.path == "/metrics":
|
||||
payload = METRICS.render()
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "text/plain; version=0.0.4")
|
||||
self.send_header("Content-Length", str(len(payload)))
|
||||
self.end_headers()
|
||||
self.wfile.write(payload)
|
||||
elif self.path == "/healthz":
|
||||
self.send_response(200)
|
||||
self.end_headers()
|
||||
else:
|
||||
self.send_error(404)
|
||||
|
||||
def log_message(self, _format: str, *_args: object) -> None:
|
||||
return
|
||||
|
||||
try:
|
||||
server = ThreadingHTTPServer(("0.0.0.0", selected_port), Handler)
|
||||
except OSError as error:
|
||||
print(
|
||||
f"cli lane metrics server unavailable: {error}",
|
||||
file=sys.stderr,
|
||||
flush=True,
|
||||
)
|
||||
return None
|
||||
threading.Thread(target=server.serve_forever, daemon=True).start()
|
||||
return server
|
||||
@ -197,6 +197,10 @@ def _terminate_worker_process(
|
||||
def _base_env() -> dict[str, str]:
|
||||
"""Build a worker environment while preserving Vault-backed CLI homes."""
|
||||
env = os.environ.copy()
|
||||
# Subscription CLIs only: a vendor API key in the environment could
|
||||
# silently flip a worker onto a metered path, so drop every key here.
|
||||
for secret in ("ANTHROPIC_API_KEY", "CLAUDE_API_KEY", "OPENAI_API_KEY", "API_SERVER_KEY"):
|
||||
env.pop(secret, None)
|
||||
env.update(
|
||||
{
|
||||
"HOME": str(DATA_ROOT / "home"),
|
||||
|
||||
205
services/hermes/scripts/cli_lane_quota.py
Normal file
205
services/hermes/scripts/cli_lane_quota.py
Normal file
@ -0,0 +1,205 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Quota-aware soft routing for automatic CLI lane provider selection."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import sys
|
||||
import urllib.request
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable
|
||||
|
||||
from cli_lane_config import (
|
||||
QUOTA_METRICS_URL,
|
||||
QUOTA_MIN_REMAINING_PERCENT_DEFAULT,
|
||||
kanban_setting,
|
||||
)
|
||||
from cli_lane_metrics import (
|
||||
record_provider_quota,
|
||||
record_quota_fetch_failure,
|
||||
record_soft_exclusion,
|
||||
)
|
||||
from cli_lane_routing import fresh_unavailable_provider
|
||||
|
||||
|
||||
# Exporter accounts are labeled by vendor; the lane routes by CLI provider.
|
||||
QUOTA_PROVIDER_LABELS = {"openai": "codex", "anthropic": "claude"}
|
||||
# Gate only on account-wide windows: a model-specific Claude window (for
|
||||
# example seven_day_opus) must not veto the whole provider.
|
||||
GATED_QUOTA_WINDOWS = {
|
||||
"codex": None,
|
||||
"claude": ("five_hour", "seven_day"),
|
||||
}
|
||||
_SAMPLE_PATTERN = re.compile(
|
||||
r"^atlas_ai_quota_(remaining_percent|reset_timestamp_seconds|fetch_success)"
|
||||
r"\{([^}]*)\} (-?[0-9.eE+]+)$"
|
||||
)
|
||||
_LABEL_PATTERN = re.compile(r'([a-zA-Z_][a-zA-Z0-9_]*)="([^"]*)"')
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ProviderQuota:
|
||||
"""One provider's binding remaining quota as seen by the lane."""
|
||||
|
||||
remaining_percent: float
|
||||
reset_timestamp: float | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SelectionConstraint:
|
||||
"""One provider exclusion decided at a routing boundary, with evidence."""
|
||||
|
||||
exclude_provider: str | None
|
||||
exclude_reason: str | None
|
||||
source: str | None
|
||||
notes: tuple[str, ...]
|
||||
|
||||
|
||||
def _gated_window(provider: str, labels: dict[str, str]) -> str | None:
|
||||
"""Return the window key when a sample belongs to a gated account limit."""
|
||||
if labels.get("limit") != provider:
|
||||
return None
|
||||
window = labels.get("window", "")
|
||||
allowed = GATED_QUOTA_WINDOWS[provider]
|
||||
if allowed is not None and window not in allowed:
|
||||
return None
|
||||
return window
|
||||
|
||||
|
||||
def parse_quota_metrics(text: str) -> dict[str, ProviderQuota]:
|
||||
"""Reduce exporter samples to each provider's tightest account window."""
|
||||
remaining: dict[tuple[str, str], float] = {}
|
||||
resets: dict[tuple[str, str], float] = {}
|
||||
fetch_success: dict[str, bool] = {}
|
||||
for line in text.splitlines():
|
||||
match = _SAMPLE_PATTERN.match(line.strip())
|
||||
if not match:
|
||||
continue
|
||||
metric, raw_labels, raw_value = match.groups()
|
||||
labels = dict(_LABEL_PATTERN.findall(raw_labels))
|
||||
provider = QUOTA_PROVIDER_LABELS.get(labels.get("provider", ""))
|
||||
if provider is None:
|
||||
continue
|
||||
value = float(raw_value)
|
||||
if metric == "fetch_success":
|
||||
fetch_success[provider] = value >= 1.0
|
||||
continue
|
||||
window = _gated_window(provider, labels)
|
||||
if window is None:
|
||||
continue
|
||||
if metric == "remaining_percent":
|
||||
remaining[(provider, window)] = value
|
||||
else:
|
||||
resets[(provider, window)] = value
|
||||
snapshot: dict[str, ProviderQuota] = {}
|
||||
for provider in QUOTA_PROVIDER_LABELS.values():
|
||||
windows = {
|
||||
window: value
|
||||
for (candidate, window), value in remaining.items()
|
||||
if candidate == provider
|
||||
}
|
||||
if not windows or not fetch_success.get(provider, True):
|
||||
# Failed or stale exporter fetches leave this provider ungated.
|
||||
continue
|
||||
binding = min(windows, key=lambda window: (windows[window], window))
|
||||
snapshot[provider] = ProviderQuota(
|
||||
remaining_percent=windows[binding],
|
||||
reset_timestamp=resets.get((provider, binding)),
|
||||
)
|
||||
return snapshot
|
||||
|
||||
|
||||
def fetch_quota_snapshot(
|
||||
url: str = QUOTA_METRICS_URL,
|
||||
open_url: Callable = urllib.request.urlopen,
|
||||
) -> dict[str, ProviderQuota]:
|
||||
"""Read the in-pod usage exporter; empty means the signal is unavailable."""
|
||||
try:
|
||||
with open_url(url, timeout=10) as response:
|
||||
text = response.read(1 << 20).decode("utf-8", errors="replace")
|
||||
snapshot = parse_quota_metrics(text)
|
||||
except (OSError, ValueError) as error:
|
||||
record_quota_fetch_failure()
|
||||
print(
|
||||
f"quota snapshot unavailable; routing fails open: {type(error).__name__}",
|
||||
file=sys.stderr,
|
||||
flush=True,
|
||||
)
|
||||
return {}
|
||||
if not snapshot:
|
||||
record_quota_fetch_failure()
|
||||
print(
|
||||
"quota snapshot carried no gated windows; routing fails open",
|
||||
file=sys.stderr,
|
||||
flush=True,
|
||||
)
|
||||
return snapshot
|
||||
|
||||
|
||||
def quota_soft_exclusion(
|
||||
snapshot: dict[str, ProviderQuota], threshold: float
|
||||
) -> tuple[str | None, str | None]:
|
||||
"""Choose at most one provider to soft-exclude from new automatic work."""
|
||||
below = {
|
||||
provider: quota.remaining_percent
|
||||
for provider, quota in snapshot.items()
|
||||
if quota.remaining_percent < threshold
|
||||
}
|
||||
if not below:
|
||||
return None, None
|
||||
excluded = min(below, key=lambda provider: (below[provider], provider))
|
||||
if len(below) == len(snapshot) and len(snapshot) > 1:
|
||||
preferred = next(name for name in snapshot if name != excluded)
|
||||
note = (
|
||||
"Quota degraded: every provider is below the "
|
||||
f"{threshold:.6g}% remaining threshold; preferring {preferred} "
|
||||
f"({snapshot[preferred].remaining_percent:.6g}% left) over "
|
||||
f"{excluded} ({below[excluded]:.6g}% left) for new auto work."
|
||||
)
|
||||
return excluded, note
|
||||
note = (
|
||||
f"Quota guard: {excluded} has {below[excluded]:.6g}% remaining "
|
||||
f"(threshold {threshold:.6g}%); routing new auto work to the other "
|
||||
"provider while active work finishes."
|
||||
)
|
||||
return excluded, note
|
||||
|
||||
|
||||
def selection_constraint(
|
||||
assignee: str,
|
||||
*,
|
||||
snapshot_from: Callable[[], dict[str, ProviderQuota]] | None = None,
|
||||
now: float | None = None,
|
||||
) -> SelectionConstraint:
|
||||
"""Combine provider health and quota gating for one selection boundary."""
|
||||
if assignee != "cli-auto":
|
||||
# Explicit lanes are operator decisions; they are never re-routed.
|
||||
return SelectionConstraint(None, None, None, ())
|
||||
snapshot = (snapshot_from or fetch_quota_snapshot)()
|
||||
resets: dict[str, float] = {}
|
||||
for provider, quota in snapshot.items():
|
||||
record_provider_quota(provider, quota.remaining_percent, quota.reset_timestamp)
|
||||
if quota.reset_timestamp is not None:
|
||||
resets[provider] = quota.reset_timestamp
|
||||
health_excluded = fresh_unavailable_provider(now=now, quota_resets=resets)
|
||||
if health_excluded is not None:
|
||||
return SelectionConstraint(
|
||||
health_excluded,
|
||||
"is unavailable according to fresh native health",
|
||||
"health",
|
||||
(),
|
||||
)
|
||||
threshold = kanban_setting(
|
||||
"provider_quota_min_remaining_percent", QUOTA_MIN_REMAINING_PERCENT_DEFAULT
|
||||
)
|
||||
excluded, note = quota_soft_exclusion(snapshot, threshold)
|
||||
for provider in snapshot:
|
||||
record_soft_exclusion(provider, provider == excluded)
|
||||
if excluded is None or note is None:
|
||||
return SelectionConstraint(None, None, None, ())
|
||||
return SelectionConstraint(
|
||||
excluded,
|
||||
"is below its remaining-quota routing threshold",
|
||||
"quota",
|
||||
(note,),
|
||||
)
|
||||
@ -31,12 +31,49 @@ def parse_assignee(assignee: str) -> tuple[str | None, str | None]:
|
||||
raise ValueError(f"unsupported external lane assignee: {assignee}")
|
||||
return match.group(1), match.group(2)
|
||||
|
||||
def fresh_unavailable_provider(now: float | None = None) -> str | None:
|
||||
def _health_number(value: Any) -> float | None:
|
||||
"""Return a plain numeric health field, rejecting booleans and strings."""
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
||||
return None
|
||||
return float(value)
|
||||
|
||||
def _quota_reset_readmits(
|
||||
health: dict[str, Any], current: float, reset_timestamp: float | None
|
||||
) -> bool:
|
||||
"""A passed provider quota reset ends a capacity cooldown early."""
|
||||
if health.get("authenticated") is False:
|
||||
# An authentication failure is not repaired by a quota window reset.
|
||||
return False
|
||||
failed_at = _health_number(health.get("failed_at"))
|
||||
return (
|
||||
reset_timestamp is not None
|
||||
and failed_at is not None
|
||||
and failed_at < reset_timestamp <= current
|
||||
)
|
||||
|
||||
def fresh_unavailable_provider(
|
||||
now: float | None = None,
|
||||
quota_resets: dict[str, float] | None = None,
|
||||
) -> str | None:
|
||||
"""Return one recently proven-down provider for automatic route exclusion."""
|
||||
current = time.time() if now is None else now
|
||||
resets = quota_resets or {}
|
||||
unavailable: list[str] = []
|
||||
for provider, path in PROVIDER_HEALTH_PATHS.items():
|
||||
health = load_json(path)
|
||||
if health.get("state") not in ("unavailable", "capacity-limited"):
|
||||
continue
|
||||
cooldown_until = _health_number(health.get("cooldown_until"))
|
||||
if cooldown_until is not None:
|
||||
# Lane-recorded failures re-enter only after the full cooldown has
|
||||
# elapsed (hysteresis), a quota reset passed, or fresh success
|
||||
# rewrote the state above.
|
||||
if current >= cooldown_until or _quota_reset_readmits(
|
||||
health, current, resets.get(provider)
|
||||
):
|
||||
continue
|
||||
unavailable.append(provider)
|
||||
continue
|
||||
try:
|
||||
age = current - path.stat().st_mtime
|
||||
except OSError:
|
||||
@ -46,10 +83,7 @@ def fresh_unavailable_provider(now: float | None = None) -> str | None:
|
||||
if health.get("authenticated") is False
|
||||
else PROVIDER_HEALTH_MAX_AGE_SECONDS
|
||||
)
|
||||
if (
|
||||
0 <= age <= max_age
|
||||
and health.get("state") == "unavailable"
|
||||
):
|
||||
if 0 <= age <= max_age:
|
||||
unavailable.append(provider)
|
||||
# If both providers are down, keep the normal attempt/fallback path so the
|
||||
# card records authoritative current errors instead of trusting snapshots.
|
||||
|
||||
@ -18,6 +18,8 @@ from cli_lane_capabilities import (
|
||||
detect_kanban_capabilities,
|
||||
initialize_kanban_capabilities,
|
||||
kanban_capabilities,
|
||||
readiness_issue,
|
||||
refresh_kanban_capabilities,
|
||||
runtime_health,
|
||||
)
|
||||
from cli_lane_board import (
|
||||
@ -30,6 +32,11 @@ from cli_lane_board import (
|
||||
)
|
||||
from cli_lane_config import (
|
||||
ARTIFACT_GC_INTERVAL_SECONDS,
|
||||
AUTH_COOLDOWN_SECONDS_DEFAULT,
|
||||
CAPACITY_COOLDOWN_SECONDS_DEFAULT,
|
||||
QUOTA_METRICS_URL,
|
||||
QUOTA_MIN_REMAINING_PERCENT_DEFAULT,
|
||||
kanban_setting,
|
||||
ARTIFACT_RETENTION_AGE_SECONDS,
|
||||
ARTIFACT_RETENTION_BYTES,
|
||||
ARTIFACT_RETENTION_COUNT,
|
||||
@ -65,6 +72,35 @@ from cli_lane_evidence import (
|
||||
_write_json_noreplace,
|
||||
)
|
||||
from cli_lane_execution import execute_claim
|
||||
from cli_lane_failover import (
|
||||
FailoverOutcome,
|
||||
_block_transient,
|
||||
_routed_or_blocked,
|
||||
capacity_failover,
|
||||
)
|
||||
from cli_lane_health import (
|
||||
classify_capacity_failure,
|
||||
record_provider_failure,
|
||||
record_provider_success,
|
||||
)
|
||||
from cli_lane_metrics import (
|
||||
METRICS,
|
||||
record_provider_fallback,
|
||||
record_provider_quota,
|
||||
record_quota_fetch_failure,
|
||||
record_route_decision,
|
||||
record_router_selection_failure,
|
||||
record_soft_exclusion,
|
||||
start_metrics_server,
|
||||
)
|
||||
from cli_lane_quota import (
|
||||
ProviderQuota,
|
||||
SelectionConstraint,
|
||||
fetch_quota_snapshot,
|
||||
parse_quota_metrics,
|
||||
quota_soft_exclusion,
|
||||
selection_constraint,
|
||||
)
|
||||
from cli_lane_files import (
|
||||
_candidate_path,
|
||||
_fsync_directory,
|
||||
|
||||
112
services/hermes/scripts/jenkins_image_build_trigger.py
Executable file
112
services/hermes/scripts/jenkins_image_build_trigger.py
Executable file
@ -0,0 +1,112 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Trigger only the reviewed-main Hermes agent image release job."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
JENKINS_ORIGIN = "https://ci.bstein.dev"
|
||||
JENKINS_BUILD_URL = f"{JENKINS_ORIGIN}/buildByToken/buildWithParameters"
|
||||
JOB_NAME = "hermes-agent-image"
|
||||
TOKEN_FILE = Path("/runtime-access/jenkins-image-build-token")
|
||||
REVISION_PATTERN = re.compile(r"^[0-9a-f]{40}$")
|
||||
QUEUE_PATH_PATTERN = re.compile(r"^/queue/item/[0-9]+/?$")
|
||||
|
||||
|
||||
class _NoRedirect(urllib.request.HTTPRedirectHandler):
|
||||
"""Keep a queued-build redirect from becoming an unauthorized job read."""
|
||||
|
||||
def redirect_request(self, _request, _file, _code, _message, _headers, _url):
|
||||
return None
|
||||
|
||||
|
||||
def _open_without_redirect(request: urllib.request.Request, timeout: int):
|
||||
"""Return the Build Token Root response, including its expected HTTP 303."""
|
||||
opener = urllib.request.build_opener(_NoRedirect())
|
||||
try:
|
||||
return opener.open(request, timeout=timeout)
|
||||
except urllib.error.HTTPError as exc:
|
||||
if exc.code == 303:
|
||||
return exc
|
||||
raise
|
||||
|
||||
|
||||
def trigger_build(
|
||||
revision: str,
|
||||
*,
|
||||
token_file: Path = TOKEN_FILE,
|
||||
opener=_open_without_redirect,
|
||||
) -> dict[str, str | int]:
|
||||
"""Post the fixed job parameters using its job-scoped build token."""
|
||||
revision = revision.strip()
|
||||
if not REVISION_PATTERN.fullmatch(revision):
|
||||
raise ValueError("revision must be a lowercase full 40-character commit")
|
||||
token = token_file.read_text(encoding="utf-8").strip()
|
||||
if not token:
|
||||
raise RuntimeError("Jenkins image-build token is empty")
|
||||
payload = urllib.parse.urlencode(
|
||||
{
|
||||
"job": JOB_NAME,
|
||||
"token": token,
|
||||
"PUBLISH_IMAGE": "true",
|
||||
"EXPECTED_SOURCE_REVISION": revision,
|
||||
"CONFIRM_PUBLISH": "PUBLISH HERMES AGENT",
|
||||
}
|
||||
).encode("utf-8")
|
||||
request = urllib.request.Request(
|
||||
JENKINS_BUILD_URL,
|
||||
data=payload,
|
||||
headers={"Content-Type": "application/x-www-form-urlencoded"},
|
||||
method="POST",
|
||||
)
|
||||
with opener(request, timeout=20) as response:
|
||||
status = int(response.status)
|
||||
location = response.headers.get("Location", "")
|
||||
if status not in {201, 303}:
|
||||
raise RuntimeError(f"Jenkins trigger returned HTTP {status}")
|
||||
if not location:
|
||||
raise RuntimeError("Jenkins trigger omitted the queue Location")
|
||||
queue_url = urllib.parse.urljoin(f"{JENKINS_ORIGIN}/", location)
|
||||
parsed_queue = urllib.parse.urlsplit(queue_url)
|
||||
expected_origin = urllib.parse.urlsplit(JENKINS_ORIGIN)
|
||||
if (
|
||||
parsed_queue.scheme != expected_origin.scheme
|
||||
or parsed_queue.netloc != expected_origin.netloc
|
||||
or parsed_queue.query
|
||||
or parsed_queue.fragment
|
||||
or not QUEUE_PATH_PATTERN.fullmatch(parsed_queue.path)
|
||||
):
|
||||
raise RuntimeError("Jenkins returned an invalid queue Location")
|
||||
# Never return the submitted URL: its form body contains the job token.
|
||||
queue_path = parsed_queue.path
|
||||
return {
|
||||
"job": JOB_NAME,
|
||||
"queue_path": queue_path,
|
||||
"source_revision": revision,
|
||||
"status": status,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Validate one revision, trigger the bounded job, and print safe metadata."""
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("revision", help="reviewed full commit currently on main")
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
result = trigger_build(args.revision)
|
||||
except (OSError, ValueError, RuntimeError, urllib.error.URLError) as exc:
|
||||
print(json.dumps({"error": str(exc)}, sort_keys=True))
|
||||
return 1
|
||||
print(json.dumps(result, indent=2, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover - exercised through main()
|
||||
raise SystemExit(main())
|
||||
@ -9,6 +9,7 @@ import struct
|
||||
from pathlib import Path
|
||||
|
||||
from node_account_io import HardeningError, read_regular
|
||||
from node_polkit_audit import audit_polkit_policy
|
||||
|
||||
|
||||
def _members(field: str, context: str) -> set[str]:
|
||||
@ -88,13 +89,22 @@ def _active_sudo_policy(text: str) -> str:
|
||||
stripped = line.lstrip()
|
||||
if not stripped:
|
||||
continue
|
||||
include = re.fullmatch(
|
||||
r"(?i)([#@]include|[#@]includedir)\s+([^\s]+)", stripped
|
||||
)
|
||||
if include:
|
||||
directive, target = include.groups()
|
||||
if directive.lower().endswith("includedir") and target == "/etc/sudoers.d":
|
||||
continue
|
||||
# Detect any include spelling before the comment rule below. sudo
|
||||
# honours both #include and @include; a quoted or space-bearing path
|
||||
# must never fall through the "#" comment branch and leave a second
|
||||
# authority file unaudited. Only the exact bare includedir into the
|
||||
# audited /etc/sudoers.d passes; everything else fails closed.
|
||||
if re.match(r"(?i)[#@]include(?:dir)?(?=\s|$)", stripped):
|
||||
include = re.fullmatch(
|
||||
r"(?i)([#@]include(?:dir)?)\s+([^\s\"'\\]+)", stripped
|
||||
)
|
||||
if include:
|
||||
directive, target = include.groups()
|
||||
if (
|
||||
directive.lower().endswith("includedir")
|
||||
and target == "/etc/sudoers.d"
|
||||
):
|
||||
continue
|
||||
raise HardeningError("sudo policy includes an unaudited authority source")
|
||||
if stripped.startswith("#") and not re.match(r"#\d+(?:\s|$)", stripped):
|
||||
continue
|
||||
@ -108,6 +118,87 @@ def _mentions_dedicated_identity(text: str, account: str, account_uid: int) -> b
|
||||
return bool(re.search(account_pattern, text) or re.search(numeric_pattern, text))
|
||||
|
||||
|
||||
SUDO_ALIAS_NAME_RE = re.compile(r"[A-Z][A-Z0-9_]*\Z")
|
||||
|
||||
|
||||
def _joined_sudo_lines(active: str) -> list[str]:
|
||||
"""Merge sudoers backslash continuations into whole logical lines."""
|
||||
lines: list[str] = []
|
||||
pending = ""
|
||||
for line in active.splitlines():
|
||||
if line.endswith("\\"):
|
||||
pending += line[:-1] + " "
|
||||
continue
|
||||
lines.append(pending + line)
|
||||
pending = ""
|
||||
if pending:
|
||||
raise HardeningError("sudo policy ends inside a line continuation")
|
||||
return lines
|
||||
|
||||
|
||||
def _sudo_user_aliases(lines: list[str]) -> dict[str, list[str]]:
|
||||
"""Collect User_Alias definitions so grant principals can be expanded."""
|
||||
aliases: dict[str, list[str]] = {}
|
||||
for line in lines:
|
||||
stripped = line.strip()
|
||||
if not stripped.startswith("User_Alias"):
|
||||
continue
|
||||
for definition in stripped.removeprefix("User_Alias").split(":"):
|
||||
name, separator, members = definition.partition("=")
|
||||
name = name.strip()
|
||||
values = [item.strip() for item in members.split(",")]
|
||||
if (
|
||||
not separator
|
||||
or not SUDO_ALIAS_NAME_RE.fullmatch(name)
|
||||
or name in aliases
|
||||
or not all(values)
|
||||
):
|
||||
raise HardeningError("sudo alias definition is not auditable")
|
||||
aliases[name] = values
|
||||
return aliases
|
||||
|
||||
|
||||
def _expand_sudo_principal(
|
||||
principal: str, aliases: dict[str, list[str]], seen: frozenset[str]
|
||||
) -> None:
|
||||
"""Reject grant principals that reach Hermes through aliases or wildcards."""
|
||||
value = principal.strip()
|
||||
while value.startswith("!"):
|
||||
value = value[1:].strip()
|
||||
value = value.strip('"')
|
||||
if not value:
|
||||
raise HardeningError("sudo grant principal is malformed")
|
||||
if value in aliases:
|
||||
if value in seen:
|
||||
raise HardeningError("sudo alias expansion is cyclic")
|
||||
for member in aliases[value]:
|
||||
_expand_sudo_principal(member, aliases, seen | {value})
|
||||
return
|
||||
if value in {"ALL", "%ALL"}:
|
||||
raise HardeningError("broad sudo authority includes Hermes")
|
||||
if value.startswith("+") or value.startswith("%:"):
|
||||
raise HardeningError("sudo netgroup authority cannot be audited")
|
||||
if SUDO_ALIAS_NAME_RE.fullmatch(value):
|
||||
raise HardeningError("sudo alias is undefined")
|
||||
|
||||
|
||||
def _audit_sudo_grants(active: str) -> None:
|
||||
"""Expand every grant's user list; only literal non-Hermes principals pass."""
|
||||
lines = _joined_sudo_lines(active)
|
||||
aliases = _sudo_user_aliases(lines)
|
||||
skip = ("Defaults", "User_Alias", "Runas_Alias", "Host_Alias", "Cmnd_Alias")
|
||||
for line in lines:
|
||||
stripped = line.strip()
|
||||
if not stripped or stripped.startswith(skip):
|
||||
continue
|
||||
head, separator, _rest = stripped.partition("=")
|
||||
fields = head.split()
|
||||
if not separator or len(fields) < 2:
|
||||
raise HardeningError("sudo grant line is not auditable")
|
||||
for principal in " ".join(fields[:-1]).split(","):
|
||||
_expand_sudo_principal(principal, aliases, frozenset())
|
||||
|
||||
|
||||
def audit_privilege_policies(
|
||||
account: str,
|
||||
account_uid: int,
|
||||
@ -160,10 +251,7 @@ def audit_privilege_policies(
|
||||
active = _active_sudo_policy(text)
|
||||
if _mentions_dedicated_identity(active, account, account_uid):
|
||||
raise HardeningError("dedicated Hermes account has sudo authority")
|
||||
for line in active.splitlines():
|
||||
fields = line.split()
|
||||
if fields and fields[0] in {"ALL", "%ALL"} and "=" in line:
|
||||
raise HardeningError("broad sudo authority includes Hermes")
|
||||
_audit_sudo_grants(active)
|
||||
continue
|
||||
active = "\n".join(
|
||||
line for line in text.splitlines() if not line.lstrip().startswith("#")
|
||||
@ -176,3 +264,4 @@ def audit_privilege_policies(
|
||||
broad = "unix-user:*" in active or "Identity=unix-user:*" in active
|
||||
if broad and grants and any(item in active for item in dangerous_polkit):
|
||||
raise HardeningError("root-equivalent broad polkit authority includes Hermes")
|
||||
audit_polkit_policy(path, text)
|
||||
|
||||
168
services/hermes/scripts/node_polkit_audit.py
Normal file
168
services/hermes/scripts/node_polkit_audit.py
Normal file
@ -0,0 +1,168 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Literal-identity proof for polkit policies on hardened Atlas nodes.
|
||||
|
||||
Polkit rules are JavaScript, so a grant can name its subject indirectly:
|
||||
concatenated strings, escape sequences, bracket lookups, or arithmetic can
|
||||
compute an identity that plain text matching never sees. Rather than model
|
||||
the language, every grant must scope itself through exact literal identity
|
||||
comparisons. Anything the audit cannot prove literal fails closed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from node_account_io import HardeningError
|
||||
|
||||
_ALLOWED_ESCAPES = {'"', "'", "\\", "/", "n", "r", "t"}
|
||||
_STRING_MARK = r"\x00[0-9]+\x00"
|
||||
_GRANT_RE = re.compile(r"\bResult\s*\.\s*YES\b")
|
||||
_PKLA_GRANT_RE = re.compile(r"(?im)^\s*Result(?:Any|Inactive|Active)\s*=\s*yes\s*$")
|
||||
_PKLA_IDENTITY_RE = re.compile(r"unix-(?:user|group):[A-Za-z0-9][A-Za-z0-9._-]{0,99}\Z")
|
||||
_IDENTITY_USE_RE = re.compile(r"subject\s*(\[|\.\s*(?:user|uid|isInGroup)\b)")
|
||||
_COMPUTED_TOKEN_RE = re.compile(
|
||||
r"\b(?:eval|Function|arguments|fromCharCode|fromCodePoint|atob|btoa|unescape"
|
||||
r"|escape|decodeURI[A-Za-z]*|encodeURI[A-Za-z]*|charCodeAt|codePointAt"
|
||||
r"|normalize|constructor|prototype|globalThis|Reflect|Proxy)\b"
|
||||
)
|
||||
# Logical && and || stay legal; every value-building operator next to a
|
||||
# string or number literal is rejected so comparisons stay whole literals.
|
||||
_OPERATOR_ADJACENT_RE = re.compile(
|
||||
rf"(?:{_STRING_MARK}|[0-9])\s*[+\-*/%<>^~](?![&|=])"
|
||||
rf"|(?<![&|=!<>])[+\-*/%<>^~]\s*(?:{_STRING_MARK}|[0-9])"
|
||||
)
|
||||
_USER_BEFORE_RE = re.compile(rf"{_STRING_MARK}\s*[=!]==?\s*\Z")
|
||||
_USER_AFTER_RE = re.compile(rf"\s*[=!]==?\s*{_STRING_MARK}")
|
||||
_UID_BEFORE_RE = re.compile(r"(?<![0-9A-Za-z_.$])[0-9]{1,10}\s*[=!]==?\s*\Z")
|
||||
_UID_AFTER_RE = re.compile(r"\s*[=!]==?\s*[0-9]{1,10}(?![0-9A-Za-z_.$])")
|
||||
_GROUP_AFTER_RE = re.compile(rf"\s*\(\s*{_STRING_MARK}\s*\)")
|
||||
_BARE_SUBJECT_RE = re.compile(r"\bsubject\b(?!\s*[.\[])")
|
||||
_PARAMETER_BEFORE_RE = re.compile(r"function\s*\(\s*[A-Za-z_$][A-Za-z0-9_$]*\s*,\s*\Z")
|
||||
_RESULT_USE_RE = re.compile(r"\bResult\b(?!\s*\.\s*[A-Z_]{2,32}\b)")
|
||||
|
||||
|
||||
def _split_strings(text: str) -> tuple[str, list[str]]:
|
||||
"""Replace string literals with markers and drop JavaScript comments."""
|
||||
code: list[str] = []
|
||||
strings: list[str] = []
|
||||
index = 0
|
||||
length = len(text)
|
||||
while index < length:
|
||||
character = text[index]
|
||||
if character == "`":
|
||||
raise HardeningError("polkit rule uses computed template strings")
|
||||
if character in {'"', "'"}:
|
||||
quote = character
|
||||
index += 1
|
||||
value: list[str] = []
|
||||
while True:
|
||||
if index >= length or text[index] in "\r\n":
|
||||
raise HardeningError("polkit rule string is unterminated")
|
||||
character = text[index]
|
||||
if character == "\\":
|
||||
if index + 1 >= length or text[index + 1] not in _ALLOWED_ESCAPES:
|
||||
raise HardeningError("polkit rule string uses computed escapes")
|
||||
escaped = text[index + 1]
|
||||
value.append({"n": "\n", "r": "\r", "t": "\t"}.get(escaped, escaped))
|
||||
index += 2
|
||||
continue
|
||||
index += 1
|
||||
if character == quote:
|
||||
break
|
||||
value.append(character)
|
||||
strings.append("".join(value))
|
||||
code.append(f"\x00{len(strings) - 1}\x00")
|
||||
continue
|
||||
if text.startswith("//", index):
|
||||
newline = text.find("\n", index)
|
||||
index = length if newline < 0 else newline
|
||||
continue
|
||||
if text.startswith("/*", index):
|
||||
closing = text.find("*/", index + 2)
|
||||
if closing < 0:
|
||||
raise HardeningError("polkit rule comment is unterminated")
|
||||
code.append(" ")
|
||||
index = closing + 2
|
||||
continue
|
||||
if character == "/":
|
||||
# Any remaining slash is a regex literal or a division operator.
|
||||
# A regex literal may embed a quote character, which would desync
|
||||
# the string scanner and swallow an unconditional grant. Neither
|
||||
# regex nor division has a place in an identity-scoped grant, so
|
||||
# both fail closed before they can hide a Result.YES.
|
||||
raise HardeningError("polkit rule uses a regex or division operator")
|
||||
code.append(character)
|
||||
index += 1
|
||||
return "".join(code), strings
|
||||
|
||||
|
||||
def _require_literal_identity_use(code: str, match: re.Match[str]) -> None:
|
||||
"""Reject one identity reference unless it is an exact literal comparison."""
|
||||
form = match.group(1)
|
||||
if form.startswith("["):
|
||||
raise HardeningError("polkit rule reads identity through a computed lookup")
|
||||
member = form.removeprefix(".").strip()
|
||||
before = code[max(match.start() - 48, 0) : match.start()]
|
||||
after = code[match.end() : match.end() + 48]
|
||||
if member == "user":
|
||||
literal = _USER_BEFORE_RE.search(before) or _USER_AFTER_RE.match(after)
|
||||
elif member == "uid":
|
||||
literal = _UID_BEFORE_RE.search(before) or _UID_AFTER_RE.match(after)
|
||||
else:
|
||||
literal = _GROUP_AFTER_RE.match(after)
|
||||
if not literal:
|
||||
raise HardeningError("polkit rule identity test is not an exact literal")
|
||||
|
||||
|
||||
def _audit_rules_javascript(text: str) -> None:
|
||||
"""Require granting rules to gate on exact literal subject identities.
|
||||
|
||||
With computed tokens, bracket lookups, and subject aliasing rejected in
|
||||
every rule file, a YES decision can only be written as the literal grant
|
||||
token, so the grant test below cannot be evaded by construction.
|
||||
"""
|
||||
code, _strings = _split_strings(text)
|
||||
if _COMPUTED_TOKEN_RE.search(code):
|
||||
raise HardeningError("polkit rule computes values the audit cannot prove")
|
||||
if _RESULT_USE_RE.search(code):
|
||||
raise HardeningError("polkit rule reads results through a computed lookup")
|
||||
for bare in _BARE_SUBJECT_RE.finditer(code):
|
||||
if not _PARAMETER_BEFORE_RE.search(code[: bare.start()]):
|
||||
raise HardeningError("polkit rule passes the subject to computed code")
|
||||
if not _GRANT_RE.search(code):
|
||||
return
|
||||
if _OPERATOR_ADJACENT_RE.search(code):
|
||||
raise HardeningError("polkit grant builds values with operators")
|
||||
uses = 0
|
||||
for match in _IDENTITY_USE_RE.finditer(code):
|
||||
uses += 1
|
||||
_require_literal_identity_use(code, match)
|
||||
if not uses:
|
||||
raise HardeningError("polkit grant is not scoped to an explicit identity")
|
||||
|
||||
|
||||
def _audit_pkla(text: str) -> None:
|
||||
"""Require granting localauthority entries to name exact literal identities."""
|
||||
if not _PKLA_GRANT_RE.search(text):
|
||||
return
|
||||
identities: list[str] = []
|
||||
for line in text.splitlines():
|
||||
name, separator, value = line.strip().partition("=")
|
||||
if name.strip() != "Identity" or not separator:
|
||||
continue
|
||||
entries = [entry.strip() for entry in value.split(";") if entry.strip()]
|
||||
identities.extend(entries)
|
||||
if not identities:
|
||||
raise HardeningError("polkit authority grant lacks an identity scope")
|
||||
for entry in identities:
|
||||
if not _PKLA_IDENTITY_RE.fullmatch(entry):
|
||||
raise HardeningError("polkit authority identity is not an exact literal")
|
||||
|
||||
|
||||
def audit_polkit_policy(path: Path, text: str) -> None:
|
||||
"""Prove one polkit policy file cannot grant through a computed identity."""
|
||||
if "localauthority" in path.parts or path.suffix == ".pkla":
|
||||
_audit_pkla(text)
|
||||
else:
|
||||
_audit_rules_javascript(text)
|
||||
@ -186,6 +186,7 @@ def stage_agent() -> None:
|
||||
"agent-api-key",
|
||||
"execution-pool-key",
|
||||
"chat-relay-key",
|
||||
"jenkins-image-build-token",
|
||||
"node-ssh-private-key",
|
||||
"node-ssh-known-hosts",
|
||||
):
|
||||
|
||||
@ -181,3 +181,24 @@ spec:
|
||||
port: 9006
|
||||
targetPort: claude-broker
|
||||
protocol: TCP
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: hermes-cli-lane-metrics
|
||||
namespace: hermes
|
||||
labels:
|
||||
app: hermes-agent
|
||||
annotations:
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/port: "9011"
|
||||
prometheus.io/path: /metrics
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app: hermes-agent
|
||||
ports:
|
||||
- name: lane-metrics
|
||||
port: 9011
|
||||
targetPort: lane-metrics
|
||||
protocol: TCP
|
||||
|
||||
@ -27,7 +27,7 @@ spec:
|
||||
prometheus.io/port: "9005"
|
||||
prometheus.io/path: /metrics
|
||||
vault.hashicorp.com/agent-inject: "true"
|
||||
vault.hashicorp.com/role: hermes-agent
|
||||
vault.hashicorp.com/role: hermes-switchyard
|
||||
vault.hashicorp.com/agent-inject-secret-relay-key: kv/data/atlas/hermes/chat-telegram
|
||||
vault.hashicorp.com/agent-inject-template-relay-key: |
|
||||
{{- with secret "kv/data/atlas/hermes/chat-telegram" -}}
|
||||
|
||||
@ -652,6 +652,25 @@ data:
|
||||
}
|
||||
}
|
||||
}
|
||||
pipelineJob('hermes-agent-image') {
|
||||
disabled(false)
|
||||
description('Human-gated, daemonless Kaniko build for the reviewed atlas/titan-iac main revision. Publishes a content-addressed Hermes agent image and archives a Flux digest patch; it never mutates Git or deploys.')
|
||||
authenticationToken(System.getenv('HERMES_AGENT_IMAGE_BUILD_TOKEN'))
|
||||
definition {
|
||||
cpsScm {
|
||||
scm {
|
||||
git {
|
||||
remote {
|
||||
url('https://scm.bstein.dev/atlas/titan-iac.git')
|
||||
credentials('gitea-pat')
|
||||
}
|
||||
branches('*/main')
|
||||
}
|
||||
}
|
||||
scriptPath('ci/Jenkinsfile.hermes-agent-image')
|
||||
}
|
||||
}
|
||||
}
|
||||
multibranchPipelineJob('titan-iac-quality-gate') {
|
||||
branchSources {
|
||||
branchSource {
|
||||
|
||||
@ -20,6 +20,7 @@ data:
|
||||
gitea:268.v75e47974c01d
|
||||
gitea-checks:603.621.vc708da_fb_371d
|
||||
multibranch-scan-webhook-trigger:1.0.11
|
||||
build-token-root:365.v717f8685a_09e
|
||||
# Structured test evidence. Without junit the `junit` step throws
|
||||
# NoSuchMethodError, jenkins.failed_tests is always empty, and triage
|
||||
# has only raw console text to work from. Pinned to the newest release
|
||||
|
||||
@ -65,6 +65,9 @@ spec:
|
||||
ARIADNE_JENKINS_API_USER={{ .Data.data.username }}
|
||||
ARIADNE_JENKINS_API_TOKEN={{ .Data.data.token }}
|
||||
{{ end }}
|
||||
{{ with secret "kv/data/atlas/hermes/developer-jenkins" }}
|
||||
HERMES_AGENT_IMAGE_BUILD_TOKEN={{ .Data.data.build_token }}
|
||||
{{ end }}
|
||||
bstein.dev/restarted-at: "2026-05-20T09:40:31Z"
|
||||
spec:
|
||||
serviceAccountName: jenkins
|
||||
|
||||
@ -0,0 +1,7 @@
|
||||
# services/jenkins/hermes-image-builder-serviceaccount.yaml
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: hermes-image-builder
|
||||
namespace: jenkins
|
||||
automountServiceAccountToken: false
|
||||
@ -5,6 +5,7 @@ namespace: jenkins
|
||||
resources:
|
||||
- namespace.yaml
|
||||
- serviceaccount.yaml
|
||||
- hermes-image-builder-serviceaccount.yaml
|
||||
- vault-serviceaccount.yaml
|
||||
- pvc.yaml
|
||||
- cache-pvc.yaml
|
||||
|
||||
@ -8,7 +8,7 @@ spec:
|
||||
provider: vault
|
||||
parameters:
|
||||
vaultAddress: "http://vault.vault.svc.cluster.local:8200"
|
||||
roleName: "jenkins"
|
||||
roleName: "jenkins-vault-sync"
|
||||
objects: |
|
||||
- objectName: "harbor-pull__dockerconfigjson"
|
||||
secretPath: "kv/data/atlas/shared/harbor-pull"
|
||||
|
||||
@ -314,7 +314,7 @@ def main():
|
||||
conn.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if __name__ == "__main__": # pragma: no cover - exercised through main()
|
||||
try:
|
||||
main()
|
||||
except Exception as exc:
|
||||
|
||||
@ -100,6 +100,6 @@ class Handler(http.server.BaseHTTPRequestHandler):
|
||||
return
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if __name__ == "__main__": # pragma: no cover - server loop
|
||||
server = http.server.ThreadingHTTPServer(("", 8080), Handler)
|
||||
server.serve_forever()
|
||||
|
||||
@ -15,8 +15,8 @@ data:
|
||||
METIS_MAX_DEVICE_BYTES: "1000000000000"
|
||||
METIS_NAMESPACE: maintenance
|
||||
METIS_REMOTE_POD_TIMEOUT_SEC: "14400"
|
||||
METIS_RUNNER_IMAGE_AMD64: registry.bstein.dev/bstein/metis:0.1.0-296-amd64 # {"$imagepolicy": "maintenance:metis-amd64"}
|
||||
METIS_RUNNER_IMAGE_ARM64: registry.bstein.dev/bstein/metis:0.1.0-295-arm64 # {"$imagepolicy": "maintenance:metis-arm64"}
|
||||
METIS_RUNNER_IMAGE_AMD64: registry.bstein.dev/bstein/metis:0.1.0-298-amd64 # {"$imagepolicy": "maintenance:metis-amd64"}
|
||||
METIS_RUNNER_IMAGE_ARM64: registry.bstein.dev/bstein/metis:0.1.0-298-arm64 # {"$imagepolicy": "maintenance:metis-arm64"}
|
||||
METIS_HARBOR_REGISTRY: registry.bstein.dev
|
||||
METIS_HARBOR_PROJECT: metis
|
||||
METIS_HARBOR_API_BASE: https://registry.bstein.dev/api/v2.0
|
||||
|
||||
@ -13,6 +13,6 @@ images:
|
||||
- name: registry.bstein.dev/bstein/ariadne
|
||||
newTag: 0.1.0-461 # {"$imagepolicy": "maintenance:ariadne:tag"}
|
||||
- name: registry.bstein.dev/bstein/metis
|
||||
newTag: 0.1.0-295-arm64 # {"$imagepolicy": "maintenance:metis-arm64:tag"}
|
||||
newTag: 0.1.0-298-arm64 # {"$imagepolicy": "maintenance:metis-arm64:tag"}
|
||||
- name: registry.bstein.dev/bstein/soteria
|
||||
newTag: 0.1.0-120 # {"$imagepolicy": "maintenance:soteria:tag"}
|
||||
|
||||
@ -32,7 +32,7 @@ spec:
|
||||
kubernetes.io/arch: amd64
|
||||
containers:
|
||||
- name: metis-sentinel
|
||||
image: registry.bstein.dev/bstein/metis-sentinel:0.1.0-296-amd64 # {"$imagepolicy": "maintenance:metis-sentinel-amd64"}
|
||||
image: registry.bstein.dev/bstein/metis-sentinel:0.1.0-298-amd64 # {"$imagepolicy": "maintenance:metis-sentinel-amd64"}
|
||||
imagePullPolicy: Always
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
|
||||
@ -32,7 +32,7 @@ spec:
|
||||
kubernetes.io/arch: arm64
|
||||
containers:
|
||||
- name: metis-sentinel
|
||||
image: registry.bstein.dev/bstein/metis-sentinel:0.1.0-295-arm64 # {"$imagepolicy": "maintenance:metis-sentinel-arm64"}
|
||||
image: registry.bstein.dev/bstein/metis-sentinel:0.1.0-298-arm64 # {"$imagepolicy": "maintenance:metis-sentinel-arm64"}
|
||||
imagePullPolicy: Always
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
|
||||
68
services/vault-hermes-jenkins-token-seed/job.yaml
Normal file
68
services/vault-hermes-jenkins-token-seed/job.yaml
Normal file
@ -0,0 +1,68 @@
|
||||
# services/vault-hermes-jenkins-token-seed/job.yaml
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: vault-hermes-jenkins-build-token-seed-2
|
||||
namespace: vault
|
||||
spec:
|
||||
backoffLimit: 2
|
||||
template:
|
||||
spec:
|
||||
serviceAccountName: hermes-jenkins-token-seed
|
||||
enableServiceLinks: false
|
||||
restartPolicy: Never
|
||||
nodeSelector:
|
||||
hardware: rpi5
|
||||
kubernetes.io/arch: arm64
|
||||
node-role.kubernetes.io/worker: "true"
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- key: kubernetes.io/hostname
|
||||
operator: NotIn
|
||||
values: [titan-04, titan-14, titan-18, titan-19, titan-24]
|
||||
securityContext:
|
||||
fsGroup: 1000
|
||||
fsGroupChangePolicy: OnRootMismatch
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: seed
|
||||
image: docker.io/hashicorp/vault@sha256:4e33b126a59c0c333b76fb4e894722462659a6bec7c48c9ee8cea56fccfd2569
|
||||
imagePullPolicy: IfNotPresent
|
||||
command: [sh, /scripts/vault_hermes_jenkins_build_token_ensure.sh]
|
||||
env:
|
||||
- name: HOME
|
||||
value: /tmp
|
||||
- name: VAULT_ADDR
|
||||
value: http://vault.vault.svc.cluster.local:8200
|
||||
- name: VAULT_K8S_ROLE
|
||||
value: hermes-jenkins-token-seed
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
readOnlyRootFilesystem: true
|
||||
runAsGroup: 1000
|
||||
runAsNonRoot: true
|
||||
runAsUser: 100
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
volumeMounts:
|
||||
- name: scripts
|
||||
mountPath: /scripts
|
||||
readOnly: true
|
||||
- name: tmp
|
||||
mountPath: /tmp
|
||||
resources:
|
||||
requests: {cpu: 25m, memory: 32Mi}
|
||||
limits: {cpu: 250m, memory: 128Mi}
|
||||
volumes:
|
||||
- name: scripts
|
||||
configMap:
|
||||
name: vault-hermes-jenkins-token-seed-script
|
||||
defaultMode: 0555
|
||||
- name: tmp
|
||||
emptyDir: {}
|
||||
13
services/vault-hermes-jenkins-token-seed/kustomization.yaml
Normal file
13
services/vault-hermes-jenkins-token-seed/kustomization.yaml
Normal file
@ -0,0 +1,13 @@
|
||||
# services/vault-hermes-jenkins-token-seed/kustomization.yaml
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: vault
|
||||
resources:
|
||||
- serviceaccount.yaml
|
||||
- job.yaml
|
||||
generatorOptions:
|
||||
disableNameSuffixHash: true
|
||||
configMapGenerator:
|
||||
- name: vault-hermes-jenkins-token-seed-script
|
||||
files:
|
||||
- vault_hermes_jenkins_build_token_ensure.sh=scripts/vault_hermes_jenkins_build_token_ensure.sh
|
||||
@ -0,0 +1,167 @@
|
||||
#!/usr/bin/env sh
|
||||
set -eu
|
||||
|
||||
secret_path="kv/atlas/hermes/developer-jenkins"
|
||||
secret_api_path="kv/data/atlas/hermes/developer-jenkins"
|
||||
jwt_file="${VAULT_K8S_JWT_FILE:-/var/run/secrets/kubernetes.io/serviceaccount/token}"
|
||||
vault_role="${VAULT_K8S_ROLE:-hermes-jenkins-token-seed}"
|
||||
payload_file="${TMPDIR:-/tmp}/hermes-jenkins-token.json"
|
||||
|
||||
log() { printf '[hermes-jenkins-token] %s\n' "$*" >&2; }
|
||||
cleanup() { rm -f "${payload_file}"; }
|
||||
trap cleanup EXIT HUP INT TERM
|
||||
|
||||
retry_command() {
|
||||
attempt=1
|
||||
while [ "${attempt}" -le 5 ]; do
|
||||
set +e
|
||||
command_output="$("$@" 2>&1)"
|
||||
command_status=$?
|
||||
set -e
|
||||
if [ "${command_status}" -eq 0 ]; then
|
||||
printf '%s' "${command_output}"
|
||||
return 0
|
||||
fi
|
||||
if [ "${attempt}" -lt 5 ]; then
|
||||
sleep "${attempt}"
|
||||
fi
|
||||
attempt=$((attempt + 1))
|
||||
done
|
||||
return "${command_status}"
|
||||
}
|
||||
|
||||
ensure_token() {
|
||||
if [ -n "${VAULT_TOKEN:-}" ]; then
|
||||
return
|
||||
fi
|
||||
jwt="$(cat "${jwt_file}")"
|
||||
VAULT_TOKEN="$(retry_command vault write -field=token \
|
||||
auth/kubernetes/login role="${vault_role}" jwt="${jwt}")" || {
|
||||
log "Vault Kubernetes login failed after retries"
|
||||
exit 1
|
||||
}
|
||||
unset jwt
|
||||
if [ -z "${VAULT_TOKEN}" ]; then
|
||||
log "Vault Kubernetes login returned an empty token"
|
||||
exit 1
|
||||
fi
|
||||
export VAULT_TOKEN
|
||||
}
|
||||
|
||||
# Return 0 for the complete field, 10 when absent, 11 when present but
|
||||
# incomplete, and 12 for a persistent authorization/transport failure.
|
||||
read_build_token() {
|
||||
attempt=1
|
||||
while [ "${attempt}" -le 5 ]; do
|
||||
set +e
|
||||
secret_json="$(vault read -format=json "${secret_api_path}" 2>&1)"
|
||||
secret_status=$?
|
||||
set -e
|
||||
if [ "${secret_status}" -eq 0 ]; then
|
||||
unset secret_json
|
||||
set +e
|
||||
build_token="$(vault kv get -field=build_token "${secret_path}" 2>&1)"
|
||||
field_status=$?
|
||||
set -e
|
||||
if [ "${field_status}" -eq 0 ]; then
|
||||
case "${build_token}" in
|
||||
''|*[!0-9a-f]*) unset build_token; return 11 ;;
|
||||
esac
|
||||
if [ "${#build_token}" -eq 64 ]; then
|
||||
unset build_token
|
||||
return 0
|
||||
fi
|
||||
unset build_token
|
||||
return 11
|
||||
fi
|
||||
if printf '%s' "${build_token}" | grep -q 'No value found'; then
|
||||
unset build_token
|
||||
return 11
|
||||
fi
|
||||
unset build_token
|
||||
elif printf '%s' "${secret_json}" | grep -Eq 'Code: 404|No value found at'; then
|
||||
unset secret_json
|
||||
return 10
|
||||
fi
|
||||
if [ "${attempt}" -lt 5 ]; then
|
||||
sleep "${attempt}"
|
||||
fi
|
||||
attempt=$((attempt + 1))
|
||||
done
|
||||
unset secret_json
|
||||
return 12
|
||||
}
|
||||
|
||||
ensure_token
|
||||
|
||||
if read_build_token; then
|
||||
read_status=0
|
||||
else
|
||||
read_status=$?
|
||||
fi
|
||||
case "${read_status}" in
|
||||
0)
|
||||
log "job-scoped token already present; no write performed"
|
||||
exit 0
|
||||
;;
|
||||
11)
|
||||
log "secret exists without a valid build_token; refusing to overwrite existing fields"
|
||||
exit 1
|
||||
;;
|
||||
12)
|
||||
log "secret read failed after retries; refusing to seed"
|
||||
exit 1
|
||||
;;
|
||||
10) ;;
|
||||
*)
|
||||
log "unexpected secret read state ${read_status}"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
new_token="$(retry_command vault write -field=random_bytes \
|
||||
sys/tools/random/32 format=hex)" || {
|
||||
log "Vault random token generation failed after retries"
|
||||
exit 1
|
||||
}
|
||||
case "${new_token}" in
|
||||
''|*[!0-9a-f]*)
|
||||
unset new_token
|
||||
log "Vault returned an invalid random token"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
if [ "${#new_token}" -ne 64 ]; then
|
||||
unset new_token
|
||||
log "Vault returned an invalid random token"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
umask 077
|
||||
printf '{"options":{"cas":0},"data":{"build_token":"%s"}}\n' \
|
||||
"${new_token}" > "${payload_file}"
|
||||
unset new_token
|
||||
|
||||
set +e
|
||||
create_error="$(vault write "${secret_api_path}" @"${payload_file}" 2>&1 >/dev/null)"
|
||||
create_status=$?
|
||||
set -e
|
||||
cleanup
|
||||
if [ "${create_status}" -eq 0 ]; then
|
||||
log "job-scoped token created with KV-v2 CAS create-only semantics"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if printf '%s' "${create_error}" | grep -qi 'check-and-set'; then
|
||||
unset create_error
|
||||
if read_build_token; then
|
||||
log "another seeder won CAS; existing token preserved"
|
||||
exit 0
|
||||
fi
|
||||
log "CAS race did not produce a readable build_token; refusing to continue"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
unset create_error
|
||||
log "KV-v2 CAS create failed; refusing to retry a write"
|
||||
exit 1
|
||||
@ -0,0 +1,7 @@
|
||||
# services/vault-hermes-jenkins-token-seed/serviceaccount.yaml
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: hermes-jenkins-token-seed
|
||||
namespace: vault
|
||||
automountServiceAccountToken: true
|
||||
@ -3,7 +3,7 @@
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: vault-k8s-auth-hermes-9
|
||||
name: vault-k8s-auth-hermes-10
|
||||
namespace: vault
|
||||
spec:
|
||||
backoffLimit: 2
|
||||
@ -15,16 +15,26 @@ spec:
|
||||
hardware: rpi5
|
||||
kubernetes.io/arch: arm64
|
||||
node-role.kubernetes.io/worker: "true"
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- key: kubernetes.io/hostname
|
||||
operator: NotIn
|
||||
values: [titan-04, titan-14, titan-18, titan-19, titan-24]
|
||||
containers:
|
||||
- name: configure-k8s-auth
|
||||
image: hashicorp/vault:1.21.4
|
||||
image: docker.io/hashicorp/vault@sha256:4e33b126a59c0c333b76fb4e894722462659a6bec7c48c9ee8cea56fccfd2569
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- sh
|
||||
- /scripts/vault_k8s_auth_configure.sh
|
||||
env:
|
||||
- name: HOME
|
||||
value: /tmp
|
||||
- name: VAULT_ADDR
|
||||
value: http://10.43.57.249:8200
|
||||
value: http://vault.vault.svc.cluster.local:8200
|
||||
- name: VAULT_K8S_ROLE
|
||||
value: vault-admin
|
||||
- name: VAULT_K8S_TOKEN_REVIEWER_JWT_FILE
|
||||
@ -38,6 +48,18 @@ spec:
|
||||
- name: token-reviewer
|
||||
mountPath: /var/run/secrets/vault-token-reviewer
|
||||
readOnly: true
|
||||
- name: tmp
|
||||
mountPath: /tmp
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
readOnlyRootFilesystem: true
|
||||
runAsGroup: 1000
|
||||
runAsNonRoot: true
|
||||
runAsUser: 100
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
resources:
|
||||
requests:
|
||||
cpu: 25m
|
||||
@ -53,3 +75,5 @@ spec:
|
||||
- name: token-reviewer
|
||||
secret:
|
||||
secretName: vault-admin-token-reviewer
|
||||
- name: tmp
|
||||
emptyDir: {}
|
||||
|
||||
@ -237,12 +237,26 @@ write_policy_and_role "mailu-mailserver" "mailu-mailserver" "mailu-vault-sync" \
|
||||
"mailu/* shared/postmark-relay shared/harbor-pull" ""
|
||||
write_policy_and_role "harbor" "harbor" "harbor-vault-sync" \
|
||||
"harbor/* shared/harbor-pull" "hermes/developer-harbor"
|
||||
harbor_policy_bootstrap_policy='
|
||||
path "kv/data/atlas/harbor/harbor-core" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
'
|
||||
write_raw_policy "harbor-policy-bootstrap" "${harbor_policy_bootstrap_policy}"
|
||||
log "writing role harbor-policy-bootstrap"
|
||||
vault_cmd write "auth/kubernetes/role/harbor-policy-bootstrap" \
|
||||
bound_service_account_names="harbor-policy-bootstrap" \
|
||||
bound_service_account_namespaces="harbor" \
|
||||
policies="harbor-policy-bootstrap" \
|
||||
ttl="${role_ttl}"
|
||||
write_policy_and_role "nextcloud" "nextcloud" "nextcloud-vault" \
|
||||
"nextcloud/* shared/keycloak-admin shared/postmark-relay" ""
|
||||
write_policy_and_role "comms" "comms" "comms-vault,atlasbot" \
|
||||
"comms/* shared/chat-ai-keys-runtime shared/harbor-pull" ""
|
||||
write_policy_and_role "jenkins" "jenkins" "jenkins,jenkins-vault-sync" \
|
||||
"jenkins/* shared/harbor-pull quality/sonarqube-oidc" "hermes/developer-jenkins"
|
||||
write_policy_and_role "jenkins" "jenkins" "jenkins" \
|
||||
"jenkins/* shared/harbor-pull quality/sonarqube-oidc hermes/developer-jenkins" ""
|
||||
write_policy_and_role "jenkins-vault-sync" "jenkins" "jenkins-vault-sync" \
|
||||
"shared/harbor-pull" ""
|
||||
write_policy_and_role "monitoring" "monitoring" "monitoring-vault-sync" \
|
||||
"monitoring/* shared/postmark-relay shared/harbor-pull" ""
|
||||
write_policy_and_role "logging" "logging" "logging-vault-sync" \
|
||||
@ -257,8 +271,28 @@ write_policy_and_role "game-stream" "game-stream" "game-stream-vault" \
|
||||
"game-stream/*" ""
|
||||
write_policy_and_role "hermes" "hermes" "hermes-vault,hermes-triage" \
|
||||
"hermes/triage-oidc hermes/agent-tokens hermes/triage-api" ""
|
||||
write_policy_and_role "hermes-agent" "hermes" "hermes-agent,hermes-switchyard" \
|
||||
# The agent keeps its own credentials but no longer holds developer-gitea:
|
||||
# the least-authority hermes-scm-broker role (below) carries the Gitea token
|
||||
# so a compromised agent cannot reach the SCM write path directly.
|
||||
write_policy_and_role "hermes-agent" "hermes" "hermes-agent" \
|
||||
"hermes/agent-oidc hermes/agent-tokens hermes/chat-telegram hermes/developer-keycloak hermes/developer-harbor hermes/developer-jenkins hermes/developer-ssh" ""
|
||||
write_policy_and_role "hermes-switchyard" "hermes" "hermes-switchyard" \
|
||||
"hermes/chat-telegram" ""
|
||||
hermes_jenkins_token_seed_policy='
|
||||
path "kv/data/atlas/hermes/developer-jenkins" {
|
||||
capabilities = ["create", "read"]
|
||||
}
|
||||
path "sys/tools/random/32" {
|
||||
capabilities = ["update"]
|
||||
}
|
||||
'
|
||||
write_raw_policy "hermes-jenkins-token-seed" "${hermes_jenkins_token_seed_policy}"
|
||||
log "writing role hermes-jenkins-token-seed"
|
||||
vault_cmd write "auth/kubernetes/role/hermes-jenkins-token-seed" \
|
||||
bound_service_account_names="hermes-jenkins-token-seed" \
|
||||
bound_service_account_namespaces="vault" \
|
||||
policies="hermes-jenkins-token-seed" \
|
||||
ttl="${role_ttl}"
|
||||
write_policy_and_role "hermes-scm-broker" "hermes-scm" "hermes-scm-broker" \
|
||||
"hermes/developer-gitea" ""
|
||||
write_policy_and_role "hermes-credential-sync" "hermes" "hermes-agent" \
|
||||
|
||||
@ -14,12 +14,50 @@
|
||||
}
|
||||
],
|
||||
"managed_modules": [
|
||||
"ci/scripts/hermes_image_release.py",
|
||||
"dockerfiles/hermes-kaniko-heredoc-runner.py",
|
||||
"services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py",
|
||||
"services/hermes/scripts/jenkins_image_build_trigger.py",
|
||||
"ci/scripts/publish_test_metrics.py",
|
||||
"ci/scripts/publish_test_metrics_quality.py",
|
||||
"ci/scripts/semgrep_report.py",
|
||||
"ci/scripts/supply_chain_report.py",
|
||||
"services/mailu/scripts/mailu_sync.py",
|
||||
"services/mailu/scripts/mailu_sync_listener.py",
|
||||
"services/hermes/scripts/cli_lane_board.py",
|
||||
"services/hermes/scripts/cli_lane_capabilities.py",
|
||||
"services/hermes/scripts/cli_lane_config.py",
|
||||
"services/hermes/scripts/cli_lane_dispatch.py",
|
||||
"services/hermes/scripts/cli_lane_evidence.py",
|
||||
"services/hermes/scripts/cli_lane_execution.py",
|
||||
"services/hermes/scripts/cli_lane_failover.py",
|
||||
"services/hermes/scripts/cli_lane_files.py",
|
||||
"services/hermes/scripts/cli_lane_finalization.py",
|
||||
"services/hermes/scripts/cli_lane_goal.py",
|
||||
"services/hermes/scripts/cli_lane_health.py",
|
||||
"services/hermes/scripts/cli_lane_metrics.py",
|
||||
"services/hermes/scripts/cli_lane_prompt.py",
|
||||
"services/hermes/scripts/cli_lane_provider.py",
|
||||
"services/hermes/scripts/cli_lane_quarantine.py",
|
||||
"services/hermes/scripts/cli_lane_quota.py",
|
||||
"services/hermes/scripts/cli_lane_records.py",
|
||||
"services/hermes/scripts/cli_lane_recovery.py",
|
||||
"services/hermes/scripts/cli_lane_retention.py",
|
||||
"services/hermes/scripts/cli_lane_routing.py",
|
||||
"services/hermes/scripts/cli_lane_runner.py",
|
||||
"testing/__init__.py",
|
||||
"testing/quality_contract.py",
|
||||
"testing/quality_docs.py",
|
||||
"testing/quality_hygiene.py",
|
||||
"testing/quality_coverage.py",
|
||||
"testing/quality_gate.py",
|
||||
"ci/tests/glue/test_ariadne_schedules.py",
|
||||
"ci/tests/glue/test_glue_metrics.py",
|
||||
"testing/tests/test_publish_test_metrics.py",
|
||||
"testing/tests/test_supply_chain_report.py",
|
||||
"testing/tests/test_semgrep_report.py",
|
||||
"testing/tests/test_quality_contract.py",
|
||||
"testing/tests/test_quality_gate.py",
|
||||
"services/gitea/scripts/gitea_branch_protection_check.py",
|
||||
"services/hermes/scm-common/scripts/gitea_api.py",
|
||||
"services/hermes/scm-common/scripts/gitea_api_policy.py",
|
||||
@ -41,38 +79,16 @@
|
||||
"services/hermes/scripts/node_account_hardening.py",
|
||||
"services/hermes/scripts/node_account_io.py",
|
||||
"services/hermes/scripts/stage_runtime_access.py",
|
||||
"services/hermes/scripts/cli_lane_board.py",
|
||||
"services/hermes/scripts/cli_lane_capabilities.py",
|
||||
"services/hermes/scripts/cli_lane_config.py",
|
||||
"services/hermes/scripts/cli_lane_dispatch.py",
|
||||
"services/hermes/scripts/cli_lane_evidence.py",
|
||||
"services/hermes/scripts/cli_lane_execution.py",
|
||||
"services/hermes/scripts/cli_lane_files.py",
|
||||
"services/hermes/scripts/cli_lane_finalization.py",
|
||||
"services/hermes/scripts/cli_lane_goal.py",
|
||||
"services/hermes/scripts/cli_lane_prompt.py",
|
||||
"services/hermes/scripts/cli_lane_provider.py",
|
||||
"services/hermes/scripts/cli_lane_quarantine.py",
|
||||
"services/hermes/scripts/cli_lane_records.py",
|
||||
"services/hermes/scripts/cli_lane_recovery.py",
|
||||
"services/hermes/scripts/cli_lane_retention.py",
|
||||
"services/hermes/scripts/cli_lane_routing.py",
|
||||
"services/hermes/scripts/cli_lane_runner.py",
|
||||
"testing/__init__.py",
|
||||
"testing/quality_contract.py",
|
||||
"testing/quality_docs.py",
|
||||
"testing/quality_hygiene.py",
|
||||
"testing/quality_coverage.py",
|
||||
"testing/quality_gate.py",
|
||||
"ci/tests/glue/test_ariadne_schedules.py",
|
||||
"ci/tests/glue/test_glue_metrics.py",
|
||||
"testing/tests/test_publish_test_metrics.py",
|
||||
"testing/tests/test_supply_chain_report.py",
|
||||
"testing/tests/test_semgrep_report.py",
|
||||
"testing/tests/test_quality_contract.py",
|
||||
"testing/tests/test_quality_gate.py"
|
||||
"services/hermes/scm-common/scripts/deadline_http.py",
|
||||
"services/hermes/scm-common/scripts/git_pack_objects.py",
|
||||
"services/hermes/scm-common/scripts/receive_pack_scan.py",
|
||||
"services/hermes/scripts/node_polkit_audit.py"
|
||||
],
|
||||
"lint_paths": [
|
||||
"ci/scripts/hermes_image_release.py",
|
||||
"dockerfiles/hermes-kaniko-heredoc-runner.py",
|
||||
"services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py",
|
||||
"services/hermes/scripts/jenkins_image_build_trigger.py",
|
||||
"ci/scripts/publish_test_metrics.py",
|
||||
"ci/scripts/publish_test_metrics_quality.py",
|
||||
"ci/scripts/semgrep_report.py",
|
||||
@ -81,6 +97,29 @@
|
||||
"services/comms/scripts/tests",
|
||||
"services/mailu/scripts/mailu_sync.py",
|
||||
"services/mailu/scripts/mailu_sync_listener.py",
|
||||
"services/hermes/scripts/cli_lane_board.py",
|
||||
"services/hermes/scripts/cli_lane_capabilities.py",
|
||||
"services/hermes/scripts/cli_lane_config.py",
|
||||
"services/hermes/scripts/cli_lane_dispatch.py",
|
||||
"services/hermes/scripts/cli_lane_evidence.py",
|
||||
"services/hermes/scripts/cli_lane_execution.py",
|
||||
"services/hermes/scripts/cli_lane_failover.py",
|
||||
"services/hermes/scripts/cli_lane_files.py",
|
||||
"services/hermes/scripts/cli_lane_finalization.py",
|
||||
"services/hermes/scripts/cli_lane_goal.py",
|
||||
"services/hermes/scripts/cli_lane_health.py",
|
||||
"services/hermes/scripts/cli_lane_metrics.py",
|
||||
"services/hermes/scripts/cli_lane_prompt.py",
|
||||
"services/hermes/scripts/cli_lane_provider.py",
|
||||
"services/hermes/scripts/cli_lane_quarantine.py",
|
||||
"services/hermes/scripts/cli_lane_quota.py",
|
||||
"services/hermes/scripts/cli_lane_records.py",
|
||||
"services/hermes/scripts/cli_lane_recovery.py",
|
||||
"services/hermes/scripts/cli_lane_retention.py",
|
||||
"services/hermes/scripts/cli_lane_routing.py",
|
||||
"services/hermes/scripts/cli_lane_runner.py",
|
||||
"testing/tests",
|
||||
"testing",
|
||||
"services/gitea/scripts/gitea_branch_protection_check.py",
|
||||
"services/hermes/scm-common/scripts",
|
||||
"services/hermes/scripts/execution_pool_client.py",
|
||||
@ -97,25 +136,7 @@
|
||||
"services/hermes/scripts/node_account_hardening.py",
|
||||
"services/hermes/scripts/node_account_io.py",
|
||||
"services/hermes/scripts/stage_runtime_access.py",
|
||||
"services/hermes/scripts/cli_lane_board.py",
|
||||
"services/hermes/scripts/cli_lane_capabilities.py",
|
||||
"services/hermes/scripts/cli_lane_config.py",
|
||||
"services/hermes/scripts/cli_lane_dispatch.py",
|
||||
"services/hermes/scripts/cli_lane_evidence.py",
|
||||
"services/hermes/scripts/cli_lane_execution.py",
|
||||
"services/hermes/scripts/cli_lane_files.py",
|
||||
"services/hermes/scripts/cli_lane_finalization.py",
|
||||
"services/hermes/scripts/cli_lane_goal.py",
|
||||
"services/hermes/scripts/cli_lane_prompt.py",
|
||||
"services/hermes/scripts/cli_lane_provider.py",
|
||||
"services/hermes/scripts/cli_lane_quarantine.py",
|
||||
"services/hermes/scripts/cli_lane_records.py",
|
||||
"services/hermes/scripts/cli_lane_recovery.py",
|
||||
"services/hermes/scripts/cli_lane_retention.py",
|
||||
"services/hermes/scripts/cli_lane_routing.py",
|
||||
"services/hermes/scripts/cli_lane_runner.py",
|
||||
"testing/tests",
|
||||
"testing"
|
||||
"services/hermes/scripts/node_polkit_audit.py"
|
||||
],
|
||||
"pytest_suites": {
|
||||
"unit": {
|
||||
@ -128,12 +149,14 @@
|
||||
"junit": "build/junit-unit.xml",
|
||||
"coverage_sources": [
|
||||
"ci/scripts",
|
||||
"dockerfiles",
|
||||
"scripts.render.dashboards_render_atlas",
|
||||
"services/gitea/scripts",
|
||||
"services/hermes/scm-common/scripts",
|
||||
"services/harbor/scripts",
|
||||
"services/hermes/scripts",
|
||||
"services/mailu/scripts",
|
||||
"testing"
|
||||
"testing",
|
||||
"services/gitea/scripts",
|
||||
"services/hermes/scm-common/scripts"
|
||||
],
|
||||
"coverage_xml": "build/coverage-unit.xml"
|
||||
},
|
||||
@ -198,6 +221,9 @@
|
||||
"ci/tests/**/*.py",
|
||||
"scripts/tests/**/*.py",
|
||||
"services/*/scripts/tests/**/*.py",
|
||||
"services/hermes/scripts/cli_lane_*.py",
|
||||
"services/mailu/scripts/mailu_sync.py",
|
||||
"services/mailu/scripts/mailu_sync_listener.py",
|
||||
"services/gitea/scripts/gitea_branch_protection_check.py",
|
||||
"services/hermes/scm-common/scripts/*.py",
|
||||
"services/hermes/scripts/execution_pool_*.py",
|
||||
@ -206,9 +232,7 @@
|
||||
"services/hermes/scripts/node_account_hardening.py",
|
||||
"services/hermes/scripts/node_account_io.py",
|
||||
"services/hermes/scripts/stage_runtime_access.py",
|
||||
"services/hermes/scripts/cli_lane_*.py",
|
||||
"services/mailu/scripts/mailu_sync.py",
|
||||
"services/mailu/scripts/mailu_sync_listener.py"
|
||||
"services/hermes/scripts/node_polkit_audit.py"
|
||||
],
|
||||
"naming_rules": [
|
||||
{
|
||||
@ -250,88 +274,33 @@
|
||||
},
|
||||
"coverage": {
|
||||
"minimum_percent": 95.0,
|
||||
"minimum_branch_percent": 95.0,
|
||||
"branch_tracked_files": [
|
||||
"services/gitea/scripts/gitea_branch_protection_check.py",
|
||||
"services/hermes/scm-common/scripts/gitea_api.py",
|
||||
"services/hermes/scm-common/scripts/gitea_api_policy.py",
|
||||
"services/hermes/scm-common/scripts/scm_broker.py",
|
||||
"services/hermes/scm-common/scripts/scm_broker_client.py",
|
||||
"services/hermes/scm-common/scripts/scm_broker_io.py",
|
||||
"services/hermes/scm-common/scripts/scm_broker_server.py",
|
||||
"services/hermes/scripts/hermes_coordinator.py",
|
||||
"services/hermes/scripts/node_account_audit.py",
|
||||
"services/hermes/scripts/node_account_hardening.py",
|
||||
"services/hermes/scripts/node_account_io.py",
|
||||
"services/hermes/scripts/stage_runtime_access.py",
|
||||
"services/hermes/scripts/execution_pool_client.py",
|
||||
"services/hermes/scripts/execution_pool_coordinator.py",
|
||||
"services/hermes/scripts/execution_pool_maintenance.py",
|
||||
"services/hermes/scripts/execution_pool_project.py",
|
||||
"services/hermes/scripts/execution_pool_protocol.py",
|
||||
"services/hermes/scripts/execution_pool_scm.py",
|
||||
"services/hermes/scripts/execution_pool_server.py",
|
||||
"services/hermes/scripts/execution_pool_store.py",
|
||||
"services/hermes/scripts/execution_pool_worker.py",
|
||||
"services/hermes/scripts/cli_lane_board.py",
|
||||
"services/hermes/scripts/cli_lane_capabilities.py",
|
||||
"services/hermes/scripts/cli_lane_config.py",
|
||||
"services/hermes/scripts/cli_lane_dispatch.py",
|
||||
"services/hermes/scripts/cli_lane_evidence.py",
|
||||
"services/hermes/scripts/cli_lane_execution.py",
|
||||
"services/hermes/scripts/cli_lane_files.py",
|
||||
"services/hermes/scripts/cli_lane_finalization.py",
|
||||
"services/hermes/scripts/cli_lane_goal.py",
|
||||
"services/hermes/scripts/cli_lane_prompt.py",
|
||||
"services/hermes/scripts/cli_lane_provider.py",
|
||||
"services/hermes/scripts/cli_lane_quarantine.py",
|
||||
"services/hermes/scripts/cli_lane_records.py",
|
||||
"services/hermes/scripts/cli_lane_recovery.py",
|
||||
"services/hermes/scripts/cli_lane_retention.py",
|
||||
"services/hermes/scripts/cli_lane_routing.py",
|
||||
"services/hermes/scripts/cli_lane_runner.py",
|
||||
"testing/quality_coverage.py"
|
||||
],
|
||||
"tracked_files": [
|
||||
"ci/scripts/hermes_image_release.py",
|
||||
"dockerfiles/hermes-kaniko-heredoc-runner.py",
|
||||
"services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py",
|
||||
"services/hermes/scripts/jenkins_image_build_trigger.py",
|
||||
"ci/scripts/publish_test_metrics.py",
|
||||
"ci/scripts/publish_test_metrics_quality.py",
|
||||
"ci/scripts/semgrep_report.py",
|
||||
"ci/scripts/supply_chain_report.py",
|
||||
"services/mailu/scripts/mailu_sync.py",
|
||||
"services/mailu/scripts/mailu_sync_listener.py",
|
||||
"services/gitea/scripts/gitea_branch_protection_check.py",
|
||||
"services/hermes/scm-common/scripts/gitea_api.py",
|
||||
"services/hermes/scm-common/scripts/gitea_api_policy.py",
|
||||
"services/hermes/scm-common/scripts/scm_broker.py",
|
||||
"services/hermes/scm-common/scripts/scm_broker_client.py",
|
||||
"services/hermes/scm-common/scripts/scm_broker_io.py",
|
||||
"services/hermes/scm-common/scripts/scm_broker_server.py",
|
||||
"services/hermes/scripts/execution_pool_client.py",
|
||||
"services/hermes/scripts/execution_pool_coordinator.py",
|
||||
"services/hermes/scripts/execution_pool_maintenance.py",
|
||||
"services/hermes/scripts/execution_pool_project.py",
|
||||
"services/hermes/scripts/execution_pool_protocol.py",
|
||||
"services/hermes/scripts/execution_pool_scm.py",
|
||||
"services/hermes/scripts/execution_pool_server.py",
|
||||
"services/hermes/scripts/execution_pool_store.py",
|
||||
"services/hermes/scripts/execution_pool_worker.py",
|
||||
"services/hermes/scripts/hermes_coordinator.py",
|
||||
"services/hermes/scripts/node_account_audit.py",
|
||||
"services/hermes/scripts/node_account_hardening.py",
|
||||
"services/hermes/scripts/node_account_io.py",
|
||||
"services/hermes/scripts/stage_runtime_access.py",
|
||||
"services/hermes/scripts/cli_lane_board.py",
|
||||
"services/hermes/scripts/cli_lane_capabilities.py",
|
||||
"services/hermes/scripts/cli_lane_config.py",
|
||||
"services/hermes/scripts/cli_lane_dispatch.py",
|
||||
"services/hermes/scripts/cli_lane_evidence.py",
|
||||
"services/hermes/scripts/cli_lane_execution.py",
|
||||
"services/hermes/scripts/cli_lane_failover.py",
|
||||
"services/hermes/scripts/cli_lane_files.py",
|
||||
"services/hermes/scripts/cli_lane_finalization.py",
|
||||
"services/hermes/scripts/cli_lane_goal.py",
|
||||
"services/hermes/scripts/cli_lane_health.py",
|
||||
"services/hermes/scripts/cli_lane_metrics.py",
|
||||
"services/hermes/scripts/cli_lane_prompt.py",
|
||||
"services/hermes/scripts/cli_lane_provider.py",
|
||||
"services/hermes/scripts/cli_lane_quarantine.py",
|
||||
"services/hermes/scripts/cli_lane_quota.py",
|
||||
"services/hermes/scripts/cli_lane_records.py",
|
||||
"services/hermes/scripts/cli_lane_recovery.py",
|
||||
"services/hermes/scripts/cli_lane_retention.py",
|
||||
@ -341,7 +310,32 @@
|
||||
"testing/quality_docs.py",
|
||||
"testing/quality_hygiene.py",
|
||||
"testing/quality_coverage.py",
|
||||
"testing/quality_gate.py"
|
||||
"testing/quality_gate.py",
|
||||
"services/gitea/scripts/gitea_branch_protection_check.py",
|
||||
"services/hermes/scm-common/scripts/gitea_api.py",
|
||||
"services/hermes/scm-common/scripts/gitea_api_policy.py",
|
||||
"services/hermes/scm-common/scripts/scm_broker.py",
|
||||
"services/hermes/scm-common/scripts/scm_broker_client.py",
|
||||
"services/hermes/scm-common/scripts/scm_broker_io.py",
|
||||
"services/hermes/scm-common/scripts/scm_broker_server.py",
|
||||
"services/hermes/scripts/execution_pool_client.py",
|
||||
"services/hermes/scripts/execution_pool_coordinator.py",
|
||||
"services/hermes/scripts/execution_pool_maintenance.py",
|
||||
"services/hermes/scripts/execution_pool_project.py",
|
||||
"services/hermes/scripts/execution_pool_protocol.py",
|
||||
"services/hermes/scripts/execution_pool_scm.py",
|
||||
"services/hermes/scripts/execution_pool_server.py",
|
||||
"services/hermes/scripts/execution_pool_store.py",
|
||||
"services/hermes/scripts/execution_pool_worker.py",
|
||||
"services/hermes/scripts/hermes_coordinator.py",
|
||||
"services/hermes/scripts/node_account_audit.py",
|
||||
"services/hermes/scripts/node_account_hardening.py",
|
||||
"services/hermes/scripts/node_account_io.py",
|
||||
"services/hermes/scripts/stage_runtime_access.py",
|
||||
"services/hermes/scm-common/scripts/deadline_http.py",
|
||||
"services/hermes/scm-common/scripts/git_pack_objects.py",
|
||||
"services/hermes/scm-common/scripts/receive_pack_scan.py",
|
||||
"services/hermes/scripts/node_polkit_audit.py"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@ -3,31 +3,23 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import xml.etree.ElementTree as ET
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CoverageRates:
|
||||
"""Line and branch percentages reported for one source file."""
|
||||
|
||||
line: float
|
||||
branch: float | None
|
||||
|
||||
|
||||
def _load_percentages(xml_path: Path, root: Path) -> dict[str, CoverageRates]:
|
||||
"""Load per-file line and branch percentages from a Cobertura report."""
|
||||
def _load_rates(xml_path: Path, root: Path) -> dict[str, dict[str, float | None]]:
|
||||
"""Load per-file line and branch percentages from a Cobertura XML report."""
|
||||
tree = ET.parse(xml_path)
|
||||
xml_root = tree.getroot()
|
||||
source_roots = [
|
||||
Path(node.text) for node in xml_root.findall("./sources/source") if node.text
|
||||
Path(node.text)
|
||||
for node in xml_root.findall("./sources/source")
|
||||
if node.text
|
||||
]
|
||||
percentages: dict[str, CoverageRates] = {}
|
||||
rates: dict[str, dict[str, float | None]] = {}
|
||||
for class_node in xml_root.findall(".//class"):
|
||||
filename = class_node.attrib.get("filename")
|
||||
line_rate = class_node.attrib.get("line-rate")
|
||||
branch_rate = class_node.attrib.get("branch-rate")
|
||||
if not filename or line_rate is None:
|
||||
continue
|
||||
normalized = filename.replace("\\", "/")
|
||||
@ -40,58 +32,44 @@ def _load_percentages(xml_path: Path, root: Path) -> dict[str, CoverageRates]:
|
||||
if candidate.exists():
|
||||
key = candidate.relative_to(root).as_posix()
|
||||
break
|
||||
percentages[key] = CoverageRates(
|
||||
line=float(line_rate) * 100.0,
|
||||
branch=float(branch_rate) * 100.0 if branch_rate is not None else None,
|
||||
)
|
||||
return percentages
|
||||
branch_rate = class_node.attrib.get("branch-rate")
|
||||
rates[key] = {
|
||||
"line": float(line_rate) * 100.0,
|
||||
"branch": None if branch_rate is None else float(branch_rate) * 100.0,
|
||||
}
|
||||
return rates
|
||||
|
||||
|
||||
def run_check(contract: dict[str, Any], root: Path, xml_path: Path) -> list[str]:
|
||||
"""Return human-readable issues for tracked files below the coverage floor.
|
||||
|
||||
The report is intentionally per-file so a single weak module cannot hide
|
||||
behind aggregate suite coverage.
|
||||
Line and branch rates are both enforced per file so a single weak module
|
||||
cannot hide behind aggregate suite coverage or line-only reporting.
|
||||
"""
|
||||
if not xml_path.exists():
|
||||
return [f"coverage xml missing: {xml_path.relative_to(root)}"]
|
||||
|
||||
percentages = _load_percentages(xml_path, root)
|
||||
rates = _load_rates(xml_path, root)
|
||||
minimum = float(contract.get("coverage", {}).get("minimum_percent", 95.0))
|
||||
coverage_contract = contract.get("coverage", {})
|
||||
branch_minimum = coverage_contract.get("minimum_branch_percent")
|
||||
if branch_minimum is not None:
|
||||
branch_minimum = float(branch_minimum)
|
||||
branch_tracked = {
|
||||
path.replace("\\", "/")
|
||||
for path in coverage_contract.get(
|
||||
"branch_tracked_files",
|
||||
coverage_contract.get("tracked_files", [])
|
||||
if branch_minimum is not None
|
||||
else [],
|
||||
)
|
||||
}
|
||||
issues: list[str] = []
|
||||
|
||||
for relative_path in contract.get("coverage", {}).get("tracked_files", []):
|
||||
normalized = relative_path.replace("\\", "/")
|
||||
rates = percentages.get(normalized)
|
||||
if rates is None:
|
||||
file_rates = rates.get(normalized)
|
||||
if file_rates is None:
|
||||
issues.append(f"coverage missing for tracked file: {relative_path}")
|
||||
continue
|
||||
if rates.line + 1e-9 < minimum:
|
||||
issues.append(
|
||||
f"coverage below {minimum:.1f}%: {relative_path} ({rates.line:.1f}%)"
|
||||
)
|
||||
if branch_minimum is None or normalized not in branch_tracked:
|
||||
continue
|
||||
if rates.branch is None:
|
||||
issues.append(f"branch coverage missing for tracked file: {relative_path}")
|
||||
elif rates.branch + 1e-9 < branch_minimum:
|
||||
issues.append(
|
||||
"branch coverage below "
|
||||
f"{branch_minimum:.1f}%: {relative_path} ({rates.branch:.1f}%)"
|
||||
)
|
||||
for metric in ("line", "branch"):
|
||||
percent = file_rates[metric]
|
||||
if percent is None:
|
||||
issues.append(
|
||||
f"{metric} coverage missing for tracked file: {relative_path}"
|
||||
)
|
||||
elif percent + 1e-9 < minimum:
|
||||
issues.append(
|
||||
f"{metric} coverage below {minimum:.1f}%: "
|
||||
f"{relative_path} ({percent:.1f}%)"
|
||||
)
|
||||
|
||||
return issues
|
||||
|
||||
@ -106,13 +84,13 @@ def compute_workspace_line_coverage(
|
||||
if not xml_path.exists():
|
||||
return 0.0
|
||||
|
||||
percentages = _load_percentages(xml_path, root)
|
||||
rates = _load_rates(xml_path, root)
|
||||
samples: list[float] = []
|
||||
for relative_path in contract.get("coverage", {}).get("tracked_files", []):
|
||||
normalized = relative_path.replace("\\", "/")
|
||||
rates = percentages.get(normalized)
|
||||
if rates is not None:
|
||||
samples.append(rates.line)
|
||||
file_rates = rates.get(normalized)
|
||||
if file_rates is not None:
|
||||
samples.append(file_rates["line"])
|
||||
if not samples:
|
||||
return 0.0
|
||||
return round(sum(samples) / len(samples), 3)
|
||||
|
||||
@ -332,7 +332,7 @@ def run_profile(
|
||||
results.append(
|
||||
_result(
|
||||
"coverage",
|
||||
"Per-file 95% coverage floor for tracked quality-managed modules.",
|
||||
"Per-file 95% line and branch coverage floor for tracked quality-managed modules.",
|
||||
_status_from_issues(issues),
|
||||
issues=issues,
|
||||
coverage_xml=str(coverage_xml.relative_to(root)),
|
||||
|
||||
78
testing/tests/data/hermes_kanban_card_corpus.jsonl
Normal file
78
testing/tests/data/hermes_kanban_card_corpus.jsonl
Normal file
@ -0,0 +1,78 @@
|
||||
{"board": "cassandra", "body": "In /opt/data/workspace/projects/cassandra-hermes-v69, add focused regression tests using only generic fictional card/deck/commander metadata. Before changing production code, verify each new test fails against the current implementation. Cover: (1) reject unsupported exact-cost wording including \"requires {3}{G}{G} to cast\" and \"need five mana to cast\"; (2) reject exhaustive whole-deck statements such as \"every card in the deck leaves the commander cost unchanged,\" including superficial supplied-metadata hedges that still quantify over the entire deck; (3) prove that deterministic claim repair which corrects a commander name is followed by semantic-compliance validation, so a newly introduced exact-cost claim cannot bypass checks; and (4) permit explicitly polarity-reversed caveats/negations such as \"do not assume no card in the whole deck changes the cost.\" Preserve and extend nearby test style and assertions without weakening fail-closed entry-hazard, unknown-card, legal-target, or provenance coverage. Do not implement production fixes, commit, push, deploy, or provide a final structured task result; leave the RED tests and report their locations and observed failures to the implementation owner.", "expected_role": "implementation", "id": "t_14985b68", "title": "Add RED regressions for the four semantic-retry compliance bypasses"}
|
||||
{"board": "cassandra", "body": "First read docs/agent_collaboration.md and docs/handoff_hermes_claude_20260808.md. Perform an independent high-tier Claude review of the Codex-authored implementation and live evidence for COLLAB-001, COLLAB-002, and COLLAB-011. Check persistence and integrity of both hash types, determinism and exactness of action-ranking replay, worker_app_version provenance, deployment/version alignment with 0.9.63, and whether recorded evidence actually supports each claim. Do not author implementation; identify findings with severity, exact reproduction or inspection commands, and explicit certify/conditional-certify/reject recommendation. Record review artifacts, model/provider/effort, and capacity blockers.", "expected_role": "implementation", "id": "t_15498b8f", "title": "Independently review v69 certification implementation and evidence"}
|
||||
{"board": "cassandra", "body": "Repair the SAME open atlas/cassandra PR #1 branch handoff/generated-strategy-audit-20260813 starting from exact independently reviewed head b684e6ab6abebd19cbb4a434a8c4fa0e5b7015d6 and API-confirmed base ee1a5f96b62e38e1179eff3cbacd4bcf5ce4464c. Parent t_15f2a75c is authoritative; reproduce every exact finding RED end-to-end through section_pipeline with PYTHONPATH explicitly pinned to the intended checked-out tree before modifying production code. Make the smallest structural repair for all four findings without regressing previously verified behavior: (1) normalize/strip Markdown emphasis consistently BEFORE table numeric/cost scanning so _nine_, __nine__, _9_, _nine mana_ cannot hide false values while ordinary identifiers/words and correct decorated/symbolic values remain correct. (2) Do not use a five-name closed header list to decide whether a cost table is scoped. Recognize realistic card identity headers such as Commander, Cards, Spell Name and variants, and preferably bind rows by exact known-card occurrence when a casting/mana-value/CMC cost column exists; ambiguous cost rows must fail closed, activation-only rows remain occurrence-locally exempt. (3) Analyze exact-cost claim text in every relevant table cell, including the nominal card cell and GFM-absorbed pipe-bearing prose/list rows directly following a table, so the exact reviewer examples block; do not join headings, fences, blank-separated paragraphs or disconnected genuine tables, and do not create false positives for ordinary non-claim cell text. (4) Replace the two-word sixth/tenth special case with a grammar/fail-closed exact-cost cue: any lexical/numeric phrase asserted as mana value/CMC/casting cost that is not a proved supported value must block, including zeroth/fourth/seventh/eighth/eleventh and cardinals twenty/twenty-five/thirty/one hundred, without treating often/attention/none/someone/Stoneforge/one-shot/twofold or cautious/negated prose as an exact claim. Add focused tracked regressions for all parent repros, nearby header/emphasis/GFM/number boundaries, and positive controls. Preserve the prior fixes for four/fourteen, six/sixteen, ten/tenth/often, malformed width, activation scope, snapshot fail-closed, one index per call, bounded runtime, unrelated tracked/untracked files and inherited audit hashes. Run focused compliance, semantic/entry, Program, deterministic acceptance and full dynamic authoring suites. For every head comparison and test invocation, pin PYTHONPATH to that exact worktree and prove the loaded module __file__ belongs to it; repeat exact old-head RED and neutered-gate RED. Run py_compile, discovery, diff/LOC, and Ruff if truly available. Commit and push one focused follow-up to the same existing PR branch, verify local/remote/refs-pull/API exact head equality and open/unmerged/undeployed state. Do not merge, deploy, comment on the PR, or modify unrelated files.", "expected_role": "implementation", "id": "t_15ed4f67", "title": "Repair Cassandra PR 1 after b684e6ab review"}
|
||||
{"board": "cassandra", "body": "Independently review Cassandra PR #1 at exact head b684e6ab6abebd19cbb4a434a8c4fa0e5b7015d6 against base 4a0ea87da71491d02a5ea65de39fa7c613de886d and the four findings in parent review t_b3a79a92. Use Claude Code subscription OAuth only through the assigned cli-claude-xhigh route; do not use any metered API key. Work read-only in a newly created clean detached worktree at the exact head; never edit the implementation workspace, commit, push, comment, merge, deploy, or mutate Kanban except returning this task result. Do not trust the parent completion result, docs, done labels, claimed counts, or existing tests as truth. Inspect the complete base-to-head diff including every new and untracked implementation/test file in the PR. Reproduce the exact prior false-acceptance and bypass inputs: four versus fourteen, six versus sixteen, ten versus tenth and often, ordinary words/card names containing number substrings, wrapped continuation lines containing ordinary pipes, and under-width/wide/misaligned Markdown rows. Add temporary independent probes outside the repo when needed; do not retain them. Verify correct decorated/prose costs, activation exemptions, genuine tables, headings/blank paragraphs/fences, fail-closed snapshot and exception paths, bounded/cardinality/index reuse/runtime behavior, test discovery and non-vacuity. Run focused compliance, semantic/entry, Program, deterministic acceptance, broad authoring suites, old-head RED and neutered-gate RED where independently meaningful. Verify exact local/remote/pull/API head equality and that PR remains open/unmerged/undeployed without exposing credentials. Return a strict findings-first SHIP or BLOCK result with exact file/line and executable repro evidence for every issue; if no defect is found, state residual risks and the evidence supporting SHIP.", "expected_role": "review", "id": "t_15f2a75c", "title": "Independent release review of Cassandra b684e6ab"}
|
||||
{"board": "cassandra", "body": "Repair every material finding from parent review t_4a0eb5f2 on private atlas/cassandra PR #1, exact starting head 49411de1df18cb8f8f9f792875175a65e9000392. Work on the existing handoff/generated-strategy-audit-20260813 branch and canonical atlas/cassandra remote.\n\nPreserve the four inherited untracked files byte-for-byte and do not stage or commit them:\n- docs/generated_strategy_job_audit.md 0e5c9160...\n- scripts/audit_generated_strategy_job.py 8259a7d8...\n- tests/test_audit_generated_strategy_job.py dbd2d776...\n- uv.lock f2d7ace7...\n\nImplement general, bounded repairs rather than phrase-specific patches:\n1. Parse valid Markdown tables structurally with optional outer pipes, escaped-pipe awareness, column-order independence, and per-cell occurrence scope. Distinguish activation-cost columns from casting/mana-cost columns even in mixed rows and misleading compound headers. Keep malformed input fail-closed.\n2. Cover common exact casting-cost constructions, including subject-first, passive, CMC/mana-value, colon, and existing forms, without flagging activation-cost claims.\n3. Bound unknown/caveat scope across subordinating and coordinating clause boundaries including although/while/but/semicolon.\n4. Cover common whole-deck closure constructions under incomplete metadata.\n5. Never let a strategy antecedent waive explicitly cost-typed anaphors; accept natural explicit strategy antecedents such as “Our strategy centers on … changes it” while keeping bare or ambiguous anaphors fail-closed.\n6. Correct documentation claims to match actual generalized behavior.\n\nBefore production edits, add tracked fictional regressions that reproduce every HIGH/MEDIUM finding and the cautious controls. Preserve ordinary valid prose/table positives. Keep every changed production/test file below 500 LOC by cohesive domain splits, not part/chunk naming. Avoid named-card production branches.\n\nRun the expanded focused compliance and release-blocker suites, semantic-review suite, Program-quality suite, structure/migration/docs/LOC/static/frontend/diff checks, bounded runtime probes, and a conclusive broad strategy-authoring discovery suite because the grammar surface changes broadly. Clearly distinguish newly run evidence from inherited evidence.\n\nWhen all required gates pass, update the collaboration/devlog evidence accurately, commit only intended tracked files, push the same branch, and verify local HEAD, remote branch, and open PR head match. Do not merge, deploy, certify live guides, or comment in Gitea. If work remains or any required test is still running, return incomplete so the goal loop continues.", "expected_role": "implementation", "id": "t_1e473bd7", "title": "Repair Cassandra PR 1 findings from independent v4 review"}
|
||||
{"board": "cassandra", "body": "Runtime-only acceptance canary for the external Kanban goal loop. Do not run tools or commands, read or edit files, access the network, or mutate any external state. On the first provider turn, return the required structured result with status=incomplete, summary exactly PHASE_ONE_INCOMPLETE, empty changed_files, empty artifacts, empty blockers, and tests_run containing goal-loop canary phase one. On the resumed provider turn, return status=completed, summary exactly GOAL_LOOP_OK, empty changed_files, empty artifacts, empty blockers, and tests_run containing goal-loop canary: passed. Do not return completed on the first turn. Completion is valid only after Hermes rejects the first report and resumes this same card.", "expected_role": "implementation", "id": "t_21496dc5", "title": "Goal-loop completion canary"}
|
||||
{"board": "cassandra", "body": "Review the implemented semantic-repair correction in /opt/data/workspace/projects/cassandra-hermes-v69 against the regression requirements and run the required validation without changing behavior unless a test infrastructure issue requires a clearly justified minimal fix. Execute the focused semantic-repair/generator tests, the relevant broad generator suite, and the complete 9/9 acceptance check required by project guidance. Confirm that the repair remains generic, bounded, fail-closed for entry hazards and unknown cards, restricted to legal targets, and provenance-backed. Capture exact commands, pass/fail counts, and any skips or environmental limitations for handoff. Do not commit, push, or deploy.", "expected_role": "implementation", "id": "t_237244f2", "title": "Run semantic-repair generator and acceptance verification"}
|
||||
{"board": "cassandra", "body": "In /opt/data/workspace/projects/cassandra-hermes-v69, read AGENTS.md and docs/agent_collaboration.md before editing. Locate the semantic-repair and generator test harnesses, then add metadata-driven fictional regression coverage that first fails against the current behavior. Cover: (1) retry repair must fail closed when a candidate route retains or introduces an entry hazard, including a hazardous ETB/entry route analogous to the reported Lazav failure; and (2) retry repair must not retain or introduce unsupported exact commander-cost or whole-deck closure claims when required facts/provenance are incomplete, analogous to the Goreclaw failure. Ensure fixtures and assertions are generic and do not name or special-case production cards. Preserve coverage for unknown-card, legal-target, provenance, and fail-closed hazard gates. Run the focused new tests and record the pre-implementation failing result in your task report. Do not commit, push, or deploy.", "expected_role": "implementation", "id": "t_28c3fe7a", "title": "Add generic semantic-retry compliance regression tests"}
|
||||
{"board": "cassandra", "body": "Repair the same open Gitea atlas/cassandra PR #1 on branch handoff/generated-strategy-audit-20260813 in the existing workspace, starting from exact reviewed head 8c0118fc9d73878e6fc1e40a845529aa7647557a. Treat parent review t_b3a79a92 and its exact repro inputs as authoritative. Reproduce RED first, then make the smallest structural fixes for all four new findings: 1) anchor every spelled-number alternative as a complete token so fourteen cannot prove four, sixteen cannot prove six, tenth cannot prove ten, and ordinary words or card names containing one/two/three/four/five/six/seven/eight/nine/ten cannot create false cost claims; apply consistently to prose and table cost parsing. 2) Preserve correct proved decorated cells and ordinary prose such as often, attention, none, someone, component and Stoneforge while still blocking actual contradicted numeric and symbolic costs end-to-end. 3) Replace the any-pipe-is-structure heuristic with real Markdown table-row recognition so a pipe on an ordinary wrapped continuation line cannot split the claim, without joining headings, blank paragraphs, fences or genuine tables. 4) Make every table row whose width disagrees with its header fail closed, especially under-width rows that shift or omit a Casting Cost cell; preserve activation exemptions and proved well-formed rows. Add focused adversarial regressions for every exact reviewer input and nearby boundaries. Preserve all unrelated tracked/untracked files and the inherited audit-file hashes. Run the parent-specified focused compliance, semantic/entry, Program, deterministic acceptance and broad authoring suites; repeat old-head RED and neutered-gate non-vacuity; verify dynamic discovery, snapshot failure, index reuse/cardinality/runtime, py_compile, diff/LOC and Ruff if available. Commit and push one focused follow-up to the same branch. Verify local/remote branch, refs/pull/1/head and authenticated Gitea PR head all equal the new commit; PR must remain open, unmerged and undeployed. Do not merge or deploy. Status completed only after push and verification.", "expected_role": "implementation", "id": "t_2e07f244", "title": "Repair Cassandra PR 1 after 8c0118fc review"}
|
||||
{"board": "cassandra", "body": "Perform a consequential independent read-only release review of private atlas/cassandra PR #1 at exact head 825d8a46b24eb1f1dc60d705afff82eefb37d512 against exact base ee1a5f96b62e38e1179eff3cbacd4bcf5ce4464c. Do not trust prior done labels, prose, or claimed counts. Do not edit files, mutate Kanban, comment, commit, push, merge, deploy, or launch a certification job. Preserve all existing untracked files. Read AGENTS.md, the PR diff, and the COLLAB-013/COLLAB-014 sections; keep context bounded to relevant files and hunks unless evidence requires expansion. Review the full semantic repair compliance change for fail-open paths, false positives on supported or cautious prose, unsafe entry and exact-cost behavior, scope and metadata consistency, exception handling, regex/runtime risk, named-card production branches, 500-line constraints, regression coverage, and whether tests actually exercise the production path. Run small executable probes and focused tests needed to reproduce findings. Verify the PR body and exact topology. Return prioritized findings with exact file and line references and reproducible inputs. If no release-blocking defect is found, explicitly state merge readiness, residual risks, and the exact fresh persisted Lazav/Goreclaw certification still required after deployment. Do not call the live certification complete.", "expected_role": "implementation", "id": "t_35d7bb2e", "title": "Independently review Cassandra semantic-repair PR 1 for release"}
|
||||
{"board": "cassandra", "body": "First read docs/agent_collaboration.md and docs/handoff_hermes_claude_20260808.md. Using the Codex-authored certification implementation, execute the live v69 certification workflow against deployed 0.9.63 and collect durable evidence for COLLAB-001, COLLAB-002, and COLLAB-011. Verify persisted certification and semantic-review hashes can be retrieved and matched; run exact action-ranking replay and demonstrate equivalence; prove the recorded worker_app_version corresponds to the executing worker. Use small batches where games or provider work is involved, and classify zero-token provider failures as capacity blockers rather than product defects. Attach commands, raw outputs, artifact paths/hashes, model, provider, effort, and blockers; clearly distinguish successful proof from unavailable-capacity cases.", "expected_role": "implementation", "id": "t_3a16b9a2", "title": "Run live v69 certification and capture reproducible evidence"}
|
||||
{"board": "cassandra", "body": "Perform a fresh consequential independent release review of private atlas/cassandra PR #1 at exact head e6bbd147aa710bc4406129bc25dacf094e01ab5c. Use the existing /opt/data/workspace/projects/cassandra-hermes-v69 checkout read-only. First verify local HEAD, canonical branch handoff/generated-strategy-audit-20260813, refs/pull/1/head, and the PR API head are byte-identical; verify the PR base from the API and review the entire current PR diff against that base, not only the latest commit and not any parent summary.\n\nDo not edit files, stage, commit, push, comment, merge, deploy, mutate Kanban other than normal completion, or alter the four inherited untracked files. Treat all prior test claims and done labels as untrusted evidence to reproduce where proportionate.\n\nConcentrate on release correctness around all historical review findings: Markdown table header/data/separator scanning and malformed fail-closed behavior; exact-cost grammar with N-mana source/rock/pip/resource prose versus true cost/mana-value/CMC/symbolic claims; caveat colon/semicolon/coordinating-clause/no-retroactive-waiver scoping; entry-hazard safety versus positive safe/caveated controls and the documented non-repair policy; snapshot reuse and adversarial runtime bounds; import/export/public API hygiene; production placement and regression coverage. Probe for bypasses, false positives, wording variants, multiline/table escapes, cardinality/performance regressions, and unintended changed semantics outside the target.\n\nIndependently run the final focused regression modules and broader strategy-authoring suite proportionate to this central parser change, plus deterministic acceptance, static/compile/diff checks available in the workspace. If a broad test failure is unrelated, reproduce and trace it before classifying it. Do not install new unreviewed dependencies or claim unavailable tools ran.\n\nPreserve and independently re-hash these untracked files without reading sensitive content into logs:\n- docs/generated_strategy_job_audit.md expected prefix 0e5c9160\n- scripts/audit_generated_strategy_job.py expected prefix 8259a7d8\n- tests/test_audit_generated_strategy_job.py expected prefix dbd2d776\n- uv.lock expected prefix f2d7ace7\n\nReturn a prioritized finding list with exact file/line references and reproducible fictional inputs for every defect. End with an explicit VERDICT: SHIP only if no blocking/material defect remains; otherwise VERDICT: BLOCK. State residual risks and exact tests. Keep PR #1 open, unmerged, and undeployed for human review.", "expected_role": "review", "id": "t_43d91361", "title": "Independent final review of Cassandra PR 1 at e6bbd147"}
|
||||
{"board": "cassandra", "body": "Act as the consequential independent release reviewer for private atlas/cassandra PR 1 at exact head 49411de1df18cb8f8f9f792875175a65e9000392, base main ee1a5f96b62e38e1179eff3cbacd4bcf5ce4464c, branch handoff/generated-strategy-audit-20260813. Read-only review only: do not edit, stage, commit, push, merge, deploy, comment on Gitea, or mutate Kanban outside this card. First verify the canonical origin, exact local and remote heads, PR state/topology, and clean tracked worktree. Preserve and exclude from the PR review these four inherited untracked files, verifying their hashes remain unchanged: docs/generated_strategy_job_audit.md 0e5c9160adbdad370a85b186fb52f8224fa708fa2c2ec935be46c35ba65f22ee; scripts/audit_generated_strategy_job.py 8259a7d8f2c5b2029fd0f8759419ecdcfa5495878bb5985daa3930f635a8ed46; tests/test_audit_generated_strategy_job.py dbd2d77691265c2fb74c7e6805ab5743a4d7bf3c715f9b4663d4cf19f1dea97c; uv.lock f2d7ace7c76ce3d910dc5d7c6b4c6345dd245068bc66981ae835772830469580. Independently inspect every production and test diff from origin/main through the exact head; do not trust prior done labels, docs, or test claims as truth. Reproduce or refute all six original review areas: caveat/unknown-prefix assertion scoping, common whole-deck closure grammar, colon-form exact costs, ambiguous card-name prefixes, activation versus casting tables, and explicit strategy antecedents. Adversarially audit the new occurrence-scoped table logic, including identical row text in activation and casting tables, mana-only activations, embedded ordinary exact-cost prose, casting-cost headers, valid Markdown tables with outer pipes omitted or columns reordered, misleading headers, fenced examples, and escaped or malformed cells. Distinguish safety fail-open defects from cautious false positives. Run the 61 focused compliance tests, 9 release-blocker tests, 7 semantic-review tests, 9 Program-quality tests, plus small production-path probes for any uncovered variants; do not duplicate the already-conclusive 618-test full suite unless focused evidence reveals a regression requiring it. Check changed-file LOC, static/diff hygiene, docs accuracy, and bounded runtime at intended input scale. Return a prioritized review with exact file/line evidence and reproducible inputs. Say merge-ready only if no material defect remains; otherwise state not merge-ready and enumerate the minimal general repairs and missing regressions. No certification or deployment claims.", "expected_role": "implementation", "id": "t_4620ed5c", "title": "Independently review Cassandra PR 1 at 49411de1"}
|
||||
{"board": "cassandra", "body": "Finalize the independent read-only verdict for private atlas/cassandra PR #1 at exact head 49411de1df18cb8f8f9f792875175a65e9000392 and base ee1a5f96. The technical audit evidence is preserved at /opt/data/home/.hermes/kanban/boards/cassandra/logs/t_4620ed5c.log. The later t_7aee0c6c attempt changed nothing and failed only because its strict JSON schema omitted findings from required; that platform defect is now fixed.\n\nYour assigned task is the review itself. If defects reproduce, complete the review with status=completed, put each defect/risk in findings, keep blockers empty, and state that the PR is not merge-ready. Use status=blocked only if an obstacle prevents you from performing this review.\n\nDo not edit, stage, commit, push, merge, deploy, comment in Gitea, or mutate other Kanban cards. Revalidate:\n- canonical origin, exact local/remote/PR heads, tracked-clean state, and the four inherited untracked hashes;\n- exact source/line causes and representative production-path inputs from the prior audit;\n- table handling for omitted outer pipes, reordered columns, misleading headers containing Activation Cost, and a row containing both Activation Cost and Casting Cost;\n- unsupported exact-cost grammar such as “has a mana cost of five mana”, “is cast for five mana”, “CMC is five”, plus unknown-clause scoping across although/while versus but/semicolon;\n- the explicit strategy antecedent boundary “Our strategy centers on curving out, and no card in the deck changes it”;\n- cautious false positives separately (escaped activation cells, Cost to Activate, malformed cells).\n\nRerun small executable probes and one representative focused regression command; do not repeat the 618-test baseline or all expensive gates. Confirm prior green suite/hygiene evidence from the preserved log as prior evidence, not newly run. Return a prioritized release verdict with exact file/line references and reproducible inputs.", "expected_role": "review", "id": "t_4a0eb5f2", "title": "Finalize Cassandra PR 1 independent verdict under v4 contract"}
|
||||
{"board": "cassandra", "body": "Independently review private atlas/cassandra PR #1 at exact head 4a0540501e3fbee1d931af042942f0dc30d429e0 against base ee1a5f96b62e38e1179eff3cbacd4bcf5ce4464c. Read-only consequential final release review: do not edit files, mutate Kanban beyond your own task result, comment, commit, push, merge, deploy, or certify live guides. Verify the remote/PR head first. Inspect the entire diff and every changed production/test/documentation file. Do not trust prior done labels or claimed test evidence as truth; use focused read-only probes/tests as needed. Prioritize fail-open table parsing and per-cell scoping, exact-cost grammar coverage/false positives, caveat clause boundaries, whole-deck closure, anaphor/antecedent precedence, malformed input, ReDoS/runtime bounds, production-path placement, documentation accuracy, and missing adversarial tests. Report prioritized findings with exact file/line evidence and reproducible inputs. If no material defect exists, explicitly state residual risks and why evidence is sufficient. Preserve all inherited untracked files byte-for-byte.", "expected_role": "implementation", "id": "t_53cd8b4d", "title": "Independent final review of Cassandra PR 1 repair"}
|
||||
{"board": "cassandra", "body": "Research and document the exact current path to perform (after a successful certified guide exists) the remaining scorecard gates in docs/handoff_hermes_codex_20260809.md: retrieve full job export and saved guide safely from the production PostgreSQL generation queue/strategy store; create a generic saved-Program action-ranking counterfactual suite covering each lane; run the audit harness; separately queue small Forge measurements. Inspect source, deployment/IaC, and existing reports only; do not change code, queue jobs, or expose credentials. Return exact source locations, operational commands/procedures that have evidence in repo, and blockers/unknowns. It must distinguish what can be automated locally vs requires live job artifacts.", "expected_role": "implementation", "id": "t_551203be", "title": "Map certified-job retrieval and action-ranking proof workflow"}
|
||||
{"board": "cassandra", "body": "Using the completed implementation and validation evidence in /opt/data/workspace/projects/cassandra-hermes-v69, update docs/agent_collaboration.md and DEVLOG with the semantic-retry correction, the generic regression scenarios, the bounded/fail-closed behavior, and exact focused, broad-generator, and 9/9 acceptance results. Inspect git diff/status and prepare a review-blocked handoff stating changed files, exact test results/commands, and concise diff evidence that no named-card production branch was introduced. Leave all changes uncommitted; do not push or deploy.", "expected_role": "implementation", "id": "t_5b926b7c", "title": "Document results and prepare review-blocked handoff"}
|
||||
{"board": "cassandra", "body": "Finalize the independent read-only verdict for private atlas/cassandra PR #1 at exact head 49411de1df18cb8f8f9f792875175a65e9000392 and base ee1a5f96. The previous audit card t_4620ed5c completed its technical review but was parked by an old result-contract ambiguity; its full worker log is authoritative audit input at /opt/data/home/.hermes/kanban/boards/cassandra/logs/t_4620ed5c.log.\n\nYour assigned task is the review itself. If defects reproduce, complete the review with status=completed, put each defect/risk in findings, keep blockers empty, and state that the PR is not merge-ready. Use status=blocked only if an obstacle prevents you from performing this review.\n\nDo not edit, stage, commit, push, merge, deploy, comment in Gitea, or mutate other Kanban cards. Revalidate:\n- canonical origin, exact local/remote/PR heads, tracked-clean state, and the four inherited untracked hashes;\n- exact source/line causes and representative production-path inputs from the prior audit;\n- table handling for omitted outer pipes, reordered columns, misleading headers containing Activation Cost, and a row containing both Activation Cost and Casting Cost;\n- unsupported exact-cost grammar such as “has a mana cost of five mana”, “is cast for five mana”, “CMC is five”, plus unknown-clause scoping across although/while versus but/semicolon;\n- the explicit strategy antecedent boundary such as “Our strategy centers on curving out, and no card in the deck changes it”;\n- cautious false positives separately (escaped activation cells, Cost to Activate, malformed cells).\n\nRerun small executable probes and a representative focused regression command; do not repeat the 618-test baseline or all expensive gates. Confirm prior green suite/hygiene evidence from the preserved log, but do not claim it as newly run. Return a prioritized release verdict with exact file/line references and reproducible inputs.", "expected_role": "review", "id": "t_7aee0c6c", "title": "Finalize Cassandra PR 1 independent verdict at 49411de1"}
|
||||
{"board": "cassandra", "body": "Act as the sole implementation owner in /opt/data/workspace/projects/cassandra-hermes-v69. Consume the architecture findings and RED regressions from parent tasks, then implement the smallest robust production fix that closes all four bypasses: reject unsupported exact commander-cost assertions including symbolic and word-number forms; detect unqualified exhaustive whole-deck paraphrases even when padded with supplied-metadata hedges; rerun semantic compliance after deterministic claim repair before accepting repaired output; and distinguish explicitly negated/caveated universal statements so legitimate polarity-aware wording is permitted. Maintain fail-closed behavior for entry hazards, unknown cards, legal targets, and provenance. Run the new regressions and relevant nearby semantic/entry suites, iterating until GREEN. Do not run the expensive full broad suite or downstream 9/9 acceptance suite. Do not commit, push, deploy, or delegate final completion to a reviewer. Return the final structured work result yourself, including changed files, test commands/results, and any remaining risks.", "expected_role": "implementation", "id": "t_92e8e57c", "title": "Implement and validate semantic-retry compliance hardening"}
|
||||
{"board": "cassandra", "body": "Resume from clean main at 382b2c1c and deployed 0.9.63. Read docs/agent_collaboration.md and docs/handoff_hermes_claude_20260808.md first. Finish live v69 certification evidence for COLLAB-001, COLLAB-002, and COLLAB-011: persisted certification and semantic-review hashes, exact action-ranking replay, and worker_app_version proof. Preserve cross-provider separation: Codex authors or implements; Claude performs independent high-tier review; Hermes synthesizes. Only after certification is clean, unblock the 4+ decided-game measurement set. Respect small game batches and treat zero-token provider errors as capacity, not defects. Record commands, artifacts, model, provider, effort, and blockers on this task.", "expected_role": "implementation", "id": "t_97df755a", "title": "Certify and prove the deployed Lazav v69 Program"}
|
||||
{"board": "cassandra", "body": "Repair atlas/cassandra PR #1 after the independent BLOCK review t_43d91361. Work in the existing /opt/data/workspace/projects/cassandra-hermes-v69 checkout on branch handoff/generated-strategy-audit-20260813, starting at exact head e6bbd147aa710bc4406129bc25dacf094e01ab5c. First verify local HEAD, canonical remote branch, refs/pull/1/head, and PR API head/base. Preserve all unrelated and untracked work; re-hash these four inherited files before and after without logging contents: docs/generated_strategy_job_audit.md prefix 0e5c9160, scripts/audit_generated_strategy_job.py 8259a7d8, tests/test_audit_generated_strategy_job.py dbd2d776, uv.lock f2d7ace7. Do not merge or deploy.\n\nTreat the exact durable completed result of review t_43d91361 as authoritative findings to reproduce, not as implementation instructions to copy blindly. Add RED tests first and make the smallest structural fixes for all four material findings:\n1. Cost claims split by ordinary Markdown soft-wrap newlines currently bypass every grammar family. Scan paragraph-continuation whitespace without joining headings, blank-separated paragraphs, or table structure; test symbolic, CMC/mana-value, and numeric claims end-to-end through the accepting-critic/reused-chunk path.\n2. Casting-cost table cells only block when the value is the entire cell. Parse the already escape-aware cell value semantically so leading/trailing prose, parentheticals, reductions notes, numeric mana text, and escaped-pipe suffixes cannot hide a contradicted cost; preserve activation-cost table exemptions and fail closed on malformed tables.\n3. The resource-noun tail heuristic incorrectly cancels claims that explicitly say N mana when a later adjunct mentions turns/lands/creatures/cards/opponents. Explicit mana units must remain cost assertions; preserve positive non-cost controls for mana sources, pips, rocks, dorks, life, and other activation/resource prose.\n4. Safe conditional entry-hazard guidance such as “Only cast Catastrophic Colossus with two expendable bodies already on board” and “Cast ... only when you can pay its entry sacrifice” is a pre-existing classifier false positive that this PR newly turns into an unrecoverable hard failure on accepted/reused chunks. Refine the classifier or compliance routing so correct protective conditions survive while truly unsafe “without paying/ignoring the sacrifice/entry drawback” guidance still blocks. Ensure a repair path exists where appropriate without weakening fail-closed entry-hazard enforcement.\n\nAlso assess and fix the low adjacent-table inheritance defect if it can be done safely in the same parser change; do not broaden lexical grammar merely to chase the documented idiom boundary. Remove or replace the _NON_CASE_MODULES support-module exclusion if it can silently hide future tests. Do not add grammar-only patches for structural escapes.\n\nAcceptance: reproduce each reported case RED before implementation; add adversarial positive/negative controls; run the seven focused compliance modules (prior baseline 90), Program quality (9), semantic-review plus entry-hazard modules (55), deterministic acceptance (9/9), and the broad tests.test_strategy_authoring_ai suite (647 baseline). Repeat vacuity/structural probes proportionately, py_compile/diff/LOC/ruff if available, and runtime/cardinality bounds including aggregate cost-index reuse if changed. Keep every source/test file below repository limits. Commit and push one focused repair to the same branch so PR #1 advances; verify remote branch/pull ref/API head equal the new commit and PR remains open, unmerged, undeployed. Report exact changed files/tests and any unavailable tools. Status is completed only after push and verification; use incomplete if tests still run.", "expected_role": "implementation", "id": "t_9826e3ce", "title": "Repair Cassandra PR 1 structural compliance escapes after review 53"}
|
||||
{"board": "cassandra", "body": "Repair the four reproduced semantic-retry compliance failures in /opt/data/workspace/projects/cassandra-hermes-v69 using TDD and generic fictional metadata only. Add regressions for: (1) unsupported commander exact-cost phrasing such as requires {3}{G}{G} to cast and need five mana to cast; (2) exhaustive whole-deck paraphrases such as every card in the deck leaves the commander cost unchanged, including superficial supplied-metadata hedges that still quantify over the whole deck; (3) recomputing semantic compliance after deterministic claim repair so a corrected commander name cannot introduce an unchecked exact-cost claim; and (4) polarity-aware handling that permits explicit caveats/negations such as do not assume no card in the whole deck changes the cost. Preserve fail-closed entry-hazard, unknown-card, legal-target, and provenance behavior. Run the new RED/GREEN regressions plus nearby semantic/entry suites. Do not run the expensive full broad suite here; the downstream acceptance task will run focused, broad, and 9/9 after this fix. Do not commit, push, or deploy. Do not delegate Kanban task completion to a review child: delegated reviewers may only return findings to the foreground worker; only the foreground worker may produce the final structured task result.", "expected_role": "implementation", "id": "t_abafcb10", "title": "Close reproduced semantic-retry compliance bypasses"}
|
||||
{"board": "cassandra", "body": "Independently review the untracked generated Strategy audit harness in `/opt/data/workspace/projects/cassandra-hermes-v69`: `scripts/audit_generated_strategy_job.py`, `tests/test_audit_generated_strategy_job.py`, and `docs/generated_strategy_job_audit.md`. Review against docs/handoff_hermes_codex_20260809.md acceptance scorecard 5/6 and actual repository contracts. Identify correctness gaps, especially dynamic semantic-review batch requirements, exact Program extraction/hashing/equality, ranking-suite validity/coverage, and whether an audit can falsely return OK. Do not edit files. Record an evidence-backed verdict with paths/lines and exact test commands; if sound, say so. This must not claim live certification proof.", "expected_role": "review", "id": "t_aefd1147", "title": "Independently review untracked generated-strategy audit harness"}
|
||||
{"board": "cassandra", "body": "Implement the current handoff critical-path correction on `/opt/data/workspace/projects/cassandra-hermes-v69`, following AGENTS.md and docs/agent_collaboration.md. Current deployed 0.9.65 failures: Lazav retry 8 job `4691c3a7-357b-43a3-88ec-d8a923d9708c` failed because semantic repair retained/introduced unsafe Phyrexian Dreadnought and Phage entry routes; Goreclaw retry 7 `982e698d-17a2-4515-9263-13267566626d` failed because repair retained unsupported exact commander cost and whole-deck closure claims with incomplete facts. Work strictly TDD: reproduce generic fictional metadata-driven failures first, see the new tests fail, then implement minimal generic bounded semantic-repair compliance. Preserve fail-closed entry hazard, unknown-card, legal-target, and provenance gates; no production named-card branches. Run focused and broad generator suites plus 9/9 acceptance, update docs/agent_collaboration.md + DEVLOG with exact results. Do not commit/push/deploy; leave reviewable diff. Block for review with changed files/test result/diff evidence. ", "expected_role": "implementation", "id": "t_af7738f5", "title": "Repair semantic-retry compliance for live Lazav and Goreclaw failures"}
|
||||
{"board": "cassandra", "body": "Review the Codex implementation/evidence and the independent Claude review under docs/agent_collaboration.md and docs/handoff_hermes_claude_20260808.md. Produce the Hermes synthesis decision for live v69 certification of COLLAB-001, COLLAB-002, and COLLAB-011: reconcile discrepancies, validate that cross-provider separation was maintained, enumerate final artifact hashes and locations, and state a clear clean/pass, conditional, or blocked verdict. Authorize the decided-game measurement set only on a clean certification verdict; otherwise specify precise remediation and retain the gate. Record commands consulted, artifacts, model/provider/effort, and blockers.", "expected_role": "implementation", "id": "t_b0a1b0b5", "title": "Synthesize certification decision and authorize measurement gate"}
|
||||
{"board": "cassandra", "body": "Perform a fresh consequential independent read-only release review of private Gitea atlas/cassandra PR #1 at exact head 8c0118fc9d73878e6fc1e40a845529aa7647557a against exact API base and merge-base ee1a5f96b62e38e1179eff3cbacd4bcf5ce4464c. This is scm.bstein.dev Gitea, not GitHub: do not use GitHub/gh skills or treat gh auth as relevant. The supplied workspace is a clean detached worktree. Do not edit files, commit, push, comment, approve, merge, deploy, mutate Kanban other than completing this assigned card, create pods, or expose credentials. Verify local head, canonical branch, pull ref, authenticated Gitea API head/base/state and merge-base before trusting the diff. Inspect the entire current base-to-head PR diff, with particular scrutiny of commit 8c0118fc and every reviewer finding from parent t_43d91361. Independently reproduce: ordinary Markdown soft-wrap bypasses across symbolic/numeric/CMC/mana-value grammar without joining headings, blank paragraphs or tables; decorated casting cells with prose, parentheses, reductions, numeric mana and escaped pipes while activation/proved costs remain accepted and malformed/adjacent tables fail closed; explicit N-mana claims with later turn/land/creature/card/opponent adjuncts while resource/pip/rock/dork/life/activation prose stays non-cost; safe two-body or pay-entry-sacrifice conditions through accepting critic/reused chunks while without-paying/ignore drawback remains blocked and repair path works. Check Oracle/snapshot consistency, exception behavior, scope/caveat boundaries, false positives and any nearby wording variants not in tests. Verify dynamic support-module discovery and no hidden tests. Reproduce old-head RED and neutered-gate non-vacuity. Run the focused compliance, semantic-review/entry, Program quality, deterministic acceptance, broad authoring suite, syntax/diff/LOC and runtime/cardinality/index-reuse checks independently. Ruff is optional only if truly unavailable. Preserve all files; use external temp state. Return findings first with exact file/line and reproducible inputs. SHIP only with no material defect and full evidence; otherwise BLOCK. Human review remains required and nothing may be merged/deployed.", "expected_role": "review", "id": "t_b3a79a92", "title": "Independent release review Cassandra PR 1 at 8c0118fc"}
|
||||
{"board": "cassandra", "body": "Prepare one reviewable Gitea pull request for the already committed Cassandra semantic-repair compliance correction. Read AGENTS.md, COLLAB-013, and COLLAB-014 first. Preserve every existing untracked file exactly: docs/generated_strategy_job_audit.md, scripts/audit_generated_strategy_job.py, tests/test_audit_generated_strategy_job.py, and uv.lock. Do not edit source files, create commits, push, merge, deploy, launch a certification job, or mutate Kanban beyond this card. Verify canonical base atlas/cassandra main is ee1a5f96b62e38e1179eff3cbacd4bcf5ce4464c, head branch handoff/generated-strategy-audit-20260813 is 825d8a46b24eb1f1dc60d705afff82eefb37d512, the branch is exactly one commit ahead and zero behind, no open PR already exists for this head, and git diff --check passes. Re-run the three focused unittest modules for semantic retry compliance. If every check passes, create exactly one pull request in private atlas/cassandra using the credential-safe gitea_api.py helper and shell-safe repeated --field arguments. The PR body must record exact commit SHAs and focused, broad, acceptance, vacuity, neutered-gate, py_compile, LOC, ReDoS, and diff-check evidence from COLLAB-014, and must state that a fresh persisted certification job and independent release review remain required before merge or deployment. Stop after verifying the PR exists and is open. Report the PR number and URL without credentials.", "expected_role": "implementation", "id": "t_b87ef6e5", "title": "Prepare Cassandra semantic-repair compliance pull request"}
|
||||
{"board": "cassandra", "body": "Proceed only if the Hermes synthesis artifact explicitly records a clean certification verdict. First read docs/agent_collaboration.md and docs/handoff_hermes_claude_20260808.md, then execute and document a measurement set containing at least four decided games using the certified v69/deployed 0.9.63 path. Keep game batches small, preserve all version and provenance metadata, and report per-game decision/outcome plus aggregate measurements. Treat zero-token provider errors as capacity events: log them, retry only within documented limits, and do not label them as defects. Attach commands, artifacts, model/provider/effort, and blockers to the task.", "expected_role": "implementation", "id": "t_cc83234b", "title": "Run gated 4+ decided-game v69 measurement set"}
|
||||
{"board": "cassandra", "body": "Starting from clean main at 382b2c1c, first read docs/agent_collaboration.md and docs/handoff_hermes_claude_20260808.md. As the Codex author, inspect the deployed 0.9.63/v69 certification path and implement or repair all missing support for COLLAB-001, COLLAB-002, and COLLAB-011: persisted certification hashes, persisted semantic-review hashes, deterministic exact action-ranking replay, and worker_app_version provenance. Preserve cross-provider separation; do not represent Claude review as Codex review. Add or update focused tests and produce commit/diff references plus artifact locations. Record every command, artifact, model/provider/effort metadata, and blockers in the task handoff.", "expected_role": "implementation", "id": "t_cda55d42", "title": "Implement persisted v69 certification and replay evidence"}
|
||||
{"board": "cassandra", "body": "Repair only the independently reproduced release blockers on private atlas/cassandra PR #1, exact current head 825d8a46b24eb1f1dc60d705afff82eefb37d512 on handoff/generated-strategy-audit-20260813. Work test-first in the existing linked checkout. Preserve the four inherited untracked files byte-for-byte and do not delete, stage, or rewrite them. Before production edits, add RED generalized regressions through the real _review_and_repair_semantics path for: sentence-scoped caveats so an unrelated trailing caution cannot waive an unsafe closure; unrelated unknown prefixes cannot waive a later exact-cost assertion; common closure forms including isn't and deck has no card; colon-form exact casting cost; ambiguous comma-name prefixes must not borrow a different card's facts; Markdown activation-cost tables such as Forest | {T} | Add {G}. must not be treated as casting-cost claims; and explicit strategy antecedents such as The plan is to curve out, and no card in the deck changes it must remain allowed without reopening unsafe pronoun closure. Use fictional/generic fixtures, not named-card patches. Split the 515-line changed test module into meaningful <=500-line modules and keep every changed production and test file <=500 lines. Update stale COLLAB-013/COLLAB-014/DEVLOG wording so it accurately describes the committed PR state. Keep the gate wired into the production repair path and preserve fail-closed exception behavior. Run the focused compliance suites, section semantic-review tests, new production-path adversarial probes, LOC checks that explicitly include changed tests, py_compile, git diff --check, and a bounded regex/runtime probe; expand to relevant broader tests if changes touch shared parsing. Commit and push the corrections to the same PR branch and verify exact remote topology/PR mergeability, but do not merge, deploy, launch certification, or claim Lazav/Goreclaw certification. Return exact files, red-to-green evidence, test counts/durations, new head SHA, preserved-untracked verification, and any residual risks.", "expected_role": "implementation", "id": "t_d248853a", "title": "Repair Cassandra PR 1 independent-review blockers"}
|
||||
{"board": "cassandra", "body": "Fresh independent release review of atlas/cassandra PR #1 at exact head 14c07111b6ed8d7fc529362eb34fa0afa0694325 against API-confirmed base ee1a5f96b62e38e1179eff3cbacd4bcf5ce4464c. Use Claude Code subscription OAuth only through cli-claude-xhigh; no metered API keys. Work read-only in a new detached worktree; do not edit implementation state, commit, push, comment, merge, deploy, or mutate external state. Treat parent t_15ed4f67 results as claims to verify, not truth. Pin PYTHONPATH to the detached tree for every test and print/assert imported module paths to avoid the editable-install trap. Reproduce every finding from t_15f2a75c end-to-end through section_pipeline: underscore-emphasized digits/words; realistic Commander/Cards/Spell Name and non-adjacent cost headers; claims in card cells; pipe-bearing prose/list rows adjacent to casting and activation tables; every affected ordinal; cardinals above nineteen and one-hundred prefix cases. Audit compositional number grammar for false proof/rejection across 0..hundreds, hyphenation, conjunctions, ordinals/cardinals, punctuation/Markdown and ordinary words/names/negated/cautious prose. Audit table scope for unknown headers, ambiguous/cardless/multiple-card rows, malformed widths, escaped pipes, GFM continuations, headings/fences/blank boundaries and occurrence-local activation exemptions. Recheck snapshot/exception fail-closed, one index/snapshot per call, deterministic cardinality, bounded runtime and no pathological regex behavior. Run focused compliance, semantic/entry, Program, deterministic acceptance, full dynamic discovery, exact old-head RED and neutered-gate RED with pinned module paths. Verify local/remote/pull/API exact head and PR open/unmerged/undeployed. Return strict findings-first SHIP/BLOCK with exact file/line and executable minimal repro; residual risks if SHIP. Remove temporary review resources and stop read-only.", "expected_role": "review", "id": "t_d3ef05d3", "title": "Independent release review of Cassandra 14c07111"}
|
||||
{"board": "cassandra", "body": "Repair every defect found by independent review task t_53cd8b4d on private atlas/cassandra PR #1, starting from exact head 4a0540501e3fbee1d931af042942f0dc30d429e0. Work only in the existing /opt/data/workspace/projects/cassandra-hermes-v69 directory and existing handoff/generated-strategy-audit-20260813 branch. Canonical remote is atlas/cassandra. Keep the PR open for human review; do not merge, deploy, or comment externally.\n\nPreserve these four inherited untracked files byte-for-byte and never stage or commit them:\n- docs/generated_strategy_job_audit.md (sha256 prefix 0e5c9160)\n- scripts/audit_generated_strategy_job.py (8259a7d8)\n- tests/test_audit_generated_strategy_job.py (dbd2d776)\n- uv.lock (f2d7ace7)\n\nBefore production edits, add and run focused fictional RED regressions that demonstrate each failure/acceptance boundary below. Then implement general bounded fixes, not phrase-specific exceptions:\n1. Fail-open Markdown header cells: a factual false cost claim in any table header cell must be scanned and blocked, while pure header labels and separator rows remain valid. Preserve optional outer pipes, escaped pipes, reordered columns, mixed activation/casting scoping, and malformed-input fail-closed behavior.\n2. False-positive N-mana resource prose: natural descriptions such as “needs two mana sources on board” and “requires three mana rocks” must not be treated as exact casting-cost claims. Correct claims like “costs two mana” and all existing symbolic/CMC/mana-value forms must remain enforced.\n3. Extend non-cost resource-noun handling for source(s), pip(s), and defensibly adjacent common resource nouns without creating an escape hatch for actual exact-cost claims. Cover “requires one white mana source” and “costs two pips of black.”\n4. Caveat colon scope: “Do not assume the following: Grave Recall costs {9}{B}.” must be accepted as caveated, without allowing a colon in unrelated prose to waive later claims. Preserve semicolon/coordinating-clause boundaries and no retroactive waiver.\n5. Hygiene: remove the unused mentions_card and claim_clauses imports; remove or make purposeful the uncalled public card_fact helper; make __all__ accurately export cross-module public functions. Since Ruff is absent, run a deterministic static unused-import/export fallback and document the explicit Ruff skip honestly.\n6. Add the missing positive control around the deliberate unsafe-entry behavior. Preserve the documented strict release choice that compliance blockers are not mechanically repaired unless evidence proves this causes an unintended safety or correctness defect; demonstrate that safe/caveated entry prose is accepted and unsafe entry guidance is rejected. Do not silently change the policy.\n7. Assess the measured ~0.57s compliance-call cost. Optimize repeated snapshot/index construction only if a small safe change can be proven equivalent; otherwise record it as residual performance risk rather than expanding scope.\n\nRun the full focused compliance/review suites and any broader grammar/strategy suite proportionate to this central parsing change. Confirm no ReDoS/runtime regression with bounded adversarial probes. Verify local HEAD, canonical remote branch, and PR #1 head are byte-identical after commit/push. Report exact commit, changed files, RED-before/GREEN-after evidence, tests, explicit skips, residual risks, PR state, and inherited-file hashes.", "expected_role": "implementation", "id": "t_d5274164", "title": "Repair Cassandra PR 1 findings from independent final review"}
|
||||
{"board": "cassandra", "body": "Runtime-only acceptance canary for the external Kanban goal loop. Do not run tools or commands, read or edit files, access the network, or mutate any external state. On the first provider turn, return the required structured result with status=incomplete, summary exactly PHASE_ONE_INCOMPLETE, empty changed_files, empty artifacts, empty blockers, and tests_run containing goal-loop canary phase one. On the resumed provider turn, return status=completed, summary exactly GOAL_LOOP_OK, empty changed_files, empty artifacts, empty blockers, and tests_run containing goal-loop canary: passed. Do not return completed on the first turn. Completion is valid only after Hermes rejects the first report and resumes this same card.", "expected_role": "implementation", "id": "t_dd6dc0a6", "title": "Goal-loop completion canary v2"}
|
||||
{"board": "cassandra", "body": "Continue the preserved Cassandra PR 1 repair from task t_d248853a in the existing workspace. Do not redo completed work. Exact workspace: /opt/data/workspace/projects/cassandra-hermes-v69. Exact branch at start should be handoff/generated-strategy-audit-20260813 with HEAD b42ebdd600710d3cc2e61290d5fb08f1c23518e1 tracking origin. Preserve all unrelated and pre-existing work. The six intended uncommitted follow-up files are docs/agent_collaboration.md, docs/history/DEVLOG.md, semantic_repair_costs.py, semantic_repair_quality.py, semantic_repair_scope.py, and semantic_retry_compliance_release_blockers.py. Preserve these four inherited untracked files byte-for-byte with hashes: docs/generated_strategy_job_audit.md 0e5c9160adbdad370a85b186fb52f8224fa708fa2c2ec935be46c35ba65f22ee; scripts/audit_generated_strategy_job.py 8259a7d8f2c5b2029fd0f8759419ecdcfa5495878bb5985daa3930f635a8ed46; tests/test_audit_generated_strategy_job.py dbd2d77691265c2fb74c7e6805ab5743a4d7bf3c715f9b4663d4cf19f1dea97c; uv.lock f2d7ace7c76ce3d910dc5d7c6b4c6345dd245068bc66981ae835772830469580. The follow-up generalizes markdown table handling: activation-cost tables including mana-only costs must not be mistaken for casting costs; casting-cost tables and embedded ordinary exact-cost prose must still block. Prior evidence is 8/8 new release blocker tests, 60 focused compliance, 7 semantic-review, 9 program-quality, pycompile, changed-file max 438 LOC, production LOC/docs/structure/diff/bounded-runtime checks green. A prior full 616 test run passed before this table-header follow-up; the follow-up full rerun was interrupted and is not evidence. Inspect the diff first. Run the focused gates and exactly one conclusive full tests.test_strategy_authoring_ai run with enough time. If any failure occurs, repair generally and rerun affected gates. Then rerun static/LOC/docs/runtime checks, verify the four hashes, commit only the intended six tracked follow-up files, push the same branch to origin, and verify the remote exact head and PR 1 topology. Do not add or commit the four inherited untracked files. Do not merge, deploy, certify, mutate Kanban outside this card, or claim completion while tests or a push are still in progress. Completion requires a pushed follow-up commit plus exact-head and test evidence.", "expected_role": "implementation", "id": "t_df41be0d", "title": "Finalize Cassandra PR 1 activation-table repair"}
|
||||
{"board": "cassandra", "body": "The full 234-module regression run under an ambient Hermes Agent environment exposed two unrelated test-isolation failures: tests.playtest_cases.manifest.PlaytestManifestTests.test_codex_decision_config_builds_worker_command assumes a `codex` binary exists instead of mocking discovery, and tests.strategy_authoring_ai_cases.claude_cli_provider.ClaudeCliExecutionTests.test_api_key_secret_maps_to_api_key_env inherits CLAUDE_CODE_OAUTH_TOKEN and dumps the whole environment on assertion failure. Harden both tests/fixtures so they are deterministic under agent credentials and never render the full environment or secret values in failure output. Verify both focused tests plus scripts/parallel_unittest.py. Do not change provider semantics merely to satisfy the tests.", "expected_role": "implementation", "id": "t_e822d35d", "title": "Harden full-suite tool and credential environment isolation"}
|
||||
{"board": "cassandra", "body": "After the regression tests are in place, implement the minimal generic production change needed for the new semantic-retry compliance tests to pass. Work in /opt/data/workspace/projects/cassandra-hermes-v69 and follow AGENTS.md plus docs/agent_collaboration.md. Make repair decisions from metadata, supported facts, and provenance only: reject unsafe entry routes, reject unknown or illegal targets, and remove/reject unsupported exact cost and whole-deck closure assertions when evidence is incomplete. Preserve existing fail-closed entry-hazard, unknown-card, legal-target, and provenance gates; do not weaken them. Do not add any named-card, named-commander, job-ID, or production-fixture branches. Keep the patch narrowly scoped, run the focused tests until green, and report changed files plus test output. Do not commit, push, or deploy.", "expected_role": "implementation", "id": "t_ef36526b", "title": "Implement bounded generic semantic-repair compliance"}
|
||||
{"board": "cassandra", "body": "In /opt/data/workspace/projects/cassandra-hermes-v69, inspect the semantic-retry pipeline, deterministic claim-repair flow, entry-hazard checks, and existing semantic/entry tests. Produce concise findings for the implementation owner identifying the exact functions, ordering constraints, existing test conventions, and safe extension points for: unsupported exact commander-cost phrasing; whole-deck universal paraphrases and metadata hedges; re-checking semantic compliance after deterministic repair; and polarity-aware caveat/negation handling. Do not modify production behavior, commit, push, deploy, or return a final task-completion result; return implementation findings only.", "expected_role": "implementation", "id": "t_fdc1a23e", "title": "Map semantic-retry compliance decision paths and repair hazards"}
|
||||
{"board": "soteria", "body": "Perform a fresh independent read-only release review of atlas/soteria PR #11. Canonical implementation head claimed by the implementer is 0143d47; verify the live PR head/base/state and review the entire base-to-head diff without trusting its result. Reproduce issue #9's original go:S3776/metrics complexity condition and prove the boolGauge extraction preserves every metric name, label, 1/0 health value, concurrency behavior, and failure path. Look for semantic drift, races, NaN/type behavior, false confidence from narrow tests, docs/LOC/style regressions, and whether the Sonar claim is actually supported. Run focused tests, go test -race ./..., go vet, gofmt, project check/quality commands, and inspect live PR checks read-only where available. Do not edit any file, comment, commit, push, merge, close the issue, deploy, or mutate Kanban outside returning this task result. Return completed with a clear SHIP or BLOCK verdict in summary/findings; review findings are findings, not task blockers. Preserve all inherited/untracked files and prove the review worktree remains clean.", "expected_role": "review", "id": "t_b1d2cd0c", "title": "Independent release review of Soteria PR 11"}
|
||||
{"board": "soteria", "body": "Resolve private atlas/soteria issue #9 (`[hermes] soteria: go:S3776`) from exact main head 47205db5b2e8041bac6a06a78f94fd3662134fc8. Work only in the isolated Hermes worktree derived from /opt/data/workspace/projects/soteria. Inspect the issue and current code independently; the issue's suggested boolGauge extraction is a hypothesis, not trusted implementation truth.\n\nAcceptance:\n- Add a focused RED-before/GREEN-after test proving the healthy PVC metric remains exactly 1 for true and 0 for false, with labels/metric identity unchanged.\n- Reduce RecordInventory cognitive complexity at internal/server/metrics.go:133 without behavior drift, phrase-specific workarounds, or unrelated refactors. If a helper is appropriate, keep it domain-named, small, documented only where useful, and tested.\n- Run targeted tests, the relevant full Go suite/race/static checks, formatting, Sonar/quality checks available locally, and any repository-specified validation. Record exact skips honestly.\n- Preserve unrelated files and verify the issue still applies at the exact starting head before editing.\n- Commit a focused branch, push to canonical atlas/soteria, and open a non-draft PR linked to issue #9 for human review. Do not merge, deploy, close the issue manually, or mutate cluster state.\n- Never expose or commit credentials. Use the runtime GIT_ASKPASS only.\n\nReport exact commit, branch, PR URL, changed files, RED/GREEN evidence, tests, skips, residual risks, and confirmation that main/issue were not modified directly.", "expected_role": "implementation", "id": "t_f1593f8c", "title": "Resolve Soteria issue 9 metrics complexity"}
|
||||
{"board": "titan-iac", "body": "Perform a fresh independent consequential review of private atlas/titan-iac PR #13 at exact head 4f8dcfbbf748e88726912bef1ae32201e1e640a6 against base 0dd6ea0f02672ecafd8c5c3bde7f5677e960c164. Work read-only in the supplied worktree. First verify local HEAD, canonical remote branch wt/t_cca008de, refs/pull/13/head, and PR API head/base exactly match. Review the complete PR diff, not prior summaries. Do not edit, stage, commit, push, comment, merge, deploy, mutate Kanban beyond normal completion, create pods, build or publish images, or expose credentials.\n\nReview the Dockerfile source-patch anchors against the pinned upstream /opt/hermes implementation and prove they fail closed under upstream drift. Reproduce the current create_task(initial_status=blocked) defect and verify the extracted patched tree makes explicit initial blocks sticky, explicit unblock reversible, dependency promotion intact, and circuit-breaker blocks non-promotable. Check whether the added build-time regression actually runs in the image build context and catches both false negatives and false positives. Review concurrency 4->2, CPU cap 3->2, and enableServiceLinks=false for resource/scheduling correctness and for any runtime dependency on generated service environment variables. Check tests, style, security, secret hygiene, line limits, kustomize render, client dry-run, and diff hygiene. Distinguish ambient live-environment test failures from PR regressions by reproducing against the base where needed. The absence of a safe image builder is a known deployment blocker, not permission to improvise privileged Docker.\n\nReturn prioritized findings with exact file/line evidence and reproducible cases. End with VERDICT: SHIP only if the code/config PR is reviewable without a material defect; otherwise VERDICT: BLOCK. Keep PR #13 open, unmerged, and undeployed for human review.", "expected_role": "review", "id": "t_06771c95", "title": "Independent review of Hermes platform PR 13 at 4f8dcfbb"}
|
||||
{"board": "titan-iac", "body": "Independent read-only xhigh security audit of the full-handoff acceptance harness produced by parent task t_6496c271.\n\nSETUP (do this first, trust nothing claimed): resolve the exact draft PR, head SHA, and base SHA dynamically rather than from any hardcoded number — list open/draft PRs for the repo, identify the one introducing the full-handoff acceptance harness, and record pr_number, head_sha, base_sha, plus the resolved changed-file list. Pin every finding to that head SHA. Read the full files, not just the diff hunks.\n\nSCOPE:\n1. Command construction: enumerate every place the harness builds a subprocess/exec/HTTP invocation. Confirm no shell=True or string interpolation of untrusted values, no injection via branch/PR/namespace/pod/image names, and that argument lists are built from validated inputs.\n2. Redaction BEFORE execution and BEFORE logging: verify the command line as logged/echoed is redacted prior to being run, not after, and that redaction lives at a single choke point rather than being reapplied per call site.\n3. Sanitization coverage: environment variables passed to children, HTTP headers, request/response bodies, captured stdout, captured stderr, and exception messages/tracebacks. Every one of these paths must be scrubbed. Look specifically for unscrubbed re-raises, repr()/f-string of a response or exception, and error paths that bypass the sanitizer.\n4. Credential containment: prove the harness cannot read or print Vault tokens/paths, cloud provider credentials, or SCM tokens. Trace every environment read and every credential-shaped file/secret access. Construct a concrete test that injects fake secrets into env/headers/responses and asserts they never appear in stdout, stderr, logs, or the JSON artifact.\n5. Ephemeral resource arming/cleanup and protected-branch rejection: verify arming is explicit and opt-in, cleanup is guaranteed on success, failure, timeout, and signal paths (no leaks), and that operations against protected branches are rejected early and unconditionally.\n\nCONSTRAINTS: keep the worktree pristine — no edits, no PR comments, no commit/push/merge/deploy/reconcile, no live mutations of any cluster or provider. Reading and running the harness's own unit tests in an isolated sandbox is allowed; anything that touches real infrastructure is not.\n\nDELIVERABLE: a findings list where each item has severity (blocking / non-blocking), exact path:line at the resolved head SHA, why it is exploitable or wrong, and an executable repro command or test snippet. Explicitly state which of the five scope areas you could NOT fully verify and why. Do not soften findings to reach a positive result.", "expected_role": "implementation", "id": "t_0908bc73", "title": "Audit harness security: command construction, redaction, credential exposure"}
|
||||
{"board": "titan-iac", "body": "Independent read-only xhigh audit proving that the default mode of the full-handoff acceptance harness (from parent task t_6496c271) performs zero mutations.\n\nSETUP: resolve the draft PR, head SHA, and base SHA dynamically (find the open/draft PR introducing the harness; do not assume a PR number). Record them and pin all findings to the resolved head SHA.\n\nSCOPE:\n1. Static command inventory: build an exhaustive inventory of every external command, API call, and SDK method the harness can reach in default mode. Walk call graphs from each entrypoint; include indirect paths through helpers, retries, and cleanup handlers. Classify each as read-only or mutating, citing path:line. A single reachable mutating call in default mode is a blocking finding.\n2. Dynamic proof via spies: run the harness in default mode with subprocess/HTTP/SDK layers replaced by spies that record invocations and hard-fail on any mutating verb (create/apply/patch/delete/rollout/scale/merge/push/comment/annotate/label, and POST/PUT/PATCH/DELETE to state-changing endpoints). Assert the recorded invocation set matches the static inventory — flag both extra calls and inventory entries never exercised.\n3. Mandatory NOT_RUN behavior: verify that any check which cannot be executed is reported as NOT_RUN and that NOT_RUN is never silently coerced into PASS or omitted from the artifact. Confirm whether the presence of any NOT_RUN forces a non-PASS overall result, and state precisely what the code actually does versus what the harness documentation claims.\n4. Escape hatches: identify every flag, env var, or config that can promote the harness out of read-only mode. Verify each requires explicit opt-in, is off by default, and is not reachable through defaulting/coercion bugs.\n\nCONSTRAINTS: worktree stays pristine — no edits, comments, commits, pushes, merges, deploys, reconciles, or live mutations. Run only in an isolated sandbox with spied/faked I/O; never point the harness at real infrastructure.\n\nDELIVERABLE: the full command inventory table, the spy transcript, and a blocking/non-blocking findings list with exact path:line and executable repro commands. State clearly whether 'default mode is read-only' is PROVEN, PARTIALLY PROVEN, or DISPROVEN, and what would be needed to close any gap.", "expected_role": "implementation", "id": "t_0d1d6c47", "title": "Prove default mode is read-only via spies and static command inventory"}
|
||||
{"board": "titan-iac", "body": "Perform a consequential independent release review of private atlas/titan-iac PR #13 at exact head 750dfa241f817179f6c0459a24651fd830e18c28 against exact base 0dd6ea0f02672ecafd8c5c3bde7f5677e960c164. The supplied workspace is a clean detached worktree at the exact head. Read-only review only: do not edit files, commit, push, comment, approve, merge, deploy, mutate Kanban other than completing this assigned card, create pods, build or publish images, or expose credentials. Verify local HEAD and all available remote PR identities before trusting the diff; fail closed if identity cannot be established. Inspect the entire six-file PR, not just the last commit. Independently reproduce the prior P1 threat models: unconditional-true created-event predicate, producer hardcoded blocked status, every non-park create mislabeled blocked, late producer reassignment after the pinned ladder, and anchor drift. Confirm the real API non-sticky oracle proves ordinary create and claim, failure limit 1 blocks, recomputation at limit 2 promotes ready, while explicit initial block/unblock, dependency promotion, and current-limit circuit breaker cases remain valid. Review malformed payload handling and textual-anchor limitations for false positives and false negatives. Verify the durable placement correction excludes titan-04 and titan-19 and changes only the main Hermes CPU request 350m to 300m. Run focused regression tests, relevant Hermes unit suites in sanitized and ambient/base-comparison modes, changed-file Ruff and static/secret/diff-scope checks, kustomize build, render, and client dry-run. No image build is required because the safe unprivileged builder is a separate capability. Return a prioritized findings-first verdict: SHIP only if there are no material defects and every explicit acceptance criterion has independent evidence; otherwise BLOCK with exact file and line evidence plus reproducible input. Clearly distinguish pre-existing base failures. Human review remains required after SHIP.", "expected_role": "review", "id": "t_12a22c7f", "title": "Independent SHIP review Hermes PR 13 at 750dfa24"}
|
||||
{"board": "titan-iac", "body": "Repair the live Atlas AI usage exporter reliability defect in private atlas/titan-iac, starting from exact main head 0dd6ea0f02672ecafd8c5c3bde7f5677e960c164. Work in the isolated linked worktree Hermes creates from /opt/data/workspace/projects/titan-iac. Create a focused branch, commit, push, and open a pull request for human review. Do not merge or deploy/reconcile an unreviewed change. Preserve unrelated files and never expose Codex/Claude credentials, OAuth contents, tokens, or provider response bodies in Git, logs, tests, PR text, or metrics.\n\nLive reproduced evidence to verify independently:\n- services/hermes/scripts/ai_usage_exporter.py query_codex() can reach the second process.wait(timeout=5) after kill and raise uncaught subprocess.TimeoutExpired.\n- Collector.refresh_provider does not catch that exception type, so the daemon polling thread terminates.\n- /healthz continues returning 200 because it does not represent poller liveness/freshness; the ai-usage-exporter container stays Ready while quota metrics become permanently stale.\n- A metrics client disconnect also emits a noisy BrokenPipe traceback.\n\nRequired behavior:\n1. Add RED regressions first for a Codex subprocess that cannot be reaped within both bounded waits. Cleanup must be strictly bounded and must never let TimeoutExpired escape in a way that kills polling; close selector/pipes and prevent child/process leakage as far as the host API permits.\n2. Isolate every provider refresh so any unexpected provider/cleanup exception records a sanitized failure and the polling loop continues to the other provider and future cycles. Never log exception messages or response bodies because they may contain sensitive material.\n3. Make health report the polling engine, not provider quota success: allow a bounded startup grace, remain healthy when an individual provider is unavailable (that is represented by fetch_success/staleness metrics), but return non-200 when the poller thread is dead or has stopped making bounded progress. Avoid restart loops caused merely by upstream quota/API failure.\n4. Handle metrics-client disconnects without server traceback noise.\n5. Preserve last-good quota values on failures and all existing low-cardinality metric contracts. Do not scrape interactive `/status`, enable API billing, add paid provider keys, or emit account identity/credentials.\n6. Ensure the ConfigMap/script change causes the agent exporter process to restart after a reviewed Flux rollout (use the existing config-revision convention or another tracked mechanism).\n\nValidation:\n- `pytest -q testing/tests/test_hermes_ai_usage_exporter.py testing/tests/test_atlas_ai_dashboard.py` plus relevant Hermes manifest/quality tests.\n- `ruff check` for changed Python when available.\n- regenerate Atlas dashboards only if the generator changes, and prove no unintended generated drift.\n- `kustomize build services/hermes`, `kustomize build services/monitoring`, client dry-runs for both, `git diff --check`.\n- Reinspect current live logs/readiness read-only to confirm the failure signature; do not mutate live workloads before review.\n\nReport RED-before/GREEN-after evidence, exact commit/branch/PR URL, changed files, validation, explicit skips, remaining risk, and why no credential can leak. Keep the PR open, non-draft unless Gitea requires otherwise, and unmerged.", "expected_role": "implementation", "id": "t_1e95ea6d", "title": "Repair AI usage exporter poller health and cleanup"}
|
||||
{"board": "titan-iac", "body": "Read the current private atlas/titan-iac main-head Hermes image patches, Dockerfile patch mechanism, and existing Kanban/Hermes scheduler tests without modifying tracked files. Trace create_task(initial_status='blocked'), event persistence, recompute_ready(), _has_sticky_block(), dependency completion handling, block_task/unblock_task, and circuit-breaker behavior. Identify the smallest patch location and exact focused tests needed to demonstrate RED-before/GREEN-after for: explicitly initially blocked tasks with no parents and with complete parents; ordinary dependency-blocked task promotion after dependencies complete; block_task/unblock_task; and sticky circuit-breaker blocks. Preserve dependency-driven promotion and official public behavior where possible. Return concrete file paths, relevant code paths, proposed assertions, and compatibility risks for the implementation worker.", "expected_role": "implementation", "id": "t_1fc75549", "title": "Analyze Hermes blocked-task state machine and regression strategy"}
|
||||
{"board": "titan-iac", "body": "Amend private atlas/titan-iac PR #13 after independent BLOCK review t_06771c95. Work only in /opt/data/workspace/projects/titan-iac/.worktrees/t_cca008de on branch wt/t_cca008de, starting from exact head 4f8dcfbbf748e88726912bef1ae32201e1e640a6 and base 0dd6ea0f02672ecafd8c5c3bde7f5677e960c164. Verify local/remote/pull-ref/API identity first. Preserve the complete existing six-file change. Do not deploy, merge, create pods, build/publish images, run privileged containers, expose credentials, or change unrelated files.\n\nFix the single P1 review finding with adversarial proof. Dockerfile.hermes-agent patches the sticky-block consumer but does not fail closed if upstream create_task stops emitting the expected created event payload {\"status\": task_status}. Add a unique fail-closed source anchor/precondition for the producer semantics on which the consumer patch relies; do not rewrite the producer unnecessarily. dockerfiles/hermes-kanban-blocked-regression.py needs a real-API non-sticky oracle: create/claim a normal task, record one spawn failure at failure_limit=1 so it is blocked, then recompute at failure_limit=2 and prove it promotes to ready. The test must fail under either an unconditional-true created-event predicate or producer drift that labels every created event blocked, while existing explicit initial block/unblock, dependency promotion, and circuit-breaker-at-current-limit cases pass. Add malformed/non-blocked created-payload controls if needed.\n\nReproduce RED against both the over-broad predicate mutant and the exact producer-drift mutant. Prove the amended Dockerfile patch rejects producer drift before build-time tests, all anchor drift remains fail-closed, and the exact patched temp tree passes. Run focused Hermes suites in a sanitized environment (baseline 107), full isolated unit surface (baseline 357), Ruff/AST/YAML/secret/diff/line checks, kustomize render, and client dry-run. Current live service-link/health variables caused four ambient failures that reproduced on base; report both evidence sets. Do not attempt an image build without the separate safe unprivileged builder capability.\n\nCommit and push the focused amendment to wt/t_cca008de so PR #13 advances. Verify remote branch and refs/pull/13/head equal the new commit and the PR remains open/unmerged/undeployed. Status completed only after commit/push/ref verification; use incomplete while anything remains.", "expected_role": "implementation", "id": "t_20f2fd16", "title": "Repair Hermes PR 13 producer-drift regression coverage"}
|
||||
{"board": "titan-iac", "body": "Perform a fresh independent read-only release review of the implementation produced by parent task t_64e1ae8d. Dynamically resolve its exact PR/head/base. Do not edit/comment/commit/push/merge/build/publish/deploy/reconcile/restart or mutate Kanban beyond returning the review.\\n\\nIndependently reproduce the zombie creation path and verify the repair prevents unreaped adopted descendants under normal completion, timeout/kill, cancellation, nested Git/SSH, concurrency, and restart without racing provider subprocess waiters or changing exit results. Audit PID1/init/subreaper/signal semantics, resource bounds and sanitized metrics. Verify compatibility with merged PR13 and then-current PR15, full quality/coverage/LOC, render/schema/client/server dry-runs, and secret scans. Return strict SHIP or BLOCK with exact executable evidence and residual rollout canary requirements.", "expected_role": "review", "id": "t_261e208f", "title": "Independent review of cli-lane process reaping"}
|
||||
{"board": "titan-iac", "body": "Independently review private atlas/titan-iac PR #12 at exact head 465d386da5237f011cb408f1a3d4b7d0c6d8ffae against exact base 0dd6ea0f02672ecafd8c5c3bde7f5677e960c164. This is a consequential read-only release review of the AI usage exporter reliability repair produced by parent task t_1e95ea6d.\n\nDo not edit files, mutate Kanban beyond returning this task result, commit, push, comment, merge, deploy, reconcile, or expose credentials/provider bodies. Verify local HEAD, canonical remote branch wt/t_1e95ea6d, and PR head are byte-identical before review. Inspect the entire three-file diff, not just the parent summary.\n\nReview especially: strictly bounded terminate/kill/reap behavior when both waits time out; selector/pipe cleanup and child leakage limits; per-provider and per-cycle exception isolation; last-good metric preservation; health semantics during startup, provider failure, dead poller, stalled provider, and interval sleep; avoidance of restart loops; client disconnect handling; thread safety; low-cardinality metric compatibility; sanitized logging; absence of OAuth/API-key/account data leakage; config-revision rollout behavior; test truthfulness and missing adversarial tests. Re-run the focused exporter/dashboard tests and proportionate manifest/render/dry-run validation independently. Treat parent tests and done status as evidence to verify, not truth.\n\nReturn prioritized findings with exact file/line references and reproducible inputs. State explicitly if no defect is found in an area, list exact tests/skips, and give a ship/not-ship verdict. Keep the PR open for human review.", "expected_role": "review", "id": "t_326d55c6", "title": "Independent release review of titan-iac PR 12"}
|
||||
{"board": "titan-iac", "body": "Implement and validate a durable least-privilege image build/publish path for Hermes agent changes. Use the existing Jenkins/Kaniko pattern or another reviewed rootless builder. Never run privileged Docker-in-Docker, never expose unauthenticated Docker or BuildKit TCP sockets, never schedule disposable builders on titan-14 or titan-18, and never print or commit registry credentials. Deliver through a focused human-reviewed PR with tests and proof that the image digest can be published and then updated through Flux. PR #13 is intentionally undeployed until this capability exists.", "expected_role": "implementation", "id": "t_404f607c", "title": "Provide a tracked unprivileged Hermes image build and publish workflow"}
|
||||
{"board": "titan-iac", "body": "After t_c31dd427 completes, independently review exact repaired PR18 head against current main. Re-run the complete t_dbdcd739 and t_5975c06a evidence, especially lease-failure persistence/restart recovery, exact-run fencing, no poison-row CrashLoop, mediator/worker drain and RWO recovery, retry branch submission, startup tooling/Flux health, focused ancestry not bundling other open drafts, all security boundaries, three-node scheduling, at least 95 percent line and branch coverage, below 500 LOC, full quality/render/dry-run/secret scans. Read-only. Return completed SHIP or completed BLOCK; a BLOCK verdict completes the review. No edits/comments/push/merge/deploy/reconcile.", "expected_role": "review", "id": "t_529c50f9", "title": "Independent re-review of final PR 18 pool repair"}
|
||||
{"board": "titan-iac", "body": "Perform a new independent read-only consequential review after parent t_8fbeb41f repairs existing PR #18. Dynamically resolve exact PR/head/base and current PR14/15/16/19 heads; distrust all claimed tests. No edits/comments/commit/push/merge/build/publish/deploy/reconcile/live mutation.\\n\\nRe-run every artifact under /opt/data/workspace/evidence/t_dbdcd739 and independently test all P0/P1/P2 integration boundaries: run ID typing/finalization/reconcile progress; workspace/branch/project routing across all Atlas boards; PR15 module/runtime compatibility; per-worker crypto isolation; sidecar/model network trust; PR14 broker-only SCM; OAuth refresh ownership; exact lease expiry/reassignment/error handling; worker-only three-node schedulability; config rollout/version skew; RBAC/network/mount isolation; canonical coverage/LOC/full gate/renders/schema/dry-runs/secret scans. Return strict SHIP or BLOCK as a completed review; findings do not require you to edit the reviewed code.", "expected_role": "review", "id": "t_5975c06a", "title": "Independent re-review of repaired PR 18 distributed pool"}
|
||||
{"board": "titan-iac", "body": "After parent repair t_69159fea completes, perform a fresh independent read-only xhigh review of draft PR #19 at its exact then-current head against then-current main. Do not trust previous claims or the earlier review. Keep the review worktree pristine and do not edit, comment, commit, push, merge, publish, deploy, reconcile, create credentials, or mutate live resources.\n\nRe-run every adversarial class from review t_e7f46d44/Codex session 01a00f52-a815-7933-9f7d-04ac16145027: structural command allowlist, credential paths, raw Secret/kubeconfig/Helm/Git controls, subprocess memory/time/process-group bounds, redact-before-truncate across JSON and human output, unknown/NaN/future/malformed/stale evidence, PATH/binary provenance, armed preflight/create uncertainty/cleanup, remote-main+PR+image+running-revision binding, three-node and chat/Telegram coverage, dependency/rollback accuracy, and zero/partial/malformed states. Prove default mode performs no mutation through spies and a complete command inventory.\n\nRequire every production module in the canonical managed/lint/line+branch coverage contract at >=95 percent and every source/test file below 500 LOC. Require full canonical quality gate, Ruff/format/compile, mutation/adversarial suite, render/dry-run, diff and secret scans all green. Return a strict completed SHIP or completed BLOCK review with exact executable evidence. A BLOCK verdict completes this review task; do not attempt implementation work.", "expected_role": "review", "id": "t_5ae177fc", "title": "Independent re-review of repaired PR 19 acceptance harness"}
|
||||
{"board": "titan-iac", "body": "Work in the isolated Titan IaC worktree created for this task. Read AGENTS.md and HERMES_ATLAS_MIGRATION_HANDOFF.md completely. Use Claude Code through Claude.ai subscription OAuth only; never configure/use ANTHROPIC_API_KEY or any Console/API-billed route.\n\nBuild the hardest remaining independent artifact: a fail-closed, read-only-by-default Hermes full-handoff acceptance harness and release runbook. Keep the change isolated to new files scripts/ops/hermes_handoff_acceptance.py, testing/tests/test_hermes_handoff_acceptance.py (split into meaningful modules if it would exceed 500 LOC), and docs/hermes_full_handoff_acceptance.md. Do not edit files changed by open PRs #14, #15, #16, #17, or the active distributed-worker task/PR. PR #13 is already merged/deployed; baseline origin/main must be ab346f55509d584e457fe26cf90be3078f7a375c or a descendant. Dynamically fetch and record exact current heads/state for #14-#17 and any distributed-worker PR; do not hard-code stale heads.\n\nThe harness must emit machine-readable JSON plus a concise human summary, redact credential-shaped values, never print environment values/tokens/private keys/cookies/Vault responses, enforce bounded timeouts/output, and classify every check PASS/FAIL/NOT_RUN/NOT_APPLICABLE. Any mandatory FAIL or NOT_RUN makes overall NO_GO. Split evidence into an external read-only operator vantage point and an actual in-pod Hermes self-probe; do not treat kubectl auth can-i --as alone as proof. Default mode must perform no mutation. Feature-branch push and draft-PR creation must be a separately armed ephemeral mode with exact unique refs, explicit confirmation, preflight, protected-main/master rejection, and verified cleanup; default reports these NOT_RUN.\n\nVerify after pending PRs merge/deploy: distinct chat/agent/triage scopes; Codex ChatGPT login and Claude claude.ai firstParty subscription with no provider API-key env vars; Switchyard provider/model/effort/fallback/latency/failure evidence; visible durable session/Kanban activity; denied cluster-admin/secrets/SA-token/impersonation/arbitrary exec-attach-portforward/workload mutation; allowed required get/list/watch/pod logs/Flux+Helm status; no worker Gitea credential and no merge/approve/close/admin route; broker clone/fetch/feature push/metadata/draft PR and protected-branch denial; every node dedicated locked hermes-agent account with no human key/sudo/disk/runtime/K3s/kubelet access; tokenless unprivileged builder and exact expected caps; immutable Harbor build tag; finalization/replay/decomposition regressions; no unexpected GitOps suspension/unhealthy state; three distinct distributed workers with coordinator-only state ownership, distinct nodes/PVCs, authenticated bounded activity/result path, stale/duplicate safety; activity visible in agent UI; chat.hermes Telegram source/topic continuity and no blank session render regression.\n\nDocument exact dependency/merge order for merged #13 plus pending #14-#17/distributed pool, image build/Flux rollout, rollback points, and final go/no-go checklist. Unit-test parsing, redaction, timeout, partial failure, zero-state, dual-vantage mismatch, mandatory skip, and ephemeral arming/cleanup behavior. Keep production files <500 LOC and achieve >=95% per-file coverage for new source. Run focused/full relevant tests, Ruff/format, py_compile, diff/secret scans and applicable render checks. Commit, push a feature/hermes-full-handoff-acceptance branch, create a draft PR only, report exact SHA/evidence. Do not merge, publish images, deploy, reconcile, create credentials, or mutate live cluster/Gitea during this implementation.", "expected_role": "implementation", "id": "t_6496c271", "title": "Build fail-closed Hermes full-handoff acceptance harness"}
|
||||
{"board": "titan-iac", "body": "Live reliability defect observed on the merged PR13 agent.hermes cli-lane-runner pod: after ~95 minutes it has 141 defunct git children, all parented to PID 1 (/opt/coordinator/cli_lane_runner.py). This is an authorized repair of our own runtime.\\n\\nDiagnose and repair subprocess lifecycle/reaping so long-lived agent.hermes workers cannot accumulate zombies or exhaust PID capacity while running Codex/Claude/Git/test commands. Work on current atlas/titan-iac main in an isolated worktree and draft PR. Do not manually mutate/deploy/restart production.\\n\\nRequirements:\\n- Reproduce the actual creation path and identify which spawn/session/process-group behavior leaves children for PID1.\\n- Implement deterministic reaping/cleanup at the owner boundary without stealing children from active provider subprocesses, corrupting streamed output, or masking exit status.\\n- Cover normal exit, timeout SIGTERM/SIGKILL, provider cancellation, nested Git/SSH helpers, concurrent lanes, runner restart, and orphan adoption. Avoid a broad signal handler if it races subprocess waiters; use an explicit supervisor/subreaper/init boundary as appropriate and justify it.\\n- Add bounded observability for active/reaped/orphaned child counts without command lines, environment, credentials, or sensitive output.\\n- Preserve PR13 blocked-task behavior and PR15 finalization/recovery compatibility. Resolve current exact PR15 head dynamically and prove clean semantic merge in the reachable order.\\n- Keep every production/test file under 500 lines, >=95% per-file coverage, quality gate green; render/dry-run Hermes and run relevant static/secret checks.\\n- Commit/push/open one draft PR, human review required. No image publication, merge, Flux reconciliation, deployment, live restart, or credential access.\\nReturn completed only with exact pushed SHA and evidence.", "expected_role": "implementation", "id": "t_64e1ae8d", "title": "Eliminate cli-lane zombie process accumulation"}
|
||||
{"board": "titan-iac", "body": "Repair existing draft PR #19 on its SAME branch and implementation worktree after independent review t_e7f46d44 returned BLOCK twice. Exact reviewed head 8f005458282269ba5c07941814e4237f5d4cf3ac; resolve current main and all dependent PR heads dynamically. Do not create another PR.\n\nTreat the completed review transcript in Codex session 01a00f52-a815-7933-9f7d-04ac16145027 as authoritative evidence. Close every release blocker:\n1. Default mode must be structurally read-only and credential-safe. Reject kubectl auth reconcile, config view --raw, Secret/raw-secret reads, client-side or post-separator dry-run tricks, git fetch/config sshCommand, Helm values, and shell paths under service-account/Vault/runtime credential roots before subprocess creation.\n2. Bound stdout/stderr in memory while the child runs; enforce absolute timeout, kill/reap the whole process group, bound descendants/pipes, validate timeout/output/concurrency inputs, and make truncation fail closed for every evaluator.\n3. Redact before truncation and screen every JSON plus human summary/error/identity field. Never expose a credential prefix.\n4. Fail closed on unknown statuses, empty catalog, NaN/non-finite numbers, future timestamps, malformed Flux/routing rows, stale required-provider evidence, generic 404/no-route strings, or mocked/self-authored PATH evidence. Add binary/path attestation or equivalent provenance.\n5. Armed mode must preflight before any networked/mutating vantage, bind actual repo to fixed expected Atlas repo, reserve cleanup time, verify exact draft/base/head state, discover uncertain creates, and require branch plus PR cleanup success. Never false-PASS malformed/failed cleanup.\n6. Freshness must bind remote main SHA, reviewed PR head, image digest/imageID, build SHA, and running deployment revision. Require all three distributed workers/nodes; no vacuous node-count=1 default. Include chat/Telegram continuity and pool assignment checks.\n7. Add all 15 production modules to canonical managed_modules, lint_paths, and coverage.tracked_files. Every new/modified source must remain below 500 LOC and at least 95 percent line and branch coverage. Fix the two environment-sensitive tests by isolating provider-health and Gitea-token fixtures. Canonical quality gate, Ruff, format, compile, mutation/adversarial tests, render/dry-run, diff/secret scans must all be green.\n8. Correct dependency/merge/conflict graph using exact current heads. Remove unsafe rollback guidance restoring cluster-admin and false PVC-pruning assumptions.\n9. Add executable regressions for every reviewer repro, including large/malformed inputs, zero state, partial failures, timeouts, descendant pipes, and no-subprocess/no-mutation spies.\n\nKeep the worktree pristine except this repair. Amend/commit and force-with-lease only the existing feature/hermes-full-handoff-acceptance branch, update PR #19, report exact head and evidence. Human review required. No merge, publish, deploy, reconcile, credential access, or live mutation.", "expected_role": "implementation", "id": "t_69159fea", "title": "Repair PR 19 full-handoff acceptance blockers"}
|
||||
{"board": "titan-iac", "body": "After t_888cdeee completes, independently review exact repaired PR19 head against current main. Re-run all prior policy, subprocess, redaction, evaluator, empty-evidence, attestation, armed cleanup, freshness, three-node/chat, dependency, mutation, coverage and format repros. Require mandatory names-absent checks to return NOT_RUN on zero evidence; exact repo/path/impersonation boundaries; at least 95 percent line and branch coverage; below 500 LOC; full canonical green gate. Read-only. Return completed SHIP or completed BLOCK; a BLOCK verdict completes the review. No edits/comments/push/merge/deploy/reconcile.", "expected_role": "review", "id": "t_699f9a2d", "title": "Independent re-review of final PR 19 harness repair"}
|
||||
{"board": "titan-iac", "body": "Live failure observed on t_dbdcd739: a read-only independent release reviewer returned a completed BLOCK review with five P0 findings, but the local goal judge rejected completion and resumed Claude with an instruction to continue because the reviewed implementation was not SHIP. A review's job is to issue a verdict, not repair the implementation; this wastes subscription capacity and can loop.\\n\\nRepair Hermes goal/result semantics so task-role completion is judged against the assigned action. For review/diagnostic/audit tasks, status=completed with a truthful SHIP or BLOCK verdict and findings must finalize even when findings show the reviewed artifact is unfit to ship. Only genuine inability to perform the review belongs in blockers. Implementation tasks must still be held to their acceptance criteria.\\n\\nRequirements:\\n- Reproduce with exact task body/result shape and local judge path.\\n- Define a deterministic task-role/expected-output contract, preferably explicit metadata/schema rather than fragile keyword-only inference; preserve backward compatibility.\\n- Make reviewer/diagnostic completion fail closed on malformed/missing verdict/evidence, but never demand code edits prohibited by read-only scope.\\n- Preserve implementation goal looping and PR15 accepted-result durability/exact-run behavior. Dynamically merge-test current PR15 head.\\n- Cover SHIP, BLOCK-with-findings, incomplete review, blocked review, implementation incomplete, ambiguous tasks, reviewer provider fallback, restart/replay, and no duplicate turn/session.\\n- Emit bounded/sanitized judge reason and correct Kanban final result; no hidden auto-mutation.\\n- All files <500, >=95% per-file canonical coverage, quality/render/dry-run/secret checks.\\n- Draft PR only, human review; no merge/build/publish/deploy/reconcile/live restart.\\nReturn exact pushed SHA and evidence.", "expected_role": "implementation", "id": "t_6da029e0", "title": "Make goal judge finalize completed review verdicts"}
|
||||
{"board": "titan-iac", "body": "Fresh independent read-only xhigh release review of chat.hermes smoothness work from parent t_83252d81. Resolve exact draft PR/head/base; do not edit/comment/push/merge/deploy/reconcile/call real Telegram or mutate session data. Reproduce blank/inaccessible session rendering, reconnect/resume, active tool progress, API/Telegram source tags and legacy migration, deterministic durable topics across hour/day/week/router restart, bounded recent-turn+summary context and duplicate suppression, malformed/oversized history, four-tenant isolation, inbound/outbound media and path safety, and truthful previous-image edit semantics. Audit Switchyard/manual route preservation and chat/agent/triage separation. Run focused/full UI/session/router Go+race/Python tests, coverage/LOC, Kustomize/schema/client+Flux-manager server dry-runs, secret/diff scans. Return strict SHIP/BLOCK with exact executable evidence and residual risks. Keep review worktree pristine.", "expected_role": "review", "id": "t_7b7eaaf5", "title": "Independent review of chat.hermes smoothness"}
|
||||
{"board": "titan-iac", "body": "Starting from the current private atlas/titan-iac main head, create an isolated linked worktree and dedicated branch. Use the two analysis reports to implement the smallest robust Hermes-only change. First run or add focused tests that fail against the current behavior, retaining RED evidence; then patch the existing Dockerfile image-patching/test mechanism so explicitly created blocked tasks acquire sticky blocked semantics without weakening normal dependency promotion, explicit unblock behavior, or circuit-breaker blocks. Add focused GREEN tests for every required scenario. Update deployment configuration to set HERMES_CLI_LANE_CONCURRENCY=2, cap cli-lane-runner CPU limit at 2, and set enableServiceLinks: false at the hermes-agent pod-spec level; add a manifest regression proving service links are disabled and check resource requests remain sensible. Preserve unrelated work and never expose or commit credentials. Do not deploy, reconcile Flux, restart the agent, or mutate live Kanban data beyond this task lifecycle. Record exact changed files and RED/GREEN commands/results for handoff.", "expected_role": "implementation", "id": "t_7ddd9eb2", "title": "Implement Hermes isolation and blocked-task regressions on a branch"}
|
||||
{"board": "titan-iac", "body": "Make chat.hermes.bstein.dev smoother and truthful without changing agent.hermes or triage.hermes scope. Work in an isolated worktree and read AGENTS.md, HERMES_ATLAS_MIGRATION_HANDOFF.md, current chat/router/session code, and exact current open PR #14-#17 plus distributed-pool diff before editing. Avoid every file changed by those PRs; if a required fix overlaps, stop with a precise dependency instead of racing it. Use subscription Codex only; no provider API keys.\n\nFirst perform a read-only live/code audit, then implement only evidence-backed fixes in a focused draft PR: eliminate blank/inaccessible session render states and ensure activity progressively renders during long work; preserve reconnect/resume and lineage; label existing and new Telegram API sessions under Telegram rather than Unassigned; keep durable topic-based Telegram continuity for roughly week-scale conversations with bounded recent turns + durable compact summary rather than resending an unbounded transcript; make source/topic/session identity deterministic across pauses and router restarts; prevent duplicate stored context; keep text/image/tool capabilities truthful; preserve actual outbound Telegram media delivery and never expose MEDIA paths; explicitly distinguish whether an image-edit request can reuse a prior image attachment versus generate a new image; do not claim inbound image editing if transport does not provide the attachment.\n\nAdd adversarial tests for existing-session migration idempotence, source labels, topic expiry/reuse, summary/recent-turn bounds, duplicate suppression, resume after an hour/day/week boundary, router restart, malformed/oversized histories, WebSocket/poll disconnect, blank render fallback, active tool progress, and Telegram media/caption safety. Preserve four-tenant isolation and Switchyard routes/manual overrides. No secrets in Git/logs. Keep production files <500 LOC and >=95% per-file coverage for new source. Run relevant full tests, Ruff/Go tests/race where applicable, Kustomize/client/Flux-manager server dry-runs, diff and secret scans. Commit/push branch feature/hermes-chat-smoothness, open draft PR for Brad. Do not merge, publish, deploy, reconcile, call real Telegram, or mutate live session data.", "expected_role": "implementation", "id": "t_83252d81", "title": "Harden chat.hermes continuity and activity rendering"}
|
||||
{"board": "titan-iac", "body": "Repair existing Titan IaC draft PR #12 on its SAME branch/worktree; do not open a replacement PR. Start by read-only fetching current origin/main ab346f55 or descendant and exact current PR14 head because PR14 is splitting baseline oversized tests. Rebase/merge safely without dropping PR13/PR14 semantics; if PR14 is not merged, validate a synthetic PR12+PR14 composition and document required merge order. The exporter reliability change remains unique and must not be closed.\n\nMandatory repair: services/hermes/scripts/ai_usage_exporter.py is now 612 LOC and violates the repository 500-LOC source rule even though the legacy quality gate does not track that script. Extract meaningful bounded Codex process/query cleanup and/or polling/HTTP engine modules so every changed production file is <500 LOC; do not add exclusions or meaningless chunks. Preserve structured subscription-only quota collection, no API keys, sanitized class-name-only errors, bounded process reaping, provider isolation, last-good samples, poller-progress health, and disconnect handling. Add/retain >=95% per-file coverage and adversarial cleanup/thread/dead-health tests. Rebase current main, update config revision without clobbering newer rollout annotations, run exporter/dashboard tests, relevant full Hermes tests, Ruff/format/compile, actual make test/quality after composing PR14 as required, kustomize Hermes+monitoring, client and Flux-manager server dry-runs, diff/secret scans. Amend/force-with-lease only existing PR12 branch, verify PR #12 stays open/unmerged with exact head, stop for fresh independent review. No merge/build/publish/deploy/reconcile/live mutation.", "expected_role": "implementation", "id": "t_87cd9076", "title": "Rebase and split AI usage exporter PR 12"}
|
||||
{"board": "titan-iac", "body": "Repair existing draft PR #19 on its SAME feature/hermes-full-handoff-acceptance branch/worktree after independent review t_5ae177fc BLOCKED exact head c808baff40a12a90f43c2e5e39b94d139f88ea29. Do not create another PR.\n\nFix the mandatory P1 exactly: evaluate_names_absent must never PASS on rc=0 with zero observations. Return NOT_RUN/NO_GO on empty evidence and on pipeline/upstream/tool/catalog/jsonpath drift. Add all five real catalog regressions plus shell-pipeline and kubectl jsonpath empty-output repros.\n\nAlso close the review's reachable hardening/evidence defects: exact repo path boundary, reject dot segments, structurally forbid impersonation in all vantages unless a narrowly explicit audited inner self-probe requires it, correct the runbook format command so it passes, remove or implement inert concurrency/Telegram flags honestly, remove unattestable unused flux/helm entries or pin them, paginate/discover uncertain draft creation and surface/manual-clean exact residue, retain executable provenance for record-false steps, and do not add broad legacy LOC exceptions in this PR; coordinate with repaired PR14/15 canonical contract instead.\n\nPreserve all previously verified read-only/default safety, bounded subprocess/redaction, fail-closed parsing, binary attestation, armed cleanup, freshness/lineage, three-node/chat checks, at least 95 percent line and branch coverage and below 500 LOC. Re-run full canonical quality and every earlier adversarial/mutation test, Ruff format/check, compile, kustomize/dry-runs, exact current dependency graph, diff/secret scans. Amend/push only existing PR19 branch and stop for fresh independent review. No merge/publish/deploy/reconcile/live mutation/credential access.", "expected_role": "implementation", "id": "t_888cdeee", "title": "Repair PR 19 zero-evidence fail-open"}
|
||||
{"board": "titan-iac", "body": "Independent read-only xhigh correctness audit of the result/verdict machinery of the full-handoff acceptance harness (from parent task t_6496c271).\n\nSETUP: resolve the draft PR, head SHA, and base SHA dynamically; do not trust any claimed coverage or claimed go/no-go behavior. Pin findings to the resolved head SHA.\n\nSCOPE:\n1. JSON artifact schema: verify the emitted artifact conforms to a declared schema, that the schema is enforced (not merely documented), and that required fields cannot be absent. Check schema versioning and behavior on unknown/extra fields.\n2. Deterministic overall verdict: derive the truth table mapping per-check states (PASS / NO_GO / NOT_RUN / error) to the overall result. Verify it is total (no unhandled combination), order-independent, and free of nondeterminism from dict/set iteration, timestamps, concurrency interleaving, or locale. Prove NO_GO cannot be masked by a later PASS and that the fail-closed default is NO_GO, not PASS.\n3. Bounded execution: verify enforced upper bounds on wall time per check and overall, on captured output size (stdout/stderr truncation without losing the verdict), and on concurrency (bounded worker pool, no unbounded fan-out). Check timeout behavior — a timed-out check must become NOT_RUN or NO_GO, never PASS.\n4. Degenerate inputs: exercise partial evidence, malformed/truncated JSON, and zero-state (no pods, no revisions, empty pool). Each must yield an explicit non-PASS state rather than a crash or a vacuous PASS. Vacuous PASS on zero-state is a blocking finding.\n5. Evidence integrity: verify dual operator-side and in-pod evidence are independently collected and cross-checked, that revision/image freshness is validated against the resolved head SHA rather than accepted from a label or claim, and that the harness cannot be made to pass on mocked or self-authored evidence. Test this concretely: craft a forged evidence bundle and confirm the harness rejects it.\n\nCONSTRAINTS: worktree pristine — no edits, comments, commits, pushes, merges, deploys, reconciles, or live mutations. Use fixtures and fakes only.\n\nDELIVERABLE: the verdict truth table you derived from the code, the degenerate-input and forged-evidence experiment results, and a blocking/non-blocking findings list with exact path:line and executable repros. Call out every place implemented behavior diverges from the documented contract.", "expected_role": "implementation", "id": "t_8ceae30f", "title": "Verify determinism, JSON schema, and evidence integrity of harness verdicts"}
|
||||
{"board": "titan-iac", "body": "Independent read-only xhigh review of the dependency, merge, rollout, and rollback plan for the full-handoff acceptance harness (from parent task t_6496c271), evaluated against actual current state rather than the plan's own claims.\n\nSETUP: resolve dynamically — the harness draft PR (number, head SHA, base SHA), and the real current status of each of PR12 through PR17 (open/draft/merged/closed, base branch, head SHA, merge conflicts, CI status). Also resolve the current distributed-pool state the rollout targets. Record what you observed with timestamps; do not rely on any status asserted in the PR description or prior task notes.\n\nSCOPE:\n1. Dependency correctness: verify the stated ordering among PR12-PR17 and the harness PR matches the actual code/config dependencies. Identify any dependency claimed but not real, or real but unclaimed. Flag any cycle or any PR that would break if merged in the stated order.\n2. Merge plan: check each PR's base branch and rebase state, detect textual and semantic conflicts between the harness PR and the others (especially shared config, schema, and entrypoint files), and assess whether the stated merge sequence is executable as written today.\n3. Rollout plan: evaluate staging against the current distributed-pool state — is the plan valid for the pool's actual size, versions, and heterogeneity? Check gating between stages, blast radius per stage, and whether the acceptance harness is a hard gate or merely advisory.\n4. Rollback plan: verify rollback is concrete and executable (exact revisions/images to revert to), covers partial-rollout states, and does not depend on the very component being rolled back. Identify any irreversible step, data/schema migration, or one-way door, and any state a rollback would strand.\n5. Freshness: confirm the plan's referenced revisions/images still exist and are current; stale references are findings.\n\nCONSTRAINTS: strictly read-only — no edits, comments, commits, pushes, merges, deploys, reconciles, or live mutations. Query state; do not change it.\n\nDELIVERABLE: an observed-state table for PR12-PR17 and the pool, then a blocking/non-blocking findings list with exact references (PR number, path:line, or resource identifier) and, for each blocker, the specific correction the plan needs. Explicitly state whether the plan is executable as written against today's state.", "expected_role": "implementation", "id": "t_8f526175", "title": "Review dependency, merge, rollout and rollback plan against PR12-PR17 state"}
|
||||
{"board": "titan-iac", "body": "Repair existing draft PR #18 on its SAME branch/worktree after independent review t_dbdcd739 returned BLOCK. Exact reviewed head: 20002527512235b29054f3636b02418168373009; current main dynamically resolve. Review evidence is durable under /opt/data/workspace/evidence/t_dbdcd739. Do not create a second PR.\\n\\nAll five P0 integration defects must close:\\n1. Coordinator.finalize compares integer current_run_id to string binding run_id, marks every result stale, then conflicting assignments wedge reconcile. Canonicalize at one boundary and use exact-run-safe DB APIs.\\n2. Existing task.workspace_path is normal for live tasks; do not capability-block every task. Define safe distributed workspace migration/ownership without shared mutable worktrees and preserve active/untracked ownership.\\n3. Accept Hermes-established safe branch conventions including wt/<task> and review/... in addition to feature/fix/etc, using actual Git ref validation and traversal denial.\\n4. Task has no repo_url/base_branch fields and current design sends every board to titan-iac. Resolve repo/base from the canonical project/board registry or explicit validated task metadata persisted by a reviewed schema migration; cover every Atlas project including Metis.\\n5. Exact merge with current PR15 must include/import every split cli_lane module and preserve its finalization semantics. No text-only compatibility claim.\\n\\nAlso close consequential handoff/security/reliability findings:\\n- Do not share one HMAC authority among all workers; derive/bind per-ordinal credentials or equivalent cryptographic isolation.\\n- Require node-role worker for execution Pods; avoid control-plane. Prefer healthy accelerator/rpi5 capacity but prove 3 distinct schedulable nodes and avoid titan-22/24/unhealthy nodes without starving Switchyard/Flux.\\n- The model container must not be able to call localhost sidecar APIs to bypass commit/untracked/result gates. Add a real mediation boundary (separate pod/network identity or unforgeable per-request capability), not instruction-only trust.\\n- Integrate the reviewed PR14 hermes-scm broker; never stage raw developer-gitea token into model-facing worker pods or bypass broker policy.\\n- Make subscription OAuth refresh ownership safe: model may use writable per-worker copies, but cannot write authoritative shared Vault credential state or the pool HMAC derivation input; prevent multi-writer refresh-token lost updates.\\n- Enforce lease expiry/reassignment with exact run fencing; surface worker exceptions to Kanban and release ordinals rather than spin; config changes must trigger controlled versioned rollout/negotiation.\\n- Remove dead/asymmetric egress, narrow credential-sync mounts, preserve tokenless SA/RBAC/network isolation.\\n- Add all new production modules to canonical lint/coverage contract, every source >=95% branch/line coverage and <500 LOC; split test baselines meaningfully if current main still needs it.\\n- Fix the env-sensitive Cassandra token test only if not already closed by PR14; do not hide failures.\\n- Run independent-style executable tests from review repros, focused/full quality, exact synthetic merges with current PR14/15/16/19, kustomize/kubeconform/client+Flux-manager server dry-runs, scheduling simulation, secret/diff scans.\\n\\nCommit/amend and force-with-lease the existing feature/hermes-distributed-worker-pool branch, update PR #18, report exact head and evidence. Human review required. No merge, image publish, Flux reconcile, deployment, live worker/node/account mutation, or credential access.", "expected_role": "implementation", "id": "t_8fbeb41f", "title": "Repair PR 18 distributed pool integration blockers"}
|
||||
{"board": "titan-iac", "body": "Independently execute and report the full quality gate for the full-handoff acceptance harness (from parent task t_6496c271). Do not accept any claimed numbers — regenerate every metric yourself.\n\nSETUP: resolve the draft PR, head SHA, and base SHA dynamically (locate the open/draft PR introducing the harness), check out the head SHA read-only, and record the exact changed-file list the gates apply to.\n\nSCOPE:\n1. LOC limit: measure every source and test file added or modified by the PR and verify each is under 500 LOC. Report the exact count per file, name any violations, and state your counting method (physical lines) explicitly.\n2. Per-file coverage: run the test suite with per-file coverage reporting and verify every source file in the PR reaches >= 95%. Report the actual per-file percentage table and every uncovered line range for files below threshold. Aggregate/total coverage is not acceptable evidence.\n3. Mutation testing: run mutation tests over the harness modules. Report the mutation score and enumerate surviving mutants with path:line — surviving mutants in verdict-computation, redaction, or read-only-enforcement code are blocking.\n4. Adversarial tests: run the adversarial/negative test suite. If gaps exist, describe the missing cases (do not commit new tests).\n5. Static and hygiene gates: run Ruff lint and Ruff format check, byte-compile all Python sources, produce the diff against the resolved base SHA, and run a secret scan over both the diff and the full tree. Report exact command lines and exit codes for each.\n\nCONSTRAINTS: keep the worktree pristine. You may run tests and read-only tooling in a sandbox, but make no edits, no formatting fixes, no comments, no commits, pushes, merges, deploys, reconciles, or live infrastructure calls. If a gate cannot run, report it as NOT_RUN with the reason — never infer a pass.\n\nDELIVERABLE: a table of gate -> exact command -> exit code -> result (PASS / FAIL / NOT_RUN), the full per-file LOC and coverage tables, the surviving-mutant list, and raw tool output for every failure. Flag each failure as blocking or non-blocking with a one-line justification.", "expected_role": "implementation", "id": "t_a585e217", "title": "Run quality gates: LOC limits, per-file coverage, mutation and adversarial tests"}
|
||||
{"board": "titan-iac", "body": "Implement the first production-safe distributed execution pool for agent.hermes in atlas/titan-iac. The purpose is to move all engineering work now done by local Codex/Claude into agent.hermes while spreading three concurrent workers across distinct cluster nodes.\n\nAuthoritative constraints:\n- Keep exactly one stateful coordinator owning Hermes Kanban/SQLite and the existing RWO hermes-agent-home. Never scale that Deployment above one and never mount its Kanban database or home PVC in a worker.\n- Add three execution workers that can be scheduled on distinct hostnames. Prefer the larger healthy ARM64 accelerator nodes, then healthy rpi5 fallback; use pod anti-affinity/topology spread and honest requests. Do not use titan-22 or titan-24. Preserve all current unhealthy-node exclusions.\n- Each worker needs its own durable RWO workspace/session volume (StatefulSet volumeClaimTemplates is acceptable) so provider sessions and Git state survive pod/node restart without cross-worker filesystem sharing.\n- The coordinator must remain the only component that claims/finalizes Kanban runs. Implement a bounded authenticated assignment/result/heartbeat channel that binds board, task_id, run_id, worker ordinal, attempt, and payload digest. Stale or replacement-run results must never complete/reclaim another run. Duplicate delivery/restart must be idempotent. Lost workers must be recoverable without double execution.\n- Workers clone/fetch only the assigned Atlas repo/feature branch through the reviewed SCM boundary; do not place a raw Gitea token in a model-facing container. Do not use a shared mutable Git worktree. Preserve unrelated/untracked files when a task explicitly owns them or fail closed with a clear blocker.\n- Use only Claude.ai subscription OAuth and ChatGPT/Codex subscription auth from Vault/runtime memory. No ANTHROPIC_API_KEY, OPENAI_API_KEY, Console/API-billed path, secret in Git/ConfigMap/log/command line. Credential refresh targets must be writable and restart-safe.\n- Give workers only the cluster/SCM capabilities required by their role. No cluster-admin, Secrets reads, service-account token creation, workload mutation, exec/attach/port-forward, or node-root access unless routed through a separately reviewed explicit operator boundary. automountServiceAccountToken must default false.\n- Preserve Switchyard automatic provider/model/effort routing and cross-provider fallback. Provider session continuity must remain attached to the exact task/run.\n- Stream sanitized worker activity, route, heartbeat, node/ordinal, and final evidence back into the existing Kanban log/session activity UI so the user can see continuous work at agent.hermes.bstein.dev.\n- Add bounded payloads, deadlines, concurrency, disk retention/GC, nofollow/atomic writes where applicable, and fail-closed startup/readiness checks.\n- Add executable adversarial tests for simultaneous claims, duplicate assignment/result, stale run, coordinator restart, worker restart, heartbeat loss, node replacement, oversized/malformed payloads, path traversal/symlinks, credential absence, cross-worker isolation, three-node scheduling contract, and visible activity.\n- Preserve compatibility with merged PR #13. Inspect open draft PRs #14/#15/#16/#17 and either avoid their surfaces or prove clean synthetic merge behavior; do not copy unreviewed assumptions from them.\n- Render services/hermes and every new Flux Kustomization, run client and Flux-equivalent server dry-runs, relevant full tests, Ruff/compile/shell/diff/secret scans. Keep source files below the repository quality limits.\n- Work only on a feature branch/worktree. Commit and push intentionally, then create one draft Gitea PR with exact validation evidence. Do not merge, publish an image, reconcile Flux, deploy, mutate live worker/node accounts, or claim live three-node proof before human review and rollout.\n\nReturn completed only when the draft PR exists at an exact pushed SHA and all implementable local acceptance checks are green. Findings belong in findings; blockers only for genuine obstacles preventing the task itself.", "expected_role": "implementation", "id": "t_af8d08d0", "title": "Implement a safe three-node agent.hermes execution pool"}
|
||||
{"board": "titan-iac", "body": "Fresh independent read-only review of the implementation produced by parent t_6da029e0. Dynamically resolve exact PR/head/base. Do not edit/comment/commit/push/merge/build/deploy/reconcile.\\n\\nReproduce the original reviewer-loop incident and verify completed review/audit/diagnostic verdicts finalize whether SHIP or BLOCK-with-findings, while malformed/incomplete reviews and unfinished implementation goals continue or fail correctly. Audit explicit role metadata, backward compatibility, restart/replay, provider fallback, session/turn duplication, PR15 semantic merge, bounded sanitized judge evidence, quality/coverage/LOC/render/dry-run. Return strict SHIP/BLOCK findings with exact evidence.", "expected_role": "review", "id": "t_b04a5e67", "title": "Independent review of goal-role completion semantics"}
|
||||
{"board": "titan-iac", "body": "Repair existing draft PR #22 on the SAME feature/hermes-review-goal-semantics branch/worktree after independent review t_b04a5e67 BLOCKED exact head b080b5f622ae0998213f3287762aea30dc931a73. Do not create another PR.\n\nClose every blocker: pass the resolved task role into unfinished_result_reason at both current and PR15 call sites so completed review BLOCK/SHIP cannot be short-circuited by implementation unfinished prose; parse explicit role/output directives from both real newlines and canonical literal backslash-n task bodies without allowing injection; make legacy inference correctly classify the real review/audit/diagnostic card corpus while conflicting or mutating cards fail closed; exclude all runner-appended goal rejection/controller history from card_scope; preserve and document deliberate single-shot semantics; prevent old accepted terminal journals from becoming invalid across upgrade or provide versioned compatibility; do not let changed-files/mutation evidence self-certify as review; use the canonical redactor extended for bare-hex Gitea PAT/basic-auth URL without reading real credentials; pin upstream context-heading contract. Add exact live-card/recovered t_dbdcd739/end-to-end execute_claim regressions. Preserve implementation goal judging, bounded safe reasons, PR15 compatibility, at least 95 percent line and branch coverage, below 500 LOC, full quality/render/dry-run/format/secret gates. Amend/push only existing PR22, no merge/deploy/reconcile.", "expected_role": "implementation", "id": "t_bc268843", "title": "Repair PR 22 review-role goal semantics"}
|
||||
{"board": "titan-iac", "body": "Provide an authenticated noninteractive Forgejo/Gitea API wrapper and concise worker guidance for scm.bstein.dev. It must use only runtime-injected credentials from the existing Vault/credential-broker path, never print tokens, never persist credentials in Git/workspaces/caches, restrict hosts and TLS verification, and make common read operations (PR head/base/state, comments/reviews/checks, branch metadata) straightforward. Mutating operations must be explicit and preserve the human-review default; no merge endpoint by default. Add tests proving URL/host allowlisting, redaction, missing-credential failure, and read-only behavior. Deliver through a focused PR and validate from an isolated Hermes worktree.", "expected_role": "implementation", "id": "t_bf2ff6ec", "title": "Give Hermes workers a first-class safe Gitea PR API client"}
|
||||
{"board": "titan-iac", "body": "Repair existing draft PR #18 on its SAME feature/hermes-distributed-worker-pool branch/worktree after independent review t_5975c06a BLOCKED exact head 689bcb6e48414854447988ba174143a307d308b9. Do not create another PR.\n\nP1 required: lease_failed must be safely retryable and must never poison the persistent SQLite store or CrashLoop the coordinator. Make state transition + Kanban recording recoverable/idempotent with exact-run fencing; catch bounded board errors; retry lease_failed; garbage-collect only after authoritative terminal evidence; ensure one poison row cannot abort later tasks/boards. Wrap startup maintenance as safely as steady state. Add the review's end-to-end wedge/conflicting-duplicate/restart/PVC-persistence repros and fault injection for DB/Kanban errors.\n\nClose consequential handoff risks too:\n- Prevent mediator/worker RWO deadlock during drain/pressure via a self-healing placement/lifecycle design and test eviction/reschedule.\n- Make retries submit safely without updating protected/existing refs: use a unique exact-run/attempt feature ref or a broker-authorized CAS update whose policy remains human-review safe. Preserve prior work and never silently discard results.\n- Avoid repeated slow global npm installs blocking 10m Flux health; use immutable image/persistent verified tooling or decouple health with bounded readiness.\n- Rebase/reconstruct PR18 so it does NOT contain open PR14/15/16/19 as ancestors. Keep PR18's own focused diff based on current main and validate disposable synthetic merges in the documented dependency order. Force-with-lease only the existing branch after exact remote lease check.\n\nPreserve all five fixed P0/security boundaries: canonical run IDs, workspace/project/branch routing, per-ordinal crypto, ConfigMap closure, no credentials/HMAC/SA token/RBAC/SCM in model container, broker-only SCM, safe OAuth ownership, three-node schedulability, >=95% line+branch, <500 LOC. Re-run every t_dbdcd739 and t_5975c06a artifact, full quality, kustomize/kubeconform/client/server dry-runs, scheduling/drain simulation, synthetic current PR merges, diff/secret scans. Amend/push only PR18 and stop for fresh human/independent review. No merge/build publish/deploy/reconcile/live mutation/credential access.", "expected_role": "implementation", "id": "t_c31dd427", "title": "Repair PR 18 lease recovery and release isolation"}
|
||||
{"board": "titan-iac", "body": "In the implementation worktree/branch, validate the completed narrow change directly rather than relying only on rendered manifests. Run all focused scheduler tests, the full relevant Hermes/Kanban regression suite, ruff/static checks, the repository-pinned Docker image build, kustomize build services/hermes, kubectl client dry-run, and git diff --check. Verify the built image contains and executes the patched scheduler behavior through the focused tests or equivalent direct image-level test path. If the tracked workflow supports authenticated internal-registry publication and publishing is necessary for PR deployability, publish only through that workflow, update the Flux manifest to the resulting immutable digest, and record it; otherwise leave a clear explicit blocker and do not invent credentials or use a public registry. Do not deploy, reconcile Flux, restart workloads, or alter live Kanban data. Return command-level pass/fail evidence, image/digest status, failures or residual risks, and any directly required corrections for the branch.", "expected_role": "implementation", "id": "t_c588a4b6", "title": "Run Hermes regression, build, and manifest validation"}
|
||||
{"board": "titan-iac", "body": "Harden agent.hermes after the live Cassandra/titan-iac proof run. Work from the current private atlas/titan-iac main head and create an isolated linked worktree/branch. This is a quality-sensitive platform change. Keep it narrowly scoped to Hermes reliability; preserve unrelated work. Open a Gitea pull request for human review, but do not merge, reconcile Flux, deploy, restart the agent, or mutate live Kanban data except this task's normal lifecycle. Never expose or commit credentials.\n\nAuthoritative live evidence to encode as regressions:\n1. Three simultaneous direct CLI workers on the 4-core hermes-agent node drove load to about 45, made the hermes and oauth2-proxy containers fail probes/restart, and temporarily left the pod 8/10 Ready. Two simultaneous workers remained stable at 10/10. Set HERMES_CLI_LANE_CONCURRENCY to 2 and cap the cli-lane-runner CPU limit at 2 so UI/auth have headroom. Verify requests remain sensible.\n2. Kubernetes service-link variables contaminate worker/test environments (for example a service named hermes-claude-broker creates names that collide with tests), while the deployment already uses DNS service names. Set enableServiceLinks: false at the hermes-agent pod-spec level and add a manifest regression proving it.\n3. Scheduler correctness defect: create_task(initial_status='blocked') records a created event with status blocked but no sticky blocked event. recompute_ready() considers blocked tasks and _has_sticky_block() only recognizes blocked/unblocked events, so an explicitly blocked task with no incomplete parent auto-promotes after the dispatcher cycle. Fix this in the existing Dockerfile patching/test mechanism without weakening dependency-driven promotion or circuit-breaker semantics. Add focused RED-before/GREEN-after tests covering: explicit initial blocked stays blocked with no parents and with already-complete parents; ordinary dependency-blocked tasks promote when dependencies complete; block_task/unblock_task remains correct; circuit-breaker blocks remain sticky. Use official public behavior where possible.\n\nInspect the existing hermes-agent image patches and tests before designing the smallest robust change. Run all focused tests, the full relevant Hermes/Kanban regression suite, ruff/static checks, Docker image build using the repository's pinned Dockerfile, kustomize build services/hermes, kubectl client dry-run, and git diff --check. If the image must be published to make the PR deployable, use only the existing authenticated internal registry workflow, update the digest in Flux manifests, and report the immutable digest; do not deploy it. If publishing is not supported by the tracked workflow, leave an explicit blocker rather than inventing credentials or using a public registry.\n\nReport exact changed files, RED/GREEN evidence, tests, image/digest status, PR URL/head/base state, residual risks, and confirm the PR is open/unmerged/undeployed. Treat a merely rendered manifest as insufficient: prove the scheduler behavior directly.", "expected_role": "implementation", "id": "t_cca008de", "title": "Harden Hermes worker isolation and blocked-task semantics"}
|
||||
{"board": "titan-iac", "body": "Inspect the current private atlas/titan-iac main-head manifests, pinned Dockerfile, image build/publish workflow, Flux image/digest conventions, and existing manifest tests without changing tracked files. Determine the precise hermes-agent pod-spec location for enableServiceLinks: false, the cli-lane-runner container location for CPU limit 2, and the configuration source for HERMES_CLI_LANE_CONCURRENCY=2. Identify how to add a regression that proves enableServiceLinks is false in the rendered/structured manifest and verify request settings remain appropriate. Document the supported authenticated internal-registry publication workflow, whether publishing is actually required for a deployable PR, and the exact validation commands for kustomize and kubectl client dry-run. Do not use credentials, publish images, reconcile Flux, deploy, restart workloads, or mutate live Kanban data.", "expected_role": "implementation", "id": "t_d06ecdaf", "title": "Audit Hermes workload isolation manifests and image workflow"}
|
||||
{"board": "titan-iac", "body": "Perform a fresh independent read-only xhigh release/security review of the completed distributed agent.hermes worker-pool implementation produced by parent task t_af8d08d0. Read AGENTS.md and the parent result, but trust neither claimed tests nor done status. Dynamically resolve its exact pushed draft PR/head/base and review the complete diff against then-current origin/main including merged PR13 and pending PR14-PR16 compatibility. Do not edit files, mutate Kanban beyond returning result, comment, commit, push, merge, publish, deploy, reconcile, or touch live credentials.\n\nPrioritize: coordinator-only SQLite/Kanban/RWO ownership; exactly three workers on distinct eligible nodes with own RWO home/workspace; no shared mutable git; authenticated/bounded assignment, heartbeat, activity, result, retry and ack bound to board/task/run/worker ordinal/attempt/digest; replay/duplicate/stale/replacement-run/restart/partition safety; no lost accepted results; worker subscription OAuth refresh semantics without API keys; broker-only SCM and no raw Gitea token; scoped RBAC/no cluster-admin/secrets/token creation/impersonation/exec-attach-portforward/workload mutation; NetworkPolicy; SSH dedicated account compatibility; Switchyard classification/escalation; agent UI activity visibility; resource requests/anti-affinity/scheduling and RWO behavior on current nodes; rolling upgrade/rollback and coordinator/worker version skew. Run adversarial executable tests, synthetic merges with exact current PR14-PR16 heads where relevant, full focused quality gates, Kustomize/schema/client+Flux-manager server dry-runs, image/runtime static inspection, and read-only live capacity checks. Return strict SHIP or BLOCK with exact file/line and reproducible inputs. Report residual risks. Keep the review worktree pristine.", "expected_role": "review", "id": "t_dbdcd739", "title": "Independent xhigh review of distributed Hermes worker pool"}
|
||||
{"board": "titan-iac", "body": "Independently review the validated branch diff against the original reliability scope and the analysis findings. Confirm scheduler tests directly prove the required blocked-task semantics; confirm manifest configuration provides two CLI lanes, a 2-CPU cli-lane-runner cap, and pod-level enableServiceLinks: false; and reject unrelated changes, credentials, deployment actions, or weakened dependency/circuit-breaker semantics. Resolve only clearly in-scope review issues, rerun affected validation if anything changes, then open a Gitea pull request from the isolated branch to the appropriate current main base for human review. Do not merge, reconcile Flux, deploy, restart the agent, or mutate live Kanban data outside normal lifecycle. Report exact changed files, RED/GREEN and full validation evidence, image and immutable digest or publication blocker, PR URL/head/base, residual risks, and explicit confirmation that the PR is open, unmerged, and undeployed.", "expected_role": "implementation", "id": "t_e3ea56cc", "title": "Review final diff and open an unmerged Gitea PR"}
|
||||
{"board": "titan-iac", "body": "Perform a fresh independent read-only xhigh review of the full-handoff acceptance harness produced by parent t_6496c271. Resolve exact draft PR/head/base dynamically. Do not trust claimed coverage or go/no-go behavior. Keep worktree pristine; no edits/comments/commit/push/merge/deploy/reconcile or live mutations. Audit security and correctness of command construction/redaction before execution, environment/header/body/stdout/stderr/exception sanitization, bounded time/output/concurrency, JSON schema and deterministic overall PASS/NO_GO, mandatory NOT_RUN behavior, dual operator/in-pod evidence, revision/image freshness, partial/malformed/zero-state handling, and explicit ephemeral arming/cleanup/protected-branch rejection. Prove default mode is genuinely read-only through spies/static command inventory. Ensure it cannot read or print Vault/provider/SCM credentials and cannot falsely pass mocked/self-authored evidence. Verify every source/test file <500 LOC and >=95% per-file coverage, run mutation/adversarial tests, full relevant quality gate, Ruff/format/compile/diff/secret scans. Review dependency/merge/rollout/rollback plan against exact current PR12-PR17 and distributed-pool state. Return strict SHIP or BLOCK with executable repros/exact lines and residual risks.", "expected_role": "review", "id": "t_e7f46d44", "title": "Independent review of full-handoff acceptance harness"}
|
||||
{"board": "titan-iac", "body": "Consolidate the five independent audits of the full-handoff acceptance harness (parent task t_6496c271) into one strict, decision-ready verdict.\n\nINPUTS: completed findings from (a) the security/redaction audit, (b) the read-only proof via static command inventory and spies, (c) the determinism/schema/evidence-integrity audit, (d) the executed quality gates (LOC, per-file coverage, mutation, Ruff/format/compile/diff/secret scans), and (e) the dependency/merge/rollout/rollback review. Each reports against a dynamically resolved draft PR, head SHA, and base SHA.\n\nWHAT TO DO:\n1. Confirm all five audits resolved the SAME PR number, head SHA, and base SHA. Divergence is itself a finding and the verdict must account for it.\n2. Deduplicate and reconcile findings. Where two audits disagree on the same code path, adjudicate by reading the code at the resolved head SHA yourself — do not average opinions or defer to the more confident report.\n3. Re-derive severity independently. Treat as blocking by default: any reachable mutation in default mode, any credential leak path (Vault/provider/SCM), any NOT_RUN coerced to PASS, any vacuous PASS on zero-state or partial evidence, any path where mocked or self-authored evidence passes, any file >= 500 LOC, any source file below 95% per-file coverage, any surviving mutant in verdict/redaction/read-only code, any failed lint/compile/secret scan, and any non-executable rollback. Downgrade one of these only with an explicit written justification.\n4. Verify claim coverage: every requirement in the original review scope must map to a finding or an explicit 'verified clean' statement. Anything neither verified nor refuted is an unverified gap, and unverified gaps in security or read-only enforcement force BLOCK.\n\nOUTPUT: a single verdict of exactly SHIP or BLOCK (no conditional or partial verdicts), followed by (a) blocking findings, each with exact path:line at the resolved head SHA and an executable repro command, (b) non-blocking findings, (c) residual risks that remain even if every blocker is fixed, and (d) the unverified-gap list. If BLOCK, give the minimal concrete change set that would flip it to SHIP.\n\nCONSTRAINTS: read-only throughout — no edits, comments, commits, pushes, merges, deploys, reconciles, or live mutations; worktree stays pristine. Do not accept any upstream audit's conclusion at face value where you can cheaply verify it against the code yourself.", "expected_role": "review", "id": "t_fa77b785", "title": "Synthesize independent review into a strict SHIP or BLOCK verdict"}
|
||||
73
testing/tests/data/hermes_t_dbdcd739_context.txt
Normal file
73
testing/tests/data/hermes_t_dbdcd739_context.txt
Normal file
@ -0,0 +1,73 @@
|
||||
# Kanban task t_dbdcd739: Independent xhigh review of distributed Hermes worker pool
|
||||
|
||||
Assignee: cli-claude-xhigh
|
||||
Status: blocked
|
||||
Workspace: worktree @ /opt/data/workspace/projects/titan-iac/.worktrees/t_dbdcd739
|
||||
Max runtime: 21600s
|
||||
Terminal timeout: 21570s
|
||||
Branch: review/hermes-distributed-worker-pool
|
||||
|
||||
## Body
|
||||
Perform a fresh independent read-only xhigh release/security review of the completed distributed agent.hermes worker-pool implementation produced by parent task t_af8d08d0. Read AGENTS.md and the parent result, but trust neither claimed tests nor done status. Dynamically resolve its exact pushed draft PR/head/base and review the complete diff against then-current origin/main including merged PR13 and pending PR14-PR16 compatibility. Do not edit files, mutate Kanban beyond returning result, comment, commit, push, merge, publish, deploy, reconcile, or touch live credentials.
|
||||
|
||||
Prioritize: coordinator-only SQLite/Kanban/RWO ownership; exactly three workers on distinct eligible nodes with own RWO home/workspace; no shared mutable git; authenticated/bounded assignment, heartbeat, activity, result, retry and ack bound to board/task/run/worker ordinal/attempt/digest; replay/duplicate/stale/replacement-run/restart/partition safety; no lost accepted results; worker subscription OAuth refresh semantics without API keys; broker-only SCM and no raw Gitea token; scoped RBAC/no cluster-admin/secrets/token creation/impersonation/exec-attach-portforward/workload mutation; NetworkPolicy; SSH dedicated account compatibility; Switchyard classification/escalation; agent UI activity visibility; resource requests/anti-affinity/scheduling and RWO behavior on current nodes; rolling upgrade/rollback and coordinator/worker version skew. Run adversarial executable tests, synthetic merges with exact current PR14-PR16 heads where relevant, full focused quality gates, Kustomize/schema/client+Flux-manager server dry-runs, image/runtime static inspection, and read-only live capacity checks. Return strict SHIP or BLOCK with exact file/line and reproducible inputs. Report residual risks. Keep the review worktree pristine.
|
||||
|
||||
## Prior attempts on this task
|
||||
### Attempt 1 — blocked (cli-claude-xhigh, 2026-08-17 10:02, 4h ago)
|
||||
Manual shepherd stop: review completed BLOCK three times with durable evidence, but goal judge incorrectly demanded implementation repairs and entered a costly continuation loop. Evidence preserved under /opt/data/workspace/evidence/t_dbdcd739; repair belongs to a separate implementation task.
|
||||
|
||||
## Parent task results
|
||||
_Handoffs from upstream tasks, captured when each parent completed (see age below). These are point-in-time snapshots, not live state — if a result drives your current work and it's not recent, re-verify against the source before acting on it as current._
|
||||
### t_af8d08d0 (completed 4h ago)
|
||||
Implemented and pushed the safe three-node agent.hermes execution pool. PR #18 is verified open and draft at the exact pushed SHA, with one coordinator, three ordinal-isolated RWO workers, authenticated/idempotent run fencing, subscription OAuth handling, narrow SCM/RBAC/network boundaries, visible activity, retention controls, and adversarial coverage.
|
||||
_metadata_: `{"artifacts": ["/opt/data/workspace/projects/titan-iac/.worktrees/t_af8d08d0/build/junit-hermes-pool-final.xml"], "blockers": [], "changed_files": ["clusters/atlas/flux-system/applications/hermes/kustomization.yaml", "services/hermes/execution-coordinator-patch.yaml", "services/hermes/execution-worker-networkpolicy.yaml", "services/hermes/execution-worker-rbac.yaml", "services/hermes/execution-worker-statefulset.yaml", "services/hermes/kustomization.yaml", "services/hermes/scripts/execution_pool_askpass.sh", "services/hermes/scripts/execution_pool_client.py", "services/hermes/scripts/execution_pool_coordinator.py", "services/hermes/scripts/execution_pool_protocol.py", "services/hermes/scripts/execution_pool_scm.py", "services/hermes/scripts/execution_pool_worker.py", "services/hermes/scripts/stage_runtime_access.py", "services/hermes/service.yaml", "services/vault/scripts/vault_k8s_auth_configure.sh", "testing/tests/test_hermes_auto_router.py", "testing/tests/test_hermes_execution_pool.py", "testing/tests/test_hermes_execution_pool_assignment.py", "testing/tests/test_hermes_runtime_access.py"], "classifier": "switchyard-classifier", "claude_session_id": null, "codex_thread_id": "01a00eeb-0327-7962-995e-1e1f235983ed", "effort": "high", "executor": "direct-cli-lane", "findings": ["The repository-wide local quality profile retains an unrelated baseline hygiene failure for four pre-existing test files over 500 lines. Docs, Ruff, unit, and coverage portions pass; all new production files are below 500 lines and the primary adversarial test file is exactly 500 lines.", "No deployment, Flux reconciliation, image publication, workload mutation, or live three-node scheduling proof was performed, as required."], "goal_judge_reason": "The worker report provides concrete evidence that all explicit requirements have been met. The implementation includes the creation of a coordinator and three ordinal-isolated RWO workers, authenticated/idempotent run fencing, subscription OAuth handling, narrow SCM/RBAC/network boundaries, visible activity, retention controls, and adversarial coverage. All tests are green, including unit tests, strict quality checks, production file scans, Kustomize renders, Flux dry-runs, and synthetic merges with open draft PRs. The report also confirms that no deployment or live three-node scheduling proof was performed as required.", "goal_mode": true, "goal_turn": 1, "model": "gpt-5.6-sol", "provider": "codex", "state_file": "/opt/data/cli-lanes/titan-iac/t_af8d08d0.json", "tests_run": ["Configured unit suite: 397 passed", "Final focused pool/lane/runtime/router suite: 150 passed", "Strict Ruff, compileall, sh -n, ShellCheck, and git diff --check: passed", "Production changed-file detect-secrets scan: 0 findings", "Hermes and Vault Kustomize renders: passed", "Hermes and Vault Flux build dry-runs: passed", "Hermes and Vault kubectl client and server-side dry-runs: passed", "PR #14 updated synthetic merge: conflict-free, rendered, 61 tests passed", "PR #15 updated synthetic merge: conflict-free, rendered, 62 tests passed", "PR #16 updated synthetic merge: conflict-free, rendered, 62 tests passed", "PR #17 synthetic merge: conflict-free, rendered, 62 tests passed", "Remote branch and PR head SHA verification: exact match"]}`
|
||||
|
||||
## Recent work by @cli-claude-xhigh
|
||||
- t_5975c06a — Independent re-review of repaired PR 18 distributed pool (2026-08-17 14:19, 11m ago): Independent read-only xhigh re-review of draft PR #18 (feature/hermes-distributed-worker-pool) at exact head 689bcb6e48414854447988ba174143a307d308b9 against current remote main d8f2d818b9a552ea6c2d7f
|
||||
- t_5ae177fc — Independent re-review of repaired PR 19 acceptance harness (2026-08-17 13:52, 38m ago): Independent read-only xhigh re-review of draft PR #19 at exact head c808baff40a12a90f43c2e5e39b94d139f88ea29 against current remote main d8f2d818b9a552ea6c2d7fe86554be829bd5ffff (merge-base 30259b52,
|
||||
- t_261e208f — Independent review of cli-lane process reaping (2026-08-17 11:34, 2h ago): Independent read-only release review of atlas/titan-iac PR #20 (feature/hermes-cli-process-reaping) at head a242dcc786576ae1a18000cb4941836ff610dff2 vs base main d8f2d818b9a552ea6c2d7fe86554be829bd5ff
|
||||
- t_6496c271 — Build fail-closed Hermes full-handoff acceptance harness (2026-08-17 10:17, 4h ago): Built a fail-closed, read-only-by-default Hermes full-handoff acceptance harness (71 checks, 15 new production modules under scripts/ops/, all <500 LOC, 100% per-file line coverage), a 12-module test
|
||||
- t_20f2fd16 — Repair Hermes PR 13 producer-drift regression coverage (2026-08-16 22:46, 15h ago): Amended PR #13 with commit 750dfa24 (pushed to wt/t_cca008de), fixing the single P1 review finding from t_06771c95 and applying the operator recovery addendum.
|
||||
|
||||
## Comment thread
|
||||
comment from worker `cli-lane-runner` at 2026-08-17 10:02, 4h ago:
|
||||
CLI route: claude/claude-opus-5 at xhigh; classifier=switchyard-manual; random routing selected target 'worker/claude/opus/xhigh'
|
||||
|
||||
comment from worker `cli-lane-runner` at 2026-08-17 10:28, 4h ago:
|
||||
Goal completion rejected; continuing turn 2/8: local goal judge requested continuation: The review identified several critical defects that need to be addressed. Specifically, there are five P0 defects which include issues with Coordinator.finalize(), workspace_path gate blocking live tasks, branch regex rejection of established conventions, missing repo_url in Task class, and incorrect import paths due to PR #15 changes. These issues indicate that the implementation is not yet ready for SHIP status as it fails to meet the explicit requirements stated in the objective.
|
||||
|
||||
comment from worker `cli-lane-runner` at 2026-08-17 10:28, 4h ago:
|
||||
Goal route 2/8: claude/claude-opus-5 at xhigh; classifier=switchyard-manual; random routing selected target 'worker/claude/opus/xhigh'
|
||||
|
||||
comment from worker `codex-shepherd` at 2026-08-17 10:29, 4h ago:
|
||||
Reviewer semantics correction: this is a read-only review. A completed review with a BLOCK verdict and concrete findings fully satisfies the task; do not implement fixes in the review worktree. Return status=completed, put release blockers in findings, blockers=[] unless an obstacle prevents reviewing. The local goal judge must not demand that a reviewer repair the reviewed PR.
|
||||
|
||||
comment from worker `cli-lane-runner` at 2026-08-17 10:36, 3h ago:
|
||||
Goal completion rejected; continuing turn 3/8: local goal judge requested continuation: The report identifies several critical defects that need to be addressed, including issues with Coordinator.finalize(), workspace_path gate blocking live tasks, branch regex rejection of established conventions, missing repo_url in Task class, and incorrect import paths due to PR #15 changes. These issues indicate that the implementation is not yet ready for SHIP status as it fails to meet the explicit requirements stated in the objective.
|
||||
|
||||
comment from worker `cli-lane-runner` at 2026-08-17 10:36, 3h ago:
|
||||
Goal route 3/8: claude/claude-opus-5 at xhigh; classifier=switchyard-manual; random routing selected target 'worker/claude/opus/xhigh'
|
||||
|
||||
comment from worker `cli-lane-runner` at 2026-08-17 10:40, 3h ago:
|
||||
Goal completion rejected; continuing turn 4/8: local goal judge requested continuation: The report identifies several critical defects that need to be addressed, including issues with Coordinator.finalize(), workspace_path gate blocking live tasks, branch regex rejection of established conventions, missing repo_url in Task class, and incorrect import paths due to PR #15 changes. These issues indicate that the implementation is not yet ready for SHIP status as it fails to meet the explicit requirements stated in the objective.
|
||||
|
||||
comment from worker `cli-lane-runner` at 2026-08-17 10:40, 3h ago:
|
||||
Goal route 4/8: claude/claude-opus-5 at xhigh; classifier=switchyard-manual; random routing selected target 'worker/claude/opus/xhigh'
|
||||
|
||||
comment from worker `cli-lane-runner` at 2026-08-17 10:42, 3h ago:
|
||||
Provider fallback: claude -> codex; Jetson reclassified the retry boundary.
|
||||
|
||||
comment from worker `cli-lane-runner` at 2026-08-17 10:43, 3h ago:
|
||||
Goal completion rejected; continuing turn 5/8: The live repository instructions override the generic connector path. I found no repo-local AGENTS.md; /opt/data/workspace/AGENTS.md is authoritative for this worktree.
|
||||
|
||||
comment from worker `cli-lane-runner` at 2026-08-17 10:43, 3h ago:
|
||||
Goal route 5/8: codex/gpt-5.6-sol at xhigh; classifier=switchyard-manual-health-guard; Switchyard selected excluded claude; preserved its xhigh effort on healthy-provider route. random routing selected target 'worker/codex/sol/xhigh'
|
||||
|
||||
comment from worker `default` at 2026-08-17 10:44, 3h ago:
|
||||
BLOCKED: Manual shepherd stop: review completed BLOCK three times with durable evidence, but goal judge incorrectly demanded implementation repairs and entered a costly continuation loop. Evidence preserved under /opt/data/workspace/evidence/t_dbdcd739; repair belongs to a separate implementation task.
|
||||
|
||||
comment from worker `cli-lane-runner` at 2026-08-17 10:44, 3h ago:
|
||||
Goal completion rejected; continuing turn 6/8: I’m continuing the review; the last metadata query was interrupted before returning data. I’ll now resolve the Gitea PR state with the mandated helper and independently rerun the inherited blockers and release gates.
|
||||
|
||||
comment from worker `cli-lane-runner` at 2026-08-17 10:44, 3h ago:
|
||||
Goal route 6/8: codex/gpt-5.6-sol at xhigh; classifier=switchyard-manual-health-guard; Switchyard selected excluded claude; preserved its xhigh effort on healthy-provider route. random routing selected target 'worker/codex/sol/xhigh'
|
||||
47
testing/tests/data/hermes_t_dbdcd739_result.json
Normal file
47
testing/tests/data/hermes_t_dbdcd739_result.json
Normal file
@ -0,0 +1,47 @@
|
||||
{
|
||||
"status": "completed",
|
||||
"summary": "Independent read-only xhigh release/security review of PR #18 (atlas/titan-iac, head 20002527512235b29054f3636b02418168373009, base origin/main d8f2d818b9a552ea6c2d7fe86554be829bd5ffff which contains merged PR #13 ab346f55). Verdict: BLOCK. Five P0 defects are executably reproducible, four of which fire on the coordinator's first tick: (1) Coordinator.finalize() compares an int Kanban current_run_id to a str run_id so every accepted result is discarded while the worker is told accepted=True, and the resulting PK collision makes reconcile() throw every cycle; (2) the workspace_path gate force-blocks every live Kanban task including the two currently running ones; (3) the branch regex rejects the established wt/<task-id> convention; (4) Task has no repo_url so every board's tasks are pushed to titan-iac; (5) the pool ConfigMap does not ship the 14 modules PR #15 splits cli_lane_runner into, so coordinator and all three workers fail at import after a clean textual merge. P1 findings cover control-plane node placement, capacity/affinity steering onto the routing model and Flux controllers, model-container access to its own signing/SCM boundaries, worker poisoning of the shared Vault credential store, and bypass of PR #14's SCM authority boundary. Architecture that is sound (coordinator-only Kanban/SQLite, per-ordinal RWO PVCs, no shared mutable git, zero worker RBAC, worker-to-worker isolation, HMAC envelope binding) is called out explicitly. Two observed failures were confirmed pre-existing and unrelated to the PR. The review worktree was restored to review/hermes-distributed-worker-pool @ 0dd6ea0f and is clean including ignored paths.",
|
||||
"changed_files": [],
|
||||
"tests_run": [
|
||||
"Focused pool suite at PR head: pytest testing/tests/test_hermes_execution_pool.py test_hermes_execution_pool_assignment.py test_hermes_runtime_access.py test_hermes_auto_router.py -> 82 passed",
|
||||
"Full suite at PR head (PYTHONPATH=.): pytest testing/tests -> 1 failed, 332 passed (failure pre-existing and environment-sensitive)",
|
||||
"Repo quality gate: python -m testing.quality_gate --profile local -> 1 failed, 396 passed (same pre-existing failure); ruff check on all changed .py -> All checks passed",
|
||||
"compileall on all changed .py -> OK; sh -n services/hermes/scripts/execution_pool_askpass.sh -> OK; git diff --check ab346f55..20002527 -> OK",
|
||||
"Adversarial repro repro_finalize_stale.py -> worker ack accepted=True, complete_task=[] block_task=[], store state=stale had_result=True, all 3 ordinals freed, reconcile() raises ProtocolError: conflicting duplicate assignment",
|
||||
"Adversarial repro repro_workspace_block.py against live titan-iac board row (mode=ro&immutable=1) -> dispatch() and reconcile() both emit kanban block_task; control case with workspace_path cleared still blocks on branch policy",
|
||||
"Adversarial repro repro_configmap_import.py over synthetic merges -> merge-main/pr14/pr16 import rc=0; merge-pr15 rc=1 ModuleNotFoundError: No module named 'cli_lane_board' for both execution_pool_coordinator and execution_pool_worker",
|
||||
"Synthetic merges via git merge-tree --write-tree of PR18 head against origin/main d8f2d818 and PR14 3cd8a17e772978fb554d88dd1c59777786be3991, PR15 7609cd77e87a64e332b2f0d670c1d7a4f62b3d9e, PR16 5ab0eba619779b3dfed2eb09d5c02f0a9c92dd0a, PR17 48cbe13ee50ce3fcb07cea3fe8d088cfed349e0c, PR19 8f005458282269ba5c07941814e4237f5d4cf3ac -> all CLEAN; each materialized tree ran the focused pool suite (81-82 passed) and kustomize build services/hermes (OK)",
|
||||
"kustomize build services/hermes and services/vault -> OK; flux build kustomization hermes --dry-run -> exit 0",
|
||||
"kubectl apply --dry-run=client -> only the pre-existing >256KiB hermes-coordinator annotation error; kubectl apply --dry-run=server --server-side --field-manager=kustomize-controller -> clean (Flux path)",
|
||||
"Read-only live cluster checks: k3s v1.33.3, node labels/taints, apps.kubernetes.io/pod-index availability, astreae StorageClass (Longhorn, Immediate, diskSelector=astreae, 24.6TiB free vs 270GiB needed), per-node free CPU/memory feasibility for the 1150m/2304Mi worker pod, vault pod labels, existing hermes NetworkPolicies, RBAC bindings in the render, image digest drift"
|
||||
],
|
||||
"artifacts": [
|
||||
"/opt/data/workspace/evidence/t_dbdcd739/RUN.md",
|
||||
"/opt/data/workspace/evidence/t_dbdcd739/repro_finalize_stale.py",
|
||||
"/opt/data/workspace/evidence/t_dbdcd739/repro_workspace_block.py",
|
||||
"/opt/data/workspace/evidence/t_dbdcd739/repro_configmap_import.py"
|
||||
],
|
||||
"findings": [
|
||||
"P0-1 BLOCKER - services/hermes/scripts/execution_pool_coordinator.py:248: finalize() evaluates `_task_value(task, \"current_run_id\") != binding[\"run_id\"]` without coercion. hermes_cli/kanban_db.py:853 and the schema at :1108 declare current_run_id as INTEGER, and live rows confirm typeof=integer (titan-iac t_dbdcd739 -> 23), while binding['run_id'] is a str produced by dispatch() at :386 and stored in a TEXT column. int 23 != str '23' is always True, so every result is marked stale: complete_task/block_task are never called, the Kanban task stays 'running' forever, and the worker receives accepted=True and writes terminal_at. Repro repro_finalize_stale.py output: ack {'accepted': True, 'duplicate': False}; complete_task []; block_task []; store state=stale had_result=True; free ordinals [0,1,2]. Follow-on: the row survives under PK (board,task_id,run_id) with a different payload digest, so reconcile() at :358 raises ProtocolError 'conflicting duplicate assignment' (execution_pool_protocol.py:307) on every 5s cycle, aborting the entire reconcile pass for all boards. Zero test coverage: the only tests touching Coordinator.finalize stub it (testing/tests/test_hermes_execution_pool_assignment.py:71 `pool.finalize = called.append`), and grep for current_run_id in both new test files returns nothing.",
|
||||
"P0-2 BLOCKER - services/hermes/scripts/execution_pool_coordinator.py:65-69: resolve_scm raises RuntimeError whenever task.workspace_path is non-empty; dispatch() (:396-406) and reconcile() (:347-356) convert that into kanban_db.block_task(kind='capability'). A read-only census of /opt/data/kanban/boards/*/kanban.db shows every task on titan-iac, cassandra and soteria carries workspace_path. On first coordinator tick this force-blocks the two currently running titan-iac tasks (t_6496c271 and t_dbdcd739, this review) and the two ready cli tasks (t_83252d81, t_87cd9076). Repro repro_workspace_block.py cases [1] and [2].",
|
||||
"P0-3 BLOCKER - services/hermes/scripts/execution_pool_coordinator.py:40,77: BRANCH = ^(?:feature|fix|chore|docs|test|refactor)/... rejects the branch names Hermes itself writes. hermes_cli/kanban_db.py:5447, :7275 and :7367 default worktree tasks to f'wt/{task.id}', and live rows carry wt/t_1e95ea6d as well as review/... names (t_dbdcd739 -> review/hermes-distributed-worker-pool). Repro repro_workspace_block.py case [3]: with workspace_path cleared the same task still blocks with 'task feature branch is outside the SCM branch policy'.",
|
||||
"P0-4 BLOCKER - services/hermes/scripts/execution_pool_coordinator.py:70,79-81: remote and base_branch come from _task_value(task, 'repo_url') / 'base_branch', but the hermes_cli.kanban_db.Task dataclass (kanban_db.py:838-916) defines neither field, so getattr always returns the DEFAULT_REPO https://scm.bstein.dev/atlas/titan-iac.git. Every task on every board is therefore cloned from and pushed to titan-iac. The metis scratch task t_8569b7b9 - the one live task that survives P0-2 - would run against a titan-iac checkout and push to feature/hermes-t_8569b7b9 there.",
|
||||
"P0-5 BLOCKER - services/hermes/kustomization.yaml:64-78 is runtime-incompatible with open PR #14... correction, PR #15 fix/hermes-result-decomposition-reliability head 7609cd77e87a64e332b2f0d670c1d7a4f62b3d9e, despite a clean textual merge. PR15 decomposes cli_lane_runner.py into 14 sibling modules (cli_lane_board, cli_lane_config, cli_lane_dispatch, cli_lane_evidence, cli_lane_execution, cli_lane_files, cli_lane_finalization, cli_lane_prompt, cli_lane_provider, cli_lane_quarantine, cli_lane_records, cli_lane_recovery, cli_lane_retention, cli_lane_routing), while the hermes-execution-pool ConfigMap ships only cli_lane_goal.py and cli_lane_runner.py. Proof repro_configmap_import.py: merge-main/pr14/pr16 import rc=0, merge-pr15 rc=1 ModuleNotFoundError: No module named 'cli_lane_board' for both execution_pool_coordinator and execution_pool_worker -> coordinator plus all three workers CrashLoopBackOff.",
|
||||
"P1 - services/hermes/execution-worker-statefulset.yaml:87-90: the second nodeSelectorTerm requires only arch=arm64, hardware=rpi5 and a hostname NotIn list, dropping the node-role.kubernetes.io/worker In ['true'] requirement that agent-deployment.yaml:109-111 uses. Measured against live capacity: 8 nodes satisfy the affinity, only 4 can fit the 1150m CPU / 2304Mi pod (titan-20 1270m free, titan-21 1520m, titan-0a 3730m, titan-0c 3630m), and titan-0a/titan-0c are k3s control-plane/master. podAntiAffinity forces 3 distinct hosts, so one worker necessarily lands on a control-plane node while running codex --dangerously-bypass-approvals-and-sandbox / claude --dangerously-skip-permissions (cli_lane_runner.py:594, :631).",
|
||||
"P1 - services/hermes/execution-worker-statefulset.yaml:91-99: preferredDuringScheduling weight 100 on node-role.kubernetes.io/accelerator steers two of three workers onto titan-20 (which runs ai/ollama, the Jetson routing model behind Switchyard AUTO) and titan-21 (which runs flux-system source-controller, image-automation-controller, image-reflector-controller and notification-controller). titan-20 retains only ~120m CPU slack after placement while the worker container's limits are 5 CPU / 6Gi, so the pool contends directly with the routing model and the GitOps control plane.",
|
||||
"P1 - services/hermes/scripts/execution_pool_client.py:142 and execution_pool_scm.py:442 bind 0.0.0.0 (required for kubelet httpGet probes). The model container holds no key but shares the pod network namespace, so it can POST 127.0.0.1:9009 {'operation':'poll'} to obtain a freshly coordinator-signed assignment envelope and replay it to 127.0.0.1:9008 /v1/scm {'operation':'submit'} and to 9009 {'operation':'result'}. Authority stays bounded to its own board/task/run/ordinal by ClientBoundary.forward (client.py:84) and _binding (scm.py:123), and worker-to-worker reach is blocked by ingress: [], but it bypasses the worker's own gates: the uncommitted/untracked check (execution_pool_worker.py:332-337) and the commits-ahead check (:342).",
|
||||
"P1 - The model container has read-write access to /runtime-access/claude/.credentials.json and /runtime-access/codex/auth.json (execution-worker-statefulset.yaml:206), and the credential-sync sidecar CAS-writes whatever it finds there into kv/data/atlas/hermes/agent-tokens (sync_runtime_credentials.py:105-116) using role hermes-execution-credential-sync, which services/vault/scripts/vault_k8s_auth_configure.sh:228-229 grants create/update/read on that path. That secret backs the coordinator's own credentials and is the derivation input for the pool HMAC key (printf \"hermes-execution-pool-v1:%s\" .Data.data.agent_api_key | sha256sum). Separately, all three workers plus the agent now refresh the same subscription OAuth documents independently; provider-side refresh-token rotation makes this a lost-update/lockout race that CAS cannot resolve.",
|
||||
"P1 - Open PR #14 feature/hermes-safe-gitea-pr-client (head 3cd8a17e772978fb554d88dd1c59777786be3991) deletes services/hermes/scripts/gitea_api.py and gitea_askpass.sh and introduces a dedicated hermes-scm-broker (own namespace, NetworkPolicy, gitea_api_policy.py, DLP tests) to bound Atlas PR authority. PR18 concurrently stages the raw kv/data/atlas/hermes/developer-gitea token into three new pods (stage_runtime_access.py:192), authenticates git with it (execution_pool_askpass.sh), calls the Gitea REST API directly (execution_pool_scm.py:355-373) and opens the egress path (execution-worker-networkpolicy.yaml:37-42). Verified on the merged tree: merge-pr14 still stages the raw token, so the broker is bypassed for the highest-volume automated PR path and the token blast radius widens by three pods on three more nodes.",
|
||||
"P2 - services/hermes/scripts/execution_pool_protocol.py:227-229,337,385: lease_seconds, lease_until and last_heartbeat are written and never read anywhere in the pool. There is no lease-expiry reaper, so a worker that dies mid-assignment holds its ordinal until Kanban's own DEFAULT_CLAIM_TTL (7 days, cli_lane_runner.py:40) causes a divergence that reconcile() can observe.",
|
||||
"P2 - services/hermes/scripts/execution_pool_worker.py:407: main() swallows OSError/RuntimeError/ValueError, sleeps 10s and re-polls the identical assignment; attempt is never incremented and Kanban is never informed. Reachable today: cli_lane_runner.parse_assignee (:152) raises ValueError for cli-codex-sol-xhigh, a live assignee on two titan-iac tasks, so such an assignment spins forever holding its ordinal. The legacy path blocked the task instead (cli_lane_runner.py:832-848).",
|
||||
"P2 - services/hermes/kustomization.yaml:77-78 sets disableNameSuffixHash: true on hermes-execution-pool, and neither execution-worker-statefulset.yaml nor execution-coordinator-patch.yaml carries a config-revision annotation (agent-deployment.yaml:28 uses ai.bstein.dev/config-rev for exactly this). Pool code changes therefore never trigger a rollout; coordinator and workers pick up new code only on unrelated restarts, producing mixed-version windows against a strict contract that has no negotiation (execution_pool_protocol.py:177 rejects any envelope whose field set differs from version 1).",
|
||||
"P2 - services/hermes/execution-worker-networkpolicy.yaml:33-36 allows worker egress to hermes-model-gate:8080, but services/hermes/networkpolicy.yaml:26-41 (hermes-model-gate-ingress) does not list hermes-execution-worker and nothing in the worker path calls model-gate (select_route targets Switchyard only). The rule is dead and asymmetric; remove it or complete the ingress side.",
|
||||
"P2 - services/hermes/execution-worker-statefulset.yaml:303 mounts the whole runtime-access volume read-write into credential-sync, whereas the agent mounts only the claude and codex subPaths read-only (agent-deployment.yaml:951-952). Minor hardening regression relative to the established pattern.",
|
||||
"PRE-EXISTING, not caused by PR18 - testing/tests/test_hermes_coordinator.py::test_cassandra_sync_repairs_origin_without_token fails wherever /runtime-access/gitea-token exists, because the test never patches hermes_coordinator._gitea_token_available (hermes_coordinator.py:148-155). Neither the module nor the test is in the PR diff and it fails standalone. This accounts for the 1 failure in both the full suite (332 passed) and the quality gate (396 passed), versus the parent task's claim of 397 passed in an environment without that file.",
|
||||
"PRE-EXISTING, not caused by PR18 - kubectl apply --dry-run=client rejects ConfigMap hermes-coordinator with 'metadata.annotations: Too long: may not be more than 262144 bytes'. Measured generator input is 341438 bytes at base ab346f55 and 343841 at head, so the threshold was already exceeded before this PR. Flux uses server-side apply and the SSA dry-run with field-manager kustomize-controller is clean.",
|
||||
"NOTE - ruff format --check would reformat 10 of the changed files, but testing/quality_gate.py:228 runs only ruff check, so this is unenforced style drift rather than a gate failure.",
|
||||
"NOTE - git/cluster image drift: both origin/main and PR18 pin hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107, while the live hermes-agent pod runs sha256:37ebf720c783ae908a602916ffccf88d43d205a157957f5dc4b487867aee45e7. Pre-existing drift, but it means the new workers and the currently running coordinator would execute different image builds until Flux recreates the agent. The pinned digest is cached only on titan-11/15/19, none of the four candidate worker nodes, so first scheduling incurs a cold pull.",
|
||||
"VERIFIED SOUND (no defect) - coordinator-only Kanban/SQLite ownership (workers mount no /opt/data; hermes-agent is replicas:1 with strategy Recreate); per-ordinal RWO 30Gi Longhorn PVCs via volumeClaimTemplates with ample astreae capacity (24624Gi schedulable free vs 270Gi needed across 87 existing volumes); no shared mutable git (per-run clone under /workspace/runs/<board>/<task>/<run>); zero Kubernetes RBAC for the hermes-execution-worker service account (no RoleBinding or ClusterRoleBinding in the render, automountServiceAccountToken: false, Vault-audience token projected only into credential-sync); worker-to-worker isolation via ingress: [] with kubelet probes proven working under the identical pattern on hermes-chat-sandbox; HMAC envelope binding to board/task/run/worker ordinal/attempt/payload digest with expiry, clock-skew bounds, delivery-id replay guard and a partial unique index enforcing one live assignment per ordinal; result idempotency by digest; heartbeat lease loss terminates the provider process (cli_lane_runner.py:46-50); no node-SSH material staged to workers (stage_runtime_access.py:179-204); Switchyard is the only routing path; heartbeat activity plus a deduplicated terminal batch reach the Kanban worker log for agent UI visibility."
|
||||
],
|
||||
"blockers": []
|
||||
}
|
||||
@ -1,257 +0,0 @@
|
||||
"""Owner and Switchyard access-boundary contracts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from testing.tests.test_hermes_cli_support import (
|
||||
HERMES,
|
||||
SCRIPTS,
|
||||
_agent_deployment,
|
||||
yaml,
|
||||
)
|
||||
|
||||
|
||||
def test_owner_agent_has_pinned_dedicated_node_ssh_access():
|
||||
deployment = _agent_deployment()
|
||||
annotations = deployment["spec"]["template"]["metadata"]["annotations"]
|
||||
assert annotations[
|
||||
"vault.hashicorp.com/agent-inject-secret-node-ssh-private-key"
|
||||
] == "kv/data/atlas/hermes/developer-ssh"
|
||||
assert annotations[
|
||||
"vault.hashicorp.com/agent-inject-secret-node-ssh-config"
|
||||
] == "kv/data/atlas/hermes/developer-ssh"
|
||||
assert annotations[
|
||||
"vault.hashicorp.com/agent-inject-secret-node-ssh-known-hosts"
|
||||
] == "kv/data/atlas/hermes/developer-ssh"
|
||||
|
||||
init = next(
|
||||
item
|
||||
for item in deployment["spec"]["template"]["spec"]["initContainers"]
|
||||
if item["name"] == "init-config"
|
||||
)
|
||||
command = init["command"][2]
|
||||
assert "ln -s /runtime-access/node-ssh-config /opt/data/home/.ssh/config" in command
|
||||
assert (
|
||||
"ln -s /runtime-access/node-ssh-known-hosts /opt/data/home/.ssh/known_hosts"
|
||||
in command
|
||||
)
|
||||
assert "ln -s home/.ssh /opt/data/.ssh" in command
|
||||
assert "chmod 0700 /opt/data/home/.ssh" in command
|
||||
assert (
|
||||
"ln -s /runtime-access/node-ssh-private-key "
|
||||
"/opt/data/home/.ssh/id_ed25519_atlas_nodes"
|
||||
) in command
|
||||
|
||||
config = yaml.safe_load((HERMES / "agent-configmap.yaml").read_text())["data"]
|
||||
assert "ssh_config" not in config
|
||||
assert "ssh_known_hosts" not in config
|
||||
assert "ssh-ed25519" not in (HERMES / "agent-configmap.yaml").read_text()
|
||||
|
||||
resources = yaml.safe_load((HERMES / "kustomization.yaml").read_text())[
|
||||
"resources"
|
||||
]
|
||||
assert "node-ssh-access.yaml" in resources
|
||||
access = [
|
||||
item
|
||||
for item in yaml.safe_load_all((HERMES / "node-ssh-access.yaml").read_text())
|
||||
if item
|
||||
]
|
||||
service_account = next(item for item in access if item["kind"] == "ServiceAccount")
|
||||
assert service_account["metadata"]["name"] == "hermes-node-ssh-access"
|
||||
provider = next(item for item in access if item["kind"] == "SecretProviderClass")
|
||||
assert provider["metadata"]["name"] == "hermes-node-ssh-access"
|
||||
assert provider["spec"]["provider"] == "vault"
|
||||
parameters = provider["spec"]["parameters"]
|
||||
assert parameters["roleName"] == "hermes-node-ssh"
|
||||
assert 'secretPath: "kv/data/atlas/hermes/developer-ssh"' in parameters["objects"]
|
||||
assert 'secretKey: "public_key"' in parameters["objects"]
|
||||
daemonset = next(item for item in access if item["kind"] == "DaemonSet")
|
||||
pod = daemonset["spec"]["template"]["spec"]
|
||||
assert pod["serviceAccountName"] == "hermes-node-ssh-access"
|
||||
assert pod["automountServiceAccountToken"] is True
|
||||
host_home = next(item for item in pod["volumes"] if item["name"] == "host-home")
|
||||
assert host_home["hostPath"] == {"path": "/home", "type": "Directory"}
|
||||
vault_secrets = next(
|
||||
item for item in pod["volumes"] if item["name"] == "vault-secrets"
|
||||
)
|
||||
assert vault_secrets["csi"]["driver"] == "secrets-store.csi.k8s.io"
|
||||
assert vault_secrets["csi"]["volumeAttributes"] == {
|
||||
"secretProviderClass": "hermes-node-ssh-access"
|
||||
}
|
||||
reconciler = pod["containers"][0]["args"][0]
|
||||
assert "/opt/node-hardener/node_account_hardening.py" in reconciler
|
||||
assert "--public-key-file /vault/secrets/node-ssh-public-key" in reconciler
|
||||
assert "grep -qxF" not in reconciler
|
||||
host_etc = next(item for item in pod["volumes"] if item["name"] == "host-etc")
|
||||
assert host_etc["hostPath"] == {"path": "/etc", "type": "Directory"}
|
||||
hardener = next(item for item in pod["volumes"] if item["name"] == "coordinator")
|
||||
assert hardener["configMap"] == {
|
||||
"name": "hermes-node-account-hardener",
|
||||
"defaultMode": 0o555,
|
||||
}
|
||||
|
||||
|
||||
def test_owner_agent_tracks_no_ssh_identity_or_host_key_material():
|
||||
"""Vault references may be tracked; SSH identities and trust data may not."""
|
||||
forbidden = (
|
||||
"BEGIN OPENSSH PRIVATE KEY",
|
||||
"ssh-ed25519 AAAA",
|
||||
"ssh-rsa AAAA",
|
||||
"IdentityFile ",
|
||||
"UserKnownHostsFile ",
|
||||
"StrictHostKeyChecking ",
|
||||
"ssh_config:",
|
||||
"ssh_known_hosts:",
|
||||
)
|
||||
text_suffixes = {
|
||||
".conf",
|
||||
".json",
|
||||
".md",
|
||||
".py",
|
||||
".sh",
|
||||
".toml",
|
||||
".yaml",
|
||||
".yml",
|
||||
}
|
||||
tracked = "\n".join(
|
||||
path.read_text(encoding="utf-8")
|
||||
for path in HERMES.rglob("*")
|
||||
if path.is_file() and path.suffix in text_suffixes
|
||||
)
|
||||
for marker in forbidden:
|
||||
assert marker not in tracked
|
||||
|
||||
|
||||
def test_switchyard_has_a_dedicated_non_owner_identity_and_read_only_catalog():
|
||||
"""Routing must not inherit the owner agent's cluster-admin capability."""
|
||||
service_accounts = [
|
||||
item
|
||||
for item in yaml.safe_load_all(
|
||||
(HERMES / "vault-serviceaccount.yaml").read_text()
|
||||
)
|
||||
if item
|
||||
]
|
||||
assert any(
|
||||
item["kind"] == "ServiceAccount"
|
||||
and item["metadata"]["name"] == "hermes-switchyard"
|
||||
for item in service_accounts
|
||||
)
|
||||
|
||||
switchyard = yaml.safe_load(
|
||||
(HERMES / "switchyard-deployment.yaml").read_text()
|
||||
)
|
||||
switchyard_pod = switchyard["spec"]["template"]["spec"]
|
||||
assert switchyard_pod["serviceAccountName"] == "hermes-switchyard"
|
||||
agent_pod = _agent_deployment()["spec"]["template"]["spec"]
|
||||
for pod, container_name in (
|
||||
(switchyard_pod, "worker-route-broker"),
|
||||
(agent_pod, "claude-broker"),
|
||||
):
|
||||
container = next(item for item in pod["containers"] if item["name"] == container_name)
|
||||
catalog = next(
|
||||
item
|
||||
for item in container["volumeMounts"]
|
||||
if item["mountPath"] == "/routing-catalog"
|
||||
)
|
||||
assert catalog["readOnly"] is True
|
||||
|
||||
rbac = [
|
||||
item
|
||||
for item in yaml.safe_load_all((HERMES / "rbac.yaml").read_text())
|
||||
if item
|
||||
]
|
||||
bindings = [item for item in rbac if item["kind"] == "ClusterRoleBinding"]
|
||||
assert bindings
|
||||
assert all(
|
||||
subject["name"] not in {"hermes-agent", "hermes-switchyard"}
|
||||
for binding in bindings
|
||||
for subject in binding["subjects"]
|
||||
)
|
||||
|
||||
|
||||
def test_switchyard_active_state_uses_a_relocatable_rwx_claim():
|
||||
"""A stale node attachment must not strand the routing authority."""
|
||||
claims = [
|
||||
item
|
||||
for item in yaml.safe_load_all((HERMES / "switchyard-pvc.yaml").read_text())
|
||||
if item
|
||||
]
|
||||
active_claim = next(
|
||||
item
|
||||
for item in claims
|
||||
if item["metadata"]["name"] == "hermes-switchyard-state-rwx"
|
||||
)
|
||||
assert active_claim["spec"]["accessModes"] == ["ReadWriteMany"]
|
||||
|
||||
deployment = yaml.safe_load((HERMES / "switchyard-deployment.yaml").read_text())
|
||||
strategy = deployment["spec"]["strategy"]
|
||||
assert strategy == {
|
||||
"type": "RollingUpdate",
|
||||
"rollingUpdate": {"maxSurge": 1, "maxUnavailable": 0},
|
||||
}
|
||||
pod = deployment["spec"]["template"]["spec"]
|
||||
state = next(item for item in pod["volumes"] if item["name"] == "state")
|
||||
assert state["persistentVolumeClaim"]["claimName"] == active_claim["metadata"][
|
||||
"name"
|
||||
]
|
||||
|
||||
|
||||
def test_worker_route_broker_accepts_pod_network_health_checks():
|
||||
"""Kubelet probes the pod IP, so the broker cannot bind to loopback only."""
|
||||
script = (SCRIPTS / "worker_route_broker.py").read_text()
|
||||
assert 'ThreadingHTTPServer(("0.0.0.0", PORT), Handler)' in script
|
||||
|
||||
|
||||
def test_switchyard_network_boundary_allows_vault_bootstrap():
|
||||
"""The pre-populate init container must reach Vault before routing starts."""
|
||||
documents = [
|
||||
item
|
||||
for item in yaml.safe_load_all((HERMES / "networkpolicy.yaml").read_text())
|
||||
if item
|
||||
]
|
||||
isolation = next(
|
||||
item
|
||||
for item in documents
|
||||
if item.get("metadata", {}).get("name") == "hermes-switchyard-isolation"
|
||||
)
|
||||
assert any(
|
||||
rule.get("to")
|
||||
== [
|
||||
{
|
||||
"namespaceSelector": {
|
||||
"matchLabels": {"kubernetes.io/metadata.name": "vault"}
|
||||
},
|
||||
"podSelector": {"matchLabels": {"app": "vault"}},
|
||||
}
|
||||
]
|
||||
and rule.get("ports") == [{"protocol": "TCP", "port": 8200}]
|
||||
for rule in isolation["spec"]["egress"]
|
||||
)
|
||||
|
||||
|
||||
def test_switchyard_network_boundary_allows_metrics_scraping():
|
||||
"""VictoriaMetrics may scrape Switchyard without widening its API boundary."""
|
||||
documents = [
|
||||
item
|
||||
for item in yaml.safe_load_all((HERMES / "networkpolicy.yaml").read_text())
|
||||
if item
|
||||
]
|
||||
isolation = next(
|
||||
item
|
||||
for item in documents
|
||||
if item.get("metadata", {}).get("name") == "hermes-switchyard-isolation"
|
||||
)
|
||||
assert any(
|
||||
rule.get("from")
|
||||
== [
|
||||
{
|
||||
"namespaceSelector": {
|
||||
"matchLabels": {
|
||||
"kubernetes.io/metadata.name": "monitoring"
|
||||
}
|
||||
},
|
||||
"podSelector": {"matchLabels": {"app": "server"}},
|
||||
}
|
||||
]
|
||||
and rule.get("ports") == [{"protocol": "TCP", "port": 9005}]
|
||||
for rule in isolation["spec"]["ingress"]
|
||||
)
|
||||
@ -36,7 +36,9 @@ def test_claude_pretool_hook_allows_normal_engineering():
|
||||
assert policy.denial_reason("vault kv get kv/atlas/hermes") is None
|
||||
|
||||
|
||||
def test_claude_settings_preserve_state_and_install_three_guardrail_layers(tmp_path: Path):
|
||||
def test_claude_settings_preserve_state_and_install_three_guardrail_layers(
|
||||
tmp_path: Path,
|
||||
):
|
||||
state = tmp_path / ".claude.json"
|
||||
settings = tmp_path / "settings.json"
|
||||
state.write_text('{"promptQueueUseCount": 4}\n', encoding="utf-8")
|
||||
@ -78,7 +80,9 @@ def test_claude_settings_preserve_state_and_install_three_guardrail_layers(tmp_p
|
||||
def test_legacy_state_is_archived_without_removing_provider_transcripts(tmp_path: Path):
|
||||
session = tmp_path / "home/.config/herdr/session.json"
|
||||
session.parent.mkdir(parents=True)
|
||||
session.write_text('{"agents":[{"agent":"claude","session_id":"abc"}]}', encoding="utf-8")
|
||||
session.write_text(
|
||||
'{"agents":[{"agent":"claude","session_id":"abc"}]}', encoding="utf-8"
|
||||
)
|
||||
binary = tmp_path / "tools/bin/herdr"
|
||||
binary.parent.mkdir(parents=True)
|
||||
binary.write_text("legacy", encoding="utf-8")
|
||||
@ -156,9 +160,7 @@ def test_agent_avoids_unhealthy_nodes_and_fits_its_remaining_capacity():
|
||||
assert hostnames["operator"] == "NotIn"
|
||||
assert set(hostnames["values"]) >= {"titan-04", "titan-19"}
|
||||
|
||||
hermes = next(
|
||||
item for item in pod["containers"] if item["name"] == "hermes"
|
||||
)
|
||||
hermes = next(item for item in pod["containers"] if item["name"] == "hermes")
|
||||
assert hermes["resources"]["requests"]["cpu"] == "300m"
|
||||
|
||||
|
||||
@ -223,8 +225,7 @@ def test_agent_root_is_stock_dashboard_and_terminal_is_a_separate_path():
|
||||
)
|
||||
|
||||
patch_init = next(
|
||||
item for item in pod["initContainers"]
|
||||
if item["name"] == "patch-tui-gateway"
|
||||
item for item in pod["initContainers"] if item["name"] == "patch-tui-gateway"
|
||||
)
|
||||
assert patch_init["command"][-1] == "/patched/server.py"
|
||||
for name in ("hermes", "terminal"):
|
||||
@ -248,20 +249,29 @@ def test_agent_root_is_stock_dashboard_and_terminal_is_a_separate_path():
|
||||
assert middlewares["hermes-agent-terminal-slash"]["spec"]["redirectRegex"][
|
||||
"replacement"
|
||||
].endswith("/terminal/")
|
||||
assert middlewares["hermes-agent-stock-dashboard-headers"]["spec"]["headers"][
|
||||
"customRequestHeaders"
|
||||
]["Origin"] == "http://127.0.0.1:9119"
|
||||
assert (
|
||||
middlewares["hermes-agent-stock-dashboard-headers"]["spec"]["headers"][
|
||||
"customRequestHeaders"
|
||||
]["Origin"]
|
||||
== "http://127.0.0.1:9119"
|
||||
)
|
||||
ingresses = {
|
||||
item["metadata"]["name"]: item
|
||||
for item in ingress_documents
|
||||
if item["kind"] == "Ingress"
|
||||
}
|
||||
assert ingresses["hermes-agent-dashboard"]["metadata"]["annotations"][
|
||||
"traefik.ingress.kubernetes.io/router.middlewares"
|
||||
] == "hermes-hermes-agent-stock-dashboard-headers@kubernetescrd"
|
||||
assert ingresses["hermes-agent-terminal"]["metadata"]["annotations"][
|
||||
"traefik.ingress.kubernetes.io/router.middlewares"
|
||||
] == "hermes-hermes-agent-terminal-slash@kubernetescrd"
|
||||
assert (
|
||||
ingresses["hermes-agent-dashboard"]["metadata"]["annotations"][
|
||||
"traefik.ingress.kubernetes.io/router.middlewares"
|
||||
]
|
||||
== "hermes-hermes-agent-stock-dashboard-headers@kubernetescrd"
|
||||
)
|
||||
assert (
|
||||
ingresses["hermes-agent-terminal"]["metadata"]["annotations"][
|
||||
"traefik.ingress.kubernetes.io/router.middlewares"
|
||||
]
|
||||
== "hermes-hermes-agent-terminal-slash@kubernetescrd"
|
||||
)
|
||||
|
||||
|
||||
def test_broker_services_survive_sibling_container_readiness_loss():
|
||||
@ -275,6 +285,19 @@ def test_broker_services_survive_sibling_container_readiness_loss():
|
||||
):
|
||||
assert services[name]["spec"]["publishNotReadyAddresses"] is True
|
||||
|
||||
# The agent web endpoints route through this Service to the same pod the
|
||||
# cli-lane-runner readiness probe gates; a deferred lane image must not
|
||||
# take down the dashboard or terminal.
|
||||
agent_auth = next(
|
||||
item
|
||||
for item in yaml.safe_load_all((HERMES / "oauth2-proxy.yaml").read_text())
|
||||
if item
|
||||
and item["kind"] == "Service"
|
||||
and item["metadata"]["name"] == "oauth2-proxy-hermes-agent"
|
||||
)
|
||||
assert agent_auth["spec"]["selector"] == {"app": "hermes-agent"}
|
||||
assert agent_auth["spec"]["publishNotReadyAddresses"] is True
|
||||
|
||||
|
||||
def test_cli_lane_domain_modules_are_all_mounted_with_the_runner():
|
||||
manifest = yaml.safe_load((HERMES / "kustomization.yaml").read_text())
|
||||
@ -322,9 +345,7 @@ def test_cli_lane_config_refresh_does_not_restart_active_work():
|
||||
for item in deployment["spec"]["template"]["spec"]["containers"]
|
||||
if item["name"] == "cli-lane-runner"
|
||||
)
|
||||
mount = next(
|
||||
item for item in lane["volumeMounts"] if item["name"] == "coordinator"
|
||||
)
|
||||
mount = next(item for item in lane["volumeMounts"] if item["name"] == "coordinator")
|
||||
|
||||
assert coordinator["options"]["disableNameSuffixHash"] is True
|
||||
assert mount == {
|
||||
@ -335,12 +356,28 @@ def test_cli_lane_config_refresh_does_not_restart_active_work():
|
||||
assert "checksum/hermes-coordinator" not in deployment["spec"]["template"].get(
|
||||
"metadata", {}
|
||||
).get("annotations", {})
|
||||
assert "startupProbe" not in lane
|
||||
assert "livenessProbe" not in lane
|
||||
assert lane["readinessProbe"] == {
|
||||
"exec": {
|
||||
"command": [
|
||||
"/opt/hermes/.venv/bin/python",
|
||||
"/opt/coordinator/cli_lane_capabilities.py",
|
||||
]
|
||||
},
|
||||
"initialDelaySeconds": 2,
|
||||
"periodSeconds": 5,
|
||||
"timeoutSeconds": 5,
|
||||
"failureThreshold": 3,
|
||||
}
|
||||
environment = {item["name"]: item["value"] for item in lane["env"]}
|
||||
assert environment["HERMES_CLI_HEALTH_MAX_AGE_SECONDS"] == "60"
|
||||
|
||||
|
||||
def test_agent_dashboard_reconnects_all_transient_websockets():
|
||||
dockerfile = (
|
||||
HERMES.parents[1] / "dockerfiles/Dockerfile.hermes-agent"
|
||||
).read_text(encoding="utf-8")
|
||||
dockerfile = (HERMES.parents[1] / "dockerfiles/Dockerfile.hermes-agent").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
assert "eventsRetryAttempt.current" in dockerfile
|
||||
assert "if (!unmounting) setVersion((v) => v + 1);" in dockerfile
|
||||
assert "events feed rejected (${ev.code}) — reload the page" in dockerfile
|
||||
@ -350,3 +387,33 @@ def test_agent_dashboard_reconnects_all_transient_websockets():
|
||||
assert ".then(() => gw.connect())" in dockerfile
|
||||
assert 'api.getSessions(1, 0, profile ?? "")' in dockerfile
|
||||
assert "dashboard token rotated by a server restart" in dockerfile
|
||||
|
||||
|
||||
def test_agent_image_runs_execution_safety_patch_and_regressions():
|
||||
dockerfiles = HERMES.parents[1] / "dockerfiles"
|
||||
dockerfile = (dockerfiles / "Dockerfile.hermes-agent").read_text(encoding="utf-8")
|
||||
dockerignore = (dockerfiles / "Dockerfile.hermes-agent.dockerignore").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
|
||||
for name in (
|
||||
"patch-hermes-execution-safety.py",
|
||||
"hermes-execution-safety-regression.py",
|
||||
"hermes_execution_patch_support.py",
|
||||
"patch_hermes_run_safety.py",
|
||||
"patch_hermes_decomposition_safety.py",
|
||||
"hermes_execution_regression_support.py",
|
||||
"hermes_run_safety_regression.py",
|
||||
"hermes_decomposition_safety_regression.py",
|
||||
):
|
||||
assert f"COPY dockerfiles/{name}" in dockerfile
|
||||
assert f"!dockerfiles/{name}" in dockerignore
|
||||
assert "/opt/hermes/.venv/bin/python /tmp/patch-hermes-execution-safety.py" in (
|
||||
dockerfile
|
||||
)
|
||||
assert "/opt/hermes/.venv/bin/python /tmp/hermes-execution-safety-regression.py" in (
|
||||
dockerfile
|
||||
)
|
||||
assert "COPY services/hermes/scripts/cli_lane_*.py" in dockerfile
|
||||
assert "!services/hermes/scripts/cli_lane_*.py" in dockerignore
|
||||
assert "HERMES_CLI_LANE_SOURCE=/tmp/hermes-lane-regression" in dockerfile
|
||||
|
||||
@ -1,235 +0,0 @@
|
||||
"""Agent authentication, networking, and Flux health contracts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from testing.tests.test_hermes_cli_support import (
|
||||
FLUX_HERMES,
|
||||
HERMES,
|
||||
KEYCLOAK,
|
||||
_agent_deployment,
|
||||
yaml,
|
||||
)
|
||||
|
||||
|
||||
def test_agent_image_runs_execution_safety_patch_and_regressions():
|
||||
dockerfiles = HERMES.parents[1] / "dockerfiles"
|
||||
dockerfile = (dockerfiles / "Dockerfile.hermes-agent").read_text(encoding="utf-8")
|
||||
dockerignore = (dockerfiles / "Dockerfile.hermes-agent.dockerignore").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
|
||||
for name in (
|
||||
"patch-hermes-execution-safety.py",
|
||||
"hermes-execution-safety-regression.py",
|
||||
"hermes_execution_patch_support.py",
|
||||
"patch_hermes_run_safety.py",
|
||||
"patch_hermes_decomposition_safety.py",
|
||||
"hermes_execution_regression_support.py",
|
||||
"hermes_run_safety_regression.py",
|
||||
"hermes_decomposition_safety_regression.py",
|
||||
):
|
||||
assert f"COPY dockerfiles/{name}" in dockerfile
|
||||
assert f"!dockerfiles/{name}" in dockerignore
|
||||
assert "/opt/hermes/.venv/bin/python /tmp/patch-hermes-execution-safety.py" in (
|
||||
dockerfile
|
||||
)
|
||||
assert "/opt/hermes/.venv/bin/python /tmp/hermes-execution-safety-regression.py" in (
|
||||
dockerfile
|
||||
)
|
||||
assert "COPY services/hermes/scripts/cli_lane_*.py" in dockerfile
|
||||
assert "!services/hermes/scripts/cli_lane_*.py" in dockerignore
|
||||
assert "HERMES_CLI_LANE_SOURCE=/tmp/hermes-lane-regression" in dockerfile
|
||||
|
||||
|
||||
def test_agent_refreshes_routes_after_restoring_cli_logins():
|
||||
deployment = _agent_deployment()
|
||||
init_containers = {
|
||||
item["name"]: item
|
||||
for item in deployment["spec"]["template"]["spec"]["initContainers"]
|
||||
}
|
||||
configure = init_containers["configure-agent-clients"]
|
||||
command = configure["command"][-1]
|
||||
assert "configure_agent_clients.py" in command
|
||||
assert command.index("configure_agent_clients.py") < command.index(
|
||||
"hermes_coordinator.py --once"
|
||||
)
|
||||
env = {item["name"]: item["value"] for item in configure["env"]}
|
||||
assert env["HERMES_AUTH_FILE"] == "/runtime-access/hermes-auth.json"
|
||||
assert env["PYTHONPATH"] == "/opt/hermes"
|
||||
for name in ("bootstrap-coordinator", "configure-agent-clients"):
|
||||
route_env = {
|
||||
item["name"]: item["value"]
|
||||
for item in init_containers[name]["env"]
|
||||
}
|
||||
assert route_env["CODEX_HOME"] == "/runtime-access/codex"
|
||||
assert route_env["CLAUDE_CONFIG_DIR"] == "/runtime-access/claude"
|
||||
assert "/opt/data/tools/bin" in route_env["PATH"]
|
||||
|
||||
assert "patch-web-session-activity" in init_containers
|
||||
web_patch = init_containers["patch-web-session-activity"]
|
||||
assert "/opt/coordinator/patch_web_session_activity.py" in web_patch["command"]
|
||||
|
||||
containers = {
|
||||
item["name"]: item
|
||||
for item in deployment["spec"]["template"]["spec"]["containers"]
|
||||
}
|
||||
steward_env = {
|
||||
item["name"]: item["value"]
|
||||
for item in containers["model-steward"]["env"]
|
||||
}
|
||||
assert steward_env["CODEX_HOME"] == "/runtime-access/codex"
|
||||
assert steward_env["CLAUDE_CONFIG_DIR"] == "/runtime-access/claude"
|
||||
assert "/opt/data/tools/bin" in steward_env["PATH"]
|
||||
hermes_mounts = {
|
||||
(item["name"], item["mountPath"], item.get("subPath"))
|
||||
for item in containers["hermes"]["volumeMounts"]
|
||||
}
|
||||
assert (
|
||||
"web-server-patch",
|
||||
"/opt/hermes/hermes_cli/web_server.py",
|
||||
"web_server.py",
|
||||
) in hermes_mounts
|
||||
|
||||
|
||||
def test_flux_health_checks_follow_the_owner_oauth_sidecar():
|
||||
flux = yaml.safe_load(FLUX_HERMES.read_text())
|
||||
checks = {
|
||||
(item["kind"], item["name"])
|
||||
for item in flux["spec"]["healthChecks"]
|
||||
}
|
||||
assert ("Deployment", "hermes-agent") in checks
|
||||
assert ("DaemonSet", "hermes-node-ssh-access") in checks
|
||||
assert ("Deployment", "oauth2-proxy-hermes-agent") not in checks
|
||||
|
||||
|
||||
def test_agent_auth_is_bstein_group_and_email_bounded():
|
||||
deployment = _agent_deployment()
|
||||
oauth = next(
|
||||
item for item in deployment["spec"]["template"]["spec"]["containers"]
|
||||
if item["name"] == "oauth2-proxy"
|
||||
)
|
||||
args = oauth["args"]
|
||||
assert "--user-id-claim=sub" in args
|
||||
assert "--oidc-groups-claim=groups" in args
|
||||
assert "--allowed-group=/hermes-owner" in args
|
||||
assert "--authenticated-emails-file=/etc/oauth2-proxy/allowed-emails" in args
|
||||
script = (KEYCLOAK / "scripts/hermes_access_oidc_ensure.sh").read_text()
|
||||
assert 'group_name="hermes-owner"' in script
|
||||
assert "username=bstein&exact=true" in script
|
||||
assert '"full.path":"true"' in script
|
||||
|
||||
|
||||
def test_agent_network_boundary_allows_only_authenticated_and_metrics_surfaces():
|
||||
documents = [
|
||||
item
|
||||
for item in yaml.safe_load_all((HERMES / "networkpolicy.yaml").read_text())
|
||||
if item
|
||||
]
|
||||
isolation = next(item for item in documents if item.get("metadata", {}).get("name") == "hermes-agent-isolation")
|
||||
assert isolation["spec"]["ingress"] == [
|
||||
{
|
||||
"from": [
|
||||
{
|
||||
"namespaceSelector": {
|
||||
"matchLabels": {
|
||||
"kubernetes.io/metadata.name": "traefik"
|
||||
}
|
||||
},
|
||||
"podSelector": {
|
||||
"matchLabels": {"app.kubernetes.io/name": "traefik"}
|
||||
},
|
||||
}
|
||||
],
|
||||
"ports": [{"protocol": "TCP", "port": 4180}],
|
||||
},
|
||||
{
|
||||
"from": [
|
||||
{
|
||||
"podSelector": {
|
||||
"matchLabels": {"app": "hermes-chat-tenant"}
|
||||
}
|
||||
}
|
||||
],
|
||||
"ports": [
|
||||
{"protocol": "TCP", "port": 9002},
|
||||
{"protocol": "TCP", "port": 9003},
|
||||
],
|
||||
},
|
||||
{
|
||||
"from": [
|
||||
{
|
||||
"podSelector": {
|
||||
"matchLabels": {"app": "hermes-switchyard"}
|
||||
}
|
||||
}
|
||||
],
|
||||
"ports": [
|
||||
{"protocol": "TCP", "port": 9003},
|
||||
{"protocol": "TCP", "port": 9006},
|
||||
],
|
||||
},
|
||||
{
|
||||
"from": [
|
||||
{
|
||||
"namespaceSelector": {
|
||||
"matchLabels": {
|
||||
"kubernetes.io/metadata.name": "monitoring"
|
||||
}
|
||||
},
|
||||
"podSelector": {"matchLabels": {"app": "server"}},
|
||||
}
|
||||
],
|
||||
"ports": [{"protocol": "TCP", "port": 9010}],
|
||||
},
|
||||
]
|
||||
egress = isolation["spec"]["egress"]
|
||||
assert {} not in egress
|
||||
broker = next(
|
||||
rule
|
||||
for rule in egress
|
||||
if rule.get("ports") == [{"protocol": "TCP", "port": 9081}]
|
||||
)
|
||||
assert broker["to"] == [
|
||||
{
|
||||
"namespaceSelector": {
|
||||
"matchLabels": {"kubernetes.io/metadata.name": "hermes-scm"}
|
||||
},
|
||||
"podSelector": {"matchLabels": {"app": "hermes-scm-broker"}},
|
||||
}
|
||||
]
|
||||
namespace_rule = next(
|
||||
rule
|
||||
for rule in egress
|
||||
if rule.get("to", [{}])[0].get("namespaceSelector", {}).get(
|
||||
"matchExpressions"
|
||||
)
|
||||
)
|
||||
assert namespace_rule["to"][0]["namespaceSelector"]["matchExpressions"] == [
|
||||
{
|
||||
"key": "kubernetes.io/metadata.name",
|
||||
"operator": "NotIn",
|
||||
"values": ["gitea", "hermes-scm"],
|
||||
}
|
||||
]
|
||||
public = next(
|
||||
rule
|
||||
for rule in egress
|
||||
if rule.get("to", [{}])[0].get("ipBlock", {}).get("cidr") == "0.0.0.0/0"
|
||||
)
|
||||
assert "192.168.0.0/16" in public["to"][0]["ipBlock"]["except"]
|
||||
|
||||
|
||||
def test_owner_agent_has_observer_kubernetes_context_without_cluster_admin():
|
||||
config = yaml.safe_load((HERMES / "agent-kubeconfig.yaml").read_text())
|
||||
assert config["current-context"] == "atlas-observer"
|
||||
assert config["contexts"][0]["context"]["namespace"] == "default"
|
||||
rbac_path = HERMES / "rbac.yaml"
|
||||
documents = [item for item in yaml.safe_load_all(rbac_path.read_text()) if item]
|
||||
bindings = [item for item in documents if item["kind"] == "ClusterRoleBinding"]
|
||||
assert bindings
|
||||
assert all(binding["roleRef"]["name"] != "cluster-admin" for binding in bindings)
|
||||
assert all(
|
||||
subject["name"] != "hermes-agent"
|
||||
for binding in bindings
|
||||
for subject in binding["subjects"]
|
||||
)
|
||||
@ -1,75 +0,0 @@
|
||||
"""Hermes chat codex broker contracts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from test_hermes_chat_support import (
|
||||
ModuleType,
|
||||
Path,
|
||||
_load_broker_module,
|
||||
base64,
|
||||
json,
|
||||
sys,
|
||||
time,
|
||||
)
|
||||
|
||||
|
||||
def test_codex_broker_refreshes_and_persists_first_party_oauth(
|
||||
tmp_path: Path, monkeypatch
|
||||
):
|
||||
"""Expired ChatGPT OAuth refreshes in the canonical Codex CLI store."""
|
||||
module = _load_broker_module(
|
||||
"hermes_codex_refresh_broker", "codex_broker.py", monkeypatch
|
||||
)
|
||||
auth_dir = tmp_path / ".codex"
|
||||
auth_dir.mkdir()
|
||||
|
||||
def jwt(expires_at: float) -> str:
|
||||
payload = base64.urlsafe_b64encode(
|
||||
json.dumps({"exp": expires_at}).encode()
|
||||
).decode().rstrip("=")
|
||||
return f"header.{payload}.signature"
|
||||
|
||||
expired = jwt(time.time() - 60)
|
||||
live = jwt(time.time() + 3600)
|
||||
auth_path = auth_dir / "auth.json"
|
||||
auth_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"auth_mode": "chatgpt",
|
||||
"tokens": {
|
||||
"access_token": expired,
|
||||
"refresh_token": "refresh-old",
|
||||
},
|
||||
}
|
||||
)
|
||||
)
|
||||
calls = []
|
||||
auth_module = ModuleType("hermes_cli.auth")
|
||||
|
||||
def refresh(access_token, refresh_token, *, timeout_seconds):
|
||||
calls.append((access_token, refresh_token, timeout_seconds))
|
||||
return {
|
||||
"access_token": live,
|
||||
"refresh_token": "refresh-new",
|
||||
"last_refresh": "2026-08-12T20:00:00Z",
|
||||
}
|
||||
|
||||
auth_module.refresh_codex_oauth_pure = refresh
|
||||
package = ModuleType("hermes_cli")
|
||||
package.auth = auth_module
|
||||
monkeypatch.setitem(sys.modules, "hermes_cli", package)
|
||||
monkeypatch.setitem(sys.modules, "hermes_cli.auth", auth_module)
|
||||
monkeypatch.setenv("CODEX_HOME", str(auth_dir))
|
||||
|
||||
assert module._access_token() == live
|
||||
persisted = json.loads(auth_path.read_text())
|
||||
assert persisted["tokens"]["access_token"] == live
|
||||
assert persisted["tokens"]["refresh_token"] == "refresh-new"
|
||||
assert persisted["last_refresh"] == "2026-08-12T20:00:00Z"
|
||||
assert calls == [(expired, "refresh-old", 30.0)]
|
||||
assert auth_path.stat().st_mode & 0o777 == 0o600
|
||||
|
||||
# A healthy token is reused, so repeated routed turns do not spend a
|
||||
# refresh token or create a second billing/authentication path.
|
||||
assert module._access_token() == live
|
||||
assert len(calls) == 1
|
||||
@ -5,7 +5,6 @@ from __future__ import annotations
|
||||
import importlib.util
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import yaml
|
||||
@ -164,6 +163,7 @@ def test_compact_image_edit_resolves_latest_tenant_artifact(tmp_path, monkeypatc
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def test_chat_reasoning_uses_switchyard_without_owner_credentials():
|
||||
"""Family pods use AUTO/manual routes without mounting owner credentials."""
|
||||
configmap = _documents(HERMES / "chat-configmap.yaml")[0]
|
||||
@ -280,105 +280,3 @@ def test_chat_reasoning_uses_switchyard_without_owner_credentials():
|
||||
assert broker_ingress["from"][0]["podSelector"]["matchLabels"] == {
|
||||
"app": "hermes-chat-tenant"
|
||||
}
|
||||
def test_image_broker_returns_bytes_and_removes_owner_cache(tmp_path: Path, monkeypatch):
|
||||
"""The broker must not retain a family user's generated image."""
|
||||
broker_path = HERMES / "scripts" / "image_broker.py"
|
||||
spec = importlib.util.spec_from_file_location("hermes_image_broker", broker_path)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
generated = tmp_path / "generated.png"
|
||||
generated.write_bytes(b"\x89PNG\r\n\x1a\nprivate-image")
|
||||
|
||||
class Provider:
|
||||
def generate(self, prompt, aspect, **kwargs):
|
||||
assert prompt == "paint a blue sphere"
|
||||
assert aspect == "square"
|
||||
return {
|
||||
"success": True,
|
||||
"image": str(generated),
|
||||
"model": "gpt-image-2-high",
|
||||
"quality": "high",
|
||||
}
|
||||
|
||||
monkeypatch.setattr(module, "_PROVIDER", Provider())
|
||||
result = module._generate(
|
||||
{
|
||||
"prompt": "paint a blue sphere",
|
||||
"aspect_ratio": "square",
|
||||
"model": "gpt-image-2-high",
|
||||
}
|
||||
)
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["image_b64"]
|
||||
assert "image" not in result
|
||||
assert not generated.exists()
|
||||
|
||||
def test_image_broker_auto_falls_back_to_local_and_honors_explicit_routes(
|
||||
monkeypatch,
|
||||
):
|
||||
"""AUTO is hosted-first while explicit local never calls the hosted lane."""
|
||||
broker_path = HERMES / "scripts" / "image_broker.py"
|
||||
spec = importlib.util.spec_from_file_location("hermes_image_router", broker_path)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
calls = []
|
||||
|
||||
def hosted(payload, model, prompt, aspect):
|
||||
calls.append(("hosted", model, prompt, aspect))
|
||||
return {"success": False, "error": "hosted refusal"}
|
||||
|
||||
def local(payload, timeout=1800.0):
|
||||
calls.append(("local", payload["model"], timeout))
|
||||
return {
|
||||
"success": True,
|
||||
"image_b64": "aW1hZ2U=",
|
||||
"model": "flux-2-klein-4b-local",
|
||||
"route": "local",
|
||||
}
|
||||
|
||||
monkeypatch.setattr(module, "_generate_hosted", hosted)
|
||||
monkeypatch.setattr(module, "_local_request", local)
|
||||
auto = module._generate(
|
||||
{
|
||||
"prompt": "colorize this family photograph",
|
||||
"aspect_ratio": "portrait",
|
||||
"model": "atlas-image-auto-high",
|
||||
}
|
||||
)
|
||||
assert auto["success"] is True
|
||||
assert auto["route"] == "local"
|
||||
assert auto["hosted_fallback_reason"] == "hosted refusal"
|
||||
assert [call[0] for call in calls] == ["hosted", "local"]
|
||||
|
||||
calls.clear()
|
||||
explicit = module._generate(
|
||||
{
|
||||
"prompt": "make a local landscape",
|
||||
"aspect_ratio": "landscape",
|
||||
"model": "flux-2-klein-4b-local",
|
||||
}
|
||||
)
|
||||
assert explicit["route"] == "local"
|
||||
assert [call[0] for call in calls] == ["local"]
|
||||
|
||||
def test_image_broker_policy_is_narrow_and_operator_extensible(tmp_path: Path, monkeypatch):
|
||||
"""Family-photo restoration stays allowed while the hard boundary remains."""
|
||||
broker_path = HERMES / "scripts" / "image_broker.py"
|
||||
spec = importlib.util.spec_from_file_location("hermes_image_policy", broker_path)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
policy = tmp_path / "policy.json"
|
||||
policy.write_text('{"additional_blocked_phrases":["site-specific block"]}')
|
||||
monkeypatch.setattr(module, "POLICY_PATH", policy)
|
||||
|
||||
assert module._policy_error(
|
||||
"Colorize my baby photograph with a lighter natural skin tone"
|
||||
) is None
|
||||
assert "minors" in module._policy_error("Create a sexual image of a child")
|
||||
assert module._policy_error("A site-specific block request") == (
|
||||
"request is blocked by the operator image policy"
|
||||
)
|
||||
|
||||
@ -1,295 +0,0 @@
|
||||
"""Hermes chat local compute contracts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from test_hermes_chat_support import (
|
||||
HERMES,
|
||||
Path,
|
||||
ROOT,
|
||||
SimpleNamespace,
|
||||
_documents,
|
||||
importlib,
|
||||
json,
|
||||
tomllib,
|
||||
)
|
||||
|
||||
|
||||
def test_local_flux_runtime_and_gpu_handoff_are_flux_managed():
|
||||
"""FLUX and Wolf share titan-24 while text stays on titan-20."""
|
||||
deployment = _documents(HERMES / "local-image-deployment.yaml")[0]
|
||||
assert deployment["metadata"]["name"] == "hermes-local-image"
|
||||
pod = deployment["spec"]["template"]["spec"]
|
||||
assert pod["serviceAccountName"] == "hermes-gpu-runtime"
|
||||
local = next(item for item in pod["containers"] if item["name"] == "local-image")
|
||||
assert len(pod["containers"]) == 1
|
||||
assert local["resources"]["requests"]["nvidia.com/gpu.shared"] == 1
|
||||
assert local["ports"] == [{"name": "local-image", "containerPort": 9004}]
|
||||
assert any(mount["mountPath"] == "/models" for mount in local["volumeMounts"])
|
||||
model_env = {item["name"]: item["value"] for item in local["env"]}
|
||||
assert model_env["HERMES_LOCAL_IMAGE_LISTEN_PORT"] == "9004"
|
||||
assert model_env["HERMES_LOCAL_IMAGE_REVISION"] == (
|
||||
"e7b7dc27f91deacad38e78976d1f2b499d76a294"
|
||||
)
|
||||
assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVITY_NODE"] == "titan-24"
|
||||
assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVE_SM_PERCENT"] == "80"
|
||||
assert model_env["HERMES_LOCAL_IMAGE_GPU_MAX_EXTERNAL_MEMORY_BYTES"] == (
|
||||
"3221225472"
|
||||
)
|
||||
assert model_env["HERMES_LOCAL_IMAGE_OFFLOAD_MODE"] == "sequential"
|
||||
assert "nvidia-process-exporter-local.monitoring.svc.cluster.local" in model_env[
|
||||
"HERMES_LOCAL_IMAGE_GPU_ACTIVITY_URL"
|
||||
]
|
||||
models_volume = next(item for item in pod["volumes"] if item["name"] == "models")
|
||||
assert models_volume["persistentVolumeClaim"]["claimName"] == (
|
||||
"hermes-image-models"
|
||||
)
|
||||
|
||||
services = _documents(HERMES / "service.yaml")
|
||||
image_service = next(
|
||||
item for item in services if item["metadata"]["name"] == "hermes-local-image"
|
||||
)
|
||||
assert image_service["spec"]["selector"] == {"app": "hermes-local-image"}
|
||||
|
||||
handoff_services = _documents(HERMES / "model-gate-deployment.yaml")
|
||||
handoff = next(
|
||||
item
|
||||
for item in handoff_services
|
||||
if item["kind"] == "Service"
|
||||
and item["metadata"]["name"] == "hermes-gpu-handoff"
|
||||
)
|
||||
assert handoff["spec"]["ports"][0]["targetPort"] == "handoff"
|
||||
|
||||
ariadne = _documents(
|
||||
Path(__file__).parents[2]
|
||||
/ "services/maintenance/apps/ariadne-deployment.yaml"
|
||||
)[0]
|
||||
env = {
|
||||
item["name"]: item["value"]
|
||||
for item in ariadne["spec"]["template"]["spec"]["containers"][0]["env"]
|
||||
if "value" in item
|
||||
}
|
||||
assert env["GAME_MODE_OLLAMA_URL"] == (
|
||||
"http://hermes-gpu-handoff.hermes.svc.cluster.local:11434"
|
||||
)
|
||||
assert env["GAME_MODE_OLLAMA_MODEL"] == "flux-2-klein-4b-local"
|
||||
|
||||
for config_name in ("configmap.yaml", "agent-configmap.yaml", "chat-configmap.yaml"):
|
||||
config = _documents(HERMES / config_name)[0]["data"]["config.yaml"]
|
||||
assert "gpt-oss:20b" not in config
|
||||
assert "atlas-switchyard" in config
|
||||
switchyard = _documents(HERMES / "switchyard-configmap.yaml")[0]["data"][
|
||||
"routes.toml"
|
||||
]
|
||||
assert 'id = "qwen2.5:14b-instruct-q4_0"' in switchyard
|
||||
assert "qwen2.5:3b-instruct-q4_0" not in switchyard
|
||||
assert "route/local/qwen2.5-14b/medium" in switchyard
|
||||
assert "Anthropic and Claude name the same provider" in switchyard
|
||||
assert "OpenAI and Codex name the same provider" in switchyard
|
||||
assert "Choose across every configured Codex and Claude family" in switchyard
|
||||
assert "Claude Fable" in switchyard
|
||||
assert switchyard.count('Treat "think hard"') == 4
|
||||
assert switchyard.count("Never choose below the") >= 5
|
||||
switchyard_config = tomllib.loads(switchyard)
|
||||
routes = switchyard_config["routes"]
|
||||
configured_targets = switchyard_config["targets"]
|
||||
for route_name in ("auto_fast", "auto_balanced", "auto_deep", "auto_maximum"):
|
||||
leading_targets = set(routes[route_name]["targets"][:2])
|
||||
assert leading_targets == {"codex_sol_xhigh", "claude_opus_xhigh"}
|
||||
assert "max_output_tokens" not in routes[route_name]
|
||||
for route_name in ("auto_fast", "auto_balanced", "auto_deep", "auto_maximum"):
|
||||
targets = routes[route_name]["targets"]
|
||||
selector_targets = routes[route_name]["response_schema"]
|
||||
assert not any(target.startswith("local_") for target in targets)
|
||||
assert any("fable" in target for target in targets)
|
||||
assert "local_qwen" not in selector_targets
|
||||
assert "not eligible for foreground" in routes[route_name]["prompt"]
|
||||
for route_name in ("auto_deep", "auto_maximum"):
|
||||
targets = routes[route_name]["targets"]
|
||||
selector_targets = routes[route_name]["response_schema"]
|
||||
assert not any(target.endswith("_low") for target in targets)
|
||||
assert "_low" not in selector_targets
|
||||
maximum_targets = routes["auto_maximum"]["targets"]
|
||||
maximum_selector_targets = routes["auto_maximum"]["response_schema"]
|
||||
assert not any(target.endswith("_medium") for target in maximum_targets)
|
||||
assert "_medium" not in maximum_selector_targets
|
||||
assert "absolute high effort floor" in routes["auto_maximum"]["prompt"]
|
||||
assert "quality mark was missed" in routes["auto_balanced"]["prompt"]
|
||||
assert "raises the next boundary to xhigh" in routes["auto_maximum"]["prompt"]
|
||||
assert "Repeated quality misses require xhigh" in routes[
|
||||
"worker_auto_maximum"
|
||||
]["prompt"]
|
||||
assert any(
|
||||
target.startswith("local_")
|
||||
for target in routes["manual_local_qwen"]["targets"]
|
||||
)
|
||||
for route_name in (
|
||||
"manual_codex_luna",
|
||||
"manual_codex_terra",
|
||||
"manual_codex_sol",
|
||||
"manual_claude_haiku",
|
||||
"manual_claude_fable",
|
||||
"manual_claude_sonnet",
|
||||
"manual_claude_opus",
|
||||
):
|
||||
assert not any(
|
||||
target.startswith("local_") for target in routes[route_name]["targets"]
|
||||
)
|
||||
for provider, families in {
|
||||
"codex": ("luna", "terra", "sol"),
|
||||
"claude": ("haiku", "fable", "sonnet", "opus"),
|
||||
}.items():
|
||||
for family in families:
|
||||
for effort in ("low", "medium", "high", "xhigh"):
|
||||
route = routes[f"manual_{provider}_{family}_{effort}"]
|
||||
assert route["id"] == f"atlas/manual/{provider}/{family}/{effort}"
|
||||
assert route["targets"][0] == f"{provider}_{family}_{effort}"
|
||||
worker_target = f"worker_{provider}_{family}_{effort}"
|
||||
assert worker_target in routes["worker_auto_maximum"]["targets"]
|
||||
assert configured_targets[worker_target]["id"] == (
|
||||
f"worker/{provider}/{family}/{effort}"
|
||||
)
|
||||
for route_name in ("auto_fast", "auto_balanced"):
|
||||
prompt = routes[route_name]["prompt"]
|
||||
assert "image tool—not the conversational model" in prompt
|
||||
assert "Do not select a" in prompt
|
||||
assert "local Qwen or Claude target" in prompt
|
||||
assert "max_output_tokens" not in routes["worker_auto_maximum"]
|
||||
model_gate = _documents(HERMES / "model-gate-configmap.yaml")[0]["data"][
|
||||
"model_gate.py"
|
||||
]
|
||||
assert "qwen2.5:14b-instruct-q4_0" in model_gate
|
||||
|
||||
def test_titan20_serializes_classifier_and_local_chat_model_residency():
|
||||
"""Classifier and local chat share one serialized resident Qwen weight."""
|
||||
deployment = _documents(
|
||||
Path(__file__).parents[2] / "services/ai-llm/deployment.yaml"
|
||||
)[0]
|
||||
pod = deployment["spec"]["template"]["spec"]
|
||||
required = pod["affinity"]["nodeAffinity"][
|
||||
"requiredDuringSchedulingIgnoredDuringExecution"
|
||||
]["nodeSelectorTerms"][0]["matchExpressions"][0]
|
||||
assert required["values"] == ["titan-20"]
|
||||
container = pod["containers"][0]
|
||||
env = {item["name"]: item["value"] for item in container["env"]}
|
||||
assert env["OLLAMA_MAX_LOADED_MODELS"] == "1"
|
||||
assert env["OLLAMA_NUM_PARALLEL"] == "1"
|
||||
assert env["OLLAMA_KEEP_ALIVE"] == "-1"
|
||||
assert env["OLLAMA_CONTEXT_LENGTH"] == "8192"
|
||||
warm_command = " ".join(container["command"])
|
||||
assert "--keepalive=-1" not in warm_command
|
||||
models = next(item for item in pod["volumes"] if item["name"] == "models")
|
||||
assert models["persistentVolumeClaim"]["claimName"] == (
|
||||
"ollama-models-titan20"
|
||||
)
|
||||
|
||||
def test_local_image_gpu_guard_distinguishes_background_and_saturated_gpu(monkeypatch):
|
||||
"""Lease-idle desktop spikes may coexist, but saturation still blocks FLUX."""
|
||||
source = ROOT / "dockerfiles" / "hermes-local-image-server.py"
|
||||
spec = importlib.util.spec_from_file_location("hermes_local_image_server", source)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
monkeypatch.setattr(module, "GPU_ACTIVITY_NODE", "titan-24")
|
||||
monkeypatch.setattr(module, "GPU_ACTIVE_SM_PERCENT", 80.0)
|
||||
monkeypatch.setattr(module, "GPU_MAX_EXTERNAL_MEMORY_BYTES", 3 << 30)
|
||||
|
||||
idle = module._parse_gpu_activity(
|
||||
'\n'.join(
|
||||
[
|
||||
'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="Xorg"} 0',
|
||||
'nvidia_process_gpu_memory_used_bytes{node="titan-24",namespace="host",process="Xorg"} 1900000000',
|
||||
'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="game-stream",process="wolf"} 3',
|
||||
'nvidia_process_gpu_memory_used_bytes{node="titan-24",namespace="hermes",process="python"} 9000000000',
|
||||
]
|
||||
)
|
||||
)
|
||||
assert idle["interactive_active"] is False
|
||||
assert idle["external_gpu_memory_bytes"] == 1900000000
|
||||
assert idle["external_gpu_sm_percent"] == 3
|
||||
|
||||
background_spike = module._parse_gpu_activity(
|
||||
'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="sway"} 41\n'
|
||||
)
|
||||
assert background_spike["interactive_active"] is False
|
||||
|
||||
active = module._parse_gpu_activity(
|
||||
'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="steam"} 91\n'
|
||||
)
|
||||
assert active["interactive_active"] is True
|
||||
assert "91%" in active["gpu_guard_reason"]
|
||||
|
||||
def test_local_flux_renderer_uses_a_disposable_cuda_worker(monkeypatch):
|
||||
"""A completed render must not leave its CUDA context in the API process."""
|
||||
source = ROOT / "dockerfiles" / "hermes-local-image-server.py"
|
||||
spec = importlib.util.spec_from_file_location("hermes_local_image_worker", source)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
calls = []
|
||||
|
||||
def run(command, **kwargs):
|
||||
calls.append((command, kwargs))
|
||||
return SimpleNamespace(
|
||||
returncode=0,
|
||||
stdout=b'{"success":true,"route":"local","image_b64":"cG5n"}',
|
||||
stderr=b"",
|
||||
)
|
||||
|
||||
monkeypatch.setattr(module.subprocess, "run", run)
|
||||
result = module._render({"prompt": "black cat", "aspect_ratio": "square"})
|
||||
|
||||
assert result["route"] == "local"
|
||||
command, options = calls[0]
|
||||
assert command[-1] == "--render-worker"
|
||||
assert json.loads(options["input"]) == {
|
||||
"prompt": "black cat",
|
||||
"aspect_ratio": "square",
|
||||
}
|
||||
assert options["timeout"] == module.RENDER_TIMEOUT_SECONDS
|
||||
assert options["check"] is False
|
||||
|
||||
def test_local_flux_uses_low_vram_offload_without_reducing_resolution():
|
||||
"""The shared 3080 lane must trade time, not image size, for headroom."""
|
||||
source = (ROOT / "dockerfiles" / "hermes-local-image-server.py").read_text()
|
||||
assert 'OFFLOAD_MODE = os.environ.get(' in source
|
||||
assert '"HERMES_LOCAL_IMAGE_OFFLOAD_MODE", "sequential"' in source
|
||||
assert "pipe.enable_sequential_cpu_offload()" in source
|
||||
assert '"square": (1024, 1024)' in source
|
||||
|
||||
def test_chat_auth_and_relay_are_pod_lifetime_only():
|
||||
statefulset = _documents(HERMES / "chat-statefulset.yaml")[0]
|
||||
pod = statefulset["spec"]["template"]["spec"]
|
||||
containers = statefulset["spec"]["template"]["spec"]["containers"]
|
||||
|
||||
for name in ("hermes", "webui"):
|
||||
container = next(item for item in containers if item["name"] == name)
|
||||
env = {item["name"]: item["value"] for item in container["env"]}
|
||||
assert env["HERMES_AUTH_FILE"] == "/runtime-access/hermes-auth.json"
|
||||
mount = next(
|
||||
item for item in container["volumeMounts"] if item["name"] == "runtime-access"
|
||||
)
|
||||
assert mount["mountPath"] == "/runtime-access"
|
||||
assert "subPath" not in mount
|
||||
|
||||
hermes_env = {
|
||||
item["name"]: item["value"]
|
||||
for item in next(item for item in containers if item["name"] == "hermes")["env"]
|
||||
}
|
||||
runtime = next(item for item in pod["volumes"] if item["name"] == "runtime-access")
|
||||
assert runtime["emptyDir"] == {"medium": "Memory", "sizeLimit": "2Mi"}
|
||||
assert not any(item["name"] == "provider-auth" for item in pod["volumes"])
|
||||
init_command = next(
|
||||
item for item in pod["initContainers"] if item["name"] == "init-config"
|
||||
)["command"][2]
|
||||
for key in (
|
||||
"ANTHROPIC_API_KEY",
|
||||
"API_SERVER_KEY",
|
||||
"CLAUDE_CODE_OAUTH_TOKEN",
|
||||
"GITEA_TOKEN",
|
||||
"HERMES_IMAGE_BROKER_KEY",
|
||||
"OPENAI_API_KEY",
|
||||
):
|
||||
assert key in init_command
|
||||
assert "printf 'API_SERVER_KEY=%s" not in init_command
|
||||
assert hermes_env["AGENT_BROWSER_EXECUTABLE_PATH"].endswith("/chrome-linux/headless_shell")
|
||||
assert "--no-sandbox" in hermes_env["AGENT_BROWSER_ARGS"]
|
||||
@ -451,3 +451,26 @@ def test_codex_broker_refreshes_and_persists_first_party_oauth(
|
||||
# refresh token or create a second billing/authentication path.
|
||||
assert module._access_token() == live
|
||||
assert len(calls) == 1
|
||||
|
||||
|
||||
def test_claude_broker_uses_native_subscription_without_api_billing(monkeypatch):
|
||||
"""Claude traffic must use the native first-party CLI subscription lane."""
|
||||
module = _load_broker_module(
|
||||
"hermes_claude_broker", "claude_oauth_broker.py", monkeypatch
|
||||
)
|
||||
monkeypatch.setenv("ANTHROPIC_API_KEY", "must-not-leak")
|
||||
monkeypatch.setenv("CLAUDE_API_KEY", "must-not-leak")
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
"resolve_route",
|
||||
lambda route: "claude-fable-5" if "/fable/" in route else route,
|
||||
)
|
||||
|
||||
model, effort = module._route(
|
||||
"route/claude/fable/xhigh", {"output_config": {"effort": "xhigh"}}
|
||||
)
|
||||
|
||||
assert (model, effort) == ("claude-fable-5", "xhigh")
|
||||
assert "ANTHROPIC_API_KEY" not in module._claude_environment()
|
||||
assert "CLAUDE_API_KEY" not in module._claude_environment()
|
||||
assert module.CAPACITY_PATTERN.search("weekly usage limit exhausted")
|
||||
|
||||
@ -1,15 +1,19 @@
|
||||
"""Hermes chat config contracts."""
|
||||
"""Core configuration and workload contracts for Hermes chat."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from test_hermes_chat_support import (
|
||||
import importlib.util
|
||||
import sys
|
||||
from types import SimpleNamespace
|
||||
|
||||
import yaml
|
||||
|
||||
from testing.tests.test_hermes_chat_support import (
|
||||
HERMES,
|
||||
ROOT,
|
||||
_documents,
|
||||
yaml,
|
||||
)
|
||||
|
||||
|
||||
def test_chat_config_enables_real_research_compute_and_delegation():
|
||||
configmap = _documents(HERMES / "chat-configmap.yaml")[0]
|
||||
config = yaml.safe_load(configmap["data"]["config.yaml"])
|
||||
@ -37,6 +41,7 @@ def test_chat_config_enables_real_research_compute_and_delegation():
|
||||
assert "terminal" not in toolsets
|
||||
assert "code_execution" not in toolsets
|
||||
|
||||
|
||||
def test_agent_config_keeps_delegated_reviewers_from_owning_task_lifecycle():
|
||||
configmap = _documents(HERMES / "agent-configmap.yaml")[0]
|
||||
instructions = configmap["data"]["AGENTS.md"]
|
||||
@ -56,7 +61,8 @@ def test_agent_config_keeps_delegated_reviewers_from_owning_task_lifecycle():
|
||||
assert "Atlas organization has private visibility" in instructions
|
||||
assert "may be public or private" in instructions
|
||||
assert "do not infer a\nrepository's visibility" in instructions
|
||||
assert "Atlas Git access is supplied by the isolated SCM\nbroker" in soul
|
||||
assert "brokered Git for clone, fetch" in instructions
|
||||
assert "client carries no repository credential" in instructions
|
||||
assert "Never call `kanban_show` without a known, non-empty task ID" in instructions
|
||||
assert "bounded ad-hoc inspection and acceptance checks may" in instructions
|
||||
assert "load implementation or TDD skills" in instructions
|
||||
@ -64,15 +70,21 @@ def test_agent_config_keeps_delegated_reviewers_from_owning_task_lifecycle():
|
||||
assert "must load a skill only when its workflow\nmaterially applies" in soul
|
||||
assert "no repository token is present in this pod" in soul
|
||||
assert "`scm.bstein.dev` is Forgejo/Gitea, not GitHub" in instructions
|
||||
assert "Never load or follow a\nGitHub/`gh`" in instructions
|
||||
assert "use `/opt/scm/gitea_api.py`" in instructions
|
||||
assert "never bypass the client with raw HTTP" in instructions
|
||||
assert "GitHub/`gh` skill for an Atlas remote" in instructions
|
||||
assert "load\n`$manage-atlas-pull-requests`" in instructions
|
||||
assert "Updates, merge,\napprove, close, delete, comments" in instructions
|
||||
assert "Leave every PR\nunmerged for Brad's review" in instructions
|
||||
assert "Use the Gitea pull-request merge endpoint" not in instructions
|
||||
assert "reconcile the open PR" not in instructions
|
||||
assert "use Flux reconciliation after" not in instructions
|
||||
assert "PR publication ends at the verified open draft" in instructions
|
||||
assert "JENKINS_BASE_URL" in instructions
|
||||
assert "Do not\nuse `git reset --hard`" in instructions
|
||||
rendered = (HERMES / "agent-deployment.yaml").read_text()
|
||||
assert 'git config --global user.name "Hermes Agent"' in rendered
|
||||
assert 'git config --global user.email "hermes@bstein.dev"' in rendered
|
||||
|
||||
|
||||
def test_agent_image_completes_parked_kanban_tasks_atomically():
|
||||
dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-agent").read_text()
|
||||
|
||||
@ -88,6 +100,7 @@ def test_agent_image_completes_parked_kanban_tasks_atomically():
|
||||
assert "payload.get(\"status\") == \"blocked\"" in dockerfile
|
||||
assert "hermes-kanban-blocked-regression.py" in dockerfile
|
||||
|
||||
|
||||
def test_sandbox_shares_only_the_tenant_workspace_without_credentials():
|
||||
sandbox_docs = _documents(HERMES / "chat-sandbox.yaml")
|
||||
deployments = [doc for doc in sandbox_docs if doc["kind"] == "Deployment"]
|
||||
@ -151,6 +164,7 @@ def test_sandbox_shares_only_the_tenant_workspace_without_credentials():
|
||||
f"hermes-chat-tenant-{ordinal}"
|
||||
)
|
||||
|
||||
|
||||
def test_gateway_image_honors_ui_model_and_caps_reasoning():
|
||||
dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-agent").read_text()
|
||||
assert (
|
||||
@ -209,6 +223,7 @@ def test_gateway_image_honors_ui_model_and_caps_reasoning():
|
||||
assert 'params.has("resume")' in dockerfile
|
||||
assert "Open Telegram setup" in dockerfile
|
||||
|
||||
|
||||
def test_telegram_router_local_build_defaults_to_cluster_architecture():
|
||||
dockerfile = (
|
||||
ROOT / "dockerfiles" / "Dockerfile.hermes-chat-router"
|
||||
@ -218,6 +233,7 @@ def test_telegram_router_local_build_defaults_to_cluster_architecture():
|
||||
assert "ARG TARGETARCH=arm64" in dockerfile
|
||||
assert "GOOS=${TARGETOS} GOARCH=${TARGETARCH}" in dockerfile
|
||||
|
||||
|
||||
def test_chat_oauth_allows_stale_service_worker_retirement():
|
||||
documents = _documents(HERMES / "oauth2-proxy.yaml")
|
||||
deployment = next(
|
||||
@ -272,3 +288,169 @@ def test_chat_oauth_allows_stale_service_worker_retirement():
|
||||
assert agent_template_volume["configMap"]["name"] == (
|
||||
"hermes-chat-oauth-templates"
|
||||
)
|
||||
|
||||
|
||||
def test_webui_recovers_auth_and_labels_session_scoped_controls():
|
||||
dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-webui").read_text()
|
||||
|
||||
assert "res.status===401||res.status===403" in dockerfile
|
||||
assert "window.location.assign('/oauth2/start?rd='" in dockerfile
|
||||
assert "childrenExpanded?'▾ ':'▸ '" in dockerfile
|
||||
assert "'atlas/auto/maximum': 'Automatic · Maximum'" in dockerfile
|
||||
assert "hermes-webui-telegram-project-patch.py" in dockerfile
|
||||
telegram_project_patch = (
|
||||
ROOT / "dockerfiles" / "hermes-webui-telegram-project-patch.py"
|
||||
).read_text()
|
||||
assert "TELEGRAM_PROJECT_NAME = 'Telegram'" in telegram_project_patch
|
||||
assert "session_key.startswith('telegram-topic-')" in telegram_project_patch
|
||||
|
||||
router = (ROOT / "dockerfiles" / "hermes-webui-router.js").read_text()
|
||||
assert "'atlas/auto/fast':'AUTO · Fast'" in router
|
||||
assert "'atlas/manual/codex/sol':'Codex · SOL'" in router
|
||||
assert "'atlas/manual/claude/opus':'Claude · Opus'" in router
|
||||
assert "watchModelOptions('modelSelect')" in router
|
||||
assert "watchModelOptions('settingsModel')" in router
|
||||
|
||||
|
||||
def test_chat_voice_uses_private_jetson_services_and_shared_auto_route():
|
||||
statefulset = _documents(HERMES / "chat-statefulset.yaml")[0]
|
||||
containers = statefulset["spec"]["template"]["spec"]["containers"]
|
||||
webui = next(item for item in containers if item["name"] == "webui")
|
||||
env = {item["name"]: item["value"] for item in webui["env"]}
|
||||
|
||||
assert env["HERMES_STT_URL"] == (
|
||||
"http://hermes-stt.hermes.svc.cluster.local:9000/v1/audio/transcriptions"
|
||||
)
|
||||
assert env["HERMES_WEBUI_ATLAS_TTS_URL"] == (
|
||||
"http://hermes-tts.hermes.svc.cluster.local:9001/v1/audio/speech"
|
||||
)
|
||||
assert "hermes_stt_client.py" in env["HERMES_LOCAL_STT_COMMAND"]
|
||||
|
||||
configmap = _documents(HERMES / "chat-configmap.yaml")[0]
|
||||
config = yaml.safe_load(configmap["data"]["config.yaml"])
|
||||
assert config["stt"] == {
|
||||
"enabled": True,
|
||||
"provider": "local_command",
|
||||
"local": {"model": "small", "language": "auto"},
|
||||
}
|
||||
|
||||
dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-webui").read_text()
|
||||
assert "hermes-webui-atlas-patch.py" in dockerfile
|
||||
assert "hermes-webui-atlas-voice.js" in dockerfile
|
||||
voice_script = (ROOT / "dockerfiles" / "hermes-webui-atlas-voice.js").read_text()
|
||||
assert "/api/transcribe/capability" in voice_script
|
||||
assert "/api/transcribe" in voice_script
|
||||
assert "/api/tts" in voice_script
|
||||
assert "speakResponse(generation)" in voice_script
|
||||
assert "window._splitForTTS(text,280)" in voice_script
|
||||
assert "pending=fetchSpeech(chunks[index+1])" in voice_script
|
||||
assert "restartSoon(token,450)" in voice_script
|
||||
assert "constraints.voiceIsolation=true" in voice_script
|
||||
assert "highpass.frequency.value=140" in voice_script
|
||||
assert "Math.max(0.04,noiseFloor*2.4+0.006)" in voice_script
|
||||
assert "while(preRoll.length>3) preRoll.shift()" in voice_script
|
||||
|
||||
stt_server = (ROOT / "dockerfiles" / "hermes-jetson-stt-server.py").read_text()
|
||||
assert "def _repetitive_token" in stt_server
|
||||
assert "compression_ratio_threshold=2.0" in stt_server
|
||||
assert "no_speech_threshold=0.5" in stt_server
|
||||
|
||||
|
||||
def test_voice_transcript_filter_removes_fan_hallucinations(monkeypatch):
|
||||
server_path = ROOT / "dockerfiles" / "hermes-jetson-stt-server.py"
|
||||
spec = importlib.util.spec_from_file_location("hermes_jetson_stt_server", server_path)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
monkeypatch.setitem(sys.modules, "cgi", SimpleNamespace())
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"torch",
|
||||
SimpleNamespace(cuda=SimpleNamespace(is_available=lambda: False)),
|
||||
)
|
||||
monkeypatch.setitem(sys.modules, "whisper", SimpleNamespace())
|
||||
spec.loader.exec_module(module)
|
||||
|
||||
result = {
|
||||
"segments": [
|
||||
{
|
||||
"text": " ththththththththth Testing.",
|
||||
"no_speech_prob": 0.12,
|
||||
"avg_logprob": -0.2,
|
||||
},
|
||||
{
|
||||
"text": " background hum",
|
||||
"no_speech_prob": 0.82,
|
||||
"avg_logprob": -0.9,
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
assert module._clean_transcript(result) == "Testing."
|
||||
|
||||
|
||||
def test_voice_models_are_baked_and_runtime_has_no_public_egress():
|
||||
stt_dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-jetson-stt").read_text()
|
||||
tts_dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-jetson-tts").read_text()
|
||||
assert "ADD --checksum=sha256:aff26ae4" in stt_dockerfile
|
||||
assert "ADD --checksum=sha256:9ecf7799" in stt_dockerfile
|
||||
assert "--chmod=0444" in stt_dockerfile
|
||||
assert "chmod 0555 /opt/models /opt/models/whisper" in stt_dockerfile
|
||||
assert "HERMES_STT_CACHE=/opt/models/whisper" in stt_dockerfile
|
||||
assert "ADD --checksum=sha256:4cabf7c3" in tts_dockerfile
|
||||
assert "ADD --checksum=sha256:db42b97d" in tts_dockerfile
|
||||
assert tts_dockerfile.count("--chmod=0444") == 6
|
||||
assert "chmod 0555 /opt/models /opt/models/piper" in tts_dockerfile
|
||||
assert "HERMES_TTS_CACHE=/opt/models/piper" in tts_dockerfile
|
||||
tts_server = (ROOT / "dockerfiles" / "hermes-jetson-tts-server.py").read_text()
|
||||
assert "download_voice" not in tts_server
|
||||
assert "baked Piper voice is missing" in tts_server
|
||||
assert "session_options.intra_op_num_threads = ONNX_THREADS" in tts_server
|
||||
|
||||
policies = _documents(HERMES / "networkpolicy.yaml")
|
||||
voice_policy = next(
|
||||
item for item in policies if item["metadata"]["name"] == "hermes-private-voice"
|
||||
)
|
||||
assert voice_policy["spec"]["policyTypes"] == ["Ingress", "Egress"]
|
||||
assert not any(
|
||||
"ipBlock" in destination
|
||||
for rule in voice_policy["spec"]["egress"]
|
||||
for destination in rule.get("to", [])
|
||||
)
|
||||
|
||||
|
||||
def test_voice_workloads_have_deliberate_xavier_placement():
|
||||
documents = _documents(HERMES / "voice-deployment.yaml")
|
||||
deployments = {
|
||||
item["metadata"]["name"]: item
|
||||
for item in documents
|
||||
if item["kind"] == "Deployment"
|
||||
}
|
||||
stt = deployments["hermes-stt"]["spec"]["template"]["spec"]
|
||||
tts = deployments["hermes-tts"]["spec"]["template"]["spec"]
|
||||
|
||||
assert "@sha256:" in stt["containers"][0]["image"]
|
||||
assert "@sha256:" in tts["containers"][0]["image"]
|
||||
assert stt["nodeSelector"] == {"kubernetes.io/hostname": "titan-21"}
|
||||
assert tts["nodeSelector"] == {"kubernetes.io/hostname": "titan-21"}
|
||||
assert stt["automountServiceAccountToken"] is False
|
||||
assert tts["automountServiceAccountToken"] is False
|
||||
assert stt["enableServiceLinks"] is False
|
||||
assert tts["enableServiceLinks"] is False
|
||||
assert stt["runtimeClassName"] == "nvidia"
|
||||
assert stt["securityContext"]["supplementalGroups"] == [44]
|
||||
stt_resources = stt["containers"][0]["resources"]
|
||||
stt_env = {
|
||||
item["name"]: item["value"] for item in stt["containers"][0]["env"]
|
||||
}
|
||||
assert stt_env["NVIDIA_DRIVER_CAPABILITIES"] == "compute,utility"
|
||||
assert stt_resources["requests"]["nvidia.com/gpu.shared"] == 1
|
||||
assert stt_resources["limits"]["nvidia.com/gpu.shared"] == 1
|
||||
assert "nvidia.com/gpu.shared" not in tts["containers"][0]["resources"]["requests"]
|
||||
tts_env = {
|
||||
item["name"]: item["value"] for item in tts["containers"][0]["env"]
|
||||
}
|
||||
assert tts_env["HERMES_TTS_VOICE"] == "en_US-lessac-medium"
|
||||
assert tts_env["HERMES_TTS_ONNX_THREADS"] == "2"
|
||||
assert tts["containers"][0]["resources"]["limits"]["cpu"] == "4"
|
||||
assert all("hostPath" not in volume for volume in stt["volumes"])
|
||||
assert all("hostPath" not in volume for volume in tts["volumes"])
|
||||
@ -1,24 +0,0 @@
|
||||
"""Hermes chat sandbox contracts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from test_hermes_chat_support import (
|
||||
Path,
|
||||
ROOT,
|
||||
importlib,
|
||||
)
|
||||
|
||||
|
||||
def test_sandbox_executes_python_with_bounded_output(tmp_path: Path, monkeypatch):
|
||||
source = ROOT / "dockerfiles" / "hermes-chat-sandbox-server.py"
|
||||
spec = importlib.util.spec_from_file_location("hermes_chat_sandbox_server", source)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
monkeypatch.setattr(module, "WORKSPACE", tmp_path)
|
||||
|
||||
result = module._execute("import math\nprint(math.comb(10, 3))")
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["stdout"] == "120\n"
|
||||
assert result["stderr"] == ""
|
||||
@ -1,392 +0,0 @@
|
||||
"""Hermes chat sessions contracts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from test_hermes_chat_support import (
|
||||
HERMES,
|
||||
Path,
|
||||
importlib,
|
||||
json,
|
||||
pytest,
|
||||
sqlite3,
|
||||
)
|
||||
|
||||
|
||||
def test_telegram_api_session_migration_is_bounded_and_idempotent(tmp_path: Path):
|
||||
"""Only named Telegram conversations receive presentation metadata."""
|
||||
module_path = HERMES / "scripts" / "migrate_telegram_api_sessions.py"
|
||||
spec = importlib.util.spec_from_file_location("migrate_telegram_sessions", module_path)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
state = tmp_path / "state.db"
|
||||
responses = tmp_path / "response_store.db"
|
||||
with sqlite3.connect(state) as connection:
|
||||
connection.execute(
|
||||
"""CREATE TABLE sessions (
|
||||
id TEXT PRIMARY KEY, source TEXT, session_key TEXT, chat_type TEXT,
|
||||
display_name TEXT, origin_json TEXT, title TEXT
|
||||
)"""
|
||||
)
|
||||
connection.executemany(
|
||||
"INSERT INTO sessions (id, source) VALUES (?, ?)",
|
||||
(("telegram-session", "api_server"), ("other-session", "api_server")),
|
||||
)
|
||||
with sqlite3.connect(responses) as connection:
|
||||
connection.execute(
|
||||
"CREATE TABLE conversations (name TEXT PRIMARY KEY, response_id TEXT NOT NULL)"
|
||||
)
|
||||
connection.execute(
|
||||
"CREATE TABLE responses (response_id TEXT PRIMARY KEY, data TEXT NOT NULL, accessed_at REAL NOT NULL)"
|
||||
)
|
||||
connection.executemany(
|
||||
"INSERT INTO responses VALUES (?, ?, 0)",
|
||||
(
|
||||
("telegram-response", json.dumps({"session_id": "telegram-session"})),
|
||||
("other-response", json.dumps({"session_id": "other-session"})),
|
||||
),
|
||||
)
|
||||
connection.executemany(
|
||||
"INSERT INTO conversations VALUES (?, ?)",
|
||||
(("telegram", "telegram-response"), ("unrelated", "other-response")),
|
||||
)
|
||||
|
||||
assert module.migrate(state, responses) == 1
|
||||
assert module.migrate(state, responses) == 0
|
||||
with sqlite3.connect(state) as connection:
|
||||
telegram = connection.execute(
|
||||
"SELECT session_key, chat_type, display_name, origin_json, title "
|
||||
"FROM sessions WHERE id = 'telegram-session'"
|
||||
).fetchone()
|
||||
other = connection.execute(
|
||||
"SELECT session_key, title FROM sessions WHERE id = 'other-session'"
|
||||
).fetchone()
|
||||
assert telegram[:3] == ("telegram", "private", "Telegram")
|
||||
assert json.loads(telegram[3]) == {
|
||||
"platform": "telegram",
|
||||
"session_key": "telegram",
|
||||
}
|
||||
assert telegram[4] == "Telegram · General"
|
||||
assert other == (None, None)
|
||||
|
||||
def test_web_session_activity_patch_projects_bounded_events(tmp_path: Path):
|
||||
"""The DOM transcript includes events without polluting agent history."""
|
||||
module_path = HERMES / "scripts" / "patch_web_session_activity.py"
|
||||
spec = importlib.util.spec_from_file_location("patch_web_activity", module_path)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
source = tmp_path / "web_server.py"
|
||||
destination = tmp_path / "patched.py"
|
||||
source.write_text(
|
||||
"prefix\n"
|
||||
+ module.LATEST_ROWS_BEFORE
|
||||
+ module.LATEST_SELECTION_BEFORE
|
||||
+ module.HELPER_MARKER
|
||||
+ " db = object()\n"
|
||||
+ module.MESSAGES_BEFORE
|
||||
+ "suffix\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
module.patch(source, destination)
|
||||
patched = destination.read_text(encoding="utf-8")
|
||||
|
||||
assert "def _run_activity_messages(" in patched
|
||||
assert 'root = get_hermes_home() / "run-activity"' in patched
|
||||
assert "path.stat().st_size > 600_000" in patched
|
||||
assert "entries[-1_000:]" in patched
|
||||
assert "*_run_activity_messages(sid)" in patched
|
||||
assert "messages.sort(" in patched
|
||||
assert "limit: Optional[int] = None" in patched
|
||||
assert "total_messages = len(messages)" in patched
|
||||
assert "min(int(limit), 10_000)" in patched
|
||||
assert '"total_messages": total_messages' in patched
|
||||
assert "SELECT id, parent_session_id, started_at, ended_at" in patched
|
||||
assert "newest still-open member of the lineage" in patched
|
||||
assert "immediate objective parent" in patched
|
||||
assert "mixes unrelated workstreams" in patched
|
||||
assert "orphaned children" in patched
|
||||
assert 'item[0].get("ended_at") is None' in patched
|
||||
|
||||
def test_web_session_lineage_returns_to_resumed_parent():
|
||||
"""An ended reviewer must not strand the live view away from its parent."""
|
||||
module_path = HERMES / "scripts" / "patch_web_session_activity.py"
|
||||
spec = importlib.util.spec_from_file_location("patch_web_lineage", module_path)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
|
||||
namespace: dict[str, object] = {}
|
||||
exec(
|
||||
"def select_active(sid, db, rows):\n" + module.LATEST_SELECTION_AFTER,
|
||||
namespace,
|
||||
)
|
||||
|
||||
class FakeDB:
|
||||
def get_session(self, session_id):
|
||||
return next((row for row in rows if row["id"] == session_id), None)
|
||||
|
||||
rows = [
|
||||
{
|
||||
"id": "umbrella",
|
||||
"parent_session_id": None,
|
||||
"started_at": 0.0,
|
||||
"ended_at": None,
|
||||
},
|
||||
{
|
||||
"id": "root",
|
||||
"parent_session_id": "umbrella",
|
||||
"started_at": 1.0,
|
||||
"ended_at": None,
|
||||
},
|
||||
{
|
||||
"id": "review-1",
|
||||
"parent_session_id": "root",
|
||||
"started_at": 2.0,
|
||||
"ended_at": 3.0,
|
||||
},
|
||||
{
|
||||
"id": "review-2",
|
||||
"parent_session_id": "root",
|
||||
"started_at": 4.0,
|
||||
"ended_at": None,
|
||||
},
|
||||
]
|
||||
select_active = namespace["select_active"]
|
||||
|
||||
assert select_active("root", FakeDB(), rows) == (
|
||||
"review-2",
|
||||
["root", "review-2"],
|
||||
)
|
||||
assert select_active("review-1", FakeDB(), rows) == (
|
||||
"review-2",
|
||||
["root", "review-2"],
|
||||
)
|
||||
rows[2]["ended_at"] = 5.0
|
||||
rows[3]["ended_at"] = 5.0
|
||||
assert select_active("root", FakeDB(), rows) == ("root", ["root"])
|
||||
|
||||
rows[1]["ended_at"] = 9.0
|
||||
rows.append(
|
||||
{
|
||||
"id": "orphaned-review",
|
||||
"parent_session_id": "root",
|
||||
"started_at": 6.0,
|
||||
"ended_at": None,
|
||||
}
|
||||
)
|
||||
rows.append(
|
||||
{
|
||||
"id": "accepted-review",
|
||||
"parent_session_id": "root",
|
||||
"started_at": 7.0,
|
||||
"ended_at": 8.0,
|
||||
}
|
||||
)
|
||||
assert select_active("review-1", FakeDB(), rows) == (
|
||||
"accepted-review",
|
||||
["root", "accepted-review"],
|
||||
)
|
||||
|
||||
def test_api_activity_patch_coalesces_streaming_heartbeats(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
"""Token callbacks stay bounded while every tool transition is retained."""
|
||||
module_path = HERMES / "scripts" / "patch_api_server_sessions.py"
|
||||
spec = importlib.util.spec_from_file_location("patch_api_activity", module_path)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
activity_body = module.EVENT_CALLBACK_SIGNATURE_AFTER.split(
|
||||
" def _make_run_event_callback(", 1
|
||||
)[0]
|
||||
namespace: dict[str, object] = {}
|
||||
exec(
|
||||
"import hashlib, json, logging, os, time\n"
|
||||
"from pathlib import Path\n"
|
||||
"logger = logging.getLogger(__name__)\n"
|
||||
"def redact_sensitive_text(value): return value\n"
|
||||
"class ActivityRecorder:\n"
|
||||
+ activity_body,
|
||||
namespace,
|
||||
)
|
||||
recorder = namespace["ActivityRecorder"]()
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
|
||||
recorder._record_run_activity("session", "_thinking")
|
||||
recorder._record_run_activity("session", "reasoning.available")
|
||||
recorder._record_run_activity("session", "subagent.thinking")
|
||||
recorder._record_run_activity("session", "tool.started", tool_name="terminal")
|
||||
recorder._record_run_activity("session", "tool.completed", tool_name="terminal")
|
||||
|
||||
journals = list((tmp_path / "run-activity").glob("*.jsonl"))
|
||||
assert len(journals) == 1
|
||||
entries = [json.loads(line) for line in journals[0].read_text().splitlines()]
|
||||
assert [entry["activity_event"] for entry in entries] == [
|
||||
"_thinking",
|
||||
"tool.started",
|
||||
"tool.completed",
|
||||
]
|
||||
|
||||
def test_legacy_api_sessions_are_nested_idempotently(tmp_path: Path):
|
||||
"""Known standalone API workers move under Cassandra without data loss."""
|
||||
module_path = HERMES / "scripts" / "migrate_api_session_lineage.py"
|
||||
spec = importlib.util.spec_from_file_location("migrate_api_sessions", module_path)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
database = tmp_path / "state.db"
|
||||
with sqlite3.connect(database) as connection:
|
||||
connection.execute(
|
||||
"CREATE TABLE sessions (id TEXT PRIMARY KEY, source TEXT, "
|
||||
"parent_session_id TEXT, title TEXT, transcript TEXT, archived INTEGER DEFAULT 0)"
|
||||
)
|
||||
connection.execute(
|
||||
"INSERT INTO sessions (id, source, parent_session_id, title, transcript) "
|
||||
"VALUES (?, 'tui', NULL, 'Cassandra', 'parent-data')",
|
||||
(module.LEGACY_CASSANDRA_PARENT,),
|
||||
)
|
||||
worker_id = next(iter(module.LEGACY_CASSANDRA_WORKERS))
|
||||
connection.execute(
|
||||
"INSERT INTO sessions (id, source, parent_session_id, title, transcript) "
|
||||
"VALUES (?, 'api_server', NULL, 'old', 'keep-me')",
|
||||
(worker_id,),
|
||||
)
|
||||
connection.executemany(
|
||||
"INSERT INTO sessions (id, source, parent_session_id, title, transcript) "
|
||||
"VALUES (?, 'api_server', NULL, 'old smoke', 'keep-smoke')",
|
||||
((orphan_id,) for orphan_id in module.LEGACY_ORPHANED_SMOKE_SESSIONS),
|
||||
)
|
||||
|
||||
assert module.migrate(database) == 1 + len(module.LEGACY_ORPHANED_SMOKE_SESSIONS)
|
||||
assert module.migrate(database) == 0
|
||||
with sqlite3.connect(database) as connection:
|
||||
row = connection.execute(
|
||||
"SELECT parent_session_id, title, transcript FROM sessions WHERE id = ?",
|
||||
(worker_id,),
|
||||
).fetchone()
|
||||
assert row == (
|
||||
module.LEGACY_CASSANDRA_PARENT,
|
||||
module.LEGACY_CASSANDRA_WORKERS[worker_id],
|
||||
"keep-me",
|
||||
)
|
||||
with sqlite3.connect(database) as connection:
|
||||
orphans = connection.execute(
|
||||
"SELECT id, archived, title, transcript FROM sessions "
|
||||
"WHERE id IN ({}) ORDER BY id".format(
|
||||
",".join("?" for _ in module.LEGACY_ORPHANED_SMOKE_SESSIONS)
|
||||
),
|
||||
tuple(module.LEGACY_ORPHANED_SMOKE_SESSIONS),
|
||||
).fetchall()
|
||||
assert orphans == sorted(
|
||||
(
|
||||
orphan_id,
|
||||
1,
|
||||
module.LEGACY_ORPHANED_SMOKE_SESSIONS[orphan_id],
|
||||
"keep-smoke",
|
||||
)
|
||||
for orphan_id in module.LEGACY_ORPHANED_SMOKE_SESSIONS
|
||||
)
|
||||
|
||||
def test_automated_triage_sessions_are_grouped_without_touching_interactive_runs(tmp_path: Path):
|
||||
"""Only the stable Ariadne contract moves below the triage parent."""
|
||||
module_path = HERMES / "scripts" / "migrate_api_session_lineage.py"
|
||||
spec = importlib.util.spec_from_file_location("migrate_triage_sessions", module_path)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
database = tmp_path / "state.db"
|
||||
with sqlite3.connect(database) as connection:
|
||||
connection.execute(
|
||||
"CREATE TABLE sessions (id TEXT PRIMARY KEY, source TEXT, "
|
||||
"parent_session_id TEXT, title TEXT, started_at REAL, archived INTEGER DEFAULT 0)"
|
||||
)
|
||||
connection.execute(
|
||||
"CREATE TABLE messages (session_id TEXT, role TEXT, content TEXT)"
|
||||
)
|
||||
connection.executemany(
|
||||
"INSERT INTO sessions (id, source, started_at) VALUES (?, 'api_server', ?)",
|
||||
(("triage-run", 1.0), ("jenkins-run", 1.5), ("interactive-run", 2.0)),
|
||||
)
|
||||
connection.executemany(
|
||||
"INSERT INTO messages (session_id, role, content) VALUES (?, 'user', ?)",
|
||||
(
|
||||
(
|
||||
"triage-run",
|
||||
module.TRIAGE_MESSAGE_PREFIXES[0]
|
||||
+ " Fix for incident sonar/bstein_home/python:S2208/finding-key.",
|
||||
),
|
||||
(
|
||||
"jenkins-run",
|
||||
module.TRIAGE_MESSAGE_PREFIXES[1]
|
||||
+ "\nAnalyze incident soteria/291 for the Jenkins job soteria.",
|
||||
),
|
||||
("interactive-run", "Please explain this alert to me."),
|
||||
),
|
||||
)
|
||||
|
||||
assert module.migrate(database, group_triage=True) == 2
|
||||
assert module.migrate(database, group_triage=True) == 0
|
||||
with sqlite3.connect(database) as connection:
|
||||
parent = connection.execute(
|
||||
"SELECT title FROM sessions WHERE id = ?", (module.TRIAGE_PARENT,)
|
||||
).fetchone()
|
||||
triage = connection.execute(
|
||||
"SELECT parent_session_id, title FROM sessions WHERE id = 'triage-run'"
|
||||
).fetchone()
|
||||
interactive = connection.execute(
|
||||
"SELECT parent_session_id FROM sessions WHERE id = 'interactive-run'"
|
||||
).fetchone()
|
||||
jenkins = connection.execute(
|
||||
"SELECT parent_session_id, title FROM sessions WHERE id = 'jenkins-run'"
|
||||
).fetchone()
|
||||
assert parent == (module.TRIAGE_PARENT_TITLE,)
|
||||
assert triage == (
|
||||
module.TRIAGE_PARENT,
|
||||
"Sonar · bstein_home · python:S2208 · iage-run",
|
||||
)
|
||||
assert interactive == (None,)
|
||||
assert jenkins == (
|
||||
module.TRIAGE_PARENT,
|
||||
"Sonar · soteria/291 · kins-run",
|
||||
)
|
||||
|
||||
def test_stale_parent_linked_api_workers_close_on_startup(tmp_path: Path):
|
||||
"""A previous gateway lifetime cannot leave phantom active workers."""
|
||||
module_path = HERMES / "scripts" / "migrate_api_session_lineage.py"
|
||||
spec = importlib.util.spec_from_file_location("close_stale_api_workers", module_path)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
database = tmp_path / "state.db"
|
||||
with sqlite3.connect(database) as connection:
|
||||
connection.execute(
|
||||
"CREATE TABLE sessions (id TEXT PRIMARY KEY, source TEXT, "
|
||||
"parent_session_id TEXT, title TEXT, started_at REAL, ended_at REAL, "
|
||||
"end_reason TEXT, archived INTEGER DEFAULT 0)"
|
||||
)
|
||||
connection.executemany(
|
||||
"INSERT INTO sessions "
|
||||
"(id, source, parent_session_id, started_at, ended_at, end_reason) "
|
||||
"VALUES (?, ?, ?, 1, ?, ?)",
|
||||
(
|
||||
("stale", "api_server", "parent", None, None),
|
||||
("root", "api_server", None, None, None),
|
||||
("finished", "api_server", "parent", 2.0, "api_run_completed"),
|
||||
("interactive", "tui", "parent", None, None),
|
||||
),
|
||||
)
|
||||
|
||||
assert module.migrate(database) == 1
|
||||
assert module.migrate(database) == 0
|
||||
with sqlite3.connect(database) as connection:
|
||||
rows = connection.execute(
|
||||
"SELECT id, ended_at, end_reason FROM sessions ORDER BY id"
|
||||
).fetchall()
|
||||
by_id = {row[0]: row[1:] for row in rows}
|
||||
assert by_id["stale"][0] is not None
|
||||
assert by_id["stale"][1] == "api_run_recovered_stale"
|
||||
assert by_id["root"] == (None, None)
|
||||
assert by_id["finished"] == (2.0, "api_run_completed")
|
||||
assert by_id["interactive"] == (None, None)
|
||||
@ -2,16 +2,10 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import importlib.util
|
||||
import json
|
||||
import sqlite3
|
||||
import sys
|
||||
import time
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
from types import ModuleType, SimpleNamespace
|
||||
import pytest
|
||||
from types import SimpleNamespace
|
||||
|
||||
import yaml
|
||||
|
||||
@ -19,9 +13,11 @@ ROOT = Path(__file__).parents[2]
|
||||
HERMES = ROOT / "services" / "hermes"
|
||||
VAULT = ROOT / "services" / "vault"
|
||||
|
||||
|
||||
def _documents(path: Path) -> list[dict]:
|
||||
return [doc for doc in yaml.safe_load_all(path.read_text()) if doc]
|
||||
|
||||
|
||||
def _load_broker_module(name: str, filename: str, monkeypatch):
|
||||
"""Load one broker with its mounted routing-catalog dependency."""
|
||||
catalog_path = HERMES / "scripts" / "routing_catalog.py"
|
||||
@ -40,22 +36,3 @@ def _load_broker_module(name: str, filename: str, monkeypatch):
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
__all__ = (
|
||||
"HERMES",
|
||||
"ModuleType",
|
||||
"Path",
|
||||
"ROOT",
|
||||
"SimpleNamespace",
|
||||
"VAULT",
|
||||
"_documents",
|
||||
"_load_broker_module",
|
||||
"base64",
|
||||
"importlib",
|
||||
"json",
|
||||
"pytest",
|
||||
"sqlite3",
|
||||
"sys",
|
||||
"time",
|
||||
"tomllib",
|
||||
"yaml",
|
||||
)
|
||||
|
||||
@ -1,483 +0,0 @@
|
||||
"""Hermes chat switchyard contracts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from test_hermes_chat_support import (
|
||||
HERMES,
|
||||
Path,
|
||||
SimpleNamespace,
|
||||
_documents,
|
||||
_load_broker_module,
|
||||
base64,
|
||||
importlib,
|
||||
json,
|
||||
pytest,
|
||||
sys,
|
||||
time,
|
||||
tomllib,
|
||||
)
|
||||
|
||||
|
||||
def test_codex_broker_auth_and_request_contract(tmp_path: Path, monkeypatch):
|
||||
"""The relay is bounded, stateless, and rejects unapproved models."""
|
||||
module = _load_broker_module(
|
||||
"hermes_codex_broker", "codex_broker.py", monkeypatch
|
||||
)
|
||||
monkeypatch.setattr(module, "TOKEN", "relay-secret")
|
||||
|
||||
assert module._authorized("Bearer relay-secret") is True
|
||||
assert module._authorized("Bearer wrong") is False
|
||||
assert module._real_model("route/codex/gpt-5.6-sol/xhigh") == "gpt-5.6-sol"
|
||||
payload = module._validate_payload(
|
||||
{
|
||||
"model": "gpt-5.6-terra",
|
||||
"input": "route this chat turn",
|
||||
"store": True,
|
||||
"stream": False,
|
||||
"max_output_tokens": 96,
|
||||
"max_completion_tokens": 96,
|
||||
"max_tokens": 96,
|
||||
"temperature": 0.7,
|
||||
"top_p": 0.9,
|
||||
}
|
||||
)
|
||||
assert payload["store"] is False
|
||||
assert payload["stream"] is True
|
||||
assert "max_output_tokens" not in payload
|
||||
assert "max_completion_tokens" not in payload
|
||||
assert "max_tokens" not in payload
|
||||
assert "temperature" not in payload
|
||||
assert "top_p" not in payload
|
||||
assert payload["input"] == [
|
||||
{
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [{"type": "input_text", "text": "route this chat turn"}],
|
||||
}
|
||||
]
|
||||
response_item = {
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [{"type": "input_text", "text": "keep this item"}],
|
||||
}
|
||||
assert module._validate_payload(
|
||||
{"model": "gpt-5.6-terra", "input": response_item}
|
||||
)["input"] == [response_item]
|
||||
response_items = [response_item]
|
||||
assert module._validate_payload(
|
||||
{"model": "gpt-5.6-terra", "input": response_items}
|
||||
)["input"] is response_items
|
||||
image_items = [
|
||||
{
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "input_text", "text": "What color is this?"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": "data:image/png;base64,cHJpdmF0ZQ==",
|
||||
"detail": "high",
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
assert module._validate_payload(
|
||||
{"model": "gpt-5.6-terra", "input": image_items}
|
||||
)["input"][0]["content"][1] == {
|
||||
"type": "input_image",
|
||||
"image_url": "data:image/png;base64,cHJpdmF0ZQ==",
|
||||
"detail": "high",
|
||||
}
|
||||
switchyard_image_items = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "input_text", "text": "What color is this?"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": "data:image/png;base64,cHJpdmF0ZQ==",
|
||||
"detail": "auto",
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
assert module._validate_payload(
|
||||
{"model": "gpt-5.6-terra", "input": switchyard_image_items}
|
||||
)["input"][0]["content"][1] == {
|
||||
"type": "input_image",
|
||||
"image_url": "data:image/png;base64,cHJpdmF0ZQ==",
|
||||
"detail": "auto",
|
||||
}
|
||||
switchyard_base64_items = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "input_text", "text": "What color is this?"},
|
||||
{
|
||||
"type": "image",
|
||||
"source": {
|
||||
"type": "base64",
|
||||
"media_type": "image/png",
|
||||
"data": "cHJpdmF0ZQ==",
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
normalized_base64 = module._validate_payload(
|
||||
{"model": "gpt-5.6-terra", "input": switchyard_base64_items}
|
||||
)["input"][0]["content"][1]
|
||||
assert normalized_base64 == {
|
||||
"type": "input_image",
|
||||
"image_url": "data:image/png;base64,cHJpdmF0ZQ==",
|
||||
}
|
||||
switchyard_enum_items = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "input_image",
|
||||
"image_url": {
|
||||
"type": "url",
|
||||
"data": {
|
||||
"url": "data:image/png;base64,cHJpdmF0ZQ==",
|
||||
"detail": "high",
|
||||
},
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
nested_image = module._validate_payload(
|
||||
{"model": "gpt-5.6-terra", "input": switchyard_enum_items}
|
||||
)["input"][0]["content"][0]
|
||||
assert nested_image["image_url"] == "data:image/png;base64,cHJpdmF0ZQ=="
|
||||
assert nested_image["detail"] == "high"
|
||||
with pytest.raises(ValueError, match=r"non-empty Responses image URL.*str\[4\]"):
|
||||
module._validate_payload(
|
||||
{
|
||||
"model": "gpt-5.6-terra",
|
||||
"input": [
|
||||
{
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "input_image",
|
||||
"image_url": {"detail": "high"},
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
routed = module._validate_payload(
|
||||
{
|
||||
"model": "route/codex/gpt-5.6-luna/low",
|
||||
"input": "use the low route",
|
||||
"stream": False,
|
||||
}
|
||||
)
|
||||
assert routed["model"] == "gpt-5.6-luna"
|
||||
with pytest.raises(ValueError, match="unsupported Codex model"):
|
||||
module._validate_payload({"model": "unapproved-model", "input": "hello"})
|
||||
with pytest.raises(ValueError, match="non-empty Responses input"):
|
||||
module._validate_payload({"model": "gpt-5.6-terra", "input": ""})
|
||||
with pytest.raises(ValueError, match="non-empty Responses input list"):
|
||||
module._validate_payload({"model": "gpt-5.6-terra", "input": []})
|
||||
|
||||
completed = {
|
||||
"id": "resp_test",
|
||||
"object": "response",
|
||||
"status": "completed",
|
||||
"output": [],
|
||||
}
|
||||
completed_item = {
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"content": [{"type": "output_text", "text": "done"}],
|
||||
}
|
||||
assert module._completed_response(
|
||||
[
|
||||
"event: response.created",
|
||||
'data: {"type":"response.created","response":{}}',
|
||||
"event: response.output_item.done",
|
||||
"data: "
|
||||
+ json.dumps(
|
||||
{
|
||||
"type": "response.output_item.done",
|
||||
"output_index": 0,
|
||||
"item": completed_item,
|
||||
}
|
||||
),
|
||||
"event: response.completed",
|
||||
"data: "
|
||||
+ json.dumps({"type": "response.completed", "response": completed}),
|
||||
"data: [DONE]",
|
||||
]
|
||||
)["output"] == [completed_item]
|
||||
raw_stream = (
|
||||
"event: response.output_item.done\n"
|
||||
"data: "
|
||||
+ json.dumps(
|
||||
{
|
||||
"type": "response.output_item.done",
|
||||
"output_index": 0,
|
||||
"item": completed_item,
|
||||
}
|
||||
)
|
||||
+ "\n\nevent: response.completed\ndata: "
|
||||
+ json.dumps({"type": "response.completed", "response": completed})
|
||||
+ "\n\n"
|
||||
).encode()
|
||||
normalized = module._normalized_stream(
|
||||
raw_stream, {**completed, "output": [completed_item]}
|
||||
).decode()
|
||||
terminal_data = next(
|
||||
line for line in normalized.splitlines() if '"response.completed"' in line
|
||||
)
|
||||
assert json.loads(terminal_data.removeprefix("data: "))["response"][
|
||||
"output"
|
||||
] == [completed_item]
|
||||
assert normalized.endswith("\n\n")
|
||||
streamed_function_item = {
|
||||
"type": "function_call",
|
||||
"name": "read_file",
|
||||
"status": "completed",
|
||||
"arguments": '{"path":"/tmp"}',
|
||||
}
|
||||
streamed_function_body = (
|
||||
"event: response.function_call_arguments.delta\n"
|
||||
'data: {"type":"response.function_call_arguments.delta",'
|
||||
'"item_id":"call_1","delta":"{\\"path\\":\\"/tmp\\"}"}\n\n'
|
||||
"event: response.function_call_arguments.done\n"
|
||||
'data: {"type":"response.function_call_arguments.done",'
|
||||
'"item_id":"call_1","arguments":"{\\"path\\":\\"/tmp\\"}"}\n\n'
|
||||
"event: response.output_item.done\n"
|
||||
'data: {"type":"response.output_item.done","output_index":0,'
|
||||
'"item":{"type":"function_call","name":"read_file",'
|
||||
'"arguments":"{\\"path\\":\\"/tmp\\"}"}}\n\n'
|
||||
"event: response.completed\n"
|
||||
"data: "
|
||||
+ json.dumps(
|
||||
{
|
||||
"type": "response.completed",
|
||||
"response": {**completed, "output": [streamed_function_item]},
|
||||
}
|
||||
)
|
||||
+ "\n\n"
|
||||
).encode()
|
||||
normalized_function_stream = module._normalized_stream(
|
||||
streamed_function_body, {**completed, "output": [streamed_function_item]}
|
||||
).decode()
|
||||
assert "response.function_call_arguments.delta" in normalized_function_stream
|
||||
assert "response.function_call_arguments.done" not in normalized_function_stream
|
||||
assert "response.output_item.done" not in normalized_function_stream
|
||||
normalized_terminal = next(
|
||||
line
|
||||
for line in normalized_function_stream.splitlines()
|
||||
if '"response.completed"' in line
|
||||
)
|
||||
assert json.loads(normalized_terminal.removeprefix("data: "))["response"][
|
||||
"output"
|
||||
] == []
|
||||
with pytest.raises(RuntimeError, match="retryable incomplete response"):
|
||||
module._completed_response(
|
||||
[
|
||||
"event: response.incomplete",
|
||||
'data: {"type":"response.incomplete","response":'
|
||||
'{"status":"incomplete","incomplete_details":'
|
||||
'{"reason":"max_output_tokens"}}}',
|
||||
]
|
||||
)
|
||||
with pytest.raises(RuntimeError, match="provider unavailable"):
|
||||
module._completed_response(
|
||||
[
|
||||
"event: error",
|
||||
'data: {"type":"error","error":{"message":"provider unavailable"}}',
|
||||
]
|
||||
)
|
||||
malformed_tool_item = {
|
||||
"type": "function_call",
|
||||
"name": "search_files",
|
||||
"status": "completed",
|
||||
"arguments": '{"path":"","offset":',
|
||||
}
|
||||
with pytest.raises(RuntimeError, match="malformed function arguments"):
|
||||
module._completed_response(
|
||||
[
|
||||
"event: response.output_item.done",
|
||||
"data: "
|
||||
+ json.dumps(
|
||||
{
|
||||
"type": "response.output_item.done",
|
||||
"output_index": 0,
|
||||
"item": malformed_tool_item,
|
||||
}
|
||||
),
|
||||
"event: response.completed",
|
||||
"data: "
|
||||
+ json.dumps({"type": "response.completed", "response": completed}),
|
||||
]
|
||||
)
|
||||
valid_tool_item = {
|
||||
**malformed_tool_item,
|
||||
"arguments": '{"path":"","offset":0}',
|
||||
}
|
||||
assert module._completed_response(
|
||||
[
|
||||
"event: response.output_item.done",
|
||||
"data: "
|
||||
+ json.dumps(
|
||||
{
|
||||
"type": "response.output_item.done",
|
||||
"output_index": 0,
|
||||
"item": valid_tool_item,
|
||||
}
|
||||
),
|
||||
"event: response.completed",
|
||||
"data: "
|
||||
+ json.dumps({"type": "response.completed", "response": completed}),
|
||||
]
|
||||
)["output"] == [valid_tool_item]
|
||||
with pytest.raises(RuntimeError, match="malformed function arguments"):
|
||||
module._completed_response(
|
||||
[
|
||||
"event: response.function_call_arguments.delta",
|
||||
'data: {"type":"response.function_call_arguments.delta",'
|
||||
'"item_id":"call_1","output_index":0,'
|
||||
'"delta":"{\\"path\\":\\"/tmp\\",\\"offset\\":"}',
|
||||
"event: response.completed",
|
||||
"data: "
|
||||
+ json.dumps({"type": "response.completed", "response": completed}),
|
||||
]
|
||||
)
|
||||
streamed_tool = module._completed_response(
|
||||
[
|
||||
"event: response.function_call_arguments.delta",
|
||||
'data: {"type":"response.function_call_arguments.delta",'
|
||||
'"item_id":"call_2","output_index":0,'
|
||||
'"delta":"{\\"path\\":\\"/tmp\\",\\"offset\\":"}',
|
||||
"event: response.function_call_arguments.done",
|
||||
'data: {"type":"response.function_call_arguments.done",'
|
||||
'"item_id":"call_2","output_index":0,'
|
||||
'"arguments":"{\\"path\\":\\"/tmp\\",\\"offset\\":0}"}',
|
||||
"event: response.completed",
|
||||
"data: "
|
||||
+ json.dumps({"type": "response.completed", "response": completed}),
|
||||
]
|
||||
)
|
||||
assert streamed_tool["status"] == "completed"
|
||||
|
||||
auth_dir = tmp_path / ".codex"
|
||||
auth_dir.mkdir()
|
||||
# The token payload need only prove the broker reads CODEX_HOME directly.
|
||||
encoded = base64.urlsafe_b64encode(
|
||||
json.dumps({"exp": time.time() + 3600}).encode()
|
||||
).decode().rstrip("=")
|
||||
(auth_dir / "auth.json").write_text(
|
||||
json.dumps({"tokens": {"access_token": f"header.{encoded}.signature"}})
|
||||
)
|
||||
monkeypatch.setenv("CODEX_HOME", str(auth_dir))
|
||||
assert module._access_token().startswith("header.")
|
||||
|
||||
def test_classifier_broker_bounds_history_without_losing_routing_intent(monkeypatch):
|
||||
"""AUTO classification must fit the Jetson context without losing intent."""
|
||||
broker_path = HERMES / "scripts" / "classifier_broker.py"
|
||||
monkeypatch.setitem(sys.modules, "httpx", SimpleNamespace())
|
||||
spec = importlib.util.spec_from_file_location("hermes_classifier_broker", broker_path)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
|
||||
payload = {
|
||||
"model": "qwen2.5:14b-instruct-q4_0",
|
||||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "routing contract\n" + ("candidate policy " * 1000),
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Build and verify the Cassandra release safely.",
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_large",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "large_tool",
|
||||
"arguments": '{"command":"' + ("x" * 5000) + '"}',
|
||||
},
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"content": "unbounded test output " * 10000,
|
||||
"tool_call_id": "call_large",
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "image_url", "image_url": {"url": "data:image/png;base64,private"}},
|
||||
{"type": "text", "text": "Turn this cat into a cute clown."},
|
||||
],
|
||||
},
|
||||
],
|
||||
"tools": [{"type": "function", "function": {"name": "large_tool"}}],
|
||||
"tool_choice": "auto",
|
||||
"parallel_tool_calls": True,
|
||||
"response_format": {"type": "json_object"},
|
||||
}
|
||||
|
||||
compacted = module.compact_payload(payload)
|
||||
encoded = json.dumps(compacted)
|
||||
|
||||
assert compacted["model"] == payload["model"]
|
||||
assert compacted["response_format"] == payload["response_format"]
|
||||
assert "tools" not in compacted
|
||||
assert "tool_choice" not in compacted
|
||||
assert "parallel_tool_calls" not in compacted
|
||||
assert all(message.get("role") != "tool" for message in compacted["messages"])
|
||||
assert all("tool_call_id" not in message for message in compacted["messages"])
|
||||
assert all("tool_calls" not in message for message in compacted["messages"])
|
||||
assert "[tool evidence]" in encoded
|
||||
assert "[assistant requested an external tool]" in encoded
|
||||
assert "Build and verify the Cassandra release safely." in encoded
|
||||
assert "Turn this cat into a cute clown." in encoded
|
||||
assert "image attachment available to the selected worker" in encoded
|
||||
assert "data:image/png;base64" not in encoded
|
||||
assert "unbounded test output " * 100 not in encoded
|
||||
assert len(encoded) < 18_000
|
||||
|
||||
def test_switchyard_classifier_is_bounded_and_fails_open_once():
|
||||
"""A sick local judge must not hold chat through repeated long retries."""
|
||||
config = tomllib.loads(
|
||||
_documents(HERMES / "switchyard-configmap.yaml")[0]["data"]["routes.toml"]
|
||||
)
|
||||
classifier = config["llm_clients"]["classifier"]
|
||||
assert classifier["base_url"] == "http://127.0.0.1:9008/v1"
|
||||
assert classifier["max_retries"] == 0
|
||||
for route in ("auto_fast", "auto_balanced"):
|
||||
assert config["routes"][route]["recent_turn_window"] == 4
|
||||
for route in ("auto_deep", "auto_maximum", "worker_auto_maximum"):
|
||||
assert config["routes"][route]["recent_turn_window"] == 6
|
||||
|
||||
deployment = _documents(HERMES / "switchyard-deployment.yaml")[0]
|
||||
containers = {
|
||||
item["name"]: item
|
||||
for item in deployment["spec"]["template"]["spec"]["containers"]
|
||||
}
|
||||
classifier_container = containers["classifier-broker"]
|
||||
env = {item["name"]: item["value"] for item in classifier_container["env"]}
|
||||
assert env["HERMES_CLASSIFIER_BROKER_READ_TIMEOUT"] == "60"
|
||||
assert classifier_container["readinessProbe"]["httpGet"]["port"] == "classifier"
|
||||
@ -1,268 +0,0 @@
|
||||
"""Hermes chat voice contracts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from test_hermes_chat_support import (
|
||||
HERMES,
|
||||
ROOT,
|
||||
SimpleNamespace,
|
||||
VAULT,
|
||||
_documents,
|
||||
importlib,
|
||||
sys,
|
||||
yaml,
|
||||
)
|
||||
|
||||
|
||||
def test_webui_recovers_auth_and_labels_session_scoped_controls():
|
||||
dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-webui").read_text()
|
||||
|
||||
assert "res.status===401||res.status===403" in dockerfile
|
||||
assert "window.location.assign('/oauth2/start?rd='" in dockerfile
|
||||
assert "childrenExpanded?'▾ ':'▸ '" in dockerfile
|
||||
assert "'atlas/auto/maximum': 'Automatic · Maximum'" in dockerfile
|
||||
assert "hermes-webui-telegram-project-patch.py" in dockerfile
|
||||
telegram_project_patch = (
|
||||
ROOT / "dockerfiles" / "hermes-webui-telegram-project-patch.py"
|
||||
).read_text()
|
||||
assert "TELEGRAM_PROJECT_NAME = 'Telegram'" in telegram_project_patch
|
||||
assert "session_key.startswith('telegram-topic-')" in telegram_project_patch
|
||||
|
||||
router = (ROOT / "dockerfiles" / "hermes-webui-router.js").read_text()
|
||||
assert "'atlas/auto/fast':'AUTO · Fast'" in router
|
||||
assert "'atlas/manual/codex/sol':'Codex · SOL'" in router
|
||||
assert "'atlas/manual/claude/opus':'Claude · Opus'" in router
|
||||
assert "watchModelOptions('modelSelect')" in router
|
||||
assert "watchModelOptions('settingsModel')" in router
|
||||
|
||||
def test_chat_voice_uses_private_jetson_services_and_shared_auto_route():
|
||||
statefulset = _documents(HERMES / "chat-statefulset.yaml")[0]
|
||||
containers = statefulset["spec"]["template"]["spec"]["containers"]
|
||||
webui = next(item for item in containers if item["name"] == "webui")
|
||||
env = {item["name"]: item["value"] for item in webui["env"]}
|
||||
|
||||
assert env["HERMES_STT_URL"] == (
|
||||
"http://hermes-stt.hermes.svc.cluster.local:9000/v1/audio/transcriptions"
|
||||
)
|
||||
assert env["HERMES_WEBUI_ATLAS_TTS_URL"] == (
|
||||
"http://hermes-tts.hermes.svc.cluster.local:9001/v1/audio/speech"
|
||||
)
|
||||
assert "hermes_stt_client.py" in env["HERMES_LOCAL_STT_COMMAND"]
|
||||
|
||||
configmap = _documents(HERMES / "chat-configmap.yaml")[0]
|
||||
config = yaml.safe_load(configmap["data"]["config.yaml"])
|
||||
assert config["stt"] == {
|
||||
"enabled": True,
|
||||
"provider": "local_command",
|
||||
"local": {"model": "small", "language": "auto"},
|
||||
}
|
||||
|
||||
dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-webui").read_text()
|
||||
assert "hermes-webui-atlas-patch.py" in dockerfile
|
||||
assert "hermes-webui-atlas-voice.js" in dockerfile
|
||||
voice_script = (ROOT / "dockerfiles" / "hermes-webui-atlas-voice.js").read_text()
|
||||
assert "/api/transcribe/capability" in voice_script
|
||||
assert "/api/transcribe" in voice_script
|
||||
assert "/api/tts" in voice_script
|
||||
assert "speakResponse(generation)" in voice_script
|
||||
assert "window._splitForTTS(text,280)" in voice_script
|
||||
assert "pending=fetchSpeech(chunks[index+1])" in voice_script
|
||||
assert "restartSoon(token,450)" in voice_script
|
||||
assert "constraints.voiceIsolation=true" in voice_script
|
||||
assert "highpass.frequency.value=140" in voice_script
|
||||
assert "Math.max(0.04,noiseFloor*2.4+0.006)" in voice_script
|
||||
assert "while(preRoll.length>3) preRoll.shift()" in voice_script
|
||||
|
||||
stt_server = (ROOT / "dockerfiles" / "hermes-jetson-stt-server.py").read_text()
|
||||
assert "def _repetitive_token" in stt_server
|
||||
assert "compression_ratio_threshold=2.0" in stt_server
|
||||
assert "no_speech_threshold=0.5" in stt_server
|
||||
|
||||
def test_voice_transcript_filter_removes_fan_hallucinations(monkeypatch):
|
||||
server_path = ROOT / "dockerfiles" / "hermes-jetson-stt-server.py"
|
||||
spec = importlib.util.spec_from_file_location("hermes_jetson_stt_server", server_path)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
monkeypatch.setitem(sys.modules, "cgi", SimpleNamespace())
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"torch",
|
||||
SimpleNamespace(cuda=SimpleNamespace(is_available=lambda: False)),
|
||||
)
|
||||
monkeypatch.setitem(sys.modules, "whisper", SimpleNamespace())
|
||||
spec.loader.exec_module(module)
|
||||
|
||||
result = {
|
||||
"segments": [
|
||||
{
|
||||
"text": " ththththththththth Testing.",
|
||||
"no_speech_prob": 0.12,
|
||||
"avg_logprob": -0.2,
|
||||
},
|
||||
{
|
||||
"text": " background hum",
|
||||
"no_speech_prob": 0.82,
|
||||
"avg_logprob": -0.9,
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
assert module._clean_transcript(result) == "Testing."
|
||||
|
||||
def test_voice_models_are_baked_and_runtime_has_no_public_egress():
|
||||
stt_dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-jetson-stt").read_text()
|
||||
tts_dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-jetson-tts").read_text()
|
||||
assert "ADD --checksum=sha256:aff26ae4" in stt_dockerfile
|
||||
assert "ADD --checksum=sha256:9ecf7799" in stt_dockerfile
|
||||
assert "--chmod=0444" in stt_dockerfile
|
||||
assert "chmod 0555 /opt/models /opt/models/whisper" in stt_dockerfile
|
||||
assert "HERMES_STT_CACHE=/opt/models/whisper" in stt_dockerfile
|
||||
assert "ADD --checksum=sha256:4cabf7c3" in tts_dockerfile
|
||||
assert "ADD --checksum=sha256:db42b97d" in tts_dockerfile
|
||||
assert tts_dockerfile.count("--chmod=0444") == 6
|
||||
assert "chmod 0555 /opt/models /opt/models/piper" in tts_dockerfile
|
||||
assert "HERMES_TTS_CACHE=/opt/models/piper" in tts_dockerfile
|
||||
tts_server = (ROOT / "dockerfiles" / "hermes-jetson-tts-server.py").read_text()
|
||||
assert "download_voice" not in tts_server
|
||||
assert "baked Piper voice is missing" in tts_server
|
||||
assert "session_options.intra_op_num_threads = ONNX_THREADS" in tts_server
|
||||
|
||||
policies = _documents(HERMES / "networkpolicy.yaml")
|
||||
voice_policy = next(
|
||||
item for item in policies if item["metadata"]["name"] == "hermes-private-voice"
|
||||
)
|
||||
assert voice_policy["spec"]["policyTypes"] == ["Ingress", "Egress"]
|
||||
assert not any(
|
||||
"ipBlock" in destination
|
||||
for rule in voice_policy["spec"]["egress"]
|
||||
for destination in rule.get("to", [])
|
||||
)
|
||||
|
||||
def test_voice_workloads_have_deliberate_xavier_placement():
|
||||
documents = _documents(HERMES / "voice-deployment.yaml")
|
||||
deployments = {
|
||||
item["metadata"]["name"]: item
|
||||
for item in documents
|
||||
if item["kind"] == "Deployment"
|
||||
}
|
||||
stt = deployments["hermes-stt"]["spec"]["template"]["spec"]
|
||||
tts = deployments["hermes-tts"]["spec"]["template"]["spec"]
|
||||
|
||||
assert "@sha256:" in stt["containers"][0]["image"]
|
||||
assert "@sha256:" in tts["containers"][0]["image"]
|
||||
assert stt["nodeSelector"] == {"kubernetes.io/hostname": "titan-21"}
|
||||
assert tts["nodeSelector"] == {"kubernetes.io/hostname": "titan-21"}
|
||||
assert stt["automountServiceAccountToken"] is False
|
||||
assert tts["automountServiceAccountToken"] is False
|
||||
assert stt["enableServiceLinks"] is False
|
||||
assert tts["enableServiceLinks"] is False
|
||||
assert stt["runtimeClassName"] == "nvidia"
|
||||
assert stt["securityContext"]["supplementalGroups"] == [44]
|
||||
stt_resources = stt["containers"][0]["resources"]
|
||||
stt_env = {
|
||||
item["name"]: item["value"] for item in stt["containers"][0]["env"]
|
||||
}
|
||||
assert stt_env["NVIDIA_DRIVER_CAPABILITIES"] == "compute,utility"
|
||||
assert stt_resources["requests"]["nvidia.com/gpu.shared"] == 1
|
||||
assert stt_resources["limits"]["nvidia.com/gpu.shared"] == 1
|
||||
assert "nvidia.com/gpu.shared" not in tts["containers"][0]["resources"]["requests"]
|
||||
tts_env = {
|
||||
item["name"]: item["value"] for item in tts["containers"][0]["env"]
|
||||
}
|
||||
assert tts_env["HERMES_TTS_VOICE"] == "en_US-lessac-medium"
|
||||
assert tts_env["HERMES_TTS_ONNX_THREADS"] == "2"
|
||||
assert tts["containers"][0]["resources"]["limits"]["cpu"] == "4"
|
||||
assert all("hostPath" not in volume for volume in stt["volumes"])
|
||||
assert all("hostPath" not in volume for volume in tts["volumes"])
|
||||
|
||||
def test_chat_image_generation_uses_private_owner_broker():
|
||||
"""Family pods get image bytes without receiving the owner's OAuth file."""
|
||||
configmap = _documents(HERMES / "chat-configmap.yaml")[0]
|
||||
assert "shared desktop/Wolf lane" in configmap["data"]["SOUL.md"]
|
||||
assert "local FLUX waits" in configmap["data"]["SOUL.md"]
|
||||
assert "Use `image_generate_local`" in configmap["data"]["SOUL.md"]
|
||||
assert "Use `image_generate_hosted`" in configmap["data"]["SOUL.md"]
|
||||
assert "ComfyUI endpoint" in configmap["data"]["SOUL.md"]
|
||||
assert "`MEDIA:` path" in configmap["data"]["SOUL.md"]
|
||||
assert "`image_edit_latest`" in configmap["data"]["SOUL.md"]
|
||||
assert "Preserve the most recently selected image lane" in configmap["data"]["SOUL.md"]
|
||||
config = yaml.safe_load(configmap["data"]["config.yaml"])
|
||||
assert config["image_gen"] == {
|
||||
"provider": "atlas-broker",
|
||||
"model": "atlas-image-auto-high",
|
||||
}
|
||||
assert config["plugins"]["enabled"] == ["atlas-broker", "auto-router"]
|
||||
|
||||
statefulset = _documents(HERMES / "chat-statefulset.yaml")[0]
|
||||
pod = statefulset["spec"]["template"]["spec"]
|
||||
hermes = next(item for item in pod["containers"] if item["name"] == "hermes")
|
||||
mounts = {item["name"]: item for item in hermes["volumeMounts"]}
|
||||
assert mounts["image-plugin"]["mountPath"] == (
|
||||
"/opt/hermes/plugins/image_gen/atlas-broker"
|
||||
)
|
||||
assert mounts["runtime-access"]["mountPath"] == "/runtime-access"
|
||||
assert "provider-auth" not in mounts
|
||||
assert not any(mount["name"] == "home" and "agent" in str(mount) for mount in hermes["volumeMounts"])
|
||||
env = {item["name"]: item["value"] for item in hermes["env"]}
|
||||
assert env["HERMES_IMAGE_BROKER_URL"].startswith("http://hermes-image-broker.")
|
||||
|
||||
plugin = (HERMES / "plugins" / "image-gen-broker" / "__init__.py").read_text()
|
||||
assert '"local": "flux-2-klein-4b-local"' in plugin
|
||||
assert '"hosted": "gpt-image-2-high"' in plugin
|
||||
assert 'name="image_generate_local"' in plugin
|
||||
assert 'name="image_generate_hosted"' in plugin
|
||||
assert '"name": "image_edit_latest"' in plugin
|
||||
assert '"name": "image_edit_latest_local"' in plugin
|
||||
assert '"name": "image_edit_latest_hosted"' in plugin
|
||||
assert "def _latest_generated_image" in plugin
|
||||
assert "newest MEDIA: path from the conversation" in plugin
|
||||
assert 'candidate.upper().startswith("MEDIA:")' in plugin
|
||||
assert "override=True" not in plugin
|
||||
|
||||
agent = _documents(HERMES / "agent-deployment.yaml")[0]
|
||||
containers = agent["spec"]["template"]["spec"]["containers"]
|
||||
broker = next(item for item in containers if item["name"] == "image-broker")
|
||||
assert broker["ports"] == [
|
||||
{"name": "image-broker", "containerPort": 9002, "protocol": "TCP"}
|
||||
]
|
||||
assert broker["securityContext"]["readOnlyRootFilesystem"] is True
|
||||
assert broker["securityContext"]["runAsNonRoot"] is True
|
||||
|
||||
services = _documents(HERMES / "service.yaml")
|
||||
service = next(
|
||||
item for item in services if item["metadata"]["name"] == "hermes-image-broker"
|
||||
)
|
||||
assert service["spec"]["selector"] == {"app": "hermes-agent"}
|
||||
|
||||
oauth_store = _documents(HERMES / "oauth-session-store.yaml")
|
||||
redis = next(item for item in oauth_store if item["kind"] == "Deployment")
|
||||
assert redis["spec"]["strategy"]["type"] == "Recreate"
|
||||
assert "--appendonly" in redis["spec"]["template"]["spec"]["containers"][0]["args"]
|
||||
|
||||
policies = _documents(HERMES / "networkpolicy.yaml")
|
||||
agent_policy = next(
|
||||
item for item in policies if item["metadata"]["name"] == "hermes-agent-isolation"
|
||||
)
|
||||
broker_ingress = next(
|
||||
rule
|
||||
for rule in agent_policy["spec"]["ingress"]
|
||||
if {port["port"] for port in rule["ports"]} == {9002, 9003}
|
||||
)
|
||||
assert broker_ingress["from"][0]["podSelector"]["matchLabels"] == {
|
||||
"app": "hermes-chat-tenant"
|
||||
}
|
||||
vault_policy = (VAULT / "scripts" / "vault_k8s_auth_configure.sh").read_text()
|
||||
assert (
|
||||
'"hermes/agent-oidc hermes/agent-tokens hermes/chat-telegram '
|
||||
'hermes/developer-keycloak hermes/developer-harbor hermes/developer-jenkins '
|
||||
'hermes/developer-ssh"'
|
||||
in vault_policy
|
||||
)
|
||||
assert (
|
||||
'write_policy_and_role "hermes-scm-broker" "hermes-scm" '
|
||||
'"hermes-scm-broker"' in vault_policy
|
||||
)
|
||||
assert (
|
||||
'write_policy_and_role "hermes-node-ssh" "hermes" '
|
||||
'"hermes-node-ssh-access"' in vault_policy
|
||||
)
|
||||
@ -3,14 +3,16 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import runpy
|
||||
import sys
|
||||
from contextlib import nullcontext
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from testing.tests.test_hermes_cli_support import _completed_result, lanes
|
||||
from testing.tests.test_hermes_cli_support import SCRIPTS, _completed_result, lanes
|
||||
|
||||
|
||||
def _old_complete(
|
||||
@ -97,6 +99,18 @@ def test_capability_detection_requires_explicit_safety_keywords():
|
||||
assert missing_completion.deferred_features == ("exact-run-completion",)
|
||||
|
||||
|
||||
def test_deferred_features_name_every_absent_capability():
|
||||
"""A fully legacy image defers all three safety features by exact name."""
|
||||
fully_legacy = lanes.KanbanCapabilities(False, False, False)
|
||||
|
||||
assert fully_legacy.ready is False
|
||||
assert fully_legacy.deferred_features == (
|
||||
"exact-run-completion",
|
||||
"ended-run-replay",
|
||||
"exact-run-reclaim",
|
||||
)
|
||||
|
||||
|
||||
def test_startup_health_moves_from_bounded_deferred_to_ready(
|
||||
tmp_path: Path,
|
||||
capsys,
|
||||
@ -109,6 +123,7 @@ def test_startup_health_moves_from_bounded_deferred_to_ready(
|
||||
|
||||
assert old.ready is False
|
||||
assert old_document["state"] == "deferred"
|
||||
assert old_document["schema_version"] == 1
|
||||
assert old_document["ready"] is False
|
||||
assert old_document["active_run_completion_safe"] is True
|
||||
assert old_document["deferred_features"] == [
|
||||
@ -127,6 +142,107 @@ def test_startup_health_moves_from_bounded_deferred_to_ready(
|
||||
assert lanes.runtime_health(health)["state"] == "ready"
|
||||
|
||||
|
||||
def test_readiness_fails_closed_without_disclosing_health_values(
|
||||
tmp_path: Path,
|
||||
capsys,
|
||||
monkeypatch,
|
||||
):
|
||||
"""Readiness distinguishes startup, malformed, deferred, stale, and ready."""
|
||||
health = tmp_path / "runtime-health.json"
|
||||
now = datetime(2026, 8, 17, 12, 0, tzinfo=timezone.utc)
|
||||
assert lanes.readiness_issue(health, now=now) == "runtime health is missing"
|
||||
|
||||
health.write_text("not-json", encoding="utf-8")
|
||||
assert lanes.readiness_issue(health, now=now) == "runtime health is malformed"
|
||||
|
||||
deferred = lanes._health_document(lanes.KanbanCapabilities(True, False, False))
|
||||
deferred["observed_at"] = now.isoformat()
|
||||
health.write_text(json.dumps(deferred), encoding="utf-8")
|
||||
assert lanes.readiness_issue(health, now=now) == "runtime compatibility is deferred"
|
||||
|
||||
ready = lanes._health_document(lanes.KanbanCapabilities(True, True, True))
|
||||
ready["observed_at"] = (now - timedelta(seconds=61)).isoformat()
|
||||
health.write_text(json.dumps(ready), encoding="utf-8")
|
||||
assert lanes.readiness_issue(health, now=now) == "runtime health is stale"
|
||||
|
||||
ready["observed_at"] = now.isoformat()
|
||||
health.write_text(json.dumps(ready), encoding="utf-8")
|
||||
assert lanes.readiness_issue(health, now=now) is None
|
||||
|
||||
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path)
|
||||
monkeypatch.setenv("HERMES_CLI_HEALTH_MAX_AGE_SECONDS", "31536000")
|
||||
assert sys.modules["cli_lane_capabilities"].main() == 0
|
||||
assert capsys.readouterr().out == ""
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"mutation",
|
||||
[
|
||||
{"schema_version": 2},
|
||||
{"state": "ready", "ready": False},
|
||||
{"state": "ready", "capabilities": []},
|
||||
{"observed_at": 5},
|
||||
{"observed_at": "not-a-date"},
|
||||
{"observed_at": "2026-08-17T12:00:00"},
|
||||
],
|
||||
)
|
||||
def test_readiness_rejects_inconsistent_documents(
|
||||
tmp_path: Path,
|
||||
mutation: dict,
|
||||
):
|
||||
"""Inconsistent ready documents remain not-ready."""
|
||||
now = datetime(2026, 8, 17, 12, 0, tzinfo=timezone.utc)
|
||||
document = lanes._health_document(lanes.KanbanCapabilities(True, True, True))
|
||||
document["observed_at"] = now.isoformat()
|
||||
document.update(mutation)
|
||||
health = tmp_path / "runtime-health.json"
|
||||
health.write_text(json.dumps(document), encoding="utf-8")
|
||||
|
||||
assert lanes.readiness_issue(health, now=now) == "runtime health is malformed"
|
||||
|
||||
|
||||
def test_readiness_command_fails_with_only_a_bounded_reason(
|
||||
tmp_path: Path,
|
||||
capsys,
|
||||
monkeypatch,
|
||||
):
|
||||
"""The probe command exits nonzero and discloses nothing but the reason."""
|
||||
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path)
|
||||
|
||||
assert sys.modules["cli_lane_capabilities"].main() == 1
|
||||
|
||||
captured = capsys.readouterr()
|
||||
assert captured.out == ""
|
||||
assert captured.err.strip() == "runtime health is missing"
|
||||
|
||||
|
||||
def test_capabilities_module_is_the_deployed_probe_entrypoint(
|
||||
tmp_path: Path,
|
||||
monkeypatch,
|
||||
):
|
||||
"""`python cli_lane_capabilities.py` runs the readiness probe directly."""
|
||||
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path)
|
||||
|
||||
with pytest.raises(SystemExit) as excinfo:
|
||||
runpy.run_path(
|
||||
str(SCRIPTS / "cli_lane_capabilities.py"),
|
||||
run_name="__main__",
|
||||
)
|
||||
|
||||
assert excinfo.value.code == 1
|
||||
|
||||
|
||||
def test_readiness_rejects_future_health(tmp_path: Path):
|
||||
"""A clock-skewed future observation cannot make a stale process ready."""
|
||||
now = datetime(2026, 8, 17, 12, 0, tzinfo=timezone.utc)
|
||||
document = lanes._health_document(lanes.KanbanCapabilities(True, True, True))
|
||||
document["observed_at"] = (now + timedelta(seconds=6)).isoformat()
|
||||
health = tmp_path / "runtime-health.json"
|
||||
health.write_text(json.dumps(document), encoding="utf-8")
|
||||
|
||||
assert lanes.readiness_issue(health, now=now) == "runtime health is stale"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("status", "run_id", "expected"),
|
||||
[("running", 7, "committed"), ("blocked", None, "deferred")],
|
||||
|
||||
@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import runpy
|
||||
import sys
|
||||
from contextlib import nullcontext
|
||||
from pathlib import Path
|
||||
@ -9,12 +10,10 @@ from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from testing.tests.test_hermes_cli_support import lanes
|
||||
from testing.tests.test_hermes_cli_support import SCRIPTS, lanes
|
||||
|
||||
|
||||
def test_board_slug_and_connection_failure_paths(monkeypatch):
|
||||
assert lanes._owns_local_workspace(SimpleNamespace(workspace_path=" /workspace "))
|
||||
assert not lanes._owns_local_workspace(SimpleNamespace(workspace_path=""))
|
||||
assert lanes._board_slug(SimpleNamespace(slug="cassandra")) == "cassandra"
|
||||
assert lanes._board_slug(SimpleNamespace(id="fallback")) == "fallback"
|
||||
failures = []
|
||||
@ -102,41 +101,6 @@ def test_claim_ready_handles_zero_limit_empty_boards_and_claim_errors(monkeypatc
|
||||
assert lanes.claim_ready(set(), 1) == []
|
||||
|
||||
|
||||
def test_dispatch_skips_unhealthy_ineligible_and_unclaimed_entries(monkeypatch):
|
||||
class Connection:
|
||||
def __init__(self, board):
|
||||
self.board = board
|
||||
|
||||
def close(self):
|
||||
return None
|
||||
|
||||
tasks = [
|
||||
SimpleNamespace(id="t_ineligible", status="ready", assignee="cli-auto"),
|
||||
SimpleNamespace(id="t_unclaimed", status="ready", assignee="cli-auto"),
|
||||
]
|
||||
db = SimpleNamespace(
|
||||
list_boards=lambda **_kwargs: [
|
||||
{"slug": "offline"},
|
||||
{"slug": "cassandra"},
|
||||
],
|
||||
scoped_current_board=lambda _board: nullcontext(),
|
||||
connect=lambda board: (_ for _ in ()).throw(OSError("offline"))
|
||||
if board == "offline"
|
||||
else Connection(board),
|
||||
recompute_ready=lambda _conn: None,
|
||||
list_tasks=lambda _conn: tasks,
|
||||
claim_task=lambda *_args, **_kwargs: None,
|
||||
)
|
||||
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db))
|
||||
assert lanes.claim_ready(
|
||||
set(),
|
||||
2,
|
||||
eligible=lambda _board, task: task.id == "t_unclaimed",
|
||||
) == []
|
||||
monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0)
|
||||
lanes.recover_orphans()
|
||||
|
||||
|
||||
class _Future:
|
||||
def __init__(self):
|
||||
self.polls = 0
|
||||
@ -166,6 +130,28 @@ class _Pool:
|
||||
return self.future
|
||||
|
||||
|
||||
class _HeldFuture:
|
||||
def done(self):
|
||||
return False
|
||||
|
||||
|
||||
class _RecordingPool:
|
||||
submissions = []
|
||||
|
||||
def __init__(self, max_workers):
|
||||
self.max_workers = max_workers
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return False
|
||||
|
||||
def submit(self, _function, board, task_id):
|
||||
self.submissions.append((board, task_id))
|
||||
return _HeldFuture()
|
||||
|
||||
|
||||
def test_ready_dispatch_loop_submits_and_reaps_failed_workers(
|
||||
tmp_path: Path,
|
||||
monkeypatch,
|
||||
@ -175,11 +161,10 @@ def test_ready_dispatch_loop_submits_and_reaps_failed_workers(
|
||||
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db))
|
||||
monkeypatch.setattr(lanes, "RESULT_SCHEMA_PATH", tmp_path / "schema.json")
|
||||
monkeypatch.setattr(lanes, "recover_orphans", lambda: None)
|
||||
monkeypatch.setattr(
|
||||
lanes,
|
||||
"initialize_kanban_capabilities",
|
||||
lambda _db: lanes.KanbanCapabilities(True, True, True),
|
||||
)
|
||||
monkeypatch.setattr(lanes, "start_metrics_server", lambda port=None: None)
|
||||
ready = lanes.KanbanCapabilities(True, True, True)
|
||||
monkeypatch.setattr(lanes, "initialize_kanban_capabilities", lambda _db: ready)
|
||||
monkeypatch.setattr(lanes, "refresh_kanban_capabilities", lambda _db: ready)
|
||||
monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0)
|
||||
monkeypatch.setattr(lanes, "maybe_gc_lane_artifacts", lambda: 0)
|
||||
claims = [[("cassandra", "t_loop")], []]
|
||||
@ -212,11 +197,10 @@ def test_deferred_dispatch_health_never_claims_new_work(
|
||||
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db))
|
||||
monkeypatch.setattr(lanes, "RESULT_SCHEMA_PATH", tmp_path / "schema.json")
|
||||
monkeypatch.setattr(lanes, "recover_orphans", lambda: None)
|
||||
monkeypatch.setattr(
|
||||
lanes,
|
||||
"initialize_kanban_capabilities",
|
||||
lambda _db: lanes.KanbanCapabilities(True, False, False),
|
||||
)
|
||||
monkeypatch.setattr(lanes, "start_metrics_server", lambda port=None: None)
|
||||
deferred = lanes.KanbanCapabilities(True, False, False)
|
||||
monkeypatch.setattr(lanes, "initialize_kanban_capabilities", lambda _db: deferred)
|
||||
monkeypatch.setattr(lanes, "refresh_kanban_capabilities", lambda _db: deferred)
|
||||
monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0)
|
||||
monkeypatch.setattr(lanes, "maybe_gc_lane_artifacts", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
@ -232,3 +216,194 @@ def test_deferred_dispatch_health_never_claims_new_work(
|
||||
)
|
||||
with pytest.raises(RuntimeError, match="stop loop"):
|
||||
lanes.main()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("start_patched", [False, True])
|
||||
def test_two_loop_api_transition_refreshes_dispatch_and_health(
|
||||
tmp_path: Path,
|
||||
monkeypatch,
|
||||
start_patched: bool,
|
||||
):
|
||||
"""Each loop observes legacy/patched API transitions without reloading modules."""
|
||||
|
||||
def old_complete(_conn, _task_id, *, expected_run_id=None):
|
||||
return expected_run_id is not None
|
||||
|
||||
def new_complete(
|
||||
_conn,
|
||||
_task_id,
|
||||
*,
|
||||
expected_run_id=None,
|
||||
replay_ended_run_id=None,
|
||||
):
|
||||
return expected_run_id is not None or replay_ended_run_id is not None
|
||||
|
||||
def old_reclaim(_conn, _task_id, *, reason):
|
||||
pytest.fail(f"unguarded legacy reclaim invoked: {reason}")
|
||||
|
||||
def new_reclaim(_conn, _task_id, *, reason, expected_run_id=None):
|
||||
return bool(reason and expected_run_id)
|
||||
|
||||
db = SimpleNamespace(
|
||||
complete_task=new_complete if start_patched else old_complete,
|
||||
reclaim_task=new_reclaim if start_patched else old_reclaim,
|
||||
)
|
||||
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db))
|
||||
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
|
||||
monkeypatch.setattr(lanes, "RESULT_SCHEMA_PATH", tmp_path / "schema.json")
|
||||
recoveries = []
|
||||
monkeypatch.setattr(lanes, "recover_orphans", lambda: recoveries.append("recover"))
|
||||
monkeypatch.setattr(lanes, "start_metrics_server", lambda port=None: None)
|
||||
monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0)
|
||||
monkeypatch.setattr(lanes, "maybe_gc_lane_artifacts", lambda: 0)
|
||||
claim_states = []
|
||||
|
||||
def claim(_active, _limit):
|
||||
ready = lanes.kanban_capabilities(db).ready
|
||||
claim_states.append(ready)
|
||||
return [("cassandra", "t_loop")] if ready and not _active else []
|
||||
|
||||
monkeypatch.setattr(lanes, "claim_ready", claim)
|
||||
_RecordingPool.submissions = []
|
||||
monkeypatch.setattr(lanes.concurrent.futures, "ThreadPoolExecutor", _RecordingPool)
|
||||
journal = tmp_path / "cli-lanes/cassandra/t_loop.terminal.json"
|
||||
journal.parent.mkdir(parents=True)
|
||||
journal.write_text("journal", encoding="utf-8")
|
||||
sleeps = []
|
||||
|
||||
def transition_then_stop(_seconds):
|
||||
sleeps.append(True)
|
||||
if len(sleeps) == 1:
|
||||
db.complete_task = old_complete if start_patched else new_complete
|
||||
db.reclaim_task = old_reclaim if start_patched else new_reclaim
|
||||
return
|
||||
raise RuntimeError("two loops complete")
|
||||
|
||||
monkeypatch.setattr(lanes.time, "sleep", transition_then_stop)
|
||||
|
||||
with pytest.raises(RuntimeError, match="two loops complete"):
|
||||
lanes.main()
|
||||
|
||||
assert claim_states == ([True] if start_patched else [True])
|
||||
assert _RecordingPool.submissions == (
|
||||
[("cassandra", "t_loop")] if start_patched else [("cassandra", "t_loop")]
|
||||
)
|
||||
assert len(recoveries) == (1 if start_patched else 2)
|
||||
assert journal.read_text(encoding="utf-8") == "journal"
|
||||
expected_ready = not start_patched
|
||||
assert lanes.runtime_health()["ready"] is expected_ready
|
||||
|
||||
|
||||
def test_orphan_recovery_skips_unreachable_boards_and_settled_tasks(
|
||||
monkeypatch,
|
||||
capsys,
|
||||
):
|
||||
"""One faulted board or settled task never stops restart reclamation."""
|
||||
running = SimpleNamespace(
|
||||
id="t_run",
|
||||
status="running",
|
||||
current_run_id=4,
|
||||
assignee="cli-auto",
|
||||
)
|
||||
settled = SimpleNamespace(
|
||||
id="t_done",
|
||||
status="done",
|
||||
current_run_id=None,
|
||||
assignee="cli-auto",
|
||||
)
|
||||
reclaims = []
|
||||
|
||||
def connect(*, board):
|
||||
if board == "broken":
|
||||
raise OSError("volume stall")
|
||||
return SimpleNamespace(close=lambda: None)
|
||||
|
||||
db = SimpleNamespace(
|
||||
list_boards=lambda include_archived=False: [
|
||||
{"slug": "broken"},
|
||||
{"slug": "healthy"},
|
||||
],
|
||||
scoped_current_board=lambda _board: nullcontext(),
|
||||
connect=connect,
|
||||
list_tasks=lambda _conn: [settled, running],
|
||||
reclaim_task=lambda _conn, task_id, **kwargs: reclaims.append(
|
||||
(task_id, kwargs["expected_run_id"])
|
||||
),
|
||||
)
|
||||
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db))
|
||||
monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0)
|
||||
monkeypatch.setattr(lanes, "_has_pending_finalization", lambda *_args: False)
|
||||
|
||||
lanes.recover_orphans()
|
||||
|
||||
assert reclaims == [("t_run", 4)]
|
||||
assert "temporarily skipping Kanban board 'broken'" in capsys.readouterr().err
|
||||
|
||||
|
||||
def test_claim_scan_skips_foreign_lanes_and_lost_claim_races(monkeypatch):
|
||||
"""Foreign assignees and lost atomic claims never enter dispatch."""
|
||||
foreign = SimpleNamespace(id="t_manual", assignee="human", status="ready")
|
||||
lost = SimpleNamespace(id="t_lost", assignee="cli-auto", status="ready")
|
||||
won = SimpleNamespace(id="t_won", assignee="cli-auto", status="ready")
|
||||
|
||||
def claim_task(_conn, task_id, **_kwargs):
|
||||
return None if task_id == "t_lost" else SimpleNamespace(id=task_id)
|
||||
|
||||
db = SimpleNamespace(
|
||||
list_boards=lambda include_archived=False: [{"slug": "cassandra"}],
|
||||
scoped_current_board=lambda _board: nullcontext(),
|
||||
connect=lambda board: SimpleNamespace(close=lambda: None),
|
||||
recompute_ready=lambda _conn: None,
|
||||
list_tasks=lambda _conn: [foreign, lost, won],
|
||||
claim_task=claim_task,
|
||||
)
|
||||
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db))
|
||||
|
||||
assert lanes.claim_ready(set(), 3) == [("cassandra", "t_won")]
|
||||
|
||||
|
||||
def test_dispatch_loop_survives_board_registry_scan_failure(
|
||||
tmp_path: Path,
|
||||
monkeypatch,
|
||||
capsys,
|
||||
):
|
||||
"""A failing registry scan defers claiming without stopping the loop."""
|
||||
db = SimpleNamespace()
|
||||
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db))
|
||||
monkeypatch.setattr(lanes, "RESULT_SCHEMA_PATH", tmp_path / "schema.json")
|
||||
monkeypatch.setattr(lanes, "recover_orphans", lambda: None)
|
||||
monkeypatch.setattr(lanes, "start_metrics_server", lambda port=None: None)
|
||||
ready = lanes.KanbanCapabilities(True, True, True)
|
||||
monkeypatch.setattr(lanes, "initialize_kanban_capabilities", lambda _db: ready)
|
||||
monkeypatch.setattr(lanes, "refresh_kanban_capabilities", lambda _db: ready)
|
||||
monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0)
|
||||
monkeypatch.setattr(lanes, "maybe_gc_lane_artifacts", lambda: 0)
|
||||
|
||||
def failing_claim(_active, _limit):
|
||||
raise RuntimeError("registry scan failed")
|
||||
|
||||
monkeypatch.setattr(lanes, "claim_ready", failing_claim)
|
||||
monkeypatch.setattr(
|
||||
lanes.time,
|
||||
"sleep",
|
||||
lambda _seconds: (_ for _ in ()).throw(RuntimeError("stop loop")),
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="stop loop"):
|
||||
lanes.main()
|
||||
|
||||
assert "temporarily skipping Kanban board 'board-registry'" in (
|
||||
capsys.readouterr().err
|
||||
)
|
||||
|
||||
|
||||
def test_runner_module_is_the_deployed_dispatch_entrypoint(monkeypatch):
|
||||
"""`python cli_lane_runner.py` hands control to the dispatch loop."""
|
||||
calls = []
|
||||
monkeypatch.setattr(lanes, "main", lambda: calls.append(True) or 0)
|
||||
|
||||
with pytest.raises(SystemExit) as excinfo:
|
||||
runpy.run_path(str(SCRIPTS / "cli_lane_runner.py"), run_name="__main__")
|
||||
|
||||
assert excinfo.value.code == 0
|
||||
assert calls == [True]
|
||||
|
||||
@ -227,3 +227,38 @@ def test_retention_failure_paths_are_bounded(tmp_path: Path, monkeypatch, capsys
|
||||
)
|
||||
assert lanes.maybe_gc_lane_artifacts(now=1000) == 0
|
||||
assert "retention deferred" in capsys.readouterr().err
|
||||
|
||||
|
||||
def test_evidence_writer_rejects_a_tampered_temporary(tmp_path: Path, monkeypatch):
|
||||
"""A shared or replaced temporary inode never becomes durable evidence."""
|
||||
real_fstat = os.fstat
|
||||
calls = []
|
||||
|
||||
def hardlinked_fstat(descriptor):
|
||||
observed = real_fstat(descriptor)
|
||||
calls.append(observed)
|
||||
if len(calls) == 2:
|
||||
return os.stat_result((*observed[:3], 2, *observed[4:]))
|
||||
return observed
|
||||
|
||||
monkeypatch.setattr(lanes.os, "fstat", hardlinked_fstat)
|
||||
|
||||
with pytest.raises(OSError, match="not private and singly linked"):
|
||||
lanes._write_json_noreplace(tmp_path / "tampered.json", {"value": "x"})
|
||||
|
||||
assert list(tmp_path.iterdir()) == []
|
||||
|
||||
|
||||
def test_evidence_writer_releases_descriptor_when_inspection_fails(
|
||||
tmp_path: Path,
|
||||
monkeypatch,
|
||||
):
|
||||
"""An fstat fault closes the created descriptor without staging cleanup."""
|
||||
|
||||
def failing_fstat(_descriptor):
|
||||
raise OSError("inspection failed")
|
||||
|
||||
monkeypatch.setattr(lanes.os, "fstat", failing_fstat)
|
||||
|
||||
with pytest.raises(OSError, match="inspection failed"):
|
||||
lanes._write_json_noreplace(tmp_path / "orphaned.json", {"value": "x"})
|
||||
|
||||
@ -16,13 +16,6 @@ class _Connection:
|
||||
return None
|
||||
|
||||
|
||||
def test_goal_rejection_history_is_list_bounded():
|
||||
execution = __import__("cli_lane_execution")
|
||||
history = ["first"]
|
||||
assert execution._goal_rejections({"goal_rejections": history}) is history
|
||||
assert execution._goal_rejections({"goal_rejections": "invalid"}) == []
|
||||
|
||||
|
||||
def test_missing_claim_is_a_noop(tmp_path: Path, monkeypatch):
|
||||
db = SimpleNamespace(
|
||||
scoped_current_board=lambda _board: nullcontext(),
|
||||
@ -65,24 +58,18 @@ def test_transient_callback_storage_errors_do_not_kill_provider(
|
||||
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db))
|
||||
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes")
|
||||
monkeypatch.setattr(lanes, "KANBAN_STORAGE_ATTEMPTS", 1)
|
||||
monkeypatch.setattr(lanes, "fresh_unavailable_provider", lambda: "claude")
|
||||
monkeypatch.setattr(lanes, "fresh_unavailable_provider", lambda *_args, **_kwargs: "claude")
|
||||
route = lanes.Route("codex", "gpt", "high", "p", "c", "r", 1, ())
|
||||
monkeypatch.setattr(lanes, "select_route", lambda *_args, **_kwargs: route)
|
||||
|
||||
def provider(*args, **_kwargs):
|
||||
assert args[6]("still working") is True
|
||||
return lanes.ProcessResult(
|
||||
1,
|
||||
"failed",
|
||||
{"status": "incomplete", "blockers": ["synthetic provider failure"]},
|
||||
False,
|
||||
)
|
||||
return lanes.ProcessResult(1, "failed", None, False)
|
||||
|
||||
monkeypatch.setattr(lanes, "run_provider", provider)
|
||||
lanes.execute_claim("cassandra", "t_callbacks")
|
||||
|
||||
assert blocks and blocks[0]["kind"] == "capability"
|
||||
assert blocks[0]["reason"] == "synthetic provider failure"
|
||||
|
||||
|
||||
def test_restart_handoff_survives_missing_prior_log(tmp_path: Path, monkeypatch):
|
||||
@ -229,3 +216,104 @@ def test_unexpected_route_exception_blocks_the_exact_run(
|
||||
|
||||
assert blocks[0]["expected_run_id"] == 23
|
||||
assert "router down" in blocks[0]["reason"]
|
||||
|
||||
|
||||
def test_capacity_fallback_without_evidence_blocks_with_provider_output(
|
||||
tmp_path: Path,
|
||||
monkeypatch,
|
||||
):
|
||||
"""A fallback turn with no structured result blocks on raw provider output."""
|
||||
task = SimpleNamespace(
|
||||
id="t_nofallback",
|
||||
status="running",
|
||||
current_run_id=22,
|
||||
assignee="cli-auto",
|
||||
max_runtime_seconds=60,
|
||||
)
|
||||
blocks = []
|
||||
db = SimpleNamespace(
|
||||
scoped_current_board=lambda _board: nullcontext(),
|
||||
connect=lambda board: _Connection(),
|
||||
get_task=lambda *_args: task,
|
||||
worker_log_path=lambda *_args, **_kwargs: tmp_path / "worker.log",
|
||||
_resolve_worktree_workspace=lambda *_args, **_kwargs: (tmp_path, "branch"),
|
||||
set_branch_name=lambda *_args: None,
|
||||
set_workspace_path=lambda *_args: None,
|
||||
build_worker_context=lambda *_args: "attempt both providers",
|
||||
add_comment=lambda *_args, **_kwargs: None,
|
||||
heartbeat_worker=lambda *_args, **_kwargs: True,
|
||||
block_task=lambda *_args, **kwargs: blocks.append(kwargs),
|
||||
)
|
||||
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db))
|
||||
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes")
|
||||
monkeypatch.setattr(lanes, "fresh_unavailable_provider", lambda *_args, **_kwargs: None)
|
||||
codex = lanes.Route("codex", "gpt", "high", "p", "c", "r", 1, ())
|
||||
claude = lanes.Route("claude", "fable", "high", "p", "c", "r", 1, ())
|
||||
monkeypatch.setattr(
|
||||
lanes,
|
||||
"select_route",
|
||||
lambda _prompt, assignee, **_kwargs: claude
|
||||
if assignee == "cli-claude-high"
|
||||
else codex,
|
||||
)
|
||||
monkeypatch.setattr(lanes, "git_handoff", lambda *_args: "handoff")
|
||||
reports = [
|
||||
lanes.ProcessResult(1, "usage limit reached", None, True),
|
||||
lanes.ProcessResult(1, "fallback also failed", None, False),
|
||||
]
|
||||
monkeypatch.setattr(lanes, "run_provider", lambda *_args, **_kwargs: reports.pop(0))
|
||||
|
||||
lanes.execute_claim("cassandra", "t_nofallback")
|
||||
|
||||
assert reports == []
|
||||
assert blocks and "fallback also failed" in blocks[0]["reason"]
|
||||
assert blocks[0]["kind"] == "capability"
|
||||
|
||||
|
||||
def test_worker_blockers_become_the_block_reason_without_goal_mode(
|
||||
tmp_path: Path,
|
||||
monkeypatch,
|
||||
):
|
||||
"""A structured non-completed report blocks with its own stated blockers."""
|
||||
task = SimpleNamespace(
|
||||
id="t_blocked",
|
||||
status="running",
|
||||
current_run_id=23,
|
||||
assignee="cli-auto",
|
||||
max_runtime_seconds=60,
|
||||
)
|
||||
blocks = []
|
||||
db = SimpleNamespace(
|
||||
scoped_current_board=lambda _board: nullcontext(),
|
||||
connect=lambda board: _Connection(),
|
||||
get_task=lambda *_args: task,
|
||||
worker_log_path=lambda *_args, **_kwargs: tmp_path / "worker.log",
|
||||
_resolve_worktree_workspace=lambda *_args, **_kwargs: (tmp_path, "branch"),
|
||||
set_branch_name=lambda *_args: None,
|
||||
set_workspace_path=lambda *_args: None,
|
||||
build_worker_context=lambda *_args: "report the obstacle",
|
||||
add_comment=lambda *_args, **_kwargs: None,
|
||||
heartbeat_worker=lambda *_args, **_kwargs: True,
|
||||
block_task=lambda *_args, **kwargs: blocks.append(kwargs),
|
||||
)
|
||||
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db))
|
||||
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes")
|
||||
monkeypatch.setattr(lanes, "fresh_unavailable_provider", lambda *_args, **_kwargs: None)
|
||||
route = lanes.Route("codex", "gpt", "high", "p", "c", "r", 1, ())
|
||||
monkeypatch.setattr(lanes, "select_route", lambda *_args, **_kwargs: route)
|
||||
structured = {
|
||||
**_completed_result("progress only"),
|
||||
"status": "blocked",
|
||||
"blockers": ["credentials expired", "remote unreachable"],
|
||||
}
|
||||
monkeypatch.setattr(
|
||||
lanes,
|
||||
"run_provider",
|
||||
lambda *_args, **_kwargs: lanes.ProcessResult(0, "out", structured, False),
|
||||
)
|
||||
|
||||
lanes.execute_claim("cassandra", "t_blocked")
|
||||
|
||||
assert blocks and blocks[0]["reason"] == (
|
||||
"credentials expired; remote unreachable"
|
||||
)
|
||||
|
||||
@ -77,7 +77,7 @@ def test_goal_card_continues_after_local_judge_rejects_progress(
|
||||
return codex_xhigh
|
||||
|
||||
monkeypatch.setattr(lanes, "select_route", select_route)
|
||||
monkeypatch.setattr(lanes, "fresh_unavailable_provider", lambda: None)
|
||||
monkeypatch.setattr(lanes, "fresh_unavailable_provider", lambda *_args, **_kwargs: None)
|
||||
reports = [
|
||||
lanes.ProcessResult(1, "authentication expired", None, True),
|
||||
lanes.ProcessResult(
|
||||
|
||||
@ -159,14 +159,71 @@ def test_terminal_finalizer_closes_supplied_invalid_snapshots(
|
||||
|
||||
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes")
|
||||
path = lanes._terminal_path(lanes.state_path("board", "task"), 8)
|
||||
lanes.atomic_json(path, {})
|
||||
empty = _Snapshot(None)
|
||||
assert lanes._finalize_terminal_record(_db(None), path, snapshot=empty) == "invalid"
|
||||
assert empty.closed == 1
|
||||
|
||||
finalization = __import__("cli_lane_finalization")
|
||||
direct = _Snapshot(None)
|
||||
assert finalization._finalize_terminal_record(
|
||||
_db(None), path, snapshot=direct
|
||||
) == "invalid"
|
||||
assert direct.closed == 1
|
||||
|
||||
def test_finalizer_rejects_invalid_names_and_unpinnable_journals(
|
||||
tmp_path: Path,
|
||||
monkeypatch,
|
||||
):
|
||||
"""Invalid identities and unreadable journals return without side effects."""
|
||||
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
|
||||
(tmp_path / "cli-lanes/cassandra").mkdir(parents=True)
|
||||
db = _db(None)
|
||||
|
||||
foreign = tmp_path / "cli-lanes/cassandra/not-a-journal.json"
|
||||
assert lanes._finalize_terminal_record(db, foreign) == "invalid"
|
||||
|
||||
missing = lanes._terminal_path(
|
||||
lanes.state_path("cassandra", "t_edge"),
|
||||
5,
|
||||
"pending",
|
||||
)
|
||||
assert lanes._finalize_terminal_record(db, missing) == "invalid"
|
||||
|
||||
|
||||
class _NullSnapshot:
|
||||
def __init__(self, document):
|
||||
self.document = document
|
||||
self.closed = 0
|
||||
|
||||
def close(self):
|
||||
self.closed += 1
|
||||
|
||||
|
||||
def test_finalizer_closes_supplied_snapshots_on_every_invalid_path(
|
||||
tmp_path: Path,
|
||||
monkeypatch,
|
||||
):
|
||||
"""Caller-pinned snapshots are released for named and payload rejections."""
|
||||
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
|
||||
(tmp_path / "cli-lanes/cassandra").mkdir(parents=True)
|
||||
db = _db(None)
|
||||
pending = lanes._terminal_path(
|
||||
lanes.state_path("cassandra", "t_edge"),
|
||||
6,
|
||||
"pending",
|
||||
)
|
||||
|
||||
misnamed = _NullSnapshot({"kanban_state": "pending"})
|
||||
assert (
|
||||
lanes._finalize_terminal_record(
|
||||
db,
|
||||
tmp_path / "cli-lanes/cassandra/not-a-journal.json",
|
||||
snapshot=misnamed,
|
||||
)
|
||||
== "invalid"
|
||||
)
|
||||
assert misnamed.closed == 1
|
||||
|
||||
empty = _NullSnapshot(None)
|
||||
assert lanes._finalize_terminal_record(db, pending, snapshot=empty) == "invalid"
|
||||
assert empty.closed == 1
|
||||
|
||||
malformed = _NullSnapshot({"kanban_state": "pending"})
|
||||
assert lanes._finalize_terminal_record(db, pending, snapshot=malformed) == (
|
||||
"invalid"
|
||||
)
|
||||
assert malformed.closed == 1
|
||||
|
||||
@ -3,8 +3,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import runpy
|
||||
import sqlite3
|
||||
from contextlib import nullcontext
|
||||
from pathlib import Path
|
||||
@ -13,7 +11,7 @@ from urllib.error import URLError
|
||||
|
||||
import pytest
|
||||
|
||||
from testing.tests.test_hermes_cli_support import SCRIPTS, _completed_result, lanes
|
||||
from testing.tests.test_hermes_cli_support import _completed_result, lanes
|
||||
|
||||
|
||||
def test_board_context_serializes_objects_and_storage_failure_is_bounded(
|
||||
@ -95,26 +93,6 @@ def test_json_and_terminal_path_helpers_fail_closed(tmp_path: Path, monkeypatch)
|
||||
)
|
||||
assert lanes._terminal_evidence_identity(missing_evidence) is None
|
||||
|
||||
with pytest.raises(ValueError, match="positive SQLite"):
|
||||
lanes.TerminalIdentity("board", "task", 0, "pending")
|
||||
|
||||
|
||||
def test_recovery_snapshot_close_is_idempotent_after_descriptors_are_released(
|
||||
tmp_path: Path,
|
||||
):
|
||||
directory = os.open(tmp_path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0))
|
||||
snapshot = lanes.TerminalRecoverySnapshot(
|
||||
None,
|
||||
tmp_path.stat(),
|
||||
None,
|
||||
directory,
|
||||
b"",
|
||||
"synthetic",
|
||||
)
|
||||
snapshot.close()
|
||||
snapshot.close()
|
||||
assert snapshot.directory_descriptor == -1
|
||||
|
||||
|
||||
def test_goal_helpers_cover_compaction_and_deterministic_failures():
|
||||
compacted = lanes.cli_lane_goal._bounded("a" * 100, 40)
|
||||
@ -144,9 +122,6 @@ def test_goal_helpers_cover_compaction_and_deterministic_failures():
|
||||
)
|
||||
assert accepted is False
|
||||
assert reason
|
||||
assert lanes.cli_lane_goal.unfinished_result_reason(
|
||||
{"status": "completed", "summary": "done", "tests_run": "not-a-list"}
|
||||
) is None
|
||||
|
||||
|
||||
def test_prompt_artifact_and_json_helpers_reject_unsafe_inputs(
|
||||
@ -172,31 +147,6 @@ def test_prompt_artifact_and_json_helpers_reject_unsafe_inputs(
|
||||
assert lanes._extract_json('prefix {"status":"blocked"} suffix') == {
|
||||
"status": "blocked"
|
||||
}
|
||||
assert lanes._extract_json('{"status":"unknown"}') is None
|
||||
|
||||
state_file = workspace / "state.json"
|
||||
state = {}
|
||||
assert lanes._event_payload(
|
||||
"codex",
|
||||
json.dumps({"type": "thread.started", "result": {"status": "blocked"}}),
|
||||
state,
|
||||
state_file,
|
||||
) == {"status": "blocked"}
|
||||
assert lanes._event_payload(
|
||||
"claude",
|
||||
json.dumps({"session_id": "session-2", "text": "not-json"}),
|
||||
state,
|
||||
state_file,
|
||||
) is None
|
||||
assert state["claude_session_id"] == "session-2"
|
||||
|
||||
prompt_module = __import__("cli_lane_prompt")
|
||||
assert prompt_module.workspace_artifacts(
|
||||
workspace, [str(directory), str(artifact)]
|
||||
) == [str(artifact.resolve())]
|
||||
assert prompt_module._event_payload(
|
||||
"other", '{"item": {}}', {}, state_file
|
||||
) is None
|
||||
|
||||
|
||||
def test_provider_event_parsing_persists_sessions_and_nested_results(
|
||||
@ -257,9 +207,48 @@ def test_switchyard_network_failure_is_explicit():
|
||||
)
|
||||
|
||||
|
||||
def test_runner_main_guard_delegates_to_dispatch(monkeypatch):
|
||||
dispatch = __import__("cli_lane_dispatch")
|
||||
monkeypatch.setattr(dispatch, "main", lambda: 17)
|
||||
with pytest.raises(SystemExit) as raised:
|
||||
runpy.run_path(str(SCRIPTS / "cli_lane_runner.py"), run_name="__main__")
|
||||
assert raised.value.code == 17
|
||||
def test_artifact_directories_and_foreign_statuses_are_filtered(
|
||||
tmp_path: Path,
|
||||
):
|
||||
"""String directory paths and non-result statuses never become evidence."""
|
||||
workspace = tmp_path / "workspace"
|
||||
(workspace / "directory").mkdir(parents=True)
|
||||
|
||||
assert lanes.workspace_artifacts(workspace, ["directory"]) == []
|
||||
assert lanes._extract_json('{"status": "unknown"}') is None
|
||||
|
||||
|
||||
def test_provider_events_without_sessions_or_results_stay_unparsed(
|
||||
tmp_path: Path,
|
||||
):
|
||||
"""Session persistence and result extraction require explicit fields."""
|
||||
state_file = tmp_path / "state.json"
|
||||
state = {}
|
||||
|
||||
assert (
|
||||
lanes._event_payload(
|
||||
"codex",
|
||||
json.dumps({"type": "thread.started", "thread": {}}),
|
||||
state,
|
||||
state_file,
|
||||
)
|
||||
is None
|
||||
)
|
||||
assert "codex_thread_id" not in state
|
||||
assert (
|
||||
lanes._event_payload(
|
||||
"codex",
|
||||
json.dumps({"item": {"text": "no structured result here"}}),
|
||||
state,
|
||||
state_file,
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
direct = lanes._event_payload(
|
||||
"codex",
|
||||
json.dumps({"result": json.dumps(_completed_result("direct"))}),
|
||||
state,
|
||||
state_file,
|
||||
)
|
||||
assert direct and direct["summary"] == "direct"
|
||||
|
||||
@ -149,3 +149,99 @@ def test_claim_requires_structured_evidence_and_surfaces_artifacts(
|
||||
else:
|
||||
assert calls[0][1]["kind"] == "capability"
|
||||
assert all(connection.closed for connection in connections)
|
||||
|
||||
|
||||
def test_goal_loop_recovers_from_corrupt_rejection_history(
|
||||
tmp_path: Path,
|
||||
monkeypatch,
|
||||
):
|
||||
"""Corrupt durable rejection history is replaced, not trusted or fatal."""
|
||||
task = SimpleNamespace(
|
||||
id="t_history",
|
||||
status="running",
|
||||
result=None,
|
||||
current_run_id=6,
|
||||
assignee="cli-auto",
|
||||
max_runtime_seconds=300,
|
||||
goal_mode=True,
|
||||
goal_max_turns=2,
|
||||
)
|
||||
calls = []
|
||||
fake_db = SimpleNamespace(
|
||||
scoped_current_board=lambda _board: nullcontext(),
|
||||
connect=lambda board: SimpleNamespace(close=lambda: None),
|
||||
get_task=lambda _conn, _task_id: task,
|
||||
worker_log_path=lambda _task_id, board: tmp_path / "worker.log",
|
||||
_resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_history"),
|
||||
set_branch_name=lambda *_args: None,
|
||||
set_workspace_path=lambda *_args: None,
|
||||
build_worker_context=lambda *_args: "achieve the goal with evidence",
|
||||
heartbeat_worker=lambda *_args, **_kwargs: True,
|
||||
add_comment=lambda *_args: None,
|
||||
complete_task=lambda *_args, **kwargs: (
|
||||
calls.append(("complete", kwargs)) or True
|
||||
),
|
||||
block_task=lambda *_args, **kwargs: calls.append(("block", kwargs)),
|
||||
)
|
||||
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
|
||||
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
|
||||
state_file = lanes.state_path("cassandra", "t_history")
|
||||
lanes.atomic_json(state_file, {"goal_rejections": "corrupt-history"})
|
||||
probes = []
|
||||
monkeypatch.setattr(
|
||||
lanes,
|
||||
"fresh_unavailable_provider",
|
||||
lambda *_args, **_kwargs: probes.append("health") and None,
|
||||
)
|
||||
route = lanes.Route(
|
||||
"codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, ()
|
||||
)
|
||||
monkeypatch.setattr(lanes, "select_route", lambda *_args, **_kwargs: route)
|
||||
reports = [
|
||||
lanes.ProcessResult(
|
||||
0,
|
||||
"first turn",
|
||||
{
|
||||
"status": "completed",
|
||||
"summary": "done",
|
||||
"changed_files": ["src/a.py"],
|
||||
"tests_run": ["pytest -q"],
|
||||
"artifacts": [],
|
||||
"findings": [],
|
||||
"blockers": [],
|
||||
},
|
||||
False,
|
||||
),
|
||||
lanes.ProcessResult(
|
||||
0,
|
||||
"second turn",
|
||||
{
|
||||
"status": "completed",
|
||||
"summary": "done with verification",
|
||||
"changed_files": ["src/a.py"],
|
||||
"tests_run": ["pytest -q: passed"],
|
||||
"artifacts": [],
|
||||
"findings": [],
|
||||
"blockers": [],
|
||||
},
|
||||
False,
|
||||
),
|
||||
]
|
||||
monkeypatch.setattr(lanes, "run_provider", lambda *_args, **_kwargs: reports.pop(0))
|
||||
verdicts = iter([(False, "verification evidence missing"), (True, "verified")])
|
||||
monkeypatch.setattr(
|
||||
lanes.cli_lane_goal,
|
||||
"judge_goal_completion",
|
||||
lambda *_args, **_kwargs: next(verdicts),
|
||||
)
|
||||
|
||||
lanes.execute_claim("cassandra", "t_history")
|
||||
|
||||
assert reports == []
|
||||
assert calls[0][0] == "complete"
|
||||
assert calls[0][1]["metadata"]["goal_turn"] == 2
|
||||
assert len(probes) == 2
|
||||
state = lanes.load_json(state_file)
|
||||
assert state["goal_rejections"] == [
|
||||
"local goal judge requested continuation: verification evidence missing"
|
||||
]
|
||||
|
||||
@ -44,13 +44,13 @@ def _result(**overrides):
|
||||
],
|
||||
)
|
||||
def test_unfinished_completion_evidence_is_rejected(result):
|
||||
assert goal.unfinished_result_reason(result)
|
||||
assert goal.unfinished_result_reason(result, role=goal.IMPLEMENTATION_ROLE)
|
||||
|
||||
|
||||
def test_completed_pending_state_test_name_is_not_a_false_positive():
|
||||
result = _result(tests_run=["pytest tests/test_pending_build_state.py: 8 passed"])
|
||||
|
||||
assert goal.unfinished_result_reason(result) is None
|
||||
assert goal.unfinished_result_reason(result, role=goal.IMPLEMENTATION_ROLE) is None
|
||||
|
||||
|
||||
def test_completed_review_findings_are_not_task_blockers():
|
||||
@ -59,7 +59,7 @@ def test_completed_review_findings_are_not_task_blockers():
|
||||
findings=["A casting-table variant fails open."],
|
||||
)
|
||||
|
||||
assert goal.unfinished_result_reason(result) is None
|
||||
assert goal.unfinished_result_reason(result, role=goal.IMPLEMENTATION_ROLE) is None
|
||||
|
||||
|
||||
class JudgeResponse:
|
||||
@ -118,3 +118,14 @@ def test_goal_judge_fails_closed_on_invalid_response():
|
||||
|
||||
assert accepted is False
|
||||
assert "local completion judge unavailable" in reason
|
||||
|
||||
|
||||
def test_completed_report_with_corrupt_test_evidence_is_still_inspected():
|
||||
"""A non-list tests_run field neither crashes nor blocks the summary gate."""
|
||||
assert goal.unfinished_result_reason(_result(tests_run=None)) is None
|
||||
assert goal.unfinished_result_reason(
|
||||
_result(
|
||||
tests_run={"suite": "still running"},
|
||||
summary="The remaining work is still running.",
|
||||
)
|
||||
)
|
||||
|
||||
116
testing/tests/test_hermes_cli_lane_metrics.py
Normal file
116
testing/tests/test_hermes_cli_lane_metrics.py
Normal file
@ -0,0 +1,116 @@
|
||||
"""Lane metrics registry semantics and the pod-local scrape endpoint."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
from testing.tests.test_hermes_cli_support import lanes
|
||||
|
||||
|
||||
def test_registry_renders_counters_and_gauges_with_stable_labels():
|
||||
registry = lanes.LaneMetricsRegistry()
|
||||
registry.increment(
|
||||
"hermes_cli_route_decisions_total",
|
||||
{"provider": "codex", "effort": "high", "classifier": "c", "outcome": "completed"},
|
||||
)
|
||||
registry.increment(
|
||||
"hermes_cli_route_decisions_total",
|
||||
{"provider": "codex", "effort": "high", "classifier": "c", "outcome": "completed"},
|
||||
)
|
||||
registry.increment("hermes_cli_quota_fetch_failures_total")
|
||||
registry.set_value(
|
||||
"hermes_cli_quota_remaining_percent", {"provider": 'we"ird\n\\'}, 12.5
|
||||
)
|
||||
rendered = registry.render().decode("utf-8")
|
||||
|
||||
assert "# TYPE hermes_cli_route_decisions_total counter" in rendered
|
||||
assert "# TYPE hermes_cli_quota_remaining_percent gauge" in rendered
|
||||
assert (
|
||||
'hermes_cli_route_decisions_total{classifier="c",effort="high",'
|
||||
'outcome="completed",provider="codex"} 2' in rendered
|
||||
)
|
||||
assert "hermes_cli_quota_fetch_failures_total 1" in rendered
|
||||
assert (
|
||||
'hermes_cli_quota_remaining_percent{provider="we\\"ird\\n\\\\"} 12.5'
|
||||
in rendered
|
||||
)
|
||||
|
||||
|
||||
def test_recording_helpers_feed_the_shared_registry():
|
||||
lanes.record_route_decision("codex", "xhigh", "metrics-test", "completed")
|
||||
lanes.record_provider_fallback("codex", "claude", "rate-limit")
|
||||
lanes.record_router_selection_failure()
|
||||
lanes.record_quota_fetch_failure()
|
||||
lanes.record_provider_quota("metrics-test-provider", 42.0, 777.0)
|
||||
lanes.record_provider_quota("metrics-test-no-reset", 41.0, None)
|
||||
lanes.record_soft_exclusion("metrics-test-provider", True)
|
||||
rendered = lanes.METRICS.render().decode("utf-8")
|
||||
|
||||
assert (
|
||||
'hermes_cli_route_decisions_total{classifier="metrics-test",'
|
||||
'effort="xhigh",outcome="completed",provider="codex"} 1' in rendered
|
||||
)
|
||||
assert (
|
||||
'hermes_cli_provider_fallbacks_total{from_provider="codex",'
|
||||
'reason="rate-limit",to_provider="claude"}' in rendered
|
||||
)
|
||||
assert "hermes_cli_router_selection_failures_total" in rendered
|
||||
assert "hermes_cli_quota_fetch_failures_total" in rendered
|
||||
assert (
|
||||
'hermes_cli_quota_remaining_percent{provider="metrics-test-provider"} 42'
|
||||
in rendered
|
||||
)
|
||||
assert (
|
||||
'hermes_cli_quota_reset_timestamp_seconds{provider="metrics-test-provider"} 777'
|
||||
in rendered
|
||||
)
|
||||
assert (
|
||||
'hermes_cli_quota_reset_timestamp_seconds{provider="metrics-test-no-reset"}'
|
||||
not in rendered
|
||||
)
|
||||
assert (
|
||||
'hermes_cli_provider_soft_excluded{provider="metrics-test-provider"} 1'
|
||||
in rendered
|
||||
)
|
||||
|
||||
|
||||
def test_metrics_server_serves_scrapes_health_and_rejects_unknown_paths():
|
||||
server = lanes.start_metrics_server(0)
|
||||
assert server is not None
|
||||
try:
|
||||
port = server.server_address[1]
|
||||
lanes.record_route_decision("codex", "low", "scrape-test", "completed")
|
||||
with urllib.request.urlopen(f"http://127.0.0.1:{port}/metrics") as response:
|
||||
assert response.status == 200
|
||||
assert b'classifier="scrape-test"' in response.read()
|
||||
with urllib.request.urlopen(f"http://127.0.0.1:{port}/healthz") as response:
|
||||
assert response.status == 200
|
||||
try:
|
||||
urllib.request.urlopen(f"http://127.0.0.1:{port}/nope")
|
||||
raise AssertionError("unknown paths must 404")
|
||||
except urllib.error.HTTPError as error:
|
||||
assert error.code == 404
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
|
||||
|
||||
def test_metrics_server_reads_port_from_environment(monkeypatch):
|
||||
monkeypatch.setenv("HERMES_CLI_LANE_METRICS_PORT", "0")
|
||||
server = lanes.start_metrics_server()
|
||||
assert server is not None
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
|
||||
|
||||
def test_metrics_server_declines_conflicting_port(capsys):
|
||||
first = lanes.start_metrics_server(0)
|
||||
assert first is not None
|
||||
try:
|
||||
second = lanes.start_metrics_server(first.server_address[1])
|
||||
assert second is None
|
||||
assert "metrics server unavailable" in capsys.readouterr().err
|
||||
finally:
|
||||
first.shutdown()
|
||||
first.server_close()
|
||||
@ -164,7 +164,10 @@ def test_agent_network_boundary_allows_only_authenticated_and_metrics_surfaces()
|
||||
"podSelector": {"matchLabels": {"app": "server"}},
|
||||
}
|
||||
],
|
||||
"ports": [{"protocol": "TCP", "port": 9010}],
|
||||
"ports": [
|
||||
{"protocol": "TCP", "port": 9010},
|
||||
{"protocol": "TCP", "port": 9011},
|
||||
],
|
||||
},
|
||||
]
|
||||
egress = isolation["spec"]["egress"]
|
||||
|
||||
@ -1,446 +0,0 @@
|
||||
"""Agent configuration and dashboard contracts for Hermes CLI lanes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from contextlib import nullcontext
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
from testing.tests.test_hermes_cli_lanes_support import (
|
||||
HERMES,
|
||||
_agent_deployment,
|
||||
client_config,
|
||||
lanes,
|
||||
migration,
|
||||
policy,
|
||||
)
|
||||
|
||||
import cli_lane_execution as lane_execution
|
||||
|
||||
|
||||
def test_workspace_preparation_failure_durably_blocks_the_claim(
|
||||
tmp_path: Path, monkeypatch
|
||||
):
|
||||
task = SimpleNamespace(id="t_bad_worktree", current_run_id=7, assignee="cli-auto")
|
||||
calls = []
|
||||
|
||||
class Connection:
|
||||
def close(self):
|
||||
return None
|
||||
|
||||
fake_db = SimpleNamespace(
|
||||
scoped_current_board=lambda _board: nullcontext(),
|
||||
connect=lambda board: Connection(),
|
||||
get_task=lambda _conn, _task_id: task,
|
||||
worker_log_path=lambda _task_id, board: tmp_path / "worker.log",
|
||||
_resolve_worktree_workspace=lambda _task, board: (_ for _ in ()).throw(
|
||||
ValueError("not a Git repository")
|
||||
),
|
||||
block_task=lambda *_args, **kwargs: calls.append(kwargs),
|
||||
)
|
||||
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
|
||||
monkeypatch.setattr(
|
||||
lane_execution, "state_path", lambda _board, _task_id: tmp_path / "state.json"
|
||||
)
|
||||
|
||||
lanes.execute_claim("cassandra", "t_bad_worktree")
|
||||
|
||||
assert calls[0]["kind"] == "capability"
|
||||
assert calls[0]["expected_run_id"] == 7
|
||||
assert "not a Git repository" in calls[0]["reason"]
|
||||
|
||||
|
||||
def test_artifacts_cannot_escape_the_task_worktree(tmp_path: Path):
|
||||
workspace = tmp_path / "workspace"
|
||||
workspace.mkdir()
|
||||
inside = workspace / "report.json"
|
||||
outside = tmp_path / "auth.json"
|
||||
inside.write_text("{}\n", encoding="utf-8")
|
||||
outside.write_text("secret\n", encoding="utf-8")
|
||||
|
||||
assert lanes.workspace_artifacts(
|
||||
workspace,
|
||||
["report.json", str(outside), "missing.json"],
|
||||
) == [str(inside)]
|
||||
|
||||
|
||||
def test_restart_provider_change_includes_explicit_workspace_handoff(
|
||||
tmp_path: Path, monkeypatch
|
||||
):
|
||||
task = SimpleNamespace(
|
||||
id="t_resume",
|
||||
current_run_id=9,
|
||||
assignee="cli-auto",
|
||||
max_runtime_seconds=60,
|
||||
)
|
||||
state_file = tmp_path / "state.json"
|
||||
state_file.write_text(
|
||||
json.dumps({"current_route": {"provider": "claude"}}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
(tmp_path / "worker.log").write_text("prior provider evidence", encoding="utf-8")
|
||||
prompts = []
|
||||
|
||||
class Connection:
|
||||
def close(self):
|
||||
return None
|
||||
|
||||
fake_db = SimpleNamespace(
|
||||
scoped_current_board=lambda _board: nullcontext(),
|
||||
connect=lambda board: Connection(),
|
||||
get_task=lambda _conn, _task_id: task,
|
||||
worker_log_path=lambda _task_id, board: tmp_path / "worker.log",
|
||||
_resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_resume"),
|
||||
set_branch_name=lambda *_args: None,
|
||||
set_workspace_path=lambda *_args: None,
|
||||
build_worker_context=lambda *_args: "resume objective",
|
||||
add_comment=lambda *_args: None,
|
||||
complete_task=lambda *_args, **_kwargs: None,
|
||||
block_task=lambda *_args, **_kwargs: None,
|
||||
)
|
||||
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
|
||||
monkeypatch.setattr(lane_execution, "state_path", lambda _board, _task_id: state_file)
|
||||
monkeypatch.setattr(
|
||||
lane_execution,
|
||||
"select_route",
|
||||
lambda *_args, **_kwargs: lanes.Route(
|
||||
"codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, ()
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
lane_execution,
|
||||
"git_handoff",
|
||||
lambda _workspace, output: f"HANDOFF:{output}",
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
lane_execution,
|
||||
"run_provider",
|
||||
lambda _route, prompt, *_args, **_kwargs: (
|
||||
prompts.append(prompt)
|
||||
or lanes.ProcessResult(
|
||||
0,
|
||||
"",
|
||||
{
|
||||
"status": "completed",
|
||||
"summary": "done",
|
||||
"changed_files": [],
|
||||
"tests_run": [],
|
||||
"artifacts": [],
|
||||
"blockers": [],
|
||||
},
|
||||
False,
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
lanes.execute_claim("cassandra", "t_resume")
|
||||
|
||||
assert "HANDOFF:prior provider evidence" in prompts[0]
|
||||
|
||||
|
||||
def test_provider_commands_are_structured_unattended_and_capped(tmp_path: Path):
|
||||
route = lanes.Route(
|
||||
"codex", "gpt-5.6-sol", "xhigh", "codex-xhigh", "jetson", "vote", 1, ()
|
||||
)
|
||||
command = lanes._codex_command(
|
||||
route, "Work.", tmp_path, {}, tmp_path / "result.json"
|
||||
)
|
||||
assert "--dangerously-bypass-approvals-and-sandbox" in command
|
||||
assert "--json" in command
|
||||
assert "--output-schema" in command
|
||||
assert 'model_reasoning_effort="xhigh"' in command
|
||||
|
||||
claude_state = {"claude_session_id": "13864642-2985-4f91-bef5-53f145f878e8"}
|
||||
claude = lanes._claude_command(
|
||||
lanes.Route(
|
||||
"claude", "claude-opus-5", "xhigh", "claude-xhigh", "jetson", "vote", 1, ()
|
||||
),
|
||||
"Review.",
|
||||
claude_state,
|
||||
False,
|
||||
)
|
||||
assert "--dangerously-skip-permissions" in claude
|
||||
assert "--output-format" in claude and "stream-json" in claude
|
||||
assert "--json-schema" in claude
|
||||
assert "--disallowedTools" in claude
|
||||
assert "Bash(kubectl apply *)" not in claude
|
||||
assert "Bash(flux reconcile *)" not in claude
|
||||
assert "max" not in claude
|
||||
|
||||
|
||||
def test_worker_contract_separates_review_findings_from_task_blockers(tmp_path: Path):
|
||||
prompt = lanes.build_prompt("Review the change.", tmp_path)
|
||||
|
||||
assert "put defects and risks in findings" in prompt
|
||||
assert "blockers array must be empty whenever status is completed" in prompt
|
||||
assert "findings" in lanes.RESULT_SCHEMA["properties"]
|
||||
assert set(lanes.RESULT_SCHEMA["required"]) == set(
|
||||
lanes.RESULT_SCHEMA["properties"]
|
||||
)
|
||||
assert (
|
||||
"assigned task itself"
|
||||
in lanes.RESULT_SCHEMA["properties"]["blockers"]["description"]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"command",
|
||||
[
|
||||
"git push --force origin main",
|
||||
"git reset --hard HEAD~1",
|
||||
"git clean -fd",
|
||||
],
|
||||
)
|
||||
def test_claude_pretool_hook_blocks_hard_denies(command: str):
|
||||
assert policy.denial_reason(command)
|
||||
|
||||
|
||||
def test_claude_pretool_hook_allows_normal_engineering():
|
||||
assert policy.denial_reason("pytest -q testing/tests") is None
|
||||
assert policy.denial_reason("git push origin feature/hermes") is None
|
||||
assert policy.denial_reason("kubectl delete pod -n cassandra stuck-worker") is None
|
||||
assert policy.denial_reason("flux reconcile kustomization hermes") is None
|
||||
assert policy.denial_reason("vault kv get kv/atlas/hermes") is None
|
||||
|
||||
|
||||
def test_claude_settings_preserve_state_and_install_three_guardrail_layers(
|
||||
tmp_path: Path,
|
||||
):
|
||||
state = tmp_path / ".claude.json"
|
||||
settings = tmp_path / "settings.json"
|
||||
state.write_text('{"promptQueueUseCount": 4}\n', encoding="utf-8")
|
||||
settings.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"theme": "dark",
|
||||
"permissions": {
|
||||
"deny": [
|
||||
"Bash(kubectl apply *)",
|
||||
"Bash(flux reconcile *)",
|
||||
"Bash(vault kv *)",
|
||||
"Bash(custom-owner-rule *)",
|
||||
]
|
||||
},
|
||||
}
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
client_config.configure_claude_state(state)
|
||||
client_config.configure_claude_settings(settings)
|
||||
|
||||
state_value = json.loads(state.read_text())
|
||||
settings_value = json.loads(settings.read_text())
|
||||
assert state_value["promptQueueUseCount"] == 4
|
||||
assert state_value["bypassPermissionsModeAccepted"] is True
|
||||
assert settings_value["theme"] == "dark"
|
||||
assert "Bash(git reset --hard *)" in settings_value["permissions"]["deny"]
|
||||
assert "Bash(custom-owner-rule *)" in settings_value["permissions"]["deny"]
|
||||
assert "Bash(kubectl apply *)" not in settings_value["permissions"]["deny"]
|
||||
assert "Bash(flux reconcile *)" not in settings_value["permissions"]["deny"]
|
||||
assert "Bash(vault kv *)" not in settings_value["permissions"]["deny"]
|
||||
hook = settings_value["hooks"]["PreToolUse"][0]["hooks"][0]
|
||||
assert "claude_command_policy.py" in hook["command"]
|
||||
|
||||
|
||||
def test_legacy_state_is_archived_without_removing_provider_transcripts(tmp_path: Path):
|
||||
session = tmp_path / "home/.config/herdr/session.json"
|
||||
session.parent.mkdir(parents=True)
|
||||
session.write_text(
|
||||
'{"agents":[{"agent":"claude","session_id":"abc"}]}', encoding="utf-8"
|
||||
)
|
||||
binary = tmp_path / "tools/bin/herdr"
|
||||
binary.parent.mkdir(parents=True)
|
||||
binary.write_text("legacy", encoding="utf-8")
|
||||
(tmp_path / "home/.claude").mkdir()
|
||||
(tmp_path / "home/.codex").mkdir()
|
||||
|
||||
archive = migration.archive_legacy_state(tmp_path)
|
||||
|
||||
value = json.loads(archive.read_text())
|
||||
assert value["legacy_session"]["agents"][0]["session_id"] == "abc"
|
||||
assert (tmp_path / "home/.claude").is_dir()
|
||||
assert (tmp_path / "home/.codex").is_dir()
|
||||
assert not binary.exists()
|
||||
assert not (tmp_path / "home/.config/herdr").exists()
|
||||
|
||||
|
||||
def test_agent_uses_one_native_kanban_control_plane():
|
||||
configmap = yaml.safe_load((HERMES / "agent-configmap.yaml").read_text())
|
||||
config = yaml.safe_load(configmap["data"]["config.yaml"])
|
||||
assert config["model"] == {
|
||||
"provider": "atlas-switchyard",
|
||||
"default": "atlas/auto/balanced",
|
||||
"model": "atlas/auto/balanced",
|
||||
}
|
||||
assert config["kanban"]["dispatch_in_gateway"] is True
|
||||
assert config["kanban"]["default_assignee"] == "cli-auto"
|
||||
assert config["plugins"]["enabled"] == ["auto-router"]
|
||||
|
||||
deployment = _agent_deployment()
|
||||
pod = deployment["spec"]["template"]["spec"]
|
||||
assert pod["enableServiceLinks"] is False
|
||||
names = {item["name"] for item in pod["containers"]}
|
||||
assert "cli-lane-runner" in names
|
||||
assert "terminal" in names
|
||||
assert not any("herdr" in name for name in names)
|
||||
rendered = (HERMES / "agent-deployment.yaml").read_text()
|
||||
assert "herdr server" not in rendered
|
||||
assert "herdr-dispatch" not in rendered
|
||||
|
||||
|
||||
def test_cli_lane_reserves_cpu_headroom_for_ui_and_auth():
|
||||
deployment = _agent_deployment()
|
||||
containers = {
|
||||
item["name"]: item
|
||||
for item in deployment["spec"]["template"]["spec"]["containers"]
|
||||
}
|
||||
lane = containers["cli-lane-runner"]
|
||||
environment = {item["name"]: item["value"] for item in lane["env"]}
|
||||
|
||||
assert environment["HERMES_CLI_LANE_CONCURRENCY"] == "2"
|
||||
assert lane["resources"] == {
|
||||
"requests": {"cpu": "100m", "memory": "256Mi"},
|
||||
"limits": {"cpu": "2", "memory": "6Gi"},
|
||||
}
|
||||
|
||||
|
||||
def test_agent_avoids_unhealthy_nodes_and_fits_its_remaining_capacity():
|
||||
"""Placement correction: keep the agent off nodes that cannot hold it.
|
||||
|
||||
titan-04 is cordoned after repeated kernel undervoltage and kubelet
|
||||
failure, and titan-19 was probe/Longhorn unstable under worker load, so
|
||||
both must join the existing hard exclusions. That leaves titan-05 as the
|
||||
healthy candidate, which is tight enough on requested CPU that the main
|
||||
container has to give back 50m to schedule there.
|
||||
"""
|
||||
pod = _agent_deployment()["spec"]["template"]["spec"]
|
||||
hostnames = next(
|
||||
item
|
||||
for item in pod["affinity"]["nodeAffinity"][
|
||||
"requiredDuringSchedulingIgnoredDuringExecution"
|
||||
]["nodeSelectorTerms"][0]["matchExpressions"]
|
||||
if item["key"] == "kubernetes.io/hostname"
|
||||
)
|
||||
|
||||
assert hostnames["operator"] == "NotIn"
|
||||
assert set(hostnames["values"]) >= {"titan-04", "titan-19"}
|
||||
|
||||
hermes = next(item for item in pod["containers"] if item["name"] == "hermes")
|
||||
assert hermes["resources"]["requests"]["cpu"] == "300m"
|
||||
|
||||
|
||||
def test_agent_root_is_stock_dashboard_and_terminal_is_a_separate_path():
|
||||
deployment = _agent_deployment()
|
||||
pod = deployment["spec"]["template"]["spec"]
|
||||
containers = {item["name"]: item for item in pod["containers"]}
|
||||
assert "webui" not in containers
|
||||
|
||||
assert "dashboard" not in containers
|
||||
hermes = containers["hermes"]
|
||||
hermes_env = {item["name"]: item["value"] for item in hermes["env"]}
|
||||
assert hermes_env["HERMES_STREAM_STALE_TIMEOUT"] == "600"
|
||||
assert hermes_env["HERMES_API_CALL_STALE_TIMEOUT"] == "600"
|
||||
assert hermes["command"] == ["/bin/sh", "-ec"]
|
||||
startup = hermes["args"][0]
|
||||
assert ". /opt/data/.env" in startup
|
||||
assert "exec /init /opt/hermes/docker/main-wrapper.sh gateway run" in startup
|
||||
hermes_env = {item["name"]: item["value"] for item in hermes["env"]}
|
||||
assert hermes_env["HERMES_DASHBOARD"] == "1"
|
||||
assert hermes_env["HERMES_DASHBOARD_HOST"] == "127.0.0.1"
|
||||
assert hermes_env["HERMES_DASHBOARD_PORT"] == "9119"
|
||||
assert hermes_env["HERMES_TUI_AGENT_INIT_TIMEOUT_S"] == "180"
|
||||
assert hermes["securityContext"]["runAsUser"] == 0
|
||||
assert hermes["securityContext"]["runAsGroup"] == 0
|
||||
for probe_name in ("startupProbe", "readinessProbe", "livenessProbe"):
|
||||
probe = hermes[probe_name]
|
||||
assert probe["exec"]["command"] == [
|
||||
"curl",
|
||||
"-fsS",
|
||||
"http://127.0.0.1:9119/api/status",
|
||||
]
|
||||
|
||||
terminal = containers["terminal"]
|
||||
command = terminal["args"][0]
|
||||
assert "--base-path /terminal" in command
|
||||
assert "--check-origin" not in command
|
||||
assert "/usr/bin/tmux new-session -A" in command
|
||||
assert "--continue" in command
|
||||
assert "--yolo" in command
|
||||
terminal_env = {item["name"]: item["value"] for item in terminal["env"]}
|
||||
assert terminal_env["HERMES_TUI_AGENT_INIT_TIMEOUT_S"] == "180"
|
||||
|
||||
claude_broker = containers["claude-broker"]
|
||||
claude_env = {item["name"]: item["value"] for item in claude_broker["env"]}
|
||||
assert claude_env["HERMES_CLAUDE_BROKER_CONCURRENCY"] == "2"
|
||||
assert claude_broker["readinessProbe"]["tcpSocket"] == {"port": "claude-broker"}
|
||||
assert claude_broker["livenessProbe"]["tcpSocket"] == {"port": "claude-broker"}
|
||||
|
||||
args = containers["oauth2-proxy"]["args"]
|
||||
terminal_upstream = "--upstream=http://127.0.0.1:7681/terminal/"
|
||||
dashboard_upstream = "--upstream=http://127.0.0.1:9119/"
|
||||
assert terminal_upstream in args
|
||||
assert dashboard_upstream in args
|
||||
assert args.index(terminal_upstream) < args.index(dashboard_upstream)
|
||||
assert "--pass-host-header=false" in args
|
||||
assert "--cookie-refresh=19m" in args
|
||||
assert "--session-store-type=redis" in args
|
||||
assert any(
|
||||
arg.startswith("--redis-connection-url=redis://hermes-oauth-sessions.")
|
||||
for arg in args
|
||||
)
|
||||
|
||||
patch_init = next(
|
||||
item for item in pod["initContainers"] if item["name"] == "patch-tui-gateway"
|
||||
)
|
||||
assert patch_init["command"][-1] == "/patched/server.py"
|
||||
for name in ("hermes", "terminal"):
|
||||
mounts = containers[name]["volumeMounts"]
|
||||
assert {
|
||||
"name": "tui-gateway-patch",
|
||||
"mountPath": "/opt/hermes/tui_gateway/server.py",
|
||||
"subPath": "server.py",
|
||||
} in mounts
|
||||
|
||||
ingress_documents = [
|
||||
item
|
||||
for item in yaml.safe_load_all((HERMES / "agent-ingress.yaml").read_text())
|
||||
if item
|
||||
]
|
||||
middlewares = {
|
||||
item["metadata"]["name"]: item
|
||||
for item in ingress_documents
|
||||
if item["kind"] == "Middleware"
|
||||
}
|
||||
assert middlewares["hermes-agent-terminal-slash"]["spec"]["redirectRegex"][
|
||||
"replacement"
|
||||
].endswith("/terminal/")
|
||||
assert (
|
||||
middlewares["hermes-agent-stock-dashboard-headers"]["spec"]["headers"][
|
||||
"customRequestHeaders"
|
||||
]["Origin"]
|
||||
== "http://127.0.0.1:9119"
|
||||
)
|
||||
ingresses = {
|
||||
item["metadata"]["name"]: item
|
||||
for item in ingress_documents
|
||||
if item["kind"] == "Ingress"
|
||||
}
|
||||
assert (
|
||||
ingresses["hermes-agent-dashboard"]["metadata"]["annotations"][
|
||||
"traefik.ingress.kubernetes.io/router.middlewares"
|
||||
]
|
||||
== "hermes-hermes-agent-stock-dashboard-headers@kubernetescrd"
|
||||
)
|
||||
assert (
|
||||
ingresses["hermes-agent-terminal"]["metadata"]["annotations"][
|
||||
"traefik.ingress.kubernetes.io/router.middlewares"
|
||||
]
|
||||
== "hermes-hermes-agent-terminal-slash@kubernetescrd"
|
||||
)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
x
Reference in New Issue
Block a user