Compare commits
54 Commits
ed43dbaa7f
...
a9ee1070be
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a9ee1070be | ||
|
|
943ecfd15b | ||
|
|
5f27e50c74 | ||
|
|
8036101f0b | ||
|
|
7db9f61659 | ||
|
|
603524ba6a | ||
|
|
a3077224e0 | ||
| be1ff33d4b | |||
|
|
8c7074ae16 | ||
|
|
6b11331520 | ||
|
|
376cc14dd1 | ||
|
|
77df54e3ba | ||
|
|
9d42ff33f5 | ||
|
|
a5d0abb678 | ||
|
|
31439f096a | ||
|
|
6c1123201e | ||
|
|
6e9e8c6ab5 | ||
| 41d10ad3f7 | |||
|
|
6df688adcc | ||
|
|
034c8372c7 | ||
|
|
62d8cd984b | ||
|
|
6f993c049e | ||
|
|
5020e49057 | ||
|
|
b8f2163af0 | ||
|
|
340ec58b68 | ||
|
|
578239a496 | ||
|
|
2019b08276 | ||
|
|
f77119b238 | ||
|
|
64272f52d2 | ||
|
|
3a57371989 | ||
|
|
df75479069 | ||
|
|
e3ecc18d0e | ||
|
|
360d8aad8f | ||
|
|
54156f839e | ||
|
|
5d963c9354 | ||
|
|
7a55b259bf | ||
|
|
03b10ff8b1 | ||
|
|
02e559d7ac | ||
|
|
d208a89d9d | ||
|
|
b080b5f622 | ||
|
|
c808baff40 | ||
|
|
9d5214f809 | ||
|
|
e8c26ecf85 | ||
|
|
73fefbb5d9 | ||
|
|
8c6e3acdac | ||
|
|
2c1d97e551 | ||
|
|
cf3e46c2b6 | ||
|
|
762784da6b | ||
|
|
1ba9f13959 | ||
|
|
8c71d55853 | ||
|
|
c20bc263e0 | ||
|
|
64ea6d685c | ||
|
|
f04a524738 | ||
| 8f00545828 |
288
ci/Jenkinsfile.hermes-agent-image
Normal file
288
ci/Jenkinsfile.hermes-agent-image
Normal file
@ -0,0 +1,288 @@
|
||||
pipeline {
|
||||
agent {
|
||||
kubernetes {
|
||||
defaultContainer 'python'
|
||||
yaml """
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
labels:
|
||||
atlas.bstein.dev/workload: hermes-agent-image-builder
|
||||
spec:
|
||||
serviceAccountName: hermes-image-builder
|
||||
automountServiceAccountToken: false
|
||||
enableServiceLinks: false
|
||||
restartPolicy: Never
|
||||
securityContext:
|
||||
fsGroup: 1000
|
||||
fsGroupChangePolicy: OnRootMismatch
|
||||
nodeSelector:
|
||||
kubernetes.io/arch: arm64
|
||||
node-role.kubernetes.io/worker: "true"
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- key: kubernetes.io/hostname
|
||||
operator: NotIn
|
||||
values:
|
||||
- titan-04
|
||||
- titan-14
|
||||
- titan-18
|
||||
- titan-19
|
||||
- titan-22
|
||||
- titan-24
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
- weight: 100
|
||||
preference:
|
||||
matchExpressions:
|
||||
- key: hardware
|
||||
operator: In
|
||||
values:
|
||||
- rpi5
|
||||
imagePullSecrets:
|
||||
- name: harbor-bstein-robot
|
||||
containers:
|
||||
- name: jnlp
|
||||
image: jenkins/inbound-agent@sha256:8eda4fe2a66bcf6a5e43436d9918fc14c306204dc8fcd75f4e15e0e6e5dc759a
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
resources:
|
||||
requests:
|
||||
cpu: 25m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
- name: python
|
||||
image: registry.bstein.dev/bstein/python@sha256:269541d3387baae008df4608ead893dba2b5cdaad1a5a380731a88992d34b808
|
||||
command: ["sleep"]
|
||||
args: ["99d"]
|
||||
tty: true
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
resources:
|
||||
requests:
|
||||
cpu: 25m
|
||||
memory: 64Mi
|
||||
limits:
|
||||
cpu: 250m
|
||||
memory: 256Mi
|
||||
- name: kaniko
|
||||
image: gcr.io/kaniko-project/executor@sha256:c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e
|
||||
command: ["/busybox/sh", "-c"]
|
||||
args: ["/busybox/sleep 99d"]
|
||||
tty: true
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"]
|
||||
privileged: false
|
||||
runAsUser: 0
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 1Gi
|
||||
limits:
|
||||
cpu: "2"
|
||||
memory: 4Gi
|
||||
"""
|
||||
}
|
||||
}
|
||||
parameters {
|
||||
booleanParam(
|
||||
name: 'PUBLISH_IMAGE',
|
||||
defaultValue: false,
|
||||
description: 'Publish the reviewed main revision to Harbor.'
|
||||
)
|
||||
string(
|
||||
name: 'EXPECTED_SOURCE_REVISION',
|
||||
defaultValue: '',
|
||||
description: 'Exact 40-character commit on atlas/titan-iac main.'
|
||||
)
|
||||
string(
|
||||
name: 'CONFIRM_PUBLISH',
|
||||
defaultValue: '',
|
||||
description: 'Enter PUBLISH HERMES AGENT to confirm the release.'
|
||||
)
|
||||
}
|
||||
environment {
|
||||
HERMES_IMAGE = 'registry.bstein.dev/bstein/hermes-agent'
|
||||
}
|
||||
options {
|
||||
disableConcurrentBuilds()
|
||||
buildDiscarder(logRotator(daysToKeepStr: '30', numToKeepStr: '100', artifactDaysToKeepStr: '30', artifactNumToKeepStr: '100'))
|
||||
skipDefaultCheckout(true)
|
||||
timeout(time: 90, unit: 'MINUTES')
|
||||
}
|
||||
stages {
|
||||
stage('Checkout reviewed source') {
|
||||
steps {
|
||||
checkout scm
|
||||
}
|
||||
}
|
||||
stage('Enforce release boundary') {
|
||||
steps {
|
||||
sh '''
|
||||
set -eu
|
||||
mkdir -p build
|
||||
test "${PUBLISH_IMAGE}" = "true"
|
||||
test "${CONFIRM_PUBLISH}" = "PUBLISH HERMES AGENT"
|
||||
case "${EXPECTED_SOURCE_REVISION}" in
|
||||
*[!0-9a-f]*|'')
|
||||
echo "EXPECTED_SOURCE_REVISION must be a lowercase full commit" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
test "${#EXPECTED_SOURCE_REVISION}" -eq 40
|
||||
actual_revision="$(git rev-parse HEAD)"
|
||||
test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}"
|
||||
test "${actual_revision}" = "$(git rev-parse origin/main)"
|
||||
test -z "$(git status --porcelain)"
|
||||
test -f dockerfiles/Dockerfile.hermes-agent
|
||||
case "${BUILD_NUMBER}" in
|
||||
''|0*|*[!0-9]*)
|
||||
echo "BUILD_NUMBER must be a positive decimal integer" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
printf '%s\n' \
|
||||
"${HERMES_IMAGE}:git-${actual_revision}-build-${BUILD_NUMBER}" \
|
||||
> build/hermes-agent.destination
|
||||
'''
|
||||
}
|
||||
}
|
||||
stage('Reject replay before publish') {
|
||||
steps {
|
||||
withCredentials([usernamePassword(
|
||||
credentialsId: 'harbor-robot',
|
||||
usernameVariable: 'HARBOR_USER',
|
||||
passwordVariable: 'HARBOR_PASSWORD'
|
||||
)]) {
|
||||
sh '''
|
||||
set -eu
|
||||
set +x
|
||||
destination="$(cat build/hermes-agent.destination)"
|
||||
python3 ci/scripts/hermes_image_release.py assert-absent \
|
||||
--source-revision "${EXPECTED_SOURCE_REVISION}" \
|
||||
--build-number "${BUILD_NUMBER}" \
|
||||
--destination "${destination}"
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Build and publish without a daemon') {
|
||||
steps {
|
||||
container('kaniko') {
|
||||
withCredentials([usernamePassword(
|
||||
credentialsId: 'harbor-robot',
|
||||
usernameVariable: 'HARBOR_USER',
|
||||
passwordVariable: 'HARBOR_PASSWORD'
|
||||
)]) {
|
||||
sh '''#!/busybox/sh
|
||||
set -eu
|
||||
set +x
|
||||
config_path=/kaniko/.docker/config.json
|
||||
destination="$(cat build/hermes-agent.destination)"
|
||||
umask 077
|
||||
auth="$(printf '%s:%s' "${HARBOR_USER}" "${HARBOR_PASSWORD}" | /busybox/base64 | /busybox/tr -d '\n')"
|
||||
/busybox/mkdir -p /kaniko/.docker
|
||||
/busybox/printf '{"auths":{"registry.bstein.dev":{"auth":"%s"}}}\n' "${auth}" > "${config_path}"
|
||||
unset HARBOR_USER HARBOR_PASSWORD auth
|
||||
trap '/busybox/rm -f "${config_path}"' EXIT HUP INT TERM
|
||||
/kaniko/executor \
|
||||
--context="dir://${WORKSPACE}" \
|
||||
--dockerfile="${WORKSPACE}/dockerfiles/Dockerfile.hermes-agent" \
|
||||
--destination="${destination}" \
|
||||
--digest-file="${WORKSPACE}/build/hermes-agent.digest" \
|
||||
--image-name-tag-with-digest-file="${WORKSPACE}/build/hermes-agent.image" \
|
||||
--build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1 \
|
||||
--label="org.opencontainers.image.revision=${EXPECTED_SOURCE_REVISION}" \
|
||||
--cleanup \
|
||||
--push-retry=3
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Render reviewed Flux handoff') {
|
||||
steps {
|
||||
withCredentials([usernamePassword(
|
||||
credentialsId: 'harbor-robot',
|
||||
usernameVariable: 'HARBOR_USER',
|
||||
passwordVariable: 'HARBOR_PASSWORD'
|
||||
)]) {
|
||||
sh '''
|
||||
set -eu
|
||||
set +x
|
||||
destination="$(cat build/hermes-agent.destination)"
|
||||
python3 ci/scripts/hermes_image_release.py render \
|
||||
--digest-file build/hermes-agent.digest \
|
||||
--image-file build/hermes-agent.image \
|
||||
--source-revision "${EXPECTED_SOURCE_REVISION}" \
|
||||
--build-number "${BUILD_NUMBER}" \
|
||||
--destination "${destination}" \
|
||||
--kustomization services/hermes/kustomization.yaml \
|
||||
--output-dir build/hermes-agent-release
|
||||
test -s build/hermes-agent-release/hermes-image-update.patch
|
||||
test -s build/hermes-agent-release/hermes-agent-image.json
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
post {
|
||||
success {
|
||||
sh '''
|
||||
set -eu
|
||||
expected_files="$(printf '%s\n' \
|
||||
build/hermes-agent.destination \
|
||||
build/hermes-agent.digest \
|
||||
build/hermes-agent.image \
|
||||
build/hermes-agent-release/hermes-agent-image.json \
|
||||
build/hermes-agent-release/hermes-image-update.patch \
|
||||
build/hermes-agent-release/hermes-kustomization.yaml \
|
||||
| LC_ALL=C sort)"
|
||||
actual_files="$(find build -type f -print | LC_ALL=C sort)"
|
||||
test "${actual_files}" = "${expected_files}"
|
||||
destination="$(cat build/hermes-agent.destination)"
|
||||
python3 ci/scripts/hermes_image_release.py verify-evidence \
|
||||
--digest-file build/hermes-agent.digest \
|
||||
--image-file build/hermes-agent.image \
|
||||
--source-revision "${EXPECTED_SOURCE_REVISION}" \
|
||||
--build-number "${BUILD_NUMBER}" \
|
||||
--destination "${destination}" \
|
||||
--kustomization services/hermes/kustomization.yaml \
|
||||
--output-dir build/hermes-agent-release
|
||||
'''
|
||||
archiveArtifacts(
|
||||
artifacts: 'build/hermes-agent.destination,build/hermes-agent.digest,build/hermes-agent.image,build/hermes-agent-release/hermes-agent-image.json,build/hermes-agent-release/hermes-image-update.patch,build/hermes-agent-release/hermes-kustomization.yaml',
|
||||
allowEmptyArchive: false,
|
||||
fingerprint: true
|
||||
)
|
||||
}
|
||||
cleanup {
|
||||
container('kaniko') {
|
||||
sh '''#!/busybox/sh
|
||||
/busybox/rm -f /kaniko/.docker/config.json
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
500
ci/scripts/hermes_image_release.py
Executable file
500
ci/scripts/hermes_image_release.py
Executable file
@ -0,0 +1,500 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify and render a reviewable Hermes agent image release."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import difflib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable
|
||||
|
||||
|
||||
DEFAULT_IMAGE = "registry.bstein.dev/bstein/hermes-agent"
|
||||
HARBOR_API_ORIGIN = "https://registry.bstein.dev/api/v2.0"
|
||||
HARBOR_PROJECT = "bstein"
|
||||
HARBOR_REPOSITORY = "hermes-agent"
|
||||
IMMUTABLE_REPOSITORY_PATTERN = "hermes-agent"
|
||||
IMMUTABLE_TAG_PATTERN = "git-*-build-*"
|
||||
DIGEST_PATTERN = re.compile(r"^sha256:[0-9a-f]{64}$")
|
||||
REVISION_PATTERN = re.compile(r"^[0-9a-f]{40}$")
|
||||
BUILD_PATTERN = re.compile(r"^[1-9][0-9]*$")
|
||||
DESTINATION_PATTERN = re.compile(
|
||||
r"^registry\.bstein\.dev/bstein/hermes-agent:"
|
||||
r"git-([0-9a-f]{40})-build-([1-9][0-9]*)$"
|
||||
)
|
||||
|
||||
|
||||
class _NoRedirect(urllib.request.HTTPRedirectHandler):
|
||||
"""Never send registry credentials to a redirect target."""
|
||||
|
||||
def redirect_request(self, _request, _file, _code, _message, _headers, _url):
|
||||
return None
|
||||
|
||||
|
||||
def _validated(value: str, pattern: re.Pattern[str], label: str) -> str:
|
||||
"""Return a normalized value when it matches the release contract."""
|
||||
normalized = value.strip()
|
||||
if not pattern.fullmatch(normalized):
|
||||
raise ValueError(f"invalid {label}: expected {pattern.pattern}")
|
||||
return normalized
|
||||
|
||||
|
||||
def validate_destination(
|
||||
destination: str, source_revision: str, build_number: str
|
||||
) -> tuple[str, str]:
|
||||
"""Bind one unique build tag to the reviewed revision and Jenkins build."""
|
||||
revision = _validated(source_revision, REVISION_PATTERN, "source revision")
|
||||
build = _validated(build_number, BUILD_PATTERN, "build number")
|
||||
match = DESTINATION_PATTERN.fullmatch(destination.strip())
|
||||
if not match or match.groups() != (revision, build):
|
||||
raise ValueError(
|
||||
"destination must bind the reviewed revision and unique Jenkins build"
|
||||
)
|
||||
return revision, build
|
||||
|
||||
|
||||
def validate_kaniko_evidence(
|
||||
*, digest_text: str, image_text: str, destination: str
|
||||
) -> str:
|
||||
"""Cross-check both independent Kaniko output files against the destination."""
|
||||
digest_lines = digest_text.splitlines()
|
||||
image_lines = image_text.splitlines()
|
||||
if len(digest_lines) != 1:
|
||||
raise ValueError("Kaniko digest evidence must contain exactly one line")
|
||||
if len(image_lines) != 1:
|
||||
raise ValueError("Kaniko image evidence must contain exactly one line")
|
||||
digest = _validated(digest_lines[0], DIGEST_PATTERN, "image digest")
|
||||
if image_lines[0].strip() != f"{destination}@{digest}":
|
||||
raise ValueError("Kaniko image evidence does not match destination and digest")
|
||||
return digest
|
||||
|
||||
|
||||
def _registry_request(request: urllib.request.Request, timeout: int) -> Any:
|
||||
"""Make a registry request without following redirects."""
|
||||
opener = urllib.request.build_opener(_NoRedirect())
|
||||
try:
|
||||
return opener.open(request, timeout=timeout)
|
||||
except urllib.error.HTTPError as exc:
|
||||
return exc
|
||||
|
||||
|
||||
def _artifact_response(
|
||||
destination: str,
|
||||
*,
|
||||
username: str,
|
||||
password: str,
|
||||
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
|
||||
) -> tuple[int, bytes]:
|
||||
"""Read one exact Harbor artifact by tag with bounded response size."""
|
||||
match = DESTINATION_PATTERN.fullmatch(destination)
|
||||
if not match:
|
||||
raise ValueError("invalid destination")
|
||||
if not username or not password:
|
||||
raise RuntimeError("Harbor credentials are empty")
|
||||
tag = destination.rsplit(":", 1)[1]
|
||||
encoded_tag = urllib.parse.quote(tag, safe="")
|
||||
auth = base64.b64encode(f"{username}:{password}".encode()).decode("ascii")
|
||||
request = urllib.request.Request(
|
||||
f"{HARBOR_API_ORIGIN}/projects/{HARBOR_PROJECT}/repositories/"
|
||||
f"{HARBOR_REPOSITORY}/artifacts/{encoded_tag}"
|
||||
"?with_immutable_status=true",
|
||||
headers={"Accept": "application/json", "Authorization": f"Basic {auth}"},
|
||||
method="GET",
|
||||
)
|
||||
with opener(request, 20) as response:
|
||||
body = response.read(1_048_577)
|
||||
if len(body) > 1_048_576:
|
||||
raise RuntimeError("Harbor artifact response exceeded the size limit")
|
||||
return int(response.status), body
|
||||
|
||||
|
||||
def _immutable_rules_response(
|
||||
*,
|
||||
username: str,
|
||||
password: str,
|
||||
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
|
||||
) -> tuple[int, bytes, dict[str, str]]:
|
||||
"""Read the project policy with the same least-privilege publish identity."""
|
||||
if not username or not password:
|
||||
raise RuntimeError("Harbor credentials are empty")
|
||||
auth = base64.b64encode(f"{username}:{password}".encode()).decode("ascii")
|
||||
request = urllib.request.Request(
|
||||
f"{HARBOR_API_ORIGIN}/projects/{HARBOR_PROJECT}/immutabletagrules"
|
||||
"?page=1&page_size=100",
|
||||
headers={"Accept": "application/json", "Authorization": f"Basic {auth}"},
|
||||
method="GET",
|
||||
)
|
||||
with opener(request, 20) as response:
|
||||
body = response.read(1_048_577)
|
||||
if len(body) > 1_048_576:
|
||||
raise RuntimeError("Harbor immutable rule response exceeded the size limit")
|
||||
return int(response.status), body, dict(response.headers)
|
||||
|
||||
|
||||
def _require_complete_rule_page(
|
||||
rules: list[dict[str, Any]], headers: dict[str, str]
|
||||
) -> None:
|
||||
"""Require proof that the bounded first page contains every rule."""
|
||||
raw_total = next(
|
||||
(value for key, value in headers.items() if key.lower() == "x-total-count"),
|
||||
None,
|
||||
)
|
||||
if raw_total is None or not str(raw_total).isdecimal():
|
||||
raise RuntimeError("Harbor immutable rule list omitted a valid total count")
|
||||
if int(raw_total) != len(rules):
|
||||
raise RuntimeError("Harbor immutable rule list was truncated")
|
||||
|
||||
|
||||
def _normalized_immutable_rule(rule: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Select only fields that bind the server-side build-tag policy."""
|
||||
return {
|
||||
"disabled": bool(rule.get("disabled", False)),
|
||||
"action": rule.get("action"),
|
||||
"template": rule.get("template"),
|
||||
"tag_selectors": [
|
||||
{
|
||||
"kind": item.get("kind"),
|
||||
"decoration": item.get("decoration"),
|
||||
"pattern": item.get("pattern"),
|
||||
}
|
||||
for item in rule.get("tag_selectors") or []
|
||||
if isinstance(item, dict)
|
||||
],
|
||||
"scope_selectors": {
|
||||
"repository": [
|
||||
{
|
||||
"kind": item.get("kind"),
|
||||
"decoration": item.get("decoration"),
|
||||
"pattern": item.get("pattern"),
|
||||
}
|
||||
for item in (rule.get("scope_selectors") or {}).get(
|
||||
"repository", []
|
||||
)
|
||||
if isinstance(item, dict)
|
||||
]
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def verify_immutable_policy(
|
||||
*,
|
||||
username: str,
|
||||
password: str,
|
||||
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
|
||||
) -> None:
|
||||
"""Fail closed before build unless the exact Harbor rule is active."""
|
||||
status, body, headers = _immutable_rules_response(
|
||||
username=username, password=password, opener=opener
|
||||
)
|
||||
if status != 200:
|
||||
raise RuntimeError(f"Harbor immutable policy preflight returned HTTP {status}")
|
||||
try:
|
||||
rules = json.loads(body.decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise RuntimeError("Harbor returned invalid immutable rule JSON") from exc
|
||||
if not isinstance(rules, list) or not all(isinstance(item, dict) for item in rules):
|
||||
raise RuntimeError("Harbor immutable rule list has an invalid shape")
|
||||
_require_complete_rule_page(rules, headers)
|
||||
expected = {
|
||||
"disabled": False,
|
||||
"action": "immutable",
|
||||
"template": "immutable_template",
|
||||
"tag_selectors": [
|
||||
{
|
||||
"kind": "doublestar",
|
||||
"decoration": "matches",
|
||||
"pattern": IMMUTABLE_TAG_PATTERN,
|
||||
}
|
||||
],
|
||||
"scope_selectors": {
|
||||
"repository": [
|
||||
{
|
||||
"kind": "doublestar",
|
||||
"decoration": "repoMatches",
|
||||
"pattern": IMMUTABLE_REPOSITORY_PATTERN,
|
||||
}
|
||||
]
|
||||
},
|
||||
}
|
||||
matches = [
|
||||
_normalized_immutable_rule(item)
|
||||
for item in rules
|
||||
if _normalized_immutable_rule(item)["tag_selectors"]
|
||||
== expected["tag_selectors"]
|
||||
and _normalized_immutable_rule(item)["scope_selectors"]
|
||||
== expected["scope_selectors"]
|
||||
]
|
||||
if matches != [expected]:
|
||||
raise RuntimeError("Harbor immutable build-tag policy is absent or not exact")
|
||||
|
||||
|
||||
def assert_tag_absent(
|
||||
destination: str,
|
||||
*,
|
||||
username: str,
|
||||
password: str,
|
||||
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
|
||||
) -> None:
|
||||
"""Reject replay before Kaniko can push an already-used immutable identity."""
|
||||
status, _body = _artifact_response(
|
||||
destination, username=username, password=password, opener=opener
|
||||
)
|
||||
if status == 404:
|
||||
return
|
||||
if status == 200:
|
||||
raise RuntimeError("Harbor destination tag already exists; refusing overwrite")
|
||||
raise RuntimeError(f"Harbor destination preflight returned HTTP {status}")
|
||||
|
||||
|
||||
def verify_registry_digest(
|
||||
destination: str,
|
||||
digest: str,
|
||||
*,
|
||||
username: str,
|
||||
password: str,
|
||||
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
|
||||
) -> None:
|
||||
"""Verify Harbor independently resolves the pushed tag to Kaniko's digest."""
|
||||
digest = _validated(digest, DIGEST_PATTERN, "image digest")
|
||||
status, body = _artifact_response(
|
||||
destination, username=username, password=password, opener=opener
|
||||
)
|
||||
if status != 200:
|
||||
raise RuntimeError(f"Harbor manifest verification returned HTTP {status}")
|
||||
try:
|
||||
artifact = json.loads(body.decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise RuntimeError("Harbor returned invalid artifact JSON") from exc
|
||||
harbor_digest = str(artifact.get("digest") or "").strip()
|
||||
if not DIGEST_PATTERN.fullmatch(harbor_digest):
|
||||
raise RuntimeError("Harbor response omitted a valid artifact digest")
|
||||
if harbor_digest != digest:
|
||||
raise RuntimeError("Harbor digest does not match Kaniko evidence")
|
||||
expected_tag = destination.rsplit(":", 1)[1]
|
||||
matching_tags = [
|
||||
item
|
||||
for item in artifact.get("tags") or []
|
||||
if isinstance(item, dict) and item.get("name") == expected_tag
|
||||
]
|
||||
if len(matching_tags) != 1:
|
||||
raise RuntimeError("Harbor artifact does not contain the expected tag")
|
||||
if matching_tags[0].get("immutable") is not True:
|
||||
raise RuntimeError("Harbor did not enforce the expected tag as immutable")
|
||||
|
||||
|
||||
def render_kustomization(source: str, digest: str, image: str = DEFAULT_IMAGE) -> str:
|
||||
"""Replace exactly one matching Kustomize image digest without reformatting."""
|
||||
digest = _validated(digest, DIGEST_PATTERN, "image digest")
|
||||
lines = source.splitlines(keepends=True)
|
||||
matches: list[int] = []
|
||||
|
||||
for index, line in enumerate(lines):
|
||||
if line.strip() != f"- name: {image}":
|
||||
continue
|
||||
name_indent = len(line) - len(line.lstrip())
|
||||
for candidate_index in range(index + 1, len(lines)):
|
||||
candidate = lines[candidate_index]
|
||||
stripped = candidate.strip()
|
||||
candidate_indent = len(candidate) - len(candidate.lstrip())
|
||||
if stripped.startswith("- name:") and candidate_indent == name_indent:
|
||||
break
|
||||
if stripped.startswith("digest:") and candidate_indent > name_indent:
|
||||
matches.append(candidate_index)
|
||||
break
|
||||
|
||||
if len(matches) != 1:
|
||||
raise ValueError(
|
||||
f"expected exactly one digest for image {image!r}; found {len(matches)}"
|
||||
)
|
||||
|
||||
index = matches[0]
|
||||
newline = "\n" if lines[index].endswith("\n") else ""
|
||||
prefix = lines[index][: len(lines[index]) - len(lines[index].lstrip())]
|
||||
lines[index] = f"{prefix}digest: {digest}{newline}"
|
||||
return "".join(lines)
|
||||
|
||||
|
||||
def write_release_artifacts(
|
||||
*,
|
||||
digest: str,
|
||||
source_revision: str,
|
||||
build_number: str,
|
||||
destination: str,
|
||||
kustomization: Path,
|
||||
output_dir: Path,
|
||||
) -> dict[str, str]:
|
||||
"""Write a rendered manifest, patch, and credential-free release metadata."""
|
||||
digest = _validated(digest, DIGEST_PATTERN, "image digest")
|
||||
source_revision, build_number = validate_destination(
|
||||
destination, source_revision, build_number
|
||||
)
|
||||
source = kustomization.read_text(encoding="utf-8")
|
||||
rendered = render_kustomization(source, digest)
|
||||
relative_name = kustomization.name
|
||||
patch = "".join(
|
||||
difflib.unified_diff(
|
||||
source.splitlines(keepends=True),
|
||||
rendered.splitlines(keepends=True),
|
||||
fromfile=f"a/services/hermes/{relative_name}",
|
||||
tofile=f"b/services/hermes/{relative_name}",
|
||||
)
|
||||
)
|
||||
if not patch:
|
||||
raise ValueError("published digest already matches the Flux manifest")
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
(output_dir / "hermes-kustomization.yaml").write_text(rendered, encoding="utf-8")
|
||||
(output_dir / "hermes-image-update.patch").write_text(patch, encoding="utf-8")
|
||||
metadata = {
|
||||
"build_number": build_number,
|
||||
"digest": digest,
|
||||
"flux_image": f"{DEFAULT_IMAGE}@{digest}",
|
||||
"image": DEFAULT_IMAGE,
|
||||
"published_tag": destination,
|
||||
"source_revision": source_revision,
|
||||
}
|
||||
(output_dir / "hermes-agent-image.json").write_text(
|
||||
json.dumps(metadata, indent=2, sort_keys=True) + "\n", encoding="utf-8"
|
||||
)
|
||||
return metadata
|
||||
|
||||
|
||||
def validate_release_artifacts(
|
||||
*,
|
||||
digest_file: Path,
|
||||
image_file: Path,
|
||||
source_revision: str,
|
||||
build_number: str,
|
||||
destination: str,
|
||||
kustomization: Path,
|
||||
output_dir: Path,
|
||||
) -> None:
|
||||
"""Revalidate the exact successful-build evidence without rewriting it."""
|
||||
digest = validate_kaniko_evidence(
|
||||
digest_text=digest_file.read_text(encoding="utf-8"),
|
||||
image_text=image_file.read_text(encoding="utf-8"),
|
||||
destination=destination,
|
||||
)
|
||||
source_revision, build_number = validate_destination(
|
||||
destination, source_revision, build_number
|
||||
)
|
||||
expected_names = {
|
||||
"hermes-agent-image.json",
|
||||
"hermes-image-update.patch",
|
||||
"hermes-kustomization.yaml",
|
||||
}
|
||||
entries = list(output_dir.iterdir())
|
||||
if {entry.name for entry in entries} != expected_names or not all(
|
||||
entry.is_file() and not entry.is_symlink() for entry in entries
|
||||
):
|
||||
raise ValueError("release output must contain exactly three evidence files")
|
||||
source = kustomization.read_text(encoding="utf-8")
|
||||
rendered = render_kustomization(source, digest)
|
||||
relative_name = kustomization.name
|
||||
patch = "".join(
|
||||
difflib.unified_diff(
|
||||
source.splitlines(keepends=True),
|
||||
rendered.splitlines(keepends=True),
|
||||
fromfile=f"a/services/hermes/{relative_name}",
|
||||
tofile=f"b/services/hermes/{relative_name}",
|
||||
)
|
||||
)
|
||||
metadata = {
|
||||
"build_number": build_number,
|
||||
"digest": digest,
|
||||
"flux_image": f"{DEFAULT_IMAGE}@{digest}",
|
||||
"image": DEFAULT_IMAGE,
|
||||
"published_tag": destination,
|
||||
"source_revision": source_revision,
|
||||
}
|
||||
expected = {
|
||||
"hermes-agent-image.json": json.dumps(metadata, indent=2, sort_keys=True) + "\n",
|
||||
"hermes-image-update.patch": patch,
|
||||
"hermes-kustomization.yaml": rendered,
|
||||
}
|
||||
for name, expected_text in expected.items():
|
||||
if (output_dir / name).read_text(encoding="utf-8") != expected_text:
|
||||
raise ValueError(f"release evidence is incomplete or mismatched: {name}")
|
||||
|
||||
|
||||
def _credentials() -> tuple[str, str]:
|
||||
"""Read the masked, runtime-only Jenkins credential environment."""
|
||||
username = os.environ.get("HARBOR_USER", "")
|
||||
password = os.environ.get("HARBOR_PASSWORD", "")
|
||||
if not username or not password:
|
||||
raise RuntimeError("Harbor credentials are unavailable")
|
||||
return username, password
|
||||
|
||||
|
||||
def _common_arguments(parser: argparse.ArgumentParser) -> None:
|
||||
parser.add_argument("--source-revision", required=True)
|
||||
parser.add_argument("--build-number", required=True)
|
||||
parser.add_argument("--destination", required=True)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Fail closed around the unique tag, then verify and render after push."""
|
||||
parser = argparse.ArgumentParser()
|
||||
commands = parser.add_subparsers(dest="command", required=True)
|
||||
absent = commands.add_parser("assert-absent")
|
||||
_common_arguments(absent)
|
||||
render = commands.add_parser("render")
|
||||
_common_arguments(render)
|
||||
render.add_argument("--digest-file", required=True, type=Path)
|
||||
render.add_argument("--image-file", required=True, type=Path)
|
||||
render.add_argument("--kustomization", required=True, type=Path)
|
||||
render.add_argument("--output-dir", required=True, type=Path)
|
||||
verify = commands.add_parser("verify-evidence")
|
||||
_common_arguments(verify)
|
||||
verify.add_argument("--digest-file", required=True, type=Path)
|
||||
verify.add_argument("--image-file", required=True, type=Path)
|
||||
verify.add_argument("--kustomization", required=True, type=Path)
|
||||
verify.add_argument("--output-dir", required=True, type=Path)
|
||||
args = parser.parse_args()
|
||||
|
||||
validate_destination(args.destination, args.source_revision, args.build_number)
|
||||
if args.command == "verify-evidence":
|
||||
validate_release_artifacts(
|
||||
digest_file=args.digest_file,
|
||||
image_file=args.image_file,
|
||||
source_revision=args.source_revision,
|
||||
build_number=args.build_number,
|
||||
destination=args.destination,
|
||||
kustomization=args.kustomization,
|
||||
output_dir=args.output_dir,
|
||||
)
|
||||
return 0
|
||||
|
||||
username, password = _credentials()
|
||||
if args.command == "assert-absent":
|
||||
verify_immutable_policy(username=username, password=password)
|
||||
assert_tag_absent(args.destination, username=username, password=password)
|
||||
return 0
|
||||
|
||||
digest = validate_kaniko_evidence(
|
||||
digest_text=args.digest_file.read_text(encoding="utf-8"),
|
||||
image_text=args.image_file.read_text(encoding="utf-8"),
|
||||
destination=args.destination,
|
||||
)
|
||||
verify_registry_digest(
|
||||
args.destination, digest, username=username, password=password
|
||||
)
|
||||
write_release_artifacts(
|
||||
digest=digest,
|
||||
source_revision=args.source_revision,
|
||||
build_number=args.build_number,
|
||||
destination=args.destination,
|
||||
kustomization=args.kustomization,
|
||||
output_dir=args.output_dir,
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover - exercised through main()
|
||||
raise SystemExit(main())
|
||||
@ -52,7 +52,7 @@ def build_report(
|
||||
results = [item for item in raw_results if isinstance(item, dict)] if isinstance(raw_results, list) else []
|
||||
errors = [item for item in raw_errors if isinstance(item, dict)] if isinstance(raw_errors, list) else []
|
||||
|
||||
severity_counts = {severity: 0 for severity in SEVERITIES}
|
||||
severity_counts = dict.fromkeys(SEVERITIES, 0)
|
||||
blocking_findings = 0
|
||||
for finding in results:
|
||||
severity = _finding_severity(finding)
|
||||
@ -93,9 +93,8 @@ def _sonar_issue_from_finding(finding: dict[str, Any]) -> dict[str, Any] | None:
|
||||
start = finding.get("start") if isinstance(finding.get("start"), dict) else {}
|
||||
end = finding.get("end") if isinstance(finding.get("end"), dict) else {}
|
||||
start_line = _line_number(start.get("line") if isinstance(start, dict) else None)
|
||||
# _line_number floors the end line at start_line, so the range never inverts.
|
||||
end_line = _line_number(end.get("line") if isinstance(end, dict) else None, start_line)
|
||||
if end_line < start_line:
|
||||
end_line = start_line
|
||||
return {
|
||||
"engineId": "semgrep",
|
||||
"ruleId": str(finding.get("check_id") or "semgrep.unknown"),
|
||||
|
||||
@ -15,7 +15,13 @@ spec:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
namespace: flux-system
|
||||
wait: false
|
||||
wait: true
|
||||
timeout: 10m
|
||||
healthChecks:
|
||||
- apiVersion: batch/v1
|
||||
kind: Job
|
||||
name: harbor-hermes-agent-immutability-ensure-1
|
||||
namespace: harbor
|
||||
dependsOn:
|
||||
- name: core
|
||||
- name: longhorn
|
||||
|
||||
@ -0,0 +1,19 @@
|
||||
# clusters/atlas/flux-system/applications/hermes-observer-bindings/kustomization.yaml
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: hermes-observer-bindings
|
||||
namespace: flux-system
|
||||
spec:
|
||||
interval: 10m
|
||||
path: ./services/hermes-observer-bindings
|
||||
prune: true
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
namespace: flux-system
|
||||
wait: true
|
||||
timeout: 5m
|
||||
dependsOn:
|
||||
- name: hermes
|
||||
- name: hermes-observer-rbac
|
||||
@ -0,0 +1,16 @@
|
||||
# clusters/atlas/flux-system/applications/hermes-observer-rbac/kustomization.yaml
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: hermes-observer-rbac
|
||||
namespace: flux-system
|
||||
spec:
|
||||
interval: 10m
|
||||
path: ./services/hermes-observer-rbac
|
||||
prune: true
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
namespace: flux-system
|
||||
wait: true
|
||||
timeout: 5m
|
||||
@ -0,0 +1,19 @@
|
||||
# clusters/atlas/flux-system/applications/hermes-scm-broker-code/kustomization.yaml
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: hermes-scm-broker-code
|
||||
namespace: flux-system
|
||||
spec:
|
||||
interval: 10m
|
||||
path: ./services/hermes/scm-common
|
||||
targetNamespace: hermes-scm
|
||||
prune: true
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
namespace: flux-system
|
||||
wait: true
|
||||
timeout: 5m
|
||||
dependsOn:
|
||||
- name: hermes-scm-namespace
|
||||
@ -0,0 +1,27 @@
|
||||
# clusters/atlas/flux-system/applications/hermes-scm-broker/kustomization.yaml
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: hermes-scm-broker
|
||||
namespace: flux-system
|
||||
spec:
|
||||
interval: 10m
|
||||
path: ./services/hermes-scm-broker
|
||||
targetNamespace: hermes-scm
|
||||
prune: true
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
namespace: flux-system
|
||||
wait: true
|
||||
timeout: 10m
|
||||
healthChecks:
|
||||
- apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
name: hermes-scm-broker
|
||||
namespace: hermes-scm
|
||||
dependsOn:
|
||||
- name: vault
|
||||
- name: gitea
|
||||
- name: hermes-scm-namespace
|
||||
- name: hermes-scm-broker-code
|
||||
@ -0,0 +1,16 @@
|
||||
# clusters/atlas/flux-system/applications/hermes-scm-namespace/kustomization.yaml
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: hermes-scm-namespace
|
||||
namespace: flux-system
|
||||
spec:
|
||||
interval: 10m
|
||||
path: ./services/hermes-scm-namespace
|
||||
prune: true
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
namespace: flux-system
|
||||
wait: true
|
||||
timeout: 5m
|
||||
@ -30,6 +30,12 @@ spec:
|
||||
kind: Deployment
|
||||
name: hermes-agent
|
||||
namespace: hermes
|
||||
# hermes-execution-worker and its mediators are deliberately absent. They run
|
||||
# at scavenger priority on a best-effort pool whose first start installs the
|
||||
# provider CLIs, so gating this Kustomization's 10m health window on them
|
||||
# would stall hermes-chat and hermes-observer-bindings, which dependsOn
|
||||
# hermes. The pool reports its own health through the worker readiness probe,
|
||||
# the per-ordinal mediator /ready endpoint, and the coordinator on :9007.
|
||||
- apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
name: hermes-node-ssh-access
|
||||
@ -60,3 +66,6 @@ spec:
|
||||
- name: keycloak
|
||||
- name: longhorn
|
||||
- name: vault
|
||||
- name: jenkins
|
||||
- name: hermes-scm-broker
|
||||
- name: hermes-observer-rbac
|
||||
|
||||
@ -6,10 +6,9 @@ metadata:
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
atlas.bstein.dev/suspend-reason: "CI controller changes are applied only during planned maintenance."
|
||||
spec:
|
||||
interval: 10m
|
||||
suspend: true
|
||||
suspend: false
|
||||
path: ./services/jenkins
|
||||
prune: true
|
||||
sourceRef:
|
||||
@ -18,6 +17,8 @@ spec:
|
||||
targetNamespace: jenkins
|
||||
dependsOn:
|
||||
- name: helm
|
||||
- name: harbor
|
||||
- name: vault-hermes-jenkins-token-seed
|
||||
healthChecks:
|
||||
- apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
|
||||
@ -4,6 +4,7 @@ kind: Kustomization
|
||||
resources:
|
||||
- gitea/kustomization.yaml
|
||||
- vault/kustomization.yaml
|
||||
- vault-hermes-jenkins-token-seed/kustomization.yaml
|
||||
- vaultwarden/kustomization.yaml
|
||||
- comms/kustomization.yaml
|
||||
- crypto/kustomization.yaml
|
||||
@ -27,7 +28,12 @@ resources:
|
||||
- jenkins/kustomization.yaml
|
||||
- ai-llm/kustomization.yaml
|
||||
- openclaw/kustomization.yaml
|
||||
- hermes-scm-namespace/kustomization.yaml
|
||||
- hermes-scm-broker-code/kustomization.yaml
|
||||
- hermes/kustomization.yaml
|
||||
- hermes-observer-rbac/kustomization.yaml
|
||||
- hermes-observer-bindings/kustomization.yaml
|
||||
- hermes-scm-broker/kustomization.yaml
|
||||
- hermes-chat/kustomization.yaml
|
||||
- hermes-triage-demo/kustomization.yaml
|
||||
- game-stream/kustomization.yaml
|
||||
|
||||
@ -0,0 +1,26 @@
|
||||
# clusters/atlas/flux-system/applications/vault-hermes-jenkins-token-seed/kustomization.yaml
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: vault-hermes-jenkins-token-seed
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
spec:
|
||||
interval: 10m
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
namespace: flux-system
|
||||
path: ./services/vault-hermes-jenkins-token-seed
|
||||
targetNamespace: vault
|
||||
prune: true
|
||||
wait: true
|
||||
timeout: 5m
|
||||
dependsOn:
|
||||
- name: vault
|
||||
healthChecks:
|
||||
- apiVersion: batch/v1
|
||||
kind: Job
|
||||
name: vault-hermes-jenkins-build-token-seed-2
|
||||
namespace: vault
|
||||
@ -16,6 +16,11 @@ spec:
|
||||
targetNamespace: vault
|
||||
prune: true
|
||||
wait: true
|
||||
healthChecks:
|
||||
- apiVersion: batch/v1
|
||||
kind: Job
|
||||
name: vault-k8s-auth-hermes-10
|
||||
namespace: vault
|
||||
dependsOn:
|
||||
- name: longhorn
|
||||
- name: helm
|
||||
|
||||
@ -15,6 +15,10 @@ RUN apt-get update \
|
||||
# Paid/provider-backed search remains selectable through normal Hermes config.
|
||||
RUN uv pip install --python /opt/hermes/.venv/bin/python ddgs==9.14.4
|
||||
|
||||
ARG HERMES_KANIKO_HEREDOC_COMPAT=0
|
||||
COPY dockerfiles/Dockerfile.hermes-agent /tmp/hermes-agent.Dockerfile
|
||||
COPY dockerfiles/hermes-kaniko-heredoc-runner.py /tmp/hermes-kaniko-heredoc-runner.py
|
||||
|
||||
# Keep dashboard chat sockets tied to the intended React mount and conversation.
|
||||
# A resumed conversation needs a different PTY attachment key from a fresh chat;
|
||||
# reconnects to that same conversation must keep using the same key.
|
||||
@ -252,6 +256,7 @@ for (const [before, after, label] of [
|
||||
}
|
||||
fs.writeFileSync(sidebarPath, sidebar);
|
||||
NODE
|
||||
RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 1 ;; *) exit 2 ;; esac
|
||||
|
||||
# The upstream OIDC gate authenticates users but deliberately treats the
|
||||
# dashboard as one shared workstation. Allow a deployment to narrow that
|
||||
@ -324,6 +329,7 @@ for before, after, label in (
|
||||
source = source.replace(before, after, 1)
|
||||
path.write_text(source)
|
||||
PY
|
||||
RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 2 ;; *) exit 2 ;; esac
|
||||
|
||||
# Give trusted plugins a pre-turn routing hook. It runs after fallback runtime
|
||||
# restoration but before Hermes builds its provider-specific system prompt.
|
||||
@ -493,6 +499,7 @@ if oneshot.count(oneshot_before) != 1:
|
||||
)
|
||||
oneshot_path.write_text(oneshot.replace(oneshot_before, oneshot_after, 1))
|
||||
PY
|
||||
RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 3 ;; *) exit 2 ;; esac
|
||||
|
||||
# Hermes WebUI sends its model/provider/reasoning selection on /v1/runs.
|
||||
# Upstream currently applies only statically declared model_routes there, so
|
||||
@ -628,6 +635,7 @@ runs_source = runs_source.replace(runs_agent_before, runs_agent_after, 1)
|
||||
source = source[:runs_start] + runs_source
|
||||
path.write_text(source)
|
||||
PY
|
||||
RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 4 ;; *) exit 2 ;; esac
|
||||
|
||||
# ``create_task(initial_status="blocked")`` is an explicit operator park, but
|
||||
# upstream only treats later block_task() events as sticky. Recognize the
|
||||
@ -779,6 +787,7 @@ if cli_source.count(help_before) != 1:
|
||||
)
|
||||
cli_path.write_text(cli_source.replace(help_before, help_after, 1))
|
||||
PY
|
||||
RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 5 ;; *) exit 2 ;; esac
|
||||
|
||||
COPY dockerfiles/hermes-kanban-blocked-regression.py /tmp/hermes-kanban-blocked-regression.py
|
||||
RUN /opt/hermes/.venv/bin/python /tmp/hermes-kanban-blocked-regression.py \
|
||||
@ -816,6 +825,7 @@ if source.count(before) != 1:
|
||||
)
|
||||
path.write_text(source.replace(before, after, 1))
|
||||
PY
|
||||
RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 6 ;; *) exit 2 ;; esac
|
||||
|
||||
# Keep the dashboard's account cards aligned with the managed provider lanes.
|
||||
# Switchyard receives Claude Code subscription OAuth through the environment;
|
||||
@ -882,6 +892,7 @@ for before, after, label in replacements:
|
||||
|
||||
path.write_text(source)
|
||||
PY
|
||||
RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 7 ;; *) exit 2 ;; esac
|
||||
|
||||
# Make parent-linked API workers first-class live dashboard sessions. They stay
|
||||
# active until the API runner closes them, even during a long final model call;
|
||||
@ -975,6 +986,7 @@ source = source.replace(row_before, row_after)
|
||||
|
||||
path.write_text(source)
|
||||
PY
|
||||
RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 8 ;; *) exit 2 ;; esac
|
||||
|
||||
# Keep the browser terminal reliable across GPU context loss, make durable
|
||||
# worker lineage visible, expose an accessible API-worker transcript, and
|
||||
@ -1543,9 +1555,42 @@ function RootRedirect() {
|
||||
fs.writeFileSync(path, source);
|
||||
}
|
||||
NODE
|
||||
RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 9 ;; *) exit 2 ;; esac
|
||||
|
||||
COPY dockerfiles/patch-hermes-execution-safety.py /tmp/patch-hermes-execution-safety.py
|
||||
COPY dockerfiles/hermes-execution-safety-regression.py /tmp/hermes-execution-safety-regression.py
|
||||
COPY dockerfiles/hermes_execution_patch_support.py /tmp/hermes_execution_patch_support.py
|
||||
COPY dockerfiles/patch_hermes_run_safety.py /tmp/patch_hermes_run_safety.py
|
||||
COPY dockerfiles/patch_hermes_decomposition_safety.py /tmp/patch_hermes_decomposition_safety.py
|
||||
COPY dockerfiles/hermes_execution_regression_support.py /tmp/hermes_execution_regression_support.py
|
||||
COPY dockerfiles/hermes_run_safety_regression.py /tmp/hermes_run_safety_regression.py
|
||||
COPY dockerfiles/hermes_decomposition_safety_regression.py /tmp/hermes_decomposition_safety_regression.py
|
||||
COPY dockerfiles/hermes_lane_compatibility_regression.py /tmp/hermes_lane_compatibility_regression.py
|
||||
COPY services/hermes/scripts/cli_lane_*.py /tmp/hermes-lane-regression/
|
||||
RUN HERMES_CLI_LANE_SOURCE=/tmp/hermes-lane-regression \
|
||||
HERMES_COMPATIBILITY_MODE=legacy \
|
||||
/opt/hermes/.venv/bin/python /tmp/hermes_lane_compatibility_regression.py \
|
||||
&& /opt/hermes/.venv/bin/python /tmp/patch-hermes-execution-safety.py \
|
||||
&& HERMES_CLI_LANE_SOURCE=/tmp/hermes-lane-regression \
|
||||
HERMES_COMPATIBILITY_MODE=patched \
|
||||
/opt/hermes/.venv/bin/python /tmp/hermes_lane_compatibility_regression.py \
|
||||
&& HERMES_CLI_LANE_SOURCE=/tmp/hermes-lane-regression \
|
||||
/opt/hermes/.venv/bin/python /tmp/hermes-execution-safety-regression.py \
|
||||
&& rm /tmp/patch-hermes-execution-safety.py \
|
||||
/tmp/hermes-execution-safety-regression.py \
|
||||
/tmp/hermes_execution_patch_support.py \
|
||||
/tmp/patch_hermes_run_safety.py \
|
||||
/tmp/patch_hermes_decomposition_safety.py \
|
||||
/tmp/hermes_execution_regression_support.py \
|
||||
/tmp/hermes_run_safety_regression.py \
|
||||
/tmp/hermes_decomposition_safety_regression.py \
|
||||
/tmp/hermes_lane_compatibility_regression.py \
|
||||
&& find /tmp/hermes-lane-regression -depth -delete
|
||||
|
||||
COPY dockerfiles/hermes-session-migrate.py /opt/hermes/bin/hermes-session-migrate
|
||||
|
||||
RUN rm -f /tmp/hermes-agent.Dockerfile /tmp/hermes-kaniko-heredoc-runner.py
|
||||
|
||||
RUN cd /opt/hermes/web \
|
||||
&& npm run build \
|
||||
&& grep -Fq 'await api.getSessions(1, 0' src/pages/ChatPage.tsx \
|
||||
@ -1602,6 +1647,8 @@ RUN cd /opt/hermes/web \
|
||||
/opt/hermes/agent/turn_context.py \
|
||||
/opt/hermes/agent/conversation_loop.py \
|
||||
/opt/hermes/hermes_cli/oneshot.py \
|
||||
/opt/hermes/hermes_cli/kanban_decompose.py \
|
||||
/opt/hermes/gateway/kanban_watchers.py \
|
||||
/opt/hermes/tools/delegate_tool.py \
|
||||
/opt/hermes/tools/web_tools.py \
|
||||
/opt/hermes/tools/python_sandbox_tool.py \
|
||||
|
||||
@ -5,3 +5,18 @@
|
||||
!dockerfiles/hermes-public-extract/**
|
||||
!dockerfiles/hermes-session-activity-panel.tsx
|
||||
!dockerfiles/hermes-session-migrate.py
|
||||
!dockerfiles/Dockerfile.hermes-agent
|
||||
!dockerfiles/hermes-kaniko-heredoc-runner.py
|
||||
!dockerfiles/patch-hermes-execution-safety.py
|
||||
!dockerfiles/hermes-execution-safety-regression.py
|
||||
!dockerfiles/hermes_execution_patch_support.py
|
||||
!dockerfiles/patch_hermes_run_safety.py
|
||||
!dockerfiles/patch_hermes_decomposition_safety.py
|
||||
!dockerfiles/hermes_execution_regression_support.py
|
||||
!dockerfiles/hermes_run_safety_regression.py
|
||||
!dockerfiles/hermes_decomposition_safety_regression.py
|
||||
!dockerfiles/hermes_lane_compatibility_regression.py
|
||||
!services/
|
||||
!services/hermes/
|
||||
!services/hermes/scripts/
|
||||
!services/hermes/scripts/cli_lane_*.py
|
||||
|
||||
23
dockerfiles/hermes-execution-safety-regression.py
Normal file
23
dockerfiles/hermes-execution-safety-regression.py
Normal file
@ -0,0 +1,23 @@
|
||||
"""Run the composable Hermes execution-safety build regressions."""
|
||||
|
||||
import unittest
|
||||
|
||||
from hermes_decomposition_safety_regression import DecompositionSafetyTests
|
||||
from hermes_run_safety_regression import ExactRunSafetyTests
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Run both transaction domains and return a build-friendly status."""
|
||||
loader = unittest.defaultTestLoader
|
||||
suite = unittest.TestSuite(
|
||||
(
|
||||
loader.loadTestsFromTestCase(ExactRunSafetyTests),
|
||||
loader.loadTestsFromTestCase(DecompositionSafetyTests),
|
||||
)
|
||||
)
|
||||
result = unittest.TextTestRunner(verbosity=2).run(suite)
|
||||
return 0 if result.wasSuccessful() else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
146
dockerfiles/hermes-kaniko-heredoc-runner.py
Normal file
146
dockerfiles/hermes-kaniko-heredoc-runner.py
Normal file
@ -0,0 +1,146 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Replay the reviewed Hermes Dockerfile heredocs for pinned Kaniko v1.23.2.
|
||||
|
||||
Kaniko v1.23.2 parses shell heredoc bodies into ``RunCommand.Files`` but its RUN
|
||||
implementation executes only ``CmdLine``. The interpreters therefore receive
|
||||
empty stdin and exit successfully. Docker and BuildKit execute these blocks
|
||||
normally, so this compatibility runner is enabled only by the Kaniko pipeline.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
MAX_DOCKERFILE_BYTES = 2_000_000
|
||||
BLOCKS = (
|
||||
("RUN node <<'NODE'", "NODE", ("node",)),
|
||||
("RUN python - <<'PY'", "PY", ("python", "-")),
|
||||
)
|
||||
EXPECTED_COMMANDS = ("node", *("python",) * 7, "node")
|
||||
PARSER_DIRECTIVE = re.compile(
|
||||
r"^\s*#\s*([a-z]+)\s*=\s*(.*?)\s*$", re.IGNORECASE
|
||||
)
|
||||
|
||||
|
||||
def _escape_character(lines: list[str]) -> str:
|
||||
"""Return the one Dockerfile parser escape directive, or its default."""
|
||||
escape = "\\"
|
||||
directives: set[str] = set()
|
||||
for line in lines:
|
||||
if not line.strip():
|
||||
break
|
||||
match = PARSER_DIRECTIVE.fullmatch(line)
|
||||
if not match:
|
||||
break
|
||||
name, value = match.group(1).lower(), match.group(2)
|
||||
if name in directives:
|
||||
raise ValueError(f"multiple Dockerfile {name} directives")
|
||||
directives.add(name)
|
||||
if name == "escape":
|
||||
if value not in {"\\", "`"}:
|
||||
raise ValueError("unsupported Dockerfile escape directive")
|
||||
escape = value
|
||||
return escape
|
||||
|
||||
|
||||
def _continued(line: str, escape: str) -> bool:
|
||||
"""Match Docker's unescaped continuation marker at physical line end."""
|
||||
stripped = line.rstrip()
|
||||
count = len(stripped) - len(stripped.rstrip(escape))
|
||||
return count % 2 == 1
|
||||
|
||||
|
||||
def _logical_instruction(
|
||||
lines: list[str], index: int, escape: str
|
||||
) -> tuple[str, int]:
|
||||
"""Normalize escape-newline pairs, including split opcodes/operators."""
|
||||
logical = lines[index]
|
||||
while _continued(logical.split("\n")[-1], escape):
|
||||
logical = logical.rstrip()
|
||||
logical = logical[:-1]
|
||||
index += 1
|
||||
while index < len(lines) and lines[index].lstrip().startswith("#"):
|
||||
index += 1
|
||||
if index >= len(lines) or not lines[index].strip():
|
||||
raise ValueError("unterminated Dockerfile line continuation")
|
||||
logical += lines[index].lstrip()
|
||||
return logical, index
|
||||
|
||||
|
||||
def extract_blocks(source: str) -> list[tuple[tuple[str, ...], str]]:
|
||||
"""Inventory every RUN heredoc, then return only the reviewed contract."""
|
||||
markers = {start: (end, command) for start, end, command in BLOCKS}
|
||||
lines = source.splitlines()
|
||||
escape = _escape_character(lines)
|
||||
blocks: list[tuple[tuple[str, ...], str]] = []
|
||||
index = 0
|
||||
while index < len(lines):
|
||||
marker = markers.get(lines[index])
|
||||
if marker is None:
|
||||
if not lines[index].strip() or lines[index].lstrip().startswith("#"):
|
||||
index += 1
|
||||
continue
|
||||
logical, index = _logical_instruction(lines, index, escape)
|
||||
stripped = logical.lstrip()
|
||||
if (
|
||||
stripped[:3].upper() == "RUN"
|
||||
and (len(stripped) == 3 or stripped[3].isspace())
|
||||
and "<<" in stripped
|
||||
):
|
||||
raise ValueError(
|
||||
"unsupported RUN heredoc outside the exact reviewed contract"
|
||||
)
|
||||
index += 1
|
||||
continue
|
||||
end, command = marker
|
||||
body_start = index + 1
|
||||
index = body_start
|
||||
while index < len(lines) and lines[index] != end:
|
||||
index += 1
|
||||
if index == len(lines):
|
||||
raise ValueError(f"unterminated {command[0]} heredoc")
|
||||
body = "\n".join(lines[body_start:index]) + "\n"
|
||||
if not body.strip():
|
||||
raise ValueError(f"empty {command[0]} heredoc")
|
||||
blocks.append((command, body))
|
||||
index += 1
|
||||
|
||||
commands = tuple(command[0] for command, _body in blocks)
|
||||
if commands != EXPECTED_COMMANDS:
|
||||
raise ValueError(
|
||||
"Hermes Dockerfile heredoc contract changed: "
|
||||
f"expected {EXPECTED_COMMANDS!r}, received {commands!r}"
|
||||
)
|
||||
return blocks
|
||||
|
||||
|
||||
def replay(dockerfile: Path, block_index: int) -> None:
|
||||
"""Execute one exact heredoc after inventorying the complete Dockerfile."""
|
||||
size = dockerfile.stat().st_size
|
||||
if size < 1 or size > MAX_DOCKERFILE_BYTES:
|
||||
raise ValueError("Hermes Dockerfile size is outside the reviewed boundary")
|
||||
source = dockerfile.read_text(encoding="utf-8")
|
||||
blocks = extract_blocks(source)
|
||||
if block_index < 1 or block_index > len(blocks):
|
||||
raise ValueError("heredoc block index is outside the reviewed contract")
|
||||
command, body = blocks[block_index - 1]
|
||||
print(f"replaying reviewed Dockerfile heredoc {block_index}/{len(blocks)}")
|
||||
subprocess.run(command, input=body, text=True, check=True)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Parse the one explicit Dockerfile path and replay its reviewed patches."""
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--dockerfile", required=True, type=Path)
|
||||
parser.add_argument("--block-index", required=True, type=int)
|
||||
args = parser.parse_args()
|
||||
replay(args.dockerfile, args.block_index)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
120
dockerfiles/hermes_decomposition_safety_regression.py
Normal file
120
dockerfiles/hermes_decomposition_safety_regression.py
Normal file
@ -0,0 +1,120 @@
|
||||
"""Regress automatic decomposition against existing execution history."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
from hermes_cli import kanban_db
|
||||
from hermes_cli import kanban_decompose
|
||||
from hermes_execution_regression_support import ExecutionSafetyTestCase
|
||||
|
||||
|
||||
class DecompositionSafetyTests(ExecutionSafetyTestCase):
|
||||
"""Verify only unexecuted triage roots may decompose automatically."""
|
||||
|
||||
def test_automatic_decomposer_skips_executed_triage_task(self) -> None:
|
||||
task_id = self._route_executed_task_to_triage()
|
||||
completions = self._client(
|
||||
{"fanout": False, "title": "must not run", "body": "must not run"}
|
||||
)
|
||||
before = len(kanban_db.list_tasks(self.connection))
|
||||
|
||||
outcome = kanban_decompose.decompose_task(task_id, automatic=True)
|
||||
|
||||
self.assertFalse(outcome.ok)
|
||||
self.assertIn("execution history", outcome.reason)
|
||||
self.assertEqual(completions.calls, 0)
|
||||
self.assertEqual(self._status(task_id), "triage")
|
||||
self.assertEqual(len(kanban_db.list_tasks(self.connection)), before)
|
||||
|
||||
def test_manual_decomposition_remains_available_after_execution(self) -> None:
|
||||
task_id = self._route_executed_task_to_triage()
|
||||
completions = self._client(
|
||||
{
|
||||
"fanout": False,
|
||||
"title": "operator-approved retry",
|
||||
"body": "Retain one bounded task.",
|
||||
"assignee": "default",
|
||||
}
|
||||
)
|
||||
|
||||
outcome = kanban_decompose.decompose_task(task_id, author="operator")
|
||||
|
||||
self.assertTrue(outcome.ok)
|
||||
self.assertFalse(outcome.fanout)
|
||||
self.assertEqual(completions.calls, 1)
|
||||
self.assertEqual(self._status(task_id), "ready")
|
||||
|
||||
def test_fresh_triage_task_still_auto_promotes(self) -> None:
|
||||
task_id = kanban_db.create_task(
|
||||
self.connection,
|
||||
title="fresh objective",
|
||||
triage=True,
|
||||
)
|
||||
completions = self._client(
|
||||
{
|
||||
"fanout": False,
|
||||
"title": "specified objective",
|
||||
"body": "One bounded task.",
|
||||
"assignee": "default",
|
||||
}
|
||||
)
|
||||
|
||||
outcome = kanban_decompose.decompose_task(task_id, automatic=True)
|
||||
|
||||
self.assertTrue(outcome.ok)
|
||||
self.assertEqual(completions.calls, 1)
|
||||
self.assertEqual(self._status(task_id), "ready")
|
||||
|
||||
def test_execution_history_gained_during_single_llm_call_blocks_commit(self) -> None:
|
||||
task_id = kanban_db.create_task(
|
||||
self.connection,
|
||||
title="concurrent single objective",
|
||||
triage=True,
|
||||
)
|
||||
completions = self._client(
|
||||
{
|
||||
"fanout": False,
|
||||
"title": "redundant rewrite",
|
||||
"body": "must not be committed",
|
||||
"assignee": "default",
|
||||
},
|
||||
before_response=lambda: self._insert_run(task_id),
|
||||
)
|
||||
|
||||
outcome = kanban_decompose.decompose_task(task_id, automatic=True)
|
||||
|
||||
self.assertFalse(outcome.ok)
|
||||
self.assertIn("gained execution history", outcome.reason)
|
||||
self.assertEqual(completions.calls, 1)
|
||||
self.assertEqual(self._status(task_id), "triage")
|
||||
|
||||
def test_execution_history_gained_during_fanout_llm_call_blocks_commit(self) -> None:
|
||||
task_id = kanban_db.create_task(
|
||||
self.connection,
|
||||
title="concurrent triage objective",
|
||||
triage=True,
|
||||
)
|
||||
|
||||
completions = self._client(
|
||||
{
|
||||
"fanout": True,
|
||||
"tasks": [
|
||||
{
|
||||
"title": "redundant child",
|
||||
"body": "must not be created",
|
||||
"assignee": "default",
|
||||
"parents": [],
|
||||
}
|
||||
],
|
||||
},
|
||||
before_response=lambda: self._insert_run(task_id),
|
||||
)
|
||||
before = len(kanban_db.list_tasks(self.connection))
|
||||
|
||||
outcome = kanban_decompose.decompose_task(task_id, automatic=True)
|
||||
|
||||
self.assertFalse(outcome.ok)
|
||||
self.assertIn("gained execution history", outcome.reason)
|
||||
self.assertEqual(completions.calls, 1)
|
||||
self.assertEqual(self._status(task_id), "triage")
|
||||
self.assertEqual(len(kanban_db.list_tasks(self.connection)), before)
|
||||
12
dockerfiles/hermes_execution_patch_support.py
Normal file
12
dockerfiles/hermes_execution_patch_support.py
Normal file
@ -0,0 +1,12 @@
|
||||
"""Fail-closed helpers shared by Hermes source patches."""
|
||||
|
||||
|
||||
|
||||
def replace_once(source: str, before: str, after: str, label: str) -> str:
|
||||
"""Replace one anchored upstream fragment and fail closed on source drift."""
|
||||
count = source.count(before)
|
||||
if count != 1:
|
||||
raise SystemExit(
|
||||
f"Hermes {label} patch context changed: expected 1, found {count}"
|
||||
)
|
||||
return source.replace(before, after, 1)
|
||||
132
dockerfiles/hermes_execution_regression_support.py
Normal file
132
dockerfiles/hermes_execution_regression_support.py
Normal file
@ -0,0 +1,132 @@
|
||||
"""Shared isolated database fixture for Hermes execution-safety regressions."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest import mock
|
||||
|
||||
_HERMES_HOME = tempfile.TemporaryDirectory(prefix="hermes-execution-test-home-")
|
||||
os.environ["HERMES_HOME"] = _HERMES_HOME.name
|
||||
_LANE_SOURCE = Path(
|
||||
os.environ.get("HERMES_CLI_LANE_SOURCE", "/tmp/hermes-lane-regression")
|
||||
)
|
||||
sys.path.insert(0, str(_LANE_SOURCE))
|
||||
|
||||
from agent import auxiliary_client # noqa: E402
|
||||
from cli_lane_config import TerminalIdentity # noqa: E402
|
||||
from cli_lane_finalization import _finalize_document_db # noqa: E402
|
||||
from hermes_cli import kanban_db # noqa: E402
|
||||
from hermes_cli import kanban_decompose # noqa: E402
|
||||
|
||||
__all__ = ("ExecutionSafetyTestCase", "TerminalIdentity", "_finalize_document_db")
|
||||
|
||||
|
||||
class _FakeCompletions:
|
||||
def __init__(self, payload: dict, before_response=None) -> None:
|
||||
self.payload = payload
|
||||
self.before_response = before_response
|
||||
self.calls = 0
|
||||
|
||||
def create(self, **_kwargs):
|
||||
self.calls += 1
|
||||
if self.before_response is not None:
|
||||
self.before_response()
|
||||
return SimpleNamespace(
|
||||
choices=[
|
||||
SimpleNamespace(
|
||||
message=SimpleNamespace(content=json.dumps(self.payload))
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
class _FakeClient:
|
||||
def __init__(self, completions: _FakeCompletions) -> None:
|
||||
self.chat = SimpleNamespace(completions=completions)
|
||||
|
||||
|
||||
class ExecutionSafetyTestCase(unittest.TestCase):
|
||||
"""Exercise patched upstream APIs against an isolated real database."""
|
||||
|
||||
def setUp(self) -> None:
|
||||
self.connection = kanban_db.connect()
|
||||
self.patches = [
|
||||
mock.patch.object(
|
||||
kanban_decompose,
|
||||
"_build_roster",
|
||||
return_value=([], {"default"}),
|
||||
),
|
||||
mock.patch.object(
|
||||
kanban_decompose,
|
||||
"_load_config",
|
||||
return_value={"kanban": {"auto_promote_children": True}},
|
||||
),
|
||||
mock.patch.object(
|
||||
auxiliary_client,
|
||||
"get_auxiliary_extra_body",
|
||||
return_value={},
|
||||
),
|
||||
]
|
||||
for patch in self.patches:
|
||||
patch.start()
|
||||
|
||||
def tearDown(self) -> None:
|
||||
for patch in reversed(self.patches):
|
||||
patch.stop()
|
||||
self.connection.close()
|
||||
|
||||
def _client(self, payload: dict, before_response=None) -> _FakeCompletions:
|
||||
completions = _FakeCompletions(payload, before_response)
|
||||
client = _FakeClient(completions)
|
||||
patch = mock.patch.object(
|
||||
auxiliary_client,
|
||||
"get_text_auxiliary_client",
|
||||
return_value=(client, "test-decomposer"),
|
||||
)
|
||||
patch.start()
|
||||
self.addCleanup(patch.stop)
|
||||
return completions
|
||||
|
||||
def _status(self, task_id: str) -> str:
|
||||
task = kanban_db.get_task(self.connection, task_id)
|
||||
self.assertIsNotNone(task)
|
||||
return task.status
|
||||
|
||||
def _route_executed_task_to_triage(self) -> str:
|
||||
task_id = kanban_db.create_task(
|
||||
self.connection,
|
||||
title="already implemented parent",
|
||||
)
|
||||
for attempt in range(kanban_db.BLOCK_RECURRENCE_LIMIT):
|
||||
if attempt:
|
||||
self.assertTrue(kanban_db.unblock_task(self.connection, task_id))
|
||||
self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
|
||||
self.assertTrue(
|
||||
kanban_db.block_task(
|
||||
self.connection,
|
||||
task_id,
|
||||
reason="same unavailable capability",
|
||||
kind="capability",
|
||||
)
|
||||
)
|
||||
self.assertEqual(self._status(task_id), "triage")
|
||||
self.assertEqual(
|
||||
len(kanban_db.list_runs(self.connection, task_id)),
|
||||
kanban_db.BLOCK_RECURRENCE_LIMIT,
|
||||
)
|
||||
return task_id
|
||||
|
||||
def _insert_run(self, task_id: str) -> None:
|
||||
with kanban_db.connect_closing() as connection:
|
||||
kanban_db._synthesize_ended_run(
|
||||
connection,
|
||||
task_id,
|
||||
outcome="reclaimed",
|
||||
summary="concurrent execution evidence",
|
||||
)
|
||||
120
dockerfiles/hermes_lane_compatibility_regression.py
Normal file
120
dockerfiles/hermes_lane_compatibility_regression.py
Normal file
@ -0,0 +1,120 @@
|
||||
"""Exercise mounted CLI scripts against the exact image Kanban API."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
|
||||
_HOME = tempfile.TemporaryDirectory(prefix="hermes-lane-compatibility-")
|
||||
os.environ["HERMES_HOME"] = _HOME.name
|
||||
os.environ["HERMES_CLI_LANE_CONCURRENCY"] = "1"
|
||||
lane_source = Path(os.environ["HERMES_CLI_LANE_SOURCE"])
|
||||
sys.path.insert(0, str(lane_source))
|
||||
|
||||
from cli_lane_capabilities import ( # noqa: E402
|
||||
detect_kanban_capabilities,
|
||||
initialize_kanban_capabilities,
|
||||
runtime_health,
|
||||
)
|
||||
from cli_lane_config import TerminalIdentity # noqa: E402
|
||||
from cli_lane_dispatch import recover_orphans # noqa: E402
|
||||
from cli_lane_finalization import _finalize_document_db # noqa: E402
|
||||
from hermes_cli import kanban_db # noqa: E402
|
||||
|
||||
|
||||
class MountedLaneCompatibilityTests(unittest.TestCase):
|
||||
"""Verify safe behavior before and after the image API patch."""
|
||||
|
||||
def setUp(self) -> None:
|
||||
self.connection = kanban_db.connect()
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.connection.close()
|
||||
|
||||
@staticmethod
|
||||
def _document(summary: str) -> dict:
|
||||
return {
|
||||
"result": json.dumps(
|
||||
{
|
||||
"status": "completed",
|
||||
"summary": summary,
|
||||
"changed_files": [],
|
||||
"tests_run": ["mixed source compatibility"],
|
||||
"artifacts": [],
|
||||
"findings": [],
|
||||
"blockers": [],
|
||||
},
|
||||
sort_keys=True,
|
||||
),
|
||||
"summary": summary,
|
||||
"metadata": {"matrix": os.environ["HERMES_COMPATIBILITY_MODE"]},
|
||||
}
|
||||
|
||||
def test_exact_image_api_matrix(self) -> None:
|
||||
mode = os.environ["HERMES_COMPATIBILITY_MODE"]
|
||||
capabilities = initialize_kanban_capabilities(kanban_db)
|
||||
self.assertTrue(capabilities.exact_run_completion)
|
||||
self.assertEqual(capabilities.ready, mode == "patched")
|
||||
self.assertEqual(runtime_health()["state"], "ready" if mode == "patched" else "deferred")
|
||||
self.assertLess(Path(_HOME.name, "cli-lanes/runtime-health.json").stat().st_size, 2048)
|
||||
|
||||
def test_current_active_run_completion_stays_safe(self) -> None:
|
||||
task_id = kanban_db.create_task(self.connection, title="active accepted result")
|
||||
self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
|
||||
run_id = kanban_db.get_task(self.connection, task_id).current_run_id
|
||||
outcome = _finalize_document_db(
|
||||
kanban_db,
|
||||
TerminalIdentity("default", task_id, run_id, "pending"),
|
||||
self._document("active result"),
|
||||
)
|
||||
self.assertEqual(outcome, "committed")
|
||||
self.assertEqual(kanban_db.get_task(self.connection, task_id).status, "done")
|
||||
|
||||
def test_ended_run_replay_is_enabled_only_after_patch(self) -> None:
|
||||
mode = os.environ["HERMES_COMPATIBILITY_MODE"]
|
||||
task_id = kanban_db.create_task(self.connection, title="ended accepted result")
|
||||
self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
|
||||
run_id = kanban_db.get_task(self.connection, task_id).current_run_id
|
||||
self.assertTrue(
|
||||
kanban_db.block_task(
|
||||
self.connection,
|
||||
task_id,
|
||||
reason="simulate post-journal error",
|
||||
expected_run_id=run_id,
|
||||
)
|
||||
)
|
||||
outcome = _finalize_document_db(
|
||||
kanban_db,
|
||||
TerminalIdentity("default", task_id, run_id, "pending"),
|
||||
self._document("ended result"),
|
||||
)
|
||||
self.assertEqual(outcome, "committed" if mode == "patched" else "deferred")
|
||||
|
||||
def test_old_reclaim_api_is_never_called_unguarded(self) -> None:
|
||||
capabilities = detect_kanban_capabilities(kanban_db)
|
||||
if capabilities.exact_run_reclaim:
|
||||
self.skipTest("patched API exercises guarded reclaim in the exact-run suite")
|
||||
with mock.patch.object(
|
||||
kanban_db,
|
||||
"reclaim_task",
|
||||
side_effect=AssertionError("legacy unguarded reclaim called"),
|
||||
), mock.patch("cli_lane_dispatch.recover_pending_finalizations", return_value=0):
|
||||
recover_orphans()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Run the compatibility matrix for the selected exact image source."""
|
||||
result = unittest.TextTestRunner(verbosity=2).run(
|
||||
unittest.defaultTestLoader.loadTestsFromTestCase(MountedLaneCompatibilityTests)
|
||||
)
|
||||
return 0 if result.wasSuccessful() else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
467
dockerfiles/hermes_run_safety_regression.py
Normal file
467
dockerfiles/hermes_run_safety_regression.py
Normal file
@ -0,0 +1,467 @@
|
||||
"""Regress exact-run replay, provenance, and reclaim safety."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from unittest import mock
|
||||
|
||||
from hermes_cli import kanban_db
|
||||
from hermes_execution_regression_support import (
|
||||
ExecutionSafetyTestCase,
|
||||
TerminalIdentity,
|
||||
_finalize_document_db,
|
||||
)
|
||||
from cli_lane_files import state_path
|
||||
from cli_lane_records import _write_terminal_record
|
||||
from cli_lane_recovery import recover_pending_finalizations
|
||||
|
||||
|
||||
class ExactRunSafetyTests(ExecutionSafetyTestCase):
|
||||
"""Verify exact-run transactions reject every stale concurrent actor."""
|
||||
|
||||
def _lane_document(self, summary: str) -> dict:
|
||||
"""Return the DB-bearing portion of one accepted lane journal."""
|
||||
return {
|
||||
"result": summary,
|
||||
"summary": summary,
|
||||
"metadata": {"source": "exact-run-regression"},
|
||||
}
|
||||
|
||||
def test_lane_marks_replaced_ended_run_stale_without_mutation(self) -> None:
|
||||
task_id = kanban_db.create_task(self.connection, title="replaced journal")
|
||||
self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
|
||||
first_run = kanban_db.get_task(self.connection, task_id).current_run_id
|
||||
self.assertTrue(
|
||||
kanban_db.block_task(
|
||||
self.connection,
|
||||
task_id,
|
||||
reason="first run ended after persistence",
|
||||
expected_run_id=first_run,
|
||||
)
|
||||
)
|
||||
self.assertTrue(kanban_db.unblock_task(self.connection, task_id))
|
||||
self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
|
||||
second_run = kanban_db.get_task(self.connection, task_id).current_run_id
|
||||
self.assertTrue(
|
||||
kanban_db.block_task(
|
||||
self.connection,
|
||||
task_id,
|
||||
reason="replacement run also ended",
|
||||
expected_run_id=second_run,
|
||||
)
|
||||
)
|
||||
before = kanban_db.get_task(self.connection, task_id)
|
||||
|
||||
outcome = _finalize_document_db(
|
||||
kanban_db,
|
||||
TerminalIdentity("default", task_id, first_run, "pending"),
|
||||
self._lane_document("stale first result"),
|
||||
)
|
||||
|
||||
after = kanban_db.get_task(self.connection, task_id)
|
||||
self.assertEqual(outcome, "stale")
|
||||
self.assertEqual(after.status, before.status)
|
||||
self.assertEqual(after.current_run_id, before.current_run_id)
|
||||
self.assertEqual(after.result, before.result)
|
||||
self.assertEqual(after.completed_run_id, before.completed_run_id)
|
||||
|
||||
def test_lane_marks_nonexistent_ended_run_stale_without_mutation(self) -> None:
|
||||
task_id = kanban_db.create_task(self.connection, title="no such run")
|
||||
before = kanban_db.get_task(self.connection, task_id)
|
||||
|
||||
outcome = _finalize_document_db(
|
||||
kanban_db,
|
||||
TerminalIdentity("default", task_id, 42, "pending"),
|
||||
self._lane_document("orphan result"),
|
||||
)
|
||||
|
||||
after = kanban_db.get_task(self.connection, task_id)
|
||||
self.assertEqual(outcome, "stale")
|
||||
self.assertEqual(after.status, before.status)
|
||||
self.assertEqual(after.result, before.result)
|
||||
self.assertEqual(kanban_db.list_runs(self.connection, task_id), [])
|
||||
|
||||
def test_full_lane_recovery_conflicts_a_replaced_ended_run(self) -> None:
|
||||
task_id = kanban_db.create_task(self.connection, title="recovery conflict")
|
||||
self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
|
||||
first_run = kanban_db.get_task(self.connection, task_id).current_run_id
|
||||
self.assertTrue(
|
||||
kanban_db.block_task(
|
||||
self.connection,
|
||||
task_id,
|
||||
reason="first accepted run ended",
|
||||
expected_run_id=first_run,
|
||||
)
|
||||
)
|
||||
self.assertTrue(kanban_db.unblock_task(self.connection, task_id))
|
||||
self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
|
||||
second_run = kanban_db.get_task(self.connection, task_id).current_run_id
|
||||
self.assertTrue(
|
||||
kanban_db.block_task(
|
||||
self.connection,
|
||||
task_id,
|
||||
reason="replacement ended",
|
||||
expected_run_id=second_run,
|
||||
)
|
||||
)
|
||||
before = kanban_db.get_task(self.connection, task_id)
|
||||
state_file = state_path("default", task_id)
|
||||
pending, _record = _write_terminal_record(
|
||||
state_file,
|
||||
board="default",
|
||||
task_id=task_id,
|
||||
run_id=first_run,
|
||||
structured={
|
||||
"status": "completed",
|
||||
"summary": "accepted stale result",
|
||||
"changed_files": [],
|
||||
"tests_run": ["real SQLite recovery regression"],
|
||||
"artifacts": [],
|
||||
"findings": [],
|
||||
"blockers": [],
|
||||
},
|
||||
summary="accepted stale result",
|
||||
metadata={"source": "exact-run-regression"},
|
||||
)
|
||||
|
||||
self.assertEqual(recover_pending_finalizations(), 0)
|
||||
|
||||
after = kanban_db.get_task(self.connection, task_id)
|
||||
self.assertEqual(after.status, before.status)
|
||||
self.assertEqual(after.current_run_id, before.current_run_id)
|
||||
self.assertEqual(after.result, before.result)
|
||||
self.assertFalse(pending.exists())
|
||||
conflicts = list(
|
||||
state_file.parent.glob(
|
||||
f"{state_file.stem}.run-{first_run}.terminal.conflict-*.json"
|
||||
)
|
||||
)
|
||||
self.assertEqual(len(conflicts), 1)
|
||||
|
||||
def test_latest_ended_run_can_replay_a_durable_completion(self) -> None:
|
||||
task_id = kanban_db.create_task(self.connection, title="journaled result")
|
||||
self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
|
||||
run_id = kanban_db.get_task(self.connection, task_id).current_run_id
|
||||
self.assertTrue(
|
||||
kanban_db.block_task(
|
||||
self.connection,
|
||||
task_id,
|
||||
reason="legacy post-journal failure",
|
||||
kind="capability",
|
||||
expected_run_id=run_id,
|
||||
)
|
||||
)
|
||||
|
||||
with mock.patch.object(
|
||||
kanban_db, "_fire_kanban_lifecycle_hook"
|
||||
) as lifecycle:
|
||||
self.assertTrue(
|
||||
kanban_db.complete_task(
|
||||
self.connection,
|
||||
task_id,
|
||||
result="durable terminal result",
|
||||
summary="durable terminal result",
|
||||
replay_ended_run_id=run_id,
|
||||
)
|
||||
)
|
||||
lifecycle.assert_called_once()
|
||||
self.assertEqual(lifecycle.call_args.args[:2], ("kanban_task_completed", task_id))
|
||||
self.assertEqual(lifecycle.call_args.kwargs["run_id"], run_id)
|
||||
task = kanban_db.get_task(self.connection, task_id)
|
||||
self.assertEqual(task.status, "done")
|
||||
self.assertEqual(task.completed_run_id, run_id)
|
||||
runs = kanban_db.list_runs(self.connection, task_id)
|
||||
self.assertEqual(len(runs), 1)
|
||||
self.assertEqual(runs[0].id, run_id)
|
||||
self.assertEqual(runs[0].status, "done")
|
||||
self.assertEqual(runs[0].outcome, "completed")
|
||||
self.assertEqual(runs[0].summary, "durable terminal result")
|
||||
completed_events = self.connection.execute(
|
||||
"SELECT run_id FROM task_events "
|
||||
"WHERE task_id = ? AND kind = 'completed' ORDER BY id",
|
||||
(task_id,),
|
||||
).fetchall()
|
||||
self.assertEqual([row["run_id"] for row in completed_events], [run_id])
|
||||
|
||||
def test_unchanged_triage_can_replay_its_exact_latest_ended_run(self) -> None:
|
||||
task_id = self._route_executed_task_to_triage()
|
||||
runs = kanban_db.list_runs(self.connection, task_id)
|
||||
run_id = runs[-1].id
|
||||
|
||||
self.assertTrue(
|
||||
kanban_db.complete_task(
|
||||
self.connection,
|
||||
task_id,
|
||||
result="accepted before the loop breaker routed triage",
|
||||
summary="accepted before the loop breaker routed triage",
|
||||
replay_ended_run_id=run_id,
|
||||
)
|
||||
)
|
||||
|
||||
task = kanban_db.get_task(self.connection, task_id)
|
||||
self.assertEqual(task.status, "done")
|
||||
self.assertEqual(task.completed_run_id, run_id)
|
||||
updated_runs = kanban_db.list_runs(self.connection, task_id)
|
||||
self.assertEqual(len(updated_runs), kanban_db.BLOCK_RECURRENCE_LIMIT)
|
||||
self.assertEqual(updated_runs[-1].id, run_id)
|
||||
self.assertEqual(updated_runs[-1].outcome, "completed")
|
||||
|
||||
def test_triage_status_transition_prevents_stale_exact_run_replay(self) -> None:
|
||||
task_id = self._route_executed_task_to_triage()
|
||||
run_id = kanban_db.list_runs(self.connection, task_id)[-1].id
|
||||
self.assertTrue(
|
||||
kanban_db.specify_triage_task(
|
||||
self.connection,
|
||||
task_id,
|
||||
title="operator changed the objective",
|
||||
author="operator",
|
||||
)
|
||||
)
|
||||
|
||||
self.assertFalse(
|
||||
kanban_db.complete_task(
|
||||
self.connection,
|
||||
task_id,
|
||||
result="stale accepted result",
|
||||
replay_ended_run_id=run_id,
|
||||
)
|
||||
)
|
||||
self.assertNotEqual(self._status(task_id), "done")
|
||||
|
||||
def test_decomposition_lineage_prevents_replay_even_if_status_returns(self) -> None:
|
||||
task_id = self._route_executed_task_to_triage()
|
||||
run_id = kanban_db.list_runs(self.connection, task_id)[-1].id
|
||||
children = kanban_db.decompose_triage_task(
|
||||
self.connection,
|
||||
task_id,
|
||||
root_assignee="default",
|
||||
children=[{"title": "new child objective"}],
|
||||
author="operator",
|
||||
auto_promote=False,
|
||||
)
|
||||
self.assertEqual(len(children or []), 1)
|
||||
with kanban_db.write_txn(self.connection):
|
||||
self.connection.execute(
|
||||
"UPDATE tasks SET status = 'ready' WHERE id = ?",
|
||||
(task_id,),
|
||||
)
|
||||
|
||||
self.assertFalse(
|
||||
kanban_db.complete_task(
|
||||
self.connection,
|
||||
task_id,
|
||||
result="pre-decomposition accepted result",
|
||||
replay_ended_run_id=run_id,
|
||||
)
|
||||
)
|
||||
self.assertEqual(self._status(task_id), "ready")
|
||||
|
||||
def test_older_ended_run_cannot_complete_over_a_replacement(self) -> None:
|
||||
task_id = kanban_db.create_task(self.connection, title="replacement guard")
|
||||
self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
|
||||
old_run = kanban_db.get_task(self.connection, task_id).current_run_id
|
||||
self.assertTrue(
|
||||
kanban_db.block_task(
|
||||
self.connection,
|
||||
task_id,
|
||||
reason="first run",
|
||||
kind="capability",
|
||||
expected_run_id=old_run,
|
||||
)
|
||||
)
|
||||
self.assertTrue(kanban_db.unblock_task(self.connection, task_id))
|
||||
self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
|
||||
replacement_run = kanban_db.get_task(self.connection, task_id).current_run_id
|
||||
self.assertNotEqual(old_run, replacement_run)
|
||||
self.assertTrue(
|
||||
kanban_db.block_task(
|
||||
self.connection,
|
||||
task_id,
|
||||
reason="replacement run",
|
||||
kind="capability",
|
||||
expected_run_id=replacement_run,
|
||||
)
|
||||
)
|
||||
|
||||
self.assertFalse(
|
||||
kanban_db.complete_task(
|
||||
self.connection,
|
||||
task_id,
|
||||
result="stale result",
|
||||
replay_ended_run_id=old_run,
|
||||
)
|
||||
)
|
||||
self.assertIn(self._status(task_id), {"blocked", "triage"})
|
||||
|
||||
def test_equal_result_bytes_remain_bound_to_the_completing_run(self) -> None:
|
||||
task_id = kanban_db.create_task(self.connection, title="same result retries")
|
||||
self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
|
||||
first_run = kanban_db.get_task(self.connection, task_id).current_run_id
|
||||
self.assertTrue(
|
||||
kanban_db.block_task(
|
||||
self.connection,
|
||||
task_id,
|
||||
reason="first attempt ended",
|
||||
expected_run_id=first_run,
|
||||
)
|
||||
)
|
||||
self.assertTrue(kanban_db.unblock_task(self.connection, task_id))
|
||||
self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
|
||||
second_run = kanban_db.get_task(self.connection, task_id).current_run_id
|
||||
self.assertTrue(
|
||||
kanban_db.complete_task(
|
||||
self.connection,
|
||||
task_id,
|
||||
result="identical result bytes",
|
||||
summary="second run wins",
|
||||
expected_run_id=second_run,
|
||||
)
|
||||
)
|
||||
|
||||
task = kanban_db.get_task(self.connection, task_id)
|
||||
self.assertEqual(task.completed_run_id, second_run)
|
||||
self.assertNotEqual(task.completed_run_id, first_run)
|
||||
runs = {run.id: run for run in kanban_db.list_runs(self.connection, task_id)}
|
||||
self.assertEqual(runs[first_run].outcome, "blocked")
|
||||
self.assertEqual(runs[second_run].outcome, "completed")
|
||||
|
||||
def test_exact_run_reclaim_succeeds_for_the_authoritative_run(self) -> None:
|
||||
task_id = kanban_db.create_task(self.connection, title="recover exact run")
|
||||
self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
|
||||
run_id = kanban_db.get_task(self.connection, task_id).current_run_id
|
||||
|
||||
self.assertTrue(
|
||||
kanban_db.reclaim_task(
|
||||
self.connection,
|
||||
task_id,
|
||||
reason="invalid exact journal",
|
||||
expected_run_id=run_id,
|
||||
)
|
||||
)
|
||||
task = kanban_db.get_task(self.connection, task_id)
|
||||
self.assertEqual(task.status, "ready")
|
||||
self.assertIsNone(task.current_run_id)
|
||||
|
||||
def test_stale_reclaim_before_transaction_preserves_replacement_run(self) -> None:
|
||||
task_id = kanban_db.create_task(self.connection, title="replacement before txn")
|
||||
self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
|
||||
old_run = kanban_db.get_task(self.connection, task_id).current_run_id
|
||||
self.assertTrue(
|
||||
kanban_db.block_task(
|
||||
self.connection,
|
||||
task_id,
|
||||
reason="old recovery run",
|
||||
expected_run_id=old_run,
|
||||
)
|
||||
)
|
||||
self.assertTrue(kanban_db.unblock_task(self.connection, task_id))
|
||||
self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
|
||||
replacement = kanban_db.get_task(self.connection, task_id)
|
||||
replacement_run = replacement.current_run_id
|
||||
self.assertNotEqual(old_run, replacement_run)
|
||||
signals = []
|
||||
|
||||
self.assertFalse(
|
||||
kanban_db.reclaim_task(
|
||||
self.connection,
|
||||
task_id,
|
||||
reason="stale journal",
|
||||
expected_run_id=old_run,
|
||||
signal_fn=lambda *args: signals.append(args),
|
||||
)
|
||||
)
|
||||
latest = kanban_db.get_task(self.connection, task_id)
|
||||
self.assertEqual(latest.status, "running")
|
||||
self.assertEqual(latest.current_run_id, replacement_run)
|
||||
self.assertEqual(latest.claim_lock, replacement.claim_lock)
|
||||
self.assertEqual(signals, [])
|
||||
|
||||
def test_reclaim_update_guard_preserves_run_changed_inside_transaction(self) -> None:
|
||||
task_id = kanban_db.create_task(self.connection, title="replacement in txn")
|
||||
self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
|
||||
old_run = kanban_db.get_task(self.connection, task_id).current_run_id
|
||||
replacement = {}
|
||||
|
||||
def install_replacement(*_args, **_kwargs):
|
||||
cursor = self.connection.execute(
|
||||
"INSERT INTO task_runs (task_id, status, claim_lock, started_at) "
|
||||
"VALUES (?, 'running', ?, strftime('%s','now'))",
|
||||
(task_id, "replacement-lock"),
|
||||
)
|
||||
replacement["run_id"] = int(cursor.lastrowid)
|
||||
self.connection.execute(
|
||||
"UPDATE tasks SET current_run_id = ?, claim_lock = ? WHERE id = ?",
|
||||
(replacement["run_id"], "replacement-lock", task_id),
|
||||
)
|
||||
return {}
|
||||
|
||||
with mock.patch.object(
|
||||
kanban_db,
|
||||
"_terminate_reclaimed_worker",
|
||||
side_effect=install_replacement,
|
||||
):
|
||||
self.assertFalse(
|
||||
kanban_db.reclaim_task(
|
||||
self.connection,
|
||||
task_id,
|
||||
reason="journal for old run",
|
||||
expected_run_id=old_run,
|
||||
)
|
||||
)
|
||||
|
||||
latest = kanban_db.get_task(self.connection, task_id)
|
||||
self.assertEqual(latest.status, "running")
|
||||
self.assertEqual(latest.current_run_id, replacement["run_id"])
|
||||
self.assertEqual(latest.claim_lock, "replacement-lock")
|
||||
|
||||
def test_concurrent_exact_reclaim_and_completion_have_one_winner(self) -> None:
|
||||
task_id = kanban_db.create_task(self.connection, title="concurrent finalizer")
|
||||
self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
|
||||
run_id = kanban_db.get_task(self.connection, task_id).current_run_id
|
||||
barrier = threading.Barrier(2)
|
||||
outcomes = {}
|
||||
errors = []
|
||||
|
||||
def reclaim() -> None:
|
||||
try:
|
||||
with kanban_db.connect_closing() as connection:
|
||||
barrier.wait()
|
||||
outcomes["reclaim"] = kanban_db.reclaim_task(
|
||||
connection,
|
||||
task_id,
|
||||
reason="concurrent invalid journal",
|
||||
expected_run_id=run_id,
|
||||
)
|
||||
except BaseException as error: # pragma: no cover - assertion relay
|
||||
errors.append(error)
|
||||
|
||||
def finalize() -> None:
|
||||
try:
|
||||
with kanban_db.connect_closing() as connection:
|
||||
barrier.wait()
|
||||
outcomes["complete"] = kanban_db.complete_task(
|
||||
connection,
|
||||
task_id,
|
||||
result="durable winner",
|
||||
summary="durable winner",
|
||||
expected_run_id=run_id,
|
||||
)
|
||||
except BaseException as error: # pragma: no cover - assertion relay
|
||||
errors.append(error)
|
||||
|
||||
threads = [threading.Thread(target=reclaim), threading.Thread(target=finalize)]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
for thread in threads:
|
||||
thread.join(timeout=10)
|
||||
self.assertFalse(thread.is_alive())
|
||||
|
||||
self.assertEqual(errors, [])
|
||||
self.assertEqual(set(outcomes), {"reclaim", "complete"})
|
||||
self.assertEqual(sum(bool(value) for value in outcomes.values()), 1)
|
||||
latest = kanban_db.get_task(self.connection, task_id)
|
||||
self.assertIn(latest.status, {"done", "ready"})
|
||||
self.assertIsNone(latest.current_run_id)
|
||||
runs = kanban_db.list_runs(self.connection, task_id)
|
||||
self.assertEqual(len(runs), 1)
|
||||
self.assertIsNotNone(runs[0].ended_at)
|
||||
6
dockerfiles/patch-hermes-execution-safety.py
Normal file
6
dockerfiles/patch-hermes-execution-safety.py
Normal file
@ -0,0 +1,6 @@
|
||||
"""Apply the composable Hermes execution-safety source patches."""
|
||||
|
||||
# Import order is intentional: the decomposition DB contract builds on the
|
||||
# exact-run schema and transaction changes. Each module fails closed on drift.
|
||||
import patch_hermes_run_safety # noqa: F401
|
||||
import patch_hermes_decomposition_safety # noqa: F401
|
||||
262
dockerfiles/patch_hermes_decomposition_safety.py
Normal file
262
dockerfiles/patch_hermes_decomposition_safety.py
Normal file
@ -0,0 +1,262 @@
|
||||
"""Patch transactional automatic-decomposition execution-history guards."""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from hermes_execution_patch_support import replace_once
|
||||
|
||||
source_root = Path(os.environ.get("HERMES_SOURCE_ROOT", "/opt/hermes"))
|
||||
db_path = source_root / "hermes_cli/kanban_db.py"
|
||||
db = db_path.read_text(encoding="utf-8")
|
||||
|
||||
specify_signature_before = '''def specify_triage_task(
|
||||
conn: sqlite3.Connection,
|
||||
task_id: str,
|
||||
*,
|
||||
title: Optional[str] = None,
|
||||
body: Optional[str] = None,
|
||||
assignee: Optional[str] = None,
|
||||
author: Optional[str] = None,
|
||||
) -> bool:
|
||||
'''
|
||||
specify_signature_after = '''def specify_triage_task(
|
||||
conn: sqlite3.Connection,
|
||||
task_id: str,
|
||||
*,
|
||||
title: Optional[str] = None,
|
||||
body: Optional[str] = None,
|
||||
assignee: Optional[str] = None,
|
||||
author: Optional[str] = None,
|
||||
require_no_runs: bool = False,
|
||||
) -> bool:
|
||||
'''
|
||||
db = replace_once(
|
||||
db,
|
||||
specify_signature_before,
|
||||
specify_signature_after,
|
||||
"transactional specify signature",
|
||||
)
|
||||
|
||||
specify_guard_before = ''' if existing is None:
|
||||
return False
|
||||
sets: list[str] = ["status = 'todo'"]
|
||||
'''
|
||||
specify_guard_after = ''' if existing is None:
|
||||
return False
|
||||
if require_no_runs and conn.execute(
|
||||
"SELECT 1 FROM task_runs WHERE task_id = ? LIMIT 1",
|
||||
(task_id,),
|
||||
).fetchone() is not None:
|
||||
return False
|
||||
sets: list[str] = ["status = 'todo'"]
|
||||
'''
|
||||
db = replace_once(
|
||||
db,
|
||||
specify_guard_before,
|
||||
specify_guard_after,
|
||||
"transactional specify execution-history guard",
|
||||
)
|
||||
|
||||
decompose_signature_before = '''def decompose_triage_task(
|
||||
conn: sqlite3.Connection,
|
||||
task_id: str,
|
||||
*,
|
||||
root_assignee: Optional[str],
|
||||
children: list[dict],
|
||||
author: Optional[str] = None,
|
||||
auto_promote: bool = True,
|
||||
) -> Optional[list[str]]:
|
||||
'''
|
||||
decompose_signature_after = '''def decompose_triage_task(
|
||||
conn: sqlite3.Connection,
|
||||
task_id: str,
|
||||
*,
|
||||
root_assignee: Optional[str],
|
||||
children: list[dict],
|
||||
author: Optional[str] = None,
|
||||
auto_promote: bool = True,
|
||||
require_no_runs: bool = False,
|
||||
) -> Optional[list[str]]:
|
||||
'''
|
||||
db = replace_once(
|
||||
db,
|
||||
decompose_signature_before,
|
||||
decompose_signature_after,
|
||||
"transactional decomposition signature",
|
||||
)
|
||||
|
||||
decompose_guard_before = ''' if root_row["status"] != "triage":
|
||||
return None
|
||||
tenant = root_row["tenant"]
|
||||
'''
|
||||
decompose_guard_after = ''' if root_row["status"] != "triage":
|
||||
return None
|
||||
if require_no_runs and conn.execute(
|
||||
"SELECT 1 FROM task_runs WHERE task_id = ? LIMIT 1",
|
||||
(task_id,),
|
||||
).fetchone() is not None:
|
||||
return None
|
||||
tenant = root_row["tenant"]
|
||||
'''
|
||||
db = replace_once(
|
||||
db,
|
||||
decompose_guard_before,
|
||||
decompose_guard_after,
|
||||
"transactional decomposition execution-history guard",
|
||||
)
|
||||
db_path.write_text(db, encoding="utf-8")
|
||||
|
||||
decompose_path = source_root / "hermes_cli/kanban_decompose.py"
|
||||
decompose = decompose_path.read_text(encoding="utf-8")
|
||||
|
||||
helper_anchor = ''' if chosen not in valid_names:
|
||||
return default_assignee
|
||||
return chosen
|
||||
|
||||
|
||||
'''
|
||||
helper_replacement = helper_anchor + '''def _has_execution_history(task_id: str) -> bool:
|
||||
"""Return whether a task has ever entered a worker run."""
|
||||
with kb.connect_closing() as conn:
|
||||
return bool(kb.list_runs(conn, task_id))
|
||||
|
||||
|
||||
'''
|
||||
decompose = replace_once(
|
||||
decompose,
|
||||
helper_anchor,
|
||||
helper_replacement,
|
||||
"auto-decompose execution-history helper",
|
||||
)
|
||||
|
||||
signature_before = '''def decompose_task(
|
||||
task_id: str,
|
||||
*,
|
||||
author: Optional[str] = None,
|
||||
timeout: Optional[int] = None,
|
||||
) -> DecomposeOutcome:
|
||||
'''
|
||||
signature_after = '''def decompose_task(
|
||||
task_id: str,
|
||||
*,
|
||||
author: Optional[str] = None,
|
||||
timeout: Optional[int] = None,
|
||||
automatic: bool = False,
|
||||
) -> DecomposeOutcome:
|
||||
'''
|
||||
decompose = replace_once(
|
||||
decompose,
|
||||
signature_before,
|
||||
signature_after,
|
||||
"auto-decompose function signature",
|
||||
)
|
||||
|
||||
status_before = ''' if task.status != "triage":
|
||||
return DecomposeOutcome(
|
||||
task_id, False, f"task is not in triage (status={task.status!r})"
|
||||
)
|
||||
|
||||
cfg = _load_config()
|
||||
'''
|
||||
status_after = ''' if task.status != "triage":
|
||||
return DecomposeOutcome(
|
||||
task_id, False, f"task is not in triage (status={task.status!r})"
|
||||
)
|
||||
if automatic and _has_execution_history(task_id):
|
||||
return DecomposeOutcome(
|
||||
task_id,
|
||||
False,
|
||||
"task has execution history and requires deliberate manual triage",
|
||||
)
|
||||
|
||||
cfg = _load_config()
|
||||
'''
|
||||
decompose = replace_once(
|
||||
decompose,
|
||||
status_before,
|
||||
status_after,
|
||||
"auto-decompose preflight guard",
|
||||
)
|
||||
|
||||
single_before = ''' author=audit_author,
|
||||
)
|
||||
if not ok:
|
||||
'''
|
||||
single_after = ''' author=audit_author,
|
||||
require_no_runs=automatic,
|
||||
)
|
||||
if not ok:
|
||||
if automatic and _has_execution_history(task_id):
|
||||
return DecomposeOutcome(
|
||||
task_id,
|
||||
False,
|
||||
"task gained execution history and requires deliberate manual triage",
|
||||
)
|
||||
'''
|
||||
decompose = replace_once(
|
||||
decompose,
|
||||
single_before,
|
||||
single_after,
|
||||
"auto-decompose single-task commit guard",
|
||||
)
|
||||
|
||||
fanout_before = ''' author=audit_author,
|
||||
auto_promote=auto_promote,
|
||||
)
|
||||
'''
|
||||
fanout_after = ''' author=audit_author,
|
||||
auto_promote=auto_promote,
|
||||
require_no_runs=automatic,
|
||||
)
|
||||
'''
|
||||
decompose = replace_once(
|
||||
decompose,
|
||||
fanout_before,
|
||||
fanout_after,
|
||||
"auto-decompose transactional fanout guard",
|
||||
)
|
||||
|
||||
fanout_outcome_before = ''' if child_ids is None:
|
||||
return DecomposeOutcome(
|
||||
task_id, False, "task moved out of triage before decomposition",
|
||||
)
|
||||
'''
|
||||
fanout_outcome_after = ''' if child_ids is None:
|
||||
if automatic and _has_execution_history(task_id):
|
||||
return DecomposeOutcome(
|
||||
task_id,
|
||||
False,
|
||||
"task gained execution history and requires deliberate manual triage",
|
||||
)
|
||||
return DecomposeOutcome(
|
||||
task_id, False, "task moved out of triage before decomposition",
|
||||
)
|
||||
'''
|
||||
decompose = replace_once(
|
||||
decompose,
|
||||
fanout_outcome_before,
|
||||
fanout_outcome_after,
|
||||
"auto-decompose fanout rejection reason",
|
||||
)
|
||||
decompose_path.write_text(decompose, encoding="utf-8")
|
||||
|
||||
|
||||
watcher_path = source_root / "gateway/kanban_watchers.py"
|
||||
watcher = watcher_path.read_text(encoding="utf-8")
|
||||
call_before = ''' outcome = _decomp.decompose_task(
|
||||
tid, author="auto-decomposer",
|
||||
)
|
||||
'''
|
||||
call_after = ''' outcome = _decomp.decompose_task(
|
||||
tid,
|
||||
author="auto-decomposer",
|
||||
automatic=True,
|
||||
)
|
||||
'''
|
||||
watcher = replace_once(
|
||||
watcher,
|
||||
call_before,
|
||||
call_after,
|
||||
"gateway automatic-decomposition call",
|
||||
)
|
||||
watcher_path.write_text(watcher, encoding="utf-8")
|
||||
480
dockerfiles/patch_hermes_run_safety.py
Normal file
480
dockerfiles/patch_hermes_run_safety.py
Normal file
@ -0,0 +1,480 @@
|
||||
"""Patch exact-run completion, provenance, and reclaim transactions."""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from hermes_execution_patch_support import replace_once
|
||||
|
||||
|
||||
|
||||
source_root = Path(os.environ.get("HERMES_SOURCE_ROOT", "/opt/hermes"))
|
||||
db_path = source_root / "hermes_cli/kanban_db.py"
|
||||
db = db_path.read_text(encoding="utf-8")
|
||||
|
||||
complete_signature_before = '''def complete_task(
|
||||
conn: sqlite3.Connection,
|
||||
task_id: str,
|
||||
*,
|
||||
result: Optional[str] = None,
|
||||
summary: Optional[str] = None,
|
||||
metadata: Optional[dict] = None,
|
||||
created_cards: Optional[Iterable[str]] = None,
|
||||
expected_run_id: Optional[int] = None,
|
||||
) -> bool:
|
||||
'''
|
||||
complete_signature_after = '''def complete_task(
|
||||
conn: sqlite3.Connection,
|
||||
task_id: str,
|
||||
*,
|
||||
result: Optional[str] = None,
|
||||
summary: Optional[str] = None,
|
||||
metadata: Optional[dict] = None,
|
||||
created_cards: Optional[Iterable[str]] = None,
|
||||
expected_run_id: Optional[int] = None,
|
||||
replay_ended_run_id: Optional[int] = None,
|
||||
) -> bool:
|
||||
'''
|
||||
db = replace_once(
|
||||
db,
|
||||
complete_signature_before,
|
||||
complete_signature_after,
|
||||
"exact ended-run completion signature",
|
||||
)
|
||||
|
||||
complete_branch_before = ''' with write_txn(conn):
|
||||
if expected_run_id is None:
|
||||
cur = conn.execute(
|
||||
"""
|
||||
UPDATE tasks
|
||||
SET status = 'done',
|
||||
result = ?,
|
||||
completed_at = ?,
|
||||
claim_lock = NULL,
|
||||
claim_expires= NULL,
|
||||
worker_pid = NULL,
|
||||
block_kind = NULL,
|
||||
block_recurrences = 0
|
||||
WHERE id = ?
|
||||
AND status IN ('running', 'ready', 'blocked', 'scheduled')
|
||||
""",
|
||||
(result, now, task_id),
|
||||
)
|
||||
else:
|
||||
'''
|
||||
complete_branch_after = ''' # Journal recovery may arrive after a legacy error path ended the
|
||||
# accepted run. Verify that it is still the latest run in this same
|
||||
# transaction; a replacement attempt must make the replay fail closed.
|
||||
with write_txn(conn):
|
||||
replayed_run_id = None
|
||||
if expected_run_id is not None and replay_ended_run_id is not None:
|
||||
raise ValueError("expected_run_id and replay_ended_run_id are mutually exclusive")
|
||||
if replay_ended_run_id is not None:
|
||||
replayed_run_id = int(replay_ended_run_id)
|
||||
# Replay is permitted only while the exact run-ending transition
|
||||
# remains the latest status mutation. Comments and diagnostics do
|
||||
# not invalidate durable accepted work, but an unblock, explicit
|
||||
# triage specification, decomposition, reclaim, or newer attempt
|
||||
# does. BEGIN IMMEDIATE keeps this lineage check and the guarded
|
||||
# task/run updates in one transaction.
|
||||
last_transition = conn.execute(
|
||||
"""
|
||||
SELECT kind, run_id
|
||||
FROM task_events
|
||||
WHERE task_id = ?
|
||||
AND kind IN (
|
||||
'archived', 'block_loop_detected', 'blocked', 'claimed',
|
||||
'completed', 'decomposed', 'dependency_wait', 'gave_up',
|
||||
'promoted', 'promoted_manual', 'reclaimed', 'scheduled',
|
||||
'specified', 'stale', 'timed_out', 'unblocked'
|
||||
)
|
||||
ORDER BY id DESC
|
||||
LIMIT 1
|
||||
""",
|
||||
(task_id,),
|
||||
).fetchone()
|
||||
if (
|
||||
last_transition is None
|
||||
or last_transition["run_id"] != replayed_run_id
|
||||
or last_transition["kind"] not in {
|
||||
"block_loop_detected",
|
||||
"blocked",
|
||||
"reclaimed",
|
||||
"stale",
|
||||
"timed_out",
|
||||
"gave_up",
|
||||
}
|
||||
):
|
||||
return False
|
||||
cur = conn.execute(
|
||||
"""
|
||||
UPDATE tasks
|
||||
SET status = 'done',
|
||||
result = ?,
|
||||
completed_at = ?,
|
||||
claim_lock = NULL,
|
||||
claim_expires= NULL,
|
||||
worker_pid = NULL,
|
||||
block_kind = NULL,
|
||||
block_recurrences = 0
|
||||
WHERE id = ?
|
||||
AND status IN ('ready', 'blocked', 'triage')
|
||||
AND current_run_id IS NULL
|
||||
AND ? = (
|
||||
SELECT id FROM task_runs
|
||||
WHERE task_id = ?
|
||||
ORDER BY id DESC
|
||||
LIMIT 1
|
||||
)
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM task_runs
|
||||
WHERE id = ?
|
||||
AND task_id = ?
|
||||
AND ended_at IS NOT NULL
|
||||
)
|
||||
""",
|
||||
(
|
||||
result,
|
||||
now,
|
||||
task_id,
|
||||
replayed_run_id,
|
||||
task_id,
|
||||
replayed_run_id,
|
||||
task_id,
|
||||
),
|
||||
)
|
||||
elif expected_run_id is None:
|
||||
cur = conn.execute(
|
||||
"""
|
||||
UPDATE tasks
|
||||
SET status = 'done',
|
||||
result = ?,
|
||||
completed_at = ?,
|
||||
claim_lock = NULL,
|
||||
claim_expires= NULL,
|
||||
worker_pid = NULL,
|
||||
block_kind = NULL,
|
||||
block_recurrences = 0
|
||||
WHERE id = ?
|
||||
AND status IN ('running', 'ready', 'blocked', 'scheduled')
|
||||
""",
|
||||
(result, now, task_id),
|
||||
)
|
||||
else:
|
||||
'''
|
||||
db = replace_once(
|
||||
db,
|
||||
complete_branch_before,
|
||||
complete_branch_after,
|
||||
"transactional exact ended-run completion",
|
||||
)
|
||||
|
||||
run_completion_before = ''' run_id = _end_run(
|
||||
conn, task_id,
|
||||
outcome="completed", status="done",
|
||||
summary=summary if summary is not None else result,
|
||||
metadata=metadata,
|
||||
)
|
||||
# If complete_task was called on a never-claimed task (ready or
|
||||
# blocked → done with no run in flight), synthesize a
|
||||
# zero-duration run so the handoff fields are persisted in
|
||||
# attempt history instead of silently lost.
|
||||
if run_id is None and (summary or metadata or result):
|
||||
run_id = _synthesize_ended_run(
|
||||
conn, task_id,
|
||||
outcome="completed",
|
||||
summary=summary if summary is not None else result,
|
||||
metadata=metadata,
|
||||
)
|
||||
'''
|
||||
run_completion_after = ''' if replayed_run_id is None:
|
||||
run_id = _end_run(
|
||||
conn, task_id,
|
||||
outcome="completed", status="done",
|
||||
summary=summary if summary is not None else result,
|
||||
metadata=metadata,
|
||||
)
|
||||
# If complete_task was called on a never-claimed task (ready or
|
||||
# blocked → done with no run in flight), synthesize a
|
||||
# zero-duration run so the handoff fields are persisted in
|
||||
# attempt history instead of silently lost.
|
||||
if run_id is None and (summary or metadata or result):
|
||||
run_id = _synthesize_ended_run(
|
||||
conn, task_id,
|
||||
outcome="completed",
|
||||
summary=summary if summary is not None else result,
|
||||
metadata=metadata,
|
||||
)
|
||||
else:
|
||||
run_id = replayed_run_id
|
||||
replayed = conn.execute(
|
||||
"""
|
||||
UPDATE task_runs
|
||||
SET status = 'done',
|
||||
outcome = 'completed',
|
||||
summary = ?,
|
||||
error = NULL,
|
||||
metadata = ?,
|
||||
claim_lock = NULL,
|
||||
claim_expires = NULL,
|
||||
worker_pid = NULL
|
||||
WHERE id = ?
|
||||
AND task_id = ?
|
||||
AND ended_at IS NOT NULL
|
||||
""",
|
||||
(
|
||||
summary if summary is not None else result,
|
||||
json.dumps(metadata, ensure_ascii=False) if metadata else None,
|
||||
run_id,
|
||||
task_id,
|
||||
),
|
||||
)
|
||||
if replayed.rowcount != 1:
|
||||
raise RuntimeError("exact ended-run completion lost its run identity")
|
||||
conn.execute(
|
||||
"UPDATE tasks SET completed_run_id = ? WHERE id = ?",
|
||||
(run_id, task_id),
|
||||
)
|
||||
'''
|
||||
db = replace_once(
|
||||
db,
|
||||
run_completion_before,
|
||||
run_completion_after,
|
||||
"exact completion run provenance",
|
||||
)
|
||||
|
||||
reclaim_before = '''def reclaim_task(
|
||||
conn: sqlite3.Connection,
|
||||
task_id: str,
|
||||
*,
|
||||
reason: Optional[str] = None,
|
||||
signal_fn=None,
|
||||
) -> bool:
|
||||
"""Operator-driven reclaim: release the claim and reset to ``ready``.
|
||||
|
||||
Unlike :func:`release_stale_claims` which only acts on tasks whose
|
||||
``claim_expires`` has passed, this function reclaims immediately
|
||||
regardless of TTL. Intended for the dashboard/CLI recovery flow
|
||||
when an operator wants to abort a running worker without waiting
|
||||
for the TTL to expire (e.g. after seeing a hallucination warning).
|
||||
|
||||
Returns True if a reclaim happened, False if the task isn't in a
|
||||
reclaimable state (not running, or doesn't exist).
|
||||
"""
|
||||
row = conn.execute(
|
||||
"SELECT status, claim_lock, worker_pid FROM tasks WHERE id = ?",
|
||||
(task_id,),
|
||||
).fetchone()
|
||||
if not row:
|
||||
return False
|
||||
if row["status"] != "running" and row["claim_lock"] is None:
|
||||
# Nothing to reclaim — already ready / blocked / done.
|
||||
return False
|
||||
prev_lock = row["claim_lock"]
|
||||
termination = _terminate_reclaimed_worker(
|
||||
row["worker_pid"], prev_lock, signal_fn=signal_fn,
|
||||
)
|
||||
with write_txn(conn):
|
||||
cur = conn.execute(
|
||||
"UPDATE tasks SET status = 'ready', claim_lock = NULL, "
|
||||
"claim_expires = NULL, worker_pid = NULL "
|
||||
"WHERE id = ? AND status IN ('running', 'ready', 'blocked') "
|
||||
"AND claim_lock IS ?",
|
||||
(task_id, prev_lock),
|
||||
)
|
||||
if cur.rowcount != 1:
|
||||
return False
|
||||
run_id = _end_run(
|
||||
conn, task_id,
|
||||
outcome="reclaimed", status="reclaimed",
|
||||
error=(
|
||||
f"manual_reclaim: {reason}" if reason
|
||||
else f"manual_reclaim lock={prev_lock}"
|
||||
),
|
||||
metadata=termination,
|
||||
)
|
||||
payload = {
|
||||
"manual": True,
|
||||
"reason": reason,
|
||||
"prev_lock": prev_lock,
|
||||
}
|
||||
payload.update(termination)
|
||||
_append_event(
|
||||
conn, task_id, "reclaimed",
|
||||
payload,
|
||||
run_id=run_id,
|
||||
)
|
||||
# Operator intervention — they've looked at the task, so the
|
||||
# consecutive-failures counter is now stale. Give the next retry
|
||||
# a fresh budget. (_clear_failure_counter opens its own write_txn,
|
||||
# so it runs after the enclosing one commits.)
|
||||
_clear_failure_counter(conn, task_id)
|
||||
return True
|
||||
'''
|
||||
reclaim_after = '''def reclaim_task(
|
||||
conn: sqlite3.Connection,
|
||||
task_id: str,
|
||||
*,
|
||||
reason: Optional[str] = None,
|
||||
signal_fn=None,
|
||||
expected_run_id: Optional[int] = None,
|
||||
) -> bool:
|
||||
"""Operator-driven reclaim: release the claim and reset to ``ready``.
|
||||
|
||||
Unlike :func:`release_stale_claims` which only acts on tasks whose
|
||||
``claim_expires`` has passed, this function reclaims immediately
|
||||
regardless of TTL. Intended for the dashboard/CLI recovery flow
|
||||
when an operator wants to abort a running worker without waiting
|
||||
for the TTL to expire (e.g. after seeing a hallucination warning).
|
||||
|
||||
When ``expected_run_id`` is supplied, the run identity is checked after
|
||||
``BEGIN IMMEDIATE`` and included in the guarded update. A replacement run
|
||||
is therefore neither signalled nor reclaimed by stale recovery evidence.
|
||||
|
||||
Returns True if a reclaim happened, False if the task isn't in a
|
||||
reclaimable state (not running, or doesn't exist), or its run changed.
|
||||
"""
|
||||
with write_txn(conn):
|
||||
row = conn.execute(
|
||||
"SELECT status, claim_lock, worker_pid, current_run_id "
|
||||
"FROM tasks WHERE id = ?",
|
||||
(task_id,),
|
||||
).fetchone()
|
||||
if not row:
|
||||
return False
|
||||
if expected_run_id is not None and row["current_run_id"] != expected_run_id:
|
||||
return False
|
||||
if row["status"] != "running" and row["claim_lock"] is None:
|
||||
# Nothing to reclaim — already ready / blocked / done.
|
||||
return False
|
||||
prev_lock = row["claim_lock"]
|
||||
termination = _terminate_reclaimed_worker(
|
||||
row["worker_pid"], prev_lock, signal_fn=signal_fn,
|
||||
)
|
||||
if expected_run_id is None:
|
||||
cur = conn.execute(
|
||||
"UPDATE tasks SET status = 'ready', claim_lock = NULL, "
|
||||
"claim_expires = NULL, worker_pid = NULL "
|
||||
"WHERE id = ? AND status IN ('running', 'ready', 'blocked') "
|
||||
"AND claim_lock IS ?",
|
||||
(task_id, prev_lock),
|
||||
)
|
||||
else:
|
||||
cur = conn.execute(
|
||||
"UPDATE tasks SET status = 'ready', claim_lock = NULL, "
|
||||
"claim_expires = NULL, worker_pid = NULL "
|
||||
"WHERE id = ? AND status IN ('running', 'ready', 'blocked') "
|
||||
"AND claim_lock IS ? AND current_run_id = ?",
|
||||
(task_id, prev_lock, int(expected_run_id)),
|
||||
)
|
||||
if cur.rowcount != 1:
|
||||
return False
|
||||
run_id = _end_run(
|
||||
conn, task_id,
|
||||
outcome="reclaimed", status="reclaimed",
|
||||
error=(
|
||||
f"manual_reclaim: {reason}" if reason
|
||||
else f"manual_reclaim lock={prev_lock}"
|
||||
),
|
||||
metadata=termination,
|
||||
)
|
||||
payload = {
|
||||
"manual": True,
|
||||
"reason": reason,
|
||||
"prev_lock": prev_lock,
|
||||
}
|
||||
payload.update(termination)
|
||||
_append_event(
|
||||
conn, task_id, "reclaimed",
|
||||
payload,
|
||||
run_id=run_id,
|
||||
)
|
||||
# Operator intervention — they've looked at the task, so the
|
||||
# consecutive-failures counter is now stale. Give the next retry
|
||||
# a fresh budget. (_clear_failure_counter opens its own write_txn,
|
||||
# so it runs after the enclosing one commits.)
|
||||
_clear_failure_counter(conn, task_id)
|
||||
return True
|
||||
'''
|
||||
db = replace_once(
|
||||
db,
|
||||
reclaim_before,
|
||||
reclaim_after,
|
||||
"transactional exact-run reclaim",
|
||||
)
|
||||
|
||||
task_field_before = ''' current_run_id: Optional[int] = None
|
||||
workflow_template_id: Optional[str] = None
|
||||
'''
|
||||
task_field_after = ''' current_run_id: Optional[int] = None
|
||||
# Durable provenance for the run that authoritatively completed this task.
|
||||
completed_run_id: Optional[int] = None
|
||||
workflow_template_id: Optional[str] = None
|
||||
'''
|
||||
db = replace_once(
|
||||
db,
|
||||
task_field_before,
|
||||
task_field_after,
|
||||
"completed-run task field",
|
||||
)
|
||||
|
||||
task_row_before = ''' current_run_id=(
|
||||
row["current_run_id"] if "current_run_id" in keys else None
|
||||
),
|
||||
workflow_template_id=(
|
||||
'''
|
||||
task_row_after = ''' current_run_id=(
|
||||
row["current_run_id"] if "current_run_id" in keys else None
|
||||
),
|
||||
completed_run_id=(
|
||||
row["completed_run_id"] if "completed_run_id" in keys else None
|
||||
),
|
||||
workflow_template_id=(
|
||||
'''
|
||||
db = replace_once(
|
||||
db,
|
||||
task_row_before,
|
||||
task_row_after,
|
||||
"completed-run row mapping",
|
||||
)
|
||||
|
||||
task_schema_before = ''' -- run is in-flight). Denormalised for cheap reads.
|
||||
current_run_id INTEGER,
|
||||
-- Forward-compat for v2 workflow routing. In v1 the kernel writes
|
||||
'''
|
||||
task_schema_after = ''' -- run is in-flight). Denormalised for cheap reads.
|
||||
current_run_id INTEGER,
|
||||
-- Immutable provenance for the task result's authoritative run.
|
||||
completed_run_id INTEGER,
|
||||
-- Forward-compat for v2 workflow routing. In v1 the kernel writes
|
||||
'''
|
||||
db = replace_once(
|
||||
db,
|
||||
task_schema_before,
|
||||
task_schema_after,
|
||||
"completed-run schema",
|
||||
)
|
||||
|
||||
task_migration_before = ''' if "current_run_id" not in cols:
|
||||
_add_column_if_missing(
|
||||
conn, "tasks", "current_run_id", "current_run_id INTEGER"
|
||||
)
|
||||
if "workflow_template_id" not in cols:
|
||||
'''
|
||||
task_migration_after = ''' if "current_run_id" not in cols:
|
||||
_add_column_if_missing(
|
||||
conn, "tasks", "current_run_id", "current_run_id INTEGER"
|
||||
)
|
||||
if "completed_run_id" not in cols:
|
||||
_add_column_if_missing(
|
||||
conn, "tasks", "completed_run_id", "completed_run_id INTEGER"
|
||||
)
|
||||
if "workflow_template_id" not in cols:
|
||||
'''
|
||||
db = replace_once(
|
||||
db,
|
||||
task_migration_before,
|
||||
task_migration_after,
|
||||
"completed-run migration",
|
||||
)
|
||||
|
||||
db_path.write_text(db, encoding="utf-8")
|
||||
341
docs/hermes_full_handoff_acceptance.md
Normal file
341
docs/hermes_full_handoff_acceptance.md
Normal file
@ -0,0 +1,341 @@
|
||||
# Hermes full-handoff acceptance and release runbook
|
||||
|
||||
`scripts/ops/hermes_handoff_acceptance.py` is a fail-closed release gate. It
|
||||
contains 63 checks in 15 groups. An empty catalog, an unknown result status, a
|
||||
mandatory `FAIL` or `NOT_RUN`, a harness error, or incomplete evidence makes the
|
||||
decision `NO_GO`. The four ephemeral checks are non-mandatory `NOT_RUN` in the
|
||||
default mode and are replaced with mandatory results only in armed mode.
|
||||
|
||||
This runbook describes evidence collection. It does not authorize a merge,
|
||||
publish, deployment, Flux reconcile, credential read, or live cluster mutation.
|
||||
Human review of PR #19 and of the resulting report is required.
|
||||
|
||||
## Required release inputs
|
||||
|
||||
Every value that can go stale must be supplied explicitly. Resolve `main`, PR
|
||||
heads, the image digest, build source SHA, deployment revision, and chat config
|
||||
revision from their authoritative read-only sources immediately before the run.
|
||||
Do not copy the snapshot in the dependency section into a later run.
|
||||
|
||||
```sh
|
||||
scripts/ops/hermes_handoff_acceptance.py \
|
||||
--context atlas-operator \
|
||||
--remote-main-sha "$REMOTE_MAIN_SHA" \
|
||||
--reviewed-head-sha "$REVIEWED_PR19_HEAD" \
|
||||
--agent-image "$AGENT_GIT_SHA_BUILD_TAG_AT_SHA256" \
|
||||
--build-sha "$IMAGE_BUILD_SOURCE_SHA" \
|
||||
--deployment-revision "$RUNNING_DEPLOYMENT_REVISION" \
|
||||
--chat-config-revision "$RUNNING_CHAT_CONFIG_REVISION" \
|
||||
--dependency-head "12=$PR12_HEAD" \
|
||||
--dependency-head "14=$PR14_HEAD" \
|
||||
--dependency-head "15=$PR15_HEAD" \
|
||||
--dependency-head "16=$PR16_HEAD" \
|
||||
--dependency-head "17=$PR17_HEAD" \
|
||||
--dependency-head "18=$PR18_HEAD" \
|
||||
--dependency-head "20=$PR20_HEAD" \
|
||||
--output build/hermes-handoff-acceptance.json
|
||||
```
|
||||
|
||||
The fixed topology is three pool workers on three distinct nodes. `--node-count`
|
||||
and `--pool-replicas` cannot lower that requirement. Pool node/ordinal assignment,
|
||||
Telegram session continuity, and the chat revision are mandatory. Deliberate Flux
|
||||
suspensions must be named with repeatable `--expected-suspension` arguments.
|
||||
|
||||
Exit status is `0` only for `GO`, `1` for `NO_GO`, and `2` for invalid execution
|
||||
bounds. JSON is written to `--output` or stdout; the human summary goes to stderr
|
||||
unless `--json-only` is passed.
|
||||
|
||||
## Structural safety boundary
|
||||
|
||||
The catalog is validated before a `Runner` exists. Invalid inputs, an empty or
|
||||
unsafe catalog, and an invalid armed preflight therefore create no subprocess.
|
||||
The command policy is checked both before and after executable resolution.
|
||||
|
||||
Default mode permits a narrow read grammar:
|
||||
|
||||
- Kubernetes reads use `name` or reviewed JSONPath projections. Secret resources,
|
||||
raw API reads, full JSON/YAML, `describe`, `cluster-info dump`, credential-bearing
|
||||
projections, `auth reconcile`, config operations, all mutations, and every
|
||||
dry-run mutation trick are rejected before spawn.
|
||||
- Git is read-only. `fetch`, configuration/helper overrides, alternate worktrees,
|
||||
external diff helpers, and push are rejected.
|
||||
- Gitea paths are pinned to `atlas/titan-iac` on an exact segment boundary, so a
|
||||
look-alike repository such as `atlas/titan-iac-evil` is refused, and any
|
||||
relative segment — including a percent-encoded `%2e%2e` — is rejected before
|
||||
the request is built.
|
||||
- Shell commands must be rendered from frozen templates, use fixed executable
|
||||
paths, and cannot name service-account, Vault, runtime-access, SSH, or other
|
||||
credential roots.
|
||||
- Impersonation is structurally impossible. `--as`, `--as-group`, and `--as-uid`
|
||||
are refused for every argv, in every mode, from every vantage, and the inner
|
||||
command of a `kubectl exec` is re-checked under the same rule rather than being
|
||||
granted an exemption. `validate_catalog` refuses an impersonating step from any
|
||||
vantage, not only the operator. A future audited self-probe that genuinely needs
|
||||
impersonation is a reviewed code change, not a configuration flag.
|
||||
- The binary allowlist is exactly the set with pinned release digests:
|
||||
`kubectl`, `git`, `sh`, `hermes`, and the Gitea client. `flux` and `helm` are
|
||||
not on it; Flux and Helm status is read through `kubectl` instead, so no
|
||||
allowlisted binary can be admitted that the executor would refuse to attest.
|
||||
- `PATH` is replaced with a fixed path. Every outer executable is resolved to an
|
||||
allowed real path, checked for unsafe write mode, and compared with a pinned
|
||||
SHA-256 digest. A caller-authored `kubectl`, `git`, or helper cannot fabricate
|
||||
evidence.
|
||||
|
||||
The executor uses a new process group, drains stdout and stderr concurrently,
|
||||
and retains at most the configured bytes while the child is running. A monotonic
|
||||
absolute timeout terminates and reaps the group, including descendants that keep
|
||||
pipes open. Timeout, deadline, and output-byte values reject booleans,
|
||||
zero/negative values, non-finite numbers, and values outside fixed bounds. There
|
||||
is no concurrency knob: `run_catalog` is strictly sequential and says so rather
|
||||
than accepting a bound it would ignore.
|
||||
Truncation discards the affected stream and makes every evaluator `NOT_RUN`.
|
||||
Bulk-evidence steps have their bytes replaced before they are recorded, but they
|
||||
keep the resolved executable path and its attested SHA-256, so withholding output
|
||||
never withholds the provenance of the binary that produced it.
|
||||
|
||||
Credential screening runs before truncation. Command, output, error, identity,
|
||||
reason, evidence, JSON, and human-summary fields are screened recursively,
|
||||
including mapping keys and short/numeric sensitive values. The complete rendered
|
||||
report is screened again; a remaining credential shape is replaced by a bounded
|
||||
`NO_GO` stub rather than being written. No credential prefix is retained at a
|
||||
truncation boundary.
|
||||
|
||||
## Vantages and evidence
|
||||
|
||||
The operator must use an external read-only kubeconfig. The self probe runs in
|
||||
the actual Hermes agent pod. Both identities are obtained with
|
||||
`kubectl auth whoami`; equal or missing principals make the run `NO_GO`. Required
|
||||
Switchyard and chat/Telegram pod vantages must also resolve. Node coverage is
|
||||
derived from all daemonset pod node names, not one selected pod.
|
||||
|
||||
Dangerous Kubernetes permissions are checked only with side-effect-free
|
||||
`kubectl auth can-i` reviews. The default catalog never constructs a Secret,
|
||||
TokenRequest, mutation, attach, port-forward, or arbitrary exec request. The one
|
||||
forge administrative denial check performs a read-only GET and accepts only an
|
||||
explicit authorization refusal; generic `404`, `not found`, and `no route` text
|
||||
are not denial evidence.
|
||||
|
||||
An absence is only evidence when the probe observed something. Both name rules
|
||||
resolve their step through one guard, so a step that exits `0` with no lines — or
|
||||
with only separators — is `NOT_RUN`, never `PASS`. This is reachable in practice
|
||||
two ways, and both are pinned by regressions: a POSIX pipeline reports the status
|
||||
of its *last* stage, so a missing upstream binary in the frozen `env_names`
|
||||
template exits `0` with no bytes; and `kubectl -o jsonpath` exits `0` when the
|
||||
structure it walks has drifted, whether a parent field, a leaf field, or a
|
||||
container filter. Because all five absence checks are mandatory, silence costs
|
||||
the release its `GO`. The pool claim projection emits one `<volume>=<claim>` line
|
||||
per template volume so a volume without a PVC still counts as an observation.
|
||||
|
||||
The strict evaluators reject duplicate-key JSON, raw control characters,
|
||||
`NaN`/infinity, boolean/number equality coercion, future timestamps, stale
|
||||
provider evidence, and every malformed routing or Flux row. Routing freshness is
|
||||
required separately for each provider, lane, and effort. Flux objects must be
|
||||
non-suspended, current at `observedGeneration`, and explicitly `Ready=True`;
|
||||
unknown or reconciling state is not healthy.
|
||||
|
||||
`release.exact-lineage-is-running` atomically binds:
|
||||
|
||||
- fixed remote `main` SHA and local `origin/main`;
|
||||
- open draft PR #19, base `main`, existing feature ref, and exact reviewed head;
|
||||
- the exact `:git-<source-SHA>-build-<n>@sha256:<digest>` image identity,
|
||||
with the source SHA equal to the release `main` SHA;
|
||||
- current Deployment generation and revision;
|
||||
- the one active ReplicaSet revision/hash; every Ready pod image and imageID;
|
||||
- the Flux-applied `main` revision.
|
||||
|
||||
The pool evaluator requires exactly ordinals 0, 1, and 2, all Ready, on three
|
||||
distinct nodes and claims, with service-account token automount disabled. The
|
||||
chat checks always require durable Telegram sessions and the exact running chat
|
||||
configuration revision.
|
||||
|
||||
## Armed ephemeral mode
|
||||
|
||||
Armed mode is the only write path and runs only after the complete default
|
||||
catalog reports `GO`.
|
||||
|
||||
```sh
|
||||
scripts/ops/hermes_handoff_acceptance.py \
|
||||
<all required release inputs above> \
|
||||
--arm-ephemeral-push \
|
||||
--confirm "ARM EPHEMERAL HERMES HANDOFF PUSH" \
|
||||
--ephemeral-token acceptance-20260817a \
|
||||
--output build/hermes-handoff-acceptance-armed.json
|
||||
```
|
||||
|
||||
Before any runner, network request, or cluster vantage is created, preflight
|
||||
binds the working directory to the existing linked PR #19 worktree, the exact
|
||||
feature branch and reviewed local HEAD, the fixed HTTPS Atlas origin, base
|
||||
`main`, confirmation phrase, and one unique ephemeral ref. The command policy is
|
||||
then pinned to that exact ref. It allows only its push, one exact draft-create
|
||||
payload, cleanup of the exactly discovered PR number, and deletion of that ref.
|
||||
|
||||
The run reserves at least 300 seconds (up to 6000) for cleanup. It verifies the
|
||||
pushed SHA, then identifies the pull request from two independent sources: the
|
||||
create response itself, and the pull index read page by page to its last page
|
||||
(50 per page, 20 pages maximum) rather than a single truncated first page. Either
|
||||
source alone names a pull request this run definitely made, because preflight
|
||||
proved the ephemeral ref did not exist. Every number found from either source is
|
||||
registered and closed, so a failed or unusable index can no longer leave an
|
||||
orphan open. A `GO` still requires exactly one open draft with the expected base,
|
||||
head ref, base SHA, and head SHA, and the two sources must agree.
|
||||
|
||||
Cleanup requires successful PR close, successful branch deletion, a successful
|
||||
empty `ls-remote`, and a strict re-read of the same closed PR. Malformed or
|
||||
partial cleanup is `FAIL`; it is never inferred from an error. When anything is
|
||||
uncertain the `draft-pull-request` evidence carries `residue_ref` and an exact
|
||||
`manual_cleanup` command list — the `PATCH .../pulls/<n> --field state=closed`
|
||||
calls for every number found, the `DELETE .../branches/<ref>` call, and, when no
|
||||
number could be determined at all, the exact ref to sweep the index for by hand.
|
||||
|
||||
Force, force-with-lease, fan-out, protected refs, other repositories, other PR
|
||||
numbers, arbitrary PATCH payloads, and repository administration remain denied.
|
||||
|
||||
## Dependency, merge, and conflict graph
|
||||
|
||||
Authoritative snapshot observed 2026-08-17 after refreshing Gitea and Git against
|
||||
remote `main` `d8f2d818b9a552ea6c2d7fe86554be829bd5ffff`:
|
||||
|
||||
| PR | Head SHA | Draft | Base | Role |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| #12 | `ed43dbaa7f094e1841aa760862e80015deb9d80c` | no | `main` | AI usage exporter repair |
|
||||
| #14 | `8c6e3acdacb223f68b1716f0a74bd543e18890e6` | yes | `main` | bounded forge client and least-privilege RBAC |
|
||||
| #15 | `73fefbb5d92ef07371a9a997f22d848acd2d06ed` | yes | `main` | result-decomposition reliability |
|
||||
| #16 | `e8c26ecf85a0c802d378fffffb98b7df43b78242` | no | `main` | unprivileged image builder |
|
||||
| #17 | `48cbe13ee50ce3fcb07cea3fe8d088cfed349e0c` | yes | `main` | three-lane placement |
|
||||
| #18 | `689bcb6e48414854447988ba174143a307d308b9` | yes | `main` | distributed three-worker pool |
|
||||
| #19 | `c808baff40a12a90f43c2e5e39b94d139f88ea29` | yes | `main` | head this repair replaces on the same branch |
|
||||
| #20 | `a242dcc786576ae1a18000cb4941836ff610dff2` | yes | `main` | CLI process-group reaping |
|
||||
|
||||
The repaired #19 SHA cannot be embedded in its own commit without changing that
|
||||
SHA. Resolve it from Gitea after push and require it as `--reviewed-head-sha`;
|
||||
the PR handoff records the exact value. All eight heads in this table, including
|
||||
the repaired #19, merge cleanly with `main` individually. Pairwise
|
||||
`git merge-tree --write-tree` over all 28 combinations found conflicts for:
|
||||
#12+#17, #12+#18, #12+#19, #14+#15, #14+#16, #14+#17, #14+#19, #14+#20, #15+#16,
|
||||
#15+#17, #15+#19, #16+#19, #16+#20, #17+#18, and #17+#20. Gitea's
|
||||
`mergeable=false` on a draft is not treated as conflict evidence.
|
||||
|
||||
This graph differs from the previous snapshot in exactly two ways. #12 and #18
|
||||
both advanced, which adds #12+#18, #17+#18, and #12+#19. And #19 no longer
|
||||
duplicates the repo-wide hygiene legacy-exception mechanism that #18 also
|
||||
carries, which makes #18+#19 clean. Recompute before every integration step;
|
||||
nothing here is durable.
|
||||
|
||||
Use this dependency order, refreshing every head and re-running merge-tree at
|
||||
each step:
|
||||
|
||||
1. #14 first so the release never restores or prolongs cluster-admin authority.
|
||||
2. #16, then #15, resolving their Dockerfile and quality-contract changes
|
||||
additively. Never take one PR's quality contract wholesale.
|
||||
3. #20 after #15; it depends on the CLI lifecycle work and must be rebased across
|
||||
the current #14/#16 contract and helper changes.
|
||||
4. #12 before #17, resolving their Deployment overlap once.
|
||||
5. #17, then #18, after the authority, image, reliability, and reaping work.
|
||||
6. #19 last as the integration/acceptance change, only after #12/#14-#18/#20 are
|
||||
merged and their exact supplied heads are ancestors of refreshed `main`.
|
||||
|
||||
An exact post-repair merge-tree check reports #19 clean with current `main`, #17,
|
||||
#18, and #20, and conflicting with #12, #14, #15, and #16 on these files:
|
||||
|
||||
| Pair | Overlapping files |
|
||||
| --- | --- |
|
||||
| #12+#19 | `testing/tests/test_hermes_auto_router.py` |
|
||||
| #14+#19 | `testing/quality_contract.json`, `testing/quality_coverage.py`, `testing/tests/test_quality_coverage_helpers.py` |
|
||||
| #15+#19 | `testing/tests/conftest.py` |
|
||||
| #16+#19 | `testing/quality_contract.json` |
|
||||
|
||||
Every one of those overlaps is additive on the #19 side. Resolve the contract by
|
||||
union — take every PR's managed modules, lint paths, tracked files, and branch
|
||||
tracked files, and never take one PR's quality contract wholesale. #19 adds two
|
||||
fixture-isolation fixtures to `conftest.py` and two lines to the auto-router test;
|
||||
keep both sides. Preserve branch coverage, every tracked module, fixture
|
||||
isolation, and all desired-state changes during conflict resolution.
|
||||
|
||||
Recompute this graph from the Gitea API immediately before integration; do not
|
||||
infer conflict status from Gitea's draft `mergeable` field.
|
||||
|
||||
The #16/#19 integration must keep both the immutable digest and the source tag
|
||||
(`newTag: git-<main-SHA>-build-<n>` plus the digest) in rendered image identity.
|
||||
Digest-only rendering loses the attested build-source link and therefore
|
||||
correctly remains `NO_GO` under the exact-lineage evaluator.
|
||||
|
||||
## Safe rollback guidance
|
||||
|
||||
Every rollback is a reviewed forward fix or selective Git revert followed by
|
||||
the normal Flux workflow. Never restore `hermes-agent-cluster-admin`, even when
|
||||
reverting unrelated portions of #14. If a rollback would widen RBAC, stop and
|
||||
write a forward least-privilege fix instead.
|
||||
|
||||
Reverting #18 removes the StatefulSet desired state, but its
|
||||
`volumeClaimTemplates` PVCs are retained by the Kubernetes default because no
|
||||
delete retention policy is set. Do not claim Flux pruning deletes those PVCs.
|
||||
Any PVC deletion is a separate, explicit, human-reviewed data-destruction task
|
||||
with exact targets and recovery implications.
|
||||
|
||||
Image rollback means a reviewed commit restoring a previously attested digest;
|
||||
never retag an immutable image. Do not use direct `kubectl` edits as a durable
|
||||
rollback.
|
||||
|
||||
## Quality and release evidence
|
||||
|
||||
The canonical quality contract explicitly manages, lints, and tracks all 16
|
||||
`scripts/ops/hermes_handoff_*.py` modules. It enforces at least 95% line and
|
||||
branch coverage per tracked file and the 500-physical-line source cap. Contract
|
||||
tests derive the module set so a future file cannot silently escape the gate.
|
||||
This change is purely additive to the contract: it registers its own modules and
|
||||
changes no repo-wide hygiene semantics.
|
||||
|
||||
The gate's `hygiene` step is red on `main` and stays red here. Exactly four
|
||||
pre-existing test files exceed the 500-line cap —
|
||||
`scripts/tests/test_dashboards_render_atlas.py` (516),
|
||||
`testing/tests/test_hermes_chat_quality.py` (2242),
|
||||
`testing/tests/test_hermes_cli_lanes.py` (1971), and
|
||||
`testing/tests/test_hermes_coordinator.py` (510). All four are byte-identical at
|
||||
merge-base `30259b52` and at current `origin/main`, none is touched by this
|
||||
change, and the same four are the only issues a pristine `main` reports. Splitting
|
||||
them, or grandfathering them, is a repo-wide contract decision that belongs to the
|
||||
canonical contract change carried by PR #14/#15, not to this acceptance harness.
|
||||
Every other gate step — docs, smell, unit, coverage — is green, and every file
|
||||
this change adds or edits is under the cap.
|
||||
|
||||
```sh
|
||||
python3 -m testing.quality_gate --profile local --build-dir build
|
||||
python3 -m testing.quality_handoff_mutation
|
||||
python3 -m ruff check --select F,B,SIM,C4,UP --ignore B017,UP015,UP035 \
|
||||
scripts/ops/hermes_handoff_*.py testing/quality_coverage.py \
|
||||
testing/quality_handoff_mutation.py testing/tests/test_hermes_handoff_*.py
|
||||
python3 -m ruff format --check \
|
||||
scripts/ops/hermes_handoff_*.py testing/quality_coverage.py \
|
||||
testing/quality_handoff_mutation.py testing/tests/test_hermes_handoff_*.py
|
||||
python3 -m compileall -q scripts/ops testing
|
||||
kustomize build services/hermes | kubectl apply --dry-run=client --validate=false -f -
|
||||
git diff --check origin/main...HEAD
|
||||
```
|
||||
|
||||
The format scope names the files this change owns. A wider `testing/quality_*.py`
|
||||
glob picks up `testing/quality_gate.py`, which is unmodified here and not
|
||||
format-clean, so the documented command would fail for a reason this change did
|
||||
not cause. `ruff format` is evidence, not a gate step: the canonical gate runs
|
||||
`ruff check` only.
|
||||
|
||||
The deterministic mutation gate must kill every policy, redaction, verdict,
|
||||
numeric, truncation, provenance, and cleanup mutant, including one that restores
|
||||
the zero-evidence pass, one that restores the bare-prefix repository pin, one
|
||||
that stops re-checking a `kubectl exec` inner command, and one that drops binary
|
||||
attestation from an unrecorded step. The adversarial suite includes large
|
||||
concurrent stdout/stderr, malformed JSON and bytes, zero state, partial failure,
|
||||
absolute timeout, descendant-held pipes, malformed cleanup, ambiguous creates,
|
||||
and no-subprocess/no-mutation spies. Render/client dry-run, exact conflict checks,
|
||||
and bounded diff/full-tree secret scans must also pass.
|
||||
|
||||
The provider-health and Gitea-token-sensitive tests use hermetic fixtures. Their
|
||||
result must be identical whether synthetic runtime files exist or are absent;
|
||||
tests must never read the mounted production token.
|
||||
|
||||
## Release decision
|
||||
|
||||
The handoff remains `NO_GO` until all dependency heads are merged, the exact
|
||||
lineage check passes from an external operator, both default and armed reports
|
||||
are complete, the branch and PR cleanup is verified, the canonical and
|
||||
adversarial gates are green, and a human has reviewed PR #19 plus the artifacts.
|
||||
The harness never merges, deploys, reconciles, publishes, or approves anything.
|
||||
354
scripts/ops/hermes_handoff_acceptance.py
Executable file
354
scripts/ops/hermes_handoff_acceptance.py
Executable file
@ -0,0 +1,354 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fail-closed acceptance harness for the Hermes full handoff.
|
||||
|
||||
Read-only by default. The harness collects evidence from two genuinely separate
|
||||
places — an external operator vantage and the Hermes agent probing itself from
|
||||
inside its own pod — classifies every check PASS / FAIL / NOT_RUN /
|
||||
NOT_APPLICABLE, and emits machine-readable JSON alongside a short human summary.
|
||||
|
||||
Any mandatory FAIL or NOT_RUN is a NO_GO, and so is a harness-level problem: an
|
||||
unreachable vantage, a catalog whose evidence no longer exists, an expired
|
||||
deadline, or two vantages that turn out to be the same principal. Nothing here
|
||||
rounds an absence of evidence up to a pass.
|
||||
|
||||
scripts/ops/hermes_handoff_acceptance.py --help
|
||||
|
||||
The release runbook lists every required freshness and lineage input.
|
||||
|
||||
Mutation lives behind a separate arming flag, needs an exact confirmation
|
||||
phrase and a unique ref, refuses protected push targets before any network
|
||||
call, and verifies its own cleanup. A default run reports those checks NOT_RUN.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import json
|
||||
import sys
|
||||
from collections.abc import Sequence
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
from hermes_handoff_catalog import Targets # noqa: E402
|
||||
from hermes_handoff_ephemeral import ( # noqa: E402
|
||||
CONFIRMATION,
|
||||
ArmingError,
|
||||
ArmRequest,
|
||||
preflight,
|
||||
run_armed,
|
||||
)
|
||||
from hermes_handoff_exec import ( # noqa: E402
|
||||
DEFAULT_COMMAND_TIMEOUT,
|
||||
DEFAULT_RUN_DEADLINE,
|
||||
Runner,
|
||||
)
|
||||
from hermes_handoff_model import GO, Report, unscreened_fields, utc_now # noqa: E402
|
||||
from hermes_handoff_policy import ( # noqa: E402
|
||||
ARMED,
|
||||
EXPECTED_BASE,
|
||||
EXPECTED_REMOTE,
|
||||
EXPECTED_REPO,
|
||||
READ_ONLY,
|
||||
)
|
||||
from hermes_handoff_redaction import DEFAULT_MAX_BYTES # noqa: E402
|
||||
from hermes_handoff_run import ( # noqa: E402
|
||||
build_catalog,
|
||||
build_report,
|
||||
resolve_vantages,
|
||||
unavailable_report,
|
||||
validate_catalog,
|
||||
validate_targets,
|
||||
)
|
||||
|
||||
EXIT_GO = 0
|
||||
EXIT_NO_GO = 1
|
||||
EXIT_USAGE = 2
|
||||
|
||||
|
||||
def dependency_head(value: str) -> tuple[int, str]:
|
||||
"""Parse one `PR=SHA` dependency-head binding."""
|
||||
try:
|
||||
number, sha = value.split("=", 1)
|
||||
parsed = int(number)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise argparse.ArgumentTypeError(
|
||||
"dependency head must be PR=40-character-SHA"
|
||||
) from exc
|
||||
if (
|
||||
parsed <= 0
|
||||
or len(sha) != 40
|
||||
or any(character not in "0123456789abcdef" for character in sha)
|
||||
):
|
||||
raise argparse.ArgumentTypeError(
|
||||
"dependency head must be PR=lowercase-40-character-SHA"
|
||||
)
|
||||
return (parsed, sha)
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
"""Return the command-line interface for the harness."""
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="hermes_handoff_acceptance",
|
||||
description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
parser.add_argument("--namespace", default="hermes", help="Hermes namespace.")
|
||||
parser.add_argument(
|
||||
"--context", help="Kubernetes context for the operator vantage."
|
||||
)
|
||||
parser.add_argument("--kubeconfig", help="Kubeconfig for the operator vantage.")
|
||||
parser.add_argument(
|
||||
"--output", type=Path, help="Write the JSON report here instead of stdout."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--json-only", action="store_true", help="Suppress the human summary."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--command-timeout",
|
||||
type=float,
|
||||
default=DEFAULT_COMMAND_TIMEOUT,
|
||||
help="Per-command seconds.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--deadline",
|
||||
type=float,
|
||||
default=DEFAULT_RUN_DEADLINE,
|
||||
help="Whole-run seconds.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-output-bytes",
|
||||
type=int,
|
||||
default=DEFAULT_MAX_BYTES,
|
||||
help="Per-capture byte budget.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--baseline-commit",
|
||||
default=Targets.baseline_commit,
|
||||
help="Commit main must descend from.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dependency-pr",
|
||||
type=int,
|
||||
action="append",
|
||||
default=None,
|
||||
help="Pull request that must be merged before this release is certifiable (repeatable).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dependency-head",
|
||||
type=dependency_head,
|
||||
action="append",
|
||||
default=None,
|
||||
metavar="PR=SHA",
|
||||
help="Exact current dependency PR head (repeat for every dependency).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--remote-main-sha", default="", help="Exact observed origin/main SHA."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--reviewed-head-sha", default="", help="Exact reviewed PR #19 head SHA."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--agent-image",
|
||||
default="",
|
||||
help="Exact agent :git-<source>-build-<n>@sha256 identity.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--build-sha", default="", help="Exact source SHA used for the image build."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--deployment-revision", default="", help="Exact running Deployment revision."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--expected-suspension",
|
||||
action="append",
|
||||
default=None,
|
||||
metavar="NAMESPACE/NAME",
|
||||
help="Flux object allowed to be suspended (repeatable).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--node-count",
|
||||
type=int,
|
||||
default=3,
|
||||
help="Nodes the fleet must cover (fixed at 3).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--pool-replicas", type=int, default=3, help="Expected pool worker count."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--chat-config-revision",
|
||||
default="",
|
||||
help="Expected chat pod-template config revision.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--chat-ordinal",
|
||||
type=int,
|
||||
default=0,
|
||||
help="Chat tenant ordinal that carries the Telegram identity.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--pool-worker-env",
|
||||
action="append",
|
||||
default=None,
|
||||
help="Environment variable the pool workers must declare (repeatable).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-evidence-age",
|
||||
type=int,
|
||||
default=86_400,
|
||||
help="Seconds before provider and routing evidence is treated as stale.",
|
||||
)
|
||||
armed = parser.add_argument_group("ephemeral mutation mode")
|
||||
armed.add_argument(
|
||||
"--arm-ephemeral-push",
|
||||
action="store_true",
|
||||
help="Permit one ephemeral branch push and draft pull request, then remove both.",
|
||||
)
|
||||
armed.add_argument(
|
||||
"--confirm", default="", help=f"Exact confirmation phrase: {CONFIRMATION!r}."
|
||||
)
|
||||
armed.add_argument(
|
||||
"--ephemeral-token", default="", help="Unique 8-64 character ref suffix."
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def targets_from_args(arguments: argparse.Namespace, now: dt.datetime) -> Targets:
|
||||
"""Return the catalog targets a parsed command line describes."""
|
||||
defaults = Targets()
|
||||
return Targets(
|
||||
namespace=arguments.namespace,
|
||||
baseline_commit=arguments.baseline_commit,
|
||||
dependency_pull_requests=tuple(
|
||||
arguments.dependency_pr or defaults.dependency_pull_requests
|
||||
),
|
||||
dependency_heads=tuple(arguments.dependency_head or ()),
|
||||
remote_main_sha=arguments.remote_main_sha,
|
||||
reviewed_head_sha=arguments.reviewed_head_sha,
|
||||
agent_image=arguments.agent_image,
|
||||
build_sha=arguments.build_sha,
|
||||
deployment_revision=arguments.deployment_revision,
|
||||
expected_suspensions=tuple(arguments.expected_suspension or ()),
|
||||
node_count=arguments.node_count,
|
||||
pool_replicas=arguments.pool_replicas,
|
||||
chat_config_revision=arguments.chat_config_revision,
|
||||
chat_ordinal=arguments.chat_ordinal,
|
||||
pool_worker_env=tuple(arguments.pool_worker_env or defaults.pool_worker_env),
|
||||
max_evidence_age_seconds=arguments.max_evidence_age,
|
||||
now=now,
|
||||
)
|
||||
|
||||
|
||||
def emit(report: Report, arguments: argparse.Namespace, stream=None) -> list[str]:
|
||||
"""Write the report, refusing to publish one that still looks credential-shaped."""
|
||||
payload = report.as_dict()
|
||||
offenders = unscreened_fields(payload)
|
||||
if offenders:
|
||||
payload = {
|
||||
"harness": payload["harness"],
|
||||
"decision": "NO_GO",
|
||||
"harness_errors": [
|
||||
"the rendered report failed its own credential screening at "
|
||||
+ ", ".join(offenders[:8])
|
||||
],
|
||||
}
|
||||
rendered = json.dumps(payload, indent=2, sort_keys=True, default=str)
|
||||
if arguments.output:
|
||||
arguments.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
arguments.output.write_text(rendered + "\n", encoding="utf-8")
|
||||
else:
|
||||
print(rendered, file=stream or sys.stdout)
|
||||
if not arguments.json_only:
|
||||
print(report.render_summary(), file=sys.stderr)
|
||||
return offenders
|
||||
|
||||
|
||||
def _armed_results(arguments: argparse.Namespace, runner: Runner, vantages) -> list:
|
||||
request = ArmRequest(
|
||||
repo=EXPECTED_REPO,
|
||||
remote=EXPECTED_REMOTE,
|
||||
token=arguments.ephemeral_token,
|
||||
confirmation=arguments.confirm,
|
||||
base=EXPECTED_BASE,
|
||||
expected_head=arguments.reviewed_head_sha,
|
||||
expected_base_sha=arguments.remote_main_sha,
|
||||
)
|
||||
preflight(request, EXPECTED_REPO, Path.cwd())
|
||||
vantage = vantages["operator"]
|
||||
return run_armed(runner, vantage, request)
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
"""Run the acceptance sweep and return the process exit status."""
|
||||
arguments = build_parser().parse_args(argv)
|
||||
now = dt.datetime.now(dt.timezone.utc)
|
||||
started_at = utc_now(lambda: now)
|
||||
targets = targets_from_args(arguments, now)
|
||||
mode = ARMED if arguments.arm_ephemeral_push else READ_ONLY
|
||||
|
||||
specs = build_catalog(targets)
|
||||
startup_problems = [*validate_targets(targets), *validate_catalog(specs)]
|
||||
if arguments.arm_ephemeral_push:
|
||||
try:
|
||||
preflight(
|
||||
ArmRequest(
|
||||
repo=EXPECTED_REPO,
|
||||
remote=EXPECTED_REMOTE,
|
||||
token=arguments.ephemeral_token,
|
||||
confirmation=arguments.confirm,
|
||||
base=EXPECTED_BASE,
|
||||
expected_head=arguments.reviewed_head_sha,
|
||||
expected_base_sha=arguments.remote_main_sha,
|
||||
),
|
||||
EXPECTED_REPO,
|
||||
Path.cwd(),
|
||||
)
|
||||
except ArmingError as exc:
|
||||
startup_problems.append(f"arming refused: {exc}")
|
||||
if startup_problems:
|
||||
report = unavailable_report(
|
||||
mode, started_at, "; ".join(startup_problems), targets
|
||||
)
|
||||
emit(report, arguments)
|
||||
return EXIT_NO_GO
|
||||
|
||||
try:
|
||||
runner = Runner(
|
||||
mode=mode,
|
||||
command_timeout=arguments.command_timeout,
|
||||
deadline_seconds=arguments.deadline,
|
||||
max_bytes=arguments.max_output_bytes,
|
||||
)
|
||||
except ValueError as exc:
|
||||
report = unavailable_report(
|
||||
mode, started_at, f"invalid execution bound: {exc}", targets
|
||||
)
|
||||
emit(report, arguments)
|
||||
return EXIT_USAGE
|
||||
vantages, records = resolve_vantages(
|
||||
runner, targets, arguments.kubeconfig, arguments.context
|
||||
)
|
||||
report = build_report(runner, targets, specs, vantages, records, mode, started_at)
|
||||
if arguments.arm_ephemeral_push:
|
||||
if report.decision != GO:
|
||||
report.harness_errors.append(
|
||||
"armed mutation was not started because read-only acceptance did not GO"
|
||||
)
|
||||
else:
|
||||
armed_results = _armed_results(arguments, runner, vantages)
|
||||
armed_ids = {result.spec.id for result in armed_results}
|
||||
report.results = [
|
||||
result for result in report.results if result.spec.id not in armed_ids
|
||||
]
|
||||
report.results.extend(armed_results)
|
||||
report.results.sort(key=lambda result: result.spec.id)
|
||||
report.finished_at = utc_now()
|
||||
offenders = emit(report, arguments)
|
||||
if offenders:
|
||||
return EXIT_NO_GO
|
||||
return EXIT_GO if report.decision == GO else EXIT_NO_GO
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover - exercised through main()
|
||||
raise SystemExit(main())
|
||||
154
scripts/ops/hermes_handoff_arming.py
Normal file
154
scripts/ops/hermes_handoff_arming.py
Normal file
@ -0,0 +1,154 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Arming, provenance, and push-target rules for the armed ephemeral mode.
|
||||
|
||||
Nothing here touches the network or spawns a process. It is the half of armed
|
||||
mode that must be true before a single command exists: the confirmation phrase,
|
||||
the fixed repository and base, the one ephemeral ref grammar, and a local
|
||||
worktree whose HEAD and ``origin/main`` are byte-exact with the reviewed release.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import configparser
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from hermes_handoff_policy import (
|
||||
EXPECTED_BASE,
|
||||
EXPECTED_REMOTE,
|
||||
EXPECTED_REPO,
|
||||
arm_ephemeral_policy,
|
||||
)
|
||||
|
||||
CONFIRMATION = "ARM EPHEMERAL HERMES HANDOFF PUSH"
|
||||
REF_PREFIX = "ephemeral/hermes-handoff-acceptance"
|
||||
REF_RE = re.compile(rf"{REF_PREFIX}/[a-z0-9][a-z0-9-]{{7,63}}\Z")
|
||||
TOKEN_RE = re.compile(r"[a-z0-9][a-z0-9-]{7,63}\Z")
|
||||
SHA_RE = re.compile(r"[0-9a-f]{40}\Z")
|
||||
EXPECTED_HEAD_REF = "feature/hermes-full-handoff-acceptance"
|
||||
EXPECTED_REMOTE_URL = "https://scm.bstein.dev/atlas/titan-iac.git"
|
||||
PROTECTED_NAMES = frozenset(
|
||||
{
|
||||
"default",
|
||||
"develop",
|
||||
"head",
|
||||
"main",
|
||||
"master",
|
||||
"prod",
|
||||
"production",
|
||||
"release",
|
||||
"stable",
|
||||
"trunk",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class ArmingError(ValueError):
|
||||
"""Raised when an ephemeral run is not safe to start."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ArmRequest:
|
||||
"""Every fixed and caller-confirmed input to one armed run."""
|
||||
|
||||
repo: str
|
||||
remote: str
|
||||
token: str
|
||||
confirmation: str
|
||||
base: str = EXPECTED_BASE
|
||||
expected_head: str = ""
|
||||
expected_base_sha: str = ""
|
||||
|
||||
@property
|
||||
def ref(self) -> str:
|
||||
return f"{REF_PREFIX}/{self.token}"
|
||||
|
||||
|
||||
def normalise_branch(name: str) -> str:
|
||||
"""Return a branch name stripped of ref decoration, for comparison."""
|
||||
return name.strip().removeprefix("refs/heads/").strip("/").lower()
|
||||
|
||||
|
||||
def assert_push_target_allowed(ref: str) -> None:
|
||||
"""Raise unless ``ref`` is the one ephemeral branch grammar."""
|
||||
if normalise_branch(ref) in PROTECTED_NAMES:
|
||||
raise ArmingError(f"refusing to push to protected branch {ref!r}")
|
||||
if not REF_RE.fullmatch(ref):
|
||||
raise ArmingError(f"push target {ref!r} is not an ephemeral acceptance ref")
|
||||
|
||||
|
||||
def bounded_read(path: Path, limit: int = 1024 * 1024) -> str:
|
||||
data = path.read_bytes()
|
||||
if not data or len(data) > limit:
|
||||
raise ArmingError(f"local Git metadata is empty or exceeds {limit} bytes")
|
||||
return data.decode("utf-8", "strict")
|
||||
|
||||
|
||||
def _attest_worktree(root: Path, expected_head: str, expected_base_sha: str) -> None:
|
||||
"""Bind cwd, common Git metadata, branch, remote, and exact local HEAD."""
|
||||
resolved = root.resolve(strict=True)
|
||||
marker = resolved / ".git"
|
||||
if not marker.is_file():
|
||||
raise ArmingError(
|
||||
"armed mode must run at the root of the existing linked worktree"
|
||||
)
|
||||
line = bounded_read(marker, 4096).strip()
|
||||
if not line.startswith("gitdir: "):
|
||||
raise ArmingError("linked-worktree Git metadata is malformed")
|
||||
gitdir = Path(line.removeprefix("gitdir: ")).resolve(strict=True)
|
||||
common = (gitdir / bounded_read(gitdir / "commondir", 4096).strip()).resolve(
|
||||
strict=True
|
||||
)
|
||||
head = bounded_read(gitdir / "HEAD", 4096).strip()
|
||||
expected_ref = f"refs/heads/{EXPECTED_HEAD_REF}"
|
||||
if head != f"ref: {expected_ref}":
|
||||
raise ArmingError("armed mode must run from the existing PR #19 feature branch")
|
||||
ref_path = common / expected_ref
|
||||
actual_head = bounded_read(ref_path, 4096).strip() if ref_path.is_file() else ""
|
||||
if actual_head != expected_head:
|
||||
raise ArmingError(
|
||||
"local worktree HEAD does not match the exact reviewed PR head"
|
||||
)
|
||||
remote_ref = common / "refs" / "remotes" / EXPECTED_REMOTE / EXPECTED_BASE
|
||||
remote_main = bounded_read(remote_ref, 4096).strip() if remote_ref.is_file() else ""
|
||||
if remote_main != expected_base_sha:
|
||||
raise ArmingError("local origin/main does not match the exact release base SHA")
|
||||
parser = configparser.RawConfigParser(interpolation=None)
|
||||
try:
|
||||
parser.read_string(bounded_read(common / "config"))
|
||||
remote_url = parser.get(f'remote "{EXPECTED_REMOTE}"', "url")
|
||||
except (configparser.Error, KeyError) as exc:
|
||||
raise ArmingError("local Git remote metadata is malformed") from exc
|
||||
if remote_url != EXPECTED_REMOTE_URL:
|
||||
raise ArmingError("origin is not the fixed Atlas titan-iac HTTPS repository")
|
||||
|
||||
|
||||
def preflight(
|
||||
request: ArmRequest, expected_repo: str, worktree: Path | None = None
|
||||
) -> None:
|
||||
"""Validate arming and local provenance before any subprocess/network call."""
|
||||
if request.confirmation != CONFIRMATION:
|
||||
raise ArmingError("confirmation phrase does not match")
|
||||
if expected_repo != EXPECTED_REPO or request.repo != EXPECTED_REPO:
|
||||
raise ArmingError(
|
||||
"armed repository is not the fixed Atlas titan-iac repository"
|
||||
)
|
||||
if (
|
||||
request.remote != EXPECTED_REMOTE
|
||||
or normalise_branch(request.base) != EXPECTED_BASE
|
||||
):
|
||||
raise ArmingError("armed remote and pull-request base are fixed to origin/main")
|
||||
if not TOKEN_RE.fullmatch(request.token):
|
||||
raise ArmingError(
|
||||
"ephemeral token must be 8-64 lowercase alphanumeric or dash characters"
|
||||
)
|
||||
if not SHA_RE.fullmatch(request.expected_head) or not SHA_RE.fullmatch(
|
||||
request.expected_base_sha
|
||||
):
|
||||
raise ArmingError("exact reviewed head and release base SHAs are required")
|
||||
assert_push_target_allowed(request.ref)
|
||||
_attest_worktree(
|
||||
worktree or Path.cwd(), request.expected_head, request.expected_base_sha
|
||||
)
|
||||
arm_ephemeral_policy(request.ref)
|
||||
202
scripts/ops/hermes_handoff_catalog.py
Normal file
202
scripts/ops/hermes_handoff_catalog.py
Normal file
@ -0,0 +1,202 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Declarative acceptance catalog: baseline, identity, routing, scopes, access.
|
||||
|
||||
The catalog is data on purpose. A reviewer should be able to read what the
|
||||
release is asserted to satisfy without reading any plumbing, and the harness
|
||||
should be able to prove structural properties over every entry — both vantages
|
||||
represented, no impersonation from outside a pod, and no mutation command of
|
||||
any kind — before it runs a single command.
|
||||
|
||||
Catalog drift is a NO_GO, not a pass. If a manifest, a field name, or a workload
|
||||
is renamed, the affected checks report ``NOT_RUN`` because the evidence they
|
||||
name is not there, and the release stops until either the platform or the
|
||||
catalog is corrected.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from hermes_handoff_model import ALWAYS, READ, CheckSpec, Step
|
||||
from hermes_handoff_policy import projection
|
||||
|
||||
OPERATOR = "operator"
|
||||
SELF = "self"
|
||||
SWITCHYARD = "switchyard"
|
||||
NODE = "node"
|
||||
CHAT = "chat"
|
||||
VANTAGE_NAMES = (OPERATOR, SELF, SWITCHYARD, NODE, CHAT)
|
||||
|
||||
# `name<TAB>generation<TAB>observedGeneration<TAB>suspend<TAB>conditions`.
|
||||
FLUX_PROJECTION = projection(
|
||||
(
|
||||
"jsonpath={range .items[*]}{.metadata.namespace}/{.metadata.name}{'\\t'}"
|
||||
"{.metadata.generation}{'\\t'}{.status.observedGeneration}{'\\t'}{.spec.suspend}{'\\t'}"
|
||||
"{range .status.conditions[*]}{.type}={.status},{end}{'\\n'}{end}"
|
||||
).replace("'", '"')
|
||||
)
|
||||
|
||||
ENV_NAME_PROJECTION = projection(
|
||||
(
|
||||
"jsonpath={range .spec.template.spec.containers[*]}{range .env[*]}{.name}{'\\n'}{end}{end}"
|
||||
).replace("'", '"')
|
||||
)
|
||||
|
||||
INIT_NAME_PROJECTION = projection(
|
||||
(
|
||||
"jsonpath={range .spec.template.spec.initContainers[*]}{.name}{'\\n'}{end}"
|
||||
).replace("'", '"')
|
||||
)
|
||||
|
||||
NODE_NAME_PROJECTION = projection(
|
||||
"jsonpath={range .items[*]}{.spec.nodeName}{'\\n'}{end}".replace("'", '"')
|
||||
)
|
||||
|
||||
POOL_PROJECTION = projection(
|
||||
(
|
||||
"jsonpath={range .items[*]}{.metadata.name}{'\\t'}{.spec.nodeName}{'\\t'}"
|
||||
"{.status.phase}{'\\t'}{range .status.conditions[?(@.type==\"Ready\")]}{.status}{end}"
|
||||
"{'\\t'}{range .spec.volumes[*]}{.persistentVolumeClaim.claimName}{','}{end}{'\\n'}{end}"
|
||||
).replace("'", '"')
|
||||
)
|
||||
|
||||
PROVIDER_API_KEY_NAMES = (
|
||||
"ANTHROPIC_API_KEY",
|
||||
"ANTHROPIC_AUTH_TOKEN",
|
||||
"ANTHROPIC_BASE_URL",
|
||||
"AZURE_OPENAI_API_KEY",
|
||||
"CLAUDE_API_KEY",
|
||||
"CLAUDE_CODE_OAUTH_TOKEN",
|
||||
"GEMINI_API_KEY",
|
||||
"GOOGLE_API_KEY",
|
||||
"OPENAI_API_KEY",
|
||||
"OPENAI_BASE_URL",
|
||||
)
|
||||
FORGE_CREDENTIAL_NAMES = (
|
||||
"GITEA_PASSWORD",
|
||||
"GITEA_TOKEN",
|
||||
"GITEA_USERNAME",
|
||||
"GIT_PASSWORD",
|
||||
)
|
||||
|
||||
CODEX_HEALTH = "/opt/data/provider-health/codex.json"
|
||||
CLAUDE_HEALTH = "/opt/data/provider-health/claude.json"
|
||||
CODEX_HEALTH_FIELDS = "authenticated,transport,state,model,effort,latency_ms,checked_at"
|
||||
CLAUDE_HEALTH_FIELDS = (
|
||||
"authenticated,api_provider,auth_method,transport,state,subscription_type,"
|
||||
"model,effort,latency_ms,checked_at"
|
||||
)
|
||||
ROUTING_LOG = "/var/lib/switchyard/routing.jsonl"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Targets:
|
||||
"""Runtime names and expected values the catalog is written against.
|
||||
|
||||
Everything a site can legitimately rename lives here so the assertions stay
|
||||
readable and a rename is a configuration change rather than a code change.
|
||||
"""
|
||||
|
||||
namespace: str = "hermes"
|
||||
agent_deployment: str = "hermes-agent"
|
||||
agent_container: str = "hermes"
|
||||
chat_statefulset: str = "hermes-chat-tenant"
|
||||
chat_container: str = "hermes"
|
||||
chat_ordinal: int = 0
|
||||
chat_router_deployment: str = "hermes-chat-router"
|
||||
triage_deployment: str = "hermes"
|
||||
triage_container: str = "hermes"
|
||||
switchyard_deployment: str = "hermes-switchyard"
|
||||
switchyard_container: str = "switchyard"
|
||||
node_daemonset: str = "hermes-node-ssh-access"
|
||||
node_container: str = "key-reconciler"
|
||||
pool_statefulset: str = "hermes-execution-worker"
|
||||
pool_replicas: int = 3
|
||||
pool_selector: str = "app=hermes-execution-worker"
|
||||
coordinator_claims: tuple[str, ...] = (
|
||||
"hermes-agent-home",
|
||||
"hermes-agent-workspace",
|
||||
)
|
||||
node_account: str = "hermes-agent"
|
||||
node_account_identity: str = "1200:1200:/home/hermes-agent:/bin/bash"
|
||||
node_count: int = 3
|
||||
repo: str = "atlas/titan-iac"
|
||||
remote: str = "origin"
|
||||
baseline_commit: str = "ab346f55509d584e457fe26cf90be3078f7a375c"
|
||||
dependency_pull_requests: tuple[int, ...] = (12, 14, 15, 16, 17, 18, 20)
|
||||
dependency_heads: tuple[tuple[int, str], ...] = ()
|
||||
remote_main_sha: str = ""
|
||||
reviewed_pr_number: int = 19
|
||||
reviewed_head_ref: str = "feature/hermes-full-handoff-acceptance"
|
||||
reviewed_head_sha: str = ""
|
||||
agent_image: str = ""
|
||||
build_sha: str = ""
|
||||
deployment_revision: str = ""
|
||||
expected_suspensions: tuple[str, ...] = ()
|
||||
harbor_namespace: str = "harbor"
|
||||
harbor_immutability_job: str = "hermes-agent-immutability"
|
||||
builder_namespace: str = "jenkins"
|
||||
builder_serviceaccount: str = "hermes-image-builder"
|
||||
builder_pipeline: str = "ci/Jenkinsfile.hermes-agent-image"
|
||||
builder_capabilities: str = (
|
||||
'add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"]'
|
||||
)
|
||||
agent_init_containers: tuple[str, ...] = ("patch-web-session-activity",)
|
||||
chat_init_containers: tuple[str, ...] = ("patch-api-server-sessions",)
|
||||
chat_config_revision: str = ""
|
||||
pool_worker_env: tuple[str, ...] = ("HERMES_WORKER_NODE", "HERMES_WORKER_ORDINAL")
|
||||
max_evidence_age_seconds: int = 86_400
|
||||
routing_tail_lines: int = 600
|
||||
routing_tail_bytes: int = 512 * 1024
|
||||
listing_bytes: int = 512 * 1024
|
||||
git_askpass: str = "/opt/coordinator/gitea_askpass.sh"
|
||||
now: Any = None
|
||||
extra: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
def step(
|
||||
key: str,
|
||||
vantage: str,
|
||||
*argv: str,
|
||||
kind: str = READ,
|
||||
optional: bool = False,
|
||||
record: bool = True,
|
||||
max_bytes: int | None = None,
|
||||
) -> Step:
|
||||
"""Return a probe step addressed to one vantage."""
|
||||
return Step(
|
||||
key=key,
|
||||
vantage=vantage,
|
||||
argv=tuple(argv),
|
||||
kind=kind,
|
||||
optional=optional,
|
||||
record=record,
|
||||
max_bytes=max_bytes,
|
||||
)
|
||||
|
||||
|
||||
def check(
|
||||
identifier: str,
|
||||
title: str,
|
||||
group: str,
|
||||
rule: str,
|
||||
steps: Sequence[Step],
|
||||
expect: dict[str, Any] | None = None,
|
||||
mandatory: bool = True,
|
||||
scope: str = ALWAYS,
|
||||
rationale: str = "",
|
||||
) -> CheckSpec:
|
||||
"""Return one catalog entry."""
|
||||
return CheckSpec(
|
||||
id=identifier,
|
||||
title=title,
|
||||
group=group,
|
||||
rule=rule,
|
||||
steps=tuple(steps),
|
||||
expect=expect or {},
|
||||
mandatory=mandatory,
|
||||
scope=scope,
|
||||
rationale=rationale,
|
||||
)
|
||||
339
scripts/ops/hermes_handoff_checks_access.py
Normal file
339
scripts/ops/hermes_handoff_checks_access.py
Normal file
@ -0,0 +1,339 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Acceptance checks for what Hermes may and may not do to the cluster.
|
||||
|
||||
Denials use Kubernetes authorization reviews only. Live denied attempts were
|
||||
removed because a policy regression could turn a supposed probe into a real
|
||||
credential read or mutation; the default harness never constructs such argv.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from hermes_handoff_catalog import (
|
||||
FLUX_PROJECTION,
|
||||
OPERATOR,
|
||||
SELF,
|
||||
Targets,
|
||||
check,
|
||||
step,
|
||||
)
|
||||
from hermes_handoff_model import REVIEW, CheckSpec
|
||||
|
||||
|
||||
def _access(targets: Targets) -> list[CheckSpec]:
|
||||
namespace = targets.namespace
|
||||
return [
|
||||
check(
|
||||
"access.no-cluster-admin-binding-for-agent",
|
||||
"No ClusterRoleBinding grants the agent service account cluster-admin",
|
||||
"access-denied",
|
||||
"names_absent",
|
||||
[
|
||||
step(
|
||||
"bindings",
|
||||
OPERATOR,
|
||||
"kubectl",
|
||||
"get",
|
||||
"clusterrolebindings",
|
||||
"-o",
|
||||
"name",
|
||||
)
|
||||
],
|
||||
{"step": "bindings", "contains": ("hermes-agent-cluster-admin",)},
|
||||
rationale="The binding inventory and in-pod authorization reviews are independent desired/enforced views.",
|
||||
),
|
||||
check(
|
||||
"access.secrets-are-denied",
|
||||
"Hermes cannot read Secrets cluster-wide",
|
||||
"access-denied",
|
||||
"review_denied",
|
||||
[
|
||||
step(
|
||||
"review",
|
||||
SELF,
|
||||
"kubectl",
|
||||
"auth",
|
||||
"can-i",
|
||||
"get",
|
||||
"secrets",
|
||||
"--all-namespaces",
|
||||
kind=REVIEW,
|
||||
)
|
||||
],
|
||||
rationale="A live Secret request is never constructed, even when an RBAC regression would allow it.",
|
||||
),
|
||||
check(
|
||||
"access.service-account-tokens-are-denied",
|
||||
"Hermes cannot mint a service-account token",
|
||||
"access-denied",
|
||||
"review_denied",
|
||||
[
|
||||
step(
|
||||
"review-token",
|
||||
SELF,
|
||||
"kubectl",
|
||||
"auth",
|
||||
"can-i",
|
||||
"create",
|
||||
"serviceaccounts/token",
|
||||
"--namespace",
|
||||
namespace,
|
||||
kind=REVIEW,
|
||||
),
|
||||
step(
|
||||
"review-serviceaccount",
|
||||
SELF,
|
||||
"kubectl",
|
||||
"auth",
|
||||
"can-i",
|
||||
"create",
|
||||
"serviceaccounts",
|
||||
"--namespace",
|
||||
namespace,
|
||||
kind=REVIEW,
|
||||
),
|
||||
],
|
||||
rationale="Token creation has no credential-safe live probe, so both relevant grants must be denied by review.",
|
||||
),
|
||||
check(
|
||||
"access.impersonation-is-denied",
|
||||
"Hermes cannot impersonate another identity",
|
||||
"access-denied",
|
||||
"review_denied",
|
||||
[
|
||||
step(
|
||||
"review",
|
||||
SELF,
|
||||
"kubectl",
|
||||
"auth",
|
||||
"can-i",
|
||||
"impersonate",
|
||||
"users",
|
||||
"--all-namespaces",
|
||||
kind=REVIEW,
|
||||
)
|
||||
],
|
||||
rationale="The default harness never constructs an impersonated request.",
|
||||
),
|
||||
check(
|
||||
"access.workload-mutation-is-denied",
|
||||
"Hermes cannot mutate its own workload",
|
||||
"access-denied",
|
||||
"review_denied",
|
||||
[
|
||||
step(
|
||||
"review",
|
||||
SELF,
|
||||
"kubectl",
|
||||
"auth",
|
||||
"can-i",
|
||||
"patch",
|
||||
"deployments",
|
||||
"--namespace",
|
||||
namespace,
|
||||
kind=REVIEW,
|
||||
)
|
||||
],
|
||||
rationale="Client-side and server-side dry-run mutation tricks are excluded structurally.",
|
||||
),
|
||||
check(
|
||||
"access.pod-exec-is-denied",
|
||||
"Hermes cannot exec into an arbitrary pod",
|
||||
"access-denied",
|
||||
"review_denied",
|
||||
[
|
||||
step(
|
||||
"review",
|
||||
SELF,
|
||||
"kubectl",
|
||||
"auth",
|
||||
"can-i",
|
||||
"create",
|
||||
"pods/exec",
|
||||
"--all-namespaces",
|
||||
kind=REVIEW,
|
||||
)
|
||||
],
|
||||
rationale="No cross-pod exec request is created by a read-only run.",
|
||||
),
|
||||
check(
|
||||
"access.attach-portforward-and-kube-system-writes-are-denied",
|
||||
"Hermes holds no attach, port-forward, or control-plane write authority",
|
||||
"access-denied",
|
||||
"review_denied",
|
||||
[
|
||||
step(
|
||||
"review-attach",
|
||||
SELF,
|
||||
"kubectl",
|
||||
"auth",
|
||||
"can-i",
|
||||
"create",
|
||||
"pods/attach",
|
||||
"--all-namespaces",
|
||||
kind=REVIEW,
|
||||
),
|
||||
step(
|
||||
"review-portforward",
|
||||
SELF,
|
||||
"kubectl",
|
||||
"auth",
|
||||
"can-i",
|
||||
"create",
|
||||
"pods/portforward",
|
||||
"--all-namespaces",
|
||||
kind=REVIEW,
|
||||
),
|
||||
step(
|
||||
"review-create",
|
||||
SELF,
|
||||
"kubectl",
|
||||
"auth",
|
||||
"can-i",
|
||||
"create",
|
||||
"configmaps",
|
||||
"--namespace",
|
||||
"kube-system",
|
||||
kind=REVIEW,
|
||||
),
|
||||
],
|
||||
rationale="Every authority is reviewed without issuing the request it would authorize.",
|
||||
),
|
||||
check(
|
||||
"access.required-reads-are-allowed",
|
||||
"Hermes retains the reads its operator role depends on",
|
||||
"access-allowed",
|
||||
"allowed",
|
||||
[
|
||||
step(
|
||||
"watch-review",
|
||||
SELF,
|
||||
"kubectl",
|
||||
"auth",
|
||||
"can-i",
|
||||
"watch",
|
||||
"pods",
|
||||
"--namespace",
|
||||
namespace,
|
||||
kind=REVIEW,
|
||||
),
|
||||
step(
|
||||
"pods",
|
||||
SELF,
|
||||
"kubectl",
|
||||
"--namespace",
|
||||
namespace,
|
||||
"get",
|
||||
"pods",
|
||||
"-o",
|
||||
"name",
|
||||
),
|
||||
step("namespaces", SELF, "kubectl", "get", "namespaces", "-o", "name"),
|
||||
step(
|
||||
"deployments",
|
||||
SELF,
|
||||
"kubectl",
|
||||
"--namespace",
|
||||
namespace,
|
||||
"get",
|
||||
"deployments",
|
||||
"-o",
|
||||
"name",
|
||||
),
|
||||
step("nodes", SELF, "kubectl", "get", "nodes", "-o", "name"),
|
||||
],
|
||||
),
|
||||
check(
|
||||
"access.flux-and-helm-status-are-allowed",
|
||||
"Hermes can read Flux and Helm reconciliation status",
|
||||
"access-allowed",
|
||||
"allowed",
|
||||
[
|
||||
step(
|
||||
"kustomizations",
|
||||
SELF,
|
||||
"kubectl",
|
||||
"get",
|
||||
"kustomizations.kustomize.toolkit.fluxcd.io",
|
||||
"--all-namespaces",
|
||||
"-o",
|
||||
"name",
|
||||
record=False,
|
||||
),
|
||||
step(
|
||||
"helmreleases",
|
||||
SELF,
|
||||
"kubectl",
|
||||
"get",
|
||||
"helmreleases.helm.toolkit.fluxcd.io",
|
||||
"--all-namespaces",
|
||||
"-o",
|
||||
"name",
|
||||
record=False,
|
||||
),
|
||||
],
|
||||
),
|
||||
check(
|
||||
"access.namespace-view-agrees-across-vantages",
|
||||
"The operator and in-pod vantages see the same namespace inventory",
|
||||
"access-allowed",
|
||||
"vantages_agree",
|
||||
[
|
||||
step(
|
||||
"operator",
|
||||
OPERATOR,
|
||||
"kubectl",
|
||||
"get",
|
||||
"namespaces",
|
||||
"-o",
|
||||
"name",
|
||||
record=False,
|
||||
),
|
||||
step(
|
||||
"self",
|
||||
SELF,
|
||||
"kubectl",
|
||||
"get",
|
||||
"namespaces",
|
||||
"-o",
|
||||
"name",
|
||||
record=False,
|
||||
),
|
||||
],
|
||||
{"steps": ("operator", "self")},
|
||||
rationale="A disagreement here means one vantage is not seeing the cluster the other is certifying.",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _gitops(targets: Targets) -> list[CheckSpec]:
|
||||
return [
|
||||
check(
|
||||
f"gitops.{kind.split('.')[0]}-reconcile-cleanly",
|
||||
f"No unexpected {label} suspension or unhealthy reconciliation",
|
||||
"gitops",
|
||||
"flux_health",
|
||||
[
|
||||
step(
|
||||
"objects",
|
||||
OPERATOR,
|
||||
"kubectl",
|
||||
"get",
|
||||
kind,
|
||||
"--all-namespaces",
|
||||
"-o",
|
||||
FLUX_PROJECTION,
|
||||
record=False,
|
||||
)
|
||||
],
|
||||
{"step": "objects", "expected_suspensions": targets.expected_suspensions},
|
||||
)
|
||||
for kind, label in (
|
||||
("kustomizations.kustomize.toolkit.fluxcd.io", "Kustomization"),
|
||||
("helmreleases.helm.toolkit.fluxcd.io", "HelmRelease"),
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def access_checks(targets: Targets) -> list[CheckSpec]:
|
||||
"""Return the access-denied, access-allowed, and GitOps health checks."""
|
||||
return [*_access(targets), *_gitops(targets)]
|
||||
374
scripts/ops/hermes_handoff_checks_delivery.py
Normal file
374
scripts/ops/hermes_handoff_checks_delivery.py
Normal file
@ -0,0 +1,374 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Acceptance checks for forge authority, node accounts, images, and regressions.
|
||||
|
||||
What the platform is allowed to do outside its own namespace: which forge
|
||||
authority a worker holds, what a node account is and is not, how an image is
|
||||
built and pinned, and whether the fixes the release depends on are actually
|
||||
running rather than merely merged.
|
||||
|
||||
Where a property has no side-effect-free live probe — a merge that must never be
|
||||
attempted, a branch protection the harness identity cannot read — the entry says
|
||||
so in its rationale and asserts the nearest fact that *is* observable, rather
|
||||
than quietly asserting nothing.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from hermes_handoff_catalog import (
|
||||
FORGE_CREDENTIAL_NAMES,
|
||||
INIT_NAME_PROJECTION,
|
||||
NODE_NAME_PROJECTION,
|
||||
OPERATOR,
|
||||
SELF,
|
||||
Targets,
|
||||
check,
|
||||
step,
|
||||
)
|
||||
from hermes_handoff_model import ATTEMPT, CheckSpec
|
||||
from hermes_handoff_policy import GITEA_CLIENT, projection, shell
|
||||
|
||||
|
||||
def _forge(targets: Targets) -> list[CheckSpec]:
|
||||
api = f"/api/v1/repos/{targets.repo}"
|
||||
return [
|
||||
check(
|
||||
"forge.worker-holds-no-git-credential-variable",
|
||||
"No forge credential is exported into the worker process",
|
||||
"forge",
|
||||
"names_absent",
|
||||
[step("env", SELF, *shell("env_names"))],
|
||||
{"step": "env", "names": FORGE_CREDENTIAL_NAMES},
|
||||
rationale="Authentication reaches Git through an askpass helper that reads runtime state, never through the environment.",
|
||||
),
|
||||
check(
|
||||
"forge.identity-is-not-an-administrator",
|
||||
"The forge identity the worker uses holds no administrative rights",
|
||||
"forge",
|
||||
"json_field",
|
||||
[step("user", SELF, GITEA_CLIENT, "GET", "/api/v1/user")],
|
||||
{"step": "user", "fields": {"is_admin": False}},
|
||||
),
|
||||
check(
|
||||
"forge.repository-authority-is-push-only",
|
||||
"The worker can push a branch but cannot administer the repository",
|
||||
"forge",
|
||||
"json_field",
|
||||
[step("repo", SELF, GITEA_CLIENT, "GET", api)],
|
||||
{
|
||||
"step": "repo",
|
||||
"fields": {
|
||||
"permissions.admin": False,
|
||||
"permissions.push": True,
|
||||
"permissions.pull": True,
|
||||
},
|
||||
},
|
||||
rationale="Merge, approve, close, and protection changes all require authority this identity is proven not to hold.",
|
||||
),
|
||||
check(
|
||||
"forge.administrative-route-is-refused",
|
||||
"An administrative forge route is refused for this identity",
|
||||
"forge",
|
||||
"denied",
|
||||
[
|
||||
step(
|
||||
"attempt",
|
||||
SELF,
|
||||
GITEA_CLIENT,
|
||||
"GET",
|
||||
f"{api}/branch_protections",
|
||||
kind=ATTEMPT,
|
||||
)
|
||||
],
|
||||
rationale="A live refusal, not a configuration reading. The same lack of authority is what blocks a merge route.",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _nodes(targets: Targets) -> list[CheckSpec]:
|
||||
return [
|
||||
check(
|
||||
"nodes.hardening-covers-every-node",
|
||||
"The node account audit runs on every node in the cluster",
|
||||
"nodes",
|
||||
"distinct_count",
|
||||
[
|
||||
step(
|
||||
"coverage",
|
||||
OPERATOR,
|
||||
"kubectl",
|
||||
"--namespace",
|
||||
targets.namespace,
|
||||
"get",
|
||||
"pods",
|
||||
"--selector",
|
||||
f"app={targets.node_daemonset}",
|
||||
"-o",
|
||||
NODE_NAME_PROJECTION,
|
||||
)
|
||||
],
|
||||
{
|
||||
"step": "coverage",
|
||||
"minimum": targets.node_count,
|
||||
"total_equals": targets.node_count,
|
||||
},
|
||||
rationale="A hardened fleet with one unenrolled node is an unhardened fleet.",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _build(targets: Targets) -> list[CheckSpec]:
|
||||
pipeline = f"{targets.remote}/main:{targets.builder_pipeline}"
|
||||
return [
|
||||
check(
|
||||
"build.builder-service-account-is-tokenless",
|
||||
"The image builder service account mounts no API token",
|
||||
"build",
|
||||
"stdout_matches",
|
||||
[
|
||||
step(
|
||||
"serviceaccount",
|
||||
OPERATOR,
|
||||
"kubectl",
|
||||
"--namespace",
|
||||
targets.builder_namespace,
|
||||
"get",
|
||||
"serviceaccount",
|
||||
targets.builder_serviceaccount,
|
||||
"-o",
|
||||
projection("jsonpath={.automountServiceAccountToken}"),
|
||||
)
|
||||
],
|
||||
{"step": "serviceaccount", "equals": "false"},
|
||||
),
|
||||
check(
|
||||
"build.builder-runs-unprivileged-with-exact-capabilities",
|
||||
"The build pod is unprivileged and adds only the expected capabilities",
|
||||
"build",
|
||||
"names_present",
|
||||
[step("pipeline", OPERATOR, "git", "show", pipeline, record=False)],
|
||||
{
|
||||
"step": "pipeline",
|
||||
"names": (
|
||||
targets.builder_capabilities,
|
||||
"privileged: false",
|
||||
"allowPrivilegeEscalation: false",
|
||||
"automountServiceAccountToken: false",
|
||||
'drop: ["ALL"]',
|
||||
),
|
||||
},
|
||||
rationale="Exact lines, not a substring sweep: a widened capability set has to change one of these lines to take effect.",
|
||||
),
|
||||
check(
|
||||
"build.harbor-immutability-rule-is-applied",
|
||||
"The Harbor immutable-tag rule for the agent image has been applied",
|
||||
"build",
|
||||
"lines_match",
|
||||
[
|
||||
step(
|
||||
"job",
|
||||
OPERATOR,
|
||||
"kubectl",
|
||||
"--namespace",
|
||||
targets.harbor_namespace,
|
||||
"get",
|
||||
"job",
|
||||
targets.harbor_immutability_job,
|
||||
"-o",
|
||||
projection("jsonpath={.status.succeeded}"),
|
||||
)
|
||||
],
|
||||
{"step": "job", "pattern": r"^[1-9][0-9]*$", "minimum": 1},
|
||||
rationale="The job verifies the rule it creates, so a completed run is evidence the rule matched the expected contract.",
|
||||
),
|
||||
_release_lineage(targets),
|
||||
]
|
||||
|
||||
|
||||
def _release_lineage(targets: Targets) -> CheckSpec:
|
||||
"""Bind the exact reviewed release from Git through the Ready pods."""
|
||||
container = targets.agent_container
|
||||
namespace = targets.namespace
|
||||
deployment = f"deploy/{targets.agent_deployment}"
|
||||
deployment_projection = projection(
|
||||
(
|
||||
"jsonpath={.metadata.generation}{'\\t'}{.status.observedGeneration}{'\\t'}"
|
||||
"{.spec.replicas}{'\\t'}{.status.readyReplicas}{'\\t'}"
|
||||
"{.metadata.annotations.deployment\\.kubernetes\\.io/revision}{'\\t'}"
|
||||
f'{{.spec.template.spec.containers[?(@.name=="{container}")].image}}'
|
||||
).replace("'", '"')
|
||||
)
|
||||
pod_projection = projection(
|
||||
(
|
||||
"jsonpath={range .items[*]}{.metadata.name}{'\\t'}{.status.phase}{'\\t'}"
|
||||
"{range .status.conditions[?(@.type==\"Ready\")]}{.status}{end}{'\\t'}"
|
||||
f'{{.spec.containers[?(@.name=="{container}")].image}}{{"\\t"}}'
|
||||
f'{{.status.containerStatuses[?(@.name=="{container}")].imageID}}{{"\\t"}}'
|
||||
"{.metadata.labels.pod-template-hash}{'\\n'}{end}"
|
||||
).replace("'", '"')
|
||||
)
|
||||
replicaset_projection = projection(
|
||||
(
|
||||
"jsonpath={range .items[*]}{.metadata.name}{'\\t'}"
|
||||
"{.metadata.annotations.deployment\\.kubernetes\\.io/revision}{'\\t'}"
|
||||
"{.status.readyReplicas}{'\\t'}{.status.availableReplicas}{'\\t'}"
|
||||
f'{{.spec.template.spec.containers[?(@.name=="{container}")].image}}{{"\\t"}}'
|
||||
"{.metadata.labels.pod-template-hash}{'\\n'}{end}"
|
||||
).replace("'", '"')
|
||||
)
|
||||
flux_projection = projection(
|
||||
(
|
||||
"jsonpath={.metadata.name}{'\\t'}{.metadata.generation}{'\\t'}"
|
||||
"{.status.observedGeneration}{'\\t'}{.spec.suspend}{'\\t'}"
|
||||
"{range .status.conditions[?(@.type==\"Ready\")]}{.status}{end}{'\\t'}"
|
||||
"{.status.lastAppliedRevision}"
|
||||
).replace("'", '"')
|
||||
)
|
||||
return check(
|
||||
"release.exact-lineage-is-running",
|
||||
"Remote main, reviewed PR, image, build, Flux, and running revision are identical",
|
||||
"release",
|
||||
"release_lineage",
|
||||
[
|
||||
step(
|
||||
"remote-main",
|
||||
OPERATOR,
|
||||
"git",
|
||||
"ls-remote",
|
||||
targets.remote,
|
||||
"refs/heads/main",
|
||||
),
|
||||
step(
|
||||
"reviewed-pr",
|
||||
OPERATOR,
|
||||
GITEA_CLIENT,
|
||||
"GET",
|
||||
f"/api/v1/repos/{targets.repo}/pulls/{targets.reviewed_pr_number}",
|
||||
),
|
||||
step(
|
||||
"build-source",
|
||||
OPERATOR,
|
||||
"git",
|
||||
"rev-parse",
|
||||
f"{targets.remote}/main",
|
||||
),
|
||||
step(
|
||||
"deployment",
|
||||
OPERATOR,
|
||||
"kubectl",
|
||||
"--namespace",
|
||||
namespace,
|
||||
"get",
|
||||
deployment,
|
||||
"-o",
|
||||
deployment_projection,
|
||||
),
|
||||
step(
|
||||
"pods",
|
||||
OPERATOR,
|
||||
"kubectl",
|
||||
"--namespace",
|
||||
namespace,
|
||||
"get",
|
||||
"pods",
|
||||
"--selector",
|
||||
f"app={targets.agent_deployment}",
|
||||
"-o",
|
||||
pod_projection,
|
||||
),
|
||||
step(
|
||||
"replicasets",
|
||||
OPERATOR,
|
||||
"kubectl",
|
||||
"--namespace",
|
||||
namespace,
|
||||
"get",
|
||||
"replicasets",
|
||||
"--selector",
|
||||
f"app={targets.agent_deployment}",
|
||||
"-o",
|
||||
replicaset_projection,
|
||||
),
|
||||
step(
|
||||
"flux",
|
||||
OPERATOR,
|
||||
"kubectl",
|
||||
"--namespace",
|
||||
"flux-system",
|
||||
"get",
|
||||
"kustomization/hermes",
|
||||
"-o",
|
||||
flux_projection,
|
||||
),
|
||||
],
|
||||
{
|
||||
"main_sha": targets.remote_main_sha,
|
||||
"head_sha": targets.reviewed_head_sha,
|
||||
"head_ref": targets.reviewed_head_ref,
|
||||
"pr_number": targets.reviewed_pr_number,
|
||||
"image": targets.agent_image,
|
||||
"build_sha": targets.build_sha,
|
||||
"deployment_revision": targets.deployment_revision,
|
||||
},
|
||||
rationale="A PASS requires one exact release identity across every source and running object.",
|
||||
)
|
||||
|
||||
|
||||
def _reliability(targets: Targets) -> list[CheckSpec]:
|
||||
return [
|
||||
check(
|
||||
"reliability.finalization-and-replay-patches-are-deployed",
|
||||
"The agent workload runs the finalization, replay, and session patches",
|
||||
"reliability",
|
||||
"names_present",
|
||||
[
|
||||
step(
|
||||
"init",
|
||||
OPERATOR,
|
||||
"kubectl",
|
||||
"--namespace",
|
||||
targets.namespace,
|
||||
"get",
|
||||
f"deploy/{targets.agent_deployment}",
|
||||
"-o",
|
||||
INIT_NAME_PROJECTION,
|
||||
)
|
||||
],
|
||||
{"step": "init", "names": targets.agent_init_containers},
|
||||
rationale="The regression suites live in the repository; this asserts the fixes they cover are actually running.",
|
||||
),
|
||||
check(
|
||||
"reliability.regression-suites-are-present-on-main",
|
||||
"The decomposition and worker-recovery regression suites are on main",
|
||||
"reliability",
|
||||
"names_present",
|
||||
[
|
||||
step(
|
||||
"suites",
|
||||
OPERATOR,
|
||||
"git",
|
||||
"ls-tree",
|
||||
"--name-only",
|
||||
f"{targets.remote}/main",
|
||||
"testing/tests/",
|
||||
)
|
||||
],
|
||||
{
|
||||
"step": "suites",
|
||||
"names": (
|
||||
"testing/tests/test_hermes_cli_lanes.py",
|
||||
"testing/tests/test_hermes_coordinator.py",
|
||||
"testing/tests/test_hermes_worker_recovery.py",
|
||||
),
|
||||
},
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def delivery_checks(targets: Targets) -> list[CheckSpec]:
|
||||
"""Return the forge, node, build, and reliability checks."""
|
||||
return [
|
||||
*_forge(targets),
|
||||
*_nodes(targets),
|
||||
*_build(targets),
|
||||
*_reliability(targets),
|
||||
]
|
||||
387
scripts/ops/hermes_handoff_checks_platform.py
Normal file
387
scripts/ops/hermes_handoff_checks_platform.py
Normal file
@ -0,0 +1,387 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Acceptance checks for the release baseline, provider identity, and routing.
|
||||
|
||||
These are the facts a handoff rests on before any authority question is asked:
|
||||
that the tree under test is the merged one, that both providers authenticate
|
||||
through the subscriptions they are supposed to, that Switchyard actually decided
|
||||
the routes it claims to, and that chat, agent, and triage are still three
|
||||
distinct scopes rather than one surface wearing three hostnames.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from hermes_handoff_catalog import (
|
||||
CLAUDE_HEALTH,
|
||||
CLAUDE_HEALTH_FIELDS,
|
||||
CODEX_HEALTH,
|
||||
CODEX_HEALTH_FIELDS,
|
||||
ENV_NAME_PROJECTION,
|
||||
OPERATOR,
|
||||
PROVIDER_API_KEY_NAMES,
|
||||
ROUTING_LOG,
|
||||
SELF,
|
||||
SWITCHYARD,
|
||||
Targets,
|
||||
check,
|
||||
step,
|
||||
)
|
||||
from hermes_handoff_model import CheckSpec
|
||||
from hermes_handoff_policy import GITEA_CLIENT, projection, shell
|
||||
|
||||
|
||||
def _baseline(targets: Targets) -> list[CheckSpec]:
|
||||
api = f"/api/v1/repos/{targets.repo}"
|
||||
dependency_heads = dict(targets.dependency_heads)
|
||||
checks = [
|
||||
check(
|
||||
"baseline.local-origin-main-is-current",
|
||||
"The local origin/main ref equals the fixed remote-main SHA",
|
||||
"baseline",
|
||||
"stdout_matches",
|
||||
[step("main", OPERATOR, "git", "rev-parse", f"{targets.remote}/main")],
|
||||
{"step": "main", "equals": targets.remote_main_sha},
|
||||
rationale="The ancestry check is meaningful only after the local tracking ref is bound to the remote observation.",
|
||||
),
|
||||
check(
|
||||
"baseline.origin-main-descends-merged-work",
|
||||
"origin/main descends from the merged Hermes isolation baseline",
|
||||
"baseline",
|
||||
"allowed",
|
||||
[
|
||||
step(
|
||||
"ancestor",
|
||||
OPERATOR,
|
||||
"git",
|
||||
"merge-base",
|
||||
"--is-ancestor",
|
||||
targets.baseline_commit,
|
||||
f"{targets.remote}/main",
|
||||
)
|
||||
],
|
||||
rationale="Everything downstream assumes the merged worker-isolation baseline is present.",
|
||||
),
|
||||
check(
|
||||
"baseline.pending-pull-requests-recorded",
|
||||
"Exact open pull-request heads are recorded, not assumed",
|
||||
"baseline",
|
||||
"json_record",
|
||||
[
|
||||
step(
|
||||
"open",
|
||||
OPERATOR,
|
||||
GITEA_CLIENT,
|
||||
"GET",
|
||||
f"{api}/pulls?state=open&limit=50",
|
||||
)
|
||||
],
|
||||
{
|
||||
"step": "open",
|
||||
"record": {
|
||||
"numbers": "[].number",
|
||||
"head_refs": "[].head.ref",
|
||||
"head_shas": "[].head.sha",
|
||||
"base_shas": "[].base.sha",
|
||||
"drafts": "[].draft",
|
||||
"mergeable": "[].mergeable",
|
||||
},
|
||||
},
|
||||
rationale="The runbook has to name the heads it was verified against; stale heads are how a release verifies the wrong tree.",
|
||||
),
|
||||
check(
|
||||
"baseline.operator-checkout-clean",
|
||||
"The operator checkout has no uncommitted drift",
|
||||
"baseline",
|
||||
"stdout_matches",
|
||||
[step("status", OPERATOR, "git", "status", "--porcelain")],
|
||||
{"step": "status", "equals": ""},
|
||||
mandatory=False,
|
||||
rationale="Advisory: a dirty checkout does not invalidate cluster evidence, but it does muddy what was compared.",
|
||||
),
|
||||
]
|
||||
checks += [
|
||||
check(
|
||||
f"baseline.dependency-pr-{number}-merged",
|
||||
f"Pull request #{number} is merged into main",
|
||||
"baseline",
|
||||
"json_field",
|
||||
[step("pr", OPERATOR, GITEA_CLIENT, "GET", f"{api}/pulls/{number}")],
|
||||
{
|
||||
"step": "pr",
|
||||
"fields": {
|
||||
"merged": True,
|
||||
"base.ref": "main",
|
||||
"head.sha": dependency_heads.get(number, ""),
|
||||
},
|
||||
},
|
||||
rationale="This harness certifies the post-merge platform; an unmerged dependency means it is measuring something else.",
|
||||
)
|
||||
for number in targets.dependency_pull_requests
|
||||
]
|
||||
return checks
|
||||
|
||||
|
||||
def _identity(targets: Targets) -> list[CheckSpec]:
|
||||
codex = shell("json_fields", path=CODEX_HEALTH, fields=CODEX_HEALTH_FIELDS)
|
||||
claude = shell("json_fields", path=CLAUDE_HEALTH, fields=CLAUDE_HEALTH_FIELDS)
|
||||
agent = f"deploy/{targets.agent_deployment}"
|
||||
return [
|
||||
check(
|
||||
"identity.codex-is-chatgpt-subscription",
|
||||
"Codex authenticates through the ChatGPT subscription, not an API key",
|
||||
"identity",
|
||||
"json_field",
|
||||
[step("health", SELF, *codex)],
|
||||
{
|
||||
"step": "health",
|
||||
"fields": {
|
||||
"authenticated": True,
|
||||
"transport": "codex-chatgpt-subscription",
|
||||
"state": "available",
|
||||
},
|
||||
},
|
||||
rationale="Only allow-listed status fields are read; the credential itself is never opened.",
|
||||
),
|
||||
check(
|
||||
"identity.codex-evidence-is-fresh",
|
||||
"Codex authentication evidence is current",
|
||||
"identity",
|
||||
"json_recent",
|
||||
[step("health", SELF, *codex)],
|
||||
{
|
||||
"step": "health",
|
||||
"now": targets.now,
|
||||
"fields": {"checked_at": targets.max_evidence_age_seconds},
|
||||
},
|
||||
rationale="Stale provider evidence describes a state the release no longer has.",
|
||||
),
|
||||
check(
|
||||
"identity.claude-is-firstparty-subscription",
|
||||
"Claude authenticates as a claude.ai first-party subscription",
|
||||
"identity",
|
||||
"json_field",
|
||||
[step("health", SELF, *claude)],
|
||||
{
|
||||
"step": "health",
|
||||
"fields": {
|
||||
"authenticated": True,
|
||||
"api_provider": "firstParty",
|
||||
"auth_method": "claude.ai",
|
||||
"transport": "claude-code-cli-subscription",
|
||||
"state": "available",
|
||||
},
|
||||
},
|
||||
),
|
||||
check(
|
||||
"identity.claude-evidence-is-fresh",
|
||||
"Claude authentication evidence is current",
|
||||
"identity",
|
||||
"json_recent",
|
||||
[step("health", SELF, *claude)],
|
||||
{
|
||||
"step": "health",
|
||||
"now": targets.now,
|
||||
"fields": {"checked_at": targets.max_evidence_age_seconds},
|
||||
},
|
||||
),
|
||||
check(
|
||||
"identity.no-provider-api-key-in-pod",
|
||||
"No provider API-key variable is set in the running agent process",
|
||||
"identity",
|
||||
"names_absent",
|
||||
[step("env", SELF, *shell("env_names"))],
|
||||
{
|
||||
"step": "env",
|
||||
"names": PROVIDER_API_KEY_NAMES,
|
||||
"contains": ("API_KEY",),
|
||||
},
|
||||
rationale="Names only. The probe lists variable names and never their values.",
|
||||
),
|
||||
check(
|
||||
"identity.no-provider-api-key-in-manifest",
|
||||
"No provider API-key variable is declared on the agent workload",
|
||||
"identity",
|
||||
"names_absent",
|
||||
[
|
||||
step(
|
||||
"env",
|
||||
OPERATOR,
|
||||
"kubectl",
|
||||
"--namespace",
|
||||
targets.namespace,
|
||||
"get",
|
||||
agent,
|
||||
"-o",
|
||||
ENV_NAME_PROJECTION,
|
||||
)
|
||||
],
|
||||
{"step": "env", "names": PROVIDER_API_KEY_NAMES, "contains": ("API_KEY",)},
|
||||
rationale="The operator counterpart to the in-pod probe: desired state and running state are asserted separately.",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _routing(targets: Targets) -> list[CheckSpec]:
|
||||
switchyard = f"deploy/{targets.switchyard_deployment}"
|
||||
tail = shell("tail_lines", path=ROUTING_LOG, limit=str(targets.routing_tail_lines))
|
||||
routing_step = step(
|
||||
"routing", SWITCHYARD, *tail, record=False, max_bytes=targets.routing_tail_bytes
|
||||
)
|
||||
return [
|
||||
check(
|
||||
"routing.switchyard-is-available",
|
||||
"Switchyard has a ready replica serving every model-call boundary",
|
||||
"routing",
|
||||
"lines_match",
|
||||
[
|
||||
step(
|
||||
"deployment",
|
||||
OPERATOR,
|
||||
"kubectl",
|
||||
"--namespace",
|
||||
targets.namespace,
|
||||
"get",
|
||||
switchyard,
|
||||
"-o",
|
||||
projection("jsonpath={.status.readyReplicas}"),
|
||||
)
|
||||
],
|
||||
{"step": "deployment", "pattern": r"^[1-9][0-9]*$", "minimum": 1},
|
||||
),
|
||||
check(
|
||||
"routing.provider-effort-and-fallback-evidence",
|
||||
"Routing evidence covers both providers, all graded efforts, and a real fallback",
|
||||
"routing",
|
||||
"routing_evidence",
|
||||
[routing_step],
|
||||
{
|
||||
"step": "routing",
|
||||
"providers": ("codex", "claude"),
|
||||
"efforts": ("medium", "high", "xhigh"),
|
||||
"lanes": ("route", "worker"),
|
||||
"require_fallback_evidence": True,
|
||||
"max_age_seconds": targets.max_evidence_age_seconds,
|
||||
"now": targets.now,
|
||||
},
|
||||
rationale="The routing log records the decided route per boundary; the tail is parsed here and never pasted into the report.",
|
||||
),
|
||||
check(
|
||||
"routing.codex-latency-recorded",
|
||||
"Codex boundary latency is measured, not assumed",
|
||||
"routing",
|
||||
"json_numeric",
|
||||
[
|
||||
step(
|
||||
"health",
|
||||
SELF,
|
||||
*shell(
|
||||
"json_fields", path=CODEX_HEALTH, fields=CODEX_HEALTH_FIELDS
|
||||
),
|
||||
)
|
||||
],
|
||||
{"step": "health", "fields": {"latency_ms": {"min": 1, "max": 600_000}}},
|
||||
),
|
||||
check(
|
||||
"routing.claude-latency-recorded",
|
||||
"Claude boundary latency is measured, not assumed",
|
||||
"routing",
|
||||
"json_numeric",
|
||||
[
|
||||
step(
|
||||
"health",
|
||||
SELF,
|
||||
*shell(
|
||||
"json_fields", path=CLAUDE_HEALTH, fields=CLAUDE_HEALTH_FIELDS
|
||||
),
|
||||
)
|
||||
],
|
||||
{"step": "health", "fields": {"latency_ms": {"min": 1, "max": 600_000}}},
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _scope_profile(
|
||||
targets: Targets, name: str, workload: str, container: str, expected: str
|
||||
) -> CheckSpec:
|
||||
profile_projection = projection(
|
||||
f'jsonpath={{.spec.template.spec.containers[?(@.name=="{container}")]'
|
||||
'.env[?(@.name=="HERMES_AUTO_ROUTER_PROFILE")].value}'
|
||||
)
|
||||
return check(
|
||||
f"scopes.{name}-profile-is-distinct",
|
||||
f"The {name} surface runs under its own routing profile",
|
||||
"scopes",
|
||||
"stdout_matches",
|
||||
[
|
||||
step(
|
||||
"profile",
|
||||
OPERATOR,
|
||||
"kubectl",
|
||||
"--namespace",
|
||||
targets.namespace,
|
||||
"get",
|
||||
workload,
|
||||
"-o",
|
||||
profile_projection,
|
||||
)
|
||||
],
|
||||
{"step": "profile", "equals": expected},
|
||||
rationale="Chat, agent, and triage must not collapse into one scope; each states its own profile.",
|
||||
)
|
||||
|
||||
|
||||
def _scopes(targets: Targets) -> list[CheckSpec]:
|
||||
return [
|
||||
_scope_profile(
|
||||
targets,
|
||||
"agent",
|
||||
f"deploy/{targets.agent_deployment}",
|
||||
targets.agent_container,
|
||||
"agent",
|
||||
),
|
||||
_scope_profile(
|
||||
targets,
|
||||
"chat",
|
||||
f"statefulset/{targets.chat_statefulset}",
|
||||
targets.chat_container,
|
||||
"chat",
|
||||
),
|
||||
_scope_profile(
|
||||
targets,
|
||||
"triage",
|
||||
f"deploy/{targets.triage_deployment}",
|
||||
targets.triage_container,
|
||||
"triage",
|
||||
),
|
||||
check(
|
||||
"scopes.surfaces-have-distinct-hosts",
|
||||
"Chat, agent, and triage are published on distinct hostnames",
|
||||
"scopes",
|
||||
"distinct_count",
|
||||
[
|
||||
step(
|
||||
"hosts",
|
||||
OPERATOR,
|
||||
"kubectl",
|
||||
"--namespace",
|
||||
targets.namespace,
|
||||
"get",
|
||||
"ingress",
|
||||
"-o",
|
||||
projection(
|
||||
'jsonpath={range .items[*]}{range .spec.rules[*]}{.host}{"\\n"}{end}{end}'
|
||||
),
|
||||
)
|
||||
],
|
||||
{"step": "hosts", "minimum": 3},
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def platform_checks(targets: Targets) -> list[CheckSpec]:
|
||||
"""Return the baseline, identity, routing, and scope checks."""
|
||||
return [
|
||||
*_baseline(targets),
|
||||
*_identity(targets),
|
||||
*_routing(targets),
|
||||
*_scopes(targets),
|
||||
]
|
||||
376
scripts/ops/hermes_handoff_checks_workers.py
Normal file
376
scripts/ops/hermes_handoff_checks_workers.py
Normal file
@ -0,0 +1,376 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Acceptance checks for the distributed worker pool and the visible surfaces.
|
||||
|
||||
A pool is only distributed if its workers occupy distinct nodes and distinct
|
||||
volumes and leave state ownership with the coordinator; a replica count alone
|
||||
satisfies none of that. A surface is only working if the work it did is visible
|
||||
in it, which is why the session and Kanban checks look at the same data the UI
|
||||
renders rather than at whether a pod is Running.
|
||||
|
||||
The ephemeral entries are placeholders in a default run. They report NOT_RUN
|
||||
until the mutation mode is separately armed, and the armed mode replaces them
|
||||
with real, mandatory results.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from hermes_handoff_catalog import (
|
||||
CHAT,
|
||||
INIT_NAME_PROJECTION,
|
||||
OPERATOR,
|
||||
POOL_PROJECTION,
|
||||
SELF,
|
||||
Targets,
|
||||
check,
|
||||
)
|
||||
from hermes_handoff_catalog import step as step
|
||||
from hermes_handoff_model import EPHEMERAL, CheckSpec
|
||||
from hermes_handoff_policy import projection
|
||||
|
||||
KANBAN_STATUSES = [
|
||||
"archived",
|
||||
"blocked",
|
||||
"done",
|
||||
"ready",
|
||||
"review",
|
||||
"running",
|
||||
"scheduled",
|
||||
"todo",
|
||||
"triage",
|
||||
]
|
||||
|
||||
|
||||
def _pool(targets: Targets) -> list[CheckSpec]:
|
||||
selector = targets.pool_selector
|
||||
return [
|
||||
check(
|
||||
"pool.three-workers-have-distinct-nodes-and-volumes",
|
||||
"Exactly three Ready ordinal workers occupy distinct nodes and claims",
|
||||
"pool",
|
||||
"pool_topology",
|
||||
[
|
||||
step(
|
||||
"statefulset",
|
||||
OPERATOR,
|
||||
"kubectl",
|
||||
"--namespace",
|
||||
targets.namespace,
|
||||
"get",
|
||||
f"statefulset/{targets.pool_statefulset}",
|
||||
"-o",
|
||||
projection(
|
||||
"jsonpath={.spec.replicas}{'\\t'}{.status.readyReplicas}{'\\t'}{.status.currentReplicas}{'\\t'}{.spec.template.spec.automountServiceAccountToken}".replace(
|
||||
"'", '"'
|
||||
)
|
||||
),
|
||||
),
|
||||
step(
|
||||
"workers",
|
||||
OPERATOR,
|
||||
"kubectl",
|
||||
"--namespace",
|
||||
targets.namespace,
|
||||
"get",
|
||||
"pods",
|
||||
"--selector",
|
||||
selector,
|
||||
"-o",
|
||||
POOL_PROJECTION,
|
||||
),
|
||||
],
|
||||
{
|
||||
"state_step": "statefulset",
|
||||
"worker_step": "workers",
|
||||
"replicas": targets.pool_replicas,
|
||||
"name": targets.pool_statefulset,
|
||||
},
|
||||
rationale="Replica count, Ready state, ordinal coverage, node placement, PVC isolation, and tokenlessness are one fail-closed assertion.",
|
||||
),
|
||||
check(
|
||||
"pool.coordinator-retains-sole-state-ownership",
|
||||
"No pool worker mounts the coordinator's durable state",
|
||||
"pool",
|
||||
"names_absent",
|
||||
[
|
||||
step(
|
||||
"claims",
|
||||
OPERATOR,
|
||||
"kubectl",
|
||||
"--namespace",
|
||||
targets.namespace,
|
||||
"get",
|
||||
f"statefulset/{targets.pool_statefulset}",
|
||||
"-o",
|
||||
projection(
|
||||
"jsonpath={range .spec.template.spec.volumes[*]}{.name}{'='}"
|
||||
"{.persistentVolumeClaim.claimName}{'\\n'}{end}".replace(
|
||||
"'", '"'
|
||||
)
|
||||
),
|
||||
)
|
||||
],
|
||||
{"step": "claims", "contains": targets.coordinator_claims},
|
||||
rationale="Two writers on one SQLite state file is the duplicate-work and lost-result failure this pool exists to avoid. One line per template volume keeps a drifted or empty projection NOT_RUN instead of a silent pass.",
|
||||
),
|
||||
*_pool_env(targets),
|
||||
]
|
||||
|
||||
|
||||
def _pool_env(targets: Targets) -> list[CheckSpec]:
|
||||
"""Return the mandatory assignment-safety configuration check."""
|
||||
return [
|
||||
check(
|
||||
"pool.assignment-safety-knobs-are-configured",
|
||||
"Pool workers carry the lease and de-duplication configuration",
|
||||
"pool",
|
||||
"names_present",
|
||||
[
|
||||
step(
|
||||
"env",
|
||||
OPERATOR,
|
||||
"kubectl",
|
||||
"--namespace",
|
||||
targets.namespace,
|
||||
"get",
|
||||
f"statefulset/{targets.pool_statefulset}",
|
||||
"-o",
|
||||
projection(
|
||||
'jsonpath={range .spec.template.spec.containers[*]}{range .env[*]}{.name}{"\\n"}{end}{end}'
|
||||
),
|
||||
)
|
||||
],
|
||||
{"step": "env", "names": targets.pool_worker_env},
|
||||
rationale="Deployed configuration only; the stale-claim and duplicate-dispatch behaviour itself is covered by the pool's own regression suite.",
|
||||
),
|
||||
check(
|
||||
"pool.assignment-fieldrefs-are-bound",
|
||||
"Worker node and ordinal identities come from fixed downward-API fields",
|
||||
"pool",
|
||||
"names_present",
|
||||
[
|
||||
step(
|
||||
"source",
|
||||
OPERATOR,
|
||||
"git",
|
||||
"show",
|
||||
f"{targets.remote}/main:services/hermes/execution-worker-statefulset.yaml",
|
||||
record=False,
|
||||
)
|
||||
],
|
||||
{
|
||||
"step": "source",
|
||||
"names": (
|
||||
"name: HERMES_WORKER_ORDINAL",
|
||||
"fieldPath: metadata.labels['apps.kubernetes.io/pod-index']",
|
||||
"name: HERMES_WORKER_NODE",
|
||||
"fieldPath: spec.nodeName",
|
||||
),
|
||||
},
|
||||
rationale="This prevents caller-authored ordinal/node values from impersonating another worker assignment.",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _surfaces(targets: Targets) -> list[CheckSpec]:
|
||||
checks = [
|
||||
check(
|
||||
"surfaces.kanban-activity-is-visible",
|
||||
"Durable Kanban activity is visible to the agent surface",
|
||||
"surfaces",
|
||||
"all_of_field",
|
||||
[
|
||||
step(
|
||||
"tasks",
|
||||
SELF,
|
||||
"hermes",
|
||||
"kanban",
|
||||
"list",
|
||||
"--json",
|
||||
record=False,
|
||||
max_bytes=targets.listing_bytes,
|
||||
)
|
||||
],
|
||||
{"step": "tasks", "key_field": "id", "fields": {"status": KANBAN_STATUSES}},
|
||||
rationale="Non-empty and well-formed. An empty board is NOT_RUN, because it cannot distinguish a quiet week from a broken read.",
|
||||
),
|
||||
check(
|
||||
"surfaces.durable-sessions-render-without-blank-rows",
|
||||
"Durable sessions are listed with a preview, a source, and an id",
|
||||
"surfaces",
|
||||
"lines_match",
|
||||
[
|
||||
step(
|
||||
"sessions",
|
||||
SELF,
|
||||
"hermes",
|
||||
"sessions",
|
||||
"list",
|
||||
"--limit",
|
||||
"20",
|
||||
record=False,
|
||||
)
|
||||
],
|
||||
{"step": "sessions", "pattern": r"^\S.*\S$", "skip": 2, "minimum": 1},
|
||||
rationale="A row whose leading column is blank is the session-render regression seen from the data side.",
|
||||
),
|
||||
check(
|
||||
"surfaces.agent-ui-session-patches-are-deployed",
|
||||
"The agent surface runs the session-activity rendering patches",
|
||||
"surfaces",
|
||||
"names_present",
|
||||
[
|
||||
step(
|
||||
"init",
|
||||
OPERATOR,
|
||||
"kubectl",
|
||||
"--namespace",
|
||||
targets.namespace,
|
||||
"get",
|
||||
f"deploy/{targets.agent_deployment}",
|
||||
"-o",
|
||||
INIT_NAME_PROJECTION,
|
||||
)
|
||||
],
|
||||
{
|
||||
"step": "init",
|
||||
"names": ("patch-web-session-activity", "patch-api-server-sessions"),
|
||||
},
|
||||
rationale="Blank session rows were a rendering regression; this asserts the patches that fixed it are in the running pod.",
|
||||
),
|
||||
check(
|
||||
"surfaces.chat-session-patches-are-deployed",
|
||||
"The chat surface runs the API-session continuity patches",
|
||||
"surfaces",
|
||||
"names_present",
|
||||
[
|
||||
step(
|
||||
"init",
|
||||
OPERATOR,
|
||||
"kubectl",
|
||||
"--namespace",
|
||||
targets.namespace,
|
||||
"get",
|
||||
f"statefulset/{targets.chat_statefulset}",
|
||||
"-o",
|
||||
INIT_NAME_PROJECTION,
|
||||
)
|
||||
],
|
||||
{"step": "init", "names": targets.chat_init_containers},
|
||||
),
|
||||
check(
|
||||
"surfaces.chat-telegram-topic-state-is-durable",
|
||||
"Telegram topic and tenant state is held on durable storage",
|
||||
"surfaces",
|
||||
"names_present",
|
||||
[
|
||||
step(
|
||||
"claims",
|
||||
OPERATOR,
|
||||
"kubectl",
|
||||
"--namespace",
|
||||
targets.namespace,
|
||||
"get",
|
||||
f"deploy/{targets.chat_router_deployment}",
|
||||
"-o",
|
||||
projection(
|
||||
'jsonpath={range .spec.template.spec.volumes[*]}{.persistentVolumeClaim.claimName}{"\\n"}{end}'
|
||||
),
|
||||
)
|
||||
],
|
||||
{"step": "claims", "contains": ("state",)},
|
||||
rationale="Topic selection survives a router restart only if the tenant state file is on a claim rather than in the pod.",
|
||||
),
|
||||
]
|
||||
checks.append(
|
||||
check(
|
||||
"surfaces.chat-telegram-sessions-are-continuous",
|
||||
"The selected chat tenant still holds durable Telegram-sourced sessions",
|
||||
"surfaces",
|
||||
"lines_match",
|
||||
[
|
||||
step(
|
||||
"telegram",
|
||||
CHAT,
|
||||
"hermes",
|
||||
"sessions",
|
||||
"list",
|
||||
"--source",
|
||||
"telegram",
|
||||
"--limit",
|
||||
"20",
|
||||
record=False,
|
||||
)
|
||||
],
|
||||
{"step": "telegram", "pattern": r"^\S.*\S$", "skip": 2, "minimum": 1},
|
||||
rationale="Session continuity is per tenant and depends on there having been Telegram traffic, so it is asserted only when the operator names the tenant that carries the identity.",
|
||||
)
|
||||
)
|
||||
checks.append(
|
||||
check(
|
||||
"surfaces.chat-runs-the-telegram-topic-revision",
|
||||
"The chat surface runs the durable Telegram topic configuration",
|
||||
"surfaces",
|
||||
"stdout_matches",
|
||||
[
|
||||
step(
|
||||
"revision",
|
||||
OPERATOR,
|
||||
"kubectl",
|
||||
"--namespace",
|
||||
targets.namespace,
|
||||
"get",
|
||||
f"statefulset/{targets.chat_statefulset}",
|
||||
"-o",
|
||||
projection(
|
||||
"jsonpath={.spec.template.metadata.annotations.ai\\.bstein\\.dev/config-rev}"
|
||||
),
|
||||
)
|
||||
],
|
||||
{"step": "revision", "equals": targets.chat_config_revision},
|
||||
rationale="Telegram topic continuity is carried by the config revision the pod template pins.",
|
||||
)
|
||||
)
|
||||
return checks
|
||||
|
||||
|
||||
def _ephemeral() -> list[CheckSpec]:
|
||||
return [
|
||||
check(
|
||||
f"ephemeral.{name}",
|
||||
title,
|
||||
"ephemeral",
|
||||
"not_armed",
|
||||
[],
|
||||
{"reason": reason},
|
||||
mandatory=False,
|
||||
scope=EPHEMERAL,
|
||||
rationale="Mutating checks are separately armed. A default run reports them NOT_RUN and never touches the forge.",
|
||||
)
|
||||
for name, title, reason in (
|
||||
(
|
||||
"feature-branch-push",
|
||||
"A unique ephemeral feature branch can be pushed and removed",
|
||||
"ephemeral mutation mode is not armed",
|
||||
),
|
||||
(
|
||||
"draft-pull-request",
|
||||
"A draft pull request can be opened and closed against the ephemeral branch",
|
||||
"ephemeral mutation mode is not armed",
|
||||
),
|
||||
(
|
||||
"protected-branch-refusal",
|
||||
"Arming against a protected branch is refused before any network call",
|
||||
"ephemeral mutation mode is not armed",
|
||||
),
|
||||
(
|
||||
"cleanup-verified",
|
||||
"Every ephemeral ref and pull request is removed and the removal is verified",
|
||||
"ephemeral mutation mode is not armed",
|
||||
),
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def worker_checks(targets: Targets) -> list[CheckSpec]:
|
||||
"""Return the pool, surface, and ephemeral-placeholder checks."""
|
||||
return [*_pool(targets), *_surfaces(targets), *_ephemeral()]
|
||||
439
scripts/ops/hermes_handoff_ephemeral.py
Normal file
439
scripts/ops/hermes_handoff_ephemeral.py
Normal file
@ -0,0 +1,439 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Discoverable, residue-surfacing, self-cleaning armed ephemeral run.
|
||||
|
||||
The create response — not the index — is the authoritative name of a pull
|
||||
request this harness made, and the index is read to its last page. When either
|
||||
source is uncertain the run fails closed and reports the exact residue plus the
|
||||
exact commands that remove it, so an operator is never left guessing what was
|
||||
created.
|
||||
|
||||
Arming, provenance, and push-target rules live in :mod:`hermes_handoff_arming`
|
||||
and are re-exported here for callers that treat armed mode as one surface.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Sequence
|
||||
from typing import Any
|
||||
|
||||
from hermes_handoff_arming import (
|
||||
CONFIRMATION,
|
||||
REF_PREFIX,
|
||||
ArmingError,
|
||||
ArmRequest,
|
||||
assert_push_target_allowed,
|
||||
bounded_read,
|
||||
normalise_branch,
|
||||
preflight,
|
||||
)
|
||||
from hermes_handoff_exec import Outcome, Runner, Vantage
|
||||
from hermes_handoff_model import EPHEMERAL, FAIL, NOT_RUN, PASS, CheckResult, CheckSpec
|
||||
from hermes_handoff_policy import (
|
||||
EXPECTED_BASE,
|
||||
GITEA_CLIENT,
|
||||
register_ephemeral_pull,
|
||||
)
|
||||
from hermes_handoff_rules import strict_json
|
||||
|
||||
__all__ = [
|
||||
"CONFIRMATION",
|
||||
"DISCOVERY_PAGE_BUDGET",
|
||||
"DISCOVERY_PAGE_SIZE",
|
||||
"REF_PREFIX",
|
||||
"ArmRequest",
|
||||
"ArmingError",
|
||||
"assert_push_target_allowed",
|
||||
"bounded_read",
|
||||
"normalise_branch",
|
||||
"preflight",
|
||||
"run_armed",
|
||||
]
|
||||
|
||||
DRAFT_TITLE_PREFIX = "WIP: "
|
||||
DISCOVERY_PAGE_SIZE = 50
|
||||
DISCOVERY_PAGE_BUDGET = 20
|
||||
|
||||
|
||||
def _spec(identifier: str, title: str) -> CheckSpec:
|
||||
return CheckSpec(
|
||||
id=f"ephemeral.{identifier}",
|
||||
title=title,
|
||||
group="ephemeral",
|
||||
rule="armed",
|
||||
steps=(),
|
||||
mandatory=True,
|
||||
scope=EPHEMERAL,
|
||||
)
|
||||
|
||||
|
||||
def _result(
|
||||
identifier: str,
|
||||
title: str,
|
||||
status: str,
|
||||
reason: str,
|
||||
outcomes: Sequence[Outcome] = (),
|
||||
evidence: dict[str, Any] | None = None,
|
||||
) -> CheckResult:
|
||||
return CheckResult(
|
||||
spec=_spec(identifier, title),
|
||||
status=status,
|
||||
reason=reason,
|
||||
evidence=evidence or {},
|
||||
outcomes=list(outcomes),
|
||||
)
|
||||
|
||||
|
||||
def _protected_refusal_result() -> CheckResult:
|
||||
accepted: list[str] = []
|
||||
for candidate in ("main", "master", "refs/heads/main", "HEAD", f"{REF_PREFIX}/x"):
|
||||
try:
|
||||
assert_push_target_allowed(candidate)
|
||||
except ArmingError:
|
||||
continue
|
||||
accepted.append(candidate)
|
||||
return _result(
|
||||
"protected-branch-refusal",
|
||||
"Arming against a protected branch is refused before any network call",
|
||||
FAIL if accepted else PASS,
|
||||
f"guard accepted {accepted}"
|
||||
if accepted
|
||||
else "every protected and malformed target was refused",
|
||||
evidence={"accepted": accepted},
|
||||
)
|
||||
|
||||
|
||||
def _detail(outcome: Outcome) -> str:
|
||||
return (
|
||||
outcome.error or outcome.combined[:200] or f"exit status {outcome.returncode}"
|
||||
)
|
||||
|
||||
|
||||
def _pr_problems(payload: Any, request: ArmRequest) -> list[str]:
|
||||
if not isinstance(payload, dict):
|
||||
return ["pull request is not an object"]
|
||||
wanted = {
|
||||
"state": "open",
|
||||
"draft": True,
|
||||
"merged": False,
|
||||
"base": {"ref": EXPECTED_BASE, "sha": request.expected_base_sha},
|
||||
"head": {"ref": request.ref, "sha": request.expected_head},
|
||||
}
|
||||
problems: list[str] = []
|
||||
if (
|
||||
not isinstance(payload.get("number"), int)
|
||||
or isinstance(payload.get("number"), bool)
|
||||
or payload["number"] <= 0
|
||||
):
|
||||
problems.append("number is malformed")
|
||||
for field in ("state", "draft", "merged"):
|
||||
if (
|
||||
type(payload.get(field)) is not type(wanted[field])
|
||||
or payload.get(field) != wanted[field]
|
||||
):
|
||||
problems.append(f"{field} does not match")
|
||||
for group, values in (("base", wanted["base"]), ("head", wanted["head"])):
|
||||
observed = payload.get(group)
|
||||
if not isinstance(observed, dict):
|
||||
problems.append(f"{group} is malformed")
|
||||
continue
|
||||
for field, value in values.items():
|
||||
if observed.get(field) != value:
|
||||
problems.append(f"{group}.{field} does not match")
|
||||
return problems
|
||||
|
||||
|
||||
def _created_number(outcome: Outcome, request: ArmRequest) -> tuple[int, str]:
|
||||
"""Return the number the create response itself reports.
|
||||
|
||||
The create response is the only source that names a pull request the harness
|
||||
definitely made. Relying on the index alone means an index that fails or
|
||||
answers ambiguously leaves an unnamed, unclosed pull request behind.
|
||||
"""
|
||||
if not outcome.ok:
|
||||
return (0, f"pull-request creation failed: {_detail(outcome)}")
|
||||
try:
|
||||
payload = strict_json(outcome.stdout)
|
||||
except ValueError as exc:
|
||||
return (0, f"pull-request creation returned malformed JSON: {exc}")
|
||||
problems = _pr_problems(payload, request)
|
||||
if problems:
|
||||
return (0, f"created pull request is invalid: {', '.join(problems)}")
|
||||
return (payload["number"], "")
|
||||
|
||||
|
||||
def _index_pages(
|
||||
runner: Runner, vantage: Vantage, api: str, reserve: float
|
||||
) -> tuple[list[Outcome], list[Any], str]:
|
||||
"""Read every page of the pull index; a truncated page can hide a match."""
|
||||
outcomes: list[Outcome] = []
|
||||
rows: list[Any] = []
|
||||
for page in range(1, DISCOVERY_PAGE_BUDGET + 1):
|
||||
outcome = runner.run(
|
||||
(
|
||||
GITEA_CLIENT,
|
||||
"GET",
|
||||
f"{api}/pulls?state=all&limit={DISCOVERY_PAGE_SIZE}&page={page}",
|
||||
),
|
||||
vantage,
|
||||
reserve_seconds=reserve,
|
||||
)
|
||||
outcomes.append(outcome)
|
||||
if not outcome.ok:
|
||||
return (outcomes, [], f"discovery page {page} failed: {_detail(outcome)}")
|
||||
try:
|
||||
payload = strict_json(outcome.stdout)
|
||||
except ValueError as exc:
|
||||
return (outcomes, [], f"discovery page {page} is malformed JSON: {exc}")
|
||||
if not isinstance(payload, list):
|
||||
return (outcomes, [], f"discovery page {page} did not return an array")
|
||||
rows.extend(payload)
|
||||
if len(payload) < DISCOVERY_PAGE_SIZE:
|
||||
return (outcomes, rows, "")
|
||||
return (outcomes, [], "pull-request discovery exceeded its page budget")
|
||||
|
||||
|
||||
def _residue(request: ArmRequest, numbers: Sequence[int], api: str) -> dict[str, Any]:
|
||||
"""Name the exact residue and the exact commands that remove it by hand."""
|
||||
manual = [
|
||||
f"{GITEA_CLIENT} PATCH {api}/pulls/{number} --field state=closed"
|
||||
for number in numbers
|
||||
]
|
||||
if not numbers:
|
||||
manual.append(
|
||||
f"{GITEA_CLIENT} GET {api}/pulls?state=all&limit={DISCOVERY_PAGE_SIZE}"
|
||||
f" then close by hand every pull whose head ref is {request.ref}"
|
||||
)
|
||||
manual.append(f"{GITEA_CLIENT} DELETE {api}/branches/{request.ref}")
|
||||
return {"residue_ref": request.ref, "manual_cleanup": manual}
|
||||
|
||||
|
||||
def _match_pulls(rows: Sequence[Any], request: ArmRequest) -> tuple[list[int], str]:
|
||||
matches: list[int] = []
|
||||
for index, item in enumerate(rows):
|
||||
if (
|
||||
not isinstance(item, dict)
|
||||
or not isinstance(item.get("head"), dict)
|
||||
or not isinstance(item.get("base"), dict)
|
||||
):
|
||||
return ([], f"pull-request discovery row {index + 1} is malformed")
|
||||
if item["head"].get("ref") != request.ref:
|
||||
continue
|
||||
problems = _pr_problems(item, request)
|
||||
if problems:
|
||||
return ([], f"discovered pull request is invalid: {', '.join(problems)}")
|
||||
matches.append(item["number"])
|
||||
if len(matches) != 1:
|
||||
return (
|
||||
matches,
|
||||
f"expected one exact draft pull request, discovered {len(matches)}",
|
||||
)
|
||||
return (matches, "")
|
||||
|
||||
|
||||
def _closed_pr_problems(payload: Any, request: ArmRequest) -> list[str]:
|
||||
"""Validate the same exact PR identity with state changed to closed."""
|
||||
problems = _pr_problems(payload, request)
|
||||
if isinstance(payload, dict) and payload.get("state") == "closed":
|
||||
problems = [
|
||||
problem for problem in problems if problem != "state does not match"
|
||||
]
|
||||
elif "state does not match" not in problems:
|
||||
problems.append("state is not closed")
|
||||
return problems
|
||||
|
||||
|
||||
def _cleanup(
|
||||
runner: Runner,
|
||||
vantage: Vantage,
|
||||
request: ArmRequest,
|
||||
api: str,
|
||||
numbers: Sequence[int],
|
||||
discovery_error: str = "",
|
||||
) -> CheckResult:
|
||||
outcomes: list[Outcome] = []
|
||||
problems = [discovery_error] if discovery_error else []
|
||||
for number in sorted(set(numbers)):
|
||||
closed = runner.run(
|
||||
(GITEA_CLIENT, "PATCH", f"{api}/pulls/{number}", "--field", "state=closed"),
|
||||
vantage,
|
||||
)
|
||||
outcomes.append(closed)
|
||||
if not closed.ok:
|
||||
problems.append(f"pull request #{number} close failed")
|
||||
continue
|
||||
try:
|
||||
payload = strict_json(closed.stdout)
|
||||
except ValueError:
|
||||
problems.append(f"pull request #{number} close response is malformed")
|
||||
continue
|
||||
expected = _closed_pr_problems(payload, request)
|
||||
if expected:
|
||||
problems.append(f"pull request #{number} close response does not match")
|
||||
deleted = runner.run(
|
||||
(GITEA_CLIENT, "DELETE", f"{api}/branches/{request.ref}"), vantage
|
||||
)
|
||||
outcomes.append(deleted)
|
||||
if not deleted.ok:
|
||||
problems.append("ephemeral branch deletion failed")
|
||||
remaining = runner.run(
|
||||
("git", "ls-remote", request.remote, f"refs/heads/{request.ref}"), vantage
|
||||
)
|
||||
outcomes.append(remaining)
|
||||
if not remaining.ok or remaining.stdout.strip():
|
||||
problems.append("ephemeral branch absence could not be verified")
|
||||
for number in sorted(set(numbers)):
|
||||
observed = runner.run((GITEA_CLIENT, "GET", f"{api}/pulls/{number}"), vantage)
|
||||
outcomes.append(observed)
|
||||
if not observed.ok:
|
||||
problems.append(f"pull request #{number} closure could not be verified")
|
||||
continue
|
||||
try:
|
||||
payload = strict_json(observed.stdout)
|
||||
except ValueError:
|
||||
problems.append(f"pull request #{number} verification is malformed")
|
||||
continue
|
||||
expected = _closed_pr_problems(payload, request)
|
||||
if expected:
|
||||
problems.append(f"pull request #{number} is not exactly closed")
|
||||
return _result(
|
||||
"cleanup-verified",
|
||||
"Every ephemeral ref and pull request is removed and the removal is verified",
|
||||
FAIL if problems else PASS,
|
||||
"; ".join(problems)
|
||||
if problems
|
||||
else "the exact branch and draft pull request are gone",
|
||||
outcomes,
|
||||
{"ref": request.ref, "pull_requests": sorted(set(numbers))},
|
||||
)
|
||||
|
||||
|
||||
def _nothing_created(branch: CheckResult, draft_reason: str) -> list[CheckResult]:
|
||||
"""Report the two results that follow a run which created no artifact."""
|
||||
return [
|
||||
branch,
|
||||
_result(
|
||||
"draft-pull-request",
|
||||
"A draft pull request can be opened and closed",
|
||||
NOT_RUN,
|
||||
draft_reason,
|
||||
),
|
||||
_result(
|
||||
"cleanup-verified",
|
||||
"Every ephemeral artifact is removed",
|
||||
NOT_RUN,
|
||||
"nothing was created",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def run_armed(
|
||||
runner: Runner, vantage: Vantage, request: ArmRequest
|
||||
) -> list[CheckResult]:
|
||||
"""Create, discover, validate, and clean one exact ephemeral branch/PR."""
|
||||
guard = _protected_refusal_result()
|
||||
branch_title = "A unique ephemeral feature branch can be pushed and removed"
|
||||
if guard.status != PASS:
|
||||
unrun = _result("feature-branch-push", branch_title, NOT_RUN, "guard failed")
|
||||
return [guard, *_nothing_created(unrun, "protected-ref guard failed")]
|
||||
api = f"/api/v1/repos/{request.repo}"
|
||||
reserve = min(6000.0, max(300.0, runner.command_timeout * 10))
|
||||
existing = runner.run(
|
||||
("git", "ls-remote", request.remote, f"refs/heads/{request.ref}"),
|
||||
vantage,
|
||||
reserve_seconds=reserve,
|
||||
)
|
||||
if not existing.ok or existing.stdout.strip():
|
||||
reason = (
|
||||
_detail(existing) if not existing.ok else "the ephemeral ref already exists"
|
||||
)
|
||||
refused = _result("feature-branch-push", branch_title, FAIL, reason, [existing])
|
||||
return [guard, *_nothing_created(refused, "branch preflight failed")]
|
||||
local_head = runner.run(
|
||||
("git", "rev-parse", "HEAD"), vantage, reserve_seconds=reserve
|
||||
)
|
||||
pushed = (
|
||||
runner.run(
|
||||
("git", "push", request.remote, f"HEAD:refs/heads/{request.ref}"),
|
||||
vantage,
|
||||
reserve_seconds=reserve,
|
||||
)
|
||||
if local_head.ok and local_head.stdout.strip() == request.expected_head
|
||||
else Outcome(
|
||||
argv=("git", "push"),
|
||||
vantage=vantage.name,
|
||||
error="local HEAD changed after static preflight",
|
||||
)
|
||||
)
|
||||
verified = runner.run(
|
||||
("git", "ls-remote", request.remote, f"refs/heads/{request.ref}"),
|
||||
vantage,
|
||||
reserve_seconds=reserve,
|
||||
)
|
||||
branch_ok = verified.ok and verified.stdout.split() == [
|
||||
request.expected_head,
|
||||
f"refs/heads/{request.ref}",
|
||||
]
|
||||
branch_result = _result(
|
||||
"feature-branch-push",
|
||||
branch_title,
|
||||
PASS if branch_ok else FAIL,
|
||||
"the exact reviewed HEAD was pushed"
|
||||
if branch_ok
|
||||
else "push could not be verified exactly",
|
||||
[local_head, pushed, verified],
|
||||
{"ref": request.ref, "head": request.expected_head},
|
||||
)
|
||||
numbers: list[int] = []
|
||||
problems: list[str] = ["branch creation was not verified"]
|
||||
create_outcomes: list[Outcome] = []
|
||||
if branch_ok:
|
||||
created = runner.run(
|
||||
(
|
||||
GITEA_CLIENT,
|
||||
"POST",
|
||||
f"{api}/pulls",
|
||||
"--field",
|
||||
f"title={DRAFT_TITLE_PREFIX}Hermes handoff acceptance ephemeral probe",
|
||||
"--field",
|
||||
f"head={request.ref}",
|
||||
"--field",
|
||||
f"base={EXPECTED_BASE}",
|
||||
"--field",
|
||||
"body=Ephemeral acceptance probe. Closed and deleted by the harness.",
|
||||
),
|
||||
vantage,
|
||||
reserve_seconds=reserve,
|
||||
)
|
||||
number, create_error = _created_number(created, request)
|
||||
pages, rows, index_error = _index_pages(runner, vantage, api, reserve)
|
||||
found, match_error = (
|
||||
_match_pulls(rows, request) if not index_error else ([], "")
|
||||
)
|
||||
create_outcomes = [created, *pages]
|
||||
# Either source alone names a pull request this run definitely made: the
|
||||
# ephemeral ref was proven absent in preflight, so any pull on it is ours.
|
||||
numbers = sorted({item for item in (number, *found) if item})
|
||||
problems = [text for text in (index_error, match_error) if text]
|
||||
if number and found and found != [number]:
|
||||
problems.append(f"created #{number} is not the discovered {found}")
|
||||
if not numbers:
|
||||
problems.append(
|
||||
create_error or "no exact draft pull request was identified"
|
||||
)
|
||||
for item in numbers:
|
||||
register_ephemeral_pull(item)
|
||||
exact = len(numbers) == 1 and not problems
|
||||
draft_result = _result(
|
||||
"draft-pull-request",
|
||||
"A draft pull request can be opened and closed against the ephemeral branch",
|
||||
PASS if exact else FAIL,
|
||||
f"discovered exact draft pull request #{numbers[0]}"
|
||||
if exact
|
||||
else "; ".join(problems),
|
||||
create_outcomes,
|
||||
{"pull_requests": numbers}
|
||||
if exact
|
||||
else {"pull_requests": numbers, **_residue(request, numbers, api)},
|
||||
)
|
||||
cleanup = _cleanup(runner, vantage, request, api, numbers, "; ".join(problems))
|
||||
return [guard, branch_result, draft_result, cleanup]
|
||||
497
scripts/ops/hermes_handoff_evaluators.py
Normal file
497
scripts/ops/hermes_handoff_evaluators.py
Normal file
@ -0,0 +1,497 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Text-shaped evaluators for the handoff acceptance harness.
|
||||
|
||||
These rules cover evidence that arrives as a refusal, an exit status, or a list
|
||||
of names. The ``denied`` rule is the load-bearing one: it refuses to classify on
|
||||
an authorization review alone — ``kubectl auth can-i`` reports what RBAC says,
|
||||
not what the API server does — so a real request must have been made and
|
||||
refused before it will pass.
|
||||
|
||||
An absence is only evidence when the probe observed something, so a name step
|
||||
that exits 0 with no lines is ``NOT_RUN``: drift produces silence, not a pass.
|
||||
|
||||
Importing this module also registers the JSON-shaped rules, so a caller only has
|
||||
to import one place to have the full catalog vocabulary available.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from hermes_handoff_exec import Outcome
|
||||
from hermes_handoff_json_rules import evaluate_flux_health # noqa: F401 (registers rules)
|
||||
from hermes_handoff_model import ATTEMPT, FAIL, NOT_RUN, PASS, REVIEW, CheckSpec
|
||||
from hermes_handoff_rules import (
|
||||
EVALUATORS,
|
||||
Evaluation,
|
||||
dotted,
|
||||
evaluate,
|
||||
evaluator,
|
||||
exact_equal,
|
||||
lines,
|
||||
looks_denied,
|
||||
missing_steps,
|
||||
step_keys,
|
||||
strict_json,
|
||||
)
|
||||
|
||||
__all__ = ["EVALUATORS", "Evaluation", "evaluate", "evaluator"]
|
||||
|
||||
|
||||
@evaluator("denied")
|
||||
def evaluate_denied(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
|
||||
"""Pass only when the review says no and a real request was refused."""
|
||||
blocked = missing_steps(spec, outcomes)
|
||||
if blocked:
|
||||
return blocked
|
||||
attempts = step_keys(spec, ATTEMPT)
|
||||
if not attempts:
|
||||
return Evaluation(
|
||||
NOT_RUN, "deny checks require a real attempt, not only a review"
|
||||
)
|
||||
evidence: dict[str, Any] = {}
|
||||
for key in step_keys(spec, REVIEW):
|
||||
verdict = outcomes[key].stdout.strip().lower()
|
||||
evidence[key] = verdict
|
||||
if verdict != "no":
|
||||
return Evaluation(
|
||||
FAIL,
|
||||
f"authorization review {key} reported {verdict or 'nothing'}",
|
||||
evidence,
|
||||
)
|
||||
for key in attempts:
|
||||
outcome = outcomes[key]
|
||||
if outcome.ok:
|
||||
return Evaluation(
|
||||
FAIL, f"attempt {key} succeeded but must be refused", evidence
|
||||
)
|
||||
if not looks_denied(outcome):
|
||||
return Evaluation(
|
||||
NOT_RUN,
|
||||
f"attempt {key} failed without a recognisable refusal (rc={outcome.returncode})",
|
||||
evidence,
|
||||
)
|
||||
evidence[key] = "refused"
|
||||
return Evaluation(PASS, "review and live attempt were both refused", evidence)
|
||||
|
||||
|
||||
@evaluator("review_denied")
|
||||
def evaluate_review_denied(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
|
||||
"""Pass only when every side-effect-free authorization review says no."""
|
||||
blocked = missing_steps(spec, outcomes)
|
||||
if blocked:
|
||||
return blocked
|
||||
reviews = step_keys(spec, REVIEW)
|
||||
if not reviews:
|
||||
return Evaluation(NOT_RUN, "a denial check requires an authorization review")
|
||||
evidence: dict[str, str] = {}
|
||||
for key in reviews:
|
||||
verdict = outcomes[key].stdout.strip().lower()
|
||||
evidence[key] = verdict
|
||||
if verdict != "no":
|
||||
return Evaluation(
|
||||
FAIL,
|
||||
f"authorization review {key} reported {verdict or 'nothing'}",
|
||||
evidence,
|
||||
)
|
||||
return Evaluation(PASS, "every authorization review was denied", evidence)
|
||||
|
||||
|
||||
@evaluator("allowed")
|
||||
def evaluate_allowed(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
|
||||
"""Pass when every required step succeeded and every review said yes."""
|
||||
blocked = missing_steps(spec, outcomes)
|
||||
if blocked:
|
||||
return blocked
|
||||
evidence: dict[str, Any] = {}
|
||||
for step in spec.steps:
|
||||
outcome = outcomes[step.key]
|
||||
if step.kind == REVIEW:
|
||||
verdict = outcome.stdout.strip().lower()
|
||||
evidence[step.key] = verdict
|
||||
if verdict != "yes":
|
||||
return Evaluation(
|
||||
FAIL,
|
||||
f"authorization review {step.key} reported {verdict or 'nothing'}",
|
||||
evidence,
|
||||
)
|
||||
continue
|
||||
if not outcome.ok:
|
||||
return Evaluation(
|
||||
FAIL, f"step {step.key} failed (rc={outcome.returncode})", evidence
|
||||
)
|
||||
evidence[step.key] = "succeeded"
|
||||
return Evaluation(PASS, "required access is present", evidence)
|
||||
|
||||
|
||||
def _matched(observed: set[str], patterns: tuple[str, ...]) -> set[str]:
|
||||
return {name for pattern in patterns for name in observed if pattern in name}
|
||||
|
||||
|
||||
def _line_step(
|
||||
spec: CheckSpec, outcomes: dict[str, Outcome]
|
||||
) -> tuple[set[str], str] | Evaluation:
|
||||
"""Return a step's distinct lines, or the NOT_RUN that silence must produce."""
|
||||
key = spec.expect["step"]
|
||||
outcome = outcomes[key]
|
||||
if not outcome.ok:
|
||||
return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})")
|
||||
observed = set(lines(outcome))
|
||||
if not observed:
|
||||
return Evaluation(NOT_RUN, f"step {key} exited 0 with no lines to examine")
|
||||
return (observed, key)
|
||||
|
||||
|
||||
@evaluator("names_absent")
|
||||
def evaluate_names_absent(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
|
||||
"""Pass when a step observed names and none of them is forbidden."""
|
||||
blocked = missing_steps(spec, outcomes)
|
||||
if blocked:
|
||||
return blocked
|
||||
resolved = _line_step(spec, outcomes)
|
||||
if isinstance(resolved, Evaluation):
|
||||
return resolved
|
||||
observed, _ = resolved
|
||||
offenders = sorted(
|
||||
(observed & set(spec.expect.get("names", ())))
|
||||
| _matched(observed, tuple(spec.expect.get("contains", ())))
|
||||
)
|
||||
evidence = {"observed_count": len(observed), "offenders": offenders}
|
||||
if offenders:
|
||||
return Evaluation(
|
||||
FAIL, f"forbidden names present: {', '.join(offenders)}", evidence
|
||||
)
|
||||
return Evaluation(PASS, "no forbidden names are present", evidence)
|
||||
|
||||
|
||||
@evaluator("names_present")
|
||||
def evaluate_names_present(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
|
||||
"""Pass when every required name, and every required substring, appears."""
|
||||
blocked = missing_steps(spec, outcomes)
|
||||
if blocked:
|
||||
return blocked
|
||||
resolved = _line_step(spec, outcomes)
|
||||
if isinstance(resolved, Evaluation):
|
||||
return resolved
|
||||
observed, _ = resolved
|
||||
missing = sorted(set(spec.expect.get("names", ())) - observed)
|
||||
missing += sorted(
|
||||
pattern
|
||||
for pattern in spec.expect.get("contains", ())
|
||||
if not any(pattern in name for name in observed)
|
||||
)
|
||||
evidence = {"observed_count": len(observed), "missing": missing}
|
||||
if missing:
|
||||
return Evaluation(
|
||||
FAIL, f"required names absent: {', '.join(missing)}", evidence
|
||||
)
|
||||
return Evaluation(PASS, "every required name is present", evidence)
|
||||
|
||||
|
||||
@evaluator("stdout_matches")
|
||||
def evaluate_stdout_matches(
|
||||
spec: CheckSpec, outcomes: dict[str, Outcome]
|
||||
) -> Evaluation:
|
||||
"""Compare a step's trimmed stdout against an expected value or set."""
|
||||
blocked = missing_steps(spec, outcomes)
|
||||
if blocked:
|
||||
return blocked
|
||||
key = spec.expect["step"]
|
||||
outcome = outcomes[key]
|
||||
if not outcome.ok:
|
||||
return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})")
|
||||
actual = outcome.stdout.strip()
|
||||
expected = spec.expect["equals"]
|
||||
allowed = expected if isinstance(expected, list) else [expected]
|
||||
evidence = {"observed": actual}
|
||||
if actual not in allowed:
|
||||
return Evaluation(
|
||||
FAIL, f"observed {actual!r}, expected one of {allowed!r}", evidence
|
||||
)
|
||||
return Evaluation(PASS, "output matches the expected value", evidence)
|
||||
|
||||
|
||||
@evaluator("distinct_count")
|
||||
def evaluate_distinct_count(
|
||||
spec: CheckSpec, outcomes: dict[str, Outcome]
|
||||
) -> Evaluation:
|
||||
"""Pass when a step yields enough distinct rows — and, optionally, exactly N.
|
||||
|
||||
Distinctness is the point for placement checks: three worker pods pinned to
|
||||
one node satisfy a replica count and defeat the isolation the replica count
|
||||
was standing in for.
|
||||
"""
|
||||
blocked = missing_steps(spec, outcomes)
|
||||
if blocked:
|
||||
return blocked
|
||||
key = spec.expect["step"]
|
||||
outcome = outcomes[key]
|
||||
if not outcome.ok:
|
||||
return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})")
|
||||
values = lines(outcome)
|
||||
distinct = sorted(set(values))
|
||||
minimum = spec.expect["minimum"]
|
||||
evidence = {
|
||||
"total": len(values),
|
||||
"distinct": len(distinct),
|
||||
"values": distinct[:16],
|
||||
}
|
||||
if not values:
|
||||
return Evaluation(NOT_RUN, f"step {key} produced no rows to count", evidence)
|
||||
if len(distinct) < minimum:
|
||||
return Evaluation(
|
||||
FAIL,
|
||||
f"{len(distinct)} distinct values, expected at least {minimum}",
|
||||
evidence,
|
||||
)
|
||||
total_equals = spec.expect.get("total_equals")
|
||||
if total_equals is not None and len(values) != total_equals:
|
||||
return Evaluation(
|
||||
FAIL, f"{len(values)} rows, expected exactly {total_equals}", evidence
|
||||
)
|
||||
return Evaluation(PASS, f"{len(distinct)} distinct values observed", evidence)
|
||||
|
||||
|
||||
@evaluator("lines_match")
|
||||
def evaluate_lines_match(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
|
||||
"""Pass when every data row of a tabular listing matches a shape.
|
||||
|
||||
Some surfaces only speak tables. A blank leading column is exactly what the
|
||||
session-render regression looked like, so asserting the shape of each row is
|
||||
a real check rather than a proxy for one.
|
||||
"""
|
||||
blocked = missing_steps(spec, outcomes)
|
||||
if blocked:
|
||||
return blocked
|
||||
key = spec.expect["step"]
|
||||
outcome = outcomes[key]
|
||||
if not outcome.ok:
|
||||
return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})")
|
||||
pattern = re.compile(spec.expect["pattern"])
|
||||
rows = outcome.stdout.splitlines()[spec.expect.get("skip", 0) :]
|
||||
rows = [row for row in rows if row.strip()]
|
||||
offenders = [row[:60] for row in rows if not pattern.search(row)]
|
||||
evidence = {"rows": len(rows), "offenders": offenders[:8]}
|
||||
if len(rows) < spec.expect.get("minimum", 1):
|
||||
return Evaluation(
|
||||
NOT_RUN,
|
||||
f"step {key} produced {len(rows)} data rows, expected at least "
|
||||
f"{spec.expect.get('minimum', 1)}",
|
||||
evidence,
|
||||
)
|
||||
if offenders:
|
||||
return Evaluation(
|
||||
FAIL, f"{len(offenders)} row(s) do not match the expected shape", evidence
|
||||
)
|
||||
return Evaluation(PASS, f"all {len(rows)} rows match the expected shape", evidence)
|
||||
|
||||
|
||||
@evaluator("vantages_agree")
|
||||
def evaluate_vantages_agree(
|
||||
spec: CheckSpec, outcomes: dict[str, Outcome]
|
||||
) -> Evaluation:
|
||||
"""Pass only when two independent observations report the same fact.
|
||||
|
||||
A disagreement is a finding in its own right: it usually means the manifest
|
||||
on record and the workload actually running have drifted apart, and either
|
||||
one alone would have looked clean.
|
||||
"""
|
||||
blocked = missing_steps(spec, outcomes)
|
||||
if blocked:
|
||||
return blocked
|
||||
left_key, right_key = spec.expect["steps"]
|
||||
observations: dict[str, list[str]] = {}
|
||||
for key in (left_key, right_key):
|
||||
outcome = outcomes[key]
|
||||
if not outcome.ok:
|
||||
return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})")
|
||||
observations[key] = sorted(lines(outcome))
|
||||
evidence = {key: value[:16] for key, value in observations.items()}
|
||||
if observations[left_key] != observations[right_key]:
|
||||
return Evaluation(
|
||||
FAIL, f"vantages disagree: {left_key} and {right_key} differ", evidence
|
||||
)
|
||||
if not observations[left_key]:
|
||||
return Evaluation(
|
||||
NOT_RUN, "both vantages returned nothing to compare", evidence
|
||||
)
|
||||
return Evaluation(PASS, "both vantages report the same state", evidence)
|
||||
|
||||
|
||||
@evaluator("not_armed")
|
||||
def evaluate_not_armed(spec: CheckSpec, _outcomes: dict[str, Outcome]) -> Evaluation:
|
||||
"""Report the fixed NOT_RUN a default, read-only run must produce."""
|
||||
reason = spec.expect.get("reason", "ephemeral mutation mode is not armed")
|
||||
return Evaluation(NOT_RUN, reason, {"armed": False})
|
||||
|
||||
|
||||
# This validator is one compact, linear state machine so every exit observes
|
||||
# only fully validated state.
|
||||
# fmt: off
|
||||
@evaluator("pool_topology")
|
||||
def evaluate_pool_topology(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
|
||||
"""Require exactly three Ready ordinal workers on distinct nodes and claims."""
|
||||
blocked = missing_steps(spec, outcomes)
|
||||
if blocked:
|
||||
return blocked
|
||||
state = outcomes[spec.expect["state_step"]].stdout.strip().split("\t")
|
||||
if len(state) != 4:
|
||||
return Evaluation(NOT_RUN, "StatefulSet projection is malformed")
|
||||
expected = spec.expect.get("replicas", 3)
|
||||
try:
|
||||
replicas, ready, current = (int(value) for value in state[:3])
|
||||
except ValueError:
|
||||
return Evaluation(NOT_RUN, "StatefulSet replica fields are malformed")
|
||||
evidence: dict[str, Any] = {"replicas": replicas, "ready": ready, "current": current,
|
||||
"automountServiceAccountToken": state[3]}
|
||||
if (replicas, ready, current, state[3].lower()) != (expected, expected, expected, "false"):
|
||||
return Evaluation(FAIL, "worker replica or tokenless state does not match the contract", evidence)
|
||||
workers: dict[int, tuple[str, str]] = {}
|
||||
names: list[str] = []
|
||||
for number, row in enumerate(lines(outcomes[spec.expect["worker_step"]]), 1):
|
||||
columns = row.split("\t")
|
||||
if len(columns) != 5:
|
||||
return Evaluation(NOT_RUN, f"worker row {number} is malformed", evidence)
|
||||
name, node, phase, ready_state, claims = columns
|
||||
match = re.fullmatch(rf"{re.escape(spec.expect['name'])}-(\d+)", name)
|
||||
claim_list = [claim for claim in claims.split(",") if claim]
|
||||
if (not match or not node or phase != "Running" or ready_state != "True"
|
||||
or len(claim_list) != 1):
|
||||
return Evaluation(FAIL, f"worker row {number} is not a Ready isolated worker", evidence)
|
||||
ordinal = int(match.group(1))
|
||||
if ordinal in workers:
|
||||
return Evaluation(FAIL, f"worker ordinal {ordinal} is duplicated", evidence)
|
||||
workers[ordinal] = (node, claim_list[0])
|
||||
names.append(name)
|
||||
evidence.update({"workers": sorted(names), "nodes": sorted({v[0] for v in workers.values()}),
|
||||
"claims": sorted({v[1] for v in workers.values()})})
|
||||
if set(workers) != set(range(expected)):
|
||||
return Evaluation(FAIL, f"worker ordinals are not exactly 0..{expected - 1}", evidence)
|
||||
if (len({v[0] for v in workers.values()}) != expected
|
||||
or len({v[1] for v in workers.values()}) != expected):
|
||||
return Evaluation(FAIL, "workers do not occupy distinct nodes and claims", evidence)
|
||||
return Evaluation(PASS, "three Ready workers have distinct nodes, ordinals, and claims", evidence)
|
||||
# fmt: on
|
||||
|
||||
|
||||
_SHA_RE = re.compile(r"[0-9a-f]{40}\Z")
|
||||
_DIGEST_RE = re.compile(r"sha256:[0-9a-f]{64}\Z")
|
||||
_IMAGE_RE = re.compile(
|
||||
r"[^@\s]+:git-([0-9a-f]{40})-build-([1-9][0-9]*)@(sha256:[0-9a-f]{64})\Z"
|
||||
)
|
||||
|
||||
|
||||
def _strict_object(outcome: Outcome, label: str) -> dict[str, Any] | Evaluation:
|
||||
try:
|
||||
payload = strict_json(outcome.stdout)
|
||||
except ValueError as exc:
|
||||
return Evaluation(NOT_RUN, f"{label} JSON is malformed: {exc}")
|
||||
if not isinstance(payload, dict):
|
||||
return Evaluation(NOT_RUN, f"{label} JSON is not an object")
|
||||
return payload
|
||||
|
||||
|
||||
# fmt: off
|
||||
@evaluator("release_lineage")
|
||||
def evaluate_release_lineage(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
|
||||
"""Atomically bind Git, PR, image, build, Flux, and running revision evidence."""
|
||||
blocked = missing_steps(spec, outcomes)
|
||||
if blocked:
|
||||
return blocked
|
||||
expected = spec.expect
|
||||
main_sha = expected["main_sha"]
|
||||
head_sha = expected["head_sha"]
|
||||
image = expected["image"]
|
||||
build_sha = expected["build_sha"]
|
||||
revision = str(expected["deployment_revision"])
|
||||
digest = image.rsplit("@", 1)[-1]
|
||||
image_match = _IMAGE_RE.fullmatch(image)
|
||||
if not (_SHA_RE.fullmatch(main_sha) and _SHA_RE.fullmatch(head_sha)
|
||||
and _SHA_RE.fullmatch(build_sha) and _DIGEST_RE.fullmatch(digest)
|
||||
and image_match and image_match.group(1) == build_sha and build_sha == main_sha):
|
||||
return Evaluation(NOT_RUN, "release lineage expectations are malformed")
|
||||
remote = outcomes["remote-main"].stdout.strip().split()
|
||||
if remote != [main_sha, "refs/heads/main"]:
|
||||
return Evaluation(FAIL, "remote main does not match the fixed expected SHA",
|
||||
{"remote_main": remote[:2]})
|
||||
pr = _strict_object(outcomes["reviewed-pr"], "reviewed PR")
|
||||
if isinstance(pr, Evaluation):
|
||||
return pr
|
||||
fields = {
|
||||
"number": expected["pr_number"],
|
||||
"state": "open",
|
||||
"draft": True,
|
||||
"merged": False,
|
||||
"base.ref": "main",
|
||||
"base.sha": main_sha,
|
||||
"head.ref": expected["head_ref"],
|
||||
"head.sha": head_sha,
|
||||
}
|
||||
for path, wanted in fields.items():
|
||||
try:
|
||||
actual = dotted(pr, path)
|
||||
except (KeyError, ValueError, IndexError):
|
||||
return Evaluation(NOT_RUN, f"reviewed PR is missing {path}")
|
||||
if not exact_equal(actual, wanted):
|
||||
return Evaluation(FAIL, f"reviewed PR {path} does not match", {path: actual})
|
||||
if outcomes["build-source"].stdout.strip() != build_sha:
|
||||
return Evaluation(FAIL, "build source SHA does not match the fixed release build SHA")
|
||||
deployment = outcomes["deployment"].stdout.strip().split("\t")
|
||||
if len(deployment) != 6:
|
||||
return Evaluation(NOT_RUN, "Deployment projection is malformed")
|
||||
generation, observed, replicas, ready, deployed_revision, deployed_image = deployment
|
||||
try:
|
||||
current = (int(generation) > 0 and generation == observed
|
||||
and int(replicas) > 0 and replicas == ready)
|
||||
except ValueError:
|
||||
return Evaluation(NOT_RUN, "Deployment numeric fields are malformed")
|
||||
if not current or (deployed_revision, deployed_image) != (revision, image):
|
||||
return Evaluation(FAIL, "Deployment lineage does not match the release contract")
|
||||
replica_hash = ""
|
||||
for number, row in enumerate(lines(outcomes["replicasets"]), 1):
|
||||
columns = row.split("\t")
|
||||
if len(columns) != 6:
|
||||
return Evaluation(NOT_RUN, f"ReplicaSet row {number} is malformed")
|
||||
_, rs_revision, rs_ready, rs_available, rs_image, rs_hash = columns
|
||||
if rs_revision != revision:
|
||||
continue
|
||||
if (rs_ready != replicas or rs_available != replicas or rs_image != image or not rs_hash):
|
||||
return Evaluation(FAIL, "active ReplicaSet does not match the Deployment lineage")
|
||||
if replica_hash:
|
||||
return Evaluation(FAIL, "more than one ReplicaSet claims the running revision")
|
||||
replica_hash = rs_hash
|
||||
if not replica_hash:
|
||||
return Evaluation(FAIL, "running Deployment revision has no matching ReplicaSet")
|
||||
pod_rows = lines(outcomes["pods"])
|
||||
if len(pod_rows) != int(replicas):
|
||||
return Evaluation(FAIL, "running pod count does not match the Deployment")
|
||||
pods: list[str] = []
|
||||
for number, row in enumerate(pod_rows, 1):
|
||||
columns = row.split("\t")
|
||||
if len(columns) != 6:
|
||||
return Evaluation(NOT_RUN, f"pod row {number} is malformed")
|
||||
name, phase, ready_state, pod_image, image_id, pod_hash = columns
|
||||
if phase != "Running" or ready_state != "True":
|
||||
return Evaluation(FAIL, f"pod {name or number} is not Ready")
|
||||
if (pod_image, pod_hash) != (image, replica_hash) or not image_id.endswith(f"@{digest}"):
|
||||
return Evaluation(FAIL, f"pod {name or number} lineage does not match")
|
||||
pods.append(name)
|
||||
flux = outcomes["flux"].stdout.strip().split("\t")
|
||||
if len(flux) != 6:
|
||||
return Evaluation(NOT_RUN, "Flux lineage projection is malformed")
|
||||
name, generation, observed, suspended, ready_state, applied = flux
|
||||
if (not name or generation != observed or suspended.lower() == "true" or ready_state != "True"):
|
||||
return Evaluation(FAIL, "Flux source is not current and Ready")
|
||||
match = re.search(r"sha1:([0-9a-f]{40})\Z", applied)
|
||||
if not match or match.group(1) != main_sha:
|
||||
return Evaluation(FAIL, "Flux applied revision is not the expected remote main")
|
||||
return Evaluation(PASS, "Git, PR, image, build, Flux, and running revision are bound",
|
||||
{"main_sha": main_sha, "head_sha": head_sha, "image_digest": digest,
|
||||
"build_sha": build_sha, "deployment_revision": revision, "pods": pods})
|
||||
# fmt: on
|
||||
|
||||
|
||||
# Two observations of the same fact agree whether or not they came from
|
||||
# different vantages; the dual-vantage case is just the important one.
|
||||
EVALUATORS["steps_agree"] = evaluate_vantages_agree
|
||||
482
scripts/ops/hermes_handoff_exec.py
Normal file
482
scripts/ops/hermes_handoff_exec.py
Normal file
@ -0,0 +1,482 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Bounded, screened, provenance-attested command execution.
|
||||
|
||||
It bounds pipes, times out and reaps the process group, and discards truncated output.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import math
|
||||
import os
|
||||
import selectors
|
||||
import shlex
|
||||
import shutil
|
||||
import signal
|
||||
import stat
|
||||
import subprocess
|
||||
import time
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from contextlib import suppress
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from hermes_handoff_policy import GITEA_CLIENT, READ_ONLY, PolicyError, check_argv
|
||||
from hermes_handoff_redaction import (
|
||||
DEFAULT_MAX_BYTES,
|
||||
TRUNCATION_NOTE,
|
||||
redact,
|
||||
safe_text,
|
||||
scrub,
|
||||
)
|
||||
|
||||
DEFAULT_COMMAND_TIMEOUT = 25.0
|
||||
DEFAULT_RUN_DEADLINE = 900.0
|
||||
DEFAULT_CLEANUP_RESERVE = 90.0
|
||||
MAX_COMMAND_TIMEOUT = 600.0
|
||||
MAX_RUN_DEADLINE = 7200.0
|
||||
MAX_OUTPUT_BYTES = 8 * 1024 * 1024
|
||||
READ_CHUNK = 64 * 1024
|
||||
PIPE_DRAIN_SECONDS = 0.25
|
||||
|
||||
SAFE_PATH = "/opt/data/tools/bin:/opt/hermes/.venv/bin:/opt/coordinator:/usr/local/bin:/usr/bin:/bin"
|
||||
# These exact allowlists are intentionally dense: reviewers compare them as a
|
||||
# single provenance boundary rather than as extensible application config.
|
||||
# fmt: off
|
||||
ENV_PASSTHROUGH = ("KUBECONFIG", "LANG", "LC_ALL", "REQUESTS_CA_BUNDLE", "SSL_CERT_FILE")
|
||||
FIXED_ENVIRONMENT = {"GIT_ASKPASS": "/opt/coordinator/gitea_askpass.sh",
|
||||
"GIT_CONFIG_GLOBAL": "/dev/null", "GIT_CONFIG_NOSYSTEM": "1",
|
||||
"GIT_TERMINAL_PROMPT": "0", "GITEA_BASE_URL": "https://scm.bstein.dev",
|
||||
"LC_ALL": "C", "PATH": SAFE_PATH}
|
||||
EXPECTED_PATHS = {
|
||||
"kubectl": {"/opt/data/tools/bin/kubectl", "/usr/local/bin/kubectl", "/usr/bin/kubectl"},
|
||||
"git": {"/usr/bin/git", "/bin/git"},
|
||||
"hermes": {"/opt/hermes/.venv/bin/hermes"},
|
||||
"sh": {"/usr/bin/sh", "/bin/sh"},
|
||||
GITEA_CLIENT: {GITEA_CLIENT},
|
||||
}
|
||||
EXPECTED_SHA256 = {
|
||||
"kubectl": {"3d514dbae5dc8c09f773df0ef0f5d449dfad05b3aca5c96b13565f886df345fd"},
|
||||
"git": {"a0e562e4bd3c4c79379e91d8c07a10104b2cefe8fac966dc6bd4874a57a807f3"},
|
||||
"sh": {"367967c823a0c391e5049b15a67c6a0a629c88b9b6dcdca75ef13ac9d65334b1"},
|
||||
"hermes": {"c8419290ef7f1a95f59eadcea9f58f446b1e6535a7f4ffee9f1c0cb1172335e4"},
|
||||
GITEA_CLIENT: {"f0943db4e81ce96908e94b0f0e78aa4b5ef95d6c4f7803f9f74a43e761576d85"},
|
||||
}
|
||||
|
||||
DEADLINE_ERROR = "deadline-exceeded"
|
||||
TIMEOUT_ERROR = "timeout"
|
||||
OPERATOR = "operator"
|
||||
SELF = "self"
|
||||
CONTEXT_AWARE_BINARIES = ("kubectl",)
|
||||
POD_COMMAND_PATHS = {"kubectl": "/usr/local/bin/kubectl", "hermes": "/opt/hermes/.venv/bin/hermes",
|
||||
"git": "/usr/bin/git", "sh": "/bin/sh", GITEA_CLIENT: GITEA_CLIENT}
|
||||
# fmt: on
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Attestation:
|
||||
"""The exact outer executable used for a probe."""
|
||||
|
||||
path: str
|
||||
sha256: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Capture:
|
||||
"""Raw bounded process result returned by an executor implementation."""
|
||||
|
||||
returncode: int
|
||||
stdout: bytes = b""
|
||||
stderr: bytes = b""
|
||||
stdout_truncated: bool = False
|
||||
stderr_truncated: bool = False
|
||||
timed_out: bool = False
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Outcome:
|
||||
"""The screened, bounded result of one probe command."""
|
||||
|
||||
argv: tuple[str, ...]
|
||||
vantage: str
|
||||
returncode: int | None = None
|
||||
stdout: str = ""
|
||||
stderr: str = ""
|
||||
truncated: bool = False
|
||||
duration_ms: int = 0
|
||||
error: str | None = None
|
||||
executable_path: str = ""
|
||||
executable_sha256: str = ""
|
||||
|
||||
@property
|
||||
def ok(self) -> bool:
|
||||
return self.error is None and self.returncode == 0 and not self.truncated
|
||||
|
||||
@property
|
||||
def ran(self) -> bool:
|
||||
return self.error is None
|
||||
|
||||
@property
|
||||
def combined(self) -> str:
|
||||
return "\n".join(part for part in (self.stdout, self.stderr) if part)
|
||||
|
||||
def as_dict(self) -> dict[str, object]:
|
||||
return scrub(
|
||||
{
|
||||
"command": redact(shlex.join(self.argv)),
|
||||
"vantage": self.vantage,
|
||||
"returncode": self.returncode,
|
||||
"stdout": self.stdout,
|
||||
"stderr": self.stderr,
|
||||
"truncated": self.truncated,
|
||||
"duration_ms": self.duration_ms,
|
||||
"error": self.error,
|
||||
"executable_path": self.executable_path,
|
||||
"executable_sha256": self.executable_sha256,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Vantage:
|
||||
"""A named identity and transport used to observe a probe."""
|
||||
|
||||
name: str
|
||||
prefix: tuple[str, ...] = ()
|
||||
inserts: Mapping[str, tuple[str, ...]] = field(default_factory=dict)
|
||||
env: Mapping[str, str] = field(default_factory=dict)
|
||||
command_paths: Mapping[str, str] = field(default_factory=dict)
|
||||
description: str = ""
|
||||
|
||||
def wrap(self, argv: Sequence[str]) -> tuple[str, ...]:
|
||||
command = tuple(argv)
|
||||
if command:
|
||||
replacement = self.command_paths.get(command[0])
|
||||
if replacement:
|
||||
command = (replacement, *command[1:])
|
||||
extra = self.inserts.get(command[0].rsplit("/", 1)[-1])
|
||||
if extra:
|
||||
command = (command[0], *extra, *command[1:])
|
||||
return (*self.prefix, *command)
|
||||
|
||||
|
||||
def _finite_number(name: str, value: Any, minimum: float, maximum: float) -> float:
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
||||
raise ValueError(f"{name} must be numeric")
|
||||
converted = float(value)
|
||||
if not math.isfinite(converted) or not minimum <= converted <= maximum:
|
||||
raise ValueError(
|
||||
f"{name} must be finite and between {minimum:g} and {maximum:g}"
|
||||
)
|
||||
return converted
|
||||
|
||||
|
||||
def _byte_budget(value: Any) -> int:
|
||||
if (
|
||||
isinstance(value, bool)
|
||||
or not isinstance(value, int)
|
||||
or not 1 <= value <= MAX_OUTPUT_BYTES
|
||||
):
|
||||
raise ValueError(
|
||||
f"max_bytes must be an integer between 1 and {MAX_OUTPUT_BYTES}"
|
||||
)
|
||||
return value
|
||||
|
||||
|
||||
def build_environment(
|
||||
vantage: Vantage, source: Mapping[str, str] | None = None
|
||||
) -> dict[str, str]:
|
||||
"""Build a deterministic child environment without caller-controlled PATH."""
|
||||
origin = os.environ if source is None else source
|
||||
environment = {name: origin[name] for name in ENV_PASSTHROUGH if name in origin}
|
||||
environment.update(FIXED_ENVIRONMENT)
|
||||
environment.update(vantage.env)
|
||||
environment["PATH"] = SAFE_PATH
|
||||
environment["GITEA_BASE_URL"] = FIXED_ENVIRONMENT["GITEA_BASE_URL"]
|
||||
environment["GIT_ASKPASS"] = FIXED_ENVIRONMENT["GIT_ASKPASS"]
|
||||
return environment
|
||||
|
||||
|
||||
# fmt: off
|
||||
def operator_vantage(kubeconfig: str | None = None, context: str | None = None) -> Vantage:
|
||||
inserts = dict.fromkeys(CONTEXT_AWARE_BINARIES, ("--context", context)) if context else {}
|
||||
detail = " ".join(part for part in (f"context={context}" if context else "",
|
||||
"kubeconfig-pinned" if kubeconfig else "") if part)
|
||||
return Vantage(name=OPERATOR, inserts=inserts,
|
||||
env={"KUBECONFIG": kubeconfig} if kubeconfig else {},
|
||||
description=f"operator {detail}".strip())
|
||||
|
||||
|
||||
def pod_vantage(namespace: str, pod: str, container: str, operator: Vantage | None = None) -> Vantage:
|
||||
reach = operator.inserts.get("kubectl", ()) if operator else ()
|
||||
return Vantage(name=SELF, prefix=("kubectl", *reach, "exec", "--namespace", namespace,
|
||||
pod, "--container", container, "--"),
|
||||
env=dict(operator.env) if operator else {}, command_paths=POD_COMMAND_PATHS,
|
||||
description=f"in-pod {namespace}/{pod}[{container}]")
|
||||
# fmt: on
|
||||
|
||||
|
||||
def attest_executable(command: str, environment: Mapping[str, str]) -> Attestation:
|
||||
"""Resolve and hash an executable from a fixed, trusted path set."""
|
||||
name = command if command == GITEA_CLIENT else command.rsplit("/", 1)[-1]
|
||||
candidate = (
|
||||
command if "/" in command else shutil.which(command, path=environment["PATH"])
|
||||
)
|
||||
if not candidate:
|
||||
raise OSError(f"trusted executable is unavailable: {name}")
|
||||
resolved = str(Path(candidate).resolve(strict=True))
|
||||
allowed = {
|
||||
str(Path(item).resolve(strict=False))
|
||||
for item in EXPECTED_PATHS.get(name, set())
|
||||
}
|
||||
if resolved not in allowed:
|
||||
raise OSError(f"executable resolved outside its trusted paths: {name}")
|
||||
details = os.stat(resolved)
|
||||
if (
|
||||
not stat.S_ISREG(details.st_mode)
|
||||
or details.st_mode & 0o022
|
||||
or not details.st_mode & 0o111
|
||||
):
|
||||
raise OSError(f"trusted executable has unsafe ownership mode: {name}")
|
||||
digest = hashlib.sha256()
|
||||
with open(resolved, "rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
observed = digest.hexdigest()
|
||||
expected = EXPECTED_SHA256.get(name)
|
||||
if not expected or observed not in expected:
|
||||
raise OSError(f"executable digest is not release-attested: {name}")
|
||||
return Attestation(resolved, observed)
|
||||
|
||||
|
||||
def _append_bounded(buffer: bytearray, chunk: bytes, limit: int) -> bool:
|
||||
remaining = limit + 1 - len(buffer)
|
||||
if remaining > 0:
|
||||
buffer.extend(chunk[:remaining])
|
||||
return len(buffer) > limit or len(chunk) > remaining
|
||||
|
||||
|
||||
def _signal_group(process: subprocess.Popen[bytes], signum: int) -> None:
|
||||
with suppress(ProcessLookupError):
|
||||
os.killpg(process.pid, signum)
|
||||
|
||||
|
||||
def execute_bounded(
|
||||
argv: Sequence[str],
|
||||
environment: Mapping[str, str],
|
||||
timeout: float,
|
||||
max_bytes: int,
|
||||
clock: Callable[[], float] = time.monotonic,
|
||||
popen: Callable[..., subprocess.Popen[bytes]] = subprocess.Popen,
|
||||
) -> Capture:
|
||||
"""Execute one process with bounded live pipe capture and group cleanup."""
|
||||
process = popen(
|
||||
list(argv),
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
env=dict(environment),
|
||||
text=False,
|
||||
close_fds=True,
|
||||
start_new_session=True,
|
||||
)
|
||||
streams = {"stdout": process.stdout, "stderr": process.stderr}
|
||||
buffers = {"stdout": bytearray(), "stderr": bytearray()}
|
||||
truncated = {"stdout": False, "stderr": False}
|
||||
selector = selectors.DefaultSelector()
|
||||
for name, stream in streams.items():
|
||||
if (
|
||||
stream is not None
|
||||
): # pragma: no branch - PIPE was requested for both streams
|
||||
os.set_blocking(stream.fileno(), False)
|
||||
selector.register(stream, selectors.EVENT_READ, name)
|
||||
expires = clock() + timeout
|
||||
timed_out = False
|
||||
try:
|
||||
while process.poll() is None:
|
||||
remaining = expires - clock()
|
||||
if remaining <= 0:
|
||||
timed_out = True
|
||||
break
|
||||
for key, _ in selector.select(min(remaining, 0.1)):
|
||||
try:
|
||||
chunk = os.read(key.fileobj.fileno(), READ_CHUNK)
|
||||
except BlockingIOError: # pragma: no cover - readiness race, retry only
|
||||
continue
|
||||
if not chunk:
|
||||
selector.unregister(key.fileobj)
|
||||
continue
|
||||
name = key.data
|
||||
truncated[name] |= _append_bounded(buffers[name], chunk, max_bytes)
|
||||
_signal_group(process, signal.SIGKILL if timed_out else signal.SIGTERM)
|
||||
drain_until = clock() + PIPE_DRAIN_SECONDS
|
||||
while selector.get_map() and clock() < drain_until:
|
||||
for key, _ in selector.select(max(0.0, drain_until - clock())):
|
||||
try:
|
||||
chunk = os.read(key.fileobj.fileno(), READ_CHUNK)
|
||||
except BlockingIOError: # pragma: no cover - readiness race, retry only
|
||||
continue
|
||||
if not chunk:
|
||||
selector.unregister(key.fileobj)
|
||||
continue
|
||||
name = key.data
|
||||
truncated[name] |= _append_bounded(buffers[name], chunk, max_bytes)
|
||||
_signal_group(process, signal.SIGKILL)
|
||||
returncode = process.wait(timeout=PIPE_DRAIN_SECONDS)
|
||||
finally:
|
||||
selector.close()
|
||||
for stream in streams.values():
|
||||
if (
|
||||
stream is not None
|
||||
): # pragma: no branch - PIPE was requested for both streams
|
||||
stream.close()
|
||||
if (
|
||||
process.poll() is None
|
||||
): # pragma: no branch - fail-safe for wait/selector exceptions
|
||||
_signal_group(process, signal.SIGKILL)
|
||||
process.wait()
|
||||
return Capture(
|
||||
returncode=returncode,
|
||||
stdout=bytes(buffers["stdout"][:max_bytes]),
|
||||
stderr=bytes(buffers["stderr"][:max_bytes]),
|
||||
stdout_truncated=truncated["stdout"],
|
||||
stderr_truncated=truncated["stderr"],
|
||||
timed_out=timed_out,
|
||||
)
|
||||
|
||||
|
||||
def _injected_executor(
|
||||
spawn: Callable[..., subprocess.CompletedProcess[Any]],
|
||||
) -> Callable[..., Capture]:
|
||||
"""Adapt deterministic test doubles without weakening the production path."""
|
||||
|
||||
def execute(
|
||||
argv: Sequence[str],
|
||||
environment: Mapping[str, str],
|
||||
timeout: float,
|
||||
max_bytes: int,
|
||||
) -> Capture:
|
||||
completed = spawn(
|
||||
list(argv),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
check=False,
|
||||
env=dict(environment),
|
||||
)
|
||||
stdout = str(completed.stdout or "").encode()
|
||||
stderr = str(completed.stderr or "").encode()
|
||||
return Capture(
|
||||
completed.returncode,
|
||||
stdout[:max_bytes],
|
||||
stderr[:max_bytes],
|
||||
len(stdout) > max_bytes,
|
||||
len(stderr) > max_bytes,
|
||||
)
|
||||
|
||||
return execute
|
||||
|
||||
|
||||
def _screen_capture(data: bytes, truncated: bool, budget: int) -> str:
|
||||
if truncated:
|
||||
return TRUNCATION_NOTE.encode()[:budget].decode()
|
||||
return safe_text(data.decode("utf-8", "replace"), budget)[0]
|
||||
|
||||
|
||||
class Runner:
|
||||
"""Run probes under policy, provenance, timeout, deadline, and byte budgets."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
mode: str = READ_ONLY,
|
||||
command_timeout: float = DEFAULT_COMMAND_TIMEOUT,
|
||||
deadline_seconds: float = DEFAULT_RUN_DEADLINE,
|
||||
max_bytes: int = DEFAULT_MAX_BYTES,
|
||||
clock: Callable[[], float] = time.monotonic,
|
||||
executor: Callable[..., Capture] | None = None,
|
||||
spawn: Callable[..., subprocess.CompletedProcess[Any]] | None = None,
|
||||
environ: Mapping[str, str] | None = None,
|
||||
attestor: Callable[[str, Mapping[str, str]], Attestation] = attest_executable,
|
||||
) -> None:
|
||||
self.mode = mode
|
||||
self.command_timeout = _finite_number(
|
||||
"command_timeout", command_timeout, 0.05, MAX_COMMAND_TIMEOUT
|
||||
)
|
||||
deadline = _finite_number(
|
||||
"deadline_seconds", deadline_seconds, 0.05, MAX_RUN_DEADLINE
|
||||
)
|
||||
self.max_bytes = _byte_budget(max_bytes)
|
||||
self._clock = clock
|
||||
self._executor = executor or (
|
||||
_injected_executor(spawn) if spawn else execute_bounded
|
||||
)
|
||||
self._environ = environ
|
||||
self._attestor = attestor
|
||||
self._deadline_at = clock() + deadline
|
||||
self.commands_run = 0
|
||||
|
||||
@property
|
||||
def remaining_seconds(self) -> float:
|
||||
return self._deadline_at - self._clock()
|
||||
|
||||
def run(
|
||||
self,
|
||||
argv: Sequence[str],
|
||||
vantage: Vantage,
|
||||
max_bytes: int | None = None,
|
||||
reserve_seconds: float = 0.0,
|
||||
) -> Outcome:
|
||||
budget_bytes = self.max_bytes if max_bytes is None else _byte_budget(max_bytes)
|
||||
reserve = _finite_number(
|
||||
"reserve_seconds", reserve_seconds, 0.0, MAX_RUN_DEADLINE
|
||||
)
|
||||
addressed = vantage.wrap(argv)
|
||||
try:
|
||||
check_argv(addressed, self.mode)
|
||||
except PolicyError as exc:
|
||||
return Outcome(argv=addressed, vantage=vantage.name, error=f"policy: {exc}")
|
||||
budget = min(self.command_timeout, self.remaining_seconds - reserve)
|
||||
if budget <= 0:
|
||||
return Outcome(argv=addressed, vantage=vantage.name, error=DEADLINE_ERROR)
|
||||
environment = build_environment(vantage, self._environ)
|
||||
try:
|
||||
attestation = self._attestor(addressed[0], environment)
|
||||
except (OSError, ValueError) as exc:
|
||||
return Outcome(
|
||||
argv=addressed, vantage=vantage.name, error=f"attestation: {exc}"
|
||||
)
|
||||
executed = (attestation.path, *addressed[1:])
|
||||
try:
|
||||
check_argv(executed, self.mode)
|
||||
except PolicyError as exc:
|
||||
return Outcome(argv=addressed, vantage=vantage.name, error=f"policy: {exc}")
|
||||
started = self._clock()
|
||||
self.commands_run += 1
|
||||
try:
|
||||
capture = self._executor(executed, environment, budget, budget_bytes)
|
||||
except subprocess.TimeoutExpired:
|
||||
capture = Capture(-signal.SIGKILL, timed_out=True)
|
||||
except OSError as exc:
|
||||
return Outcome(
|
||||
argv=addressed,
|
||||
vantage=vantage.name,
|
||||
error=f"spawn: {exc.strerror or type(exc).__name__}",
|
||||
executable_path=attestation.path,
|
||||
executable_sha256=attestation.sha256,
|
||||
)
|
||||
was_truncated = capture.stdout_truncated or capture.stderr_truncated
|
||||
stdout = _screen_capture(capture.stdout, capture.stdout_truncated, budget_bytes)
|
||||
stderr = _screen_capture(capture.stderr, capture.stderr_truncated, budget_bytes)
|
||||
return Outcome(
|
||||
argv=addressed,
|
||||
vantage=vantage.name,
|
||||
returncode=capture.returncode,
|
||||
stdout=stdout,
|
||||
stderr=stderr,
|
||||
truncated=was_truncated,
|
||||
duration_ms=int((self._clock() - started) * 1000),
|
||||
error=TIMEOUT_ERROR if capture.timed_out else None,
|
||||
executable_path=attestation.path,
|
||||
executable_sha256=attestation.sha256,
|
||||
)
|
||||
474
scripts/ops/hermes_handoff_json_rules.py
Normal file
474
scripts/ops/hermes_handoff_json_rules.py
Normal file
@ -0,0 +1,474 @@
|
||||
#!/usr/bin/env python3
|
||||
"""JSON-shaped evaluators for the handoff acceptance harness.
|
||||
|
||||
These rules cover the evidence that arrives as structured output — Kubernetes
|
||||
objects, forge pull requests, provider health files, and the two append-only
|
||||
evidence logs the platform keeps. They share the fail-closed contract of every
|
||||
other rule: an absent path, an unparsable payload, or an empty result set is
|
||||
``NOT_RUN``, never a quiet pass.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
import math
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from hermes_handoff_exec import Outcome
|
||||
from hermes_handoff_model import FAIL, NOT_RUN, PASS, CheckSpec
|
||||
from hermes_handoff_rules import (
|
||||
Evaluation,
|
||||
dotted,
|
||||
evaluator,
|
||||
exact_equal,
|
||||
is_list,
|
||||
missing_steps,
|
||||
parsed_step,
|
||||
strict_json,
|
||||
)
|
||||
|
||||
_ROUTE_PART_RE = re.compile(r"[a-z0-9][a-z0-9_.-]{0,63}\Z")
|
||||
_FLUX_NAME_RE = re.compile(
|
||||
r"[a-z0-9]([-a-z0-9.]*[a-z0-9])?/[a-z0-9]([-a-z0-9.]*[a-z0-9])?\Z"
|
||||
)
|
||||
|
||||
|
||||
def _timestamp(value: Any) -> dt.datetime:
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
raise ValueError("timestamp is not a non-empty string")
|
||||
observed = dt.datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
if observed.tzinfo is None:
|
||||
raise ValueError("timestamp has no timezone")
|
||||
return observed.astimezone(dt.timezone.utc)
|
||||
|
||||
|
||||
def _now(value: Any) -> dt.datetime:
|
||||
if not isinstance(value, dt.datetime) or value.tzinfo is None:
|
||||
raise ValueError("run clock is not timezone-aware")
|
||||
return value.astimezone(dt.timezone.utc)
|
||||
|
||||
|
||||
@evaluator("json_field")
|
||||
def evaluate_json_field(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
|
||||
"""Compare dotted JSON paths of one step's output against expected values."""
|
||||
blocked = missing_steps(spec, outcomes)
|
||||
if blocked:
|
||||
return blocked
|
||||
parsed = parsed_step(spec, outcomes)
|
||||
if isinstance(parsed, Evaluation):
|
||||
return parsed
|
||||
payload, key = parsed
|
||||
evidence: dict[str, Any] = {}
|
||||
mismatches: list[str] = []
|
||||
for path, expected in spec.expect["fields"].items():
|
||||
try:
|
||||
actual = dotted(payload, path)
|
||||
except (KeyError, ValueError, IndexError):
|
||||
return Evaluation(
|
||||
NOT_RUN, f"path {path} is absent from step {key}", evidence
|
||||
)
|
||||
evidence[path] = actual
|
||||
allowed = expected if isinstance(expected, list) else [expected]
|
||||
if not any(exact_equal(actual, item) for item in allowed):
|
||||
mismatches.append(f"{path}={actual!r} expected one of {allowed!r}")
|
||||
if mismatches:
|
||||
return Evaluation(FAIL, "; ".join(mismatches), evidence)
|
||||
return Evaluation(PASS, "observed fields match the release contract", evidence)
|
||||
|
||||
|
||||
@evaluator("json_numeric")
|
||||
def evaluate_json_numeric(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
|
||||
"""Compare numeric JSON paths against inclusive minimum/maximum bounds."""
|
||||
blocked = missing_steps(spec, outcomes)
|
||||
if blocked:
|
||||
return blocked
|
||||
parsed = parsed_step(spec, outcomes)
|
||||
if isinstance(parsed, Evaluation):
|
||||
return parsed
|
||||
payload, key = parsed
|
||||
evidence: dict[str, Any] = {}
|
||||
problems: list[str] = []
|
||||
for path, bounds in spec.expect["fields"].items():
|
||||
try:
|
||||
actual = dotted(payload, path)
|
||||
except (KeyError, ValueError, IndexError):
|
||||
return Evaluation(
|
||||
NOT_RUN, f"path {path} is absent from step {key}", evidence
|
||||
)
|
||||
if (
|
||||
not isinstance(actual, (int, float))
|
||||
or isinstance(actual, bool)
|
||||
or not math.isfinite(float(actual))
|
||||
):
|
||||
return Evaluation(
|
||||
NOT_RUN, f"path {path} is not numeric in step {key}", evidence
|
||||
)
|
||||
evidence[path] = actual
|
||||
if "min" in bounds and actual < bounds["min"]:
|
||||
problems.append(f"{path}={actual} below minimum {bounds['min']}")
|
||||
if "max" in bounds and actual > bounds["max"]:
|
||||
problems.append(f"{path}={actual} above maximum {bounds['max']}")
|
||||
if problems:
|
||||
return Evaluation(FAIL, "; ".join(problems), evidence)
|
||||
return Evaluation(PASS, "numeric fields are within the expected bounds", evidence)
|
||||
|
||||
|
||||
@evaluator("json_recent")
|
||||
def evaluate_json_recent(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
|
||||
"""Pass when a timestamp field is fresh enough to be evidence about now.
|
||||
|
||||
Stale provider or routing evidence is worse than none: it describes a state
|
||||
the release no longer has. A timestamp older than the allowed age is a FAIL,
|
||||
and one the harness cannot parse is NOT_RUN.
|
||||
"""
|
||||
blocked = missing_steps(spec, outcomes)
|
||||
if blocked:
|
||||
return blocked
|
||||
parsed = parsed_step(spec, outcomes)
|
||||
if isinstance(parsed, Evaluation):
|
||||
return parsed
|
||||
payload, key = parsed
|
||||
try:
|
||||
now = _now(spec.expect["now"])
|
||||
except (TypeError, ValueError) as exc:
|
||||
return Evaluation(NOT_RUN, f"invalid run clock: {exc}")
|
||||
evidence: dict[str, Any] = {}
|
||||
stale: list[str] = []
|
||||
for path, max_age in spec.expect["fields"].items():
|
||||
try:
|
||||
raw = dotted(payload, path)
|
||||
except (KeyError, ValueError, IndexError):
|
||||
return Evaluation(
|
||||
NOT_RUN, f"path {path} is absent from step {key}", evidence
|
||||
)
|
||||
try:
|
||||
observed = _timestamp(raw)
|
||||
except (TypeError, ValueError):
|
||||
return Evaluation(
|
||||
NOT_RUN, f"path {path} is not a timezone-aware ISO timestamp", evidence
|
||||
)
|
||||
age = (now - observed).total_seconds()
|
||||
evidence[path] = {"age_seconds": int(age)}
|
||||
if age < 0:
|
||||
stale.append(f"{path} is {abs(int(age))}s in the future")
|
||||
elif age > max_age:
|
||||
stale.append(f"{path} is {int(age)}s old, limit {max_age}s")
|
||||
if stale:
|
||||
return Evaluation(FAIL, "; ".join(stale), evidence)
|
||||
return Evaluation(PASS, "observed timestamps are fresh", evidence)
|
||||
|
||||
|
||||
@evaluator("all_of_field")
|
||||
def evaluate_all_of_field(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
|
||||
"""Pass when every selected element of a JSON array matches expected fields."""
|
||||
blocked = missing_steps(spec, outcomes)
|
||||
if blocked:
|
||||
return blocked
|
||||
parsed = parsed_step(spec, outcomes)
|
||||
if isinstance(parsed, Evaluation):
|
||||
return parsed
|
||||
payload, key = parsed
|
||||
try:
|
||||
elements = dotted(payload, spec.expect.get("array", ""))
|
||||
except (KeyError, ValueError, IndexError):
|
||||
return Evaluation(NOT_RUN, f"array path is absent from step {key}")
|
||||
if not is_list(elements):
|
||||
return Evaluation(NOT_RUN, f"step {key} did not return an array")
|
||||
key_field = spec.expect.get("key_field", "")
|
||||
wanted = spec.expect.get("keys")
|
||||
exempt = set(spec.expect.get("exempt_keys", ()))
|
||||
seen: list[Any] = []
|
||||
problems: list[str] = []
|
||||
evidence: dict[str, Any] = {}
|
||||
for element in elements:
|
||||
identity = dotted(element, key_field) if key_field else None
|
||||
if wanted is not None and identity not in wanted:
|
||||
continue
|
||||
if identity in exempt:
|
||||
continue
|
||||
seen.append(identity)
|
||||
for path in spec.expect.get("non_empty", ()):
|
||||
try:
|
||||
value = dotted(element, path)
|
||||
except (KeyError, ValueError, IndexError):
|
||||
value = None
|
||||
if value in (None, "", [], {}):
|
||||
problems.append(f"{identity}: {path} is empty")
|
||||
for path, expected in spec.expect.get("fields", {}).items():
|
||||
try:
|
||||
actual = dotted(element, path)
|
||||
except (KeyError, ValueError, IndexError):
|
||||
actual = None
|
||||
allowed = expected if isinstance(expected, list) else [expected]
|
||||
if not any(exact_equal(actual, item) for item in allowed):
|
||||
problems.append(
|
||||
f"{identity}: {path}={actual!r} expected one of {allowed!r}"
|
||||
)
|
||||
evidence["examined"] = seen[:32]
|
||||
if wanted is not None:
|
||||
absent = [item for item in wanted if item not in seen]
|
||||
if absent:
|
||||
return Evaluation(
|
||||
NOT_RUN, f"expected keys absent from step {key}: {absent}", evidence
|
||||
)
|
||||
elif not seen:
|
||||
return Evaluation(NOT_RUN, f"step {key} returned nothing to examine", evidence)
|
||||
if problems:
|
||||
return Evaluation(FAIL, "; ".join(problems[:8]), evidence)
|
||||
return Evaluation(
|
||||
PASS, f"{len(seen)} element(s) match the release contract", evidence
|
||||
)
|
||||
|
||||
|
||||
@evaluator("json_record")
|
||||
def evaluate_json_record(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
|
||||
"""Capture named JSON paths as evidence; pass when all of them resolve.
|
||||
|
||||
Recording is a release requirement in its own right — the runbook has to
|
||||
name the exact heads it was verified against — so an unreachable source is a
|
||||
NOT_RUN rather than an empty section.
|
||||
"""
|
||||
blocked = missing_steps(spec, outcomes)
|
||||
if blocked:
|
||||
return blocked
|
||||
parsed = parsed_step(spec, outcomes)
|
||||
if isinstance(parsed, Evaluation):
|
||||
return parsed
|
||||
payload, key = parsed
|
||||
evidence: dict[str, Any] = {}
|
||||
for label, path in spec.expect["record"].items():
|
||||
try:
|
||||
evidence[label] = dotted(payload, path)
|
||||
except (KeyError, ValueError, IndexError):
|
||||
return Evaluation(
|
||||
NOT_RUN, f"path {path} is absent from step {key}", evidence
|
||||
)
|
||||
return Evaluation(PASS, "state recorded", evidence)
|
||||
|
||||
|
||||
@evaluator("routing_evidence")
|
||||
def evaluate_routing_evidence(
|
||||
spec: CheckSpec, outcomes: dict[str, Outcome]
|
||||
) -> Evaluation:
|
||||
"""Summarise a Switchyard routing log tail into provider/effort coverage.
|
||||
|
||||
Route identifiers are ``<lane>/<provider>/<family>/<effort>``, so provider,
|
||||
model family, effort tier, and lane all come from the same field. Fallback
|
||||
and failure evidence is a separate ``fallback_reason`` on the record that
|
||||
was retried. Every row must be complete strict JSON; a clipped or malformed
|
||||
line makes the evidence unusable rather than being silently discarded.
|
||||
"""
|
||||
blocked = missing_steps(spec, outcomes)
|
||||
if blocked:
|
||||
return blocked
|
||||
key = spec.expect["step"]
|
||||
outcome = outcomes[key]
|
||||
if not outcome.ok:
|
||||
return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})")
|
||||
records: list[dict[str, Any]] = []
|
||||
routes: list[list[str]] = []
|
||||
stamps_by_dimension: dict[tuple[str, str], dt.datetime] = {}
|
||||
now_value = spec.expect.get("now")
|
||||
try:
|
||||
now = _now(now_value) if now_value is not None else None
|
||||
except (TypeError, ValueError) as exc:
|
||||
return Evaluation(NOT_RUN, f"invalid routing run clock: {exc}")
|
||||
for row_number, line in enumerate(outcome.stdout.splitlines(), 1):
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
record = strict_json(line)
|
||||
except ValueError as exc:
|
||||
return Evaluation(
|
||||
NOT_RUN, f"routing row {row_number} is malformed JSON: {exc}"
|
||||
)
|
||||
if not isinstance(record, dict):
|
||||
return Evaluation(NOT_RUN, f"routing row {row_number} is not an object")
|
||||
try:
|
||||
observed = _timestamp(record.get("ts"))
|
||||
except (TypeError, ValueError) as exc:
|
||||
return Evaluation(
|
||||
NOT_RUN, f"routing row {row_number} has invalid timestamp: {exc}"
|
||||
)
|
||||
if now is not None and observed > now:
|
||||
return Evaluation(FAIL, f"routing row {row_number} has a future timestamp")
|
||||
model = record.get("model")
|
||||
if not isinstance(model, str) or not model:
|
||||
return Evaluation(NOT_RUN, f"routing row {row_number} has no model")
|
||||
route = model.split("/")
|
||||
if len(route) == 4 and all(_ROUTE_PART_RE.fullmatch(part) for part in route):
|
||||
routes.append(route)
|
||||
for label, value in (
|
||||
("provider", route[1]),
|
||||
("effort", route[3]),
|
||||
("lane", route[0]),
|
||||
):
|
||||
key_dimension = (label, value)
|
||||
stamps_by_dimension[key_dimension] = max(
|
||||
observed, stamps_by_dimension.get(key_dimension, observed)
|
||||
)
|
||||
elif record.get("tier") != "classifier":
|
||||
return Evaluation(
|
||||
NOT_RUN, f"routing row {row_number} has malformed route {model!r}"
|
||||
)
|
||||
fallback = record.get("fallback_reason")
|
||||
if fallback is not None and (
|
||||
not isinstance(fallback, str) or not fallback.strip()
|
||||
):
|
||||
return Evaluation(
|
||||
NOT_RUN, f"routing row {row_number} has malformed fallback_reason"
|
||||
)
|
||||
record["_observed"] = observed
|
||||
records.append(record)
|
||||
fallbacks = [record for record in records if record.get("fallback_reason")]
|
||||
stamps = sorted(record["_observed"] for record in records)
|
||||
evidence: dict[str, Any] = {
|
||||
"records": len(records),
|
||||
"routed": len(routes),
|
||||
"lanes": sorted({route[0] for route in routes}),
|
||||
"providers": sorted({route[1] for route in routes}),
|
||||
"families": sorted({route[2] for route in routes}),
|
||||
"efforts": sorted({route[3] for route in routes}),
|
||||
"fallback_events": len(fallbacks),
|
||||
"fallback_reasons": sorted(
|
||||
{str(record["fallback_reason"]) for record in fallbacks}
|
||||
),
|
||||
"latest_ts": stamps[-1].isoformat() if stamps else "",
|
||||
}
|
||||
if not records:
|
||||
return Evaluation(
|
||||
NOT_RUN, "routing log tail contained no parsable records", evidence
|
||||
)
|
||||
missing = [
|
||||
f"{label}:{item}"
|
||||
for label, wanted in (
|
||||
("provider", spec.expect.get("providers", ())),
|
||||
("effort", spec.expect.get("efforts", ())),
|
||||
("lane", spec.expect.get("lanes", ())),
|
||||
)
|
||||
for item in wanted
|
||||
if item not in evidence[f"{label}s"]
|
||||
]
|
||||
if spec.expect.get("require_fallback_evidence") and not fallbacks:
|
||||
missing.append("fallback:none-observed")
|
||||
max_age = spec.expect.get("max_age_seconds")
|
||||
if max_age and now:
|
||||
ages: dict[str, int] = {}
|
||||
for label, wanted in (
|
||||
("provider", spec.expect.get("providers", ())),
|
||||
("effort", spec.expect.get("efforts", ())),
|
||||
("lane", spec.expect.get("lanes", ())),
|
||||
):
|
||||
for item in wanted:
|
||||
stamp = stamps_by_dimension.get((label, item))
|
||||
if stamp is None:
|
||||
continue
|
||||
age = int((now - stamp).total_seconds())
|
||||
ages[f"{label}:{item}"] = age
|
||||
if age > max_age:
|
||||
missing.append(
|
||||
f"freshness:{label}:{item}:{age}s-exceeds-{max_age}s"
|
||||
)
|
||||
evidence["age_seconds"] = ages
|
||||
if missing:
|
||||
return Evaluation(
|
||||
FAIL, f"routing evidence missing {', '.join(missing)}", evidence
|
||||
)
|
||||
return Evaluation(PASS, "routing evidence covers the required lanes", evidence)
|
||||
|
||||
|
||||
@evaluator("flux_health")
|
||||
def evaluate_flux_health(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
|
||||
"""Classify a compact Flux projection into suspensions and unhealthy objects.
|
||||
|
||||
Each row carries name, generation, observed generation, suspension, and all
|
||||
condition states. A non-suspended object must be current, explicitly
|
||||
``Ready=True``, not ``Healthy=False/Unknown``, and not reconciling. Suspended
|
||||
objects are judged against the exact expected-suspension list.
|
||||
"""
|
||||
blocked = missing_steps(spec, outcomes)
|
||||
if blocked:
|
||||
return blocked
|
||||
key = spec.expect["step"]
|
||||
outcome = outcomes[key]
|
||||
if not outcome.ok:
|
||||
return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})")
|
||||
expected = set(spec.expect.get("expected_suspensions", ()))
|
||||
suspended: list[str] = []
|
||||
unhealthy: list[str] = []
|
||||
total = 0
|
||||
for row_number, line in enumerate(outcome.stdout.splitlines(), 1):
|
||||
if not line.strip():
|
||||
continue
|
||||
columns = line.split("\t")
|
||||
if len(columns) != 5:
|
||||
return Evaluation(
|
||||
NOT_RUN, f"Flux row {row_number} has {len(columns)} columns"
|
||||
)
|
||||
name, generation, observed_generation, suspend, conditions = columns
|
||||
name = name.strip()
|
||||
if not _FLUX_NAME_RE.fullmatch(name):
|
||||
return Evaluation(
|
||||
NOT_RUN, f"Flux row {row_number} has malformed object name"
|
||||
)
|
||||
try:
|
||||
current_generation = int(generation)
|
||||
reconciled_generation = int(observed_generation)
|
||||
except ValueError:
|
||||
return Evaluation(
|
||||
NOT_RUN, f"Flux row {row_number} has malformed generations"
|
||||
)
|
||||
if current_generation <= 0 or current_generation != reconciled_generation:
|
||||
return Evaluation(
|
||||
FAIL, f"Flux row {row_number} has stale observedGeneration"
|
||||
)
|
||||
if suspend.strip().lower() not in {"", "false", "true"}:
|
||||
return Evaluation(
|
||||
NOT_RUN, f"Flux row {row_number} has malformed suspension state"
|
||||
)
|
||||
total += 1
|
||||
if suspend.strip().lower() == "true":
|
||||
suspended.append(name)
|
||||
continue
|
||||
states: dict[str, str] = {}
|
||||
for pair in filter(None, conditions.strip().strip(",").split(",")):
|
||||
if pair.count("=") != 1:
|
||||
return Evaluation(
|
||||
NOT_RUN, f"Flux row {row_number} has malformed condition"
|
||||
)
|
||||
condition, state = pair.split("=", 1)
|
||||
if (
|
||||
not condition
|
||||
or state not in {"True", "False", "Unknown"}
|
||||
or condition in states
|
||||
):
|
||||
return Evaluation(
|
||||
NOT_RUN, f"Flux row {row_number} has malformed condition state"
|
||||
)
|
||||
states[condition] = state
|
||||
healthy = states.get("Healthy")
|
||||
if (
|
||||
states.get("Ready") != "True"
|
||||
or healthy in {"False", "Unknown"}
|
||||
or states.get("Reconciling") == "True"
|
||||
):
|
||||
unhealthy.append(name)
|
||||
unexpected = sorted(set(suspended) - expected)
|
||||
evidence = {
|
||||
"total": total,
|
||||
"suspended": sorted(suspended),
|
||||
"unexpected_suspensions": unexpected,
|
||||
"unhealthy": sorted(unhealthy),
|
||||
}
|
||||
if not total:
|
||||
return Evaluation(NOT_RUN, f"step {key} returned no Flux objects", evidence)
|
||||
problems = []
|
||||
if unexpected:
|
||||
problems.append(f"unexpected suspensions: {', '.join(unexpected)}")
|
||||
if unhealthy:
|
||||
problems.append(f"unhealthy: {', '.join(sorted(unhealthy)[:8])}")
|
||||
if problems:
|
||||
return Evaluation(FAIL, "; ".join(problems), evidence)
|
||||
return Evaluation(PASS, f"{total} Flux objects reconciled as expected", evidence)
|
||||
290
scripts/ops/hermes_handoff_model.py
Normal file
290
scripts/ops/hermes_handoff_model.py
Normal file
@ -0,0 +1,290 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Result model, aggregation, and rendering for the handoff acceptance harness.
|
||||
|
||||
The classification is deliberately four-valued. ``NOT_APPLICABLE`` is the only
|
||||
way a check can be excused, and a check can only reach it by matching a scope
|
||||
rule that a reviewer wrote down; anything the harness merely failed to observe
|
||||
is ``NOT_RUN``. Because a mandatory ``NOT_RUN`` is a NO_GO exactly like a
|
||||
``FAIL``, an unreachable cluster, an exhausted deadline, or a probe nobody
|
||||
maintained can never round up to a release.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from hermes_handoff_exec import Outcome
|
||||
from hermes_handoff_redaction import contains_credential_shape, safe_text, scrub
|
||||
|
||||
PASS = "PASS"
|
||||
FAIL = "FAIL"
|
||||
NOT_RUN = "NOT_RUN"
|
||||
NOT_APPLICABLE = "NOT_APPLICABLE"
|
||||
STATUSES = (PASS, FAIL, NOT_RUN, NOT_APPLICABLE)
|
||||
|
||||
GO = "GO"
|
||||
NO_GO = "NO_GO"
|
||||
|
||||
ALWAYS = "always"
|
||||
EPHEMERAL = "ephemeral"
|
||||
|
||||
HARNESS = "hermes-handoff-acceptance"
|
||||
SCHEMA_VERSION = 1
|
||||
|
||||
|
||||
REVIEW = "review"
|
||||
ATTEMPT = "attempt"
|
||||
READ = "read"
|
||||
STEP_KINDS = (REVIEW, ATTEMPT, READ)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Step:
|
||||
"""One command a check needs, and the vantage it must be observed from.
|
||||
|
||||
``kind`` separates a self access review (``review``) from a real request the
|
||||
identity is expected to be refused (``attempt``). A deny check needs both:
|
||||
an authorization review states policy, only an attempt proves the API server
|
||||
enforces it.
|
||||
|
||||
``record`` is cleared for bulk evidence — a routing log tail, a large object
|
||||
listing — which the harness parses but must not paste into the artifact.
|
||||
``max_bytes`` raises the capture budget for that same bulk evidence, where a
|
||||
default-sized truncation would silently narrow the window being examined.
|
||||
"""
|
||||
|
||||
key: str
|
||||
vantage: str
|
||||
argv: tuple[str, ...]
|
||||
kind: str = READ
|
||||
optional: bool = False
|
||||
record: bool = True
|
||||
max_bytes: int | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CheckSpec:
|
||||
"""A declarative acceptance check.
|
||||
|
||||
``rule`` names an evaluator; ``expect`` carries its parameters. Keeping the
|
||||
catalog declarative means a reviewer reads what is asserted, not how it is
|
||||
plumbed, and the harness can prove structural properties over every check
|
||||
(no impersonation, no mutation, both vantages present) before running any.
|
||||
"""
|
||||
|
||||
id: str
|
||||
title: str
|
||||
group: str
|
||||
rule: str
|
||||
steps: tuple[Step, ...] = ()
|
||||
expect: Mapping[str, Any] = field(default_factory=dict)
|
||||
mandatory: bool = True
|
||||
scope: str = ALWAYS
|
||||
rationale: str = ""
|
||||
|
||||
@property
|
||||
def vantages(self) -> tuple[str, ...]:
|
||||
"""Return the distinct vantages this check draws evidence from."""
|
||||
seen: list[str] = []
|
||||
for step in self.steps:
|
||||
if step.vantage not in seen:
|
||||
seen.append(step.vantage)
|
||||
return tuple(seen)
|
||||
|
||||
|
||||
@dataclass
|
||||
class CheckResult:
|
||||
"""The classified outcome of one acceptance check."""
|
||||
|
||||
spec: CheckSpec
|
||||
status: str
|
||||
reason: str = ""
|
||||
evidence: dict[str, Any] = field(default_factory=dict)
|
||||
outcomes: list[Outcome] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def blocking(self) -> bool:
|
||||
"""Report whether this result alone forces NO_GO."""
|
||||
if self.status not in STATUSES:
|
||||
return True
|
||||
return self.spec.mandatory and self.status in {FAIL, NOT_RUN}
|
||||
|
||||
def as_dict(self, include_outcomes: bool = True) -> dict[str, Any]:
|
||||
"""Return a JSON-serialisable, screened view of the result."""
|
||||
payload: dict[str, Any] = {
|
||||
"id": self.spec.id,
|
||||
"title": self.spec.title,
|
||||
"group": self.spec.group,
|
||||
"rule": self.spec.rule,
|
||||
"scope": self.spec.scope,
|
||||
"mandatory": self.spec.mandatory,
|
||||
"vantages": list(self.spec.vantages),
|
||||
"status": self.status,
|
||||
"reason": self.reason,
|
||||
"blocking": self.blocking,
|
||||
"evidence": self.evidence,
|
||||
}
|
||||
if self.spec.rationale:
|
||||
payload["rationale"] = self.spec.rationale
|
||||
if include_outcomes:
|
||||
payload["commands"] = [outcome.as_dict() for outcome in self.outcomes]
|
||||
return scrub(payload)
|
||||
|
||||
|
||||
@dataclass
|
||||
class VantageRecord:
|
||||
"""What the harness proved about one vantage before trusting its evidence."""
|
||||
|
||||
name: str
|
||||
description: str
|
||||
identity: str = ""
|
||||
available: bool = False
|
||||
detail: str = ""
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
"""Return a JSON-serialisable view of the vantage."""
|
||||
return scrub(
|
||||
{
|
||||
"name": self.name,
|
||||
"description": self.description,
|
||||
"identity": self.identity,
|
||||
"available": self.available,
|
||||
"detail": self.detail,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Report:
|
||||
"""The whole acceptance run: metadata, per-check results, and a verdict."""
|
||||
|
||||
mode: str
|
||||
started_at: str
|
||||
baseline: dict[str, Any] = field(default_factory=dict)
|
||||
vantages: list[VantageRecord] = field(default_factory=list)
|
||||
results: list[CheckResult] = field(default_factory=list)
|
||||
harness_errors: list[str] = field(default_factory=list)
|
||||
finished_at: str = ""
|
||||
|
||||
@property
|
||||
def counts(self) -> dict[str, int]:
|
||||
"""Return the number of results in each status."""
|
||||
tally = dict.fromkeys(STATUSES, 0)
|
||||
for result in self.results:
|
||||
if result.status in tally:
|
||||
tally[result.status] += 1
|
||||
return tally
|
||||
|
||||
@property
|
||||
def invalid_statuses(self) -> list[str]:
|
||||
"""Return check ids whose status is outside the report schema."""
|
||||
return [
|
||||
result.spec.id for result in self.results if result.status not in STATUSES
|
||||
]
|
||||
|
||||
@property
|
||||
def blocking(self) -> list[CheckResult]:
|
||||
"""Return every result that forces NO_GO, most severe first."""
|
||||
order = {FAIL: 0, NOT_RUN: 1}
|
||||
return sorted(
|
||||
(result for result in self.results if result.blocking),
|
||||
key=lambda result: (order.get(result.status, 2), result.spec.id),
|
||||
)
|
||||
|
||||
@property
|
||||
def decision(self) -> str:
|
||||
"""Return GO only when nothing mandatory failed, skipped, or errored."""
|
||||
if (
|
||||
not self.results
|
||||
or self.invalid_statuses
|
||||
or self.harness_errors
|
||||
or self.blocking
|
||||
):
|
||||
return NO_GO
|
||||
return GO
|
||||
|
||||
def group_summary(self) -> dict[str, dict[str, int]]:
|
||||
"""Return per-group status counts, for the human summary."""
|
||||
groups: dict[str, dict[str, int]] = {}
|
||||
for result in self.results:
|
||||
bucket = groups.setdefault(result.spec.group, dict.fromkeys(STATUSES, 0))
|
||||
bucket[result.status] = bucket.get(result.status, 0) + 1
|
||||
return groups
|
||||
|
||||
def as_dict(self, include_outcomes: bool = True) -> dict[str, Any]:
|
||||
"""Return the machine-readable report."""
|
||||
return scrub(
|
||||
{
|
||||
"harness": HARNESS,
|
||||
"schema_version": SCHEMA_VERSION,
|
||||
"mode": self.mode,
|
||||
"started_at": self.started_at,
|
||||
"finished_at": self.finished_at,
|
||||
"decision": self.decision,
|
||||
"counts": self.counts,
|
||||
"baseline": scrub(self.baseline),
|
||||
"vantages": [vantage.as_dict() for vantage in self.vantages],
|
||||
"harness_errors": [scrub(error) for error in self.harness_errors],
|
||||
"invalid_statuses": self.invalid_statuses,
|
||||
"blocking": [result.spec.id for result in self.blocking],
|
||||
"checks": [result.as_dict(include_outcomes) for result in self.results],
|
||||
}
|
||||
)
|
||||
|
||||
def render_summary(self, width: int = 88) -> str:
|
||||
"""Return the concise human summary printed alongside the JSON."""
|
||||
counts = self.counts
|
||||
lines = [
|
||||
f"{HARNESS}: {self.decision}",
|
||||
f" mode={self.mode} started={self.started_at} checks={len(self.results)}",
|
||||
" " + " ".join(f"{status}={counts[status]}" for status in STATUSES),
|
||||
]
|
||||
for vantage in self.vantages:
|
||||
state = "available" if vantage.available else "unavailable"
|
||||
identity = vantage.identity or "unknown"
|
||||
lines.append(f" vantage {vantage.name}: {state} identity={identity}")
|
||||
for error in self.harness_errors:
|
||||
lines.append(f" harness error: {error}")
|
||||
if self.blocking:
|
||||
lines.append(" blocking:")
|
||||
for result in self.blocking:
|
||||
lines.append(f" {result.status:<15}{result.spec.id}")
|
||||
lines.append(f" {'':<15}{result.reason[:width]}")
|
||||
groups = self.group_summary()
|
||||
lines.append(" groups:")
|
||||
for name in sorted(groups):
|
||||
bucket = groups[name]
|
||||
detail = " ".join(
|
||||
f"{status}={bucket[status]}" for status in STATUSES if bucket[status]
|
||||
)
|
||||
lines.append(f" {name:<18}{detail}")
|
||||
return safe_text("\n".join(lines), 32 * 1024)[0]
|
||||
|
||||
|
||||
def utc_now(clock: Any = None) -> str:
|
||||
"""Return an ISO-8601 UTC timestamp, injectable for deterministic tests."""
|
||||
now = clock() if clock else dt.datetime.now(dt.timezone.utc)
|
||||
return now.replace(microsecond=0).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def unscreened_fields(payload: Any, path: str = "$") -> list[str]:
|
||||
"""Return JSON pointers whose text still looks credential-shaped.
|
||||
|
||||
The report is already screened field by field; this is the invariant check
|
||||
that runs before anything is written, so a screening gap fails the run
|
||||
instead of publishing an artifact nobody re-read.
|
||||
"""
|
||||
offenders: list[str] = []
|
||||
if isinstance(payload, str):
|
||||
if contains_credential_shape(payload):
|
||||
offenders.append(path)
|
||||
elif isinstance(payload, Mapping):
|
||||
for key, value in payload.items():
|
||||
offenders.extend(unscreened_fields(value, f"{path}.{key}"))
|
||||
elif isinstance(payload, Sequence) and not isinstance(payload, (str, bytes)):
|
||||
for index, value in enumerate(payload):
|
||||
offenders.extend(unscreened_fields(value, f"{path}[{index}]"))
|
||||
return offenders
|
||||
493
scripts/ops/hermes_handoff_policy.py
Normal file
493
scripts/ops/hermes_handoff_policy.py
Normal file
@ -0,0 +1,493 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fail-closed command policy for the Hermes handoff acceptance harness.
|
||||
|
||||
Before spawn it admits only credential-safe, structurally read-only grammar;
|
||||
shell is limited to frozen templates with absolute tool paths.
|
||||
|
||||
Impersonation is structurally impossible: ``--as`` and its siblings are refused
|
||||
for every argv, in every mode, from every vantage, and the inner command of a
|
||||
``kubectl exec`` is re-checked under the same rule rather than being trusted.
|
||||
Only binaries with a pinned release digest are on the allowlist, so nothing can
|
||||
be admitted here that the executor would then refuse to attest.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Iterator
|
||||
from urllib.parse import unquote
|
||||
|
||||
READ_ONLY = "read-only"
|
||||
ARMED = "ephemeral-armed"
|
||||
MODES = (READ_ONLY, ARMED)
|
||||
|
||||
GITEA_CLIENT = "/opt/coordinator/gitea_api.py"
|
||||
EXPECTED_REPO = "atlas/titan-iac"
|
||||
EXPECTED_REMOTE = "origin"
|
||||
EXPECTED_BASE = "main"
|
||||
|
||||
# fmt: off
|
||||
FORBIDDEN_ARGS = ("--as", "--as-group", "--as-uid", "--client-certificate",
|
||||
"--client-key", "--password", "--token", "--username")
|
||||
# fmt: on
|
||||
IMPERSONATION_ARGS = ("--as", "--as-group", "--as-uid")
|
||||
FORBIDDEN_BINARIES = frozenset(
|
||||
{"age", "curl", "gpg", "openssl", "sops", "vault", "wget"}
|
||||
)
|
||||
ALLOWED_BINARIES = frozenset({"kubectl", "git", "hermes", "sh", GITEA_CLIENT})
|
||||
TRUSTED_EXECUTABLE_ROOTS = (
|
||||
"/bin/",
|
||||
"/usr/bin/",
|
||||
"/usr/local/bin/",
|
||||
"/opt/data/tools/bin/",
|
||||
"/opt/hermes/.venv/bin/",
|
||||
"/opt/coordinator/",
|
||||
)
|
||||
TRUSTED_INNER_PATHS = {
|
||||
"/bin/sh": "sh",
|
||||
"/usr/bin/git": "git",
|
||||
"/usr/bin/sh": "sh",
|
||||
"/usr/local/bin/kubectl": "kubectl",
|
||||
"/opt/hermes/.venv/bin/hermes": "hermes",
|
||||
GITEA_CLIENT: GITEA_CLIENT,
|
||||
}
|
||||
|
||||
VALUE_FLAGS = frozenset(
|
||||
{
|
||||
"-C",
|
||||
"-c",
|
||||
"-l",
|
||||
"-n",
|
||||
"-o",
|
||||
"--cluster",
|
||||
"--container",
|
||||
"--context",
|
||||
"--field-selector",
|
||||
"--kubeconfig",
|
||||
"--namespace",
|
||||
"--output",
|
||||
"--request-timeout",
|
||||
"--selector",
|
||||
"--server",
|
||||
"--since",
|
||||
"--tail",
|
||||
"--user",
|
||||
}
|
||||
)
|
||||
|
||||
KUBECTL_READ_SUBCOMMANDS = frozenset(
|
||||
{
|
||||
"api-resources",
|
||||
"api-versions",
|
||||
"explain",
|
||||
"get",
|
||||
"top",
|
||||
"version",
|
||||
}
|
||||
)
|
||||
KUBECTL_EXEC_SUBCOMMANDS = frozenset({"exec"})
|
||||
KUBECTL_SUBCOMMANDS = KUBECTL_READ_SUBCOMMANDS | KUBECTL_EXEC_SUBCOMMANDS | {"auth"}
|
||||
SECRET_RESOURCES = frozenset({"secret", "secrets"})
|
||||
UNSAFE_PROJECTION_PARTS = (
|
||||
".args",
|
||||
".command",
|
||||
".data",
|
||||
".envfrom",
|
||||
".secretkeyref",
|
||||
".status.token",
|
||||
".stringdata",
|
||||
".valuefrom",
|
||||
"clientconfig",
|
||||
"password",
|
||||
"privatekey",
|
||||
)
|
||||
SAFE_ENV_VALUE = 'env[?(@.name=="hermes_auto_router_profile")].value'
|
||||
SAFE_JSON_FIELDS = frozenset(["authenticated", "transport", "state", "model", "effort", "latency_ms", "checked_at", "api_provider", "auth_method", "subscription_type"]) # fmt: skip
|
||||
|
||||
GIT_READ_SUBCOMMANDS = frozenset(
|
||||
{
|
||||
"cat-file",
|
||||
"diff",
|
||||
"for-each-ref",
|
||||
"log",
|
||||
"ls-remote",
|
||||
"ls-tree",
|
||||
"merge-base",
|
||||
"rev-list",
|
||||
"rev-parse",
|
||||
"show",
|
||||
"status",
|
||||
}
|
||||
)
|
||||
GIT_FORCE_FLAGS = frozenset(
|
||||
{"-f", "--force", "--force-with-lease", "--mirror", "--all", "--tags"}
|
||||
)
|
||||
GIT_UNSAFE_FLAGS = (
|
||||
"-c",
|
||||
"--config-env",
|
||||
"--exec-path",
|
||||
"--upload-pack",
|
||||
"--receive-pack",
|
||||
)
|
||||
GITEA_READ_METHODS = frozenset({"GET"})
|
||||
HERMES_READ_COMMANDS = frozenset(
|
||||
{("kanban", "list"), ("kanban", "show"), ("sessions", "list"), ("status",)}
|
||||
)
|
||||
|
||||
# These locations contain credential bytes even when a caller claims it will
|
||||
# only inspect metadata. They are rejected for every argv, including a frozen
|
||||
# shell rendering and the inner command of kubectl exec.
|
||||
SECRET_PATH_RE = re.compile(
|
||||
r"(?:^|[\s'\"=])(?:/[^\s'\"]*)?(?:"
|
||||
r"var/run/secrets|run/secrets|vault/secrets|runtime-access|scm-access|pool-access|"
|
||||
r"\.credentials\.json|auth\.json|\.git-credentials|authorized_keys|"
|
||||
r"id_[a-z0-9]+|\.netrc|shadow|gshadow|k3s\.yaml"
|
||||
r")(?:$|[/\s'\"])",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_PARAMETER_RE = re.compile(r"[A-Za-z0-9/][A-Za-z0-9._/@:,-]{0,255}\Z")
|
||||
_PLACEHOLDER_RE = re.compile(r"\{([a-z_]+)\}")
|
||||
_SAFE_API_PATH_RE = re.compile(r"/[A-Za-z0-9][A-Za-z0-9._~:/?&=%+-]{0,511}\Z")
|
||||
|
||||
SHELL_TEMPLATES: dict[str, str] = {
|
||||
"env_names": (
|
||||
"/usr/bin/env | /usr/bin/sed -n "
|
||||
"'s/^\\([A-Za-z_][A-Za-z0-9_]*\\)=.*/\\1/p' | /usr/bin/sort"
|
||||
),
|
||||
"json_fields": (
|
||||
'/opt/hermes/.venv/bin/python -c "import json,sys;d=json.load(open(sys.argv[1]));'
|
||||
'print(json.dumps({{k:d.get(k) for k in sys.argv[2].split(chr(44))}}))" '
|
||||
"{path} {fields}"
|
||||
),
|
||||
"tail_lines": "/usr/bin/tail -n {limit} {path}",
|
||||
}
|
||||
|
||||
|
||||
class PolicyError(ValueError):
|
||||
"""Raised when a requested command falls outside the safe boundary."""
|
||||
|
||||
|
||||
def template_parameters(template_name: str) -> set[str]:
|
||||
"""Return the parameter names a frozen template requires."""
|
||||
template = SHELL_TEMPLATES.get(template_name)
|
||||
if template is None:
|
||||
raise PolicyError(f"unknown shell template: {template_name}")
|
||||
return set(_PLACEHOLDER_RE.findall(template))
|
||||
|
||||
|
||||
def render_shell(template_name: str, **parameters: str) -> str:
|
||||
"""Return a frozen shell template with validated parameters substituted."""
|
||||
required = template_parameters(template_name)
|
||||
if set(parameters) != required:
|
||||
raise PolicyError(
|
||||
f"shell template {template_name} expects parameters {sorted(required)}"
|
||||
)
|
||||
for key, value in parameters.items():
|
||||
if not _PARAMETER_RE.fullmatch(value):
|
||||
raise PolicyError(f"unsafe shell parameter for {template_name}.{key}")
|
||||
if SECRET_PATH_RE.search(value):
|
||||
raise PolicyError("shell templates may not name credential paths")
|
||||
if (
|
||||
template_name == "json_fields"
|
||||
and not set(parameters["fields"].split(",")) <= SAFE_JSON_FIELDS
|
||||
):
|
||||
raise PolicyError("json_fields requested a credential-bearing or unknown field")
|
||||
return SHELL_TEMPLATES[template_name].format(**parameters)
|
||||
|
||||
|
||||
_RENDERED: set[str] = set()
|
||||
_APPROVED_PROJECTIONS: set[str] = set()
|
||||
_ARMED_REF = ""
|
||||
_ARMED_PULLS: set[int] = set()
|
||||
|
||||
|
||||
# fmt: off
|
||||
def arm_ephemeral_policy(ref: str) -> None:
|
||||
global _ARMED_REF
|
||||
if not re.fullmatch(r"ephemeral/hermes-handoff-acceptance/[a-z0-9][a-z0-9-]{7,63}", ref):
|
||||
raise PolicyError("armed policy requires one exact ephemeral ref")
|
||||
_ARMED_REF = ref
|
||||
_ARMED_PULLS.clear()
|
||||
|
||||
|
||||
def register_ephemeral_pull(number: int) -> None:
|
||||
if not _ARMED_REF or isinstance(number, bool) or not isinstance(number, int) or number <= 0:
|
||||
raise PolicyError("ephemeral pull registration is malformed")
|
||||
_ARMED_PULLS.add(number)
|
||||
|
||||
|
||||
def shell(template_name: str, **parameters: str) -> tuple[str, ...]:
|
||||
"""Return a validated ``sh -c`` argv built from a frozen template."""
|
||||
script = render_shell(template_name, **parameters)
|
||||
_RENDERED.add(script)
|
||||
return ("sh", "-c", script)
|
||||
|
||||
|
||||
def projection(value: str) -> str:
|
||||
normalized = value.lower().replace(" ", "")
|
||||
if not normalized.startswith("jsonpath=") or any(part in normalized for part in UNSAFE_PROJECTION_PARTS):
|
||||
raise PolicyError("Kubernetes projection is not credential-safe")
|
||||
if ".metadata.annotations" in normalized and not any(key in normalized for key in (
|
||||
"deployment\\.kubernetes\\.io/revision", "ai\\.bstein\\.dev/config-rev")):
|
||||
raise PolicyError("Kubernetes annotations are not on the reviewed allowlist")
|
||||
if ".metadata.labels" in normalized and "pod-template-hash" not in normalized:
|
||||
raise PolicyError("Kubernetes labels are not on the reviewed allowlist")
|
||||
if ".env" in normalized and ".value" in normalized and SAFE_ENV_VALUE not in normalized:
|
||||
raise PolicyError("Kubernetes projection may expose environment values")
|
||||
_APPROVED_PROJECTIONS.add(normalized)
|
||||
return value
|
||||
# fmt: on
|
||||
|
||||
|
||||
def positionals(argv: tuple[str, ...], start: int = 1) -> Iterator[str]:
|
||||
"""Yield positional arguments, skipping flags and their values."""
|
||||
index = start
|
||||
while index < len(argv):
|
||||
item = argv[index]
|
||||
if item == "--":
|
||||
return
|
||||
if item.startswith("-"):
|
||||
index += 2 if item in VALUE_FLAGS else 1
|
||||
continue
|
||||
yield item
|
||||
index += 1
|
||||
|
||||
|
||||
def _binary_name(value: str) -> str:
|
||||
if value == GITEA_CLIENT or (
|
||||
value.startswith("/opt/coordinator/") and value.endswith("/gitea_api.py")
|
||||
):
|
||||
return GITEA_CLIENT
|
||||
if "/" not in value:
|
||||
return value
|
||||
mapped = TRUSTED_INNER_PATHS.get(value)
|
||||
if mapped:
|
||||
return mapped
|
||||
if "/../" in value or value.endswith("/..") or "/./" in value:
|
||||
raise PolicyError(f"executable path is not normalized: {value}")
|
||||
if not value.startswith(TRUSTED_EXECUTABLE_ROOTS):
|
||||
raise PolicyError(f"executable path is not trusted: {value}")
|
||||
return value.rsplit("/", 1)[-1]
|
||||
|
||||
|
||||
def _reject_forbidden_arguments(argv: tuple[str, ...]) -> None:
|
||||
"""Refuse credential and impersonation arguments up to the ``--`` boundary.
|
||||
|
||||
Everything after ``--`` is the inner command of a ``kubectl exec``, which
|
||||
:func:`_check_kubectl` submits to :func:`check_argv` again, so it is held to
|
||||
exactly this rule rather than being exempted from it.
|
||||
"""
|
||||
boundary = argv.index("--") if "--" in argv else len(argv)
|
||||
for argument in argv[:boundary]:
|
||||
for forbidden in FORBIDDEN_ARGS:
|
||||
if argument == forbidden or argument.startswith(f"{forbidden}="):
|
||||
raise PolicyError(f"forbidden argument: {forbidden}")
|
||||
if argument == "--raw" or argument.startswith("--raw="):
|
||||
raise PolicyError("raw API reads are forbidden")
|
||||
for argument in argv:
|
||||
if argument == "--dry-run" or argument.startswith("--dry-run="):
|
||||
raise PolicyError("dry-run mutation commands are forbidden")
|
||||
|
||||
|
||||
def _reject_secret_paths(argv: tuple[str, ...]) -> None:
|
||||
if any(SECRET_PATH_RE.search(argument) for argument in argv):
|
||||
raise PolicyError("probes may not name a credential path")
|
||||
|
||||
|
||||
def _resource_name(value: str) -> str:
|
||||
return value.lower().split("/", 1)[0].split(".", 1)[0]
|
||||
|
||||
|
||||
def _output_value(argv: tuple[str, ...]) -> str:
|
||||
"""Return kubectl's output argument in a single canonical form."""
|
||||
for index, item in enumerate(argv[1:], 1):
|
||||
if item in {"-o", "--output"} and index + 1 < len(argv):
|
||||
return argv[index + 1]
|
||||
if item.startswith(("-o=", "--output=")):
|
||||
return item.split("=", 1)[1]
|
||||
return ""
|
||||
|
||||
|
||||
def _check_projection(output: str) -> None:
|
||||
"""Reject full or credential-bearing Kubernetes output projections."""
|
||||
lowered = output.lower().replace(" ", "")
|
||||
if lowered in {"json", "yaml", "wide"} or lowered.startswith("custom-columns"):
|
||||
raise PolicyError(
|
||||
"full Kubernetes object output is forbidden; use a safe projection"
|
||||
)
|
||||
if lowered and not (lowered == "name" or lowered.startswith("jsonpath=")):
|
||||
raise PolicyError(
|
||||
"Kubernetes output must be name or an approved JSONPath projection"
|
||||
)
|
||||
if lowered.startswith("jsonpath="):
|
||||
if any(part in lowered for part in UNSAFE_PROJECTION_PARTS):
|
||||
raise PolicyError("Kubernetes projection may expose runtime credentials")
|
||||
if ".env" in lowered and SAFE_ENV_VALUE not in lowered:
|
||||
# Environment names are safe; arbitrary `.value` reads are not.
|
||||
without_names = lowered.replace("{.name}", "")
|
||||
if ".value" in without_names:
|
||||
raise PolicyError("Kubernetes projection may expose environment values")
|
||||
if lowered not in _APPROVED_PROJECTIONS:
|
||||
raise PolicyError(
|
||||
"Kubernetes JSONPath projection is not on the reviewed allowlist"
|
||||
)
|
||||
|
||||
|
||||
def _check_kubectl(argv: tuple[str, ...], mode: str) -> None:
|
||||
words = list(positionals(argv))
|
||||
if not words or words[0] not in KUBECTL_SUBCOMMANDS:
|
||||
raise PolicyError("kubectl subcommand is outside the read-only boundary")
|
||||
subcommand = words[0]
|
||||
if subcommand == "auth":
|
||||
if len(words) < 2 or words[1] not in {"can-i", "whoami"}:
|
||||
raise PolicyError("kubectl auth is limited to can-i and whoami")
|
||||
return
|
||||
if subcommand == "exec":
|
||||
if "--" not in argv:
|
||||
raise PolicyError("kubectl exec requires an explicit -- command boundary")
|
||||
inner = argv[argv.index("--") + 1 :]
|
||||
if not inner:
|
||||
raise PolicyError("kubectl exec requires an inner command")
|
||||
if inner[0] not in TRUSTED_INNER_PATHS:
|
||||
raise PolicyError(
|
||||
"kubectl exec inner command must use a fixed trusted path"
|
||||
)
|
||||
check_argv(inner, mode)
|
||||
return
|
||||
resources = words[1].split(",") if len(words) > 1 else []
|
||||
if any(_resource_name(resource) in SECRET_RESOURCES for resource in resources):
|
||||
raise PolicyError("Secret objects may not be read by the harness")
|
||||
output = _output_value(argv)
|
||||
if subcommand == "get" and not output:
|
||||
raise PolicyError("kubectl get requires a reviewed output projection")
|
||||
_check_projection(output)
|
||||
|
||||
|
||||
def _check_git(argv: tuple[str, ...], mode: str) -> None:
|
||||
unsafe = False
|
||||
for item in argv[1:]:
|
||||
if item == "-C" or item.startswith(("--git-dir", "--work-tree", "--namespace")):
|
||||
unsafe = True
|
||||
if item == "-c" or (item.startswith("-c") and len(item) > 2):
|
||||
unsafe = True
|
||||
if any(
|
||||
item == flag or item.startswith(f"{flag}=") for flag in GIT_UNSAFE_FLAGS[1:]
|
||||
):
|
||||
unsafe = True
|
||||
if unsafe:
|
||||
raise PolicyError("git configuration and helper overrides are forbidden")
|
||||
subcommand = next(positionals(argv), None)
|
||||
if subcommand is None:
|
||||
raise PolicyError("git requires a subcommand")
|
||||
if subcommand in GIT_READ_SUBCOMMANDS:
|
||||
return
|
||||
if subcommand == "push" and mode == ARMED:
|
||||
if GIT_FORCE_FLAGS & set(argv):
|
||||
raise PolicyError("git push may not force or fan out across refs")
|
||||
words = list(positionals(argv))
|
||||
if len(words) != 3 or words[1] != EXPECTED_REMOTE:
|
||||
raise PolicyError("armed git push is pinned to origin and one refspec")
|
||||
destination = words[2].partition(":")[2]
|
||||
if not _ARMED_REF or destination != f"refs/heads/{_ARMED_REF}":
|
||||
raise PolicyError("armed git push is not the preflighted ephemeral ref")
|
||||
return
|
||||
raise PolicyError(f"git {subcommand} is not permitted in {mode} mode")
|
||||
|
||||
|
||||
def _check_hermes(argv: tuple[str, ...]) -> None:
|
||||
words = tuple(positionals(argv))
|
||||
if not words:
|
||||
raise PolicyError("hermes requires a subcommand")
|
||||
if words[:2] not in HERMES_READ_COMMANDS and words[:1] not in HERMES_READ_COMMANDS:
|
||||
raise PolicyError(
|
||||
f"hermes {' '.join(words[:2])} is outside the read-only boundary"
|
||||
)
|
||||
|
||||
|
||||
def _reject_dot_segments(path: str) -> None:
|
||||
"""Refuse any relative segment, including a percent-encoded one.
|
||||
|
||||
The repository pin below is a textual prefix test, so a single ``..`` would
|
||||
let a pinned path address another repository or the token endpoints once a
|
||||
router normalised it. Decode first: ``%2e%2e`` is the same segment.
|
||||
"""
|
||||
decoded = unquote(unquote(path)).split("?", 1)[0]
|
||||
if any(segment in {"", ".", ".."} for segment in decoded.split("/")[1:]):
|
||||
raise PolicyError("gitea_api.py paths may not contain relative segments")
|
||||
|
||||
|
||||
def _check_gitea(argv: tuple[str, ...], mode: str) -> None:
|
||||
if len(argv) < 3:
|
||||
raise PolicyError("gitea_api.py requires a method and a path")
|
||||
method, path = argv[1].upper(), argv[2]
|
||||
if not path.startswith("/api/v1/") or not _SAFE_API_PATH_RE.fullmatch(path):
|
||||
raise PolicyError("gitea_api.py requires a bounded API path")
|
||||
_reject_dot_segments(path)
|
||||
repo_root = f"/api/v1/repos/{EXPECTED_REPO}"
|
||||
if (
|
||||
path != "/api/v1/user"
|
||||
and path != repo_root
|
||||
and not path.startswith((f"{repo_root}/", f"{repo_root}?"))
|
||||
):
|
||||
raise PolicyError("gitea_api.py is pinned to the Atlas titan-iac repository")
|
||||
if method in GITEA_READ_METHODS:
|
||||
return
|
||||
if mode == ARMED:
|
||||
fields = [
|
||||
argv[index + 1] for index, item in enumerate(argv[:-1]) if item == "--field"
|
||||
]
|
||||
if method == "POST" and path == f"{repo_root}/pulls" and _ARMED_REF:
|
||||
required = {
|
||||
"title=WIP: Hermes handoff acceptance ephemeral probe",
|
||||
f"head={_ARMED_REF}",
|
||||
f"base={EXPECTED_BASE}",
|
||||
"body=Ephemeral acceptance probe. Closed and deleted by the harness.",
|
||||
}
|
||||
if len(fields) == len(required) and set(fields) == required:
|
||||
return
|
||||
match = re.fullmatch(rf"{re.escape(repo_root)}/pulls/([1-9][0-9]*)", path)
|
||||
if (
|
||||
method == "PATCH"
|
||||
and match
|
||||
and int(match.group(1)) in _ARMED_PULLS
|
||||
and fields == ["state=closed"]
|
||||
):
|
||||
return
|
||||
if (
|
||||
method == "DELETE"
|
||||
and _ARMED_REF
|
||||
and path == f"{repo_root}/branches/{_ARMED_REF}"
|
||||
and len(argv) == 3
|
||||
):
|
||||
return
|
||||
raise PolicyError(f"gitea_api.py {method} is not permitted in {mode} mode")
|
||||
|
||||
|
||||
def _check_shell(argv: tuple[str, ...]) -> None:
|
||||
if len(argv) != 3 or argv[1] != "-c" or argv[2] not in _RENDERED:
|
||||
raise PolicyError("sh is only reachable as sh -c <frozen template>")
|
||||
|
||||
|
||||
def check_argv(argv: tuple[str, ...], mode: str = READ_ONLY) -> None:
|
||||
"""Raise :class:`PolicyError` unless ``argv`` is safe in ``mode``."""
|
||||
if mode not in MODES:
|
||||
raise PolicyError(f"unknown mode: {mode}")
|
||||
if not argv:
|
||||
raise PolicyError("empty command")
|
||||
binary = _binary_name(argv[0])
|
||||
if binary in FORBIDDEN_BINARIES:
|
||||
raise PolicyError(f"forbidden binary: {binary}")
|
||||
if binary not in ALLOWED_BINARIES:
|
||||
raise PolicyError(f"binary is not on the harness allowlist: {binary}")
|
||||
_reject_forbidden_arguments(argv)
|
||||
_reject_secret_paths(argv)
|
||||
if binary == "sh":
|
||||
_check_shell(argv)
|
||||
elif binary == "kubectl":
|
||||
_check_kubectl(argv, mode)
|
||||
elif binary == "git":
|
||||
_check_git(argv, mode)
|
||||
elif binary == "hermes":
|
||||
_check_hermes(argv)
|
||||
else:
|
||||
_check_gitea(argv, mode)
|
||||
243
scripts/ops/hermes_handoff_redaction.py
Normal file
243
scripts/ops/hermes_handoff_redaction.py
Normal file
@ -0,0 +1,243 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Credential redaction and output bounding for the handoff acceptance harness.
|
||||
|
||||
Every string the harness records — command output, failure reasons, evidence
|
||||
values — passes through :func:`redact` before it reaches a report. Screening is
|
||||
a fail-closed accident barrier, not proof that text is secret-free: a short
|
||||
secret under an innocuous key is not reliably distinguishable from prose. The
|
||||
harness therefore also never asks for a secret in the first place. Probes read
|
||||
names, shapes, and status fields; they never read environment values, token
|
||||
files, private keys, cookies, or Vault responses.
|
||||
|
||||
Bounding is part of the same barrier. Screening always precedes truncation so a
|
||||
credential that crosses the byte boundary cannot leave a useful prefix behind.
|
||||
The process runner additionally discards all captured bytes when a live stream
|
||||
exceeds its cap; the fixed truncation marker is the only recorded output.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
REDACTED = "[redacted]"
|
||||
DEFAULT_MAX_BYTES = 64 * 1024
|
||||
TRUNCATION_NOTE = "[truncated]"
|
||||
|
||||
# Keys whose value is never safe to record, matched against a normalised
|
||||
# (lowercased, separator-stripped) form of the key.
|
||||
SENSITIVE_KEY_SUFFIXES = (
|
||||
"accesskey",
|
||||
"accesskeyid",
|
||||
"apikey",
|
||||
"authtoken",
|
||||
"clientsecret",
|
||||
"connectionstring",
|
||||
"cookie",
|
||||
"credential",
|
||||
"credentials",
|
||||
"passphrase",
|
||||
"passwd",
|
||||
"password",
|
||||
"privatekey",
|
||||
"secret",
|
||||
"secretkey",
|
||||
"sessionkey",
|
||||
"signature",
|
||||
"signingkey",
|
||||
"token",
|
||||
"webhook",
|
||||
)
|
||||
SENSITIVE_KEY_EXACT = frozenset(
|
||||
{
|
||||
"auth",
|
||||
"authorization",
|
||||
"bearer",
|
||||
"pat",
|
||||
"pw",
|
||||
"sas",
|
||||
"setcookie",
|
||||
}
|
||||
)
|
||||
# A bare `key:` is a structural field name across Kubernetes and TOML far more
|
||||
# often than it is a credential, so it is screened by value shape and by the
|
||||
# token patterns rather than by name. The compound forms above still match.
|
||||
|
||||
# Ordinary prose values shorter than this threshold are not independently
|
||||
# treated as secret-shaped. Values under sensitive keys are screened at every
|
||||
# length, including numeric values.
|
||||
MIN_SECRET_LENGTH = 8
|
||||
NON_SECRET_LITERALS = frozenset(
|
||||
{"true", "false", "null", "none", "yes", "no", "0", "1", ""}
|
||||
)
|
||||
_KEY_NOISE_RE = re.compile(r"[^a-z0-9]+")
|
||||
|
||||
# Ordered: structural matches first so a nested token is not partially rewritten
|
||||
# by a looser rule.
|
||||
_PATTERNS: tuple[re.Pattern[str], ...] = (
|
||||
re.compile(
|
||||
r"-----BEGIN[ A-Z0-9-]{0,64}-----.*?-----END[ A-Z0-9-]{0,64}-----",
|
||||
re.DOTALL,
|
||||
),
|
||||
re.compile(r"-----BEGIN[ A-Z0-9-]{0,64}-----.*", re.DOTALL),
|
||||
re.compile(r"(?i)\b(?:authorization|proxy-authorization)\s*:\s*\S.*"),
|
||||
re.compile(r"(?i)\b(?:set-)?cookie\s*:\s*\S.*"),
|
||||
re.compile(r"(?i)\b(?:bearer|basic|token)\s+[A-Za-z0-9+/_=.:~-]{8,}"),
|
||||
re.compile(r"(?i)\b[a-z][a-z0-9+.-]{1,20}://[^\s/@:]{1,128}:[^\s/@]{1,256}@"),
|
||||
re.compile(
|
||||
r"\bssh-(?:rsa|ed25519|dss|ecdsa-[A-Za-z0-9-]+)\s+[A-Za-z0-9+/]{16,}={0,3}"
|
||||
),
|
||||
re.compile(r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{4,}"),
|
||||
re.compile(r"(?<![A-Za-z0-9])gh[pousr]_[A-Za-z0-9]*"),
|
||||
re.compile(r"(?<![A-Za-z0-9])github_pat_[A-Za-z0-9_]*"),
|
||||
re.compile(r"(?<![A-Za-z0-9])glpat-[A-Za-z0-9_-]*"),
|
||||
re.compile(r"(?<![A-Za-z0-9])xox[baprs]-[A-Za-z0-9-]*"),
|
||||
re.compile(r"(?<![A-Za-z0-9])sk-(?:ant-|proj-)?[A-Za-z0-9_-]*"),
|
||||
re.compile(r"(?<![A-Za-z0-9])(?:gta|gto|gitea|forgejo)_[A-Za-z0-9_-]*"),
|
||||
re.compile(r"(?<![A-Za-z0-9])hvs\.[A-Za-z0-9_-]*"),
|
||||
re.compile(r"(?<![A-Za-z0-9])(?:AKIA|ASIA)[0-9A-Z]*"),
|
||||
re.compile(r"(?<![A-Za-z0-9])AIza[0-9A-Za-z_-]{35}"),
|
||||
re.compile(r"(?<![A-Za-z0-9])npm_[A-Za-z0-9]{16,}"),
|
||||
)
|
||||
|
||||
# `key: value` / `key=value` where the key names a credential.
|
||||
_ASSIGNMENT_RE = re.compile(
|
||||
r"""(?x)
|
||||
(?P<quote>["']?)
|
||||
(?P<key>[A-Za-z][A-Za-z0-9_.\ -]{0,63})
|
||||
(?P=quote)
|
||||
(?P<gap>\s{0,8}[:=]\s{0,8})
|
||||
(?P<value>"[^"\n]{0,4096}"|'[^'\n]{0,4096}'|[^\s,;}\n][^,;}\n]{0,4096})
|
||||
"""
|
||||
)
|
||||
|
||||
# Long, mixed-class runs. Deliberately conservative: prose and hex digests of
|
||||
# fewer than 44 characters are left alone so commit SHAs stay readable.
|
||||
_ENTROPY_RE = re.compile(
|
||||
r"(?<![A-Za-z0-9+/_=-])[A-Za-z0-9+/_=-]{44,}(?![A-Za-z0-9+/_=-])"
|
||||
)
|
||||
|
||||
|
||||
def normalise_key(key: str) -> str:
|
||||
"""Return a key lowercased with separators removed, for suffix matching."""
|
||||
return _KEY_NOISE_RE.sub("", key.lower())
|
||||
|
||||
|
||||
def is_sensitive_key(key: str) -> bool:
|
||||
"""Report whether a key name means its value must never be recorded."""
|
||||
compact = normalise_key(key)
|
||||
if not compact:
|
||||
return False
|
||||
if compact in SENSITIVE_KEY_EXACT:
|
||||
return True
|
||||
return compact.endswith(SENSITIVE_KEY_SUFFIXES)
|
||||
|
||||
|
||||
def _is_high_entropy(candidate: str) -> bool:
|
||||
"""Report whether a long run mixes enough character classes to be a token."""
|
||||
classes = sum(
|
||||
(
|
||||
any(character.islower() for character in candidate),
|
||||
any(character.isupper() for character in candidate),
|
||||
any(character.isdigit() for character in candidate),
|
||||
any(character in "+/_=-" for character in candidate),
|
||||
)
|
||||
)
|
||||
return classes >= 4
|
||||
|
||||
|
||||
def could_hold_secret(value: str) -> bool:
|
||||
"""Report whether a raw value is long and opaque enough to be a credential."""
|
||||
stripped = value.strip().strip("\"'")
|
||||
if stripped.lower() in NON_SECRET_LITERALS or len(stripped) < MIN_SECRET_LENGTH:
|
||||
return False
|
||||
try:
|
||||
float(stripped)
|
||||
except ValueError:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _redact_assignment(match: re.Match[str]) -> str:
|
||||
value = match.group("value")
|
||||
stripped = value.strip().strip("\"'").lower()
|
||||
if not is_sensitive_key(match.group("key")) or stripped in NON_SECRET_LITERALS:
|
||||
return match.group(0)
|
||||
# Preserve the original quoting so screened JSON stays parsable JSON.
|
||||
quoted = value.strip()[:1] in {'"', "'"}
|
||||
replacement = f'"{REDACTED}"' if quoted else REDACTED
|
||||
key_quote = match.group("quote")
|
||||
return (
|
||||
f"{key_quote}{match.group('key')}{key_quote}{match.group('gap')}{replacement}"
|
||||
)
|
||||
|
||||
|
||||
def _redact_entropy(match: re.Match[str]) -> str:
|
||||
return REDACTED if _is_high_entropy(match.group(0)) else match.group(0)
|
||||
|
||||
|
||||
def redact(text: str) -> str:
|
||||
"""Return ``text`` with credential-shaped substrings replaced."""
|
||||
if not text:
|
||||
return text
|
||||
for pattern in _PATTERNS:
|
||||
text = pattern.sub(REDACTED, text)
|
||||
text = _ASSIGNMENT_RE.sub(_redact_assignment, text)
|
||||
return _ENTROPY_RE.sub(_redact_entropy, text)
|
||||
|
||||
|
||||
def bound(text: str, max_bytes: int = DEFAULT_MAX_BYTES) -> tuple[str, bool]:
|
||||
"""Truncate ``text`` to a UTF-8 byte budget, reporting whether it was cut."""
|
||||
if max_bytes <= 0:
|
||||
return ("", bool(text))
|
||||
encoded = text.encode("utf-8", "replace")
|
||||
if len(encoded) <= max_bytes:
|
||||
return (text, False)
|
||||
marker = TRUNCATION_NOTE.encode()[:max_bytes]
|
||||
content_budget = max(
|
||||
0, max_bytes - len(marker) - (1 if len(marker) < max_bytes else 0)
|
||||
)
|
||||
kept = encoded[:content_budget].decode("utf-8", "ignore")
|
||||
separator = "\n" if kept and len(marker) < max_bytes else ""
|
||||
return (f"{kept}{separator}{marker.decode()}", True)
|
||||
|
||||
|
||||
def safe_text(text: str, max_bytes: int = DEFAULT_MAX_BYTES) -> tuple[str, bool]:
|
||||
"""Redact then bound, so truncation never exposes a credential prefix."""
|
||||
return bound(redact(text), max_bytes)
|
||||
|
||||
|
||||
def scrub(value: Any, max_bytes: int = DEFAULT_MAX_BYTES) -> Any:
|
||||
"""Recursively bound and redact a JSON-shaped value.
|
||||
|
||||
Mapping keys are themselves screened; values under a sensitive key are
|
||||
dropped outright rather than pattern-screened, because a short secret
|
||||
would survive screening.
|
||||
"""
|
||||
if isinstance(value, str):
|
||||
return safe_text(value, max_bytes)[0]
|
||||
if isinstance(value, dict):
|
||||
screened: dict[str, Any] = {}
|
||||
for key, item in value.items():
|
||||
raw_key = str(key)
|
||||
safe_key = safe_text(raw_key, max_bytes)[0]
|
||||
sensitive_value = (
|
||||
is_sensitive_key(raw_key)
|
||||
and item is not None
|
||||
and not isinstance(item, bool)
|
||||
and item != ""
|
||||
)
|
||||
screened[safe_key] = REDACTED if sensitive_value else scrub(item, max_bytes)
|
||||
return screened
|
||||
if isinstance(value, (list, tuple)):
|
||||
return [scrub(item, max_bytes) for item in value]
|
||||
return value
|
||||
|
||||
|
||||
def contains_credential_shape(text: str) -> bool:
|
||||
"""Report whether screening would rewrite ``text``.
|
||||
|
||||
Used by the harness self-test to prove a rendered report is clean before it
|
||||
is written, and by the unit suite as an assertion helper.
|
||||
"""
|
||||
return redact(text) != text
|
||||
199
scripts/ops/hermes_handoff_rules.py
Normal file
199
scripts/ops/hermes_handoff_rules.py
Normal file
@ -0,0 +1,199 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Shared primitives and dispatch for handoff acceptance evaluators.
|
||||
|
||||
Evaluators are registered by name so the check catalog stays declarative: an
|
||||
entry says *what* it asserts, not how to plumb it. The dispatch here is the one
|
||||
place that turns an unexpected exception into ``NOT_RUN`` — a harness bug must
|
||||
never be indistinguishable from a passing release.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from hermes_handoff_exec import Outcome
|
||||
from hermes_handoff_model import NOT_RUN, STATUSES, CheckSpec
|
||||
|
||||
DENIAL_MARKERS = (
|
||||
"forbidden",
|
||||
"is not allowed",
|
||||
"unauthorized",
|
||||
"permission denied",
|
||||
"cannot get",
|
||||
"cannot list",
|
||||
"cannot create",
|
||||
"cannot delete",
|
||||
"cannot patch",
|
||||
"not permitted",
|
||||
"gitea api returned http 403",
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Evaluation:
|
||||
"""A classification plus the small facts that justify it."""
|
||||
|
||||
status: str
|
||||
reason: str
|
||||
evidence: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
Evaluator = Callable[[CheckSpec, dict[str, Outcome]], Evaluation]
|
||||
EVALUATORS: dict[str, Evaluator] = {}
|
||||
|
||||
|
||||
def evaluator(name: str) -> Callable[[Evaluator], Evaluator]:
|
||||
"""Register an evaluator under the rule name catalog entries use."""
|
||||
|
||||
def register(function: Evaluator) -> Evaluator:
|
||||
EVALUATORS[name] = function
|
||||
return function
|
||||
|
||||
return register
|
||||
|
||||
|
||||
def missing_steps(
|
||||
spec: CheckSpec, outcomes: Mapping[str, Outcome]
|
||||
) -> Evaluation | None:
|
||||
"""Return NOT_RUN when a required step never produced usable evidence."""
|
||||
for step in spec.steps:
|
||||
if step.optional:
|
||||
continue
|
||||
outcome = outcomes.get(step.key)
|
||||
if outcome is None:
|
||||
return Evaluation(NOT_RUN, f"step {step.key} was never executed")
|
||||
if outcome.truncated:
|
||||
return Evaluation(NOT_RUN, f"step {step.key} output was truncated")
|
||||
if not outcome.ran:
|
||||
return Evaluation(NOT_RUN, f"step {step.key} did not run: {outcome.error}")
|
||||
return None
|
||||
|
||||
|
||||
def looks_denied(outcome: Outcome) -> bool:
|
||||
"""Report whether a real request was refused by policy rather than failing."""
|
||||
if outcome.ok:
|
||||
return False
|
||||
haystack = outcome.combined.lower()
|
||||
return any(marker in haystack for marker in DENIAL_MARKERS)
|
||||
|
||||
|
||||
def step_keys(spec: CheckSpec, kind: str) -> tuple[str, ...]:
|
||||
"""Return the keys of every step of one kind."""
|
||||
return tuple(step.key for step in spec.steps if step.kind == kind)
|
||||
|
||||
|
||||
def is_list(value: Any) -> bool:
|
||||
"""Report whether a value is a JSON array rather than a string."""
|
||||
return isinstance(value, Sequence) and not isinstance(value, (str, bytes))
|
||||
|
||||
|
||||
def exact_equal(actual: Any, expected: Any) -> bool:
|
||||
"""Compare JSON values without Python's bool/int type confusion."""
|
||||
return type(actual) is type(expected) and actual == expected
|
||||
|
||||
|
||||
def dotted(payload: Any, path: str) -> Any:
|
||||
"""Return a value addressed by a dotted path.
|
||||
|
||||
``a.b[0].c`` indexes; ``a[].b`` maps the remainder of the path over a list,
|
||||
which is how the catalog records one field from every pull request or every
|
||||
Flux object without a bespoke parser per check.
|
||||
"""
|
||||
current = payload
|
||||
segments = [segment for segment in path.split(".") if segment]
|
||||
for position, raw in enumerate(segments):
|
||||
name, bracket, indexes = raw.partition("[")
|
||||
if name:
|
||||
if not isinstance(current, Mapping) or name not in current:
|
||||
raise KeyError(path)
|
||||
current = current[name]
|
||||
if not bracket:
|
||||
continue
|
||||
for chunk in indexes.rstrip("]").split("]["):
|
||||
if not is_list(current):
|
||||
raise KeyError(path)
|
||||
if chunk == "":
|
||||
remainder = ".".join(segments[position + 1 :])
|
||||
return [
|
||||
dotted(item, remainder) if remainder else item for item in current
|
||||
]
|
||||
offset = int(chunk)
|
||||
if offset >= len(current):
|
||||
raise KeyError(path)
|
||||
current = current[offset]
|
||||
return current
|
||||
|
||||
|
||||
def strict_json(text: str) -> Any:
|
||||
"""Decode strict JSON, rejecting constants and duplicate object keys."""
|
||||
text = text.strip()
|
||||
if not text:
|
||||
raise ValueError("empty output")
|
||||
|
||||
def reject_constant(value: str) -> None:
|
||||
raise ValueError(f"non-finite JSON constant {value}")
|
||||
|
||||
def unique_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
|
||||
result: dict[str, Any] = {}
|
||||
for key, value in pairs:
|
||||
if key in result:
|
||||
raise ValueError(f"duplicate JSON key {key!r}")
|
||||
result[key] = value
|
||||
return result
|
||||
|
||||
return json.loads(
|
||||
text,
|
||||
strict=True,
|
||||
parse_constant=reject_constant,
|
||||
object_pairs_hook=unique_object,
|
||||
)
|
||||
|
||||
|
||||
def parse_json(outcome: Outcome) -> Any:
|
||||
"""Parse one outcome as strict JSON."""
|
||||
return strict_json(outcome.stdout)
|
||||
|
||||
|
||||
def lines(outcome: Outcome) -> list[str]:
|
||||
"""Return non-empty, stripped stdout lines."""
|
||||
return [line.strip() for line in outcome.stdout.splitlines() if line.strip()]
|
||||
|
||||
|
||||
def parsed_step(
|
||||
spec: CheckSpec, outcomes: dict[str, Outcome]
|
||||
) -> tuple[Any, str] | Evaluation:
|
||||
"""Return the parsed payload of ``expect['step']`` or the NOT_RUN to report."""
|
||||
key = spec.expect["step"]
|
||||
outcome = outcomes[key]
|
||||
if not outcome.ok:
|
||||
return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})")
|
||||
try:
|
||||
return (parse_json(outcome), key)
|
||||
except ValueError as exc:
|
||||
return Evaluation(NOT_RUN, f"step {key} did not return JSON: {exc}")
|
||||
|
||||
|
||||
def evaluate(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
|
||||
"""Dispatch to the named evaluator, converting any surprise into NOT_RUN."""
|
||||
rule = EVALUATORS.get(spec.rule)
|
||||
if rule is None:
|
||||
return Evaluation(NOT_RUN, f"unknown evaluator: {spec.rule}")
|
||||
truncated = sorted(key for key, outcome in outcomes.items() if outcome.truncated)
|
||||
if truncated:
|
||||
return Evaluation(
|
||||
NOT_RUN, f"truncated evidence from step(s): {', '.join(truncated)}"
|
||||
)
|
||||
try:
|
||||
result = rule(spec, outcomes)
|
||||
except Exception as exc: # noqa: BLE001 - a harness bug must never read as PASS
|
||||
return Evaluation(
|
||||
NOT_RUN, f"evaluator {spec.rule} raised {type(exc).__name__}: {exc}"
|
||||
)
|
||||
if result.status not in STATUSES:
|
||||
return Evaluation(
|
||||
NOT_RUN, f"evaluator {spec.rule} returned unknown status {result.status!r}"
|
||||
)
|
||||
return result
|
||||
484
scripts/ops/hermes_handoff_run.py
Normal file
484
scripts/ops/hermes_handoff_run.py
Normal file
@ -0,0 +1,484 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Orchestration for the Hermes full-handoff acceptance harness.
|
||||
|
||||
Three things happen before any check runs, and each of them can stop the run on
|
||||
its own:
|
||||
|
||||
* the catalog is validated structurally — every rule exists, every live-denial
|
||||
evaluator carries a read-only attempt, no operator-side step can impersonate;
|
||||
* each vantage is resolved and asked who it is; and
|
||||
* the operator and in-pod identities are compared. If they are the same
|
||||
principal there is only one vantage, the dual-vantage evidence would be a
|
||||
restatement rather than a corroboration, and the run is a NO_GO.
|
||||
|
||||
Everything after that is uniform: run a check's steps against its vantage, hand
|
||||
the outcomes to the named evaluator, and record the classification. A vantage
|
||||
that could not be resolved yields ``NOT_RUN`` for every check that needs it,
|
||||
which is exactly the fail-closed answer.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
|
||||
from hermes_handoff_catalog import (
|
||||
CHAT,
|
||||
NODE,
|
||||
OPERATOR,
|
||||
SELF,
|
||||
SWITCHYARD,
|
||||
VANTAGE_NAMES,
|
||||
Targets,
|
||||
)
|
||||
from hermes_handoff_checks_access import access_checks
|
||||
from hermes_handoff_checks_delivery import delivery_checks
|
||||
from hermes_handoff_checks_platform import platform_checks
|
||||
from hermes_handoff_checks_workers import worker_checks
|
||||
from hermes_handoff_evaluators import EVALUATORS, evaluate
|
||||
from hermes_handoff_exec import Outcome, Runner, Vantage, operator_vantage, pod_vantage
|
||||
from hermes_handoff_model import (
|
||||
ATTEMPT,
|
||||
EPHEMERAL,
|
||||
NOT_RUN,
|
||||
STEP_KINDS,
|
||||
CheckResult,
|
||||
CheckSpec,
|
||||
Report,
|
||||
VantageRecord,
|
||||
utc_now,
|
||||
)
|
||||
from hermes_handoff_policy import (
|
||||
EXPECTED_REMOTE,
|
||||
EXPECTED_REPO,
|
||||
IMPERSONATION_ARGS,
|
||||
PolicyError,
|
||||
check_argv,
|
||||
projection,
|
||||
)
|
||||
|
||||
UNRECORDED = "[output not recorded]"
|
||||
|
||||
|
||||
def build_catalog(targets: Targets) -> list[CheckSpec]:
|
||||
"""Return the whole acceptance catalog for a set of targets."""
|
||||
return [
|
||||
*platform_checks(targets),
|
||||
*access_checks(targets),
|
||||
*delivery_checks(targets),
|
||||
*worker_checks(targets),
|
||||
]
|
||||
|
||||
|
||||
def validate_catalog(specs: Sequence[CheckSpec]) -> list[str]:
|
||||
"""Return the structural problems that make a catalog unsafe to run."""
|
||||
problems: list[str] = []
|
||||
if not specs:
|
||||
return ["catalog is empty"]
|
||||
seen: set[str] = set()
|
||||
fake_operator = operator_vantage()
|
||||
fake_pod = pod_vantage("hermes", "acceptance-probe", "probe", fake_operator)
|
||||
for spec in specs:
|
||||
if spec.id in seen:
|
||||
problems.append(f"{spec.id}: duplicate check id")
|
||||
seen.add(spec.id)
|
||||
if spec.rule not in EVALUATORS and spec.scope != EPHEMERAL:
|
||||
problems.append(f"{spec.id}: unknown rule {spec.rule!r}")
|
||||
step_names: set[str] = set()
|
||||
for step in spec.steps:
|
||||
if step.key in step_names:
|
||||
problems.append(f"{spec.id}: duplicate step key {step.key!r}")
|
||||
step_names.add(step.key)
|
||||
if step.vantage not in VANTAGE_NAMES:
|
||||
problems.append(
|
||||
f"{spec.id}: step {step.key} names unknown vantage {step.vantage!r}"
|
||||
)
|
||||
if step.kind not in STEP_KINDS:
|
||||
problems.append(
|
||||
f"{spec.id}: step {step.key} has unknown kind {step.kind!r}"
|
||||
)
|
||||
if set(step.argv) & set(IMPERSONATION_ARGS):
|
||||
problems.append(
|
||||
f"{spec.id}: step {step.key} impersonates from the "
|
||||
f"{step.vantage} vantage"
|
||||
)
|
||||
try:
|
||||
addressed = (
|
||||
fake_operator if step.vantage == OPERATOR else fake_pod
|
||||
).wrap(step.argv)
|
||||
check_argv(addressed)
|
||||
except PolicyError as exc:
|
||||
problems.append(f"{spec.id}: step {step.key} violates policy: {exc}")
|
||||
if spec.rule == "denied" and not any(
|
||||
step.kind == ATTEMPT for step in spec.steps
|
||||
):
|
||||
problems.append(
|
||||
f"{spec.id}: a deny check needs a real attempt, not only a review"
|
||||
)
|
||||
return problems
|
||||
|
||||
|
||||
def validate_targets(targets: Targets) -> list[str]:
|
||||
"""Return release-input problems before a runner or network call exists."""
|
||||
problems: list[str] = []
|
||||
sha = re.compile(r"[0-9a-f]{40}\Z")
|
||||
image = re.compile(
|
||||
r"[^@\s]+:git-([0-9a-f]{40})-build-[1-9][0-9]*@sha256:[0-9a-f]{64}\Z"
|
||||
)
|
||||
if targets.repo != EXPECTED_REPO or targets.remote != EXPECTED_REMOTE:
|
||||
problems.append(
|
||||
"repository and remote must be the fixed Atlas titan-iac origin"
|
||||
)
|
||||
if (
|
||||
targets.reviewed_pr_number != 19
|
||||
or targets.reviewed_head_ref != "feature/hermes-full-handoff-acceptance"
|
||||
):
|
||||
problems.append(
|
||||
"reviewed PR identity must remain fixed to draft PR #19 and its existing branch"
|
||||
)
|
||||
for label, value in (
|
||||
("baseline commit", targets.baseline_commit),
|
||||
("remote main SHA", targets.remote_main_sha),
|
||||
("reviewed head SHA", targets.reviewed_head_sha),
|
||||
("build SHA", targets.build_sha),
|
||||
):
|
||||
if not sha.fullmatch(value):
|
||||
problems.append(f"{label} must be an exact lowercase 40-character SHA")
|
||||
image_match = image.fullmatch(targets.agent_image)
|
||||
if not image_match or image_match.group(1) != targets.build_sha:
|
||||
problems.append("agent image tag/digest must bind the exact build SHA")
|
||||
if targets.build_sha != targets.remote_main_sha:
|
||||
problems.append("build SHA must equal the exact release main SHA")
|
||||
if not re.fullmatch(r"[1-9][0-9]{0,8}", str(targets.deployment_revision)):
|
||||
problems.append("deployment revision must be a positive integer string")
|
||||
if targets.node_count != 3 or targets.pool_replicas != 3:
|
||||
problems.append("node and pool counts are fixed at exactly three")
|
||||
if not targets.chat_config_revision.strip():
|
||||
problems.append(
|
||||
"Telegram continuity and its chat config revision are mandatory"
|
||||
)
|
||||
if set(targets.pool_worker_env) != {"HERMES_WORKER_NODE", "HERMES_WORKER_ORDINAL"}:
|
||||
problems.append("pool assignment evidence must include worker node and ordinal")
|
||||
heads = dict(targets.dependency_heads)
|
||||
if len(heads) != len(targets.dependency_heads) or set(heads) != set(
|
||||
targets.dependency_pull_requests
|
||||
):
|
||||
problems.append("every dependency PR must have one exact current head")
|
||||
elif any(not sha.fullmatch(value) for value in heads.values()):
|
||||
problems.append("dependency heads must be exact lowercase 40-character SHAs")
|
||||
if (
|
||||
not isinstance(targets.max_evidence_age_seconds, int)
|
||||
or targets.max_evidence_age_seconds <= 0
|
||||
):
|
||||
problems.append("maximum evidence age must be a positive integer")
|
||||
return problems
|
||||
|
||||
|
||||
def resolve_identity(runner: Runner, vantage: Vantage) -> tuple[str, str]:
|
||||
"""Return the authenticated username of a vantage and any failure detail."""
|
||||
outcome = runner.run(
|
||||
(
|
||||
"kubectl",
|
||||
"auth",
|
||||
"whoami",
|
||||
"-o",
|
||||
projection("jsonpath={.status.userInfo.username}"),
|
||||
),
|
||||
vantage,
|
||||
)
|
||||
if not outcome.ok:
|
||||
return ("", outcome.error or outcome.combined[:200])
|
||||
username = outcome.stdout.strip()
|
||||
if not username or any(character.isspace() for character in username):
|
||||
return ("", "identity response carried no username")
|
||||
return (username, "")
|
||||
|
||||
|
||||
def _first_pod(
|
||||
runner: Runner, operator: Vantage, namespace: str, selector: str
|
||||
) -> tuple[str, str]:
|
||||
outcome = runner.run(
|
||||
(
|
||||
"kubectl",
|
||||
"--namespace",
|
||||
namespace,
|
||||
"get",
|
||||
"pods",
|
||||
"--selector",
|
||||
selector,
|
||||
"--field-selector",
|
||||
"status.phase=Running",
|
||||
"-o",
|
||||
"name",
|
||||
),
|
||||
operator,
|
||||
)
|
||||
if not outcome.ok:
|
||||
return ("", outcome.error or outcome.combined[:200])
|
||||
names = [
|
||||
line.strip().removeprefix("pod/")
|
||||
for line in outcome.stdout.splitlines()
|
||||
if line.strip()
|
||||
]
|
||||
if not names:
|
||||
return ("", f"no running pod matches {selector}")
|
||||
return (sorted(names)[0], "")
|
||||
|
||||
|
||||
def resolve_vantages(
|
||||
runner: Runner,
|
||||
targets: Targets,
|
||||
kubeconfig: str | None = None,
|
||||
context: str | None = None,
|
||||
) -> tuple[dict[str, Vantage], list[VantageRecord]]:
|
||||
"""Resolve every vantage the catalog names, recording what each one is."""
|
||||
operator = operator_vantage(kubeconfig, context)
|
||||
username, detail = resolve_identity(runner, operator)
|
||||
records = [
|
||||
VantageRecord(
|
||||
name=OPERATOR,
|
||||
description=operator.description or "external read-only operator",
|
||||
identity=username,
|
||||
available=bool(username),
|
||||
detail=detail,
|
||||
)
|
||||
]
|
||||
vantages: dict[str, Vantage] = {OPERATOR: operator}
|
||||
if not username:
|
||||
return (vantages, records)
|
||||
|
||||
for name, selector, container, description in (
|
||||
(
|
||||
SELF,
|
||||
f"app={targets.agent_deployment}",
|
||||
targets.agent_container,
|
||||
"the Hermes agent itself",
|
||||
),
|
||||
(
|
||||
SWITCHYARD,
|
||||
f"app={targets.switchyard_deployment}",
|
||||
targets.switchyard_container,
|
||||
"the Switchyard routing evidence log",
|
||||
),
|
||||
(
|
||||
NODE,
|
||||
f"app={targets.node_daemonset}",
|
||||
targets.node_container,
|
||||
"a node hardening probe",
|
||||
),
|
||||
):
|
||||
pod, failure = _first_pod(runner, operator, targets.namespace, selector)
|
||||
if not pod:
|
||||
records.append(
|
||||
VantageRecord(
|
||||
name=name, description=description, available=False, detail=failure
|
||||
)
|
||||
)
|
||||
continue
|
||||
vantage = pod_vantage(targets.namespace, pod, container, operator)
|
||||
identity, identity_detail = (
|
||||
resolve_identity(runner, vantage) if name == SELF else ("", "")
|
||||
)
|
||||
records.append(
|
||||
VantageRecord(
|
||||
name=name,
|
||||
description=f"{description} ({targets.namespace}/{pod})",
|
||||
identity=identity,
|
||||
available=True,
|
||||
detail=identity_detail,
|
||||
)
|
||||
)
|
||||
vantages[name] = vantage
|
||||
|
||||
chat_pod = f"{targets.chat_statefulset}-{targets.chat_ordinal}"
|
||||
present = runner.run(
|
||||
(
|
||||
"kubectl",
|
||||
"--namespace",
|
||||
targets.namespace,
|
||||
"get",
|
||||
f"pod/{chat_pod}",
|
||||
"-o",
|
||||
"name",
|
||||
),
|
||||
operator,
|
||||
)
|
||||
if present.ok and present.stdout.strip():
|
||||
vantages[CHAT] = pod_vantage(
|
||||
targets.namespace, chat_pod, targets.chat_container, operator
|
||||
)
|
||||
records.append(
|
||||
VantageRecord(
|
||||
name=CHAT,
|
||||
description=f"the chat surface ({targets.namespace}/{chat_pod})",
|
||||
available=True,
|
||||
)
|
||||
)
|
||||
else:
|
||||
records.append(
|
||||
VantageRecord(
|
||||
name=CHAT,
|
||||
description=f"the chat surface ({targets.namespace}/{chat_pod})",
|
||||
available=False,
|
||||
detail=present.error
|
||||
or present.combined[:200]
|
||||
or "chat tenant pod not found",
|
||||
)
|
||||
)
|
||||
return (vantages, records)
|
||||
|
||||
|
||||
def vantage_problems(records: Iterable[VantageRecord]) -> list[str]:
|
||||
"""Return the identity problems that invalidate dual-vantage evidence."""
|
||||
by_name = {record.name: record for record in records}
|
||||
operator = by_name.get(OPERATOR)
|
||||
hermes = by_name.get(SELF)
|
||||
problems: list[str] = []
|
||||
if operator is None or not operator.available:
|
||||
problems.append(
|
||||
"the operator vantage is unavailable; no evidence can be collected"
|
||||
)
|
||||
return problems
|
||||
if hermes is None or not hermes.available:
|
||||
problems.append(
|
||||
"the in-pod Hermes vantage is unavailable; self-probe evidence is missing"
|
||||
)
|
||||
return problems
|
||||
if not hermes.identity:
|
||||
problems.append("the in-pod vantage did not report an identity")
|
||||
elif operator.identity == hermes.identity:
|
||||
problems.append(
|
||||
"the operator and in-pod vantages authenticate as the same principal "
|
||||
f"({operator.identity}); run the harness with a separate operator kubeconfig"
|
||||
)
|
||||
for name, label in ((SWITCHYARD, "Switchyard"), (CHAT, "chat/Telegram")):
|
||||
record = by_name.get(name)
|
||||
if record is None or not record.available:
|
||||
problems.append(f"the required {label} vantage is unavailable")
|
||||
return problems
|
||||
|
||||
|
||||
def run_check(
|
||||
runner: Runner, spec: CheckSpec, vantages: Mapping[str, Vantage]
|
||||
) -> CheckResult:
|
||||
"""Run one check's steps and classify the result."""
|
||||
outcomes: dict[str, Outcome] = {}
|
||||
recorded: list[Outcome] = []
|
||||
for step in spec.steps:
|
||||
vantage = vantages.get(step.vantage)
|
||||
if vantage is None:
|
||||
outcome = Outcome(
|
||||
argv=step.argv,
|
||||
vantage=step.vantage,
|
||||
error=f"vantage {step.vantage} is unavailable",
|
||||
)
|
||||
else:
|
||||
outcome = runner.run(step.argv, vantage, step.max_bytes)
|
||||
outcomes[step.key] = outcome
|
||||
recorded.append(
|
||||
outcome
|
||||
if step.record
|
||||
else Outcome(
|
||||
argv=outcome.argv,
|
||||
vantage=outcome.vantage,
|
||||
returncode=outcome.returncode,
|
||||
stdout=UNRECORDED if outcome.stdout else "",
|
||||
stderr=UNRECORDED if outcome.stderr else "",
|
||||
truncated=outcome.truncated,
|
||||
duration_ms=outcome.duration_ms,
|
||||
error=outcome.error,
|
||||
executable_path=outcome.executable_path,
|
||||
executable_sha256=outcome.executable_sha256,
|
||||
)
|
||||
)
|
||||
evaluation = evaluate(spec, outcomes)
|
||||
return CheckResult(
|
||||
spec=spec,
|
||||
status=evaluation.status,
|
||||
reason=evaluation.reason,
|
||||
evidence=evaluation.evidence,
|
||||
outcomes=recorded,
|
||||
)
|
||||
|
||||
|
||||
def run_catalog(
|
||||
runner: Runner, specs: Sequence[CheckSpec], vantages: Mapping[str, Vantage]
|
||||
) -> list[CheckResult]:
|
||||
"""Run every check in the catalog.
|
||||
|
||||
Nothing is skipped once the deadline passes: the runner returns a
|
||||
deadline outcome for each remaining command, which classifies as
|
||||
``NOT_RUN``, so a truncated sweep is visible in the report rather than
|
||||
silently shorter.
|
||||
"""
|
||||
return [run_check(runner, spec, vantages) for spec in specs]
|
||||
|
||||
|
||||
def build_report(
|
||||
runner: Runner,
|
||||
targets: Targets,
|
||||
specs: Sequence[CheckSpec],
|
||||
vantages: Mapping[str, Vantage],
|
||||
records: Sequence[VantageRecord],
|
||||
mode: str,
|
||||
started_at: str,
|
||||
extra_results: Sequence[CheckResult] = (),
|
||||
) -> Report:
|
||||
"""Run the catalog and assemble the report, including structural problems."""
|
||||
problems = validate_catalog(specs) + vantage_problems(records)
|
||||
report = Report(
|
||||
mode=mode,
|
||||
started_at=started_at,
|
||||
baseline={
|
||||
"namespace": targets.namespace,
|
||||
"baseline_commit": targets.baseline_commit,
|
||||
"dependency_pull_requests": list(targets.dependency_pull_requests),
|
||||
"expected_suspensions": list(targets.expected_suspensions),
|
||||
"node_count": targets.node_count,
|
||||
"pool_replicas": targets.pool_replicas,
|
||||
},
|
||||
vantages=list(records),
|
||||
harness_errors=problems,
|
||||
)
|
||||
armed_ids = {result.spec.id for result in extra_results}
|
||||
runnable = [spec for spec in specs if spec.id not in armed_ids]
|
||||
if problems:
|
||||
report.results = [
|
||||
CheckResult(spec=spec, status=NOT_RUN, reason="harness preflight failed")
|
||||
for spec in runnable
|
||||
]
|
||||
report.results.extend(extra_results)
|
||||
else:
|
||||
report.results = [*run_catalog(runner, runnable, vantages), *extra_results]
|
||||
report.results.sort(key=lambda result: result.spec.id)
|
||||
if runner.remaining_seconds <= 0:
|
||||
report.harness_errors.append(
|
||||
"the run deadline expired before every check completed"
|
||||
)
|
||||
report.finished_at = utc_now()
|
||||
return report
|
||||
|
||||
|
||||
def unavailable_report(
|
||||
mode: str, started_at: str, reason: str, targets: Targets
|
||||
) -> Report:
|
||||
"""Return a NO_GO report for a run that could not start at all."""
|
||||
return Report(
|
||||
mode=mode,
|
||||
started_at=started_at,
|
||||
finished_at=utc_now(),
|
||||
baseline={"namespace": targets.namespace},
|
||||
harness_errors=[reason],
|
||||
results=[
|
||||
CheckResult(
|
||||
spec=CheckSpec(
|
||||
id="harness.startup",
|
||||
title="The harness could start and resolve its vantages",
|
||||
group="harness",
|
||||
rule="allowed",
|
||||
),
|
||||
status=NOT_RUN,
|
||||
reason=reason,
|
||||
)
|
||||
],
|
||||
)
|
||||
25
scripts/tests/test_dashboard_render_support.py
Normal file
25
scripts/tests/test_dashboard_render_support.py
Normal file
@ -0,0 +1,25 @@
|
||||
import importlib.util
|
||||
|
||||
import pathlib
|
||||
|
||||
def load_module():
|
||||
path = pathlib.Path(__file__).resolve().parents[1] / "render" / "dashboards_render_atlas.py"
|
||||
spec = importlib.util.spec_from_file_location("scripts.dashboards_render_atlas", path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
assert spec.loader is not None
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
def flatten_panels(panels):
|
||||
flat = []
|
||||
for panel in panels:
|
||||
flat.append(panel)
|
||||
flat.extend(panel.get("panels", []))
|
||||
return flat
|
||||
|
||||
__all__ = (
|
||||
"flatten_panels",
|
||||
"importlib",
|
||||
"load_module",
|
||||
"pathlib",
|
||||
)
|
||||
127
scripts/tests/test_dashboards_render_atlas_drilldowns.py
Normal file
127
scripts/tests/test_dashboards_render_atlas_drilldowns.py
Normal file
@ -0,0 +1,127 @@
|
||||
"""Detailed Atlas Jobs dashboard collapse and drilldown contracts."""
|
||||
|
||||
from scripts.tests.test_dashboard_render_support import load_module
|
||||
|
||||
|
||||
def test_jobs_dashboard_collapses_heavy_drilldowns_for_light_first_paint():
|
||||
mod = load_module()
|
||||
dashboard = mod.build_jobs_dashboard()
|
||||
panels = dashboard["panels"]
|
||||
rows = [panel for panel in panels if panel["type"] == "row"]
|
||||
visible_query_panels = [panel for panel in panels if panel["type"] != "row"]
|
||||
nested_panels_by_title = {
|
||||
child["title"]: child for row in rows for child in row.get("panels", [])
|
||||
}
|
||||
|
||||
assert len(panels) == 18
|
||||
assert len(visible_query_panels) == 12
|
||||
assert sum(len(panel.get("targets", [])) for panel in visible_query_panels) == 12
|
||||
assert all(
|
||||
panel["title"] != "Coverage Gap to 95% by Suite"
|
||||
for panel in visible_query_panels
|
||||
)
|
||||
assert [row["title"] for row in rows] == [
|
||||
"CI Runs And Test Result History",
|
||||
"Check Failure Rates By Suite",
|
||||
"Check Healthy Rates By Suite",
|
||||
"Test Drilldowns And Problem Tests",
|
||||
"Telemetry Completeness And Branches",
|
||||
"SonarQube Project Health",
|
||||
]
|
||||
assert all(row["collapsed"] for row in rows)
|
||||
|
||||
assert "Coverage Failure Rate" in nested_panels_by_title
|
||||
assert "Supply Chain Healthy Rate" in nested_panels_by_title
|
||||
assert "Test Category Health History" in nested_panels_by_title
|
||||
assert "Selected Test Pass Rate History" in nested_panels_by_title
|
||||
assert "Coverage Metrics Present by Suite" in nested_panels_by_title
|
||||
assert "SonarQube API Up" in nested_panels_by_title
|
||||
|
||||
failure_rate_panel = nested_panels_by_title["Coverage Failure Rate"]
|
||||
assert failure_rate_panel["type"] == "state-timeline"
|
||||
assert failure_rate_panel["fieldConfig"]["defaults"]["unit"] == "percent"
|
||||
assert failure_rate_panel["fieldConfig"]["defaults"]["max"] == 100
|
||||
assert (
|
||||
failure_rate_panel["fieldConfig"]["defaults"]["thresholds"]["steps"][0]["color"]
|
||||
== "dark-blue"
|
||||
)
|
||||
assert "increase(" not in failure_rate_panel["targets"][0]["expr"]
|
||||
assert (
|
||||
"platform_quality:check_failed_flag:present_1h"
|
||||
in failure_rate_panel["targets"][0]["expr"]
|
||||
)
|
||||
assert (
|
||||
"platform_quality:check_seen_flag:present_1h"
|
||||
in failure_rate_panel["targets"][0]["expr"]
|
||||
)
|
||||
assert (
|
||||
"platform_quality:check_status:present_1h"
|
||||
not in failure_rate_panel["targets"][0]["expr"]
|
||||
)
|
||||
assert (
|
||||
".*_quality_gate_checks_total" not in failure_rate_panel["targets"][0]["expr"]
|
||||
)
|
||||
assert "last_over_time" not in failure_rate_panel["targets"][0]["expr"]
|
||||
assert "label_replace" not in failure_rate_panel["targets"][0]["expr"]
|
||||
assert "0 *" in failure_rate_panel["targets"][0]["expr"]
|
||||
assert "and on(suite)" not in failure_rate_panel["targets"][0]["expr"]
|
||||
|
||||
pass_rate_panel = nested_panels_by_title["Selected Test Pass Rate History"]
|
||||
assert pass_rate_panel["type"] == "state-timeline"
|
||||
assert (
|
||||
"platform_quality:test_case_pass_rate:percent_1h"
|
||||
in pass_rate_panel["targets"][0]["expr"]
|
||||
)
|
||||
assert (
|
||||
"platform_quality_gate_test_case_result"
|
||||
not in pass_rate_panel["targets"][0]["expr"]
|
||||
)
|
||||
|
||||
pass_fail_panel = nested_panels_by_title["Selected Test Pass/Fail History"]
|
||||
assert pass_fail_panel["fieldConfig"]["defaults"]["custom"]["drawStyle"] == "bars"
|
||||
assert all(
|
||||
"platform_quality:test_case_status:count_1h" in target["expr"]
|
||||
for target in pass_fail_panel["targets"]
|
||||
)
|
||||
|
||||
problematic_panel = nested_panels_by_title[
|
||||
"Problematic Tests Over Time (Top failures)"
|
||||
]
|
||||
assert problematic_panel["type"] == "state-timeline"
|
||||
assert problematic_panel["gridPos"]["w"] == 24
|
||||
assert 'test!=""' in problematic_panel["targets"][0]["expr"]
|
||||
assert "vector(0)" not in problematic_panel["targets"][0]["expr"]
|
||||
assert problematic_panel["fieldConfig"]["defaults"]["thresholds"]["steps"] == [
|
||||
{"color": "dark-blue", "value": None},
|
||||
{"color": "dark-green", "value": 2},
|
||||
{"color": "dark-yellow", "value": 3},
|
||||
{"color": "dark-orange", "value": 5},
|
||||
{"color": "dark-red", "value": 8},
|
||||
]
|
||||
assert "rolling 24h failure count" in problematic_panel["description"]
|
||||
assert "at least two recent failures" in problematic_panel["description"]
|
||||
|
||||
sonar_mix_panel = nested_panels_by_title["Sonar Gate Status Mix (Selected)"]
|
||||
sonar_health_panel = nested_panels_by_title["Sonar Gate Health by Project"]
|
||||
assert sonar_mix_panel["gridPos"]["w"] == 4
|
||||
assert sonar_health_panel["gridPos"]["w"] == 8
|
||||
assert sonar_health_panel["type"] == "state-timeline"
|
||||
assert (
|
||||
"platform_quality:sonar_gate_health_percent:latest_1h"
|
||||
in sonar_health_panel["targets"][0]["expr"]
|
||||
)
|
||||
assert (
|
||||
"sonarqube_project_quality_gate_pass"
|
||||
not in sonar_health_panel["targets"][0]["expr"]
|
||||
)
|
||||
|
||||
branch_panel = nested_panels_by_title["Primary Branch Clean by Suite (7d)"]
|
||||
recent_branch_panel = nested_panels_by_title["Recent Branch Evidence by Suite (7d)"]
|
||||
assert branch_panel["gridPos"]["x"] == 12
|
||||
assert recent_branch_panel["gridPos"]["x"] == 18
|
||||
assert "[7d:1h]" in recent_branch_panel["targets"][0]["expr"]
|
||||
assert "[7d:1h]" in branch_panel["targets"][0]["expr"]
|
||||
assert branch_panel["fieldConfig"]["defaults"]["unit"] == "percent"
|
||||
assert "unless on(suite)" in branch_panel["targets"][0]["expr"]
|
||||
assert "> bool 0" in branch_panel["targets"][0]["expr"]
|
||||
assert branch_panel["targets"][0]["expr"].startswith("sort(")
|
||||
@ -1,241 +1,11 @@
|
||||
import importlib.util
|
||||
import pathlib
|
||||
"""Atlas testing and jobs dashboard rendering contracts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
def load_module():
|
||||
path = pathlib.Path(__file__).resolve().parents[1] / "render" / "dashboards_render_atlas.py"
|
||||
spec = importlib.util.spec_from_file_location("scripts.dashboards_render_atlas", path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
assert spec.loader is not None
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def flatten_panels(panels):
|
||||
flat = []
|
||||
for panel in panels:
|
||||
flat.append(panel)
|
||||
flat.extend(panel.get("panels", []))
|
||||
return flat
|
||||
|
||||
|
||||
def test_table_panel_options_and_filterable():
|
||||
mod = load_module()
|
||||
panel = mod.table_panel(
|
||||
1,
|
||||
"test",
|
||||
"metric",
|
||||
{"h": 1, "w": 1, "x": 0, "y": 0},
|
||||
unit="percent",
|
||||
transformations=[{"id": "labelsToFields", "options": {}}],
|
||||
instant=True,
|
||||
options={"showColumnFilters": False},
|
||||
filterable=False,
|
||||
footer={"show": False, "fields": "", "calcs": []},
|
||||
format="table",
|
||||
)
|
||||
assert panel["fieldConfig"]["defaults"]["unit"] == "percent"
|
||||
assert panel["fieldConfig"]["defaults"]["custom"]["filterable"] is False
|
||||
assert panel["options"]["showHeader"] is True
|
||||
assert panel["targets"][0]["format"] == "table"
|
||||
|
||||
|
||||
def test_node_filter_and_expr_helpers():
|
||||
mod = load_module()
|
||||
expr = mod.node_filter("titan-.*")
|
||||
assert "label_replace" in expr
|
||||
cpu_expr = mod.node_cpu_expr("titan-.*")
|
||||
mem_expr = mod.node_mem_expr("titan-.*")
|
||||
assert "node_cpu_seconds_total" in cpu_expr
|
||||
assert "clamp_max(clamp_min(" in cpu_expr
|
||||
assert "* 100, 0), 100)" in cpu_expr
|
||||
assert "node_memory_MemAvailable_bytes" in mem_expr
|
||||
|
||||
|
||||
def test_overview_availability_panel_uses_recorded_365d_rollup():
|
||||
mod = load_module()
|
||||
dashboard = mod.build_overview()
|
||||
panel = next(panel for panel in flatten_panels(dashboard["panels"]) if panel["id"] == 27)
|
||||
|
||||
assert panel["title"] == "Atlas Availability (365d)"
|
||||
availability_expr = panel["targets"][0]["expr"]
|
||||
assert (
|
||||
'last_over_time(atlas:availability:ratio_365d{scope="atlas",definition="request-v4"}[48h])'
|
||||
in availability_expr
|
||||
)
|
||||
assert 'code=~"5.."' in availability_expr
|
||||
assert 'code=~"[1-5].."' in availability_expr
|
||||
assert "atlas:availability:failures_1d" not in availability_expr
|
||||
assert "atlas:availability:requests_1d" not in availability_expr
|
||||
assert "sum_over_time" not in availability_expr
|
||||
assert "kube_node_status_condition" not in availability_expr
|
||||
assert "kube_deployment_status_replicas_available" not in availability_expr
|
||||
assert panel["targets"][0]["instant"] is True
|
||||
assert "Every server-side 5xx" in panel["description"]
|
||||
assert "Replica counts, Grafana health" in panel["description"]
|
||||
assert "daily rollup job publishes one annual sample" in panel["description"]
|
||||
assert "keeps it for up to 48 hours" in panel["description"]
|
||||
|
||||
|
||||
def test_overview_uses_readable_quality_power_and_gitops_panels():
|
||||
mod = load_module()
|
||||
dashboard = mod.build_overview()
|
||||
panels_by_title = {panel["title"]: panel for panel in flatten_panels(dashboard["panels"])}
|
||||
|
||||
assert dashboard["links"] == [
|
||||
{"title": "Atlas Testing", "url": "/d/atlas-testing", "targetBlank": True}
|
||||
]
|
||||
assert "atlas-jobs" not in repr(dashboard)
|
||||
assert "Platform Test Success Rate" not in panels_by_title
|
||||
assert panels_by_title["Test Category Health"]["type"] == "state-timeline"
|
||||
assert panels_by_title["Test Category Health"]["gridPos"] == {"h": 6, "w": 6, "x": 15, "y": 13}
|
||||
assert panels_by_title["Test Category Health"]["targets"][0]["legendFormat"] == "{{category}}"
|
||||
assert panels_by_title["Test Category Health"]["targets"][0]["format"] == "time_series"
|
||||
assert panels_by_title["Test Category Health"]["targets"][0]["range"] is True
|
||||
assert "${overview_suite:regex}" not in panels_by_title["Test Category Health"]["targets"][0]["expr"]
|
||||
assert mod.PLATFORM_TEST_SUITE_CANONICAL_MATCHER in panels_by_title["Test Category Health"]["targets"][0]["expr"]
|
||||
assert "platform_quality:test_category_health_rate:percent_1h" in panels_by_title["Test Category Health"]["targets"][0]["expr"]
|
||||
assert panels_by_title["Test Category Health"]["timeFrom"] == "24h"
|
||||
assert f'category=~"{mod.PLATFORM_TEST_OVERVIEW_CATEGORY_REGEX}"' in panels_by_title["Test Category Health"]["targets"][0]["expr"]
|
||||
assert "manual" not in mod.PLATFORM_TEST_OVERVIEW_CATEGORY_REGEX
|
||||
assert "unit" not in mod.PLATFORM_TEST_OVERVIEW_CATEGORY_REGEX
|
||||
assert panels_by_title["UPS History (Power Draw)"]["gridPos"] == {"h": 6, "w": 6, "x": 3, "y": 7}
|
||||
assert panels_by_title["Ariadne Run Volume"]["gridPos"] == {"h": 6, "w": 6, "x": 9, "y": 7}
|
||||
assert panels_by_title["Pyrphoros UPS Current"]["gridPos"]["w"] == 3
|
||||
assert panels_by_title["Current Enclosure Climate"]["gridPos"]["w"] == 3
|
||||
assert panels_by_title["UPS History (Power Draw)"]["options"]["legend"]["placement"] == "bottom"
|
||||
assert panels_by_title["UPS History (Power Draw)"]["options"]["legend"]["displayMode"] == "list"
|
||||
assert panels_by_title["UPS History (Power Draw)"]["fieldConfig"]["defaults"]["custom"]["drawStyle"] == "line"
|
||||
assert panels_by_title["UPS History (Power Draw)"]["fieldConfig"]["defaults"]["custom"]["fillOpacity"] == 22
|
||||
assert all(target["expr"].startswith("max(") for target in panels_by_title["UPS History (Power Draw)"]["targets"])
|
||||
ups_overrides = panels_by_title["UPS History (Power Draw)"]["fieldConfig"]["overrides"]
|
||||
ups_override_by_name = {override["matcher"]["options"]: override for override in ups_overrides}
|
||||
assert ups_override_by_name["Pyrphoros"]["properties"] == [
|
||||
{"id": "color", "value": {"mode": "fixed", "fixedColor": "dark-blue"}},
|
||||
]
|
||||
assert ups_override_by_name["Statera"]["properties"] == [
|
||||
{"id": "color", "value": {"mode": "fixed", "fixedColor": "dark-yellow"}},
|
||||
]
|
||||
assert panels_by_title["Ariadne Run Volume"]["fieldConfig"]["defaults"]["custom"]["drawStyle"] == "bars"
|
||||
assert panels_by_title["Ariadne Run Volume"]["options"]["legend"]["placement"] == "bottom"
|
||||
assert panels_by_title["Ariadne Run Volume"]["options"]["legend"]["displayMode"] == "list"
|
||||
assert "Fan History (0-10)" not in panels_by_title
|
||||
assert panels_by_title["Fan Intensity History"]["type"] == "state-timeline"
|
||||
assert panels_by_title["Fan Intensity History"]["gridPos"] == {"h": 6, "w": 6, "x": 9, "y": 13}
|
||||
assert panels_by_title["Fan Intensity History"]["fieldConfig"]["defaults"]["max"] == 10
|
||||
assert panels_by_title["Fan Intensity History"]["targets"][0]["legendFormat"] == "{{fan}}"
|
||||
fan_steps = panels_by_title["Fan Intensity History"]["fieldConfig"]["defaults"]["thresholds"]["steps"]
|
||||
assert [step["value"] for step in fan_steps] == [None, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
|
||||
assert fan_steps[0]["color"] == "#8f1d1d"
|
||||
assert fan_steps[5]["color"] == "#d4b106"
|
||||
assert fan_steps[-1]["color"] == "#1f60c4"
|
||||
fan_mappings = panels_by_title["Fan Intensity History"]["fieldConfig"]["defaults"]["mappings"][0]["options"]
|
||||
assert fan_mappings["0"]["text"] == "Off"
|
||||
assert fan_mappings["10"]["text"] == "10"
|
||||
assert "Tent Interior" in panels_by_title["Fan Intensity History"]["targets"][0]["expr"]
|
||||
assert panels_by_title["Fan Intensity History"]["options"]["legend"]["displayMode"] == "list"
|
||||
assert panels_by_title["Fan Intensity History"]["options"]["legend"]["showLegend"] is False
|
||||
assert panels_by_title["Fan Intensity History"]["options"]["mergeValues"] is False
|
||||
assert panels_by_title["Fan Intensity History"]["options"]["showValue"] == "auto"
|
||||
|
||||
assert panels_by_title["Flux Source"]["type"] == "stat"
|
||||
assert panels_by_title["Flux Source"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 7}
|
||||
assert panels_by_title["Flux Source"]["targets"][0]["legendFormat"] == "{{branch}}"
|
||||
assert panels_by_title["Current Gate Health"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 9}
|
||||
assert "platform_quality:test_category_health_rate:percent_1h" in panels_by_title["Current Gate Health"]["targets"][0]["expr"]
|
||||
assert panels_by_title["CI Run Success (24h)"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 11}
|
||||
assert panels_by_title["Suites With Runs (24h)"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 15}
|
||||
suites_reporting_expr = panels_by_title["Suites With Runs (24h)"]["targets"][0]["expr"]
|
||||
assert "> bool 0" in suites_reporting_expr
|
||||
assert mod.PLATFORM_TEST_SUITE_CANONICAL_MATCHER in suites_reporting_expr
|
||||
assert "bstein-home" not in suites_reporting_expr
|
||||
assert "published quality-gate run" in panels_by_title["Suites With Runs (24h)"]["description"]
|
||||
assert panels_by_title["Avg Coverage"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 17}
|
||||
assert "LOC Clean Suites" not in panels_by_title
|
||||
assert panels_by_title["GitOps Health"]["type"] == "state-timeline"
|
||||
assert panels_by_title["GitOps Health"]["gridPos"] == {"h": 6, "w": 6, "x": 15, "y": 7}
|
||||
gitops_expr = panels_by_title["GitOps Health"]["targets"][0]["expr"]
|
||||
assert "Kustomizations Not Suspended" in gitops_expr
|
||||
assert "HelmReleases Not Suspended" in gitops_expr
|
||||
assert panels_by_title["Test Category Health"]["type"] == "state-timeline"
|
||||
assert panels_by_title["Test Category Health"]["options"]["legend"]["showLegend"] is False
|
||||
assert panels_by_title["Test Category Health"]["options"]["mergeValues"] is False
|
||||
assert panels_by_title["Test Category Health"]["options"]["showValue"] == "auto"
|
||||
assert panels_by_title["Test Category Health"]["options"]["rowHeight"] == 0.9
|
||||
assert panels_by_title["Test Category Health"]["targets"][0]["legendFormat"] == "{{category}}"
|
||||
assert 'label_set(vector(0), "category", "none")' in panels_by_title["Test Category Health"]["targets"][0]["expr"]
|
||||
assert not any(variable["name"] == "overview_suite" for variable in dashboard["templating"]["list"])
|
||||
|
||||
pvc_backup_expr = panels_by_title["PVC Backup Health / Age"]["targets"][0]["expr"]
|
||||
assert "backup-telemetry-missing" in pvc_backup_expr
|
||||
assert 'pvc_backup_(count|last_success_timestamp_seconds|health_reason)' in pvc_backup_expr
|
||||
|
||||
gpu_expr = panels_by_title["Namespace GPU Utilization"]["targets"][0]["expr"]
|
||||
assert "nvidia_namespace_gpu_sm_util_percent" in gpu_expr
|
||||
assert "nvidia_gpu_device_utilization_percent" in gpu_expr
|
||||
assert "jetson_gr3d_active_seconds_total" in gpu_expr
|
||||
assert "increase" in gpu_expr
|
||||
assert "sum_over_time" not in gpu_expr
|
||||
assert "count_over_time" not in gpu_expr
|
||||
assert "avg_over_time" in gpu_expr
|
||||
assert "$__range" in gpu_expr
|
||||
assert "$__range_s" in gpu_expr
|
||||
assert "sum by (namespace)" in gpu_expr
|
||||
assert 'namespace", "shared"' not in gpu_expr
|
||||
assert "kube_pod_container_resource_requests" in gpu_expr
|
||||
assert mod.GPU_RESOURCE_REGEX in gpu_expr
|
||||
assert '"gpu_source", "nvidia"' in gpu_expr
|
||||
assert '"gpu_source", "jetson"' in gpu_expr
|
||||
assert "100 *" in gpu_expr
|
||||
assert "100 -" not in gpu_expr
|
||||
assert 'namespace", "unattributed"' in gpu_expr
|
||||
assert 'namespace", "idle"' in gpu_expr
|
||||
assert panels_by_title["Namespace GPU Utilization"]["targets"][0]["instant"] is True
|
||||
assert "selected time range" in panels_by_title["Namespace GPU Utilization"]["description"]
|
||||
assert "continuously sampled" in panels_by_title["Namespace GPU Utilization"]["description"]
|
||||
assert "Kubernetes shared-GPU allocations" in panels_by_title["Namespace GPU Utilization"]["description"]
|
||||
|
||||
|
||||
def test_gpu_node_panel_prefers_stable_process_metrics_and_covers_all_gpu_families():
|
||||
mod = load_module()
|
||||
dashboard = mod.build_gpu_dashboard()
|
||||
panels_by_title = {panel["title"]: panel for panel in flatten_panels(dashboard["panels"])}
|
||||
|
||||
node_expr = panels_by_title["GPU Util by Node"]["targets"][0]["expr"]
|
||||
assert "nvidia_gpu_device_utilization_percent" in node_expr
|
||||
assert "DCGM_FI_DEV_GPU_UTIL" in node_expr
|
||||
assert "jetson_gr3d_freq_percent" in node_expr
|
||||
assert "unless on(node)" in node_expr
|
||||
assert mod.GPU_NODES == ["titan-20", "titan-21", "titan-22", "titan-24"]
|
||||
|
||||
process_expr = panels_by_title["GPU Processes by Pod"]["targets"][0]["expr"]
|
||||
assert "nvidia_process_gpu_sm_util_percent" in process_expr
|
||||
assert "DCGM_FI_DEV_GPU_UTIL" not in process_expr
|
||||
|
||||
|
||||
def test_overview_and_testing_panels_all_have_concise_descriptions():
|
||||
mod = load_module()
|
||||
|
||||
for dashboard in [mod.build_overview(), mod.build_jobs_dashboard(), mod.build_testing_dashboard()]:
|
||||
panels = flatten_panels(dashboard["panels"])
|
||||
assert all(panel.get("description") for panel in panels if panel["type"] != "row")
|
||||
assert "85.7" not in repr(dashboard)
|
||||
|
||||
|
||||
def test_render_configmap_writes(tmp_path):
|
||||
mod = load_module()
|
||||
mod.DASHBOARD_DIR = tmp_path / "dash"
|
||||
mod.ROOT = tmp_path
|
||||
uid = "atlas-test"
|
||||
info = {"configmap": tmp_path / "cm.yaml"}
|
||||
data = {"title": "Atlas Test"}
|
||||
mod.write_json(uid, data)
|
||||
mod.render_configmap(uid, info)
|
||||
json_path = mod.DASHBOARD_DIR / f"{uid}.json"
|
||||
assert json_path.exists()
|
||||
content = (tmp_path / "cm.yaml").read_text()
|
||||
assert "kind: ConfigMap" in content
|
||||
assert f"{uid}.json" in content
|
||||
from test_dashboard_render_support import (
|
||||
flatten_panels,
|
||||
load_module,
|
||||
)
|
||||
|
||||
|
||||
def test_testing_suite_variable_uses_configured_suite_values_only():
|
||||
@ -257,7 +27,6 @@ def test_testing_suite_variable_uses_configured_suite_values_only():
|
||||
option["value"] in out_of_scope_names for option in variable["options"]
|
||||
)
|
||||
|
||||
|
||||
def test_testing_dashboard_is_public_but_jobs_dashboard_remains_internal():
|
||||
mod = load_module()
|
||||
jobs = mod.build_jobs_dashboard()
|
||||
@ -269,7 +38,6 @@ def test_testing_dashboard_is_public_but_jobs_dashboard_remains_internal():
|
||||
assert testing["folderUid"] == mod.PUBLIC_DASHBOARD_FOLDER
|
||||
assert testing["editable"] is False
|
||||
|
||||
|
||||
def test_jobs_dashboard_separates_current_gate_health_from_reliability():
|
||||
mod = load_module()
|
||||
dashboard = mod.build_jobs_dashboard()
|
||||
@ -395,7 +163,6 @@ def test_jobs_dashboard_separates_current_gate_health_from_reliability():
|
||||
assert run_volume_panel["fieldConfig"]["defaults"]["custom"]["drawStyle"] == "bars"
|
||||
assert "[$__interval]" not in run_volume_panel["targets"][0]["expr"]
|
||||
|
||||
|
||||
def test_jobs_dashboard_bar_gauges_use_solid_threshold_colors():
|
||||
mod = load_module()
|
||||
dashboard = mod.build_jobs_dashboard()
|
||||
@ -416,8 +183,6 @@ def test_jobs_dashboard_bar_gauges_use_solid_threshold_colors():
|
||||
|
||||
assert {"color": "dark-yellow", "value": 93} in threshold_steps
|
||||
assert {"color": "dark-blue", "value": 100} in threshold_steps
|
||||
|
||||
|
||||
def test_jobs_dashboard_collapses_heavy_drilldowns_for_light_first_paint():
|
||||
mod = load_module()
|
||||
dashboard = mod.build_jobs_dashboard()
|
||||
221
scripts/tests/test_dashboards_render_overview.py
Normal file
221
scripts/tests/test_dashboards_render_overview.py
Normal file
@ -0,0 +1,221 @@
|
||||
"""Atlas overview dashboard rendering contracts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from test_dashboard_render_support import (
|
||||
flatten_panels,
|
||||
load_module,
|
||||
)
|
||||
|
||||
|
||||
def test_table_panel_options_and_filterable():
|
||||
mod = load_module()
|
||||
panel = mod.table_panel(
|
||||
1,
|
||||
"test",
|
||||
"metric",
|
||||
{"h": 1, "w": 1, "x": 0, "y": 0},
|
||||
unit="percent",
|
||||
transformations=[{"id": "labelsToFields", "options": {}}],
|
||||
instant=True,
|
||||
options={"showColumnFilters": False},
|
||||
filterable=False,
|
||||
footer={"show": False, "fields": "", "calcs": []},
|
||||
format="table",
|
||||
)
|
||||
assert panel["fieldConfig"]["defaults"]["unit"] == "percent"
|
||||
assert panel["fieldConfig"]["defaults"]["custom"]["filterable"] is False
|
||||
assert panel["options"]["showHeader"] is True
|
||||
assert panel["targets"][0]["format"] == "table"
|
||||
|
||||
def test_node_filter_and_expr_helpers():
|
||||
mod = load_module()
|
||||
expr = mod.node_filter("titan-.*")
|
||||
assert "label_replace" in expr
|
||||
cpu_expr = mod.node_cpu_expr("titan-.*")
|
||||
mem_expr = mod.node_mem_expr("titan-.*")
|
||||
assert "node_cpu_seconds_total" in cpu_expr
|
||||
assert "clamp_max(clamp_min(" in cpu_expr
|
||||
assert "* 100, 0), 100)" in cpu_expr
|
||||
assert "node_memory_MemAvailable_bytes" in mem_expr
|
||||
|
||||
def test_overview_availability_panel_uses_recorded_365d_rollup():
|
||||
mod = load_module()
|
||||
dashboard = mod.build_overview()
|
||||
panel = next(panel for panel in flatten_panels(dashboard["panels"]) if panel["id"] == 27)
|
||||
|
||||
assert panel["title"] == "Atlas Availability (365d)"
|
||||
availability_expr = panel["targets"][0]["expr"]
|
||||
assert (
|
||||
'last_over_time(atlas:availability:ratio_365d{scope="atlas",definition="request-v4"}[48h])'
|
||||
in availability_expr
|
||||
)
|
||||
assert 'code=~"5.."' in availability_expr
|
||||
assert 'code=~"[1-5].."' in availability_expr
|
||||
assert "atlas:availability:failures_1d" not in availability_expr
|
||||
assert "atlas:availability:requests_1d" not in availability_expr
|
||||
assert "sum_over_time" not in availability_expr
|
||||
assert "kube_node_status_condition" not in availability_expr
|
||||
assert "kube_deployment_status_replicas_available" not in availability_expr
|
||||
assert panel["targets"][0]["instant"] is True
|
||||
assert "Every server-side 5xx" in panel["description"]
|
||||
assert "Replica counts, Grafana health" in panel["description"]
|
||||
assert "daily rollup job publishes one annual sample" in panel["description"]
|
||||
assert "keeps it for up to 48 hours" in panel["description"]
|
||||
|
||||
def test_overview_uses_readable_quality_power_and_gitops_panels():
|
||||
mod = load_module()
|
||||
dashboard = mod.build_overview()
|
||||
panels_by_title = {panel["title"]: panel for panel in flatten_panels(dashboard["panels"])}
|
||||
|
||||
assert dashboard["links"] == [
|
||||
{"title": "Atlas Testing", "url": "/d/atlas-testing", "targetBlank": True}
|
||||
]
|
||||
assert "atlas-jobs" not in repr(dashboard)
|
||||
assert "Platform Test Success Rate" not in panels_by_title
|
||||
assert panels_by_title["Test Category Health"]["type"] == "state-timeline"
|
||||
assert panels_by_title["Test Category Health"]["gridPos"] == {"h": 6, "w": 6, "x": 15, "y": 13}
|
||||
assert panels_by_title["Test Category Health"]["targets"][0]["legendFormat"] == "{{category}}"
|
||||
assert panels_by_title["Test Category Health"]["targets"][0]["format"] == "time_series"
|
||||
assert panels_by_title["Test Category Health"]["targets"][0]["range"] is True
|
||||
assert "${overview_suite:regex}" not in panels_by_title["Test Category Health"]["targets"][0]["expr"]
|
||||
assert mod.PLATFORM_TEST_SUITE_CANONICAL_MATCHER in panels_by_title["Test Category Health"]["targets"][0]["expr"]
|
||||
assert "platform_quality:test_category_health_rate:percent_1h" in panels_by_title["Test Category Health"]["targets"][0]["expr"]
|
||||
assert panels_by_title["Test Category Health"]["timeFrom"] == "24h"
|
||||
assert f'category=~"{mod.PLATFORM_TEST_OVERVIEW_CATEGORY_REGEX}"' in panels_by_title["Test Category Health"]["targets"][0]["expr"]
|
||||
assert "manual" not in mod.PLATFORM_TEST_OVERVIEW_CATEGORY_REGEX
|
||||
assert "unit" not in mod.PLATFORM_TEST_OVERVIEW_CATEGORY_REGEX
|
||||
assert panels_by_title["UPS History (Power Draw)"]["gridPos"] == {"h": 6, "w": 6, "x": 3, "y": 7}
|
||||
assert panels_by_title["Ariadne Run Volume"]["gridPos"] == {"h": 6, "w": 6, "x": 9, "y": 7}
|
||||
assert panels_by_title["Pyrphoros UPS Current"]["gridPos"]["w"] == 3
|
||||
assert panels_by_title["Current Enclosure Climate"]["gridPos"]["w"] == 3
|
||||
assert panels_by_title["UPS History (Power Draw)"]["options"]["legend"]["placement"] == "bottom"
|
||||
assert panels_by_title["UPS History (Power Draw)"]["options"]["legend"]["displayMode"] == "list"
|
||||
assert panels_by_title["UPS History (Power Draw)"]["fieldConfig"]["defaults"]["custom"]["drawStyle"] == "line"
|
||||
assert panels_by_title["UPS History (Power Draw)"]["fieldConfig"]["defaults"]["custom"]["fillOpacity"] == 22
|
||||
assert all(target["expr"].startswith("max(") for target in panels_by_title["UPS History (Power Draw)"]["targets"])
|
||||
ups_overrides = panels_by_title["UPS History (Power Draw)"]["fieldConfig"]["overrides"]
|
||||
ups_override_by_name = {override["matcher"]["options"]: override for override in ups_overrides}
|
||||
assert ups_override_by_name["Pyrphoros"]["properties"] == [
|
||||
{"id": "color", "value": {"mode": "fixed", "fixedColor": "dark-blue"}},
|
||||
]
|
||||
assert ups_override_by_name["Statera"]["properties"] == [
|
||||
{"id": "color", "value": {"mode": "fixed", "fixedColor": "dark-yellow"}},
|
||||
]
|
||||
assert panels_by_title["Ariadne Run Volume"]["fieldConfig"]["defaults"]["custom"]["drawStyle"] == "bars"
|
||||
assert panels_by_title["Ariadne Run Volume"]["options"]["legend"]["placement"] == "bottom"
|
||||
assert panels_by_title["Ariadne Run Volume"]["options"]["legend"]["displayMode"] == "list"
|
||||
assert "Fan History (0-10)" not in panels_by_title
|
||||
assert panels_by_title["Fan Intensity History"]["type"] == "state-timeline"
|
||||
assert panels_by_title["Fan Intensity History"]["gridPos"] == {"h": 6, "w": 6, "x": 9, "y": 13}
|
||||
assert panels_by_title["Fan Intensity History"]["fieldConfig"]["defaults"]["max"] == 10
|
||||
assert panels_by_title["Fan Intensity History"]["targets"][0]["legendFormat"] == "{{fan}}"
|
||||
fan_steps = panels_by_title["Fan Intensity History"]["fieldConfig"]["defaults"]["thresholds"]["steps"]
|
||||
assert [step["value"] for step in fan_steps] == [None, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
|
||||
assert fan_steps[0]["color"] == "#8f1d1d"
|
||||
assert fan_steps[5]["color"] == "#d4b106"
|
||||
assert fan_steps[-1]["color"] == "#1f60c4"
|
||||
fan_mappings = panels_by_title["Fan Intensity History"]["fieldConfig"]["defaults"]["mappings"][0]["options"]
|
||||
assert fan_mappings["0"]["text"] == "Off"
|
||||
assert fan_mappings["10"]["text"] == "10"
|
||||
assert "Tent Interior" in panels_by_title["Fan Intensity History"]["targets"][0]["expr"]
|
||||
assert panels_by_title["Fan Intensity History"]["options"]["legend"]["displayMode"] == "list"
|
||||
assert panels_by_title["Fan Intensity History"]["options"]["legend"]["showLegend"] is False
|
||||
assert panels_by_title["Fan Intensity History"]["options"]["mergeValues"] is False
|
||||
assert panels_by_title["Fan Intensity History"]["options"]["showValue"] == "auto"
|
||||
|
||||
assert panels_by_title["Flux Source"]["type"] == "stat"
|
||||
assert panels_by_title["Flux Source"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 7}
|
||||
assert panels_by_title["Flux Source"]["targets"][0]["legendFormat"] == "{{branch}}"
|
||||
assert panels_by_title["Current Gate Health"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 9}
|
||||
assert "platform_quality:test_category_health_rate:percent_1h" in panels_by_title["Current Gate Health"]["targets"][0]["expr"]
|
||||
assert panels_by_title["CI Run Success (24h)"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 11}
|
||||
assert panels_by_title["Suites With Runs (24h)"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 15}
|
||||
suites_reporting_expr = panels_by_title["Suites With Runs (24h)"]["targets"][0]["expr"]
|
||||
assert "> bool 0" in suites_reporting_expr
|
||||
assert mod.PLATFORM_TEST_SUITE_CANONICAL_MATCHER in suites_reporting_expr
|
||||
assert "bstein-home" not in suites_reporting_expr
|
||||
assert "published quality-gate run" in panels_by_title["Suites With Runs (24h)"]["description"]
|
||||
assert panels_by_title["Avg Coverage"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 17}
|
||||
assert "LOC Clean Suites" not in panels_by_title
|
||||
assert panels_by_title["GitOps Health"]["type"] == "state-timeline"
|
||||
assert panels_by_title["GitOps Health"]["gridPos"] == {"h": 6, "w": 6, "x": 15, "y": 7}
|
||||
gitops_expr = panels_by_title["GitOps Health"]["targets"][0]["expr"]
|
||||
assert "Kustomizations Not Suspended" in gitops_expr
|
||||
assert "HelmReleases Not Suspended" in gitops_expr
|
||||
assert panels_by_title["Test Category Health"]["type"] == "state-timeline"
|
||||
assert panels_by_title["Test Category Health"]["options"]["legend"]["showLegend"] is False
|
||||
assert panels_by_title["Test Category Health"]["options"]["mergeValues"] is False
|
||||
assert panels_by_title["Test Category Health"]["options"]["showValue"] == "auto"
|
||||
assert panels_by_title["Test Category Health"]["options"]["rowHeight"] == 0.9
|
||||
assert panels_by_title["Test Category Health"]["targets"][0]["legendFormat"] == "{{category}}"
|
||||
assert 'label_set(vector(0), "category", "none")' in panels_by_title["Test Category Health"]["targets"][0]["expr"]
|
||||
assert not any(variable["name"] == "overview_suite" for variable in dashboard["templating"]["list"])
|
||||
|
||||
pvc_backup_expr = panels_by_title["PVC Backup Health / Age"]["targets"][0]["expr"]
|
||||
assert "backup-telemetry-missing" in pvc_backup_expr
|
||||
assert 'pvc_backup_(count|last_success_timestamp_seconds|health_reason)' in pvc_backup_expr
|
||||
|
||||
gpu_expr = panels_by_title["Namespace GPU Utilization"]["targets"][0]["expr"]
|
||||
assert "nvidia_namespace_gpu_sm_util_percent" in gpu_expr
|
||||
assert "nvidia_gpu_device_utilization_percent" in gpu_expr
|
||||
assert "jetson_gr3d_active_seconds_total" in gpu_expr
|
||||
assert "increase" in gpu_expr
|
||||
assert "sum_over_time" not in gpu_expr
|
||||
assert "count_over_time" not in gpu_expr
|
||||
assert "avg_over_time" in gpu_expr
|
||||
assert "$__range" in gpu_expr
|
||||
assert "$__range_s" in gpu_expr
|
||||
assert "sum by (namespace)" in gpu_expr
|
||||
assert 'namespace", "shared"' not in gpu_expr
|
||||
assert "kube_pod_container_resource_requests" in gpu_expr
|
||||
assert mod.GPU_RESOURCE_REGEX in gpu_expr
|
||||
assert '"gpu_source", "nvidia"' in gpu_expr
|
||||
assert '"gpu_source", "jetson"' in gpu_expr
|
||||
assert "100 *" in gpu_expr
|
||||
assert "100 -" not in gpu_expr
|
||||
assert 'namespace", "unattributed"' in gpu_expr
|
||||
assert 'namespace", "idle"' in gpu_expr
|
||||
assert panels_by_title["Namespace GPU Utilization"]["targets"][0]["instant"] is True
|
||||
assert "selected time range" in panels_by_title["Namespace GPU Utilization"]["description"]
|
||||
assert "continuously sampled" in panels_by_title["Namespace GPU Utilization"]["description"]
|
||||
assert "Kubernetes shared-GPU allocations" in panels_by_title["Namespace GPU Utilization"]["description"]
|
||||
|
||||
def test_gpu_node_panel_prefers_stable_process_metrics_and_covers_all_gpu_families():
|
||||
mod = load_module()
|
||||
dashboard = mod.build_gpu_dashboard()
|
||||
panels_by_title = {panel["title"]: panel for panel in flatten_panels(dashboard["panels"])}
|
||||
|
||||
node_expr = panels_by_title["GPU Util by Node"]["targets"][0]["expr"]
|
||||
assert "nvidia_gpu_device_utilization_percent" in node_expr
|
||||
assert "DCGM_FI_DEV_GPU_UTIL" in node_expr
|
||||
assert "jetson_gr3d_freq_percent" in node_expr
|
||||
assert "unless on(node)" in node_expr
|
||||
assert mod.GPU_NODES == ["titan-20", "titan-21", "titan-22", "titan-24"]
|
||||
|
||||
process_expr = panels_by_title["GPU Processes by Pod"]["targets"][0]["expr"]
|
||||
assert "nvidia_process_gpu_sm_util_percent" in process_expr
|
||||
assert "DCGM_FI_DEV_GPU_UTIL" not in process_expr
|
||||
|
||||
def test_overview_and_testing_panels_all_have_concise_descriptions():
|
||||
mod = load_module()
|
||||
|
||||
for dashboard in [mod.build_overview(), mod.build_jobs_dashboard(), mod.build_testing_dashboard()]:
|
||||
panels = flatten_panels(dashboard["panels"])
|
||||
assert all(panel.get("description") for panel in panels if panel["type"] != "row")
|
||||
assert "85.7" not in repr(dashboard)
|
||||
|
||||
def test_render_configmap_writes(tmp_path):
|
||||
mod = load_module()
|
||||
mod.DASHBOARD_DIR = tmp_path / "dash"
|
||||
mod.ROOT = tmp_path
|
||||
uid = "atlas-test"
|
||||
info = {"configmap": tmp_path / "cm.yaml"}
|
||||
data = {"title": "Atlas Test"}
|
||||
mod.write_json(uid, data)
|
||||
mod.render_configmap(uid, info)
|
||||
json_path = mod.DASHBOARD_DIR / f"{uid}.json"
|
||||
assert json_path.exists()
|
||||
content = (tmp_path / "cm.yaml").read_text()
|
||||
assert "kind: ConfigMap" in content
|
||||
assert f"{uid}.json" in content
|
||||
@ -232,6 +232,24 @@ def test_retry_db_connect_reraises_final_error(monkeypatch):
|
||||
sync.retry_db_connect(attempts=1)
|
||||
|
||||
|
||||
def test_retry_helpers_return_none_without_attempts(monkeypatch):
|
||||
sync = load_sync_module(monkeypatch)
|
||||
|
||||
assert sync.retry_request("request", lambda: "never", attempts=0) is None
|
||||
assert sync.retry_db_connect(attempts=0) is None
|
||||
|
||||
|
||||
def test_kc_update_attributes_replaces_non_dict_attributes(monkeypatch):
|
||||
sync = load_sync_module(monkeypatch)
|
||||
current_resp = _FakeResponse({"attributes": "not-a-dict"})
|
||||
ok_resp = _FakeResponse({"attributes": {"mailu_app_password": ["abc"]}})
|
||||
sync.SESSION = _FakeSession(_FakeResponse({}), [current_resp, ok_resp])
|
||||
|
||||
sync.kc_update_attributes("token", {"id": "u1", "username": "u1"}, {"mailu_app_password": "abc"})
|
||||
|
||||
assert sync.SESSION.put_called
|
||||
|
||||
|
||||
def test_ensure_mailu_user_skips_foreign_domain(monkeypatch):
|
||||
sync = load_sync_module(monkeypatch)
|
||||
executed = []
|
||||
@ -405,3 +423,62 @@ def test_main_generates_password_and_upserts(monkeypatch):
|
||||
# Only mail-enabled users (or legacy users with a mailbox) are synced and backfilled.
|
||||
assert len(updated) == 3
|
||||
assert conns and len(conns[0]._cursor.executions) == 3
|
||||
|
||||
|
||||
def test_main_skips_disabled_users_and_provisioned_users_without_update(monkeypatch):
|
||||
sync = load_sync_module(monkeypatch)
|
||||
monkeypatch.setattr(sync.bcrypt_sha256, "hash", lambda password: f"hash:{password}")
|
||||
users = [
|
||||
{
|
||||
"id": "u1",
|
||||
"username": "disabled",
|
||||
"email": "disabled@example.com",
|
||||
"enabled": False,
|
||||
"attributes": {"mailu_enabled": ["true"]},
|
||||
},
|
||||
{
|
||||
"id": "u2",
|
||||
"username": "settled",
|
||||
"email": "settled@example.com",
|
||||
"attributes": {
|
||||
"mailu_enabled": ["true"],
|
||||
"mailu_email": ["settled@example.com"],
|
||||
"mailu_app_password": ["existing"],
|
||||
},
|
||||
},
|
||||
]
|
||||
updated = []
|
||||
|
||||
class _Cursor:
|
||||
def __init__(self):
|
||||
self.executions = []
|
||||
|
||||
def execute(self, sql, params):
|
||||
self.executions.append(params)
|
||||
|
||||
def close(self):
|
||||
return None
|
||||
|
||||
class _Conn:
|
||||
def __init__(self):
|
||||
self.autocommit = False
|
||||
self._cursor = _Cursor()
|
||||
|
||||
def cursor(self, cursor_factory=None):
|
||||
return self._cursor
|
||||
|
||||
def close(self):
|
||||
return None
|
||||
|
||||
conn = _Conn()
|
||||
monkeypatch.setattr(sync, "get_kc_token", lambda: "tok")
|
||||
monkeypatch.setattr(sync, "kc_get_users", lambda token: users)
|
||||
monkeypatch.setattr(sync, "kc_update_attributes", lambda token, user, attrs: updated.append(user["id"]))
|
||||
monkeypatch.setattr(sync.psycopg2, "connect", lambda **kwargs: conn)
|
||||
|
||||
sync.main()
|
||||
|
||||
# The already-provisioned user is synced without a Keycloak write; the
|
||||
# disabled user never reaches the mailbox upsert.
|
||||
assert updated == []
|
||||
assert len(conn._cursor.executions) == 1
|
||||
|
||||
96
scripts/tests/test_mailu_sync_edges.py
Normal file
96
scripts/tests/test_mailu_sync_edges.py
Normal file
@ -0,0 +1,96 @@
|
||||
"""Retry, attribute-corruption, and skip-path edges for the Mailu sync job."""
|
||||
|
||||
from test_mailu_sync import _FakeResponse, _FakeSession, load_sync_module
|
||||
|
||||
def test_retry_helpers_with_zero_attempts_do_nothing(monkeypatch):
|
||||
sync = load_sync_module(monkeypatch)
|
||||
monkeypatch.setattr(
|
||||
sync.psycopg2,
|
||||
"connect",
|
||||
lambda **kwargs: (_ for _ in ()).throw(AssertionError("must not connect")),
|
||||
)
|
||||
|
||||
assert sync.retry_request(
|
||||
"request",
|
||||
lambda: (_ for _ in ()).throw(AssertionError("must not run")),
|
||||
attempts=0,
|
||||
) is None
|
||||
assert sync.retry_db_connect(attempts=0) is None
|
||||
|
||||
|
||||
def test_kc_update_attributes_replaces_a_corrupt_attribute_map(monkeypatch):
|
||||
sync = load_sync_module(monkeypatch)
|
||||
current_resp = _FakeResponse({"username": "u1", "attributes": "corrupt"})
|
||||
ok_resp = _FakeResponse({"attributes": {"mailu_app_password": ["abc"]}})
|
||||
sync.SESSION = _FakeSession(_FakeResponse({}), [current_resp, ok_resp])
|
||||
|
||||
sync.kc_update_attributes(
|
||||
"token",
|
||||
{"id": "u1", "username": "u1"},
|
||||
{"mailu_app_password": "abc"},
|
||||
)
|
||||
|
||||
assert sync.SESSION.put_called and sync.SESSION.get_calls == 2
|
||||
|
||||
|
||||
def test_main_skips_disabled_users_and_settled_attribute_sets(monkeypatch):
|
||||
sync = load_sync_module(monkeypatch)
|
||||
monkeypatch.setattr(sync.bcrypt_sha256, "hash", lambda password: f"hash:{password}")
|
||||
users = [
|
||||
{
|
||||
"id": "u_disabled",
|
||||
"username": "disabled",
|
||||
"email": "disabled@example.com",
|
||||
"enabled": False,
|
||||
"attributes": {"mailu_enabled": ["true"]},
|
||||
},
|
||||
{
|
||||
"id": "u_settled",
|
||||
"username": "settled",
|
||||
"email": "settled@example.com",
|
||||
"attributes": {
|
||||
"mailu_enabled": ["true"],
|
||||
"mailu_email": ["settled@example.com"],
|
||||
"mailu_app_password": ["keepme"],
|
||||
},
|
||||
},
|
||||
]
|
||||
updated = []
|
||||
|
||||
class _Cursor:
|
||||
def __init__(self):
|
||||
self.executions = []
|
||||
|
||||
def execute(self, sql, params):
|
||||
self.executions.append(params)
|
||||
|
||||
def close(self):
|
||||
return None
|
||||
|
||||
class _Conn:
|
||||
def __init__(self):
|
||||
self.autocommit = False
|
||||
self._cursor = _Cursor()
|
||||
|
||||
def cursor(self, cursor_factory=None):
|
||||
return self._cursor
|
||||
|
||||
def close(self):
|
||||
return None
|
||||
|
||||
conn = _Conn()
|
||||
monkeypatch.setattr(sync, "get_kc_token", lambda: "tok")
|
||||
monkeypatch.setattr(sync, "kc_get_users", lambda token: users)
|
||||
monkeypatch.setattr(
|
||||
sync,
|
||||
"kc_update_attributes",
|
||||
lambda token, user, attrs: updated.append(user["id"]),
|
||||
)
|
||||
monkeypatch.setattr(sync.psycopg2, "connect", lambda **kwargs: conn)
|
||||
|
||||
sync.main()
|
||||
|
||||
assert updated == []
|
||||
assert [params["email"] for params in conn._cursor.executions] == [
|
||||
"settled@example.com"
|
||||
]
|
||||
@ -127,8 +127,64 @@ def test_listener_post_wait_keeps_running_request_successful(monkeypatch):
|
||||
assert handler.responses == [200]
|
||||
|
||||
|
||||
def test_listener_post_treats_non_object_json_as_plain_trigger(monkeypatch):
|
||||
listener = load_listener_module(monkeypatch)
|
||||
called = []
|
||||
monkeypatch.setattr(listener, "_trigger_sync_async", lambda force=False: called.append(force) or True)
|
||||
handler = _handler_for(listener, "[1, 2]")
|
||||
|
||||
handler.do_POST()
|
||||
|
||||
assert called == [False]
|
||||
assert handler.responses == [202]
|
||||
|
||||
|
||||
def test_listener_log_message_is_quiet(monkeypatch):
|
||||
listener = load_listener_module(monkeypatch)
|
||||
handler = listener.Handler.__new__(listener.Handler)
|
||||
|
||||
assert handler.log_message("ignored %s", "value") is None
|
||||
|
||||
|
||||
def test_listener_post_treats_non_object_payloads_as_plain_triggers(monkeypatch):
|
||||
listener = load_listener_module(monkeypatch)
|
||||
called = []
|
||||
monkeypatch.setattr(
|
||||
listener,
|
||||
"_trigger_sync_async",
|
||||
lambda force=False: called.append(force) or True,
|
||||
)
|
||||
handler = _handler_for(listener, "[1, 2, 3]")
|
||||
|
||||
handler.do_POST()
|
||||
|
||||
assert called == [False]
|
||||
assert handler.responses == [202]
|
||||
|
||||
|
||||
def test_listener_module_serves_forever_when_run_directly(monkeypatch):
|
||||
import http.server
|
||||
import runpy
|
||||
|
||||
served = []
|
||||
|
||||
class _FakeServer:
|
||||
def __init__(self, address, handler):
|
||||
served.append((address, handler.__name__))
|
||||
|
||||
def serve_forever(self):
|
||||
served.append("serving")
|
||||
|
||||
monkeypatch.setenv("MAILU_SYNC_WAIT_TIMEOUT_SEC", "0")
|
||||
monkeypatch.setattr(http.server, "ThreadingHTTPServer", _FakeServer)
|
||||
module_path = (
|
||||
pathlib.Path(__file__).resolve().parents[2]
|
||||
/ "services"
|
||||
/ "mailu"
|
||||
/ "scripts"
|
||||
/ "mailu_sync_listener.py"
|
||||
)
|
||||
|
||||
runpy.run_path(str(module_path), run_name="__main__")
|
||||
|
||||
assert served == [(("", 8080), "Handler"), "serving"]
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: gitea-atlas-identity-bootstrap-3
|
||||
name: gitea-atlas-identity-bootstrap-5
|
||||
namespace: gitea
|
||||
labels:
|
||||
app.kubernetes.io/name: gitea
|
||||
@ -34,6 +34,20 @@ spec:
|
||||
nodeSelector:
|
||||
node-role.kubernetes.io/worker: "true"
|
||||
hardware: rpi5
|
||||
initContainers:
|
||||
- name: python-runtime
|
||||
image: python@sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df
|
||||
command: [/bin/sh, -ec]
|
||||
args: ["cp -R /usr/local/. /python/"]
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities: {drop: [ALL]}
|
||||
readOnlyRootFilesystem: true
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
runAsGroup: 1000
|
||||
volumeMounts:
|
||||
- {name: python-runtime, mountPath: /python}
|
||||
containers:
|
||||
- name: bootstrap
|
||||
image: gitea/gitea:1.23
|
||||
@ -80,6 +94,7 @@ spec:
|
||||
- name: bootstrap
|
||||
mountPath: /opt/bootstrap
|
||||
readOnly: true
|
||||
- {name: python-runtime, mountPath: /opt/python, readOnly: true}
|
||||
volumes:
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
@ -88,3 +103,6 @@ spec:
|
||||
configMap:
|
||||
name: gitea-atlas-identity-bootstrap
|
||||
defaultMode: 0555
|
||||
- name: python-runtime
|
||||
emptyDir:
|
||||
sizeLimit: 128Mi
|
||||
|
||||
@ -15,6 +15,7 @@ configMapGenerator:
|
||||
- name: gitea-atlas-identity-bootstrap
|
||||
files:
|
||||
- gitea_atlas_identity_ensure.sh=scripts/gitea_atlas_identity_ensure.sh
|
||||
- gitea_branch_protection_check.py=scripts/gitea_branch_protection_check.py
|
||||
|
||||
generatorOptions:
|
||||
disableNameSuffixHash: true
|
||||
|
||||
@ -19,6 +19,7 @@ source_owner="${GITEA_SOURCE_OWNER:-bstein}"
|
||||
managed_repositories="${GITEA_MANAGED_REPOSITORIES:-hermes-code-demo cassandra soteria pegasus metis ananke ariadne typhon titan-iac}"
|
||||
contributor_team=Contributors
|
||||
contributor_units='["repo.actions","repo.packages","repo.code","repo.issues","repo.wiki","repo.pulls","repo.releases","repo.projects"]'
|
||||
protected_reviewer=bstein
|
||||
|
||||
die() {
|
||||
echo "Gitea Atlas identity bootstrap failed: $*" >&2
|
||||
@ -272,6 +273,46 @@ transfer_managed_repositories() {
|
||||
trap - EXIT HUP INT TERM
|
||||
}
|
||||
|
||||
ensure_default_branch_protection() {
|
||||
repository=$1
|
||||
repo_file=$(mktemp)
|
||||
protections_file=$(mktemp)
|
||||
response_file=$(mktemp)
|
||||
|
||||
status=$(api_request GET "/repos/${organization}/${repository}" '' "${repo_file}")
|
||||
expect_status "${status}" 200 "read ${repository} metadata for branch protection"
|
||||
default_branch=$(sed -n 's/.*"default_branch":"\([^"]*\)".*/\1/p' "${repo_file}")
|
||||
case "${default_branch}" in
|
||||
main|master) ;;
|
||||
*) die "${repository} has unsupported default branch ${default_branch:-missing}" ;;
|
||||
esac
|
||||
|
||||
status=$(api_request GET "/repos/${organization}/${repository}/branch_protections" '' "${protections_file}")
|
||||
expect_status "${status}" 200 "list ${repository} branch protections"
|
||||
if ! protection_state=$(
|
||||
/opt/python/bin/python3 /opt/bootstrap/gitea_branch_protection_check.py \
|
||||
"${protections_file}" "${default_branch}" "${protected_reviewer}"
|
||||
); then
|
||||
die "${repository} ${default_branch} protection differs from the human-review policy"
|
||||
fi
|
||||
|
||||
if [ "${protection_state}" = ABSENT ]; then
|
||||
payload="{\"rule_name\":\"${default_branch}\",\"enable_push\":true,\"enable_push_whitelist\":true,\"push_whitelist_usernames\":[\"${protected_reviewer}\"],\"push_whitelist_teams\":[],\"push_whitelist_deploy_keys\":false,\"enable_force_push\":false,\"enable_force_push_allowlist\":false,\"force_push_allowlist_usernames\":[],\"force_push_allowlist_teams\":[],\"force_push_allowlist_deploy_keys\":false,\"enable_merge_whitelist\":true,\"merge_whitelist_usernames\":[\"${protected_reviewer}\"],\"merge_whitelist_teams\":[],\"enable_approvals_whitelist\":true,\"approvals_whitelist_username\":[\"${protected_reviewer}\"],\"approvals_whitelist_teams\":[],\"required_approvals\":1,\"block_on_rejected_reviews\":true,\"block_on_outdated_branch\":true,\"dismiss_stale_approvals\":true,\"block_admin_merge_override\":true}"
|
||||
status=$(api_request POST "/repos/${organization}/${repository}/branch_protections" "${payload}" "${response_file}")
|
||||
expect_status "${status}" 201 "protect ${repository} ${default_branch}"
|
||||
status=$(api_request GET "/repos/${organization}/${repository}/branch_protections" '' "${protections_file}")
|
||||
expect_status "${status}" 200 "read back ${repository} branch protections"
|
||||
protection_state=$(
|
||||
/opt/python/bin/python3 /opt/bootstrap/gitea_branch_protection_check.py \
|
||||
"${protections_file}" "${default_branch}" "${protected_reviewer}"
|
||||
) || die "${repository} ${default_branch} protection differs from the human-review policy"
|
||||
fi
|
||||
[ "${protection_state}" = PRESENT ] || \
|
||||
die "${repository} ${default_branch} protection did not become effective"
|
||||
|
||||
rm -f "${repo_file}" "${protections_file}" "${response_file}"
|
||||
}
|
||||
|
||||
wait_for_gitea
|
||||
ensure_user "${reconciler_user}" "${reconciler_email}" true
|
||||
ensure_user "${hermes_user}" "${hermes_email}" false
|
||||
@ -318,6 +359,10 @@ expect_status "${status}" 204 "add Hermes to Atlas contributors"
|
||||
|
||||
transfer_managed_repositories
|
||||
|
||||
for repository in ${managed_repositories}; do
|
||||
ensure_default_branch_protection "${repository}"
|
||||
done
|
||||
|
||||
vault_write gitea/atlas-reconciler "{\"data\":{\"username\":\"${reconciler_user}\",\"token\":\"${reconciler_token}\",\"base_url\":\"${public_url}\"}}"
|
||||
vault_write hermes/developer-gitea "{\"data\":{\"username\":\"${hermes_user}\",\"email\":\"${hermes_email}\",\"token\":\"${hermes_token}\",\"base_url\":\"${public_url}\"}}"
|
||||
|
||||
|
||||
146
services/gitea/scripts/gitea_branch_protection_check.py
Normal file
146
services/gitea/scripts/gitea_branch_protection_check.py
Normal file
@ -0,0 +1,146 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate the first effective Gitea protection for an Atlas primary branch."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
import stat
|
||||
from pathlib import Path
|
||||
|
||||
MAX_INPUT = 1024 * 1024
|
||||
MAX_RULES = 100
|
||||
MAX_RULE_NAME = 255
|
||||
SPECIAL = frozenset("*?\\[]{}")
|
||||
|
||||
|
||||
class PolicyError(RuntimeError):
|
||||
"""The protection response is ambiguous or violates human review policy."""
|
||||
|
||||
|
||||
def _read_bounded(path: Path) -> bytes:
|
||||
descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
|
||||
try:
|
||||
metadata = os.fstat(descriptor)
|
||||
if not stat.S_ISREG(metadata.st_mode) or metadata.st_size > MAX_INPUT:
|
||||
raise PolicyError("branch protection response exceeds the safe limit")
|
||||
value = os.read(descriptor, MAX_INPUT + 1)
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
if len(value) != metadata.st_size:
|
||||
raise PolicyError("branch protection response changed while reading")
|
||||
return value
|
||||
|
||||
|
||||
def _is_plain(pattern: str) -> bool:
|
||||
return not any(character in SPECIAL for character in pattern)
|
||||
|
||||
|
||||
def _literal_rule(pattern: str) -> str:
|
||||
"""Return one exact rule name or reject Gitea glob interpretation.
|
||||
|
||||
Gitea 1.23.8 delegates matching to gobwas/glob. Reimplementing that
|
||||
grammar and its error quoting rules here would create a second policy
|
||||
engine. The reconciler instead accepts only plain ASCII literals. This
|
||||
deliberately rejects braces, ranges, escapes, malformed globs, and even
|
||||
special rules that appear unrelated: an operator must remove ambiguity
|
||||
before this job may attest the primary-branch boundary.
|
||||
"""
|
||||
if not pattern.isascii() or not 1 <= len(pattern) <= MAX_RULE_NAME:
|
||||
raise PolicyError("branch protection rule name is invalid")
|
||||
if not _is_plain(pattern):
|
||||
raise PolicyError("glob branch protection rules require human review")
|
||||
return pattern
|
||||
|
||||
|
||||
def _created(value: object) -> datetime.datetime:
|
||||
if not isinstance(value, str) or len(value) > 64:
|
||||
raise PolicyError("branch protection creation time is invalid")
|
||||
try:
|
||||
result = datetime.datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
except ValueError as exc:
|
||||
raise PolicyError("branch protection creation time is invalid") from exc
|
||||
if result.tzinfo is None:
|
||||
raise PolicyError("branch protection creation time is invalid")
|
||||
return result
|
||||
|
||||
|
||||
def _required(reviewer: str) -> dict[str, object]:
|
||||
return {
|
||||
"enable_push": True,
|
||||
"enable_push_whitelist": True,
|
||||
"push_whitelist_usernames": [reviewer],
|
||||
"push_whitelist_deploy_keys": False,
|
||||
"enable_force_push": False,
|
||||
"enable_merge_whitelist": True,
|
||||
"merge_whitelist_usernames": [reviewer],
|
||||
"enable_approvals_whitelist": True,
|
||||
"approvals_whitelist_username": [reviewer],
|
||||
"required_approvals": 1,
|
||||
"block_on_rejected_reviews": True,
|
||||
"block_on_outdated_branch": True,
|
||||
"dismiss_stale_approvals": True,
|
||||
"block_admin_merge_override": True,
|
||||
}
|
||||
|
||||
|
||||
def evaluate(value: bytes, branch: str, reviewer: str) -> str:
|
||||
"""Return PRESENT/ABSENT or fail on ambiguous/effective policy drift."""
|
||||
if len(value) > MAX_INPUT or branch not in {"main", "master"}:
|
||||
raise PolicyError("branch protection input is invalid")
|
||||
data = json.loads(value)
|
||||
if not isinstance(data, list) or len(data) > MAX_RULES:
|
||||
raise PolicyError("branch protection response is invalid")
|
||||
ordered: list[tuple[int, datetime.datetime, int, dict[str, object]]] = []
|
||||
for index, raw in enumerate(data):
|
||||
if not isinstance(raw, dict):
|
||||
raise PolicyError("branch protection entry is invalid")
|
||||
priority = raw.get("priority")
|
||||
rule_name = raw.get("rule_name")
|
||||
if (
|
||||
not isinstance(priority, int)
|
||||
or isinstance(priority, bool)
|
||||
or not 0 <= priority <= 1_000_000
|
||||
or not isinstance(rule_name, str)
|
||||
):
|
||||
raise PolicyError("branch protection priority is ambiguous")
|
||||
created = _created(raw.get("created_at"))
|
||||
literal = _literal_rule(rule_name)
|
||||
if literal == branch:
|
||||
# With every rule constrained to a plain literal, Gitea's glob
|
||||
# comparator cannot affect this result. Preserve API ordering:
|
||||
# priority first (including legacy zero), creation time, then the
|
||||
# stable response order for an otherwise exact tie.
|
||||
ordered.append((priority, created, index, raw))
|
||||
if not ordered:
|
||||
return "ABSENT"
|
||||
if len(ordered) != 1:
|
||||
raise PolicyError("duplicate primary-branch protections are ambiguous")
|
||||
effective = ordered[0][3]
|
||||
for key, expected in _required(reviewer).items():
|
||||
if effective.get(key) != expected:
|
||||
raise PolicyError(
|
||||
f"effective {branch} protection differs from human-review policy"
|
||||
)
|
||||
return "PRESENT"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("response", type=Path)
|
||||
parser.add_argument("branch", choices=("main", "master"))
|
||||
parser.add_argument("reviewer")
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
value = _read_bounded(args.response)
|
||||
print(evaluate(value, args.branch, args.reviewer))
|
||||
except (OSError, json.JSONDecodeError, PolicyError) as exc:
|
||||
print(f"branch protection check failed: {exc}", file=__import__("sys").stderr)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
79
services/harbor/hermes-agent-immutability-job.yaml
Normal file
79
services/harbor/hermes-agent-immutability-job.yaml
Normal file
@ -0,0 +1,79 @@
|
||||
# services/harbor/hermes-agent-immutability-job.yaml
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: harbor-hermes-agent-immutability-ensure-1
|
||||
namespace: harbor
|
||||
spec:
|
||||
backoffLimit: 2
|
||||
activeDeadlineSeconds: 600
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
vault.hashicorp.com/agent-inject: "true"
|
||||
vault.hashicorp.com/agent-pre-populate-only: "true"
|
||||
vault.hashicorp.com/agent-run-as-user: "65532"
|
||||
vault.hashicorp.com/agent-run-as-group: "65532"
|
||||
vault.hashicorp.com/role: harbor-policy-bootstrap
|
||||
vault.hashicorp.com/agent-inject-secret-harbor-admin-password: kv/data/atlas/harbor/harbor-core
|
||||
vault.hashicorp.com/agent-inject-template-harbor-admin-password: |
|
||||
{{- with secret "kv/data/atlas/harbor/harbor-core" -}}
|
||||
{{ .Data.data.harbor_admin_password }}
|
||||
{{- end -}}
|
||||
spec:
|
||||
serviceAccountName: harbor-policy-bootstrap
|
||||
enableServiceLinks: false
|
||||
restartPolicy: Never
|
||||
nodeSelector:
|
||||
hardware: rpi5
|
||||
kubernetes.io/arch: arm64
|
||||
node-role.kubernetes.io/worker: "true"
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- key: kubernetes.io/hostname
|
||||
operator: NotIn
|
||||
values: [titan-04, titan-14, titan-18, titan-19, titan-24]
|
||||
securityContext:
|
||||
fsGroup: 65532
|
||||
fsGroupChangePolicy: OnRootMismatch
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: ensure
|
||||
image: docker.io/library/python@sha256:efcdfa6a6b2fd2afb9c7dfa9a5b288a6f68338b5cfdebe6b637d986067d85757
|
||||
imagePullPolicy: IfNotPresent
|
||||
command: [python3, /scripts/harbor_hermes_agent_immutability_ensure.py]
|
||||
env:
|
||||
- name: HARBOR_API_ORIGIN
|
||||
value: https://registry.bstein.dev/api/v2.0
|
||||
- name: HARBOR_ADMIN_PASSWORD_FILE
|
||||
value: /vault/secrets/harbor-admin-password
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
readOnlyRootFilesystem: true
|
||||
runAsGroup: 65532
|
||||
runAsNonRoot: true
|
||||
runAsUser: 65532
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
volumeMounts:
|
||||
- name: scripts
|
||||
mountPath: /scripts
|
||||
readOnly: true
|
||||
- name: tmp
|
||||
mountPath: /tmp
|
||||
resources:
|
||||
requests: {cpu: 25m, memory: 32Mi}
|
||||
limits: {cpu: 250m, memory: 128Mi}
|
||||
volumes:
|
||||
- name: scripts
|
||||
configMap:
|
||||
name: harbor-hermes-agent-immutability-script
|
||||
defaultMode: 0555
|
||||
- name: tmp
|
||||
emptyDir: {}
|
||||
@ -12,9 +12,14 @@ resources:
|
||||
- certificate.yaml
|
||||
- helmrelease.yaml
|
||||
- vault-sync-deployment.yaml
|
||||
- policy-bootstrap-serviceaccount.yaml
|
||||
- hermes-agent-immutability-job.yaml
|
||||
- bootstrap-jobs/cassandra-registry-ensure-job.yaml
|
||||
- image.yaml
|
||||
configMapGenerator:
|
||||
- name: harbor-vault-entrypoint
|
||||
files:
|
||||
- scripts/vault-entrypoint.sh
|
||||
- name: harbor-hermes-agent-immutability-script
|
||||
files:
|
||||
- harbor_hermes_agent_immutability_ensure.py=scripts/harbor_hermes_agent_immutability_ensure.py
|
||||
|
||||
7
services/harbor/policy-bootstrap-serviceaccount.yaml
Normal file
7
services/harbor/policy-bootstrap-serviceaccount.yaml
Normal file
@ -0,0 +1,7 @@
|
||||
# services/harbor/policy-bootstrap-serviceaccount.yaml
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: harbor-policy-bootstrap
|
||||
namespace: harbor
|
||||
automountServiceAccountToken: true
|
||||
@ -0,0 +1,378 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create and verify the narrowly scoped Hermes agent immutable-tag rule."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
PROJECT = "bstein"
|
||||
REPOSITORY_PATTERN = "hermes-agent"
|
||||
TAG_PATTERN = "git-*-build-*"
|
||||
PUBLISH_ROBOT = "robot$jenkins-pipelines"
|
||||
EXPECTED_ORIGIN = "https://registry.bstein.dev/api/v2.0"
|
||||
MAX_RESPONSE = 1_048_576
|
||||
TRANSIENT_STATUSES = {429, 502, 503, 504}
|
||||
EXPECTED_RULE = {
|
||||
"disabled": False,
|
||||
"action": "immutable",
|
||||
"template": "immutable_template",
|
||||
"tag_selectors": [
|
||||
{
|
||||
"kind": "doublestar",
|
||||
"decoration": "matches",
|
||||
"pattern": TAG_PATTERN,
|
||||
}
|
||||
],
|
||||
"scope_selectors": {
|
||||
"repository": [
|
||||
{
|
||||
"kind": "doublestar",
|
||||
"decoration": "repoMatches",
|
||||
"pattern": REPOSITORY_PATTERN,
|
||||
}
|
||||
]
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class NoRedirect(urllib.request.HTTPRedirectHandler):
|
||||
"""Prevent Basic credentials from following an unexpected redirect."""
|
||||
|
||||
def redirect_request(self, _request, _file, _code, _message, _headers, _url):
|
||||
return None
|
||||
|
||||
|
||||
class HarborUnavailable(RuntimeError):
|
||||
"""Harbor is not ready yet, rather than returning a policy decision."""
|
||||
|
||||
|
||||
def normalized_rule(rule: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Return only the immutable contract fields Harbor must preserve."""
|
||||
return {
|
||||
"disabled": bool(rule.get("disabled", False)),
|
||||
"action": rule.get("action"),
|
||||
"template": rule.get("template"),
|
||||
"tag_selectors": [
|
||||
{
|
||||
"kind": selector.get("kind"),
|
||||
"decoration": selector.get("decoration"),
|
||||
"pattern": selector.get("pattern"),
|
||||
}
|
||||
for selector in rule.get("tag_selectors") or []
|
||||
if isinstance(selector, dict)
|
||||
],
|
||||
"scope_selectors": {
|
||||
"repository": [
|
||||
{
|
||||
"kind": selector.get("kind"),
|
||||
"decoration": selector.get("decoration"),
|
||||
"pattern": selector.get("pattern"),
|
||||
}
|
||||
for selector in (rule.get("scope_selectors") or {}).get(
|
||||
"repository", []
|
||||
)
|
||||
if isinstance(selector, dict)
|
||||
]
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def targets_hermes_builds(rule: dict[str, Any]) -> bool:
|
||||
"""Detect a rule that claims this exact repository and tag selector."""
|
||||
normalized = normalized_rule(rule)
|
||||
return (
|
||||
normalized["tag_selectors"] == EXPECTED_RULE["tag_selectors"]
|
||||
and normalized["scope_selectors"] == EXPECTED_RULE["scope_selectors"]
|
||||
)
|
||||
|
||||
|
||||
class HarborClient:
|
||||
"""Bounded same-origin client for Harbor's immutable-tag API."""
|
||||
|
||||
def __init__(self, origin: str, username: str, password: str) -> None:
|
||||
normalized_origin = origin.rstrip("/")
|
||||
if normalized_origin != EXPECTED_ORIGIN:
|
||||
raise ValueError("Harbor API origin is not the pinned production API")
|
||||
self.origin = normalized_origin
|
||||
token = base64.b64encode(f"{username}:{password}".encode()).decode("ascii")
|
||||
self.headers = {"Authorization": f"Basic {token}"}
|
||||
self.opener = urllib.request.build_opener(NoRedirect())
|
||||
|
||||
def request(
|
||||
self, method: str, path: str, payload: dict[str, Any] | None = None
|
||||
) -> tuple[int, bytes, dict[str, str]]:
|
||||
"""Issue one request, returning even non-2xx responses for strict checks."""
|
||||
data = None
|
||||
headers = dict(self.headers)
|
||||
if payload is not None:
|
||||
data = json.dumps(payload, separators=(",", ":")).encode()
|
||||
headers["Content-Type"] = "application/json"
|
||||
request = urllib.request.Request(
|
||||
f"{self.origin}{path}", data=data, headers=headers, method=method
|
||||
)
|
||||
try:
|
||||
response = self.opener.open(request, timeout=20)
|
||||
except urllib.error.HTTPError as exc:
|
||||
response = exc
|
||||
except (urllib.error.URLError, TimeoutError) as exc:
|
||||
raise HarborUnavailable("Harbor policy API is unavailable") from exc
|
||||
with response:
|
||||
body = response.read(MAX_RESPONSE + 1)
|
||||
if len(body) > MAX_RESPONSE:
|
||||
raise RuntimeError("Harbor response exceeded the size limit")
|
||||
return int(response.status), body, dict(response.headers)
|
||||
|
||||
|
||||
def list_rules(client: HarborClient) -> list[dict[str, Any]]:
|
||||
"""Read and validate the complete small rule set for the project."""
|
||||
path = f"/projects/{PROJECT}/immutabletagrules?page=1&page_size=100"
|
||||
status, body, headers = client.request("GET", path)
|
||||
if status in TRANSIENT_STATUSES:
|
||||
raise HarborUnavailable(f"Harbor immutable rule list returned HTTP {status}")
|
||||
if status != 200:
|
||||
raise RuntimeError(f"Harbor immutable rule list returned HTTP {status}")
|
||||
values = _json_list(body, "immutable rule")
|
||||
_require_complete_page(values, headers, "immutable rule")
|
||||
return values
|
||||
|
||||
|
||||
def _json_list(body: bytes, label: str) -> list[dict[str, Any]]:
|
||||
"""Decode one bounded Harbor list without accepting a partial shape."""
|
||||
try:
|
||||
value = json.loads(body.decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise RuntimeError(f"Harbor returned invalid {label} JSON") from exc
|
||||
if not isinstance(value, list) or not all(isinstance(item, dict) for item in value):
|
||||
raise RuntimeError(f"Harbor {label} list has an invalid shape")
|
||||
return value
|
||||
|
||||
|
||||
def _require_complete_page(
|
||||
values: list[dict[str, Any]], headers: dict[str, str], label: str
|
||||
) -> None:
|
||||
"""Reject a truncated first page or a proxy that strips count evidence."""
|
||||
raw_total = next(
|
||||
(value for key, value in headers.items() if key.lower() == "x-total-count"),
|
||||
None,
|
||||
)
|
||||
if raw_total is None or not str(raw_total).isdecimal():
|
||||
raise RuntimeError(f"Harbor {label} list omitted a valid total count")
|
||||
if int(raw_total) != len(values):
|
||||
raise RuntimeError(f"Harbor {label} list was truncated")
|
||||
|
||||
|
||||
def _allowed(access: dict[str, Any], resource: str, action: str) -> bool:
|
||||
"""Match one positive robot permission, accepting Harbor's default effect."""
|
||||
return (
|
||||
access.get("resource") == resource
|
||||
and access.get("action") == action
|
||||
and access.get("effect") in (None, "", "allow")
|
||||
)
|
||||
|
||||
|
||||
def _publisher_scope(robot: dict[str, Any]) -> tuple[dict[str, Any], list[dict[str, Any]]]:
|
||||
"""Return the existing bstein scope after validating its push boundary."""
|
||||
permissions = robot.get("permissions")
|
||||
if not isinstance(permissions, list) or not all(
|
||||
isinstance(item, dict) for item in permissions
|
||||
):
|
||||
raise RuntimeError("Jenkins Harbor robot permissions have an invalid shape")
|
||||
matches = [
|
||||
item
|
||||
for item in permissions
|
||||
if item.get("kind") == "project" and item.get("namespace") == PROJECT
|
||||
]
|
||||
if len(matches) != 1:
|
||||
raise RuntimeError("Jenkins Harbor robot must have one existing bstein scope")
|
||||
access = matches[0].get("access")
|
||||
if not isinstance(access, list) or not all(isinstance(item, dict) for item in access):
|
||||
raise RuntimeError("Jenkins Harbor robot bstein access has an invalid shape")
|
||||
for required in (("repository", "pull"), ("repository", "push")):
|
||||
if not any(_allowed(item, *required) for item in access):
|
||||
raise RuntimeError("Jenkins Harbor robot lacks its existing pull/push boundary")
|
||||
immutable_access = [
|
||||
item for item in access if item.get("resource") == "immutable-tag"
|
||||
]
|
||||
if immutable_access and not (
|
||||
len(immutable_access) == 1
|
||||
and _allowed(immutable_access[0], "immutable-tag", "list")
|
||||
):
|
||||
raise RuntimeError("Jenkins Harbor robot has broader immutable-tag access")
|
||||
return matches[0], access
|
||||
|
||||
|
||||
def _verified_publisher(robot: dict[str, Any]) -> bool:
|
||||
"""Check the exact identity and least-privilege policy after persistence."""
|
||||
if (
|
||||
robot.get("name") != PUBLISH_ROBOT
|
||||
or robot.get("level") != "system"
|
||||
or robot.get("editable") is not True
|
||||
or robot.get("disable") is not False
|
||||
):
|
||||
return False
|
||||
try:
|
||||
_scope, access = _publisher_scope(robot)
|
||||
except RuntimeError:
|
||||
return False
|
||||
return any(_allowed(item, "immutable-tag", "list") for item in access)
|
||||
|
||||
|
||||
def ensure_publisher_can_read_rule(client: HarborClient) -> int:
|
||||
"""Grant only immutable-tag:list to the existing Jenkins push robot."""
|
||||
status, body, headers = client.request("GET", "/robots?page=1&page_size=100")
|
||||
if status in TRANSIENT_STATUSES:
|
||||
raise HarborUnavailable(f"Harbor robot list returned HTTP {status}")
|
||||
if status != 200:
|
||||
raise RuntimeError(f"Harbor robot list returned HTTP {status}")
|
||||
robots = _json_list(body, "robot")
|
||||
_require_complete_page(robots, headers, "robot")
|
||||
matches = [
|
||||
item
|
||||
for item in robots
|
||||
if item.get("name") == PUBLISH_ROBOT
|
||||
]
|
||||
if len(matches) != 1:
|
||||
raise RuntimeError("expected exactly one Jenkins Harbor publisher robot")
|
||||
robot_id = matches[0].get("id")
|
||||
if not isinstance(robot_id, int) or robot_id < 1:
|
||||
raise RuntimeError("Jenkins Harbor robot omitted a valid ID")
|
||||
status, body, _headers = client.request("GET", f"/robots/{robot_id}")
|
||||
if status in TRANSIENT_STATUSES:
|
||||
raise HarborUnavailable(f"Harbor robot read returned HTTP {status}")
|
||||
if status != 200:
|
||||
raise RuntimeError(f"Harbor robot read returned HTTP {status}")
|
||||
try:
|
||||
robot = json.loads(body.decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise RuntimeError("Harbor returned invalid robot JSON") from exc
|
||||
if not isinstance(robot, dict):
|
||||
raise RuntimeError("Harbor robot response has an invalid shape")
|
||||
if (
|
||||
robot.get("name") != PUBLISH_ROBOT
|
||||
or robot.get("level") != "system"
|
||||
or robot.get("editable") is not True
|
||||
or robot.get("disable") is not False
|
||||
):
|
||||
raise RuntimeError("Jenkins Harbor publisher identity is not active and exact")
|
||||
scope, access = _publisher_scope(robot)
|
||||
if any(_allowed(item, "immutable-tag", "list") for item in access):
|
||||
return robot_id
|
||||
|
||||
# Harbor updates permissions through PUT /robots/{id}; its separate PATCH
|
||||
# endpoint is the only secret-rotation operation. Preserve every current
|
||||
# scope and field while adding the one read-only action.
|
||||
access.append({"resource": "immutable-tag", "action": "list"})
|
||||
payload = {
|
||||
"name": robot["name"],
|
||||
"description": robot.get("description") or "",
|
||||
"level": robot["level"],
|
||||
"disable": robot["disable"],
|
||||
"permissions": robot["permissions"],
|
||||
}
|
||||
duration = robot.get("duration")
|
||||
if duration is not None:
|
||||
if not isinstance(duration, int):
|
||||
raise RuntimeError("Jenkins Harbor robot duration has an invalid shape")
|
||||
payload["duration"] = duration
|
||||
# Keep the validated object reference live in the preserved permission list.
|
||||
if scope.get("access") is not access:
|
||||
raise RuntimeError("Jenkins Harbor robot permission normalization drifted")
|
||||
status, _body, _headers = client.request(
|
||||
"PUT", f"/robots/{robot_id}", payload
|
||||
)
|
||||
if status in TRANSIENT_STATUSES:
|
||||
raise HarborUnavailable(f"Harbor robot update returned HTTP {status}")
|
||||
if status != 200:
|
||||
raise RuntimeError(f"Harbor robot policy update returned HTTP {status}")
|
||||
|
||||
for attempt in range(1, 6):
|
||||
status, body, _headers = client.request("GET", f"/robots/{robot_id}")
|
||||
if status == 200:
|
||||
try:
|
||||
persisted = json.loads(body.decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError):
|
||||
persisted = None
|
||||
if isinstance(persisted, dict) and _verified_publisher(persisted):
|
||||
return robot_id
|
||||
if attempt < 5:
|
||||
time.sleep(attempt)
|
||||
raise RuntimeError("Jenkins Harbor robot policy did not verify exactly")
|
||||
|
||||
|
||||
def ensure_rule(client: HarborClient) -> int:
|
||||
"""Create once, or validate the one exact enabled rule already present."""
|
||||
rules = list_rules(client)
|
||||
matches = [rule for rule in rules if targets_hermes_builds(rule)]
|
||||
if len(matches) > 1:
|
||||
raise RuntimeError("multiple Hermes agent immutable rules exist")
|
||||
if matches:
|
||||
if normalized_rule(matches[0]) != EXPECTED_RULE:
|
||||
raise RuntimeError("Hermes agent immutable rule exists but is not enabled/exact")
|
||||
rule_id = matches[0].get("id")
|
||||
if not isinstance(rule_id, int) or rule_id < 1:
|
||||
raise RuntimeError("Harbor immutable rule omitted a valid ID")
|
||||
return rule_id
|
||||
|
||||
path = f"/projects/{PROJECT}/immutabletagrules"
|
||||
status, _body, headers = client.request("POST", path, EXPECTED_RULE)
|
||||
if status in TRANSIENT_STATUSES:
|
||||
raise HarborUnavailable(f"Harbor immutable rule create returned HTTP {status}")
|
||||
if status != 201:
|
||||
raise RuntimeError(f"Harbor immutable rule create returned HTTP {status}")
|
||||
location = headers.get("Location") or headers.get("location") or ""
|
||||
api_path = urllib.parse.urlsplit(client.origin).path.rstrip("/")
|
||||
expected_prefix = f"{api_path}{path}/"
|
||||
if not location.startswith(expected_prefix):
|
||||
raise RuntimeError("Harbor immutable rule create omitted the exact Location")
|
||||
suffix = location[len(expected_prefix) :]
|
||||
if not suffix.isdecimal() or int(suffix) < 1:
|
||||
raise RuntimeError("Harbor immutable rule Location has an invalid ID")
|
||||
|
||||
# Harbor persists synchronously, but bounded retries distinguish a stale
|
||||
# read from accepting an unverified policy.
|
||||
for attempt in range(1, 6):
|
||||
matches = [rule for rule in list_rules(client) if targets_hermes_builds(rule)]
|
||||
if len(matches) == 1 and normalized_rule(matches[0]) == EXPECTED_RULE:
|
||||
rule_id = matches[0].get("id")
|
||||
if rule_id == int(suffix):
|
||||
return rule_id
|
||||
if attempt < 5:
|
||||
time.sleep(attempt)
|
||||
raise RuntimeError("created Harbor immutable rule did not verify exactly")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Load the runtime-only admin credential and enforce the tracked policy."""
|
||||
origin = os.environ.get("HARBOR_API_ORIGIN", "")
|
||||
password_file = Path(os.environ.get("HARBOR_ADMIN_PASSWORD_FILE", ""))
|
||||
password = password_file.read_text(encoding="utf-8").strip()
|
||||
if not password:
|
||||
raise RuntimeError("Harbor admin password is empty")
|
||||
client = HarborClient(origin, "admin", password)
|
||||
for attempt in range(1, 13):
|
||||
try:
|
||||
rule_id = ensure_rule(client)
|
||||
robot_id = ensure_publisher_can_read_rule(client)
|
||||
break
|
||||
except HarborUnavailable:
|
||||
if attempt == 12:
|
||||
raise
|
||||
time.sleep(min(attempt * 2, 15))
|
||||
print(
|
||||
"Hermes agent immutable build-tag rule is active "
|
||||
f"(id={rule_id}); publisher preflight access is active (robot={robot_id})"
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover - exercised through main()
|
||||
raise SystemExit(main())
|
||||
5
services/hermes-observer-bindings/kustomization.yaml
Normal file
5
services/hermes-observer-bindings/kustomization.yaml
Normal file
@ -0,0 +1,5 @@
|
||||
# services/hermes-observer-bindings/kustomization.yaml
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- rolebindings.yaml
|
||||
204
services/hermes-observer-bindings/rolebindings.yaml
Normal file
204
services/hermes-observer-bindings/rolebindings.yaml
Normal file
@ -0,0 +1,204 @@
|
||||
# services/hermes-observer-bindings/rolebindings.yaml
|
||||
apiVersion: v1
|
||||
kind: List
|
||||
items:
|
||||
- apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata: {name: hermes-agent-observer, namespace: ai}
|
||||
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
|
||||
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
|
||||
- apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata: {name: hermes-agent-observer, namespace: bstein-dev-home}
|
||||
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
|
||||
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
|
||||
- apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata: {name: hermes-agent-observer, namespace: cassandra}
|
||||
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
|
||||
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
|
||||
- apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata: {name: hermes-agent-observer, namespace: cert-manager}
|
||||
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
|
||||
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
|
||||
- apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata: {name: hermes-agent-observer, namespace: climate}
|
||||
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
|
||||
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
|
||||
- apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata: {name: hermes-agent-observer, namespace: comms}
|
||||
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
|
||||
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
|
||||
- apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata: {name: hermes-agent-observer, namespace: crypto}
|
||||
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
|
||||
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
|
||||
- apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata: {name: hermes-agent-observer, namespace: default}
|
||||
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
|
||||
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
|
||||
- apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata: {name: hermes-agent-observer, namespace: finance}
|
||||
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
|
||||
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
|
||||
- apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata: {name: hermes-agent-observer, namespace: flux-system}
|
||||
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
|
||||
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
|
||||
- apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata: {name: hermes-agent-observer, namespace: game-stream}
|
||||
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
|
||||
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
|
||||
- apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata: {name: hermes-agent-observer, namespace: gitea}
|
||||
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
|
||||
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
|
||||
- apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata: {name: hermes-agent-observer, namespace: harbor}
|
||||
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
|
||||
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
|
||||
- apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata: {name: hermes-agent-observer, namespace: health}
|
||||
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
|
||||
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
|
||||
- apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata: {name: hermes-agent-observer, namespace: hermes}
|
||||
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
|
||||
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
|
||||
- apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata: {name: hermes-agent-observer, namespace: hermes-chat}
|
||||
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
|
||||
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
|
||||
- apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata: {name: hermes-agent-observer, namespace: hermes-triage-demo}
|
||||
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
|
||||
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
|
||||
- apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata: {name: hermes-agent-observer, namespace: jellyfin}
|
||||
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
|
||||
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
|
||||
- apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata: {name: hermes-agent-observer, namespace: jenkins}
|
||||
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
|
||||
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
|
||||
- apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata: {name: hermes-agent-observer, namespace: kube-node-lease}
|
||||
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
|
||||
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
|
||||
- apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata: {name: hermes-agent-observer, namespace: kube-public}
|
||||
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
|
||||
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
|
||||
- apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata: {name: hermes-agent-observer, namespace: kube-system}
|
||||
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
|
||||
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
|
||||
- apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata: {name: hermes-agent-observer, namespace: logging}
|
||||
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
|
||||
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
|
||||
- apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata: {name: hermes-agent-observer, namespace: longhorn-system}
|
||||
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
|
||||
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
|
||||
- apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata: {name: hermes-agent-observer, namespace: mailu-mailserver}
|
||||
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
|
||||
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
|
||||
- apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata: {name: hermes-agent-observer, namespace: maintenance}
|
||||
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
|
||||
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
|
||||
- apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata: {name: hermes-agent-observer, namespace: metallb-system}
|
||||
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
|
||||
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
|
||||
- apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata: {name: hermes-agent-observer, namespace: monitoring}
|
||||
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
|
||||
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
|
||||
- apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata: {name: hermes-agent-observer, namespace: nextcloud}
|
||||
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
|
||||
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
|
||||
- apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata: {name: hermes-agent-observer, namespace: openclaw}
|
||||
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
|
||||
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
|
||||
- apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata: {name: hermes-agent-observer, namespace: outline}
|
||||
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
|
||||
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
|
||||
- apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata: {name: hermes-agent-observer, namespace: planka}
|
||||
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
|
||||
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
|
||||
- apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata: {name: hermes-agent-observer, namespace: postgres}
|
||||
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
|
||||
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
|
||||
- apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata: {name: hermes-agent-observer, namespace: quality}
|
||||
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
|
||||
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
|
||||
- apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata: {name: hermes-agent-observer, namespace: sso}
|
||||
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
|
||||
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
|
||||
- apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata: {name: hermes-agent-observer, namespace: sui-metrics}
|
||||
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
|
||||
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
|
||||
- apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata: {name: hermes-agent-observer, namespace: traefik}
|
||||
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
|
||||
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
|
||||
- apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata: {name: hermes-agent-observer, namespace: vault}
|
||||
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
|
||||
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
|
||||
- apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata: {name: hermes-agent-observer, namespace: vaultwarden}
|
||||
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
|
||||
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
|
||||
- apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata: {name: hermes-agent-observer, namespace: veles}
|
||||
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
|
||||
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
|
||||
5
services/hermes-observer-rbac/kustomization.yaml
Normal file
5
services/hermes-observer-rbac/kustomization.yaml
Normal file
@ -0,0 +1,5 @@
|
||||
# services/hermes-observer-rbac/kustomization.yaml
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- rbac.yaml
|
||||
112
services/hermes-observer-rbac/rbac.yaml
Normal file
112
services/hermes-observer-rbac/rbac.yaml
Normal file
@ -0,0 +1,112 @@
|
||||
# services/hermes-observer-rbac/rbac.yaml
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: hermes-agent-cluster-observer-v2
|
||||
labels:
|
||||
app.kubernetes.io/name: hermes-agent
|
||||
app.kubernetes.io/part-of: hermes
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources:
|
||||
- namespaces
|
||||
- nodes
|
||||
- persistentvolumes
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: ["networking.k8s.io"]
|
||||
resources: ["ingressclasses"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: ["storage.k8s.io"]
|
||||
resources:
|
||||
- csidrivers
|
||||
- csinodes
|
||||
- storageclasses
|
||||
- volumeattachments
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: ["metrics.k8s.io"]
|
||||
resources: ["nodes"]
|
||||
verbs: ["get", "list"]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: hermes-agent-namespaced-observer-v2
|
||||
labels:
|
||||
app.kubernetes.io/name: hermes-agent
|
||||
app.kubernetes.io/part-of: hermes
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources:
|
||||
- configmaps
|
||||
- endpoints
|
||||
- events
|
||||
- persistentvolumeclaims
|
||||
- pods
|
||||
- pods/log
|
||||
- replicationcontrollers
|
||||
- resourcequotas
|
||||
- services
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: ["apps"]
|
||||
resources:
|
||||
- controllerrevisions
|
||||
- daemonsets
|
||||
- deployments
|
||||
- replicasets
|
||||
- statefulsets
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: ["batch"]
|
||||
resources: ["cronjobs", "jobs"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: ["autoscaling"]
|
||||
resources: ["horizontalpodautoscalers"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: ["networking.k8s.io"]
|
||||
resources: ["ingresses", "networkpolicies"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: ["discovery.k8s.io"]
|
||||
resources: ["endpointslices"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: ["policy"]
|
||||
resources: ["poddisruptionbudgets"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: ["storage.k8s.io"]
|
||||
resources: ["csistoragecapacities"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: ["metrics.k8s.io"]
|
||||
resources: ["pods"]
|
||||
verbs: ["get", "list"]
|
||||
- apiGroups:
|
||||
- helm.toolkit.fluxcd.io
|
||||
- image.toolkit.fluxcd.io
|
||||
- kustomize.toolkit.fluxcd.io
|
||||
- notification.toolkit.fluxcd.io
|
||||
- source.toolkit.fluxcd.io
|
||||
resources: ["*"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: ["longhorn.io"]
|
||||
resources:
|
||||
- backingimages
|
||||
- engines
|
||||
- instancemanagers
|
||||
- nodes
|
||||
- replicas
|
||||
- settings
|
||||
- volumes
|
||||
verbs: ["get", "list", "watch"]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: hermes-agent-cluster-observer-v2
|
||||
labels:
|
||||
app.kubernetes.io/name: hermes-agent
|
||||
app.kubernetes.io/part-of: hermes
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: hermes-agent
|
||||
namespace: hermes
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: hermes-agent-cluster-observer-v2
|
||||
99
services/hermes-scm-broker/deployment.yaml
Normal file
99
services/hermes-scm-broker/deployment.yaml
Normal file
@ -0,0 +1,99 @@
|
||||
# services/hermes-scm-broker/deployment.yaml
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: hermes-scm-broker
|
||||
namespace: hermes-scm
|
||||
labels:
|
||||
app: hermes-scm-broker
|
||||
spec:
|
||||
replicas: 1
|
||||
revisionHistoryLimit: 2
|
||||
selector:
|
||||
matchLabels:
|
||||
app: hermes-scm-broker
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: hermes-scm-broker
|
||||
annotations:
|
||||
ai.bstein.dev/config-rev: scm-boundary-v2
|
||||
vault.hashicorp.com/agent-inject: "true"
|
||||
vault.hashicorp.com/role: hermes-scm-broker
|
||||
vault.hashicorp.com/agent-inject-secret-gitea-token: kv/data/atlas/hermes/developer-gitea
|
||||
vault.hashicorp.com/agent-inject-template-gitea-token: |
|
||||
{{- with secret "kv/data/atlas/hermes/developer-gitea" -}}
|
||||
{{ .Data.data.token }}
|
||||
{{- end }}
|
||||
vault.hashicorp.com/agent-pre-populate-only: "true"
|
||||
vault.hashicorp.com/agent-init-first: "true"
|
||||
vault.hashicorp.com/agent-requests-cpu: 10m
|
||||
vault.hashicorp.com/agent-requests-mem: 32Mi
|
||||
vault.hashicorp.com/agent-limits-cpu: 50m
|
||||
vault.hashicorp.com/agent-limits-mem: 64Mi
|
||||
spec:
|
||||
serviceAccountName: hermes-scm-broker
|
||||
automountServiceAccountToken: true
|
||||
securityContext:
|
||||
fsGroup: 10000
|
||||
fsGroupChangePolicy: OnRootMismatch
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- key: kubernetes.io/arch
|
||||
operator: In
|
||||
values: [arm64]
|
||||
- key: node-role.kubernetes.io/worker
|
||||
operator: In
|
||||
values: ["true"]
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
- weight: 100
|
||||
preference:
|
||||
matchExpressions:
|
||||
- key: hardware
|
||||
operator: In
|
||||
values: [rpi5]
|
||||
containers:
|
||||
- name: broker
|
||||
image: registry.bstein.dev/bstein/hermes-agent
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- /opt/hermes/.venv/bin/python
|
||||
- /opt/broker/scm_broker.py
|
||||
ports:
|
||||
- {name: http, containerPort: 9081, protocol: TCP}
|
||||
readinessProbe:
|
||||
httpGet: {path: /healthz, port: http}
|
||||
initialDelaySeconds: 3
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
httpGet: {path: /healthz, port: http}
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 30
|
||||
failureThreshold: 10
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: [ALL]
|
||||
readOnlyRootFilesystem: true
|
||||
runAsNonRoot: true
|
||||
runAsUser: 10000
|
||||
runAsGroup: 10000
|
||||
volumeMounts:
|
||||
- {name: broker-code, mountPath: /opt/broker, readOnly: true}
|
||||
- {name: tmp, mountPath: /tmp}
|
||||
resources:
|
||||
requests: {cpu: 50m, memory: 128Mi}
|
||||
limits: {cpu: "1", memory: 768Mi}
|
||||
volumes:
|
||||
- name: broker-code
|
||||
configMap:
|
||||
name: hermes-scm-boundary-v2
|
||||
defaultMode: 0555
|
||||
- name: tmp
|
||||
emptyDir:
|
||||
sizeLimit: 1Gi
|
||||
11
services/hermes-scm-broker/kustomization.yaml
Normal file
11
services/hermes-scm-broker/kustomization.yaml
Normal file
@ -0,0 +1,11 @@
|
||||
# services/hermes-scm-broker/kustomization.yaml
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: hermes-scm
|
||||
resources:
|
||||
- service.yaml
|
||||
- deployment.yaml
|
||||
- networkpolicy.yaml
|
||||
images:
|
||||
- name: registry.bstein.dev/bstein/hermes-agent
|
||||
digest: sha256:37ebf720c783ae908a602916ffccf88d43d205a157957f5dc4b487867aee45e7
|
||||
52
services/hermes-scm-broker/networkpolicy.yaml
Normal file
52
services/hermes-scm-broker/networkpolicy.yaml
Normal file
@ -0,0 +1,52 @@
|
||||
# services/hermes-scm-broker/networkpolicy.yaml
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: hermes-scm-broker-isolation
|
||||
namespace: hermes-scm
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app: hermes-scm-broker
|
||||
policyTypes: [Ingress, Egress]
|
||||
ingress:
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: hermes
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app: hermes-agent
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: hermes
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app: hermes-execution-mediator
|
||||
ports:
|
||||
- {protocol: TCP, port: 9081}
|
||||
egress:
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: kube-system
|
||||
podSelector:
|
||||
matchLabels:
|
||||
k8s-app: kube-dns
|
||||
ports:
|
||||
- {protocol: UDP, port: 53}
|
||||
- {protocol: TCP, port: 53}
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: vault
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app: vault
|
||||
ports:
|
||||
- {protocol: TCP, port: 8200}
|
||||
- to:
|
||||
- ipBlock:
|
||||
cidr: 192.168.22.9/32
|
||||
ports:
|
||||
- {protocol: TCP, port: 443}
|
||||
13
services/hermes-scm-broker/service.yaml
Normal file
13
services/hermes-scm-broker/service.yaml
Normal file
@ -0,0 +1,13 @@
|
||||
# services/hermes-scm-broker/service.yaml
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: hermes-scm-broker
|
||||
namespace: hermes-scm
|
||||
spec:
|
||||
selector:
|
||||
app: hermes-scm-broker
|
||||
ports:
|
||||
- name: http
|
||||
port: 9081
|
||||
targetPort: http
|
||||
5
services/hermes-scm-namespace/kustomization.yaml
Normal file
5
services/hermes-scm-namespace/kustomization.yaml
Normal file
@ -0,0 +1,5 @@
|
||||
# services/hermes-scm-namespace/kustomization.yaml
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- namespace.yaml
|
||||
14
services/hermes-scm-namespace/namespace.yaml
Normal file
14
services/hermes-scm-namespace/namespace.yaml
Normal file
@ -0,0 +1,14 @@
|
||||
# services/hermes-scm-namespace/namespace.yaml
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: hermes-scm
|
||||
labels:
|
||||
app.kubernetes.io/name: hermes-scm-broker
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: hermes-scm-broker
|
||||
namespace: hermes-scm
|
||||
automountServiceAccountToken: true
|
||||
@ -102,6 +102,110 @@ console tails and named artifact contents in its deterministic bundle. A report
|
||||
must say `retained Ariadne evidence` when that fallback is used; it must not
|
||||
pretend direct Jenkins access succeeded.
|
||||
|
||||
## Publishing an agent image after review
|
||||
|
||||
The `hermes-agent-image` Jenkins job is the only supported agent image builder.
|
||||
It runs daemonless Kaniko v1.23.2 under the unbound `hermes-image-builder`
|
||||
ServiceAccount without a service-account token, host socket,
|
||||
privileged container, or writable Git credential. It accepts only an exact
|
||||
40-character revision that is both the checked-out commit and current
|
||||
`atlas/titan-iac` `main`, so a human must merge the source PR first.
|
||||
|
||||
The build pod requires an ARM64 worker and gives `hardware=rpi5` the maximum
|
||||
scheduler preference. Healthy rpi4 workers remain valid fallback capacity when
|
||||
the rpi5 pool cannot fit the full request. Cordoned, unhealthy, amd64, and
|
||||
reserved last-resort nodes remain excluded by the required placement boundary.
|
||||
|
||||
Kaniko runs as UID 0 because it must unpack an image root filesystem and enter
|
||||
that filesystem for Dockerfile `RUN` instructions. Its capability set is the
|
||||
minimum proven by an exact ARM64 no-push build of this Dockerfile:
|
||||
`CHOWN`, `FOWNER`, `DAC_OVERRIDE`, `SETGID`, and `SETUID`. `SETGID` is required
|
||||
because pinned Kaniko applies the base image's supplementary group list before
|
||||
each Dockerfile `RUN`; `SETUID` lets apt drop privileges to its `_apt` account
|
||||
while downloading package indexes. It still has no privilege
|
||||
escalation, service-account token, host path, daemon socket, or Docker/BuildKit
|
||||
TCP endpoint and uses the runtime-default seccomp profile. This is residual
|
||||
root-in-the-build-pod risk, bounded to disposable `emptyDir` storage and a
|
||||
human-reviewed `main` revision; Dockerfile changes require the same scrutiny as
|
||||
executable cluster code.
|
||||
|
||||
Pinned Kaniko v1.23.2 parses Dockerfile `RUN` heredocs but does not materialize
|
||||
their inline files when executing a command. The Jenkins lane therefore enables
|
||||
a bounded compatibility replay after copying the reviewed Dockerfile and runner
|
||||
from the same checked-out commit. Before every replay, the runner inventories
|
||||
the entire Dockerfile as logical instructions, including split opcodes,
|
||||
operators, and either Dockerfile escape character. It rejects every heredoc
|
||||
form outside the exact nine Node/Python blocks before launching an interpreter.
|
||||
Each accepted body is then replayed immediately after its corresponding Kaniko
|
||||
`RUN`, preserving source order before any dependent regression. Docker and
|
||||
BuildKit keep their native behavior because the compatibility argument defaults
|
||||
off. The final TypeScript build, Python compile, and source assertions remain
|
||||
mandatory, so an omitted or drifted replay fails before any image can pass
|
||||
release verification.
|
||||
|
||||
From agent.hermes, trigger that one fixed job with:
|
||||
|
||||
```sh
|
||||
jenkins_image_build_trigger.py '<reviewed full main commit>'
|
||||
```
|
||||
|
||||
The helper has no general Jenkins credential or caller-selectable job name. Its
|
||||
Vault-projected token is bound by Jenkins only to `hermes-agent-image`. The job
|
||||
also requires its fixed publish confirmation. Each run claims the unique tag
|
||||
`git-<reviewed-sha>-build-<jenkins-build-number>` and refuses to overwrite an
|
||||
existing tag. It cross-checks Kaniko's digest and tagged-image evidence, then
|
||||
independently reads the pushed tag from Harbor before archiving a JSON record
|
||||
and Flux digest patch. A successful post-condition then revalidates the exact
|
||||
six-file evidence set and archives those six explicit paths with empty archives
|
||||
forbidden. Apply that patch on a new branch and submit it for human review; the
|
||||
build never changes Git, reconciles Flux, or deploys by itself.
|
||||
The helper posts only to the fixed HTTPS Build Token Root endpoint and accepts
|
||||
only its real queue responses: HTTP 201 or a non-followed HTTP 303 with an exact
|
||||
same-origin `/queue/item/<number>/` location. An unauthenticated request is
|
||||
denied, and the token cannot select, configure, read, or administer another
|
||||
Jenkins job. The pipeline independently rejects any commit that is not the
|
||||
current `origin/main`, preserving the human merge/review boundary.
|
||||
|
||||
Harbor independently enforces an enabled immutable-tag rule scoped to only the
|
||||
`bstein/hermes-agent` repository and `git-*-build-*` tags. A second manifest PUT,
|
||||
retag, or deletion is rejected by Harbor even if a caller bypasses the Jenkins
|
||||
preflight. A revisioned, Flux-tracked policy Job creates or verifies that exact
|
||||
rule with a runtime-only Vault credential; it refuses to alter a conflicting
|
||||
rule and retries only explicit transport/readiness failures while Harbor
|
||||
starts. The same Job grants the existing Jenkins publisher only
|
||||
`immutable-tag:list` on `bstein`, preserving its other project scopes and never
|
||||
calling Harbor's separate secret-rotation endpoint. The pipeline uses that
|
||||
read-only permission to require the exact enabled rule before Kaniko starts.
|
||||
That check also makes an already-running Jenkins controller fail closed during
|
||||
rollout; Jenkins cannot become Ready on the new revision before the policy Job
|
||||
succeeds.
|
||||
|
||||
The tracked rollout order is deliberate: the revisioned Vault role Job must
|
||||
complete before the Vault Kustomization becomes Ready. A separate Flux
|
||||
Kustomization then runs the seed Job under the dedicated
|
||||
`hermes-jenkins-token-seed` identity, which can only create/read the exact
|
||||
`kv/data/atlas/hermes/developer-jenkins` path and request random bytes. Jenkins
|
||||
depends on both that seed and the Harbor policy Job; Hermes depends on Jenkins.
|
||||
The seeder uses KV-v2 CAS create-only semantics. It never changes an existing
|
||||
token, never fills a missing field in an existing secret, and fails closed when
|
||||
it cannot distinguish absence from a read error. The recurring Vault
|
||||
configuration job reconciles the same narrow role and policy.
|
||||
|
||||
Rotate this fixed-job token only as a coordinated operator action:
|
||||
|
||||
1. Stop new `hermes-agent-image` triggers and wait for its queue and executor to
|
||||
drain.
|
||||
2. Read the current KV-v2 metadata version for
|
||||
`kv/atlas/hermes/developer-jenkins`.
|
||||
3. Generate a fresh value from Vault and patch only `build_token` with
|
||||
`vault kv patch -cas=<current-version>`. Keep the value in a mode-0600
|
||||
temporary file or standard input, never a command argument or log.
|
||||
4. Roll Jenkins first and wait until it is Ready, then roll `hermes-agent` and
|
||||
wait until it is Ready. Do not resume triggers between those two consumers.
|
||||
5. Run one reviewed-main canary. If rollback is required, repeat the CAS patch
|
||||
with the previous value as another coordinated rotation; never delete the
|
||||
secret to make the seeder recreate it.
|
||||
|
||||
## The actual supervised triage algorithm
|
||||
|
||||
1. Classify the request as test/build triage, service health, or alert tuning.
|
||||
@ -257,6 +361,60 @@ Use this short explanation:
|
||||
leading different real incidents, improving the skill after failures, and
|
||||
teaching the architecture without prompts.
|
||||
|
||||
## Distributed execution-pool rollout boundary
|
||||
|
||||
- Do not reconcile the pool until the approved credential owner has provisioned
|
||||
six distinct Vault fields: `execution_worker_{0,1,2}_claude_credentials_json`
|
||||
and `execution_worker_{0,1,2}_codex_auth_json`. Each ordinal needs an
|
||||
independent account or refresh-token lineage; copying one rotating refresh
|
||||
token into multiple fields recreates the lost-update failure this design
|
||||
prevents.
|
||||
- Worker credential files live on separate ordinal-owned RWO `provider-access`
|
||||
claims. Provider refresh updates these durable private copies across Pod
|
||||
restarts; they are never synchronized back to Vault. Rotate one bootstrap
|
||||
credential at a time through the reviewed Vault workflow and reinitialize
|
||||
only that ordinal after human approval.
|
||||
- Existing task worktrees remain on the single local owner lane. Only tasks
|
||||
without `workspace_path` enter the distributed pool, where each ordinal owns
|
||||
one RWO checkout. Repository and base-branch identity comes from the canonical
|
||||
board registry rather than task-supplied metadata.
|
||||
- Model Pods have no pool key, broker mount, broker egress, or Kubernetes token.
|
||||
Ordinal mediator Deployments are separate network identities that co-mount the
|
||||
matching workspace PVC; they alone authenticate exact-run results and reach the
|
||||
PR14 SCM broker.
|
||||
- The ordinal workspace claim is `ReadWriteMany` because the worker and its
|
||||
mediator both mount it. With `ReadWriteOnce` a drain or preemption that moved
|
||||
only one of them left the other holding the attachment on the old node, and
|
||||
the ordinal deadlocked on Multi-Attach until an operator deleted a Pod.
|
||||
Colocation is now a scheduling preference and both Pods run at `scavenger`
|
||||
priority, so either can be evicted and rescheduled on its own. There is
|
||||
deliberately no PodDisruptionBudget: this is a best-effort pool and a budget
|
||||
would impede node drains. Claims only one Pod mounts stay `ReadWriteOnce`.
|
||||
- `lease_failed` is a *retryable* durable state, not a terminal one. It means the
|
||||
ordinal was released but Kanban has not confirmed the outcome, so every
|
||||
maintenance pass retries it. Only `finalized` and `stale` are terminal, and
|
||||
only terminal rows are garbage-collected, so no run is dropped before the pool
|
||||
knows what happened to it. If the log keeps reporting a deferred lease park for
|
||||
the same run, Kanban is refusing an exact-run park on a task it still reports
|
||||
as running: inspect that task rather than deleting the row.
|
||||
- The broker accepts branch creation only. A pool retry that adds commits
|
||||
therefore publishes a new `<branch>-attempt-<n>` (or, failing that,
|
||||
`<branch>-<head12>`) ref in the same reviewed namespace instead of updating the
|
||||
ref a previous attempt published, and adopts an existing ref that already
|
||||
points at the exact head rather than pushing again. Expect one draft per
|
||||
published ref; each is still human-reviewed. A submission that cannot land
|
||||
blocks the run with the reason and leaves the commits on the ordinal's
|
||||
workspace, so prior work is recoverable.
|
||||
- Provider CLIs install once per pinned version onto a durable per-ordinal
|
||||
`tools` claim, re-verified against the real binaries so a pruned cache
|
||||
reinstalls. The pool is intentionally absent from the `hermes` Kustomization's
|
||||
`healthChecks`: gating that 10m window on a best-effort pool would stall
|
||||
`hermes-chat` and `hermes-observer-bindings`, which `dependsOn: hermes`.
|
||||
- A hashed execution-pool ConfigMap, protocol-version readiness checks, and
|
||||
versioned SCM boundary name make code/config changes controlled rollouts. A
|
||||
rollout is still a human-reviewed operation; this repository change does not
|
||||
reconcile or deploy it.
|
||||
|
||||
## Your shortest path to fluency
|
||||
|
||||
1. Explain the request diagram without looking.
|
||||
|
||||
@ -88,6 +88,14 @@ data:
|
||||
auto_decompose: true
|
||||
auto_decompose_per_tick: 2
|
||||
dispatch_stale_timeout_seconds: 14400
|
||||
# Steady-state quota-aware routing for the direct CLI lane: below this
|
||||
# remaining-percent a provider stops receiving NEW cli-auto work while
|
||||
# active work finishes; lane-observed capacity/auth failures exclude a
|
||||
# provider until the cooldown elapses, its quota window resets, or a
|
||||
# fresh success is recorded.
|
||||
provider_quota_min_remaining_percent: 15
|
||||
provider_capacity_cooldown_seconds: 300
|
||||
provider_auth_cooldown_seconds: 3600
|
||||
|
||||
model_catalog:
|
||||
enabled: true
|
||||
@ -189,9 +197,9 @@ data:
|
||||
not replace these coordinator-wide rules. Never call `kanban_show` without
|
||||
a known, non-empty task ID. Ad-hoc inspection and acceptance checks do not
|
||||
need a synthetic Kanban lookup, and must load a skill only when its workflow
|
||||
materially applies. Atlas HTTPS Git authentication is already supplied by
|
||||
the runtime-only `GIT_ASKPASS`; use it without reading or exposing the
|
||||
credential. Coordinator guidance lives at
|
||||
materially applies. Atlas Git access is supplied by the isolated SCM
|
||||
broker; no repository token is present in this pod. Use only configured
|
||||
broker remotes and clients. Coordinator guidance lives at
|
||||
`/opt/data/workspace/AGENTS.md` when more detail is needed.
|
||||
|
||||
The Jetson classifier is mandatory for AUTO selection. Switchyard may use
|
||||
@ -295,70 +303,75 @@ data:
|
||||
objective's difficulty warrants it; otherwise synthesize at the original
|
||||
effort.
|
||||
|
||||
This owner-only pod has cluster-admin access across Atlas, including logs,
|
||||
Secrets, exec, port-forwarding, rollout operations, and Flux reconciliation.
|
||||
Prefer the titan-iac Git/Flux workflow for every durable cluster change;
|
||||
direct operations are available for explicit operator requests, incident
|
||||
recovery, and verification, and must be followed by a matching source-of-
|
||||
truth change when they alter desired state. Never expose credentials in
|
||||
chat or logs. Triage belongs at triage.hermes.bstein.dev.
|
||||
This owner-only pod has scoped read-only Kubernetes diagnostics across
|
||||
Atlas, including resource status, events, logs, and Flux/Helm evidence. It
|
||||
cannot read Secrets, exec or attach to pods, create service-account tokens,
|
||||
mutate workloads or RBAC, or reconcile Flux. Put every durable cluster
|
||||
change on a reviewed titan-iac branch. Never expose credentials in chat or
|
||||
logs. Triage belongs at triage.hermes.bstein.dev.
|
||||
|
||||
## Atlas engineering access
|
||||
|
||||
The Atlas organization has private visibility. Repository visibility is
|
||||
preserved per project and may be public or private; do not infer a
|
||||
repository's visibility from the organization setting. Repositories are
|
||||
canonical at `https://scm.bstein.dev/atlas/<repo>.git`. HTTPS Git
|
||||
authentication is already supplied through `GIT_ASKPASS`. Verify the remote
|
||||
canonical at `https://scm.bstein.dev/atlas/<repo>.git`; worker Git traffic
|
||||
uses `http://hermes-scm-broker.hermes-scm.svc.cluster.local:9081/git/atlas/<repo>.git`.
|
||||
Verify the broker remote
|
||||
and cleanly separate pre-existing changes, create a task branch, run the
|
||||
repository's tests, use `git push --dry-run` when proving access, and push a
|
||||
real branch only when the requested implementation is review-ready. Never
|
||||
force-push.
|
||||
|
||||
`scm.bstein.dev` is Gitea, not GitHub. Never load or follow a GitHub/`gh`
|
||||
skill for an Atlas remote, and do not interpret an unauthenticated Gitea
|
||||
HTTP 404 as a missing private repository. Use authenticated Git for clone,
|
||||
fetch, and push. For pull-request metadata, comments, and merges, call
|
||||
`/opt/coordinator/gitea_api.py METHOD /api/v1/...`; it injects the runtime
|
||||
Vault token without exposing it to the command line, environment, output,
|
||||
or transcript. Before a consequential merge, independently verify the
|
||||
exact base/head ancestry and diff, run the relevant tests, and confirm the
|
||||
candidate Jenkins result. Prefer the internal endpoint in
|
||||
`JENKINS_BASE_URL`; when Jenkins API authorization prevents a read, use the
|
||||
existing cluster access to inspect the controller's job/build files and
|
||||
logs rather than guessing a public hostname. The preferred read-only path
|
||||
is `/opt/coordinator/jenkins_build_evidence.py JOB [--branch BRANCH]
|
||||
[--commit SHA] --wait`; it distinguishes a genuinely terminal build from
|
||||
nested Jenkins execution metadata and returns bounded JSON/log evidence.
|
||||
With a branch, the helper also tries the conventional `JOB-branches`
|
||||
multibranch name. For JSON Gitea mutations, prefer repeatable shell-safe
|
||||
`--field KEY=VALUE` arguments over hand-quoted JSON.
|
||||
After merging, observe the default-branch build to a terminal result and
|
||||
report exact commit, build, and test evidence.
|
||||
`scm.bstein.dev` is Forgejo/Gitea, not GitHub. Never load or follow a
|
||||
GitHub/`gh` skill for an Atlas remote, and do not interpret an
|
||||
unauthenticated Gitea HTTP 404 as a missing private repository. Use
|
||||
brokered Git for clone, fetch, and creation of a new namespaced feature
|
||||
branch. Existing-ref updates, protected refs, deletion, and force-push are
|
||||
rejected. The broker inflates and scans every pushed object, so thin
|
||||
packs are rejected; always push with `git push --no-thin` so the pack
|
||||
is self-contained. For bounded
|
||||
repository/pull-request evidence or to create a review-ready draft PR, load
|
||||
`$manage-atlas-pull-requests` and use `/opt/scm/gitea_api.py`. The
|
||||
client carries no repository credential; a separate least-authority
|
||||
workload performs the allowed operation. It permits only Atlas-scoped
|
||||
metadata reads and verified same-repository draft creation. Updates, merge,
|
||||
approve, close, delete, comments, repository administration, and force-push
|
||||
are unavailable; never bypass the client with raw HTTP. Leave every PR
|
||||
unmerged for Brad's review. Independently
|
||||
verify the exact base/head ancestry and diff, run the relevant tests, and
|
||||
confirm the candidate Jenkins result before presenting a handoff. Prefer
|
||||
the internal endpoint in
|
||||
`JENKINS_BASE_URL`; when Jenkins API authorization prevents a read, use
|
||||
Kubernetes pod logs and status evidence without exec or secret access.
|
||||
Do not bypass the boundary to inspect controller files. If Brad separately
|
||||
reports that he merged the PR, observe
|
||||
the default-branch build to a terminal result and report exact commit,
|
||||
build, and test evidence.
|
||||
|
||||
The workspace already has a non-secret Hermes Git author identity. Do not
|
||||
use `git reset --hard`, even in a fresh clone; use a detached worktree or a
|
||||
clean branch switch when comparing revisions, and preserve any unexpected
|
||||
file as user state. Use the Gitea pull-request merge endpoint for an open
|
||||
PR so both Git history and PR state remain auditable. If an authorized
|
||||
manual merge already reached the base branch, reconcile the open PR with
|
||||
`Do=manually-merged` and its exact merge commit instead of leaving stale
|
||||
review state.
|
||||
file as user state. PR publication ends at the verified open draft; Brad
|
||||
owns review and merge authority.
|
||||
|
||||
The terminal PATH contains the pinned operator tools. Start cluster work
|
||||
with `kubectl config current-context`, read-only status/events/logs, and the
|
||||
relevant `titan-iac` manifests. Put durable desired-state changes on a
|
||||
reviewable `titan-iac` branch, validate Kustomize and client dry-run, then
|
||||
use Flux reconciliation after the tracked change is published. Direct
|
||||
`kubectl` mutations are for explicit incident recovery or ephemeral
|
||||
verification, not ordinary delivery.
|
||||
relevant `titan-iac` manifests. The broker namespace is deliberately
|
||||
excluded from agent RBAC. Kubernetes cannot express a deny on one namespace
|
||||
inside an all-namespace list, so enumerate namespaces and run namespaced
|
||||
diagnostics instead of relying on `kubectl ... --all-namespaces`. Put
|
||||
durable desired-state changes on a
|
||||
reviewable `titan-iac` branch and validate Kustomize and client dry-run.
|
||||
Brad owns merge and Flux reconciliation. Direct `kubectl` mutation is not
|
||||
available to the agent; hand an explicit incident mutation back to Brad.
|
||||
|
||||
Node SSH uses a dedicated audited Hermes identity: `ssh titan-04` (or any
|
||||
current Kubernetes node name). Host keys are pinned and password fallback
|
||||
is disabled. Use SSH only for host-level evidence or repairs that cannot be
|
||||
performed through Kubernetes. Read first, identify the exact node and
|
||||
impact, avoid fleet-wide destructive commands, and reflect persistent host
|
||||
configuration in the appropriate tracked provisioning source.
|
||||
Node SSH uses the dedicated locked-password `hermes-agent` OS account:
|
||||
`ssh titan-04` (or any current Kubernetes node name). Host keys are pinned,
|
||||
password fallback is disabled, and the account has no sudo, disk, runtime,
|
||||
or Kubernetes-storage authority. Use it for unprivileged host evidence;
|
||||
hand privileged host repair back to Brad and reflect persistent changes in
|
||||
the tracked provisioning source.
|
||||
|
||||
Treat every credential, SSH identity, host-trust record, and
|
||||
credential-bearing access-client state as runtime-only Vault data. Keep
|
||||
@ -395,7 +408,7 @@ data:
|
||||
<low|medium|high|xhigh> [model]` for a persistent override. The first native
|
||||
Codex worker requires one device-code login; refreshed provider credentials
|
||||
persist through Vault while client caches remain disposable. The owner
|
||||
workspace includes cluster-admin Kubernetes
|
||||
access plus `kubectl`, `flux`, `helm`, `kustomize`, `vault`, `sops`, `age`,
|
||||
workspace includes scoped read-only Kubernetes diagnostics plus `kubectl`,
|
||||
`flux`, `helm`, `kustomize`, `vault`, `sops`, `age`,
|
||||
`terraform`, `k9s`, `jq`, `yq`, `gh`, Git, SSH, Python, Node, the
|
||||
browser/computer tools, and the native provider CLIs.
|
||||
|
||||
@ -25,7 +25,7 @@ spec:
|
||||
ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers
|
||||
ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback
|
||||
ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available
|
||||
ai.bstein.dev/config-rev: "20260817-ai-usage-exporter-split-v2"
|
||||
ai.bstein.dev/config-rev: "20260818-ai-usage-exporter-split-v3"
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/path: /metrics
|
||||
prometheus.io/port: "9010"
|
||||
@ -46,15 +46,10 @@ spec:
|
||||
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
|
||||
{{ .Data.data.codex_auth_json }}
|
||||
{{- end }}
|
||||
vault.hashicorp.com/agent-inject-secret-gitea-token: kv/data/atlas/hermes/developer-gitea
|
||||
vault.hashicorp.com/agent-inject-template-gitea-token: |
|
||||
{{- with secret "kv/data/atlas/hermes/developer-gitea" -}}
|
||||
{{ .Data.data.token }}
|
||||
{{- end }}
|
||||
vault.hashicorp.com/agent-inject-secret-gitea-username: kv/data/atlas/hermes/developer-gitea
|
||||
vault.hashicorp.com/agent-inject-template-gitea-username: |
|
||||
{{- with secret "kv/data/atlas/hermes/developer-gitea" -}}
|
||||
{{ .Data.data.username }}
|
||||
vault.hashicorp.com/agent-inject-secret-jenkins-image-build-token: kv/data/atlas/hermes/developer-jenkins
|
||||
vault.hashicorp.com/agent-inject-template-jenkins-image-build-token: |
|
||||
{{- with secret "kv/data/atlas/hermes/developer-jenkins" -}}
|
||||
{{ .Data.data.build_token }}
|
||||
{{- end }}
|
||||
vault.hashicorp.com/agent-inject-secret-node-ssh-private-key: kv/data/atlas/hermes/developer-ssh
|
||||
vault.hashicorp.com/agent-inject-template-node-ssh-private-key: |
|
||||
@ -186,9 +181,8 @@ spec:
|
||||
chmod 0600 "${profile_env}"
|
||||
chown 10000:10000 "${profile_env}"
|
||||
done
|
||||
upsert_env GIT_ASKPASS /opt/coordinator/gitea_askpass.sh
|
||||
upsert_env GIT_TERMINAL_PROMPT 0
|
||||
upsert_env GITEA_BASE_URL https://scm.bstein.dev
|
||||
upsert_env HERMES_SCM_BROKER_URL http://hermes-scm-broker.hermes-scm.svc.cluster.local:9081
|
||||
upsert_env JENKINS_BASE_URL http://jenkins.jenkins.svc.cluster.local:8080
|
||||
upsert_env ARIADNE_BASE_URL http://ariadne.maintenance.svc.cluster.local
|
||||
upsert_env VICTORIA_METRICS_URL http://victoria-metrics-single-server.monitoring.svc.cluster.local:8428
|
||||
@ -663,6 +657,8 @@ spec:
|
||||
- {name: subprocess-secret-patch, mountPath: /opt/hermes/tools/process_registry.py, subPath: process_registry.py}
|
||||
- {name: kubeconfig, mountPath: /opt/data/home/.kube/config, subPath: config, readOnly: true}
|
||||
- {name: auto-router-plugin, mountPath: /opt/data/plugins/auto-router, readOnly: true}
|
||||
- {name: atlas-pr-skill, mountPath: /opt/data/workspace/skills/manage-atlas-pull-requests, readOnly: true}
|
||||
- {name: scm-boundary, mountPath: /opt/scm, readOnly: true}
|
||||
- {name: routing-catalog, mountPath: /routing-catalog, readOnly: true}
|
||||
- {name: tmp, mountPath: /tmp}
|
||||
startupProbe:
|
||||
@ -826,6 +822,8 @@ spec:
|
||||
- {name: subprocess-secret-patch, mountPath: /opt/hermes/tools/process_registry.py, subPath: process_registry.py}
|
||||
- {name: kubeconfig, mountPath: /opt/data/home/.kube/config, subPath: config, readOnly: true}
|
||||
- {name: auto-router-plugin, mountPath: /opt/data/plugins/auto-router, readOnly: true}
|
||||
- {name: atlas-pr-skill, mountPath: /opt/data/workspace/skills/manage-atlas-pull-requests, readOnly: true}
|
||||
- {name: scm-boundary, mountPath: /opt/scm, readOnly: true}
|
||||
- {name: routing-catalog, mountPath: /routing-catalog, readOnly: true}
|
||||
- {name: tmp, mountPath: /tmp}
|
||||
- {name: ttyd-index, mountPath: /ttyd-index, readOnly: true}
|
||||
@ -866,6 +864,8 @@ spec:
|
||||
. /opt/data/.env
|
||||
set +a
|
||||
exec /opt/hermes/.venv/bin/python /opt/coordinator/cli_lane_runner.py
|
||||
ports:
|
||||
- {name: lane-metrics, containerPort: 9011, protocol: TCP}
|
||||
env:
|
||||
- {name: HERMES_HOME, value: /opt/data}
|
||||
- {name: HERMES_AUTH_FILE, value: /runtime-access/hermes-auth.json}
|
||||
@ -875,6 +875,9 @@ spec:
|
||||
- {name: KUBECONFIG, value: /opt/data/home/.kube/config}
|
||||
- {name: PYTHONPATH, value: /opt/hermes}
|
||||
- {name: HERMES_CLI_LANE_CONCURRENCY, value: "2"}
|
||||
- {name: HERMES_CLI_HEALTH_MAX_AGE_SECONDS, value: "60"}
|
||||
- {name: HERMES_CLI_LANE_METRICS_PORT, value: "9011"}
|
||||
- {name: HERMES_CLI_QUOTA_METRICS_URL, value: "http://127.0.0.1:9010/metrics"}
|
||||
- {name: HERMES_AUTO_ROUTER_PROFILE, value: agent}
|
||||
- {name: PATH, value: /opt/coordinator:/opt/data/tools/bin:/opt/data/home/.local/bin:/opt/hermes/.venv/bin:/usr/local/bin:/usr/bin:/bin}
|
||||
securityContext:
|
||||
@ -894,7 +897,18 @@ spec:
|
||||
- {name: auth-patch, mountPath: /opt/hermes/hermes_cli/auth.py, subPath: auth.py}
|
||||
- {name: kubeconfig, mountPath: /opt/data/home/.kube/config, subPath: config, readOnly: true}
|
||||
- {name: auto-router-plugin, mountPath: /opt/data/plugins/auto-router, readOnly: true}
|
||||
- {name: atlas-pr-skill, mountPath: /opt/data/workspace/skills/manage-atlas-pull-requests, readOnly: true}
|
||||
- {name: scm-boundary, mountPath: /opt/scm, readOnly: true}
|
||||
- {name: tmp, mountPath: /tmp}
|
||||
readinessProbe:
|
||||
exec:
|
||||
command:
|
||||
- /opt/hermes/.venv/bin/python
|
||||
- /opt/coordinator/cli_lane_capabilities.py
|
||||
initialDelaySeconds: 2
|
||||
periodSeconds: 5
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 3
|
||||
resources:
|
||||
requests: {cpu: 100m, memory: 256Mi}
|
||||
limits: {cpu: "2", memory: 6Gi}
|
||||
@ -1188,6 +1202,16 @@ spec:
|
||||
- {key: dashboard-api.py, path: dashboard/plugin_api.py}
|
||||
- {key: dashboard-index.js, path: dashboard/dist/index.js}
|
||||
- {key: dashboard-style.css, path: dashboard/dist/style.css}
|
||||
- name: atlas-pr-skill
|
||||
configMap:
|
||||
name: hermes-atlas-pr-skill
|
||||
items:
|
||||
- {key: SKILL.md, path: SKILL.md}
|
||||
- {key: openai.yaml, path: agents/openai.yaml}
|
||||
- name: scm-boundary
|
||||
configMap:
|
||||
name: hermes-scm-boundary-v2
|
||||
defaultMode: 0555
|
||||
- name: image-policy
|
||||
configMap:
|
||||
name: hermes-image-policy
|
||||
|
||||
@ -11,9 +11,9 @@ users:
|
||||
user:
|
||||
tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
|
||||
contexts:
|
||||
- name: atlas-owner
|
||||
- name: atlas-observer
|
||||
context:
|
||||
cluster: atlas
|
||||
user: hermes-agent
|
||||
namespace: default
|
||||
current-context: atlas-owner
|
||||
current-context: atlas-observer
|
||||
|
||||
@ -1,16 +0,0 @@
|
||||
# services/hermes/agent-rbac.yaml
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: hermes-agent-cluster-admin
|
||||
labels:
|
||||
app.kubernetes.io/name: hermes-agent
|
||||
app.kubernetes.io/part-of: hermes
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: hermes-agent
|
||||
namespace: hermes
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: cluster-admin
|
||||
66
services/hermes/execution-coordinator-patch.yaml
Normal file
66
services/hermes/execution-coordinator-patch.yaml
Normal file
@ -0,0 +1,66 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: hermes-agent
|
||||
namespace: hermes
|
||||
spec:
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
vault.hashicorp.com/agent-inject-secret-execution-pool-key: kv/data/atlas/hermes/agent-tokens
|
||||
vault.hashicorp.com/agent-inject-perms-execution-pool-key: "0600"
|
||||
vault.hashicorp.com/agent-inject-template-execution-pool-key: |
|
||||
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
|
||||
{{ printf "hermes-execution-pool-root-v2:%s" .Data.data.agent_api_key | sha256sum }}
|
||||
{{- end }}
|
||||
spec:
|
||||
containers:
|
||||
- name: cli-lane-runner
|
||||
env:
|
||||
- {name: HERMES_CLI_LANE_OWNED_WORKSPACES_ONLY, value: "true"}
|
||||
- {name: HERMES_CLI_LANE_CONCURRENCY, value: "1"}
|
||||
- name: execution-pool-coordinator
|
||||
image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
|
||||
imagePullPolicy: IfNotPresent
|
||||
command: [/opt/hermes/.venv/bin/python, /opt/coordinator/execution_pool_coordinator.py]
|
||||
env:
|
||||
- {name: HERMES_HOME, value: /opt/data}
|
||||
- {name: HOME, value: /opt/data/home}
|
||||
- {name: PYTHONPATH, value: /opt/hermes}
|
||||
- {name: HERMES_EXECUTION_POOL_KEY_FILE, value: /pool-access/execution-pool-key}
|
||||
- {name: PATH, value: /opt/coordinator:/opt/hermes/.venv/bin:/usr/local/bin:/usr/bin:/bin}
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities: {drop: [ALL]}
|
||||
readOnlyRootFilesystem: true
|
||||
runAsNonRoot: true
|
||||
runAsUser: 10000
|
||||
runAsGroup: 10000
|
||||
seccompProfile: {type: RuntimeDefault}
|
||||
volumeMounts:
|
||||
- {name: home, mountPath: /opt/data}
|
||||
- {name: runtime-access, mountPath: /pool-access/execution-pool-key, subPath: execution-pool-key, readOnly: true}
|
||||
- {name: execution-pool-code, mountPath: /opt/coordinator, readOnly: true}
|
||||
- {name: auth-patch, mountPath: /opt/hermes/hermes_cli/auth.py, subPath: auth.py}
|
||||
- {name: tmp, mountPath: /tmp}
|
||||
ports:
|
||||
- {name: execution-pool, containerPort: 9007, protocol: TCP}
|
||||
startupProbe:
|
||||
httpGet: {path: /ready, port: execution-pool}
|
||||
periodSeconds: 5
|
||||
failureThreshold: 60
|
||||
readinessProbe:
|
||||
httpGet: {path: /ready, port: execution-pool}
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
httpGet: {path: /ready, port: execution-pool}
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 30
|
||||
resources:
|
||||
requests: {cpu: 50m, memory: 128Mi}
|
||||
limits: {cpu: 500m, memory: 512Mi}
|
||||
volumes:
|
||||
- name: execution-pool-code
|
||||
configMap:
|
||||
name: hermes-execution-pool
|
||||
defaultMode: 0555
|
||||
476
services/hermes/execution-mediator.yaml
Normal file
476
services/hermes/execution-mediator.yaml
Normal file
@ -0,0 +1,476 @@
|
||||
# Privileged HMAC and SCM mediation runs outside the model Pods. Each mediator is
|
||||
# colocated with exactly one ordinal's RWO workspace but has a distinct network
|
||||
# identity, private HMAC derivation, and durable integrity state.
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata: {name: hermes-execution-mediator-state-0, namespace: hermes}
|
||||
spec:
|
||||
accessModes: [ReadWriteOnce]
|
||||
storageClassName: astreae
|
||||
resources: {requests: {storage: 1Gi}}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata: {name: hermes-execution-mediator-state-1, namespace: hermes}
|
||||
spec:
|
||||
accessModes: [ReadWriteOnce]
|
||||
storageClassName: astreae
|
||||
resources: {requests: {storage: 1Gi}}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata: {name: hermes-execution-mediator-state-2, namespace: hermes}
|
||||
spec:
|
||||
accessModes: [ReadWriteOnce]
|
||||
storageClassName: astreae
|
||||
resources: {requests: {storage: 1Gi}}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata: {name: hermes-execution-mediator-0, namespace: hermes}
|
||||
spec:
|
||||
selector: {app: hermes-execution-mediator, pool-ordinal: "0"}
|
||||
ports: [{name: mediator, port: 9009, targetPort: mediator}]
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata: {name: hermes-execution-mediator-1, namespace: hermes}
|
||||
spec:
|
||||
selector: {app: hermes-execution-mediator, pool-ordinal: "1"}
|
||||
ports: [{name: mediator, port: 9009, targetPort: mediator}]
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata: {name: hermes-execution-mediator-2, namespace: hermes}
|
||||
spec:
|
||||
selector: {app: hermes-execution-mediator, pool-ordinal: "2"}
|
||||
ports: [{name: mediator, port: 9009, targetPort: mediator}]
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: hermes-execution-mediator-0
|
||||
namespace: hermes
|
||||
labels: {app: hermes-execution-mediator, pool-ordinal: "0"}
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy: {type: Recreate}
|
||||
selector:
|
||||
matchLabels: {app: hermes-execution-mediator, pool-ordinal: "0"}
|
||||
template:
|
||||
metadata:
|
||||
labels: {app: hermes-execution-mediator, pool-ordinal: "0"}
|
||||
annotations:
|
||||
ai.bstein.dev/config-rev: execution-pool-v2-mediated
|
||||
ai.bstein.dev/security-boundary: model-pod-has-no-hmac-or-scm-network-authority
|
||||
vault.hashicorp.com/agent-inject: "true"
|
||||
vault.hashicorp.com/role: hermes-execution-worker
|
||||
vault.hashicorp.com/agent-inject-containers: stage-mediator-access
|
||||
vault.hashicorp.com/agent-service-account-token-volume-name: vault-auth-token
|
||||
vault.hashicorp.com/agent-inject-secret-execution-pool-key: kv/data/atlas/hermes/agent-tokens
|
||||
vault.hashicorp.com/agent-inject-perms-execution-pool-key: "0600"
|
||||
vault.hashicorp.com/agent-inject-template-execution-pool-key: |
|
||||
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
|
||||
{{ printf "hermes-execution-pool-root-v2:%s" .Data.data.agent_api_key | sha256sum }}
|
||||
{{- end }}
|
||||
vault.hashicorp.com/agent-pre-populate-only: "true"
|
||||
vault.hashicorp.com/agent-init-first: "true"
|
||||
vault.hashicorp.com/agent-requests-cpu: 2m
|
||||
vault.hashicorp.com/agent-requests-mem: 16Mi
|
||||
vault.hashicorp.com/agent-limits-cpu: 100m
|
||||
vault.hashicorp.com/agent-limits-mem: 128Mi
|
||||
spec:
|
||||
serviceAccountName: hermes-execution-worker
|
||||
automountServiceAccountToken: false
|
||||
enableServiceLinks: false
|
||||
# Priority parity with the worker: a pressured node evicts the pair
|
||||
# symmetrically instead of moving only the lower-priority half, and the
|
||||
# pool never preempts anything else in the cluster.
|
||||
priorityClassName: scavenger
|
||||
securityContext:
|
||||
fsGroup: 10000
|
||||
fsGroupChangePolicy: OnRootMismatch
|
||||
seccompProfile: {type: RuntimeDefault}
|
||||
affinity:
|
||||
# Colocation with the worker is a throughput preference, not a
|
||||
# requirement. The shared workspace claim is ReadWriteMany, so this
|
||||
# mediator still schedules and serves when its worker is Pending or
|
||||
# placed elsewhere -- which is what makes a drain self-healing.
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- {key: kubernetes.io/arch, operator: In, values: [arm64]}
|
||||
- {key: node-role.kubernetes.io/worker, operator: In, values: ["true"]}
|
||||
- {key: kubernetes.io/hostname, operator: NotIn, values: [titan-04, titan-13, titan-14, titan-17, titan-18, titan-19, titan-22, titan-24]}
|
||||
podAffinity:
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
- weight: 100
|
||||
podAffinityTerm:
|
||||
labelSelector:
|
||||
matchLabels:
|
||||
statefulset.kubernetes.io/pod-name: hermes-execution-worker-0
|
||||
topologyKey: kubernetes.io/hostname
|
||||
initContainers:
|
||||
- name: stage-mediator-access
|
||||
image: registry.bstein.dev/bstein/hermes-agent
|
||||
command: [/opt/hermes/.venv/bin/python, /opt/coordinator/stage_runtime_access.py, execution-mediator]
|
||||
env:
|
||||
- {name: HERMES_WORKER_ORDINAL, value: "0"}
|
||||
- {name: HERMES_POOL_ACCESS_ROOT, value: /pool-access}
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
runAsUser: 0
|
||||
runAsGroup: 0
|
||||
seccompProfile: {type: RuntimeDefault}
|
||||
volumeMounts:
|
||||
- {name: pool-access, mountPath: /pool-access}
|
||||
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
|
||||
resources:
|
||||
requests: {cpu: 2m, memory: 16Mi}
|
||||
limits: {cpu: 100m, memory: 64Mi}
|
||||
containers:
|
||||
- name: execution-mediator
|
||||
image: registry.bstein.dev/bstein/hermes-agent
|
||||
command: [/opt/hermes/.venv/bin/python, /opt/coordinator/execution_pool_client.py]
|
||||
env:
|
||||
- {name: HERMES_WORKER_ORDINAL, value: "0"}
|
||||
- {name: HERMES_WORKER_ROOT, value: /workspace}
|
||||
- {name: HERMES_SCM_STATE_ROOT, value: /scm-state}
|
||||
- {name: HERMES_EXECUTION_POOL_KEY_FILE, value: /pool-access/execution-pool-key}
|
||||
- {name: PYTHONPATH, value: /opt/scm:/opt/coordinator:/opt/hermes}
|
||||
ports: [{name: mediator, containerPort: 9009, protocol: TCP}]
|
||||
startupProbe:
|
||||
httpGet: {path: /ready, port: mediator}
|
||||
periodSeconds: 5
|
||||
failureThreshold: 60
|
||||
readinessProbe:
|
||||
httpGet: {path: /ready, port: mediator}
|
||||
periodSeconds: 10
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities: {drop: [ALL]}
|
||||
readOnlyRootFilesystem: true
|
||||
runAsNonRoot: true
|
||||
runAsUser: 10000
|
||||
runAsGroup: 10000
|
||||
seccompProfile: {type: RuntimeDefault}
|
||||
volumeMounts:
|
||||
- {name: workspace, mountPath: /workspace}
|
||||
- {name: scm-state, mountPath: /scm-state}
|
||||
- {name: pool-access, mountPath: /pool-access, readOnly: true}
|
||||
- {name: scm-broker-client, mountPath: /opt/scm, readOnly: true}
|
||||
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
|
||||
- {name: tmp, mountPath: /tmp}
|
||||
resources:
|
||||
requests: {cpu: 2m, memory: 64Mi}
|
||||
limits: {cpu: 250m, memory: 256Mi}
|
||||
volumes:
|
||||
- name: workspace
|
||||
persistentVolumeClaim: {claimName: workspace-hermes-execution-worker-0}
|
||||
- name: scm-state
|
||||
persistentVolumeClaim: {claimName: hermes-execution-mediator-state-0}
|
||||
- name: pool-access
|
||||
emptyDir: {medium: Memory, sizeLimit: 1Mi}
|
||||
- name: scm-broker-client
|
||||
configMap: {name: hermes-scm-boundary-v2, defaultMode: 0555}
|
||||
- name: coordinator
|
||||
configMap: {name: hermes-execution-pool, defaultMode: 0555}
|
||||
- name: tmp
|
||||
emptyDir: {sizeLimit: 64Mi}
|
||||
- name: vault-auth-token
|
||||
projected:
|
||||
defaultMode: 0600
|
||||
sources:
|
||||
- serviceAccountToken: {audience: vault, expirationSeconds: 3600, path: token}
|
||||
- configMap:
|
||||
name: kube-root-ca.crt
|
||||
items: [{key: ca.crt, path: ca.crt}]
|
||||
- downwardAPI:
|
||||
items: [{path: namespace, fieldRef: {fieldPath: metadata.namespace}}]
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: hermes-execution-mediator-1
|
||||
namespace: hermes
|
||||
labels: {app: hermes-execution-mediator, pool-ordinal: "1"}
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy: {type: Recreate}
|
||||
selector:
|
||||
matchLabels: {app: hermes-execution-mediator, pool-ordinal: "1"}
|
||||
template:
|
||||
metadata:
|
||||
labels: {app: hermes-execution-mediator, pool-ordinal: "1"}
|
||||
annotations:
|
||||
ai.bstein.dev/config-rev: execution-pool-v2-mediated
|
||||
ai.bstein.dev/security-boundary: model-pod-has-no-hmac-or-scm-network-authority
|
||||
vault.hashicorp.com/agent-inject: "true"
|
||||
vault.hashicorp.com/role: hermes-execution-worker
|
||||
vault.hashicorp.com/agent-inject-containers: stage-mediator-access
|
||||
vault.hashicorp.com/agent-service-account-token-volume-name: vault-auth-token
|
||||
vault.hashicorp.com/agent-inject-secret-execution-pool-key: kv/data/atlas/hermes/agent-tokens
|
||||
vault.hashicorp.com/agent-inject-perms-execution-pool-key: "0600"
|
||||
vault.hashicorp.com/agent-inject-template-execution-pool-key: |
|
||||
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
|
||||
{{ printf "hermes-execution-pool-root-v2:%s" .Data.data.agent_api_key | sha256sum }}
|
||||
{{- end }}
|
||||
vault.hashicorp.com/agent-pre-populate-only: "true"
|
||||
vault.hashicorp.com/agent-init-first: "true"
|
||||
vault.hashicorp.com/agent-requests-cpu: 2m
|
||||
vault.hashicorp.com/agent-requests-mem: 16Mi
|
||||
vault.hashicorp.com/agent-limits-cpu: 100m
|
||||
vault.hashicorp.com/agent-limits-mem: 128Mi
|
||||
spec:
|
||||
serviceAccountName: hermes-execution-worker
|
||||
automountServiceAccountToken: false
|
||||
enableServiceLinks: false
|
||||
# Priority parity with the worker: a pressured node evicts the pair
|
||||
# symmetrically instead of moving only the lower-priority half, and the
|
||||
# pool never preempts anything else in the cluster.
|
||||
priorityClassName: scavenger
|
||||
securityContext:
|
||||
fsGroup: 10000
|
||||
fsGroupChangePolicy: OnRootMismatch
|
||||
seccompProfile: {type: RuntimeDefault}
|
||||
affinity:
|
||||
# Colocation with the worker is a throughput preference, not a
|
||||
# requirement. The shared workspace claim is ReadWriteMany, so this
|
||||
# mediator still schedules and serves when its worker is Pending or
|
||||
# placed elsewhere -- which is what makes a drain self-healing.
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- {key: kubernetes.io/arch, operator: In, values: [arm64]}
|
||||
- {key: node-role.kubernetes.io/worker, operator: In, values: ["true"]}
|
||||
- {key: kubernetes.io/hostname, operator: NotIn, values: [titan-04, titan-13, titan-14, titan-17, titan-18, titan-19, titan-22, titan-24]}
|
||||
podAffinity:
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
- weight: 100
|
||||
podAffinityTerm:
|
||||
labelSelector:
|
||||
matchLabels:
|
||||
statefulset.kubernetes.io/pod-name: hermes-execution-worker-1
|
||||
topologyKey: kubernetes.io/hostname
|
||||
initContainers:
|
||||
- name: stage-mediator-access
|
||||
image: registry.bstein.dev/bstein/hermes-agent
|
||||
command: [/opt/hermes/.venv/bin/python, /opt/coordinator/stage_runtime_access.py, execution-mediator]
|
||||
env:
|
||||
- {name: HERMES_WORKER_ORDINAL, value: "1"}
|
||||
- {name: HERMES_POOL_ACCESS_ROOT, value: /pool-access}
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
runAsUser: 0
|
||||
runAsGroup: 0
|
||||
seccompProfile: {type: RuntimeDefault}
|
||||
volumeMounts:
|
||||
- {name: pool-access, mountPath: /pool-access}
|
||||
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
|
||||
resources:
|
||||
requests: {cpu: 2m, memory: 16Mi}
|
||||
limits: {cpu: 100m, memory: 64Mi}
|
||||
containers:
|
||||
- name: execution-mediator
|
||||
image: registry.bstein.dev/bstein/hermes-agent
|
||||
command: [/opt/hermes/.venv/bin/python, /opt/coordinator/execution_pool_client.py]
|
||||
env:
|
||||
- {name: HERMES_WORKER_ORDINAL, value: "1"}
|
||||
- {name: HERMES_WORKER_ROOT, value: /workspace}
|
||||
- {name: HERMES_SCM_STATE_ROOT, value: /scm-state}
|
||||
- {name: HERMES_EXECUTION_POOL_KEY_FILE, value: /pool-access/execution-pool-key}
|
||||
- {name: PYTHONPATH, value: /opt/scm:/opt/coordinator:/opt/hermes}
|
||||
ports: [{name: mediator, containerPort: 9009, protocol: TCP}]
|
||||
startupProbe:
|
||||
httpGet: {path: /ready, port: mediator}
|
||||
periodSeconds: 5
|
||||
failureThreshold: 60
|
||||
readinessProbe:
|
||||
httpGet: {path: /ready, port: mediator}
|
||||
periodSeconds: 10
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities: {drop: [ALL]}
|
||||
readOnlyRootFilesystem: true
|
||||
runAsNonRoot: true
|
||||
runAsUser: 10000
|
||||
runAsGroup: 10000
|
||||
seccompProfile: {type: RuntimeDefault}
|
||||
volumeMounts:
|
||||
- {name: workspace, mountPath: /workspace}
|
||||
- {name: scm-state, mountPath: /scm-state}
|
||||
- {name: pool-access, mountPath: /pool-access, readOnly: true}
|
||||
- {name: scm-broker-client, mountPath: /opt/scm, readOnly: true}
|
||||
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
|
||||
- {name: tmp, mountPath: /tmp}
|
||||
resources:
|
||||
requests: {cpu: 2m, memory: 64Mi}
|
||||
limits: {cpu: 250m, memory: 256Mi}
|
||||
volumes:
|
||||
- name: workspace
|
||||
persistentVolumeClaim: {claimName: workspace-hermes-execution-worker-1}
|
||||
- name: scm-state
|
||||
persistentVolumeClaim: {claimName: hermes-execution-mediator-state-1}
|
||||
- name: pool-access
|
||||
emptyDir: {medium: Memory, sizeLimit: 1Mi}
|
||||
- name: scm-broker-client
|
||||
configMap: {name: hermes-scm-boundary-v2, defaultMode: 0555}
|
||||
- name: coordinator
|
||||
configMap: {name: hermes-execution-pool, defaultMode: 0555}
|
||||
- name: tmp
|
||||
emptyDir: {sizeLimit: 64Mi}
|
||||
- name: vault-auth-token
|
||||
projected:
|
||||
defaultMode: 0600
|
||||
sources:
|
||||
- serviceAccountToken: {audience: vault, expirationSeconds: 3600, path: token}
|
||||
- configMap:
|
||||
name: kube-root-ca.crt
|
||||
items: [{key: ca.crt, path: ca.crt}]
|
||||
- downwardAPI:
|
||||
items: [{path: namespace, fieldRef: {fieldPath: metadata.namespace}}]
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: hermes-execution-mediator-2
|
||||
namespace: hermes
|
||||
labels: {app: hermes-execution-mediator, pool-ordinal: "2"}
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy: {type: Recreate}
|
||||
selector:
|
||||
matchLabels: {app: hermes-execution-mediator, pool-ordinal: "2"}
|
||||
template:
|
||||
metadata:
|
||||
labels: {app: hermes-execution-mediator, pool-ordinal: "2"}
|
||||
annotations:
|
||||
ai.bstein.dev/config-rev: execution-pool-v2-mediated
|
||||
ai.bstein.dev/security-boundary: model-pod-has-no-hmac-or-scm-network-authority
|
||||
vault.hashicorp.com/agent-inject: "true"
|
||||
vault.hashicorp.com/role: hermes-execution-worker
|
||||
vault.hashicorp.com/agent-inject-containers: stage-mediator-access
|
||||
vault.hashicorp.com/agent-service-account-token-volume-name: vault-auth-token
|
||||
vault.hashicorp.com/agent-inject-secret-execution-pool-key: kv/data/atlas/hermes/agent-tokens
|
||||
vault.hashicorp.com/agent-inject-perms-execution-pool-key: "0600"
|
||||
vault.hashicorp.com/agent-inject-template-execution-pool-key: |
|
||||
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
|
||||
{{ printf "hermes-execution-pool-root-v2:%s" .Data.data.agent_api_key | sha256sum }}
|
||||
{{- end }}
|
||||
vault.hashicorp.com/agent-pre-populate-only: "true"
|
||||
vault.hashicorp.com/agent-init-first: "true"
|
||||
vault.hashicorp.com/agent-requests-cpu: 2m
|
||||
vault.hashicorp.com/agent-requests-mem: 16Mi
|
||||
vault.hashicorp.com/agent-limits-cpu: 100m
|
||||
vault.hashicorp.com/agent-limits-mem: 128Mi
|
||||
spec:
|
||||
serviceAccountName: hermes-execution-worker
|
||||
automountServiceAccountToken: false
|
||||
enableServiceLinks: false
|
||||
# Priority parity with the worker: a pressured node evicts the pair
|
||||
# symmetrically instead of moving only the lower-priority half, and the
|
||||
# pool never preempts anything else in the cluster.
|
||||
priorityClassName: scavenger
|
||||
securityContext:
|
||||
fsGroup: 10000
|
||||
fsGroupChangePolicy: OnRootMismatch
|
||||
seccompProfile: {type: RuntimeDefault}
|
||||
affinity:
|
||||
# Colocation with the worker is a throughput preference, not a
|
||||
# requirement. The shared workspace claim is ReadWriteMany, so this
|
||||
# mediator still schedules and serves when its worker is Pending or
|
||||
# placed elsewhere -- which is what makes a drain self-healing.
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- {key: kubernetes.io/arch, operator: In, values: [arm64]}
|
||||
- {key: node-role.kubernetes.io/worker, operator: In, values: ["true"]}
|
||||
- {key: kubernetes.io/hostname, operator: NotIn, values: [titan-04, titan-13, titan-14, titan-17, titan-18, titan-19, titan-22, titan-24]}
|
||||
podAffinity:
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
- weight: 100
|
||||
podAffinityTerm:
|
||||
labelSelector:
|
||||
matchLabels:
|
||||
statefulset.kubernetes.io/pod-name: hermes-execution-worker-2
|
||||
topologyKey: kubernetes.io/hostname
|
||||
initContainers:
|
||||
- name: stage-mediator-access
|
||||
image: registry.bstein.dev/bstein/hermes-agent
|
||||
command: [/opt/hermes/.venv/bin/python, /opt/coordinator/stage_runtime_access.py, execution-mediator]
|
||||
env:
|
||||
- {name: HERMES_WORKER_ORDINAL, value: "2"}
|
||||
- {name: HERMES_POOL_ACCESS_ROOT, value: /pool-access}
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
runAsUser: 0
|
||||
runAsGroup: 0
|
||||
seccompProfile: {type: RuntimeDefault}
|
||||
volumeMounts:
|
||||
- {name: pool-access, mountPath: /pool-access}
|
||||
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
|
||||
resources:
|
||||
requests: {cpu: 2m, memory: 16Mi}
|
||||
limits: {cpu: 100m, memory: 64Mi}
|
||||
containers:
|
||||
- name: execution-mediator
|
||||
image: registry.bstein.dev/bstein/hermes-agent
|
||||
command: [/opt/hermes/.venv/bin/python, /opt/coordinator/execution_pool_client.py]
|
||||
env:
|
||||
- {name: HERMES_WORKER_ORDINAL, value: "2"}
|
||||
- {name: HERMES_WORKER_ROOT, value: /workspace}
|
||||
- {name: HERMES_SCM_STATE_ROOT, value: /scm-state}
|
||||
- {name: HERMES_EXECUTION_POOL_KEY_FILE, value: /pool-access/execution-pool-key}
|
||||
- {name: PYTHONPATH, value: /opt/scm:/opt/coordinator:/opt/hermes}
|
||||
ports: [{name: mediator, containerPort: 9009, protocol: TCP}]
|
||||
startupProbe:
|
||||
httpGet: {path: /ready, port: mediator}
|
||||
periodSeconds: 5
|
||||
failureThreshold: 60
|
||||
readinessProbe:
|
||||
httpGet: {path: /ready, port: mediator}
|
||||
periodSeconds: 10
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities: {drop: [ALL]}
|
||||
readOnlyRootFilesystem: true
|
||||
runAsNonRoot: true
|
||||
runAsUser: 10000
|
||||
runAsGroup: 10000
|
||||
seccompProfile: {type: RuntimeDefault}
|
||||
volumeMounts:
|
||||
- {name: workspace, mountPath: /workspace}
|
||||
- {name: scm-state, mountPath: /scm-state}
|
||||
- {name: pool-access, mountPath: /pool-access, readOnly: true}
|
||||
- {name: scm-broker-client, mountPath: /opt/scm, readOnly: true}
|
||||
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
|
||||
- {name: tmp, mountPath: /tmp}
|
||||
resources:
|
||||
requests: {cpu: 2m, memory: 64Mi}
|
||||
limits: {cpu: 250m, memory: 256Mi}
|
||||
volumes:
|
||||
- name: workspace
|
||||
persistentVolumeClaim: {claimName: workspace-hermes-execution-worker-2}
|
||||
- name: scm-state
|
||||
persistentVolumeClaim: {claimName: hermes-execution-mediator-state-2}
|
||||
- name: pool-access
|
||||
emptyDir: {medium: Memory, sizeLimit: 1Mi}
|
||||
- name: scm-broker-client
|
||||
configMap: {name: hermes-scm-boundary-v2, defaultMode: 0555}
|
||||
- name: coordinator
|
||||
configMap: {name: hermes-execution-pool, defaultMode: 0555}
|
||||
- name: tmp
|
||||
emptyDir: {sizeLimit: 64Mi}
|
||||
- name: vault-auth-token
|
||||
projected:
|
||||
defaultMode: 0600
|
||||
sources:
|
||||
- serviceAccountToken: {audience: vault, expirationSeconds: 3600, path: token}
|
||||
- configMap:
|
||||
name: kube-root-ca.crt
|
||||
items: [{key: ca.crt, path: ca.crt}]
|
||||
- downwardAPI:
|
||||
items: [{path: namespace, fieldRef: {fieldPath: metadata.namespace}}]
|
||||
206
services/hermes/execution-worker-networkpolicy.yaml
Normal file
206
services/hermes/execution-worker-networkpolicy.yaml
Normal file
@ -0,0 +1,206 @@
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: hermes-execution-worker-isolation
|
||||
namespace: hermes
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app: hermes-execution-worker
|
||||
policyTypes: [Ingress, Egress]
|
||||
ingress: []
|
||||
egress:
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels: {kubernetes.io/metadata.name: kube-system}
|
||||
podSelector:
|
||||
matchLabels: {k8s-app: kube-dns}
|
||||
ports: [{protocol: UDP, port: 53}, {protocol: TCP, port: 53}]
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels: {kubernetes.io/metadata.name: vault}
|
||||
podSelector:
|
||||
matchLabels: {app: vault}
|
||||
ports: [{protocol: TCP, port: 8200}]
|
||||
- to:
|
||||
- podSelector:
|
||||
matchLabels: {app: hermes-switchyard}
|
||||
ports: [{protocol: TCP, port: 9005}]
|
||||
- to:
|
||||
- ipBlock:
|
||||
cidr: 0.0.0.0/0
|
||||
except:
|
||||
- 10.0.0.0/8
|
||||
- 100.64.0.0/10
|
||||
- 127.0.0.0/8
|
||||
- 169.254.0.0/16
|
||||
- 172.16.0.0/12
|
||||
- 192.168.0.0/16
|
||||
- 224.0.0.0/4
|
||||
ports: [{protocol: TCP, port: 443}]
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: hermes-execution-worker-mediator-0
|
||||
namespace: hermes
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app: hermes-execution-worker
|
||||
apps.kubernetes.io/pod-index: "0"
|
||||
policyTypes: [Egress]
|
||||
egress:
|
||||
- to:
|
||||
- podSelector:
|
||||
matchLabels: {app: hermes-execution-mediator, pool-ordinal: "0"}
|
||||
ports: [{protocol: TCP, port: 9009}]
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: hermes-execution-worker-mediator-1
|
||||
namespace: hermes
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app: hermes-execution-worker
|
||||
apps.kubernetes.io/pod-index: "1"
|
||||
policyTypes: [Egress]
|
||||
egress:
|
||||
- to:
|
||||
- podSelector:
|
||||
matchLabels: {app: hermes-execution-mediator, pool-ordinal: "1"}
|
||||
ports: [{protocol: TCP, port: 9009}]
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: hermes-execution-worker-mediator-2
|
||||
namespace: hermes
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app: hermes-execution-worker
|
||||
apps.kubernetes.io/pod-index: "2"
|
||||
policyTypes: [Egress]
|
||||
egress:
|
||||
- to:
|
||||
- podSelector:
|
||||
matchLabels: {app: hermes-execution-mediator, pool-ordinal: "2"}
|
||||
ports: [{protocol: TCP, port: 9009}]
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: hermes-execution-pool-ingress
|
||||
namespace: hermes
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels: {app: hermes-agent}
|
||||
policyTypes: [Ingress]
|
||||
ingress:
|
||||
- from:
|
||||
- podSelector:
|
||||
matchLabels: {app: hermes-execution-mediator}
|
||||
ports: [{protocol: TCP, port: 9007}]
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: hermes-execution-mediator-isolation
|
||||
namespace: hermes
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels: {app: hermes-execution-mediator}
|
||||
policyTypes: [Ingress, Egress]
|
||||
ingress: []
|
||||
egress:
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels: {kubernetes.io/metadata.name: kube-system}
|
||||
podSelector:
|
||||
matchLabels: {k8s-app: kube-dns}
|
||||
ports: [{protocol: UDP, port: 53}, {protocol: TCP, port: 53}]
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels: {kubernetes.io/metadata.name: vault}
|
||||
podSelector:
|
||||
matchLabels: {app: vault}
|
||||
ports: [{protocol: TCP, port: 8200}]
|
||||
- to:
|
||||
- podSelector:
|
||||
matchLabels: {app: hermes-agent}
|
||||
ports: [{protocol: TCP, port: 9007}]
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels: {kubernetes.io/metadata.name: hermes-scm}
|
||||
podSelector:
|
||||
matchLabels: {app: hermes-scm-broker}
|
||||
ports: [{protocol: TCP, port: 9081}]
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: hermes-execution-mediator-worker-0
|
||||
namespace: hermes
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels: {app: hermes-execution-mediator, pool-ordinal: "0"}
|
||||
policyTypes: [Ingress]
|
||||
ingress:
|
||||
- from:
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
app: hermes-execution-worker
|
||||
apps.kubernetes.io/pod-index: "0"
|
||||
ports: [{protocol: TCP, port: 9009}]
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: hermes-execution-mediator-worker-1
|
||||
namespace: hermes
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels: {app: hermes-execution-mediator, pool-ordinal: "1"}
|
||||
policyTypes: [Ingress]
|
||||
ingress:
|
||||
- from:
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
app: hermes-execution-worker
|
||||
apps.kubernetes.io/pod-index: "1"
|
||||
ports: [{protocol: TCP, port: 9009}]
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: hermes-execution-mediator-worker-2
|
||||
namespace: hermes
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels: {app: hermes-execution-mediator, pool-ordinal: "2"}
|
||||
policyTypes: [Ingress]
|
||||
ingress:
|
||||
- from:
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
app: hermes-execution-worker
|
||||
apps.kubernetes.io/pod-index: "2"
|
||||
ports: [{protocol: TCP, port: 9009}]
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: hermes-execution-switchyard-ingress
|
||||
namespace: hermes
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels: {app: hermes-switchyard}
|
||||
policyTypes: [Ingress]
|
||||
ingress:
|
||||
- from:
|
||||
- podSelector:
|
||||
matchLabels: {app: hermes-execution-worker}
|
||||
ports: [{protocol: TCP, port: 9005}]
|
||||
12
services/hermes/execution-worker-rbac.yaml
Normal file
12
services/hermes/execution-worker-rbac.yaml
Normal file
@ -0,0 +1,12 @@
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: hermes-execution-worker
|
||||
namespace: hermes
|
||||
labels:
|
||||
app.kubernetes.io/name: hermes-execution-worker
|
||||
app.kubernetes.io/part-of: hermes
|
||||
automountServiceAccountToken: false
|
||||
# Intentionally no RoleBinding or ClusterRoleBinding. The projected, bounded
|
||||
# token is mounted only into Vault-facing containers and grants no Kubernetes
|
||||
# API verbs to the model-facing execution worker.
|
||||
291
services/hermes/execution-worker-statefulset.yaml
Normal file
291
services/hermes/execution-worker-statefulset.yaml
Normal file
@ -0,0 +1,291 @@
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
name: hermes-execution-worker
|
||||
namespace: hermes
|
||||
labels:
|
||||
app: hermes-execution-worker
|
||||
spec:
|
||||
serviceName: hermes-execution-worker
|
||||
replicas: 3
|
||||
podManagementPolicy: Parallel
|
||||
revisionHistoryLimit: 2
|
||||
selector:
|
||||
matchLabels:
|
||||
app: hermes-execution-worker
|
||||
updateStrategy:
|
||||
type: RollingUpdate
|
||||
rollingUpdate:
|
||||
partition: 0
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: hermes-execution-worker
|
||||
app.kubernetes.io/name: hermes-execution-worker
|
||||
app.kubernetes.io/part-of: hermes
|
||||
annotations:
|
||||
ai.bstein.dev/role: fenced-execution-only
|
||||
ai.bstein.dev/scm-boundary: mediated-pr14-broker-with-completion-gates
|
||||
ai.bstein.dev/model-policy: Switchyard AUTO with cross-provider fallback
|
||||
ai.bstein.dev/storage: separate durable RWO workspace and OAuth refresh ownership per ordinal
|
||||
vault.hashicorp.com/agent-inject: "true"
|
||||
vault.hashicorp.com/role: hermes-execution-worker
|
||||
vault.hashicorp.com/agent-inject-containers: stage-worker-access
|
||||
vault.hashicorp.com/agent-service-account-token-volume-name: vault-auth-token
|
||||
vault.hashicorp.com/agent-inject-secret-claude-credentials-0: kv/data/atlas/hermes/agent-tokens
|
||||
vault.hashicorp.com/agent-inject-perms-claude-credentials-0: "0600"
|
||||
vault.hashicorp.com/agent-inject-template-claude-credentials-0: |
|
||||
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
|
||||
{{ .Data.data.execution_worker_0_claude_credentials_json }}
|
||||
{{- end }}
|
||||
vault.hashicorp.com/agent-inject-secret-claude-credentials-1: kv/data/atlas/hermes/agent-tokens
|
||||
vault.hashicorp.com/agent-inject-perms-claude-credentials-1: "0600"
|
||||
vault.hashicorp.com/agent-inject-template-claude-credentials-1: |
|
||||
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
|
||||
{{ .Data.data.execution_worker_1_claude_credentials_json }}
|
||||
{{- end }}
|
||||
vault.hashicorp.com/agent-inject-secret-claude-credentials-2: kv/data/atlas/hermes/agent-tokens
|
||||
vault.hashicorp.com/agent-inject-perms-claude-credentials-2: "0600"
|
||||
vault.hashicorp.com/agent-inject-template-claude-credentials-2: |
|
||||
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
|
||||
{{ .Data.data.execution_worker_2_claude_credentials_json }}
|
||||
{{- end }}
|
||||
vault.hashicorp.com/agent-inject-secret-codex-auth-0: kv/data/atlas/hermes/agent-tokens
|
||||
vault.hashicorp.com/agent-inject-perms-codex-auth-0: "0600"
|
||||
vault.hashicorp.com/agent-inject-template-codex-auth-0: |
|
||||
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
|
||||
{{ .Data.data.execution_worker_0_codex_auth_json }}
|
||||
{{- end }}
|
||||
vault.hashicorp.com/agent-inject-secret-codex-auth-1: kv/data/atlas/hermes/agent-tokens
|
||||
vault.hashicorp.com/agent-inject-perms-codex-auth-1: "0600"
|
||||
vault.hashicorp.com/agent-inject-template-codex-auth-1: |
|
||||
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
|
||||
{{ .Data.data.execution_worker_1_codex_auth_json }}
|
||||
{{- end }}
|
||||
vault.hashicorp.com/agent-inject-secret-codex-auth-2: kv/data/atlas/hermes/agent-tokens
|
||||
vault.hashicorp.com/agent-inject-perms-codex-auth-2: "0600"
|
||||
vault.hashicorp.com/agent-inject-template-codex-auth-2: |
|
||||
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
|
||||
{{ .Data.data.execution_worker_2_codex_auth_json }}
|
||||
{{- end }}
|
||||
vault.hashicorp.com/agent-pre-populate-only: "true"
|
||||
vault.hashicorp.com/agent-init-first: "true"
|
||||
vault.hashicorp.com/agent-requests-cpu: 2m
|
||||
vault.hashicorp.com/agent-requests-mem: 16Mi
|
||||
vault.hashicorp.com/agent-limits-cpu: 100m
|
||||
vault.hashicorp.com/agent-limits-mem: 128Mi
|
||||
spec:
|
||||
serviceAccountName: hermes-execution-worker
|
||||
priorityClassName: scavenger
|
||||
automountServiceAccountToken: false
|
||||
enableServiceLinks: false
|
||||
terminationGracePeriodSeconds: 30
|
||||
securityContext:
|
||||
fsGroup: 10000
|
||||
fsGroupChangePolicy: OnRootMismatch
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- {key: kubernetes.io/arch, operator: In, values: [arm64]}
|
||||
- {key: node-role.kubernetes.io/worker, operator: In, values: ["true"]}
|
||||
- {key: kubernetes.io/hostname, operator: NotIn, values: [titan-04, titan-13, titan-14, titan-17, titan-18, titan-19, titan-22, titan-24]}
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
- weight: 100
|
||||
preference:
|
||||
matchExpressions:
|
||||
- {key: node-role.kubernetes.io/accelerator, operator: Exists}
|
||||
- weight: 50
|
||||
preference:
|
||||
matchExpressions:
|
||||
- {key: hardware, operator: In, values: [rpi5]}
|
||||
podAntiAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
- labelSelector:
|
||||
matchLabels:
|
||||
app: hermes-execution-worker
|
||||
topologyKey: kubernetes.io/hostname
|
||||
topologySpreadConstraints:
|
||||
- maxSkew: 1
|
||||
topologyKey: kubernetes.io/hostname
|
||||
whenUnsatisfiable: DoNotSchedule
|
||||
labelSelector:
|
||||
matchLabels:
|
||||
app: hermes-execution-worker
|
||||
initContainers:
|
||||
- name: stage-worker-access
|
||||
image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
|
||||
imagePullPolicy: IfNotPresent
|
||||
command: [/opt/hermes/.venv/bin/python, /opt/coordinator/stage_runtime_access.py, execution-worker]
|
||||
env:
|
||||
- {name: HERMES_WORKER_ROOT, value: /workspace}
|
||||
- {name: HERMES_PROVIDER_ACCESS_ROOT, value: /provider-access}
|
||||
- name: HERMES_WORKER_ORDINAL
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.labels['apps.kubernetes.io/pod-index']
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
runAsUser: 0
|
||||
runAsGroup: 0
|
||||
seccompProfile: {type: RuntimeDefault}
|
||||
volumeMounts:
|
||||
- {name: workspace, mountPath: /workspace}
|
||||
- {name: provider-access, mountPath: /provider-access}
|
||||
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
|
||||
resources:
|
||||
requests: {cpu: 2m, memory: 32Mi}
|
||||
limits: {cpu: 100m, memory: 64Mi}
|
||||
- name: install-provider-clis
|
||||
image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
|
||||
imagePullPolicy: IfNotPresent
|
||||
command: [/bin/sh, -ec]
|
||||
args:
|
||||
# The tools volume is durable, so this install runs once per pinned
|
||||
# version rather than on every Pod start. The version marker is only
|
||||
# trusted when both binaries are actually present and executable, so
|
||||
# a partial or pruned cache reinstalls instead of failing closed
|
||||
# forever. The install itself is time-bounded so a stalled registry
|
||||
# surfaces as a fast init failure with backoff instead of an
|
||||
# open-ended wait that a Flux health timeout would have to absorb.
|
||||
- |
|
||||
tools=/worker-data/tools
|
||||
marker="${tools}/.cli-versions-0.147.0-2.1.226"
|
||||
mkdir -p "${tools}/bin"
|
||||
if [ ! -f "${marker}" ] || [ ! -x "${tools}/bin/codex" ] \
|
||||
|| [ ! -x "${tools}/bin/claude" ]; then
|
||||
rm -f "${marker}"
|
||||
timeout 900 npm install --global --omit=dev --no-audit --no-fund \
|
||||
--fetch-timeout=120000 --fetch-retries=2 --prefix "${tools}" \
|
||||
@openai/codex@0.147.0 @anthropic-ai/claude-code@2.1.226
|
||||
touch "${marker}"
|
||||
fi
|
||||
test -x "${tools}/bin/codex"
|
||||
test -x "${tools}/bin/claude"
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities: {drop: [ALL]}
|
||||
runAsNonRoot: true
|
||||
runAsUser: 10000
|
||||
runAsGroup: 10000
|
||||
seccompProfile: {type: RuntimeDefault}
|
||||
volumeMounts:
|
||||
- {name: tools, mountPath: /worker-data/tools}
|
||||
resources:
|
||||
requests: {cpu: 5m, memory: 64Mi}
|
||||
limits: {cpu: "1", memory: 1Gi}
|
||||
containers:
|
||||
- name: execution-worker
|
||||
image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
|
||||
imagePullPolicy: IfNotPresent
|
||||
command: [/opt/hermes/.venv/bin/python, /opt/coordinator/execution_pool_worker.py]
|
||||
env:
|
||||
- {name: HERMES_HOME, value: /worker-data}
|
||||
- {name: HERMES_WORKER_ROOT, value: /workspace}
|
||||
- {name: HOME, value: /worker-data/home}
|
||||
- {name: CODEX_HOME, value: /provider-access/codex}
|
||||
- {name: CLAUDE_CONFIG_DIR, value: /provider-access/claude}
|
||||
- {name: HERMES_AUTO_ROUTER_PROFILE, value: agent}
|
||||
- {name: PYTHONPATH, value: /opt/hermes}
|
||||
- {name: PATH, value: /worker-data/tools/bin:/opt/coordinator:/opt/hermes/.venv/bin:/usr/local/bin:/usr/bin:/bin}
|
||||
- name: HERMES_WORKER_ORDINAL
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.labels['apps.kubernetes.io/pod-index']
|
||||
- name: HERMES_WORKER_NODE
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
startupProbe:
|
||||
exec:
|
||||
command: [/bin/sh, -ec, "test -w /workspace && test -w /provider-access/codex/auth.json && test -w /provider-access/claude/.credentials.json"]
|
||||
periodSeconds: 5
|
||||
failureThreshold: 60
|
||||
readinessProbe:
|
||||
exec:
|
||||
command: [/bin/sh, -ec, "test -w /workspace && test -w /provider-access/codex/auth.json"]
|
||||
periodSeconds: 10
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities: {drop: [ALL]}
|
||||
readOnlyRootFilesystem: true
|
||||
runAsNonRoot: true
|
||||
runAsUser: 10000
|
||||
runAsGroup: 10000
|
||||
seccompProfile: {type: RuntimeDefault}
|
||||
volumeMounts:
|
||||
- {name: workspace, mountPath: /workspace}
|
||||
- {name: worker-data, mountPath: /worker-data}
|
||||
- {name: tools, mountPath: /worker-data/tools, readOnly: true}
|
||||
- {name: provider-access, mountPath: /provider-access}
|
||||
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
|
||||
- {name: tmp, mountPath: /tmp}
|
||||
resources:
|
||||
requests: {cpu: 5m, memory: 128Mi, ephemeral-storage: 1Gi}
|
||||
limits: {cpu: "2", memory: 4Gi, ephemeral-storage: 8Gi}
|
||||
volumes:
|
||||
- name: worker-data
|
||||
emptyDir: {sizeLimit: 128Mi}
|
||||
- name: coordinator
|
||||
configMap:
|
||||
name: hermes-execution-pool
|
||||
defaultMode: 0555
|
||||
- name: tmp
|
||||
emptyDir: {sizeLimit: 2Gi}
|
||||
- name: vault-auth-token
|
||||
projected:
|
||||
defaultMode: 0600
|
||||
sources:
|
||||
- serviceAccountToken:
|
||||
audience: vault
|
||||
expirationSeconds: 3600
|
||||
path: token
|
||||
- configMap:
|
||||
name: kube-root-ca.crt
|
||||
items:
|
||||
- {key: ca.crt, path: ca.crt}
|
||||
- downwardAPI:
|
||||
items:
|
||||
- {path: namespace, fieldRef: {fieldPath: metadata.namespace}}
|
||||
volumeClaimTemplates:
|
||||
# The workspace is shared with this ordinal's mediator, which performs every
|
||||
# SCM operation on it. ReadWriteMany removes the cross-node Multi-Attach
|
||||
# deadlock that a drain or preemption used to create between the two Pods,
|
||||
# so each can be evicted and rescheduled independently. Longhorn already
|
||||
# serves the hermes-chat tenant workspaces this way on the same class.
|
||||
- metadata:
|
||||
name: workspace
|
||||
labels:
|
||||
app: hermes-execution-worker
|
||||
spec:
|
||||
accessModes: [ReadWriteMany]
|
||||
storageClassName: astreae
|
||||
resources:
|
||||
requests:
|
||||
storage: 30Gi
|
||||
# Provider CLIs are installed once per pinned version onto a durable volume
|
||||
# instead of being re-downloaded into an emptyDir on every Pod start.
|
||||
- metadata:
|
||||
name: tools
|
||||
labels:
|
||||
app: hermes-execution-worker
|
||||
spec:
|
||||
accessModes: [ReadWriteOnce]
|
||||
storageClassName: astreae
|
||||
resources:
|
||||
requests:
|
||||
storage: 2Gi
|
||||
# OAuth refresh ownership is single-mounter by design: keep it ReadWriteOnce.
|
||||
- metadata:
|
||||
name: provider-access
|
||||
labels:
|
||||
app: hermes-execution-worker
|
||||
spec:
|
||||
accessModes: [ReadWriteOnce]
|
||||
storageClassName: astreae
|
||||
resources:
|
||||
requests:
|
||||
storage: 1Gi
|
||||
@ -7,13 +7,13 @@ images:
|
||||
digest: sha256:37ebf720c783ae908a602916ffccf88d43d205a157957f5dc4b487867aee45e7
|
||||
resources:
|
||||
- namespace.yaml
|
||||
- scm-common
|
||||
- vault-serviceaccount.yaml
|
||||
- configmap.yaml
|
||||
- agent-configmap.yaml
|
||||
- chat-configmap.yaml
|
||||
- switchyard-configmap.yaml
|
||||
- rbac.yaml
|
||||
- agent-rbac.yaml
|
||||
- node-ssh-access.yaml
|
||||
- pvc.yaml
|
||||
- switchyard-pvc.yaml
|
||||
@ -41,6 +41,13 @@ resources:
|
||||
- oauth2-proxy.yaml
|
||||
- agent-certificate.yaml
|
||||
- agent-ingress.yaml
|
||||
- execution-worker-rbac.yaml
|
||||
- execution-worker-statefulset.yaml
|
||||
- execution-mediator.yaml
|
||||
- execution-worker-networkpolicy.yaml
|
||||
|
||||
patches:
|
||||
- path: execution-coordinator-patch.yaml
|
||||
|
||||
configMapGenerator:
|
||||
- name: hermes-chat-oauth-templates
|
||||
@ -55,6 +62,38 @@ configMapGenerator:
|
||||
- OPERATOR-RUNBOOK.md=NOTES.md
|
||||
options:
|
||||
disableNameSuffixHash: true
|
||||
- name: hermes-execution-pool
|
||||
namespace: hermes
|
||||
files:
|
||||
- cli_lane_board.py=scripts/cli_lane_board.py
|
||||
- cli_lane_capabilities.py=scripts/cli_lane_capabilities.py
|
||||
- cli_lane_config.py=scripts/cli_lane_config.py
|
||||
- cli_lane_dispatch.py=scripts/cli_lane_dispatch.py
|
||||
- cli_lane_evidence.py=scripts/cli_lane_evidence.py
|
||||
- cli_lane_execution.py=scripts/cli_lane_execution.py
|
||||
- cli_lane_files.py=scripts/cli_lane_files.py
|
||||
- cli_lane_finalization.py=scripts/cli_lane_finalization.py
|
||||
- cli_lane_goal.py=scripts/cli_lane_goal.py
|
||||
- cli_lane_prompt.py=scripts/cli_lane_prompt.py
|
||||
- cli_lane_provider.py=scripts/cli_lane_provider.py
|
||||
- cli_lane_quarantine.py=scripts/cli_lane_quarantine.py
|
||||
- cli_lane_records.py=scripts/cli_lane_records.py
|
||||
- cli_lane_recovery.py=scripts/cli_lane_recovery.py
|
||||
- cli_lane_retention.py=scripts/cli_lane_retention.py
|
||||
- cli_lane_routing.py=scripts/cli_lane_routing.py
|
||||
- cli_lane_runner.py=scripts/cli_lane_runner.py
|
||||
- execution_pool_protocol.py=scripts/execution_pool_protocol.py
|
||||
- execution_pool_store.py=scripts/execution_pool_store.py
|
||||
- execution_pool_project.py=scripts/execution_pool_project.py
|
||||
- execution_pool_coordinator.py=scripts/execution_pool_coordinator.py
|
||||
- execution_pool_maintenance.py=scripts/execution_pool_maintenance.py
|
||||
- execution_pool_server.py=scripts/execution_pool_server.py
|
||||
- execution_pool_client.py=scripts/execution_pool_client.py
|
||||
- execution_pool_worker.py=scripts/execution_pool_worker.py
|
||||
- execution_pool_scm.py=scripts/execution_pool_scm.py
|
||||
- gitea_api_policy.py=scm-common/scripts/gitea_api_policy.py
|
||||
- scm_broker_client.py=scm-common/scripts/scm_broker_client.py
|
||||
- stage_runtime_access.py=scripts/stage_runtime_access.py
|
||||
- name: hermes-coordinator
|
||||
namespace: hermes
|
||||
files:
|
||||
@ -63,7 +102,22 @@ configMapGenerator:
|
||||
- ai_usage_http.py=scripts/ai_usage_http.py
|
||||
- claude=scripts/claude
|
||||
- claude_command_policy.py=scripts/claude_command_policy.py
|
||||
- cli_lane_board.py=scripts/cli_lane_board.py
|
||||
- cli_lane_capabilities.py=scripts/cli_lane_capabilities.py
|
||||
- cli_lane_config.py=scripts/cli_lane_config.py
|
||||
- cli_lane_dispatch.py=scripts/cli_lane_dispatch.py
|
||||
- cli_lane_evidence.py=scripts/cli_lane_evidence.py
|
||||
- cli_lane_execution.py=scripts/cli_lane_execution.py
|
||||
- cli_lane_files.py=scripts/cli_lane_files.py
|
||||
- cli_lane_finalization.py=scripts/cli_lane_finalization.py
|
||||
- cli_lane_goal.py=scripts/cli_lane_goal.py
|
||||
- cli_lane_prompt.py=scripts/cli_lane_prompt.py
|
||||
- cli_lane_provider.py=scripts/cli_lane_provider.py
|
||||
- cli_lane_quarantine.py=scripts/cli_lane_quarantine.py
|
||||
- cli_lane_records.py=scripts/cli_lane_records.py
|
||||
- cli_lane_recovery.py=scripts/cli_lane_recovery.py
|
||||
- cli_lane_retention.py=scripts/cli_lane_retention.py
|
||||
- cli_lane_routing.py=scripts/cli_lane_routing.py
|
||||
- cli_lane_runner.py=scripts/cli_lane_runner.py
|
||||
- codex=scripts/codex
|
||||
- configure_agent_clients.py=scripts/configure_agent_clients.py
|
||||
@ -71,14 +125,13 @@ configMapGenerator:
|
||||
- classifier_broker.py=scripts/classifier_broker.py
|
||||
- claude_oauth_broker.py=scripts/claude_oauth_broker.py
|
||||
- worker_route_broker.py=scripts/worker_route_broker.py
|
||||
- gitea_api.py=scripts/gitea_api.py
|
||||
- gitea_askpass.sh=scripts/gitea_askpass.sh
|
||||
- hermes_coordinator.py=scripts/hermes_coordinator.py
|
||||
- hermes_model_routing.py=scripts/hermes_model_routing.py
|
||||
- hermes_stt_client.py=scripts/hermes_stt_client.py
|
||||
- image_broker.py=scripts/image_broker.py
|
||||
- install_agent_tools.sh=scripts/install_agent_tools.sh
|
||||
- jenkins_build_evidence.py=scripts/jenkins_build_evidence.py
|
||||
- jenkins_image_build_trigger.py=scripts/jenkins_image_build_trigger.py
|
||||
- kanban_status_recovery.py=scripts/kanban_status_recovery.py
|
||||
- migrate_herdr_state.py=scripts/migrate_herdr_state.py
|
||||
- migrate_api_session_lineage.py=scripts/migrate_api_session_lineage.py
|
||||
@ -106,6 +159,15 @@ configMapGenerator:
|
||||
- config=agent-kubeconfig.yaml
|
||||
options:
|
||||
disableNameSuffixHash: true
|
||||
- name: hermes-node-account-hardener
|
||||
namespace: hermes
|
||||
files:
|
||||
- node_account_hardening.py=scripts/node_account_hardening.py
|
||||
- node_account_audit.py=scripts/node_account_audit.py
|
||||
- node_account_io.py=scripts/node_account_io.py
|
||||
- node_polkit_audit.py=scripts/node_polkit_audit.py
|
||||
options:
|
||||
disableNameSuffixHash: true
|
||||
- name: hermes-auto-router-plugin
|
||||
namespace: hermes
|
||||
files:
|
||||
@ -160,3 +222,10 @@ configMapGenerator:
|
||||
- alert-review.md=skills/tune-atlas-alerts/references/alert-review.md
|
||||
options:
|
||||
disableNameSuffixHash: true
|
||||
- name: hermes-atlas-pr-skill
|
||||
namespace: hermes
|
||||
files:
|
||||
- SKILL.md=skills/manage-atlas-pull-requests/SKILL.md
|
||||
- openai.yaml=skills/manage-atlas-pull-requests/agents/openai.yaml
|
||||
options:
|
||||
disableNameSuffixHash: true
|
||||
|
||||
@ -119,11 +119,66 @@ spec:
|
||||
app: server
|
||||
ports:
|
||||
- {protocol: TCP, port: 9010}
|
||||
# agent.hermes.bstein.dev is an owner-only engineering workstation. The
|
||||
# browser boundary remains OAuth-protected, while its workers need to reach
|
||||
# every cluster namespace, Atlas LAN service, and hosted provider endpoint.
|
||||
- {protocol: TCP, port: 9011}
|
||||
egress:
|
||||
- {}
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: kube-system
|
||||
podSelector:
|
||||
matchLabels:
|
||||
k8s-app: kube-dns
|
||||
ports:
|
||||
- {protocol: UDP, port: 53}
|
||||
- {protocol: TCP, port: 53}
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchExpressions:
|
||||
- key: kubernetes.io/metadata.name
|
||||
operator: NotIn
|
||||
values: [gitea, hermes-scm]
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: hermes-scm
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app: hermes-scm-broker
|
||||
ports:
|
||||
- {protocol: TCP, port: 9081}
|
||||
- to:
|
||||
- ipBlock:
|
||||
cidr: 10.43.0.1/32
|
||||
ports:
|
||||
- {protocol: TCP, port: 443}
|
||||
- to:
|
||||
- ipBlock:
|
||||
cidr: 192.168.0.0/16
|
||||
except:
|
||||
- 192.168.22.9/32
|
||||
- to:
|
||||
- ipBlock:
|
||||
cidr: 0.0.0.0/0
|
||||
except:
|
||||
- 10.0.0.0/8
|
||||
- 100.64.0.0/10
|
||||
- 127.0.0.0/8
|
||||
- 169.254.0.0/16
|
||||
- 172.16.0.0/12
|
||||
- 192.168.0.0/16
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: hermes-node-ssh-access-isolation
|
||||
namespace: hermes
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app: hermes-node-ssh-access
|
||||
policyTypes: [Ingress, Egress]
|
||||
ingress: []
|
||||
egress: []
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
|
||||
@ -45,32 +45,16 @@ spec:
|
||||
- operator: Exists
|
||||
containers:
|
||||
- name: key-reconciler
|
||||
image: busybox:1.37
|
||||
# The pinned index supports the cluster's amd64 and arm64 nodes.
|
||||
image: python@sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df
|
||||
imagePullPolicy: IfNotPresent
|
||||
command: [/bin/sh, -ec]
|
||||
args:
|
||||
- |
|
||||
reconcile() {
|
||||
key="$(cat /vault/secrets/node-ssh-public-key)"
|
||||
found=0
|
||||
for user in atlas oceanus; do
|
||||
home="/host-home/${user}"
|
||||
[ -d "${home}" ] || continue
|
||||
found=1
|
||||
identity="$(awk -F: -v name="${user}" '$1 == name {print $3 ":" $4; exit}' /host-etc/passwd)"
|
||||
[ -n "${identity}" ] || identity="$(stat -c %u:%g "${home}")"
|
||||
uid="${identity%%:*}"
|
||||
gid="${identity##*:}"
|
||||
install -d -m 0700 -o "${uid}" -g "${gid}" "${home}/.ssh"
|
||||
touch "${home}/.ssh/authorized_keys"
|
||||
grep -qxF "${key}" "${home}/.ssh/authorized_keys" || printf '%s\n' "${key}" >> "${home}/.ssh/authorized_keys"
|
||||
chown "${uid}:${gid}" "${home}/.ssh/authorized_keys"
|
||||
chmod 0600 "${home}/.ssh/authorized_keys"
|
||||
done
|
||||
[ "${found}" = 1 ] || { echo "no supported node SSH account found" >&2; return 1; }
|
||||
}
|
||||
while true; do
|
||||
reconcile
|
||||
/usr/local/bin/python \
|
||||
/opt/node-hardener/node_account_hardening.py \
|
||||
--public-key-file /vault/secrets/node-ssh-public-key
|
||||
sleep 300
|
||||
done
|
||||
securityContext:
|
||||
@ -83,8 +67,21 @@ spec:
|
||||
volumeMounts:
|
||||
- name: host-home
|
||||
mountPath: /host-home
|
||||
- name: host-passwd
|
||||
mountPath: /host-etc/passwd
|
||||
- name: host-etc
|
||||
mountPath: /host-etc
|
||||
- name: host-polkit-share
|
||||
mountPath: /host-polkit-share
|
||||
readOnly: true
|
||||
- name: host-k3s
|
||||
mountPath: /host-k3s
|
||||
- name: host-kubelet
|
||||
mountPath: /host-kubelet
|
||||
- name: host-run-k3s
|
||||
mountPath: /host-run-k3s
|
||||
- name: host-run-containerd
|
||||
mountPath: /host-run-containerd
|
||||
- name: coordinator
|
||||
mountPath: /opt/node-hardener
|
||||
readOnly: true
|
||||
- name: vault-secrets
|
||||
mountPath: /vault/secrets
|
||||
@ -94,19 +91,43 @@ spec:
|
||||
resources:
|
||||
requests:
|
||||
cpu: 5m
|
||||
memory: 8Mi
|
||||
memory: 32Mi
|
||||
limits:
|
||||
cpu: 50m
|
||||
memory: 32Mi
|
||||
memory: 128Mi
|
||||
volumes:
|
||||
- name: host-home
|
||||
hostPath:
|
||||
path: /home
|
||||
type: Directory
|
||||
- name: host-passwd
|
||||
- name: host-etc
|
||||
hostPath:
|
||||
path: /etc/passwd
|
||||
type: File
|
||||
path: /etc
|
||||
type: Directory
|
||||
- name: host-polkit-share
|
||||
hostPath:
|
||||
path: /usr/share/polkit-1
|
||||
type: Directory
|
||||
- name: host-k3s
|
||||
hostPath:
|
||||
path: /var/lib/rancher/k3s
|
||||
type: Directory
|
||||
- name: host-kubelet
|
||||
hostPath:
|
||||
path: /var/lib/kubelet
|
||||
type: Directory
|
||||
- name: host-run-k3s
|
||||
hostPath:
|
||||
path: /run/k3s
|
||||
type: Directory
|
||||
- name: host-run-containerd
|
||||
hostPath:
|
||||
path: /run/containerd
|
||||
type: Directory
|
||||
- name: coordinator
|
||||
configMap:
|
||||
name: hermes-node-account-hardener
|
||||
defaultMode: 0555
|
||||
- name: vault-secrets
|
||||
csi:
|
||||
driver: secrets-store.csi.k8s.io
|
||||
|
||||
@ -16,6 +16,9 @@ metadata:
|
||||
spec:
|
||||
selector:
|
||||
app: hermes-agent
|
||||
# The agent dashboard and terminal must stay reachable while the
|
||||
# cli-lane-runner readiness probe holds the shared pod NotReady.
|
||||
publishNotReadyAddresses: true
|
||||
ports:
|
||||
- {name: http, port: 80, targetPort: auth-http}
|
||||
---
|
||||
|
||||
17
services/hermes/scm-common/kustomization.yaml
Normal file
17
services/hermes/scm-common/kustomization.yaml
Normal file
@ -0,0 +1,17 @@
|
||||
# services/hermes/scm-common/kustomization.yaml
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
configMapGenerator:
|
||||
- name: hermes-scm-boundary-v2
|
||||
files:
|
||||
- deadline_http.py=scripts/deadline_http.py
|
||||
- git_pack_objects.py=scripts/git_pack_objects.py
|
||||
- gitea_api.py=scripts/gitea_api.py
|
||||
- gitea_api_policy.py=scripts/gitea_api_policy.py
|
||||
- receive_pack_scan.py=scripts/receive_pack_scan.py
|
||||
- scm_broker.py=scripts/scm_broker.py
|
||||
- scm_broker_io.py=scripts/scm_broker_io.py
|
||||
- scm_broker_server.py=scripts/scm_broker_server.py
|
||||
- scm_broker_client.py=scripts/scm_broker_client.py
|
||||
options:
|
||||
disableNameSuffixHash: true
|
||||
344
services/hermes/scm-common/scripts/deadline_http.py
Normal file
344
services/hermes/scm-common/scripts/deadline_http.py
Normal file
@ -0,0 +1,344 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Absolute wall-clock deadlines for every outbound Hermes HTTP exchange.
|
||||
|
||||
Small control exchanges run inside a killable helper process, so connect,
|
||||
send, read, and DNS all share one hard bound that no blocking socket state
|
||||
can outlive. Large streaming Git exchanges cannot cross a process boundary;
|
||||
they instead register their connection with a watchdog that force-closes the
|
||||
socket when the same kind of absolute deadline expires.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import contextlib
|
||||
import email.message
|
||||
import http.client
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from dataclasses import dataclass
|
||||
|
||||
from gitea_api_policy import PolicyError
|
||||
|
||||
MAX_CONTROL_BYTES = 128 * 1024
|
||||
MAX_PROTOCOL_BYTES = 4 * 1024 * 1024
|
||||
MAX_STREAM_SECONDS = 900.0
|
||||
READ_CHUNK = 64 * 1024
|
||||
|
||||
|
||||
class _RejectRedirect(urllib.request.HTTPRedirectHandler):
|
||||
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
||||
raise urllib.error.URLError("redirect rejected")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Result:
|
||||
"""Minimal HTTP evidence returned across the process boundary."""
|
||||
|
||||
status: int
|
||||
content_type: str
|
||||
body: bytes
|
||||
|
||||
|
||||
def _read_response(response, maximum: int, deadline: float) -> bytes:
|
||||
chunks: list[bytes] = []
|
||||
total = 0
|
||||
while True:
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
raise TimeoutError
|
||||
raw = getattr(getattr(response, "fp", None), "raw", None)
|
||||
sock = getattr(raw, "_sock", raw)
|
||||
if hasattr(sock, "settimeout"):
|
||||
sock.settimeout(remaining)
|
||||
chunk = response.read(min(READ_CHUNK, maximum + 1 - total))
|
||||
if not chunk:
|
||||
break
|
||||
chunks.append(chunk)
|
||||
total += len(chunk)
|
||||
if total > maximum:
|
||||
raise ValueError("response too large")
|
||||
return b"".join(chunks)
|
||||
|
||||
|
||||
def _child_exchange(control: dict[str, object]) -> Result:
|
||||
url = control.get("url")
|
||||
method = control.get("method")
|
||||
headers = control.get("headers")
|
||||
encoded = control.get("body")
|
||||
maximum = control.get("maximum")
|
||||
timeout = control.get("timeout")
|
||||
if (
|
||||
not isinstance(url, str)
|
||||
or not isinstance(method, str)
|
||||
or not isinstance(headers, dict)
|
||||
or not all(isinstance(key, str) and isinstance(value, str) for key, value in headers.items())
|
||||
or not isinstance(encoded, str)
|
||||
or not isinstance(maximum, int)
|
||||
or not 0 <= maximum <= 2 * 1024 * 1024
|
||||
or not isinstance(timeout, (int, float))
|
||||
or not 0 < timeout <= 120
|
||||
):
|
||||
raise ValueError("invalid control")
|
||||
body = base64.b64decode(encoded, validate=True) if encoded else None
|
||||
if body is not None and len(body) > MAX_CONTROL_BYTES:
|
||||
raise ValueError("request too large")
|
||||
deadline = time.monotonic() + float(timeout)
|
||||
request = urllib.request.Request(
|
||||
url, data=body, method=method, headers=headers # type: ignore[arg-type]
|
||||
)
|
||||
opener = urllib.request.build_opener(_RejectRedirect())
|
||||
try:
|
||||
response = opener.open(request, timeout=max(deadline - time.monotonic(), 0.001))
|
||||
except urllib.error.HTTPError as error:
|
||||
response = error
|
||||
with response:
|
||||
status = getattr(response, "status", response.getcode())
|
||||
content_type = response.headers.get_content_type()
|
||||
value = _read_response(response, maximum, deadline)
|
||||
return Result(int(status), str(content_type), value)
|
||||
|
||||
|
||||
def _child_main() -> int:
|
||||
try:
|
||||
raw = sys.stdin.buffer.read(MAX_CONTROL_BYTES + 1)
|
||||
if len(raw) > MAX_CONTROL_BYTES:
|
||||
raise ValueError("control too large")
|
||||
control = json.loads(raw)
|
||||
if not isinstance(control, dict):
|
||||
raise ValueError("invalid control")
|
||||
result = _child_exchange(control)
|
||||
output = {
|
||||
"ok": True,
|
||||
"status": result.status,
|
||||
"content_type": result.content_type,
|
||||
"body": base64.b64encode(result.body).decode("ascii"),
|
||||
}
|
||||
except Exception:
|
||||
output = {"ok": False}
|
||||
sys.stdout.write(json.dumps(output, separators=(",", ":")))
|
||||
return 0 if output["ok"] else 1
|
||||
|
||||
|
||||
def _terminate(process: subprocess.Popen[bytes]) -> None:
|
||||
with contextlib.suppress(ProcessLookupError):
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
process.communicate()
|
||||
|
||||
|
||||
def exchange(
|
||||
request: urllib.request.Request,
|
||||
*,
|
||||
maximum: int,
|
||||
timeout: float,
|
||||
popen=subprocess.Popen,
|
||||
) -> Result:
|
||||
"""Execute a whole request under one monotonic deadline and size limit."""
|
||||
if not 0 < timeout <= 120 or not 0 <= maximum <= 2 * 1024 * 1024:
|
||||
raise PolicyError("HTTP control request bounds are invalid")
|
||||
body = request.data or b""
|
||||
if not isinstance(body, bytes) or len(body) > MAX_CONTROL_BYTES:
|
||||
raise PolicyError("HTTP control request exceeds the safe size limit")
|
||||
control = json.dumps(
|
||||
{
|
||||
"url": request.full_url,
|
||||
"method": request.get_method(),
|
||||
"headers": dict(request.header_items()),
|
||||
"body": base64.b64encode(body).decode("ascii"),
|
||||
"maximum": maximum,
|
||||
"timeout": timeout,
|
||||
},
|
||||
separators=(",", ":"),
|
||||
).encode()
|
||||
deadline = time.monotonic() + timeout
|
||||
process = popen(
|
||||
[sys.executable, os.path.realpath(__file__), "--child"],
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
start_new_session=True,
|
||||
env={"LANG": "C.UTF-8", "PATH": "/usr/bin:/bin"},
|
||||
)
|
||||
try:
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
raise subprocess.TimeoutExpired(process.args, timeout)
|
||||
output, _error = process.communicate(control, timeout=remaining)
|
||||
except subprocess.TimeoutExpired as exc:
|
||||
_terminate(process)
|
||||
raise PolicyError("HTTP control request deadline exceeded") from exc
|
||||
if process.returncode != 0 or len(output) > MAX_PROTOCOL_BYTES:
|
||||
raise PolicyError("HTTP control request failed")
|
||||
try:
|
||||
result = json.loads(output)
|
||||
if not isinstance(result, dict) or result.get("ok") is not True:
|
||||
raise ValueError
|
||||
status = result["status"]
|
||||
content_type = result["content_type"]
|
||||
encoded = result["body"]
|
||||
if not isinstance(status, int) or not isinstance(content_type, str) or not isinstance(encoded, str):
|
||||
raise ValueError
|
||||
value = base64.b64decode(encoded, validate=True)
|
||||
except (KeyError, ValueError, TypeError, json.JSONDecodeError) as exc:
|
||||
raise PolicyError("HTTP control helper returned invalid evidence") from exc
|
||||
if len(value) > maximum:
|
||||
raise PolicyError("HTTP control response exceeds the safe size limit")
|
||||
return Result(status, content_type, value)
|
||||
|
||||
|
||||
class _AdaptedResponse:
|
||||
"""Present one finished exchange through the urlopen reader interface."""
|
||||
|
||||
def __init__(self, result: Result):
|
||||
self.status = result.status
|
||||
self.headers = email.message.Message()
|
||||
self.headers["Content-Type"] = result.content_type
|
||||
self._stream = io.BytesIO(result.body)
|
||||
|
||||
def read(self, limit: int = -1) -> bytes:
|
||||
return self._stream.read(limit)
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_excinfo: object) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def open_bounded(
|
||||
request: urllib.request.Request,
|
||||
*,
|
||||
maximum: int,
|
||||
timeout: float,
|
||||
popen=subprocess.Popen,
|
||||
) -> _AdaptedResponse:
|
||||
"""Open one control exchange whose whole lifetime shares one deadline."""
|
||||
return _AdaptedResponse(
|
||||
exchange(request, maximum=maximum, timeout=timeout, popen=popen)
|
||||
)
|
||||
|
||||
|
||||
def _resolve_within(host: str, port: int, timeout: float) -> None:
|
||||
"""Resolve one host under a hard timeout the socket layer never covers.
|
||||
|
||||
``getaddrinfo`` ignores socket timeouts and runs before any socket
|
||||
exists, so a slow or hostile resolver would otherwise outlive the whole
|
||||
stream deadline. Running it in a joinable worker bounds resolution by the
|
||||
same wall clock; a stuck lookup leaks only one daemon thread.
|
||||
"""
|
||||
outcome: dict[str, object] = {}
|
||||
|
||||
def run() -> None:
|
||||
try:
|
||||
outcome["value"] = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM)
|
||||
except OSError as exc:
|
||||
outcome["error"] = exc
|
||||
|
||||
worker = threading.Thread(target=run, daemon=True)
|
||||
worker.start()
|
||||
worker.join(timeout)
|
||||
if worker.is_alive():
|
||||
raise PolicyError("HTTP stream DNS resolution deadline exceeded")
|
||||
if "error" in outcome:
|
||||
raise outcome["error"] # type: ignore[misc]
|
||||
|
||||
|
||||
class StreamDeadline:
|
||||
"""Force-close tracked connections once an absolute deadline passes."""
|
||||
|
||||
def __init__(self, timeout: float, *, timer=threading.Timer):
|
||||
if not 0 < timeout <= MAX_STREAM_SECONDS:
|
||||
raise PolicyError("HTTP stream deadline bounds are invalid")
|
||||
self._deadline = time.monotonic() + timeout
|
||||
self._lock = threading.Lock()
|
||||
self._connections: list[http.client.HTTPConnection] = []
|
||||
self.expired = False
|
||||
self._timer = timer(timeout, self._expire)
|
||||
self._timer.daemon = True
|
||||
self._timer.start()
|
||||
|
||||
def remaining(self) -> float:
|
||||
value = self._deadline - time.monotonic()
|
||||
if value <= 0 or self.expired:
|
||||
raise PolicyError("HTTP stream deadline exceeded")
|
||||
return value
|
||||
|
||||
def _expire(self) -> None:
|
||||
with self._lock:
|
||||
self.expired = True
|
||||
connections = list(self._connections)
|
||||
for connection in connections:
|
||||
# Shut the raw socket down first so a blocked send or recv wakes
|
||||
# immediately; close alone leaves peers waiting on the old fd.
|
||||
sock = getattr(connection, "sock", None)
|
||||
if sock is not None:
|
||||
with contextlib.suppress(OSError):
|
||||
sock.shutdown(socket.SHUT_RDWR)
|
||||
connection.close()
|
||||
|
||||
def _track(self, connection: http.client.HTTPConnection):
|
||||
with self._lock:
|
||||
expired = self.expired
|
||||
if not expired:
|
||||
self._connections.append(connection)
|
||||
if expired:
|
||||
connection.close()
|
||||
raise PolicyError("HTTP stream deadline exceeded")
|
||||
return connection
|
||||
|
||||
def _bounded_connection(self, connection: http.client.HTTPConnection):
|
||||
"""Register one connection and bound its DNS phase by the deadline.
|
||||
|
||||
The watchdog can only close a live socket, but ``connect`` resolves
|
||||
the host before any socket exists. Wrapping ``connect`` runs that
|
||||
resolution under the same absolute deadline first, then hands off to
|
||||
the real connect for the socket phases the watchdog already covers.
|
||||
"""
|
||||
original_connect = connection.connect
|
||||
|
||||
def connect() -> None:
|
||||
_resolve_within(connection.host, connection.port, self.remaining())
|
||||
original_connect()
|
||||
|
||||
connection.connect = connect # type: ignore[method-assign]
|
||||
return self._track(connection)
|
||||
|
||||
def cancel(self) -> None:
|
||||
self._timer.cancel()
|
||||
|
||||
def handlers(self) -> tuple[urllib.request.BaseHandler, ...]:
|
||||
"""Build urllib handlers whose connections obey this deadline."""
|
||||
deadline = self
|
||||
|
||||
class GuardedHTTPHandler(urllib.request.HTTPHandler):
|
||||
def http_open(self, req):
|
||||
return self.do_open(
|
||||
lambda host, **kwargs: deadline._bounded_connection(
|
||||
http.client.HTTPConnection(host, **kwargs)
|
||||
),
|
||||
req,
|
||||
)
|
||||
|
||||
class GuardedHTTPSHandler(urllib.request.HTTPSHandler):
|
||||
def https_open(self, req):
|
||||
return self.do_open(
|
||||
lambda host, **kwargs: deadline._bounded_connection(
|
||||
http.client.HTTPSConnection(host, **kwargs)
|
||||
),
|
||||
req,
|
||||
)
|
||||
|
||||
return (GuardedHTTPHandler(), GuardedHTTPSHandler())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(_child_main() if sys.argv[1:] == ["--child"] else 2)
|
||||
308
services/hermes/scm-common/scripts/git_pack_objects.py
Normal file
308
services/hermes/scm-common/scripts/git_pack_objects.py
Normal file
@ -0,0 +1,308 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Bounded Git pack inflation and delta resolution for quarantine scanning.
|
||||
|
||||
Pushed objects arrive zlib-compressed inside a packfile, so any content
|
||||
screening must inflate the real payloads first. This module unpacks one
|
||||
self-contained pack under explicit object, size, and checksum bounds and
|
||||
resolves every delta against bases inside the same pack. Thin packs are
|
||||
rejected outright: a delta against a base the broker cannot see would leave
|
||||
part of the pushed content unscanned.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import tempfile
|
||||
import zlib
|
||||
from typing import BinaryIO
|
||||
|
||||
from gitea_api_policy import PolicyError
|
||||
|
||||
OBJECT_TYPE_NAMES = {1: b"commit", 2: b"tree", 3: b"blob", 4: b"tag"}
|
||||
OFS_DELTA = 6
|
||||
REF_DELTA = 7
|
||||
MAX_PACK_OBJECTS = 50_000
|
||||
MAX_OBJECT_BYTES = 32 * 1024 * 1024
|
||||
MAX_TOTAL_BYTES = 256 * 1024 * 1024
|
||||
MAX_RESOLVE_PASSES = 64
|
||||
SPOOL_MEMORY_LIMIT = 256 * 1024
|
||||
READ_CHUNK = 64 * 1024
|
||||
|
||||
|
||||
def _read_exact(stream: BinaryIO, size: int) -> bytes:
|
||||
value = stream.read(size)
|
||||
if len(value) != size:
|
||||
raise PolicyError("Git pack data ended early")
|
||||
return value
|
||||
|
||||
|
||||
def _object_header(stream: BinaryIO) -> tuple[int, int]:
|
||||
"""Parse one object's type and declared inflated size."""
|
||||
byte = _read_exact(stream, 1)[0]
|
||||
type_code = (byte >> 4) & 0x7
|
||||
size = byte & 0x0F
|
||||
shift = 4
|
||||
while byte & 0x80:
|
||||
if shift > 60:
|
||||
raise PolicyError("Git object header is invalid")
|
||||
byte = _read_exact(stream, 1)[0]
|
||||
size |= (byte & 0x7F) << shift
|
||||
shift += 7
|
||||
return type_code, size
|
||||
|
||||
|
||||
def _delta_distance(stream: BinaryIO) -> int:
|
||||
"""Parse one ofs-delta base distance."""
|
||||
byte = _read_exact(stream, 1)[0]
|
||||
value = byte & 0x7F
|
||||
while byte & 0x80:
|
||||
byte = _read_exact(stream, 1)[0]
|
||||
value = ((value + 1) << 7) | (byte & 0x7F)
|
||||
if value > 1 << 48:
|
||||
raise PolicyError("Git delta base distance is invalid")
|
||||
return value
|
||||
|
||||
|
||||
def _inflate(stream: BinaryIO, declared: int) -> BinaryIO:
|
||||
"""Inflate one object into a quarantine spool, bounded by its header."""
|
||||
spool = tempfile.SpooledTemporaryFile( # noqa: SIM115 - caller owns returned spool
|
||||
max_size=SPOOL_MEMORY_LIMIT, dir="/tmp"
|
||||
)
|
||||
decompressor = zlib.decompressobj()
|
||||
produced = 0
|
||||
try:
|
||||
while not decompressor.eof:
|
||||
data = stream.read(READ_CHUNK)
|
||||
if not data:
|
||||
raise PolicyError("Git pack object data ended early")
|
||||
while True:
|
||||
value = decompressor.decompress(data, READ_CHUNK)
|
||||
produced += len(value)
|
||||
if produced > declared:
|
||||
raise PolicyError("Git object size does not match its header")
|
||||
spool.write(value)
|
||||
data = decompressor.unconsumed_tail
|
||||
if decompressor.eof or not data:
|
||||
break
|
||||
if decompressor.unused_data:
|
||||
stream.seek(-len(decompressor.unused_data), 1)
|
||||
if produced != declared:
|
||||
raise PolicyError("Git object size does not match its header")
|
||||
spool.seek(0)
|
||||
return spool
|
||||
except zlib.error as exc:
|
||||
spool.close()
|
||||
raise PolicyError("Git pack object data is corrupt") from exc
|
||||
except Exception:
|
||||
spool.close()
|
||||
raise
|
||||
|
||||
|
||||
def _delta_size(delta: bytes, position: int) -> tuple[int, int]:
|
||||
size = 0
|
||||
shift = 0
|
||||
while True:
|
||||
if position >= len(delta) or shift > 60:
|
||||
raise PolicyError("Git delta header is invalid")
|
||||
byte = delta[position]
|
||||
position += 1
|
||||
size |= (byte & 0x7F) << shift
|
||||
shift += 7
|
||||
if not byte & 0x80:
|
||||
return size, position
|
||||
|
||||
|
||||
def _apply_delta(base: bytes, delta: bytes) -> bytes:
|
||||
"""Apply one Git delta and require exact declared sizes throughout."""
|
||||
source_size, position = _delta_size(delta, 0)
|
||||
target_size, position = _delta_size(delta, position)
|
||||
if source_size != len(base) or target_size > MAX_OBJECT_BYTES:
|
||||
raise PolicyError("Git delta sizes are invalid")
|
||||
parts: list[bytes] = []
|
||||
produced = 0
|
||||
while position < len(delta):
|
||||
opcode = delta[position]
|
||||
position += 1
|
||||
if opcode & 0x80:
|
||||
operands = [bit for bit in (1, 2, 4, 8, 16, 32, 64) if opcode & bit]
|
||||
if position + len(operands) > len(delta):
|
||||
raise PolicyError("Git delta copy is truncated")
|
||||
offset = 0
|
||||
size = 0
|
||||
for shift, bit in enumerate((1, 2, 4, 8)):
|
||||
if opcode & bit:
|
||||
offset |= delta[position] << (8 * shift)
|
||||
position += 1
|
||||
for shift, bit in enumerate((16, 32, 64)):
|
||||
if opcode & bit:
|
||||
size |= delta[position] << (8 * shift)
|
||||
position += 1
|
||||
size = size or 0x10000
|
||||
if offset + size > len(base):
|
||||
raise PolicyError("Git delta copy is out of bounds")
|
||||
parts.append(base[offset : offset + size])
|
||||
produced += size
|
||||
elif opcode:
|
||||
if position + opcode > len(delta):
|
||||
raise PolicyError("Git delta insert is truncated")
|
||||
parts.append(delta[position : position + opcode])
|
||||
position += opcode
|
||||
produced += opcode
|
||||
else:
|
||||
raise PolicyError("Git delta instruction is invalid")
|
||||
if produced > target_size:
|
||||
raise PolicyError("Git delta result exceeds its declared size")
|
||||
if produced != target_size:
|
||||
raise PolicyError("Git delta result size is invalid")
|
||||
return b"".join(parts)
|
||||
|
||||
|
||||
def _payload_sha(type_code: int, payload: BinaryIO, size: int) -> bytes:
|
||||
digest = hashlib.sha1(OBJECT_TYPE_NAMES[type_code] + b" %d\x00" % size)
|
||||
payload.seek(0)
|
||||
while True:
|
||||
chunk = payload.read(READ_CHUNK)
|
||||
if not chunk:
|
||||
break
|
||||
digest.update(chunk)
|
||||
payload.seek(0)
|
||||
return digest.digest()
|
||||
|
||||
|
||||
def _verify_checksum(stream: BinaryIO, start: int, end: int) -> None:
|
||||
trailer = _read_exact(stream, 20)
|
||||
stream.seek(start)
|
||||
digest = hashlib.sha1()
|
||||
remaining = end - start
|
||||
while remaining:
|
||||
chunk = stream.read(min(READ_CHUNK, remaining))
|
||||
digest.update(chunk)
|
||||
remaining -= len(chunk)
|
||||
if digest.digest() != trailer:
|
||||
raise PolicyError("Git pack checksum is invalid")
|
||||
stream.seek(end + 20)
|
||||
|
||||
|
||||
class _Entry:
|
||||
"""One pack entry: either resolved content or an unresolved delta."""
|
||||
|
||||
def __init__(self, type_code: int, payload: BinaryIO, size: int, base):
|
||||
self.type_code = type_code
|
||||
self.payload = payload
|
||||
self.size = size
|
||||
self.base = base
|
||||
|
||||
|
||||
def _resolve(entries: dict[int, _Entry], total: list[int]) -> None:
|
||||
"""Resolve every delta against in-pack bases only, to a fixed point."""
|
||||
by_sha: dict[bytes, int] = {}
|
||||
for offset, entry in entries.items():
|
||||
if entry.base is None:
|
||||
by_sha.setdefault(
|
||||
_payload_sha(entry.type_code, entry.payload, entry.size), offset
|
||||
)
|
||||
for _ in range(MAX_RESOLVE_PASSES):
|
||||
pending = [item for item in entries.items() if item[1].base is not None]
|
||||
if not pending:
|
||||
return
|
||||
progressed = False
|
||||
for offset, entry in pending:
|
||||
base_key = entry.base
|
||||
base_offset = base_key if isinstance(base_key, int) else by_sha.get(base_key)
|
||||
base = entries.get(base_offset) if base_offset is not None else None
|
||||
if base is None or base.base is not None:
|
||||
if isinstance(base_key, int) and base is None:
|
||||
raise PolicyError("Git delta base offset is invalid")
|
||||
continue
|
||||
base.payload.seek(0)
|
||||
entry.payload.seek(0)
|
||||
value = _apply_delta(base.payload.read(), entry.payload.read())
|
||||
total[0] += len(value)
|
||||
if total[0] > MAX_TOTAL_BYTES:
|
||||
raise PolicyError("Git pack contents exceed the safe size limit")
|
||||
entry.payload.close()
|
||||
spool = tempfile.SpooledTemporaryFile( # noqa: SIM115 - entry owns spool
|
||||
max_size=SPOOL_MEMORY_LIMIT, dir="/tmp"
|
||||
)
|
||||
spool.write(value)
|
||||
spool.seek(0)
|
||||
entry.type_code = base.type_code
|
||||
entry.payload = spool
|
||||
entry.size = len(value)
|
||||
entry.base = None
|
||||
by_sha.setdefault(_payload_sha(entry.type_code, spool, entry.size), offset)
|
||||
progressed = True
|
||||
if not progressed:
|
||||
raise PolicyError("Git delta base is outside the pack; push full packs")
|
||||
raise PolicyError("Git delta chains exceed the safe resolution limit")
|
||||
|
||||
|
||||
def unpack_objects(stream: BinaryIO) -> list[tuple[int, BinaryIO]]:
|
||||
"""Inflate one self-contained pack and return every resolved payload."""
|
||||
start = stream.tell()
|
||||
header = _read_exact(stream, 12)
|
||||
version = int.from_bytes(header[4:8], "big")
|
||||
count = int.from_bytes(header[8:12], "big")
|
||||
if header[:4] != b"PACK" or version not in {2, 3}:
|
||||
raise PolicyError("Git pack header is invalid")
|
||||
if count > MAX_PACK_OBJECTS:
|
||||
raise PolicyError("Git pack contains too many objects")
|
||||
entries: dict[int, _Entry] = {}
|
||||
total = [0]
|
||||
try:
|
||||
for _ in range(count):
|
||||
offset = stream.tell() - start
|
||||
type_code, declared = _object_header(stream)
|
||||
if declared > MAX_OBJECT_BYTES:
|
||||
raise PolicyError("Git object exceeds the safe size limit")
|
||||
total[0] += declared
|
||||
if total[0] > MAX_TOTAL_BYTES:
|
||||
raise PolicyError("Git pack contents exceed the safe size limit")
|
||||
base: int | bytes | None = None
|
||||
if type_code == OFS_DELTA:
|
||||
distance = _delta_distance(stream)
|
||||
base = offset - distance
|
||||
if base < 0:
|
||||
raise PolicyError("Git delta base offset is invalid")
|
||||
elif type_code == REF_DELTA:
|
||||
base = _read_exact(stream, 20)
|
||||
elif type_code not in OBJECT_TYPE_NAMES:
|
||||
raise PolicyError("Git pack object type is invalid")
|
||||
entries[offset] = _Entry(type_code, _inflate(stream, declared), declared, base)
|
||||
_verify_checksum(stream, start, stream.tell())
|
||||
_resolve(entries, total)
|
||||
except Exception:
|
||||
for entry in entries.values():
|
||||
entry.payload.close()
|
||||
raise
|
||||
for entry in entries.values():
|
||||
entry.payload.seek(0)
|
||||
return [
|
||||
(entries[offset].type_code, entries[offset].payload)
|
||||
for offset in sorted(entries)
|
||||
]
|
||||
|
||||
|
||||
def parse_loose_object(value: bytes) -> tuple[int, bytes]:
|
||||
"""Inflate one quarantined loose object and return its typed payload."""
|
||||
if len(value) > MAX_OBJECT_BYTES:
|
||||
raise PolicyError("Git object exceeds the safe size limit")
|
||||
decompressor = zlib.decompressobj()
|
||||
try:
|
||||
inflated = decompressor.decompress(value, MAX_OBJECT_BYTES + 1)
|
||||
except zlib.error as exc:
|
||||
raise PolicyError("Git loose object data is corrupt") from exc
|
||||
if not decompressor.eof or decompressor.unused_data or len(inflated) > MAX_OBJECT_BYTES:
|
||||
raise PolicyError("Git loose object framing is invalid")
|
||||
kind, separator, remainder = inflated.partition(b"\x00")
|
||||
name, space, size = kind.partition(b" ")
|
||||
types = {label: code for code, label in OBJECT_TYPE_NAMES.items()}
|
||||
if (
|
||||
not separator
|
||||
or not space
|
||||
or name not in types
|
||||
or not size.isdigit()
|
||||
or int(size) != len(remainder)
|
||||
):
|
||||
raise PolicyError("Git loose object header is invalid")
|
||||
return types[name], remainder
|
||||
492
services/hermes/scm-common/scripts/gitea_api.py
Normal file
492
services/hermes/scm-common/scripts/gitea_api.py
Normal file
@ -0,0 +1,492 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Use the private Atlas Forgejo API through a least-authority PR boundary."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
|
||||
import deadline_http
|
||||
from gitea_api_policy import (
|
||||
PolicyError,
|
||||
_draft_title,
|
||||
_reject_forbidden,
|
||||
_validate_body,
|
||||
_validate_pr_number,
|
||||
_validate_pr_number_segment,
|
||||
_validate_query,
|
||||
_validate_ref,
|
||||
_validate_ref_bounds,
|
||||
_validate_repo,
|
||||
_validate_sha,
|
||||
)
|
||||
|
||||
CANONICAL_BASE_URL = "https://scm.bstein.dev"
|
||||
ALLOWED_OWNER = "atlas"
|
||||
DEFAULT_TOKEN_FILE = Path("/vault/secrets/gitea-token")
|
||||
CREATE_PATH_RE = re.compile(r"/api/v1/repos/atlas/([^/]+)/pulls\Z")
|
||||
MAX_ERROR_BYTES = 8192
|
||||
MAX_RESPONSE_BYTES = 2 * 1024 * 1024
|
||||
MAX_API_TARGET_LENGTH = 768
|
||||
MAX_API_PATH_LENGTH = 512
|
||||
RAW_API_TARGET_RE = re.compile(r"[A-Za-z0-9/_.?&=-]+\Z")
|
||||
|
||||
|
||||
def _safe_urlopen(request: urllib.request.Request, timeout: int):
|
||||
"""Open one exchange under a killable absolute wall-clock deadline.
|
||||
|
||||
The helper process rejects redirects before urllib can copy
|
||||
authentication headers, and its whole connect/send/read lifetime
|
||||
shares one hard deadline.
|
||||
"""
|
||||
return deadline_http.open_bounded(
|
||||
request, maximum=MAX_RESPONSE_BYTES, timeout=timeout
|
||||
)
|
||||
|
||||
|
||||
def read_token(path: Path = DEFAULT_TOKEN_FILE) -> str:
|
||||
"""Read the token from the pod-lifetime Vault projection."""
|
||||
token = path.read_text(encoding="utf-8").strip()
|
||||
if not token:
|
||||
raise ValueError(f"runtime credential is empty: {path}")
|
||||
return token
|
||||
|
||||
|
||||
def configured_base_url() -> str:
|
||||
"""Reject attempts to redirect credentials away from the canonical origin."""
|
||||
configured = os.environ.get("GITEA_BASE_URL", CANONICAL_BASE_URL)
|
||||
if configured != CANONICAL_BASE_URL:
|
||||
raise PolicyError("Forgejo origin is fixed to the private Atlas SCM service")
|
||||
return configured
|
||||
|
||||
|
||||
def _split_api_path(path: str) -> urllib.parse.SplitResult:
|
||||
if not isinstance(path, str) or len(path) > MAX_API_TARGET_LENGTH:
|
||||
raise PolicyError("API target exceeds the safe size limit")
|
||||
if (
|
||||
not path.isascii()
|
||||
or any(ord(character) < 32 or ord(character) == 127 for character in path)
|
||||
or "\\" in path
|
||||
or not RAW_API_TARGET_RE.fullmatch(path)
|
||||
):
|
||||
raise PolicyError("API target contains non-canonical raw characters")
|
||||
target = urllib.parse.urlsplit(path)
|
||||
canonical = urllib.parse.urlunsplit(("", "", target.path, target.query, ""))
|
||||
if canonical != path:
|
||||
raise PolicyError("API target is not in exact canonical form")
|
||||
if target.scheme or target.netloc or target.fragment:
|
||||
raise PolicyError("API path must be relative to the Atlas SCM origin")
|
||||
if not target.path.startswith("/api/v1/"):
|
||||
raise PolicyError("API path must start with /api/v1/")
|
||||
if len(target.path) > MAX_API_PATH_LENGTH:
|
||||
raise PolicyError("API path exceeds the safe size limit")
|
||||
segments = target.path.split("/")
|
||||
if "//" in target.path or any(segment in {".", ".."} for segment in segments):
|
||||
raise PolicyError("encoded or non-canonical API paths are not allowed")
|
||||
return target
|
||||
|
||||
|
||||
def _authorize_read(
|
||||
target: urllib.parse.SplitResult, *, forbidden: tuple[str, ...] = ()
|
||||
) -> str:
|
||||
"""Allow only repository, PR, branch, commit, and status metadata reads."""
|
||||
prefix = "/api/v1/repos/atlas/"
|
||||
remainder = target.path.removeprefix(prefix)
|
||||
if remainder == target.path:
|
||||
raise PolicyError("reads are limited to explicit Atlas repository metadata")
|
||||
repo, separator, suffix = remainder.partition("/")
|
||||
_reject_forbidden(repo, "repository", forbidden)
|
||||
_validate_repo(repo)
|
||||
|
||||
if not separator:
|
||||
_validate_query(target, set())
|
||||
return "repository"
|
||||
if suffix == "pulls":
|
||||
_validate_query(target, {"page", "limit", "state"})
|
||||
return "pull-list"
|
||||
content_match = re.fullmatch(r"pulls/([^/]+)\.(?:patch|diff)", suffix)
|
||||
if content_match:
|
||||
_validate_pr_number_segment(content_match.group(1))
|
||||
raise PolicyError("repository API route is outside the metadata read allowlist")
|
||||
pull_match = re.fullmatch(r"pulls/([^/]+)", suffix)
|
||||
if pull_match:
|
||||
_validate_pr_number_segment(pull_match.group(1))
|
||||
_validate_query(target, set())
|
||||
return "pull"
|
||||
evidence_match = re.fullmatch(r"pulls/([^/]+)/(?:commits|files)", suffix)
|
||||
if evidence_match:
|
||||
_validate_pr_number_segment(evidence_match.group(1))
|
||||
_validate_query(target, {"page", "limit"})
|
||||
return "pull-evidence"
|
||||
if suffix == "branches":
|
||||
_validate_query(target, {"page", "limit"})
|
||||
return "branch-list"
|
||||
branch_match = re.fullmatch(r"branches/([^/]+)", suffix)
|
||||
if branch_match:
|
||||
_validate_ref(branch_match.group(1), "branch")
|
||||
_validate_query(target, set())
|
||||
return "branch"
|
||||
if suffix == "commits":
|
||||
_validate_query(target, {"page", "limit"})
|
||||
return "commit-list"
|
||||
if re.fullmatch(r"git/commits/[0-9a-fA-F]{40}", suffix):
|
||||
_validate_query(target, set())
|
||||
return "commit"
|
||||
if re.fullmatch(
|
||||
r"(?:commits/[0-9a-fA-F]{40}/status|commits/[0-9a-fA-F]{40}/statuses|statuses/[0-9a-fA-F]{40})",
|
||||
suffix,
|
||||
):
|
||||
_validate_query(target, {"page", "limit"})
|
||||
return "status"
|
||||
raise PolicyError("repository API route is outside the metadata read allowlist")
|
||||
|
||||
|
||||
def api_url(base_url: str, path: str) -> str:
|
||||
"""Return a canonical same-origin URL without forwarding credentials."""
|
||||
if base_url != CANONICAL_BASE_URL:
|
||||
raise PolicyError("Forgejo origin is fixed to the private Atlas SCM service")
|
||||
target = _split_api_path(path)
|
||||
return urllib.parse.urlunsplit(
|
||||
("https", "scm.bstein.dev", target.path, target.query, "")
|
||||
)
|
||||
|
||||
|
||||
def authorize_request(
|
||||
method: str,
|
||||
path: str,
|
||||
data: object | None,
|
||||
*,
|
||||
forbidden: tuple[str, ...] = (),
|
||||
) -> str:
|
||||
"""Validate one request and return its bounded operation name."""
|
||||
normalized_method = method.upper()
|
||||
target = _split_api_path(path)
|
||||
if normalized_method == "GET":
|
||||
if data is not None:
|
||||
raise PolicyError("read operations cannot include a request body")
|
||||
return _authorize_read(target, forbidden=forbidden)
|
||||
if target.query:
|
||||
raise PolicyError("mutating operations cannot include query parameters")
|
||||
if normalized_method != "POST":
|
||||
raise PolicyError("only read and create-draft operations are available")
|
||||
match = CREATE_PATH_RE.fullmatch(target.path)
|
||||
if not match:
|
||||
raise PolicyError("POST is limited to creating an Atlas draft pull request")
|
||||
_reject_forbidden(match.group(1), "repository", forbidden)
|
||||
_validate_repo(match.group(1))
|
||||
if not isinstance(data, dict) or set(data) != {"base", "body", "head", "title"}:
|
||||
raise PolicyError("draft creation accepts only base, body, head, and title")
|
||||
if data["title"] != _draft_title(data["title"], forbidden=forbidden):
|
||||
raise PolicyError("new pull requests must use the Gitea draft-title prefix")
|
||||
_validate_body(data["body"], forbidden=forbidden)
|
||||
_validate_ref_bounds(data["base"], "base", forbidden=forbidden)
|
||||
_validate_ref_bounds(data["head"], "head", forbidden=forbidden)
|
||||
_validate_ref(data["base"], "base", forbidden=forbidden)
|
||||
_validate_ref(data["head"], "head", forbidden=forbidden)
|
||||
return "create-draft"
|
||||
|
||||
|
||||
def build_request(
|
||||
method: str,
|
||||
path: str,
|
||||
*,
|
||||
base_url: str,
|
||||
token: str,
|
||||
data: object | None = None,
|
||||
) -> urllib.request.Request:
|
||||
"""Build an authorized request without putting the token in its URL or body."""
|
||||
authorize_request(method, path, data, forbidden=(token,))
|
||||
return _build_request(method, path, base_url=base_url, token=token, data=data)
|
||||
|
||||
|
||||
def _build_request(
|
||||
method: str,
|
||||
path: str,
|
||||
*,
|
||||
base_url: str,
|
||||
token: str,
|
||||
data: object | None = None,
|
||||
) -> urllib.request.Request:
|
||||
"""Build one already-authorized upstream request."""
|
||||
payload = None
|
||||
if data is not None:
|
||||
payload = json.dumps(data, separators=(",", ":")).encode("utf-8")
|
||||
return urllib.request.Request(
|
||||
api_url(base_url, path),
|
||||
data=payload,
|
||||
method=method.upper(),
|
||||
headers={
|
||||
"Accept": "application/json",
|
||||
"Authorization": f"token {token}",
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent": "hermes-atlas-pr-client/2",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def redact_bytes(value: bytes, token: str) -> bytes:
|
||||
"""Remove exact runtime-token and authorization-header values from output."""
|
||||
redacted = value.replace(token.encode("utf-8"), b"[REDACTED]")
|
||||
return re.sub(
|
||||
rb"(?i)(authorization\s*[:=]\s*)(?:token|bearer)?\s*[^\s,;\"}]+",
|
||||
rb"\1[REDACTED]",
|
||||
redacted,
|
||||
)
|
||||
|
||||
|
||||
def _reject_response_credential(value: bytes, token: str) -> None:
|
||||
"""Fail closed if an upstream body reflects any ordinary token encoding."""
|
||||
raw = token.encode("utf-8")
|
||||
forms = {
|
||||
raw,
|
||||
base64.b64encode(raw),
|
||||
base64.b64encode(b"hermes-automation:" + raw),
|
||||
urllib.parse.quote(token, safe="").encode("ascii"),
|
||||
}
|
||||
if any(form and form in value for form in forms):
|
||||
raise PolicyError("Forgejo response contains runtime credential material")
|
||||
|
||||
|
||||
def _request(
|
||||
method: str,
|
||||
path: str,
|
||||
data: object | None,
|
||||
*,
|
||||
token: str,
|
||||
opener: Callable[..., object] = _safe_urlopen,
|
||||
authorized: bool = False,
|
||||
expected_status: int,
|
||||
) -> bytes:
|
||||
builder = _build_request if authorized else build_request
|
||||
request = builder(method, path, base_url=configured_base_url(), token=token, data=data)
|
||||
with opener(request, timeout=30) as response: # type: ignore[attr-defined]
|
||||
status = getattr(response, "status", None)
|
||||
if status is None and hasattr(response, "getcode"):
|
||||
status = response.getcode() # type: ignore[attr-defined]
|
||||
if status != expected_status:
|
||||
raise PolicyError("Forgejo returned an unexpected HTTP status")
|
||||
if response.headers.get_content_type() != "application/json": # type: ignore[attr-defined]
|
||||
raise PolicyError("Forgejo returned an unexpected response type")
|
||||
body = response.read(MAX_RESPONSE_BYTES + 1) # type: ignore[attr-defined]
|
||||
if len(body) > MAX_RESPONSE_BYTES:
|
||||
raise PolicyError("Forgejo response exceeds the safe size limit")
|
||||
_reject_response_credential(body, token)
|
||||
return redact_bytes(body, token)
|
||||
|
||||
|
||||
def read(
|
||||
path: str, *, token: str, opener: Callable[..., object] = _safe_urlopen
|
||||
) -> bytes:
|
||||
"""Read one explicitly allowed Atlas repository metadata resource."""
|
||||
return _request(
|
||||
"GET", path, None, token=token, opener=opener, expected_status=200
|
||||
)
|
||||
|
||||
|
||||
def _nested(document: dict[str, object], *keys: str) -> object:
|
||||
value: object = document
|
||||
for key in keys:
|
||||
if not isinstance(value, dict) or key not in value:
|
||||
raise PolicyError("Forgejo omitted required pull-request evidence")
|
||||
value = value[key]
|
||||
return value
|
||||
|
||||
|
||||
def _require_create_response(
|
||||
value: bytes,
|
||||
*,
|
||||
repo: str,
|
||||
base: str,
|
||||
head: str,
|
||||
head_sha: str,
|
||||
title: str,
|
||||
body: str,
|
||||
) -> dict[str, object]:
|
||||
"""Require exact server evidence for the intended open draft handoff."""
|
||||
try:
|
||||
document = json.loads(value)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise PolicyError("Forgejo returned invalid pull-request metadata") from exc
|
||||
if not isinstance(document, dict):
|
||||
raise PolicyError("Forgejo returned invalid pull-request metadata")
|
||||
number = _validate_pr_number(document.get("number"))
|
||||
full_name = f"{ALLOWED_OWNER}/{repo}"
|
||||
expected_html = f"{CANONICAL_BASE_URL}/{full_name}/pulls/{number}"
|
||||
checks = (
|
||||
document.get("state") == "open",
|
||||
document.get("draft") is True,
|
||||
document.get("merged") is False,
|
||||
document.get("html_url") == expected_html,
|
||||
document.get("url") == expected_html,
|
||||
document.get("title") == title,
|
||||
document.get("body") == body,
|
||||
_nested(document, "base", "ref") == base,
|
||||
_nested(document, "base", "repo", "full_name") == full_name,
|
||||
_nested(document, "head", "ref") == head,
|
||||
_nested(document, "head", "sha") == head_sha,
|
||||
_nested(document, "head", "repo", "full_name") == full_name,
|
||||
)
|
||||
if not all(checks):
|
||||
raise PolicyError("Forgejo did not confirm the exact draft pull-request state")
|
||||
return document
|
||||
|
||||
|
||||
def create_draft(
|
||||
repo: str,
|
||||
*,
|
||||
base: str,
|
||||
head: str,
|
||||
head_sha: str,
|
||||
title: str,
|
||||
body: str,
|
||||
token: str,
|
||||
opener: Callable[..., object] = _safe_urlopen,
|
||||
) -> bytes:
|
||||
"""Create and verify a same-repository draft pull request for human review."""
|
||||
_reject_forbidden(repo, "repository", (token,))
|
||||
repo = _validate_repo(repo)
|
||||
title = _draft_title(title, forbidden=(token,))
|
||||
body = _validate_body(body, forbidden=(token,))
|
||||
head_sha = _validate_sha(head_sha)
|
||||
_validate_ref_bounds(base, "base", forbidden=(token,))
|
||||
_validate_ref_bounds(head, "head", forbidden=(token,))
|
||||
base = _validate_ref(base, "base", forbidden=(token,))
|
||||
head = _validate_ref(head, "head", forbidden=(token,))
|
||||
data = {"base": base, "body": body, "head": head, "title": title}
|
||||
result = _request(
|
||||
"POST",
|
||||
f"/api/v1/repos/{ALLOWED_OWNER}/{repo}/pulls",
|
||||
data,
|
||||
token=token,
|
||||
opener=opener,
|
||||
authorized=True,
|
||||
expected_status=201,
|
||||
)
|
||||
_require_create_response(
|
||||
result,
|
||||
repo=repo,
|
||||
base=base,
|
||||
head=head,
|
||||
head_sha=head_sha,
|
||||
title=title,
|
||||
body=body,
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||
"""Parse the intentionally narrow read and create-draft operations."""
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="validate and describe the request without reading a token or using the network",
|
||||
)
|
||||
subparsers = parser.add_subparsers(dest="operation", required=True)
|
||||
read_parser = subparsers.add_parser("read", help="read allowed Atlas metadata")
|
||||
read_parser.add_argument("path")
|
||||
create_parser = subparsers.add_parser(
|
||||
"create-draft", help="create a same-repository draft pull request"
|
||||
)
|
||||
create_parser.add_argument("repo")
|
||||
create_parser.add_argument("--base", required=True)
|
||||
create_parser.add_argument("--head", required=True)
|
||||
create_parser.add_argument("--head-sha", required=True)
|
||||
create_parser.add_argument("--title", required=True)
|
||||
create_parser.add_argument("--body", default="")
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def _operation(
|
||||
args: argparse.Namespace, *, validate_refs: bool = True
|
||||
) -> tuple[str, str, object | None]:
|
||||
if args.operation == "read":
|
||||
return "GET", args.path, None
|
||||
repo = _validate_repo(args.repo)
|
||||
_validate_sha(args.head_sha)
|
||||
body = _validate_body(args.body)
|
||||
title = _draft_title(args.title)
|
||||
_validate_ref_bounds(args.base, "base")
|
||||
_validate_ref_bounds(args.head, "head")
|
||||
base = _validate_ref(args.base, "base") if validate_refs else args.base
|
||||
head = _validate_ref(args.head, "head") if validate_refs else args.head
|
||||
data = {
|
||||
"base": base,
|
||||
"body": body,
|
||||
"head": head,
|
||||
"title": title,
|
||||
}
|
||||
return "POST", f"/api/v1/repos/{ALLOWED_OWNER}/{repo}/pulls", data
|
||||
|
||||
|
||||
def _write_body(body: bytes) -> None:
|
||||
if body:
|
||||
sys.stdout.buffer.write(body)
|
||||
if not body.endswith(b"\n"):
|
||||
sys.stdout.buffer.write(b"\n")
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
"""Validate, optionally describe, and execute one safe Forgejo operation."""
|
||||
try:
|
||||
args = parse_args(argv)
|
||||
if args.dry_run:
|
||||
method, path, data = _operation(args, validate_refs=False)
|
||||
operation = authorize_request(method, path, data)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"dry_run": True,
|
||||
"host": "scm.bstein.dev",
|
||||
"operation": operation,
|
||||
"owner": ALLOWED_OWNER,
|
||||
"path": path,
|
||||
"request_fields": sorted(data)
|
||||
if isinstance(data, dict)
|
||||
else [],
|
||||
},
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
method, path, data = _operation(args, validate_refs=False)
|
||||
from scm_broker_client import create_draft as broker_create_draft
|
||||
from scm_broker_client import read as broker_read
|
||||
|
||||
if args.operation == "read":
|
||||
body = broker_read(path)
|
||||
else:
|
||||
assert isinstance(data, dict)
|
||||
body = broker_create_draft(
|
||||
args.repo,
|
||||
base=str(data["base"]),
|
||||
head=str(data["head"]),
|
||||
head_sha=args.head_sha,
|
||||
title=str(data["title"]),
|
||||
body=str(data["body"]),
|
||||
)
|
||||
_write_body(body)
|
||||
return 0
|
||||
except urllib.error.HTTPError as exc:
|
||||
exc.read(MAX_ERROR_BYTES)
|
||||
print(f"SCM broker request failed with HTTP {exc.code}", file=sys.stderr)
|
||||
return 1
|
||||
except (OSError, PolicyError, ValueError, json.JSONDecodeError):
|
||||
print(
|
||||
"Forgejo request rejected or unavailable; no credential was disclosed",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
490
services/hermes/scm-common/scripts/gitea_api_policy.py
Normal file
490
services/hermes/scm-common/scripts/gitea_api_policy.py
Normal file
@ -0,0 +1,490 @@
|
||||
"""Validation policy for Hermes' least-authority Atlas Forgejo client.
|
||||
|
||||
The screening catches structured credential assignments, known token formats,
|
||||
and long high-entropy values. It is a fail-closed accident barrier, not proof
|
||||
that text is secret-free: a short or multiword secret under an innocuous key is
|
||||
not reliably distinguishable from prose and must never be supplied by callers.
|
||||
JSON depth/nodes/documents and multiline gaps are deliberately bounded; large
|
||||
or unusual configuration blobs are not a supported pull-request field format.
|
||||
"""
|
||||
|
||||
# ruff: noqa: SIM905
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import urllib.parse
|
||||
from collections import Counter
|
||||
from math import log2
|
||||
|
||||
REPO_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,99}\Z")
|
||||
SHA_RE = re.compile(r"[0-9a-fA-F]{40}\Z")
|
||||
DRAFT_TITLE_PREFIX = "WIP: "
|
||||
GIT_BIN = "/usr/bin/git"
|
||||
MAX_QUERY_LENGTH = 128
|
||||
MAX_QUERY_ITEMS = 3
|
||||
MAX_QUERY_KEY_LENGTH = 16
|
||||
MAX_QUERY_VALUE_LENGTH = 16
|
||||
MAX_PAGE = 10_000
|
||||
MAX_LIMIT = 50
|
||||
MAX_PR_NUMBER = 2_147_483_647
|
||||
MAX_PR_NUMBER_DIGITS = 10
|
||||
MAX_REF_CHARACTERS = 200
|
||||
MAX_REF_UTF8_BYTES = 255
|
||||
MAX_TITLE_UTF8_BYTES = 512
|
||||
MAX_BODY_UTF8_BYTES = 32_768
|
||||
MAX_JSON_DOCUMENTS = 32
|
||||
MAX_JSON_NODES = 2_048
|
||||
MAX_JSON_DEPTH = 32
|
||||
CANONICAL_QUERY_RE = re.compile(r"[A-Za-z0-9_=&-]*\Z")
|
||||
ASSIGNMENT_RE = re.compile(
|
||||
r"""(?mx)
|
||||
(?<![A-Za-z0-9_.-])
|
||||
(?P<key_quote>[\"']?)
|
||||
(?P<key>\.?[A-Za-z][A-Za-z0-9_.-]{0,127})
|
||||
(?P=key_quote)
|
||||
[ \t]*(?P<separator>[:=])[ \t]*
|
||||
(?P<value>
|
||||
\"(?:\\.|[^\"\\\r\n])*\" |
|
||||
'(?:\\.|[^'\\\r\n])*' |
|
||||
[^\r\n,;}&]{0,2048}
|
||||
)
|
||||
"""
|
||||
)
|
||||
JSON_KEY_RE = re.compile(r'"(?P<key>[^"]{1,512})"(?P<spacing>[\x00-\x20\x7f]{0,256}):')
|
||||
MULTILINE_ASSIGNMENT_RE = re.compile(
|
||||
r"""(?mx)
|
||||
(?<![A-Za-z0-9_.-])
|
||||
(?P<key_quote>["']?)
|
||||
(?P<key>\.?[A-Za-z][A-Za-z0-9_.-]{0,127})
|
||||
(?P=key_quote)
|
||||
[ \t]{0,64}(?:\r?\n[ \t]{0,64}){0,4}
|
||||
(?P<separator>[:=])
|
||||
[ \t]{0,64}(?:\r?\n[ \t]{0,64}){1,4}
|
||||
(?P<value>[^\r\n,;}&]{1,2048})
|
||||
"""
|
||||
)
|
||||
CAMEL_BOUNDARY_RE = re.compile(r"(?<=[a-z0-9])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])")
|
||||
CREDENTIAL_SCHEME_RE = re.compile(
|
||||
r"(?i)\b(?:bearer|basic|token)\s+"
|
||||
r"(?=[A-Za-z0-9+/_=.:-]{8,}\b)(?=[A-Za-z0-9+/_=.:-]*[0-9+/_=.-])"
|
||||
r"[A-Za-z0-9+/_=.:-]{8,}"
|
||||
)
|
||||
STANDALONE_SECRET_PATTERNS = tuple(
|
||||
re.compile(pattern, re.IGNORECASE)
|
||||
for pattern in (
|
||||
r"(?<![A-Za-z0-9])gh[pousr]_[A-Za-z0-9]{20,}(?![A-Za-z0-9])",
|
||||
r"(?<![A-Za-z0-9])github_pat_[A-Za-z0-9_]{20,}(?![A-Za-z0-9])",
|
||||
r"(?<![A-Za-z0-9])glpat-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9])",
|
||||
r"(?<![A-Za-z0-9])xox[baprs]-[A-Za-z0-9-]{20,}(?![A-Za-z0-9])",
|
||||
r"(?<![A-Za-z0-9])sk-(?:ant-|proj-)?[A-Za-z0-9_-]{20,}(?![A-Za-z0-9])",
|
||||
r"(?<![A-Za-z0-9])[rs]k_(?:test|live)_[A-Za-z0-9]{20,}(?![A-Za-z0-9])",
|
||||
r"(?<![A-Za-z0-9])whsec_[A-Za-z0-9]{20,}(?![A-Za-z0-9])",
|
||||
r"(?<![A-Za-z0-9])npm_[A-Za-z0-9]{20,}(?![A-Za-z0-9])",
|
||||
r"(?<![A-Za-z0-9])pypi-[A-Za-z0-9_-]{30,}(?![A-Za-z0-9])",
|
||||
r"(?<![A-Za-z0-9])hf_[A-Za-z0-9]{20,}(?![A-Za-z0-9])",
|
||||
r"(?<![A-Za-z0-9])(?:gta|gto|gitea|forgejo)_[A-Za-z0-9_-]{20,}(?![A-Za-z0-9])",
|
||||
r"(?<![A-Za-z0-9])ya29\.[A-Za-z0-9_-]{20,}(?![A-Za-z0-9])",
|
||||
r"(?<![A-Za-z0-9])oy2[A-Za-z0-9]{40,}(?![A-Za-z0-9])",
|
||||
r"(?<![A-Za-z0-9])SG\.[A-Za-z0-9_-]{16,}\.[A-Za-z0-9_-]{20,}(?![A-Za-z0-9])",
|
||||
r"(?<![A-Za-z0-9])SK[0-9a-f]{32}(?![A-Za-z0-9])",
|
||||
r"(?<![A-Za-z0-9])(?:AKIA|ASIA)[0-9A-Z]{16}(?![A-Za-z0-9])",
|
||||
r"(?<![A-Za-z0-9])AIza[0-9A-Za-z_-]{35}(?![A-Za-z0-9])",
|
||||
r"(?<![A-Za-z0-9])eyJ[A-Za-z0-9_-]{12,}\.[A-Za-z0-9_-]{12,}\.[A-Za-z0-9_-]{8,}(?![A-Za-z0-9])",
|
||||
r"-----BEGIN\s+[A-Z0-9 ][A-Z0-9 -]{1,62}-----",
|
||||
r"\bssh-(?:rsa|ed25519|dss|ecdsa-[A-Za-z0-9-]+)\s+[A-Za-z0-9+/]{16,}={0,3}",
|
||||
r"https://hooks\.slack\.com/services/[A-Za-z0-9/_-]{20,}",
|
||||
r"https://(?:discord(?:app)?\.com)/api/webhooks/[0-9]+/[A-Za-z0-9._-]{20,}",
|
||||
r"https://[^\s/]*webhook\.office\.com/[^\s]{20,}",
|
||||
)
|
||||
)
|
||||
HIGH_ENTROPY_TOKEN_RE = re.compile(
|
||||
r"(?<![A-Za-z0-9+/_=-])[A-Za-z0-9+/_=-]{48,16384}(?![A-Za-z0-9+/_=-])"
|
||||
)
|
||||
PROSE_LEAD_WORDS = frozenset(
|
||||
"add allow change check describe document ensure explain fix handle keep preserve "
|
||||
"prevent reject remove review support test validate".split()
|
||||
)
|
||||
SENSITIVE_KEY_WORDS = frozenset(
|
||||
"auth authentication authorization auths basic bearer credential credentials "
|
||||
"passwd password pat private sas secret secrets sig signature signing token tokens "
|
||||
"webhook".split()
|
||||
)
|
||||
KEY_MODIFIER_WORDS = frozenset(
|
||||
"access account api auth client encryption identity private registry secret service "
|
||||
"session signing ssh".split()
|
||||
)
|
||||
SENSITIVE_COMPACT_KEYS = frozenset(
|
||||
"accessid accesskeyid accountkey clientemail clientid connectionstring "
|
||||
"dockerconfigjson privatekeyid serviceaccountkey sharedaccesssignature".split()
|
||||
)
|
||||
SENSITIVE_COMPACT_SUFFIXES = tuple(
|
||||
"accesskey accesskeyid accountkey apikey authkey clientemail clientid clientkey "
|
||||
"connectionstring credential credentials encryptionkey identitykey password passwd "
|
||||
"privatekeyid privatekey secret secretkey servicekey sessionkey signature signingkey "
|
||||
"sshkey token webhook".split()
|
||||
)
|
||||
|
||||
|
||||
class PolicyError(ValueError):
|
||||
"""Raised when a requested Forgejo operation exceeds the safe boundary."""
|
||||
|
||||
|
||||
def _validate_repo(repo: str) -> str:
|
||||
if not REPO_RE.fullmatch(repo) or repo in {".", ".."}:
|
||||
raise PolicyError("repository name is outside the Atlas allowlist")
|
||||
return repo
|
||||
|
||||
|
||||
def _reject_forbidden(value: object, name: str, forbidden: tuple[str, ...]) -> None:
|
||||
"""Reject exact runtime credentials before invoking helpers or the network."""
|
||||
if not isinstance(value, str) or any(secret and secret in value for secret in forbidden):
|
||||
raise PolicyError(f"{name} contains runtime credential material")
|
||||
|
||||
|
||||
def _validate_ref_bounds(
|
||||
value: object, name: str, *, forbidden: tuple[str, ...] = ()
|
||||
) -> str:
|
||||
"""Reject oversized or non-encodable refs before invoking Git."""
|
||||
_reject_forbidden(value, name, forbidden)
|
||||
if not value or len(value) > MAX_REF_CHARACTERS:
|
||||
raise PolicyError(f"{name} exceeds the safe branch-name limit")
|
||||
try:
|
||||
encoded = value.encode("utf-8", errors="strict")
|
||||
except UnicodeEncodeError as exc:
|
||||
raise PolicyError(f"{name} is not valid UTF-8 text") from exc
|
||||
if len(encoded) > MAX_REF_UTF8_BYTES:
|
||||
raise PolicyError(f"{name} exceeds the safe UTF-8 branch-name limit")
|
||||
return value
|
||||
|
||||
|
||||
def _validate_ref(
|
||||
value: object, name: str, *, forbidden: tuple[str, ...] = ()
|
||||
) -> str:
|
||||
"""Validate the complete Git ref grammar using Git itself."""
|
||||
value = _validate_ref_bounds(value, name, forbidden=forbidden)
|
||||
if value.startswith("-"):
|
||||
raise PolicyError(f"{name} must be a same-repository branch name")
|
||||
if any(ord(character) < 32 or ord(character) == 127 for character in value):
|
||||
raise PolicyError(f"{name} must be a safe same-repository branch name")
|
||||
result = subprocess.run(
|
||||
[GIT_BIN, "check-ref-format", f"refs/heads/{value}"],
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
check=False,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise PolicyError(f"{name} must be a safe same-repository branch name")
|
||||
return value
|
||||
|
||||
|
||||
def _validate_sha(value: object, name: str = "head SHA") -> str:
|
||||
if not isinstance(value, str) or not SHA_RE.fullmatch(value):
|
||||
raise PolicyError(f"{name} must be a full Git SHA-1")
|
||||
return value.lower()
|
||||
|
||||
|
||||
def _validate_pr_number_segment(value: object) -> int:
|
||||
"""Validate a canonical bounded pull-request number from an API path."""
|
||||
if (
|
||||
not isinstance(value, str)
|
||||
or len(value) > MAX_PR_NUMBER_DIGITS
|
||||
or not re.fullmatch(r"[1-9][0-9]*", value)
|
||||
):
|
||||
raise PolicyError("pull-request number must use bounded canonical ASCII digits")
|
||||
number = int(value)
|
||||
if number > MAX_PR_NUMBER:
|
||||
raise PolicyError("pull-request number is outside its allowed range")
|
||||
return number
|
||||
|
||||
|
||||
def _validate_pr_number(value: object) -> int:
|
||||
"""Validate a bounded pull-request number returned by Forgejo."""
|
||||
if (
|
||||
not isinstance(value, int)
|
||||
or isinstance(value, bool)
|
||||
or not 1 <= value <= MAX_PR_NUMBER
|
||||
):
|
||||
raise PolicyError("Forgejo omitted a valid pull-request number")
|
||||
return value
|
||||
|
||||
|
||||
def _validate_text(
|
||||
value: object,
|
||||
name: str,
|
||||
maximum: int,
|
||||
maximum_bytes: int,
|
||||
*,
|
||||
required: bool,
|
||||
) -> str:
|
||||
if not isinstance(value, str):
|
||||
raise PolicyError(f"{name} must be text")
|
||||
if required and not value.strip():
|
||||
raise PolicyError(f"{name} must not be empty")
|
||||
try:
|
||||
encoded = value.encode("utf-8", errors="strict")
|
||||
except UnicodeEncodeError as exc:
|
||||
raise PolicyError(f"{name} is not valid UTF-8 text") from exc
|
||||
if len(value) > maximum or len(encoded) > maximum_bytes or "\x00" in value:
|
||||
raise PolicyError(f"{name} exceeds the safe request limit")
|
||||
return value
|
||||
|
||||
|
||||
def _normalized_key(value: str) -> tuple[tuple[str, ...], str]:
|
||||
"""Split camelCase and separator-based assignment keys into semantics."""
|
||||
separated = CAMEL_BOUNDARY_RE.sub(" ", value.strip(".\"'"))
|
||||
words = tuple(re.findall(r"[A-Za-z0-9]+", separated.lower()))
|
||||
return words, "".join(words)
|
||||
|
||||
|
||||
def _is_sensitive_key(value: str) -> bool:
|
||||
words, compact = _normalized_key(value)
|
||||
word_set = set(words)
|
||||
if word_set & SENSITIVE_KEY_WORDS:
|
||||
return True
|
||||
if compact in SENSITIVE_COMPACT_KEYS:
|
||||
return True
|
||||
if compact.endswith(SENSITIVE_COMPACT_SUFFIXES):
|
||||
return True
|
||||
if "key" in word_set and word_set & KEY_MODIFIER_WORDS:
|
||||
return True
|
||||
if {"client", "email"} <= word_set or {"client", "id"} <= word_set:
|
||||
return True
|
||||
if {"access", "id"} <= word_set or {"connection", "string"} <= word_set:
|
||||
return True
|
||||
return compact.endswith("configjson")
|
||||
|
||||
|
||||
def _strip_assignment_value(value: str) -> tuple[str, bool]:
|
||||
stripped = value.strip()
|
||||
quoted = (
|
||||
len(stripped) >= 2 and stripped[0] in {'"', "'"} and stripped[-1] == stripped[0]
|
||||
)
|
||||
if quoted:
|
||||
stripped = stripped[1:-1].strip()
|
||||
return stripped, quoted
|
||||
|
||||
|
||||
def _looks_like_prose(value: str) -> bool:
|
||||
words = re.findall(r"[A-Za-z]+(?:[-'][A-Za-z]+)?", value)
|
||||
if len(words) < 2:
|
||||
return False
|
||||
if re.search(r"[$`{}\[\]\\@/:=+_]", value):
|
||||
return False
|
||||
return words[0].lower() in PROSE_LEAD_WORDS
|
||||
|
||||
|
||||
def _looks_sensitive_assignment_value(value: str) -> bool:
|
||||
stripped, quoted = _strip_assignment_value(value)
|
||||
if not stripped:
|
||||
return False
|
||||
if quoted or stripped.startswith(("{", "[", "|", ">", "!!", "&", "*")):
|
||||
return True
|
||||
if CREDENTIAL_SCHEME_RE.search(stripped):
|
||||
return True
|
||||
if any(pattern.search(stripped) for pattern in STANDALONE_SECRET_PATTERNS):
|
||||
return True
|
||||
if re.search(r"(?:https?://|[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+)", stripped):
|
||||
return True
|
||||
if re.search(r"(?:\$\{|\$\(|\\[nrt]|[A-Za-z0-9+/]{16,}={0,3}\Z)", stripped):
|
||||
return True
|
||||
return not _looks_like_prose(stripped)
|
||||
|
||||
|
||||
def _is_structural_credential_assignment(key: str, value: str) -> bool:
|
||||
words, compact_key = _normalized_key(key)
|
||||
stripped, _quoted = _strip_assignment_value(value)
|
||||
_value_words, compact_value = _normalized_key(stripped)
|
||||
return (
|
||||
compact_key == "type" and compact_value in {"serviceaccount", "credential"}
|
||||
) or ("service" in words and "account" in words and bool(stripped))
|
||||
|
||||
|
||||
def _decode_json_key(raw: str) -> tuple[str, bool]:
|
||||
"""Decode one bounded JSON key without silently accepting bad escapes."""
|
||||
try:
|
||||
decoded = json.loads(f'"{raw}"')
|
||||
except (json.JSONDecodeError, UnicodeError):
|
||||
approximate = re.sub(r"[^A-Za-z0-9]+", "_", raw.replace("\\", ""))
|
||||
return approximate, False
|
||||
return decoded if isinstance(decoded, str) else "", True
|
||||
|
||||
|
||||
def _json_value_has_sensitive_assignment(
|
||||
value: object, *, depth: int = 0, nodes: list[int] | None = None
|
||||
) -> bool:
|
||||
"""Walk one decoded JSON value with explicit depth and node ceilings."""
|
||||
if nodes is None:
|
||||
nodes = [0]
|
||||
nodes[0] += 1
|
||||
if nodes[0] > MAX_JSON_NODES or depth > MAX_JSON_DEPTH:
|
||||
raise PolicyError("structured pull-request text exceeds the scan limit")
|
||||
if isinstance(value, dict):
|
||||
for key, assigned in value.items():
|
||||
if not isinstance(key, str):
|
||||
continue
|
||||
if _is_sensitive_key(key):
|
||||
return True
|
||||
assigned_text = assigned if isinstance(assigned, str) else ""
|
||||
if _is_structural_credential_assignment(key, assigned_text):
|
||||
return True
|
||||
if _json_value_has_sensitive_assignment(
|
||||
assigned, depth=depth + 1, nodes=nodes
|
||||
):
|
||||
return True
|
||||
elif isinstance(value, list):
|
||||
return any(
|
||||
_json_value_has_sensitive_assignment(item, depth=depth + 1, nodes=nodes)
|
||||
for item in value
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
def _decoded_json_documents(value: str):
|
||||
"""Yield bounded embedded JSON object or array fragments."""
|
||||
decoder = json.JSONDecoder()
|
||||
index = 0
|
||||
attempts = 0
|
||||
while attempts < MAX_JSON_DOCUMENTS:
|
||||
positions = [
|
||||
position for token in "{[" if (position := value.find(token, index)) >= 0
|
||||
]
|
||||
if not positions:
|
||||
return
|
||||
start = min(positions)
|
||||
attempts += 1
|
||||
try:
|
||||
document, end = decoder.raw_decode(value, start)
|
||||
except (json.JSONDecodeError, RecursionError, ValueError):
|
||||
index = start + 1
|
||||
continue
|
||||
yield document
|
||||
index = max(end, start + 1)
|
||||
if "{" in value[index:] or "[" in value[index:]:
|
||||
raise PolicyError("structured pull-request text exceeds the scan limit")
|
||||
|
||||
|
||||
def _has_structured_sensitive_assignment(value: str) -> bool:
|
||||
"""Detect bounded JSON and multiline YAML/env credential assignments."""
|
||||
for candidate in JSON_KEY_RE.finditer(value):
|
||||
decoded, valid = _decode_json_key(candidate.group("key"))
|
||||
if _is_sensitive_key(decoded):
|
||||
return True
|
||||
if not valid and _is_sensitive_key(candidate.group("key").replace("\\", "")):
|
||||
return True
|
||||
for candidate in MULTILINE_ASSIGNMENT_RE.finditer(value):
|
||||
key = candidate.group("key")
|
||||
assigned = candidate.group("value")
|
||||
if _is_structural_credential_assignment(key, assigned) or (
|
||||
_is_sensitive_key(key) and _looks_sensitive_assignment_value(assigned)
|
||||
):
|
||||
return True
|
||||
return any(
|
||||
_json_value_has_sensitive_assignment(item)
|
||||
for item in _decoded_json_documents(value)
|
||||
)
|
||||
|
||||
|
||||
def _has_high_entropy_token(value: str) -> bool:
|
||||
for match in HIGH_ENTROPY_TOKEN_RE.finditer(value):
|
||||
token = match.group(0)
|
||||
if len(token) > 256:
|
||||
return True
|
||||
groups = sum(
|
||||
bool(re.search(pattern, token))
|
||||
for pattern in (r"[a-z]", r"[A-Z]", r"[0-9]", r"[+/_=-]")
|
||||
)
|
||||
counts = Counter(token)
|
||||
entropy = -sum(
|
||||
(count / len(token)) * log2(count / len(token)) for count in counts.values()
|
||||
)
|
||||
if groups >= 3 and entropy >= 4.0:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _reject_sensitive(value: str, name: str, forbidden: tuple[str, ...]) -> None:
|
||||
if any(secret and secret in value for secret in forbidden):
|
||||
raise PolicyError(f"pull-request {name} contains runtime credential material")
|
||||
if CREDENTIAL_SCHEME_RE.search(value):
|
||||
raise PolicyError(f"pull-request {name} resembles credential material")
|
||||
if any(pattern.search(value) for pattern in STANDALONE_SECRET_PATTERNS):
|
||||
raise PolicyError(f"pull-request {name} resembles credential material")
|
||||
if _has_structured_sensitive_assignment(value):
|
||||
raise PolicyError(f"pull-request {name} resembles credential material")
|
||||
for candidate in ASSIGNMENT_RE.finditer(value):
|
||||
key = candidate.group("key")
|
||||
assigned = candidate.group("value")
|
||||
if _is_structural_credential_assignment(key, assigned) or (
|
||||
_is_sensitive_key(key) and _looks_sensitive_assignment_value(assigned)
|
||||
):
|
||||
raise PolicyError(f"pull-request {name} resembles credential material")
|
||||
if _has_high_entropy_token(value):
|
||||
raise PolicyError(f"pull-request {name} resembles credential material")
|
||||
|
||||
|
||||
def _validate_body(value: object, *, forbidden: tuple[str, ...] = ()) -> str:
|
||||
body = _validate_text(value, "body", 16384, MAX_BODY_UTF8_BYTES, required=False)
|
||||
_reject_sensitive(body, "body", forbidden)
|
||||
return body
|
||||
|
||||
|
||||
def _draft_title(value: object, *, forbidden: tuple[str, ...] = ()) -> str:
|
||||
"""Return a bounded, secret-screened Gitea draft title."""
|
||||
title = _validate_text(
|
||||
value, "title", 251, MAX_TITLE_UTF8_BYTES, required=True
|
||||
).strip()
|
||||
_reject_sensitive(title, "title", forbidden)
|
||||
for prefix in ("WIP:", "[WIP]"):
|
||||
if title.upper().startswith(prefix):
|
||||
title = title[len(prefix) :].lstrip()
|
||||
break
|
||||
if not title:
|
||||
raise PolicyError("title must contain text after the draft prefix")
|
||||
return DRAFT_TITLE_PREFIX + title
|
||||
|
||||
|
||||
def _validate_query(target: urllib.parse.SplitResult, allowed: set[str]) -> None:
|
||||
"""Accept only short canonical ASCII query strings with bounded pagination."""
|
||||
raw = target.query
|
||||
if (
|
||||
len(raw) > MAX_QUERY_LENGTH
|
||||
or not raw.isascii()
|
||||
or not CANONICAL_QUERY_RE.fullmatch(raw)
|
||||
or "%" in raw
|
||||
):
|
||||
raise PolicyError("API query must use short canonical ASCII form")
|
||||
try:
|
||||
pairs = urllib.parse.parse_qsl(raw, keep_blank_values=True, strict_parsing=True)
|
||||
except ValueError as exc:
|
||||
raise PolicyError("invalid API query") from exc
|
||||
if len(pairs) > MAX_QUERY_ITEMS:
|
||||
raise PolicyError("too many API query parameters")
|
||||
if len({key for key, _ in pairs}) != len(pairs):
|
||||
raise PolicyError("duplicate API query parameters are not allowed")
|
||||
if any(
|
||||
not key
|
||||
or len(key) > MAX_QUERY_KEY_LENGTH
|
||||
or len(value) > MAX_QUERY_VALUE_LENGTH
|
||||
for key, value in pairs
|
||||
):
|
||||
raise PolicyError("API query key or value exceeds its safe limit")
|
||||
if any(key not in allowed for key, _ in pairs):
|
||||
raise PolicyError("API query parameter is outside the read allowlist")
|
||||
values = dict(pairs)
|
||||
for name in ("page", "limit"):
|
||||
if name not in values:
|
||||
continue
|
||||
if not re.fullmatch(r"[0-9]+", values[name]):
|
||||
raise PolicyError(f"{name} must use ASCII decimal digits")
|
||||
number = int(values[name])
|
||||
if values[name] != str(number):
|
||||
raise PolicyError(f"{name} must use canonical ASCII decimal form")
|
||||
ceiling = MAX_PAGE if name == "page" else MAX_LIMIT
|
||||
if not 1 <= number <= ceiling:
|
||||
raise PolicyError(f"{name} is outside its allowed range")
|
||||
if "state" in values and values["state"] not in {"open", "closed", "all"}:
|
||||
raise PolicyError("pull-request state is invalid")
|
||||
143
services/hermes/scm-common/scripts/receive_pack_scan.py
Normal file
143
services/hermes/scm-common/scripts/receive_pack_scan.py
Normal file
@ -0,0 +1,143 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate Git receive-pack requests and scan pushed object content.
|
||||
|
||||
Ref commands may only create new namespaced feature branches. Every pushed
|
||||
object is then inflated in quarantine and its decompressed payload scanned,
|
||||
so credentials cannot ride through the broker inside compressed pack data.
|
||||
The screening is a fail-closed accident barrier for credential-shaped
|
||||
content: exact runtime-token forms, private keys, SSH key material, and
|
||||
known provider token formats. Entropy heuristics are deliberately excluded
|
||||
so SOPS ciphertext, pinned digests, and lock files remain pushable; a short
|
||||
secret disguised as prose is not reliably detectable and must never be
|
||||
supplied by callers.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import re
|
||||
from typing import BinaryIO
|
||||
|
||||
import git_pack_objects
|
||||
from gitea_api_policy import (
|
||||
STANDALONE_SECRET_PATTERNS,
|
||||
PolicyError,
|
||||
_validate_ref,
|
||||
)
|
||||
|
||||
ZERO_SHA = b"0" * 40
|
||||
MAX_PUSH_COMMANDS = 16
|
||||
SCAN_CHUNK = 256 * 1024
|
||||
SCAN_CARRY = 8 * 1024
|
||||
FEATURE_REF_RE = re.compile(
|
||||
r"refs/heads/(?:(?:feature|fix|hermes|handoff)/[A-Za-z0-9][A-Za-z0-9._/-]{0,190})\Z"
|
||||
)
|
||||
PKT_HEADER_RE = re.compile(rb"[0-9a-f]{4}\Z")
|
||||
CONTENT_SECRET_PATTERNS = tuple(
|
||||
re.compile(pattern.pattern.encode("ascii"), re.IGNORECASE)
|
||||
for pattern in STANDALONE_SECRET_PATTERNS
|
||||
) + (
|
||||
re.compile(rb"(?i)(?<![A-Za-z0-9])AGE-SECRET-KEY-1[A-Z0-9]{20,}"),
|
||||
re.compile(rb"(?i)PuTTY-User-Key-File"),
|
||||
re.compile(
|
||||
rb"(?im)^[ \t]*authorization[ \t]*:[ \t]*(?:basic|bearer|token)[ \t]+"
|
||||
rb"(?=[A-Za-z0-9+/_=.:-]*[0-9+/=.-])[A-Za-z0-9+/_=.:-]{8,}"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _read_pkt_line(stream: BinaryIO) -> bytes | None:
|
||||
"""Read one pkt-line; None marks the flush packet before the pack."""
|
||||
header = stream.read(4)
|
||||
if not header:
|
||||
raise PolicyError("Git receive-pack request omitted its command terminator")
|
||||
if not PKT_HEADER_RE.fullmatch(header):
|
||||
raise PolicyError("Git receive-pack command framing is invalid")
|
||||
size = int(header, 16)
|
||||
if size == 0:
|
||||
return None
|
||||
if size < 4:
|
||||
raise PolicyError("Git receive-pack command framing is invalid")
|
||||
value = stream.read(size - 4)
|
||||
if len(value) != size - 4:
|
||||
raise PolicyError("Git receive-pack command framing is invalid")
|
||||
return value
|
||||
|
||||
|
||||
def _validate_command(line: bytes, token: str, forbidden: tuple[bytes, ...]) -> None:
|
||||
"""Permit only creation of one new, namespaced feature branch."""
|
||||
if any(form in line for form in forbidden):
|
||||
raise PolicyError("Git request contains runtime credential material")
|
||||
command = line.rstrip(b"\n").split(b"\x00", 1)[0]
|
||||
fields = command.split(b" ")
|
||||
if len(fields) != 3 or not all(
|
||||
re.fullmatch(rb"[0-9a-f]{40}", item) for item in fields[:2]
|
||||
):
|
||||
raise PolicyError("Git receive-pack ref command is invalid")
|
||||
old_sha, new_sha, raw_ref = fields
|
||||
if old_sha != ZERO_SHA or new_sha == ZERO_SHA:
|
||||
raise PolicyError("Git broker permits only new feature-branch creation")
|
||||
try:
|
||||
ref = raw_ref.decode("ascii")
|
||||
except UnicodeDecodeError as exc:
|
||||
raise PolicyError("Git ref must be canonical ASCII") from exc
|
||||
if not FEATURE_REF_RE.fullmatch(ref):
|
||||
raise PolicyError("Git push is limited to namespaced feature branches")
|
||||
_validate_ref(ref.removeprefix("refs/heads/"), "head", forbidden=(token,))
|
||||
|
||||
|
||||
def _scan_payload(payload: BinaryIO, forbidden: tuple[bytes, ...]) -> None:
|
||||
"""Scan one decompressed object payload across chunk boundaries."""
|
||||
carry = b""
|
||||
while True:
|
||||
chunk = payload.read(SCAN_CHUNK)
|
||||
if not chunk:
|
||||
return
|
||||
window = carry + chunk
|
||||
if any(form in window for form in forbidden):
|
||||
raise PolicyError("Git push contains runtime credential material")
|
||||
if any(pattern.search(window) for pattern in CONTENT_SECRET_PATTERNS):
|
||||
raise PolicyError("Git push contains credential-shaped content")
|
||||
carry = window[-SCAN_CARRY:]
|
||||
|
||||
|
||||
def _scan_pack(stream: BinaryIO, forbidden: tuple[bytes, ...]) -> None:
|
||||
objects = git_pack_objects.unpack_objects(stream)
|
||||
try:
|
||||
if stream.read(1):
|
||||
raise PolicyError("Git request has bytes after its pack")
|
||||
for _type_code, payload in objects:
|
||||
_scan_payload(payload, forbidden)
|
||||
finally:
|
||||
for _type_code, payload in objects:
|
||||
payload.close()
|
||||
|
||||
|
||||
def validate_receive_pack(
|
||||
body: bytes | BinaryIO, token: str, forbidden: tuple[bytes, ...]
|
||||
) -> None:
|
||||
"""Validate one whole receive-pack request: commands, pack, and content."""
|
||||
stream = io.BytesIO(body) if isinstance(body, bytes) else body
|
||||
stream.seek(0)
|
||||
try:
|
||||
commands = 0
|
||||
while True:
|
||||
line = _read_pkt_line(stream)
|
||||
if line is None:
|
||||
break
|
||||
_validate_command(line, token, forbidden)
|
||||
commands += 1
|
||||
if commands > MAX_PUSH_COMMANDS:
|
||||
raise PolicyError("Git push contains too many ref commands")
|
||||
if commands == 0:
|
||||
raise PolicyError("Git receive-pack request has no ref command")
|
||||
_scan_pack(stream, forbidden)
|
||||
finally:
|
||||
stream.seek(0)
|
||||
|
||||
|
||||
def scan_loose_object(value: bytes, forbidden: tuple[bytes, ...]) -> int:
|
||||
"""Scan one quarantined loose object and return its parsed type code."""
|
||||
type_code, payload = git_pack_objects.parse_loose_object(value)
|
||||
_scan_payload(io.BytesIO(payload), forbidden)
|
||||
return type_code
|
||||
455
services/hermes/scm-common/scripts/scm_broker.py
Normal file
455
services/hermes/scm-common/scripts/scm_broker.py
Normal file
@ -0,0 +1,455 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Serve the credential-isolated Atlas API and Git smart-HTTP boundary."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import json
|
||||
import re
|
||||
import socket
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from http.server import BaseHTTPRequestHandler
|
||||
from typing import BinaryIO
|
||||
|
||||
import deadline_http
|
||||
from gitea_api import (
|
||||
CANONICAL_BASE_URL,
|
||||
PolicyError,
|
||||
create_draft,
|
||||
read,
|
||||
read_token,
|
||||
)
|
||||
from gitea_api_policy import _reject_forbidden, _validate_repo
|
||||
from receive_pack_scan import validate_receive_pack
|
||||
from scm_broker_io import RejectRedirect, response_status as _status, spool_response
|
||||
from scm_broker_server import AbsoluteHeaderDeadlineMixin, BoundedThreadingHTTPServer
|
||||
|
||||
BROKER_PORT = 9081
|
||||
GIT_USER = "hermes-automation"
|
||||
MAX_CONTROL_BODY = 64 * 1024
|
||||
MAX_GIT_REQUEST = 128 * 1024 * 1024
|
||||
MAX_GIT_RESPONSE = 128 * 1024 * 1024
|
||||
MAX_HEADERS = 32
|
||||
MAX_HEADER_BYTES = 16 * 1024
|
||||
SPOOL_MEMORY_LIMIT = 1024 * 1024
|
||||
STREAM_CHUNK = 64 * 1024
|
||||
INBOUND_HEADER_TIMEOUT = 10.0
|
||||
INBOUND_BODY_TIMEOUT = 120.0
|
||||
UPSTREAM_DEADLINE_SECONDS = 300.0
|
||||
GIT_PATH_RE = re.compile(
|
||||
r"/git/atlas/(?P<repo>[A-Za-z0-9][A-Za-z0-9._-]{0,99})\.git/"
|
||||
r"(?P<operation>info/refs|git-upload-pack|git-receive-pack)\Z"
|
||||
)
|
||||
|
||||
|
||||
def _read_bounded(
|
||||
stream,
|
||||
maximum: int,
|
||||
length: int | None = None,
|
||||
*,
|
||||
deadline: float | None = None,
|
||||
set_timeout=None,
|
||||
) -> bytes:
|
||||
if length is not None and not 0 <= length <= maximum:
|
||||
raise PolicyError("SCM request exceeds the safe size limit")
|
||||
wanted = maximum + 1 if length is None else length
|
||||
chunks: list[bytes] = []
|
||||
total = 0
|
||||
while total < wanted:
|
||||
if deadline is not None:
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
raise PolicyError("SCM request body deadline exceeded")
|
||||
if set_timeout is not None:
|
||||
set_timeout(remaining)
|
||||
chunk = stream.read(min(STREAM_CHUNK, wanted - total))
|
||||
if not chunk:
|
||||
break
|
||||
chunks.append(chunk)
|
||||
total += len(chunk)
|
||||
value = b"".join(chunks)
|
||||
if len(value) > maximum or (length is not None and len(value) != length):
|
||||
raise PolicyError("SCM request exceeds the safe size limit")
|
||||
return value
|
||||
|
||||
|
||||
def _spool_bounded(
|
||||
stream,
|
||||
maximum: int,
|
||||
length: int,
|
||||
*,
|
||||
token: str,
|
||||
context: str,
|
||||
deadline: float,
|
||||
set_timeout=None,
|
||||
) -> tuple[BinaryIO, int]:
|
||||
"""Copy a fixed-length exchange through a bounded-memory disk spool."""
|
||||
if not 0 <= length <= maximum:
|
||||
raise PolicyError("SCM request exceeds the safe size limit")
|
||||
spool = tempfile.SpooledTemporaryFile( # noqa: SIM115 - caller owns returned spool
|
||||
max_size=SPOOL_MEMORY_LIMIT, dir="/tmp"
|
||||
)
|
||||
remaining = length
|
||||
forms = _credential_forms(token)
|
||||
carry = b""
|
||||
try:
|
||||
while remaining:
|
||||
remaining_time = deadline - time.monotonic()
|
||||
if remaining_time <= 0:
|
||||
raise PolicyError("SCM request body deadline exceeded")
|
||||
if set_timeout is not None:
|
||||
set_timeout(remaining_time)
|
||||
chunk = stream.read(min(STREAM_CHUNK, remaining))
|
||||
if not chunk:
|
||||
raise PolicyError("SCM request body ended early")
|
||||
candidate = carry + chunk
|
||||
if any(form in candidate for form in forms):
|
||||
raise PolicyError(f"{context} contains runtime credential material")
|
||||
width = max(len(form) for form in forms) - 1
|
||||
carry = candidate[-width:] if width else b""
|
||||
spool.write(chunk)
|
||||
remaining -= len(chunk)
|
||||
spool.seek(0)
|
||||
return spool, length
|
||||
except Exception:
|
||||
spool.close()
|
||||
raise
|
||||
|
||||
|
||||
def _spool_response(
|
||||
response, maximum: int, token: str, deadline_seconds: float
|
||||
) -> tuple[BinaryIO, int]:
|
||||
return spool_response(
|
||||
response,
|
||||
maximum,
|
||||
_credential_forms(token),
|
||||
memory_limit=SPOOL_MEMORY_LIMIT,
|
||||
chunk_size=STREAM_CHUNK,
|
||||
deadline_seconds=deadline_seconds,
|
||||
)
|
||||
|
||||
|
||||
def _load_json(handler: BaseHTTPRequestHandler) -> dict[str, object]:
|
||||
if handler.headers.get("Transfer-Encoding"):
|
||||
raise PolicyError("chunked broker control requests are not allowed")
|
||||
if handler.headers.get_content_type() != "application/json":
|
||||
raise PolicyError("broker control request must be JSON")
|
||||
length = _content_length(handler.headers, MAX_CONTROL_BODY)
|
||||
deadline = time.monotonic() + INBOUND_BODY_TIMEOUT
|
||||
value = json.loads(
|
||||
_read_bounded(
|
||||
handler.rfile,
|
||||
MAX_CONTROL_BODY,
|
||||
length,
|
||||
deadline=deadline,
|
||||
set_timeout=handler.connection.settimeout,
|
||||
)
|
||||
)
|
||||
if not isinstance(value, dict):
|
||||
raise PolicyError("broker control request must be an object")
|
||||
return value
|
||||
|
||||
|
||||
def _git_target(raw: str) -> tuple[str, str, str]:
|
||||
if (
|
||||
not raw.isascii()
|
||||
or any(ord(character) < 32 or ord(character) == 127 for character in raw)
|
||||
or "%" in raw
|
||||
or "\\" in raw
|
||||
or len(raw) > 512
|
||||
):
|
||||
raise PolicyError("Git target is not canonical ASCII")
|
||||
target = urllib.parse.urlsplit(raw)
|
||||
canonical = urllib.parse.urlunsplit(("", "", target.path, target.query, ""))
|
||||
if canonical != raw:
|
||||
raise PolicyError("Git target is not in exact canonical form")
|
||||
if target.scheme or target.netloc or target.fragment:
|
||||
raise PolicyError("Git target must be relative")
|
||||
match = GIT_PATH_RE.fullmatch(target.path)
|
||||
if not match:
|
||||
raise PolicyError("Git target is outside the Atlas allowlist")
|
||||
repo = _validate_repo(match.group("repo"))
|
||||
operation = match.group("operation")
|
||||
query = target.query
|
||||
if operation == "info/refs":
|
||||
if query not in {"service=git-upload-pack", "service=git-receive-pack"}:
|
||||
raise PolicyError("Git discovery service is outside the allowlist")
|
||||
service = query.removeprefix("service=")
|
||||
elif query:
|
||||
raise PolicyError("Git RPC query parameters are not allowed")
|
||||
else:
|
||||
service = operation
|
||||
return repo, operation, service
|
||||
|
||||
|
||||
def _content_length(headers: object, maximum: int) -> int:
|
||||
"""Parse one short canonical bounded Content-Length header."""
|
||||
raw = headers.get("Content-Length", "") # type: ignore[attr-defined]
|
||||
if (
|
||||
not isinstance(raw, str)
|
||||
or not raw.isascii()
|
||||
or len(raw) > 10
|
||||
or not raw.isdigit()
|
||||
):
|
||||
raise PolicyError("SCM request length is invalid")
|
||||
length = int(raw)
|
||||
if raw != str(length) or not 0 <= length <= maximum:
|
||||
raise PolicyError("SCM request length is invalid")
|
||||
return length
|
||||
|
||||
|
||||
def _credential_forms(token: str) -> tuple[bytes, ...]:
|
||||
basic = base64.b64encode(f"{GIT_USER}:{token}".encode())
|
||||
return token.encode("utf-8"), basic, b"Basic " + basic
|
||||
|
||||
|
||||
def _validate_receive_pack(body: bytes | BinaryIO, token: str) -> None:
|
||||
"""Permit only vetted new-feature-branch pushes with scanned contents."""
|
||||
validate_receive_pack(body, token, _credential_forms(token))
|
||||
|
||||
|
||||
def _guarded_opener(guard: deadline_http.StreamDeadline):
|
||||
"""Build an opener whose connection obeys one absolute stream deadline."""
|
||||
return urllib.request.build_opener(RejectRedirect(), *guard.handlers()).open
|
||||
|
||||
|
||||
def _upstream_git_request(
|
||||
target: str,
|
||||
*,
|
||||
method: str,
|
||||
body: bytes | BinaryIO | None,
|
||||
content_type: str | None,
|
||||
expected_type: str,
|
||||
token: str,
|
||||
body_length: int | None = None,
|
||||
stream_result: bool = False,
|
||||
opener=None,
|
||||
) -> bytes | tuple[BinaryIO, int]:
|
||||
credentials = base64.b64encode(f"{GIT_USER}:{token}".encode()).decode("ascii")
|
||||
headers = {"Accept": expected_type, "Authorization": f"Basic {credentials}"}
|
||||
if content_type:
|
||||
headers["Content-Type"] = content_type
|
||||
if body is not None:
|
||||
if body_length is None:
|
||||
if not isinstance(body, bytes):
|
||||
raise PolicyError("Git upstream request length is missing")
|
||||
body_length = len(body)
|
||||
headers["Content-Length"] = str(body_length)
|
||||
request = urllib.request.Request(
|
||||
CANONICAL_BASE_URL + target,
|
||||
data=body,
|
||||
method=method,
|
||||
headers=headers,
|
||||
)
|
||||
# One wall-clock budget covers connect, send, headers, and body read; the
|
||||
# watchdog closes the upstream socket if any phase overstays it.
|
||||
guard = deadline_http.StreamDeadline(UPSTREAM_DEADLINE_SECONDS)
|
||||
try:
|
||||
open_upstream = opener if opener is not None else _guarded_opener(guard)
|
||||
with open_upstream(request, timeout=guard.remaining()) as response:
|
||||
if _status(response) != 200:
|
||||
raise PolicyError("Git upstream returned an unexpected HTTP status")
|
||||
if response.headers.get_content_type() != expected_type:
|
||||
raise PolicyError("Git upstream returned an unexpected response type")
|
||||
result, result_length = _spool_response(
|
||||
response, MAX_GIT_RESPONSE, token, guard.remaining()
|
||||
)
|
||||
finally:
|
||||
guard.cancel()
|
||||
if stream_result:
|
||||
return result, result_length
|
||||
try:
|
||||
return result.read()
|
||||
finally:
|
||||
result.close()
|
||||
|
||||
|
||||
class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler):
|
||||
"""Expose only bounded metadata, draft creation, and smart-HTTP Git."""
|
||||
|
||||
server_version = "HermesSCMBroker/1"
|
||||
sys_version = ""
|
||||
header_deadline_seconds = INBOUND_HEADER_TIMEOUT
|
||||
|
||||
def _validate_headers(self) -> None:
|
||||
items = list(self.headers.items())
|
||||
if len(items) > MAX_HEADERS:
|
||||
raise PolicyError("SCM request has too many headers")
|
||||
total = 0
|
||||
for name, value in items:
|
||||
if not name.isascii() or not value.isascii():
|
||||
raise PolicyError("SCM request headers must be ASCII")
|
||||
total += len(name) + len(value) + 4
|
||||
if any(ord(character) < 32 and character != "\t" for character in value):
|
||||
raise PolicyError("SCM request header contains controls")
|
||||
if total > MAX_HEADER_BYTES:
|
||||
raise PolicyError("SCM request headers exceed the safe limit")
|
||||
if len(self.headers.get_all("Content-Length", [])) > 1:
|
||||
raise PolicyError("SCM request has duplicate Content-Length")
|
||||
|
||||
def _stream(
|
||||
self, status: int, content_type: str, body: BinaryIO, length: int
|
||||
) -> None:
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", content_type)
|
||||
self.send_header("Content-Length", str(length))
|
||||
self.end_headers()
|
||||
deadline = time.monotonic() + INBOUND_BODY_TIMEOUT
|
||||
while True:
|
||||
chunk = body.read(STREAM_CHUNK)
|
||||
if not chunk:
|
||||
break
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
raise PolicyError("SCM response write deadline exceeded")
|
||||
self.connection.settimeout(remaining)
|
||||
self.wfile.write(chunk)
|
||||
|
||||
def log_message(self, _format: str, *_args: object) -> None:
|
||||
return
|
||||
|
||||
def _json(self, status: int, body: bytes) -> None:
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.send_header("Cache-Control", "no-store")
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def _reject(self, status: int = 400) -> None:
|
||||
self._json(status, b'{"error":"request rejected"}')
|
||||
|
||||
def do_GET(self) -> None:
|
||||
try:
|
||||
self._validate_headers()
|
||||
if self.path == "/healthz":
|
||||
self._json(200, b'{"status":"ok"}')
|
||||
return
|
||||
repo, operation, service = _git_target(self.path)
|
||||
if operation != "info/refs":
|
||||
raise PolicyError("Git RPC requires POST")
|
||||
token = read_token()
|
||||
_reject_forbidden(repo, "repository", (token,))
|
||||
expected = f"application/x-{service}-advertisement"
|
||||
streamed = _upstream_git_request(
|
||||
f"/atlas/{repo}.git/info/refs?service={service}",
|
||||
method="GET",
|
||||
body=None,
|
||||
content_type=None,
|
||||
expected_type=expected,
|
||||
token=token,
|
||||
stream_result=True,
|
||||
)
|
||||
body, length = streamed
|
||||
try:
|
||||
self._stream(200, expected, body, length)
|
||||
finally:
|
||||
body.close()
|
||||
except (
|
||||
OSError,
|
||||
PolicyError,
|
||||
socket.timeout,
|
||||
urllib.error.URLError,
|
||||
ValueError,
|
||||
):
|
||||
self._reject()
|
||||
|
||||
def do_POST(self) -> None:
|
||||
try:
|
||||
self._validate_headers()
|
||||
self.connection.settimeout(INBOUND_BODY_TIMEOUT)
|
||||
if self.path in {"/v1/metadata", "/v1/drafts"}:
|
||||
self._control()
|
||||
else:
|
||||
self._git_rpc()
|
||||
except (
|
||||
OSError,
|
||||
PolicyError,
|
||||
socket.timeout,
|
||||
urllib.error.URLError,
|
||||
ValueError,
|
||||
json.JSONDecodeError,
|
||||
):
|
||||
self._reject()
|
||||
|
||||
def _control(self) -> None:
|
||||
data = _load_json(self)
|
||||
token = read_token()
|
||||
if self.path == "/v1/metadata":
|
||||
if set(data) != {"path"} or not isinstance(data["path"], str):
|
||||
raise PolicyError("metadata request fields are invalid")
|
||||
result = read(data["path"], token=token)
|
||||
else:
|
||||
expected = {"base", "body", "head", "head_sha", "repo", "title"}
|
||||
if set(data) != expected or not all(
|
||||
isinstance(data[key], str) for key in expected
|
||||
):
|
||||
raise PolicyError("draft request fields are invalid")
|
||||
result = create_draft(token=token, **data) # type: ignore[arg-type]
|
||||
if token.encode("utf-8") in result:
|
||||
raise PolicyError("SCM upstream reflected credential material")
|
||||
self._json(200, result)
|
||||
|
||||
def _git_rpc(self) -> None:
|
||||
repo, operation, service = _git_target(self.path)
|
||||
if (
|
||||
operation not in {"git-upload-pack", "git-receive-pack"}
|
||||
or service != operation
|
||||
):
|
||||
raise PolicyError("Git RPC operation is outside the allowlist")
|
||||
expected_request = f"application/x-{service}-request"
|
||||
if self.headers.get_content_type() != expected_request or self.headers.get(
|
||||
"Transfer-Encoding"
|
||||
):
|
||||
raise PolicyError("Git RPC request type is invalid")
|
||||
length = _content_length(self.headers, MAX_GIT_REQUEST)
|
||||
token = read_token()
|
||||
_reject_forbidden(repo, "repository", (token,))
|
||||
body, body_length = _spool_bounded(
|
||||
self.rfile,
|
||||
MAX_GIT_REQUEST,
|
||||
length,
|
||||
token=token,
|
||||
context="Git request",
|
||||
deadline=time.monotonic() + INBOUND_BODY_TIMEOUT,
|
||||
set_timeout=self.connection.settimeout,
|
||||
)
|
||||
try:
|
||||
if service == "git-receive-pack":
|
||||
_validate_receive_pack(body, token)
|
||||
expected = f"application/x-{service}-result"
|
||||
streamed = _upstream_git_request(
|
||||
f"/atlas/{repo}.git/{service}",
|
||||
method="POST",
|
||||
body=body,
|
||||
body_length=body_length,
|
||||
content_type=expected_request,
|
||||
expected_type=expected,
|
||||
token=token,
|
||||
stream_result=True,
|
||||
)
|
||||
result, result_length = streamed
|
||||
try:
|
||||
self._stream(200, expected, result, result_length)
|
||||
finally:
|
||||
result.close()
|
||||
finally:
|
||||
body.close()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--listen", default="0.0.0.0")
|
||||
parser.add_argument("--port", type=int, default=BROKER_PORT)
|
||||
args = parser.parse_args()
|
||||
BoundedThreadingHTTPServer((args.listen, args.port), BrokerHandler).serve_forever()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
93
services/hermes/scm-common/scripts/scm_broker_client.py
Normal file
93
services/hermes/scm-common/scripts/scm_broker_client.py
Normal file
@ -0,0 +1,93 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Call the credential-isolated Hermes SCM broker over its fixed cluster origin."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import urllib.request
|
||||
from collections.abc import Callable
|
||||
|
||||
import deadline_http
|
||||
from gitea_api_policy import PolicyError
|
||||
|
||||
BROKER_ORIGIN = "http://hermes-scm-broker.hermes-scm.svc.cluster.local:9081"
|
||||
MAX_RESPONSE_BYTES = 2 * 1024 * 1024
|
||||
|
||||
|
||||
def _open(request: urllib.request.Request, timeout: int):
|
||||
"""Open one broker exchange under a killable absolute deadline.
|
||||
|
||||
The helper process keeps every request on its fixed in-cluster origin by
|
||||
rejecting redirects before they are followed.
|
||||
"""
|
||||
return deadline_http.open_bounded(
|
||||
request, maximum=MAX_RESPONSE_BYTES, timeout=timeout
|
||||
)
|
||||
|
||||
|
||||
def request(
|
||||
endpoint: str,
|
||||
payload: dict[str, object],
|
||||
*,
|
||||
opener: Callable[..., object] = _open,
|
||||
) -> bytes:
|
||||
"""Send one bounded broker operation without any repository credential."""
|
||||
if endpoint not in {"/v1/metadata", "/v1/drafts"}:
|
||||
raise PolicyError("SCM broker operation is outside the client allowlist")
|
||||
body = json.dumps(payload, separators=(",", ":")).encode("utf-8")
|
||||
if len(body) > 64 * 1024:
|
||||
raise PolicyError("SCM broker request exceeds the safe size limit")
|
||||
outgoing = urllib.request.Request(
|
||||
BROKER_ORIGIN + endpoint,
|
||||
data=body,
|
||||
method="POST",
|
||||
headers={
|
||||
"Accept": "application/json",
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent": "hermes-scm-broker-client/1",
|
||||
},
|
||||
)
|
||||
with opener(outgoing, timeout=30) as response: # type: ignore[attr-defined]
|
||||
status = getattr(response, "status", None)
|
||||
if status is None and hasattr(response, "getcode"):
|
||||
status = response.getcode() # type: ignore[attr-defined]
|
||||
if status != 200:
|
||||
raise PolicyError("SCM broker returned an unexpected HTTP status")
|
||||
content_type = response.headers.get_content_type() # type: ignore[attr-defined]
|
||||
if content_type != "application/json":
|
||||
raise PolicyError("SCM broker returned an unexpected response type")
|
||||
result = response.read(MAX_RESPONSE_BYTES + 1) # type: ignore[attr-defined]
|
||||
if len(result) > MAX_RESPONSE_BYTES:
|
||||
raise PolicyError("SCM broker response exceeds the safe size limit")
|
||||
json.loads(result)
|
||||
return result
|
||||
|
||||
|
||||
def read(path: str, *, opener: Callable[..., object] = _open) -> bytes:
|
||||
"""Read explicitly allowed Atlas metadata through the broker."""
|
||||
return request("/v1/metadata", {"path": path}, opener=opener)
|
||||
|
||||
|
||||
def create_draft(
|
||||
repo: str,
|
||||
*,
|
||||
base: str,
|
||||
head: str,
|
||||
head_sha: str,
|
||||
title: str,
|
||||
body: str,
|
||||
opener: Callable[..., object] = _open,
|
||||
) -> bytes:
|
||||
"""Create one verified draft through the broker for human review."""
|
||||
return request(
|
||||
"/v1/drafts",
|
||||
{
|
||||
"base": base,
|
||||
"body": body,
|
||||
"head": head,
|
||||
"head_sha": head_sha,
|
||||
"repo": repo,
|
||||
"title": title,
|
||||
},
|
||||
opener=opener,
|
||||
)
|
||||
72
services/hermes/scm-common/scripts/scm_broker_io.py
Normal file
72
services/hermes/scm-common/scripts/scm_broker_io.py
Normal file
@ -0,0 +1,72 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Bounded-memory stream helpers for the Hermes SCM broker."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.request
|
||||
from typing import BinaryIO
|
||||
|
||||
from gitea_api_policy import PolicyError
|
||||
|
||||
|
||||
class RejectRedirect(urllib.request.HTTPRedirectHandler):
|
||||
"""Reject every upstream redirect before credentials can be forwarded."""
|
||||
|
||||
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
||||
raise PolicyError("SCM upstream redirects are not allowed")
|
||||
|
||||
|
||||
def response_status(response: object) -> int | None:
|
||||
"""Read an HTTP status from either urllib response interface."""
|
||||
value = getattr(response, "status", None)
|
||||
if value is None and hasattr(response, "getcode"):
|
||||
value = response.getcode() # type: ignore[attr-defined]
|
||||
return value
|
||||
|
||||
|
||||
def spool_response(
|
||||
response,
|
||||
maximum: int,
|
||||
forbidden: tuple[bytes, ...],
|
||||
*,
|
||||
memory_limit: int,
|
||||
chunk_size: int,
|
||||
deadline_seconds: float,
|
||||
) -> tuple[BinaryIO, int]:
|
||||
"""Spool an upstream response to disk while scanning chunk boundaries."""
|
||||
spool = tempfile.SpooledTemporaryFile( # noqa: SIM115 - caller owns returned spool
|
||||
max_size=memory_limit, dir="/tmp"
|
||||
)
|
||||
carry = b""
|
||||
total = 0
|
||||
deadline = time.monotonic() + deadline_seconds
|
||||
try:
|
||||
while True:
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
raise PolicyError("SCM upstream response deadline exceeded")
|
||||
raw = getattr(getattr(response, "fp", None), "raw", None)
|
||||
sock = getattr(raw, "_sock", raw)
|
||||
if hasattr(sock, "settimeout"):
|
||||
sock.settimeout(remaining)
|
||||
chunk = response.read(chunk_size)
|
||||
if not chunk:
|
||||
break
|
||||
total += len(chunk)
|
||||
if total > maximum:
|
||||
raise PolicyError("SCM upstream response exceeds the safe size limit")
|
||||
candidate = carry + chunk
|
||||
if any(value in candidate for value in forbidden):
|
||||
raise PolicyError(
|
||||
"Git upstream response contains runtime credential material"
|
||||
)
|
||||
width = max(len(value) for value in forbidden) - 1
|
||||
carry = candidate[-width:] if width else b""
|
||||
spool.write(chunk)
|
||||
spool.seek(0)
|
||||
return spool, total
|
||||
except Exception:
|
||||
spool.close()
|
||||
raise
|
||||
118
services/hermes/scm-common/scripts/scm_broker_server.py
Normal file
118
services/hermes/scm-common/scripts/scm_broker_server.py
Normal file
@ -0,0 +1,118 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Concurrency-bounded threaded HTTP server for the Hermes SCM broker."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import time
|
||||
from http.server import ThreadingHTTPServer
|
||||
|
||||
MAX_CONCURRENT_REQUESTS = 3
|
||||
|
||||
|
||||
class _AbsoluteDeadlineReader:
|
||||
"""Read header lines against one wall-clock deadline, not idle timeouts."""
|
||||
|
||||
def __init__(self, stream, connection):
|
||||
self._stream = stream
|
||||
self._connection = connection
|
||||
self._deadline: float | None = None
|
||||
|
||||
def begin(self, timeout: float) -> None:
|
||||
self._deadline = time.monotonic() + timeout
|
||||
|
||||
def end(self) -> None:
|
||||
self._deadline = None
|
||||
|
||||
def readline(self, limit: int = -1) -> bytes:
|
||||
if self._deadline is None:
|
||||
return self._stream.readline(limit)
|
||||
value = bytearray()
|
||||
while limit < 0 or len(value) < limit:
|
||||
remaining = self._deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
raise TimeoutError("absolute SCM header deadline exceeded")
|
||||
self._connection.settimeout(remaining)
|
||||
character = self._stream.read(1)
|
||||
if not character:
|
||||
break
|
||||
value.extend(character)
|
||||
if character == b"\n":
|
||||
break
|
||||
return bytes(value)
|
||||
|
||||
def __getattr__(self, name: str):
|
||||
return getattr(self._stream, name)
|
||||
|
||||
|
||||
class AbsoluteHeaderDeadlineMixin:
|
||||
"""Apply an absolute request-line plus header deadline to HTTP handlers."""
|
||||
|
||||
header_deadline_seconds = 10.0
|
||||
|
||||
def setup(self) -> None:
|
||||
super().setup()
|
||||
self._header_reader = _AbsoluteDeadlineReader(self.rfile, self.connection)
|
||||
self.rfile = self._header_reader
|
||||
|
||||
def handle_one_request(self) -> None:
|
||||
try:
|
||||
self._header_reader.begin(self.header_deadline_seconds)
|
||||
try:
|
||||
self.raw_requestline = self.rfile.readline(65537)
|
||||
if len(self.raw_requestline) > 65536:
|
||||
self.requestline = ""
|
||||
self.request_version = ""
|
||||
self.command = ""
|
||||
self.send_error(414)
|
||||
return
|
||||
if not self.raw_requestline:
|
||||
self.close_connection = True
|
||||
return
|
||||
if not self.parse_request():
|
||||
return
|
||||
finally:
|
||||
self._header_reader.end()
|
||||
method_name = "do_" + self.command
|
||||
if not hasattr(self, method_name):
|
||||
self.send_error(501, "Unsupported method")
|
||||
return
|
||||
getattr(self, method_name)()
|
||||
self.wfile.flush()
|
||||
except TimeoutError:
|
||||
self.close_connection = True
|
||||
|
||||
|
||||
class BoundedThreadingHTTPServer(ThreadingHTTPServer):
|
||||
"""Cap active handlers so large Git exchanges cannot exhaust memory."""
|
||||
|
||||
daemon_threads = True
|
||||
block_on_close = True
|
||||
request_queue_size = 16
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self._slots = threading.BoundedSemaphore(MAX_CONCURRENT_REQUESTS)
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
def process_request(self, request, client_address) -> None:
|
||||
if not self._slots.acquire(blocking=False):
|
||||
try:
|
||||
request.sendall(
|
||||
b"HTTP/1.1 503 Service Unavailable\r\n"
|
||||
b"Connection: close\r\nContent-Length: 0\r\n\r\n"
|
||||
)
|
||||
finally:
|
||||
self.shutdown_request(request)
|
||||
return
|
||||
try:
|
||||
super().process_request(request, client_address)
|
||||
except Exception:
|
||||
self._slots.release()
|
||||
self.shutdown_request(request)
|
||||
raise
|
||||
|
||||
def process_request_thread(self, request, client_address) -> None:
|
||||
try:
|
||||
super().process_request_thread(request, client_address)
|
||||
finally:
|
||||
self._slots.release()
|
||||
82
services/hermes/scripts/cli_lane_board.py
Normal file
82
services/hermes/scripts/cli_lane_board.py
Normal file
@ -0,0 +1,82 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Bounded Kanban access and workspace resolution for Hermes CLI workers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable
|
||||
|
||||
from cli_lane_config import (
|
||||
BOARD_CORRUPTION_ERRORS,
|
||||
EXTERNAL_PREFIX,
|
||||
KANBAN_STORAGE_ATTEMPTS,
|
||||
WORKTREE_LOCK,
|
||||
)
|
||||
|
||||
|
||||
def _task_value(task: Any, name: str, default: Any = None) -> Any:
|
||||
return getattr(task, name, default)
|
||||
|
||||
def _task_context(kanban_db: Any, conn: Any, task_id: str) -> str:
|
||||
value = kanban_db.build_worker_context(conn, task_id)
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
return json.dumps(value, indent=2, default=str)
|
||||
|
||||
def _resolve_workspace(kanban_db: Any, conn: Any, task: Any, board: str) -> Path:
|
||||
# External coding lanes always receive their own linked worktree. A task
|
||||
# that does not resolve to a Git repository is blocked rather than sharing
|
||||
# a mutable checkout with another unattended worker.
|
||||
# Git worktree creation updates common-repository metadata. Serialize that
|
||||
# short materialization step while allowing the provider workers themselves
|
||||
# to run concurrently in independent worktrees.
|
||||
with WORKTREE_LOCK:
|
||||
value, branch_name = kanban_db._resolve_worktree_workspace(task, board=board)
|
||||
kanban_db.set_branch_name(conn, str(_task_value(task, "id")), branch_name)
|
||||
return Path(value).resolve()
|
||||
|
||||
def _record_board_access_error(board: str, error: Exception) -> None:
|
||||
"""Report one board access failure once while allowing other lanes to run."""
|
||||
failure_kind = "storage" if isinstance(error, (OSError, sqlite3.Error)) else "access"
|
||||
detail = f"{failure_kind} {type(error).__name__}: {error}"
|
||||
if BOARD_CORRUPTION_ERRORS.get(board) == detail:
|
||||
return
|
||||
print(
|
||||
f"temporarily skipping Kanban board {board!r}: {detail}",
|
||||
file=sys.stderr,
|
||||
flush=True,
|
||||
)
|
||||
BOARD_CORRUPTION_ERRORS[board] = detail
|
||||
|
||||
def _board_call(
|
||||
kanban_db: Any,
|
||||
board: str,
|
||||
operation: Callable[[Any], Any],
|
||||
) -> Any:
|
||||
"""Run one bounded Kanban operation on a fresh, promptly closed connection."""
|
||||
last_storage_error: Exception | None = None
|
||||
for attempt in range(KANBAN_STORAGE_ATTEMPTS):
|
||||
conn = None
|
||||
try:
|
||||
with kanban_db.scoped_current_board(board):
|
||||
conn = kanban_db.connect(board=board)
|
||||
result = operation(conn)
|
||||
BOARD_CORRUPTION_ERRORS.pop(board, None)
|
||||
return result
|
||||
except (OSError, sqlite3.Error) as error:
|
||||
last_storage_error = error
|
||||
_record_board_access_error(board, error)
|
||||
if attempt + 1 < KANBAN_STORAGE_ATTEMPTS:
|
||||
time.sleep(0.2 * (attempt + 1))
|
||||
finally:
|
||||
if conn is not None:
|
||||
conn.close()
|
||||
assert last_storage_error is not None
|
||||
raise last_storage_error
|
||||
|
||||
def _external(task: Any) -> bool:
|
||||
return str(_task_value(task, "assignee", "") or "").startswith(EXTERNAL_PREFIX)
|
||||
238
services/hermes/scripts/cli_lane_capabilities.py
Normal file
238
services/hermes/scripts/cli_lane_capabilities.py
Normal file
@ -0,0 +1,238 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Detect and publish the Kanban API boundary required by CLI lanes.
|
||||
|
||||
During a rollout the coordinator container and the cli-lane-runner worker can
|
||||
run different hermes-agent image generations against one shared Kanban DB.
|
||||
The image APIs that differ on the decomposition/finalization path:
|
||||
|
||||
* ``complete_task``: both generations accept ``expected_run_id`` (active
|
||||
exact-run completion); only the patched image accepts
|
||||
``replay_ended_run_id`` (ended-run replay).
|
||||
* ``reclaim_task``: only the patched image accepts ``expected_run_id``
|
||||
(guarded orphan reclaim after a runner restart).
|
||||
* ``specify_triage_task``/``decompose_triage_task``: only the patched image
|
||||
accepts ``require_no_runs`` (refuses redefining tasks with run history).
|
||||
|
||||
Worker-side handling per combination (proven in
|
||||
testing/tests/test_hermes_cli_capabilities.py):
|
||||
|
||||
* legacy worker: health/readiness stay deferred, no new claims, no unguarded
|
||||
reclaim, replay-needing results stay journaled for a patched successor.
|
||||
* patched worker: ready health, exact completion, guarded replay/reclaim; a
|
||||
run voided by a legacy coordinator's unguarded mid-run decomposition
|
||||
converges to stale/conflict evidence instead of completing.
|
||||
* patched coordinator: mid-run decomposition is refused entirely, so the
|
||||
active exact run completes on either worker generation.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from dataclasses import asdict, dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable
|
||||
|
||||
from cli_lane_config import STATE_ROOT, utc_now
|
||||
from cli_lane_files import atomic_json, load_json
|
||||
|
||||
|
||||
_CAPABILITIES: KanbanCapabilities | None = None
|
||||
_CAPABILITY_SOURCE: tuple[int, int] | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class KanbanCapabilities:
|
||||
"""Safety-relevant keyword support exposed by the image's Kanban DB."""
|
||||
|
||||
exact_run_completion: bool
|
||||
ended_run_replay: bool
|
||||
exact_run_reclaim: bool
|
||||
|
||||
@property
|
||||
def ready(self) -> bool:
|
||||
"""Return whether dispatch and every recovery transition are safe."""
|
||||
return (
|
||||
self.exact_run_completion
|
||||
and self.ended_run_replay
|
||||
and self.exact_run_reclaim
|
||||
)
|
||||
|
||||
@property
|
||||
def deferred_features(self) -> tuple[str, ...]:
|
||||
"""List fixed, bounded capability names absent from the image API."""
|
||||
missing = []
|
||||
if not self.exact_run_completion:
|
||||
missing.append("exact-run-completion")
|
||||
if not self.ended_run_replay:
|
||||
missing.append("ended-run-replay")
|
||||
if not self.exact_run_reclaim:
|
||||
missing.append("exact-run-reclaim")
|
||||
return tuple(missing)
|
||||
|
||||
|
||||
def _explicit_keyword(function: Callable[..., Any], keyword: str) -> bool:
|
||||
"""Require an explicit keyword parameter rather than trusting ``**kwargs``."""
|
||||
try:
|
||||
parameter = inspect.signature(function).parameters.get(keyword)
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
return parameter is not None and parameter.kind in {
|
||||
inspect.Parameter.POSITIONAL_OR_KEYWORD,
|
||||
inspect.Parameter.KEYWORD_ONLY,
|
||||
}
|
||||
|
||||
|
||||
def detect_kanban_capabilities(kanban_db: Any) -> KanbanCapabilities:
|
||||
"""Inspect the loaded image API without invoking a mutating DB operation."""
|
||||
complete_task = getattr(kanban_db, "complete_task", None)
|
||||
reclaim_task = getattr(kanban_db, "reclaim_task", None)
|
||||
return KanbanCapabilities(
|
||||
exact_run_completion=callable(complete_task)
|
||||
and _explicit_keyword(complete_task, "expected_run_id"),
|
||||
ended_run_replay=callable(complete_task)
|
||||
and _explicit_keyword(complete_task, "replay_ended_run_id"),
|
||||
exact_run_reclaim=callable(reclaim_task)
|
||||
and _explicit_keyword(reclaim_task, "expected_run_id"),
|
||||
)
|
||||
|
||||
|
||||
def _source_identity(kanban_db: Any) -> tuple[int, int]:
|
||||
"""Bind cached capability evidence to the two inspected callables."""
|
||||
return (
|
||||
id(getattr(kanban_db, "complete_task", None)),
|
||||
id(getattr(kanban_db, "reclaim_task", None)),
|
||||
)
|
||||
|
||||
|
||||
def _health_document(capabilities: KanbanCapabilities) -> dict[str, Any]:
|
||||
"""Return a bounded operator-readable compatibility health document."""
|
||||
return {
|
||||
"component": "cli-lane-runner",
|
||||
"schema_version": 1,
|
||||
"state": "ready" if capabilities.ready else "deferred",
|
||||
"ready": capabilities.ready,
|
||||
"active_run_completion_safe": capabilities.exact_run_completion,
|
||||
"deferred_features": list(capabilities.deferred_features),
|
||||
"capabilities": asdict(capabilities),
|
||||
"observed_at": utc_now(),
|
||||
}
|
||||
|
||||
|
||||
def initialize_kanban_capabilities(
|
||||
kanban_db: Any,
|
||||
*,
|
||||
health_path: Path | None = None,
|
||||
) -> KanbanCapabilities:
|
||||
"""Detect startup compatibility and durably expose ready/deferred health."""
|
||||
global _CAPABILITIES, _CAPABILITY_SOURCE
|
||||
capabilities = detect_kanban_capabilities(kanban_db)
|
||||
_CAPABILITIES = capabilities
|
||||
_CAPABILITY_SOURCE = _source_identity(kanban_db)
|
||||
destination = health_path or STATE_ROOT / "runtime-health.json"
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
atomic_json(destination, _health_document(capabilities), 0o600)
|
||||
if not capabilities.ready:
|
||||
print(
|
||||
"CLI lane safety compatibility deferred; missing image APIs: "
|
||||
+ ", ".join(capabilities.deferred_features),
|
||||
file=sys.stderr,
|
||||
flush=True,
|
||||
)
|
||||
return capabilities
|
||||
|
||||
|
||||
def refresh_kanban_capabilities(
|
||||
kanban_db: Any,
|
||||
*,
|
||||
health_path: Path | None = None,
|
||||
) -> KanbanCapabilities:
|
||||
"""Reinspect the live callables and refresh bounded readiness evidence.
|
||||
|
||||
ConfigMap script updates and image changes are deliberately not imported or
|
||||
reloaded here. A running process observes only the already-loaded module,
|
||||
which prevents a half-old/half-new Python module graph. Tests may replace
|
||||
the two callables to model an image API transition at a loop boundary.
|
||||
"""
|
||||
return initialize_kanban_capabilities(kanban_db, health_path=health_path)
|
||||
|
||||
|
||||
def kanban_capabilities(kanban_db: Any) -> KanbanCapabilities:
|
||||
"""Return startup evidence, redetecting only when the loaded API changes."""
|
||||
if _CAPABILITIES is None or _source_identity(kanban_db) != _CAPABILITY_SOURCE:
|
||||
return initialize_kanban_capabilities(kanban_db)
|
||||
return _CAPABILITIES
|
||||
|
||||
|
||||
def runtime_health(path: Path | None = None) -> dict[str, Any]:
|
||||
"""Read the bounded health document for probes and operator diagnostics."""
|
||||
document = load_json(path or STATE_ROOT / "runtime-health.json")
|
||||
return document if isinstance(document, dict) else {}
|
||||
|
||||
|
||||
def readiness_issue(
|
||||
path: Path | None = None,
|
||||
*,
|
||||
now: datetime | None = None,
|
||||
maximum_age_seconds: float = 60.0,
|
||||
) -> str | None:
|
||||
"""Return a bounded readiness reason without exposing health contents."""
|
||||
destination = path or STATE_ROOT / "runtime-health.json"
|
||||
try:
|
||||
document = json.loads(destination.read_text(encoding="utf-8"))
|
||||
except FileNotFoundError:
|
||||
return "runtime health is missing"
|
||||
except (OSError, UnicodeError, json.JSONDecodeError):
|
||||
return "runtime health is malformed"
|
||||
if not isinstance(document, dict) or document.get("schema_version") != 1:
|
||||
return "runtime health is malformed"
|
||||
if document.get("state") == "deferred" and document.get("ready") is False:
|
||||
return "runtime compatibility is deferred"
|
||||
capabilities = document.get("capabilities")
|
||||
if (
|
||||
document.get("state") != "ready"
|
||||
or document.get("ready") is not True
|
||||
or not isinstance(capabilities, dict)
|
||||
or any(
|
||||
capabilities.get(name) is not True
|
||||
for name in (
|
||||
"exact_run_completion",
|
||||
"ended_run_replay",
|
||||
"exact_run_reclaim",
|
||||
)
|
||||
)
|
||||
):
|
||||
return "runtime health is malformed"
|
||||
observed_at = document.get("observed_at")
|
||||
if not isinstance(observed_at, str):
|
||||
return "runtime health is malformed"
|
||||
try:
|
||||
observed = datetime.fromisoformat(observed_at.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return "runtime health is malformed"
|
||||
if observed.tzinfo is None:
|
||||
return "runtime health is malformed"
|
||||
current = now or datetime.now(timezone.utc)
|
||||
age = (
|
||||
current.astimezone(timezone.utc) - observed.astimezone(timezone.utc)
|
||||
).total_seconds()
|
||||
if age < -5.0 or age > maximum_age_seconds:
|
||||
return "runtime health is stale"
|
||||
return None
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Provide the non-mutating Kubernetes readiness command."""
|
||||
maximum_age = float(os.environ.get("HERMES_CLI_HEALTH_MAX_AGE_SECONDS", "60"))
|
||||
issue = readiness_issue(maximum_age_seconds=maximum_age)
|
||||
if issue:
|
||||
print(issue, file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
220
services/hermes/scripts/cli_lane_config.py
Normal file
220
services/hermes/scripts/cli_lane_config.py
Normal file
@ -0,0 +1,220 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Shared immutable configuration and result types for Hermes CLI lanes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import threading
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
import cli_lane_goal
|
||||
|
||||
|
||||
DATA_ROOT = Path(os.environ.get("HERMES_HOME", "/opt/data"))
|
||||
SWITCHYARD_URL = os.environ.get(
|
||||
"HERMES_SWITCHYARD_URL",
|
||||
"http://hermes-switchyard.hermes.svc.cluster.local:9005/v1/chat/completions",
|
||||
)
|
||||
STATE_ROOT = DATA_ROOT / "cli-lanes"
|
||||
CODEX_BIN = DATA_ROOT / "tools/bin/codex"
|
||||
CLAUDE_BIN = DATA_ROOT / "tools/bin/claude"
|
||||
CLAUDE_SETTINGS = DATA_ROOT / "home/.claude/settings.json"
|
||||
RESULT_SCHEMA_PATH = STATE_ROOT / "worker-result.schema.json"
|
||||
EFFORTS = ("low", "medium", "high", "xhigh")
|
||||
EXTERNAL_PREFIX = "cli-"
|
||||
DEFAULT_CLAIM_TTL = 7 * 24 * 60 * 60
|
||||
DEFAULT_MAX_RUNTIME = 12 * 60 * 60
|
||||
HEARTBEAT_SECONDS = 20
|
||||
PROVIDER_HEALTH_MAX_AGE_SECONDS = 5 * 60
|
||||
PROVIDER_AUTH_FAILURE_MAX_AGE_SECONDS = 12 * 60 * 60
|
||||
QUOTA_MIN_REMAINING_PERCENT_DEFAULT = 15.0
|
||||
CAPACITY_COOLDOWN_SECONDS_DEFAULT = 300.0
|
||||
AUTH_COOLDOWN_SECONDS_DEFAULT = 3600.0
|
||||
QUOTA_METRICS_URL = os.environ.get(
|
||||
"HERMES_CLI_QUOTA_METRICS_URL", "http://127.0.0.1:9010/metrics"
|
||||
)
|
||||
KANBAN_STORAGE_ATTEMPTS = 5
|
||||
ARTIFACT_GC_INTERVAL_SECONDS = 5 * 60
|
||||
ARTIFACT_RETENTION_AGE_SECONDS = 30 * 24 * 60 * 60
|
||||
ARTIFACT_RETENTION_COUNT = 256
|
||||
ARTIFACT_RETENTION_BYTES = 128 * 1024 * 1024
|
||||
MAX_TERMINAL_RECORD_BYTES = 1024 * 1024
|
||||
QUARANTINE_HASH_BYTES = 64 * 1024
|
||||
SQLITE_RUN_ID_MAX = 2**63 - 1
|
||||
PROVIDER_HEALTH_PATHS = {
|
||||
"codex": DATA_ROOT / "provider-health/codex.json",
|
||||
"claude": DATA_ROOT / "provider-health/claude.json",
|
||||
}
|
||||
WORKTREE_LOCK = threading.Lock()
|
||||
BOARD_CORRUPTION_ERRORS: dict[str, str] = {}
|
||||
LAST_ARTIFACT_GC = 0.0
|
||||
CAPACITY_PATTERN = re.compile(
|
||||
r"(?:rate.?limit|capacity|overload|usage.?limit|quota|credit|exhaust|429|529"
|
||||
r"|authentication|unauthorized|forbidden|oauth|token.*expired|401|403)",
|
||||
re.I,
|
||||
)
|
||||
NO_CLAUDE_SESSION = "No conversation found with session ID:"
|
||||
CLAUDE_SESSION_COLLISION = "Session ID already in use"
|
||||
NO_CODEX_THREAD = "no rollout found for thread id"
|
||||
|
||||
|
||||
RESULT_SCHEMA: dict[str, Any] = {
|
||||
"type": "object",
|
||||
"additionalProperties": False,
|
||||
"required": [
|
||||
"status",
|
||||
"summary",
|
||||
"changed_files",
|
||||
"tests_run",
|
||||
"artifacts",
|
||||
"findings",
|
||||
"blockers",
|
||||
],
|
||||
"properties": {
|
||||
"status": {
|
||||
"type": "string",
|
||||
"enum": sorted(cli_lane_goal.RESULT_STATUSES),
|
||||
},
|
||||
"summary": {"type": "string"},
|
||||
"changed_files": {"type": "array", "items": {"type": "string"}},
|
||||
"tests_run": {"type": "array", "items": {"type": "string"}},
|
||||
"artifacts": {"type": "array", "items": {"type": "string"}},
|
||||
"findings": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": (
|
||||
"Defects, risks, or observations discovered by a review or diagnosis. "
|
||||
"Findings do not prevent the assigned review or diagnosis from completing."
|
||||
),
|
||||
},
|
||||
"blockers": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": (
|
||||
"Concrete obstacles that prevent completion of the assigned task itself. "
|
||||
"This must be empty when status is completed; review findings belong in findings."
|
||||
),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Route:
|
||||
"""One provider/model/effort decision for an observable worker boundary."""
|
||||
|
||||
provider: str
|
||||
model: str
|
||||
effort: str
|
||||
profile: str
|
||||
classifier: str
|
||||
reason: str
|
||||
latency_ms: int
|
||||
fallback_chain: tuple[str, ...]
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProcessResult:
|
||||
"""Captured outcome from one provider CLI invocation."""
|
||||
|
||||
returncode: int
|
||||
output: str
|
||||
structured: dict[str, Any] | None
|
||||
capacity_failure: bool
|
||||
|
||||
|
||||
class TerminalFinalizationPending(RuntimeError):
|
||||
"""An accepted worker result is durable but not committed to Kanban yet."""
|
||||
|
||||
|
||||
def canonical_run_id(value: object) -> int | None:
|
||||
"""Return one positive SQLite-safe run ID from an int or canonical decimal."""
|
||||
if type(value) is int:
|
||||
candidate = value
|
||||
elif type(value) is str:
|
||||
# Bound conversion before ``int`` so an attacker cannot hand Python an
|
||||
# arbitrarily large decimal from an otherwise lexically valid name.
|
||||
if not re.fullmatch(r"[1-9][0-9]{0,18}", value):
|
||||
return None
|
||||
candidate = int(value)
|
||||
else:
|
||||
return None
|
||||
if not 1 <= candidate <= SQLITE_RUN_ID_MAX:
|
||||
return None
|
||||
return candidate
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TerminalIdentity:
|
||||
"""Replay authority derived only from a journal's directory and filename."""
|
||||
|
||||
board: str
|
||||
task_id: str
|
||||
run_id: int
|
||||
state: str
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
"""Reject noncanonical identities before they can reach SQLite."""
|
||||
if type(self.run_id) is not int or canonical_run_id(self.run_id) is None:
|
||||
raise ValueError("terminal run identity is not a positive SQLite int64")
|
||||
|
||||
|
||||
@dataclass
|
||||
class TerminalSnapshot:
|
||||
"""One inode-bound, bounded journal read held open across finalization."""
|
||||
|
||||
document: dict[str, Any]
|
||||
file_stat: os.stat_result
|
||||
descriptor: int
|
||||
directory_descriptor: int
|
||||
|
||||
def close(self) -> None:
|
||||
"""Release the pinned file and directory descriptors."""
|
||||
os.close(self.descriptor)
|
||||
os.close(self.directory_descriptor)
|
||||
|
||||
|
||||
@dataclass
|
||||
class TerminalRecoverySnapshot:
|
||||
"""One raw directory entry pinned before recovery classifies its payload."""
|
||||
|
||||
document: dict[str, Any] | None
|
||||
file_stat: os.stat_result
|
||||
descriptor: int | None
|
||||
directory_descriptor: int
|
||||
prefix: bytes
|
||||
invalid_reason: str | None
|
||||
|
||||
def close(self) -> None:
|
||||
"""Release the descriptors retained across classification/quarantine."""
|
||||
if self.descriptor is not None:
|
||||
os.close(self.descriptor)
|
||||
self.descriptor = None
|
||||
if self.directory_descriptor >= 0:
|
||||
os.close(self.directory_descriptor)
|
||||
self.directory_descriptor = -1
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def kanban_setting(name: str, default: float) -> float:
|
||||
"""Read one numeric routing setting from the deployed kanban config block."""
|
||||
try:
|
||||
document = yaml.safe_load(
|
||||
(DATA_ROOT / "config.yaml").read_text(encoding="utf-8")
|
||||
)
|
||||
except (OSError, yaml.YAMLError):
|
||||
return default
|
||||
kanban = document.get("kanban") if isinstance(document, dict) else None
|
||||
value = kanban.get(name) if isinstance(kanban, dict) else None
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
||||
return default
|
||||
return float(value)
|
||||
203
services/hermes/scripts/cli_lane_dispatch.py
Normal file
203
services/hermes/scripts/cli_lane_dispatch.py
Normal file
@ -0,0 +1,203 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Concurrent board recovery, claim, and dispatch loop for CLI workers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import concurrent.futures
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from cli_lane_board import _external, _record_board_access_error, _task_value
|
||||
from cli_lane_capabilities import (
|
||||
initialize_kanban_capabilities,
|
||||
kanban_capabilities,
|
||||
refresh_kanban_capabilities,
|
||||
)
|
||||
from cli_lane_config import (
|
||||
BOARD_CORRUPTION_ERRORS,
|
||||
DEFAULT_CLAIM_TTL,
|
||||
EXTERNAL_PREFIX,
|
||||
RESULT_SCHEMA,
|
||||
RESULT_SCHEMA_PATH,
|
||||
)
|
||||
from cli_lane_execution import execute_claim
|
||||
from cli_lane_files import atomic_json
|
||||
from cli_lane_metrics import start_metrics_server
|
||||
from cli_lane_recovery import _has_pending_finalization, recover_pending_finalizations
|
||||
from cli_lane_retention import maybe_gc_lane_artifacts
|
||||
|
||||
|
||||
def _board_slug(board: Any) -> str:
|
||||
if isinstance(board, dict):
|
||||
return str(board.get("slug") or board.get("id") or "")
|
||||
return str(getattr(board, "slug", None) or getattr(board, "id", None) or board)
|
||||
|
||||
|
||||
def _connect_healthy_board(kanban_db: Any, board: str) -> Any | None:
|
||||
"""Open one board without letting localized storage faults stop other lanes."""
|
||||
try:
|
||||
return kanban_db.connect(board=board)
|
||||
except Exception as error:
|
||||
_record_board_access_error(board, error)
|
||||
return None
|
||||
|
||||
|
||||
def recover_orphans() -> None:
|
||||
"""Return external running tasks to ready after a runner/pod restart."""
|
||||
from hermes_cli import kanban_db
|
||||
|
||||
recover_pending_finalizations()
|
||||
if not kanban_capabilities(kanban_db).exact_run_reclaim:
|
||||
return
|
||||
try:
|
||||
boards = kanban_db.list_boards(include_archived=False)
|
||||
except Exception as error:
|
||||
_record_board_access_error("board-registry", error)
|
||||
return
|
||||
for raw_board in boards:
|
||||
board = _board_slug(raw_board)
|
||||
if not board:
|
||||
continue
|
||||
with kanban_db.scoped_current_board(board):
|
||||
conn = _connect_healthy_board(kanban_db, board)
|
||||
if conn is None:
|
||||
continue
|
||||
try:
|
||||
for task in kanban_db.list_tasks(conn):
|
||||
if (
|
||||
_external(task)
|
||||
and str(_task_value(task, "status", "")) == "running"
|
||||
):
|
||||
task_id = str(_task_value(task, "id"))
|
||||
run_id = _task_value(task, "current_run_id", None)
|
||||
if not isinstance(run_id, int):
|
||||
continue
|
||||
if _has_pending_finalization(board, task_id, run_id):
|
||||
continue
|
||||
kanban_db.reclaim_task(
|
||||
conn,
|
||||
task_id,
|
||||
reason="direct CLI lane restarted; provider session will resume",
|
||||
expected_run_id=run_id,
|
||||
)
|
||||
BOARD_CORRUPTION_ERRORS.pop(board, None)
|
||||
except Exception as error:
|
||||
_record_board_access_error(board, error)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def claim_ready(
|
||||
active: set[tuple[str, str]],
|
||||
limit: int,
|
||||
eligible: Callable[[str, Any], bool] | None = None,
|
||||
) -> list[tuple[str, str]]:
|
||||
"""Atomically claim external ready tasks across all non-archived boards."""
|
||||
from hermes_cli import kanban_db
|
||||
|
||||
claimed: list[tuple[str, str]] = []
|
||||
if limit <= 0:
|
||||
return claimed
|
||||
for raw_board in kanban_db.list_boards(include_archived=False):
|
||||
board = _board_slug(raw_board)
|
||||
if not board:
|
||||
continue
|
||||
with kanban_db.scoped_current_board(board):
|
||||
conn = _connect_healthy_board(kanban_db, board)
|
||||
if conn is None:
|
||||
continue
|
||||
try:
|
||||
kanban_db.recompute_ready(conn)
|
||||
tasks = kanban_db.list_tasks(conn)
|
||||
BOARD_CORRUPTION_ERRORS.pop(board, None)
|
||||
for task in tasks:
|
||||
task_id = str(_task_value(task, "id", ""))
|
||||
assignee = str(_task_value(task, "assignee", "") or "")
|
||||
if (
|
||||
task_id
|
||||
and not assignee
|
||||
and str(_task_value(task, "status", "")) == "ready"
|
||||
and kanban_db.assign_task(conn, task_id, "cli-auto")
|
||||
):
|
||||
task = kanban_db.get_task(conn, task_id)
|
||||
assignee = "cli-auto"
|
||||
if (
|
||||
not task_id
|
||||
or (board, task_id) in active
|
||||
or not assignee.startswith(EXTERNAL_PREFIX)
|
||||
or str(_task_value(task, "status", "")) != "ready"
|
||||
or (eligible is not None and not eligible(board, task))
|
||||
):
|
||||
continue
|
||||
try:
|
||||
result = kanban_db.claim_task(
|
||||
conn,
|
||||
task_id,
|
||||
ttl_seconds=DEFAULT_CLAIM_TTL,
|
||||
claimer="direct-cli-lane",
|
||||
)
|
||||
except Exception:
|
||||
continue
|
||||
if result is not None:
|
||||
claimed.append((board, task_id))
|
||||
if len(claimed) >= limit:
|
||||
return claimed
|
||||
except Exception as error:
|
||||
_record_board_access_error(board, error)
|
||||
finally:
|
||||
conn.close()
|
||||
return claimed
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Continuously bridge external Kanban lanes to provider CLIs.
|
||||
|
||||
The loop is exited only by process signals or unrecoverable exceptions;
|
||||
every per-board and per-claim fault is bounded inside one pass.
|
||||
"""
|
||||
from hermes_cli import kanban_db
|
||||
|
||||
RESULT_SCHEMA_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
atomic_json(RESULT_SCHEMA_PATH, RESULT_SCHEMA, 0o644)
|
||||
start_metrics_server()
|
||||
capabilities = initialize_kanban_capabilities(kanban_db)
|
||||
recover_orphans()
|
||||
workers = max(1, min(int(os.environ.get("HERMES_CLI_LANE_CONCURRENCY", "4")), 8))
|
||||
futures: dict[concurrent.futures.Future[None], tuple[str, str]] = {}
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as pool:
|
||||
while True:
|
||||
previous_capabilities = capabilities
|
||||
capabilities = refresh_kanban_capabilities(kanban_db)
|
||||
if capabilities.ready and not previous_capabilities.ready:
|
||||
recover_orphans()
|
||||
for future in list(futures):
|
||||
if future.done():
|
||||
try:
|
||||
future.result()
|
||||
except Exception as error:
|
||||
print(
|
||||
f"worker future failed: {error}",
|
||||
file=sys.stderr,
|
||||
flush=True,
|
||||
)
|
||||
del futures[future]
|
||||
recover_pending_finalizations()
|
||||
maybe_gc_lane_artifacts()
|
||||
active = set(futures.values())
|
||||
try:
|
||||
newly_claimed = (
|
||||
claim_ready(active, workers - len(futures))
|
||||
if capabilities.ready
|
||||
else []
|
||||
)
|
||||
BOARD_CORRUPTION_ERRORS.pop("board-registry", None)
|
||||
except Exception as error:
|
||||
_record_board_access_error("board-registry", error)
|
||||
newly_claimed = []
|
||||
for board, task_id in newly_claimed:
|
||||
future = pool.submit(execute_claim, board, task_id)
|
||||
futures[future] = (board, task_id)
|
||||
time.sleep(5)
|
||||
294
services/hermes/scripts/cli_lane_evidence.py
Normal file
294
services/hermes/scripts/cli_lane_evidence.py
Normal file
@ -0,0 +1,294 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Immutable terminal evidence creation and identity-safe retirement."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ctypes
|
||||
import contextlib
|
||||
import errno
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import stat
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from cli_lane_config import MAX_TERMINAL_RECORD_BYTES, TerminalIdentity, utc_now
|
||||
from cli_lane_files import _terminal_evidence_identity, state_path
|
||||
from cli_lane_records import _load_small_json, _terminal_record_valid
|
||||
|
||||
|
||||
def _rename_noreplace(
|
||||
source: str,
|
||||
destination: str,
|
||||
*,
|
||||
source_dir: int,
|
||||
destination_dir: int,
|
||||
) -> None:
|
||||
"""Atomically rename a directory entry without replacing a collision."""
|
||||
renameat2 = getattr(ctypes.CDLL(None, use_errno=True), "renameat2", None)
|
||||
if renameat2 is None:
|
||||
raise OSError(errno.ENOSYS, "renameat2 is unavailable")
|
||||
renameat2.argtypes = (
|
||||
ctypes.c_int,
|
||||
ctypes.c_char_p,
|
||||
ctypes.c_int,
|
||||
ctypes.c_char_p,
|
||||
ctypes.c_uint,
|
||||
)
|
||||
renameat2.restype = ctypes.c_int
|
||||
result = renameat2(
|
||||
source_dir,
|
||||
ctypes.c_char_p(os.fsencode(source)),
|
||||
destination_dir,
|
||||
ctypes.c_char_p(os.fsencode(destination)),
|
||||
1, # RENAME_NOREPLACE
|
||||
)
|
||||
if result != 0:
|
||||
error = ctypes.get_errno()
|
||||
raise OSError(error, os.strerror(error), source, destination)
|
||||
|
||||
def _retire_terminal_entry(
|
||||
path: Path,
|
||||
source_stat: os.stat_result,
|
||||
*,
|
||||
board_descriptor: int,
|
||||
quarantine_descriptor: int,
|
||||
authority_name: str | None = None,
|
||||
source_descriptor: int | None = None,
|
||||
) -> str:
|
||||
"""Retire only the inode previously inspected; preserve any replacement."""
|
||||
pinned = os.fstat(source_descriptor) if source_descriptor is not None else source_stat
|
||||
if pinned.st_dev != source_stat.st_dev or pinned.st_ino != source_stat.st_ino:
|
||||
return "replacement"
|
||||
try:
|
||||
current = os.stat(
|
||||
path.name,
|
||||
dir_fd=board_descriptor,
|
||||
follow_symlinks=False,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
return "missing"
|
||||
if current.st_dev != pinned.st_dev or current.st_ino != pinned.st_ino:
|
||||
return "replacement"
|
||||
# A replacement can itself be staged repeatedly while recovery races a
|
||||
# writer. Keep every generation bound to the original canonical pending
|
||||
# name instead of hashing an intermediate .retire name and hiding it from
|
||||
# the next recovery pass.
|
||||
path_digest = hashlib.sha256(
|
||||
(authority_name or path.name).encode("utf-8")
|
||||
).hexdigest()[:16]
|
||||
source_digest = hashlib.sha256(
|
||||
f"{pinned.st_dev:x}\0{pinned.st_ino:x}".encode()
|
||||
).hexdigest()[:16]
|
||||
for sequence in range(32):
|
||||
staging = f".retire.{path_digest}.{source_digest}.{sequence}"
|
||||
try:
|
||||
_rename_noreplace(
|
||||
path.name,
|
||||
staging,
|
||||
source_dir=board_descriptor,
|
||||
destination_dir=quarantine_descriptor,
|
||||
)
|
||||
except FileExistsError:
|
||||
continue
|
||||
break
|
||||
else:
|
||||
return "collision"
|
||||
staged = os.stat(
|
||||
staging,
|
||||
dir_fd=quarantine_descriptor,
|
||||
follow_symlinks=False,
|
||||
)
|
||||
if staged.st_dev == pinned.st_dev and staged.st_ino == pinned.st_ino:
|
||||
os.unlink(staging, dir_fd=quarantine_descriptor)
|
||||
os.fsync(quarantine_descriptor)
|
||||
os.fsync(board_descriptor)
|
||||
return "retired"
|
||||
# The path changed after the pre-check but before the rename. Put the
|
||||
# replacement back without overwriting an even newer entry.
|
||||
try:
|
||||
_rename_noreplace(
|
||||
staging,
|
||||
path.name,
|
||||
source_dir=quarantine_descriptor,
|
||||
destination_dir=board_descriptor,
|
||||
)
|
||||
except FileExistsError:
|
||||
return "replacement-staged"
|
||||
os.fsync(quarantine_descriptor)
|
||||
os.fsync(board_descriptor)
|
||||
return "replacement"
|
||||
|
||||
def _write_json_noreplace(path: Path, value: dict[str, Any]) -> bool:
|
||||
"""Durably create one immutable JSON artifact without replacing a peer."""
|
||||
payload = (json.dumps(value, indent=2, sort_keys=True) + "\n").encode("utf-8")
|
||||
if len(payload) > MAX_TERMINAL_RECORD_BYTES + 4096:
|
||||
raise ValueError("terminal evidence exceeds the bounded artifact limit")
|
||||
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
os.chmod(path.parent, 0o700, follow_symlinks=False)
|
||||
directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
|
||||
directory_flags |= getattr(os, "O_NOFOLLOW", 0)
|
||||
create_flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
|
||||
create_flags |= getattr(os, "O_NOFOLLOW", 0)
|
||||
directory = os.open(path.parent, directory_flags)
|
||||
temporary = f".{path.name}.{uuid.uuid4().hex}.tmp"
|
||||
temporary_stat = None
|
||||
descriptor = None
|
||||
cleanup_descriptor = None
|
||||
try:
|
||||
descriptor = os.open(temporary, create_flags, 0o600, dir_fd=directory)
|
||||
temporary_stat = os.fstat(descriptor)
|
||||
with os.fdopen(descriptor, "wb") as stream:
|
||||
descriptor = None
|
||||
stream.write(payload)
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
written = os.fstat(stream.fileno())
|
||||
if stat.S_IMODE(written.st_mode) != 0o600 or written.st_nlink != 1:
|
||||
raise OSError("terminal evidence is not private and singly linked")
|
||||
cleanup_descriptor = os.dup(stream.fileno())
|
||||
try:
|
||||
_rename_noreplace(
|
||||
temporary,
|
||||
path.name,
|
||||
source_dir=directory,
|
||||
destination_dir=directory,
|
||||
)
|
||||
except FileExistsError:
|
||||
return False
|
||||
os.fsync(directory)
|
||||
return True
|
||||
finally:
|
||||
if descriptor is not None:
|
||||
os.close(descriptor)
|
||||
if temporary_stat is not None:
|
||||
with contextlib.suppress(OSError):
|
||||
_retire_terminal_entry(
|
||||
Path(temporary),
|
||||
temporary_stat,
|
||||
board_descriptor=directory,
|
||||
quarantine_descriptor=directory,
|
||||
source_descriptor=cleanup_descriptor,
|
||||
)
|
||||
if cleanup_descriptor is not None:
|
||||
os.close(cleanup_descriptor)
|
||||
os.close(directory)
|
||||
|
||||
def _terminal_document_digest(document: dict[str, Any]) -> str:
|
||||
"""Return the stable identity for one exact accepted result document."""
|
||||
immutable = {
|
||||
key: document.get(key)
|
||||
for key in (
|
||||
"board",
|
||||
"task_id",
|
||||
"expected_run_id",
|
||||
"result",
|
||||
"summary",
|
||||
"metadata",
|
||||
)
|
||||
}
|
||||
canonical = json.dumps(immutable, sort_keys=True, separators=(",", ":"))
|
||||
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()[:32]
|
||||
|
||||
def _terminal_evidence_path(
|
||||
identity: TerminalIdentity,
|
||||
state: str,
|
||||
document: dict[str, Any],
|
||||
) -> Path:
|
||||
"""Return a deterministic immutable prepared or conflict evidence path."""
|
||||
if state not in {"prepared", "conflict"}:
|
||||
raise ValueError(f"invalid terminal evidence state: {state}")
|
||||
base = state_path(identity.board, identity.task_id)
|
||||
digest = _terminal_document_digest(document)
|
||||
return base.with_name(
|
||||
f"{base.stem}.run-{identity.run_id}.terminal.{state}-{digest}.json"
|
||||
)
|
||||
|
||||
def _same_terminal_document(left: dict[str, Any], right: dict[str, Any]) -> bool:
|
||||
"""Compare the immutable result-bearing fields of terminal evidence."""
|
||||
keys = ("board", "task_id", "expected_run_id", "result", "summary", "metadata")
|
||||
return all(left.get(key) == right.get(key) for key in keys)
|
||||
|
||||
def _terminal_evidence_valid(
|
||||
document: dict[str, Any],
|
||||
identity: TerminalIdentity,
|
||||
state: str,
|
||||
) -> bool:
|
||||
"""Validate semantic terminal content plus its immutable evidence state."""
|
||||
if document.get("kanban_state") != state:
|
||||
return False
|
||||
semantic = dict(document)
|
||||
semantic_identity = identity
|
||||
if state == "conflict":
|
||||
semantic["kanban_state"] = "prepared"
|
||||
semantic_identity = TerminalIdentity(
|
||||
identity.board,
|
||||
identity.task_id,
|
||||
identity.run_id,
|
||||
"prepared",
|
||||
)
|
||||
return _terminal_record_valid(semantic, semantic_identity)
|
||||
|
||||
def _persist_prepared_evidence(
|
||||
identity: TerminalIdentity,
|
||||
document: dict[str, Any],
|
||||
) -> Path:
|
||||
"""Fsync an immutable accepted result before entering the DB boundary."""
|
||||
prepared = dict(document)
|
||||
prepared["kanban_state"] = "prepared"
|
||||
prepared["prepared_at"] = utc_now()
|
||||
path = _terminal_evidence_path(identity, "prepared", document)
|
||||
if _write_json_noreplace(path, prepared):
|
||||
return path
|
||||
if _terminal_evidence_identity(path) != TerminalIdentity(
|
||||
identity.board, identity.task_id, identity.run_id, "prepared"
|
||||
):
|
||||
raise OSError("prepared terminal evidence identity is invalid")
|
||||
existing = _load_small_json(path)
|
||||
prepared_identity = TerminalIdentity(
|
||||
identity.board,
|
||||
identity.task_id,
|
||||
identity.run_id,
|
||||
"prepared",
|
||||
)
|
||||
if not _terminal_evidence_valid(
|
||||
existing,
|
||||
prepared_identity,
|
||||
"prepared",
|
||||
) or not _same_terminal_document(existing, prepared):
|
||||
raise OSError("prepared terminal evidence path contains a conflicting result")
|
||||
return path
|
||||
|
||||
def _persist_conflict_evidence(
|
||||
identity: TerminalIdentity,
|
||||
document: dict[str, Any],
|
||||
reason: str,
|
||||
) -> Path:
|
||||
"""Preserve a losing valid result in full under deterministic retention."""
|
||||
conflict = dict(document)
|
||||
conflict["kanban_state"] = "conflict"
|
||||
conflict["conflict_reason"] = reason
|
||||
conflict["conflicted_at"] = utc_now()
|
||||
path = _terminal_evidence_path(identity, "conflict", document)
|
||||
if _write_json_noreplace(path, conflict):
|
||||
return path
|
||||
if _terminal_evidence_identity(path) != TerminalIdentity(
|
||||
identity.board, identity.task_id, identity.run_id, "conflict"
|
||||
):
|
||||
raise OSError("terminal conflict evidence identity is invalid")
|
||||
existing = _load_small_json(path)
|
||||
conflict_identity = TerminalIdentity(
|
||||
identity.board,
|
||||
identity.task_id,
|
||||
identity.run_id,
|
||||
"conflict",
|
||||
)
|
||||
if not _terminal_evidence_valid(
|
||||
existing,
|
||||
conflict_identity,
|
||||
"conflict",
|
||||
) or not _same_terminal_document(existing, conflict):
|
||||
raise OSError("terminal conflict evidence path contains a different result")
|
||||
return path
|
||||
462
services/hermes/scripts/cli_lane_execution.py
Normal file
462
services/hermes/scripts/cli_lane_execution.py
Normal file
@ -0,0 +1,462 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Single-claim provider execution and durable Kanban result handoff."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import json
|
||||
import sqlite3
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import cli_lane_goal
|
||||
from cli_lane_board import _board_call, _resolve_workspace, _task_context, _task_value
|
||||
from cli_lane_config import (
|
||||
DEFAULT_MAX_RUNTIME,
|
||||
TerminalFinalizationPending,
|
||||
)
|
||||
from cli_lane_failover import _routed_or_blocked, capacity_failover
|
||||
from cli_lane_files import (
|
||||
_terminal_identity,
|
||||
_terminal_path,
|
||||
atomic_json,
|
||||
load_json,
|
||||
state_path,
|
||||
)
|
||||
from cli_lane_finalization import _finalize_terminal_record, _recover_exact_run
|
||||
from cli_lane_health import record_provider_success
|
||||
from cli_lane_metrics import record_route_decision
|
||||
from cli_lane_prompt import build_prompt, git_handoff, workspace_artifacts
|
||||
from cli_lane_provider import run_provider
|
||||
from cli_lane_quota import selection_constraint
|
||||
from cli_lane_records import _persist_candidate, _write_terminal_record
|
||||
from cli_lane_recovery import _has_pending_finalization
|
||||
from cli_lane_routing import fresh_unavailable_provider, select_route
|
||||
|
||||
|
||||
def execute_claim(board: str, task_id: str) -> None:
|
||||
"""Execute one already-claimed task and commit its outcome to Kanban."""
|
||||
from hermes_cli import kanban_db
|
||||
|
||||
run_id = None
|
||||
with kanban_db.scoped_current_board(board):
|
||||
conn = kanban_db.connect(board=board)
|
||||
try:
|
||||
task = kanban_db.get_task(conn, task_id)
|
||||
if task is None:
|
||||
return
|
||||
run_id = _task_value(task, "current_run_id", None)
|
||||
assignee = str(_task_value(task, "assignee", "cli-auto") or "cli-auto")
|
||||
state_file = state_path(board, task_id)
|
||||
state = load_json(state_file)
|
||||
state.update(
|
||||
{
|
||||
"board": board,
|
||||
"task_id": task_id,
|
||||
"run_id": run_id,
|
||||
"assignee": assignee,
|
||||
}
|
||||
)
|
||||
atomic_json(state_file, state)
|
||||
log_path = Path(kanban_db.worker_log_path(task_id, board=board))
|
||||
workspace = _resolve_workspace(kanban_db, conn, task, board)
|
||||
kanban_db.set_workspace_path(conn, task_id, str(workspace))
|
||||
context = _task_context(kanban_db, conn, task_id)
|
||||
except Exception as error:
|
||||
failure_reason = (
|
||||
f"Direct CLI lane preparation failed: {type(error).__name__}: {error}"
|
||||
)
|
||||
conn.close()
|
||||
_board_call(
|
||||
kanban_db,
|
||||
board,
|
||||
lambda fresh: kanban_db.block_task(
|
||||
fresh,
|
||||
task_id,
|
||||
reason=failure_reason,
|
||||
kind="capability",
|
||||
expected_run_id=run_id,
|
||||
),
|
||||
)
|
||||
return
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def heartbeat(note: str) -> bool:
|
||||
try:
|
||||
return bool(
|
||||
_board_call(
|
||||
kanban_db,
|
||||
board,
|
||||
lambda fresh: kanban_db.heartbeat_worker(
|
||||
fresh,
|
||||
task_id,
|
||||
note=note,
|
||||
expected_run_id=run_id,
|
||||
),
|
||||
)
|
||||
)
|
||||
except (OSError, sqlite3.Error):
|
||||
# The claim TTL is deliberately long. Keep the expensive
|
||||
# provider process alive during a transient volume stall and
|
||||
# retry on the next heartbeat instead of losing its work.
|
||||
return True
|
||||
|
||||
def comment(body: str) -> None:
|
||||
with contextlib.suppress(OSError, sqlite3.Error):
|
||||
_board_call(
|
||||
kanban_db,
|
||||
board,
|
||||
lambda fresh: kanban_db.add_comment(
|
||||
fresh,
|
||||
task_id,
|
||||
"cli-lane-runner",
|
||||
body,
|
||||
),
|
||||
)
|
||||
# Route state is also written to the durable lane-state file;
|
||||
# a later heartbeat or terminal result remains authoritative.
|
||||
|
||||
try:
|
||||
previous_route = state.get("current_route")
|
||||
constraint = selection_constraint(assignee)
|
||||
unavailable_provider = (
|
||||
constraint.exclude_provider if constraint.source == "health" else None
|
||||
)
|
||||
route = _routed_or_blocked(
|
||||
kanban_db,
|
||||
board,
|
||||
task_id,
|
||||
run_id,
|
||||
lambda: select_route(
|
||||
context,
|
||||
assignee,
|
||||
exclude_provider=constraint.exclude_provider,
|
||||
exclude_reason=constraint.exclude_reason,
|
||||
),
|
||||
)
|
||||
if route is None:
|
||||
return
|
||||
for note in constraint.notes:
|
||||
comment(note)
|
||||
if constraint.exclude_provider:
|
||||
comment(
|
||||
f"Provider routing guard ({constraint.source}) excluded "
|
||||
f"{constraint.exclude_provider} before automatic routing.",
|
||||
)
|
||||
comment(
|
||||
f"CLI route: {route.provider}/{route.model} at {route.effort}; classifier={route.classifier}; {route.reason}",
|
||||
)
|
||||
resume_handoff = ""
|
||||
if (
|
||||
isinstance(previous_route, dict)
|
||||
and previous_route.get("provider")
|
||||
and previous_route.get("provider") != route.provider
|
||||
):
|
||||
try:
|
||||
previous_output = log_path.read_text(encoding="utf-8")[-10000:]
|
||||
except OSError:
|
||||
previous_output = "Previous provider log was unavailable after restart."
|
||||
resume_handoff = git_handoff(workspace, previous_output)
|
||||
comment(
|
||||
f"Restart-time provider change: {previous_route.get('provider')} -> {route.provider}; explicit workspace handoff attached.",
|
||||
)
|
||||
prompt = build_prompt(context, workspace, resume_handoff)
|
||||
max_runtime = int(
|
||||
_task_value(task, "max_runtime_seconds", 0) or DEFAULT_MAX_RUNTIME
|
||||
)
|
||||
goal_mode = bool(_task_value(task, "goal_mode", False))
|
||||
goal_max_turns = max(1, int(_task_value(task, "goal_max_turns", 1) or 1))
|
||||
goal_turn = max(1, int(state.get("goal_turn", 0) or 0) + 1)
|
||||
deadline = time.monotonic() + max_runtime
|
||||
while True:
|
||||
state["goal_turn"] = goal_turn
|
||||
atomic_json(state_file, state)
|
||||
remaining = max(1, int(deadline - time.monotonic()))
|
||||
result = run_provider(
|
||||
route,
|
||||
prompt,
|
||||
workspace,
|
||||
state,
|
||||
state_file,
|
||||
log_path,
|
||||
heartbeat,
|
||||
remaining,
|
||||
)
|
||||
candidate_file = None
|
||||
if result.structured:
|
||||
candidate_file = _persist_candidate(
|
||||
state,
|
||||
state_file,
|
||||
dict(result.structured),
|
||||
route=route,
|
||||
returncode=result.returncode,
|
||||
goal_turn=goal_turn,
|
||||
)
|
||||
if result.capacity_failure:
|
||||
boundary = capacity_failover(
|
||||
kanban_db,
|
||||
board=board,
|
||||
task_id=task_id,
|
||||
run_id=run_id,
|
||||
assignee=assignee,
|
||||
comment=comment,
|
||||
context=context,
|
||||
workspace=workspace,
|
||||
state=state,
|
||||
state_file=state_file,
|
||||
log_path=log_path,
|
||||
heartbeat=heartbeat,
|
||||
deadline=deadline,
|
||||
route=route,
|
||||
result=result,
|
||||
candidate_file=candidate_file,
|
||||
goal_turn=goal_turn,
|
||||
)
|
||||
route = boundary.route
|
||||
result = boundary.result
|
||||
candidate_file = boundary.candidate_file
|
||||
if boundary.unavailable_provider is not None:
|
||||
unavailable_provider = boundary.unavailable_provider
|
||||
if boundary.blocked:
|
||||
break
|
||||
if result.returncode == 0:
|
||||
record_provider_success(route.provider)
|
||||
structured = result.structured
|
||||
if structured:
|
||||
structured = dict(structured)
|
||||
structured["artifacts"] = workspace_artifacts(
|
||||
workspace,
|
||||
structured.get("artifacts"),
|
||||
)
|
||||
metadata = {
|
||||
"executor": "direct-cli-lane",
|
||||
"provider": route.provider,
|
||||
"model": route.model,
|
||||
"effort": route.effort,
|
||||
"classifier": route.classifier,
|
||||
"state_file": str(state_file),
|
||||
"codex_thread_id": state.get("codex_thread_id"),
|
||||
"claude_session_id": state.get("claude_session_id"),
|
||||
"goal_mode": goal_mode,
|
||||
"goal_turn": goal_turn,
|
||||
}
|
||||
if candidate_file is not None:
|
||||
metadata["candidate_file"] = str(candidate_file)
|
||||
if structured:
|
||||
for key in (
|
||||
"changed_files",
|
||||
"tests_run",
|
||||
"artifacts",
|
||||
"findings",
|
||||
"blockers",
|
||||
):
|
||||
value = structured.get(key)
|
||||
metadata[key] = value if isinstance(value, list) else []
|
||||
|
||||
completion_problem = None
|
||||
if structured and result.returncode == 0:
|
||||
# Resolve the role from the card itself, before any
|
||||
# controller evidence is appended, and hand it to the gate
|
||||
# so a review is judged on its verdict rather than on prose
|
||||
# about the artifact it reviewed.
|
||||
task_role, role_source = cli_lane_goal.task_role(
|
||||
context,
|
||||
structured,
|
||||
)
|
||||
metadata["task_role"] = task_role
|
||||
metadata["task_role_source"] = role_source
|
||||
completion_problem = cli_lane_goal.unfinished_result_reason(
|
||||
structured,
|
||||
role=task_role,
|
||||
)
|
||||
if (
|
||||
structured.get("status") == "completed"
|
||||
and completion_problem is None
|
||||
and goal_mode
|
||||
):
|
||||
heartbeat("local goal-completion judge active")
|
||||
rejection_history = state.get("goal_rejections", [])
|
||||
if not isinstance(rejection_history, list):
|
||||
rejection_history = []
|
||||
judge_context = context
|
||||
if goal_turn > 1 or rejection_history:
|
||||
judge_context += (
|
||||
f"\n\n{cli_lane_goal.CONTROLLER_EVIDENCE_HEADING}\n"
|
||||
f"current turn {goal_turn}/{goal_max_turns}; prior rejected "
|
||||
f"reports: {json.dumps(rejection_history[-5:])}."
|
||||
)
|
||||
accepted, judge_reason = cli_lane_goal.judge_goal_completion(
|
||||
judge_context,
|
||||
structured,
|
||||
)
|
||||
metadata["goal_judge_reason"] = judge_reason
|
||||
if not accepted:
|
||||
completion_problem = f"local goal judge requested continuation: {judge_reason}"
|
||||
|
||||
if (
|
||||
structured
|
||||
and structured.get("status") == "completed"
|
||||
and result.returncode == 0
|
||||
and completion_problem is None
|
||||
):
|
||||
terminal_file = _terminal_path(state_file, run_id, "pending")
|
||||
metadata["terminal_record"] = str(
|
||||
_terminal_path(state_file, run_id, "committed")
|
||||
)
|
||||
try:
|
||||
terminal_file, terminal_record = _write_terminal_record(
|
||||
state_file,
|
||||
board=board,
|
||||
task_id=task_id,
|
||||
run_id=run_id,
|
||||
structured=structured,
|
||||
summary=str(structured.get("summary") or "Completed"),
|
||||
metadata=metadata,
|
||||
)
|
||||
outcome = _finalize_terminal_record(
|
||||
kanban_db,
|
||||
terminal_file,
|
||||
terminal_record,
|
||||
)
|
||||
except Exception as error:
|
||||
identity = _terminal_identity(terminal_file)
|
||||
replayable = bool(
|
||||
identity is not None
|
||||
and _has_pending_finalization(
|
||||
identity.board,
|
||||
identity.task_id,
|
||||
identity.run_id,
|
||||
)
|
||||
)
|
||||
if not replayable:
|
||||
_recover_exact_run(
|
||||
kanban_db,
|
||||
identity,
|
||||
f"persistence raised {type(error).__name__}",
|
||||
)
|
||||
raise TerminalFinalizationPending(
|
||||
"accepted worker result remains outside the generic block path; "
|
||||
f"terminal replayable={replayable}; persistence/finalization "
|
||||
f"raised {type(error).__name__}"
|
||||
) from error
|
||||
if outcome != "committed":
|
||||
raise TerminalFinalizationPending(
|
||||
"accepted worker result is durably journaled but "
|
||||
f"Kanban finalization is {outcome}"
|
||||
)
|
||||
record_route_decision(
|
||||
route.provider, route.effort, route.classifier, "completed"
|
||||
)
|
||||
break
|
||||
|
||||
can_continue = (
|
||||
goal_mode
|
||||
and completion_problem is not None
|
||||
and goal_turn < goal_max_turns
|
||||
and deadline - time.monotonic() > 30
|
||||
)
|
||||
if can_continue:
|
||||
rejection_history = state.get("goal_rejections", [])
|
||||
if not isinstance(rejection_history, list):
|
||||
rejection_history = []
|
||||
state["goal_rejections"] = [
|
||||
*rejection_history[-4:],
|
||||
completion_problem,
|
||||
]
|
||||
goal_turn += 1
|
||||
comment(
|
||||
f"Goal completion rejected; continuing turn {goal_turn}/{goal_max_turns}: {completion_problem}",
|
||||
)
|
||||
escalation_context = (
|
||||
context
|
||||
+ "\n\nThe previous worker turn missed its completion quality mark: "
|
||||
+ completion_problem
|
||||
+ "\nSelect a route that can finish and verify the remaining work."
|
||||
)
|
||||
excluded = unavailable_provider
|
||||
if excluded is None and assignee == "cli-auto":
|
||||
excluded = fresh_unavailable_provider()
|
||||
record_route_decision(
|
||||
route.provider, route.effort, route.classifier, "goal-continued"
|
||||
)
|
||||
next_route = _routed_or_blocked(
|
||||
kanban_db,
|
||||
board,
|
||||
task_id,
|
||||
run_id,
|
||||
lambda boundary=escalation_context, blocked=excluded: select_route(
|
||||
boundary,
|
||||
assignee,
|
||||
exclude_provider=blocked,
|
||||
exclude_reason="is unavailable according to fresh native health"
|
||||
if blocked
|
||||
else None,
|
||||
),
|
||||
)
|
||||
if next_route is None:
|
||||
break
|
||||
comment(
|
||||
f"Goal route {goal_turn}/{goal_max_turns}: "
|
||||
f"{next_route.provider}/{next_route.model} at {next_route.effort}; "
|
||||
f"classifier={next_route.classifier}; {next_route.reason}",
|
||||
)
|
||||
handoff = (
|
||||
git_handoff(workspace, result.output)
|
||||
if next_route.provider != route.provider
|
||||
else ""
|
||||
)
|
||||
route = next_route
|
||||
prompt = build_prompt(
|
||||
context,
|
||||
workspace,
|
||||
handoff
|
||||
+ "\nGoal-loop continuation: the previous final report was rejected because "
|
||||
+ completion_problem
|
||||
+ ". Reinspect live state, finish the outstanding work, and return new final evidence.",
|
||||
)
|
||||
continue
|
||||
|
||||
reason = completion_problem
|
||||
if reason is None and structured:
|
||||
blockers = structured.get("blockers", [])
|
||||
reason = "; ".join(str(item) for item in blockers)
|
||||
reason = reason or str(structured.get("summary") or "")
|
||||
if reason is None:
|
||||
reason = result.output[-4000:]
|
||||
failure_reason = reason or (
|
||||
f"{route.provider} worker failed with exit {result.returncode}"
|
||||
)
|
||||
failure_kind = (
|
||||
"transient" if result.capacity_failure else "capability"
|
||||
)
|
||||
record_route_decision(
|
||||
route.provider, route.effort, route.classifier, f"blocked-{failure_kind}"
|
||||
)
|
||||
_board_call(
|
||||
kanban_db,
|
||||
board,
|
||||
lambda fresh, failure_reason=failure_reason, failure_kind=failure_kind: kanban_db.block_task(
|
||||
fresh,
|
||||
task_id,
|
||||
reason=failure_reason,
|
||||
kind=failure_kind,
|
||||
expected_run_id=run_id,
|
||||
),
|
||||
)
|
||||
break
|
||||
except TerminalFinalizationPending as error:
|
||||
comment(str(error))
|
||||
except Exception as error:
|
||||
failure_reason = f"Direct CLI lane failed: {type(error).__name__}: {error}"
|
||||
_board_call(
|
||||
kanban_db,
|
||||
board,
|
||||
lambda fresh: kanban_db.block_task(
|
||||
fresh,
|
||||
task_id,
|
||||
reason=failure_reason,
|
||||
kind="capability",
|
||||
expected_run_id=run_id,
|
||||
),
|
||||
)
|
||||
181
services/hermes/scripts/cli_lane_failover.py
Normal file
181
services/hermes/scripts/cli_lane_failover.py
Normal file
@ -0,0 +1,181 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Bounded, fail-closed cross-provider failover for one capacity boundary."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable
|
||||
|
||||
from cli_lane_board import _board_call
|
||||
from cli_lane_config import ProcessResult, Route
|
||||
from cli_lane_health import classify_capacity_failure, record_provider_failure
|
||||
from cli_lane_metrics import (
|
||||
record_provider_fallback,
|
||||
record_route_decision,
|
||||
record_router_selection_failure,
|
||||
)
|
||||
from cli_lane_prompt import build_prompt, git_handoff
|
||||
from cli_lane_provider import run_provider
|
||||
from cli_lane_records import _persist_candidate
|
||||
from cli_lane_routing import select_route
|
||||
|
||||
|
||||
@dataclass
|
||||
class FailoverOutcome:
|
||||
"""Where one capacity boundary left the claim's route and result."""
|
||||
|
||||
route: Route
|
||||
result: ProcessResult
|
||||
candidate_file: Path | None
|
||||
unavailable_provider: str | None
|
||||
blocked: bool
|
||||
|
||||
|
||||
def _block_transient(
|
||||
kanban_db: Any, board: str, task_id: str, run_id: Any, reason: str
|
||||
) -> None:
|
||||
"""Park one card as retryable so it re-enters when providers recover."""
|
||||
_board_call(
|
||||
kanban_db,
|
||||
board,
|
||||
lambda fresh: kanban_db.block_task(
|
||||
fresh,
|
||||
task_id,
|
||||
reason=reason,
|
||||
kind="transient",
|
||||
expected_run_id=run_id,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _routed_or_blocked(
|
||||
kanban_db: Any,
|
||||
board: str,
|
||||
task_id: str,
|
||||
run_id: Any,
|
||||
select: Callable[[], Route],
|
||||
) -> Route | None:
|
||||
"""Turn a Switchyard outage into a transient block, never a capability one."""
|
||||
try:
|
||||
return select()
|
||||
except RuntimeError as error:
|
||||
record_router_selection_failure()
|
||||
_block_transient(
|
||||
kanban_db,
|
||||
board,
|
||||
task_id,
|
||||
run_id,
|
||||
"Switchyard route selection is unavailable; the card retries "
|
||||
f"when the router recovers: {error}",
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def capacity_failover(
|
||||
kanban_db: Any,
|
||||
*,
|
||||
board: str,
|
||||
task_id: str,
|
||||
run_id: Any,
|
||||
assignee: str,
|
||||
comment: Callable[[str], None],
|
||||
context: str,
|
||||
workspace: Path,
|
||||
state: dict[str, Any],
|
||||
state_file: Path,
|
||||
log_path: Path,
|
||||
heartbeat: Callable[[str], bool],
|
||||
deadline: float,
|
||||
route: Route,
|
||||
result: ProcessResult,
|
||||
candidate_file: Path | None,
|
||||
goal_turn: int,
|
||||
) -> FailoverOutcome:
|
||||
"""Apply the capacity policy once: record, fail closed, or fail over."""
|
||||
failure_class = classify_capacity_failure(result.output)
|
||||
record_provider_failure(route.provider, failure_class)
|
||||
record_route_decision(
|
||||
route.provider, route.effort, route.classifier, "capacity-failure"
|
||||
)
|
||||
if assignee != "cli-auto":
|
||||
# Fail closed: an operator pinned this provider, so a capacity or
|
||||
# auth failure must wait for that provider, never switch away.
|
||||
_block_transient(
|
||||
kanban_db,
|
||||
board,
|
||||
task_id,
|
||||
run_id,
|
||||
f"Manually pinned provider {route.provider} hit a "
|
||||
f"{failure_class} failure; the explicit {assignee} lane never "
|
||||
"switches providers. Retry after recovery or reassign to cli-auto.",
|
||||
)
|
||||
return FailoverOutcome(route, result, candidate_file, None, True)
|
||||
retry_context = (
|
||||
context
|
||||
+ "\n\nRouting boundary: the first provider failed from capacity/authentication. "
|
||||
+ "Select the alternate hosted provider at an appropriate effort."
|
||||
)
|
||||
alternate = "claude" if route.provider == "codex" else "codex"
|
||||
fallback = _routed_or_blocked(
|
||||
kanban_db,
|
||||
board,
|
||||
task_id,
|
||||
run_id,
|
||||
lambda: select_route(retry_context, f"cli-{alternate}-{route.effort}"),
|
||||
)
|
||||
if fallback is None:
|
||||
return FailoverOutcome(route, result, candidate_file, route.provider, True)
|
||||
record_provider_fallback(route.provider, fallback.provider, failure_class)
|
||||
comment(
|
||||
f"Provider fallback: {route.provider} -> {fallback.provider} "
|
||||
f"after a {failure_class} failure; Jetson reclassified the retry boundary.",
|
||||
)
|
||||
fallback_result = run_provider(
|
||||
fallback,
|
||||
build_prompt(
|
||||
context,
|
||||
workspace,
|
||||
git_handoff(workspace, result.output),
|
||||
),
|
||||
workspace,
|
||||
state,
|
||||
state_file,
|
||||
log_path,
|
||||
heartbeat,
|
||||
max(1, int(deadline - time.monotonic())),
|
||||
)
|
||||
if fallback_result.capacity_failure:
|
||||
# Bounded failover depth: two hosted providers, never a third
|
||||
# metered or local path.
|
||||
fallback_class = classify_capacity_failure(fallback_result.output)
|
||||
record_provider_failure(fallback.provider, fallback_class)
|
||||
record_route_decision(
|
||||
fallback.provider, fallback.effort, fallback.classifier, "capacity-failure"
|
||||
)
|
||||
_block_transient(
|
||||
kanban_db,
|
||||
board,
|
||||
task_id,
|
||||
run_id,
|
||||
"Both hosted providers failed at this boundary "
|
||||
f"({route.provider}: {failure_class}; {fallback.provider}: "
|
||||
f"{fallback_class}); the lane never falls to a metered or "
|
||||
"local path. Waiting for provider recovery.",
|
||||
)
|
||||
return FailoverOutcome(
|
||||
fallback, fallback_result, candidate_file, route.provider, True
|
||||
)
|
||||
if fallback_result.structured:
|
||||
candidate_file = _persist_candidate(
|
||||
state,
|
||||
state_file,
|
||||
dict(fallback_result.structured),
|
||||
route=fallback,
|
||||
returncode=fallback_result.returncode,
|
||||
goal_turn=goal_turn,
|
||||
)
|
||||
return FailoverOutcome(
|
||||
fallback, fallback_result, candidate_file, route.provider, False
|
||||
)
|
||||
173
services/hermes/scripts/cli_lane_files.py
Normal file
173
services/hermes/scripts/cli_lane_files.py
Normal file
@ -0,0 +1,173 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Durable path and filesystem primitives for Hermes CLI lane journals."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import stat
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from cli_lane_config import STATE_ROOT, TerminalIdentity, canonical_run_id
|
||||
|
||||
|
||||
def atomic_json(path: Path, value: dict[str, Any], mode: int = 0o600) -> None:
|
||||
"""Durably replace a small state document without following temp symlinks."""
|
||||
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
os.chmod(path.parent, 0o700, follow_symlinks=False)
|
||||
temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp")
|
||||
flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
|
||||
flags |= getattr(os, "O_NOFOLLOW", 0)
|
||||
descriptor = None
|
||||
try:
|
||||
descriptor = os.open(temporary, flags, mode)
|
||||
with os.fdopen(descriptor, "w", encoding="utf-8") as stream:
|
||||
descriptor = None
|
||||
json.dump(value, stream, indent=2, sort_keys=True)
|
||||
stream.write("\n")
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
os.chmod(temporary, mode, follow_symlinks=False)
|
||||
os.replace(temporary, path)
|
||||
_fsync_directory(path.parent)
|
||||
finally:
|
||||
if descriptor is not None:
|
||||
os.close(descriptor)
|
||||
with contextlib.suppress(FileNotFoundError):
|
||||
temporary.unlink()
|
||||
|
||||
def _fsync_directory(directory: Path) -> None:
|
||||
"""Persist directory-entry changes after an atomic rename or quarantine."""
|
||||
flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
|
||||
flags |= getattr(os, "O_NOFOLLOW", 0)
|
||||
descriptor = os.open(directory, flags)
|
||||
try:
|
||||
os.fsync(descriptor)
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
try:
|
||||
flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
|
||||
descriptor = os.open(path, flags)
|
||||
with os.fdopen(descriptor, "r", encoding="utf-8") as stream:
|
||||
value = json.load(stream)
|
||||
except (OSError, UnicodeError, json.JSONDecodeError):
|
||||
return {}
|
||||
return value if isinstance(value, dict) else {}
|
||||
|
||||
def state_path(board: str, task_id: str) -> Path:
|
||||
safe_board = re.sub(r"[^a-zA-Z0-9_.-]+", "-", board)
|
||||
safe_task = re.sub(r"[^a-zA-Z0-9_.-]+", "-", task_id)
|
||||
return STATE_ROOT / safe_board / f"{safe_task}.json"
|
||||
|
||||
def _result_path(state_file: Path, run_id: Any, sequence: int) -> Path:
|
||||
"""Return a unique provider-result path; completed turns are never reused."""
|
||||
safe_run = re.sub(r"[^a-zA-Z0-9_.-]+", "-", str(run_id or "unknown"))
|
||||
return state_file.with_name(
|
||||
f"{state_file.stem}.run-{safe_run}.provider-{sequence}.result.json"
|
||||
)
|
||||
|
||||
def _candidate_path(state_file: Path, run_id: Any, sequence: int) -> Path:
|
||||
"""Return the durable path for one exact structured worker response."""
|
||||
safe_run = re.sub(r"[^a-zA-Z0-9_.-]+", "-", str(run_id or "unknown"))
|
||||
return state_file.with_name(
|
||||
f"{state_file.stem}.run-{safe_run}.candidate-{sequence}.json"
|
||||
)
|
||||
|
||||
def _terminal_path(
|
||||
state_file: Path,
|
||||
run_id: Any,
|
||||
state: str = "pending",
|
||||
) -> Path:
|
||||
"""Return the identity-bound replay path for an accepted terminal response."""
|
||||
if state not in {"pending", "committed"}:
|
||||
raise ValueError(f"invalid terminal journal state: {state}")
|
||||
canonical = canonical_run_id(run_id)
|
||||
if type(run_id) is not int or canonical is None:
|
||||
raise ValueError("terminal run ID must be a positive SQLite int64")
|
||||
return state_file.with_name(
|
||||
f"{state_file.stem}.run-{canonical}.terminal.{state}.json"
|
||||
)
|
||||
|
||||
_TERMINAL_NAME = re.compile(
|
||||
r"^(?P<task>[a-zA-Z0-9_.-]+)\.run-(?P<run>[0-9]+)\."
|
||||
r"terminal\.(?P<state>pending|committed)\.json$"
|
||||
)
|
||||
_TERMINAL_EVIDENCE_NAME = re.compile(
|
||||
r"^(?P<task>[a-zA-Z0-9_.-]+)\.run-(?P<run>[0-9]+)\."
|
||||
r"terminal\.(?P<state>prepared|conflict)-(?P<digest>[a-f0-9]{32})\.json$"
|
||||
)
|
||||
_RETIRE_NAME = re.compile(
|
||||
r"^\.retire\.(?P<path_digest>[a-f0-9]{16})\."
|
||||
r"(?P<source_digest>[a-f0-9]{16})\.(?P<sequence>[0-9]+)$"
|
||||
)
|
||||
_LEGACY_RETIRE_NAME = re.compile(r"^\.retire\.[a-f0-9]{32}\.[0-9]+$")
|
||||
_SAFE_BOARD = re.compile(r"^[a-zA-Z0-9_.-]+$")
|
||||
|
||||
|
||||
def _terminal_identity(path: Path) -> TerminalIdentity | None:
|
||||
"""Parse replay authority lexically before opening a journal or board."""
|
||||
try:
|
||||
relative = path.relative_to(STATE_ROOT)
|
||||
except ValueError:
|
||||
return None
|
||||
if len(relative.parts) != 2:
|
||||
return None
|
||||
board, filename = relative.parts
|
||||
match = _TERMINAL_NAME.fullmatch(filename)
|
||||
run_id = canonical_run_id(match.group("run")) if match else None
|
||||
try:
|
||||
board_stat = path.parent.stat(follow_symlinks=False)
|
||||
except OSError:
|
||||
return None
|
||||
if (
|
||||
not match
|
||||
or run_id is None
|
||||
or not stat.S_ISDIR(board_stat.st_mode)
|
||||
or not _SAFE_BOARD.fullmatch(board)
|
||||
or board in {".", ".."}
|
||||
or match.group("task") in {".", ".."}
|
||||
):
|
||||
return None
|
||||
return TerminalIdentity(
|
||||
board=board,
|
||||
task_id=match.group("task"),
|
||||
run_id=run_id,
|
||||
state=match.group("state"),
|
||||
)
|
||||
|
||||
def _terminal_evidence_identity(path: Path) -> TerminalIdentity | None:
|
||||
"""Parse immutable prepared/conflict evidence authority from its path."""
|
||||
try:
|
||||
relative = path.relative_to(STATE_ROOT)
|
||||
except ValueError:
|
||||
return None
|
||||
if len(relative.parts) != 2:
|
||||
return None
|
||||
board, filename = relative.parts
|
||||
match = _TERMINAL_EVIDENCE_NAME.fullmatch(filename)
|
||||
run_id = canonical_run_id(match.group("run")) if match else None
|
||||
try:
|
||||
board_stat = path.parent.stat(follow_symlinks=False)
|
||||
except OSError:
|
||||
return None
|
||||
if (
|
||||
not match
|
||||
or run_id is None
|
||||
or not stat.S_ISDIR(board_stat.st_mode)
|
||||
or not _SAFE_BOARD.fullmatch(board)
|
||||
or board in {".", ".."}
|
||||
or match.group("task") in {".", ".."}
|
||||
):
|
||||
return None
|
||||
return TerminalIdentity(
|
||||
board=board,
|
||||
task_id=match.group("task"),
|
||||
run_id=run_id,
|
||||
state=match.group("state"),
|
||||
)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
x
Reference in New Issue
Block a user