titan/atlas-iac
3
0
Fork 0
Code Issues Pull Requests 9 Actions Packages Projects Releases Wiki Activity

hermes: add daemonless agent image release lane

Browse Source
This commit is contained in:
jenkins 2026-08-16 20:04:17 -03:00
parent 9bf1ab8a9c
commit 4236538c8f
37 changed files with 3376 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

258
ci/Jenkinsfile.hermes-agent-image Normal file
View File

@ -0,0 +1,258 @@
pipeline {
agent {
kubernetes {
defaultContainer 'python'
yaml """
apiVersion: v1
kind: Pod
metadata:
labels:
atlas.bstein.dev/workload: hermes-agent-image-builder
spec:
automountServiceAccountToken: false
enableServiceLinks: false
restartPolicy: Never
securityContext:
fsGroup: 1000
fsGroupChangePolicy: OnRootMismatch
nodeSelector:
kubernetes.io/arch: arm64
node-role.kubernetes.io/worker: "true"
hardware: rpi5
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: NotIn
values:
- titan-04
- titan-14
- titan-18
- titan-19
- titan-22
- titan-24
imagePullSecrets:
- name: harbor-bstein-robot
containers:
- name: jnlp
image: jenkins/inbound-agent@sha256:8eda4fe2a66bcf6a5e43436d9918fc14c306204dc8fcd75f4e15e0e6e5dc759a
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
runAsNonRoot: true
runAsUser: 1000
seccompProfile:
type: RuntimeDefault
resources:
requests:
cpu: 25m
memory: 256Mi
limits:
cpu: 500m
memory: 512Mi
- name: python
image: registry.bstein.dev/bstein/python@sha256:269541d3387baae008df4608ead893dba2b5cdaad1a5a380731a88992d34b808
command: ["sleep"]
args: ["99d"]
tty: true
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
runAsNonRoot: true
runAsUser: 1000
seccompProfile:
type: RuntimeDefault
resources:
requests:
cpu: 25m
memory: 64Mi
limits:
cpu: 250m
memory: 256Mi
- name: kaniko
image: gcr.io/kaniko-project/executor@sha256:c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e
command: ["/busybox/sh", "-c"]
args: ["/busybox/sleep 99d"]
tty: true
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"]
privileged: false
runAsUser: 0
seccompProfile:
type: RuntimeDefault
resources:
requests:
cpu: 250m
memory: 1Gi
limits:
cpu: "2"
memory: 4Gi
"""
}
}
parameters {
booleanParam(
name: 'PUBLISH_IMAGE',
defaultValue: false,
description: 'Publish the reviewed main revision to Harbor.'
)
string(
name: 'EXPECTED_SOURCE_REVISION',
defaultValue: '',
description: 'Exact 40-character commit on atlas/titan-iac main.'
)
string(
name: 'CONFIRM_PUBLISH',
defaultValue: '',
description: 'Enter PUBLISH HERMES AGENT to confirm the release.'
)
}
environment {
HERMES_IMAGE = 'registry.bstein.dev/bstein/hermes-agent'
}
options {
disableConcurrentBuilds()
buildDiscarder(logRotator(daysToKeepStr: '30', numToKeepStr: '100', artifactDaysToKeepStr: '30', artifactNumToKeepStr: '100'))
skipDefaultCheckout(true)
timeout(time: 90, unit: 'MINUTES')
}
stages {
stage('Checkout reviewed source') {
steps {
checkout scm
}
}
stage('Enforce release boundary') {
steps {
sh '''
set -eu
mkdir -p build
test "${PUBLISH_IMAGE}" = "true"
test "${CONFIRM_PUBLISH}" = "PUBLISH HERMES AGENT"
case "${EXPECTED_SOURCE_REVISION}" in
*[!0-9a-f]*|'')
echo "EXPECTED_SOURCE_REVISION must be a lowercase full commit" >&2
exit 2
;;
esac
test "${#EXPECTED_SOURCE_REVISION}" -eq 40
actual_revision="$(git rev-parse HEAD)"
test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}"
test "${actual_revision}" = "$(git rev-parse origin/main)"
test -z "$(git status --porcelain)"
test -f dockerfiles/Dockerfile.hermes-agent
case "${BUILD_NUMBER}" in
''|0*|*[!0-9]*)
echo "BUILD_NUMBER must be a positive decimal integer" >&2
exit 2
;;
esac
printf '%s\n' \
"${HERMES_IMAGE}:git-${actual_revision}-build-${BUILD_NUMBER}" \
> build/hermes-agent.destination
'''
}
}
stage('Reject replay before publish') {
steps {
withCredentials([usernamePassword(
credentialsId: 'harbor-robot',
usernameVariable: 'HARBOR_USER',
passwordVariable: 'HARBOR_PASSWORD'
)]) {
sh '''
set -eu
set +x
destination="$(cat build/hermes-agent.destination)"
python3 ci/scripts/hermes_image_release.py assert-absent \
--source-revision "${EXPECTED_SOURCE_REVISION}" \
--build-number "${BUILD_NUMBER}" \
--destination "${destination}"
'''
}
}
}
stage('Build and publish without a daemon') {
steps {
container('kaniko') {
withCredentials([usernamePassword(
credentialsId: 'harbor-robot',
usernameVariable: 'HARBOR_USER',
passwordVariable: 'HARBOR_PASSWORD'
)]) {
sh '''#!/busybox/sh
set -eu
set +x
config_path=/kaniko/.docker/config.json
destination="$(cat build/hermes-agent.destination)"
umask 077
auth="$(printf '%s:%s' "${HARBOR_USER}" "${HARBOR_PASSWORD}" | /busybox/base64 | /busybox/tr -d '\n')"
/busybox/mkdir -p /kaniko/.docker
/busybox/printf '{"auths":{"registry.bstein.dev":{"auth":"%s"}}}\n' "${auth}" > "${config_path}"
unset HARBOR_USER HARBOR_PASSWORD auth
trap '/busybox/rm -f "${config_path}"' EXIT HUP INT TERM
/kaniko/executor \
--context="dir://${WORKSPACE}" \
--dockerfile="${WORKSPACE}/dockerfiles/Dockerfile.hermes-agent" \
--destination="${destination}" \
--digest-file="${WORKSPACE}/build/hermes-agent.digest" \
--image-name-tag-with-digest-file="${WORKSPACE}/build/hermes-agent.image" \
--build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1 \
--label="org.opencontainers.image.revision=${EXPECTED_SOURCE_REVISION}" \
--cleanup \
--push-retry=3
'''
}
}
}
}
stage('Render reviewed Flux handoff') {
steps {
withCredentials([usernamePassword(
credentialsId: 'harbor-robot',
usernameVariable: 'HARBOR_USER',
passwordVariable: 'HARBOR_PASSWORD'
)]) {
sh '''
set -eu
set +x
destination="$(cat build/hermes-agent.destination)"
python3 ci/scripts/hermes_image_release.py render \
--digest-file build/hermes-agent.digest \
--image-file build/hermes-agent.image \
--source-revision "${EXPECTED_SOURCE_REVISION}" \
--build-number "${BUILD_NUMBER}" \
--destination "${destination}" \
--kustomization services/hermes/kustomization.yaml \
--output-dir build/hermes-agent-release
test -s build/hermes-agent-release/hermes-image-update.patch
test -s build/hermes-agent-release/hermes-agent-image.json
'''
}
}
}
}
post {
always {
archiveArtifacts(
artifacts: 'build/hermes-agent.digest,build/hermes-agent.image,build/hermes-agent.destination,build/hermes-agent-release/**',
allowEmptyArchive: true,
fingerprint: true
)
}
cleanup {
container('kaniko') {
sh '''#!/busybox/sh
/busybox/rm -f /kaniko/.docker/config.json
'''
}
}
}
}

426
ci/scripts/hermes_image_release.py Executable file
View File

@ -0,0 +1,426 @@
#!/usr/bin/env python3
"""Verify and render a reviewable Hermes agent image release."""
from __future__ import annotations
import argparse
import base64
import difflib
import json
import os
import re
import urllib.error
import urllib.parse
import urllib.request
from pathlib import Path
from typing import Any, Callable
DEFAULT_IMAGE = "registry.bstein.dev/bstein/hermes-agent"
HARBOR_API_ORIGIN = "https://registry.bstein.dev/api/v2.0"
HARBOR_PROJECT = "bstein"
HARBOR_REPOSITORY = "hermes-agent"
IMMUTABLE_REPOSITORY_PATTERN = "hermes-agent"
IMMUTABLE_TAG_PATTERN = "git-*-build-*"
DIGEST_PATTERN = re.compile(r"^sha256:[0-9a-f]{64}$")
REVISION_PATTERN = re.compile(r"^[0-9a-f]{40}$")
BUILD_PATTERN = re.compile(r"^[1-9][0-9]*$")
DESTINATION_PATTERN = re.compile(
r"^registry\.bstein\.dev/bstein/hermes-agent:"
r"git-([0-9a-f]{40})-build-([1-9][0-9]*)$"
)
class _NoRedirect(urllib.request.HTTPRedirectHandler):
"""Never send registry credentials to a redirect target."""
def redirect_request(self, _request, _file, _code, _message, _headers, _url):
return None
def _validated(value: str, pattern: re.Pattern[str], label: str) -> str:
"""Return a normalized value when it matches the release contract."""
normalized = value.strip()
if not pattern.fullmatch(normalized):
raise ValueError(f"invalid {label}: expected {pattern.pattern}")
return normalized
def validate_destination(
destination: str, source_revision: str, build_number: str
) -> tuple[str, str]:
"""Bind one unique build tag to the reviewed revision and Jenkins build."""
revision = _validated(source_revision, REVISION_PATTERN, "source revision")
build = _validated(build_number, BUILD_PATTERN, "build number")
match = DESTINATION_PATTERN.fullmatch(destination.strip())
if not match or match.groups() != (revision, build):
raise ValueError(
"destination must bind the reviewed revision and unique Jenkins build"
)
return revision, build
def validate_kaniko_evidence(
*, digest_text: str, image_text: str, destination: str
) -> str:
"""Cross-check both independent Kaniko output files against the destination."""
digest_lines = digest_text.splitlines()
image_lines = image_text.splitlines()
if len(digest_lines) != 1:
raise ValueError("Kaniko digest evidence must contain exactly one line")
if len(image_lines) != 1:
raise ValueError("Kaniko image evidence must contain exactly one line")
digest = _validated(digest_lines[0], DIGEST_PATTERN, "image digest")
if image_lines[0].strip() != f"{destination}@{digest}":
raise ValueError("Kaniko image evidence does not match destination and digest")
return digest
def _registry_request(request: urllib.request.Request, timeout: int) -> Any:
"""Make a registry request without following redirects."""
opener = urllib.request.build_opener(_NoRedirect())
try:
return opener.open(request, timeout=timeout)
except urllib.error.HTTPError as exc:
return exc
def _artifact_response(
destination: str,
*,
username: str,
password: str,
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
) -> tuple[int, bytes]:
"""Read one exact Harbor artifact by tag with bounded response size."""
match = DESTINATION_PATTERN.fullmatch(destination)
if not match:
raise ValueError("invalid destination")
if not username or not password:
raise RuntimeError("Harbor credentials are empty")
tag = destination.rsplit(":", 1)[1]
encoded_tag = urllib.parse.quote(tag, safe="")
auth = base64.b64encode(f"{username}:{password}".encode()).decode("ascii")
request = urllib.request.Request(
f"{HARBOR_API_ORIGIN}/projects/{HARBOR_PROJECT}/repositories/"
f"{HARBOR_REPOSITORY}/artifacts/{encoded_tag}"
"?with_immutable_status=true",
headers={"Accept": "application/json", "Authorization": f"Basic {auth}"},
method="GET",
)
with opener(request, 20) as response:
body = response.read(1_048_577)
if len(body) > 1_048_576:
raise RuntimeError("Harbor artifact response exceeded the size limit")
return int(response.status), body
def _immutable_rules_response(
*,
username: str,
password: str,
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
) -> tuple[int, bytes, dict[str, str]]:
"""Read the project policy with the same least-privilege publish identity."""
if not username or not password:
raise RuntimeError("Harbor credentials are empty")
auth = base64.b64encode(f"{username}:{password}".encode()).decode("ascii")
request = urllib.request.Request(
f"{HARBOR_API_ORIGIN}/projects/{HARBOR_PROJECT}/immutabletagrules"
"?page=1&page_size=100",
headers={"Accept": "application/json", "Authorization": f"Basic {auth}"},
method="GET",
)
with opener(request, 20) as response:
body = response.read(1_048_577)
if len(body) > 1_048_576:
raise RuntimeError("Harbor immutable rule response exceeded the size limit")
return int(response.status), body, dict(response.headers)
def _require_complete_rule_page(
rules: list[dict[str, Any]], headers: dict[str, str]
) -> None:
"""Require proof that the bounded first page contains every rule."""
raw_total = next(
(value for key, value in headers.items() if key.lower() == "x-total-count"),
None,
)
if raw_total is None or not str(raw_total).isdecimal():
raise RuntimeError("Harbor immutable rule list omitted a valid total count")
if int(raw_total) != len(rules):
raise RuntimeError("Harbor immutable rule list was truncated")
def _normalized_immutable_rule(rule: dict[str, Any]) -> dict[str, Any]:
"""Select only fields that bind the server-side build-tag policy."""
return {
"disabled": bool(rule.get("disabled", False)),
"action": rule.get("action"),
"template": rule.get("template"),
"tag_selectors": [
{
"kind": item.get("kind"),
"decoration": item.get("decoration"),
"pattern": item.get("pattern"),
}
for item in rule.get("tag_selectors") or []
if isinstance(item, dict)
],
"scope_selectors": {
"repository": [
{
"kind": item.get("kind"),
"decoration": item.get("decoration"),
"pattern": item.get("pattern"),
}
for item in (rule.get("scope_selectors") or {}).get(
"repository", []
)
if isinstance(item, dict)
]
},
}
def verify_immutable_policy(
*,
username: str,
password: str,
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
) -> None:
"""Fail closed before build unless the exact Harbor rule is active."""
status, body, headers = _immutable_rules_response(
username=username, password=password, opener=opener
)
if status != 200:
raise RuntimeError(f"Harbor immutable policy preflight returned HTTP {status}")
try:
rules = json.loads(body.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise RuntimeError("Harbor returned invalid immutable rule JSON") from exc
if not isinstance(rules, list) or not all(isinstance(item, dict) for item in rules):
raise RuntimeError("Harbor immutable rule list has an invalid shape")
_require_complete_rule_page(rules, headers)
expected = {
"disabled": False,
"action": "immutable",
"template": "immutable_template",
"tag_selectors": [
{
"kind": "doublestar",
"decoration": "matches",
"pattern": IMMUTABLE_TAG_PATTERN,
}
],
"scope_selectors": {
"repository": [
{
"kind": "doublestar",
"decoration": "repoMatches",
"pattern": IMMUTABLE_REPOSITORY_PATTERN,
}
]
},
}
matches = [
_normalized_immutable_rule(item)
for item in rules
if _normalized_immutable_rule(item)["tag_selectors"]
== expected["tag_selectors"]
and _normalized_immutable_rule(item)["scope_selectors"]
== expected["scope_selectors"]
]
if matches != [expected]:
raise RuntimeError("Harbor immutable build-tag policy is absent or not exact")
def assert_tag_absent(
destination: str,
*,
username: str,
password: str,
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
) -> None:
"""Reject replay before Kaniko can push an already-used immutable identity."""
status, _body = _artifact_response(
destination, username=username, password=password, opener=opener
)
if status == 404:
return
if status == 200:
raise RuntimeError("Harbor destination tag already exists; refusing overwrite")
raise RuntimeError(f"Harbor destination preflight returned HTTP {status}")
def verify_registry_digest(
destination: str,
digest: str,
*,
username: str,
password: str,
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
) -> None:
"""Verify Harbor independently resolves the pushed tag to Kaniko's digest."""
digest = _validated(digest, DIGEST_PATTERN, "image digest")
status, body = _artifact_response(
destination, username=username, password=password, opener=opener
)
if status != 200:
raise RuntimeError(f"Harbor manifest verification returned HTTP {status}")
try:
artifact = json.loads(body.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise RuntimeError("Harbor returned invalid artifact JSON") from exc
harbor_digest = str(artifact.get("digest") or "").strip()
if not DIGEST_PATTERN.fullmatch(harbor_digest):
raise RuntimeError("Harbor response omitted a valid artifact digest")
if harbor_digest != digest:
raise RuntimeError("Harbor digest does not match Kaniko evidence")
expected_tag = destination.rsplit(":", 1)[1]
matching_tags = [
item
for item in artifact.get("tags") or []
if isinstance(item, dict) and item.get("name") == expected_tag
]
if len(matching_tags) != 1:
raise RuntimeError("Harbor artifact does not contain the expected tag")
if matching_tags[0].get("immutable") is not True:
raise RuntimeError("Harbor did not enforce the expected tag as immutable")
def render_kustomization(source: str, digest: str, image: str = DEFAULT_IMAGE) -> str:
"""Replace exactly one matching Kustomize image digest without reformatting."""
digest = _validated(digest, DIGEST_PATTERN, "image digest")
lines = source.splitlines(keepends=True)
matches: list[int] = []
for index, line in enumerate(lines):
if line.strip() != f"- name: {image}":
continue
name_indent = len(line) - len(line.lstrip())
for candidate_index in range(index + 1, len(lines)):
candidate = lines[candidate_index]
stripped = candidate.strip()
candidate_indent = len(candidate) - len(candidate.lstrip())
if stripped.startswith("- name:") and candidate_indent == name_indent:
break
if stripped.startswith("digest:") and candidate_indent > name_indent:
matches.append(candidate_index)
break
if len(matches) != 1:
raise ValueError(
f"expected exactly one digest for image {image!r}; found {len(matches)}"
)
index = matches[0]
newline = "\n" if lines[index].endswith("\n") else ""
prefix = lines[index][: len(lines[index]) - len(lines[index].lstrip())]
lines[index] = f"{prefix}digest: {digest}{newline}"
return "".join(lines)
def write_release_artifacts(
*,
digest: str,
source_revision: str,
build_number: str,
destination: str,
kustomization: Path,
output_dir: Path,
) -> dict[str, str]:
"""Write a rendered manifest, patch, and credential-free release metadata."""
digest = _validated(digest, DIGEST_PATTERN, "image digest")
source_revision, build_number = validate_destination(
destination, source_revision, build_number
)
source = kustomization.read_text(encoding="utf-8")
rendered = render_kustomization(source, digest)
relative_name = kustomization.name
patch = "".join(
difflib.unified_diff(
source.splitlines(keepends=True),
rendered.splitlines(keepends=True),
fromfile=f"a/services/hermes/{relative_name}",
tofile=f"b/services/hermes/{relative_name}",
)
)
if not patch:
raise ValueError("published digest already matches the Flux manifest")
output_dir.mkdir(parents=True, exist_ok=True)
(output_dir / "hermes-kustomization.yaml").write_text(rendered, encoding="utf-8")
(output_dir / "hermes-image-update.patch").write_text(patch, encoding="utf-8")
metadata = {
"build_number": build_number,
"digest": digest,
"flux_image": f"{DEFAULT_IMAGE}@{digest}",
"image": DEFAULT_IMAGE,
"published_tag": destination,
"source_revision": source_revision,
}
(output_dir / "hermes-agent-image.json").write_text(
json.dumps(metadata, indent=2, sort_keys=True) + "\n", encoding="utf-8"
)
return metadata
def _credentials() -> tuple[str, str]:
"""Read the masked, runtime-only Jenkins credential environment."""
username = os.environ.get("HARBOR_USER", "")
password = os.environ.get("HARBOR_PASSWORD", "")
if not username or not password:
raise RuntimeError("Harbor credentials are unavailable")
return username, password
def _common_arguments(parser: argparse.ArgumentParser) -> None:
parser.add_argument("--source-revision", required=True)
parser.add_argument("--build-number", required=True)
parser.add_argument("--destination", required=True)
def main() -> int:
"""Fail closed around the unique tag, then verify and render after push."""
parser = argparse.ArgumentParser()
commands = parser.add_subparsers(dest="command", required=True)
absent = commands.add_parser("assert-absent")
_common_arguments(absent)
render = commands.add_parser("render")
_common_arguments(render)
render.add_argument("--digest-file", required=True, type=Path)
render.add_argument("--image-file", required=True, type=Path)
render.add_argument("--kustomization", required=True, type=Path)
render.add_argument("--output-dir", required=True, type=Path)
args = parser.parse_args()
validate_destination(args.destination, args.source_revision, args.build_number)
username, password = _credentials()
if args.command == "assert-absent":
verify_immutable_policy(username=username, password=password)
assert_tag_absent(args.destination, username=username, password=password)
return 0
digest = validate_kaniko_evidence(
digest_text=args.digest_file.read_text(encoding="utf-8"),
image_text=args.image_file.read_text(encoding="utf-8"),
destination=args.destination,
)
verify_registry_digest(
args.destination, digest, username=username, password=password
)
write_release_artifacts(
digest=digest,
source_revision=args.source_revision,
build_number=args.build_number,
destination=args.destination,
kustomization=args.kustomization,
output_dir=args.output_dir,
)
return 0
if __name__ == "__main__": # pragma: no cover - exercised through main()
raise SystemExit(main())

8
clusters/atlas/flux-system/applications/harbor/kustomization.yaml
View File

@ -15,7 +15,13 @@ spec:
kind: GitRepository
name: flux-system
namespace: flux-system
wait: false
wait: true
timeout: 10m
healthChecks:
- apiVersion: batch/v1
kind: Job
name: harbor-hermes-agent-immutability-ensure-1
namespace: harbor
dependsOn:
- name: core
- name: longhorn

1
clusters/atlas/flux-system/applications/hermes/kustomization.yaml
View File

@ -60,3 +60,4 @@ spec:
- name: keycloak
- name: longhorn
- name: vault
- name: jenkins

5
clusters/atlas/flux-system/applications/jenkins/kustomization.yaml
View File

@ -6,10 +6,9 @@ metadata:
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "CI controller changes are applied only during planned maintenance."
spec:
interval: 10m
suspend: true
suspend: false
path: ./services/jenkins
prune: true
sourceRef:
@ -18,6 +17,8 @@ spec:
targetNamespace: jenkins
dependsOn:
- name: helm
- name: harbor
- name: vault-hermes-jenkins-token-seed
healthChecks:
- apiVersion: apps/v1
kind: Deployment

1
clusters/atlas/flux-system/applications/kustomization.yaml
View File

@ -4,6 +4,7 @@ kind: Kustomization
resources:
- gitea/kustomization.yaml
- vault/kustomization.yaml
- vault-hermes-jenkins-token-seed/kustomization.yaml
- vaultwarden/kustomization.yaml
- comms/kustomization.yaml
- crypto/kustomization.yaml

26
clusters/atlas/flux-system/applications/vault-hermes-jenkins-token-seed/kustomization.yaml Normal file
View File

@ -0,0 +1,26 @@
# clusters/atlas/flux-system/applications/vault-hermes-jenkins-token-seed/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: vault-hermes-jenkins-token-seed
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec:
interval: 10m
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
path: ./services/vault-hermes-jenkins-token-seed
targetNamespace: vault
prune: true
wait: true
timeout: 5m
dependsOn:
- name: vault
healthChecks:
- apiVersion: batch/v1
kind: Job
name: vault-hermes-jenkins-build-token-seed-2
namespace: vault

5
clusters/atlas/flux-system/applications/vault/kustomization.yaml
View File

@ -16,6 +16,11 @@ spec:
targetNamespace: vault
prune: true
wait: true
healthChecks:
- apiVersion: batch/v1
kind: Job
name: vault-k8s-auth-hermes-9
namespace: vault
dependsOn:
- name: longhorn
- name: helm

10
dockerfiles/Dockerfile.hermes-agent
View File

@ -1433,6 +1433,16 @@ NODE
COPY dockerfiles/hermes-session-migrate.py /opt/hermes/bin/hermes-session-migrate
ARG HERMES_KANIKO_HEREDOC_COMPAT=0
COPY dockerfiles/Dockerfile.hermes-agent /tmp/hermes-agent.Dockerfile
COPY dockerfiles/hermes-kaniko-heredoc-runner.py /tmp/hermes-kaniko-heredoc-runner.py
RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0|1) ;; *) exit 2 ;; esac \
&& if [ "${HERMES_KANIKO_HEREDOC_COMPAT}" = 1 ]; then \
python /tmp/hermes-kaniko-heredoc-runner.py \
--dockerfile /tmp/hermes-agent.Dockerfile; \
fi \
&& rm -f /tmp/hermes-agent.Dockerfile /tmp/hermes-kaniko-heredoc-runner.py
RUN cd /opt/hermes/web \
&& npm run build \
&& grep -Fq 'await api.getSessions(1, 0' src/pages/ChatPage.tsx \

2
dockerfiles/Dockerfile.hermes-agent.dockerignore
View File

@ -4,3 +4,5 @@
!dockerfiles/hermes-public-extract/**
!dockerfiles/hermes-session-activity-panel.tsx
!dockerfiles/hermes-session-migrate.py
!dockerfiles/Dockerfile.hermes-agent
!dockerfiles/hermes-kaniko-heredoc-runner.py

79
dockerfiles/hermes-kaniko-heredoc-runner.py Normal file
View File

@ -0,0 +1,79 @@
#!/usr/bin/env python3
"""Replay the reviewed Hermes Dockerfile heredocs for pinned Kaniko.
Kaniko v1.24 parses shell heredoc bodies into ``RunCommand.Files`` but its RUN
implementation executes only ``CmdLine``. The interpreters therefore receive
empty stdin and exit successfully. Docker and BuildKit execute these blocks
normally, so this compatibility runner is enabled only by the Kaniko pipeline.
"""
from __future__ import annotations
import argparse
import subprocess
from pathlib import Path
MAX_DOCKERFILE_BYTES = 2_000_000
BLOCKS = (
("RUN node <<'NODE'", "NODE", ("node",)),
("RUN python - <<'PY'", "PY", ("python", "-")),
)
EXPECTED_COMMANDS = ("node", *("python",) * 7, "node")
def extract_blocks(source: str) -> list[tuple[tuple[str, ...], str]]:
"""Return only exact supported RUN heredocs in their reviewed order."""
markers = {start: (end, command) for start, end, command in BLOCKS}
lines = source.splitlines()
blocks: list[tuple[tuple[str, ...], str]] = []
index = 0
while index < len(lines):
marker = markers.get(lines[index])
if marker is None:
index += 1
continue
end, command = marker
body_start = index + 1
index = body_start
while index < len(lines) and lines[index] != end:
index += 1
if index == len(lines):
raise ValueError(f"unterminated {command[0]} heredoc")
body = "\n".join(lines[body_start:index]) + "\n"
if not body.strip():
raise ValueError(f"empty {command[0]} heredoc")
blocks.append((command, body))
index += 1
commands = tuple(command[0] for command, _body in blocks)
if commands != EXPECTED_COMMANDS:
raise ValueError(
"Hermes Dockerfile heredoc contract changed: "
f"expected {EXPECTED_COMMANDS!r}, received {commands!r}"
)
return blocks
def replay(dockerfile: Path) -> None:
"""Execute each exact heredoc in an isolated interpreter process."""
size = dockerfile.stat().st_size
if size < 1 or size > MAX_DOCKERFILE_BYTES:
raise ValueError("Hermes Dockerfile size is outside the reviewed boundary")
source = dockerfile.read_text(encoding="utf-8")
for index, (command, body) in enumerate(extract_blocks(source), start=1):
print(f"replaying reviewed Dockerfile heredoc {index}/{len(EXPECTED_COMMANDS)}")
subprocess.run(command, input=body, text=True, check=True)
def main() -> int:
"""Parse the one explicit Dockerfile path and replay its reviewed patches."""
parser = argparse.ArgumentParser()
parser.add_argument("--dockerfile", required=True, type=Path)
args = parser.parse_args()
replay(args.dockerfile)
return 0
if __name__ == "__main__":
raise SystemExit(main())

79
services/harbor/hermes-agent-immutability-job.yaml Normal file
View File

@ -0,0 +1,79 @@
# services/harbor/hermes-agent-immutability-job.yaml
apiVersion: batch/v1
kind: Job
metadata:
name: harbor-hermes-agent-immutability-ensure-1
namespace: harbor
spec:
backoffLimit: 2
activeDeadlineSeconds: 600
template:
metadata:
annotations:
vault.hashicorp.com/agent-inject: "true"
vault.hashicorp.com/agent-pre-populate-only: "true"
vault.hashicorp.com/agent-run-as-user: "65532"
vault.hashicorp.com/agent-run-as-group: "65532"
vault.hashicorp.com/role: harbor-policy-bootstrap
vault.hashicorp.com/agent-inject-secret-harbor-admin-password: kv/data/atlas/harbor/harbor-core
vault.hashicorp.com/agent-inject-template-harbor-admin-password: |
{{- with secret "kv/data/atlas/harbor/harbor-core" -}}
{{ .Data.data.harbor_admin_password }}
{{- end -}}
spec:
serviceAccountName: harbor-policy-bootstrap
enableServiceLinks: false
restartPolicy: Never
nodeSelector:
hardware: rpi5
kubernetes.io/arch: arm64
node-role.kubernetes.io/worker: "true"
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: NotIn
values: [titan-04, titan-14, titan-18, titan-19, titan-24]
securityContext:
fsGroup: 65532
fsGroupChangePolicy: OnRootMismatch
seccompProfile:
type: RuntimeDefault
containers:
- name: ensure
image: docker.io/library/python@sha256:efcdfa6a6b2fd2afb9c7dfa9a5b288a6f68338b5cfdebe6b637d986067d85757
imagePullPolicy: IfNotPresent
command: [python3, /scripts/harbor_hermes_agent_immutability_ensure.py]
env:
- name: HARBOR_API_ORIGIN
value: https://registry.bstein.dev/api/v2.0
- name: HARBOR_ADMIN_PASSWORD_FILE
value: /vault/secrets/harbor-admin-password
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
readOnlyRootFilesystem: true
runAsGroup: 65532
runAsNonRoot: true
runAsUser: 65532
seccompProfile:
type: RuntimeDefault
volumeMounts:
- name: scripts
mountPath: /scripts
readOnly: true
- name: tmp
mountPath: /tmp
resources:
requests: {cpu: 25m, memory: 32Mi}
limits: {cpu: 250m, memory: 128Mi}
volumes:
- name: scripts
configMap:
name: harbor-hermes-agent-immutability-script
defaultMode: 0555
- name: tmp
emptyDir: {}

5
services/harbor/kustomization.yaml
View File

@ -12,9 +12,14 @@ resources:
- certificate.yaml
- helmrelease.yaml
- vault-sync-deployment.yaml
- policy-bootstrap-serviceaccount.yaml
- hermes-agent-immutability-job.yaml
- bootstrap-jobs/cassandra-registry-ensure-job.yaml
- image.yaml
configMapGenerator:
- name: harbor-vault-entrypoint
files:
- scripts/vault-entrypoint.sh
- name: harbor-hermes-agent-immutability-script
files:
- harbor_hermes_agent_immutability_ensure.py=scripts/harbor_hermes_agent_immutability_ensure.py

7
services/harbor/policy-bootstrap-serviceaccount.yaml Normal file
View File

@ -0,0 +1,7 @@
# services/harbor/policy-bootstrap-serviceaccount.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: harbor-policy-bootstrap
namespace: harbor
automountServiceAccountToken: true

378
services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py Normal file
View File

@ -0,0 +1,378 @@
#!/usr/bin/env python3
"""Create and verify the narrowly scoped Hermes agent immutable-tag rule."""
from __future__ import annotations
import base64
import json
import os
import time
import urllib.error
import urllib.parse
import urllib.request
from pathlib import Path
from typing import Any
PROJECT = "bstein"
REPOSITORY_PATTERN = "hermes-agent"
TAG_PATTERN = "git-*-build-*"
PUBLISH_ROBOT = "robot$jenkins-pipelines"
EXPECTED_ORIGIN = "https://registry.bstein.dev/api/v2.0"
MAX_RESPONSE = 1_048_576
TRANSIENT_STATUSES = {429, 502, 503, 504}
EXPECTED_RULE = {
"disabled": False,
"action": "immutable",
"template": "immutable_template",
"tag_selectors": [
{
"kind": "doublestar",
"decoration": "matches",
"pattern": TAG_PATTERN,
}
],
"scope_selectors": {
"repository": [
{
"kind": "doublestar",
"decoration": "repoMatches",
"pattern": REPOSITORY_PATTERN,
}
]
},
}
class NoRedirect(urllib.request.HTTPRedirectHandler):
"""Prevent Basic credentials from following an unexpected redirect."""
def redirect_request(self, _request, _file, _code, _message, _headers, _url):
return None
class HarborUnavailable(RuntimeError):
"""Harbor is not ready yet, rather than returning a policy decision."""
def normalized_rule(rule: dict[str, Any]) -> dict[str, Any]:
"""Return only the immutable contract fields Harbor must preserve."""
return {
"disabled": bool(rule.get("disabled", False)),
"action": rule.get("action"),
"template": rule.get("template"),
"tag_selectors": [
{
"kind": selector.get("kind"),
"decoration": selector.get("decoration"),
"pattern": selector.get("pattern"),
}
for selector in rule.get("tag_selectors") or []
if isinstance(selector, dict)
],
"scope_selectors": {
"repository": [
{
"kind": selector.get("kind"),
"decoration": selector.get("decoration"),
"pattern": selector.get("pattern"),
}
for selector in (rule.get("scope_selectors") or {}).get(
"repository", []
)
if isinstance(selector, dict)
]
},
}
def targets_hermes_builds(rule: dict[str, Any]) -> bool:
"""Detect a rule that claims this exact repository and tag selector."""
normalized = normalized_rule(rule)
return (
normalized["tag_selectors"] == EXPECTED_RULE["tag_selectors"]
and normalized["scope_selectors"] == EXPECTED_RULE["scope_selectors"]
)
class HarborClient:
"""Bounded same-origin client for Harbor's immutable-tag API."""
def __init__(self, origin: str, username: str, password: str) -> None:
normalized_origin = origin.rstrip("/")
if normalized_origin != EXPECTED_ORIGIN:
raise ValueError("Harbor API origin is not the pinned production API")
self.origin = normalized_origin
token = base64.b64encode(f"{username}:{password}".encode()).decode("ascii")
self.headers = {"Authorization": f"Basic {token}"}
self.opener = urllib.request.build_opener(NoRedirect())
def request(
self, method: str, path: str, payload: dict[str, Any] | None = None
) -> tuple[int, bytes, dict[str, str]]:
"""Issue one request, returning even non-2xx responses for strict checks."""
data = None
headers = dict(self.headers)
if payload is not None:
data = json.dumps(payload, separators=(",", ":")).encode()
headers["Content-Type"] = "application/json"
request = urllib.request.Request(
f"{self.origin}{path}", data=data, headers=headers, method=method
)
try:
response = self.opener.open(request, timeout=20)
except urllib.error.HTTPError as exc:
response = exc
except (urllib.error.URLError, TimeoutError) as exc:
raise HarborUnavailable("Harbor policy API is unavailable") from exc
with response:
body = response.read(MAX_RESPONSE + 1)
if len(body) > MAX_RESPONSE:
raise RuntimeError("Harbor response exceeded the size limit")
return int(response.status), body, dict(response.headers)
def list_rules(client: HarborClient) -> list[dict[str, Any]]:
"""Read and validate the complete small rule set for the project."""
path = f"/projects/{PROJECT}/immutabletagrules?page=1&page_size=100"
status, body, headers = client.request("GET", path)
if status in TRANSIENT_STATUSES:
raise HarborUnavailable(f"Harbor immutable rule list returned HTTP {status}")
if status != 200:
raise RuntimeError(f"Harbor immutable rule list returned HTTP {status}")
values = _json_list(body, "immutable rule")
_require_complete_page(values, headers, "immutable rule")
return values
def _json_list(body: bytes, label: str) -> list[dict[str, Any]]:
"""Decode one bounded Harbor list without accepting a partial shape."""
try:
value = json.loads(body.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise RuntimeError(f"Harbor returned invalid {label} JSON") from exc
if not isinstance(value, list) or not all(isinstance(item, dict) for item in value):
raise RuntimeError(f"Harbor {label} list has an invalid shape")
return value
def _require_complete_page(
values: list[dict[str, Any]], headers: dict[str, str], label: str
) -> None:
"""Reject a truncated first page or a proxy that strips count evidence."""
raw_total = next(
(value for key, value in headers.items() if key.lower() == "x-total-count"),
None,
)
if raw_total is None or not str(raw_total).isdecimal():
raise RuntimeError(f"Harbor {label} list omitted a valid total count")
if int(raw_total) != len(values):
raise RuntimeError(f"Harbor {label} list was truncated")
def _allowed(access: dict[str, Any], resource: str, action: str) -> bool:
"""Match one positive robot permission, accepting Harbor's default effect."""
return (
access.get("resource") == resource
and access.get("action") == action
and access.get("effect") in (None, "", "allow")
)
def _publisher_scope(robot: dict[str, Any]) -> tuple[dict[str, Any], list[dict[str, Any]]]:
"""Return the existing bstein scope after validating its push boundary."""
permissions = robot.get("permissions")
if not isinstance(permissions, list) or not all(
isinstance(item, dict) for item in permissions
):
raise RuntimeError("Jenkins Harbor robot permissions have an invalid shape")
matches = [
item
for item in permissions
if item.get("kind") == "project" and item.get("namespace") == PROJECT
]
if len(matches) != 1:
raise RuntimeError("Jenkins Harbor robot must have one existing bstein scope")
access = matches[0].get("access")
if not isinstance(access, list) or not all(isinstance(item, dict) for item in access):
raise RuntimeError("Jenkins Harbor robot bstein access has an invalid shape")
for required in (("repository", "pull"), ("repository", "push")):
if not any(_allowed(item, *required) for item in access):
raise RuntimeError("Jenkins Harbor robot lacks its existing pull/push boundary")
immutable_access = [
item for item in access if item.get("resource") == "immutable-tag"
]
if immutable_access and not (
len(immutable_access) == 1
and _allowed(immutable_access[0], "immutable-tag", "list")
):
raise RuntimeError("Jenkins Harbor robot has broader immutable-tag access")
return matches[0], access
def _verified_publisher(robot: dict[str, Any]) -> bool:
"""Check the exact identity and least-privilege policy after persistence."""
if (
robot.get("name") != PUBLISH_ROBOT
or robot.get("level") != "system"
or robot.get("editable") is not True
or robot.get("disable") is not False
):
return False
try:
_scope, access = _publisher_scope(robot)
except RuntimeError:
return False
return any(_allowed(item, "immutable-tag", "list") for item in access)
def ensure_publisher_can_read_rule(client: HarborClient) -> int:
"""Grant only immutable-tag:list to the existing Jenkins push robot."""
status, body, headers = client.request("GET", "/robots?page=1&page_size=100")
if status in TRANSIENT_STATUSES:
raise HarborUnavailable(f"Harbor robot list returned HTTP {status}")
if status != 200:
raise RuntimeError(f"Harbor robot list returned HTTP {status}")
robots = _json_list(body, "robot")
_require_complete_page(robots, headers, "robot")
matches = [
item
for item in robots
if item.get("name") == PUBLISH_ROBOT
]
if len(matches) != 1:
raise RuntimeError("expected exactly one Jenkins Harbor publisher robot")
robot_id = matches[0].get("id")
if not isinstance(robot_id, int) or robot_id < 1:
raise RuntimeError("Jenkins Harbor robot omitted a valid ID")
status, body, _headers = client.request("GET", f"/robots/{robot_id}")
if status in TRANSIENT_STATUSES:
raise HarborUnavailable(f"Harbor robot read returned HTTP {status}")
if status != 200:
raise RuntimeError(f"Harbor robot read returned HTTP {status}")
try:
robot = json.loads(body.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise RuntimeError("Harbor returned invalid robot JSON") from exc
if not isinstance(robot, dict):
raise RuntimeError("Harbor robot response has an invalid shape")
if (
robot.get("name") != PUBLISH_ROBOT
or robot.get("level") != "system"
or robot.get("editable") is not True
or robot.get("disable") is not False
):
raise RuntimeError("Jenkins Harbor publisher identity is not active and exact")
scope, access = _publisher_scope(robot)
if any(_allowed(item, "immutable-tag", "list") for item in access):
return robot_id
# Harbor updates permissions through PUT /robots/{id}; its separate PATCH
# endpoint is the only secret-rotation operation. Preserve every current
# scope and field while adding the one read-only action.
access.append({"resource": "immutable-tag", "action": "list"})
payload = {
"name": robot["name"],
"description": robot.get("description") or "",
"level": robot["level"],
"disable": robot["disable"],
"permissions": robot["permissions"],
}
duration = robot.get("duration")
if duration is not None:
if not isinstance(duration, int):
raise RuntimeError("Jenkins Harbor robot duration has an invalid shape")
payload["duration"] = duration
# Keep the validated object reference live in the preserved permission list.
if scope.get("access") is not access:
raise RuntimeError("Jenkins Harbor robot permission normalization drifted")
status, _body, _headers = client.request(
"PUT", f"/robots/{robot_id}", payload
)
if status in TRANSIENT_STATUSES:
raise HarborUnavailable(f"Harbor robot update returned HTTP {status}")
if status != 200:
raise RuntimeError(f"Harbor robot policy update returned HTTP {status}")
for attempt in range(1, 6):
status, body, _headers = client.request("GET", f"/robots/{robot_id}")
if status == 200:
try:
persisted = json.loads(body.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError):
persisted = None
if isinstance(persisted, dict) and _verified_publisher(persisted):
return robot_id
if attempt < 5:
time.sleep(attempt)
raise RuntimeError("Jenkins Harbor robot policy did not verify exactly")
def ensure_rule(client: HarborClient) -> int:
"""Create once, or validate the one exact enabled rule already present."""
rules = list_rules(client)
matches = [rule for rule in rules if targets_hermes_builds(rule)]
if len(matches) > 1:
raise RuntimeError("multiple Hermes agent immutable rules exist")
if matches:
if normalized_rule(matches[0]) != EXPECTED_RULE:
raise RuntimeError("Hermes agent immutable rule exists but is not enabled/exact")
rule_id = matches[0].get("id")
if not isinstance(rule_id, int) or rule_id < 1:
raise RuntimeError("Harbor immutable rule omitted a valid ID")
return rule_id
path = f"/projects/{PROJECT}/immutabletagrules"
status, _body, headers = client.request("POST", path, EXPECTED_RULE)
if status in TRANSIENT_STATUSES:
raise HarborUnavailable(f"Harbor immutable rule create returned HTTP {status}")
if status != 201:
raise RuntimeError(f"Harbor immutable rule create returned HTTP {status}")
location = headers.get("Location") or headers.get("location") or ""
api_path = urllib.parse.urlsplit(client.origin).path.rstrip("/")
expected_prefix = f"{api_path}{path}/"
if not location.startswith(expected_prefix):
raise RuntimeError("Harbor immutable rule create omitted the exact Location")
suffix = location[len(expected_prefix) :]
if not suffix.isdecimal() or int(suffix) < 1:
raise RuntimeError("Harbor immutable rule Location has an invalid ID")
# Harbor persists synchronously, but bounded retries distinguish a stale
# read from accepting an unverified policy.
for attempt in range(1, 6):
matches = [rule for rule in list_rules(client) if targets_hermes_builds(rule)]
if len(matches) == 1 and normalized_rule(matches[0]) == EXPECTED_RULE:
rule_id = matches[0].get("id")
if rule_id == int(suffix):
return rule_id
if attempt < 5:
time.sleep(attempt)
raise RuntimeError("created Harbor immutable rule did not verify exactly")
def main() -> int:
"""Load the runtime-only admin credential and enforce the tracked policy."""
origin = os.environ.get("HARBOR_API_ORIGIN", "")
password_file = Path(os.environ.get("HARBOR_ADMIN_PASSWORD_FILE", ""))
password = password_file.read_text(encoding="utf-8").strip()
if not password:
raise RuntimeError("Harbor admin password is empty")
client = HarborClient(origin, "admin", password)
for attempt in range(1, 13):
try:
rule_id = ensure_rule(client)
robot_id = ensure_publisher_can_read_rule(client)
break
except HarborUnavailable:
if attempt == 12:
raise
time.sleep(min(attempt * 2, 15))
print(
"Hermes agent immutable build-tag rule is active "
f"(id={rule_id}); publisher preflight access is active (robot={robot_id})"
)
return 0
if __name__ == "__main__":
raise SystemExit(main())

92
services/hermes/NOTES.md
View File

@ -102,6 +102,98 @@ console tails and named artifact contents in its deterministic bundle. A report
must say `retained Ariadne evidence` when that fallback is used; it must not
pretend direct Jenkins access succeeded.
## Publishing an agent image after review
The `hermes-agent-image` Jenkins job is the only supported agent image builder.
It runs daemonless Kaniko without a service-account token, host socket,
privileged container, or writable Git credential. It accepts only an exact
40-character revision that is both the checked-out commit and current
`atlas/titan-iac` `main`, so a human must merge the source PR first.
Kaniko runs as UID 0 because it must unpack an image root filesystem and enter
that filesystem for Dockerfile `RUN` instructions. Its capability set is the
minimum proven by an exact ARM64 no-push build of this Dockerfile:
`CHOWN`, `FOWNER`, `DAC_OVERRIDE`, `SETGID`, and `SETUID`. `SETGID` is required
because pinned Kaniko applies the base image's supplementary group list before
each Dockerfile `RUN`; `SETUID` lets apt drop privileges to its `_apt` account
while downloading package indexes. It still has no privilege
escalation, service-account token, host path, daemon socket, or Docker/BuildKit
TCP endpoint and uses the runtime-default seccomp profile. This is residual
root-in-the-build-pod risk, bounded to disposable `emptyDir` storage and a
human-reviewed `main` revision; Dockerfile changes require the same scrutiny as
executable cluster code.
Pinned Kaniko parses Dockerfile `RUN` heredocs but does not materialize their
inline files when executing a command. The Jenkins lane therefore enables a
bounded compatibility replay after copying the reviewed Dockerfile and runner
from the same checked-out commit. The runner accepts only the exact nine
Node/Python heredocs in their expected order and launches each in a separate
interpreter process. Docker and BuildKit keep their native behavior because the
compatibility argument defaults off. The final TypeScript build, Python compile,
and source assertions remain mandatory, so an omitted or drifted replay fails
before any image can pass release verification.
From agent.hermes, trigger that one fixed job with:
```sh
jenkins_image_build_trigger.py '<reviewed full main commit>'
```
The helper has no general Jenkins credential or caller-selectable job name. Its
Vault-projected token is bound by Jenkins only to `hermes-agent-image`. The job
also requires its fixed publish confirmation. Each run claims the unique tag
`git-<reviewed-sha>-build-<jenkins-build-number>` and refuses to overwrite an
existing tag. It cross-checks Kaniko's digest and tagged-image evidence, then
independently reads the pushed tag from Harbor before archiving a JSON record
and Flux digest patch. Apply that patch on a new branch and submit it for human
review; the build never changes Git, reconciles Flux, or deploys by itself.
The helper posts only to the fixed HTTPS Build Token Root endpoint and accepts
only its real queue responses: HTTP 201 or a non-followed HTTP 303 with an exact
same-origin `/queue/item/<number>/` location. An unauthenticated request is
denied, and the token cannot select, configure, read, or administer another
Jenkins job. The pipeline independently rejects any commit that is not the
current `origin/main`, preserving the human merge/review boundary.
Harbor independently enforces an enabled immutable-tag rule scoped to only the
`bstein/hermes-agent` repository and `git-*-build-*` tags. A second manifest PUT,
retag, or deletion is rejected by Harbor even if a caller bypasses the Jenkins
preflight. A revisioned, Flux-tracked policy Job creates or verifies that exact
rule with a runtime-only Vault credential; it refuses to alter a conflicting
rule and retries only explicit transport/readiness failures while Harbor
starts. The same Job grants the existing Jenkins publisher only
`immutable-tag:list` on `bstein`, preserving its other project scopes and never
calling Harbor's separate secret-rotation endpoint. The pipeline uses that
read-only permission to require the exact enabled rule before Kaniko starts.
That check also makes an already-running Jenkins controller fail closed during
rollout; Jenkins cannot become Ready on the new revision before the policy Job
succeeds.
The tracked rollout order is deliberate: the revisioned Vault role Job must
complete before the Vault Kustomization becomes Ready. A separate Flux
Kustomization then runs the seed Job under the dedicated
`hermes-jenkins-token-seed` identity, which can only create/read the exact
`kv/data/atlas/hermes/developer-jenkins` path and request random bytes. Jenkins
depends on both that seed and the Harbor policy Job; Hermes depends on Jenkins.
The seeder uses KV-v2 CAS create-only semantics. It never changes an existing
token, never fills a missing field in an existing secret, and fails closed when
it cannot distinguish absence from a read error. The recurring Vault
configuration job reconciles the same narrow role and policy.
Rotate this fixed-job token only as a coordinated operator action:
1. Stop new `hermes-agent-image` triggers and wait for its queue and executor to
drain.
2. Read the current KV-v2 metadata version for
`kv/atlas/hermes/developer-jenkins`.
3. Generate a fresh value from Vault and patch only `build_token` with
`vault kv patch -cas=<current-version>`. Keep the value in a mode-0600
temporary file or standard input, never a command argument or log.
4. Roll Jenkins first and wait until it is Ready, then roll `hermes-agent` and
wait until it is Ready. Do not resume triggers between those two consumers.
5. Run one reviewed-main canary. If rollback is required, repeat the CAS patch
with the previous value as another coordinated rotation; never delete the
secret to make the seeder recreate it.
## The actual supervised triage algorithm
1. Classify the request as test/build triage, service health, or alert tuning.

5
services/hermes/agent-deployment.yaml
View File

@ -56,6 +56,11 @@ spec:
{{- with secret "kv/data/atlas/hermes/developer-gitea" -}}
{{ .Data.data.username }}
{{- end }}
vault.hashicorp.com/agent-inject-secret-jenkins-image-build-token: kv/data/atlas/hermes/developer-jenkins
vault.hashicorp.com/agent-inject-template-jenkins-image-build-token: |
{{- with secret "kv/data/atlas/hermes/developer-jenkins" -}}
{{ .Data.data.build_token }}
{{- end }}
vault.hashicorp.com/agent-inject-secret-node-ssh-private-key: kv/data/atlas/hermes/developer-ssh
vault.hashicorp.com/agent-inject-template-node-ssh-private-key: |
{{- with secret "kv/data/atlas/hermes/developer-ssh" -}}

1
services/hermes/kustomization.yaml
View File

@ -77,6 +77,7 @@ configMapGenerator:
- image_broker.py=scripts/image_broker.py
- install_agent_tools.sh=scripts/install_agent_tools.sh
- jenkins_build_evidence.py=scripts/jenkins_build_evidence.py
- jenkins_image_build_trigger.py=scripts/jenkins_image_build_trigger.py
- kanban_status_recovery.py=scripts/kanban_status_recovery.py
- migrate_herdr_state.py=scripts/migrate_herdr_state.py
- migrate_api_session_lineage.py=scripts/migrate_api_session_lineage.py

112
services/hermes/scripts/jenkins_image_build_trigger.py Executable file
View File

@ -0,0 +1,112 @@
#!/usr/bin/env python3
"""Trigger only the reviewed-main Hermes agent image release job."""
from __future__ import annotations
import argparse
import json
import re
import urllib.error
import urllib.parse
import urllib.request
from pathlib import Path
JENKINS_ORIGIN = "https://ci.bstein.dev"
JENKINS_BUILD_URL = f"{JENKINS_ORIGIN}/buildByToken/buildWithParameters"
JOB_NAME = "hermes-agent-image"
TOKEN_FILE = Path("/runtime-access/jenkins-image-build-token")
REVISION_PATTERN = re.compile(r"^[0-9a-f]{40}$")
QUEUE_PATH_PATTERN = re.compile(r"^/queue/item/[0-9]+/?$")
class _NoRedirect(urllib.request.HTTPRedirectHandler):
"""Keep a queued-build redirect from becoming an unauthorized job read."""
def redirect_request(self, _request, _file, _code, _message, _headers, _url):
return None
def _open_without_redirect(request: urllib.request.Request, timeout: int):
"""Return the Build Token Root response, including its expected HTTP 303."""
opener = urllib.request.build_opener(_NoRedirect())
try:
return opener.open(request, timeout=timeout)
except urllib.error.HTTPError as exc:
if exc.code == 303:
return exc
raise
def trigger_build(
revision: str,
*,
token_file: Path = TOKEN_FILE,
opener=_open_without_redirect,
) -> dict[str, str | int]:
"""Post the fixed job parameters using its job-scoped build token."""
revision = revision.strip()
if not REVISION_PATTERN.fullmatch(revision):
raise ValueError("revision must be a lowercase full 40-character commit")
token = token_file.read_text(encoding="utf-8").strip()
if not token:
raise RuntimeError("Jenkins image-build token is empty")
payload = urllib.parse.urlencode(
{
"job": JOB_NAME,
"token": token,
"PUBLISH_IMAGE": "true",
"EXPECTED_SOURCE_REVISION": revision,
"CONFIRM_PUBLISH": "PUBLISH HERMES AGENT",
}
).encode("utf-8")
request = urllib.request.Request(
JENKINS_BUILD_URL,
data=payload,
headers={"Content-Type": "application/x-www-form-urlencoded"},
method="POST",
)
with opener(request, timeout=20) as response:
status = int(response.status)
location = response.headers.get("Location", "")
if status not in {201, 303}:
raise RuntimeError(f"Jenkins trigger returned HTTP {status}")
if not location:
raise RuntimeError("Jenkins trigger omitted the queue Location")
queue_url = urllib.parse.urljoin(f"{JENKINS_ORIGIN}/", location)
parsed_queue = urllib.parse.urlsplit(queue_url)
expected_origin = urllib.parse.urlsplit(JENKINS_ORIGIN)
if (
parsed_queue.scheme != expected_origin.scheme
or parsed_queue.netloc != expected_origin.netloc
or parsed_queue.query
or parsed_queue.fragment
or not QUEUE_PATH_PATTERN.fullmatch(parsed_queue.path)
):
raise RuntimeError("Jenkins returned an invalid queue Location")
# Never return the submitted URL: its form body contains the job token.
queue_path = parsed_queue.path
return {
"job": JOB_NAME,
"queue_path": queue_path,
"source_revision": revision,
"status": status,
}
def main() -> int:
"""Validate one revision, trigger the bounded job, and print safe metadata."""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("revision", help="reviewed full commit currently on main")
args = parser.parse_args()
try:
result = trigger_build(args.revision)
except (OSError, ValueError, RuntimeError, urllib.error.URLError) as exc:
print(json.dumps({"error": str(exc)}, sort_keys=True))
return 1
print(json.dumps(result, indent=2, sort_keys=True))
return 0
if __name__ == "__main__": # pragma: no cover - exercised through main()
raise SystemExit(main())

1
services/hermes/scripts/stage_runtime_access.py
View File

@ -95,6 +95,7 @@ def stage_agent() -> None:
"chat-relay-key",
"gitea-token",
"gitea-username",
"jenkins-image-build-token",
"node-ssh-private-key",
"node-ssh-config",
"node-ssh-known-hosts",

2
services/hermes/switchyard-deployment.yaml
View File

@ -27,7 +27,7 @@ spec:
prometheus.io/port: "9005"
prometheus.io/path: /metrics
vault.hashicorp.com/agent-inject: "true"
vault.hashicorp.com/role: hermes-agent
vault.hashicorp.com/role: hermes-switchyard
vault.hashicorp.com/agent-inject-secret-relay-key: kv/data/atlas/hermes/chat-telegram
vault.hashicorp.com/agent-inject-template-relay-key: |
{{- with secret "kv/data/atlas/hermes/chat-telegram" -}}

19
services/jenkins/configmap-jcasc.yaml
View File

@ -652,6 +652,25 @@ data:
}
}
}
pipelineJob('hermes-agent-image') {
disabled(false)
description('Human-gated, daemonless Kaniko build for the reviewed atlas/titan-iac main revision. Publishes a content-addressed Hermes agent image and archives a Flux digest patch; it never mutates Git or deploys.')
authenticationToken(System.getenv('HERMES_AGENT_IMAGE_BUILD_TOKEN'))
definition {
cpsScm {
scm {
git {
remote {
url('https://scm.bstein.dev/atlas/titan-iac.git')
credentials('gitea-pat')
}
branches('*/main')
}
}
scriptPath('ci/Jenkinsfile.hermes-agent-image')
}
}
}
multibranchPipelineJob('titan-iac-quality-gate') {
branchSources {
branchSource {

1
services/jenkins/configmap-plugins.yaml
View File

@ -20,6 +20,7 @@ data:
gitea:268.v75e47974c01d
gitea-checks:603.621.vc708da_fb_371d
multibranch-scan-webhook-trigger:1.0.11
build-token-root:365.v717f8685a_09e
# Structured test evidence. Without junit the `junit` step throws
# NoSuchMethodError, jenkins.failed_tests is always empty, and triage
# has only raw console text to work from. Pinned to the newest release

3
services/jenkins/deployment.yaml
View File

@ -65,6 +65,9 @@ spec:
ARIADNE_JENKINS_API_USER={{ .Data.data.username }}
ARIADNE_JENKINS_API_TOKEN={{ .Data.data.token }}
{{ end }}
{{ with secret "kv/data/atlas/hermes/developer-jenkins" }}
HERMES_AGENT_IMAGE_BUILD_TOKEN={{ .Data.data.build_token }}
{{ end }}
bstein.dev/restarted-at: "2026-05-20T09:40:31Z"
spec:
serviceAccountName: jenkins

2
services/jenkins/secretproviderclass.yaml
View File

@ -8,7 +8,7 @@ spec:
provider: vault
parameters:
vaultAddress: "http://vault.vault.svc.cluster.local:8200"
roleName: "jenkins"
roleName: "jenkins-vault-sync"
objects: |
- objectName: "harbor-pull__dockerconfigjson"
secretPath: "kv/data/atlas/shared/harbor-pull"

68
services/vault-hermes-jenkins-token-seed/job.yaml Normal file
View File

@ -0,0 +1,68 @@
# services/vault-hermes-jenkins-token-seed/job.yaml
apiVersion: batch/v1
kind: Job
metadata:
name: vault-hermes-jenkins-build-token-seed-2
namespace: vault
spec:
backoffLimit: 2
template:
spec:
serviceAccountName: hermes-jenkins-token-seed
enableServiceLinks: false
restartPolicy: Never
nodeSelector:
hardware: rpi5
kubernetes.io/arch: arm64
node-role.kubernetes.io/worker: "true"
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: NotIn
values: [titan-04, titan-14, titan-18, titan-19, titan-24]
securityContext:
fsGroup: 1000
fsGroupChangePolicy: OnRootMismatch
seccompProfile:
type: RuntimeDefault
containers:
- name: seed
image: docker.io/hashicorp/vault@sha256:4e33b126a59c0c333b76fb4e894722462659a6bec7c48c9ee8cea56fccfd2569
imagePullPolicy: IfNotPresent
command: [sh, /scripts/vault_hermes_jenkins_build_token_ensure.sh]
env:
- name: HOME
value: /tmp
- name: VAULT_ADDR
value: http://vault.vault.svc.cluster.local:8200
- name: VAULT_K8S_ROLE
value: hermes-jenkins-token-seed
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
readOnlyRootFilesystem: true
runAsGroup: 1000
runAsNonRoot: true
runAsUser: 100
seccompProfile:
type: RuntimeDefault
volumeMounts:
- name: scripts
mountPath: /scripts
readOnly: true
- name: tmp
mountPath: /tmp
resources:
requests: {cpu: 25m, memory: 32Mi}
limits: {cpu: 250m, memory: 128Mi}
volumes:
- name: scripts
configMap:
name: vault-hermes-jenkins-token-seed-script
defaultMode: 0555
- name: tmp
emptyDir: {}

13
services/vault-hermes-jenkins-token-seed/kustomization.yaml Normal file
View File

@ -0,0 +1,13 @@
# services/vault-hermes-jenkins-token-seed/kustomization.yaml
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: vault
resources:
- serviceaccount.yaml
- job.yaml
generatorOptions:
disableNameSuffixHash: true
configMapGenerator:
- name: vault-hermes-jenkins-token-seed-script
files:
- vault_hermes_jenkins_build_token_ensure.sh=scripts/vault_hermes_jenkins_build_token_ensure.sh

145
services/vault-hermes-jenkins-token-seed/scripts/vault_hermes_jenkins_build_token_ensure.sh Normal file
View File

@ -0,0 +1,145 @@
#!/usr/bin/env sh
set -eu
secret_api_path="kv/data/atlas/hermes/developer-jenkins"
jwt_file="${VAULT_K8S_JWT_FILE:-/var/run/secrets/kubernetes.io/serviceaccount/token}"
vault_role="${VAULT_K8S_ROLE:-hermes-jenkins-token-seed}"
payload_file="${TMPDIR:-/tmp}/hermes-jenkins-token.json"
log() { printf '[hermes-jenkins-token] %s\n' "$*" >&2; }
cleanup() { rm -f "${payload_file}"; }
trap cleanup EXIT HUP INT TERM
retry_command() {
attempt=1
while [ "${attempt}" -le 5 ]; do
set +e
command_output="$("$@" 2>&1)"
command_status=$?
set -e
if [ "${command_status}" -eq 0 ]; then
printf '%s' "${command_output}"
return 0
fi
if [ "${attempt}" -lt 5 ]; then
sleep "${attempt}"
fi
attempt=$((attempt + 1))
done
return "${command_status}"
}
ensure_token() {
if [ -n "${VAULT_TOKEN:-}" ]; then
return
fi
jwt="$(cat "${jwt_file}")"
VAULT_TOKEN="$(retry_command vault write -field=token \
auth/kubernetes/login role="${vault_role}" jwt="${jwt}")" || {
log "Vault Kubernetes login failed after retries"
exit 1
}
unset jwt
if [ -z "${VAULT_TOKEN}" ]; then
log "Vault Kubernetes login returned an empty token"
exit 1
fi
export VAULT_TOKEN
}
# Return 0 for the complete field, 10 when absent, 11 when present but
# incomplete, and 12 for a persistent authorization/transport failure.
read_build_token() {
attempt=1
while [ "${attempt}" -le 5 ]; do
set +e
secret_json="$(vault read -format=json "${secret_api_path}" 2>&1)"
secret_status=$?
set -e
if [ "${secret_status}" -eq 0 ]; then
if printf '%s' "${secret_json}" | grep -Eq '"build_token"[[:space:]]*:[[:space:]]*"[0-9a-f]{64}"'; then
unset secret_json
return 0
fi
unset secret_json
return 11
fi
if printf '%s' "${secret_json}" | grep -Eq 'Code: 404|No value found at'; then
unset secret_json
return 10
fi
if [ "${attempt}" -lt 5 ]; then
sleep "${attempt}"
fi
attempt=$((attempt + 1))
done
unset secret_json
return 12
}
ensure_token
if read_build_token; then
read_status=0
else
read_status=$?
fi
case "${read_status}" in
0)
log "job-scoped token already present; no write performed"
exit 0
;;
11)
log "secret exists without a valid build_token; refusing to overwrite existing fields"
exit 1
;;
12)
log "secret read failed after retries; refusing to seed"
exit 1
;;
10) ;;
*)
log "unexpected secret read state ${read_status}"
exit 1
;;
esac
new_token="$(retry_command vault write -field=random_bytes \
sys/tools/random/32 format=hex)" || {
log "Vault random token generation failed after retries"
exit 1
}
if ! printf '%s' "${new_token}" | grep -Eq '^[0-9a-f]{64}$'; then
unset new_token
log "Vault returned an invalid random token"
exit 1
fi
umask 077
printf '{"options":{"cas":0},"data":{"build_token":"%s"}}\n' \
"${new_token}" > "${payload_file}"
unset new_token
set +e
create_error="$(vault write "${secret_api_path}" @"${payload_file}" 2>&1 >/dev/null)"
create_status=$?
set -e
cleanup
if [ "${create_status}" -eq 0 ]; then
log "job-scoped token created with KV-v2 CAS create-only semantics"
exit 0
fi
if printf '%s' "${create_error}" | grep -qi 'check-and-set'; then
unset create_error
if read_build_token; then
log "another seeder won CAS; existing token preserved"
exit 0
fi
log "CAS race did not produce a readable build_token; refusing to continue"
exit 1
fi
unset create_error
log "KV-v2 CAS create failed; refusing to retry a write"
exit 1

7
services/vault-hermes-jenkins-token-seed/serviceaccount.yaml Normal file
View File

@ -0,0 +1,7 @@
# services/vault-hermes-jenkins-token-seed/serviceaccount.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: hermes-jenkins-token-seed
namespace: vault
automountServiceAccountToken: true

30
services/vault/hermes-auth-role-bootstrap-job.yaml
View File

@ -3,7 +3,7 @@
apiVersion: batch/v1
kind: Job
metadata:
name: vault-k8s-auth-hermes-8
name: vault-k8s-auth-hermes-9
namespace: vault
spec:
backoffLimit: 2
@ -15,16 +15,26 @@ spec:
hardware: rpi5
kubernetes.io/arch: arm64
node-role.kubernetes.io/worker: "true"
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: NotIn
values: [titan-04, titan-14, titan-18, titan-19, titan-24]
containers:
- name: configure-k8s-auth
image: hashicorp/vault:1.21.4
image: docker.io/hashicorp/vault@sha256:4e33b126a59c0c333b76fb4e894722462659a6bec7c48c9ee8cea56fccfd2569
imagePullPolicy: IfNotPresent
command:
- sh
- /scripts/vault_k8s_auth_configure.sh
env:
- name: HOME
value: /tmp
- name: VAULT_ADDR
value: http://10.43.57.249:8200
value: http://vault.vault.svc.cluster.local:8200
- name: VAULT_K8S_ROLE
value: vault-admin
- name: VAULT_K8S_TOKEN_REVIEWER_JWT_FILE
@ -38,6 +48,18 @@ spec:
- name: token-reviewer
mountPath: /var/run/secrets/vault-token-reviewer
readOnly: true
- name: tmp
mountPath: /tmp
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
readOnlyRootFilesystem: true
runAsGroup: 1000
runAsNonRoot: true
runAsUser: 100
seccompProfile:
type: RuntimeDefault
resources:
requests:
cpu: 25m
@ -53,3 +75,5 @@ spec:
- name: token-reviewer
secret:
secretName: vault-admin-token-reviewer
- name: tmp
emptyDir: {}

37
services/vault/scripts/vault_k8s_auth_configure.sh
View File

@ -235,12 +235,26 @@ write_policy_and_role "mailu-mailserver" "mailu-mailserver" "mailu-vault-sync" \
"mailu/* shared/postmark-relay shared/harbor-pull" ""
write_policy_and_role "harbor" "harbor" "harbor-vault-sync" \
"harbor/* shared/harbor-pull" "hermes/developer-harbor"
harbor_policy_bootstrap_policy='
path "kv/data/atlas/harbor/harbor-core" {
capabilities = ["read"]
}
'
write_raw_policy "harbor-policy-bootstrap" "${harbor_policy_bootstrap_policy}"
log "writing role harbor-policy-bootstrap"
vault_cmd write "auth/kubernetes/role/harbor-policy-bootstrap" \
bound_service_account_names="harbor-policy-bootstrap" \
bound_service_account_namespaces="harbor" \
policies="harbor-policy-bootstrap" \
ttl="${role_ttl}"
write_policy_and_role "nextcloud" "nextcloud" "nextcloud-vault" \
"nextcloud/* shared/keycloak-admin shared/postmark-relay" ""
write_policy_and_role "comms" "comms" "comms-vault,atlasbot" \
"comms/* shared/chat-ai-keys-runtime shared/harbor-pull" ""
write_policy_and_role "jenkins" "jenkins" "jenkins,jenkins-vault-sync" \
"jenkins/* shared/harbor-pull quality/sonarqube-oidc" "hermes/developer-jenkins"
write_policy_and_role "jenkins" "jenkins" "jenkins" \
"jenkins/* shared/harbor-pull quality/sonarqube-oidc hermes/developer-jenkins" ""
write_policy_and_role "jenkins-vault-sync" "jenkins" "jenkins-vault-sync" \
"shared/harbor-pull" ""
write_policy_and_role "monitoring" "monitoring" "monitoring-vault-sync" \
"monitoring/* shared/postmark-relay shared/harbor-pull" ""
write_policy_and_role "logging" "logging" "logging-vault-sync" \
@ -255,8 +269,25 @@ write_policy_and_role "game-stream" "game-stream" "game-stream-vault" \
"game-stream/*" ""
write_policy_and_role "hermes" "hermes" "hermes-vault,hermes-triage" \
"hermes/triage-oidc hermes/agent-tokens hermes/triage-api" ""
write_policy_and_role "hermes-agent" "hermes" "hermes-agent,hermes-switchyard" \
write_policy_and_role "hermes-agent" "hermes" "hermes-agent" \
"hermes/agent-oidc hermes/agent-tokens hermes/chat-telegram hermes/developer-keycloak hermes/developer-gitea hermes/developer-harbor hermes/developer-jenkins hermes/developer-ssh" ""
write_policy_and_role "hermes-switchyard" "hermes" "hermes-switchyard" \
"hermes/chat-telegram" ""
hermes_jenkins_token_seed_policy='
path "kv/data/atlas/hermes/developer-jenkins" {
capabilities = ["create", "read"]
}
path "sys/tools/random/32" {
capabilities = ["update"]
}
'
write_raw_policy "hermes-jenkins-token-seed" "${hermes_jenkins_token_seed_policy}"
log "writing role hermes-jenkins-token-seed"
vault_cmd write "auth/kubernetes/role/hermes-jenkins-token-seed" \
bound_service_account_names="hermes-jenkins-token-seed" \
bound_service_account_namespaces="vault" \
policies="hermes-jenkins-token-seed" \
ttl="${role_ttl}"
write_policy_and_role "hermes-credential-sync" "hermes" "hermes-agent" \
"" "hermes/agent-tokens"
write_policy_and_role "hermes-node-ssh" "hermes" "hermes-node-ssh-access" \

3
testing/quality_contract.json
View File

@ -14,6 +14,7 @@
}
],
"managed_modules": [
"ci/scripts/hermes_image_release.py",
"ci/scripts/publish_test_metrics.py",
"ci/scripts/publish_test_metrics_quality.py",
"ci/scripts/semgrep_report.py",
@ -35,6 +36,7 @@
"testing/tests/test_quality_gate.py"
],
"lint_paths": [
"ci/scripts/hermes_image_release.py",
"ci/scripts/publish_test_metrics.py",
"ci/scripts/publish_test_metrics_quality.py",
"ci/scripts/semgrep_report.py",
@ -168,6 +170,7 @@
"coverage": {
"minimum_percent": 95.0,
"tracked_files": [
"ci/scripts/hermes_image_release.py",
"ci/scripts/publish_test_metrics.py",
"ci/scripts/publish_test_metrics_quality.py",
"ci/scripts/semgrep_report.py",

488
testing/tests/test_hermes_image_builder.py Normal file
View File

@ -0,0 +1,488 @@
"""Safety and artifact contracts for the Hermes agent image release lane."""
from __future__ import annotations
import importlib.util
import io
import json
import urllib.parse
from pathlib import Path
import pytest
import yaml
REPO_ROOT = Path(__file__).resolve().parents[2]
PIPELINE_PATH = REPO_ROOT / "ci/Jenkinsfile.hermes-agent-image"
RELEASE_SCRIPT = REPO_ROOT / "ci/scripts/hermes_image_release.py"
TRIGGER_SCRIPT = REPO_ROOT / "services/hermes/scripts/jenkins_image_build_trigger.py"
HEREDOC_RUNNER = REPO_ROOT / "dockerfiles/hermes-kaniko-heredoc-runner.py"
def _load_release_module():
spec = importlib.util.spec_from_file_location(
"hermes_image_release", RELEASE_SCRIPT
)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def _load_trigger_module():
spec = importlib.util.spec_from_file_location(
"jenkins_image_build_trigger", TRIGGER_SCRIPT
)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def _load_heredoc_runner():
spec = importlib.util.spec_from_file_location(
"hermes_kaniko_heredoc_runner", HEREDOC_RUNNER
)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def _pod_spec() -> dict:
source = PIPELINE_PATH.read_text(encoding="utf-8")
pod_yaml = source.split('yaml """', 1)[1].split('"""', 1)[0]
return yaml.safe_load(pod_yaml)["spec"]
def test_builder_pod_is_daemonless_and_kernel_bounded() -> None:
"""The builder gets only proven build caps, never host, daemon, or K8s access."""
source = PIPELINE_PATH.read_text(encoding="utf-8")
spec = _pod_spec()
assert spec["automountServiceAccountToken"] is False
assert spec["enableServiceLinks"] is False
assert "hostPath" not in source
assert "docker.sock" not in source
assert "tcp://" not in source
assert "buildkitd" not in source.lower()
assert "dind" not in source.lower()
containers = {item["name"]: item for item in spec["containers"]}
kaniko = containers["kaniko"]
assert kaniko["image"] == (
"gcr.io/kaniko-project/executor@sha256:"
"c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e"
)
assert kaniko["securityContext"]["capabilities"]["add"] == [
"CHOWN",
"FOWNER",
"DAC_OVERRIDE",
"SETGID",
"SETUID",
]
for container in containers.values():
security = container["securityContext"]
assert security["allowPrivilegeEscalation"] is False
assert security["capabilities"]["drop"] == ["ALL"]
assert security["seccompProfile"]["type"] == "RuntimeDefault"
assert security.get("privileged", False) is False
assert "add" not in containers["jnlp"]["securityContext"]["capabilities"]
assert "add" not in containers["python"]["securityContext"]["capabilities"]
def test_builder_is_restricted_to_healthy_rpi5_capacity() -> None:
"""Disposable builds must stay off unhealthy and reserved Atlas nodes."""
spec = _pod_spec()
assert spec["nodeSelector"]["hardware"] == "rpi5"
expressions = spec["affinity"]["nodeAffinity"][
"requiredDuringSchedulingIgnoredDuringExecution"
]["nodeSelectorTerms"][0]["matchExpressions"]
host_rule = next(
rule for rule in expressions if rule["key"] == "kubernetes.io/hostname"
)
assert host_rule["operator"] == "NotIn"
assert set(host_rule["values"]) >= {
"titan-04",
"titan-14",
"titan-18",
"titan-19",
"titan-22",
"titan-24",
}
def test_pipeline_requires_reviewed_main_and_runtime_credentials() -> None:
"""Publishing requires explicit confirmation and a reviewed main commit."""
source = PIPELINE_PATH.read_text(encoding="utf-8")
assert 'test "${PUBLISH_IMAGE}" = "true"' in source
assert 'test "${CONFIRM_PUBLISH}" = "PUBLISH HERMES AGENT"' in source
assert "git rev-parse origin/main" in source
assert "credentialsId: 'harbor-robot'" in source
assert "set +x" in source
assert "umask 077" in source
assert "unset HARBOR_USER HARBOR_PASSWORD auth" in source
assert "/busybox/rm -f /kaniko/.docker/config.json" in source
assert "--digest-file=" in source
assert "--image-name-tag-with-digest-file=" in source
assert "--destination=" in source
assert "assert-absent" in source
assert "git-${actual_revision}-build-${BUILD_NUMBER}" in source
assert source.count("--build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1") == 1
def test_kaniko_replays_only_the_exact_reviewed_heredoc_contract() -> None:
"""Pinned Kaniko's ignored inline files are replayed in exact source order."""
module = _load_heredoc_runner()
dockerfile = (REPO_ROOT / "dockerfiles/Dockerfile.hermes-agent").read_text()
blocks = module.extract_blocks(dockerfile)
assert tuple(command[0] for command, _body in blocks) == (
"node",
"python",
"python",
"python",
"python",
"python",
"python",
"python",
"node",
)
assert "session message total" in blocks[-1][1]
compat = dockerfile.split("ARG HERMES_KANIKO_HEREDOC_COMPAT=0", 1)[1]
assert "case \"${HERMES_KANIKO_HEREDOC_COMPAT}\" in 0|1)" in compat
assert "python /tmp/hermes-kaniko-heredoc-runner.py" in compat
ignored = (
REPO_ROOT / "dockerfiles/Dockerfile.hermes-agent.dockerignore"
).read_text()
assert "!dockerfiles/Dockerfile.hermes-agent" in ignored
assert "!dockerfiles/hermes-kaniko-heredoc-runner.py" in ignored
def test_kaniko_heredoc_runner_rejects_drift_and_executes_separately(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
"""Missing blocks fail closed and each reviewed body gets a fresh process."""
module = _load_heredoc_runner()
source = (REPO_ROOT / "dockerfiles/Dockerfile.hermes-agent").read_text()
with pytest.raises(ValueError, match="contract changed"):
module.extract_blocks(source.replace("RUN node <<'NODE'", "RUN node", 1))
calls = []
def run(command, **kwargs):
calls.append((command, kwargs))
monkeypatch.setattr(module.subprocess, "run", run)
dockerfile = tmp_path / "Dockerfile"
dockerfile.write_text(source, encoding="utf-8")
module.replay(dockerfile)
assert len(calls) == 9
assert all(call[1]["check"] is True for call in calls)
assert all(call[1]["text"] is True for call in calls)
assert all(call[1]["input"].endswith("\n") for call in calls)
def test_jenkins_job_is_manual_and_reads_pipeline_from_main() -> None:
"""JCasC must not publish unreviewed branch contents or poll automatically."""
config = yaml.safe_load(
(REPO_ROOT / "services/jenkins/configmap-jcasc.yaml").read_text(
encoding="utf-8"
)
)
jobs = config["data"]["jobs.yaml"]
block = jobs.split("pipelineJob('hermes-agent-image')", 1)[1].split(
"pipelineJob(", 1
)[0]
assert "branches('*/main')" in block
assert "scriptPath('ci/Jenkinsfile.hermes-agent-image')" in block
assert (
"authenticationToken(System.getenv('HERMES_AGENT_IMAGE_BUILD_TOKEN'))" in block
)
assert "pipelineTriggers" not in block
assert "scmTrigger" not in block
def test_agent_trigger_is_limited_to_the_image_job(tmp_path: Path) -> None:
"""Agent Hermes gets one job token, fixed parameters, and no Jenkins admin API."""
module = _load_trigger_module()
revision = "a" * 40
token_path = tmp_path / "token"
token_path.write_text("private-job-token\n", encoding="utf-8")
captured = {}
class Response(io.BytesIO):
status = 201
headers = {"Location": "https://ci.bstein.dev/queue/item/42/"}
def __enter__(self):
return self
def __exit__(self, *_args):
self.close()
def opener(request, timeout):
captured["request"] = request
captured["timeout"] = timeout
return Response(b"")
result = module.trigger_build(revision, token_file=token_path, opener=opener)
request = captured["request"]
fields = urllib.parse.parse_qs(request.data.decode("utf-8"))
assert request.full_url == module.JENKINS_BUILD_URL
assert fields == {
"CONFIRM_PUBLISH": ["PUBLISH HERMES AGENT"],
"EXPECTED_SOURCE_REVISION": [revision],
"PUBLISH_IMAGE": ["true"],
"job": ["hermes-agent-image"],
"token": ["private-job-token"],
}
assert captured["timeout"] == 20
assert "private-job-token" not in json.dumps(result)
assert result["source_revision"] == revision
def test_agent_trigger_rejects_unsafe_revision_and_empty_token(tmp_path: Path) -> None:
"""No user-controlled job, URL, or abbreviated revision reaches Jenkins."""
module = _load_trigger_module()
token_path = tmp_path / "token"
token_path.write_text("token\n", encoding="utf-8")
with pytest.raises(ValueError):
module.trigger_build("main", token_file=token_path)
token_path.write_text("\n", encoding="utf-8")
with pytest.raises(RuntimeError, match="empty"):
module.trigger_build("a" * 40, token_file=token_path)
def test_agent_trigger_accepts_existing_queue_redirect(tmp_path: Path) -> None:
"""HTTP 303 means the exact release is already queued, not a trigger failure."""
module = _load_trigger_module()
token_path = tmp_path / "token"
token_path.write_text("token\n", encoding="utf-8")
class Response(io.BytesIO):
status = 303
headers = {"Location": "https://ci.bstein.dev/queue/item/7/"}
def __enter__(self):
return self
def __exit__(self, *_args):
self.close()
result = module.trigger_build(
"e" * 40, token_file=token_path, opener=lambda *_args, **_kwargs: Response()
)
assert result["status"] == 303
assert result["queue_path"] == "/queue/item/7/"
def test_job_token_is_generated_and_injected_only_at_runtime() -> None:
"""The fixed-job credential stays in Vault and pod-lifetime memory."""
plugins = (REPO_ROOT / "services/jenkins/configmap-plugins.yaml").read_text(
encoding="utf-8"
)
vault = (
REPO_ROOT / "services/vault/scripts/vault_k8s_auth_configure.sh"
).read_text(encoding="utf-8")
seeder = (
REPO_ROOT
/ "services/vault-hermes-jenkins-token-seed/scripts/vault_hermes_jenkins_build_token_ensure.sh"
).read_text(encoding="utf-8")
jenkins = (REPO_ROOT / "services/jenkins/deployment.yaml").read_text(
encoding="utf-8"
)
agent = (REPO_ROOT / "services/hermes/agent-deployment.yaml").read_text(
encoding="utf-8"
)
stage = (REPO_ROOT / "services/hermes/scripts/stage_runtime_access.py").read_text(
encoding="utf-8"
)
assert "build-token-root:365.v717f8685a_09e" in plugins
assert 'write_raw_policy "hermes-jenkins-token-seed"' in vault
assert "sys/tools/random/32 format=hex" in seeder
assert '"options":{"cas":0}' in seeder
assert "kv/data/atlas/hermes/developer-jenkins" in seeder
assert "HERMES_AGENT_IMAGE_BUILD_TOKEN={{ .Data.data.build_token }}" in jenkins
assert "agent-inject-secret-jenkins-image-build-token" in agent
assert '"jenkins-image-build-token"' in stage
def test_release_renderer_preserves_manifest_and_emits_safe_artifacts(
tmp_path: Path,
) -> None:
"""The release artifact is exact, reviewable, and contains no credentials."""
module = _load_release_module()
old_digest = "sha256:" + "1" * 64
new_digest = "sha256:" + "2" * 64
revision = "a" * 40
build_number = "17"
manifest = tmp_path / "kustomization.yaml"
manifest.write_text(
"apiVersion: kustomize.config.k8s.io/v1beta1\n"
"kind: Kustomization\n"
"images:\n"
f" - name: {module.DEFAULT_IMAGE}\n"
f" digest: {old_digest}\n",
encoding="utf-8",
)
output = tmp_path / "out"
metadata = module.write_release_artifacts(
digest=f"{new_digest}\n",
source_revision=revision,
build_number=build_number,
destination=f"{module.DEFAULT_IMAGE}:git-{revision}-build-{build_number}",
kustomization=manifest,
output_dir=output,
)
assert manifest.read_text(encoding="utf-8").endswith(f"{old_digest}\n")
assert (
(output / "hermes-kustomization.yaml")
.read_text(encoding="utf-8")
.endswith(f"{new_digest}\n")
)
patch = (output / "hermes-image-update.patch").read_text(encoding="utf-8")
assert f"- digest: {old_digest}" in patch
assert f"+ digest: {new_digest}" in patch
assert (
json.loads((output / "hermes-agent-image.json").read_text(encoding="utf-8"))
== metadata
)
assert set(metadata) == {
"digest",
"build_number",
"flux_image",
"image",
"published_tag",
"source_revision",
}
@pytest.mark.parametrize(
("digest", "revision", "destination"),
[
(
"latest",
"a" * 40,
"registry.bstein.dev/bstein/hermes-agent:git-" + "a" * 40 + "-build-1",
),
(
"sha256:" + "1" * 64,
"short",
"registry.bstein.dev/bstein/hermes-agent:git-short-build-1",
),
(
"sha256:" + "1" * 64,
"a" * 40,
"registry.bstein.dev/bstein/hermes-agent:latest",
),
],
)
def test_release_renderer_rejects_unpinned_inputs(
tmp_path: Path, digest: str, revision: str, destination: str
) -> None:
"""Only exact digests and immutable source-derived tags are accepted."""
module = _load_release_module()
manifest = tmp_path / "kustomization.yaml"
manifest.write_text(
"images:\n"
f" - name: {module.DEFAULT_IMAGE}\n"
" digest: sha256:" + "0" * 64 + "\n",
encoding="utf-8",
)
with pytest.raises(ValueError):
module.write_release_artifacts(
digest=digest,
source_revision=revision,
build_number="1",
destination=destination,
kustomization=manifest,
output_dir=tmp_path / "out",
)
def test_release_renderer_fails_closed_on_manifest_drift(tmp_path: Path) -> None:
"""Missing, duplicate, or already-current image entries require human review."""
module = _load_release_module()
digest = "sha256:" + "f" * 64
missing = "images:\n - name: example.invalid/other\n digest: " + digest + "\n"
with pytest.raises(ValueError, match="found 0"):
module.render_kustomization(missing, digest)
duplicate = (
"images:\n"
f" - name: {module.DEFAULT_IMAGE}\n digest: {digest}\n"
f" - name: {module.DEFAULT_IMAGE}\n digest: {digest}\n"
)
with pytest.raises(ValueError, match="found 2"):
module.render_kustomization(duplicate, digest)
manifest = tmp_path / "kustomization.yaml"
manifest.write_text(
f"images:\n - name: {module.DEFAULT_IMAGE}\n digest: {digest}\n",
encoding="utf-8",
)
revision = "b" * 40
with pytest.raises(ValueError, match="already matches"):
module.write_release_artifacts(
digest=digest,
source_revision=revision,
build_number="1",
destination=f"{module.DEFAULT_IMAGE}:git-{revision}-build-1",
kustomization=manifest,
output_dir=tmp_path / "out",
)
def test_release_cli_reads_digest_file(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""The pipeline CLI uses the Kaniko digest file as its sole digest input."""
module = _load_release_module()
digest = "sha256:" + "c" * 64
revision = "d" * 40
manifest = tmp_path / "kustomization.yaml"
manifest.write_text(
f"images:\n - name: {module.DEFAULT_IMAGE}\n digest: sha256:"
+ "0" * 64
+ "\n",
encoding="utf-8",
)
digest_file = tmp_path / "digest"
digest_file.write_text(digest + "\n", encoding="utf-8")
destination = f"{module.DEFAULT_IMAGE}:git-{revision}-build-9"
image_file = tmp_path / "image"
image_file.write_text(f"{destination}@{digest}\n", encoding="utf-8")
output = tmp_path / "out"
monkeypatch.setenv("HARBOR_USER", "robot")
monkeypatch.setenv("HARBOR_PASSWORD", "secret")
monkeypatch.setattr(
module, "verify_registry_digest", lambda *_args, **_kwargs: None
)
monkeypatch.setattr(
"sys.argv",
[
"hermes_image_release.py",
"render",
"--digest-file",
str(digest_file),
"--image-file",
str(image_file),
"--source-revision",
revision,
"--build-number",
"9",
"--destination",
destination,
"--kustomization",
str(manifest),
"--output-dir",
str(output),
],
)
assert module.main() == 0
assert (
json.loads((output / "hermes-agent-image.json").read_text(encoding="utf-8"))[
"digest"
]
== digest
)

469
testing/tests/test_hermes_image_builder_adversarial.py Normal file
View File

@ -0,0 +1,469 @@
"""Adversarial boundaries for the Hermes image publisher and trigger."""
from __future__ import annotations
import importlib.util
import io
import json
from pathlib import Path
import pytest
REPO_ROOT = Path(__file__).resolve().parents[2]
RELEASE_SCRIPT = REPO_ROOT / "ci/scripts/hermes_image_release.py"
TRIGGER_SCRIPT = REPO_ROOT / "services/hermes/scripts/jenkins_image_build_trigger.py"
def _load(path: Path, name: str):
spec = importlib.util.spec_from_file_location(name, path)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
class Response(io.BytesIO):
"""Small context-managed HTTP response fixture."""
def __init__(
self,
status: int,
headers: dict[str, str] | None = None,
body: bytes = b"",
):
super().__init__(body)
self.status = status
self.headers = headers or {}
def test_kaniko_evidence_binds_digest_destination_and_unique_build() -> None:
"""Both Kaniko artifacts must describe one exact non-replayable tag."""
module = _load(RELEASE_SCRIPT, "hermes_image_release_evidence")
revision = "a" * 40
digest = "sha256:" + "b" * 64
destination = f"{module.DEFAULT_IMAGE}:git-{revision}-build-42"
assert module.validate_destination(destination, revision, "42") == (
revision,
"42",
)
assert (
module.validate_kaniko_evidence(
digest_text=f"{digest}\n",
image_text=f"{destination}@{digest}\n",
destination=destination,
)
== digest
)
@pytest.mark.parametrize(
("digest_text", "image_template"),
[
("sha256:" + "a" * 64 + "\nextra\n", "{destination}@{digest}"),
("sha256:" + "a" * 64, "{destination}@sha256:" + "b" * 64),
("sha256:" + "a" * 64, "registry.invalid/x:y@{digest}"),
("sha256:" + "a" * 64, "{destination}@{digest}\nextra"),
],
)
def test_kaniko_evidence_rejects_cross_artifact_mismatch(
digest_text: str, image_template: str
) -> None:
"""A digest, tag, repository, or cardinality mismatch stops rendering."""
module = _load(RELEASE_SCRIPT, "hermes_image_release_mismatch")
revision = "c" * 40
digest = "sha256:" + "a" * 64
destination = f"{module.DEFAULT_IMAGE}:git-{revision}-build-8"
image_text = image_template.format(destination=destination, digest=digest)
with pytest.raises(ValueError):
module.validate_kaniko_evidence(
digest_text=digest_text,
image_text=image_text,
destination=destination,
)
def test_registry_preflight_rejects_existing_tag_and_auth_failures() -> None:
"""Only an authenticated 404 permits Kaniko to claim the unique tag."""
module = _load(RELEASE_SCRIPT, "hermes_image_release_preflight")
destination = f"{module.DEFAULT_IMAGE}:git-{'d' * 40}-build-3"
captured = {}
def missing(request, timeout):
captured["request"] = request
captured["timeout"] = timeout
return Response(404)
module.assert_tag_absent(
destination, username="robot", password="private", opener=missing
)
request = captured["request"]
assert request.method == "GET"
assert request.full_url.startswith(
"https://registry.bstein.dev/api/v2.0/projects/bstein/repositories/"
"hermes-agent/artifacts/"
)
assert "private" not in request.full_url
assert captured["timeout"] == 20
for status, message in (
(200, "already exists"),
(401, "HTTP 401"),
(503, "HTTP 503"),
):
with pytest.raises(RuntimeError, match=message):
module.assert_tag_absent(
destination,
username="robot",
password="private",
opener=lambda *_args, code=status: Response(code),
)
def test_registry_preflight_requires_exact_server_immutable_policy() -> None:
"""The already-running controller cannot publish before Flux installs the rule."""
module = _load(RELEASE_SCRIPT, "hermes_image_release_policy_preflight")
expected = {
"disabled": False,
"action": "immutable",
"template": "immutable_template",
"tag_selectors": [
{
"kind": "doublestar",
"decoration": "matches",
"pattern": "git-*-build-*",
}
],
"scope_selectors": {
"repository": [
{
"kind": "doublestar",
"decoration": "repoMatches",
"pattern": "hermes-agent",
}
]
},
}
captured = {}
def exact(request, timeout):
captured["url"] = request.full_url
captured["timeout"] = timeout
return Response(
200,
{"X-Total-Count": "1"},
body=json.dumps([{"id": 9, **expected}]).encode(),
)
module.verify_immutable_policy(username="robot", password="private", opener=exact)
assert captured["url"].endswith(
"/projects/bstein/immutabletagrules?page=1&page_size=100"
)
assert captured["timeout"] == 20
for status, body, headers, message in (
(403, b"", {}, "HTTP 403"),
(200, b"[]", {"X-Total-Count": "0"}, "absent"),
(
200,
json.dumps([{**expected, "disabled": True}]).encode(),
{"X-Total-Count": "1"},
"not exact",
),
(200, b"not-json", {}, "invalid"),
(
200,
json.dumps([expected]).encode(),
{"X-Total-Count": "2"},
"truncated",
),
):
with pytest.raises(RuntimeError, match=message):
module.verify_immutable_policy(
username="robot",
password="private",
opener=lambda *_args, code=status, value=body, evidence=headers: Response(
code, evidence, body=value
),
)
def test_harbor_client_fails_closed_on_input_size_auth_and_json(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Malformed inputs, oversized bodies, missing auth, and bad JSON all fail."""
module = _load(RELEASE_SCRIPT, "hermes_image_release_harbor_errors")
destination = f"{module.DEFAULT_IMAGE}:git-{'1' * 40}-build-6"
with pytest.raises(ValueError, match="destination"):
module.assert_tag_absent(
"registry.invalid/x:tag", username="robot", password="private"
)
with pytest.raises(RuntimeError, match="credentials"):
module.assert_tag_absent(destination, username="", password="private")
with pytest.raises(RuntimeError, match="size limit"):
module.assert_tag_absent(
destination,
username="robot",
password="private",
opener=lambda *_args: Response(200, body=b"x" * 1_048_577),
)
with pytest.raises(RuntimeError, match="invalid artifact JSON"):
module.verify_registry_digest(
destination,
"sha256:" + "2" * 64,
username="robot",
password="private",
opener=lambda *_args: Response(200, body=b"\xff"),
)
monkeypatch.delenv("HARBOR_USER", raising=False)
monkeypatch.delenv("HARBOR_PASSWORD", raising=False)
with pytest.raises(RuntimeError, match="unavailable"):
module._credentials()
def test_default_harbor_opener_returns_http_responses(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""The real wrapper returns both normal and non-redirect HTTP responses."""
module = _load(RELEASE_SCRIPT, "hermes_image_release_opener")
request = module.urllib.request.Request("https://registry.bstein.dev/test")
class SuccessOpener:
def open(self, _request, timeout):
assert timeout == 7
return Response(204)
monkeypatch.setattr(
module.urllib.request, "build_opener", lambda *_handlers: SuccessOpener()
)
assert module._registry_request(request, 7).status == 204
class ErrorOpener:
def open(self, _request, timeout):
assert timeout == 8
raise module.urllib.error.HTTPError(
_request.full_url, 404, "missing", {}, None
)
monkeypatch.setattr(
module.urllib.request, "build_opener", lambda *_handlers: ErrorOpener()
)
assert module._registry_request(request, 8).code == 404
@pytest.mark.parametrize(
("status", "artifact", "message"),
[
(404, {}, "HTTP 404"),
(200, {}, "omitted"),
(200, {"digest": "sha256:" + "2" * 64}, "does not match"),
(200, {"digest": "latest"}, "omitted"),
],
)
def test_registry_verification_rejects_missing_or_mismatched_digest(
status: int, artifact: dict[str, str], message: str
) -> None:
"""Rendering requires an independent exact Harbor digest response."""
module = _load(RELEASE_SCRIPT, "hermes_image_release_registry")
digest = "sha256:" + "1" * 64
destination = f"{module.DEFAULT_IMAGE}:git-{'e' * 40}-build-4"
body = json.dumps(artifact).encode("utf-8")
with pytest.raises(RuntimeError, match=message):
module.verify_registry_digest(
destination,
digest,
username="robot",
password="private",
opener=lambda *_args: Response(status, body=body),
)
def test_registry_verification_accepts_exact_pushed_digest() -> None:
"""An exact authenticated Harbor digest allows artifact rendering."""
module = _load(RELEASE_SCRIPT, "hermes_image_release_registry_ok")
digest = "sha256:" + "4" * 64
destination = f"{module.DEFAULT_IMAGE}:git-{'f' * 40}-build-5"
with pytest.raises(RuntimeError, match="expected tag"):
module.verify_registry_digest(
destination,
digest,
username="robot",
password="private",
opener=lambda *_args: Response(
200,
body=json.dumps(
{"digest": digest, "tags": [{"name": "different-tag"}]}
).encode("utf-8"),
),
)
module.verify_registry_digest(
destination,
digest,
username="robot",
password="private",
opener=lambda *_args: Response(
200,
body=json.dumps(
{
"digest": digest,
"tags": [
{
"name": destination.rsplit(":", 1)[1],
"immutable": True,
}
],
}
).encode("utf-8"),
),
)
def test_registry_verification_rejects_tag_without_server_immutability() -> None:
"""Digest evidence is insufficient unless Harbor reports the build tag immutable."""
module = _load(RELEASE_SCRIPT, "hermes_image_release_registry_mutable")
digest = "sha256:" + "7" * 64
destination = f"{module.DEFAULT_IMAGE}:git-{'8' * 40}-build-19"
captured = {}
def opener(request, _timeout):
captured["url"] = request.full_url
return Response(
200,
body=json.dumps(
{
"digest": digest,
"tags": [
{
"name": destination.rsplit(":", 1)[1],
"immutable": False,
}
],
}
).encode(),
)
with pytest.raises(RuntimeError, match="immutable"):
module.verify_registry_digest(
destination,
digest,
username="robot",
password="private",
opener=opener,
)
assert captured["url"].endswith("?with_immutable_status=true")
def test_release_main_preflight_uses_fixed_credentials_and_destination(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""The preflight CLI validates and checks exactly the requested build tag."""
module = _load(RELEASE_SCRIPT, "hermes_image_release_main_absent")
revision = "3" * 40
destination = f"{module.DEFAULT_IMAGE}:git-{revision}-build-12"
captured = {}
monkeypatch.setenv("HARBOR_USER", "robot")
monkeypatch.setenv("HARBOR_PASSWORD", "private")
def assert_absent(value, **credentials):
captured["destination"] = value
captured["credentials"] = credentials
def verify_policy(**credentials):
captured["policy_credentials"] = credentials
monkeypatch.setattr(module, "assert_tag_absent", assert_absent)
monkeypatch.setattr(module, "verify_immutable_policy", verify_policy)
monkeypatch.setattr(
"sys.argv",
[
"hermes_image_release.py",
"assert-absent",
"--source-revision",
revision,
"--build-number",
"12",
"--destination",
destination,
],
)
assert module.main() == 0
assert captured == {
"destination": destination,
"credentials": {"username": "robot", "password": "private"},
"policy_credentials": {"username": "robot", "password": "private"},
}
def test_renderer_skips_a_matching_entry_without_a_digest() -> None:
"""Only the one complete image entry is changed when an earlier entry drifts."""
module = _load(RELEASE_SCRIPT, "hermes_image_release_manifest_scan")
digest = "sha256:" + "5" * 64
source = (
"images:\n"
f" - name: {module.DEFAULT_IMAGE}\n"
" newTag: ignored\n"
" - name: example.invalid/other\n"
" newTag: stable\n"
f" - name: {module.DEFAULT_IMAGE}\n"
" digest: sha256:" + "0" * 64 + "\n"
)
assert module.render_kustomization(source, digest).endswith(f"{digest}\n")
@pytest.mark.parametrize("status", [200, 202, 204, 301, 302, 307, 308])
def test_trigger_rejects_non_plugin_success_status(tmp_path: Path, status: int) -> None:
"""Generic proxy successes and redirects are not proof of a queued build."""
module = _load(TRIGGER_SCRIPT, f"jenkins_trigger_status_{status}")
token = tmp_path / "token"
token.write_text("private\n", encoding="utf-8")
response = lambda *_args, **_kwargs: Response( # noqa: E731
status, {"Location": "https://ci.bstein.dev/queue/item/9/"}
)
with pytest.raises(RuntimeError, match=f"HTTP {status}"):
module.trigger_build("a" * 40, token_file=token, opener=response)
@pytest.mark.parametrize(
"location",
[
"",
"https://evil.invalid/queue/item/9/",
"http://ci.bstein.dev/queue/item/9/",
"https://ci.bstein.dev/job/hermes-agent-image/9/",
"https://ci.bstein.dev/queue/item/9/?token=leak",
"https://ci.bstein.dev/queue/item/9/#fragment",
"https://ci.bstein.dev/queue/item/not-a-number/",
],
)
def test_trigger_rejects_missing_malformed_or_cross_origin_queue_location(
tmp_path: Path, location: str
) -> None:
"""A real accepted status still needs the exact same-origin queue resource."""
module = _load(TRIGGER_SCRIPT, "jenkins_trigger_location")
token = tmp_path / "token"
token.write_text("private\n", encoding="utf-8")
response = lambda *_args, **_kwargs: Response( # noqa: E731
201, {"Location": location}
)
with pytest.raises(RuntimeError, match="Location"):
module.trigger_build("b" * 40, token_file=token, opener=response)
def test_trigger_uses_https_and_accepts_exact_relative_queue_path(
tmp_path: Path,
) -> None:
"""A relative plugin Location is resolved only against the fixed HTTPS origin."""
module = _load(TRIGGER_SCRIPT, "jenkins_trigger_origin")
token = tmp_path / "token"
token.write_text("private\n", encoding="utf-8")
captured = {}
def opener(request, timeout):
captured["url"] = request.full_url
captured["timeout"] = timeout
return Response(201, {"Location": "/queue/item/11/"})
result = module.trigger_build("c" * 40, token_file=token, opener=opener)
assert captured["url"].startswith("https://ci.bstein.dev/")
assert captured["timeout"] == 20
assert result["queue_path"] == "/queue/item/11/"

325
testing/tests/test_hermes_image_builder_harbor.py Normal file
View File

@ -0,0 +1,325 @@
"""Server-side Harbor immutability contracts for Hermes agent releases."""
from __future__ import annotations
import importlib.util
import json
from pathlib import Path
import pytest
import yaml
REPO_ROOT = Path(__file__).resolve().parents[2]
SCRIPT = (
REPO_ROOT
/ "services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py"
)
def _load_module():
spec = importlib.util.spec_from_file_location("harbor_immutability", SCRIPT)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def _rule(module, *, rule_id: int = 17, disabled: bool = False) -> dict:
return {"id": rule_id, **module.EXPECTED_RULE, "disabled": disabled}
def _robot(module, *, immutable: bool = False, extra_immutable: bool = False) -> dict:
access = [
{"resource": "repository", "action": "pull", "effect": "allow"},
{"resource": "repository", "action": "push", "effect": "allow"},
]
if immutable:
access.append({"resource": "immutable-tag", "action": "list"})
if extra_immutable:
access.append({"resource": "immutable-tag", "action": "delete"})
return {
"id": 41,
"name": module.PUBLISH_ROBOT,
"description": "Jenkins publisher",
"level": "system",
"duration": -1,
"editable": True,
"disable": False,
"permissions": [
{
"kind": "project",
"namespace": "other",
"access": [{"resource": "repository", "action": "pull"}],
},
{"kind": "project", "namespace": "bstein", "access": access},
],
}
def _count(value: int) -> dict[str, str]:
return {"X-Total-Count": str(value)}
class FakeClient:
"""Small deterministic Harbor API fake."""
def __init__(self, responses):
self.origin = "https://registry.bstein.dev/api/v2.0"
self.responses = list(responses)
self.calls = []
def request(self, method, path, payload=None):
self.calls.append((method, path, payload))
return self.responses.pop(0)
def test_flux_tracks_exact_immutable_rule_before_jenkins() -> None:
"""The Harbor rule is a reviewed prerequisite, not a Jenkins preflight only."""
harbor = yaml.safe_load(
(
REPO_ROOT
/ "clusters/atlas/flux-system/applications/harbor/kustomization.yaml"
).read_text()
)
jenkins = yaml.safe_load(
(
REPO_ROOT
/ "clusters/atlas/flux-system/applications/jenkins/kustomization.yaml"
).read_text()
)
check = {
"apiVersion": "batch/v1",
"kind": "Job",
"name": "harbor-hermes-agent-immutability-ensure-1",
"namespace": "harbor",
}
assert check in harbor["spec"]["healthChecks"]
assert "harbor" in {item["name"] for item in jenkins["spec"]["dependsOn"]}
def test_policy_job_uses_runtime_vault_secret_and_hardened_pinned_image() -> None:
"""No Harbor credential is committed or retained in a mutable workload."""
job = yaml.safe_load(
(REPO_ROOT / "services/harbor/hermes-agent-immutability-job.yaml").read_text()
)
template = job["spec"]["template"]
annotations = template["metadata"]["annotations"]
assert annotations["vault.hashicorp.com/role"] == "harbor-policy-bootstrap"
assert (
annotations["vault.hashicorp.com/agent-inject-secret-harbor-admin-password"]
== "kv/data/atlas/harbor/harbor-core"
)
pod = template["spec"]
assert pod["serviceAccountName"] == "harbor-policy-bootstrap"
assert pod["enableServiceLinks"] is False
container = pod["containers"][0]
assert "@sha256:" in container["image"]
security = container["securityContext"]
assert security["runAsNonRoot"] is True
assert security["readOnlyRootFilesystem"] is True
assert security["allowPrivilegeEscalation"] is False
assert security["capabilities"]["drop"] == ["ALL"]
vault = (
REPO_ROOT / "services/vault/scripts/vault_k8s_auth_configure.sh"
).read_text()
policy = vault.split("harbor_policy_bootstrap_policy='", 1)[1].split("'", 1)[0]
assert 'path "kv/data/atlas/harbor/harbor-core"' in policy
assert 'capabilities = ["read"]' in policy
assert "*" not in policy
assert 'bound_service_account_names="harbor-policy-bootstrap"' in vault
script = SCRIPT.read_text()
assert 'PUBLISH_ROBOT = "robot$jenkins-pipelines"' in script
assert '{"resource": "immutable-tag", "action": "list"}' in script
assert 'client.request(\n "PUT", f"/robots/{robot_id}", payload' in script
assert '"secret"' not in script.split("payload = {", 1)[1].split("}", 1)[0]
def test_rule_contract_is_exact_repository_and_unique_build_tags() -> None:
"""Unrelated bstein repositories and ordinary Hermes tags stay mutable."""
module = _load_module()
assert module.EXPECTED_RULE == {
"disabled": False,
"action": "immutable",
"template": "immutable_template",
"tag_selectors": [
{
"kind": "doublestar",
"decoration": "matches",
"pattern": "git-*-build-*",
}
],
"scope_selectors": {
"repository": [
{
"kind": "doublestar",
"decoration": "repoMatches",
"pattern": "hermes-agent",
}
]
},
}
def test_existing_exact_rule_is_idempotent_without_mutation() -> None:
"""A rerun only validates the exact enabled rule."""
module = _load_module()
body = json.dumps([_rule(module)]).encode()
client = FakeClient([(200, body, _count(1))])
assert module.ensure_rule(client) == 17
assert [call[0] for call in client.calls] == ["GET"]
def test_create_requires_201_exact_location_and_verified_reread() -> None:
"""Policy bootstrap fails closed until Harbor returns the exact persisted rule."""
module = _load_module()
body = json.dumps([_rule(module, rule_id=23)]).encode()
client = FakeClient(
[
(200, b"[]", _count(0)),
(
201,
b"",
{"Location": "/api/v2.0/projects/bstein/immutabletagrules/23"},
),
(200, body, _count(1)),
]
)
assert module.ensure_rule(client) == 23
assert client.calls[1] == (
"POST",
"/projects/bstein/immutabletagrules",
module.EXPECTED_RULE,
)
@pytest.mark.parametrize(
"responses,match",
[
([(200, b"[]", _count(0)), (200, b"", {})], "HTTP 200"),
(
[
(200, b"[]", _count(0)),
(201, b"", {"Location": "https://evil.invalid/1"}),
],
"Location",
),
],
)
def test_create_rejects_noncanonical_responses(responses, match: str) -> None:
"""Proxy success pages and foreign locations can never count as enforcement."""
module = _load_module()
with pytest.raises(RuntimeError, match=match):
module.ensure_rule(FakeClient(responses))
def test_readiness_failures_are_distinct_from_policy_rejections() -> None:
"""The tracked Job may wait for Harbor without retrying an auth denial."""
module = _load_module()
with pytest.raises(module.HarborUnavailable):
module.ensure_rule(FakeClient([(503, b"", {})]))
with pytest.raises(RuntimeError, match="HTTP 403") as exc:
module.ensure_rule(FakeClient([(403, b"", {})]))
assert not isinstance(exc.value, module.HarborUnavailable)
def test_disabled_or_duplicate_exact_scope_fails_closed() -> None:
"""Bootstrap never silently edits a conflicting security policy."""
module = _load_module()
disabled = json.dumps([_rule(module, disabled=True)]).encode()
with pytest.raises(RuntimeError, match="not enabled"):
module.ensure_rule(FakeClient([(200, disabled, _count(1))]))
duplicate = json.dumps([_rule(module), _rule(module, rule_id=18)]).encode()
with pytest.raises(RuntimeError, match="multiple"):
module.ensure_rule(FakeClient([(200, duplicate, _count(2))]))
def test_policy_lists_reject_missing_or_truncated_count_evidence() -> None:
"""A hidden second page can never produce a duplicate rule or robot update."""
module = _load_module()
body = json.dumps([_rule(module)]).encode()
for headers in ({}, _count(2)):
with pytest.raises(RuntimeError, match="count|truncated"):
module.ensure_rule(FakeClient([(200, body, headers)]))
robot_body = json.dumps(
[{"id": 41, "name": module.PUBLISH_ROBOT}]
).encode()
with pytest.raises(RuntimeError, match="truncated"):
module.ensure_publisher_can_read_rule(
FakeClient([(200, robot_body, _count(2))])
)
def test_publisher_policy_adds_only_read_access_and_preserves_robot() -> None:
"""Bootstrap preserves every existing scope and never touches robot secret state."""
module = _load_module()
original = _robot(module)
persisted = _robot(module, immutable=True)
client = FakeClient(
[
(
200,
json.dumps([{"id": 41, "name": module.PUBLISH_ROBOT}]).encode(),
_count(1),
),
(200, json.dumps(original).encode(), {}),
(200, b"", {}),
(200, json.dumps(persisted).encode(), {}),
]
)
assert module.ensure_publisher_can_read_rule(client) == 41
method, path, payload = client.calls[2]
assert (method, path) == ("PUT", "/robots/41")
assert "secret" not in payload
assert payload["permissions"][0] == original["permissions"][0]
assert payload["permissions"][1]["access"][-1] == {
"resource": "immutable-tag",
"action": "list",
}
def test_publisher_policy_is_idempotent_and_rejects_broad_access() -> None:
"""An exact read policy is stable; mutation-capable immutable access is blocked."""
module = _load_module()
exact = _robot(module, immutable=True)
client = FakeClient(
[
(
200,
json.dumps([{"id": 41, "name": module.PUBLISH_ROBOT}]).encode(),
_count(1),
),
(200, json.dumps(exact).encode(), {}),
]
)
assert module.ensure_publisher_can_read_rule(client) == 41
assert [call[0] for call in client.calls] == ["GET", "GET"]
broad = _robot(module, immutable=True, extra_immutable=True)
client = FakeClient(
[
(
200,
json.dumps([{"id": 41, "name": module.PUBLISH_ROBOT}]).encode(),
_count(1),
),
(200, json.dumps(broad).encode(), {}),
]
)
with pytest.raises(RuntimeError, match="broader"):
module.ensure_publisher_can_read_rule(client)
def test_harbor_client_rejects_plaintext_or_ambiguous_origins() -> None:
"""Runtime admin credentials are never sent over HTTP or a URL with query state."""
module = _load_module()
for origin in (
"http://registry.bstein.dev/api/v2.0",
"https://registry.bstein.dev/api/v2.0?next=evil",
"https://other.invalid/api/v2.0",
"",
):
with pytest.raises(ValueError):
module.HarborClient(origin, "admin", "secret")

272
testing/tests/test_hermes_image_builder_vault.py Normal file
View File

@ -0,0 +1,272 @@
"""Vault least-privilege and fail-closed seeding tests for image releases."""
from __future__ import annotations
import os
import stat
import subprocess
from pathlib import Path
import pytest
import yaml
REPO_ROOT = Path(__file__).resolve().parents[2]
VAULT_CONFIG = REPO_ROOT / "services/vault/scripts/vault_k8s_auth_configure.sh"
SEEDER = (
REPO_ROOT
/ "services/vault-hermes-jenkins-token-seed/scripts/vault_hermes_jenkins_build_token_ensure.sh"
)
def test_developer_jenkins_access_is_read_only_and_bound_to_two_consumers() -> None:
"""Only the Jenkins controller and Hermes agent can read the fixed-job token."""
source = VAULT_CONFIG.read_text(encoding="utf-8")
assert 'write_policy_and_role "jenkins" "jenkins" "jenkins"' in source
assert (
'"jenkins/* shared/harbor-pull quality/sonarqube-oidc '
'hermes/developer-jenkins" ""'
) in source
assert 'write_policy_and_role "hermes-agent" "hermes" "hermes-agent"' in source
assert 'hermes/developer-jenkins hermes/developer-ssh" ""' in source
assert 'write_policy_and_role "hermes-switchyard" "hermes"' in source
assert '"hermes/chat-telegram" ""' in source
assert '"hermes/developer-jenkins"' not in source
jenkins_spc = yaml.safe_load(
(REPO_ROOT / "services/jenkins/secretproviderclass.yaml").read_text()
)
assert jenkins_spc["spec"]["parameters"]["roleName"] == "jenkins-vault-sync"
switchyard = yaml.safe_load(
(REPO_ROOT / "services/hermes/switchyard-deployment.yaml").read_text()
)
annotations = switchyard["spec"]["template"]["metadata"]["annotations"]
assert annotations["vault.hashicorp.com/role"] == "hermes-switchyard"
def test_flux_orders_seed_then_jenkins_then_hermes() -> None:
"""The fixed token must be Ready before either consumer is rolled."""
app_root = REPO_ROOT / "clusters/atlas/flux-system/applications"
vault = yaml.safe_load((app_root / "vault/kustomization.yaml").read_text())
seed = yaml.safe_load(
(app_root / "vault-hermes-jenkins-token-seed/kustomization.yaml").read_text()
)
jenkins = yaml.safe_load((app_root / "jenkins/kustomization.yaml").read_text())
hermes = yaml.safe_load((app_root / "hermes/kustomization.yaml").read_text())
role_check = {
"apiVersion": "batch/v1",
"kind": "Job",
"name": "vault-k8s-auth-hermes-9",
"namespace": "vault",
}
seed_check = {
"apiVersion": "batch/v1",
"kind": "Job",
"name": "vault-hermes-jenkins-build-token-seed-2",
"namespace": "vault",
}
assert role_check in vault["spec"]["healthChecks"]
assert {item["name"] for item in seed["spec"]["dependsOn"]} == {"vault"}
assert seed_check in seed["spec"]["healthChecks"]
assert jenkins["spec"]["suspend"] is False
assert {item["name"] for item in jenkins["spec"]["dependsOn"]} >= {
"helm",
"harbor",
"vault-hermes-jenkins-token-seed",
}
assert "jenkins" in {item["name"] for item in hermes["spec"]["dependsOn"]}
def test_seed_job_is_revisioned_bounded_and_tracks_fail_closed_script() -> None:
"""The prerequisite is a tracked one-shot on healthy non-reserved capacity."""
job = yaml.safe_load(
(
REPO_ROOT / "services/vault-hermes-jenkins-token-seed/job.yaml"
).read_text()
)
assert job["metadata"]["name"] == "vault-hermes-jenkins-build-token-seed-2"
pod = job["spec"]["template"]["spec"]
assert pod["serviceAccountName"] == "hermes-jenkins-token-seed"
assert pod["enableServiceLinks"] is False
assert pod["restartPolicy"] == "Never"
assert pod["nodeSelector"]["hardware"] == "rpi5"
expression = pod["affinity"]["nodeAffinity"][
"requiredDuringSchedulingIgnoredDuringExecution"
]["nodeSelectorTerms"][0]["matchExpressions"][0]
assert set(expression["values"]) >= {
"titan-04",
"titan-14",
"titan-18",
"titan-19",
"titan-24",
}
container = pod["containers"][0]
assert "@sha256:" in container["image"]
security = container["securityContext"]
assert security["runAsNonRoot"] is True
assert security["readOnlyRootFilesystem"] is True
assert security["allowPrivilegeEscalation"] is False
assert security["capabilities"]["drop"] == ["ALL"]
source = SEEDER.read_text(encoding="utf-8")
assert '"options":{"cas":0}' in source
assert "kv/data/atlas/hermes/developer-jenkins" in source
assert "kv patch" not in source
assert "kv delete" not in source
assert SEEDER.name not in VAULT_CONFIG.read_text(encoding="utf-8")
def test_seed_vault_policy_is_exact_create_read_and_rng_only() -> None:
"""The seed identity cannot rotate, delete, list, or touch unrelated KV."""
source = VAULT_CONFIG.read_text(encoding="utf-8")
policy = source.split("hermes_jenkins_token_seed_policy='", 1)[1].split("'", 1)[0]
assert 'path "kv/data/atlas/hermes/developer-jenkins"' in policy
assert 'capabilities = ["create", "read"]' in policy
assert 'path "sys/tools/random/32"' in policy
assert 'capabilities = ["update"]' in policy
for forbidden in ("delete", "patch", "list", 'kv/data/atlas/hermes/*'):
assert forbidden not in policy
assert 'bound_service_account_names="hermes-jenkins-token-seed"' in source
assert 'bound_service_account_namespaces="vault"' in source
def _fake_vault_tools(tmp_path: Path) -> tuple[Path, Path, Path]:
"""Create deterministic Vault/sleep fakes and return their evidence paths."""
binary_dir = tmp_path / "bin"
binary_dir.mkdir(parents=True)
calls = tmp_path / "calls"
stdin_capture = tmp_path / "stdin"
vault = binary_dir / "vault"
vault.write_text(
"""#!/bin/sh
set -eu
printf '%s\n' "$*" >> "$FAKE_CALL_LOG"
scenario="$FAKE_SCENARIO"
if [ "$1" = "write" ] && [ "$2" = "-field=random_bytes" ] && [ "$3" = "sys/tools/random/32" ]; then
printf '%064d\n' 0
exit 0
fi
if [ "$1" = "write" ] && [ "$2" = "kv/data/atlas/hermes/developer-jenkins" ]; then
payload="${3#@}"
cp "$payload" "$FAKE_STDIN_CAPTURE"
if [ "$scenario" = "cas-race" ]; then
: > "$FAKE_WINNER"
echo 'check-and-set parameter did not match' >&2
exit 2
fi
exit 0
fi
if [ "$1" = "read" ] && [ "$2" = "-format=json" ]; then
case "$scenario" in
existing)
printf '{"data":{"data":{"build_token":"%064d"}}}\n' 1
exit 0
;;
missing-field)
printf '{"data":{"data":{"other":"preserve-me"}}}\n'
exit 0
;;
absent)
echo 'Code: 404' >&2
exit 2
;;
cas-race)
if [ -f "$FAKE_WINNER" ]; then
printf '{"data":{"data":{"build_token":"%064d"}}}\n' 2
exit 0
fi
echo 'Code: 404' >&2
exit 2
;;
transient)
count=0
if [ -f "$FAKE_COUNT" ]; then count="$(cat "$FAKE_COUNT")"; fi
count=$((count + 1))
printf '%s' "$count" > "$FAKE_COUNT"
if [ "$count" -lt 3 ]; then echo 'temporary upstream error' >&2; exit 2; fi
echo 'Code: 404' >&2
exit 2
;;
read-error)
echo 'permission denied' >&2
exit 2
;;
esac
fi
echo "unexpected fake Vault call: $*" >&2
exit 99
""",
encoding="utf-8",
)
sleep = binary_dir / "sleep"
sleep.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8")
for executable in (vault, sleep):
executable.chmod(executable.stat().st_mode | stat.S_IXUSR)
return binary_dir, calls, stdin_capture
def _run_seeder(
tmp_path: Path, scenario: str
) -> tuple[subprocess.CompletedProcess, str]:
binary_dir, calls, stdin_capture = _fake_vault_tools(tmp_path)
env = {
**os.environ,
"PATH": f"{binary_dir}:{os.environ['PATH']}",
"VAULT_TOKEN": "test-token",
"FAKE_SCENARIO": scenario,
"FAKE_CALL_LOG": str(calls),
"FAKE_STDIN_CAPTURE": str(stdin_capture),
"FAKE_WINNER": str(tmp_path / "winner"),
"FAKE_COUNT": str(tmp_path / "count"),
"TMPDIR": str(tmp_path),
}
result = subprocess.run(
["sh", str(SEEDER)],
env=env,
text=True,
capture_output=True,
check=False,
)
call_text = calls.read_text(encoding="utf-8") if calls.exists() else ""
return result, call_text
@pytest.mark.parametrize("scenario", ["existing", "missing-field", "read-error"])
def test_seeder_never_overwrites_existing_or_ambiguous_state(
tmp_path: Path, scenario: str
) -> None:
"""Existing fields and persistent read failures can never become a put."""
result, calls = _run_seeder(tmp_path, scenario)
assert (result.returncode == 0) is (scenario == "existing")
assert "write kv/data/atlas/hermes/developer-jenkins" not in calls
assert "sys/tools/random" not in calls
def test_seeder_uses_single_cas_create_and_stdin_for_absent_secret(
tmp_path: Path,
) -> None:
"""A confirmed 404 produces one create-only write without a token argument."""
result, calls = _run_seeder(tmp_path, "absent")
assert result.returncode == 0, result.stderr
assert calls.count("write kv/data/atlas/hermes/developer-jenkins") == 1
assert "@" in calls
assert "0000000000000000000000000000000000000000000000000000000000000000" not in calls
payload = (tmp_path / "stdin").read_text(encoding="utf-8")
assert '"options":{"cas":0}' in payload
assert '"build_token":"' in payload
assert not (tmp_path / "hermes-jenkins-token.json").exists()
def test_seeder_retries_reads_and_accepts_only_a_verified_cas_winner(
tmp_path: Path,
) -> None:
"""Transient reads retry; a competing create is accepted only after reread."""
transient, transient_calls = _run_seeder(tmp_path / "transient", "transient")
assert transient.returncode == 0, transient.stderr
assert transient_calls.count("read -format=json kv/data/atlas/hermes/developer-jenkins") == 3
assert transient_calls.count("write kv/data/atlas/hermes/developer-jenkins") == 1
race, race_calls = _run_seeder(tmp_path / "race", "cas-race")
assert race.returncode == 0, race.stderr
assert race_calls.count("write kv/data/atlas/hermes/developer-jenkins") == 1
assert race_calls.count("read -format=json kv/data/atlas/hermes/developer-jenkins") == 2
assert "another seeder won CAS" in race.stderr

2
testing/tests/test_hermes_runtime_access.py
View File

@ -177,6 +177,7 @@ def test_agent_runtime_stage_keeps_credentials_in_memory(tmp_path: Path, monkeyp
"chat-relay-key": "relay-key",
"gitea-token": "gitea-key",
"gitea-username": "hermes-automation",
"jenkins-image-build-token": "job-scoped-token",
"node-ssh-private-key": "private-key",
"node-ssh-config": "host-config",
"node-ssh-known-hosts": "known-hosts",
@ -196,6 +197,7 @@ def test_agent_runtime_stage_keeps_credentials_in_memory(tmp_path: Path, monkeyp
assert (runtime / "claude/.credentials.json").stat().st_mode & 0o777 == 0o600
assert (runtime / "codex/auth.json").stat().st_mode & 0o777 == 0o600
assert (runtime / "jenkins-image-build-token").stat().st_mode & 0o777 == 0o600
assert (runtime / "claude/settings.json").is_symlink()
assert (runtime / "codex/skills").is_symlink()
assert not (runtime / "claude/backups").exists()