From 0881e9eb45a7014d9798510fae4a34e52ffeb6ee Mon Sep 17 00:00:00 2001 From: jenkins Date: Sun, 16 Aug 2026 20:04:17 -0300 Subject: [PATCH] hermes: add daemonless agent image release lane --- ci/Jenkinsfile.hermes-agent-image | 256 +++++++++++ ci/scripts/hermes_image_release.py | 300 +++++++++++++ .../applications/hermes/kustomization.yaml | 1 + .../applications/jenkins/kustomization.yaml | 4 +- .../applications/vault/kustomization.yaml | 5 + services/hermes/NOTES.md | 51 +++ services/hermes/agent-deployment.yaml | 5 + services/hermes/kustomization.yaml | 1 + .../scripts/jenkins_image_build_trigger.py | 112 +++++ .../hermes/scripts/stage_runtime_access.py | 1 + services/hermes/switchyard-deployment.yaml | 2 +- services/jenkins/configmap-jcasc.yaml | 19 + services/jenkins/configmap-plugins.yaml | 1 + services/jenkins/deployment.yaml | 3 + services/jenkins/secretproviderclass.yaml | 2 +- .../hermes-jenkins-build-token-seed-job.yaml | 47 ++ services/vault/kustomization.yaml | 2 + ...vault_hermes_jenkins_build_token_ensure.sh | 153 +++++++ .../vault/scripts/vault_k8s_auth_configure.sh | 14 +- testing/quality_contract.json | 3 + testing/tests/test_hermes_image_builder.py | 415 ++++++++++++++++++ .../test_hermes_image_builder_adversarial.py | 356 +++++++++++++++ .../tests/test_hermes_image_builder_vault.py | 233 ++++++++++ testing/tests/test_hermes_runtime_access.py | 2 + 24 files changed, 1981 insertions(+), 7 deletions(-) create mode 100644 ci/Jenkinsfile.hermes-agent-image create mode 100755 ci/scripts/hermes_image_release.py create mode 100755 services/hermes/scripts/jenkins_image_build_trigger.py create mode 100644 services/vault/hermes-jenkins-build-token-seed-job.yaml create mode 100755 services/vault/scripts/vault_hermes_jenkins_build_token_ensure.sh create mode 100644 testing/tests/test_hermes_image_builder.py create mode 100644 testing/tests/test_hermes_image_builder_adversarial.py create mode 100644 testing/tests/test_hermes_image_builder_vault.py diff --git a/ci/Jenkinsfile.hermes-agent-image b/ci/Jenkinsfile.hermes-agent-image new file mode 100644 index 00000000..42db38df --- /dev/null +++ b/ci/Jenkinsfile.hermes-agent-image @@ -0,0 +1,256 @@ +pipeline { + agent { + kubernetes { + defaultContainer 'python' + yaml """ +apiVersion: v1 +kind: Pod +metadata: + labels: + atlas.bstein.dev/workload: hermes-agent-image-builder +spec: + automountServiceAccountToken: false + enableServiceLinks: false + restartPolicy: Never + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + nodeSelector: + kubernetes.io/arch: arm64 + node-role.kubernetes.io/worker: "true" + hardware: rpi5 + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: NotIn + values: + - titan-04 + - titan-14 + - titan-18 + - titan-19 + - titan-22 + - titan-24 + imagePullSecrets: + - name: harbor-bstein-robot + containers: + - name: jnlp + image: jenkins/inbound-agent@sha256:8eda4fe2a66bcf6a5e43436d9918fc14c306204dc8fcd75f4e15e0e6e5dc759a + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + runAsNonRoot: true + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault + resources: + requests: + cpu: 25m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + - name: python + image: registry.bstein.dev/bstein/python@sha256:269541d3387baae008df4608ead893dba2b5cdaad1a5a380731a88992d34b808 + command: ["sleep"] + args: ["99d"] + tty: true + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + runAsNonRoot: true + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault + resources: + requests: + cpu: 25m + memory: 64Mi + limits: + cpu: 250m + memory: 256Mi + - name: kaniko + image: gcr.io/kaniko-project/executor@sha256:c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e + command: ["/busybox/sh", "-c"] + args: ["/busybox/sleep 99d"] + tty: true + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + privileged: false + runAsUser: 0 + seccompProfile: + type: RuntimeDefault + resources: + requests: + cpu: 250m + memory: 1Gi + limits: + cpu: "2" + memory: 4Gi +""" + } + } + parameters { + booleanParam( + name: 'PUBLISH_IMAGE', + defaultValue: false, + description: 'Publish the reviewed main revision to Harbor.' + ) + string( + name: 'EXPECTED_SOURCE_REVISION', + defaultValue: '', + description: 'Exact 40-character commit on atlas/titan-iac main.' + ) + string( + name: 'CONFIRM_PUBLISH', + defaultValue: '', + description: 'Enter PUBLISH HERMES AGENT to confirm the release.' + ) + } + environment { + HERMES_IMAGE = 'registry.bstein.dev/bstein/hermes-agent' + } + options { + disableConcurrentBuilds() + buildDiscarder(logRotator(daysToKeepStr: '30', numToKeepStr: '100', artifactDaysToKeepStr: '30', artifactNumToKeepStr: '100')) + skipDefaultCheckout(true) + timeout(time: 90, unit: 'MINUTES') + } + stages { + stage('Checkout reviewed source') { + steps { + checkout scm + } + } + stage('Enforce release boundary') { + steps { + sh ''' + set -eu + mkdir -p build + test "${PUBLISH_IMAGE}" = "true" + test "${CONFIRM_PUBLISH}" = "PUBLISH HERMES AGENT" + case "${EXPECTED_SOURCE_REVISION}" in + *[!0-9a-f]*|'') + echo "EXPECTED_SOURCE_REVISION must be a lowercase full commit" >&2 + exit 2 + ;; + esac + test "${#EXPECTED_SOURCE_REVISION}" -eq 40 + actual_revision="$(git rev-parse HEAD)" + test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}" + test "${actual_revision}" = "$(git rev-parse origin/main)" + test -z "$(git status --porcelain)" + test -f dockerfiles/Dockerfile.hermes-agent + case "${BUILD_NUMBER}" in + ''|0*|*[!0-9]*) + echo "BUILD_NUMBER must be a positive decimal integer" >&2 + exit 2 + ;; + esac + printf '%s\n' \ + "${HERMES_IMAGE}:git-${actual_revision}-build-${BUILD_NUMBER}" \ + > build/hermes-agent.destination + ''' + } + } + stage('Reject replay before publish') { + steps { + withCredentials([usernamePassword( + credentialsId: 'harbor-robot', + usernameVariable: 'HARBOR_USER', + passwordVariable: 'HARBOR_PASSWORD' + )]) { + sh ''' + set -eu + set +x + destination="$(cat build/hermes-agent.destination)" + python3 ci/scripts/hermes_image_release.py assert-absent \ + --source-revision "${EXPECTED_SOURCE_REVISION}" \ + --build-number "${BUILD_NUMBER}" \ + --destination "${destination}" + ''' + } + } + } + stage('Build and publish without a daemon') { + steps { + container('kaniko') { + withCredentials([usernamePassword( + credentialsId: 'harbor-robot', + usernameVariable: 'HARBOR_USER', + passwordVariable: 'HARBOR_PASSWORD' + )]) { + sh '''#!/busybox/sh + set -eu + set +x + config_path=/kaniko/.docker/config.json + destination="$(cat build/hermes-agent.destination)" + umask 077 + auth="$(printf '%s:%s' "${HARBOR_USER}" "${HARBOR_PASSWORD}" | /busybox/base64 | /busybox/tr -d '\n')" + /busybox/mkdir -p /kaniko/.docker + /busybox/printf '{"auths":{"registry.bstein.dev":{"auth":"%s"}}}\n' "${auth}" > "${config_path}" + unset HARBOR_USER HARBOR_PASSWORD auth + trap '/busybox/rm -f "${config_path}"' EXIT HUP INT TERM + /kaniko/executor \ + --context="dir://${WORKSPACE}" \ + --dockerfile="${WORKSPACE}/dockerfiles/Dockerfile.hermes-agent" \ + --destination="${destination}" \ + --digest-file="${WORKSPACE}/build/hermes-agent.digest" \ + --image-name-tag-with-digest-file="${WORKSPACE}/build/hermes-agent.image" \ + --label="org.opencontainers.image.revision=${EXPECTED_SOURCE_REVISION}" \ + --cleanup \ + --push-retry=3 + ''' + } + } + } + } + stage('Render reviewed Flux handoff') { + steps { + withCredentials([usernamePassword( + credentialsId: 'harbor-robot', + usernameVariable: 'HARBOR_USER', + passwordVariable: 'HARBOR_PASSWORD' + )]) { + sh ''' + set -eu + set +x + destination="$(cat build/hermes-agent.destination)" + python3 ci/scripts/hermes_image_release.py render \ + --digest-file build/hermes-agent.digest \ + --image-file build/hermes-agent.image \ + --source-revision "${EXPECTED_SOURCE_REVISION}" \ + --build-number "${BUILD_NUMBER}" \ + --destination "${destination}" \ + --kustomization services/hermes/kustomization.yaml \ + --output-dir build/hermes-agent-release + test -s build/hermes-agent-release/hermes-image-update.patch + test -s build/hermes-agent-release/hermes-agent-image.json + ''' + } + } + } + } + post { + always { + archiveArtifacts( + artifacts: 'build/hermes-agent.digest,build/hermes-agent.image,build/hermes-agent.destination,build/hermes-agent-release/**', + allowEmptyArchive: true, + fingerprint: true + ) + } + cleanup { + container('kaniko') { + sh '''#!/busybox/sh + /busybox/rm -f /kaniko/.docker/config.json + ''' + } + } + } +} diff --git a/ci/scripts/hermes_image_release.py b/ci/scripts/hermes_image_release.py new file mode 100755 index 00000000..7d35242a --- /dev/null +++ b/ci/scripts/hermes_image_release.py @@ -0,0 +1,300 @@ +#!/usr/bin/env python3 +"""Verify and render a reviewable Hermes agent image release.""" + +from __future__ import annotations + +import argparse +import base64 +import difflib +import json +import os +import re +import urllib.error +import urllib.parse +import urllib.request +from pathlib import Path +from typing import Any, Callable + + +DEFAULT_IMAGE = "registry.bstein.dev/bstein/hermes-agent" +HARBOR_API_ORIGIN = "https://registry.bstein.dev/api/v2.0" +HARBOR_PROJECT = "bstein" +HARBOR_REPOSITORY = "hermes-agent" +DIGEST_PATTERN = re.compile(r"^sha256:[0-9a-f]{64}$") +REVISION_PATTERN = re.compile(r"^[0-9a-f]{40}$") +BUILD_PATTERN = re.compile(r"^[1-9][0-9]*$") +DESTINATION_PATTERN = re.compile( + r"^registry\.bstein\.dev/bstein/hermes-agent:" + r"git-([0-9a-f]{40})-build-([1-9][0-9]*)$" +) + + +class _NoRedirect(urllib.request.HTTPRedirectHandler): + """Never send registry credentials to a redirect target.""" + + def redirect_request(self, _request, _file, _code, _message, _headers, _url): + return None + + +def _validated(value: str, pattern: re.Pattern[str], label: str) -> str: + """Return a normalized value when it matches the release contract.""" + normalized = value.strip() + if not pattern.fullmatch(normalized): + raise ValueError(f"invalid {label}: expected {pattern.pattern}") + return normalized + + +def validate_destination( + destination: str, source_revision: str, build_number: str +) -> tuple[str, str]: + """Bind one unique build tag to the reviewed revision and Jenkins build.""" + revision = _validated(source_revision, REVISION_PATTERN, "source revision") + build = _validated(build_number, BUILD_PATTERN, "build number") + match = DESTINATION_PATTERN.fullmatch(destination.strip()) + if not match or match.groups() != (revision, build): + raise ValueError( + "destination must bind the reviewed revision and unique Jenkins build" + ) + return revision, build + + +def validate_kaniko_evidence( + *, digest_text: str, image_text: str, destination: str +) -> str: + """Cross-check both independent Kaniko output files against the destination.""" + digest_lines = digest_text.splitlines() + image_lines = image_text.splitlines() + if len(digest_lines) != 1: + raise ValueError("Kaniko digest evidence must contain exactly one line") + if len(image_lines) != 1: + raise ValueError("Kaniko image evidence must contain exactly one line") + digest = _validated(digest_lines[0], DIGEST_PATTERN, "image digest") + if image_lines[0].strip() != f"{destination}@{digest}": + raise ValueError("Kaniko image evidence does not match destination and digest") + return digest + + +def _registry_request(request: urllib.request.Request, timeout: int) -> Any: + """Make a registry request without following redirects.""" + opener = urllib.request.build_opener(_NoRedirect()) + try: + return opener.open(request, timeout=timeout) + except urllib.error.HTTPError as exc: + return exc + + +def _artifact_response( + destination: str, + *, + username: str, + password: str, + opener: Callable[[urllib.request.Request, int], Any] = _registry_request, +) -> tuple[int, bytes]: + """Read one exact Harbor artifact by tag with bounded response size.""" + match = DESTINATION_PATTERN.fullmatch(destination) + if not match: + raise ValueError("invalid destination") + if not username or not password: + raise RuntimeError("Harbor credentials are empty") + tag = destination.rsplit(":", 1)[1] + encoded_tag = urllib.parse.quote(tag, safe="") + auth = base64.b64encode(f"{username}:{password}".encode()).decode("ascii") + request = urllib.request.Request( + f"{HARBOR_API_ORIGIN}/projects/{HARBOR_PROJECT}/repositories/" + f"{HARBOR_REPOSITORY}/artifacts/{encoded_tag}", + headers={"Accept": "application/json", "Authorization": f"Basic {auth}"}, + method="GET", + ) + with opener(request, 20) as response: + body = response.read(1_048_577) + if len(body) > 1_048_576: + raise RuntimeError("Harbor artifact response exceeded the size limit") + return int(response.status), body + + +def assert_tag_absent( + destination: str, + *, + username: str, + password: str, + opener: Callable[[urllib.request.Request, int], Any] = _registry_request, +) -> None: + """Reject replay before Kaniko can push an already-used immutable identity.""" + status, _body = _artifact_response( + destination, username=username, password=password, opener=opener + ) + if status == 404: + return + if status == 200: + raise RuntimeError("Harbor destination tag already exists; refusing overwrite") + raise RuntimeError(f"Harbor destination preflight returned HTTP {status}") + + +def verify_registry_digest( + destination: str, + digest: str, + *, + username: str, + password: str, + opener: Callable[[urllib.request.Request, int], Any] = _registry_request, +) -> None: + """Verify Harbor independently resolves the pushed tag to Kaniko's digest.""" + digest = _validated(digest, DIGEST_PATTERN, "image digest") + status, body = _artifact_response( + destination, username=username, password=password, opener=opener + ) + if status != 200: + raise RuntimeError(f"Harbor manifest verification returned HTTP {status}") + try: + artifact = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise RuntimeError("Harbor returned invalid artifact JSON") from exc + harbor_digest = str(artifact.get("digest") or "").strip() + if not DIGEST_PATTERN.fullmatch(harbor_digest): + raise RuntimeError("Harbor response omitted a valid artifact digest") + if harbor_digest != digest: + raise RuntimeError("Harbor digest does not match Kaniko evidence") + expected_tag = destination.rsplit(":", 1)[1] + tag_names = { + str(item.get("name")) + for item in artifact.get("tags") or [] + if isinstance(item, dict) and item.get("name") + } + if expected_tag not in tag_names: + raise RuntimeError("Harbor artifact does not contain the expected tag") + + +def render_kustomization(source: str, digest: str, image: str = DEFAULT_IMAGE) -> str: + """Replace exactly one matching Kustomize image digest without reformatting.""" + digest = _validated(digest, DIGEST_PATTERN, "image digest") + lines = source.splitlines(keepends=True) + matches: list[int] = [] + + for index, line in enumerate(lines): + if line.strip() != f"- name: {image}": + continue + name_indent = len(line) - len(line.lstrip()) + for candidate_index in range(index + 1, len(lines)): + candidate = lines[candidate_index] + stripped = candidate.strip() + candidate_indent = len(candidate) - len(candidate.lstrip()) + if stripped.startswith("- name:") and candidate_indent == name_indent: + break + if stripped.startswith("digest:") and candidate_indent > name_indent: + matches.append(candidate_index) + break + + if len(matches) != 1: + raise ValueError( + f"expected exactly one digest for image {image!r}; found {len(matches)}" + ) + + index = matches[0] + newline = "\n" if lines[index].endswith("\n") else "" + prefix = lines[index][: len(lines[index]) - len(lines[index].lstrip())] + lines[index] = f"{prefix}digest: {digest}{newline}" + return "".join(lines) + + +def write_release_artifacts( + *, + digest: str, + source_revision: str, + build_number: str, + destination: str, + kustomization: Path, + output_dir: Path, +) -> dict[str, str]: + """Write a rendered manifest, patch, and credential-free release metadata.""" + digest = _validated(digest, DIGEST_PATTERN, "image digest") + source_revision, build_number = validate_destination( + destination, source_revision, build_number + ) + + source = kustomization.read_text(encoding="utf-8") + rendered = render_kustomization(source, digest) + relative_name = kustomization.name + patch = "".join( + difflib.unified_diff( + source.splitlines(keepends=True), + rendered.splitlines(keepends=True), + fromfile=f"a/services/hermes/{relative_name}", + tofile=f"b/services/hermes/{relative_name}", + ) + ) + if not patch: + raise ValueError("published digest already matches the Flux manifest") + + output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / "hermes-kustomization.yaml").write_text(rendered, encoding="utf-8") + (output_dir / "hermes-image-update.patch").write_text(patch, encoding="utf-8") + metadata = { + "build_number": build_number, + "digest": digest, + "flux_image": f"{DEFAULT_IMAGE}@{digest}", + "image": DEFAULT_IMAGE, + "published_tag": destination, + "source_revision": source_revision, + } + (output_dir / "hermes-agent-image.json").write_text( + json.dumps(metadata, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + return metadata + + +def _credentials() -> tuple[str, str]: + """Read the masked, runtime-only Jenkins credential environment.""" + username = os.environ.get("HARBOR_USER", "") + password = os.environ.get("HARBOR_PASSWORD", "") + if not username or not password: + raise RuntimeError("Harbor credentials are unavailable") + return username, password + + +def _common_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--source-revision", required=True) + parser.add_argument("--build-number", required=True) + parser.add_argument("--destination", required=True) + + +def main() -> int: + """Fail closed around the unique tag, then verify and render after push.""" + parser = argparse.ArgumentParser() + commands = parser.add_subparsers(dest="command", required=True) + absent = commands.add_parser("assert-absent") + _common_arguments(absent) + render = commands.add_parser("render") + _common_arguments(render) + render.add_argument("--digest-file", required=True, type=Path) + render.add_argument("--image-file", required=True, type=Path) + render.add_argument("--kustomization", required=True, type=Path) + render.add_argument("--output-dir", required=True, type=Path) + args = parser.parse_args() + + validate_destination(args.destination, args.source_revision, args.build_number) + username, password = _credentials() + if args.command == "assert-absent": + assert_tag_absent(args.destination, username=username, password=password) + return 0 + + digest = validate_kaniko_evidence( + digest_text=args.digest_file.read_text(encoding="utf-8"), + image_text=args.image_file.read_text(encoding="utf-8"), + destination=args.destination, + ) + verify_registry_digest( + args.destination, digest, username=username, password=password + ) + write_release_artifacts( + digest=digest, + source_revision=args.source_revision, + build_number=args.build_number, + destination=args.destination, + kustomization=args.kustomization, + output_dir=args.output_dir, + ) + return 0 + + +if __name__ == "__main__": # pragma: no cover - exercised through main() + raise SystemExit(main()) diff --git a/clusters/atlas/flux-system/applications/hermes/kustomization.yaml b/clusters/atlas/flux-system/applications/hermes/kustomization.yaml index 3949282b..9a8365ea 100644 --- a/clusters/atlas/flux-system/applications/hermes/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/hermes/kustomization.yaml @@ -60,3 +60,4 @@ spec: - name: keycloak - name: longhorn - name: vault + - name: jenkins diff --git a/clusters/atlas/flux-system/applications/jenkins/kustomization.yaml b/clusters/atlas/flux-system/applications/jenkins/kustomization.yaml index dce79ab7..5d80d148 100644 --- a/clusters/atlas/flux-system/applications/jenkins/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/jenkins/kustomization.yaml @@ -6,10 +6,9 @@ metadata: namespace: flux-system annotations: kustomize.toolkit.fluxcd.io/ssa: IfNotPresent - atlas.bstein.dev/suspend-reason: "CI controller changes are applied only during planned maintenance." spec: interval: 10m - suspend: true + suspend: false path: ./services/jenkins prune: true sourceRef: @@ -18,6 +17,7 @@ spec: targetNamespace: jenkins dependsOn: - name: helm + - name: vault healthChecks: - apiVersion: apps/v1 kind: Deployment diff --git a/clusters/atlas/flux-system/applications/vault/kustomization.yaml b/clusters/atlas/flux-system/applications/vault/kustomization.yaml index 7dd03da8..ade968ed 100644 --- a/clusters/atlas/flux-system/applications/vault/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/vault/kustomization.yaml @@ -16,6 +16,11 @@ spec: targetNamespace: vault prune: true wait: true + healthChecks: + - apiVersion: batch/v1 + kind: Job + name: vault-hermes-jenkins-build-token-seed-1 + namespace: vault dependsOn: - name: longhorn - name: helm diff --git a/services/hermes/NOTES.md b/services/hermes/NOTES.md index c70e528a..7aacb357 100644 --- a/services/hermes/NOTES.md +++ b/services/hermes/NOTES.md @@ -102,6 +102,57 @@ console tails and named artifact contents in its deterministic bundle. A report must say `retained Ariadne evidence` when that fallback is used; it must not pretend direct Jenkins access succeeded. +## Publishing an agent image after review + +The `hermes-agent-image` Jenkins job is the only supported agent image builder. +It runs daemonless Kaniko without a service-account token, host socket, +privileged container, or writable Git credential. It accepts only an exact +40-character revision that is both the checked-out commit and current +`atlas/titan-iac` `main`, so a human must merge the source PR first. + +From agent.hermes, trigger that one fixed job with: + +```sh +jenkins_image_build_trigger.py '' +``` + +The helper has no general Jenkins credential or caller-selectable job name. Its +Vault-projected token is bound by Jenkins only to `hermes-agent-image`. The job +also requires its fixed publish confirmation. Each run claims the unique tag +`git--build-` and refuses to overwrite an +existing tag. It cross-checks Kaniko's digest and tagged-image evidence, then +independently reads the pushed tag from Harbor before archiving a JSON record +and Flux digest patch. Apply that patch on a new branch and submit it for human +review; the build never changes Git, reconciles Flux, or deploys by itself. +The helper posts only to the fixed HTTPS Build Token Root endpoint and accepts +only its real queue responses: HTTP 201 or a non-followed HTTP 303 with an exact +same-origin `/queue/item//` location. An unauthenticated request is +denied, and the token cannot select, configure, read, or administer another +Jenkins job. The pipeline independently rejects any commit that is not the +current `origin/main`, preserving the human merge/review boundary. + +The tracked rollout order is deliberate: the revisioned Vault seed Job must +complete before the Vault Kustomization becomes Ready, Jenkins depends on +Vault, and Hermes depends on Jenkins. The seeder uses KV-v2 CAS create-only +semantics. It never changes an existing token, never fills a missing field in +an existing secret, and fails closed when it cannot distinguish absence from a +read error. The recurring Vault configuration job verifies the same invariant. + +Rotate this fixed-job token only as a coordinated operator action: + +1. Stop new `hermes-agent-image` triggers and wait for its queue and executor to + drain. +2. Read the current KV-v2 metadata version for + `kv/atlas/hermes/developer-jenkins`. +3. Generate a fresh value from Vault and patch only `build_token` with + `vault kv patch -cas=`. Keep the value in a mode-0600 + temporary file or standard input, never a command argument or log. +4. Roll Jenkins first and wait until it is Ready, then roll `hermes-agent` and + wait until it is Ready. Do not resume triggers between those two consumers. +5. Run one reviewed-main canary. If rollback is required, repeat the CAS patch + with the previous value as another coordinated rotation; never delete the + secret to make the seeder recreate it. + ## The actual supervised triage algorithm 1. Classify the request as test/build triage, service health, or alert tuning. diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml index 845eb6f9..de202788 100644 --- a/services/hermes/agent-deployment.yaml +++ b/services/hermes/agent-deployment.yaml @@ -56,6 +56,11 @@ spec: {{- with secret "kv/data/atlas/hermes/developer-gitea" -}} {{ .Data.data.username }} {{- end }} + vault.hashicorp.com/agent-inject-secret-jenkins-image-build-token: kv/data/atlas/hermes/developer-jenkins + vault.hashicorp.com/agent-inject-template-jenkins-image-build-token: | + {{- with secret "kv/data/atlas/hermes/developer-jenkins" -}} + {{ .Data.data.build_token }} + {{- end }} vault.hashicorp.com/agent-inject-secret-node-ssh-private-key: kv/data/atlas/hermes/developer-ssh vault.hashicorp.com/agent-inject-template-node-ssh-private-key: | {{- with secret "kv/data/atlas/hermes/developer-ssh" -}} diff --git a/services/hermes/kustomization.yaml b/services/hermes/kustomization.yaml index 3b9c2345..6009cd7b 100644 --- a/services/hermes/kustomization.yaml +++ b/services/hermes/kustomization.yaml @@ -77,6 +77,7 @@ configMapGenerator: - image_broker.py=scripts/image_broker.py - install_agent_tools.sh=scripts/install_agent_tools.sh - jenkins_build_evidence.py=scripts/jenkins_build_evidence.py + - jenkins_image_build_trigger.py=scripts/jenkins_image_build_trigger.py - kanban_status_recovery.py=scripts/kanban_status_recovery.py - migrate_herdr_state.py=scripts/migrate_herdr_state.py - migrate_api_session_lineage.py=scripts/migrate_api_session_lineage.py diff --git a/services/hermes/scripts/jenkins_image_build_trigger.py b/services/hermes/scripts/jenkins_image_build_trigger.py new file mode 100755 index 00000000..6cd91ed8 --- /dev/null +++ b/services/hermes/scripts/jenkins_image_build_trigger.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +"""Trigger only the reviewed-main Hermes agent image release job.""" + +from __future__ import annotations + +import argparse +import json +import re +import urllib.error +import urllib.parse +import urllib.request +from pathlib import Path + + +JENKINS_ORIGIN = "https://ci.bstein.dev" +JENKINS_BUILD_URL = f"{JENKINS_ORIGIN}/buildByToken/buildWithParameters" +JOB_NAME = "hermes-agent-image" +TOKEN_FILE = Path("/runtime-access/jenkins-image-build-token") +REVISION_PATTERN = re.compile(r"^[0-9a-f]{40}$") +QUEUE_PATH_PATTERN = re.compile(r"^/queue/item/[0-9]+/?$") + + +class _NoRedirect(urllib.request.HTTPRedirectHandler): + """Keep a queued-build redirect from becoming an unauthorized job read.""" + + def redirect_request(self, _request, _file, _code, _message, _headers, _url): + return None + + +def _open_without_redirect(request: urllib.request.Request, timeout: int): + """Return the Build Token Root response, including its expected HTTP 303.""" + opener = urllib.request.build_opener(_NoRedirect()) + try: + return opener.open(request, timeout=timeout) + except urllib.error.HTTPError as exc: + if exc.code == 303: + return exc + raise + + +def trigger_build( + revision: str, + *, + token_file: Path = TOKEN_FILE, + opener=_open_without_redirect, +) -> dict[str, str | int]: + """Post the fixed job parameters using its job-scoped build token.""" + revision = revision.strip() + if not REVISION_PATTERN.fullmatch(revision): + raise ValueError("revision must be a lowercase full 40-character commit") + token = token_file.read_text(encoding="utf-8").strip() + if not token: + raise RuntimeError("Jenkins image-build token is empty") + payload = urllib.parse.urlencode( + { + "job": JOB_NAME, + "token": token, + "PUBLISH_IMAGE": "true", + "EXPECTED_SOURCE_REVISION": revision, + "CONFIRM_PUBLISH": "PUBLISH HERMES AGENT", + } + ).encode("utf-8") + request = urllib.request.Request( + JENKINS_BUILD_URL, + data=payload, + headers={"Content-Type": "application/x-www-form-urlencoded"}, + method="POST", + ) + with opener(request, timeout=20) as response: + status = int(response.status) + location = response.headers.get("Location", "") + if status not in {201, 303}: + raise RuntimeError(f"Jenkins trigger returned HTTP {status}") + if not location: + raise RuntimeError("Jenkins trigger omitted the queue Location") + queue_url = urllib.parse.urljoin(f"{JENKINS_ORIGIN}/", location) + parsed_queue = urllib.parse.urlsplit(queue_url) + expected_origin = urllib.parse.urlsplit(JENKINS_ORIGIN) + if ( + parsed_queue.scheme != expected_origin.scheme + or parsed_queue.netloc != expected_origin.netloc + or parsed_queue.query + or parsed_queue.fragment + or not QUEUE_PATH_PATTERN.fullmatch(parsed_queue.path) + ): + raise RuntimeError("Jenkins returned an invalid queue Location") + # Never return the submitted URL: its form body contains the job token. + queue_path = parsed_queue.path + return { + "job": JOB_NAME, + "queue_path": queue_path, + "source_revision": revision, + "status": status, + } + + +def main() -> int: + """Validate one revision, trigger the bounded job, and print safe metadata.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("revision", help="reviewed full commit currently on main") + args = parser.parse_args() + try: + result = trigger_build(args.revision) + except (OSError, ValueError, RuntimeError, urllib.error.URLError) as exc: + print(json.dumps({"error": str(exc)}, sort_keys=True)) + return 1 + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": # pragma: no cover - exercised through main() + raise SystemExit(main()) diff --git a/services/hermes/scripts/stage_runtime_access.py b/services/hermes/scripts/stage_runtime_access.py index f4d73160..e8276a5b 100644 --- a/services/hermes/scripts/stage_runtime_access.py +++ b/services/hermes/scripts/stage_runtime_access.py @@ -95,6 +95,7 @@ def stage_agent() -> None: "chat-relay-key", "gitea-token", "gitea-username", + "jenkins-image-build-token", "node-ssh-private-key", "node-ssh-config", "node-ssh-known-hosts", diff --git a/services/hermes/switchyard-deployment.yaml b/services/hermes/switchyard-deployment.yaml index 48143682..456d0a1a 100644 --- a/services/hermes/switchyard-deployment.yaml +++ b/services/hermes/switchyard-deployment.yaml @@ -27,7 +27,7 @@ spec: prometheus.io/port: "9005" prometheus.io/path: /metrics vault.hashicorp.com/agent-inject: "true" - vault.hashicorp.com/role: hermes-agent + vault.hashicorp.com/role: hermes-switchyard vault.hashicorp.com/agent-inject-secret-relay-key: kv/data/atlas/hermes/chat-telegram vault.hashicorp.com/agent-inject-template-relay-key: | {{- with secret "kv/data/atlas/hermes/chat-telegram" -}} diff --git a/services/jenkins/configmap-jcasc.yaml b/services/jenkins/configmap-jcasc.yaml index e9aa57b9..2a1e81eb 100644 --- a/services/jenkins/configmap-jcasc.yaml +++ b/services/jenkins/configmap-jcasc.yaml @@ -652,6 +652,25 @@ data: } } } + pipelineJob('hermes-agent-image') { + disabled(false) + description('Human-gated, daemonless Kaniko build for the reviewed atlas/titan-iac main revision. Publishes a content-addressed Hermes agent image and archives a Flux digest patch; it never mutates Git or deploys.') + authenticationToken(System.getenv('HERMES_AGENT_IMAGE_BUILD_TOKEN')) + definition { + cpsScm { + scm { + git { + remote { + url('https://scm.bstein.dev/atlas/titan-iac.git') + credentials('gitea-pat') + } + branches('*/main') + } + } + scriptPath('ci/Jenkinsfile.hermes-agent-image') + } + } + } multibranchPipelineJob('titan-iac-quality-gate') { branchSources { branchSource { diff --git a/services/jenkins/configmap-plugins.yaml b/services/jenkins/configmap-plugins.yaml index 049bc1bd..ea4b450d 100644 --- a/services/jenkins/configmap-plugins.yaml +++ b/services/jenkins/configmap-plugins.yaml @@ -20,6 +20,7 @@ data: gitea:268.v75e47974c01d gitea-checks:603.621.vc708da_fb_371d multibranch-scan-webhook-trigger:1.0.11 + build-token-root:365.v717f8685a_09e # Structured test evidence. Without junit the `junit` step throws # NoSuchMethodError, jenkins.failed_tests is always empty, and triage # has only raw console text to work from. Pinned to the newest release diff --git a/services/jenkins/deployment.yaml b/services/jenkins/deployment.yaml index 1fa22374..46109da7 100644 --- a/services/jenkins/deployment.yaml +++ b/services/jenkins/deployment.yaml @@ -65,6 +65,9 @@ spec: ARIADNE_JENKINS_API_USER={{ .Data.data.username }} ARIADNE_JENKINS_API_TOKEN={{ .Data.data.token }} {{ end }} + {{ with secret "kv/data/atlas/hermes/developer-jenkins" }} + HERMES_AGENT_IMAGE_BUILD_TOKEN={{ .Data.data.build_token }} + {{ end }} bstein.dev/restarted-at: "2026-05-20T09:40:31Z" spec: serviceAccountName: jenkins diff --git a/services/jenkins/secretproviderclass.yaml b/services/jenkins/secretproviderclass.yaml index a9d9dd50..148f193f 100644 --- a/services/jenkins/secretproviderclass.yaml +++ b/services/jenkins/secretproviderclass.yaml @@ -8,7 +8,7 @@ spec: provider: vault parameters: vaultAddress: "http://vault.vault.svc.cluster.local:8200" - roleName: "jenkins" + roleName: "jenkins-vault-sync" objects: | - objectName: "harbor-pull__dockerconfigjson" secretPath: "kv/data/atlas/shared/harbor-pull" diff --git a/services/vault/hermes-jenkins-build-token-seed-job.yaml b/services/vault/hermes-jenkins-build-token-seed-job.yaml new file mode 100644 index 00000000..5c4432e3 --- /dev/null +++ b/services/vault/hermes-jenkins-build-token-seed-job.yaml @@ -0,0 +1,47 @@ +# services/vault/hermes-jenkins-build-token-seed-job.yaml +# Revisioned prerequisite: seed the job-scoped token before Jenkins/Hermes roll. +apiVersion: batch/v1 +kind: Job +metadata: + name: vault-hermes-jenkins-build-token-seed-1 + namespace: vault +spec: + backoffLimit: 2 + template: + spec: + serviceAccountName: vault-admin + restartPolicy: Never + nodeSelector: + hardware: rpi5 + kubernetes.io/arch: arm64 + node-role.kubernetes.io/worker: "true" + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: NotIn + values: [titan-04, titan-14, titan-18, titan-19, titan-24] + containers: + - name: seed + image: hashicorp/vault:1.21.4 + imagePullPolicy: IfNotPresent + command: [sh, /scripts/vault_hermes_jenkins_build_token_ensure.sh] + env: + - name: VAULT_ADDR + value: http://vault.vault.svc.cluster.local:8200 + - name: VAULT_K8S_ROLE + value: vault-admin + volumeMounts: + - name: scripts + mountPath: /scripts + readOnly: true + resources: + requests: {cpu: 25m, memory: 32Mi} + limits: {cpu: 250m, memory: 128Mi} + volumes: + - name: scripts + configMap: + name: vault-k8s-auth-config-script + defaultMode: 0555 diff --git a/services/vault/kustomization.yaml b/services/vault/kustomization.yaml index d370373c..a6e72123 100644 --- a/services/vault/kustomization.yaml +++ b/services/vault/kustomization.yaml @@ -12,6 +12,7 @@ resources: - statefulset.yaml - k8s-auth-config-cronjob.yaml - hermes-auth-role-bootstrap-job.yaml + - hermes-jenkins-build-token-seed-job.yaml - oidc-config-cronjob.yaml - service.yaml - certificate.yaml @@ -27,6 +28,7 @@ configMapGenerator: - name: vault-k8s-auth-config-script files: - vault_k8s_auth_configure.sh=scripts/vault_k8s_auth_configure.sh + - vault_hermes_jenkins_build_token_ensure.sh=scripts/vault_hermes_jenkins_build_token_ensure.sh - name: vault-entrypoint files: - vault-entrypoint.sh=scripts/vault-entrypoint.sh diff --git a/services/vault/scripts/vault_hermes_jenkins_build_token_ensure.sh b/services/vault/scripts/vault_hermes_jenkins_build_token_ensure.sh new file mode 100755 index 00000000..1d9ddbf4 --- /dev/null +++ b/services/vault/scripts/vault_hermes_jenkins_build_token_ensure.sh @@ -0,0 +1,153 @@ +#!/usr/bin/env sh +set -eu + +secret_path="kv/atlas/hermes/developer-jenkins" +jwt_file="${VAULT_K8S_JWT_FILE:-/var/run/secrets/kubernetes.io/serviceaccount/token}" +vault_role="${VAULT_K8S_ROLE:-vault-admin}" + +log() { printf '[hermes-jenkins-token] %s\n' "$*" >&2; } + +retry_command() { + attempt=1 + while [ "${attempt}" -le 5 ]; do + set +e + command_output="$("$@" 2>&1)" + command_status=$? + set -e + if [ "${command_status}" -eq 0 ]; then + printf '%s' "${command_output}" + return 0 + fi + if [ "${attempt}" -lt 5 ]; then + sleep "${attempt}" + fi + attempt=$((attempt + 1)) + done + return "${command_status}" +} + +ensure_token() { + if [ -n "${VAULT_TOKEN:-}" ]; then + return + fi + jwt="$(cat "${jwt_file}")" + VAULT_TOKEN="$(retry_command vault write -field=token \ + auth/kubernetes/login role="${vault_role}" jwt="${jwt}")" || { + log "Vault Kubernetes login failed after retries" + exit 1 + } + unset jwt + if [ -z "${VAULT_TOKEN}" ]; then + log "Vault Kubernetes login returned an empty token" + exit 1 + fi + export VAULT_TOKEN +} + +# Return 0 when the field exists, 10 when the secret is absent, 11 when the +# secret exists without the field, and 12 for a persistent read error. +read_build_token() { + attempt=1 + while [ "${attempt}" -le 5 ]; do + set +e + secret_error="$(vault kv get "${secret_path}" 2>&1 >/dev/null)" + secret_status=$? + set -e + if [ "${secret_status}" -eq 0 ]; then + set +e + build_token="$(vault kv get -field=build_token "${secret_path}" 2>&1)" + field_status=$? + set -e + if [ "${field_status}" -eq 0 ] && [ -n "${build_token}" ]; then + return 0 + fi + if printf '%s' "${build_token}" | grep -q 'No value found'; then + unset build_token + return 11 + fi + if [ "${field_status}" -eq 0 ]; then + unset build_token + return 11 + fi + elif printf '%s' "${secret_error}" | grep -Eq 'Code: 404|No value found at'; then + return 10 + fi + if [ "${attempt}" -lt 5 ]; then + sleep "${attempt}" + fi + attempt=$((attempt + 1)) + done + unset secret_error build_token + return 12 +} + +ensure_token + +if read_build_token; then + read_status=0 +else + read_status=$? +fi +case "${read_status}" in + 0) + unset build_token + log "job-scoped token already present; no write performed" + exit 0 + ;; + 11) + log "secret exists without build_token; refusing to overwrite existing fields" + exit 1 + ;; + 12) + log "secret read failed after retries; refusing to seed" + exit 1 + ;; + 10) ;; + *) + log "unexpected secret read state ${read_status}" + exit 1 + ;; +esac + +new_token="$(retry_command vault write -field=random_bytes \ + sys/tools/random/32 format=hex)" || { + log "Vault random token generation failed after retries" + exit 1 + } +if [ -z "${new_token}" ]; then + log "Vault returned an empty random token" + exit 1 +fi + +set +e +create_error="$(printf '%s' "${new_token}" | \ + vault kv put -cas=0 "${secret_path}" build_token=- 2>&1 >/dev/null)" +create_status=$? +set -e +unset new_token +if [ "${create_status}" -eq 0 ]; then + log "job-scoped token created with KV-v2 CAS create-only semantics" + exit 0 +fi + +# A simultaneous revisioned Job/Cron run may win the create. Accept that race +# only if a fresh retried read finds the complete field; never rotate it. +if printf '%s' "${create_error}" | grep -qi 'check-and-set'; then + unset create_error + if read_build_token; then + read_status=0 + else + read_status=$? + fi + if [ "${read_status}" -eq 0 ]; then + unset build_token + log "another seeder won CAS; existing token preserved" + exit 0 + fi + log "CAS race did not produce a readable build_token; refusing to continue" + exit 1 +fi + +unset create_error +log "KV-v2 CAS create failed; refusing to retry a write" +exit 1 diff --git a/services/vault/scripts/vault_k8s_auth_configure.sh b/services/vault/scripts/vault_k8s_auth_configure.sh index 73572e62..61291c39 100644 --- a/services/vault/scripts/vault_k8s_auth_configure.sh +++ b/services/vault/scripts/vault_k8s_auth_configure.sh @@ -239,8 +239,10 @@ write_policy_and_role "nextcloud" "nextcloud" "nextcloud-vault" \ "nextcloud/* shared/keycloak-admin shared/postmark-relay" "" write_policy_and_role "comms" "comms" "comms-vault,atlasbot" \ "comms/* shared/chat-ai-keys-runtime shared/harbor-pull" "" -write_policy_and_role "jenkins" "jenkins" "jenkins,jenkins-vault-sync" \ - "jenkins/* shared/harbor-pull quality/sonarqube-oidc" "hermes/developer-jenkins" +write_policy_and_role "jenkins" "jenkins" "jenkins" \ + "jenkins/* shared/harbor-pull quality/sonarqube-oidc hermes/developer-jenkins" "" +write_policy_and_role "jenkins-vault-sync" "jenkins" "jenkins-vault-sync" \ + "shared/harbor-pull" "" write_policy_and_role "monitoring" "monitoring" "monitoring-vault-sync" \ "monitoring/* shared/postmark-relay shared/harbor-pull" "" write_policy_and_role "logging" "logging" "logging-vault-sync" \ @@ -255,8 +257,10 @@ write_policy_and_role "game-stream" "game-stream" "game-stream-vault" \ "game-stream/*" "" write_policy_and_role "hermes" "hermes" "hermes-vault,hermes-triage" \ "hermes/triage-oidc hermes/agent-tokens hermes/triage-api" "" -write_policy_and_role "hermes-agent" "hermes" "hermes-agent,hermes-switchyard" \ +write_policy_and_role "hermes-agent" "hermes" "hermes-agent" \ "hermes/agent-oidc hermes/agent-tokens hermes/chat-telegram hermes/developer-keycloak hermes/developer-gitea hermes/developer-harbor hermes/developer-jenkins hermes/developer-ssh" "" +write_policy_and_role "hermes-switchyard" "hermes" "hermes-switchyard" \ + "hermes/chat-telegram" "" write_policy_and_role "hermes-credential-sync" "hermes" "hermes-agent" \ "" "hermes/agent-tokens" write_policy_and_role "hermes-node-ssh" "hermes" "hermes-node-ssh-access" \ @@ -304,6 +308,10 @@ write_policy_and_role "postgres" "postgres" "postgres-vault" \ write_policy_and_role "vault" "vault" "vault" \ "vault/*" "" +# This verifier is also run by a revisioned prerequisite Job. It creates only +# when the entire KV-v2 secret is absent and never rotates an existing value. +/scripts/vault_hermes_jenkins_build_token_ensure.sh + write_policy_and_role "sso-secrets" "sso" "mas-secrets-ensure" \ "shared/keycloak-admin shared/postmark-relay maintenance/metis-ssh-keys" \ "harbor/harbor-oidc vault/vault-oidc-config comms/synapse-oidc logging/oauth2-proxy-logs-oidc finance/actual-oidc maintenance/metis-oidc maintenance/soteria-oidc maintenance/metis-ssh-keys veles/veles-oidc cassandra/cassandra-oidc gitea/gitea-veles-oidc gitea/gitea-cassandra-oidc hermes/chat-oidc hermes/chat-telegram hermes/agent-oidc hermes/triage-oidc hermes/developer-keycloak" \ diff --git a/testing/quality_contract.json b/testing/quality_contract.json index e8793d84..e2797e05 100644 --- a/testing/quality_contract.json +++ b/testing/quality_contract.json @@ -14,6 +14,7 @@ } ], "managed_modules": [ + "ci/scripts/hermes_image_release.py", "ci/scripts/publish_test_metrics.py", "ci/scripts/publish_test_metrics_quality.py", "ci/scripts/semgrep_report.py", @@ -35,6 +36,7 @@ "testing/tests/test_quality_gate.py" ], "lint_paths": [ + "ci/scripts/hermes_image_release.py", "ci/scripts/publish_test_metrics.py", "ci/scripts/publish_test_metrics_quality.py", "ci/scripts/semgrep_report.py", @@ -168,6 +170,7 @@ "coverage": { "minimum_percent": 95.0, "tracked_files": [ + "ci/scripts/hermes_image_release.py", "ci/scripts/publish_test_metrics.py", "ci/scripts/publish_test_metrics_quality.py", "ci/scripts/semgrep_report.py", diff --git a/testing/tests/test_hermes_image_builder.py b/testing/tests/test_hermes_image_builder.py new file mode 100644 index 00000000..0459f699 --- /dev/null +++ b/testing/tests/test_hermes_image_builder.py @@ -0,0 +1,415 @@ +"""Safety and artifact contracts for the Hermes agent image release lane.""" + +from __future__ import annotations + +import importlib.util +import io +import json +import urllib.parse +from pathlib import Path + +import pytest +import yaml + + +REPO_ROOT = Path(__file__).resolve().parents[2] +PIPELINE_PATH = REPO_ROOT / "ci/Jenkinsfile.hermes-agent-image" +RELEASE_SCRIPT = REPO_ROOT / "ci/scripts/hermes_image_release.py" +TRIGGER_SCRIPT = REPO_ROOT / "services/hermes/scripts/jenkins_image_build_trigger.py" + + +def _load_release_module(): + spec = importlib.util.spec_from_file_location( + "hermes_image_release", RELEASE_SCRIPT + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _load_trigger_module(): + spec = importlib.util.spec_from_file_location( + "jenkins_image_build_trigger", TRIGGER_SCRIPT + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _pod_spec() -> dict: + source = PIPELINE_PATH.read_text(encoding="utf-8") + pod_yaml = source.split('yaml """', 1)[1].split('"""', 1)[0] + return yaml.safe_load(pod_yaml)["spec"] + + +def test_builder_pod_is_daemonless_and_kernel_unprivileged() -> None: + """The builder must not gain host, daemon, service-token, or Linux privileges.""" + source = PIPELINE_PATH.read_text(encoding="utf-8") + spec = _pod_spec() + assert spec["automountServiceAccountToken"] is False + assert spec["enableServiceLinks"] is False + assert "hostPath" not in source + assert "docker.sock" not in source + assert "tcp://" not in source + assert "buildkitd" not in source.lower() + assert "dind" not in source.lower() + + containers = {item["name"]: item for item in spec["containers"]} + kaniko = containers["kaniko"] + assert kaniko["image"] == ( + "gcr.io/kaniko-project/executor@sha256:" + "c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e" + ) + for container in containers.values(): + security = container["securityContext"] + assert security["allowPrivilegeEscalation"] is False + assert security["capabilities"]["drop"] == ["ALL"] + assert security["seccompProfile"]["type"] == "RuntimeDefault" + assert security.get("privileged", False) is False + + +def test_builder_is_restricted_to_healthy_rpi5_capacity() -> None: + """Disposable builds must stay off unhealthy and reserved Atlas nodes.""" + spec = _pod_spec() + assert spec["nodeSelector"]["hardware"] == "rpi5" + expressions = spec["affinity"]["nodeAffinity"][ + "requiredDuringSchedulingIgnoredDuringExecution" + ]["nodeSelectorTerms"][0]["matchExpressions"] + host_rule = next( + rule for rule in expressions if rule["key"] == "kubernetes.io/hostname" + ) + assert host_rule["operator"] == "NotIn" + assert set(host_rule["values"]) >= { + "titan-04", + "titan-14", + "titan-18", + "titan-19", + "titan-22", + "titan-24", + } + + +def test_pipeline_requires_reviewed_main_and_runtime_credentials() -> None: + """Publishing requires explicit confirmation and a reviewed main commit.""" + source = PIPELINE_PATH.read_text(encoding="utf-8") + assert 'test "${PUBLISH_IMAGE}" = "true"' in source + assert 'test "${CONFIRM_PUBLISH}" = "PUBLISH HERMES AGENT"' in source + assert "git rev-parse origin/main" in source + assert "credentialsId: 'harbor-robot'" in source + assert "set +x" in source + assert "umask 077" in source + assert "unset HARBOR_USER HARBOR_PASSWORD auth" in source + assert "/busybox/rm -f /kaniko/.docker/config.json" in source + assert "--digest-file=" in source + assert "--image-name-tag-with-digest-file=" in source + assert "--destination=" in source + assert "assert-absent" in source + assert "git-${actual_revision}-build-${BUILD_NUMBER}" in source + + +def test_jenkins_job_is_manual_and_reads_pipeline_from_main() -> None: + """JCasC must not publish unreviewed branch contents or poll automatically.""" + config = yaml.safe_load( + (REPO_ROOT / "services/jenkins/configmap-jcasc.yaml").read_text( + encoding="utf-8" + ) + ) + jobs = config["data"]["jobs.yaml"] + block = jobs.split("pipelineJob('hermes-agent-image')", 1)[1].split( + "pipelineJob(", 1 + )[0] + assert "branches('*/main')" in block + assert "scriptPath('ci/Jenkinsfile.hermes-agent-image')" in block + assert ( + "authenticationToken(System.getenv('HERMES_AGENT_IMAGE_BUILD_TOKEN'))" in block + ) + assert "pipelineTriggers" not in block + assert "scmTrigger" not in block + + +def test_agent_trigger_is_limited_to_the_image_job(tmp_path: Path) -> None: + """Agent Hermes gets one job token, fixed parameters, and no Jenkins admin API.""" + module = _load_trigger_module() + revision = "a" * 40 + token_path = tmp_path / "token" + token_path.write_text("private-job-token\n", encoding="utf-8") + captured = {} + + class Response(io.BytesIO): + status = 201 + headers = {"Location": "https://ci.bstein.dev/queue/item/42/"} + + def __enter__(self): + return self + + def __exit__(self, *_args): + self.close() + + def opener(request, timeout): + captured["request"] = request + captured["timeout"] = timeout + return Response(b"") + + result = module.trigger_build(revision, token_file=token_path, opener=opener) + request = captured["request"] + fields = urllib.parse.parse_qs(request.data.decode("utf-8")) + assert request.full_url == module.JENKINS_BUILD_URL + assert fields == { + "CONFIRM_PUBLISH": ["PUBLISH HERMES AGENT"], + "EXPECTED_SOURCE_REVISION": [revision], + "PUBLISH_IMAGE": ["true"], + "job": ["hermes-agent-image"], + "token": ["private-job-token"], + } + assert captured["timeout"] == 20 + assert "private-job-token" not in json.dumps(result) + assert result["source_revision"] == revision + + +def test_agent_trigger_rejects_unsafe_revision_and_empty_token(tmp_path: Path) -> None: + """No user-controlled job, URL, or abbreviated revision reaches Jenkins.""" + module = _load_trigger_module() + token_path = tmp_path / "token" + token_path.write_text("token\n", encoding="utf-8") + with pytest.raises(ValueError): + module.trigger_build("main", token_file=token_path) + token_path.write_text("\n", encoding="utf-8") + with pytest.raises(RuntimeError, match="empty"): + module.trigger_build("a" * 40, token_file=token_path) + + +def test_agent_trigger_accepts_existing_queue_redirect(tmp_path: Path) -> None: + """HTTP 303 means the exact release is already queued, not a trigger failure.""" + module = _load_trigger_module() + token_path = tmp_path / "token" + token_path.write_text("token\n", encoding="utf-8") + + class Response(io.BytesIO): + status = 303 + headers = {"Location": "https://ci.bstein.dev/queue/item/7/"} + + def __enter__(self): + return self + + def __exit__(self, *_args): + self.close() + + result = module.trigger_build( + "e" * 40, token_file=token_path, opener=lambda *_args, **_kwargs: Response() + ) + assert result["status"] == 303 + assert result["queue_path"] == "/queue/item/7/" + + +def test_job_token_is_generated_and_injected_only_at_runtime() -> None: + """The fixed-job credential stays in Vault and pod-lifetime memory.""" + plugins = (REPO_ROOT / "services/jenkins/configmap-plugins.yaml").read_text( + encoding="utf-8" + ) + vault = ( + REPO_ROOT / "services/vault/scripts/vault_k8s_auth_configure.sh" + ).read_text(encoding="utf-8") + seeder = ( + REPO_ROOT / "services/vault/scripts/vault_hermes_jenkins_build_token_ensure.sh" + ).read_text(encoding="utf-8") + jenkins = (REPO_ROOT / "services/jenkins/deployment.yaml").read_text( + encoding="utf-8" + ) + agent = (REPO_ROOT / "services/hermes/agent-deployment.yaml").read_text( + encoding="utf-8" + ) + stage = (REPO_ROOT / "services/hermes/scripts/stage_runtime_access.py").read_text( + encoding="utf-8" + ) + assert "build-token-root:365.v717f8685a_09e" in plugins + assert "vault_hermes_jenkins_build_token_ensure.sh" in vault + assert "sys/tools/random/32 format=hex" in seeder + assert "kv put -cas=0" in seeder + assert "kv/atlas/hermes/developer-jenkins" in seeder + assert "HERMES_AGENT_IMAGE_BUILD_TOKEN={{ .Data.data.build_token }}" in jenkins + assert "agent-inject-secret-jenkins-image-build-token" in agent + assert '"jenkins-image-build-token"' in stage + + +def test_release_renderer_preserves_manifest_and_emits_safe_artifacts( + tmp_path: Path, +) -> None: + """The release artifact is exact, reviewable, and contains no credentials.""" + module = _load_release_module() + old_digest = "sha256:" + "1" * 64 + new_digest = "sha256:" + "2" * 64 + revision = "a" * 40 + build_number = "17" + manifest = tmp_path / "kustomization.yaml" + manifest.write_text( + "apiVersion: kustomize.config.k8s.io/v1beta1\n" + "kind: Kustomization\n" + "images:\n" + f" - name: {module.DEFAULT_IMAGE}\n" + f" digest: {old_digest}\n", + encoding="utf-8", + ) + output = tmp_path / "out" + metadata = module.write_release_artifacts( + digest=f"{new_digest}\n", + source_revision=revision, + build_number=build_number, + destination=f"{module.DEFAULT_IMAGE}:git-{revision}-build-{build_number}", + kustomization=manifest, + output_dir=output, + ) + + assert manifest.read_text(encoding="utf-8").endswith(f"{old_digest}\n") + assert ( + (output / "hermes-kustomization.yaml") + .read_text(encoding="utf-8") + .endswith(f"{new_digest}\n") + ) + patch = (output / "hermes-image-update.patch").read_text(encoding="utf-8") + assert f"- digest: {old_digest}" in patch + assert f"+ digest: {new_digest}" in patch + assert ( + json.loads((output / "hermes-agent-image.json").read_text(encoding="utf-8")) + == metadata + ) + assert set(metadata) == { + "digest", + "build_number", + "flux_image", + "image", + "published_tag", + "source_revision", + } + + +@pytest.mark.parametrize( + ("digest", "revision", "destination"), + [ + ( + "latest", + "a" * 40, + "registry.bstein.dev/bstein/hermes-agent:git-" + "a" * 40 + "-build-1", + ), + ( + "sha256:" + "1" * 64, + "short", + "registry.bstein.dev/bstein/hermes-agent:git-short-build-1", + ), + ( + "sha256:" + "1" * 64, + "a" * 40, + "registry.bstein.dev/bstein/hermes-agent:latest", + ), + ], +) +def test_release_renderer_rejects_unpinned_inputs( + tmp_path: Path, digest: str, revision: str, destination: str +) -> None: + """Only exact digests and immutable source-derived tags are accepted.""" + module = _load_release_module() + manifest = tmp_path / "kustomization.yaml" + manifest.write_text( + "images:\n" + f" - name: {module.DEFAULT_IMAGE}\n" + " digest: sha256:" + "0" * 64 + "\n", + encoding="utf-8", + ) + with pytest.raises(ValueError): + module.write_release_artifacts( + digest=digest, + source_revision=revision, + build_number="1", + destination=destination, + kustomization=manifest, + output_dir=tmp_path / "out", + ) + + +def test_release_renderer_fails_closed_on_manifest_drift(tmp_path: Path) -> None: + """Missing, duplicate, or already-current image entries require human review.""" + module = _load_release_module() + digest = "sha256:" + "f" * 64 + missing = "images:\n - name: example.invalid/other\n digest: " + digest + "\n" + with pytest.raises(ValueError, match="found 0"): + module.render_kustomization(missing, digest) + + duplicate = ( + "images:\n" + f" - name: {module.DEFAULT_IMAGE}\n digest: {digest}\n" + f" - name: {module.DEFAULT_IMAGE}\n digest: {digest}\n" + ) + with pytest.raises(ValueError, match="found 2"): + module.render_kustomization(duplicate, digest) + + manifest = tmp_path / "kustomization.yaml" + manifest.write_text( + f"images:\n - name: {module.DEFAULT_IMAGE}\n digest: {digest}\n", + encoding="utf-8", + ) + revision = "b" * 40 + with pytest.raises(ValueError, match="already matches"): + module.write_release_artifacts( + digest=digest, + source_revision=revision, + build_number="1", + destination=f"{module.DEFAULT_IMAGE}:git-{revision}-build-1", + kustomization=manifest, + output_dir=tmp_path / "out", + ) + + +def test_release_cli_reads_digest_file( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The pipeline CLI uses the Kaniko digest file as its sole digest input.""" + module = _load_release_module() + digest = "sha256:" + "c" * 64 + revision = "d" * 40 + manifest = tmp_path / "kustomization.yaml" + manifest.write_text( + f"images:\n - name: {module.DEFAULT_IMAGE}\n digest: sha256:" + + "0" * 64 + + "\n", + encoding="utf-8", + ) + digest_file = tmp_path / "digest" + digest_file.write_text(digest + "\n", encoding="utf-8") + destination = f"{module.DEFAULT_IMAGE}:git-{revision}-build-9" + image_file = tmp_path / "image" + image_file.write_text(f"{destination}@{digest}\n", encoding="utf-8") + output = tmp_path / "out" + monkeypatch.setenv("HARBOR_USER", "robot") + monkeypatch.setenv("HARBOR_PASSWORD", "secret") + monkeypatch.setattr( + module, "verify_registry_digest", lambda *_args, **_kwargs: None + ) + monkeypatch.setattr( + "sys.argv", + [ + "hermes_image_release.py", + "render", + "--digest-file", + str(digest_file), + "--image-file", + str(image_file), + "--source-revision", + revision, + "--build-number", + "9", + "--destination", + destination, + "--kustomization", + str(manifest), + "--output-dir", + str(output), + ], + ) + assert module.main() == 0 + assert ( + json.loads((output / "hermes-agent-image.json").read_text(encoding="utf-8"))[ + "digest" + ] + == digest + ) diff --git a/testing/tests/test_hermes_image_builder_adversarial.py b/testing/tests/test_hermes_image_builder_adversarial.py new file mode 100644 index 00000000..b8fb3103 --- /dev/null +++ b/testing/tests/test_hermes_image_builder_adversarial.py @@ -0,0 +1,356 @@ +"""Adversarial boundaries for the Hermes image publisher and trigger.""" + +from __future__ import annotations + +import importlib.util +import io +import json +from pathlib import Path + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parents[2] +RELEASE_SCRIPT = REPO_ROOT / "ci/scripts/hermes_image_release.py" +TRIGGER_SCRIPT = REPO_ROOT / "services/hermes/scripts/jenkins_image_build_trigger.py" + + +def _load(path: Path, name: str): + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class Response(io.BytesIO): + """Small context-managed HTTP response fixture.""" + + def __init__( + self, + status: int, + headers: dict[str, str] | None = None, + body: bytes = b"", + ): + super().__init__(body) + self.status = status + self.headers = headers or {} + + +def test_kaniko_evidence_binds_digest_destination_and_unique_build() -> None: + """Both Kaniko artifacts must describe one exact non-replayable tag.""" + module = _load(RELEASE_SCRIPT, "hermes_image_release_evidence") + revision = "a" * 40 + digest = "sha256:" + "b" * 64 + destination = f"{module.DEFAULT_IMAGE}:git-{revision}-build-42" + assert module.validate_destination(destination, revision, "42") == ( + revision, + "42", + ) + assert ( + module.validate_kaniko_evidence( + digest_text=f"{digest}\n", + image_text=f"{destination}@{digest}\n", + destination=destination, + ) + == digest + ) + + +@pytest.mark.parametrize( + ("digest_text", "image_template"), + [ + ("sha256:" + "a" * 64 + "\nextra\n", "{destination}@{digest}"), + ("sha256:" + "a" * 64, "{destination}@sha256:" + "b" * 64), + ("sha256:" + "a" * 64, "registry.invalid/x:y@{digest}"), + ("sha256:" + "a" * 64, "{destination}@{digest}\nextra"), + ], +) +def test_kaniko_evidence_rejects_cross_artifact_mismatch( + digest_text: str, image_template: str +) -> None: + """A digest, tag, repository, or cardinality mismatch stops rendering.""" + module = _load(RELEASE_SCRIPT, "hermes_image_release_mismatch") + revision = "c" * 40 + digest = "sha256:" + "a" * 64 + destination = f"{module.DEFAULT_IMAGE}:git-{revision}-build-8" + image_text = image_template.format(destination=destination, digest=digest) + with pytest.raises(ValueError): + module.validate_kaniko_evidence( + digest_text=digest_text, + image_text=image_text, + destination=destination, + ) + + +def test_registry_preflight_rejects_existing_tag_and_auth_failures() -> None: + """Only an authenticated 404 permits Kaniko to claim the unique tag.""" + module = _load(RELEASE_SCRIPT, "hermes_image_release_preflight") + destination = f"{module.DEFAULT_IMAGE}:git-{'d' * 40}-build-3" + captured = {} + + def missing(request, timeout): + captured["request"] = request + captured["timeout"] = timeout + return Response(404) + + module.assert_tag_absent( + destination, username="robot", password="private", opener=missing + ) + request = captured["request"] + assert request.method == "GET" + assert request.full_url.startswith( + "https://registry.bstein.dev/api/v2.0/projects/bstein/repositories/" + "hermes-agent/artifacts/" + ) + assert "private" not in request.full_url + assert captured["timeout"] == 20 + + for status, message in ( + (200, "already exists"), + (401, "HTTP 401"), + (503, "HTTP 503"), + ): + with pytest.raises(RuntimeError, match=message): + module.assert_tag_absent( + destination, + username="robot", + password="private", + opener=lambda *_args, code=status: Response(code), + ) + + +def test_harbor_client_fails_closed_on_input_size_auth_and_json( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Malformed inputs, oversized bodies, missing auth, and bad JSON all fail.""" + module = _load(RELEASE_SCRIPT, "hermes_image_release_harbor_errors") + destination = f"{module.DEFAULT_IMAGE}:git-{'1' * 40}-build-6" + with pytest.raises(ValueError, match="destination"): + module.assert_tag_absent( + "registry.invalid/x:tag", username="robot", password="private" + ) + with pytest.raises(RuntimeError, match="credentials"): + module.assert_tag_absent(destination, username="", password="private") + with pytest.raises(RuntimeError, match="size limit"): + module.assert_tag_absent( + destination, + username="robot", + password="private", + opener=lambda *_args: Response(200, body=b"x" * 1_048_577), + ) + with pytest.raises(RuntimeError, match="invalid artifact JSON"): + module.verify_registry_digest( + destination, + "sha256:" + "2" * 64, + username="robot", + password="private", + opener=lambda *_args: Response(200, body=b"\xff"), + ) + monkeypatch.delenv("HARBOR_USER", raising=False) + monkeypatch.delenv("HARBOR_PASSWORD", raising=False) + with pytest.raises(RuntimeError, match="unavailable"): + module._credentials() + + +def test_default_harbor_opener_returns_http_responses( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The real wrapper returns both normal and non-redirect HTTP responses.""" + module = _load(RELEASE_SCRIPT, "hermes_image_release_opener") + request = module.urllib.request.Request("https://registry.bstein.dev/test") + + class SuccessOpener: + def open(self, _request, timeout): + assert timeout == 7 + return Response(204) + + monkeypatch.setattr( + module.urllib.request, "build_opener", lambda *_handlers: SuccessOpener() + ) + assert module._registry_request(request, 7).status == 204 + + class ErrorOpener: + def open(self, _request, timeout): + assert timeout == 8 + raise module.urllib.error.HTTPError( + _request.full_url, 404, "missing", {}, None + ) + + monkeypatch.setattr( + module.urllib.request, "build_opener", lambda *_handlers: ErrorOpener() + ) + assert module._registry_request(request, 8).code == 404 + + +@pytest.mark.parametrize( + ("status", "artifact", "message"), + [ + (404, {}, "HTTP 404"), + (200, {}, "omitted"), + (200, {"digest": "sha256:" + "2" * 64}, "does not match"), + (200, {"digest": "latest"}, "omitted"), + ], +) +def test_registry_verification_rejects_missing_or_mismatched_digest( + status: int, artifact: dict[str, str], message: str +) -> None: + """Rendering requires an independent exact Harbor digest response.""" + module = _load(RELEASE_SCRIPT, "hermes_image_release_registry") + digest = "sha256:" + "1" * 64 + destination = f"{module.DEFAULT_IMAGE}:git-{'e' * 40}-build-4" + body = json.dumps(artifact).encode("utf-8") + with pytest.raises(RuntimeError, match=message): + module.verify_registry_digest( + destination, + digest, + username="robot", + password="private", + opener=lambda *_args: Response(status, body=body), + ) + + +def test_registry_verification_accepts_exact_pushed_digest() -> None: + """An exact authenticated Harbor digest allows artifact rendering.""" + module = _load(RELEASE_SCRIPT, "hermes_image_release_registry_ok") + digest = "sha256:" + "4" * 64 + destination = f"{module.DEFAULT_IMAGE}:git-{'f' * 40}-build-5" + with pytest.raises(RuntimeError, match="expected tag"): + module.verify_registry_digest( + destination, + digest, + username="robot", + password="private", + opener=lambda *_args: Response( + 200, + body=json.dumps( + {"digest": digest, "tags": [{"name": "different-tag"}]} + ).encode("utf-8"), + ), + ) + module.verify_registry_digest( + destination, + digest, + username="robot", + password="private", + opener=lambda *_args: Response( + 200, + body=json.dumps( + { + "digest": digest, + "tags": [{"name": destination.rsplit(":", 1)[1]}], + } + ).encode("utf-8"), + ), + ) + + +def test_release_main_preflight_uses_fixed_credentials_and_destination( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The preflight CLI validates and checks exactly the requested build tag.""" + module = _load(RELEASE_SCRIPT, "hermes_image_release_main_absent") + revision = "3" * 40 + destination = f"{module.DEFAULT_IMAGE}:git-{revision}-build-12" + captured = {} + monkeypatch.setenv("HARBOR_USER", "robot") + monkeypatch.setenv("HARBOR_PASSWORD", "private") + + def assert_absent(value, **credentials): + captured["destination"] = value + captured["credentials"] = credentials + + monkeypatch.setattr(module, "assert_tag_absent", assert_absent) + monkeypatch.setattr( + "sys.argv", + [ + "hermes_image_release.py", + "assert-absent", + "--source-revision", + revision, + "--build-number", + "12", + "--destination", + destination, + ], + ) + assert module.main() == 0 + assert captured == { + "destination": destination, + "credentials": {"username": "robot", "password": "private"}, + } + + +def test_renderer_skips_a_matching_entry_without_a_digest() -> None: + """Only the one complete image entry is changed when an earlier entry drifts.""" + module = _load(RELEASE_SCRIPT, "hermes_image_release_manifest_scan") + digest = "sha256:" + "5" * 64 + source = ( + "images:\n" + f" - name: {module.DEFAULT_IMAGE}\n" + " newTag: ignored\n" + " - name: example.invalid/other\n" + " newTag: stable\n" + f" - name: {module.DEFAULT_IMAGE}\n" + " digest: sha256:" + "0" * 64 + "\n" + ) + assert module.render_kustomization(source, digest).endswith(f"{digest}\n") + + +@pytest.mark.parametrize("status", [200, 202, 204, 301, 302, 307, 308]) +def test_trigger_rejects_non_plugin_success_status(tmp_path: Path, status: int) -> None: + """Generic proxy successes and redirects are not proof of a queued build.""" + module = _load(TRIGGER_SCRIPT, f"jenkins_trigger_status_{status}") + token = tmp_path / "token" + token.write_text("private\n", encoding="utf-8") + response = lambda *_args, **_kwargs: Response( # noqa: E731 + status, {"Location": "https://ci.bstein.dev/queue/item/9/"} + ) + with pytest.raises(RuntimeError, match=f"HTTP {status}"): + module.trigger_build("a" * 40, token_file=token, opener=response) + + +@pytest.mark.parametrize( + "location", + [ + "", + "https://evil.invalid/queue/item/9/", + "http://ci.bstein.dev/queue/item/9/", + "https://ci.bstein.dev/job/hermes-agent-image/9/", + "https://ci.bstein.dev/queue/item/9/?token=leak", + "https://ci.bstein.dev/queue/item/9/#fragment", + "https://ci.bstein.dev/queue/item/not-a-number/", + ], +) +def test_trigger_rejects_missing_malformed_or_cross_origin_queue_location( + tmp_path: Path, location: str +) -> None: + """A real accepted status still needs the exact same-origin queue resource.""" + module = _load(TRIGGER_SCRIPT, "jenkins_trigger_location") + token = tmp_path / "token" + token.write_text("private\n", encoding="utf-8") + response = lambda *_args, **_kwargs: Response( # noqa: E731 + 201, {"Location": location} + ) + with pytest.raises(RuntimeError, match="Location"): + module.trigger_build("b" * 40, token_file=token, opener=response) + + +def test_trigger_uses_https_and_accepts_exact_relative_queue_path( + tmp_path: Path, +) -> None: + """A relative plugin Location is resolved only against the fixed HTTPS origin.""" + module = _load(TRIGGER_SCRIPT, "jenkins_trigger_origin") + token = tmp_path / "token" + token.write_text("private\n", encoding="utf-8") + captured = {} + + def opener(request, timeout): + captured["url"] = request.full_url + captured["timeout"] = timeout + return Response(201, {"Location": "/queue/item/11/"}) + + result = module.trigger_build("c" * 40, token_file=token, opener=opener) + assert captured["url"].startswith("https://ci.bstein.dev/") + assert captured["timeout"] == 20 + assert result["queue_path"] == "/queue/item/11/" diff --git a/testing/tests/test_hermes_image_builder_vault.py b/testing/tests/test_hermes_image_builder_vault.py new file mode 100644 index 00000000..49b568dd --- /dev/null +++ b/testing/tests/test_hermes_image_builder_vault.py @@ -0,0 +1,233 @@ +"""Vault least-privilege and fail-closed seeding tests for image releases.""" + +from __future__ import annotations + +import os +import stat +import subprocess +from pathlib import Path + +import pytest +import yaml + + +REPO_ROOT = Path(__file__).resolve().parents[2] +VAULT_CONFIG = REPO_ROOT / "services/vault/scripts/vault_k8s_auth_configure.sh" +SEEDER = REPO_ROOT / "services/vault/scripts/vault_hermes_jenkins_build_token_ensure.sh" + + +def test_developer_jenkins_access_is_read_only_and_bound_to_two_consumers() -> None: + """Only the Jenkins controller and Hermes agent can read the fixed-job token.""" + source = VAULT_CONFIG.read_text(encoding="utf-8") + assert 'write_policy_and_role "jenkins" "jenkins" "jenkins"' in source + assert ( + '"jenkins/* shared/harbor-pull quality/sonarqube-oidc ' + 'hermes/developer-jenkins" ""' + ) in source + assert 'write_policy_and_role "hermes-agent" "hermes" "hermes-agent"' in source + assert 'hermes/developer-jenkins hermes/developer-ssh" ""' in source + assert 'write_policy_and_role "hermes-switchyard" "hermes"' in source + assert '"hermes/chat-telegram" ""' in source + assert '"hermes/developer-jenkins"' not in source + + jenkins_spc = yaml.safe_load( + (REPO_ROOT / "services/jenkins/secretproviderclass.yaml").read_text() + ) + assert jenkins_spc["spec"]["parameters"]["roleName"] == "jenkins-vault-sync" + switchyard = yaml.safe_load( + (REPO_ROOT / "services/hermes/switchyard-deployment.yaml").read_text() + ) + annotations = switchyard["spec"]["template"]["metadata"]["annotations"] + assert annotations["vault.hashicorp.com/role"] == "hermes-switchyard" + + +def test_flux_orders_seed_then_jenkins_then_hermes() -> None: + """The fixed token must be Ready before either consumer is rolled.""" + app_root = REPO_ROOT / "clusters/atlas/flux-system/applications" + vault = yaml.safe_load((app_root / "vault/kustomization.yaml").read_text()) + jenkins = yaml.safe_load((app_root / "jenkins/kustomization.yaml").read_text()) + hermes = yaml.safe_load((app_root / "hermes/kustomization.yaml").read_text()) + seed_check = { + "apiVersion": "batch/v1", + "kind": "Job", + "name": "vault-hermes-jenkins-build-token-seed-1", + "namespace": "vault", + } + assert seed_check in vault["spec"]["healthChecks"] + assert jenkins["spec"]["suspend"] is False + assert {item["name"] for item in jenkins["spec"]["dependsOn"]} >= { + "helm", + "vault", + } + assert "jenkins" in {item["name"] for item in hermes["spec"]["dependsOn"]} + + +def test_seed_job_is_revisioned_bounded_and_tracks_fail_closed_script() -> None: + """The prerequisite is a tracked one-shot on healthy non-reserved capacity.""" + job = yaml.safe_load( + ( + REPO_ROOT / "services/vault/hermes-jenkins-build-token-seed-job.yaml" + ).read_text() + ) + assert job["metadata"]["name"] == "vault-hermes-jenkins-build-token-seed-1" + pod = job["spec"]["template"]["spec"] + assert pod["serviceAccountName"] == "vault-admin" + assert pod["restartPolicy"] == "Never" + assert pod["nodeSelector"]["hardware"] == "rpi5" + expression = pod["affinity"]["nodeAffinity"][ + "requiredDuringSchedulingIgnoredDuringExecution" + ]["nodeSelectorTerms"][0]["matchExpressions"][0] + assert set(expression["values"]) >= { + "titan-04", + "titan-14", + "titan-18", + "titan-19", + "titan-24", + } + source = SEEDER.read_text(encoding="utf-8") + assert "kv put -cas=0" in source + assert "build_token=-" in source + assert "kv patch" not in source + assert "kv delete" not in source + + +def _fake_vault_tools(tmp_path: Path) -> tuple[Path, Path, Path]: + """Create deterministic Vault/sleep fakes and return their evidence paths.""" + binary_dir = tmp_path / "bin" + binary_dir.mkdir(parents=True) + calls = tmp_path / "calls" + stdin_capture = tmp_path / "stdin" + vault = binary_dir / "vault" + vault.write_text( + """#!/bin/sh +set -eu +printf '%s\n' "$*" >> "$FAKE_CALL_LOG" +scenario="$FAKE_SCENARIO" +if [ "$1" = "write" ]; then + printf '%s\n' generated-token-value + exit 0 +fi +if [ "$1" = "kv" ] && [ "$2" = "put" ]; then + cat > "$FAKE_STDIN_CAPTURE" + if [ "$scenario" = "cas-race" ]; then + : > "$FAKE_WINNER" + echo 'check-and-set parameter did not match' >&2 + exit 2 + fi + exit 0 +fi +if [ "$1" = "kv" ] && [ "$2" = "get" ]; then + if [ "${3:-}" = "-field=build_token" ]; then + case "$scenario" in + existing|cas-race) + printf '%s\n' existing-token + exit 0 + ;; + missing-field) + echo 'No value found at kv/atlas/hermes/developer-jenkins' >&2 + exit 2 + ;; + esac + fi + case "$scenario" in + existing|missing-field) + exit 0 + ;; + absent) + echo 'Code: 404' >&2 + exit 2 + ;; + cas-race) + if [ -f "$FAKE_WINNER" ]; then exit 0; fi + echo 'Code: 404' >&2 + exit 2 + ;; + transient) + count=0 + if [ -f "$FAKE_COUNT" ]; then count="$(cat "$FAKE_COUNT")"; fi + count=$((count + 1)) + printf '%s' "$count" > "$FAKE_COUNT" + if [ "$count" -lt 3 ]; then echo 'temporary upstream error' >&2; exit 2; fi + echo 'Code: 404' >&2 + exit 2 + ;; + read-error) + echo 'permission denied' >&2 + exit 2 + ;; + esac +fi +echo "unexpected fake Vault call: $*" >&2 +exit 99 +""", + encoding="utf-8", + ) + sleep = binary_dir / "sleep" + sleep.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + for executable in (vault, sleep): + executable.chmod(executable.stat().st_mode | stat.S_IXUSR) + return binary_dir, calls, stdin_capture + + +def _run_seeder( + tmp_path: Path, scenario: str +) -> tuple[subprocess.CompletedProcess, str]: + binary_dir, calls, stdin_capture = _fake_vault_tools(tmp_path) + env = { + **os.environ, + "PATH": f"{binary_dir}:{os.environ['PATH']}", + "VAULT_TOKEN": "test-token", + "FAKE_SCENARIO": scenario, + "FAKE_CALL_LOG": str(calls), + "FAKE_STDIN_CAPTURE": str(stdin_capture), + "FAKE_WINNER": str(tmp_path / "winner"), + "FAKE_COUNT": str(tmp_path / "count"), + } + result = subprocess.run( + ["sh", str(SEEDER)], + env=env, + text=True, + capture_output=True, + check=False, + ) + call_text = calls.read_text(encoding="utf-8") if calls.exists() else "" + return result, call_text + + +@pytest.mark.parametrize("scenario", ["existing", "missing-field", "read-error"]) +def test_seeder_never_overwrites_existing_or_ambiguous_state( + tmp_path: Path, scenario: str +) -> None: + """Existing fields and persistent read failures can never become a put.""" + result, calls = _run_seeder(tmp_path, scenario) + assert (result.returncode == 0) is (scenario == "existing") + assert "kv put" not in calls + assert "sys/tools/random" not in calls + + +def test_seeder_uses_single_cas_create_and_stdin_for_absent_secret( + tmp_path: Path, +) -> None: + """A confirmed 404 produces one create-only write without a token argument.""" + result, calls = _run_seeder(tmp_path, "absent") + assert result.returncode == 0, result.stderr + assert calls.count("kv put") == 1 + assert "kv put -cas=0 kv/atlas/hermes/developer-jenkins build_token=-" in calls + assert "generated-token-value" not in calls + assert (tmp_path / "stdin").read_text(encoding="utf-8") == "generated-token-value" + + +def test_seeder_retries_reads_and_accepts_only_a_verified_cas_winner( + tmp_path: Path, +) -> None: + """Transient reads retry; a competing create is accepted only after reread.""" + transient, transient_calls = _run_seeder(tmp_path / "transient", "transient") + assert transient.returncode == 0, transient.stderr + assert transient_calls.count("kv get kv/atlas/hermes/developer-jenkins") == 3 + assert transient_calls.count("kv put") == 1 + + race, race_calls = _run_seeder(tmp_path / "race", "cas-race") + assert race.returncode == 0, race.stderr + assert race_calls.count("kv put") == 1 + assert "kv get -field=build_token" in race_calls + assert "another seeder won CAS" in race.stderr diff --git a/testing/tests/test_hermes_runtime_access.py b/testing/tests/test_hermes_runtime_access.py index 7e455eb6..fa4c8021 100644 --- a/testing/tests/test_hermes_runtime_access.py +++ b/testing/tests/test_hermes_runtime_access.py @@ -177,6 +177,7 @@ def test_agent_runtime_stage_keeps_credentials_in_memory(tmp_path: Path, monkeyp "chat-relay-key": "relay-key", "gitea-token": "gitea-key", "gitea-username": "hermes-automation", + "jenkins-image-build-token": "job-scoped-token", "node-ssh-private-key": "private-key", "node-ssh-config": "host-config", "node-ssh-known-hosts": "known-hosts", @@ -196,6 +197,7 @@ def test_agent_runtime_stage_keeps_credentials_in_memory(tmp_path: Path, monkeyp assert (runtime / "claude/.credentials.json").stat().st_mode & 0o777 == 0o600 assert (runtime / "codex/auth.json").stat().st_mode & 0o777 == 0o600 + assert (runtime / "jenkins-image-build-token").stat().st_mode & 0o777 == 0o600 assert (runtime / "claude/settings.json").is_symlink() assert (runtime / "codex/skills").is_symlink() assert not (runtime / "claude/backups").exists()