From e8c26ecf85a0c802d378fffffb98b7df43b78242 Mon Sep 17 00:00:00 2001 From: jenkins Date: Sun, 16 Aug 2026 20:04:17 -0300 Subject: [PATCH 1/4] hermes: add daemonless agent image release lane --- ci/Jenkinsfile.hermes-agent-image | 288 ++++++++++ ci/scripts/hermes_image_release.py | 500 ++++++++++++++++++ .../applications/harbor/kustomization.yaml | 8 +- .../applications/hermes/kustomization.yaml | 1 + .../applications/jenkins/kustomization.yaml | 5 +- .../applications/kustomization.yaml | 1 + .../kustomization.yaml | 26 + .../applications/vault/kustomization.yaml | 5 + dockerfiles/Dockerfile.hermes-agent | 15 + .../Dockerfile.hermes-agent.dockerignore | 2 + dockerfiles/hermes-kaniko-heredoc-runner.py | 146 +++++ .../harbor/hermes-agent-immutability-job.yaml | 79 +++ services/harbor/kustomization.yaml | 5 + .../policy-bootstrap-serviceaccount.yaml | 7 + ...harbor_hermes_agent_immutability_ensure.py | 378 +++++++++++++ services/hermes/NOTES.md | 104 ++++ services/hermes/agent-deployment.yaml | 5 + services/hermes/kustomization.yaml | 1 + .../scripts/jenkins_image_build_trigger.py | 112 ++++ .../hermes/scripts/stage_runtime_access.py | 1 + services/hermes/switchyard-deployment.yaml | 2 +- services/jenkins/configmap-jcasc.yaml | 19 + services/jenkins/configmap-plugins.yaml | 1 + services/jenkins/deployment.yaml | 3 + .../hermes-image-builder-serviceaccount.yaml | 7 + services/jenkins/kustomization.yaml | 1 + services/jenkins/secretproviderclass.yaml | 2 +- .../vault-hermes-jenkins-token-seed/job.yaml | 68 +++ .../kustomization.yaml | 13 + ...vault_hermes_jenkins_build_token_ensure.sh | 167 ++++++ .../serviceaccount.yaml | 7 + .../vault/hermes-auth-role-bootstrap-job.yaml | 30 +- .../vault/scripts/vault_k8s_auth_configure.sh | 37 +- testing/quality_contract.json | 15 + testing/tests/test_hermes_image_builder.py | 473 +++++++++++++++++ .../test_hermes_image_builder_adversarial.py | 469 ++++++++++++++++ .../test_hermes_image_builder_coverage.py | 456 ++++++++++++++++ .../test_hermes_image_builder_fresh_review.py | 335 ++++++++++++ .../tests/test_hermes_image_builder_harbor.py | 325 ++++++++++++ .../tests/test_hermes_image_builder_vault.py | 329 ++++++++++++ testing/tests/test_hermes_runtime_access.py | 2 + 41 files changed, 4439 insertions(+), 11 deletions(-) create mode 100644 ci/Jenkinsfile.hermes-agent-image create mode 100755 ci/scripts/hermes_image_release.py create mode 100644 clusters/atlas/flux-system/applications/vault-hermes-jenkins-token-seed/kustomization.yaml create mode 100644 dockerfiles/hermes-kaniko-heredoc-runner.py create mode 100644 services/harbor/hermes-agent-immutability-job.yaml create mode 100644 services/harbor/policy-bootstrap-serviceaccount.yaml create mode 100644 services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py create mode 100755 services/hermes/scripts/jenkins_image_build_trigger.py create mode 100644 services/jenkins/hermes-image-builder-serviceaccount.yaml create mode 100644 services/vault-hermes-jenkins-token-seed/job.yaml create mode 100644 services/vault-hermes-jenkins-token-seed/kustomization.yaml create mode 100644 services/vault-hermes-jenkins-token-seed/scripts/vault_hermes_jenkins_build_token_ensure.sh create mode 100644 services/vault-hermes-jenkins-token-seed/serviceaccount.yaml create mode 100644 testing/tests/test_hermes_image_builder.py create mode 100644 testing/tests/test_hermes_image_builder_adversarial.py create mode 100644 testing/tests/test_hermes_image_builder_coverage.py create mode 100644 testing/tests/test_hermes_image_builder_fresh_review.py create mode 100644 testing/tests/test_hermes_image_builder_harbor.py create mode 100644 testing/tests/test_hermes_image_builder_vault.py diff --git a/ci/Jenkinsfile.hermes-agent-image b/ci/Jenkinsfile.hermes-agent-image new file mode 100644 index 00000000..2d350ac7 --- /dev/null +++ b/ci/Jenkinsfile.hermes-agent-image @@ -0,0 +1,288 @@ +pipeline { + agent { + kubernetes { + defaultContainer 'python' + yaml """ +apiVersion: v1 +kind: Pod +metadata: + labels: + atlas.bstein.dev/workload: hermes-agent-image-builder +spec: + serviceAccountName: hermes-image-builder + automountServiceAccountToken: false + enableServiceLinks: false + restartPolicy: Never + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + nodeSelector: + kubernetes.io/arch: arm64 + node-role.kubernetes.io/worker: "true" + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: NotIn + values: + - titan-04 + - titan-14 + - titan-18 + - titan-19 + - titan-22 + - titan-24 + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: hardware + operator: In + values: + - rpi5 + imagePullSecrets: + - name: harbor-bstein-robot + containers: + - name: jnlp + image: jenkins/inbound-agent@sha256:8eda4fe2a66bcf6a5e43436d9918fc14c306204dc8fcd75f4e15e0e6e5dc759a + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + runAsNonRoot: true + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault + resources: + requests: + cpu: 25m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + - name: python + image: registry.bstein.dev/bstein/python@sha256:269541d3387baae008df4608ead893dba2b5cdaad1a5a380731a88992d34b808 + command: ["sleep"] + args: ["99d"] + tty: true + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + runAsNonRoot: true + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault + resources: + requests: + cpu: 25m + memory: 64Mi + limits: + cpu: 250m + memory: 256Mi + - name: kaniko + image: gcr.io/kaniko-project/executor@sha256:c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e + command: ["/busybox/sh", "-c"] + args: ["/busybox/sleep 99d"] + tty: true + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"] + privileged: false + runAsUser: 0 + seccompProfile: + type: RuntimeDefault + resources: + requests: + cpu: 250m + memory: 1Gi + limits: + cpu: "2" + memory: 4Gi +""" + } + } + parameters { + booleanParam( + name: 'PUBLISH_IMAGE', + defaultValue: false, + description: 'Publish the reviewed main revision to Harbor.' + ) + string( + name: 'EXPECTED_SOURCE_REVISION', + defaultValue: '', + description: 'Exact 40-character commit on atlas/titan-iac main.' + ) + string( + name: 'CONFIRM_PUBLISH', + defaultValue: '', + description: 'Enter PUBLISH HERMES AGENT to confirm the release.' + ) + } + environment { + HERMES_IMAGE = 'registry.bstein.dev/bstein/hermes-agent' + } + options { + disableConcurrentBuilds() + buildDiscarder(logRotator(daysToKeepStr: '30', numToKeepStr: '100', artifactDaysToKeepStr: '30', artifactNumToKeepStr: '100')) + skipDefaultCheckout(true) + timeout(time: 90, unit: 'MINUTES') + } + stages { + stage('Checkout reviewed source') { + steps { + checkout scm + } + } + stage('Enforce release boundary') { + steps { + sh ''' + set -eu + mkdir -p build + test "${PUBLISH_IMAGE}" = "true" + test "${CONFIRM_PUBLISH}" = "PUBLISH HERMES AGENT" + case "${EXPECTED_SOURCE_REVISION}" in + *[!0-9a-f]*|'') + echo "EXPECTED_SOURCE_REVISION must be a lowercase full commit" >&2 + exit 2 + ;; + esac + test "${#EXPECTED_SOURCE_REVISION}" -eq 40 + actual_revision="$(git rev-parse HEAD)" + test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}" + test "${actual_revision}" = "$(git rev-parse origin/main)" + test -z "$(git status --porcelain)" + test -f dockerfiles/Dockerfile.hermes-agent + case "${BUILD_NUMBER}" in + ''|0*|*[!0-9]*) + echo "BUILD_NUMBER must be a positive decimal integer" >&2 + exit 2 + ;; + esac + printf '%s\n' \ + "${HERMES_IMAGE}:git-${actual_revision}-build-${BUILD_NUMBER}" \ + > build/hermes-agent.destination + ''' + } + } + stage('Reject replay before publish') { + steps { + withCredentials([usernamePassword( + credentialsId: 'harbor-robot', + usernameVariable: 'HARBOR_USER', + passwordVariable: 'HARBOR_PASSWORD' + )]) { + sh ''' + set -eu + set +x + destination="$(cat build/hermes-agent.destination)" + python3 ci/scripts/hermes_image_release.py assert-absent \ + --source-revision "${EXPECTED_SOURCE_REVISION}" \ + --build-number "${BUILD_NUMBER}" \ + --destination "${destination}" + ''' + } + } + } + stage('Build and publish without a daemon') { + steps { + container('kaniko') { + withCredentials([usernamePassword( + credentialsId: 'harbor-robot', + usernameVariable: 'HARBOR_USER', + passwordVariable: 'HARBOR_PASSWORD' + )]) { + sh '''#!/busybox/sh + set -eu + set +x + config_path=/kaniko/.docker/config.json + destination="$(cat build/hermes-agent.destination)" + umask 077 + auth="$(printf '%s:%s' "${HARBOR_USER}" "${HARBOR_PASSWORD}" | /busybox/base64 | /busybox/tr -d '\n')" + /busybox/mkdir -p /kaniko/.docker + /busybox/printf '{"auths":{"registry.bstein.dev":{"auth":"%s"}}}\n' "${auth}" > "${config_path}" + unset HARBOR_USER HARBOR_PASSWORD auth + trap '/busybox/rm -f "${config_path}"' EXIT HUP INT TERM + /kaniko/executor \ + --context="dir://${WORKSPACE}" \ + --dockerfile="${WORKSPACE}/dockerfiles/Dockerfile.hermes-agent" \ + --destination="${destination}" \ + --digest-file="${WORKSPACE}/build/hermes-agent.digest" \ + --image-name-tag-with-digest-file="${WORKSPACE}/build/hermes-agent.image" \ + --build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1 \ + --label="org.opencontainers.image.revision=${EXPECTED_SOURCE_REVISION}" \ + --cleanup \ + --push-retry=3 + ''' + } + } + } + } + stage('Render reviewed Flux handoff') { + steps { + withCredentials([usernamePassword( + credentialsId: 'harbor-robot', + usernameVariable: 'HARBOR_USER', + passwordVariable: 'HARBOR_PASSWORD' + )]) { + sh ''' + set -eu + set +x + destination="$(cat build/hermes-agent.destination)" + python3 ci/scripts/hermes_image_release.py render \ + --digest-file build/hermes-agent.digest \ + --image-file build/hermes-agent.image \ + --source-revision "${EXPECTED_SOURCE_REVISION}" \ + --build-number "${BUILD_NUMBER}" \ + --destination "${destination}" \ + --kustomization services/hermes/kustomization.yaml \ + --output-dir build/hermes-agent-release + test -s build/hermes-agent-release/hermes-image-update.patch + test -s build/hermes-agent-release/hermes-agent-image.json + ''' + } + } + } + } + post { + success { + sh ''' + set -eu + expected_files="$(printf '%s\n' \ + build/hermes-agent.destination \ + build/hermes-agent.digest \ + build/hermes-agent.image \ + build/hermes-agent-release/hermes-agent-image.json \ + build/hermes-agent-release/hermes-image-update.patch \ + build/hermes-agent-release/hermes-kustomization.yaml \ + | LC_ALL=C sort)" + actual_files="$(find build -type f -print | LC_ALL=C sort)" + test "${actual_files}" = "${expected_files}" + destination="$(cat build/hermes-agent.destination)" + python3 ci/scripts/hermes_image_release.py verify-evidence \ + --digest-file build/hermes-agent.digest \ + --image-file build/hermes-agent.image \ + --source-revision "${EXPECTED_SOURCE_REVISION}" \ + --build-number "${BUILD_NUMBER}" \ + --destination "${destination}" \ + --kustomization services/hermes/kustomization.yaml \ + --output-dir build/hermes-agent-release + ''' + archiveArtifacts( + artifacts: 'build/hermes-agent.destination,build/hermes-agent.digest,build/hermes-agent.image,build/hermes-agent-release/hermes-agent-image.json,build/hermes-agent-release/hermes-image-update.patch,build/hermes-agent-release/hermes-kustomization.yaml', + allowEmptyArchive: false, + fingerprint: true + ) + } + cleanup { + container('kaniko') { + sh '''#!/busybox/sh + /busybox/rm -f /kaniko/.docker/config.json + ''' + } + } + } +} diff --git a/ci/scripts/hermes_image_release.py b/ci/scripts/hermes_image_release.py new file mode 100755 index 00000000..2f0f4166 --- /dev/null +++ b/ci/scripts/hermes_image_release.py @@ -0,0 +1,500 @@ +#!/usr/bin/env python3 +"""Verify and render a reviewable Hermes agent image release.""" + +from __future__ import annotations + +import argparse +import base64 +import difflib +import json +import os +import re +import urllib.error +import urllib.parse +import urllib.request +from pathlib import Path +from typing import Any, Callable + + +DEFAULT_IMAGE = "registry.bstein.dev/bstein/hermes-agent" +HARBOR_API_ORIGIN = "https://registry.bstein.dev/api/v2.0" +HARBOR_PROJECT = "bstein" +HARBOR_REPOSITORY = "hermes-agent" +IMMUTABLE_REPOSITORY_PATTERN = "hermes-agent" +IMMUTABLE_TAG_PATTERN = "git-*-build-*" +DIGEST_PATTERN = re.compile(r"^sha256:[0-9a-f]{64}$") +REVISION_PATTERN = re.compile(r"^[0-9a-f]{40}$") +BUILD_PATTERN = re.compile(r"^[1-9][0-9]*$") +DESTINATION_PATTERN = re.compile( + r"^registry\.bstein\.dev/bstein/hermes-agent:" + r"git-([0-9a-f]{40})-build-([1-9][0-9]*)$" +) + + +class _NoRedirect(urllib.request.HTTPRedirectHandler): + """Never send registry credentials to a redirect target.""" + + def redirect_request(self, _request, _file, _code, _message, _headers, _url): + return None + + +def _validated(value: str, pattern: re.Pattern[str], label: str) -> str: + """Return a normalized value when it matches the release contract.""" + normalized = value.strip() + if not pattern.fullmatch(normalized): + raise ValueError(f"invalid {label}: expected {pattern.pattern}") + return normalized + + +def validate_destination( + destination: str, source_revision: str, build_number: str +) -> tuple[str, str]: + """Bind one unique build tag to the reviewed revision and Jenkins build.""" + revision = _validated(source_revision, REVISION_PATTERN, "source revision") + build = _validated(build_number, BUILD_PATTERN, "build number") + match = DESTINATION_PATTERN.fullmatch(destination.strip()) + if not match or match.groups() != (revision, build): + raise ValueError( + "destination must bind the reviewed revision and unique Jenkins build" + ) + return revision, build + + +def validate_kaniko_evidence( + *, digest_text: str, image_text: str, destination: str +) -> str: + """Cross-check both independent Kaniko output files against the destination.""" + digest_lines = digest_text.splitlines() + image_lines = image_text.splitlines() + if len(digest_lines) != 1: + raise ValueError("Kaniko digest evidence must contain exactly one line") + if len(image_lines) != 1: + raise ValueError("Kaniko image evidence must contain exactly one line") + digest = _validated(digest_lines[0], DIGEST_PATTERN, "image digest") + if image_lines[0].strip() != f"{destination}@{digest}": + raise ValueError("Kaniko image evidence does not match destination and digest") + return digest + + +def _registry_request(request: urllib.request.Request, timeout: int) -> Any: + """Make a registry request without following redirects.""" + opener = urllib.request.build_opener(_NoRedirect()) + try: + return opener.open(request, timeout=timeout) + except urllib.error.HTTPError as exc: + return exc + + +def _artifact_response( + destination: str, + *, + username: str, + password: str, + opener: Callable[[urllib.request.Request, int], Any] = _registry_request, +) -> tuple[int, bytes]: + """Read one exact Harbor artifact by tag with bounded response size.""" + match = DESTINATION_PATTERN.fullmatch(destination) + if not match: + raise ValueError("invalid destination") + if not username or not password: + raise RuntimeError("Harbor credentials are empty") + tag = destination.rsplit(":", 1)[1] + encoded_tag = urllib.parse.quote(tag, safe="") + auth = base64.b64encode(f"{username}:{password}".encode()).decode("ascii") + request = urllib.request.Request( + f"{HARBOR_API_ORIGIN}/projects/{HARBOR_PROJECT}/repositories/" + f"{HARBOR_REPOSITORY}/artifacts/{encoded_tag}" + "?with_immutable_status=true", + headers={"Accept": "application/json", "Authorization": f"Basic {auth}"}, + method="GET", + ) + with opener(request, 20) as response: + body = response.read(1_048_577) + if len(body) > 1_048_576: + raise RuntimeError("Harbor artifact response exceeded the size limit") + return int(response.status), body + + +def _immutable_rules_response( + *, + username: str, + password: str, + opener: Callable[[urllib.request.Request, int], Any] = _registry_request, +) -> tuple[int, bytes, dict[str, str]]: + """Read the project policy with the same least-privilege publish identity.""" + if not username or not password: + raise RuntimeError("Harbor credentials are empty") + auth = base64.b64encode(f"{username}:{password}".encode()).decode("ascii") + request = urllib.request.Request( + f"{HARBOR_API_ORIGIN}/projects/{HARBOR_PROJECT}/immutabletagrules" + "?page=1&page_size=100", + headers={"Accept": "application/json", "Authorization": f"Basic {auth}"}, + method="GET", + ) + with opener(request, 20) as response: + body = response.read(1_048_577) + if len(body) > 1_048_576: + raise RuntimeError("Harbor immutable rule response exceeded the size limit") + return int(response.status), body, dict(response.headers) + + +def _require_complete_rule_page( + rules: list[dict[str, Any]], headers: dict[str, str] +) -> None: + """Require proof that the bounded first page contains every rule.""" + raw_total = next( + (value for key, value in headers.items() if key.lower() == "x-total-count"), + None, + ) + if raw_total is None or not str(raw_total).isdecimal(): + raise RuntimeError("Harbor immutable rule list omitted a valid total count") + if int(raw_total) != len(rules): + raise RuntimeError("Harbor immutable rule list was truncated") + + +def _normalized_immutable_rule(rule: dict[str, Any]) -> dict[str, Any]: + """Select only fields that bind the server-side build-tag policy.""" + return { + "disabled": bool(rule.get("disabled", False)), + "action": rule.get("action"), + "template": rule.get("template"), + "tag_selectors": [ + { + "kind": item.get("kind"), + "decoration": item.get("decoration"), + "pattern": item.get("pattern"), + } + for item in rule.get("tag_selectors") or [] + if isinstance(item, dict) + ], + "scope_selectors": { + "repository": [ + { + "kind": item.get("kind"), + "decoration": item.get("decoration"), + "pattern": item.get("pattern"), + } + for item in (rule.get("scope_selectors") or {}).get( + "repository", [] + ) + if isinstance(item, dict) + ] + }, + } + + +def verify_immutable_policy( + *, + username: str, + password: str, + opener: Callable[[urllib.request.Request, int], Any] = _registry_request, +) -> None: + """Fail closed before build unless the exact Harbor rule is active.""" + status, body, headers = _immutable_rules_response( + username=username, password=password, opener=opener + ) + if status != 200: + raise RuntimeError(f"Harbor immutable policy preflight returned HTTP {status}") + try: + rules = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise RuntimeError("Harbor returned invalid immutable rule JSON") from exc + if not isinstance(rules, list) or not all(isinstance(item, dict) for item in rules): + raise RuntimeError("Harbor immutable rule list has an invalid shape") + _require_complete_rule_page(rules, headers) + expected = { + "disabled": False, + "action": "immutable", + "template": "immutable_template", + "tag_selectors": [ + { + "kind": "doublestar", + "decoration": "matches", + "pattern": IMMUTABLE_TAG_PATTERN, + } + ], + "scope_selectors": { + "repository": [ + { + "kind": "doublestar", + "decoration": "repoMatches", + "pattern": IMMUTABLE_REPOSITORY_PATTERN, + } + ] + }, + } + matches = [ + _normalized_immutable_rule(item) + for item in rules + if _normalized_immutable_rule(item)["tag_selectors"] + == expected["tag_selectors"] + and _normalized_immutable_rule(item)["scope_selectors"] + == expected["scope_selectors"] + ] + if matches != [expected]: + raise RuntimeError("Harbor immutable build-tag policy is absent or not exact") + + +def assert_tag_absent( + destination: str, + *, + username: str, + password: str, + opener: Callable[[urllib.request.Request, int], Any] = _registry_request, +) -> None: + """Reject replay before Kaniko can push an already-used immutable identity.""" + status, _body = _artifact_response( + destination, username=username, password=password, opener=opener + ) + if status == 404: + return + if status == 200: + raise RuntimeError("Harbor destination tag already exists; refusing overwrite") + raise RuntimeError(f"Harbor destination preflight returned HTTP {status}") + + +def verify_registry_digest( + destination: str, + digest: str, + *, + username: str, + password: str, + opener: Callable[[urllib.request.Request, int], Any] = _registry_request, +) -> None: + """Verify Harbor independently resolves the pushed tag to Kaniko's digest.""" + digest = _validated(digest, DIGEST_PATTERN, "image digest") + status, body = _artifact_response( + destination, username=username, password=password, opener=opener + ) + if status != 200: + raise RuntimeError(f"Harbor manifest verification returned HTTP {status}") + try: + artifact = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise RuntimeError("Harbor returned invalid artifact JSON") from exc + harbor_digest = str(artifact.get("digest") or "").strip() + if not DIGEST_PATTERN.fullmatch(harbor_digest): + raise RuntimeError("Harbor response omitted a valid artifact digest") + if harbor_digest != digest: + raise RuntimeError("Harbor digest does not match Kaniko evidence") + expected_tag = destination.rsplit(":", 1)[1] + matching_tags = [ + item + for item in artifact.get("tags") or [] + if isinstance(item, dict) and item.get("name") == expected_tag + ] + if len(matching_tags) != 1: + raise RuntimeError("Harbor artifact does not contain the expected tag") + if matching_tags[0].get("immutable") is not True: + raise RuntimeError("Harbor did not enforce the expected tag as immutable") + + +def render_kustomization(source: str, digest: str, image: str = DEFAULT_IMAGE) -> str: + """Replace exactly one matching Kustomize image digest without reformatting.""" + digest = _validated(digest, DIGEST_PATTERN, "image digest") + lines = source.splitlines(keepends=True) + matches: list[int] = [] + + for index, line in enumerate(lines): + if line.strip() != f"- name: {image}": + continue + name_indent = len(line) - len(line.lstrip()) + for candidate_index in range(index + 1, len(lines)): + candidate = lines[candidate_index] + stripped = candidate.strip() + candidate_indent = len(candidate) - len(candidate.lstrip()) + if stripped.startswith("- name:") and candidate_indent == name_indent: + break + if stripped.startswith("digest:") and candidate_indent > name_indent: + matches.append(candidate_index) + break + + if len(matches) != 1: + raise ValueError( + f"expected exactly one digest for image {image!r}; found {len(matches)}" + ) + + index = matches[0] + newline = "\n" if lines[index].endswith("\n") else "" + prefix = lines[index][: len(lines[index]) - len(lines[index].lstrip())] + lines[index] = f"{prefix}digest: {digest}{newline}" + return "".join(lines) + + +def write_release_artifacts( + *, + digest: str, + source_revision: str, + build_number: str, + destination: str, + kustomization: Path, + output_dir: Path, +) -> dict[str, str]: + """Write a rendered manifest, patch, and credential-free release metadata.""" + digest = _validated(digest, DIGEST_PATTERN, "image digest") + source_revision, build_number = validate_destination( + destination, source_revision, build_number + ) + source = kustomization.read_text(encoding="utf-8") + rendered = render_kustomization(source, digest) + relative_name = kustomization.name + patch = "".join( + difflib.unified_diff( + source.splitlines(keepends=True), + rendered.splitlines(keepends=True), + fromfile=f"a/services/hermes/{relative_name}", + tofile=f"b/services/hermes/{relative_name}", + ) + ) + if not patch: + raise ValueError("published digest already matches the Flux manifest") + output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / "hermes-kustomization.yaml").write_text(rendered, encoding="utf-8") + (output_dir / "hermes-image-update.patch").write_text(patch, encoding="utf-8") + metadata = { + "build_number": build_number, + "digest": digest, + "flux_image": f"{DEFAULT_IMAGE}@{digest}", + "image": DEFAULT_IMAGE, + "published_tag": destination, + "source_revision": source_revision, + } + (output_dir / "hermes-agent-image.json").write_text( + json.dumps(metadata, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + return metadata + + +def validate_release_artifacts( + *, + digest_file: Path, + image_file: Path, + source_revision: str, + build_number: str, + destination: str, + kustomization: Path, + output_dir: Path, +) -> None: + """Revalidate the exact successful-build evidence without rewriting it.""" + digest = validate_kaniko_evidence( + digest_text=digest_file.read_text(encoding="utf-8"), + image_text=image_file.read_text(encoding="utf-8"), + destination=destination, + ) + source_revision, build_number = validate_destination( + destination, source_revision, build_number + ) + expected_names = { + "hermes-agent-image.json", + "hermes-image-update.patch", + "hermes-kustomization.yaml", + } + entries = list(output_dir.iterdir()) + if {entry.name for entry in entries} != expected_names or not all( + entry.is_file() and not entry.is_symlink() for entry in entries + ): + raise ValueError("release output must contain exactly three evidence files") + source = kustomization.read_text(encoding="utf-8") + rendered = render_kustomization(source, digest) + relative_name = kustomization.name + patch = "".join( + difflib.unified_diff( + source.splitlines(keepends=True), + rendered.splitlines(keepends=True), + fromfile=f"a/services/hermes/{relative_name}", + tofile=f"b/services/hermes/{relative_name}", + ) + ) + metadata = { + "build_number": build_number, + "digest": digest, + "flux_image": f"{DEFAULT_IMAGE}@{digest}", + "image": DEFAULT_IMAGE, + "published_tag": destination, + "source_revision": source_revision, + } + expected = { + "hermes-agent-image.json": json.dumps(metadata, indent=2, sort_keys=True) + "\n", + "hermes-image-update.patch": patch, + "hermes-kustomization.yaml": rendered, + } + for name, expected_text in expected.items(): + if (output_dir / name).read_text(encoding="utf-8") != expected_text: + raise ValueError(f"release evidence is incomplete or mismatched: {name}") + + +def _credentials() -> tuple[str, str]: + """Read the masked, runtime-only Jenkins credential environment.""" + username = os.environ.get("HARBOR_USER", "") + password = os.environ.get("HARBOR_PASSWORD", "") + if not username or not password: + raise RuntimeError("Harbor credentials are unavailable") + return username, password + + +def _common_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--source-revision", required=True) + parser.add_argument("--build-number", required=True) + parser.add_argument("--destination", required=True) + + +def main() -> int: + """Fail closed around the unique tag, then verify and render after push.""" + parser = argparse.ArgumentParser() + commands = parser.add_subparsers(dest="command", required=True) + absent = commands.add_parser("assert-absent") + _common_arguments(absent) + render = commands.add_parser("render") + _common_arguments(render) + render.add_argument("--digest-file", required=True, type=Path) + render.add_argument("--image-file", required=True, type=Path) + render.add_argument("--kustomization", required=True, type=Path) + render.add_argument("--output-dir", required=True, type=Path) + verify = commands.add_parser("verify-evidence") + _common_arguments(verify) + verify.add_argument("--digest-file", required=True, type=Path) + verify.add_argument("--image-file", required=True, type=Path) + verify.add_argument("--kustomization", required=True, type=Path) + verify.add_argument("--output-dir", required=True, type=Path) + args = parser.parse_args() + + validate_destination(args.destination, args.source_revision, args.build_number) + if args.command == "verify-evidence": + validate_release_artifacts( + digest_file=args.digest_file, + image_file=args.image_file, + source_revision=args.source_revision, + build_number=args.build_number, + destination=args.destination, + kustomization=args.kustomization, + output_dir=args.output_dir, + ) + return 0 + + username, password = _credentials() + if args.command == "assert-absent": + verify_immutable_policy(username=username, password=password) + assert_tag_absent(args.destination, username=username, password=password) + return 0 + + digest = validate_kaniko_evidence( + digest_text=args.digest_file.read_text(encoding="utf-8"), + image_text=args.image_file.read_text(encoding="utf-8"), + destination=args.destination, + ) + verify_registry_digest( + args.destination, digest, username=username, password=password + ) + write_release_artifacts( + digest=digest, + source_revision=args.source_revision, + build_number=args.build_number, + destination=args.destination, + kustomization=args.kustomization, + output_dir=args.output_dir, + ) + return 0 + + +if __name__ == "__main__": # pragma: no cover - exercised through main() + raise SystemExit(main()) diff --git a/clusters/atlas/flux-system/applications/harbor/kustomization.yaml b/clusters/atlas/flux-system/applications/harbor/kustomization.yaml index 5bbe2e8f..225214fe 100644 --- a/clusters/atlas/flux-system/applications/harbor/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/harbor/kustomization.yaml @@ -15,7 +15,13 @@ spec: kind: GitRepository name: flux-system namespace: flux-system - wait: false + wait: true + timeout: 10m + healthChecks: + - apiVersion: batch/v1 + kind: Job + name: harbor-hermes-agent-immutability-ensure-1 + namespace: harbor dependsOn: - name: core - name: longhorn diff --git a/clusters/atlas/flux-system/applications/hermes/kustomization.yaml b/clusters/atlas/flux-system/applications/hermes/kustomization.yaml index 3949282b..9a8365ea 100644 --- a/clusters/atlas/flux-system/applications/hermes/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/hermes/kustomization.yaml @@ -60,3 +60,4 @@ spec: - name: keycloak - name: longhorn - name: vault + - name: jenkins diff --git a/clusters/atlas/flux-system/applications/jenkins/kustomization.yaml b/clusters/atlas/flux-system/applications/jenkins/kustomization.yaml index dce79ab7..5cf8d56d 100644 --- a/clusters/atlas/flux-system/applications/jenkins/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/jenkins/kustomization.yaml @@ -6,10 +6,9 @@ metadata: namespace: flux-system annotations: kustomize.toolkit.fluxcd.io/ssa: IfNotPresent - atlas.bstein.dev/suspend-reason: "CI controller changes are applied only during planned maintenance." spec: interval: 10m - suspend: true + suspend: false path: ./services/jenkins prune: true sourceRef: @@ -18,6 +17,8 @@ spec: targetNamespace: jenkins dependsOn: - name: helm + - name: harbor + - name: vault-hermes-jenkins-token-seed healthChecks: - apiVersion: apps/v1 kind: Deployment diff --git a/clusters/atlas/flux-system/applications/kustomization.yaml b/clusters/atlas/flux-system/applications/kustomization.yaml index eb8bc86a..3303f4fa 100644 --- a/clusters/atlas/flux-system/applications/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/kustomization.yaml @@ -4,6 +4,7 @@ kind: Kustomization resources: - gitea/kustomization.yaml - vault/kustomization.yaml + - vault-hermes-jenkins-token-seed/kustomization.yaml - vaultwarden/kustomization.yaml - comms/kustomization.yaml - crypto/kustomization.yaml diff --git a/clusters/atlas/flux-system/applications/vault-hermes-jenkins-token-seed/kustomization.yaml b/clusters/atlas/flux-system/applications/vault-hermes-jenkins-token-seed/kustomization.yaml new file mode 100644 index 00000000..561179c8 --- /dev/null +++ b/clusters/atlas/flux-system/applications/vault-hermes-jenkins-token-seed/kustomization.yaml @@ -0,0 +1,26 @@ +# clusters/atlas/flux-system/applications/vault-hermes-jenkins-token-seed/kustomization.yaml +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: vault-hermes-jenkins-token-seed + namespace: flux-system + annotations: + kustomize.toolkit.fluxcd.io/ssa: IfNotPresent +spec: + interval: 10m + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + path: ./services/vault-hermes-jenkins-token-seed + targetNamespace: vault + prune: true + wait: true + timeout: 5m + dependsOn: + - name: vault + healthChecks: + - apiVersion: batch/v1 + kind: Job + name: vault-hermes-jenkins-build-token-seed-2 + namespace: vault diff --git a/clusters/atlas/flux-system/applications/vault/kustomization.yaml b/clusters/atlas/flux-system/applications/vault/kustomization.yaml index 7dd03da8..58f8ef7d 100644 --- a/clusters/atlas/flux-system/applications/vault/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/vault/kustomization.yaml @@ -16,6 +16,11 @@ spec: targetNamespace: vault prune: true wait: true + healthChecks: + - apiVersion: batch/v1 + kind: Job + name: vault-k8s-auth-hermes-9 + namespace: vault dependsOn: - name: longhorn - name: helm diff --git a/dockerfiles/Dockerfile.hermes-agent b/dockerfiles/Dockerfile.hermes-agent index ce9ceb46..493c31a4 100644 --- a/dockerfiles/Dockerfile.hermes-agent +++ b/dockerfiles/Dockerfile.hermes-agent @@ -15,6 +15,10 @@ RUN apt-get update \ # Paid/provider-backed search remains selectable through normal Hermes config. RUN uv pip install --python /opt/hermes/.venv/bin/python ddgs==9.14.4 +ARG HERMES_KANIKO_HEREDOC_COMPAT=0 +COPY dockerfiles/Dockerfile.hermes-agent /tmp/hermes-agent.Dockerfile +COPY dockerfiles/hermes-kaniko-heredoc-runner.py /tmp/hermes-kaniko-heredoc-runner.py + # Keep dashboard chat sockets tied to the intended React mount and conversation. # A resumed conversation needs a different PTY attachment key from a fresh chat; # reconnects to that same conversation must keep using the same key. @@ -252,6 +256,7 @@ for (const [before, after, label] of [ } fs.writeFileSync(sidebarPath, sidebar); NODE +RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 1 ;; *) exit 2 ;; esac # The upstream OIDC gate authenticates users but deliberately treats the # dashboard as one shared workstation. Allow a deployment to narrow that @@ -324,6 +329,7 @@ for before, after, label in ( source = source.replace(before, after, 1) path.write_text(source) PY +RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 2 ;; *) exit 2 ;; esac # Give trusted plugins a pre-turn routing hook. It runs after fallback runtime # restoration but before Hermes builds its provider-specific system prompt. @@ -493,6 +499,7 @@ if oneshot.count(oneshot_before) != 1: ) oneshot_path.write_text(oneshot.replace(oneshot_before, oneshot_after, 1)) PY +RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 3 ;; *) exit 2 ;; esac # Hermes WebUI sends its model/provider/reasoning selection on /v1/runs. # Upstream currently applies only statically declared model_routes there, so @@ -628,6 +635,7 @@ runs_source = runs_source.replace(runs_agent_before, runs_agent_after, 1) source = source[:runs_start] + runs_source path.write_text(source) PY +RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 4 ;; *) exit 2 ;; esac # ``create_task(initial_status="blocked")`` is an explicit operator park, but # upstream only treats later block_task() events as sticky. Recognize the @@ -779,6 +787,7 @@ if cli_source.count(help_before) != 1: ) cli_path.write_text(cli_source.replace(help_before, help_after, 1)) PY +RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 5 ;; *) exit 2 ;; esac COPY dockerfiles/hermes-kanban-blocked-regression.py /tmp/hermes-kanban-blocked-regression.py RUN /opt/hermes/.venv/bin/python /tmp/hermes-kanban-blocked-regression.py \ @@ -816,6 +825,7 @@ if source.count(before) != 1: ) path.write_text(source.replace(before, after, 1)) PY +RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 6 ;; *) exit 2 ;; esac # Keep the dashboard's account cards aligned with the managed provider lanes. # Switchyard receives Claude Code subscription OAuth through the environment; @@ -882,6 +892,7 @@ for before, after, label in replacements: path.write_text(source) PY +RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 7 ;; *) exit 2 ;; esac # Make parent-linked API workers first-class live dashboard sessions. They stay # active until the API runner closes them, even during a long final model call; @@ -975,6 +986,7 @@ source = source.replace(row_before, row_after) path.write_text(source) PY +RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 8 ;; *) exit 2 ;; esac # Keep the browser terminal reliable across GPU context loss, make durable # worker lineage visible, expose an accessible API-worker transcript, and @@ -1543,9 +1555,12 @@ function RootRedirect() { fs.writeFileSync(path, source); } NODE +RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 9 ;; *) exit 2 ;; esac COPY dockerfiles/hermes-session-migrate.py /opt/hermes/bin/hermes-session-migrate +RUN rm -f /tmp/hermes-agent.Dockerfile /tmp/hermes-kaniko-heredoc-runner.py + RUN cd /opt/hermes/web \ && npm run build \ && grep -Fq 'await api.getSessions(1, 0' src/pages/ChatPage.tsx \ diff --git a/dockerfiles/Dockerfile.hermes-agent.dockerignore b/dockerfiles/Dockerfile.hermes-agent.dockerignore index 1327bda4..6c13b88f 100644 --- a/dockerfiles/Dockerfile.hermes-agent.dockerignore +++ b/dockerfiles/Dockerfile.hermes-agent.dockerignore @@ -5,3 +5,5 @@ !dockerfiles/hermes-public-extract/** !dockerfiles/hermes-session-activity-panel.tsx !dockerfiles/hermes-session-migrate.py +!dockerfiles/Dockerfile.hermes-agent +!dockerfiles/hermes-kaniko-heredoc-runner.py diff --git a/dockerfiles/hermes-kaniko-heredoc-runner.py b/dockerfiles/hermes-kaniko-heredoc-runner.py new file mode 100644 index 00000000..91a1330a --- /dev/null +++ b/dockerfiles/hermes-kaniko-heredoc-runner.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +"""Replay the reviewed Hermes Dockerfile heredocs for pinned Kaniko v1.23.2. + +Kaniko v1.23.2 parses shell heredoc bodies into ``RunCommand.Files`` but its RUN +implementation executes only ``CmdLine``. The interpreters therefore receive +empty stdin and exit successfully. Docker and BuildKit execute these blocks +normally, so this compatibility runner is enabled only by the Kaniko pipeline. +""" + +from __future__ import annotations + +import argparse +import re +import subprocess +from pathlib import Path + + +MAX_DOCKERFILE_BYTES = 2_000_000 +BLOCKS = ( + ("RUN node <<'NODE'", "NODE", ("node",)), + ("RUN python - <<'PY'", "PY", ("python", "-")), +) +EXPECTED_COMMANDS = ("node", *("python",) * 7, "node") +PARSER_DIRECTIVE = re.compile( + r"^\s*#\s*([a-z]+)\s*=\s*(.*?)\s*$", re.IGNORECASE +) + + +def _escape_character(lines: list[str]) -> str: + """Return the one Dockerfile parser escape directive, or its default.""" + escape = "\\" + directives: set[str] = set() + for line in lines: + if not line.strip(): + break + match = PARSER_DIRECTIVE.fullmatch(line) + if not match: + break + name, value = match.group(1).lower(), match.group(2) + if name in directives: + raise ValueError(f"multiple Dockerfile {name} directives") + directives.add(name) + if name == "escape": + if value not in {"\\", "`"}: + raise ValueError("unsupported Dockerfile escape directive") + escape = value + return escape + + +def _continued(line: str, escape: str) -> bool: + """Match Docker's unescaped continuation marker at physical line end.""" + stripped = line.rstrip() + count = len(stripped) - len(stripped.rstrip(escape)) + return count % 2 == 1 + + +def _logical_instruction( + lines: list[str], index: int, escape: str +) -> tuple[str, int]: + """Normalize escape-newline pairs, including split opcodes/operators.""" + logical = lines[index] + while _continued(logical.split("\n")[-1], escape): + logical = logical.rstrip() + logical = logical[:-1] + index += 1 + while index < len(lines) and lines[index].lstrip().startswith("#"): + index += 1 + if index >= len(lines) or not lines[index].strip(): + raise ValueError("unterminated Dockerfile line continuation") + logical += lines[index].lstrip() + return logical, index + + +def extract_blocks(source: str) -> list[tuple[tuple[str, ...], str]]: + """Inventory every RUN heredoc, then return only the reviewed contract.""" + markers = {start: (end, command) for start, end, command in BLOCKS} + lines = source.splitlines() + escape = _escape_character(lines) + blocks: list[tuple[tuple[str, ...], str]] = [] + index = 0 + while index < len(lines): + marker = markers.get(lines[index]) + if marker is None: + if not lines[index].strip() or lines[index].lstrip().startswith("#"): + index += 1 + continue + logical, index = _logical_instruction(lines, index, escape) + stripped = logical.lstrip() + if ( + stripped[:3].upper() == "RUN" + and (len(stripped) == 3 or stripped[3].isspace()) + and "<<" in stripped + ): + raise ValueError( + "unsupported RUN heredoc outside the exact reviewed contract" + ) + index += 1 + continue + end, command = marker + body_start = index + 1 + index = body_start + while index < len(lines) and lines[index] != end: + index += 1 + if index == len(lines): + raise ValueError(f"unterminated {command[0]} heredoc") + body = "\n".join(lines[body_start:index]) + "\n" + if not body.strip(): + raise ValueError(f"empty {command[0]} heredoc") + blocks.append((command, body)) + index += 1 + + commands = tuple(command[0] for command, _body in blocks) + if commands != EXPECTED_COMMANDS: + raise ValueError( + "Hermes Dockerfile heredoc contract changed: " + f"expected {EXPECTED_COMMANDS!r}, received {commands!r}" + ) + return blocks + + +def replay(dockerfile: Path, block_index: int) -> None: + """Execute one exact heredoc after inventorying the complete Dockerfile.""" + size = dockerfile.stat().st_size + if size < 1 or size > MAX_DOCKERFILE_BYTES: + raise ValueError("Hermes Dockerfile size is outside the reviewed boundary") + source = dockerfile.read_text(encoding="utf-8") + blocks = extract_blocks(source) + if block_index < 1 or block_index > len(blocks): + raise ValueError("heredoc block index is outside the reviewed contract") + command, body = blocks[block_index - 1] + print(f"replaying reviewed Dockerfile heredoc {block_index}/{len(blocks)}") + subprocess.run(command, input=body, text=True, check=True) + + +def main() -> int: + """Parse the one explicit Dockerfile path and replay its reviewed patches.""" + parser = argparse.ArgumentParser() + parser.add_argument("--dockerfile", required=True, type=Path) + parser.add_argument("--block-index", required=True, type=int) + args = parser.parse_args() + replay(args.dockerfile, args.block_index) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/harbor/hermes-agent-immutability-job.yaml b/services/harbor/hermes-agent-immutability-job.yaml new file mode 100644 index 00000000..5df877bb --- /dev/null +++ b/services/harbor/hermes-agent-immutability-job.yaml @@ -0,0 +1,79 @@ +# services/harbor/hermes-agent-immutability-job.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: harbor-hermes-agent-immutability-ensure-1 + namespace: harbor +spec: + backoffLimit: 2 + activeDeadlineSeconds: 600 + template: + metadata: + annotations: + vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/agent-pre-populate-only: "true" + vault.hashicorp.com/agent-run-as-user: "65532" + vault.hashicorp.com/agent-run-as-group: "65532" + vault.hashicorp.com/role: harbor-policy-bootstrap + vault.hashicorp.com/agent-inject-secret-harbor-admin-password: kv/data/atlas/harbor/harbor-core + vault.hashicorp.com/agent-inject-template-harbor-admin-password: | + {{- with secret "kv/data/atlas/harbor/harbor-core" -}} + {{ .Data.data.harbor_admin_password }} + {{- end -}} + spec: + serviceAccountName: harbor-policy-bootstrap + enableServiceLinks: false + restartPolicy: Never + nodeSelector: + hardware: rpi5 + kubernetes.io/arch: arm64 + node-role.kubernetes.io/worker: "true" + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: NotIn + values: [titan-04, titan-14, titan-18, titan-19, titan-24] + securityContext: + fsGroup: 65532 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: + type: RuntimeDefault + containers: + - name: ensure + image: docker.io/library/python@sha256:efcdfa6a6b2fd2afb9c7dfa9a5b288a6f68338b5cfdebe6b637d986067d85757 + imagePullPolicy: IfNotPresent + command: [python3, /scripts/harbor_hermes_agent_immutability_ensure.py] + env: + - name: HARBOR_API_ORIGIN + value: https://registry.bstein.dev/api/v2.0 + - name: HARBOR_ADMIN_PASSWORD_FILE + value: /vault/secrets/harbor-admin-password + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + readOnlyRootFilesystem: true + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 + seccompProfile: + type: RuntimeDefault + volumeMounts: + - name: scripts + mountPath: /scripts + readOnly: true + - name: tmp + mountPath: /tmp + resources: + requests: {cpu: 25m, memory: 32Mi} + limits: {cpu: 250m, memory: 128Mi} + volumes: + - name: scripts + configMap: + name: harbor-hermes-agent-immutability-script + defaultMode: 0555 + - name: tmp + emptyDir: {} diff --git a/services/harbor/kustomization.yaml b/services/harbor/kustomization.yaml index 6a4ce002..7784bb50 100644 --- a/services/harbor/kustomization.yaml +++ b/services/harbor/kustomization.yaml @@ -12,9 +12,14 @@ resources: - certificate.yaml - helmrelease.yaml - vault-sync-deployment.yaml + - policy-bootstrap-serviceaccount.yaml + - hermes-agent-immutability-job.yaml - bootstrap-jobs/cassandra-registry-ensure-job.yaml - image.yaml configMapGenerator: - name: harbor-vault-entrypoint files: - scripts/vault-entrypoint.sh + - name: harbor-hermes-agent-immutability-script + files: + - harbor_hermes_agent_immutability_ensure.py=scripts/harbor_hermes_agent_immutability_ensure.py diff --git a/services/harbor/policy-bootstrap-serviceaccount.yaml b/services/harbor/policy-bootstrap-serviceaccount.yaml new file mode 100644 index 00000000..4839196f --- /dev/null +++ b/services/harbor/policy-bootstrap-serviceaccount.yaml @@ -0,0 +1,7 @@ +# services/harbor/policy-bootstrap-serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: harbor-policy-bootstrap + namespace: harbor +automountServiceAccountToken: true diff --git a/services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py b/services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py new file mode 100644 index 00000000..a313cc80 --- /dev/null +++ b/services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py @@ -0,0 +1,378 @@ +#!/usr/bin/env python3 +"""Create and verify the narrowly scoped Hermes agent immutable-tag rule.""" + +from __future__ import annotations + +import base64 +import json +import os +import time +import urllib.error +import urllib.parse +import urllib.request +from pathlib import Path +from typing import Any + + +PROJECT = "bstein" +REPOSITORY_PATTERN = "hermes-agent" +TAG_PATTERN = "git-*-build-*" +PUBLISH_ROBOT = "robot$jenkins-pipelines" +EXPECTED_ORIGIN = "https://registry.bstein.dev/api/v2.0" +MAX_RESPONSE = 1_048_576 +TRANSIENT_STATUSES = {429, 502, 503, 504} +EXPECTED_RULE = { + "disabled": False, + "action": "immutable", + "template": "immutable_template", + "tag_selectors": [ + { + "kind": "doublestar", + "decoration": "matches", + "pattern": TAG_PATTERN, + } + ], + "scope_selectors": { + "repository": [ + { + "kind": "doublestar", + "decoration": "repoMatches", + "pattern": REPOSITORY_PATTERN, + } + ] + }, +} + + +class NoRedirect(urllib.request.HTTPRedirectHandler): + """Prevent Basic credentials from following an unexpected redirect.""" + + def redirect_request(self, _request, _file, _code, _message, _headers, _url): + return None + + +class HarborUnavailable(RuntimeError): + """Harbor is not ready yet, rather than returning a policy decision.""" + + +def normalized_rule(rule: dict[str, Any]) -> dict[str, Any]: + """Return only the immutable contract fields Harbor must preserve.""" + return { + "disabled": bool(rule.get("disabled", False)), + "action": rule.get("action"), + "template": rule.get("template"), + "tag_selectors": [ + { + "kind": selector.get("kind"), + "decoration": selector.get("decoration"), + "pattern": selector.get("pattern"), + } + for selector in rule.get("tag_selectors") or [] + if isinstance(selector, dict) + ], + "scope_selectors": { + "repository": [ + { + "kind": selector.get("kind"), + "decoration": selector.get("decoration"), + "pattern": selector.get("pattern"), + } + for selector in (rule.get("scope_selectors") or {}).get( + "repository", [] + ) + if isinstance(selector, dict) + ] + }, + } + + +def targets_hermes_builds(rule: dict[str, Any]) -> bool: + """Detect a rule that claims this exact repository and tag selector.""" + normalized = normalized_rule(rule) + return ( + normalized["tag_selectors"] == EXPECTED_RULE["tag_selectors"] + and normalized["scope_selectors"] == EXPECTED_RULE["scope_selectors"] + ) + + +class HarborClient: + """Bounded same-origin client for Harbor's immutable-tag API.""" + + def __init__(self, origin: str, username: str, password: str) -> None: + normalized_origin = origin.rstrip("/") + if normalized_origin != EXPECTED_ORIGIN: + raise ValueError("Harbor API origin is not the pinned production API") + self.origin = normalized_origin + token = base64.b64encode(f"{username}:{password}".encode()).decode("ascii") + self.headers = {"Authorization": f"Basic {token}"} + self.opener = urllib.request.build_opener(NoRedirect()) + + def request( + self, method: str, path: str, payload: dict[str, Any] | None = None + ) -> tuple[int, bytes, dict[str, str]]: + """Issue one request, returning even non-2xx responses for strict checks.""" + data = None + headers = dict(self.headers) + if payload is not None: + data = json.dumps(payload, separators=(",", ":")).encode() + headers["Content-Type"] = "application/json" + request = urllib.request.Request( + f"{self.origin}{path}", data=data, headers=headers, method=method + ) + try: + response = self.opener.open(request, timeout=20) + except urllib.error.HTTPError as exc: + response = exc + except (urllib.error.URLError, TimeoutError) as exc: + raise HarborUnavailable("Harbor policy API is unavailable") from exc + with response: + body = response.read(MAX_RESPONSE + 1) + if len(body) > MAX_RESPONSE: + raise RuntimeError("Harbor response exceeded the size limit") + return int(response.status), body, dict(response.headers) + + +def list_rules(client: HarborClient) -> list[dict[str, Any]]: + """Read and validate the complete small rule set for the project.""" + path = f"/projects/{PROJECT}/immutabletagrules?page=1&page_size=100" + status, body, headers = client.request("GET", path) + if status in TRANSIENT_STATUSES: + raise HarborUnavailable(f"Harbor immutable rule list returned HTTP {status}") + if status != 200: + raise RuntimeError(f"Harbor immutable rule list returned HTTP {status}") + values = _json_list(body, "immutable rule") + _require_complete_page(values, headers, "immutable rule") + return values + + +def _json_list(body: bytes, label: str) -> list[dict[str, Any]]: + """Decode one bounded Harbor list without accepting a partial shape.""" + try: + value = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise RuntimeError(f"Harbor returned invalid {label} JSON") from exc + if not isinstance(value, list) or not all(isinstance(item, dict) for item in value): + raise RuntimeError(f"Harbor {label} list has an invalid shape") + return value + + +def _require_complete_page( + values: list[dict[str, Any]], headers: dict[str, str], label: str +) -> None: + """Reject a truncated first page or a proxy that strips count evidence.""" + raw_total = next( + (value for key, value in headers.items() if key.lower() == "x-total-count"), + None, + ) + if raw_total is None or not str(raw_total).isdecimal(): + raise RuntimeError(f"Harbor {label} list omitted a valid total count") + if int(raw_total) != len(values): + raise RuntimeError(f"Harbor {label} list was truncated") + + +def _allowed(access: dict[str, Any], resource: str, action: str) -> bool: + """Match one positive robot permission, accepting Harbor's default effect.""" + return ( + access.get("resource") == resource + and access.get("action") == action + and access.get("effect") in (None, "", "allow") + ) + + +def _publisher_scope(robot: dict[str, Any]) -> tuple[dict[str, Any], list[dict[str, Any]]]: + """Return the existing bstein scope after validating its push boundary.""" + permissions = robot.get("permissions") + if not isinstance(permissions, list) or not all( + isinstance(item, dict) for item in permissions + ): + raise RuntimeError("Jenkins Harbor robot permissions have an invalid shape") + matches = [ + item + for item in permissions + if item.get("kind") == "project" and item.get("namespace") == PROJECT + ] + if len(matches) != 1: + raise RuntimeError("Jenkins Harbor robot must have one existing bstein scope") + access = matches[0].get("access") + if not isinstance(access, list) or not all(isinstance(item, dict) for item in access): + raise RuntimeError("Jenkins Harbor robot bstein access has an invalid shape") + for required in (("repository", "pull"), ("repository", "push")): + if not any(_allowed(item, *required) for item in access): + raise RuntimeError("Jenkins Harbor robot lacks its existing pull/push boundary") + immutable_access = [ + item for item in access if item.get("resource") == "immutable-tag" + ] + if immutable_access and not ( + len(immutable_access) == 1 + and _allowed(immutable_access[0], "immutable-tag", "list") + ): + raise RuntimeError("Jenkins Harbor robot has broader immutable-tag access") + return matches[0], access + + +def _verified_publisher(robot: dict[str, Any]) -> bool: + """Check the exact identity and least-privilege policy after persistence.""" + if ( + robot.get("name") != PUBLISH_ROBOT + or robot.get("level") != "system" + or robot.get("editable") is not True + or robot.get("disable") is not False + ): + return False + try: + _scope, access = _publisher_scope(robot) + except RuntimeError: + return False + return any(_allowed(item, "immutable-tag", "list") for item in access) + + +def ensure_publisher_can_read_rule(client: HarborClient) -> int: + """Grant only immutable-tag:list to the existing Jenkins push robot.""" + status, body, headers = client.request("GET", "/robots?page=1&page_size=100") + if status in TRANSIENT_STATUSES: + raise HarborUnavailable(f"Harbor robot list returned HTTP {status}") + if status != 200: + raise RuntimeError(f"Harbor robot list returned HTTP {status}") + robots = _json_list(body, "robot") + _require_complete_page(robots, headers, "robot") + matches = [ + item + for item in robots + if item.get("name") == PUBLISH_ROBOT + ] + if len(matches) != 1: + raise RuntimeError("expected exactly one Jenkins Harbor publisher robot") + robot_id = matches[0].get("id") + if not isinstance(robot_id, int) or robot_id < 1: + raise RuntimeError("Jenkins Harbor robot omitted a valid ID") + status, body, _headers = client.request("GET", f"/robots/{robot_id}") + if status in TRANSIENT_STATUSES: + raise HarborUnavailable(f"Harbor robot read returned HTTP {status}") + if status != 200: + raise RuntimeError(f"Harbor robot read returned HTTP {status}") + try: + robot = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise RuntimeError("Harbor returned invalid robot JSON") from exc + if not isinstance(robot, dict): + raise RuntimeError("Harbor robot response has an invalid shape") + if ( + robot.get("name") != PUBLISH_ROBOT + or robot.get("level") != "system" + or robot.get("editable") is not True + or robot.get("disable") is not False + ): + raise RuntimeError("Jenkins Harbor publisher identity is not active and exact") + scope, access = _publisher_scope(robot) + if any(_allowed(item, "immutable-tag", "list") for item in access): + return robot_id + + # Harbor updates permissions through PUT /robots/{id}; its separate PATCH + # endpoint is the only secret-rotation operation. Preserve every current + # scope and field while adding the one read-only action. + access.append({"resource": "immutable-tag", "action": "list"}) + payload = { + "name": robot["name"], + "description": robot.get("description") or "", + "level": robot["level"], + "disable": robot["disable"], + "permissions": robot["permissions"], + } + duration = robot.get("duration") + if duration is not None: + if not isinstance(duration, int): + raise RuntimeError("Jenkins Harbor robot duration has an invalid shape") + payload["duration"] = duration + # Keep the validated object reference live in the preserved permission list. + if scope.get("access") is not access: + raise RuntimeError("Jenkins Harbor robot permission normalization drifted") + status, _body, _headers = client.request( + "PUT", f"/robots/{robot_id}", payload + ) + if status in TRANSIENT_STATUSES: + raise HarborUnavailable(f"Harbor robot update returned HTTP {status}") + if status != 200: + raise RuntimeError(f"Harbor robot policy update returned HTTP {status}") + + for attempt in range(1, 6): + status, body, _headers = client.request("GET", f"/robots/{robot_id}") + if status == 200: + try: + persisted = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError): + persisted = None + if isinstance(persisted, dict) and _verified_publisher(persisted): + return robot_id + if attempt < 5: + time.sleep(attempt) + raise RuntimeError("Jenkins Harbor robot policy did not verify exactly") + + +def ensure_rule(client: HarborClient) -> int: + """Create once, or validate the one exact enabled rule already present.""" + rules = list_rules(client) + matches = [rule for rule in rules if targets_hermes_builds(rule)] + if len(matches) > 1: + raise RuntimeError("multiple Hermes agent immutable rules exist") + if matches: + if normalized_rule(matches[0]) != EXPECTED_RULE: + raise RuntimeError("Hermes agent immutable rule exists but is not enabled/exact") + rule_id = matches[0].get("id") + if not isinstance(rule_id, int) or rule_id < 1: + raise RuntimeError("Harbor immutable rule omitted a valid ID") + return rule_id + + path = f"/projects/{PROJECT}/immutabletagrules" + status, _body, headers = client.request("POST", path, EXPECTED_RULE) + if status in TRANSIENT_STATUSES: + raise HarborUnavailable(f"Harbor immutable rule create returned HTTP {status}") + if status != 201: + raise RuntimeError(f"Harbor immutable rule create returned HTTP {status}") + location = headers.get("Location") or headers.get("location") or "" + api_path = urllib.parse.urlsplit(client.origin).path.rstrip("/") + expected_prefix = f"{api_path}{path}/" + if not location.startswith(expected_prefix): + raise RuntimeError("Harbor immutable rule create omitted the exact Location") + suffix = location[len(expected_prefix) :] + if not suffix.isdecimal() or int(suffix) < 1: + raise RuntimeError("Harbor immutable rule Location has an invalid ID") + + # Harbor persists synchronously, but bounded retries distinguish a stale + # read from accepting an unverified policy. + for attempt in range(1, 6): + matches = [rule for rule in list_rules(client) if targets_hermes_builds(rule)] + if len(matches) == 1 and normalized_rule(matches[0]) == EXPECTED_RULE: + rule_id = matches[0].get("id") + if rule_id == int(suffix): + return rule_id + if attempt < 5: + time.sleep(attempt) + raise RuntimeError("created Harbor immutable rule did not verify exactly") + + +def main() -> int: + """Load the runtime-only admin credential and enforce the tracked policy.""" + origin = os.environ.get("HARBOR_API_ORIGIN", "") + password_file = Path(os.environ.get("HARBOR_ADMIN_PASSWORD_FILE", "")) + password = password_file.read_text(encoding="utf-8").strip() + if not password: + raise RuntimeError("Harbor admin password is empty") + client = HarborClient(origin, "admin", password) + for attempt in range(1, 13): + try: + rule_id = ensure_rule(client) + robot_id = ensure_publisher_can_read_rule(client) + break + except HarborUnavailable: + if attempt == 12: + raise + time.sleep(min(attempt * 2, 15)) + print( + "Hermes agent immutable build-tag rule is active " + f"(id={rule_id}); publisher preflight access is active (robot={robot_id})" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/hermes/NOTES.md b/services/hermes/NOTES.md index c70e528a..aec53d0c 100644 --- a/services/hermes/NOTES.md +++ b/services/hermes/NOTES.md @@ -102,6 +102,110 @@ console tails and named artifact contents in its deterministic bundle. A report must say `retained Ariadne evidence` when that fallback is used; it must not pretend direct Jenkins access succeeded. +## Publishing an agent image after review + +The `hermes-agent-image` Jenkins job is the only supported agent image builder. +It runs daemonless Kaniko v1.23.2 under the unbound `hermes-image-builder` +ServiceAccount without a service-account token, host socket, +privileged container, or writable Git credential. It accepts only an exact +40-character revision that is both the checked-out commit and current +`atlas/titan-iac` `main`, so a human must merge the source PR first. + +The build pod requires an ARM64 worker and gives `hardware=rpi5` the maximum +scheduler preference. Healthy rpi4 workers remain valid fallback capacity when +the rpi5 pool cannot fit the full request. Cordoned, unhealthy, amd64, and +reserved last-resort nodes remain excluded by the required placement boundary. + +Kaniko runs as UID 0 because it must unpack an image root filesystem and enter +that filesystem for Dockerfile `RUN` instructions. Its capability set is the +minimum proven by an exact ARM64 no-push build of this Dockerfile: +`CHOWN`, `FOWNER`, `DAC_OVERRIDE`, `SETGID`, and `SETUID`. `SETGID` is required +because pinned Kaniko applies the base image's supplementary group list before +each Dockerfile `RUN`; `SETUID` lets apt drop privileges to its `_apt` account +while downloading package indexes. It still has no privilege +escalation, service-account token, host path, daemon socket, or Docker/BuildKit +TCP endpoint and uses the runtime-default seccomp profile. This is residual +root-in-the-build-pod risk, bounded to disposable `emptyDir` storage and a +human-reviewed `main` revision; Dockerfile changes require the same scrutiny as +executable cluster code. + +Pinned Kaniko v1.23.2 parses Dockerfile `RUN` heredocs but does not materialize +their inline files when executing a command. The Jenkins lane therefore enables +a bounded compatibility replay after copying the reviewed Dockerfile and runner +from the same checked-out commit. Before every replay, the runner inventories +the entire Dockerfile as logical instructions, including split opcodes, +operators, and either Dockerfile escape character. It rejects every heredoc +form outside the exact nine Node/Python blocks before launching an interpreter. +Each accepted body is then replayed immediately after its corresponding Kaniko +`RUN`, preserving source order before any dependent regression. Docker and +BuildKit keep their native behavior because the compatibility argument defaults +off. The final TypeScript build, Python compile, and source assertions remain +mandatory, so an omitted or drifted replay fails before any image can pass +release verification. + +From agent.hermes, trigger that one fixed job with: + +```sh +jenkins_image_build_trigger.py '' +``` + +The helper has no general Jenkins credential or caller-selectable job name. Its +Vault-projected token is bound by Jenkins only to `hermes-agent-image`. The job +also requires its fixed publish confirmation. Each run claims the unique tag +`git--build-` and refuses to overwrite an +existing tag. It cross-checks Kaniko's digest and tagged-image evidence, then +independently reads the pushed tag from Harbor before archiving a JSON record +and Flux digest patch. A successful post-condition then revalidates the exact +six-file evidence set and archives those six explicit paths with empty archives +forbidden. Apply that patch on a new branch and submit it for human review; the +build never changes Git, reconciles Flux, or deploys by itself. +The helper posts only to the fixed HTTPS Build Token Root endpoint and accepts +only its real queue responses: HTTP 201 or a non-followed HTTP 303 with an exact +same-origin `/queue/item//` location. An unauthenticated request is +denied, and the token cannot select, configure, read, or administer another +Jenkins job. The pipeline independently rejects any commit that is not the +current `origin/main`, preserving the human merge/review boundary. + +Harbor independently enforces an enabled immutable-tag rule scoped to only the +`bstein/hermes-agent` repository and `git-*-build-*` tags. A second manifest PUT, +retag, or deletion is rejected by Harbor even if a caller bypasses the Jenkins +preflight. A revisioned, Flux-tracked policy Job creates or verifies that exact +rule with a runtime-only Vault credential; it refuses to alter a conflicting +rule and retries only explicit transport/readiness failures while Harbor +starts. The same Job grants the existing Jenkins publisher only +`immutable-tag:list` on `bstein`, preserving its other project scopes and never +calling Harbor's separate secret-rotation endpoint. The pipeline uses that +read-only permission to require the exact enabled rule before Kaniko starts. +That check also makes an already-running Jenkins controller fail closed during +rollout; Jenkins cannot become Ready on the new revision before the policy Job +succeeds. + +The tracked rollout order is deliberate: the revisioned Vault role Job must +complete before the Vault Kustomization becomes Ready. A separate Flux +Kustomization then runs the seed Job under the dedicated +`hermes-jenkins-token-seed` identity, which can only create/read the exact +`kv/data/atlas/hermes/developer-jenkins` path and request random bytes. Jenkins +depends on both that seed and the Harbor policy Job; Hermes depends on Jenkins. +The seeder uses KV-v2 CAS create-only semantics. It never changes an existing +token, never fills a missing field in an existing secret, and fails closed when +it cannot distinguish absence from a read error. The recurring Vault +configuration job reconciles the same narrow role and policy. + +Rotate this fixed-job token only as a coordinated operator action: + +1. Stop new `hermes-agent-image` triggers and wait for its queue and executor to + drain. +2. Read the current KV-v2 metadata version for + `kv/atlas/hermes/developer-jenkins`. +3. Generate a fresh value from Vault and patch only `build_token` with + `vault kv patch -cas=`. Keep the value in a mode-0600 + temporary file or standard input, never a command argument or log. +4. Roll Jenkins first and wait until it is Ready, then roll `hermes-agent` and + wait until it is Ready. Do not resume triggers between those two consumers. +5. Run one reviewed-main canary. If rollback is required, repeat the CAS patch + with the previous value as another coordinated rotation; never delete the + secret to make the seeder recreate it. + ## The actual supervised triage algorithm 1. Classify the request as test/build triage, service health, or alert tuning. diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml index 5ed758cf..028c5d70 100644 --- a/services/hermes/agent-deployment.yaml +++ b/services/hermes/agent-deployment.yaml @@ -56,6 +56,11 @@ spec: {{- with secret "kv/data/atlas/hermes/developer-gitea" -}} {{ .Data.data.username }} {{- end }} + vault.hashicorp.com/agent-inject-secret-jenkins-image-build-token: kv/data/atlas/hermes/developer-jenkins + vault.hashicorp.com/agent-inject-template-jenkins-image-build-token: | + {{- with secret "kv/data/atlas/hermes/developer-jenkins" -}} + {{ .Data.data.build_token }} + {{- end }} vault.hashicorp.com/agent-inject-secret-node-ssh-private-key: kv/data/atlas/hermes/developer-ssh vault.hashicorp.com/agent-inject-template-node-ssh-private-key: | {{- with secret "kv/data/atlas/hermes/developer-ssh" -}} diff --git a/services/hermes/kustomization.yaml b/services/hermes/kustomization.yaml index 3b9c2345..6009cd7b 100644 --- a/services/hermes/kustomization.yaml +++ b/services/hermes/kustomization.yaml @@ -77,6 +77,7 @@ configMapGenerator: - image_broker.py=scripts/image_broker.py - install_agent_tools.sh=scripts/install_agent_tools.sh - jenkins_build_evidence.py=scripts/jenkins_build_evidence.py + - jenkins_image_build_trigger.py=scripts/jenkins_image_build_trigger.py - kanban_status_recovery.py=scripts/kanban_status_recovery.py - migrate_herdr_state.py=scripts/migrate_herdr_state.py - migrate_api_session_lineage.py=scripts/migrate_api_session_lineage.py diff --git a/services/hermes/scripts/jenkins_image_build_trigger.py b/services/hermes/scripts/jenkins_image_build_trigger.py new file mode 100755 index 00000000..6cd91ed8 --- /dev/null +++ b/services/hermes/scripts/jenkins_image_build_trigger.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +"""Trigger only the reviewed-main Hermes agent image release job.""" + +from __future__ import annotations + +import argparse +import json +import re +import urllib.error +import urllib.parse +import urllib.request +from pathlib import Path + + +JENKINS_ORIGIN = "https://ci.bstein.dev" +JENKINS_BUILD_URL = f"{JENKINS_ORIGIN}/buildByToken/buildWithParameters" +JOB_NAME = "hermes-agent-image" +TOKEN_FILE = Path("/runtime-access/jenkins-image-build-token") +REVISION_PATTERN = re.compile(r"^[0-9a-f]{40}$") +QUEUE_PATH_PATTERN = re.compile(r"^/queue/item/[0-9]+/?$") + + +class _NoRedirect(urllib.request.HTTPRedirectHandler): + """Keep a queued-build redirect from becoming an unauthorized job read.""" + + def redirect_request(self, _request, _file, _code, _message, _headers, _url): + return None + + +def _open_without_redirect(request: urllib.request.Request, timeout: int): + """Return the Build Token Root response, including its expected HTTP 303.""" + opener = urllib.request.build_opener(_NoRedirect()) + try: + return opener.open(request, timeout=timeout) + except urllib.error.HTTPError as exc: + if exc.code == 303: + return exc + raise + + +def trigger_build( + revision: str, + *, + token_file: Path = TOKEN_FILE, + opener=_open_without_redirect, +) -> dict[str, str | int]: + """Post the fixed job parameters using its job-scoped build token.""" + revision = revision.strip() + if not REVISION_PATTERN.fullmatch(revision): + raise ValueError("revision must be a lowercase full 40-character commit") + token = token_file.read_text(encoding="utf-8").strip() + if not token: + raise RuntimeError("Jenkins image-build token is empty") + payload = urllib.parse.urlencode( + { + "job": JOB_NAME, + "token": token, + "PUBLISH_IMAGE": "true", + "EXPECTED_SOURCE_REVISION": revision, + "CONFIRM_PUBLISH": "PUBLISH HERMES AGENT", + } + ).encode("utf-8") + request = urllib.request.Request( + JENKINS_BUILD_URL, + data=payload, + headers={"Content-Type": "application/x-www-form-urlencoded"}, + method="POST", + ) + with opener(request, timeout=20) as response: + status = int(response.status) + location = response.headers.get("Location", "") + if status not in {201, 303}: + raise RuntimeError(f"Jenkins trigger returned HTTP {status}") + if not location: + raise RuntimeError("Jenkins trigger omitted the queue Location") + queue_url = urllib.parse.urljoin(f"{JENKINS_ORIGIN}/", location) + parsed_queue = urllib.parse.urlsplit(queue_url) + expected_origin = urllib.parse.urlsplit(JENKINS_ORIGIN) + if ( + parsed_queue.scheme != expected_origin.scheme + or parsed_queue.netloc != expected_origin.netloc + or parsed_queue.query + or parsed_queue.fragment + or not QUEUE_PATH_PATTERN.fullmatch(parsed_queue.path) + ): + raise RuntimeError("Jenkins returned an invalid queue Location") + # Never return the submitted URL: its form body contains the job token. + queue_path = parsed_queue.path + return { + "job": JOB_NAME, + "queue_path": queue_path, + "source_revision": revision, + "status": status, + } + + +def main() -> int: + """Validate one revision, trigger the bounded job, and print safe metadata.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("revision", help="reviewed full commit currently on main") + args = parser.parse_args() + try: + result = trigger_build(args.revision) + except (OSError, ValueError, RuntimeError, urllib.error.URLError) as exc: + print(json.dumps({"error": str(exc)}, sort_keys=True)) + return 1 + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": # pragma: no cover - exercised through main() + raise SystemExit(main()) diff --git a/services/hermes/scripts/stage_runtime_access.py b/services/hermes/scripts/stage_runtime_access.py index f4d73160..e8276a5b 100644 --- a/services/hermes/scripts/stage_runtime_access.py +++ b/services/hermes/scripts/stage_runtime_access.py @@ -95,6 +95,7 @@ def stage_agent() -> None: "chat-relay-key", "gitea-token", "gitea-username", + "jenkins-image-build-token", "node-ssh-private-key", "node-ssh-config", "node-ssh-known-hosts", diff --git a/services/hermes/switchyard-deployment.yaml b/services/hermes/switchyard-deployment.yaml index 48143682..456d0a1a 100644 --- a/services/hermes/switchyard-deployment.yaml +++ b/services/hermes/switchyard-deployment.yaml @@ -27,7 +27,7 @@ spec: prometheus.io/port: "9005" prometheus.io/path: /metrics vault.hashicorp.com/agent-inject: "true" - vault.hashicorp.com/role: hermes-agent + vault.hashicorp.com/role: hermes-switchyard vault.hashicorp.com/agent-inject-secret-relay-key: kv/data/atlas/hermes/chat-telegram vault.hashicorp.com/agent-inject-template-relay-key: | {{- with secret "kv/data/atlas/hermes/chat-telegram" -}} diff --git a/services/jenkins/configmap-jcasc.yaml b/services/jenkins/configmap-jcasc.yaml index e9aa57b9..2a1e81eb 100644 --- a/services/jenkins/configmap-jcasc.yaml +++ b/services/jenkins/configmap-jcasc.yaml @@ -652,6 +652,25 @@ data: } } } + pipelineJob('hermes-agent-image') { + disabled(false) + description('Human-gated, daemonless Kaniko build for the reviewed atlas/titan-iac main revision. Publishes a content-addressed Hermes agent image and archives a Flux digest patch; it never mutates Git or deploys.') + authenticationToken(System.getenv('HERMES_AGENT_IMAGE_BUILD_TOKEN')) + definition { + cpsScm { + scm { + git { + remote { + url('https://scm.bstein.dev/atlas/titan-iac.git') + credentials('gitea-pat') + } + branches('*/main') + } + } + scriptPath('ci/Jenkinsfile.hermes-agent-image') + } + } + } multibranchPipelineJob('titan-iac-quality-gate') { branchSources { branchSource { diff --git a/services/jenkins/configmap-plugins.yaml b/services/jenkins/configmap-plugins.yaml index 049bc1bd..ea4b450d 100644 --- a/services/jenkins/configmap-plugins.yaml +++ b/services/jenkins/configmap-plugins.yaml @@ -20,6 +20,7 @@ data: gitea:268.v75e47974c01d gitea-checks:603.621.vc708da_fb_371d multibranch-scan-webhook-trigger:1.0.11 + build-token-root:365.v717f8685a_09e # Structured test evidence. Without junit the `junit` step throws # NoSuchMethodError, jenkins.failed_tests is always empty, and triage # has only raw console text to work from. Pinned to the newest release diff --git a/services/jenkins/deployment.yaml b/services/jenkins/deployment.yaml index 1fa22374..46109da7 100644 --- a/services/jenkins/deployment.yaml +++ b/services/jenkins/deployment.yaml @@ -65,6 +65,9 @@ spec: ARIADNE_JENKINS_API_USER={{ .Data.data.username }} ARIADNE_JENKINS_API_TOKEN={{ .Data.data.token }} {{ end }} + {{ with secret "kv/data/atlas/hermes/developer-jenkins" }} + HERMES_AGENT_IMAGE_BUILD_TOKEN={{ .Data.data.build_token }} + {{ end }} bstein.dev/restarted-at: "2026-05-20T09:40:31Z" spec: serviceAccountName: jenkins diff --git a/services/jenkins/hermes-image-builder-serviceaccount.yaml b/services/jenkins/hermes-image-builder-serviceaccount.yaml new file mode 100644 index 00000000..98279417 --- /dev/null +++ b/services/jenkins/hermes-image-builder-serviceaccount.yaml @@ -0,0 +1,7 @@ +# services/jenkins/hermes-image-builder-serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: hermes-image-builder + namespace: jenkins +automountServiceAccountToken: false diff --git a/services/jenkins/kustomization.yaml b/services/jenkins/kustomization.yaml index b69d7642..5f3e0554 100644 --- a/services/jenkins/kustomization.yaml +++ b/services/jenkins/kustomization.yaml @@ -5,6 +5,7 @@ namespace: jenkins resources: - namespace.yaml - serviceaccount.yaml + - hermes-image-builder-serviceaccount.yaml - vault-serviceaccount.yaml - pvc.yaml - cache-pvc.yaml diff --git a/services/jenkins/secretproviderclass.yaml b/services/jenkins/secretproviderclass.yaml index a9d9dd50..148f193f 100644 --- a/services/jenkins/secretproviderclass.yaml +++ b/services/jenkins/secretproviderclass.yaml @@ -8,7 +8,7 @@ spec: provider: vault parameters: vaultAddress: "http://vault.vault.svc.cluster.local:8200" - roleName: "jenkins" + roleName: "jenkins-vault-sync" objects: | - objectName: "harbor-pull__dockerconfigjson" secretPath: "kv/data/atlas/shared/harbor-pull" diff --git a/services/vault-hermes-jenkins-token-seed/job.yaml b/services/vault-hermes-jenkins-token-seed/job.yaml new file mode 100644 index 00000000..17f7a4ca --- /dev/null +++ b/services/vault-hermes-jenkins-token-seed/job.yaml @@ -0,0 +1,68 @@ +# services/vault-hermes-jenkins-token-seed/job.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: vault-hermes-jenkins-build-token-seed-2 + namespace: vault +spec: + backoffLimit: 2 + template: + spec: + serviceAccountName: hermes-jenkins-token-seed + enableServiceLinks: false + restartPolicy: Never + nodeSelector: + hardware: rpi5 + kubernetes.io/arch: arm64 + node-role.kubernetes.io/worker: "true" + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: NotIn + values: [titan-04, titan-14, titan-18, titan-19, titan-24] + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: + type: RuntimeDefault + containers: + - name: seed + image: docker.io/hashicorp/vault@sha256:4e33b126a59c0c333b76fb4e894722462659a6bec7c48c9ee8cea56fccfd2569 + imagePullPolicy: IfNotPresent + command: [sh, /scripts/vault_hermes_jenkins_build_token_ensure.sh] + env: + - name: HOME + value: /tmp + - name: VAULT_ADDR + value: http://vault.vault.svc.cluster.local:8200 + - name: VAULT_K8S_ROLE + value: hermes-jenkins-token-seed + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + readOnlyRootFilesystem: true + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 100 + seccompProfile: + type: RuntimeDefault + volumeMounts: + - name: scripts + mountPath: /scripts + readOnly: true + - name: tmp + mountPath: /tmp + resources: + requests: {cpu: 25m, memory: 32Mi} + limits: {cpu: 250m, memory: 128Mi} + volumes: + - name: scripts + configMap: + name: vault-hermes-jenkins-token-seed-script + defaultMode: 0555 + - name: tmp + emptyDir: {} diff --git a/services/vault-hermes-jenkins-token-seed/kustomization.yaml b/services/vault-hermes-jenkins-token-seed/kustomization.yaml new file mode 100644 index 00000000..920caa50 --- /dev/null +++ b/services/vault-hermes-jenkins-token-seed/kustomization.yaml @@ -0,0 +1,13 @@ +# services/vault-hermes-jenkins-token-seed/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: vault +resources: + - serviceaccount.yaml + - job.yaml +generatorOptions: + disableNameSuffixHash: true +configMapGenerator: + - name: vault-hermes-jenkins-token-seed-script + files: + - vault_hermes_jenkins_build_token_ensure.sh=scripts/vault_hermes_jenkins_build_token_ensure.sh diff --git a/services/vault-hermes-jenkins-token-seed/scripts/vault_hermes_jenkins_build_token_ensure.sh b/services/vault-hermes-jenkins-token-seed/scripts/vault_hermes_jenkins_build_token_ensure.sh new file mode 100644 index 00000000..1514b248 --- /dev/null +++ b/services/vault-hermes-jenkins-token-seed/scripts/vault_hermes_jenkins_build_token_ensure.sh @@ -0,0 +1,167 @@ +#!/usr/bin/env sh +set -eu + +secret_path="kv/atlas/hermes/developer-jenkins" +secret_api_path="kv/data/atlas/hermes/developer-jenkins" +jwt_file="${VAULT_K8S_JWT_FILE:-/var/run/secrets/kubernetes.io/serviceaccount/token}" +vault_role="${VAULT_K8S_ROLE:-hermes-jenkins-token-seed}" +payload_file="${TMPDIR:-/tmp}/hermes-jenkins-token.json" + +log() { printf '[hermes-jenkins-token] %s\n' "$*" >&2; } +cleanup() { rm -f "${payload_file}"; } +trap cleanup EXIT HUP INT TERM + +retry_command() { + attempt=1 + while [ "${attempt}" -le 5 ]; do + set +e + command_output="$("$@" 2>&1)" + command_status=$? + set -e + if [ "${command_status}" -eq 0 ]; then + printf '%s' "${command_output}" + return 0 + fi + if [ "${attempt}" -lt 5 ]; then + sleep "${attempt}" + fi + attempt=$((attempt + 1)) + done + return "${command_status}" +} + +ensure_token() { + if [ -n "${VAULT_TOKEN:-}" ]; then + return + fi + jwt="$(cat "${jwt_file}")" + VAULT_TOKEN="$(retry_command vault write -field=token \ + auth/kubernetes/login role="${vault_role}" jwt="${jwt}")" || { + log "Vault Kubernetes login failed after retries" + exit 1 + } + unset jwt + if [ -z "${VAULT_TOKEN}" ]; then + log "Vault Kubernetes login returned an empty token" + exit 1 + fi + export VAULT_TOKEN +} + +# Return 0 for the complete field, 10 when absent, 11 when present but +# incomplete, and 12 for a persistent authorization/transport failure. +read_build_token() { + attempt=1 + while [ "${attempt}" -le 5 ]; do + set +e + secret_json="$(vault read -format=json "${secret_api_path}" 2>&1)" + secret_status=$? + set -e + if [ "${secret_status}" -eq 0 ]; then + unset secret_json + set +e + build_token="$(vault kv get -field=build_token "${secret_path}" 2>&1)" + field_status=$? + set -e + if [ "${field_status}" -eq 0 ]; then + case "${build_token}" in + ''|*[!0-9a-f]*) unset build_token; return 11 ;; + esac + if [ "${#build_token}" -eq 64 ]; then + unset build_token + return 0 + fi + unset build_token + return 11 + fi + if printf '%s' "${build_token}" | grep -q 'No value found'; then + unset build_token + return 11 + fi + unset build_token + elif printf '%s' "${secret_json}" | grep -Eq 'Code: 404|No value found at'; then + unset secret_json + return 10 + fi + if [ "${attempt}" -lt 5 ]; then + sleep "${attempt}" + fi + attempt=$((attempt + 1)) + done + unset secret_json + return 12 +} + +ensure_token + +if read_build_token; then + read_status=0 +else + read_status=$? +fi +case "${read_status}" in + 0) + log "job-scoped token already present; no write performed" + exit 0 + ;; + 11) + log "secret exists without a valid build_token; refusing to overwrite existing fields" + exit 1 + ;; + 12) + log "secret read failed after retries; refusing to seed" + exit 1 + ;; + 10) ;; + *) + log "unexpected secret read state ${read_status}" + exit 1 + ;; +esac + +new_token="$(retry_command vault write -field=random_bytes \ + sys/tools/random/32 format=hex)" || { + log "Vault random token generation failed after retries" + exit 1 + } +case "${new_token}" in + ''|*[!0-9a-f]*) + unset new_token + log "Vault returned an invalid random token" + exit 1 + ;; +esac +if [ "${#new_token}" -ne 64 ]; then + unset new_token + log "Vault returned an invalid random token" + exit 1 +fi + +umask 077 +printf '{"options":{"cas":0},"data":{"build_token":"%s"}}\n' \ + "${new_token}" > "${payload_file}" +unset new_token + +set +e +create_error="$(vault write "${secret_api_path}" @"${payload_file}" 2>&1 >/dev/null)" +create_status=$? +set -e +cleanup +if [ "${create_status}" -eq 0 ]; then + log "job-scoped token created with KV-v2 CAS create-only semantics" + exit 0 +fi + +if printf '%s' "${create_error}" | grep -qi 'check-and-set'; then + unset create_error + if read_build_token; then + log "another seeder won CAS; existing token preserved" + exit 0 + fi + log "CAS race did not produce a readable build_token; refusing to continue" + exit 1 +fi + +unset create_error +log "KV-v2 CAS create failed; refusing to retry a write" +exit 1 diff --git a/services/vault-hermes-jenkins-token-seed/serviceaccount.yaml b/services/vault-hermes-jenkins-token-seed/serviceaccount.yaml new file mode 100644 index 00000000..962471bd --- /dev/null +++ b/services/vault-hermes-jenkins-token-seed/serviceaccount.yaml @@ -0,0 +1,7 @@ +# services/vault-hermes-jenkins-token-seed/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: hermes-jenkins-token-seed + namespace: vault +automountServiceAccountToken: true diff --git a/services/vault/hermes-auth-role-bootstrap-job.yaml b/services/vault/hermes-auth-role-bootstrap-job.yaml index 07c223f0..4bcee103 100644 --- a/services/vault/hermes-auth-role-bootstrap-job.yaml +++ b/services/vault/hermes-auth-role-bootstrap-job.yaml @@ -3,7 +3,7 @@ apiVersion: batch/v1 kind: Job metadata: - name: vault-k8s-auth-hermes-8 + name: vault-k8s-auth-hermes-9 namespace: vault spec: backoffLimit: 2 @@ -15,16 +15,26 @@ spec: hardware: rpi5 kubernetes.io/arch: arm64 node-role.kubernetes.io/worker: "true" + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: NotIn + values: [titan-04, titan-14, titan-18, titan-19, titan-24] containers: - name: configure-k8s-auth - image: hashicorp/vault:1.21.4 + image: docker.io/hashicorp/vault@sha256:4e33b126a59c0c333b76fb4e894722462659a6bec7c48c9ee8cea56fccfd2569 imagePullPolicy: IfNotPresent command: - sh - /scripts/vault_k8s_auth_configure.sh env: + - name: HOME + value: /tmp - name: VAULT_ADDR - value: http://10.43.57.249:8200 + value: http://vault.vault.svc.cluster.local:8200 - name: VAULT_K8S_ROLE value: vault-admin - name: VAULT_K8S_TOKEN_REVIEWER_JWT_FILE @@ -38,6 +48,18 @@ spec: - name: token-reviewer mountPath: /var/run/secrets/vault-token-reviewer readOnly: true + - name: tmp + mountPath: /tmp + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + readOnlyRootFilesystem: true + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 100 + seccompProfile: + type: RuntimeDefault resources: requests: cpu: 25m @@ -53,3 +75,5 @@ spec: - name: token-reviewer secret: secretName: vault-admin-token-reviewer + - name: tmp + emptyDir: {} diff --git a/services/vault/scripts/vault_k8s_auth_configure.sh b/services/vault/scripts/vault_k8s_auth_configure.sh index 73572e62..48e839e6 100644 --- a/services/vault/scripts/vault_k8s_auth_configure.sh +++ b/services/vault/scripts/vault_k8s_auth_configure.sh @@ -235,12 +235,26 @@ write_policy_and_role "mailu-mailserver" "mailu-mailserver" "mailu-vault-sync" \ "mailu/* shared/postmark-relay shared/harbor-pull" "" write_policy_and_role "harbor" "harbor" "harbor-vault-sync" \ "harbor/* shared/harbor-pull" "hermes/developer-harbor" +harbor_policy_bootstrap_policy=' +path "kv/data/atlas/harbor/harbor-core" { + capabilities = ["read"] +} +' +write_raw_policy "harbor-policy-bootstrap" "${harbor_policy_bootstrap_policy}" +log "writing role harbor-policy-bootstrap" +vault_cmd write "auth/kubernetes/role/harbor-policy-bootstrap" \ + bound_service_account_names="harbor-policy-bootstrap" \ + bound_service_account_namespaces="harbor" \ + policies="harbor-policy-bootstrap" \ + ttl="${role_ttl}" write_policy_and_role "nextcloud" "nextcloud" "nextcloud-vault" \ "nextcloud/* shared/keycloak-admin shared/postmark-relay" "" write_policy_and_role "comms" "comms" "comms-vault,atlasbot" \ "comms/* shared/chat-ai-keys-runtime shared/harbor-pull" "" -write_policy_and_role "jenkins" "jenkins" "jenkins,jenkins-vault-sync" \ - "jenkins/* shared/harbor-pull quality/sonarqube-oidc" "hermes/developer-jenkins" +write_policy_and_role "jenkins" "jenkins" "jenkins" \ + "jenkins/* shared/harbor-pull quality/sonarqube-oidc hermes/developer-jenkins" "" +write_policy_and_role "jenkins-vault-sync" "jenkins" "jenkins-vault-sync" \ + "shared/harbor-pull" "" write_policy_and_role "monitoring" "monitoring" "monitoring-vault-sync" \ "monitoring/* shared/postmark-relay shared/harbor-pull" "" write_policy_and_role "logging" "logging" "logging-vault-sync" \ @@ -255,8 +269,25 @@ write_policy_and_role "game-stream" "game-stream" "game-stream-vault" \ "game-stream/*" "" write_policy_and_role "hermes" "hermes" "hermes-vault,hermes-triage" \ "hermes/triage-oidc hermes/agent-tokens hermes/triage-api" "" -write_policy_and_role "hermes-agent" "hermes" "hermes-agent,hermes-switchyard" \ +write_policy_and_role "hermes-agent" "hermes" "hermes-agent" \ "hermes/agent-oidc hermes/agent-tokens hermes/chat-telegram hermes/developer-keycloak hermes/developer-gitea hermes/developer-harbor hermes/developer-jenkins hermes/developer-ssh" "" +write_policy_and_role "hermes-switchyard" "hermes" "hermes-switchyard" \ + "hermes/chat-telegram" "" +hermes_jenkins_token_seed_policy=' +path "kv/data/atlas/hermes/developer-jenkins" { + capabilities = ["create", "read"] +} +path "sys/tools/random/32" { + capabilities = ["update"] +} +' +write_raw_policy "hermes-jenkins-token-seed" "${hermes_jenkins_token_seed_policy}" +log "writing role hermes-jenkins-token-seed" +vault_cmd write "auth/kubernetes/role/hermes-jenkins-token-seed" \ + bound_service_account_names="hermes-jenkins-token-seed" \ + bound_service_account_namespaces="vault" \ + policies="hermes-jenkins-token-seed" \ + ttl="${role_ttl}" write_policy_and_role "hermes-credential-sync" "hermes" "hermes-agent" \ "" "hermes/agent-tokens" write_policy_and_role "hermes-node-ssh" "hermes" "hermes-node-ssh-access" \ diff --git a/testing/quality_contract.json b/testing/quality_contract.json index e8793d84..73d68fa2 100644 --- a/testing/quality_contract.json +++ b/testing/quality_contract.json @@ -14,6 +14,10 @@ } ], "managed_modules": [ + "ci/scripts/hermes_image_release.py", + "dockerfiles/hermes-kaniko-heredoc-runner.py", + "services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py", + "services/hermes/scripts/jenkins_image_build_trigger.py", "ci/scripts/publish_test_metrics.py", "ci/scripts/publish_test_metrics_quality.py", "ci/scripts/semgrep_report.py", @@ -35,6 +39,10 @@ "testing/tests/test_quality_gate.py" ], "lint_paths": [ + "ci/scripts/hermes_image_release.py", + "dockerfiles/hermes-kaniko-heredoc-runner.py", + "services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py", + "services/hermes/scripts/jenkins_image_build_trigger.py", "ci/scripts/publish_test_metrics.py", "ci/scripts/publish_test_metrics_quality.py", "ci/scripts/semgrep_report.py", @@ -57,7 +65,10 @@ "junit": "build/junit-unit.xml", "coverage_sources": [ "ci/scripts", + "dockerfiles", "scripts.render.dashboards_render_atlas", + "services/harbor/scripts", + "services/hermes/scripts", "services/mailu/scripts", "testing" ], @@ -168,6 +179,10 @@ "coverage": { "minimum_percent": 95.0, "tracked_files": [ + "ci/scripts/hermes_image_release.py", + "dockerfiles/hermes-kaniko-heredoc-runner.py", + "services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py", + "services/hermes/scripts/jenkins_image_build_trigger.py", "ci/scripts/publish_test_metrics.py", "ci/scripts/publish_test_metrics_quality.py", "ci/scripts/semgrep_report.py", diff --git a/testing/tests/test_hermes_image_builder.py b/testing/tests/test_hermes_image_builder.py new file mode 100644 index 00000000..8d7418d3 --- /dev/null +++ b/testing/tests/test_hermes_image_builder.py @@ -0,0 +1,473 @@ +"""Safety and artifact contracts for the Hermes agent image release lane.""" + +from __future__ import annotations + +import importlib.util +import io +import json +import re +import urllib.parse +from pathlib import Path + +import pytest +import yaml + + +REPO_ROOT = Path(__file__).resolve().parents[2] +PIPELINE_PATH = REPO_ROOT / "ci/Jenkinsfile.hermes-agent-image" +RELEASE_SCRIPT = REPO_ROOT / "ci/scripts/hermes_image_release.py" +TRIGGER_SCRIPT = REPO_ROOT / "services/hermes/scripts/jenkins_image_build_trigger.py" +HEREDOC_RUNNER = REPO_ROOT / "dockerfiles/hermes-kaniko-heredoc-runner.py" + + +def _load_release_module(): + spec = importlib.util.spec_from_file_location( + "hermes_image_release", RELEASE_SCRIPT + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _load_trigger_module(): + spec = importlib.util.spec_from_file_location( + "jenkins_image_build_trigger", TRIGGER_SCRIPT + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _load_heredoc_runner(): + spec = importlib.util.spec_from_file_location( + "hermes_kaniko_heredoc_runner", HEREDOC_RUNNER + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _pod_spec() -> dict: + source = PIPELINE_PATH.read_text(encoding="utf-8") + pod_yaml = source.split('yaml """', 1)[1].split('"""', 1)[0] + return yaml.safe_load(pod_yaml)["spec"] + + +def test_builder_pod_is_daemonless_and_kernel_bounded() -> None: + """The builder gets only proven build caps, never host, daemon, or K8s access.""" + source = PIPELINE_PATH.read_text(encoding="utf-8") + spec = _pod_spec() + assert spec["serviceAccountName"] == "hermes-image-builder" + assert spec["automountServiceAccountToken"] is False + assert spec["enableServiceLinks"] is False + assert "hostPath" not in source + assert "docker.sock" not in source + assert "tcp://" not in source + assert "buildkitd" not in source.lower() + assert "dind" not in source.lower() + assert "serviceAccountToken" not in source + + containers = {item["name"]: item for item in spec["containers"]} + kaniko = containers["kaniko"] + assert kaniko["image"] == ( + "gcr.io/kaniko-project/executor@sha256:" + "c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e" + ) + assert kaniko["securityContext"]["capabilities"]["add"] == [ + "CHOWN", + "FOWNER", + "DAC_OVERRIDE", + "SETGID", + "SETUID", + ] + for container in containers.values(): + security = container["securityContext"] + assert security["allowPrivilegeEscalation"] is False + assert security["capabilities"]["drop"] == ["ALL"] + assert security["seccompProfile"]["type"] == "RuntimeDefault" + assert security.get("privileged", False) is False + assert "add" not in containers["jnlp"]["securityContext"]["capabilities"] + assert "add" not in containers["python"]["securityContext"]["capabilities"] + + +def test_pipeline_requires_reviewed_main_and_runtime_credentials() -> None: + """Publishing requires explicit confirmation and a reviewed main commit.""" + source = PIPELINE_PATH.read_text(encoding="utf-8") + assert 'test "${PUBLISH_IMAGE}" = "true"' in source + assert 'test "${CONFIRM_PUBLISH}" = "PUBLISH HERMES AGENT"' in source + assert "git rev-parse origin/main" in source + assert "credentialsId: 'harbor-robot'" in source + assert "set +x" in source + assert "umask 077" in source + assert "unset HARBOR_USER HARBOR_PASSWORD auth" in source + assert "/busybox/rm -f /kaniko/.docker/config.json" in source + assert "--digest-file=" in source + assert "--image-name-tag-with-digest-file=" in source + assert "--destination=" in source + assert "assert-absent" in source + assert "git-${actual_revision}-build-${BUILD_NUMBER}" in source + assert source.count("--build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1") == 1 + + +def test_kaniko_replays_only_the_exact_reviewed_heredoc_contract() -> None: + """Pinned Kaniko's ignored inline files are replayed in exact source order.""" + module = _load_heredoc_runner() + dockerfile = (REPO_ROOT / "dockerfiles/Dockerfile.hermes-agent").read_text() + blocks = module.extract_blocks(dockerfile) + assert tuple(command[0] for command, _body in blocks) == ( + "node", + "python", + "python", + "python", + "python", + "python", + "python", + "python", + "node", + ) + assert "session message total" in blocks[-1][1] + compat = dockerfile.split("ARG HERMES_KANIKO_HEREDOC_COMPAT=0", 1)[1] + assert compat.count("python /tmp/hermes-kaniko-heredoc-runner.py") == 9 + assert [int(value) for value in re.findall(r"--block-index (\d+)", compat)] == list( + range(1, 10) + ) + ignored = ( + REPO_ROOT / "dockerfiles/Dockerfile.hermes-agent.dockerignore" + ).read_text() + assert "!dockerfiles/Dockerfile.hermes-agent" in ignored + assert "!dockerfiles/hermes-kaniko-heredoc-runner.py" in ignored + + +def test_kaniko_heredoc_runner_rejects_drift_and_executes_separately( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """Missing blocks fail closed and each reviewed body gets a fresh process.""" + module = _load_heredoc_runner() + source = (REPO_ROOT / "dockerfiles/Dockerfile.hermes-agent").read_text() + with pytest.raises(ValueError, match="contract changed"): + module.extract_blocks(source.replace("RUN node <<'NODE'", "RUN node", 1)) + + calls = [] + + def run(command, **kwargs): + calls.append((command, kwargs)) + + monkeypatch.setattr(module.subprocess, "run", run) + dockerfile = tmp_path / "Dockerfile" + dockerfile.write_text(source, encoding="utf-8") + for block_index in range(1, 10): + module.replay(dockerfile, block_index) + assert len(calls) == 9 + assert all(call[1]["check"] is True for call in calls) + assert all(call[1]["text"] is True for call in calls) + assert all(call[1]["input"].endswith("\n") for call in calls) + + +def test_jenkins_job_is_manual_and_reads_pipeline_from_main() -> None: + """JCasC must not publish unreviewed branch contents or poll automatically.""" + config = yaml.safe_load( + (REPO_ROOT / "services/jenkins/configmap-jcasc.yaml").read_text( + encoding="utf-8" + ) + ) + jobs = config["data"]["jobs.yaml"] + block = jobs.split("pipelineJob('hermes-agent-image')", 1)[1].split( + "pipelineJob(", 1 + )[0] + assert "branches('*/main')" in block + assert "scriptPath('ci/Jenkinsfile.hermes-agent-image')" in block + assert ( + "authenticationToken(System.getenv('HERMES_AGENT_IMAGE_BUILD_TOKEN'))" in block + ) + assert "pipelineTriggers" not in block + assert "scmTrigger" not in block + + +def test_agent_trigger_is_limited_to_the_image_job(tmp_path: Path) -> None: + """Agent Hermes gets one job token, fixed parameters, and no Jenkins admin API.""" + module = _load_trigger_module() + revision = "a" * 40 + token_path = tmp_path / "token" + token_path.write_text("private-job-token\n", encoding="utf-8") + captured = {} + + class Response(io.BytesIO): + status = 201 + headers = {"Location": "https://ci.bstein.dev/queue/item/42/"} + + def __enter__(self): + return self + + def __exit__(self, *_args): + self.close() + + def opener(request, timeout): + captured["request"] = request + captured["timeout"] = timeout + return Response(b"") + + result = module.trigger_build(revision, token_file=token_path, opener=opener) + request = captured["request"] + fields = urllib.parse.parse_qs(request.data.decode("utf-8")) + assert request.full_url == module.JENKINS_BUILD_URL + assert fields == { + "CONFIRM_PUBLISH": ["PUBLISH HERMES AGENT"], + "EXPECTED_SOURCE_REVISION": [revision], + "PUBLISH_IMAGE": ["true"], + "job": ["hermes-agent-image"], + "token": ["private-job-token"], + } + assert captured["timeout"] == 20 + assert "private-job-token" not in json.dumps(result) + assert result["source_revision"] == revision + + +def test_agent_trigger_rejects_unsafe_revision_and_empty_token(tmp_path: Path) -> None: + """No user-controlled job, URL, or abbreviated revision reaches Jenkins.""" + module = _load_trigger_module() + token_path = tmp_path / "token" + token_path.write_text("token\n", encoding="utf-8") + with pytest.raises(ValueError): + module.trigger_build("main", token_file=token_path) + token_path.write_text("\n", encoding="utf-8") + with pytest.raises(RuntimeError, match="empty"): + module.trigger_build("a" * 40, token_file=token_path) + + +def test_agent_trigger_accepts_existing_queue_redirect(tmp_path: Path) -> None: + """HTTP 303 means the exact release is already queued, not a trigger failure.""" + module = _load_trigger_module() + token_path = tmp_path / "token" + token_path.write_text("token\n", encoding="utf-8") + + class Response(io.BytesIO): + status = 303 + headers = {"Location": "https://ci.bstein.dev/queue/item/7/"} + + def __enter__(self): + return self + + def __exit__(self, *_args): + self.close() + + result = module.trigger_build( + "e" * 40, token_file=token_path, opener=lambda *_args, **_kwargs: Response() + ) + assert result["status"] == 303 + assert result["queue_path"] == "/queue/item/7/" + + +def test_job_token_is_generated_and_injected_only_at_runtime() -> None: + """The fixed-job credential stays in Vault and pod-lifetime memory.""" + plugins = (REPO_ROOT / "services/jenkins/configmap-plugins.yaml").read_text( + encoding="utf-8" + ) + vault = ( + REPO_ROOT / "services/vault/scripts/vault_k8s_auth_configure.sh" + ).read_text(encoding="utf-8") + seeder = ( + REPO_ROOT + / "services/vault-hermes-jenkins-token-seed/scripts/vault_hermes_jenkins_build_token_ensure.sh" + ).read_text(encoding="utf-8") + jenkins = (REPO_ROOT / "services/jenkins/deployment.yaml").read_text( + encoding="utf-8" + ) + agent = (REPO_ROOT / "services/hermes/agent-deployment.yaml").read_text( + encoding="utf-8" + ) + stage = (REPO_ROOT / "services/hermes/scripts/stage_runtime_access.py").read_text( + encoding="utf-8" + ) + assert "build-token-root:365.v717f8685a_09e" in plugins + assert 'write_raw_policy "hermes-jenkins-token-seed"' in vault + assert "sys/tools/random/32 format=hex" in seeder + assert '"options":{"cas":0}' in seeder + assert "kv/data/atlas/hermes/developer-jenkins" in seeder + assert "HERMES_AGENT_IMAGE_BUILD_TOKEN={{ .Data.data.build_token }}" in jenkins + assert "agent-inject-secret-jenkins-image-build-token" in agent + assert '"jenkins-image-build-token"' in stage + + +def test_release_renderer_preserves_manifest_and_emits_safe_artifacts( + tmp_path: Path, +) -> None: + """The release artifact is exact, reviewable, and contains no credentials.""" + module = _load_release_module() + old_digest = "sha256:" + "1" * 64 + new_digest = "sha256:" + "2" * 64 + revision = "a" * 40 + build_number = "17" + manifest = tmp_path / "kustomization.yaml" + manifest.write_text( + "apiVersion: kustomize.config.k8s.io/v1beta1\n" + "kind: Kustomization\n" + "images:\n" + f" - name: {module.DEFAULT_IMAGE}\n" + f" digest: {old_digest}\n", + encoding="utf-8", + ) + output = tmp_path / "out" + metadata = module.write_release_artifacts( + digest=f"{new_digest}\n", + source_revision=revision, + build_number=build_number, + destination=f"{module.DEFAULT_IMAGE}:git-{revision}-build-{build_number}", + kustomization=manifest, + output_dir=output, + ) + + assert manifest.read_text(encoding="utf-8").endswith(f"{old_digest}\n") + assert ( + (output / "hermes-kustomization.yaml") + .read_text(encoding="utf-8") + .endswith(f"{new_digest}\n") + ) + patch = (output / "hermes-image-update.patch").read_text(encoding="utf-8") + assert f"- digest: {old_digest}" in patch + assert f"+ digest: {new_digest}" in patch + assert ( + json.loads((output / "hermes-agent-image.json").read_text(encoding="utf-8")) + == metadata + ) + assert set(metadata) == { + "digest", + "build_number", + "flux_image", + "image", + "published_tag", + "source_revision", + } + + +@pytest.mark.parametrize( + ("digest", "revision", "destination"), + [ + ( + "latest", + "a" * 40, + "registry.bstein.dev/bstein/hermes-agent:git-" + "a" * 40 + "-build-1", + ), + ( + "sha256:" + "1" * 64, + "short", + "registry.bstein.dev/bstein/hermes-agent:git-short-build-1", + ), + ( + "sha256:" + "1" * 64, + "a" * 40, + "registry.bstein.dev/bstein/hermes-agent:latest", + ), + ], +) +def test_release_renderer_rejects_unpinned_inputs( + tmp_path: Path, digest: str, revision: str, destination: str +) -> None: + """Only exact digests and immutable source-derived tags are accepted.""" + module = _load_release_module() + manifest = tmp_path / "kustomization.yaml" + manifest.write_text( + "images:\n" + f" - name: {module.DEFAULT_IMAGE}\n" + " digest: sha256:" + "0" * 64 + "\n", + encoding="utf-8", + ) + with pytest.raises(ValueError): + module.write_release_artifacts( + digest=digest, + source_revision=revision, + build_number="1", + destination=destination, + kustomization=manifest, + output_dir=tmp_path / "out", + ) + + +def test_release_renderer_fails_closed_on_manifest_drift(tmp_path: Path) -> None: + """Missing, duplicate, or already-current image entries require human review.""" + module = _load_release_module() + digest = "sha256:" + "f" * 64 + missing = "images:\n - name: example.invalid/other\n digest: " + digest + "\n" + with pytest.raises(ValueError, match="found 0"): + module.render_kustomization(missing, digest) + + duplicate = ( + "images:\n" + f" - name: {module.DEFAULT_IMAGE}\n digest: {digest}\n" + f" - name: {module.DEFAULT_IMAGE}\n digest: {digest}\n" + ) + with pytest.raises(ValueError, match="found 2"): + module.render_kustomization(duplicate, digest) + + manifest = tmp_path / "kustomization.yaml" + manifest.write_text( + f"images:\n - name: {module.DEFAULT_IMAGE}\n digest: {digest}\n", + encoding="utf-8", + ) + revision = "b" * 40 + with pytest.raises(ValueError, match="already matches"): + module.write_release_artifacts( + digest=digest, + source_revision=revision, + build_number="1", + destination=f"{module.DEFAULT_IMAGE}:git-{revision}-build-1", + kustomization=manifest, + output_dir=tmp_path / "out", + ) + + +def test_release_cli_reads_digest_file( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The pipeline CLI uses the Kaniko digest file as its sole digest input.""" + module = _load_release_module() + digest = "sha256:" + "c" * 64 + revision = "d" * 40 + manifest = tmp_path / "kustomization.yaml" + manifest.write_text( + f"images:\n - name: {module.DEFAULT_IMAGE}\n digest: sha256:" + + "0" * 64 + + "\n", + encoding="utf-8", + ) + digest_file = tmp_path / "digest" + digest_file.write_text(digest + "\n", encoding="utf-8") + destination = f"{module.DEFAULT_IMAGE}:git-{revision}-build-9" + image_file = tmp_path / "image" + image_file.write_text(f"{destination}@{digest}\n", encoding="utf-8") + output = tmp_path / "out" + monkeypatch.setenv("HARBOR_USER", "robot") + monkeypatch.setenv("HARBOR_PASSWORD", "secret") + monkeypatch.setattr( + module, "verify_registry_digest", lambda *_args, **_kwargs: None + ) + monkeypatch.setattr( + "sys.argv", + [ + "hermes_image_release.py", + "render", + "--digest-file", + str(digest_file), + "--image-file", + str(image_file), + "--source-revision", + revision, + "--build-number", + "9", + "--destination", + destination, + "--kustomization", + str(manifest), + "--output-dir", + str(output), + ], + ) + assert module.main() == 0 + assert ( + json.loads((output / "hermes-agent-image.json").read_text(encoding="utf-8"))[ + "digest" + ] + == digest + ) diff --git a/testing/tests/test_hermes_image_builder_adversarial.py b/testing/tests/test_hermes_image_builder_adversarial.py new file mode 100644 index 00000000..be65e1de --- /dev/null +++ b/testing/tests/test_hermes_image_builder_adversarial.py @@ -0,0 +1,469 @@ +"""Adversarial boundaries for the Hermes image publisher and trigger.""" + +from __future__ import annotations + +import importlib.util +import io +import json +from pathlib import Path + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parents[2] +RELEASE_SCRIPT = REPO_ROOT / "ci/scripts/hermes_image_release.py" +TRIGGER_SCRIPT = REPO_ROOT / "services/hermes/scripts/jenkins_image_build_trigger.py" + + +def _load(path: Path, name: str): + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class Response(io.BytesIO): + """Small context-managed HTTP response fixture.""" + + def __init__( + self, + status: int, + headers: dict[str, str] | None = None, + body: bytes = b"", + ): + super().__init__(body) + self.status = status + self.headers = headers or {} + + +def test_kaniko_evidence_binds_digest_destination_and_unique_build() -> None: + """Both Kaniko artifacts must describe one exact non-replayable tag.""" + module = _load(RELEASE_SCRIPT, "hermes_image_release_evidence") + revision = "a" * 40 + digest = "sha256:" + "b" * 64 + destination = f"{module.DEFAULT_IMAGE}:git-{revision}-build-42" + assert module.validate_destination(destination, revision, "42") == ( + revision, + "42", + ) + assert ( + module.validate_kaniko_evidence( + digest_text=f"{digest}\n", + image_text=f"{destination}@{digest}\n", + destination=destination, + ) + == digest + ) + + +@pytest.mark.parametrize( + ("digest_text", "image_template"), + [ + ("sha256:" + "a" * 64 + "\nextra\n", "{destination}@{digest}"), + ("sha256:" + "a" * 64, "{destination}@sha256:" + "b" * 64), + ("sha256:" + "a" * 64, "registry.invalid/x:y@{digest}"), + ("sha256:" + "a" * 64, "{destination}@{digest}\nextra"), + ], +) +def test_kaniko_evidence_rejects_cross_artifact_mismatch( + digest_text: str, image_template: str +) -> None: + """A digest, tag, repository, or cardinality mismatch stops rendering.""" + module = _load(RELEASE_SCRIPT, "hermes_image_release_mismatch") + revision = "c" * 40 + digest = "sha256:" + "a" * 64 + destination = f"{module.DEFAULT_IMAGE}:git-{revision}-build-8" + image_text = image_template.format(destination=destination, digest=digest) + with pytest.raises(ValueError): + module.validate_kaniko_evidence( + digest_text=digest_text, + image_text=image_text, + destination=destination, + ) + + +def test_registry_preflight_rejects_existing_tag_and_auth_failures() -> None: + """Only an authenticated 404 permits Kaniko to claim the unique tag.""" + module = _load(RELEASE_SCRIPT, "hermes_image_release_preflight") + destination = f"{module.DEFAULT_IMAGE}:git-{'d' * 40}-build-3" + captured = {} + + def missing(request, timeout): + captured["request"] = request + captured["timeout"] = timeout + return Response(404) + + module.assert_tag_absent( + destination, username="robot", password="private", opener=missing + ) + request = captured["request"] + assert request.method == "GET" + assert request.full_url.startswith( + "https://registry.bstein.dev/api/v2.0/projects/bstein/repositories/" + "hermes-agent/artifacts/" + ) + assert "private" not in request.full_url + assert captured["timeout"] == 20 + + for status, message in ( + (200, "already exists"), + (401, "HTTP 401"), + (503, "HTTP 503"), + ): + with pytest.raises(RuntimeError, match=message): + module.assert_tag_absent( + destination, + username="robot", + password="private", + opener=lambda *_args, code=status: Response(code), + ) + + +def test_registry_preflight_requires_exact_server_immutable_policy() -> None: + """The already-running controller cannot publish before Flux installs the rule.""" + module = _load(RELEASE_SCRIPT, "hermes_image_release_policy_preflight") + expected = { + "disabled": False, + "action": "immutable", + "template": "immutable_template", + "tag_selectors": [ + { + "kind": "doublestar", + "decoration": "matches", + "pattern": "git-*-build-*", + } + ], + "scope_selectors": { + "repository": [ + { + "kind": "doublestar", + "decoration": "repoMatches", + "pattern": "hermes-agent", + } + ] + }, + } + captured = {} + + def exact(request, timeout): + captured["url"] = request.full_url + captured["timeout"] = timeout + return Response( + 200, + {"X-Total-Count": "1"}, + body=json.dumps([{"id": 9, **expected}]).encode(), + ) + + module.verify_immutable_policy(username="robot", password="private", opener=exact) + assert captured["url"].endswith( + "/projects/bstein/immutabletagrules?page=1&page_size=100" + ) + assert captured["timeout"] == 20 + + for status, body, headers, message in ( + (403, b"", {}, "HTTP 403"), + (200, b"[]", {"X-Total-Count": "0"}, "absent"), + ( + 200, + json.dumps([{**expected, "disabled": True}]).encode(), + {"X-Total-Count": "1"}, + "not exact", + ), + (200, b"not-json", {}, "invalid"), + ( + 200, + json.dumps([expected]).encode(), + {"X-Total-Count": "2"}, + "truncated", + ), + ): + with pytest.raises(RuntimeError, match=message): + module.verify_immutable_policy( + username="robot", + password="private", + opener=lambda *_args, code=status, value=body, evidence=headers: Response( + code, evidence, body=value + ), + ) + + +def test_harbor_client_fails_closed_on_input_size_auth_and_json( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Malformed inputs, oversized bodies, missing auth, and bad JSON all fail.""" + module = _load(RELEASE_SCRIPT, "hermes_image_release_harbor_errors") + destination = f"{module.DEFAULT_IMAGE}:git-{'1' * 40}-build-6" + with pytest.raises(ValueError, match="destination"): + module.assert_tag_absent( + "registry.invalid/x:tag", username="robot", password="private" + ) + with pytest.raises(RuntimeError, match="credentials"): + module.assert_tag_absent(destination, username="", password="private") + with pytest.raises(RuntimeError, match="size limit"): + module.assert_tag_absent( + destination, + username="robot", + password="private", + opener=lambda *_args: Response(200, body=b"x" * 1_048_577), + ) + with pytest.raises(RuntimeError, match="invalid artifact JSON"): + module.verify_registry_digest( + destination, + "sha256:" + "2" * 64, + username="robot", + password="private", + opener=lambda *_args: Response(200, body=b"\xff"), + ) + monkeypatch.delenv("HARBOR_USER", raising=False) + monkeypatch.delenv("HARBOR_PASSWORD", raising=False) + with pytest.raises(RuntimeError, match="unavailable"): + module._credentials() + + +def test_default_harbor_opener_returns_http_responses( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The real wrapper returns both normal and non-redirect HTTP responses.""" + module = _load(RELEASE_SCRIPT, "hermes_image_release_opener") + request = module.urllib.request.Request("https://registry.bstein.dev/test") + + class SuccessOpener: + def open(self, _request, timeout): + assert timeout == 7 + return Response(204) + + monkeypatch.setattr( + module.urllib.request, "build_opener", lambda *_handlers: SuccessOpener() + ) + assert module._registry_request(request, 7).status == 204 + + class ErrorOpener: + def open(self, _request, timeout): + assert timeout == 8 + raise module.urllib.error.HTTPError( + _request.full_url, 404, "missing", {}, None + ) + + monkeypatch.setattr( + module.urllib.request, "build_opener", lambda *_handlers: ErrorOpener() + ) + assert module._registry_request(request, 8).code == 404 + + +@pytest.mark.parametrize( + ("status", "artifact", "message"), + [ + (404, {}, "HTTP 404"), + (200, {}, "omitted"), + (200, {"digest": "sha256:" + "2" * 64}, "does not match"), + (200, {"digest": "latest"}, "omitted"), + ], +) +def test_registry_verification_rejects_missing_or_mismatched_digest( + status: int, artifact: dict[str, str], message: str +) -> None: + """Rendering requires an independent exact Harbor digest response.""" + module = _load(RELEASE_SCRIPT, "hermes_image_release_registry") + digest = "sha256:" + "1" * 64 + destination = f"{module.DEFAULT_IMAGE}:git-{'e' * 40}-build-4" + body = json.dumps(artifact).encode("utf-8") + with pytest.raises(RuntimeError, match=message): + module.verify_registry_digest( + destination, + digest, + username="robot", + password="private", + opener=lambda *_args: Response(status, body=body), + ) + + +def test_registry_verification_accepts_exact_pushed_digest() -> None: + """An exact authenticated Harbor digest allows artifact rendering.""" + module = _load(RELEASE_SCRIPT, "hermes_image_release_registry_ok") + digest = "sha256:" + "4" * 64 + destination = f"{module.DEFAULT_IMAGE}:git-{'f' * 40}-build-5" + with pytest.raises(RuntimeError, match="expected tag"): + module.verify_registry_digest( + destination, + digest, + username="robot", + password="private", + opener=lambda *_args: Response( + 200, + body=json.dumps( + {"digest": digest, "tags": [{"name": "different-tag"}]} + ).encode("utf-8"), + ), + ) + module.verify_registry_digest( + destination, + digest, + username="robot", + password="private", + opener=lambda *_args: Response( + 200, + body=json.dumps( + { + "digest": digest, + "tags": [ + { + "name": destination.rsplit(":", 1)[1], + "immutable": True, + } + ], + } + ).encode("utf-8"), + ), + ) + + +def test_registry_verification_rejects_tag_without_server_immutability() -> None: + """Digest evidence is insufficient unless Harbor reports the build tag immutable.""" + module = _load(RELEASE_SCRIPT, "hermes_image_release_registry_mutable") + digest = "sha256:" + "7" * 64 + destination = f"{module.DEFAULT_IMAGE}:git-{'8' * 40}-build-19" + captured = {} + + def opener(request, _timeout): + captured["url"] = request.full_url + return Response( + 200, + body=json.dumps( + { + "digest": digest, + "tags": [ + { + "name": destination.rsplit(":", 1)[1], + "immutable": False, + } + ], + } + ).encode(), + ) + + with pytest.raises(RuntimeError, match="immutable"): + module.verify_registry_digest( + destination, + digest, + username="robot", + password="private", + opener=opener, + ) + assert captured["url"].endswith("?with_immutable_status=true") + + +def test_release_main_preflight_uses_fixed_credentials_and_destination( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The preflight CLI validates and checks exactly the requested build tag.""" + module = _load(RELEASE_SCRIPT, "hermes_image_release_main_absent") + revision = "3" * 40 + destination = f"{module.DEFAULT_IMAGE}:git-{revision}-build-12" + captured = {} + monkeypatch.setenv("HARBOR_USER", "robot") + monkeypatch.setenv("HARBOR_PASSWORD", "private") + + def assert_absent(value, **credentials): + captured["destination"] = value + captured["credentials"] = credentials + + def verify_policy(**credentials): + captured["policy_credentials"] = credentials + + monkeypatch.setattr(module, "assert_tag_absent", assert_absent) + monkeypatch.setattr(module, "verify_immutable_policy", verify_policy) + monkeypatch.setattr( + "sys.argv", + [ + "hermes_image_release.py", + "assert-absent", + "--source-revision", + revision, + "--build-number", + "12", + "--destination", + destination, + ], + ) + assert module.main() == 0 + assert captured == { + "destination": destination, + "credentials": {"username": "robot", "password": "private"}, + "policy_credentials": {"username": "robot", "password": "private"}, + } + + +def test_renderer_skips_a_matching_entry_without_a_digest() -> None: + """Only the one complete image entry is changed when an earlier entry drifts.""" + module = _load(RELEASE_SCRIPT, "hermes_image_release_manifest_scan") + digest = "sha256:" + "5" * 64 + source = ( + "images:\n" + f" - name: {module.DEFAULT_IMAGE}\n" + " newTag: ignored\n" + " - name: example.invalid/other\n" + " newTag: stable\n" + f" - name: {module.DEFAULT_IMAGE}\n" + " digest: sha256:" + "0" * 64 + "\n" + ) + assert module.render_kustomization(source, digest).endswith(f"{digest}\n") + + +@pytest.mark.parametrize("status", [200, 202, 204, 301, 302, 307, 308]) +def test_trigger_rejects_non_plugin_success_status(tmp_path: Path, status: int) -> None: + """Generic proxy successes and redirects are not proof of a queued build.""" + module = _load(TRIGGER_SCRIPT, f"jenkins_trigger_status_{status}") + token = tmp_path / "token" + token.write_text("private\n", encoding="utf-8") + response = lambda *_args, **_kwargs: Response( # noqa: E731 + status, {"Location": "https://ci.bstein.dev/queue/item/9/"} + ) + with pytest.raises(RuntimeError, match=f"HTTP {status}"): + module.trigger_build("a" * 40, token_file=token, opener=response) + + +@pytest.mark.parametrize( + "location", + [ + "", + "https://evil.invalid/queue/item/9/", + "http://ci.bstein.dev/queue/item/9/", + "https://ci.bstein.dev/job/hermes-agent-image/9/", + "https://ci.bstein.dev/queue/item/9/?token=leak", + "https://ci.bstein.dev/queue/item/9/#fragment", + "https://ci.bstein.dev/queue/item/not-a-number/", + ], +) +def test_trigger_rejects_missing_malformed_or_cross_origin_queue_location( + tmp_path: Path, location: str +) -> None: + """A real accepted status still needs the exact same-origin queue resource.""" + module = _load(TRIGGER_SCRIPT, "jenkins_trigger_location") + token = tmp_path / "token" + token.write_text("private\n", encoding="utf-8") + response = lambda *_args, **_kwargs: Response( # noqa: E731 + 201, {"Location": location} + ) + with pytest.raises(RuntimeError, match="Location"): + module.trigger_build("b" * 40, token_file=token, opener=response) + + +def test_trigger_uses_https_and_accepts_exact_relative_queue_path( + tmp_path: Path, +) -> None: + """A relative plugin Location is resolved only against the fixed HTTPS origin.""" + module = _load(TRIGGER_SCRIPT, "jenkins_trigger_origin") + token = tmp_path / "token" + token.write_text("private\n", encoding="utf-8") + captured = {} + + def opener(request, timeout): + captured["url"] = request.full_url + captured["timeout"] = timeout + return Response(201, {"Location": "/queue/item/11/"}) + + result = module.trigger_build("c" * 40, token_file=token, opener=opener) + assert captured["url"].startswith("https://ci.bstein.dev/") + assert captured["timeout"] == 20 + assert result["queue_path"] == "/queue/item/11/" diff --git a/testing/tests/test_hermes_image_builder_coverage.py b/testing/tests/test_hermes_image_builder_coverage.py new file mode 100644 index 00000000..13a33177 --- /dev/null +++ b/testing/tests/test_hermes_image_builder_coverage.py @@ -0,0 +1,456 @@ +"""Boundary coverage for the three security-critical image-lane helpers.""" + +from __future__ import annotations + +import importlib.util +import io +import json +import sys +import urllib.error +from pathlib import Path + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parents[2] +RUNNER = REPO_ROOT / "dockerfiles/hermes-kaniko-heredoc-runner.py" +DOCKERFILE = REPO_ROOT / "dockerfiles/Dockerfile.hermes-agent" +HARBOR = ( + REPO_ROOT + / "services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py" +) +TRIGGER = REPO_ROOT / "services/hermes/scripts/jenkins_image_build_trigger.py" + + +def _load(path: Path, name: str): + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class Response(io.BytesIO): + """Context-managed urllib response used without network access.""" + + def __init__( + self, + status: int, + body: bytes = b"", + headers: dict[str, str] | None = None, + ) -> None: + super().__init__(body) + self.status = status + self.headers = headers or {} + + def __enter__(self): + return self + + def __exit__(self, *_args): + self.close() + + +class FakeClient: + """Ordered Harbor response fake with the pinned API origin.""" + + origin = "https://registry.bstein.dev/api/v2.0" + + def __init__(self, responses) -> None: + self.responses = list(responses) + self.calls = [] + + def request(self, method, path, payload=None): + self.calls.append((method, path, payload)) + return self.responses.pop(0) + + +def _count(value: int) -> dict[str, str]: + return {"X-Total-Count": str(value)} + + +def _robot(module, *, immutable: bool = False, duration=-1) -> dict: + access = [ + {"resource": "repository", "action": "pull"}, + {"resource": "repository", "action": "push"}, + ] + if immutable: + access.append({"resource": "immutable-tag", "action": "list"}) + return { + "id": 41, + "name": module.PUBLISH_ROBOT, + "description": "publisher", + "level": "system", + "duration": duration, + "editable": True, + "disable": False, + "permissions": [ + { + "kind": "project", + "namespace": module.PROJECT, + "access": access, + } + ], + } + + +def _robot_list(module, *, robot_id=41, name=None): + return json.dumps( + [{"id": robot_id, "name": name or module.PUBLISH_ROBOT}] + ).encode() + + +def test_runner_rejects_directive_and_heredoc_boundaries(tmp_path: Path) -> None: + """Malformed parser state, bodies, sizes, and indices all fail closed.""" + module = _load(RUNNER, "runner_boundary_coverage") + assert module._escape_character([""]) == "\\" + with pytest.raises(ValueError, match="multiple"): + module._escape_character(["# escape=\\", "# escape=\\"]) + with pytest.raises(ValueError, match="unsupported"): + module._escape_character(["# escape=^"]) + with pytest.raises(ValueError, match="unterminated.*continuation"): + module._logical_instruction(["R\\"], 0, "\\") + with pytest.raises(ValueError, match="unterminated node"): + module.extract_blocks("RUN node <<'NODE'\nbody") + + source = DOCKERFILE.read_text(encoding="utf-8") + marker = "RUN node <<'NODE'\n" + body_start = source.index(marker) + len(marker) + body_end = source.index("\nNODE", body_start) + empty = source[:body_start] + source[body_end + 1 :] + with pytest.raises(ValueError, match="empty node"): + module.extract_blocks(empty) + + dockerfile = tmp_path / "Dockerfile" + dockerfile.write_text("", encoding="utf-8") + with pytest.raises(ValueError, match="size"): + module.replay(dockerfile, 1) + with dockerfile.open("wb") as stream: + stream.truncate(module.MAX_DOCKERFILE_BYTES + 1) + with pytest.raises(ValueError, match="size"): + module.replay(dockerfile, 1) + dockerfile.write_text(source, encoding="utf-8") + for index in (0, 10): + with pytest.raises(ValueError, match="index"): + module.replay(dockerfile, index) + + +def test_runner_main_dispatches_the_exact_path_and_index( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """The command-line wrapper cannot omit either bounded input.""" + module = _load(RUNNER, "runner_main_coverage") + dockerfile = tmp_path / "Dockerfile" + seen = [] + monkeypatch.setattr(module, "replay", lambda path, index: seen.append((path, index))) + monkeypatch.setattr( + sys, + "argv", + ["runner", "--dockerfile", str(dockerfile), "--block-index", "4"], + ) + assert module.main() == 0 + assert seen == [(dockerfile, 4)] + + +def test_trigger_redirect_wrapper_accepts_only_plugin_303( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The low-level opener returns a real response or the one expected redirect.""" + module = _load(TRIGGER, "trigger_opener_coverage") + assert module._NoRedirect().redirect_request(None, None, 0, None, None, None) is None + request = module.urllib.request.Request(module.JENKINS_BUILD_URL) + + class Opener: + def __init__(self, result) -> None: + self.result = result + + def open(self, _request, timeout): + assert timeout == 9 + if isinstance(self.result, BaseException): + raise self.result + return self.result + + success = Response(201) + monkeypatch.setattr( + module.urllib.request, "build_opener", lambda *_args: Opener(success) + ) + assert module._open_without_redirect(request, 9) is success + redirect = module.urllib.error.HTTPError( + request.full_url, 303, "queued", {"Location": "/queue/item/1/"}, None + ) + monkeypatch.setattr( + module.urllib.request, "build_opener", lambda *_args: Opener(redirect) + ) + assert module._open_without_redirect(request, 9) is redirect + denied = module.urllib.error.HTTPError(request.full_url, 403, "denied", {}, None) + monkeypatch.setattr( + module.urllib.request, "build_opener", lambda *_args: Opener(denied) + ) + with pytest.raises(urllib.error.HTTPError): + module._open_without_redirect(request, 9) + + +def test_trigger_main_reports_safe_success_and_errors( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """CLI output contains safe metadata and converts expected failures to JSON.""" + module = _load(TRIGGER, "trigger_main_coverage") + revision = "a" * 40 + monkeypatch.setattr(sys, "argv", ["trigger", revision]) + monkeypatch.setattr( + module, + "trigger_build", + lambda value: {"job": module.JOB_NAME, "source_revision": value, "status": 201}, + ) + assert module.main() == 0 + assert json.loads(capsys.readouterr().out)["source_revision"] == revision + monkeypatch.setattr( + module, "trigger_build", lambda _value: (_ for _ in ()).throw(OSError("closed")) + ) + assert module.main() == 1 + assert json.loads(capsys.readouterr().out) == {"error": "closed"} + + +def test_harbor_http_client_bounds_requests_and_failures( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Credentials stay same-origin and all response/error paths remain bounded.""" + module = _load(HARBOR, "harbor_client_coverage") + assert module.NoRedirect().redirect_request(None, None, 0, None, None, None) is None + + class Opener: + result = Response(200, b"ok", {"X-Test": "yes"}) + + def open(self, request, timeout): + self.request = request + assert timeout == 20 + if isinstance(self.result, BaseException): + raise self.result + return self.result + + opener = Opener() + monkeypatch.setattr(module.urllib.request, "build_opener", lambda *_args: opener) + client = module.HarborClient(module.EXPECTED_ORIGIN + "/", "admin", "private") + status, body, headers = client.request("POST", "/rules", {"value": 1}) + assert (status, body, headers["X-Test"]) == (200, b"ok", "yes") + assert opener.request.get_header("Content-type") == "application/json" + + opener.result = module.urllib.error.HTTPError( + module.EXPECTED_ORIGIN, 404, "missing", {}, io.BytesIO(b"missing") + ) + assert client.request("GET", "/missing")[:2] == (404, b"missing") + opener.result = module.urllib.error.URLError("offline") + with pytest.raises(module.HarborUnavailable): + client.request("GET", "/rules") + opener.result = Response(200, b"x" * (module.MAX_RESPONSE + 1)) + with pytest.raises(RuntimeError, match="size"): + client.request("GET", "/rules") + + +@pytest.mark.parametrize("body", [b"\xff", b"not-json", b"{}", b"[1]"]) +def test_harbor_json_lists_reject_invalid_shapes(body: bytes) -> None: + """Malformed encodings, JSON, objects, and scalar members are never lists.""" + module = _load(HARBOR, f"harbor_json_{body!r}") + with pytest.raises(RuntimeError, match="invalid"): + module._json_list(body, "fixture") + + +@pytest.mark.parametrize( + "robot,match", + [ + ({"permissions": "bad"}, "permissions"), + ({"permissions": [1]}, "permissions"), + ({"permissions": []}, "one existing"), + ( + { + "permissions": [ + {"kind": "project", "namespace": "bstein", "access": []}, + {"kind": "project", "namespace": "bstein", "access": []}, + ] + }, + "one existing", + ), + ( + { + "permissions": [ + {"kind": "project", "namespace": "bstein", "access": "bad"} + ] + }, + "access", + ), + ( + { + "permissions": [ + {"kind": "project", "namespace": "bstein", "access": []} + ] + }, + "pull/push", + ), + ], +) +def test_publisher_scope_rejects_malformed_or_incomplete_permissions( + robot: dict, match: str +) -> None: + """Robot mutations require one structurally exact existing push scope.""" + module = _load(HARBOR, f"harbor_scope_{match}") + with pytest.raises(RuntimeError, match=match): + module._publisher_scope(robot) + assert module._verified_publisher(robot) is False + + +@pytest.mark.parametrize( + "responses,match,unavailable", + [ + ([(503, b"", {})], "503", True), + ([(403, b"", {})], "403", False), + ([(200, b"[]", _count(0))], "exactly one", False), + ([(200, b"[]", _count(0))], "exactly one", False), + ([(200, b'[{\"name\":\"robot$jenkins-pipelines\",\"id\":0}]', _count(1))], "valid ID", False), + ], +) +def test_publisher_list_rejects_status_count_and_id( + responses, match: str, unavailable: bool +) -> None: + """Publisher discovery distinguishes readiness from permanent policy errors.""" + module = _load(HARBOR, f"harbor_publisher_list_{match}_{unavailable}") + error = module.HarborUnavailable if unavailable else RuntimeError + with pytest.raises(error, match=match): + module.ensure_publisher_can_read_rule(FakeClient(responses)) + + +@pytest.mark.parametrize( + "second,match,unavailable", + [ + ((503, b"", {}), "503", True), + ((403, b"", {}), "403", False), + ((200, b"not-json", {}), "invalid robot JSON", False), + ((200, b"[]", {}), "invalid shape", False), + ((200, b'{}', {}), "not active and exact", False), + ], +) +def test_publisher_read_rejects_status_and_identity( + second, match: str, unavailable: bool +) -> None: + """The detailed robot reread must be available, valid, and exact.""" + module = _load(HARBOR, f"harbor_publisher_read_{match}_{unavailable}") + responses = [(200, _robot_list(module), _count(1)), second] + error = module.HarborUnavailable if unavailable else RuntimeError + with pytest.raises(error, match=match): + module.ensure_publisher_can_read_rule(FakeClient(responses)) + + +def test_publisher_update_rejects_duration_reference_and_status( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Only the preserved scope and a successful policy PUT can be accepted.""" + module = _load(HARBOR, "harbor_publisher_update_coverage") + for duration in ("forever",): + responses = [ + (200, _robot_list(module), _count(1)), + (200, json.dumps(_robot(module, duration=duration)).encode(), {}), + ] + with pytest.raises(RuntimeError, match="duration"): + module.ensure_publisher_can_read_rule(FakeClient(responses)) + + robot = _robot(module) + monkeypatch.setattr(module, "_publisher_scope", lambda _robot: ({}, [])) + responses = [ + (200, _robot_list(module), _count(1)), + (200, json.dumps(robot).encode(), {}), + ] + with pytest.raises(RuntimeError, match="normalization"): + module.ensure_publisher_can_read_rule(FakeClient(responses)) + + monkeypatch.undo() + for status, error in ((503, module.HarborUnavailable), (409, RuntimeError)): + responses = [ + (200, _robot_list(module), _count(1)), + (200, json.dumps(_robot(module)).encode(), {}), + (status, b"", {}), + ] + with pytest.raises(error): + module.ensure_publisher_can_read_rule(FakeClient(responses)) + + +def test_publisher_and_rule_verification_retries_are_bounded( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Stale reads may retry, but never silently become successful evidence.""" + module = _load(HARBOR, "harbor_retry_coverage") + monkeypatch.setattr(module.time, "sleep", lambda _seconds: None) + responses = [ + (200, _robot_list(module), _count(1)), + (200, json.dumps(_robot(module)).encode(), {}), + (200, b"", {}), + *((500, b"", {}) for _ in range(5)), + ] + with pytest.raises(RuntimeError, match="did not verify"): + module.ensure_publisher_can_read_rule(FakeClient(responses)) + + rule_path = "/api/v2.0/projects/bstein/immutabletagrules/23" + responses = [ + (200, b"[]", _count(0)), + (201, b"", {"Location": rule_path}), + *((200, b"[]", _count(0)) for _ in range(5)), + ] + with pytest.raises(RuntimeError, match="did not verify"): + module.ensure_rule(FakeClient(responses)) + + +def test_rule_rejects_ids_create_status_and_location_suffix() -> None: + """Neither existing nor newly created rules can omit their exact positive ID.""" + module = _load(HARBOR, "harbor_rule_id_coverage") + bad_rule = {"id": 0, **module.EXPECTED_RULE} + with pytest.raises(RuntimeError, match="valid ID"): + module.ensure_rule( + FakeClient([(200, json.dumps([bad_rule]).encode(), _count(1))]) + ) + with pytest.raises(module.HarborUnavailable): + module.ensure_rule( + FakeClient([(200, b"[]", _count(0)), (503, b"", {})]) + ) + with pytest.raises(RuntimeError, match="invalid ID"): + module.ensure_rule( + FakeClient( + [ + (200, b"[]", _count(0)), + ( + 201, + b"", + { + "location": "/api/v2.0/projects/bstein/immutabletagrules/nope" + }, + ), + ] + ) + ) + + +def test_harbor_main_retries_then_reports_exact_ids( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + """Runtime credentials are required and transient startup failures are bounded.""" + module = _load(HARBOR, "harbor_main_coverage") + password_file = tmp_path / "password" + password_file.write_text("private\n", encoding="utf-8") + monkeypatch.setenv("HARBOR_API_ORIGIN", module.EXPECTED_ORIGIN) + monkeypatch.setenv("HARBOR_ADMIN_PASSWORD_FILE", str(password_file)) + monkeypatch.setattr(module, "HarborClient", lambda *_args: object()) + attempts = iter([module.HarborUnavailable("warming"), 17]) + + def ensure_rule(_client): + result = next(attempts) + if isinstance(result, BaseException): + raise result + return result + + monkeypatch.setattr(module, "ensure_rule", ensure_rule) + monkeypatch.setattr(module, "ensure_publisher_can_read_rule", lambda _client: 41) + monkeypatch.setattr(module.time, "sleep", lambda seconds: None) + assert module.main() == 0 + assert "id=17" in capsys.readouterr().out + + password_file.write_text("\n", encoding="utf-8") + with pytest.raises(RuntimeError, match="empty"): + module.main() diff --git a/testing/tests/test_hermes_image_builder_fresh_review.py b/testing/tests/test_hermes_image_builder_fresh_review.py new file mode 100644 index 00000000..27ccc9b3 --- /dev/null +++ b/testing/tests/test_hermes_image_builder_fresh_review.py @@ -0,0 +1,335 @@ +"""Fresh fail-closed review cases for the Hermes image builder.""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +import pytest +import yaml + + +REPO_ROOT = Path(__file__).resolve().parents[2] +PIPELINE = REPO_ROOT / "ci/Jenkinsfile.hermes-agent-image" +DOCKERFILE = REPO_ROOT / "dockerfiles/Dockerfile.hermes-agent" +RUNNER = REPO_ROOT / "dockerfiles/hermes-kaniko-heredoc-runner.py" +RELEASE = REPO_ROOT / "ci/scripts/hermes_image_release.py" +BUILDER_SA = REPO_ROOT / "services/jenkins/hermes-image-builder-serviceaccount.yaml" + + +def _load(path: Path, name: str): + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _pod_spec() -> dict: + source = PIPELINE.read_text(encoding="utf-8") + pod_yaml = source.split('yaml """', 1)[1].split('"""', 1)[0] + return yaml.safe_load(pod_yaml)["spec"] + + +def test_builder_prefers_rpi5_with_healthy_arm64_worker_fallback() -> None: + """Disposable builds prefer rpi5 without excluding schedulable rpi4 workers.""" + spec = _pod_spec() + assert spec["nodeSelector"] == { + "kubernetes.io/arch": "arm64", + "node-role.kubernetes.io/worker": "true", + } + affinity = spec["affinity"]["nodeAffinity"] + assert affinity["preferredDuringSchedulingIgnoredDuringExecution"] == [ + { + "weight": 100, + "preference": { + "matchExpressions": [ + {"key": "hardware", "operator": "In", "values": ["rpi5"]} + ] + }, + } + ] + expressions = affinity["requiredDuringSchedulingIgnoredDuringExecution"][ + "nodeSelectorTerms" + ][0]["matchExpressions"] + host_rule = next( + rule for rule in expressions if rule["key"] == "kubernetes.io/hostname" + ) + assert host_rule["operator"] == "NotIn" + assert set(host_rule["values"]) >= { + "titan-04", + "titan-14", + "titan-18", + "titan-19", + "titan-22", + "titan-24", + } + + +@pytest.mark.parametrize( + "unsupported", + [ + "RUN cat < None: + """Any RUN heredoc outside the reviewed nine is rejected before replay.""" + module = _load(RUNNER, f"heredoc_reject_{abs(hash(unsupported))}") + dockerfile = tmp_path / "Dockerfile" + dockerfile.write_text( + DOCKERFILE.read_text(encoding="utf-8") + "\n" + unsupported + "\n", + encoding="utf-8", + ) + calls = [] + monkeypatch.setattr(module.subprocess, "run", lambda *_args, **_kwargs: calls.append(1)) + with pytest.raises(ValueError, match="unsupported RUN heredoc"): + module.replay(dockerfile, 1) + assert calls == [] + + +def test_exact_reviewed_heredocs_remain_buildkit_native_by_default() -> None: + """Current source inventories cleanly and enables replay only for Kaniko.""" + module = _load(RUNNER, "heredoc_positive_contract") + source = DOCKERFILE.read_text(encoding="utf-8") + assert len(module.extract_blocks(source)) == 9 + assert source.count("ARG HERMES_KANIKO_HEREDOC_COMPAT=0") == 1 + assert "Kaniko v1.23.2" in RUNNER.read_text(encoding="utf-8") + pipeline = PIPELINE.read_text(encoding="utf-8") + assert pipeline.count("--build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1") == 1 + + +def test_appended_tenth_reviewed_form_rejects_before_execution( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """Even an otherwise supported marker cannot expand the nine-block set.""" + module = _load(RUNNER, "heredoc_tenth_block") + dockerfile = tmp_path / "Dockerfile" + dockerfile.write_text( + DOCKERFILE.read_text(encoding="utf-8") + + "\nRUN node <<'NODE'\nconsole.log('tenth');\nNODE\n", + encoding="utf-8", + ) + calls = [] + monkeypatch.setattr(module.subprocess, "run", lambda *_args, **_kwargs: calls.append(1)) + with pytest.raises(ValueError, match="contract changed"): + module.replay(dockerfile, 1) + assert calls == [] + + +@pytest.mark.parametrize( + ("prefix", "unsupported"), + [ + ("", "R\\\nUN cat < None: + """Split RUN/opcode operators and backtick escapes cannot evade inventory.""" + module = _load(RUNNER, f"heredoc_logical_{abs(hash((prefix, unsupported)))}") + dockerfile = tmp_path / "Dockerfile" + dockerfile.write_text( + prefix + DOCKERFILE.read_text(encoding="utf-8") + "\n" + unsupported + "\n", + encoding="utf-8", + ) + calls = [] + monkeypatch.setattr(module.subprocess, "run", lambda *_args, **_kwargs: calls.append(1)) + with pytest.raises(ValueError, match="unsupported RUN heredoc"): + module.replay(dockerfile, 1) + assert calls == [] + + +def test_merged_main_replays_each_block_before_dependent_work() -> None: + """PR13's blocked-task regression runs only after its source patch replay.""" + source = DOCKERFILE.read_text(encoding="utf-8") + replay_positions = [ + source.index(f"--block-index {index}") for index in range(1, 10) + ] + # Locate repeated markers rather than trusting one synthetic occurrence. + block_positions = [] + for marker in ("RUN node <<'NODE'", "RUN python - <<'PY'"): + start = 0 + while True: + position = source.find(marker, start) + if position < 0: + break + block_positions.append(position) + start = position + len(marker) + block_positions.sort() + assert len(block_positions) == len(replay_positions) == 9 + for index, (block, replay) in enumerate(zip(block_positions, replay_positions)): + assert block < replay + if index + 1 < len(block_positions): + assert replay < block_positions[index + 1] + regression = source.index( + "RUN /opt/hermes/.venv/bin/python /tmp/hermes-kanban-blocked-regression.py" + ) + assert replay_positions[4] < regression < replay_positions[5] + + +def test_builder_service_account_is_explicit_tokenless_and_unbound() -> None: + """The build Pod selects one tokenless SA that no tracked RBAC grants bind.""" + account = yaml.safe_load(BUILDER_SA.read_text(encoding="utf-8")) + assert account == { + "apiVersion": "v1", + "kind": "ServiceAccount", + "metadata": {"name": "hermes-image-builder", "namespace": "jenkins"}, + "automountServiceAccountToken": False, + } + kustomization = yaml.safe_load( + (REPO_ROOT / "services/jenkins/kustomization.yaml").read_text(encoding="utf-8") + ) + assert "hermes-image-builder-serviceaccount.yaml" in kustomization["resources"] + + spec = _pod_spec() + assert spec["serviceAccountName"] == "hermes-image-builder" + assert spec["automountServiceAccountToken"] is False + for volume in spec.get("volumes", []): + projected = volume.get("projected", {}) + assert all("serviceAccountToken" not in item for item in projected.get("sources", [])) + + for manifest in (REPO_ROOT / "services/jenkins").glob("*.yaml"): + for document in yaml.safe_load_all(manifest.read_text(encoding="utf-8")): + if not isinstance(document, dict) or document.get("kind") not in { + "RoleBinding", + "ClusterRoleBinding", + }: + continue + subjects = document.get("subjects") or [] + assert not any( + item.get("kind") == "ServiceAccount" + and item.get("name") == "hermes-image-builder" + for item in subjects + ) + + +def _release_fixture(tmp_path: Path): + module = _load(RELEASE, f"release_evidence_{tmp_path.name}") + digest = "sha256:" + "7" * 64 + revision = "8" * 40 + build = "23" + destination = f"{module.DEFAULT_IMAGE}:git-{revision}-build-{build}" + digest_file = tmp_path / "hermes-agent.digest" + image_file = tmp_path / "hermes-agent.image" + manifest = tmp_path / "kustomization.yaml" + output = tmp_path / "release" + digest_file.write_text(digest + "\n", encoding="utf-8") + image_file.write_text(f"{destination}@{digest}\n", encoding="utf-8") + manifest.write_text( + "images:\n" + f" - name: {module.DEFAULT_IMAGE}\n" + " digest: sha256:" + "0" * 64 + "\n", + encoding="utf-8", + ) + module.write_release_artifacts( + digest=digest, + source_revision=revision, + build_number=build, + destination=destination, + kustomization=manifest, + output_dir=output, + ) + kwargs = { + "digest_file": digest_file, + "image_file": image_file, + "source_revision": revision, + "build_number": build, + "destination": destination, + "kustomization": manifest, + "output_dir": output, + } + return module, kwargs + + +def test_success_evidence_revalidation_accepts_only_exact_complete_set( + tmp_path: Path, +) -> None: + """Exact evidence passes; an extra, missing, or altered file fails closed.""" + module, kwargs = _release_fixture(tmp_path) + module.validate_release_artifacts(**kwargs) + + extra = kwargs["output_dir"] / "unexpected" + extra.write_text("surprise\n", encoding="utf-8") + with pytest.raises(ValueError, match="exactly three"): + module.validate_release_artifacts(**kwargs) + extra.unlink() + + metadata = kwargs["output_dir"] / "hermes-agent-image.json" + original = metadata.read_text(encoding="utf-8") + metadata.write_text(original.replace('"build_number": "23"', '"build_number": "24"')) + with pytest.raises(ValueError, match="incomplete or mismatched"): + module.validate_release_artifacts(**kwargs) + metadata.write_text(original, encoding="utf-8") + (kwargs["output_dir"] / "hermes-image-update.patch").unlink() + with pytest.raises(ValueError, match="exactly three"): + module.validate_release_artifacts(**kwargs) + + +def test_verify_evidence_cli_needs_no_runtime_registry_credential( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Post-success validation is deterministic and cannot mask missing creds.""" + module, kwargs = _release_fixture(tmp_path) + monkeypatch.delenv("HARBOR_USER", raising=False) + monkeypatch.delenv("HARBOR_PASSWORD", raising=False) + monkeypatch.setattr( + sys, + "argv", + [ + "hermes_image_release.py", + "verify-evidence", + "--digest-file", + str(kwargs["digest_file"]), + "--image-file", + str(kwargs["image_file"]), + "--source-revision", + kwargs["source_revision"], + "--build-number", + kwargs["build_number"], + "--destination", + kwargs["destination"], + "--kustomization", + str(kwargs["kustomization"]), + "--output-dir", + str(kwargs["output_dir"]), + ], + ) + assert module.main() == 0 + + +def test_pipeline_success_post_requires_and_archives_exact_six_files() -> None: + """Missing or partial post-success evidence must change the build to failed.""" + source = PIPELINE.read_text(encoding="utf-8") + post = source.split(" post {", 1)[1] + assert "success {" in post + assert "always {" not in post + assert "verify-evidence" in post + assert 'allowEmptyArchive: false' in post + archive = post.split("artifacts: '", 1)[1].split("'", 1)[0] + paths = archive.split(",") + assert len(paths) == 6 + assert len(set(paths)) == 6 + assert all("*" not in path for path in paths) + assert "find build -type f" in post diff --git a/testing/tests/test_hermes_image_builder_harbor.py b/testing/tests/test_hermes_image_builder_harbor.py new file mode 100644 index 00000000..9a64e24a --- /dev/null +++ b/testing/tests/test_hermes_image_builder_harbor.py @@ -0,0 +1,325 @@ +"""Server-side Harbor immutability contracts for Hermes agent releases.""" + +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path + +import pytest +import yaml + + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ( + REPO_ROOT + / "services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py" +) + + +def _load_module(): + spec = importlib.util.spec_from_file_location("harbor_immutability", SCRIPT) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _rule(module, *, rule_id: int = 17, disabled: bool = False) -> dict: + return {"id": rule_id, **module.EXPECTED_RULE, "disabled": disabled} + + +def _robot(module, *, immutable: bool = False, extra_immutable: bool = False) -> dict: + access = [ + {"resource": "repository", "action": "pull", "effect": "allow"}, + {"resource": "repository", "action": "push", "effect": "allow"}, + ] + if immutable: + access.append({"resource": "immutable-tag", "action": "list"}) + if extra_immutable: + access.append({"resource": "immutable-tag", "action": "delete"}) + return { + "id": 41, + "name": module.PUBLISH_ROBOT, + "description": "Jenkins publisher", + "level": "system", + "duration": -1, + "editable": True, + "disable": False, + "permissions": [ + { + "kind": "project", + "namespace": "other", + "access": [{"resource": "repository", "action": "pull"}], + }, + {"kind": "project", "namespace": "bstein", "access": access}, + ], + } + + +def _count(value: int) -> dict[str, str]: + return {"X-Total-Count": str(value)} + + +class FakeClient: + """Small deterministic Harbor API fake.""" + + def __init__(self, responses): + self.origin = "https://registry.bstein.dev/api/v2.0" + self.responses = list(responses) + self.calls = [] + + def request(self, method, path, payload=None): + self.calls.append((method, path, payload)) + return self.responses.pop(0) + + +def test_flux_tracks_exact_immutable_rule_before_jenkins() -> None: + """The Harbor rule is a reviewed prerequisite, not a Jenkins preflight only.""" + harbor = yaml.safe_load( + ( + REPO_ROOT + / "clusters/atlas/flux-system/applications/harbor/kustomization.yaml" + ).read_text() + ) + jenkins = yaml.safe_load( + ( + REPO_ROOT + / "clusters/atlas/flux-system/applications/jenkins/kustomization.yaml" + ).read_text() + ) + check = { + "apiVersion": "batch/v1", + "kind": "Job", + "name": "harbor-hermes-agent-immutability-ensure-1", + "namespace": "harbor", + } + assert check in harbor["spec"]["healthChecks"] + assert "harbor" in {item["name"] for item in jenkins["spec"]["dependsOn"]} + + +def test_policy_job_uses_runtime_vault_secret_and_hardened_pinned_image() -> None: + """No Harbor credential is committed or retained in a mutable workload.""" + job = yaml.safe_load( + (REPO_ROOT / "services/harbor/hermes-agent-immutability-job.yaml").read_text() + ) + template = job["spec"]["template"] + annotations = template["metadata"]["annotations"] + assert annotations["vault.hashicorp.com/role"] == "harbor-policy-bootstrap" + assert ( + annotations["vault.hashicorp.com/agent-inject-secret-harbor-admin-password"] + == "kv/data/atlas/harbor/harbor-core" + ) + pod = template["spec"] + assert pod["serviceAccountName"] == "harbor-policy-bootstrap" + assert pod["enableServiceLinks"] is False + container = pod["containers"][0] + assert "@sha256:" in container["image"] + security = container["securityContext"] + assert security["runAsNonRoot"] is True + assert security["readOnlyRootFilesystem"] is True + assert security["allowPrivilegeEscalation"] is False + assert security["capabilities"]["drop"] == ["ALL"] + + vault = ( + REPO_ROOT / "services/vault/scripts/vault_k8s_auth_configure.sh" + ).read_text() + policy = vault.split("harbor_policy_bootstrap_policy='", 1)[1].split("'", 1)[0] + assert 'path "kv/data/atlas/harbor/harbor-core"' in policy + assert 'capabilities = ["read"]' in policy + assert "*" not in policy + assert 'bound_service_account_names="harbor-policy-bootstrap"' in vault + script = SCRIPT.read_text() + assert 'PUBLISH_ROBOT = "robot$jenkins-pipelines"' in script + assert '{"resource": "immutable-tag", "action": "list"}' in script + assert 'client.request(\n "PUT", f"/robots/{robot_id}", payload' in script + assert '"secret"' not in script.split("payload = {", 1)[1].split("}", 1)[0] + + +def test_rule_contract_is_exact_repository_and_unique_build_tags() -> None: + """Unrelated bstein repositories and ordinary Hermes tags stay mutable.""" + module = _load_module() + assert module.EXPECTED_RULE == { + "disabled": False, + "action": "immutable", + "template": "immutable_template", + "tag_selectors": [ + { + "kind": "doublestar", + "decoration": "matches", + "pattern": "git-*-build-*", + } + ], + "scope_selectors": { + "repository": [ + { + "kind": "doublestar", + "decoration": "repoMatches", + "pattern": "hermes-agent", + } + ] + }, + } + + +def test_existing_exact_rule_is_idempotent_without_mutation() -> None: + """A rerun only validates the exact enabled rule.""" + module = _load_module() + body = json.dumps([_rule(module)]).encode() + client = FakeClient([(200, body, _count(1))]) + assert module.ensure_rule(client) == 17 + assert [call[0] for call in client.calls] == ["GET"] + + +def test_create_requires_201_exact_location_and_verified_reread() -> None: + """Policy bootstrap fails closed until Harbor returns the exact persisted rule.""" + module = _load_module() + body = json.dumps([_rule(module, rule_id=23)]).encode() + client = FakeClient( + [ + (200, b"[]", _count(0)), + ( + 201, + b"", + {"Location": "/api/v2.0/projects/bstein/immutabletagrules/23"}, + ), + (200, body, _count(1)), + ] + ) + assert module.ensure_rule(client) == 23 + assert client.calls[1] == ( + "POST", + "/projects/bstein/immutabletagrules", + module.EXPECTED_RULE, + ) + + +@pytest.mark.parametrize( + "responses,match", + [ + ([(200, b"[]", _count(0)), (200, b"", {})], "HTTP 200"), + ( + [ + (200, b"[]", _count(0)), + (201, b"", {"Location": "https://evil.invalid/1"}), + ], + "Location", + ), + ], +) +def test_create_rejects_noncanonical_responses(responses, match: str) -> None: + """Proxy success pages and foreign locations can never count as enforcement.""" + module = _load_module() + with pytest.raises(RuntimeError, match=match): + module.ensure_rule(FakeClient(responses)) + + +def test_readiness_failures_are_distinct_from_policy_rejections() -> None: + """The tracked Job may wait for Harbor without retrying an auth denial.""" + module = _load_module() + with pytest.raises(module.HarborUnavailable): + module.ensure_rule(FakeClient([(503, b"", {})])) + with pytest.raises(RuntimeError, match="HTTP 403") as exc: + module.ensure_rule(FakeClient([(403, b"", {})])) + assert not isinstance(exc.value, module.HarborUnavailable) + + +def test_disabled_or_duplicate_exact_scope_fails_closed() -> None: + """Bootstrap never silently edits a conflicting security policy.""" + module = _load_module() + disabled = json.dumps([_rule(module, disabled=True)]).encode() + with pytest.raises(RuntimeError, match="not enabled"): + module.ensure_rule(FakeClient([(200, disabled, _count(1))])) + + duplicate = json.dumps([_rule(module), _rule(module, rule_id=18)]).encode() + with pytest.raises(RuntimeError, match="multiple"): + module.ensure_rule(FakeClient([(200, duplicate, _count(2))])) + + +def test_policy_lists_reject_missing_or_truncated_count_evidence() -> None: + """A hidden second page can never produce a duplicate rule or robot update.""" + module = _load_module() + body = json.dumps([_rule(module)]).encode() + for headers in ({}, _count(2)): + with pytest.raises(RuntimeError, match="count|truncated"): + module.ensure_rule(FakeClient([(200, body, headers)])) + robot_body = json.dumps( + [{"id": 41, "name": module.PUBLISH_ROBOT}] + ).encode() + with pytest.raises(RuntimeError, match="truncated"): + module.ensure_publisher_can_read_rule( + FakeClient([(200, robot_body, _count(2))]) + ) + + +def test_publisher_policy_adds_only_read_access_and_preserves_robot() -> None: + """Bootstrap preserves every existing scope and never touches robot secret state.""" + module = _load_module() + original = _robot(module) + persisted = _robot(module, immutable=True) + client = FakeClient( + [ + ( + 200, + json.dumps([{"id": 41, "name": module.PUBLISH_ROBOT}]).encode(), + _count(1), + ), + (200, json.dumps(original).encode(), {}), + (200, b"", {}), + (200, json.dumps(persisted).encode(), {}), + ] + ) + assert module.ensure_publisher_can_read_rule(client) == 41 + method, path, payload = client.calls[2] + assert (method, path) == ("PUT", "/robots/41") + assert "secret" not in payload + assert payload["permissions"][0] == original["permissions"][0] + assert payload["permissions"][1]["access"][-1] == { + "resource": "immutable-tag", + "action": "list", + } + + +def test_publisher_policy_is_idempotent_and_rejects_broad_access() -> None: + """An exact read policy is stable; mutation-capable immutable access is blocked.""" + module = _load_module() + exact = _robot(module, immutable=True) + client = FakeClient( + [ + ( + 200, + json.dumps([{"id": 41, "name": module.PUBLISH_ROBOT}]).encode(), + _count(1), + ), + (200, json.dumps(exact).encode(), {}), + ] + ) + assert module.ensure_publisher_can_read_rule(client) == 41 + assert [call[0] for call in client.calls] == ["GET", "GET"] + + broad = _robot(module, immutable=True, extra_immutable=True) + client = FakeClient( + [ + ( + 200, + json.dumps([{"id": 41, "name": module.PUBLISH_ROBOT}]).encode(), + _count(1), + ), + (200, json.dumps(broad).encode(), {}), + ] + ) + with pytest.raises(RuntimeError, match="broader"): + module.ensure_publisher_can_read_rule(client) + + +def test_harbor_client_rejects_plaintext_or_ambiguous_origins() -> None: + """Runtime admin credentials are never sent over HTTP or a URL with query state.""" + module = _load_module() + for origin in ( + "http://registry.bstein.dev/api/v2.0", + "https://registry.bstein.dev/api/v2.0?next=evil", + "https://other.invalid/api/v2.0", + "", + ): + with pytest.raises(ValueError): + module.HarborClient(origin, "admin", "secret") diff --git a/testing/tests/test_hermes_image_builder_vault.py b/testing/tests/test_hermes_image_builder_vault.py new file mode 100644 index 00000000..f46823f5 --- /dev/null +++ b/testing/tests/test_hermes_image_builder_vault.py @@ -0,0 +1,329 @@ +"""Vault least-privilege and fail-closed seeding tests for image releases.""" + +from __future__ import annotations + +import os +import stat +import subprocess +from pathlib import Path + +import pytest +import yaml + + +REPO_ROOT = Path(__file__).resolve().parents[2] +VAULT_CONFIG = REPO_ROOT / "services/vault/scripts/vault_k8s_auth_configure.sh" +SEEDER = ( + REPO_ROOT + / "services/vault-hermes-jenkins-token-seed/scripts/vault_hermes_jenkins_build_token_ensure.sh" +) + + +def test_developer_jenkins_access_is_read_only_and_bound_to_two_consumers() -> None: + """Only the Jenkins controller and Hermes agent can read the fixed-job token.""" + source = VAULT_CONFIG.read_text(encoding="utf-8") + assert 'write_policy_and_role "jenkins" "jenkins" "jenkins"' in source + assert ( + '"jenkins/* shared/harbor-pull quality/sonarqube-oidc ' + 'hermes/developer-jenkins" ""' + ) in source + assert 'write_policy_and_role "hermes-agent" "hermes" "hermes-agent"' in source + assert 'hermes/developer-jenkins hermes/developer-ssh" ""' in source + assert 'write_policy_and_role "hermes-switchyard" "hermes"' in source + assert '"hermes/chat-telegram" ""' in source + assert '"hermes/developer-jenkins"' not in source + + jenkins_spc = yaml.safe_load( + (REPO_ROOT / "services/jenkins/secretproviderclass.yaml").read_text() + ) + assert jenkins_spc["spec"]["parameters"]["roleName"] == "jenkins-vault-sync" + switchyard = yaml.safe_load( + (REPO_ROOT / "services/hermes/switchyard-deployment.yaml").read_text() + ) + annotations = switchyard["spec"]["template"]["metadata"]["annotations"] + assert annotations["vault.hashicorp.com/role"] == "hermes-switchyard" + + +def test_flux_orders_seed_then_jenkins_then_hermes() -> None: + """The fixed token must be Ready before either consumer is rolled.""" + app_root = REPO_ROOT / "clusters/atlas/flux-system/applications" + vault = yaml.safe_load((app_root / "vault/kustomization.yaml").read_text()) + seed = yaml.safe_load( + (app_root / "vault-hermes-jenkins-token-seed/kustomization.yaml").read_text() + ) + jenkins = yaml.safe_load((app_root / "jenkins/kustomization.yaml").read_text()) + hermes = yaml.safe_load((app_root / "hermes/kustomization.yaml").read_text()) + role_check = { + "apiVersion": "batch/v1", + "kind": "Job", + "name": "vault-k8s-auth-hermes-9", + "namespace": "vault", + } + seed_check = { + "apiVersion": "batch/v1", + "kind": "Job", + "name": "vault-hermes-jenkins-build-token-seed-2", + "namespace": "vault", + } + assert role_check in vault["spec"]["healthChecks"] + assert {item["name"] for item in seed["spec"]["dependsOn"]} == {"vault"} + assert seed_check in seed["spec"]["healthChecks"] + assert jenkins["spec"]["suspend"] is False + assert {item["name"] for item in jenkins["spec"]["dependsOn"]} >= { + "helm", + "harbor", + "vault-hermes-jenkins-token-seed", + } + assert "jenkins" in {item["name"] for item in hermes["spec"]["dependsOn"]} + + +def test_seed_job_is_revisioned_bounded_and_tracks_fail_closed_script() -> None: + """The prerequisite is a tracked one-shot on healthy non-reserved capacity.""" + job = yaml.safe_load( + ( + REPO_ROOT / "services/vault-hermes-jenkins-token-seed/job.yaml" + ).read_text() + ) + assert job["metadata"]["name"] == "vault-hermes-jenkins-build-token-seed-2" + pod = job["spec"]["template"]["spec"] + assert pod["serviceAccountName"] == "hermes-jenkins-token-seed" + assert pod["enableServiceLinks"] is False + assert pod["restartPolicy"] == "Never" + assert pod["nodeSelector"]["hardware"] == "rpi5" + expression = pod["affinity"]["nodeAffinity"][ + "requiredDuringSchedulingIgnoredDuringExecution" + ]["nodeSelectorTerms"][0]["matchExpressions"][0] + assert set(expression["values"]) >= { + "titan-04", + "titan-14", + "titan-18", + "titan-19", + "titan-24", + } + container = pod["containers"][0] + assert "@sha256:" in container["image"] + security = container["securityContext"] + assert security["runAsNonRoot"] is True + assert security["readOnlyRootFilesystem"] is True + assert security["allowPrivilegeEscalation"] is False + assert security["capabilities"]["drop"] == ["ALL"] + source = SEEDER.read_text(encoding="utf-8") + assert '"options":{"cas":0}' in source + assert 'vault kv get -field=build_token "${secret_path}"' in source + assert "kv/data/atlas/hermes/developer-jenkins" in source + assert '"build_token"[[:space:]]' not in source + assert "kv patch" not in source + assert "kv delete" not in source + assert SEEDER.name not in VAULT_CONFIG.read_text(encoding="utf-8") + + +def test_seed_vault_policy_is_exact_create_read_and_rng_only() -> None: + """The seed identity cannot rotate, delete, list, or touch unrelated KV.""" + source = VAULT_CONFIG.read_text(encoding="utf-8") + policy = source.split("hermes_jenkins_token_seed_policy='", 1)[1].split("'", 1)[0] + assert 'path "kv/data/atlas/hermes/developer-jenkins"' in policy + assert 'capabilities = ["create", "read"]' in policy + assert 'path "sys/tools/random/32"' in policy + assert 'capabilities = ["update"]' in policy + for forbidden in ("delete", "patch", "list", 'kv/data/atlas/hermes/*'): + assert forbidden not in policy + assert 'bound_service_account_names="hermes-jenkins-token-seed"' in source + assert 'bound_service_account_namespaces="vault"' in source + + +def _fake_vault_tools(tmp_path: Path) -> tuple[Path, Path, Path]: + """Create deterministic Vault/sleep fakes and return their evidence paths.""" + binary_dir = tmp_path / "bin" + binary_dir.mkdir(parents=True) + calls = tmp_path / "calls" + stdin_capture = tmp_path / "stdin" + vault = binary_dir / "vault" + vault.write_text( + """#!/bin/sh +set -eu +printf '%s\n' "$*" >> "$FAKE_CALL_LOG" +scenario="$FAKE_SCENARIO" +if [ "$1" = "write" ] && [ "$2" = "-field=random_bytes" ] && [ "$3" = "sys/tools/random/32" ]; then + printf '%064d\n' 0 + exit 0 +fi +if [ "$1" = "kv" ] && [ "$2" = "get" ] && [ "$3" = "-field=build_token" ]; then + case "$scenario" in + existing) + printf '%064d\n' 1 + exit 0 + ;; + cas-race) + printf '%064d\n' 2 + exit 0 + ;; + field-prefix) + printf 'prefix%064d\n' 3 + exit 0 + ;; + field-suffix) + printf '%064dsuffix\n' 4 + exit 0 + ;; + field-multiline) + printf '%064d\nextra\n' 5 + exit 0 + ;; + missing-field|metadata-only|custom-metadata) + echo 'No value found at build_token' >&2 + exit 2 + ;; + esac +fi +if [ "$1" = "write" ] && [ "$2" = "kv/data/atlas/hermes/developer-jenkins" ]; then + payload="${3#@}" + cp "$payload" "$FAKE_STDIN_CAPTURE" + if [ "$scenario" = "cas-race" ]; then + : > "$FAKE_WINNER" + echo 'check-and-set parameter did not match' >&2 + exit 2 + fi + exit 0 +fi +if [ "$1" = "read" ] && [ "$2" = "-format=json" ]; then + case "$scenario" in + existing) + printf '{"data":{"data":{"build_token":"%064d"},"metadata":{"build_token":"decoy"}}}\n' 1 + exit 0 + ;; + missing-field) + printf '{"data":{"data":{"other":"preserve-me"}}}\n' + exit 0 + ;; + metadata-only) + printf '{"data":{"data":{"other":"preserve-me"},"metadata":{"build_token":"%064d"}}}\n' 6 + exit 0 + ;; + custom-metadata) + printf '{"data":{"data":{"other":"preserve-me"},"metadata":{"custom_metadata":{"build_token":"%064d"}}}}\n' 7 + exit 0 + ;; + field-prefix|field-suffix|field-multiline) + printf '{"data":{"data":{"build_token":"present"}}}\n' + exit 0 + ;; + absent) + echo 'Code: 404' >&2 + exit 2 + ;; + cas-race) + if [ -f "$FAKE_WINNER" ]; then + printf '{"data":{"data":{"build_token":"%064d"}}}\n' 2 + exit 0 + fi + echo 'Code: 404' >&2 + exit 2 + ;; + transient) + count=0 + if [ -f "$FAKE_COUNT" ]; then count="$(cat "$FAKE_COUNT")"; fi + count=$((count + 1)) + printf '%s' "$count" > "$FAKE_COUNT" + if [ "$count" -lt 3 ]; then echo 'temporary upstream error' >&2; exit 2; fi + echo 'Code: 404' >&2 + exit 2 + ;; + read-error) + echo 'permission denied' >&2 + exit 2 + ;; + esac +fi +echo "unexpected fake Vault call: $*" >&2 +exit 99 +""", + encoding="utf-8", + ) + sleep = binary_dir / "sleep" + sleep.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + for executable in (vault, sleep): + executable.chmod(executable.stat().st_mode | stat.S_IXUSR) + return binary_dir, calls, stdin_capture + + +def _run_seeder( + tmp_path: Path, scenario: str +) -> tuple[subprocess.CompletedProcess, str]: + binary_dir, calls, stdin_capture = _fake_vault_tools(tmp_path) + env = { + **os.environ, + "PATH": f"{binary_dir}:{os.environ['PATH']}", + "VAULT_TOKEN": "test-token", + "FAKE_SCENARIO": scenario, + "FAKE_CALL_LOG": str(calls), + "FAKE_STDIN_CAPTURE": str(stdin_capture), + "FAKE_WINNER": str(tmp_path / "winner"), + "FAKE_COUNT": str(tmp_path / "count"), + "TMPDIR": str(tmp_path), + } + result = subprocess.run( + ["sh", str(SEEDER)], + env=env, + text=True, + capture_output=True, + check=False, + ) + call_text = calls.read_text(encoding="utf-8") if calls.exists() else "" + return result, call_text + + +@pytest.mark.parametrize( + "scenario", + ["existing", "missing-field", "metadata-only", "custom-metadata", "read-error"], +) +def test_seeder_never_overwrites_existing_or_ambiguous_state( + tmp_path: Path, scenario: str +) -> None: + """Existing fields and persistent read failures can never become a put.""" + result, calls = _run_seeder(tmp_path, scenario) + assert (result.returncode == 0) is (scenario == "existing") + assert "write kv/data/atlas/hermes/developer-jenkins" not in calls + assert "sys/tools/random" not in calls + + +@pytest.mark.parametrize("scenario", ["field-prefix", "field-suffix", "field-multiline"]) +def test_seeder_requires_the_entire_exact_data_field( + tmp_path: Path, scenario: str +) -> None: + """Prefix, suffix, or multiline values cannot satisfy readiness.""" + result, calls = _run_seeder(tmp_path, scenario) + assert result.returncode != 0 + assert "kv get -field=build_token kv/atlas/hermes/developer-jenkins" in calls + assert "write kv/data/atlas/hermes/developer-jenkins" not in calls + assert "sys/tools/random" not in calls + + +def test_seeder_uses_single_cas_create_and_stdin_for_absent_secret( + tmp_path: Path, +) -> None: + """A confirmed 404 produces one create-only write without a token argument.""" + result, calls = _run_seeder(tmp_path, "absent") + assert result.returncode == 0, result.stderr + assert calls.count("write kv/data/atlas/hermes/developer-jenkins") == 1 + assert "@" in calls + assert "0000000000000000000000000000000000000000000000000000000000000000" not in calls + payload = (tmp_path / "stdin").read_text(encoding="utf-8") + assert '"options":{"cas":0}' in payload + assert '"build_token":"' in payload + assert not (tmp_path / "hermes-jenkins-token.json").exists() + + +def test_seeder_retries_reads_and_accepts_only_a_verified_cas_winner( + tmp_path: Path, +) -> None: + """Transient reads retry; a competing create is accepted only after reread.""" + transient, transient_calls = _run_seeder(tmp_path / "transient", "transient") + assert transient.returncode == 0, transient.stderr + assert transient_calls.count("read -format=json kv/data/atlas/hermes/developer-jenkins") == 3 + assert transient_calls.count("write kv/data/atlas/hermes/developer-jenkins") == 1 + + race, race_calls = _run_seeder(tmp_path / "race", "cas-race") + assert race.returncode == 0, race.stderr + assert race_calls.count("write kv/data/atlas/hermes/developer-jenkins") == 1 + assert race_calls.count("read -format=json kv/data/atlas/hermes/developer-jenkins") == 2 + assert "another seeder won CAS" in race.stderr diff --git a/testing/tests/test_hermes_runtime_access.py b/testing/tests/test_hermes_runtime_access.py index 7e455eb6..fa4c8021 100644 --- a/testing/tests/test_hermes_runtime_access.py +++ b/testing/tests/test_hermes_runtime_access.py @@ -177,6 +177,7 @@ def test_agent_runtime_stage_keeps_credentials_in_memory(tmp_path: Path, monkeyp "chat-relay-key": "relay-key", "gitea-token": "gitea-key", "gitea-username": "hermes-automation", + "jenkins-image-build-token": "job-scoped-token", "node-ssh-private-key": "private-key", "node-ssh-config": "host-config", "node-ssh-known-hosts": "known-hosts", @@ -196,6 +197,7 @@ def test_agent_runtime_stage_keeps_credentials_in_memory(tmp_path: Path, monkeyp assert (runtime / "claude/.credentials.json").stat().st_mode & 0o777 == 0o600 assert (runtime / "codex/auth.json").stat().st_mode & 0o777 == 0o600 + assert (runtime / "jenkins-image-build-token").stat().st_mode & 0o777 == 0o600 assert (runtime / "claude/settings.json").is_symlink() assert (runtime / "codex/skills").is_symlink() assert not (runtime / "claude/backups").exists() From 3a57371989a0f76ad5aa363455a2dc2d9561b009 Mon Sep 17 00:00:00 2001 From: jenkins Date: Mon, 17 Aug 2026 15:10:17 -0300 Subject: [PATCH 2/4] hermes: close release-lane branch coverage gaps Cover the uncovered branches in the image release script (redirect handler, policy-read credential and size bounds, count and shape rejections, EOF manifest scan) and in the Harbor immutability helper (scope-failure verification, absent robot duration, stale-read retries, created-rule ID binding, bounded startup retries). Exclude the __main__ guard like the sibling release script. Co-Authored-By: Claude Fable 5 --- ...harbor_hermes_agent_immutability_ensure.py | 2 +- .../test_hermes_image_builder_branch_edges.py | 227 ++++++++++++++++++ 2 files changed, 228 insertions(+), 1 deletion(-) create mode 100644 testing/tests/test_hermes_image_builder_branch_edges.py diff --git a/services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py b/services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py index a313cc80..06475c50 100644 --- a/services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py +++ b/services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py @@ -374,5 +374,5 @@ def main() -> int: return 0 -if __name__ == "__main__": +if __name__ == "__main__": # pragma: no cover - exercised through main() raise SystemExit(main()) diff --git a/testing/tests/test_hermes_image_builder_branch_edges.py b/testing/tests/test_hermes_image_builder_branch_edges.py new file mode 100644 index 00000000..8ada5cbb --- /dev/null +++ b/testing/tests/test_hermes_image_builder_branch_edges.py @@ -0,0 +1,227 @@ +"""Branch-edge coverage for the Hermes release and Harbor policy scripts.""" + +from __future__ import annotations + +import importlib.util +import io +import json +from pathlib import Path + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parents[2] +RELEASE = REPO_ROOT / "ci/scripts/hermes_image_release.py" +HARBOR = ( + REPO_ROOT + / "services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py" +) + + +def _load(path: Path, name: str): + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class Response(io.BytesIO): + """Context-managed urllib response used without network access.""" + + def __init__( + self, + status: int, + body: bytes = b"", + headers: dict[str, str] | None = None, + ) -> None: + super().__init__(body) + self.status = status + self.headers = headers or {} + + +class FakeClient: + """Ordered Harbor response fake with the pinned API origin.""" + + origin = "https://registry.bstein.dev/api/v2.0" + + def __init__(self, responses) -> None: + self.responses = list(responses) + self.calls = [] + + def request(self, method, path, payload=None): + self.calls.append((method, path, payload)) + return self.responses.pop(0) + + +def _count(value: int) -> dict[str, str]: + return {"X-Total-Count": str(value)} + + +def _publisher(module, *, immutable: bool = False, with_duration: bool = True) -> dict: + access = [ + {"resource": "repository", "action": "pull"}, + {"resource": "repository", "action": "push"}, + ] + if immutable: + access.append({"resource": "immutable-tag", "action": "list"}) + robot = { + "id": 41, + "name": module.PUBLISH_ROBOT, + "description": "publisher", + "level": "system", + "editable": True, + "disable": False, + "permissions": [ + {"kind": "project", "namespace": module.PROJECT, "access": access} + ], + } + if with_duration: + robot["duration"] = -1 + return robot + + +def test_release_redirect_handler_never_forwards_credentials() -> None: + """The registry opener refuses to follow any redirect target.""" + module = _load(RELEASE, "release_redirect_edges") + handler = module._NoRedirect() + assert handler.redirect_request(None, None, 0, None, None, None) is None + + +def test_release_policy_read_requires_credentials_and_bounded_body() -> None: + """The policy preflight rejects empty credentials and oversized responses.""" + module = _load(RELEASE, "release_policy_read_edges") + with pytest.raises(RuntimeError, match="credentials are empty"): + module._immutable_rules_response(username="", password="private") + with pytest.raises(RuntimeError, match="size limit"): + module._immutable_rules_response( + username="robot", + password="private", + opener=lambda *_args: Response(200, b"x" * 1_048_577), + ) + + +@pytest.mark.parametrize("headers", [{}, {"X-Total-Count": "many"}]) +def test_release_policy_rejects_missing_or_invalid_count(headers: dict) -> None: + """A proxy that strips or mangles count evidence cannot prove completeness.""" + module = _load(RELEASE, "release_policy_count_edges") + with pytest.raises(RuntimeError, match="total count"): + module.verify_immutable_policy( + username="robot", + password="private", + opener=lambda *_args, evidence=headers: Response(200, b"[]", evidence), + ) + + +@pytest.mark.parametrize("body", [b"{}", b"[1]"]) +def test_release_policy_rejects_non_rule_list_shapes(body: bytes) -> None: + """Valid JSON that is not a list of rule objects fails the preflight.""" + module = _load(RELEASE, "release_policy_shape_edges") + with pytest.raises(RuntimeError, match="invalid shape"): + module.verify_immutable_policy( + username="robot", + password="private", + opener=lambda *_args, value=body: Response(200, value, _count(1)), + ) + + +def test_renderer_ignores_trailing_entry_without_digest_at_eof() -> None: + """A dangling final image entry cannot shadow the one complete digest.""" + module = _load(RELEASE, "release_render_eof_edges") + digest = "sha256:" + "6" * 64 + source = ( + "images:\n" + f" - name: {module.DEFAULT_IMAGE}\n" + " digest: sha256:" + "0" * 64 + "\n" + f" - name: {module.DEFAULT_IMAGE}\n" + " newTag: dangling\n" + ) + rendered = module.render_kustomization(source, digest) + assert f"digest: {digest}\n" in rendered + assert rendered.endswith("newTag: dangling\n") + + +def test_harbor_verified_publisher_rejects_exact_identity_with_bad_scope() -> None: + """An exact robot identity still fails verification on a malformed scope.""" + module = _load(HARBOR, "harbor_verified_scope_edges") + robot = _publisher(module) + robot["permissions"] = "bad" + assert module._verified_publisher(robot) is False + + +def test_harbor_publisher_update_omits_absent_duration(monkeypatch) -> None: + """A robot without a configured duration is preserved without inventing one.""" + module = _load(HARBOR, "harbor_duration_edges") + monkeypatch.setattr(module.time, "sleep", lambda _seconds: None) + verified = _publisher(module, immutable=True) + client = FakeClient( + [ + (200, json.dumps([{"id": 41, "name": module.PUBLISH_ROBOT}]).encode(), _count(1)), + (200, json.dumps(_publisher(module, with_duration=False)).encode(), {}), + (200, b"", {}), + (200, json.dumps(verified).encode(), {}), + ] + ) + assert module.ensure_publisher_can_read_rule(client) == 41 + method, path, payload = client.calls[2] + assert (method, path) == ("PUT", "/robots/41") + assert "duration" not in payload + + +def test_harbor_publisher_verify_retries_past_invalid_json(monkeypatch) -> None: + """One stale unreadable reread retries instead of passing or failing outright.""" + module = _load(HARBOR, "harbor_verify_retry_edges") + sleeps = [] + monkeypatch.setattr(module.time, "sleep", sleeps.append) + verified = _publisher(module, immutable=True) + client = FakeClient( + [ + (200, json.dumps([{"id": 41, "name": module.PUBLISH_ROBOT}]).encode(), _count(1)), + (200, json.dumps(_publisher(module)).encode(), {}), + (200, b"", {}), + (200, b"not-json", {}), + (200, json.dumps(verified).encode(), {}), + ] + ) + assert module.ensure_publisher_can_read_rule(client) == 41 + assert sleeps == [1] + + +def test_harbor_rule_verify_requires_the_created_rule_id(monkeypatch) -> None: + """A matching rule under a different ID is a stale read, not creation proof.""" + module = _load(HARBOR, "harbor_rule_id_retry_edges") + sleeps = [] + monkeypatch.setattr(module.time, "sleep", sleeps.append) + location = "/api/v2.0/projects/bstein/immutabletagrules/23" + client = FakeClient( + [ + (200, b"[]", _count(0)), + (201, b"", {"Location": location}), + (200, json.dumps([{"id": 99, **module.EXPECTED_RULE}]).encode(), _count(1)), + (200, json.dumps([{"id": 23, **module.EXPECTED_RULE}]).encode(), _count(1)), + ] + ) + assert module.ensure_rule(client) == 23 + assert sleeps == [1] + + +def test_harbor_main_reraises_after_bounded_unavailable_retries( + monkeypatch, tmp_path: Path +) -> None: + """Persistent Harbor unavailability surfaces instead of looping forever.""" + module = _load(HARBOR, "harbor_main_retry_edges") + password_file = tmp_path / "password" + password_file.write_text("private\n", encoding="utf-8") + monkeypatch.setenv("HARBOR_API_ORIGIN", module.EXPECTED_ORIGIN) + monkeypatch.setenv("HARBOR_ADMIN_PASSWORD_FILE", str(password_file)) + monkeypatch.setattr(module, "HarborClient", lambda *_args: object()) + + def never_ready(_client): + raise module.HarborUnavailable("still warming") + + monkeypatch.setattr(module, "ensure_rule", never_ready) + sleeps = [] + monkeypatch.setattr(module.time, "sleep", sleeps.append) + with pytest.raises(module.HarborUnavailable, match="still warming"): + module.main() + assert len(sleeps) == 11 From 64272f52d2e6f2eef728c38432722a249777574b Mon Sep 17 00:00:00 2001 From: jenkins Date: Mon, 17 Aug 2026 15:10:17 -0300 Subject: [PATCH 3/4] hermes: lift remaining tracked modules to the branch floor Exercise the mailu sync retry, attribute, and skip branches, the listener non-object JSON path, and the hygiene conftest skip; drop the unreachable inverted-range clamp in the semgrep report (the line helper already floors the end line) and pin that behavior with a test. Exclude the mailu __main__ guards from measurement. Co-Authored-By: Claude Fable 5 --- ci/scripts/semgrep_report.py | 3 +- scripts/tests/test_mailu_sync.py | 77 +++++++++++++++++++ scripts/tests/test_mailu_sync_listener.py | 12 +++ services/mailu/scripts/mailu_sync.py | 2 +- services/mailu/scripts/mailu_sync_listener.py | 2 +- testing/tests/test_quality_hygiene_helpers.py | 28 ++++++- testing/tests/test_semgrep_report.py | 23 ++++++ 7 files changed, 142 insertions(+), 5 deletions(-) diff --git a/ci/scripts/semgrep_report.py b/ci/scripts/semgrep_report.py index b4ed5a2d..80fb5953 100644 --- a/ci/scripts/semgrep_report.py +++ b/ci/scripts/semgrep_report.py @@ -93,9 +93,8 @@ def _sonar_issue_from_finding(finding: dict[str, Any]) -> dict[str, Any] | None: start = finding.get("start") if isinstance(finding.get("start"), dict) else {} end = finding.get("end") if isinstance(finding.get("end"), dict) else {} start_line = _line_number(start.get("line") if isinstance(start, dict) else None) + # _line_number floors the end line at start_line, so the range never inverts. end_line = _line_number(end.get("line") if isinstance(end, dict) else None, start_line) - if end_line < start_line: - end_line = start_line return { "engineId": "semgrep", "ruleId": str(finding.get("check_id") or "semgrep.unknown"), diff --git a/scripts/tests/test_mailu_sync.py b/scripts/tests/test_mailu_sync.py index 4d6207fe..c5a35954 100644 --- a/scripts/tests/test_mailu_sync.py +++ b/scripts/tests/test_mailu_sync.py @@ -232,6 +232,24 @@ def test_retry_db_connect_reraises_final_error(monkeypatch): sync.retry_db_connect(attempts=1) +def test_retry_helpers_return_none_without_attempts(monkeypatch): + sync = load_sync_module(monkeypatch) + + assert sync.retry_request("request", lambda: "never", attempts=0) is None + assert sync.retry_db_connect(attempts=0) is None + + +def test_kc_update_attributes_replaces_non_dict_attributes(monkeypatch): + sync = load_sync_module(monkeypatch) + current_resp = _FakeResponse({"attributes": "not-a-dict"}) + ok_resp = _FakeResponse({"attributes": {"mailu_app_password": ["abc"]}}) + sync.SESSION = _FakeSession(_FakeResponse({}), [current_resp, ok_resp]) + + sync.kc_update_attributes("token", {"id": "u1", "username": "u1"}, {"mailu_app_password": "abc"}) + + assert sync.SESSION.put_called + + def test_ensure_mailu_user_skips_foreign_domain(monkeypatch): sync = load_sync_module(monkeypatch) executed = [] @@ -405,3 +423,62 @@ def test_main_generates_password_and_upserts(monkeypatch): # Only mail-enabled users (or legacy users with a mailbox) are synced and backfilled. assert len(updated) == 3 assert conns and len(conns[0]._cursor.executions) == 3 + + +def test_main_skips_disabled_users_and_provisioned_users_without_update(monkeypatch): + sync = load_sync_module(monkeypatch) + monkeypatch.setattr(sync.bcrypt_sha256, "hash", lambda password: f"hash:{password}") + users = [ + { + "id": "u1", + "username": "disabled", + "email": "disabled@example.com", + "enabled": False, + "attributes": {"mailu_enabled": ["true"]}, + }, + { + "id": "u2", + "username": "settled", + "email": "settled@example.com", + "attributes": { + "mailu_enabled": ["true"], + "mailu_email": ["settled@example.com"], + "mailu_app_password": ["existing"], + }, + }, + ] + updated = [] + + class _Cursor: + def __init__(self): + self.executions = [] + + def execute(self, sql, params): + self.executions.append(params) + + def close(self): + return None + + class _Conn: + def __init__(self): + self.autocommit = False + self._cursor = _Cursor() + + def cursor(self, cursor_factory=None): + return self._cursor + + def close(self): + return None + + conn = _Conn() + monkeypatch.setattr(sync, "get_kc_token", lambda: "tok") + monkeypatch.setattr(sync, "kc_get_users", lambda token: users) + monkeypatch.setattr(sync, "kc_update_attributes", lambda token, user, attrs: updated.append(user["id"])) + monkeypatch.setattr(sync.psycopg2, "connect", lambda **kwargs: conn) + + sync.main() + + # The already-provisioned user is synced without a Keycloak write; the + # disabled user never reaches the mailbox upsert. + assert updated == [] + assert len(conn._cursor.executions) == 1 diff --git a/scripts/tests/test_mailu_sync_listener.py b/scripts/tests/test_mailu_sync_listener.py index eff7ff34..d9d99229 100644 --- a/scripts/tests/test_mailu_sync_listener.py +++ b/scripts/tests/test_mailu_sync_listener.py @@ -127,6 +127,18 @@ def test_listener_post_wait_keeps_running_request_successful(monkeypatch): assert handler.responses == [200] +def test_listener_post_treats_non_object_json_as_plain_trigger(monkeypatch): + listener = load_listener_module(monkeypatch) + called = [] + monkeypatch.setattr(listener, "_trigger_sync_async", lambda force=False: called.append(force) or True) + handler = _handler_for(listener, "[1, 2]") + + handler.do_POST() + + assert called == [False] + assert handler.responses == [202] + + def test_listener_log_message_is_quiet(monkeypatch): listener = load_listener_module(monkeypatch) handler = listener.Handler.__new__(listener.Handler) diff --git a/services/mailu/scripts/mailu_sync.py b/services/mailu/scripts/mailu_sync.py index d04ee25a..5b8837a3 100644 --- a/services/mailu/scripts/mailu_sync.py +++ b/services/mailu/scripts/mailu_sync.py @@ -314,7 +314,7 @@ def main(): conn.close() -if __name__ == "__main__": +if __name__ == "__main__": # pragma: no cover - exercised through main() try: main() except Exception as exc: diff --git a/services/mailu/scripts/mailu_sync_listener.py b/services/mailu/scripts/mailu_sync_listener.py index 48f3d4fe..63c9824f 100644 --- a/services/mailu/scripts/mailu_sync_listener.py +++ b/services/mailu/scripts/mailu_sync_listener.py @@ -100,6 +100,6 @@ class Handler(http.server.BaseHTTPRequestHandler): return -if __name__ == "__main__": +if __name__ == "__main__": # pragma: no cover - server loop server = http.server.ThreadingHTTPServer(("", 8080), Handler) server.serve_forever() diff --git a/testing/tests/test_quality_hygiene_helpers.py b/testing/tests/test_quality_hygiene_helpers.py index a303c650..9a96c86f 100644 --- a/testing/tests/test_quality_hygiene_helpers.py +++ b/testing/tests/test_quality_hygiene_helpers.py @@ -4,7 +4,33 @@ from __future__ import annotations from pathlib import Path -from testing.quality_hygiene import count_files_over_line_limit +from testing.quality_hygiene import count_files_over_line_limit, run_check + + +def test_run_check_skips_conftest_files_in_naming_rules(tmp_path: Path) -> None: + """A conftest.py is pytest plumbing, never a naming-rule violation.""" + + tests_dir = tmp_path / "tests" + tests_dir.mkdir() + (tests_dir / "conftest.py").write_text("fixtures = True\n", encoding="utf-8") + (tests_dir / "helper.py").write_text("value = 1\n", encoding="utf-8") + + contract = { + "hygiene": { + "naming_rules": [ + { + "glob": "tests/*.py", + "pattern": "^test_[a-z0-9_]+\\.py$", + "description": "Pytest files use test_*.py names.", + } + ] + } + } + + issues = run_check(contract, tmp_path) + + assert len(issues) == 1 + assert "helper.py" in issues[0] def test_count_files_over_line_limit_counts_only_long_matches(tmp_path: Path) -> None: diff --git a/testing/tests/test_semgrep_report.py b/testing/tests/test_semgrep_report.py index e0cf1735..7fc62346 100644 --- a/testing/tests/test_semgrep_report.py +++ b/testing/tests/test_semgrep_report.py @@ -100,6 +100,29 @@ def test_build_sonar_issues_uses_safe_defaults_for_sparse_findings() -> None: assert issue["primaryLocation"]["textRange"] == {"startLine": 1, "endLine": 1} +def test_build_sonar_issues_clamps_inverted_line_ranges() -> None: + """An end line before the start line should collapse to a valid range.""" + + issues = semgrep_report.build_sonar_issues( + { + "results": [ + { + "check_id": "python.lang.correctness.range", + "path": "app/main.py", + "start": {"line": 9}, + "end": {"line": 3}, + "extra": {"severity": "WARNING", "message": "inverted"}, + } + ] + } + ) + + assert issues["issues"][0]["primaryLocation"]["textRange"] == { + "startLine": 9, + "endLine": 9, + } + + def test_read_json_handles_invalid_json_and_non_object(tmp_path: Path) -> None: """Report loading should fail closed for bad JSON and wrong top-level shapes.""" From f77119b23855f8d90559bc1bbf42bae8d8509d29 Mon Sep 17 00:00:00 2001 From: jenkins Date: Mon, 17 Aug 2026 15:10:17 -0300 Subject: [PATCH 4/4] hermes: enforce per-file branch coverage in the quality gate The coverage check read only line-rate, so a file could pass with weak branch coverage. Load both Cobertura rates and fail any tracked file below the 95% floor on either metric, failing closed when branch evidence is absent. Prove enforcement end to end with a synthetic fully-lined but branch-weak file failing run_profile. Co-Authored-By: Claude Fable 5 --- testing/quality_coverage.py | 47 +++++++++----- testing/quality_gate.py | 2 +- testing/tests/test_quality_contract.py | 14 ++-- .../tests/test_quality_coverage_helpers.py | 65 ++++++++++++++++++- testing/tests/test_quality_gate.py | 26 ++++++++ 5 files changed, 127 insertions(+), 27 deletions(-) diff --git a/testing/quality_coverage.py b/testing/quality_coverage.py index 78d6649f..d99aea03 100644 --- a/testing/quality_coverage.py +++ b/testing/quality_coverage.py @@ -7,8 +7,8 @@ from pathlib import Path from typing import Any -def _load_percentages(xml_path: Path, root: Path) -> dict[str, float]: - """Load per-file line-rate percentages from a Cobertura XML report.""" +def _load_rates(xml_path: Path, root: Path) -> dict[str, dict[str, float | None]]: + """Load per-file line and branch percentages from a Cobertura XML report.""" tree = ET.parse(xml_path) xml_root = tree.getroot() source_roots = [ @@ -16,7 +16,7 @@ def _load_percentages(xml_path: Path, root: Path) -> dict[str, float]: for node in xml_root.findall("./sources/source") if node.text ] - percentages: dict[str, float] = {} + rates: dict[str, dict[str, float | None]] = {} for class_node in xml_root.findall(".//class"): filename = class_node.attrib.get("filename") line_rate = class_node.attrib.get("line-rate") @@ -32,33 +32,44 @@ def _load_percentages(xml_path: Path, root: Path) -> dict[str, float]: if candidate.exists(): key = candidate.relative_to(root).as_posix() break - percentages[key] = float(line_rate) * 100.0 - return percentages + branch_rate = class_node.attrib.get("branch-rate") + rates[key] = { + "line": float(line_rate) * 100.0, + "branch": None if branch_rate is None else float(branch_rate) * 100.0, + } + return rates def run_check(contract: dict[str, Any], root: Path, xml_path: Path) -> list[str]: """Return human-readable issues for tracked files below the coverage floor. - The report is intentionally per-file so a single weak module cannot hide - behind aggregate suite coverage. + Line and branch rates are both enforced per file so a single weak module + cannot hide behind aggregate suite coverage or line-only reporting. """ if not xml_path.exists(): return [f"coverage xml missing: {xml_path.relative_to(root)}"] - percentages = _load_percentages(xml_path, root) + rates = _load_rates(xml_path, root) minimum = float(contract.get("coverage", {}).get("minimum_percent", 95.0)) issues: list[str] = [] for relative_path in contract.get("coverage", {}).get("tracked_files", []): normalized = relative_path.replace("\\", "/") - percent = percentages.get(normalized) - if percent is None: + file_rates = rates.get(normalized) + if file_rates is None: issues.append(f"coverage missing for tracked file: {relative_path}") continue - if percent + 1e-9 < minimum: - issues.append( - f"coverage below {minimum:.1f}%: {relative_path} ({percent:.1f}%)" - ) + for metric in ("line", "branch"): + percent = file_rates[metric] + if percent is None: + issues.append( + f"{metric} coverage missing for tracked file: {relative_path}" + ) + elif percent + 1e-9 < minimum: + issues.append( + f"{metric} coverage below {minimum:.1f}%: " + f"{relative_path} ({percent:.1f}%)" + ) return issues @@ -73,13 +84,13 @@ def compute_workspace_line_coverage( if not xml_path.exists(): return 0.0 - percentages = _load_percentages(xml_path, root) + rates = _load_rates(xml_path, root) samples: list[float] = [] for relative_path in contract.get("coverage", {}).get("tracked_files", []): normalized = relative_path.replace("\\", "/") - percent = percentages.get(normalized) - if percent is not None: - samples.append(percent) + file_rates = rates.get(normalized) + if file_rates is not None: + samples.append(file_rates["line"]) if not samples: return 0.0 return round(sum(samples) / len(samples), 3) diff --git a/testing/quality_gate.py b/testing/quality_gate.py index dc5dd1e4..d436281c 100644 --- a/testing/quality_gate.py +++ b/testing/quality_gate.py @@ -332,7 +332,7 @@ def run_profile( results.append( _result( "coverage", - "Per-file 95% coverage floor for tracked quality-managed modules.", + "Per-file 95% line and branch coverage floor for tracked quality-managed modules.", _status_from_issues(issues), issues=issues, coverage_xml=str(coverage_xml.relative_to(root)), diff --git a/testing/tests/test_quality_contract.py b/testing/tests/test_quality_contract.py index 6e260a1b..f7bf38b7 100644 --- a/testing/tests/test_quality_contract.py +++ b/testing/tests/test_quality_contract.py @@ -98,8 +98,9 @@ def test_coverage_check_enforces_per_file_floor(tmp_path: Path): - - + + + @@ -112,13 +113,14 @@ def test_coverage_check_enforces_per_file_floor(tmp_path: Path): contract = { "coverage": { "minimum_percent": 95.0, - "tracked_files": ["ok.py", "low.py", "missing.py"], + "tracked_files": ["ok.py", "low.py", "weak_branches.py", "missing.py"], } } issues = run_coverage_check(contract, tmp_path, coverage_xml) - assert "coverage below 95.0%: low.py (90.0%)" in issues + assert "line coverage below 95.0%: low.py (90.0%)" in issues + assert "branch coverage below 95.0%: weak_branches.py (90.0%)" in issues assert "coverage missing for tracked file: missing.py" in issues @@ -142,8 +144,8 @@ def test_coverage_check_handles_missing_xml_and_source_root_mapping(tmp_path: Pa - - + + diff --git a/testing/tests/test_quality_coverage_helpers.py b/testing/tests/test_quality_coverage_helpers.py index 5eaa624e..7e6d90f3 100644 --- a/testing/tests/test_quality_coverage_helpers.py +++ b/testing/tests/test_quality_coverage_helpers.py @@ -57,7 +57,7 @@ def test_run_check_keeps_relative_names_when_source_roots_do_not_match(tmp_path: - + @@ -73,4 +73,65 @@ def test_run_check_keeps_relative_names_when_source_roots_do_not_match(tmp_path: coverage_xml, ) - assert issues == ["coverage below 95.0%: relative.py (80.0%)"] + assert issues == ["line coverage below 95.0%: relative.py (80.0%)"] + + +def test_run_check_fails_a_file_below_the_branch_floor(tmp_path: Path) -> None: + """Full line coverage must never hide a file whose branch coverage is weak.""" + + coverage_xml = tmp_path / "coverage.xml" + coverage_xml.write_text( + textwrap.dedent( + """\ + + + + + + + + + + + """ + ), + encoding="utf-8", + ) + + issues = run_check( + {"coverage": {"minimum_percent": 95.0, "tracked_files": ["weak.py", "solid.py"]}}, + tmp_path, + coverage_xml, + ) + + assert issues == ["branch coverage below 95.0%: weak.py (90.0%)"] + + +def test_run_check_fails_closed_when_branch_evidence_is_absent(tmp_path: Path) -> None: + """A report generated without branch measurement cannot satisfy the gate.""" + + coverage_xml = tmp_path / "coverage.xml" + coverage_xml.write_text( + textwrap.dedent( + """\ + + + + + + + + + + """ + ), + encoding="utf-8", + ) + + issues = run_check( + {"coverage": {"minimum_percent": 95.0, "tracked_files": ["unmeasured.py"]}}, + tmp_path, + coverage_xml, + ) + + assert issues == ["branch coverage missing for tracked file: unmeasured.py"] diff --git a/testing/tests/test_quality_gate.py b/testing/tests/test_quality_gate.py index 256a78e6..fd334d85 100644 --- a/testing/tests/test_quality_gate.py +++ b/testing/tests/test_quality_gate.py @@ -60,6 +60,32 @@ def test_run_profile_aggregates_internal_and_pytest_results(tmp_path: Path, monk assert any(result.get("junit") == "build/junit-unit.xml" for result in summary["results"]) +def test_run_profile_fails_synthetic_file_below_branch_floor(tmp_path: Path): + """The gate itself must fail a tracked file with full lines but weak branches.""" + build_dir = tmp_path / "build" + build_dir.mkdir() + (build_dir / "coverage-unit.xml").write_text( + "" + '' + "", + encoding="utf-8", + ) + + contract = { + "profiles": {"local": ["coverage"]}, + "pytest_suites": {"unit": {"coverage_xml": "build/coverage-unit.xml"}}, + "coverage": {"minimum_percent": 95.0, "tracked_files": ["weak.py"]}, + } + + summary = quality_gate.run_profile(contract, tmp_path, "local", build_dir) + + assert summary["status"] == "failed" + coverage_result = summary["results"][0] + assert coverage_result["name"] == "coverage" + assert coverage_result["issues"] == ["branch coverage below 95.0%: weak.py (90.0%)"] + assert summary["workspace_line_coverage_percent"] == 100.0 + + def test_main_writes_summary_file(tmp_path: Path, monkeypatch): summary = {"status": "ok", "profile": "local", "results": [], "manual_scripts": []} monkeypatch.chdir(tmp_path)