diff --git a/ci/Jenkinsfile.hermes-agent-image b/ci/Jenkinsfile.hermes-agent-image index 2adc096e..8a177434 100644 --- a/ci/Jenkinsfile.hermes-agent-image +++ b/ci/Jenkinsfile.hermes-agent-image @@ -114,7 +114,7 @@ spec: string( name: 'EXPECTED_SOURCE_REVISION', defaultValue: '', - description: 'Exact 40-character commit on atlas/titan-iac main.' + description: 'Full reviewed commit that must be contained by atlas/titan-iac main.' ) string( name: 'CONFIRM_PUBLISH', @@ -152,9 +152,11 @@ spec: ;; esac test "${#EXPECTED_SOURCE_REVISION}" -eq 40 + git fetch --no-tags origin main + git checkout --detach origin/main actual_revision="$(git rev-parse HEAD)" - test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}" test "${actual_revision}" = "$(git rev-parse origin/main)" + git merge-base --is-ancestor "${EXPECTED_SOURCE_REVISION}" "${actual_revision}" test -z "$(git status --porcelain)" test -f dockerfiles/Dockerfile.hermes-agent case "${BUILD_NUMBER}" in @@ -166,10 +168,31 @@ spec: printf '%s\n' \ "${HERMES_IMAGE}:git-${actual_revision}-build-${BUILD_NUMBER}" \ > build/hermes-agent.destination + printf '%s\n' "${actual_revision}" > build/hermes-agent.source-revision ''' } } } + stage('Validate reviewed release source') { + steps { + container('python') { + sh ''' + set -eu + python3 -m pip install --disable-pip-version-check --no-cache-dir \ + --target=/tmp/hermes-agent-release-test-deps \ + pytest==8.3.4 PyYAML==6.0.2 + PYTHONPATH=/tmp/hermes-agent-release-test-deps \ + python3 -m pytest -q \ + testing/tests/test_hermes_image_builder.py \ + testing/tests/test_hermes_image_builder_adversarial.py \ + testing/tests/test_hermes_image_builder_coverage.py \ + testing/tests/test_hermes_image_builder_fresh_review.py \ + testing/tests/test_hermes_oci_promote.py \ + testing/tests/test_hermes_image_automation.py + ''' + } + } + } stage('Reject replay before publish') { steps { withCredentials([usernamePassword( @@ -181,8 +204,9 @@ spec: set -eu set +x destination="$(cat build/hermes-agent.destination)" + source_revision="$(cat build/hermes-agent.source-revision)" python3 ci/scripts/hermes_image_release.py assert-absent \ - --source-revision "${EXPECTED_SOURCE_REVISION}" \ + --source-revision "${source_revision}" \ --build-number "${BUILD_NUMBER}" \ --destination "${destination}" ''' @@ -202,6 +226,7 @@ spec: set +x config_path=/kaniko/.docker/config.json destination="$(cat build/hermes-agent.destination)" + source_revision="$(cat build/hermes-agent.source-revision)" umask 077 auth="$(printf '%s:%s' "${HARBOR_USER}" "${HARBOR_PASSWORD}" | /busybox/base64 | /busybox/tr -d '\n')" /busybox/mkdir -p /kaniko/.docker @@ -218,7 +243,7 @@ spec: --digest-file="${WORKSPACE}/build/hermes-agent.digest" \ --image-name-tag-with-digest-file="${WORKSPACE}/build/hermes-agent.image" \ --build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1 \ - --label="org.opencontainers.image.revision=${EXPECTED_SOURCE_REVISION}" \ + --label="org.opencontainers.image.revision=${source_revision}" \ --cleanup \ --push-retry=3 /busybox/chmod 644 build/hermes-agent.digest build/hermes-agent.image @@ -238,10 +263,11 @@ spec: set -eu set +x destination="$(cat build/hermes-agent.destination)" + source_revision="$(cat build/hermes-agent.source-revision)" python3 ci/scripts/hermes_image_release.py render \ --digest-file build/hermes-agent.digest \ --image-file build/hermes-agent.image \ - --source-revision "${EXPECTED_SOURCE_REVISION}" \ + --source-revision "${source_revision}" \ --build-number "${BUILD_NUMBER}" \ --destination "${destination}" \ --kustomization services/hermes/kustomization.yaml \ @@ -252,15 +278,15 @@ spec: } } } - } - post { - success { + stage('Verify and archive release evidence') { + steps { sh ''' set -eu expected_files="$(printf '%s\n' \ build/hermes-agent.destination \ build/hermes-agent.digest \ build/hermes-agent.image \ + build/hermes-agent.source-revision \ build/hermes-agent-release/hermes-agent-image.json \ build/hermes-agent-release/hermes-image-update.patch \ build/hermes-agent-release/hermes-kustomization.yaml \ @@ -268,26 +294,42 @@ spec: actual_files="$(find build -type f -print | LC_ALL=C sort)" test "${actual_files}" = "${expected_files}" destination="$(cat build/hermes-agent.destination)" + source_revision="$(cat build/hermes-agent.source-revision)" python3 ci/scripts/hermes_image_release.py verify-evidence \ --digest-file build/hermes-agent.digest \ --image-file build/hermes-agent.image \ - --source-revision "${EXPECTED_SOURCE_REVISION}" \ + --source-revision "${source_revision}" \ --build-number "${BUILD_NUMBER}" \ --destination "${destination}" \ --kustomization services/hermes/kustomization.yaml \ --output-dir build/hermes-agent-release ''' archiveArtifacts( - artifacts: 'build/hermes-agent.destination,build/hermes-agent.digest,build/hermes-agent.image,build/hermes-agent-release/hermes-agent-image.json,build/hermes-agent-release/hermes-image-update.patch,build/hermes-agent-release/hermes-kustomization.yaml', + artifacts: 'build/hermes-agent.destination,build/hermes-agent.digest,build/hermes-agent.image,build/hermes-agent.source-revision,build/hermes-agent-release/hermes-agent-image.json,build/hermes-agent-release/hermes-image-update.patch,build/hermes-agent-release/hermes-kustomization.yaml', allowEmptyArchive: false, fingerprint: true ) + } } - cleanup { - container('kaniko') { - sh '''#!/busybox/sh - /busybox/rm -f /kaniko/.docker/config.json - ''' + stage('Publish Flux release tag') { + steps { + withCredentials([usernamePassword( + credentialsId: 'harbor-robot', + usernameVariable: 'HARBOR_USER', + passwordVariable: 'HARBOR_PASSWORD' + )]) { + sh ''' + set -eu + set +x + destination="$(cat build/hermes-agent.destination)" + source_revision="$(cat build/hermes-agent.source-revision)" + python3 ci/scripts/hermes_oci_promote.py \ + --destination "${destination}" \ + --digest-file build/hermes-agent.digest \ + --source-revision "${source_revision}" \ + --build-number "${BUILD_NUMBER}" + ''' + } } } } diff --git a/ci/Jenkinsfile.hermes-webui-image b/ci/Jenkinsfile.hermes-webui-image index 859f9809..7e6c45c3 100644 --- a/ci/Jenkinsfile.hermes-webui-image +++ b/ci/Jenkinsfile.hermes-webui-image @@ -114,7 +114,7 @@ spec: string( name: 'EXPECTED_SOURCE_REVISION', defaultValue: '', - description: 'Exact 40-character commit on atlas/titan-iac main.' + description: 'Full reviewed commit that must be contained by atlas/titan-iac main.' ) string( name: 'CONFIRM_PUBLISH', @@ -152,9 +152,11 @@ spec: ;; esac test "${#EXPECTED_SOURCE_REVISION}" -eq 40 + git fetch --no-tags origin main + git checkout --detach origin/main actual_revision="$(git rev-parse HEAD)" - test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}" test "${actual_revision}" = "$(git rev-parse origin/main)" + git merge-base --is-ancestor "${EXPECTED_SOURCE_REVISION}" "${actual_revision}" test -z "$(git status --porcelain)" test -f dockerfiles/Dockerfile.hermes-webui case "${BUILD_NUMBER}" in @@ -166,6 +168,7 @@ spec: printf '%s\n' \ "${HERMES_IMAGE}:git-${actual_revision}-build-${BUILD_NUMBER}" \ > build/hermes-webui.destination + printf '%s\n' "${actual_revision}" > build/hermes-webui.source-revision ''' } } @@ -181,7 +184,9 @@ spec: PYTHONPATH=/tmp/hermes-webui-release-test-deps \ python3 -m pytest -q \ testing/tests/test_hermes_webui_brand.py \ - testing/tests/test_hermes_webui_release.py + testing/tests/test_hermes_webui_release.py \ + testing/tests/test_hermes_oci_promote.py \ + testing/tests/test_hermes_image_automation.py ''' } } @@ -197,8 +202,9 @@ spec: set -eu set +x destination="$(cat build/hermes-webui.destination)" + source_revision="$(cat build/hermes-webui.source-revision)" python3 ci/scripts/hermes_webui_release.py assert-absent \ - --source-revision "${EXPECTED_SOURCE_REVISION}" \ + --source-revision "${source_revision}" \ --build-number "${BUILD_NUMBER}" \ --destination "${destination}" ''' @@ -218,6 +224,7 @@ spec: set +x config_path=/kaniko/.docker/config.json destination="$(cat build/hermes-webui.destination)" + source_revision="$(cat build/hermes-webui.source-revision)" umask 077 auth="$(printf '%s:%s' "${HARBOR_USER}" "${HARBOR_PASSWORD}" | /busybox/base64 | /busybox/tr -d '\n')" /busybox/mkdir -p /kaniko/.docker @@ -233,7 +240,7 @@ spec: --destination="${destination}" \ --digest-file="${WORKSPACE}/build/hermes-webui.digest" \ --image-name-tag-with-digest-file="${WORKSPACE}/build/hermes-webui.image" \ - --label="org.opencontainers.image.revision=${EXPECTED_SOURCE_REVISION}" \ + --label="org.opencontainers.image.revision=${source_revision}" \ --label="org.opencontainers.image.source=https://scm.bstein.dev/atlas/titan-iac" \ --label="org.opencontainers.image.title=hermes-webui" \ --cleanup \ @@ -255,10 +262,11 @@ spec: set -eu set +x destination="$(cat build/hermes-webui.destination)" + source_revision="$(cat build/hermes-webui.source-revision)" python3 ci/scripts/hermes_webui_release.py render \ --digest-file build/hermes-webui.digest \ --image-file build/hermes-webui.image \ - --source-revision "${EXPECTED_SOURCE_REVISION}" \ + --source-revision "${source_revision}" \ --build-number "${BUILD_NUMBER}" \ --destination "${destination}" \ --chat-manifest services/hermes/chat-statefulset.yaml \ @@ -270,15 +278,15 @@ spec: } } } - } - post { - success { + stage('Verify and archive release evidence') { + steps { sh ''' set -eu expected_files="$(printf '%s\n' \ build/hermes-webui.destination \ build/hermes-webui.digest \ build/hermes-webui.image \ + build/hermes-webui.source-revision \ build/hermes-webui-release/hermes-chat-statefulset.yaml \ build/hermes-webui-release/hermes-dashboard-deployment.yaml \ build/hermes-webui-release/hermes-webui-image.json \ @@ -287,10 +295,11 @@ spec: actual_files="$(find build -type f -print | LC_ALL=C sort)" test "${actual_files}" = "${expected_files}" destination="$(cat build/hermes-webui.destination)" + source_revision="$(cat build/hermes-webui.source-revision)" python3 ci/scripts/hermes_webui_release.py verify-evidence \ --digest-file build/hermes-webui.digest \ --image-file build/hermes-webui.image \ - --source-revision "${EXPECTED_SOURCE_REVISION}" \ + --source-revision "${source_revision}" \ --build-number "${BUILD_NUMBER}" \ --destination "${destination}" \ --chat-manifest services/hermes/chat-statefulset.yaml \ @@ -298,16 +307,31 @@ spec: --output-dir build/hermes-webui-release ''' archiveArtifacts( - artifacts: 'build/hermes-webui.destination,build/hermes-webui.digest,build/hermes-webui.image,build/hermes-webui-release/hermes-chat-statefulset.yaml,build/hermes-webui-release/hermes-dashboard-deployment.yaml,build/hermes-webui-release/hermes-webui-image.json,build/hermes-webui-release/hermes-webui-image-update.patch', + artifacts: 'build/hermes-webui.destination,build/hermes-webui.digest,build/hermes-webui.image,build/hermes-webui.source-revision,build/hermes-webui-release/hermes-chat-statefulset.yaml,build/hermes-webui-release/hermes-dashboard-deployment.yaml,build/hermes-webui-release/hermes-webui-image.json,build/hermes-webui-release/hermes-webui-image-update.patch', allowEmptyArchive: false, fingerprint: true ) + } } - cleanup { - container('kaniko') { - sh '''#!/busybox/sh - /busybox/rm -f /kaniko/.docker/config.json - ''' + stage('Publish Flux release tag') { + steps { + withCredentials([usernamePassword( + credentialsId: 'harbor-robot', + usernameVariable: 'HARBOR_USER', + passwordVariable: 'HARBOR_PASSWORD' + )]) { + sh ''' + set -eu + set +x + destination="$(cat build/hermes-webui.destination)" + source_revision="$(cat build/hermes-webui.source-revision)" + python3 ci/scripts/hermes_oci_promote.py \ + --destination "${destination}" \ + --digest-file build/hermes-webui.digest \ + --source-revision "${source_revision}" \ + --build-number "${BUILD_NUMBER}" + ''' + } } } } diff --git a/ci/scripts/hermes_image_release.py b/ci/scripts/hermes_image_release.py index 2f0f4166..c2f6918e 100755 --- a/ci/scripts/hermes_image_release.py +++ b/ci/scripts/hermes_image_release.py @@ -174,9 +174,7 @@ def _normalized_immutable_rule(rule: dict[str, Any]) -> dict[str, Any]: "decoration": item.get("decoration"), "pattern": item.get("pattern"), } - for item in (rule.get("scope_selectors") or {}).get( - "repository", [] - ) + for item in (rule.get("scope_selectors") or {}).get("repository", []) if isinstance(item, dict) ] }, @@ -317,7 +315,10 @@ def render_kustomization(source: str, digest: str, image: str = DEFAULT_IMAGE) - index = matches[0] newline = "\n" if lines[index].endswith("\n") else "" prefix = lines[index][: len(lines[index]) - len(lines[index].lstrip())] - lines[index] = f"{prefix}digest: {digest}{newline}" + value = lines[index].strip().removeprefix("digest:").strip() + _current_digest, separator, comment = value.partition(" #") + suffix = f" #{comment}" if separator else "" + lines[index] = f"{prefix}digest: {digest}{suffix}{newline}" return "".join(lines) @@ -414,7 +415,8 @@ def validate_release_artifacts( "source_revision": source_revision, } expected = { - "hermes-agent-image.json": json.dumps(metadata, indent=2, sort_keys=True) + "\n", + "hermes-agent-image.json": json.dumps(metadata, indent=2, sort_keys=True) + + "\n", "hermes-image-update.patch": patch, "hermes-kustomization.yaml": rendered, } diff --git a/ci/scripts/hermes_oci_promote.py b/ci/scripts/hermes_oci_promote.py new file mode 100644 index 00000000..aa698b87 --- /dev/null +++ b/ci/scripts/hermes_oci_promote.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python3 +"""Publish a validated Hermes candidate manifest under its Flux release tag.""" + +from __future__ import annotations + +import argparse +import base64 +import json +import os +import re +import urllib.error +import urllib.parse +import urllib.request +from pathlib import Path +from typing import Any, Callable + + +REGISTRY_ORIGIN = "https://registry.bstein.dev" +DESTINATION_PATTERN = re.compile( + r"^registry\.bstein\.dev/bstein/(?Phermes-(?:agent|webui)):" + r"git-(?P[0-9a-f]{40})-build-(?P[1-9][0-9]*)$" +) +DIGEST_PATTERN = re.compile(r"^sha256:[0-9a-f]{64}$") +MANIFEST_TYPES = { + "application/vnd.docker.distribution.manifest.v2+json", + "application/vnd.docker.distribution.manifest.list.v2+json", + "application/vnd.oci.image.manifest.v1+json", + "application/vnd.oci.image.index.v1+json", +} +MAX_MANIFEST_BYTES = 4 * 1024 * 1024 + + +class _NoRedirect(urllib.request.HTTPRedirectHandler): + """Never forward registry credentials to another origin.""" + + def redirect_request(self, _request, _file, _code, _message, _headers, _url): + return None + + +def _registry_request(request: urllib.request.Request, timeout: int) -> Any: + """Return normal and HTTP error responses without following redirects.""" + opener = urllib.request.build_opener(_NoRedirect()) + try: + return opener.open(request, timeout=timeout) + except urllib.error.HTTPError as exc: + return exc + + +def _status(response: Any) -> int: + """Normalize urllib response and HTTPError status fields.""" + return int(getattr(response, "status", getattr(response, "code", 0))) + + +def _authorization(username: str, password: str) -> str: + """Build a Basic authorization value without placing it in a URL.""" + if not username or not password: + raise RuntimeError("Harbor credentials are unavailable") + encoded = base64.b64encode(f"{username}:{password}".encode()).decode("ascii") + return f"Basic {encoded}" + + +def _validated_release( + destination: str, + digest: str, + source_revision: str, + build_number: str, +) -> tuple[str, str, str]: + """Bind the candidate and release tag to one reviewed source and build.""" + match = DESTINATION_PATTERN.fullmatch(destination.strip()) + if not match: + raise ValueError("invalid Hermes candidate destination") + if match.group("revision") != source_revision.strip(): + raise ValueError("candidate source revision does not match evidence") + if match.group("build") != build_number.strip(): + raise ValueError("candidate build number does not match evidence") + normalized_digest = digest.strip() + if not DIGEST_PATTERN.fullmatch(normalized_digest): + raise ValueError("invalid candidate digest") + candidate_tag = destination.rsplit(":", 1)[1] + return match.group("component"), candidate_tag, normalized_digest + + +def _manifest_url(component: str, tag: str) -> str: + """Return one same-origin, path-escaped Docker Registry manifest URL.""" + encoded_tag = urllib.parse.quote(tag, safe="") + return f"{REGISTRY_ORIGIN}/v2/bstein/{component}/manifests/{encoded_tag}" + + +def promote_candidate( + *, + destination: str, + digest: str, + source_revision: str, + build_number: str, + username: str, + password: str, + opener: Callable[[urllib.request.Request, int], Any] = _registry_request, +) -> dict[str, str]: + """Copy an exact candidate manifest to the immutable ``-release`` tag.""" + component, candidate_tag, normalized_digest = _validated_release( + destination, digest, source_revision, build_number + ) + release_tag = f"{candidate_tag}-release" + authorization = _authorization(username, password) + accept = ", ".join(sorted(MANIFEST_TYPES)) + candidate_request = urllib.request.Request( + _manifest_url(component, candidate_tag), + headers={"Accept": accept, "Authorization": authorization}, + method="GET", + ) + with opener(candidate_request, 30) as response: + if _status(response) != 200: + raise RuntimeError(f"candidate manifest returned HTTP {_status(response)}") + manifest = response.read(MAX_MANIFEST_BYTES + 1) + if len(manifest) > MAX_MANIFEST_BYTES: + raise RuntimeError("candidate manifest exceeded the size limit") + observed_digest = response.headers.get("Docker-Content-Digest", "") + content_type = response.headers.get("Content-Type", "").split(";", 1)[0] + if observed_digest != normalized_digest: + raise RuntimeError("candidate manifest digest does not match build evidence") + if content_type not in MANIFEST_TYPES: + raise RuntimeError("candidate manifest returned an unsupported content type") + + release_url = _manifest_url(component, release_tag) + head_request = urllib.request.Request( + release_url, + headers={"Accept": accept, "Authorization": authorization}, + method="HEAD", + ) + with opener(head_request, 20) as response: + head_status = _status(response) + existing_digest = response.headers.get("Docker-Content-Digest", "") + if head_status == 200: + if existing_digest != normalized_digest: + raise RuntimeError("release tag already exists with another digest") + result = "already-present" + elif head_status == 404: + put_request = urllib.request.Request( + release_url, + data=manifest, + headers={ + "Authorization": authorization, + "Content-Type": content_type, + }, + method="PUT", + ) + with opener(put_request, 30) as response: + put_status = _status(response) + promoted_digest = response.headers.get("Docker-Content-Digest", "") + if put_status not in {201, 202}: + raise RuntimeError(f"release manifest returned HTTP {put_status}") + if promoted_digest and promoted_digest != normalized_digest: + raise RuntimeError("release manifest digest changed during promotion") + result = "published" + else: + raise RuntimeError(f"release tag preflight returned HTTP {head_status}") + + return { + "component": component, + "digest": normalized_digest, + "release_tag": release_tag, + "result": result, + "source_revision": source_revision, + } + + +def main() -> int: + """Promote one evidence-bound candidate and print credential-free metadata.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--destination", required=True) + parser.add_argument("--digest-file", required=True, type=Path) + parser.add_argument("--source-revision", required=True) + parser.add_argument("--build-number", required=True) + args = parser.parse_args() + try: + result = promote_candidate( + destination=args.destination, + digest=args.digest_file.read_text(encoding="utf-8"), + source_revision=args.source_revision, + build_number=args.build_number, + username=os.environ.get("HARBOR_USER", ""), + password=os.environ.get("HARBOR_PASSWORD", ""), + ) + except (OSError, ValueError, RuntimeError, urllib.error.URLError) as exc: + print(json.dumps({"error": str(exc)}, sort_keys=True)) + return 1 + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": # pragma: no cover - exercised through main() + raise SystemExit(main()) diff --git a/ci/scripts/hermes_webui_flux_release.py b/ci/scripts/hermes_webui_flux_release.py index 5483b333..cfcefc35 100644 --- a/ci/scripts/hermes_webui_flux_release.py +++ b/ci/scripts/hermes_webui_flux_release.py @@ -60,13 +60,16 @@ def render_workload( raise ValueError(f"Flux target identity changed: expected {kind}/{name}") lines = source.splitlines(keepends=True) matches: list[int] = [] + suffixes: dict[int, str] = {} for index, line in enumerate(lines): stripped = line.strip() if not stripped.startswith(f"image: {image}@"): continue - current_digest = stripped.removeprefix(f"image: {image}@") + value = stripped.removeprefix(f"image: {image}@") + current_digest, separator, comment = value.partition(" #") validated(current_digest, DIGEST_PATTERN, "current Flux image digest") matches.append(index) + suffixes[index] = f" #{comment}" if separator else "" if len(matches) != 1: raise ValueError( f"expected exactly one {image!r} image in {kind}/{name}; " @@ -75,7 +78,7 @@ def render_workload( index = matches[0] newline = "\n" if lines[index].endswith("\n") else "" prefix = lines[index][: len(lines[index]) - len(lines[index].lstrip())] - lines[index] = f"{prefix}image: {image}@{digest}{newline}" + lines[index] = f"{prefix}image: {image}@{digest}{suffixes[index]}{newline}" return "".join(lines) diff --git a/clusters/atlas/flux-system/applications/hermes/image-automation.yaml b/clusters/atlas/flux-system/applications/hermes/image-automation.yaml new file mode 100644 index 00000000..c1b4e246 --- /dev/null +++ b/clusters/atlas/flux-system/applications/hermes/image-automation.yaml @@ -0,0 +1,26 @@ +# clusters/atlas/flux-system/applications/hermes/image-automation.yaml +apiVersion: image.toolkit.fluxcd.io/v1 +kind: ImageUpdateAutomation +metadata: + name: hermes + namespace: hermes +spec: + interval: 1m0s + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + git: + checkout: + ref: + branch: main + commit: + author: + email: ops@bstein.dev + name: flux-bot + messageTemplate: "chore(hermes): promote validated image release" + push: + branch: main + update: + strategy: Setters + path: services/hermes diff --git a/clusters/atlas/flux-system/applications/kustomization.yaml b/clusters/atlas/flux-system/applications/kustomization.yaml index faac8c41..d1ec51fd 100644 --- a/clusters/atlas/flux-system/applications/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/kustomization.yaml @@ -31,6 +31,7 @@ resources: - hermes-scm-namespace/kustomization.yaml - hermes-scm-broker-code/kustomization.yaml - hermes/kustomization.yaml + - hermes/image-automation.yaml - hermes-observer-rbac/kustomization.yaml - hermes-observer-bindings/kustomization.yaml - hermes-scm-broker/kustomization.yaml diff --git a/services/hermes/agent-configmap.yaml b/services/hermes/agent-configmap.yaml index a7bee49d..55d2e549 100644 --- a/services/hermes/agent-configmap.yaml +++ b/services/hermes/agent-configmap.yaml @@ -369,6 +369,17 @@ data: file as user state. PR publication ends at the verified open draft; Brad owns review and merge authority. + For an explicitly requested Hermes runtime release, trigger only the + reviewed image lanes with `/opt/coordinator/jenkins_image_build_trigger.py`. + Use `--component agent` for backend/runtime changes and `--component webui` + for chat UI changes, passing a full commit already contained by `main`. + Jenkins builds the newest main containing that commit, publishes a final + immutable release tag only after its evidence passes, and Flux applies the + resulting digest. Candidate tags and failed builds never deploy. Follow the + returned queue item with `jenkins_build_evidence.py`, then verify the Flux + revision, pod image digest, rollout health, and public SSO redirect before + reporting the release complete. + The terminal PATH contains the pinned operator tools. Start cluster work with `kubectl config current-context`, read-only status/events/logs, and the relevant `titan-iac` manifests. The broker namespace is deliberately diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml index 868ed00f..8998abf6 100644 --- a/services/hermes/agent-deployment.yaml +++ b/services/hermes/agent-deployment.yaml @@ -25,7 +25,7 @@ spec: ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback ai.bstein.dev/placement: titan-08 rpi5; storage-backbone nodes excluded - ai.bstein.dev/config-rev: "20260823-claude-auth-hysteresis" + ai.bstein.dev/config-rev: "20260823-image-release-automation" prometheus.io/scrape: "true" prometheus.io/path: /metrics prometheus.io/port: "9010" diff --git a/services/hermes/chat-statefulset.yaml b/services/hermes/chat-statefulset.yaml index f362dfb9..0a2c2e30 100644 --- a/services/hermes/chat-statefulset.yaml +++ b/services/hermes/chat-statefulset.yaml @@ -320,7 +320,7 @@ spec: requests: {cpu: 250m, memory: 256Mi} limits: {cpu: "1", memory: 2Gi} - name: webui - image: registry.bstein.dev/bstein/hermes-webui@sha256:c276a9e17c9057237472f39640c9ebac9d4359d9f62a34df52a759eef43167bf + image: registry.bstein.dev/bstein/hermes-webui@sha256:c276a9e17c9057237472f39640c9ebac9d4359d9f62a34df52a759eef43167bf # {"$imagepolicy": "hermes:hermes-webui-release:digest"} imagePullPolicy: IfNotPresent command: [/bin/sh, -ec] args: diff --git a/services/hermes/deployment.yaml b/services/hermes/deployment.yaml index 08c2046e..329b5b5f 100644 --- a/services/hermes/deployment.yaml +++ b/services/hermes/deployment.yaml @@ -385,7 +385,7 @@ spec: cpu: "2" memory: 4Gi - name: webui - image: registry.bstein.dev/bstein/hermes-webui@sha256:ac6ba7bfd8a86227f31a9a96ebea41227ccf70391f7dcf34206f4d58e835e50e + image: registry.bstein.dev/bstein/hermes-webui@sha256:ac6ba7bfd8a86227f31a9a96ebea41227ccf70391f7dcf34206f4d58e835e50e # {"$imagepolicy": "hermes:hermes-webui-release:digest"} imagePullPolicy: IfNotPresent command: [/bin/sh, -ec] args: diff --git a/services/hermes/image.yaml b/services/hermes/image.yaml new file mode 100644 index 00000000..d460d3c7 --- /dev/null +++ b/services/hermes/image.yaml @@ -0,0 +1,52 @@ +# services/hermes/image.yaml +apiVersion: image.toolkit.fluxcd.io/v1 +kind: ImageRepository +metadata: + name: hermes-agent-release + namespace: hermes +spec: + image: registry.bstein.dev/bstein/hermes-agent + interval: 1m0s +--- +apiVersion: image.toolkit.fluxcd.io/v1 +kind: ImagePolicy +metadata: + name: hermes-agent-release + namespace: hermes +spec: + imageRepositoryRef: + name: hermes-agent-release + filterTags: + pattern: '^git-[0-9a-f]{40}-build-(?P[1-9][0-9]*)-release$' + extract: '$build' + policy: + numerical: + order: asc + digestReflectionPolicy: Always + interval: 1m0s +--- +apiVersion: image.toolkit.fluxcd.io/v1 +kind: ImageRepository +metadata: + name: hermes-webui-release + namespace: hermes +spec: + image: registry.bstein.dev/bstein/hermes-webui + interval: 1m0s +--- +apiVersion: image.toolkit.fluxcd.io/v1 +kind: ImagePolicy +metadata: + name: hermes-webui-release + namespace: hermes +spec: + imageRepositoryRef: + name: hermes-webui-release + filterTags: + pattern: '^git-[0-9a-f]{40}-build-(?P[1-9][0-9]*)-release$' + extract: '$build' + policy: + numerical: + order: asc + digestReflectionPolicy: Always + interval: 1m0s diff --git a/services/hermes/kustomization.yaml b/services/hermes/kustomization.yaml index 185cffaf..e2281ca5 100644 --- a/services/hermes/kustomization.yaml +++ b/services/hermes/kustomization.yaml @@ -4,9 +4,10 @@ kind: Kustomization namespace: hermes images: - name: registry.bstein.dev/bstein/hermes-agent - digest: sha256:4a385fbd04ae1e3e8ca0237db98a318606a2a03786ef467757b8da5eee4e4b37 + digest: sha256:4a385fbd04ae1e3e8ca0237db98a318606a2a03786ef467757b8da5eee4e4b37 # {"$imagepolicy": "hermes:hermes-agent-release:digest"} resources: - namespace.yaml + - image.yaml - scm-common - vault-serviceaccount.yaml - configmap.yaml diff --git a/services/hermes/scripts/jenkins_image_build_trigger.py b/services/hermes/scripts/jenkins_image_build_trigger.py index 6cd91ed8..2f74cdc5 100755 --- a/services/hermes/scripts/jenkins_image_build_trigger.py +++ b/services/hermes/scripts/jenkins_image_build_trigger.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Trigger only the reviewed-main Hermes agent image release job.""" +"""Trigger a bounded reviewed-main Hermes image release job.""" from __future__ import annotations @@ -14,7 +14,17 @@ from pathlib import Path JENKINS_ORIGIN = "https://ci.bstein.dev" JENKINS_BUILD_URL = f"{JENKINS_ORIGIN}/buildByToken/buildWithParameters" -JOB_NAME = "hermes-agent-image" +JOBS = { + "agent": { + "job": "hermes-agent-image", + "confirmation": "PUBLISH HERMES AGENT", + }, + "webui": { + "job": "hermes-webui-image", + "confirmation": "PUBLISH HERMES WEBUI", + }, +} +JOB_NAME = JOBS["agent"]["job"] TOKEN_FILE = Path("/runtime-access/jenkins-image-build-token") REVISION_PATTERN = re.compile(r"^[0-9a-f]{40}$") QUEUE_PATH_PATTERN = re.compile(r"^/queue/item/[0-9]+/?$") @@ -41,23 +51,27 @@ def _open_without_redirect(request: urllib.request.Request, timeout: int): def trigger_build( revision: str, *, + component: str = "agent", token_file: Path = TOKEN_FILE, opener=_open_without_redirect, ) -> dict[str, str | int]: - """Post the fixed job parameters using its job-scoped build token.""" + """Post one allow-listed job's fixed parameters using the release token.""" revision = revision.strip() if not REVISION_PATTERN.fullmatch(revision): raise ValueError("revision must be a lowercase full 40-character commit") + job = JOBS.get(component) + if job is None: + raise ValueError("component must be agent or webui") token = token_file.read_text(encoding="utf-8").strip() if not token: raise RuntimeError("Jenkins image-build token is empty") payload = urllib.parse.urlencode( { - "job": JOB_NAME, + "job": job["job"], "token": token, "PUBLISH_IMAGE": "true", "EXPECTED_SOURCE_REVISION": revision, - "CONFIRM_PUBLISH": "PUBLISH HERMES AGENT", + "CONFIRM_PUBLISH": job["confirmation"], } ).encode("utf-8") request = urllib.request.Request( @@ -87,7 +101,8 @@ def trigger_build( # Never return the submitted URL: its form body contains the job token. queue_path = parsed_queue.path return { - "job": JOB_NAME, + "component": component, + "job": job["job"], "queue_path": queue_path, "source_revision": revision, "status": status, @@ -97,10 +112,13 @@ def trigger_build( def main() -> int: """Validate one revision, trigger the bounded job, and print safe metadata.""" parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("revision", help="reviewed full commit currently on main") + parser.add_argument( + "--component", choices=sorted(JOBS), default="agent", help="image to release" + ) + parser.add_argument("revision", help="reviewed full commit contained by main") args = parser.parse_args() try: - result = trigger_build(args.revision) + result = trigger_build(args.revision, component=args.component) except (OSError, ValueError, RuntimeError, urllib.error.URLError) as exc: print(json.dumps({"error": str(exc)}, sort_keys=True)) return 1 diff --git a/services/jenkins/configmap-jcasc.yaml b/services/jenkins/configmap-jcasc.yaml index fcb87234..b02be8c5 100644 --- a/services/jenkins/configmap-jcasc.yaml +++ b/services/jenkins/configmap-jcasc.yaml @@ -654,7 +654,7 @@ data: } pipelineJob('hermes-agent-image') { disabled(false) - description('Human-gated, daemonless Kaniko build for the reviewed atlas/titan-iac main revision. Publishes a content-addressed Hermes agent image and archives a Flux digest patch; it never mutates Git or deploys.') + description('Bounded daemonless Kaniko release for the latest atlas/titan-iac main containing a reviewed commit. Archives exact evidence, then publishes an immutable tag for Flux deployment.') authenticationToken(System.getenv('HERMES_AGENT_IMAGE_BUILD_TOKEN')) definition { cpsScm { @@ -673,7 +673,8 @@ data: } pipelineJob('hermes-webui-image') { disabled(false) - description('Human-gated, daemonless Kaniko build for the reviewed atlas/titan-iac main revision. Publishes an immutable Hermes WebUI image and archives a narrow two-workload Flux digest patch; it never mutates Git or deploys.') + description('Bounded daemonless Kaniko release for the latest atlas/titan-iac main containing a reviewed commit. Archives exact evidence, then publishes an immutable tag for Flux deployment.') + authenticationToken(System.getenv('HERMES_AGENT_IMAGE_BUILD_TOKEN')) definition { cpsScm { scm { diff --git a/services/jenkins/deployment.yaml b/services/jenkins/deployment.yaml index 258725c0..acffb3f7 100644 --- a/services/jenkins/deployment.yaml +++ b/services/jenkins/deployment.yaml @@ -68,7 +68,7 @@ spec: {{ with secret "kv/data/atlas/hermes/developer-jenkins" }} HERMES_AGENT_IMAGE_BUILD_TOKEN={{ .Data.data.build_token }} {{ end }} - bstein.dev/restarted-at: "2026-05-20T09:40:31Z" + bstein.dev/restarted-at: "2026-08-23T16:40:25Z" spec: serviceAccountName: jenkins priorityClassName: scavenger diff --git a/testing/tests/test_hermes_image_automation.py b/testing/tests/test_hermes_image_automation.py new file mode 100644 index 00000000..12b5fc9c --- /dev/null +++ b/testing/tests/test_hermes_image_automation.py @@ -0,0 +1,67 @@ +"""Contracts for automatic deployment of validated Hermes image releases.""" + +from __future__ import annotations + +from pathlib import Path + +import yaml + + +ROOT = Path(__file__).resolve().parents[2] +SERVICE = ROOT / "services/hermes" +APPLICATIONS = ROOT / "clusters/atlas/flux-system/applications" + + +def test_image_policies_observe_only_validated_release_tags() -> None: + """Candidates remain invisible until Jenkins publishes the release suffix.""" + documents = list( + yaml.safe_load_all((SERVICE / "image.yaml").read_text(encoding="utf-8")) + ) + repositories = { + item["metadata"]["name"]: item + for item in documents + if item["kind"] == "ImageRepository" + } + policies = { + item["metadata"]["name"]: item + for item in documents + if item["kind"] == "ImagePolicy" + } + assert set(repositories) == {"hermes-agent-release", "hermes-webui-release"} + assert set(policies) == set(repositories) + for name, policy in policies.items(): + assert policy["metadata"]["namespace"] == "hermes" + assert policy["spec"]["imageRepositoryRef"]["name"] == name + assert policy["spec"]["filterTags"] == { + "pattern": ("^git-[0-9a-f]{40}-build-" "(?P[1-9][0-9]*)-release$"), + "extract": "$build", + } + assert policy["spec"]["policy"] == {"numerical": {"order": "asc"}} + assert policy["spec"]["digestReflectionPolicy"] == "Always" + + +def test_flux_updates_only_the_three_hermes_image_digests() -> None: + """Flux persists selected digests to Git and rolls all matching workloads.""" + service_kustomization = (SERVICE / "kustomization.yaml").read_text(encoding="utf-8") + applications_kustomization = (APPLICATIONS / "kustomization.yaml").read_text( + encoding="utf-8" + ) + automation = yaml.safe_load( + (APPLICATIONS / "hermes/image-automation.yaml").read_text(encoding="utf-8") + ) + agent = (SERVICE / "kustomization.yaml").read_text(encoding="utf-8") + chat = (SERVICE / "chat-statefulset.yaml").read_text(encoding="utf-8") + dashboard = (SERVICE / "deployment.yaml").read_text(encoding="utf-8") + + assert " - image.yaml" in service_kustomization + assert " - hermes/image-automation.yaml" in applications_kustomization + assert automation["spec"]["git"]["checkout"]["ref"]["branch"] == "main" + assert automation["spec"]["git"]["push"]["branch"] == "main" + assert automation["spec"]["update"] == { + "strategy": "Setters", + "path": "services/hermes", + } + assert agent.count('"$imagepolicy": "hermes:hermes-agent-release:digest"') == 1 + webui_marker = '"$imagepolicy": "hermes:hermes-webui-release:digest"' + assert chat.count(webui_marker) == 1 + assert dashboard.count(webui_marker) == 1 diff --git a/testing/tests/test_hermes_image_builder.py b/testing/tests/test_hermes_image_builder.py index 8d7418d3..419a3d3f 100644 --- a/testing/tests/test_hermes_image_builder.py +++ b/testing/tests/test_hermes_image_builder.py @@ -103,7 +103,7 @@ def test_pipeline_requires_reviewed_main_and_runtime_credentials() -> None: assert "set +x" in source assert "umask 077" in source assert "unset HARBOR_USER HARBOR_PASSWORD auth" in source - assert "/busybox/rm -f /kaniko/.docker/config.json" in source + assert '/busybox/rm -f "${config_path}"' in source assert "--digest-file=" in source assert "--image-name-tag-with-digest-file=" in source assert "--destination=" in source @@ -237,6 +237,41 @@ def test_agent_trigger_rejects_unsafe_revision_and_empty_token(tmp_path: Path) - module.trigger_build("a" * 40, token_file=token_path) +def test_agent_trigger_can_select_only_the_bounded_webui_job(tmp_path: Path) -> None: + """Hermes can release WebUI without gaining a caller-controlled Jenkins job.""" + module = _load_trigger_module() + token_path = tmp_path / "token" + token_path.write_text("private-job-token\n", encoding="utf-8") + captured = {} + + class Response(io.BytesIO): + status = 201 + headers = {"Location": "https://ci.bstein.dev/queue/item/84/"} + + def __enter__(self): + return self + + def __exit__(self, *_args): + self.close() + + def opener(request, timeout): + captured["request"] = request + captured["timeout"] = timeout + return Response(b"") + + result = module.trigger_build( + "b" * 40, component="webui", token_file=token_path, opener=opener + ) + fields = urllib.parse.parse_qs(captured["request"].data.decode("utf-8")) + assert fields["job"] == ["hermes-webui-image"] + assert fields["CONFIRM_PUBLISH"] == ["PUBLISH HERMES WEBUI"] + assert result["component"] == "webui" + with pytest.raises(ValueError, match="agent or webui"): + module.trigger_build( + "b" * 40, component="other", token_file=token_path, opener=opener + ) + + def test_agent_trigger_accepts_existing_queue_redirect(tmp_path: Path) -> None: """HTTP 303 means the exact release is already queued, not a trigger failure.""" module = _load_trigger_module() diff --git a/testing/tests/test_hermes_image_builder_coverage.py b/testing/tests/test_hermes_image_builder_coverage.py index 13a33177..f5a6c3a6 100644 --- a/testing/tests/test_hermes_image_builder_coverage.py +++ b/testing/tests/test_hermes_image_builder_coverage.py @@ -16,8 +16,7 @@ REPO_ROOT = Path(__file__).resolve().parents[2] RUNNER = REPO_ROOT / "dockerfiles/hermes-kaniko-heredoc-runner.py" DOCKERFILE = REPO_ROOT / "dockerfiles/Dockerfile.hermes-agent" HARBOR = ( - REPO_ROOT - / "services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py" + REPO_ROOT / "services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py" ) TRIGGER = REPO_ROOT / "services/hermes/scripts/jenkins_image_build_trigger.py" @@ -94,9 +93,7 @@ def _robot(module, *, immutable: bool = False, duration=-1) -> dict: def _robot_list(module, *, robot_id=41, name=None): - return json.dumps( - [{"id": robot_id, "name": name or module.PUBLISH_ROBOT}] - ).encode() + return json.dumps([{"id": robot_id, "name": name or module.PUBLISH_ROBOT}]).encode() def test_runner_rejects_directive_and_heredoc_boundaries(tmp_path: Path) -> None: @@ -141,7 +138,9 @@ def test_runner_main_dispatches_the_exact_path_and_index( module = _load(RUNNER, "runner_main_coverage") dockerfile = tmp_path / "Dockerfile" seen = [] - monkeypatch.setattr(module, "replay", lambda path, index: seen.append((path, index))) + monkeypatch.setattr( + module, "replay", lambda path, index: seen.append((path, index)) + ) monkeypatch.setattr( sys, "argv", @@ -156,7 +155,9 @@ def test_trigger_redirect_wrapper_accepts_only_plugin_303( ) -> None: """The low-level opener returns a real response or the one expected redirect.""" module = _load(TRIGGER, "trigger_opener_coverage") - assert module._NoRedirect().redirect_request(None, None, 0, None, None, None) is None + assert ( + module._NoRedirect().redirect_request(None, None, 0, None, None, None) is None + ) request = module.urllib.request.Request(module.JENKINS_BUILD_URL) class Opener: @@ -199,12 +200,19 @@ def test_trigger_main_reports_safe_success_and_errors( monkeypatch.setattr( module, "trigger_build", - lambda value: {"job": module.JOB_NAME, "source_revision": value, "status": 201}, + lambda value, component="agent": { + "component": component, + "job": module.JOBS[component]["job"], + "source_revision": value, + "status": 201, + }, ) assert module.main() == 0 assert json.loads(capsys.readouterr().out)["source_revision"] == revision monkeypatch.setattr( - module, "trigger_build", lambda _value: (_ for _ in ()).throw(OSError("closed")) + module, + "trigger_build", + lambda _value, component="agent": (_ for _ in ()).throw(OSError("closed")), ) assert module.main() == 1 assert json.loads(capsys.readouterr().out) == {"error": "closed"} @@ -278,11 +286,7 @@ def test_harbor_json_lists_reject_invalid_shapes(body: bytes) -> None: "access", ), ( - { - "permissions": [ - {"kind": "project", "namespace": "bstein", "access": []} - ] - }, + {"permissions": [{"kind": "project", "namespace": "bstein", "access": []}]}, "pull/push", ), ], @@ -304,7 +308,11 @@ def test_publisher_scope_rejects_malformed_or_incomplete_permissions( ([(403, b"", {})], "403", False), ([(200, b"[]", _count(0))], "exactly one", False), ([(200, b"[]", _count(0))], "exactly one", False), - ([(200, b'[{\"name\":\"robot$jenkins-pipelines\",\"id\":0}]', _count(1))], "valid ID", False), + ( + [(200, b'[{"name":"robot$jenkins-pipelines","id":0}]', _count(1))], + "valid ID", + False, + ), ], ) def test_publisher_list_rejects_status_count_and_id( @@ -324,7 +332,7 @@ def test_publisher_list_rejects_status_count_and_id( ((403, b"", {}), "403", False), ((200, b"not-json", {}), "invalid robot JSON", False), ((200, b"[]", {}), "invalid shape", False), - ((200, b'{}', {}), "not active and exact", False), + ((200, b"{}", {}), "not active and exact", False), ], ) def test_publisher_read_rejects_status_and_identity( @@ -405,9 +413,7 @@ def test_rule_rejects_ids_create_status_and_location_suffix() -> None: FakeClient([(200, json.dumps([bad_rule]).encode(), _count(1))]) ) with pytest.raises(module.HarborUnavailable): - module.ensure_rule( - FakeClient([(200, b"[]", _count(0)), (503, b"", {})]) - ) + module.ensure_rule(FakeClient([(200, b"[]", _count(0)), (503, b"", {})])) with pytest.raises(RuntimeError, match="invalid ID"): module.ensure_rule( FakeClient( diff --git a/testing/tests/test_hermes_image_builder_fresh_review.py b/testing/tests/test_hermes_image_builder_fresh_review.py index 27ccc9b3..5581f683 100644 --- a/testing/tests/test_hermes_image_builder_fresh_review.py +++ b/testing/tests/test_hermes_image_builder_fresh_review.py @@ -90,7 +90,9 @@ def test_all_unsupported_run_heredoc_forms_reject_before_execution( encoding="utf-8", ) calls = [] - monkeypatch.setattr(module.subprocess, "run", lambda *_args, **_kwargs: calls.append(1)) + monkeypatch.setattr( + module.subprocess, "run", lambda *_args, **_kwargs: calls.append(1) + ) with pytest.raises(ValueError, match="unsupported RUN heredoc"): module.replay(dockerfile, 1) assert calls == [] @@ -119,7 +121,9 @@ def test_appended_tenth_reviewed_form_rejects_before_execution( encoding="utf-8", ) calls = [] - monkeypatch.setattr(module.subprocess, "run", lambda *_args, **_kwargs: calls.append(1)) + monkeypatch.setattr( + module.subprocess, "run", lambda *_args, **_kwargs: calls.append(1) + ) with pytest.raises(ValueError, match="contract changed"): module.replay(dockerfile, 1) assert calls == [] @@ -155,7 +159,9 @@ def test_logical_instruction_normalization_rejects_split_heredocs_before_executi encoding="utf-8", ) calls = [] - monkeypatch.setattr(module.subprocess, "run", lambda *_args, **_kwargs: calls.append(1)) + monkeypatch.setattr( + module.subprocess, "run", lambda *_args, **_kwargs: calls.append(1) + ) with pytest.raises(ValueError, match="unsupported RUN heredoc"): module.replay(dockerfile, 1) assert calls == [] @@ -208,7 +214,9 @@ def test_builder_service_account_is_explicit_tokenless_and_unbound() -> None: assert spec["automountServiceAccountToken"] is False for volume in spec.get("volumes", []): projected = volume.get("projected", {}) - assert all("serviceAccountToken" not in item for item in projected.get("sources", [])) + assert all( + "serviceAccountToken" not in item for item in projected.get("sources", []) + ) for manifest in (REPO_ROOT / "services/jenkins").glob("*.yaml"): for document in yaml.safe_load_all(manifest.read_text(encoding="utf-8")): @@ -278,7 +286,9 @@ def test_success_evidence_revalidation_accepts_only_exact_complete_set( metadata = kwargs["output_dir"] / "hermes-agent-image.json" original = metadata.read_text(encoding="utf-8") - metadata.write_text(original.replace('"build_number": "23"', '"build_number": "24"')) + metadata.write_text( + original.replace('"build_number": "23"', '"build_number": "24"') + ) with pytest.raises(ValueError, match="incomplete or mismatched"): module.validate_release_artifacts(**kwargs) metadata.write_text(original, encoding="utf-8") @@ -319,17 +329,20 @@ def test_verify_evidence_cli_needs_no_runtime_registry_credential( assert module.main() == 0 -def test_pipeline_success_post_requires_and_archives_exact_six_files() -> None: - """Missing or partial post-success evidence must change the build to failed.""" +def test_pipeline_requires_and_archives_exact_release_evidence() -> None: + """Missing evidence must fail before the candidate becomes a release.""" source = PIPELINE.read_text(encoding="utf-8") - post = source.split(" post {", 1)[1] - assert "success {" in post - assert "always {" not in post - assert "verify-evidence" in post - assert 'allowEmptyArchive: false' in post - archive = post.split("artifacts: '", 1)[1].split("'", 1)[0] + evidence = source.split("stage('Verify and archive release evidence')", 1)[1] + evidence = evidence.split("stage('Publish Flux release tag')", 1)[0] + assert "verify-evidence" in evidence + assert "allowEmptyArchive: false" in evidence + archive = evidence.split("artifacts: '", 1)[1].split("'", 1)[0] paths = archive.split(",") - assert len(paths) == 6 - assert len(set(paths)) == 6 + assert len(paths) == 7 + assert len(set(paths)) == 7 assert all("*" not in path for path in paths) - assert "find build -type f" in post + assert "find build -type f" in evidence + assert "build/hermes-agent.source-revision" in paths + promotion = source.split("stage('Publish Flux release tag')", 1)[1] + assert "ci/scripts/hermes_oci_promote.py" in promotion + assert " post {" not in source diff --git a/testing/tests/test_hermes_oci_promote.py b/testing/tests/test_hermes_oci_promote.py new file mode 100644 index 00000000..4e6d1ee7 --- /dev/null +++ b/testing/tests/test_hermes_oci_promote.py @@ -0,0 +1,205 @@ +"""Safety tests for the evidence-bound Hermes OCI release promotion.""" + +from __future__ import annotations + +import importlib.util +import io +import json +import sys +import urllib.request +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / "ci/scripts/hermes_oci_promote.py" +REVISION = "a" * 40 +DIGEST = "sha256:" + "b" * 64 +DESTINATION = f"registry.bstein.dev/bstein/hermes-agent:git-{REVISION}-build-17" +CONTENT_TYPE = "application/vnd.docker.distribution.manifest.v2+json" +MANIFEST = b'{"schemaVersion":2}' + + +def _load(): + spec = importlib.util.spec_from_file_location("hermes_oci_promote", SCRIPT) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class Response(io.BytesIO): + """Minimal context-managed urllib response.""" + + def __init__(self, status: int, body: bytes = b"", headers=None): + super().__init__(body) + self.status = status + self.headers = headers or {} + + def __enter__(self): + return self + + def __exit__(self, *_args): + self.close() + + +def _candidate(headers=None, body: bytes = MANIFEST) -> Response: + values = { + "Docker-Content-Digest": DIGEST, + "Content-Type": CONTENT_TYPE, + } + values.update(headers or {}) + return Response(200, body, values) + + +def _promote(module, opener): + return module.promote_candidate( + destination=DESTINATION, + digest=DIGEST, + source_revision=REVISION, + build_number="17", + username="robot", + password="private", + opener=opener, + ) + + +def test_promotion_copies_exact_candidate_to_release_tag() -> None: + """Only the exact evidence digest is copied to the Flux-visible tag.""" + module = _load() + calls: list[urllib.request.Request] = [] + responses = iter( + [ + _candidate(), + Response(404), + Response(201, headers={"Docker-Content-Digest": DIGEST}), + ] + ) + + def opener(request, timeout): + calls.append(request) + assert timeout in {20, 30} + return next(responses) + + result = _promote(module, opener) + assert [request.method for request in calls] == ["GET", "HEAD", "PUT"] + assert calls[0].full_url.endswith(f"/manifests/git-{REVISION}-build-17") + assert calls[2].full_url.endswith(f"/manifests/git-{REVISION}-build-17-release") + assert calls[2].data == MANIFEST + assert result["result"] == "published" + assert result["digest"] == DIGEST + assert "private" not in json.dumps(result) + + +def test_promotion_is_idempotent_for_the_same_digest() -> None: + """A replay succeeds only when the immutable release already matches.""" + module = _load() + responses = iter( + [_candidate(), Response(200, headers={"Docker-Content-Digest": DIGEST})] + ) + result = _promote(module, lambda *_args: next(responses)) + assert result["result"] == "already-present" + + +@pytest.mark.parametrize( + ("changes", "message"), + [ + ({"destination": "registry.invalid/hermes:latest"}, "destination"), + ({"source_revision": "c" * 40}, "revision"), + ({"build_number": "18"}, "build number"), + ({"digest": "sha256:bad"}, "digest"), + ({"username": ""}, "credentials"), + ], +) +def test_promotion_rejects_unbound_inputs(changes: dict, message: str) -> None: + """Destination, source, build, digest, and credentials fail closed.""" + module = _load() + kwargs = { + "destination": DESTINATION, + "digest": DIGEST, + "source_revision": REVISION, + "build_number": "17", + "username": "robot", + "password": "private", + "opener": lambda *_args: _candidate(), + } + kwargs.update(changes) + with pytest.raises((ValueError, RuntimeError), match=message): + module.promote_candidate(**kwargs) + + +@pytest.mark.parametrize( + ("responses", "message"), + [ + ([Response(401)], "candidate manifest returned HTTP 401"), + ([_candidate({"Docker-Content-Digest": "sha256:" + "c" * 64})], "digest"), + ([_candidate({"Content-Type": "text/plain"})], "content type"), + ( + [ + _candidate(), + Response(200, headers={"Docker-Content-Digest": "sha256:" + "c" * 64}), + ], + "another digest", + ), + ([_candidate(), Response(500)], "preflight returned HTTP 500"), + ( + [_candidate(), Response(404), Response(500)], + "release manifest returned HTTP 500", + ), + ( + [ + _candidate(), + Response(404), + Response(201, headers={"Docker-Content-Digest": "sha256:" + "c" * 64}), + ], + "changed", + ), + ], +) +def test_registry_failures_cannot_create_a_valid_release( + responses: list[Response], message: str +) -> None: + """Registry ambiguity or mismatch always fails the final release boundary.""" + module = _load() + queued = iter(responses) + with pytest.raises(RuntimeError, match=message): + _promote(module, lambda *_args: next(queued)) + + +def test_cli_reports_credential_free_success_and_error( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """The command emits bounded metadata and converts expected failures to JSON.""" + module = _load() + digest_file = tmp_path / "digest" + digest_file.write_text(DIGEST + "\n", encoding="utf-8") + monkeypatch.setattr( + sys, + "argv", + [ + "promote", + "--destination", + DESTINATION, + "--digest-file", + str(digest_file), + "--source-revision", + REVISION, + "--build-number", + "17", + ], + ) + monkeypatch.setattr( + module, + "promote_candidate", + lambda **_kwargs: {"result": "published", "digest": DIGEST}, + ) + assert module.main() == 0 + assert json.loads(capsys.readouterr().out)["result"] == "published" + monkeypatch.setattr( + module, + "promote_candidate", + lambda **_kwargs: (_ for _ in ()).throw(RuntimeError("denied")), + ) + assert module.main() == 1 + assert json.loads(capsys.readouterr().out) == {"error": "denied"} diff --git a/testing/tests/test_hermes_webui_release.py b/testing/tests/test_hermes_webui_release.py index 0ab35dfa..6b7431ab 100644 --- a/testing/tests/test_hermes_webui_release.py +++ b/testing/tests/test_hermes_webui_release.py @@ -69,8 +69,8 @@ def _release_fixture(tmp_path: Path): return module, digest, kwargs -def test_webui_job_is_independent_manual_and_main_only() -> None: - """WebUI has its own job and never widens the existing agent-only lane.""" +def test_webui_job_is_independent_bounded_and_main_only() -> None: + """WebUI has its own bounded job using the same Vault-injected release token.""" config = yaml.safe_load( (ROOT / "services/jenkins/configmap-jcasc.yaml").read_text(encoding="utf-8") ) @@ -83,14 +83,17 @@ def test_webui_job_is_independent_manual_and_main_only() -> None: assert "branches('*/main')" in block assert "scriptPath('ci/Jenkinsfile.hermes-webui-image')" in block assert "pipelineTriggers" not in block - assert "HERMES_AGENT_IMAGE_BUILD_TOKEN" not in block + assert ( + "authenticationToken(System.getenv('HERMES_AGENT_IMAGE_BUILD_TOKEN'))" in block + ) -def test_pipeline_builds_exact_reviewed_main_and_never_deploys() -> None: - """Publish is explicit, immutable, evidence-producing, and Git/Flux review only.""" +def test_pipeline_builds_latest_main_containing_reviewed_anchor() -> None: + """Publish is explicit, evidence-bound, and handed only to Flux.""" source = PIPELINE.read_text(encoding="utf-8") assert 'test "${CONFIRM_PUBLISH}" = "PUBLISH HERMES WEBUI"' in source assert 'test "${actual_revision}" = "$(git rev-parse origin/main)"' in source + assert 'git merge-base --is-ancestor "${EXPECTED_SOURCE_REVISION}"' in source assert "dockerfiles/Dockerfile.hermes-webui" in source assert "ci/scripts/hermes_webui_release.py" in source assert "registry.bstein.dev/bstein/hermes-webui" in source @@ -99,8 +102,9 @@ def test_pipeline_builds_exact_reviewed_main_and_never_deploys() -> None: assert "HERMES_KANIKO_HEREDOC_COMPAT" not in source assert "--digest-file=" in source assert "--image-name-tag-with-digest-file=" in source - assert "org.opencontainers.image.revision=${EXPECTED_SOURCE_REVISION}" in source + assert "org.opencontainers.image.revision=${source_revision}" in source assert "assert-absent" in source and "verify-evidence" in source + assert "ci/scripts/hermes_oci_promote.py" in source assert "test_hermes_webui_brand.py" in source assert "test_hermes_webui_release.py" in source for forbidden in ("kubectl ", "flux reconcile", "git push", "git commit"): @@ -300,7 +304,9 @@ def test_release_verifies_exact_webui_harbor_artifact_and_policy() -> None: ) -@pytest.mark.parametrize("labels", [None, {}, {"org.opencontainers.image.revision": "c" * 40}]) +@pytest.mark.parametrize( + "labels", [None, {}, {"org.opencontainers.image.revision": "c" * 40}] +) def test_release_rejects_missing_or_wrong_harbor_source_revision(labels) -> None: """A tag derived from Git cannot substitute for the persisted OCI label.""" module = _load(RELEASE, f"webui_registry_revision_{labels!r}") @@ -379,9 +385,7 @@ def test_flux_tracks_webui_policy_before_jenkins() -> None: assert container["securityContext"]["runAsNonRoot"] is True assert container["securityContext"]["capabilities"]["drop"] == ["ALL"] - release_docs = (ROOT / "docs/hermes_webui_release.md").read_text( - encoding="utf-8" - ) + release_docs = (ROOT / "docs/hermes_webui_release.md").read_text(encoding="utf-8") assert "immutable-tag:list" in release_docs assert "harbor-hermes-agent-immutability-ensure-1" in release_docs @@ -466,14 +470,17 @@ def test_webui_policy_rejects_disabled_duplicate_or_truncated_rules() -> None: policy.list_rules(_FakePolicyClient([(200, b"[]", {"X-Total-Count": "1"})])) -def test_pipeline_archives_exact_seven_files() -> None: - """Publication cannot pass with missing digest, workload, or metadata evidence.""" +def test_pipeline_archives_exact_eight_files_before_release() -> None: + """Release cannot pass with missing digest, workload, or metadata evidence.""" source = PIPELINE.read_text(encoding="utf-8") - post = source.split(" post {", 1)[1] - archive = post.split("artifacts: '", 1)[1].split("'", 1)[0].split(",") - assert len(archive) == len(set(archive)) == 7 + evidence = source.split("stage('Verify and archive release evidence')", 1)[1] + evidence = evidence.split("stage('Publish Flux release tag')", 1)[0] + archive = evidence.split("artifacts: '", 1)[1].split("'", 1)[0].split(",") + assert len(archive) == len(set(archive)) == 8 assert all("*" not in path for path in archive) - assert "find build -type f" in post - assert "allowEmptyArchive: false" in post - assert "hermes-chat-statefulset.yaml" in post - assert "hermes-dashboard-deployment.yaml" in post + assert "find build -type f" in evidence + assert "allowEmptyArchive: false" in evidence + assert "build/hermes-webui.source-revision" in archive + assert "hermes-chat-statefulset.yaml" in evidence + assert "hermes-dashboard-deployment.yaml" in evidence + assert " post {" not in source