diff --git a/ci/Jenkinsfile.hermes-chat-router-image b/ci/Jenkinsfile.hermes-chat-router-image new file mode 100644 index 00000000..30ea5b5d --- /dev/null +++ b/ci/Jenkinsfile.hermes-chat-router-image @@ -0,0 +1,339 @@ +pipeline { + agent { + kubernetes { + defaultContainer 'python' + yaml """ +apiVersion: v1 +kind: Pod +metadata: + labels: + atlas.bstein.dev/workload: hermes-chat-router-image-builder +spec: + serviceAccountName: hermes-image-builder + automountServiceAccountToken: false + enableServiceLinks: false + restartPolicy: Never + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + nodeSelector: + kubernetes.io/arch: arm64 + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: [titan-20] + imagePullSecrets: + - name: harbor-bstein-robot + containers: + - name: jnlp + image: jenkins/inbound-agent@sha256:8eda4fe2a66bcf6a5e43436d9918fc14c306204dc8fcd75f4e15e0e6e5dc759a + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + runAsNonRoot: true + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault + resources: + requests: {cpu: 25m, memory: 128Mi} + limits: {cpu: 500m, memory: 512Mi} + - name: python + image: registry.bstein.dev/bstein/python@sha256:269541d3387baae008df4608ead893dba2b5cdaad1a5a380731a88992d34b808 + command: ["sleep"] + args: ["99d"] + tty: true + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + runAsNonRoot: true + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault + resources: + requests: {cpu: 25m, memory: 64Mi} + limits: {cpu: 250m, memory: 256Mi} + - name: golang + image: golang:1.24-alpine@sha256:8bee1901f1e530bfb4a7850aa7a479d17ae3a18beb6e09064ed54cfd245b7191 + command: ["sleep"] + args: ["99d"] + tty: true + env: + - {name: GOCACHE, value: /tmp/go-cache} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + runAsNonRoot: true + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault + resources: + requests: {cpu: 100m, memory: 128Mi} + limits: {cpu: "1", memory: 1Gi} + - name: kaniko + image: gcr.io/kaniko-project/executor@sha256:c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e + command: ["/busybox/sh", "-c"] + args: ["/busybox/sleep 99d"] + tty: true + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"] + privileged: false + runAsUser: 0 + seccompProfile: + type: RuntimeDefault + resources: + requests: + cpu: 100m + memory: 256Mi + ephemeral-storage: 2Gi + limits: + cpu: "1" + memory: 2Gi + ephemeral-storage: 5Gi +""" + } + } + parameters { + booleanParam( + name: 'PUBLISH_IMAGE', + defaultValue: false, + description: 'Publish the reviewed main revision to Harbor.' + ) + string( + name: 'EXPECTED_SOURCE_REVISION', + defaultValue: '', + description: 'Full reviewed commit that must be contained by atlas/titan-iac main.' + ) + string( + name: 'CONFIRM_PUBLISH', + defaultValue: '', + description: 'Enter PUBLISH HERMES CHAT ROUTER to confirm the release.' + ) + } + environment { + HERMES_IMAGE = 'registry.bstein.dev/bstein/hermes-chat-router' + } + options { + disableConcurrentBuilds() + buildDiscarder(logRotator(daysToKeepStr: '30', numToKeepStr: '100', artifactDaysToKeepStr: '30', artifactNumToKeepStr: '100')) + skipDefaultCheckout(true) + timeout(time: 45, unit: 'MINUTES') + } + stages { + stage('Checkout reviewed source') { + steps { + checkout scm + } + } + stage('Enforce release boundary') { + steps { + container('jnlp') { + sh ''' + set -eu + mkdir -p build + test "${PUBLISH_IMAGE}" = "true" + test "${CONFIRM_PUBLISH}" = "PUBLISH HERMES CHAT ROUTER" + case "${EXPECTED_SOURCE_REVISION}" in + *[!0-9a-f]*|'') + echo "EXPECTED_SOURCE_REVISION must be a lowercase full commit" >&2 + exit 2 + ;; + esac + test "${#EXPECTED_SOURCE_REVISION}" -eq 40 + main_revision="$(git rev-parse HEAD)" + test "${main_revision}" = "$(git rev-parse origin/main)" + git merge-base --is-ancestor "${EXPECTED_SOURCE_REVISION}" "${main_revision}" + git checkout --detach "${EXPECTED_SOURCE_REVISION}" + actual_revision="$(git rev-parse HEAD)" + test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}" + test -z "$(git status --porcelain)" + test -f dockerfiles/Dockerfile.hermes-chat-router + case "${BUILD_NUMBER}" in + ''|0*|*[!0-9]*) + echo "BUILD_NUMBER must be a positive decimal integer" >&2 + exit 2 + ;; + esac + printf '%s\n' \ + "${HERMES_IMAGE}:git-${actual_revision}-build-${BUILD_NUMBER}" \ + > build/hermes-chat-router.destination + printf '%s\n' "${actual_revision}" \ + > build/hermes-chat-router.source-revision + ''' + } + } + } + stage('Validate reviewed router source') { + steps { + container('python') { + sh ''' + set -eu + python3 -m pip install --disable-pip-version-check --no-cache-dir \ + --target=/tmp/hermes-chat-router-test-deps \ + pytest==8.3.4 PyYAML==6.0.2 + PYTHONPATH=/tmp/hermes-chat-router-test-deps \ + python3 -m pytest -q \ + testing/tests/test_hermes_chat_router_release.py \ + testing/tests/test_hermes_oci_promote.py \ + testing/tests/test_hermes_image_automation.py + ''' + } + container('golang') { + sh ''' + set -eu + cd services/hermes/router + GO111MODULE=off CGO_ENABLED=0 go test ./... + ''' + } + } + } + stage('Reject replay before publish') { + steps { + withCredentials([usernamePassword( + credentialsId: 'harbor-robot', + usernameVariable: 'HARBOR_USER', + passwordVariable: 'HARBOR_PASSWORD' + )]) { + sh ''' + set -eu + set +x + destination="$(cat build/hermes-chat-router.destination)" + source_revision="$(cat build/hermes-chat-router.source-revision)" + python3 ci/scripts/hermes_chat_router_release.py assert-absent \ + --source-revision "${source_revision}" \ + --build-number "${BUILD_NUMBER}" \ + --destination "${destination}" + ''' + } + } + } + stage('Build and publish without a daemon') { + steps { + container('kaniko') { + withCredentials([usernamePassword( + credentialsId: 'harbor-robot', + usernameVariable: 'HARBOR_USER', + passwordVariable: 'HARBOR_PASSWORD' + )]) { + sh '''#!/busybox/sh + set -eu + set +x + config_path=/kaniko/.docker/config.json + destination="$(cat build/hermes-chat-router.destination)" + source_revision="$(cat build/hermes-chat-router.source-revision)" + umask 077 + auth="$(printf '%s:%s' "${HARBOR_USER}" "${HARBOR_PASSWORD}" | /busybox/base64 | /busybox/tr -d '\n')" + /busybox/mkdir -p /kaniko/.docker + /busybox/printf '{"auths":{"registry.bstein.dev":{"auth":"%s"}}}\n' "${auth}" > "${config_path}" + unset HARBOR_USER HARBOR_PASSWORD auth + trap '/busybox/rm -f "${config_path}"' EXIT HUP INT TERM + umask 022 + /kaniko/executor \ + --registry-mirror=harbor-core.harbor.svc.cluster.local \ + --insecure-registry=harbor-core.harbor.svc.cluster.local \ + --context="dir://${WORKSPACE}" \ + --dockerfile="${WORKSPACE}/dockerfiles/Dockerfile.hermes-chat-router" \ + --destination="${destination}" \ + --digest-file="${WORKSPACE}/build/hermes-chat-router.digest" \ + --image-name-tag-with-digest-file="${WORKSPACE}/build/hermes-chat-router.image" \ + --label="org.opencontainers.image.revision=${source_revision}" \ + --label="org.opencontainers.image.source=https://scm.bstein.dev/atlas/titan-iac" \ + --label="org.opencontainers.image.title=hermes-chat-router" \ + --cleanup \ + --push-retry=3 + /busybox/chmod 644 \ + build/hermes-chat-router.digest build/hermes-chat-router.image + ''' + } + } + } + } + stage('Render reviewed Flux handoff') { + steps { + withCredentials([usernamePassword( + credentialsId: 'harbor-robot', + usernameVariable: 'HARBOR_USER', + passwordVariable: 'HARBOR_PASSWORD' + )]) { + sh ''' + set -eu + set +x + destination="$(cat build/hermes-chat-router.destination)" + source_revision="$(cat build/hermes-chat-router.source-revision)" + python3 ci/scripts/hermes_chat_router_release.py render \ + --digest-file build/hermes-chat-router.digest \ + --image-file build/hermes-chat-router.image \ + --source-revision "${source_revision}" \ + --build-number "${BUILD_NUMBER}" \ + --destination "${destination}" \ + --manifest services/hermes/chat-router.yaml \ + --output-dir build/hermes-chat-router-release + ''' + } + } + } + stage('Verify and archive release evidence') { + steps { + sh ''' + set -eu + expected_files="$(printf '%s\n' \ + build/hermes-chat-router.destination \ + build/hermes-chat-router.digest \ + build/hermes-chat-router.image \ + build/hermes-chat-router.source-revision \ + build/hermes-chat-router-release/hermes-chat-router-deployment.yaml \ + build/hermes-chat-router-release/hermes-chat-router-image-update.patch \ + build/hermes-chat-router-release/hermes-chat-router-image.json \ + | LC_ALL=C sort)" + actual_files="$(find build -type f -print | LC_ALL=C sort)" + test "${actual_files}" = "${expected_files}" + destination="$(cat build/hermes-chat-router.destination)" + source_revision="$(cat build/hermes-chat-router.source-revision)" + python3 ci/scripts/hermes_chat_router_release.py verify-evidence \ + --digest-file build/hermes-chat-router.digest \ + --image-file build/hermes-chat-router.image \ + --source-revision "${source_revision}" \ + --build-number "${BUILD_NUMBER}" \ + --destination "${destination}" \ + --manifest services/hermes/chat-router.yaml \ + --output-dir build/hermes-chat-router-release + ''' + archiveArtifacts( + artifacts: 'build/hermes-chat-router.destination,build/hermes-chat-router.digest,build/hermes-chat-router.image,build/hermes-chat-router.source-revision,build/hermes-chat-router-release/hermes-chat-router-deployment.yaml,build/hermes-chat-router-release/hermes-chat-router-image-update.patch,build/hermes-chat-router-release/hermes-chat-router-image.json', + allowEmptyArchive: false, + fingerprint: true + ) + } + } + stage('Publish Flux release tag') { + steps { + withCredentials([usernamePassword( + credentialsId: 'harbor-robot', + usernameVariable: 'HARBOR_USER', + passwordVariable: 'HARBOR_PASSWORD' + )]) { + sh ''' + set -eu + set +x + destination="$(cat build/hermes-chat-router.destination)" + source_revision="$(cat build/hermes-chat-router.source-revision)" + python3 ci/scripts/hermes_oci_promote.py \ + --destination "${destination}" \ + --digest-file build/hermes-chat-router.digest \ + --source-revision "${source_revision}" \ + --build-number "${BUILD_NUMBER}" + ''' + } + } + } + } +} diff --git a/ci/scripts/hermes_chat_router_release.py b/ci/scripts/hermes_chat_router_release.py new file mode 100644 index 00000000..7ee37e8e --- /dev/null +++ b/ci/scripts/hermes_chat_router_release.py @@ -0,0 +1,479 @@ +#!/usr/bin/env python3 +"""Verify and render a reviewable Hermes chat-router image release.""" + +from __future__ import annotations + +import argparse +import base64 +import difflib +import json +import os +import re +import urllib.error +import urllib.parse +import urllib.request +from pathlib import Path +from typing import Any, Callable + + +DEFAULT_IMAGE = "registry.bstein.dev/bstein/hermes-chat-router" +HARBOR_API_ORIGIN = "https://registry.bstein.dev/api/v2.0" +HARBOR_PROJECT = "bstein" +HARBOR_REPOSITORY = "hermes-chat-router" +IMMUTABLE_REPOSITORY_PATTERN = "hermes-chat-router" +IMMUTABLE_TAG_PATTERN = "git-*-build-*" +DIGEST_PATTERN = re.compile(r"^sha256:[0-9a-f]{64}$") +REVISION_PATTERN = re.compile(r"^[0-9a-f]{40}$") +BUILD_PATTERN = re.compile(r"^[1-9][0-9]*$") +DESTINATION_PATTERN = re.compile( + r"^registry\.bstein\.dev/bstein/hermes-chat-router:" + r"git-([0-9a-f]{40})-build-([1-9][0-9]*)$" +) + + +class _NoRedirect(urllib.request.HTTPRedirectHandler): + """Never forward registry credentials to a redirect target.""" + + def redirect_request(self, _request, _file, _code, _message, _headers, _url): + return None + + +def _validated(value: str, pattern: re.Pattern[str], label: str) -> str: + """Return a normalized value only when it matches the release contract.""" + normalized = value.strip() + if not pattern.fullmatch(normalized): + raise ValueError(f"invalid {label}: expected {pattern.pattern}") + return normalized + + +def validate_destination( + destination: str, source_revision: str, build_number: str +) -> tuple[str, str]: + """Bind the unique build tag to one reviewed source and Jenkins build.""" + revision = _validated(source_revision, REVISION_PATTERN, "source revision") + build = _validated(build_number, BUILD_PATTERN, "build number") + match = DESTINATION_PATTERN.fullmatch(destination.strip()) + if not match or match.groups() != (revision, build): + raise ValueError("destination does not match the reviewed revision and build") + return revision, build + + +def validate_kaniko_evidence( + *, digest_text: str, image_text: str, destination: str +) -> str: + """Cross-check Kaniko's two independent output files.""" + digests = digest_text.splitlines() + images = image_text.splitlines() + if len(digests) != 1 or len(images) != 1: + raise ValueError("Kaniko evidence must contain exactly one line per file") + digest = _validated(digests[0], DIGEST_PATTERN, "image digest") + if images[0].strip() != f"{destination}@{digest}": + raise ValueError("Kaniko image evidence does not match destination and digest") + return digest + + +def _registry_request(request: urllib.request.Request, timeout: int) -> Any: + """Return same-origin registry responses without following redirects.""" + opener = urllib.request.build_opener(_NoRedirect()) + try: + return opener.open(request, timeout=timeout) + except urllib.error.HTTPError as exc: + return exc + + +def _authorization(username: str, password: str) -> str: + """Build a Basic header without placing credentials in a URL.""" + if not username or not password: + raise RuntimeError("Harbor credentials are unavailable") + token = base64.b64encode(f"{username}:{password}".encode()).decode("ascii") + return f"Basic {token}" + + +def _artifact_response( + destination: str, + *, + username: str, + password: str, + opener: Callable[[urllib.request.Request, int], Any] = _registry_request, +) -> tuple[int, bytes]: + """Read one exact Harbor artifact by candidate tag.""" + if not DESTINATION_PATTERN.fullmatch(destination): + raise ValueError("invalid destination") + tag = urllib.parse.quote(destination.rsplit(":", 1)[1], safe="") + request = urllib.request.Request( + f"{HARBOR_API_ORIGIN}/projects/{HARBOR_PROJECT}/repositories/" + f"{HARBOR_REPOSITORY}/artifacts/{tag}?with_immutable_status=true", + headers={ + "Accept": "application/json", + "Authorization": _authorization(username, password), + }, + method="GET", + ) + with opener(request, 20) as response: + body = response.read(1_048_577) + if len(body) > 1_048_576: + raise RuntimeError("Harbor artifact response exceeded the size limit") + return int(response.status), body + + +def _rules_response( + *, + username: str, + password: str, + opener: Callable[[urllib.request.Request, int], Any] = _registry_request, +) -> tuple[int, bytes, dict[str, str]]: + """Read the complete bounded Harbor immutable-rule page.""" + request = urllib.request.Request( + f"{HARBOR_API_ORIGIN}/projects/{HARBOR_PROJECT}/immutabletagrules" + "?page=1&page_size=100", + headers={ + "Accept": "application/json", + "Authorization": _authorization(username, password), + }, + method="GET", + ) + with opener(request, 20) as response: + body = response.read(1_048_577) + if len(body) > 1_048_576: + raise RuntimeError("Harbor immutable rule response exceeded the size limit") + return int(response.status), body, dict(response.headers) + + +def _normalized_rule(rule: dict[str, Any]) -> dict[str, Any]: + """Select only immutable-policy fields used by this lane.""" + return { + "disabled": bool(rule.get("disabled", False)), + "action": rule.get("action"), + "template": rule.get("template"), + "tag_selectors": [ + {key: item.get(key) for key in ("kind", "decoration", "pattern")} + for item in rule.get("tag_selectors") or [] + if isinstance(item, dict) + ], + "scope_selectors": { + "repository": [ + {key: item.get(key) for key in ("kind", "decoration", "pattern")} + for item in (rule.get("scope_selectors") or {}).get("repository", []) + if isinstance(item, dict) + ] + }, + } + + +def verify_immutable_policy( + *, + username: str, + password: str, + opener: Callable[[urllib.request.Request, int], Any] = _registry_request, +) -> None: + """Fail closed unless one exact active router immutability rule exists.""" + status, body, headers = _rules_response( + username=username, password=password, opener=opener + ) + if status != 200: + raise RuntimeError(f"Harbor immutable policy preflight returned HTTP {status}") + try: + rules = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise RuntimeError("Harbor returned invalid immutable rule JSON") from exc + if not isinstance(rules, list) or not all(isinstance(item, dict) for item in rules): + raise RuntimeError("Harbor immutable rule list has an invalid shape") + total = next( + (value for key, value in headers.items() if key.lower() == "x-total-count"), + None, + ) + if total is None or not str(total).isdecimal() or int(total) != len(rules): + raise RuntimeError("Harbor immutable rule page is incomplete") + expected = { + "disabled": False, + "action": "immutable", + "template": "immutable_template", + "tag_selectors": [ + { + "kind": "doublestar", + "decoration": "matches", + "pattern": IMMUTABLE_TAG_PATTERN, + } + ], + "scope_selectors": { + "repository": [ + { + "kind": "doublestar", + "decoration": "repoMatches", + "pattern": IMMUTABLE_REPOSITORY_PATTERN, + } + ] + }, + } + matches = [ + value + for value in map(_normalized_rule, rules) + if value["tag_selectors"] == expected["tag_selectors"] + and value["scope_selectors"] == expected["scope_selectors"] + ] + if matches != [expected]: + raise RuntimeError("Harbor router immutable build-tag policy is not exact") + + +def assert_tag_absent( + destination: str, + *, + username: str, + password: str, + opener: Callable[[urllib.request.Request, int], Any] = _registry_request, +) -> None: + """Reject replay before Kaniko can target an already-used build tag.""" + status, _ = _artifact_response( + destination, username=username, password=password, opener=opener + ) + if status == 404: + return + if status == 200: + raise RuntimeError("Harbor destination tag already exists") + raise RuntimeError(f"Harbor destination preflight returned HTTP {status}") + + +def verify_registry_digest( + destination: str, + digest: str, + source_revision: str, + *, + username: str, + password: str, + opener: Callable[[urllib.request.Request, int], Any] = _registry_request, +) -> None: + """Verify Harbor's digest, immutable tag, and persisted source label.""" + digest = _validated(digest, DIGEST_PATTERN, "image digest") + revision = _validated(source_revision, REVISION_PATTERN, "source revision") + status, body = _artifact_response( + destination, username=username, password=password, opener=opener + ) + if status != 200: + raise RuntimeError(f"Harbor manifest verification returned HTTP {status}") + try: + artifact = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise RuntimeError("Harbor returned invalid artifact JSON") from exc + tag = destination.rsplit(":", 1)[1] + matching = [ + item + for item in artifact.get("tags") or [] + if isinstance(item, dict) and item.get("name") == tag + ] + labels = ((artifact.get("extra_attrs") or {}).get("config") or {}).get("Labels") + if artifact.get("digest") != digest: + raise RuntimeError("Harbor digest does not match Kaniko evidence") + if len(matching) != 1 or matching[0].get("immutable") is not True: + raise RuntimeError("Harbor did not enforce the candidate tag as immutable") + if not isinstance(labels, dict) or labels.get( + "org.opencontainers.image.revision" + ) != revision: + raise RuntimeError("Harbor OCI source-revision label does not match") + + +def render_workload(source: str, digest: str) -> str: + """Replace the single exact router image while preserving the Flux marker.""" + digest = _validated(digest, DIGEST_PATTERN, "image digest") + identity = "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: hermes-chat-router\n" + if not source.startswith("# services/hermes/chat-router.yaml\n" + identity): + raise ValueError("Flux target identity changed") + lines = source.splitlines(keepends=True) + matches: list[int] = [] + for index, line in enumerate(lines): + stripped = line.strip() + if not stripped.startswith("image: "): + continue + value = stripped.removeprefix("image: ").split(" #", 1)[0] + image, separator, current_digest = value.rpartition("@") + if separator and re.fullmatch( + rf"{re.escape(DEFAULT_IMAGE)}(?::[A-Za-z0-9_][A-Za-z0-9_.-]{{0,127}})?", + image, + ): + _validated(current_digest, DIGEST_PATTERN, "current Flux image digest") + matches.append(index) + if len(matches) != 1: + raise ValueError(f"expected exactly one router image; found {len(matches)}") + index = matches[0] + indent = lines[index][: len(lines[index]) - len(lines[index].lstrip())] + comment = "" + if " #" in lines[index]: + comment = " #" + lines[index].split(" #", 1)[1].rstrip("\n") + newline = "\n" if lines[index].endswith("\n") else "" + lines[index] = f"{indent}image: {DEFAULT_IMAGE}@{digest}{comment}{newline}" + return "".join(lines) + + +def _metadata( + digest: str, revision: str, build: str, destination: str +) -> dict[str, Any]: + return { + "build_number": build, + "digest": digest, + "flux_image": f"{DEFAULT_IMAGE}@{digest}", + "flux_targets": ["apps/Deployment/hermes/hermes-chat-router"], + "image": DEFAULT_IMAGE, + "published_tag": destination, + "source_revision": revision, + } + + +def _expected_artifacts( + *, + digest: str, + source_revision: str, + build_number: str, + destination: str, + manifest: Path, +) -> dict[str, str]: + source = manifest.read_text(encoding="utf-8") + rendered = render_workload(source, digest) + patch = "".join( + difflib.unified_diff( + source.splitlines(keepends=True), + rendered.splitlines(keepends=True), + fromfile="a/services/hermes/chat-router.yaml", + tofile="b/services/hermes/chat-router.yaml", + ) + ) + if not patch: + raise ValueError("published digest already matches the Flux target") + metadata = _metadata(digest, source_revision, build_number, destination) + return { + "hermes-chat-router-deployment.yaml": rendered, + "hermes-chat-router-image-update.patch": patch, + "hermes-chat-router-image.json": json.dumps( + metadata, indent=2, sort_keys=True + ) + + "\n", + } + + +def write_release_artifacts( + *, + digest: str, + source_revision: str, + build_number: str, + destination: str, + manifest: Path, + output_dir: Path, +) -> None: + """Write deterministic, credential-free Flux handoff evidence.""" + digest = _validated(digest, DIGEST_PATTERN, "image digest") + revision, build = validate_destination(destination, source_revision, build_number) + expected = _expected_artifacts( + digest=digest, + source_revision=revision, + build_number=build, + destination=destination, + manifest=manifest, + ) + output_dir.mkdir(parents=True, exist_ok=True) + for name, content in expected.items(): + (output_dir / name).write_text(content, encoding="utf-8") + + +def validate_release_artifacts( + *, + digest: str, + source_revision: str, + build_number: str, + destination: str, + manifest: Path, + output_dir: Path, +) -> None: + """Recompute and compare every archived handoff byte.""" + digest = _validated(digest, DIGEST_PATTERN, "image digest") + revision, build = validate_destination(destination, source_revision, build_number) + expected = _expected_artifacts( + digest=digest, + source_revision=revision, + build_number=build, + destination=destination, + manifest=manifest, + ) + entries = list(output_dir.iterdir()) + if {entry.name for entry in entries} != set(expected) or not all( + entry.is_file() and not entry.is_symlink() for entry in entries + ): + raise ValueError("release output must contain exactly three evidence files") + for name, content in expected.items(): + if (output_dir / name).read_text(encoding="utf-8") != content: + raise ValueError(f"release evidence is incomplete or mismatched: {name}") + + +def _credentials() -> tuple[str, str]: + username = os.environ.get("HARBOR_USER", "") + password = os.environ.get("HARBOR_PASSWORD", "") + if not username or not password: + raise RuntimeError("Harbor credentials are unavailable") + return username, password + + +def _add_common(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--source-revision", required=True) + parser.add_argument("--build-number", required=True) + parser.add_argument("--destination", required=True) + + +def main() -> int: + """Run one fail-closed candidate or evidence operation.""" + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + absent = commands.add_parser("assert-absent") + _add_common(absent) + for name in ("render", "verify-evidence"): + command = commands.add_parser(name) + _add_common(command) + command.add_argument("--digest-file", required=True, type=Path) + command.add_argument("--image-file", required=True, type=Path) + command.add_argument("--manifest", required=True, type=Path) + command.add_argument("--output-dir", required=True, type=Path) + args = parser.parse_args() + validate_destination(args.destination, args.source_revision, args.build_number) + if args.command == "verify-evidence": + digest = validate_kaniko_evidence( + digest_text=args.digest_file.read_text(encoding="utf-8"), + image_text=args.image_file.read_text(encoding="utf-8"), + destination=args.destination, + ) + validate_release_artifacts( + digest=digest, + source_revision=args.source_revision, + build_number=args.build_number, + destination=args.destination, + manifest=args.manifest, + output_dir=args.output_dir, + ) + return 0 + username, password = _credentials() + if args.command == "assert-absent": + verify_immutable_policy(username=username, password=password) + assert_tag_absent( + args.destination, username=username, password=password + ) + return 0 + digest = validate_kaniko_evidence( + digest_text=args.digest_file.read_text(encoding="utf-8"), + image_text=args.image_file.read_text(encoding="utf-8"), + destination=args.destination, + ) + verify_registry_digest( + args.destination, + digest, + args.source_revision, + username=username, + password=password, + ) + write_release_artifacts( + digest=digest, + source_revision=args.source_revision, + build_number=args.build_number, + destination=args.destination, + manifest=args.manifest, + output_dir=args.output_dir, + ) + return 0 + + +if __name__ == "__main__": # pragma: no cover - exercised through main() + raise SystemExit(main()) diff --git a/ci/scripts/hermes_oci_promote.py b/ci/scripts/hermes_oci_promote.py index 7b384bbf..9d19809e 100644 --- a/ci/scripts/hermes_oci_promote.py +++ b/ci/scripts/hermes_oci_promote.py @@ -18,7 +18,7 @@ from typing import Any, Callable REGISTRY_ORIGIN = "https://registry.bstein.dev" DESTINATION_PATTERN = re.compile( r"^registry\.bstein\.dev/bstein/" - r"(?Phermes-(?:agent|webui|jetson-(?:stt|tts))):" + r"(?Phermes-(?:agent|webui|chat-router|jetson-(?:stt|tts))):" r"git-(?P[0-9a-f]{40})-build-(?P[1-9][0-9]*)$" ) DIGEST_PATTERN = re.compile(r"^sha256:[0-9a-f]{64}$") diff --git a/clusters/atlas/flux-system/applications/harbor/kustomization.yaml b/clusters/atlas/flux-system/applications/harbor/kustomization.yaml index a1782f58..d8a283e4 100644 --- a/clusters/atlas/flux-system/applications/harbor/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/harbor/kustomization.yaml @@ -26,6 +26,10 @@ spec: kind: Job name: harbor-hermes-webui-immutability-ensure-1 namespace: harbor + - apiVersion: batch/v1 + kind: Job + name: harbor-hermes-chat-router-immutability-ensure-1 + namespace: harbor dependsOn: - name: core - name: longhorn diff --git a/services/harbor/hermes-chat-router-immutability-job.yaml b/services/harbor/hermes-chat-router-immutability-job.yaml new file mode 100644 index 00000000..3c6996dd --- /dev/null +++ b/services/harbor/hermes-chat-router-immutability-job.yaml @@ -0,0 +1,79 @@ +# services/harbor/hermes-chat-router-immutability-job.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: harbor-hermes-chat-router-immutability-ensure-1 + namespace: harbor +spec: + backoffLimit: 2 + activeDeadlineSeconds: 600 + template: + metadata: + annotations: + vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/agent-pre-populate-only: "true" + vault.hashicorp.com/agent-run-as-user: "65532" + vault.hashicorp.com/agent-run-as-group: "65532" + vault.hashicorp.com/role: harbor-policy-bootstrap + vault.hashicorp.com/agent-inject-secret-harbor-admin-password: kv/data/atlas/harbor/harbor-core + vault.hashicorp.com/agent-inject-template-harbor-admin-password: | + {{- with secret "kv/data/atlas/harbor/harbor-core" -}} + {{ .Data.data.harbor_admin_password }} + {{- end -}} + spec: + serviceAccountName: harbor-policy-bootstrap + enableServiceLinks: false + restartPolicy: Never + nodeSelector: + hardware: rpi5 + kubernetes.io/arch: arm64 + node-role.kubernetes.io/worker: "true" + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: NotIn + values: [titan-04, titan-14, titan-18, titan-19, titan-24] + securityContext: + fsGroup: 65532 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: + type: RuntimeDefault + containers: + - name: ensure + image: docker.io/library/python@sha256:efcdfa6a6b2fd2afb9c7dfa9a5b288a6f68338b5cfdebe6b637d986067d85757 + imagePullPolicy: IfNotPresent + command: [python3, /scripts/harbor_hermes_chat_router_immutability_ensure.py] + env: + - name: HARBOR_API_ORIGIN + value: https://registry.bstein.dev/api/v2.0 + - name: HARBOR_ADMIN_PASSWORD_FILE + value: /vault/secrets/harbor-admin-password + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + readOnlyRootFilesystem: true + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 + seccompProfile: + type: RuntimeDefault + volumeMounts: + - name: scripts + mountPath: /scripts + readOnly: true + - name: tmp + mountPath: /tmp + resources: + requests: {cpu: 25m, memory: 32Mi} + limits: {cpu: 250m, memory: 128Mi} + volumes: + - name: scripts + configMap: + name: harbor-hermes-chat-router-immutability-script + defaultMode: 0555 + - name: tmp + emptyDir: {} diff --git a/services/harbor/kustomization.yaml b/services/harbor/kustomization.yaml index 750e808d..b221025f 100644 --- a/services/harbor/kustomization.yaml +++ b/services/harbor/kustomization.yaml @@ -15,6 +15,7 @@ resources: - policy-bootstrap-serviceaccount.yaml - hermes-agent-immutability-job.yaml - hermes-webui-immutability-job.yaml + - hermes-chat-router-immutability-job.yaml - bootstrap-jobs/cassandra-registry-ensure-job.yaml - image.yaml configMapGenerator: @@ -27,3 +28,7 @@ configMapGenerator: - name: harbor-hermes-webui-immutability-script files: - harbor_hermes_webui_immutability_ensure.py=scripts/harbor_hermes_webui_immutability_ensure.py + - name: harbor-hermes-chat-router-immutability-script + files: + - harbor_immutable_rule_ensure.py=scripts/harbor_immutable_rule_ensure.py + - harbor_hermes_chat_router_immutability_ensure.py=scripts/harbor_hermes_chat_router_immutability_ensure.py diff --git a/services/harbor/scripts/harbor_hermes_chat_router_immutability_ensure.py b/services/harbor/scripts/harbor_hermes_chat_router_immutability_ensure.py new file mode 100644 index 00000000..d3032c2e --- /dev/null +++ b/services/harbor/scripts/harbor_hermes_chat_router_immutability_ensure.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +"""Create and verify the Hermes chat-router immutable build-tag rule.""" + +from __future__ import annotations + +import os +import time +from pathlib import Path + +from harbor_immutable_rule_ensure import HarborClient, HarborUnavailable, ensure_rule + + +PROJECT = "bstein" +REPOSITORY_PATTERN = "hermes-chat-router" +TAG_PATTERN = "git-*-build-*" + + +def main() -> int: + """Load the runtime-only admin credential and enforce tracked policy.""" + origin = os.environ.get("HARBOR_API_ORIGIN", "") + password_file = Path(os.environ.get("HARBOR_ADMIN_PASSWORD_FILE", "")) + password = password_file.read_text(encoding="utf-8").strip() + client = HarborClient(origin, "admin", password) + for attempt in range(1, 13): + try: + rule_id = ensure_rule( + client, + project=PROJECT, + repository=REPOSITORY_PATTERN, + tag_pattern=TAG_PATTERN, + ) + break + except HarborUnavailable: + if attempt == 12: + raise + time.sleep(min(attempt * 2, 15)) + print(f"Hermes chat-router immutable build-tag rule is active (id={rule_id})") + return 0 + + +if __name__ == "__main__": # pragma: no cover - exercised through main() + raise SystemExit(main()) diff --git a/services/harbor/scripts/harbor_immutable_rule_ensure.py b/services/harbor/scripts/harbor_immutable_rule_ensure.py new file mode 100644 index 00000000..7ae497d4 --- /dev/null +++ b/services/harbor/scripts/harbor_immutable_rule_ensure.py @@ -0,0 +1,184 @@ +"""Bounded helpers for enforcing one exact Harbor immutable-tag rule.""" + +from __future__ import annotations + +import base64 +import json +import time +import urllib.error +import urllib.parse +import urllib.request +from typing import Any + + +EXPECTED_ORIGIN = "https://registry.bstein.dev/api/v2.0" +MAX_RESPONSE = 1_048_576 +TRANSIENT_STATUSES = {429, 502, 503, 504} + + +class NoRedirect(urllib.request.HTTPRedirectHandler): + """Prevent Basic credentials from following an unexpected redirect.""" + + def redirect_request(self, _request, _file, _code, _message, _headers, _url): + return None + + +class HarborUnavailable(RuntimeError): + """Harbor is temporarily unavailable rather than denying policy.""" + + +def expected_rule(repository: str, tag_pattern: str) -> dict[str, Any]: + """Build the exact enabled immutable rule for one literal repository.""" + return { + "disabled": False, + "action": "immutable", + "template": "immutable_template", + "tag_selectors": [ + { + "kind": "doublestar", + "decoration": "matches", + "pattern": tag_pattern, + } + ], + "scope_selectors": { + "repository": [ + { + "kind": "doublestar", + "decoration": "repoMatches", + "pattern": repository, + } + ] + }, + } + + +def normalized_rule(rule: dict[str, Any]) -> dict[str, Any]: + """Return only the immutable contract fields Harbor must preserve.""" + return { + "disabled": bool(rule.get("disabled", False)), + "action": rule.get("action"), + "template": rule.get("template"), + "tag_selectors": [ + {key: item.get(key) for key in ("kind", "decoration", "pattern")} + for item in rule.get("tag_selectors") or [] + if isinstance(item, dict) + ], + "scope_selectors": { + "repository": [ + {key: item.get(key) for key in ("kind", "decoration", "pattern")} + for item in (rule.get("scope_selectors") or {}).get("repository", []) + if isinstance(item, dict) + ] + }, + } + + +class HarborClient: + """Small same-origin client for Harbor's immutable-tag API.""" + + def __init__(self, origin: str, username: str, password: str) -> None: + if origin.rstrip("/") != EXPECTED_ORIGIN: + raise ValueError("Harbor API origin is not the pinned production API") + if not username or not password: + raise RuntimeError("Harbor admin credential is empty") + token = base64.b64encode(f"{username}:{password}".encode()).decode("ascii") + self.origin = EXPECTED_ORIGIN + self.headers = {"Authorization": f"Basic {token}"} + self.opener = urllib.request.build_opener(NoRedirect()) + + def request( + self, method: str, path: str, payload: dict[str, Any] | None = None + ) -> tuple[int, bytes, dict[str, str]]: + """Issue a bounded request and preserve non-2xx responses for checks.""" + data = None + headers = dict(self.headers) + if payload is not None: + data = json.dumps(payload, separators=(",", ":")).encode() + headers["Content-Type"] = "application/json" + request = urllib.request.Request( + f"{self.origin}{path}", data=data, headers=headers, method=method + ) + try: + response = self.opener.open(request, timeout=20) + except urllib.error.HTTPError as exc: + response = exc + except (urllib.error.URLError, TimeoutError) as exc: + raise HarborUnavailable("Harbor policy API is unavailable") from exc + with response: + body = response.read(MAX_RESPONSE + 1) + if len(body) > MAX_RESPONSE: + raise RuntimeError("Harbor response exceeded the size limit") + return int(response.status), body, dict(response.headers) + + +def list_rules(client: HarborClient, project: str) -> list[dict[str, Any]]: + """Read and validate the complete bounded project rule list.""" + path = f"/projects/{project}/immutabletagrules?page=1&page_size=100" + status, body, headers = client.request("GET", path) + if status in TRANSIENT_STATUSES: + raise HarborUnavailable(f"Harbor immutable rule list returned HTTP {status}") + if status != 200: + raise RuntimeError(f"Harbor immutable rule list returned HTTP {status}") + try: + rules = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise RuntimeError("Harbor returned invalid immutable rule JSON") from exc + if not isinstance(rules, list) or not all(isinstance(item, dict) for item in rules): + raise RuntimeError("Harbor immutable rule list has an invalid shape") + total = next( + (value for key, value in headers.items() if key.lower() == "x-total-count"), + None, + ) + if total is None or not str(total).isdecimal() or int(total) != len(rules): + raise RuntimeError("Harbor immutable rule list is incomplete") + return rules + + +def ensure_rule( + client: HarborClient, *, project: str, repository: str, tag_pattern: str +) -> int: + """Create once or verify the one exact enabled repository rule.""" + expected = expected_rule(repository, tag_pattern) + + def matches() -> list[dict[str, Any]]: + return [ + item + for item in list_rules(client, project) + if normalized_rule(item)["tag_selectors"] == expected["tag_selectors"] + and normalized_rule(item)["scope_selectors"] + == expected["scope_selectors"] + ] + + existing = matches() + if len(existing) > 1: + raise RuntimeError("multiple matching immutable rules exist") + if existing: + if normalized_rule(existing[0]) != expected: + raise RuntimeError("matching immutable rule is not enabled and exact") + rule_id = existing[0].get("id") + if not isinstance(rule_id, int) or rule_id < 1: + raise RuntimeError("Harbor immutable rule omitted a valid ID") + return rule_id + path = f"/projects/{project}/immutabletagrules" + status, _body, headers = client.request("POST", path, expected) + if status in TRANSIENT_STATUSES: + raise HarborUnavailable(f"Harbor immutable rule create returned HTTP {status}") + if status != 201: + raise RuntimeError(f"Harbor immutable rule create returned HTTP {status}") + location = headers.get("Location") or headers.get("location") or "" + api_path = urllib.parse.urlsplit(client.origin).path.rstrip("/") + prefix = f"{api_path}{path}/" + suffix = location.removeprefix(prefix) if location.startswith(prefix) else "" + if not suffix.isdecimal() or int(suffix) < 1: + raise RuntimeError("Harbor immutable rule create omitted the exact Location") + for attempt in range(1, 6): + created = matches() + if ( + len(created) == 1 + and normalized_rule(created[0]) == expected + and created[0].get("id") == int(suffix) + ): + return int(suffix) + if attempt < 5: + time.sleep(attempt) + raise RuntimeError("created Harbor immutable rule did not verify exactly") diff --git a/services/hermes/chat-router.yaml b/services/hermes/chat-router.yaml index 980cf43f..9358425c 100644 --- a/services/hermes/chat-router.yaml +++ b/services/hermes/chat-router.yaml @@ -62,7 +62,7 @@ spec: values: [rpi5] containers: - name: router - image: registry.bstein.dev/bstein/hermes-chat-router@sha256:6744cb7b87c6050f1b97c0675cd280b8295b3b5826ba1d6d92e37aee6fd0b8c4 + image: registry.bstein.dev/bstein/hermes-chat-router@sha256:6744cb7b87c6050f1b97c0675cd280b8295b3b5826ba1d6d92e37aee6fd0b8c4 # {"$imagepolicy": "hermes:hermes-chat-router-release"} imagePullPolicy: IfNotPresent ports: - {name: http, containerPort: 8080, protocol: TCP} diff --git a/services/hermes/image.yaml b/services/hermes/image.yaml index 41e09aa4..c2453335 100644 --- a/services/hermes/image.yaml +++ b/services/hermes/image.yaml @@ -53,6 +53,32 @@ spec: --- apiVersion: image.toolkit.fluxcd.io/v1 kind: ImageRepository +metadata: + name: hermes-chat-router-release + namespace: hermes +spec: + image: registry.bstein.dev/bstein/hermes-chat-router + interval: 1m0s +--- +apiVersion: image.toolkit.fluxcd.io/v1 +kind: ImagePolicy +metadata: + name: hermes-chat-router-release + namespace: hermes +spec: + imageRepositoryRef: + name: hermes-chat-router-release + filterTags: + pattern: '^git-[0-9a-f]{40}-build-(?P[1-9][0-9]*)-release$' + extract: '$build' + policy: + numerical: + order: asc + digestReflectionPolicy: Always + interval: 1m0s +--- +apiVersion: image.toolkit.fluxcd.io/v1 +kind: ImageRepository metadata: name: hermes-stt-release namespace: hermes diff --git a/services/hermes/scripts/hermes_image_release_status.py b/services/hermes/scripts/hermes_image_release_status.py index a87cfbae..85a918af 100755 --- a/services/hermes/scripts/hermes_image_release_status.py +++ b/services/hermes/scripts/hermes_image_release_status.py @@ -34,6 +34,13 @@ COMPONENTS = { ("statefulset", "hermes-chat-tenant", "hermes-chat-tenant"), ), }, + "router": { + "policy": "hermes-chat-router-release", + "repository": "registry.bstein.dev/bstein/hermes-chat-router", + "workloads": ( + ("deployment", "hermes-chat-router", "hermes-chat-router"), + ), + }, "stt": { "policy": "hermes-stt-release", "repository": "registry.bstein.dev/bstein/hermes-jetson-stt", @@ -192,7 +199,7 @@ def _workload_status( def inspect_release(component: str, revision: str) -> dict: """Return safe end-to-end release state for one exact reviewed revision.""" if component not in COMPONENTS: - raise ValueError("component must be agent, webui, stt, or tts") + raise ValueError("component must be agent, router, webui, stt, or tts") if not REVISION_PATTERN.fullmatch(revision): raise ValueError("revision must be a lowercase full 40-character commit") config = COMPONENTS[component] diff --git a/services/hermes/scripts/jenkins_image_build_trigger.py b/services/hermes/scripts/jenkins_image_build_trigger.py index b83a13a7..14aefe2d 100755 --- a/services/hermes/scripts/jenkins_image_build_trigger.py +++ b/services/hermes/scripts/jenkins_image_build_trigger.py @@ -27,6 +27,10 @@ JOBS = { "job": "hermes-webui-image", "confirmation": "PUBLISH HERMES WEBUI", }, + "router": { + "job": "hermes-chat-router-image", + "confirmation": "PUBLISH HERMES CHAT ROUTER", + }, "stt": { "job": "hermes-voice-image", "confirmation": "PUBLISH HERMES STT", @@ -108,7 +112,7 @@ def trigger_build( raise ValueError("revision must be a lowercase full 40-character commit") job = JOBS.get(component) if job is None: - raise ValueError("component must be agent, webui, stt, or tts") + raise ValueError("component must be agent, router, webui, stt, or tts") token = token_file.read_text(encoding="utf-8").strip() if not token: raise RuntimeError("Jenkins image-build token is empty") diff --git a/services/jenkins/configmap-jcasc.yaml b/services/jenkins/configmap-jcasc.yaml index af7e139e..2cbb6daa 100644 --- a/services/jenkins/configmap-jcasc.yaml +++ b/services/jenkins/configmap-jcasc.yaml @@ -700,6 +700,30 @@ data: } } } + pipelineJob('hermes-chat-router-image') { + disabled(false) + description('Bounded daemonless Kaniko release for one exact reviewed Hermes chat-router revision already contained by main. Archives exact evidence, then publishes an immutable tag for Flux deployment.') + authenticationToken(System.getenv('HERMES_AGENT_IMAGE_BUILD_TOKEN')) + parameters { + booleanParam('PUBLISH_IMAGE', false, 'Publish the reviewed Hermes chat-router image.') + stringParam('EXPECTED_SOURCE_REVISION', '', 'Full reviewed commit that must be contained by atlas/titan-iac main.') + stringParam('CONFIRM_PUBLISH', '', 'Exact confirmation: PUBLISH HERMES CHAT ROUTER') + } + definition { + cpsScm { + scm { + git { + remote { + url('https://scm.bstein.dev/atlas/titan-iac.git') + credentials('gitea-pat') + } + branches('*/main') + } + } + scriptPath('ci/Jenkinsfile.hermes-chat-router-image') + } + } + } pipelineJob('hermes-voice-image') { disabled(false) description('Bounded daemonless Kaniko release for reviewed Hermes STT or TTS source. A validated immutable release tag is consumed by Flux.') diff --git a/testing/tests/test_hermes_chat_router_release.py b/testing/tests/test_hermes_chat_router_release.py new file mode 100644 index 00000000..d4765c68 --- /dev/null +++ b/testing/tests/test_hermes_chat_router_release.py @@ -0,0 +1,416 @@ +"""Exact-source, Harbor, and Flux contracts for the chat-router release lane.""" + +from __future__ import annotations + +import importlib.util +import io +import json +from pathlib import Path +import sys +import urllib.parse + +import pytest +import yaml + + +ROOT = Path(__file__).resolve().parents[2] +PIPELINE = ROOT / "ci/Jenkinsfile.hermes-chat-router-image" +RELEASE = ROOT / "ci/scripts/hermes_chat_router_release.py" +PROMOTE = ROOT / "ci/scripts/hermes_oci_promote.py" +MANIFEST = ROOT / "services/hermes/chat-router.yaml" +IMAGE_POLICY = ROOT / "services/hermes/image.yaml" +JENKINS = ROOT / "services/jenkins/configmap-jcasc.yaml" +TRIGGER = ROOT / "services/hermes/scripts/jenkins_image_build_trigger.py" +STATUS = ROOT / "services/hermes/scripts/hermes_image_release_status.py" +HARBOR_JOB = ROOT / "services/harbor/hermes-chat-router-immutability-job.yaml" +HARBOR_POLICY = ( + ROOT / "services/harbor/scripts/harbor_hermes_chat_router_immutability_ensure.py" +) +HARBOR_GENERIC = ROOT / "services/harbor/scripts/harbor_immutable_rule_ensure.py" + + +def _load(path: Path, name: str): + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +class Response(io.BytesIO): + """Minimal context-managed urllib response.""" + + def __init__(self, status: int, body: bytes = b"", headers=None): + super().__init__(body) + self.status = status + self.headers = headers or {} + + def __enter__(self): + return self + + def __exit__(self, *_args): + self.close() + + +def _fixture(tmp_path: Path): + module = _load(RELEASE, f"router_release_{tmp_path.name}") + digest = "sha256:" + "7" * 64 + revision = "8" * 40 + build = "23" + destination = f"{module.DEFAULT_IMAGE}:git-{revision}-build-{build}" + digest_file = tmp_path / "digest" + image_file = tmp_path / "image" + digest_file.write_text(digest + "\n", encoding="utf-8") + image_file.write_text(f"{destination}@{digest}\n", encoding="utf-8") + output = tmp_path / "release" + kwargs = { + "digest": digest, + "source_revision": revision, + "build_number": build, + "destination": destination, + "manifest": MANIFEST, + "output_dir": output, + } + module.write_release_artifacts(**kwargs) + return module, digest_file, image_file, kwargs + + +def test_pipeline_builds_one_exact_reviewed_router_revision() -> None: + """The job detaches the reviewed main ancestor and never mutates Git or K8s.""" + source = PIPELINE.read_text(encoding="utf-8") + assert 'test "${CONFIRM_PUBLISH}" = "PUBLISH HERMES CHAT ROUTER"' in source + assert 'test "${main_revision}" = "$(git rev-parse origin/main)"' in source + assert 'git merge-base --is-ancestor "${EXPECTED_SOURCE_REVISION}"' in source + assert 'git checkout --detach "${EXPECTED_SOURCE_REVISION}"' in source + assert 'test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}"' in source + assert "dockerfiles/Dockerfile.hermes-chat-router" in source + assert "GO111MODULE=off CGO_ENABLED=0 go test ./..." in source + assert "ci/scripts/hermes_chat_router_release.py assert-absent" in source + assert "ci/scripts/hermes_oci_promote.py" in source + assert "--digest-file=" in source + assert "--image-name-tag-with-digest-file=" in source + for label in ( + "org.opencontainers.image.revision=${source_revision}", + "org.opencontainers.image.source=https://scm.bstein.dev/atlas/titan-iac", + "org.opencontainers.image.title=hermes-chat-router", + ): + assert label in source + for forbidden in ("kubectl ", "flux reconcile", "git push", "git commit"): + assert forbidden not in source + pod_yaml = source.split('yaml """', 1)[1].split('"""', 1)[0] + pod = yaml.safe_load(pod_yaml)["spec"] + assert pod["serviceAccountName"] == "hermes-image-builder" + assert pod["automountServiceAccountToken"] is False + assert pod["enableServiceLinks"] is False + containers = {item["name"]: item for item in pod["containers"]} + assert set(containers) == {"jnlp", "python", "golang", "kaniko"} + assert pod["affinity"]["nodeAffinity"][ + "requiredDuringSchedulingIgnoredDuringExecution" + ]["nodeSelectorTerms"][0]["matchExpressions"][0]["values"] == ["titan-20"] + assert "docker.sock" not in source and "hostPath" not in source + for container in containers.values(): + security = container["securityContext"] + assert security["allowPrivilegeEscalation"] is False + assert security["capabilities"]["drop"] == ["ALL"] + + +def test_jenkins_job_is_token_guarded_main_only_and_non_concurrent() -> None: + """Only the fixed router job and fixed confirmation can enter the lane.""" + config = yaml.safe_load(JENKINS.read_text(encoding="utf-8")) + jobs = config["data"]["jobs.yaml"] + assert jobs.count("pipelineJob('hermes-chat-router-image')") == 1 + block = jobs.split("pipelineJob('hermes-chat-router-image')", 1)[1].split( + "pipelineJob('hermes-voice-image')", 1 + )[0] + assert "branches('*/main')" in block + assert "scriptPath('ci/Jenkinsfile.hermes-chat-router-image')" in block + assert "authenticationToken(System.getenv('HERMES_AGENT_IMAGE_BUILD_TOKEN'))" in block + assert "PUBLISH HERMES CHAT ROUTER" in block + assert "pipelineTriggers" not in block + pipeline = PIPELINE.read_text(encoding="utf-8") + assert "disableConcurrentBuilds()" in pipeline + assert "artifactDaysToKeepStr: '30'" in pipeline + + +def test_release_renders_exactly_one_flux_consumer_and_revalidates(tmp_path: Path) -> None: + """A release handoff contains only the one router Deployment mutation.""" + module, digest_file, image_file, kwargs = _fixture(tmp_path) + output = kwargs["output_dir"] + patch = (output / "hermes-chat-router-image-update.patch").read_text() + assert patch.count(f"+ image: {module.DEFAULT_IMAGE}@{kwargs['digest']}") == 1 + assert "services/hermes/chat-router.yaml" in patch + assert "chat-statefulset.yaml" not in patch + metadata = json.loads((output / "hermes-chat-router-image.json").read_text()) + assert metadata["source_revision"] == kwargs["source_revision"] + assert metadata["flux_targets"] == [ + "apps/Deployment/hermes/hermes-chat-router" + ] + module.validate_release_artifacts(**kwargs) + assert module.validate_kaniko_evidence( + digest_text=digest_file.read_text(), + image_text=image_file.read_text(), + destination=kwargs["destination"], + ) == kwargs["digest"] + (output / "extra").write_text("no\n") + with pytest.raises(ValueError, match="exactly three"): + module.validate_release_artifacts(**kwargs) + + +@pytest.mark.parametrize( + "source", + [ + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: wrong\n", + "# services/hermes/chat-router.yaml\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: hermes-chat-router\n", + ], +) +def test_renderer_fails_closed_on_target_drift(source: str) -> None: + """Identity or consumer-count drift cannot silently broaden promotion.""" + module = _load(RELEASE, f"router_drift_{len(source)}") + with pytest.raises(ValueError): + module.render_workload(source, "sha256:" + "a" * 64) + + +def test_harbor_candidate_evidence_and_policy_are_exact() -> None: + """Candidate acceptance requires an immutable tag, digest, and OCI revision.""" + module = _load(RELEASE, "router_registry_contract") + revision = "a" * 40 + digest = "sha256:" + "b" * 64 + destination = f"{module.DEFAULT_IMAGE}:git-{revision}-build-9" + seen = [] + + def artifact_open(request, timeout): + seen.append((request, timeout)) + return Response( + 200, + json.dumps( + { + "digest": digest, + "tags": [ + {"name": f"git-{revision}-build-9", "immutable": True} + ], + "extra_attrs": { + "config": { + "Labels": { + "org.opencontainers.image.revision": revision + } + } + }, + } + ).encode(), + ) + + module.verify_registry_digest( + destination, + digest, + revision, + username="robot", + password="private", + opener=artifact_open, + ) + assert "/repositories/hermes-chat-router/artifacts/" in seen[0][0].full_url + assert seen[0][0].get_header("Authorization").startswith("Basic ") + + def rule_open(_request, _timeout): + expected = { + "disabled": False, + "action": "immutable", + "template": "immutable_template", + "tag_selectors": [ + { + "kind": "doublestar", + "decoration": "matches", + "pattern": "git-*-build-*", + } + ], + "scope_selectors": { + "repository": [ + { + "kind": "doublestar", + "decoration": "repoMatches", + "pattern": "hermes-chat-router", + } + ] + }, + } + return Response( + 200, json.dumps([expected]).encode(), {"X-Total-Count": "1"} + ) + + module.verify_immutable_policy( + username="robot", password="private", opener=rule_open + ) + assert module.assert_tag_absent( + destination, + username="robot", + password="private", + opener=lambda *_args: Response(404), + ) is None + with pytest.raises(RuntimeError, match="already exists"): + module.assert_tag_absent( + destination, + username="robot", + password="private", + opener=lambda *_args: Response(200), + ) + + +def test_harbor_policy_is_flux_tracked_and_vault_injected() -> None: + """The preflight rule is desired state and no credential is stored in Git.""" + app = yaml.safe_load( + (ROOT / "clusters/atlas/flux-system/applications/harbor/kustomization.yaml") + .read_text(encoding="utf-8") + ) + assert { + "apiVersion": "batch/v1", + "kind": "Job", + "name": "harbor-hermes-chat-router-immutability-ensure-1", + "namespace": "harbor", + } in app["spec"]["healthChecks"] + _load(HARBOR_GENERIC, "harbor_immutable_rule_ensure") + policy = _load(HARBOR_POLICY, "router_harbor_policy") + assert policy.REPOSITORY_PATTERN == "hermes-chat-router" + assert policy.TAG_PATTERN == "git-*-build-*" + job = yaml.safe_load(HARBOR_JOB.read_text(encoding="utf-8")) + template = job["spec"]["template"] + annotations = template["metadata"]["annotations"] + assert annotations["vault.hashicorp.com/role"] == "harbor-policy-bootstrap" + assert annotations[ + "vault.hashicorp.com/agent-inject-secret-harbor-admin-password" + ] == "kv/data/atlas/harbor/harbor-core" + container = template["spec"]["containers"][0] + assert "@sha256:" in container["image"] + assert container["securityContext"]["readOnlyRootFilesystem"] is True + assert container["securityContext"]["runAsNonRoot"] is True + source = HARBOR_POLICY.read_text().lower() + HARBOR_GENERIC.read_text().lower() + assert "private-token" not in source and "password123" not in source + + +def test_harbor_policy_helper_creates_once_and_rejects_weakened_rule() -> None: + """Creation is verified by ID; a disabled matching rule fails closed.""" + module = _load(HARBOR_GENERIC, "router_harbor_helper") + expected = module.expected_rule("hermes-chat-router", "git-*-build-*") + created = {**expected, "id": 19} + + class Client: + origin = module.EXPECTED_ORIGIN + + def __init__(self): + self.calls = 0 + + def request(self, method, path, payload=None): + self.calls += 1 + if self.calls == 1: + return 200, b"[]", {"X-Total-Count": "0"} + if self.calls == 2: + assert method == "POST" and payload == expected + return 201, b"", {"Location": "/api/v2.0" + path + "/19"} + return 200, json.dumps([created]).encode(), {"X-Total-Count": "1"} + + client = Client() + assert module.ensure_rule( + client, + project="bstein", + repository="hermes-chat-router", + tag_pattern="git-*-build-*", + ) == 19 + weakened = {**created, "disabled": True} + + class Weakened: + origin = module.EXPECTED_ORIGIN + + def request(self, *_args, **_kwargs): + return 200, json.dumps([weakened]).encode(), {"X-Total-Count": "1"} + + with pytest.raises(RuntimeError, match="not enabled and exact"): + module.ensure_rule( + Weakened(), + project="bstein", + repository="hermes-chat-router", + tag_pattern="git-*-build-*", + ) + + +def test_flux_policy_and_marker_select_only_router_releases() -> None: + """Candidates remain invisible until the exact release suffix exists.""" + documents = list(yaml.safe_load_all(IMAGE_POLICY.read_text(encoding="utf-8"))) + repository = next( + item + for item in documents + if item["kind"] == "ImageRepository" + and item["metadata"]["name"] == "hermes-chat-router-release" + ) + policy = next( + item + for item in documents + if item["kind"] == "ImagePolicy" + and item["metadata"]["name"] == "hermes-chat-router-release" + ) + assert repository["spec"]["image"] == ( + "registry.bstein.dev/bstein/hermes-chat-router" + ) + assert policy["spec"]["filterTags"]["pattern"].endswith("-release$") + assert policy["spec"]["digestReflectionPolicy"] == "Always" + marker = '"$imagepolicy": "hermes:hermes-chat-router-release"' + assert MANIFEST.read_text(encoding="utf-8").count(marker) == 1 + + +def test_runtime_trigger_and_status_support_only_fixed_router_contract( + tmp_path: Path, +) -> None: + """Hermes can request and follow router delivery without choosing a job.""" + trigger = _load(TRIGGER, "router_release_trigger") + token = tmp_path / "token" + token.write_text("private-token\n", encoding="utf-8") + captured = {} + + def opener(request, timeout): + captured["request"] = request + assert timeout == 20 + return Response( + 201, headers={"Location": "https://ci.bstein.dev/queue/item/42/"} + ) + + revision = "c" * 40 + result = trigger.trigger_build( + revision, component="router", token_file=token, opener=opener + ) + fields = urllib.parse.parse_qs(captured["request"].data.decode()) + assert fields["job"] == ["hermes-chat-router-image"] + assert fields["CONFIRM_PUBLISH"] == ["PUBLISH HERMES CHAT ROUTER"] + assert "private-token" not in json.dumps(result) + assert result["follow_command"].endswith( + f"--component router --revision {revision} --wait" + ) + status = _load(STATUS, "router_release_status") + assert status.COMPONENTS["router"] == { + "policy": "hermes-chat-router-release", + "repository": "registry.bstein.dev/bstein/hermes-chat-router", + "workloads": ( + ("deployment", "hermes-chat-router", "hermes-chat-router"), + ), + } + + +def test_generic_promoter_accepts_router_and_rejects_other_repositories() -> None: + """The manifest copier allow-list adds only chat-router.""" + module = _load(PROMOTE, "router_promote_contract") + revision = "d" * 40 + digest = "sha256:" + "e" * 64 + destination = ( + "registry.bstein.dev/bstein/hermes-chat-router:" + f"git-{revision}-build-7" + ) + component, tag, observed = module._validated_release( + destination, digest, revision, "7" + ) + assert component == "hermes-chat-router" + assert tag == f"git-{revision}-build-7" + assert observed == digest + with pytest.raises(ValueError): + module._validated_release( + "registry.bstein.dev/bstein/not-router:git-" + revision + "-build-7", + digest, + revision, + "7", + ) diff --git a/testing/tests/test_hermes_image_automation.py b/testing/tests/test_hermes_image_automation.py index ab6d94f9..7cd5d978 100644 --- a/testing/tests/test_hermes_image_automation.py +++ b/testing/tests/test_hermes_image_automation.py @@ -29,6 +29,7 @@ def test_image_policies_observe_only_validated_release_tags() -> None: } assert set(repositories) == { "hermes-agent-release", + "hermes-chat-router-release", "hermes-webui-release", "hermes-stt-release", "hermes-tts-release", @@ -58,6 +59,7 @@ def test_flux_updates_only_the_reviewed_hermes_image_digests() -> None: chat = (SERVICE / "chat-statefulset.yaml").read_text(encoding="utf-8") dashboard = (SERVICE / "deployment.yaml").read_text(encoding="utf-8") voice = (SERVICE / "voice-deployment.yaml").read_text(encoding="utf-8") + router = (SERVICE / "chat-router.yaml").read_text(encoding="utf-8") assert " - image.yaml" in service_kustomization assert " - hermes/image-automation.yaml" in applications_kustomization @@ -69,7 +71,7 @@ def test_flux_updates_only_the_reviewed_hermes_image_digests() -> None: } assert agent.count('"$imagepolicy": "hermes:hermes-agent-release:digest"') == 1 webui_marker = '"$imagepolicy": "hermes:hermes-webui-release"' - assert chat.count(webui_marker) == 1 + assert chat.count(webui_marker) == 2 assert dashboard.count(webui_marker) == 1 # A digest-only setter replaces the complete YAML scalar with ``sha256:...``. # Whole-image setters must retain the registry and repository in pod specs. @@ -85,3 +87,8 @@ def test_flux_updates_only_the_reviewed_hermes_image_digests() -> None: marked_line = next(line for line in voice.splitlines() if marker in line) assert f"registry.bstein.dev/bstein/hermes-jetson-{component}" in marked_line assert "@sha256:" in marked_line + router_marker = '"$imagepolicy": "hermes:hermes-chat-router-release"' + assert router.count(router_marker) == 1 + marked_line = next(line for line in router.splitlines() if router_marker in line) + assert "registry.bstein.dev/bstein/hermes-chat-router" in marked_line + assert "@sha256:" in marked_line diff --git a/testing/tests/test_hermes_image_builder.py b/testing/tests/test_hermes_image_builder.py index 9c6fdd92..248f1ba2 100644 --- a/testing/tests/test_hermes_image_builder.py +++ b/testing/tests/test_hermes_image_builder.py @@ -273,7 +273,7 @@ def test_agent_trigger_can_select_only_the_bounded_webui_job(tmp_path: Path) -> assert fields["job"] == ["hermes-webui-image"] assert fields["CONFIRM_PUBLISH"] == ["PUBLISH HERMES WEBUI"] assert result["component"] == "webui" - with pytest.raises(ValueError, match="agent, webui, stt, or tts"): + with pytest.raises(ValueError, match="agent, router, webui, stt, or tts"): module.trigger_build( "b" * 40, component="other", token_file=token_path, opener=opener ) diff --git a/testing/tests/test_hermes_release_followthrough.py b/testing/tests/test_hermes_release_followthrough.py index 333180eb..fad75111 100644 --- a/testing/tests/test_hermes_release_followthrough.py +++ b/testing/tests/test_hermes_release_followthrough.py @@ -138,7 +138,7 @@ def _install_fake(module, monkeypatch: pytest.MonkeyPatch, objects: dict) -> Non monkeypatch.setattr(module, "_kubectl_json", fake) -@pytest.mark.parametrize("component", ["agent", "webui", "stt", "tts"]) +@pytest.mark.parametrize("component", ["agent", "router", "webui", "stt", "tts"]) def test_exact_release_convergence_covers_every_component( monkeypatch: pytest.MonkeyPatch, component: str ) -> None: