diff --git a/ci/Jenkinsfile.hermes-agent-image b/ci/Jenkinsfile.hermes-agent-image new file mode 100644 index 00000000..2d350ac7 --- /dev/null +++ b/ci/Jenkinsfile.hermes-agent-image @@ -0,0 +1,288 @@ +pipeline { + agent { + kubernetes { + defaultContainer 'python' + yaml """ +apiVersion: v1 +kind: Pod +metadata: + labels: + atlas.bstein.dev/workload: hermes-agent-image-builder +spec: + serviceAccountName: hermes-image-builder + automountServiceAccountToken: false + enableServiceLinks: false + restartPolicy: Never + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + nodeSelector: + kubernetes.io/arch: arm64 + node-role.kubernetes.io/worker: "true" + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: NotIn + values: + - titan-04 + - titan-14 + - titan-18 + - titan-19 + - titan-22 + - titan-24 + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: hardware + operator: In + values: + - rpi5 + imagePullSecrets: + - name: harbor-bstein-robot + containers: + - name: jnlp + image: jenkins/inbound-agent@sha256:8eda4fe2a66bcf6a5e43436d9918fc14c306204dc8fcd75f4e15e0e6e5dc759a + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + runAsNonRoot: true + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault + resources: + requests: + cpu: 25m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + - name: python + image: registry.bstein.dev/bstein/python@sha256:269541d3387baae008df4608ead893dba2b5cdaad1a5a380731a88992d34b808 + command: ["sleep"] + args: ["99d"] + tty: true + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + runAsNonRoot: true + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault + resources: + requests: + cpu: 25m + memory: 64Mi + limits: + cpu: 250m + memory: 256Mi + - name: kaniko + image: gcr.io/kaniko-project/executor@sha256:c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e + command: ["/busybox/sh", "-c"] + args: ["/busybox/sleep 99d"] + tty: true + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"] + privileged: false + runAsUser: 0 + seccompProfile: + type: RuntimeDefault + resources: + requests: + cpu: 250m + memory: 1Gi + limits: + cpu: "2" + memory: 4Gi +""" + } + } + parameters { + booleanParam( + name: 'PUBLISH_IMAGE', + defaultValue: false, + description: 'Publish the reviewed main revision to Harbor.' + ) + string( + name: 'EXPECTED_SOURCE_REVISION', + defaultValue: '', + description: 'Exact 40-character commit on atlas/titan-iac main.' + ) + string( + name: 'CONFIRM_PUBLISH', + defaultValue: '', + description: 'Enter PUBLISH HERMES AGENT to confirm the release.' + ) + } + environment { + HERMES_IMAGE = 'registry.bstein.dev/bstein/hermes-agent' + } + options { + disableConcurrentBuilds() + buildDiscarder(logRotator(daysToKeepStr: '30', numToKeepStr: '100', artifactDaysToKeepStr: '30', artifactNumToKeepStr: '100')) + skipDefaultCheckout(true) + timeout(time: 90, unit: 'MINUTES') + } + stages { + stage('Checkout reviewed source') { + steps { + checkout scm + } + } + stage('Enforce release boundary') { + steps { + sh ''' + set -eu + mkdir -p build + test "${PUBLISH_IMAGE}" = "true" + test "${CONFIRM_PUBLISH}" = "PUBLISH HERMES AGENT" + case "${EXPECTED_SOURCE_REVISION}" in + *[!0-9a-f]*|'') + echo "EXPECTED_SOURCE_REVISION must be a lowercase full commit" >&2 + exit 2 + ;; + esac + test "${#EXPECTED_SOURCE_REVISION}" -eq 40 + actual_revision="$(git rev-parse HEAD)" + test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}" + test "${actual_revision}" = "$(git rev-parse origin/main)" + test -z "$(git status --porcelain)" + test -f dockerfiles/Dockerfile.hermes-agent + case "${BUILD_NUMBER}" in + ''|0*|*[!0-9]*) + echo "BUILD_NUMBER must be a positive decimal integer" >&2 + exit 2 + ;; + esac + printf '%s\n' \ + "${HERMES_IMAGE}:git-${actual_revision}-build-${BUILD_NUMBER}" \ + > build/hermes-agent.destination + ''' + } + } + stage('Reject replay before publish') { + steps { + withCredentials([usernamePassword( + credentialsId: 'harbor-robot', + usernameVariable: 'HARBOR_USER', + passwordVariable: 'HARBOR_PASSWORD' + )]) { + sh ''' + set -eu + set +x + destination="$(cat build/hermes-agent.destination)" + python3 ci/scripts/hermes_image_release.py assert-absent \ + --source-revision "${EXPECTED_SOURCE_REVISION}" \ + --build-number "${BUILD_NUMBER}" \ + --destination "${destination}" + ''' + } + } + } + stage('Build and publish without a daemon') { + steps { + container('kaniko') { + withCredentials([usernamePassword( + credentialsId: 'harbor-robot', + usernameVariable: 'HARBOR_USER', + passwordVariable: 'HARBOR_PASSWORD' + )]) { + sh '''#!/busybox/sh + set -eu + set +x + config_path=/kaniko/.docker/config.json + destination="$(cat build/hermes-agent.destination)" + umask 077 + auth="$(printf '%s:%s' "${HARBOR_USER}" "${HARBOR_PASSWORD}" | /busybox/base64 | /busybox/tr -d '\n')" + /busybox/mkdir -p /kaniko/.docker + /busybox/printf '{"auths":{"registry.bstein.dev":{"auth":"%s"}}}\n' "${auth}" > "${config_path}" + unset HARBOR_USER HARBOR_PASSWORD auth + trap '/busybox/rm -f "${config_path}"' EXIT HUP INT TERM + /kaniko/executor \ + --context="dir://${WORKSPACE}" \ + --dockerfile="${WORKSPACE}/dockerfiles/Dockerfile.hermes-agent" \ + --destination="${destination}" \ + --digest-file="${WORKSPACE}/build/hermes-agent.digest" \ + --image-name-tag-with-digest-file="${WORKSPACE}/build/hermes-agent.image" \ + --build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1 \ + --label="org.opencontainers.image.revision=${EXPECTED_SOURCE_REVISION}" \ + --cleanup \ + --push-retry=3 + ''' + } + } + } + } + stage('Render reviewed Flux handoff') { + steps { + withCredentials([usernamePassword( + credentialsId: 'harbor-robot', + usernameVariable: 'HARBOR_USER', + passwordVariable: 'HARBOR_PASSWORD' + )]) { + sh ''' + set -eu + set +x + destination="$(cat build/hermes-agent.destination)" + python3 ci/scripts/hermes_image_release.py render \ + --digest-file build/hermes-agent.digest \ + --image-file build/hermes-agent.image \ + --source-revision "${EXPECTED_SOURCE_REVISION}" \ + --build-number "${BUILD_NUMBER}" \ + --destination "${destination}" \ + --kustomization services/hermes/kustomization.yaml \ + --output-dir build/hermes-agent-release + test -s build/hermes-agent-release/hermes-image-update.patch + test -s build/hermes-agent-release/hermes-agent-image.json + ''' + } + } + } + } + post { + success { + sh ''' + set -eu + expected_files="$(printf '%s\n' \ + build/hermes-agent.destination \ + build/hermes-agent.digest \ + build/hermes-agent.image \ + build/hermes-agent-release/hermes-agent-image.json \ + build/hermes-agent-release/hermes-image-update.patch \ + build/hermes-agent-release/hermes-kustomization.yaml \ + | LC_ALL=C sort)" + actual_files="$(find build -type f -print | LC_ALL=C sort)" + test "${actual_files}" = "${expected_files}" + destination="$(cat build/hermes-agent.destination)" + python3 ci/scripts/hermes_image_release.py verify-evidence \ + --digest-file build/hermes-agent.digest \ + --image-file build/hermes-agent.image \ + --source-revision "${EXPECTED_SOURCE_REVISION}" \ + --build-number "${BUILD_NUMBER}" \ + --destination "${destination}" \ + --kustomization services/hermes/kustomization.yaml \ + --output-dir build/hermes-agent-release + ''' + archiveArtifacts( + artifacts: 'build/hermes-agent.destination,build/hermes-agent.digest,build/hermes-agent.image,build/hermes-agent-release/hermes-agent-image.json,build/hermes-agent-release/hermes-image-update.patch,build/hermes-agent-release/hermes-kustomization.yaml', + allowEmptyArchive: false, + fingerprint: true + ) + } + cleanup { + container('kaniko') { + sh '''#!/busybox/sh + /busybox/rm -f /kaniko/.docker/config.json + ''' + } + } + } +} diff --git a/ci/scripts/hermes_image_release.py b/ci/scripts/hermes_image_release.py new file mode 100755 index 00000000..2f0f4166 --- /dev/null +++ b/ci/scripts/hermes_image_release.py @@ -0,0 +1,500 @@ +#!/usr/bin/env python3 +"""Verify and render a reviewable Hermes agent image release.""" + +from __future__ import annotations + +import argparse +import base64 +import difflib +import json +import os +import re +import urllib.error +import urllib.parse +import urllib.request +from pathlib import Path +from typing import Any, Callable + + +DEFAULT_IMAGE = "registry.bstein.dev/bstein/hermes-agent" +HARBOR_API_ORIGIN = "https://registry.bstein.dev/api/v2.0" +HARBOR_PROJECT = "bstein" +HARBOR_REPOSITORY = "hermes-agent" +IMMUTABLE_REPOSITORY_PATTERN = "hermes-agent" +IMMUTABLE_TAG_PATTERN = "git-*-build-*" +DIGEST_PATTERN = re.compile(r"^sha256:[0-9a-f]{64}$") +REVISION_PATTERN = re.compile(r"^[0-9a-f]{40}$") +BUILD_PATTERN = re.compile(r"^[1-9][0-9]*$") +DESTINATION_PATTERN = re.compile( + r"^registry\.bstein\.dev/bstein/hermes-agent:" + r"git-([0-9a-f]{40})-build-([1-9][0-9]*)$" +) + + +class _NoRedirect(urllib.request.HTTPRedirectHandler): + """Never send registry credentials to a redirect target.""" + + def redirect_request(self, _request, _file, _code, _message, _headers, _url): + return None + + +def _validated(value: str, pattern: re.Pattern[str], label: str) -> str: + """Return a normalized value when it matches the release contract.""" + normalized = value.strip() + if not pattern.fullmatch(normalized): + raise ValueError(f"invalid {label}: expected {pattern.pattern}") + return normalized + + +def validate_destination( + destination: str, source_revision: str, build_number: str +) -> tuple[str, str]: + """Bind one unique build tag to the reviewed revision and Jenkins build.""" + revision = _validated(source_revision, REVISION_PATTERN, "source revision") + build = _validated(build_number, BUILD_PATTERN, "build number") + match = DESTINATION_PATTERN.fullmatch(destination.strip()) + if not match or match.groups() != (revision, build): + raise ValueError( + "destination must bind the reviewed revision and unique Jenkins build" + ) + return revision, build + + +def validate_kaniko_evidence( + *, digest_text: str, image_text: str, destination: str +) -> str: + """Cross-check both independent Kaniko output files against the destination.""" + digest_lines = digest_text.splitlines() + image_lines = image_text.splitlines() + if len(digest_lines) != 1: + raise ValueError("Kaniko digest evidence must contain exactly one line") + if len(image_lines) != 1: + raise ValueError("Kaniko image evidence must contain exactly one line") + digest = _validated(digest_lines[0], DIGEST_PATTERN, "image digest") + if image_lines[0].strip() != f"{destination}@{digest}": + raise ValueError("Kaniko image evidence does not match destination and digest") + return digest + + +def _registry_request(request: urllib.request.Request, timeout: int) -> Any: + """Make a registry request without following redirects.""" + opener = urllib.request.build_opener(_NoRedirect()) + try: + return opener.open(request, timeout=timeout) + except urllib.error.HTTPError as exc: + return exc + + +def _artifact_response( + destination: str, + *, + username: str, + password: str, + opener: Callable[[urllib.request.Request, int], Any] = _registry_request, +) -> tuple[int, bytes]: + """Read one exact Harbor artifact by tag with bounded response size.""" + match = DESTINATION_PATTERN.fullmatch(destination) + if not match: + raise ValueError("invalid destination") + if not username or not password: + raise RuntimeError("Harbor credentials are empty") + tag = destination.rsplit(":", 1)[1] + encoded_tag = urllib.parse.quote(tag, safe="") + auth = base64.b64encode(f"{username}:{password}".encode()).decode("ascii") + request = urllib.request.Request( + f"{HARBOR_API_ORIGIN}/projects/{HARBOR_PROJECT}/repositories/" + f"{HARBOR_REPOSITORY}/artifacts/{encoded_tag}" + "?with_immutable_status=true", + headers={"Accept": "application/json", "Authorization": f"Basic {auth}"}, + method="GET", + ) + with opener(request, 20) as response: + body = response.read(1_048_577) + if len(body) > 1_048_576: + raise RuntimeError("Harbor artifact response exceeded the size limit") + return int(response.status), body + + +def _immutable_rules_response( + *, + username: str, + password: str, + opener: Callable[[urllib.request.Request, int], Any] = _registry_request, +) -> tuple[int, bytes, dict[str, str]]: + """Read the project policy with the same least-privilege publish identity.""" + if not username or not password: + raise RuntimeError("Harbor credentials are empty") + auth = base64.b64encode(f"{username}:{password}".encode()).decode("ascii") + request = urllib.request.Request( + f"{HARBOR_API_ORIGIN}/projects/{HARBOR_PROJECT}/immutabletagrules" + "?page=1&page_size=100", + headers={"Accept": "application/json", "Authorization": f"Basic {auth}"}, + method="GET", + ) + with opener(request, 20) as response: + body = response.read(1_048_577) + if len(body) > 1_048_576: + raise RuntimeError("Harbor immutable rule response exceeded the size limit") + return int(response.status), body, dict(response.headers) + + +def _require_complete_rule_page( + rules: list[dict[str, Any]], headers: dict[str, str] +) -> None: + """Require proof that the bounded first page contains every rule.""" + raw_total = next( + (value for key, value in headers.items() if key.lower() == "x-total-count"), + None, + ) + if raw_total is None or not str(raw_total).isdecimal(): + raise RuntimeError("Harbor immutable rule list omitted a valid total count") + if int(raw_total) != len(rules): + raise RuntimeError("Harbor immutable rule list was truncated") + + +def _normalized_immutable_rule(rule: dict[str, Any]) -> dict[str, Any]: + """Select only fields that bind the server-side build-tag policy.""" + return { + "disabled": bool(rule.get("disabled", False)), + "action": rule.get("action"), + "template": rule.get("template"), + "tag_selectors": [ + { + "kind": item.get("kind"), + "decoration": item.get("decoration"), + "pattern": item.get("pattern"), + } + for item in rule.get("tag_selectors") or [] + if isinstance(item, dict) + ], + "scope_selectors": { + "repository": [ + { + "kind": item.get("kind"), + "decoration": item.get("decoration"), + "pattern": item.get("pattern"), + } + for item in (rule.get("scope_selectors") or {}).get( + "repository", [] + ) + if isinstance(item, dict) + ] + }, + } + + +def verify_immutable_policy( + *, + username: str, + password: str, + opener: Callable[[urllib.request.Request, int], Any] = _registry_request, +) -> None: + """Fail closed before build unless the exact Harbor rule is active.""" + status, body, headers = _immutable_rules_response( + username=username, password=password, opener=opener + ) + if status != 200: + raise RuntimeError(f"Harbor immutable policy preflight returned HTTP {status}") + try: + rules = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise RuntimeError("Harbor returned invalid immutable rule JSON") from exc + if not isinstance(rules, list) or not all(isinstance(item, dict) for item in rules): + raise RuntimeError("Harbor immutable rule list has an invalid shape") + _require_complete_rule_page(rules, headers) + expected = { + "disabled": False, + "action": "immutable", + "template": "immutable_template", + "tag_selectors": [ + { + "kind": "doublestar", + "decoration": "matches", + "pattern": IMMUTABLE_TAG_PATTERN, + } + ], + "scope_selectors": { + "repository": [ + { + "kind": "doublestar", + "decoration": "repoMatches", + "pattern": IMMUTABLE_REPOSITORY_PATTERN, + } + ] + }, + } + matches = [ + _normalized_immutable_rule(item) + for item in rules + if _normalized_immutable_rule(item)["tag_selectors"] + == expected["tag_selectors"] + and _normalized_immutable_rule(item)["scope_selectors"] + == expected["scope_selectors"] + ] + if matches != [expected]: + raise RuntimeError("Harbor immutable build-tag policy is absent or not exact") + + +def assert_tag_absent( + destination: str, + *, + username: str, + password: str, + opener: Callable[[urllib.request.Request, int], Any] = _registry_request, +) -> None: + """Reject replay before Kaniko can push an already-used immutable identity.""" + status, _body = _artifact_response( + destination, username=username, password=password, opener=opener + ) + if status == 404: + return + if status == 200: + raise RuntimeError("Harbor destination tag already exists; refusing overwrite") + raise RuntimeError(f"Harbor destination preflight returned HTTP {status}") + + +def verify_registry_digest( + destination: str, + digest: str, + *, + username: str, + password: str, + opener: Callable[[urllib.request.Request, int], Any] = _registry_request, +) -> None: + """Verify Harbor independently resolves the pushed tag to Kaniko's digest.""" + digest = _validated(digest, DIGEST_PATTERN, "image digest") + status, body = _artifact_response( + destination, username=username, password=password, opener=opener + ) + if status != 200: + raise RuntimeError(f"Harbor manifest verification returned HTTP {status}") + try: + artifact = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise RuntimeError("Harbor returned invalid artifact JSON") from exc + harbor_digest = str(artifact.get("digest") or "").strip() + if not DIGEST_PATTERN.fullmatch(harbor_digest): + raise RuntimeError("Harbor response omitted a valid artifact digest") + if harbor_digest != digest: + raise RuntimeError("Harbor digest does not match Kaniko evidence") + expected_tag = destination.rsplit(":", 1)[1] + matching_tags = [ + item + for item in artifact.get("tags") or [] + if isinstance(item, dict) and item.get("name") == expected_tag + ] + if len(matching_tags) != 1: + raise RuntimeError("Harbor artifact does not contain the expected tag") + if matching_tags[0].get("immutable") is not True: + raise RuntimeError("Harbor did not enforce the expected tag as immutable") + + +def render_kustomization(source: str, digest: str, image: str = DEFAULT_IMAGE) -> str: + """Replace exactly one matching Kustomize image digest without reformatting.""" + digest = _validated(digest, DIGEST_PATTERN, "image digest") + lines = source.splitlines(keepends=True) + matches: list[int] = [] + + for index, line in enumerate(lines): + if line.strip() != f"- name: {image}": + continue + name_indent = len(line) - len(line.lstrip()) + for candidate_index in range(index + 1, len(lines)): + candidate = lines[candidate_index] + stripped = candidate.strip() + candidate_indent = len(candidate) - len(candidate.lstrip()) + if stripped.startswith("- name:") and candidate_indent == name_indent: + break + if stripped.startswith("digest:") and candidate_indent > name_indent: + matches.append(candidate_index) + break + + if len(matches) != 1: + raise ValueError( + f"expected exactly one digest for image {image!r}; found {len(matches)}" + ) + + index = matches[0] + newline = "\n" if lines[index].endswith("\n") else "" + prefix = lines[index][: len(lines[index]) - len(lines[index].lstrip())] + lines[index] = f"{prefix}digest: {digest}{newline}" + return "".join(lines) + + +def write_release_artifacts( + *, + digest: str, + source_revision: str, + build_number: str, + destination: str, + kustomization: Path, + output_dir: Path, +) -> dict[str, str]: + """Write a rendered manifest, patch, and credential-free release metadata.""" + digest = _validated(digest, DIGEST_PATTERN, "image digest") + source_revision, build_number = validate_destination( + destination, source_revision, build_number + ) + source = kustomization.read_text(encoding="utf-8") + rendered = render_kustomization(source, digest) + relative_name = kustomization.name + patch = "".join( + difflib.unified_diff( + source.splitlines(keepends=True), + rendered.splitlines(keepends=True), + fromfile=f"a/services/hermes/{relative_name}", + tofile=f"b/services/hermes/{relative_name}", + ) + ) + if not patch: + raise ValueError("published digest already matches the Flux manifest") + output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / "hermes-kustomization.yaml").write_text(rendered, encoding="utf-8") + (output_dir / "hermes-image-update.patch").write_text(patch, encoding="utf-8") + metadata = { + "build_number": build_number, + "digest": digest, + "flux_image": f"{DEFAULT_IMAGE}@{digest}", + "image": DEFAULT_IMAGE, + "published_tag": destination, + "source_revision": source_revision, + } + (output_dir / "hermes-agent-image.json").write_text( + json.dumps(metadata, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + return metadata + + +def validate_release_artifacts( + *, + digest_file: Path, + image_file: Path, + source_revision: str, + build_number: str, + destination: str, + kustomization: Path, + output_dir: Path, +) -> None: + """Revalidate the exact successful-build evidence without rewriting it.""" + digest = validate_kaniko_evidence( + digest_text=digest_file.read_text(encoding="utf-8"), + image_text=image_file.read_text(encoding="utf-8"), + destination=destination, + ) + source_revision, build_number = validate_destination( + destination, source_revision, build_number + ) + expected_names = { + "hermes-agent-image.json", + "hermes-image-update.patch", + "hermes-kustomization.yaml", + } + entries = list(output_dir.iterdir()) + if {entry.name for entry in entries} != expected_names or not all( + entry.is_file() and not entry.is_symlink() for entry in entries + ): + raise ValueError("release output must contain exactly three evidence files") + source = kustomization.read_text(encoding="utf-8") + rendered = render_kustomization(source, digest) + relative_name = kustomization.name + patch = "".join( + difflib.unified_diff( + source.splitlines(keepends=True), + rendered.splitlines(keepends=True), + fromfile=f"a/services/hermes/{relative_name}", + tofile=f"b/services/hermes/{relative_name}", + ) + ) + metadata = { + "build_number": build_number, + "digest": digest, + "flux_image": f"{DEFAULT_IMAGE}@{digest}", + "image": DEFAULT_IMAGE, + "published_tag": destination, + "source_revision": source_revision, + } + expected = { + "hermes-agent-image.json": json.dumps(metadata, indent=2, sort_keys=True) + "\n", + "hermes-image-update.patch": patch, + "hermes-kustomization.yaml": rendered, + } + for name, expected_text in expected.items(): + if (output_dir / name).read_text(encoding="utf-8") != expected_text: + raise ValueError(f"release evidence is incomplete or mismatched: {name}") + + +def _credentials() -> tuple[str, str]: + """Read the masked, runtime-only Jenkins credential environment.""" + username = os.environ.get("HARBOR_USER", "") + password = os.environ.get("HARBOR_PASSWORD", "") + if not username or not password: + raise RuntimeError("Harbor credentials are unavailable") + return username, password + + +def _common_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--source-revision", required=True) + parser.add_argument("--build-number", required=True) + parser.add_argument("--destination", required=True) + + +def main() -> int: + """Fail closed around the unique tag, then verify and render after push.""" + parser = argparse.ArgumentParser() + commands = parser.add_subparsers(dest="command", required=True) + absent = commands.add_parser("assert-absent") + _common_arguments(absent) + render = commands.add_parser("render") + _common_arguments(render) + render.add_argument("--digest-file", required=True, type=Path) + render.add_argument("--image-file", required=True, type=Path) + render.add_argument("--kustomization", required=True, type=Path) + render.add_argument("--output-dir", required=True, type=Path) + verify = commands.add_parser("verify-evidence") + _common_arguments(verify) + verify.add_argument("--digest-file", required=True, type=Path) + verify.add_argument("--image-file", required=True, type=Path) + verify.add_argument("--kustomization", required=True, type=Path) + verify.add_argument("--output-dir", required=True, type=Path) + args = parser.parse_args() + + validate_destination(args.destination, args.source_revision, args.build_number) + if args.command == "verify-evidence": + validate_release_artifacts( + digest_file=args.digest_file, + image_file=args.image_file, + source_revision=args.source_revision, + build_number=args.build_number, + destination=args.destination, + kustomization=args.kustomization, + output_dir=args.output_dir, + ) + return 0 + + username, password = _credentials() + if args.command == "assert-absent": + verify_immutable_policy(username=username, password=password) + assert_tag_absent(args.destination, username=username, password=password) + return 0 + + digest = validate_kaniko_evidence( + digest_text=args.digest_file.read_text(encoding="utf-8"), + image_text=args.image_file.read_text(encoding="utf-8"), + destination=args.destination, + ) + verify_registry_digest( + args.destination, digest, username=username, password=password + ) + write_release_artifacts( + digest=digest, + source_revision=args.source_revision, + build_number=args.build_number, + destination=args.destination, + kustomization=args.kustomization, + output_dir=args.output_dir, + ) + return 0 + + +if __name__ == "__main__": # pragma: no cover - exercised through main() + raise SystemExit(main()) diff --git a/ci/scripts/semgrep_report.py b/ci/scripts/semgrep_report.py index 5175e2e6..80fb5953 100644 --- a/ci/scripts/semgrep_report.py +++ b/ci/scripts/semgrep_report.py @@ -93,7 +93,7 @@ def _sonar_issue_from_finding(finding: dict[str, Any]) -> dict[str, Any] | None: start = finding.get("start") if isinstance(finding.get("start"), dict) else {} end = finding.get("end") if isinstance(finding.get("end"), dict) else {} start_line = _line_number(start.get("line") if isinstance(start, dict) else None) - # _line_number never returns below its default, so end_line >= start_line. + # _line_number floors the end line at start_line, so the range never inverts. end_line = _line_number(end.get("line") if isinstance(end, dict) else None, start_line) return { "engineId": "semgrep", diff --git a/clusters/atlas/flux-system/applications/harbor/kustomization.yaml b/clusters/atlas/flux-system/applications/harbor/kustomization.yaml index 5bbe2e8f..225214fe 100644 --- a/clusters/atlas/flux-system/applications/harbor/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/harbor/kustomization.yaml @@ -15,7 +15,13 @@ spec: kind: GitRepository name: flux-system namespace: flux-system - wait: false + wait: true + timeout: 10m + healthChecks: + - apiVersion: batch/v1 + kind: Job + name: harbor-hermes-agent-immutability-ensure-1 + namespace: harbor dependsOn: - name: core - name: longhorn diff --git a/clusters/atlas/flux-system/applications/hermes-observer-bindings/kustomization.yaml b/clusters/atlas/flux-system/applications/hermes-observer-bindings/kustomization.yaml new file mode 100644 index 00000000..b64913d5 --- /dev/null +++ b/clusters/atlas/flux-system/applications/hermes-observer-bindings/kustomization.yaml @@ -0,0 +1,19 @@ +# clusters/atlas/flux-system/applications/hermes-observer-bindings/kustomization.yaml +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: hermes-observer-bindings + namespace: flux-system +spec: + interval: 10m + path: ./services/hermes-observer-bindings + prune: true + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + wait: true + timeout: 5m + dependsOn: + - name: hermes + - name: hermes-observer-rbac diff --git a/clusters/atlas/flux-system/applications/hermes-observer-rbac/kustomization.yaml b/clusters/atlas/flux-system/applications/hermes-observer-rbac/kustomization.yaml new file mode 100644 index 00000000..c2749739 --- /dev/null +++ b/clusters/atlas/flux-system/applications/hermes-observer-rbac/kustomization.yaml @@ -0,0 +1,16 @@ +# clusters/atlas/flux-system/applications/hermes-observer-rbac/kustomization.yaml +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: hermes-observer-rbac + namespace: flux-system +spec: + interval: 10m + path: ./services/hermes-observer-rbac + prune: true + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + wait: true + timeout: 5m diff --git a/clusters/atlas/flux-system/applications/hermes-scm-broker-code/kustomization.yaml b/clusters/atlas/flux-system/applications/hermes-scm-broker-code/kustomization.yaml new file mode 100644 index 00000000..20ae82e2 --- /dev/null +++ b/clusters/atlas/flux-system/applications/hermes-scm-broker-code/kustomization.yaml @@ -0,0 +1,19 @@ +# clusters/atlas/flux-system/applications/hermes-scm-broker-code/kustomization.yaml +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: hermes-scm-broker-code + namespace: flux-system +spec: + interval: 10m + path: ./services/hermes/scm-common + targetNamespace: hermes-scm + prune: true + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + wait: true + timeout: 5m + dependsOn: + - name: hermes-scm-namespace diff --git a/clusters/atlas/flux-system/applications/hermes-scm-broker/kustomization.yaml b/clusters/atlas/flux-system/applications/hermes-scm-broker/kustomization.yaml new file mode 100644 index 00000000..70b3989e --- /dev/null +++ b/clusters/atlas/flux-system/applications/hermes-scm-broker/kustomization.yaml @@ -0,0 +1,27 @@ +# clusters/atlas/flux-system/applications/hermes-scm-broker/kustomization.yaml +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: hermes-scm-broker + namespace: flux-system +spec: + interval: 10m + path: ./services/hermes-scm-broker + targetNamespace: hermes-scm + prune: true + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + wait: true + timeout: 10m + healthChecks: + - apiVersion: apps/v1 + kind: Deployment + name: hermes-scm-broker + namespace: hermes-scm + dependsOn: + - name: vault + - name: gitea + - name: hermes-scm-namespace + - name: hermes-scm-broker-code diff --git a/clusters/atlas/flux-system/applications/hermes-scm-namespace/kustomization.yaml b/clusters/atlas/flux-system/applications/hermes-scm-namespace/kustomization.yaml new file mode 100644 index 00000000..9f066709 --- /dev/null +++ b/clusters/atlas/flux-system/applications/hermes-scm-namespace/kustomization.yaml @@ -0,0 +1,16 @@ +# clusters/atlas/flux-system/applications/hermes-scm-namespace/kustomization.yaml +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: hermes-scm-namespace + namespace: flux-system +spec: + interval: 10m + path: ./services/hermes-scm-namespace + prune: true + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + wait: true + timeout: 5m diff --git a/clusters/atlas/flux-system/applications/hermes/kustomization.yaml b/clusters/atlas/flux-system/applications/hermes/kustomization.yaml index 3949282b..b4b2759f 100644 --- a/clusters/atlas/flux-system/applications/hermes/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/hermes/kustomization.yaml @@ -60,3 +60,6 @@ spec: - name: keycloak - name: longhorn - name: vault + - name: jenkins + - name: hermes-scm-broker + - name: hermes-observer-rbac diff --git a/clusters/atlas/flux-system/applications/jenkins/kustomization.yaml b/clusters/atlas/flux-system/applications/jenkins/kustomization.yaml index dce79ab7..5cf8d56d 100644 --- a/clusters/atlas/flux-system/applications/jenkins/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/jenkins/kustomization.yaml @@ -6,10 +6,9 @@ metadata: namespace: flux-system annotations: kustomize.toolkit.fluxcd.io/ssa: IfNotPresent - atlas.bstein.dev/suspend-reason: "CI controller changes are applied only during planned maintenance." spec: interval: 10m - suspend: true + suspend: false path: ./services/jenkins prune: true sourceRef: @@ -18,6 +17,8 @@ spec: targetNamespace: jenkins dependsOn: - name: helm + - name: harbor + - name: vault-hermes-jenkins-token-seed healthChecks: - apiVersion: apps/v1 kind: Deployment diff --git a/clusters/atlas/flux-system/applications/kustomization.yaml b/clusters/atlas/flux-system/applications/kustomization.yaml index eb8bc86a..794f0869 100644 --- a/clusters/atlas/flux-system/applications/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/kustomization.yaml @@ -4,6 +4,7 @@ kind: Kustomization resources: - gitea/kustomization.yaml - vault/kustomization.yaml + - vault-hermes-jenkins-token-seed/kustomization.yaml - vaultwarden/kustomization.yaml - comms/kustomization.yaml - crypto/kustomization.yaml @@ -27,7 +28,12 @@ resources: - jenkins/kustomization.yaml - ai-llm/kustomization.yaml - openclaw/kustomization.yaml + - hermes-scm-namespace/kustomization.yaml + - hermes-scm-broker-code/kustomization.yaml - hermes/kustomization.yaml + - hermes-observer-rbac/kustomization.yaml + - hermes-observer-bindings/kustomization.yaml + - hermes-scm-broker/kustomization.yaml - hermes-chat/kustomization.yaml - hermes-triage-demo/kustomization.yaml - game-stream/kustomization.yaml diff --git a/clusters/atlas/flux-system/applications/vault-hermes-jenkins-token-seed/kustomization.yaml b/clusters/atlas/flux-system/applications/vault-hermes-jenkins-token-seed/kustomization.yaml new file mode 100644 index 00000000..561179c8 --- /dev/null +++ b/clusters/atlas/flux-system/applications/vault-hermes-jenkins-token-seed/kustomization.yaml @@ -0,0 +1,26 @@ +# clusters/atlas/flux-system/applications/vault-hermes-jenkins-token-seed/kustomization.yaml +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: vault-hermes-jenkins-token-seed + namespace: flux-system + annotations: + kustomize.toolkit.fluxcd.io/ssa: IfNotPresent +spec: + interval: 10m + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + path: ./services/vault-hermes-jenkins-token-seed + targetNamespace: vault + prune: true + wait: true + timeout: 5m + dependsOn: + - name: vault + healthChecks: + - apiVersion: batch/v1 + kind: Job + name: vault-hermes-jenkins-build-token-seed-2 + namespace: vault diff --git a/clusters/atlas/flux-system/applications/vault/kustomization.yaml b/clusters/atlas/flux-system/applications/vault/kustomization.yaml index 7dd03da8..c3408a6e 100644 --- a/clusters/atlas/flux-system/applications/vault/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/vault/kustomization.yaml @@ -16,6 +16,11 @@ spec: targetNamespace: vault prune: true wait: true + healthChecks: + - apiVersion: batch/v1 + kind: Job + name: vault-k8s-auth-hermes-10 + namespace: vault dependsOn: - name: longhorn - name: helm diff --git a/dockerfiles/Dockerfile.hermes-agent b/dockerfiles/Dockerfile.hermes-agent index 1416aae7..35a2f463 100644 --- a/dockerfiles/Dockerfile.hermes-agent +++ b/dockerfiles/Dockerfile.hermes-agent @@ -15,6 +15,10 @@ RUN apt-get update \ # Paid/provider-backed search remains selectable through normal Hermes config. RUN uv pip install --python /opt/hermes/.venv/bin/python ddgs==9.14.4 +ARG HERMES_KANIKO_HEREDOC_COMPAT=0 +COPY dockerfiles/Dockerfile.hermes-agent /tmp/hermes-agent.Dockerfile +COPY dockerfiles/hermes-kaniko-heredoc-runner.py /tmp/hermes-kaniko-heredoc-runner.py + # Keep dashboard chat sockets tied to the intended React mount and conversation. # A resumed conversation needs a different PTY attachment key from a fresh chat; # reconnects to that same conversation must keep using the same key. @@ -252,6 +256,7 @@ for (const [before, after, label] of [ } fs.writeFileSync(sidebarPath, sidebar); NODE +RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 1 ;; *) exit 2 ;; esac # The upstream OIDC gate authenticates users but deliberately treats the # dashboard as one shared workstation. Allow a deployment to narrow that @@ -324,6 +329,7 @@ for before, after, label in ( source = source.replace(before, after, 1) path.write_text(source) PY +RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 2 ;; *) exit 2 ;; esac # Give trusted plugins a pre-turn routing hook. It runs after fallback runtime # restoration but before Hermes builds its provider-specific system prompt. @@ -493,6 +499,7 @@ if oneshot.count(oneshot_before) != 1: ) oneshot_path.write_text(oneshot.replace(oneshot_before, oneshot_after, 1)) PY +RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 3 ;; *) exit 2 ;; esac # Hermes WebUI sends its model/provider/reasoning selection on /v1/runs. # Upstream currently applies only statically declared model_routes there, so @@ -628,6 +635,7 @@ runs_source = runs_source.replace(runs_agent_before, runs_agent_after, 1) source = source[:runs_start] + runs_source path.write_text(source) PY +RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 4 ;; *) exit 2 ;; esac # ``create_task(initial_status="blocked")`` is an explicit operator park, but # upstream only treats later block_task() events as sticky. Recognize the @@ -779,6 +787,7 @@ if cli_source.count(help_before) != 1: ) cli_path.write_text(cli_source.replace(help_before, help_after, 1)) PY +RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 5 ;; *) exit 2 ;; esac COPY dockerfiles/hermes-kanban-blocked-regression.py /tmp/hermes-kanban-blocked-regression.py RUN /opt/hermes/.venv/bin/python /tmp/hermes-kanban-blocked-regression.py \ @@ -816,6 +825,7 @@ if source.count(before) != 1: ) path.write_text(source.replace(before, after, 1)) PY +RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 6 ;; *) exit 2 ;; esac # Keep the dashboard's account cards aligned with the managed provider lanes. # Switchyard receives Claude Code subscription OAuth through the environment; @@ -882,6 +892,7 @@ for before, after, label in replacements: path.write_text(source) PY +RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 7 ;; *) exit 2 ;; esac # Make parent-linked API workers first-class live dashboard sessions. They stay # active until the API runner closes them, even during a long final model call; @@ -975,6 +986,7 @@ source = source.replace(row_before, row_after) path.write_text(source) PY +RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 8 ;; *) exit 2 ;; esac # Keep the browser terminal reliable across GPU context loss, make durable # worker lineage visible, expose an accessible API-worker transcript, and @@ -1543,6 +1555,7 @@ function RootRedirect() { fs.writeFileSync(path, source); } NODE +RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 9 ;; *) exit 2 ;; esac COPY dockerfiles/patch-hermes-execution-safety.py /tmp/patch-hermes-execution-safety.py COPY dockerfiles/hermes-execution-safety-regression.py /tmp/hermes-execution-safety-regression.py @@ -1576,6 +1589,8 @@ RUN HERMES_CLI_LANE_SOURCE=/tmp/hermes-lane-regression \ COPY dockerfiles/hermes-session-migrate.py /opt/hermes/bin/hermes-session-migrate +RUN rm -f /tmp/hermes-agent.Dockerfile /tmp/hermes-kaniko-heredoc-runner.py + RUN cd /opt/hermes/web \ && npm run build \ && grep -Fq 'await api.getSessions(1, 0' src/pages/ChatPage.tsx \ diff --git a/dockerfiles/Dockerfile.hermes-agent.dockerignore b/dockerfiles/Dockerfile.hermes-agent.dockerignore index 0abc8045..e4731f78 100644 --- a/dockerfiles/Dockerfile.hermes-agent.dockerignore +++ b/dockerfiles/Dockerfile.hermes-agent.dockerignore @@ -5,6 +5,8 @@ !dockerfiles/hermes-public-extract/** !dockerfiles/hermes-session-activity-panel.tsx !dockerfiles/hermes-session-migrate.py +!dockerfiles/Dockerfile.hermes-agent +!dockerfiles/hermes-kaniko-heredoc-runner.py !dockerfiles/patch-hermes-execution-safety.py !dockerfiles/hermes-execution-safety-regression.py !dockerfiles/hermes_execution_patch_support.py diff --git a/dockerfiles/hermes-kaniko-heredoc-runner.py b/dockerfiles/hermes-kaniko-heredoc-runner.py new file mode 100644 index 00000000..91a1330a --- /dev/null +++ b/dockerfiles/hermes-kaniko-heredoc-runner.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +"""Replay the reviewed Hermes Dockerfile heredocs for pinned Kaniko v1.23.2. + +Kaniko v1.23.2 parses shell heredoc bodies into ``RunCommand.Files`` but its RUN +implementation executes only ``CmdLine``. The interpreters therefore receive +empty stdin and exit successfully. Docker and BuildKit execute these blocks +normally, so this compatibility runner is enabled only by the Kaniko pipeline. +""" + +from __future__ import annotations + +import argparse +import re +import subprocess +from pathlib import Path + + +MAX_DOCKERFILE_BYTES = 2_000_000 +BLOCKS = ( + ("RUN node <<'NODE'", "NODE", ("node",)), + ("RUN python - <<'PY'", "PY", ("python", "-")), +) +EXPECTED_COMMANDS = ("node", *("python",) * 7, "node") +PARSER_DIRECTIVE = re.compile( + r"^\s*#\s*([a-z]+)\s*=\s*(.*?)\s*$", re.IGNORECASE +) + + +def _escape_character(lines: list[str]) -> str: + """Return the one Dockerfile parser escape directive, or its default.""" + escape = "\\" + directives: set[str] = set() + for line in lines: + if not line.strip(): + break + match = PARSER_DIRECTIVE.fullmatch(line) + if not match: + break + name, value = match.group(1).lower(), match.group(2) + if name in directives: + raise ValueError(f"multiple Dockerfile {name} directives") + directives.add(name) + if name == "escape": + if value not in {"\\", "`"}: + raise ValueError("unsupported Dockerfile escape directive") + escape = value + return escape + + +def _continued(line: str, escape: str) -> bool: + """Match Docker's unescaped continuation marker at physical line end.""" + stripped = line.rstrip() + count = len(stripped) - len(stripped.rstrip(escape)) + return count % 2 == 1 + + +def _logical_instruction( + lines: list[str], index: int, escape: str +) -> tuple[str, int]: + """Normalize escape-newline pairs, including split opcodes/operators.""" + logical = lines[index] + while _continued(logical.split("\n")[-1], escape): + logical = logical.rstrip() + logical = logical[:-1] + index += 1 + while index < len(lines) and lines[index].lstrip().startswith("#"): + index += 1 + if index >= len(lines) or not lines[index].strip(): + raise ValueError("unterminated Dockerfile line continuation") + logical += lines[index].lstrip() + return logical, index + + +def extract_blocks(source: str) -> list[tuple[tuple[str, ...], str]]: + """Inventory every RUN heredoc, then return only the reviewed contract.""" + markers = {start: (end, command) for start, end, command in BLOCKS} + lines = source.splitlines() + escape = _escape_character(lines) + blocks: list[tuple[tuple[str, ...], str]] = [] + index = 0 + while index < len(lines): + marker = markers.get(lines[index]) + if marker is None: + if not lines[index].strip() or lines[index].lstrip().startswith("#"): + index += 1 + continue + logical, index = _logical_instruction(lines, index, escape) + stripped = logical.lstrip() + if ( + stripped[:3].upper() == "RUN" + and (len(stripped) == 3 or stripped[3].isspace()) + and "<<" in stripped + ): + raise ValueError( + "unsupported RUN heredoc outside the exact reviewed contract" + ) + index += 1 + continue + end, command = marker + body_start = index + 1 + index = body_start + while index < len(lines) and lines[index] != end: + index += 1 + if index == len(lines): + raise ValueError(f"unterminated {command[0]} heredoc") + body = "\n".join(lines[body_start:index]) + "\n" + if not body.strip(): + raise ValueError(f"empty {command[0]} heredoc") + blocks.append((command, body)) + index += 1 + + commands = tuple(command[0] for command, _body in blocks) + if commands != EXPECTED_COMMANDS: + raise ValueError( + "Hermes Dockerfile heredoc contract changed: " + f"expected {EXPECTED_COMMANDS!r}, received {commands!r}" + ) + return blocks + + +def replay(dockerfile: Path, block_index: int) -> None: + """Execute one exact heredoc after inventorying the complete Dockerfile.""" + size = dockerfile.stat().st_size + if size < 1 or size > MAX_DOCKERFILE_BYTES: + raise ValueError("Hermes Dockerfile size is outside the reviewed boundary") + source = dockerfile.read_text(encoding="utf-8") + blocks = extract_blocks(source) + if block_index < 1 or block_index > len(blocks): + raise ValueError("heredoc block index is outside the reviewed contract") + command, body = blocks[block_index - 1] + print(f"replaying reviewed Dockerfile heredoc {block_index}/{len(blocks)}") + subprocess.run(command, input=body, text=True, check=True) + + +def main() -> int: + """Parse the one explicit Dockerfile path and replay its reviewed patches.""" + parser = argparse.ArgumentParser() + parser.add_argument("--dockerfile", required=True, type=Path) + parser.add_argument("--block-index", required=True, type=int) + args = parser.parse_args() + replay(args.dockerfile, args.block_index) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/tests/test_mailu_sync.py b/scripts/tests/test_mailu_sync.py index 4d6207fe..c5a35954 100644 --- a/scripts/tests/test_mailu_sync.py +++ b/scripts/tests/test_mailu_sync.py @@ -232,6 +232,24 @@ def test_retry_db_connect_reraises_final_error(monkeypatch): sync.retry_db_connect(attempts=1) +def test_retry_helpers_return_none_without_attempts(monkeypatch): + sync = load_sync_module(monkeypatch) + + assert sync.retry_request("request", lambda: "never", attempts=0) is None + assert sync.retry_db_connect(attempts=0) is None + + +def test_kc_update_attributes_replaces_non_dict_attributes(monkeypatch): + sync = load_sync_module(monkeypatch) + current_resp = _FakeResponse({"attributes": "not-a-dict"}) + ok_resp = _FakeResponse({"attributes": {"mailu_app_password": ["abc"]}}) + sync.SESSION = _FakeSession(_FakeResponse({}), [current_resp, ok_resp]) + + sync.kc_update_attributes("token", {"id": "u1", "username": "u1"}, {"mailu_app_password": "abc"}) + + assert sync.SESSION.put_called + + def test_ensure_mailu_user_skips_foreign_domain(monkeypatch): sync = load_sync_module(monkeypatch) executed = [] @@ -405,3 +423,62 @@ def test_main_generates_password_and_upserts(monkeypatch): # Only mail-enabled users (or legacy users with a mailbox) are synced and backfilled. assert len(updated) == 3 assert conns and len(conns[0]._cursor.executions) == 3 + + +def test_main_skips_disabled_users_and_provisioned_users_without_update(monkeypatch): + sync = load_sync_module(monkeypatch) + monkeypatch.setattr(sync.bcrypt_sha256, "hash", lambda password: f"hash:{password}") + users = [ + { + "id": "u1", + "username": "disabled", + "email": "disabled@example.com", + "enabled": False, + "attributes": {"mailu_enabled": ["true"]}, + }, + { + "id": "u2", + "username": "settled", + "email": "settled@example.com", + "attributes": { + "mailu_enabled": ["true"], + "mailu_email": ["settled@example.com"], + "mailu_app_password": ["existing"], + }, + }, + ] + updated = [] + + class _Cursor: + def __init__(self): + self.executions = [] + + def execute(self, sql, params): + self.executions.append(params) + + def close(self): + return None + + class _Conn: + def __init__(self): + self.autocommit = False + self._cursor = _Cursor() + + def cursor(self, cursor_factory=None): + return self._cursor + + def close(self): + return None + + conn = _Conn() + monkeypatch.setattr(sync, "get_kc_token", lambda: "tok") + monkeypatch.setattr(sync, "kc_get_users", lambda token: users) + monkeypatch.setattr(sync, "kc_update_attributes", lambda token, user, attrs: updated.append(user["id"])) + monkeypatch.setattr(sync.psycopg2, "connect", lambda **kwargs: conn) + + sync.main() + + # The already-provisioned user is synced without a Keycloak write; the + # disabled user never reaches the mailbox upsert. + assert updated == [] + assert len(conn._cursor.executions) == 1 diff --git a/scripts/tests/test_mailu_sync_listener.py b/scripts/tests/test_mailu_sync_listener.py index 58285591..132634e6 100644 --- a/scripts/tests/test_mailu_sync_listener.py +++ b/scripts/tests/test_mailu_sync_listener.py @@ -127,6 +127,18 @@ def test_listener_post_wait_keeps_running_request_successful(monkeypatch): assert handler.responses == [200] +def test_listener_post_treats_non_object_json_as_plain_trigger(monkeypatch): + listener = load_listener_module(monkeypatch) + called = [] + monkeypatch.setattr(listener, "_trigger_sync_async", lambda force=False: called.append(force) or True) + handler = _handler_for(listener, "[1, 2]") + + handler.do_POST() + + assert called == [False] + assert handler.responses == [202] + + def test_listener_log_message_is_quiet(monkeypatch): listener = load_listener_module(monkeypatch) handler = listener.Handler.__new__(listener.Handler) diff --git a/services/gitea/atlas-identity-bootstrap-job.yaml b/services/gitea/atlas-identity-bootstrap-job.yaml index c7e2071f..bacecd01 100644 --- a/services/gitea/atlas-identity-bootstrap-job.yaml +++ b/services/gitea/atlas-identity-bootstrap-job.yaml @@ -2,7 +2,7 @@ apiVersion: batch/v1 kind: Job metadata: - name: gitea-atlas-identity-bootstrap-3 + name: gitea-atlas-identity-bootstrap-5 namespace: gitea labels: app.kubernetes.io/name: gitea @@ -34,6 +34,20 @@ spec: nodeSelector: node-role.kubernetes.io/worker: "true" hardware: rpi5 + initContainers: + - name: python-runtime + image: python@sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df + command: [/bin/sh, -ec] + args: ["cp -R /usr/local/. /python/"] + securityContext: + allowPrivilegeEscalation: false + capabilities: {drop: [ALL]} + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + volumeMounts: + - {name: python-runtime, mountPath: /python} containers: - name: bootstrap image: gitea/gitea:1.23 @@ -80,6 +94,7 @@ spec: - name: bootstrap mountPath: /opt/bootstrap readOnly: true + - {name: python-runtime, mountPath: /opt/python, readOnly: true} volumes: - name: data persistentVolumeClaim: @@ -88,3 +103,6 @@ spec: configMap: name: gitea-atlas-identity-bootstrap defaultMode: 0555 + - name: python-runtime + emptyDir: + sizeLimit: 128Mi diff --git a/services/gitea/kustomization.yaml b/services/gitea/kustomization.yaml index 8c473a38..6511418a 100644 --- a/services/gitea/kustomization.yaml +++ b/services/gitea/kustomization.yaml @@ -15,6 +15,7 @@ configMapGenerator: - name: gitea-atlas-identity-bootstrap files: - gitea_atlas_identity_ensure.sh=scripts/gitea_atlas_identity_ensure.sh + - gitea_branch_protection_check.py=scripts/gitea_branch_protection_check.py generatorOptions: disableNameSuffixHash: true diff --git a/services/gitea/scripts/gitea_atlas_identity_ensure.sh b/services/gitea/scripts/gitea_atlas_identity_ensure.sh index f98f08f5..6d5dbd90 100644 --- a/services/gitea/scripts/gitea_atlas_identity_ensure.sh +++ b/services/gitea/scripts/gitea_atlas_identity_ensure.sh @@ -19,6 +19,7 @@ source_owner="${GITEA_SOURCE_OWNER:-bstein}" managed_repositories="${GITEA_MANAGED_REPOSITORIES:-hermes-code-demo cassandra soteria pegasus metis ananke ariadne typhon titan-iac}" contributor_team=Contributors contributor_units='["repo.actions","repo.packages","repo.code","repo.issues","repo.wiki","repo.pulls","repo.releases","repo.projects"]' +protected_reviewer=bstein die() { echo "Gitea Atlas identity bootstrap failed: $*" >&2 @@ -272,6 +273,46 @@ transfer_managed_repositories() { trap - EXIT HUP INT TERM } +ensure_default_branch_protection() { + repository=$1 + repo_file=$(mktemp) + protections_file=$(mktemp) + response_file=$(mktemp) + + status=$(api_request GET "/repos/${organization}/${repository}" '' "${repo_file}") + expect_status "${status}" 200 "read ${repository} metadata for branch protection" + default_branch=$(sed -n 's/.*"default_branch":"\([^"]*\)".*/\1/p' "${repo_file}") + case "${default_branch}" in + main|master) ;; + *) die "${repository} has unsupported default branch ${default_branch:-missing}" ;; + esac + + status=$(api_request GET "/repos/${organization}/${repository}/branch_protections" '' "${protections_file}") + expect_status "${status}" 200 "list ${repository} branch protections" + if ! protection_state=$( + /opt/python/bin/python3 /opt/bootstrap/gitea_branch_protection_check.py \ + "${protections_file}" "${default_branch}" "${protected_reviewer}" + ); then + die "${repository} ${default_branch} protection differs from the human-review policy" + fi + + if [ "${protection_state}" = ABSENT ]; then + payload="{\"rule_name\":\"${default_branch}\",\"enable_push\":true,\"enable_push_whitelist\":true,\"push_whitelist_usernames\":[\"${protected_reviewer}\"],\"push_whitelist_teams\":[],\"push_whitelist_deploy_keys\":false,\"enable_force_push\":false,\"enable_force_push_allowlist\":false,\"force_push_allowlist_usernames\":[],\"force_push_allowlist_teams\":[],\"force_push_allowlist_deploy_keys\":false,\"enable_merge_whitelist\":true,\"merge_whitelist_usernames\":[\"${protected_reviewer}\"],\"merge_whitelist_teams\":[],\"enable_approvals_whitelist\":true,\"approvals_whitelist_username\":[\"${protected_reviewer}\"],\"approvals_whitelist_teams\":[],\"required_approvals\":1,\"block_on_rejected_reviews\":true,\"block_on_outdated_branch\":true,\"dismiss_stale_approvals\":true,\"block_admin_merge_override\":true}" + status=$(api_request POST "/repos/${organization}/${repository}/branch_protections" "${payload}" "${response_file}") + expect_status "${status}" 201 "protect ${repository} ${default_branch}" + status=$(api_request GET "/repos/${organization}/${repository}/branch_protections" '' "${protections_file}") + expect_status "${status}" 200 "read back ${repository} branch protections" + protection_state=$( + /opt/python/bin/python3 /opt/bootstrap/gitea_branch_protection_check.py \ + "${protections_file}" "${default_branch}" "${protected_reviewer}" + ) || die "${repository} ${default_branch} protection differs from the human-review policy" + fi + [ "${protection_state}" = PRESENT ] || \ + die "${repository} ${default_branch} protection did not become effective" + + rm -f "${repo_file}" "${protections_file}" "${response_file}" +} + wait_for_gitea ensure_user "${reconciler_user}" "${reconciler_email}" true ensure_user "${hermes_user}" "${hermes_email}" false @@ -318,6 +359,10 @@ expect_status "${status}" 204 "add Hermes to Atlas contributors" transfer_managed_repositories +for repository in ${managed_repositories}; do + ensure_default_branch_protection "${repository}" +done + vault_write gitea/atlas-reconciler "{\"data\":{\"username\":\"${reconciler_user}\",\"token\":\"${reconciler_token}\",\"base_url\":\"${public_url}\"}}" vault_write hermes/developer-gitea "{\"data\":{\"username\":\"${hermes_user}\",\"email\":\"${hermes_email}\",\"token\":\"${hermes_token}\",\"base_url\":\"${public_url}\"}}" diff --git a/services/gitea/scripts/gitea_branch_protection_check.py b/services/gitea/scripts/gitea_branch_protection_check.py new file mode 100644 index 00000000..5cb9717d --- /dev/null +++ b/services/gitea/scripts/gitea_branch_protection_check.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +"""Validate the first effective Gitea protection for an Atlas primary branch.""" + +from __future__ import annotations + +import argparse +import datetime +import json +import os +import stat +from pathlib import Path + +MAX_INPUT = 1024 * 1024 +MAX_RULES = 100 +MAX_RULE_NAME = 255 +SPECIAL = frozenset("*?\\[]{}") + + +class PolicyError(RuntimeError): + """The protection response is ambiguous or violates human review policy.""" + + +def _read_bounded(path: Path) -> bytes: + descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + try: + metadata = os.fstat(descriptor) + if not stat.S_ISREG(metadata.st_mode) or metadata.st_size > MAX_INPUT: + raise PolicyError("branch protection response exceeds the safe limit") + value = os.read(descriptor, MAX_INPUT + 1) + finally: + os.close(descriptor) + if len(value) != metadata.st_size: + raise PolicyError("branch protection response changed while reading") + return value + + +def _is_plain(pattern: str) -> bool: + return not any(character in SPECIAL for character in pattern) + + +def _literal_rule(pattern: str) -> str: + """Return one exact rule name or reject Gitea glob interpretation. + + Gitea 1.23.8 delegates matching to gobwas/glob. Reimplementing that + grammar and its error quoting rules here would create a second policy + engine. The reconciler instead accepts only plain ASCII literals. This + deliberately rejects braces, ranges, escapes, malformed globs, and even + special rules that appear unrelated: an operator must remove ambiguity + before this job may attest the primary-branch boundary. + """ + if not pattern.isascii() or not 1 <= len(pattern) <= MAX_RULE_NAME: + raise PolicyError("branch protection rule name is invalid") + if not _is_plain(pattern): + raise PolicyError("glob branch protection rules require human review") + return pattern + + +def _created(value: object) -> datetime.datetime: + if not isinstance(value, str) or len(value) > 64: + raise PolicyError("branch protection creation time is invalid") + try: + result = datetime.datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as exc: + raise PolicyError("branch protection creation time is invalid") from exc + if result.tzinfo is None: + raise PolicyError("branch protection creation time is invalid") + return result + + +def _required(reviewer: str) -> dict[str, object]: + return { + "enable_push": True, + "enable_push_whitelist": True, + "push_whitelist_usernames": [reviewer], + "push_whitelist_deploy_keys": False, + "enable_force_push": False, + "enable_merge_whitelist": True, + "merge_whitelist_usernames": [reviewer], + "enable_approvals_whitelist": True, + "approvals_whitelist_username": [reviewer], + "required_approvals": 1, + "block_on_rejected_reviews": True, + "block_on_outdated_branch": True, + "dismiss_stale_approvals": True, + "block_admin_merge_override": True, + } + + +def evaluate(value: bytes, branch: str, reviewer: str) -> str: + """Return PRESENT/ABSENT or fail on ambiguous/effective policy drift.""" + if len(value) > MAX_INPUT or branch not in {"main", "master"}: + raise PolicyError("branch protection input is invalid") + data = json.loads(value) + if not isinstance(data, list) or len(data) > MAX_RULES: + raise PolicyError("branch protection response is invalid") + ordered: list[tuple[int, datetime.datetime, int, dict[str, object]]] = [] + for index, raw in enumerate(data): + if not isinstance(raw, dict): + raise PolicyError("branch protection entry is invalid") + priority = raw.get("priority") + rule_name = raw.get("rule_name") + if ( + not isinstance(priority, int) + or isinstance(priority, bool) + or not 0 <= priority <= 1_000_000 + or not isinstance(rule_name, str) + ): + raise PolicyError("branch protection priority is ambiguous") + created = _created(raw.get("created_at")) + literal = _literal_rule(rule_name) + if literal == branch: + # With every rule constrained to a plain literal, Gitea's glob + # comparator cannot affect this result. Preserve API ordering: + # priority first (including legacy zero), creation time, then the + # stable response order for an otherwise exact tie. + ordered.append((priority, created, index, raw)) + if not ordered: + return "ABSENT" + if len(ordered) != 1: + raise PolicyError("duplicate primary-branch protections are ambiguous") + effective = ordered[0][3] + for key, expected in _required(reviewer).items(): + if effective.get(key) != expected: + raise PolicyError( + f"effective {branch} protection differs from human-review policy" + ) + return "PRESENT" + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("response", type=Path) + parser.add_argument("branch", choices=("main", "master")) + parser.add_argument("reviewer") + args = parser.parse_args() + try: + value = _read_bounded(args.response) + print(evaluate(value, args.branch, args.reviewer)) + except (OSError, json.JSONDecodeError, PolicyError) as exc: + print(f"branch protection check failed: {exc}", file=__import__("sys").stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/harbor/hermes-agent-immutability-job.yaml b/services/harbor/hermes-agent-immutability-job.yaml new file mode 100644 index 00000000..5df877bb --- /dev/null +++ b/services/harbor/hermes-agent-immutability-job.yaml @@ -0,0 +1,79 @@ +# services/harbor/hermes-agent-immutability-job.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: harbor-hermes-agent-immutability-ensure-1 + namespace: harbor +spec: + backoffLimit: 2 + activeDeadlineSeconds: 600 + template: + metadata: + annotations: + vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/agent-pre-populate-only: "true" + vault.hashicorp.com/agent-run-as-user: "65532" + vault.hashicorp.com/agent-run-as-group: "65532" + vault.hashicorp.com/role: harbor-policy-bootstrap + vault.hashicorp.com/agent-inject-secret-harbor-admin-password: kv/data/atlas/harbor/harbor-core + vault.hashicorp.com/agent-inject-template-harbor-admin-password: | + {{- with secret "kv/data/atlas/harbor/harbor-core" -}} + {{ .Data.data.harbor_admin_password }} + {{- end -}} + spec: + serviceAccountName: harbor-policy-bootstrap + enableServiceLinks: false + restartPolicy: Never + nodeSelector: + hardware: rpi5 + kubernetes.io/arch: arm64 + node-role.kubernetes.io/worker: "true" + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: NotIn + values: [titan-04, titan-14, titan-18, titan-19, titan-24] + securityContext: + fsGroup: 65532 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: + type: RuntimeDefault + containers: + - name: ensure + image: docker.io/library/python@sha256:efcdfa6a6b2fd2afb9c7dfa9a5b288a6f68338b5cfdebe6b637d986067d85757 + imagePullPolicy: IfNotPresent + command: [python3, /scripts/harbor_hermes_agent_immutability_ensure.py] + env: + - name: HARBOR_API_ORIGIN + value: https://registry.bstein.dev/api/v2.0 + - name: HARBOR_ADMIN_PASSWORD_FILE + value: /vault/secrets/harbor-admin-password + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + readOnlyRootFilesystem: true + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 + seccompProfile: + type: RuntimeDefault + volumeMounts: + - name: scripts + mountPath: /scripts + readOnly: true + - name: tmp + mountPath: /tmp + resources: + requests: {cpu: 25m, memory: 32Mi} + limits: {cpu: 250m, memory: 128Mi} + volumes: + - name: scripts + configMap: + name: harbor-hermes-agent-immutability-script + defaultMode: 0555 + - name: tmp + emptyDir: {} diff --git a/services/harbor/kustomization.yaml b/services/harbor/kustomization.yaml index 6a4ce002..7784bb50 100644 --- a/services/harbor/kustomization.yaml +++ b/services/harbor/kustomization.yaml @@ -12,9 +12,14 @@ resources: - certificate.yaml - helmrelease.yaml - vault-sync-deployment.yaml + - policy-bootstrap-serviceaccount.yaml + - hermes-agent-immutability-job.yaml - bootstrap-jobs/cassandra-registry-ensure-job.yaml - image.yaml configMapGenerator: - name: harbor-vault-entrypoint files: - scripts/vault-entrypoint.sh + - name: harbor-hermes-agent-immutability-script + files: + - harbor_hermes_agent_immutability_ensure.py=scripts/harbor_hermes_agent_immutability_ensure.py diff --git a/services/harbor/policy-bootstrap-serviceaccount.yaml b/services/harbor/policy-bootstrap-serviceaccount.yaml new file mode 100644 index 00000000..4839196f --- /dev/null +++ b/services/harbor/policy-bootstrap-serviceaccount.yaml @@ -0,0 +1,7 @@ +# services/harbor/policy-bootstrap-serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: harbor-policy-bootstrap + namespace: harbor +automountServiceAccountToken: true diff --git a/services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py b/services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py new file mode 100644 index 00000000..06475c50 --- /dev/null +++ b/services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py @@ -0,0 +1,378 @@ +#!/usr/bin/env python3 +"""Create and verify the narrowly scoped Hermes agent immutable-tag rule.""" + +from __future__ import annotations + +import base64 +import json +import os +import time +import urllib.error +import urllib.parse +import urllib.request +from pathlib import Path +from typing import Any + + +PROJECT = "bstein" +REPOSITORY_PATTERN = "hermes-agent" +TAG_PATTERN = "git-*-build-*" +PUBLISH_ROBOT = "robot$jenkins-pipelines" +EXPECTED_ORIGIN = "https://registry.bstein.dev/api/v2.0" +MAX_RESPONSE = 1_048_576 +TRANSIENT_STATUSES = {429, 502, 503, 504} +EXPECTED_RULE = { + "disabled": False, + "action": "immutable", + "template": "immutable_template", + "tag_selectors": [ + { + "kind": "doublestar", + "decoration": "matches", + "pattern": TAG_PATTERN, + } + ], + "scope_selectors": { + "repository": [ + { + "kind": "doublestar", + "decoration": "repoMatches", + "pattern": REPOSITORY_PATTERN, + } + ] + }, +} + + +class NoRedirect(urllib.request.HTTPRedirectHandler): + """Prevent Basic credentials from following an unexpected redirect.""" + + def redirect_request(self, _request, _file, _code, _message, _headers, _url): + return None + + +class HarborUnavailable(RuntimeError): + """Harbor is not ready yet, rather than returning a policy decision.""" + + +def normalized_rule(rule: dict[str, Any]) -> dict[str, Any]: + """Return only the immutable contract fields Harbor must preserve.""" + return { + "disabled": bool(rule.get("disabled", False)), + "action": rule.get("action"), + "template": rule.get("template"), + "tag_selectors": [ + { + "kind": selector.get("kind"), + "decoration": selector.get("decoration"), + "pattern": selector.get("pattern"), + } + for selector in rule.get("tag_selectors") or [] + if isinstance(selector, dict) + ], + "scope_selectors": { + "repository": [ + { + "kind": selector.get("kind"), + "decoration": selector.get("decoration"), + "pattern": selector.get("pattern"), + } + for selector in (rule.get("scope_selectors") or {}).get( + "repository", [] + ) + if isinstance(selector, dict) + ] + }, + } + + +def targets_hermes_builds(rule: dict[str, Any]) -> bool: + """Detect a rule that claims this exact repository and tag selector.""" + normalized = normalized_rule(rule) + return ( + normalized["tag_selectors"] == EXPECTED_RULE["tag_selectors"] + and normalized["scope_selectors"] == EXPECTED_RULE["scope_selectors"] + ) + + +class HarborClient: + """Bounded same-origin client for Harbor's immutable-tag API.""" + + def __init__(self, origin: str, username: str, password: str) -> None: + normalized_origin = origin.rstrip("/") + if normalized_origin != EXPECTED_ORIGIN: + raise ValueError("Harbor API origin is not the pinned production API") + self.origin = normalized_origin + token = base64.b64encode(f"{username}:{password}".encode()).decode("ascii") + self.headers = {"Authorization": f"Basic {token}"} + self.opener = urllib.request.build_opener(NoRedirect()) + + def request( + self, method: str, path: str, payload: dict[str, Any] | None = None + ) -> tuple[int, bytes, dict[str, str]]: + """Issue one request, returning even non-2xx responses for strict checks.""" + data = None + headers = dict(self.headers) + if payload is not None: + data = json.dumps(payload, separators=(",", ":")).encode() + headers["Content-Type"] = "application/json" + request = urllib.request.Request( + f"{self.origin}{path}", data=data, headers=headers, method=method + ) + try: + response = self.opener.open(request, timeout=20) + except urllib.error.HTTPError as exc: + response = exc + except (urllib.error.URLError, TimeoutError) as exc: + raise HarborUnavailable("Harbor policy API is unavailable") from exc + with response: + body = response.read(MAX_RESPONSE + 1) + if len(body) > MAX_RESPONSE: + raise RuntimeError("Harbor response exceeded the size limit") + return int(response.status), body, dict(response.headers) + + +def list_rules(client: HarborClient) -> list[dict[str, Any]]: + """Read and validate the complete small rule set for the project.""" + path = f"/projects/{PROJECT}/immutabletagrules?page=1&page_size=100" + status, body, headers = client.request("GET", path) + if status in TRANSIENT_STATUSES: + raise HarborUnavailable(f"Harbor immutable rule list returned HTTP {status}") + if status != 200: + raise RuntimeError(f"Harbor immutable rule list returned HTTP {status}") + values = _json_list(body, "immutable rule") + _require_complete_page(values, headers, "immutable rule") + return values + + +def _json_list(body: bytes, label: str) -> list[dict[str, Any]]: + """Decode one bounded Harbor list without accepting a partial shape.""" + try: + value = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise RuntimeError(f"Harbor returned invalid {label} JSON") from exc + if not isinstance(value, list) or not all(isinstance(item, dict) for item in value): + raise RuntimeError(f"Harbor {label} list has an invalid shape") + return value + + +def _require_complete_page( + values: list[dict[str, Any]], headers: dict[str, str], label: str +) -> None: + """Reject a truncated first page or a proxy that strips count evidence.""" + raw_total = next( + (value for key, value in headers.items() if key.lower() == "x-total-count"), + None, + ) + if raw_total is None or not str(raw_total).isdecimal(): + raise RuntimeError(f"Harbor {label} list omitted a valid total count") + if int(raw_total) != len(values): + raise RuntimeError(f"Harbor {label} list was truncated") + + +def _allowed(access: dict[str, Any], resource: str, action: str) -> bool: + """Match one positive robot permission, accepting Harbor's default effect.""" + return ( + access.get("resource") == resource + and access.get("action") == action + and access.get("effect") in (None, "", "allow") + ) + + +def _publisher_scope(robot: dict[str, Any]) -> tuple[dict[str, Any], list[dict[str, Any]]]: + """Return the existing bstein scope after validating its push boundary.""" + permissions = robot.get("permissions") + if not isinstance(permissions, list) or not all( + isinstance(item, dict) for item in permissions + ): + raise RuntimeError("Jenkins Harbor robot permissions have an invalid shape") + matches = [ + item + for item in permissions + if item.get("kind") == "project" and item.get("namespace") == PROJECT + ] + if len(matches) != 1: + raise RuntimeError("Jenkins Harbor robot must have one existing bstein scope") + access = matches[0].get("access") + if not isinstance(access, list) or not all(isinstance(item, dict) for item in access): + raise RuntimeError("Jenkins Harbor robot bstein access has an invalid shape") + for required in (("repository", "pull"), ("repository", "push")): + if not any(_allowed(item, *required) for item in access): + raise RuntimeError("Jenkins Harbor robot lacks its existing pull/push boundary") + immutable_access = [ + item for item in access if item.get("resource") == "immutable-tag" + ] + if immutable_access and not ( + len(immutable_access) == 1 + and _allowed(immutable_access[0], "immutable-tag", "list") + ): + raise RuntimeError("Jenkins Harbor robot has broader immutable-tag access") + return matches[0], access + + +def _verified_publisher(robot: dict[str, Any]) -> bool: + """Check the exact identity and least-privilege policy after persistence.""" + if ( + robot.get("name") != PUBLISH_ROBOT + or robot.get("level") != "system" + or robot.get("editable") is not True + or robot.get("disable") is not False + ): + return False + try: + _scope, access = _publisher_scope(robot) + except RuntimeError: + return False + return any(_allowed(item, "immutable-tag", "list") for item in access) + + +def ensure_publisher_can_read_rule(client: HarborClient) -> int: + """Grant only immutable-tag:list to the existing Jenkins push robot.""" + status, body, headers = client.request("GET", "/robots?page=1&page_size=100") + if status in TRANSIENT_STATUSES: + raise HarborUnavailable(f"Harbor robot list returned HTTP {status}") + if status != 200: + raise RuntimeError(f"Harbor robot list returned HTTP {status}") + robots = _json_list(body, "robot") + _require_complete_page(robots, headers, "robot") + matches = [ + item + for item in robots + if item.get("name") == PUBLISH_ROBOT + ] + if len(matches) != 1: + raise RuntimeError("expected exactly one Jenkins Harbor publisher robot") + robot_id = matches[0].get("id") + if not isinstance(robot_id, int) or robot_id < 1: + raise RuntimeError("Jenkins Harbor robot omitted a valid ID") + status, body, _headers = client.request("GET", f"/robots/{robot_id}") + if status in TRANSIENT_STATUSES: + raise HarborUnavailable(f"Harbor robot read returned HTTP {status}") + if status != 200: + raise RuntimeError(f"Harbor robot read returned HTTP {status}") + try: + robot = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise RuntimeError("Harbor returned invalid robot JSON") from exc + if not isinstance(robot, dict): + raise RuntimeError("Harbor robot response has an invalid shape") + if ( + robot.get("name") != PUBLISH_ROBOT + or robot.get("level") != "system" + or robot.get("editable") is not True + or robot.get("disable") is not False + ): + raise RuntimeError("Jenkins Harbor publisher identity is not active and exact") + scope, access = _publisher_scope(robot) + if any(_allowed(item, "immutable-tag", "list") for item in access): + return robot_id + + # Harbor updates permissions through PUT /robots/{id}; its separate PATCH + # endpoint is the only secret-rotation operation. Preserve every current + # scope and field while adding the one read-only action. + access.append({"resource": "immutable-tag", "action": "list"}) + payload = { + "name": robot["name"], + "description": robot.get("description") or "", + "level": robot["level"], + "disable": robot["disable"], + "permissions": robot["permissions"], + } + duration = robot.get("duration") + if duration is not None: + if not isinstance(duration, int): + raise RuntimeError("Jenkins Harbor robot duration has an invalid shape") + payload["duration"] = duration + # Keep the validated object reference live in the preserved permission list. + if scope.get("access") is not access: + raise RuntimeError("Jenkins Harbor robot permission normalization drifted") + status, _body, _headers = client.request( + "PUT", f"/robots/{robot_id}", payload + ) + if status in TRANSIENT_STATUSES: + raise HarborUnavailable(f"Harbor robot update returned HTTP {status}") + if status != 200: + raise RuntimeError(f"Harbor robot policy update returned HTTP {status}") + + for attempt in range(1, 6): + status, body, _headers = client.request("GET", f"/robots/{robot_id}") + if status == 200: + try: + persisted = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError): + persisted = None + if isinstance(persisted, dict) and _verified_publisher(persisted): + return robot_id + if attempt < 5: + time.sleep(attempt) + raise RuntimeError("Jenkins Harbor robot policy did not verify exactly") + + +def ensure_rule(client: HarborClient) -> int: + """Create once, or validate the one exact enabled rule already present.""" + rules = list_rules(client) + matches = [rule for rule in rules if targets_hermes_builds(rule)] + if len(matches) > 1: + raise RuntimeError("multiple Hermes agent immutable rules exist") + if matches: + if normalized_rule(matches[0]) != EXPECTED_RULE: + raise RuntimeError("Hermes agent immutable rule exists but is not enabled/exact") + rule_id = matches[0].get("id") + if not isinstance(rule_id, int) or rule_id < 1: + raise RuntimeError("Harbor immutable rule omitted a valid ID") + return rule_id + + path = f"/projects/{PROJECT}/immutabletagrules" + status, _body, headers = client.request("POST", path, EXPECTED_RULE) + if status in TRANSIENT_STATUSES: + raise HarborUnavailable(f"Harbor immutable rule create returned HTTP {status}") + if status != 201: + raise RuntimeError(f"Harbor immutable rule create returned HTTP {status}") + location = headers.get("Location") or headers.get("location") or "" + api_path = urllib.parse.urlsplit(client.origin).path.rstrip("/") + expected_prefix = f"{api_path}{path}/" + if not location.startswith(expected_prefix): + raise RuntimeError("Harbor immutable rule create omitted the exact Location") + suffix = location[len(expected_prefix) :] + if not suffix.isdecimal() or int(suffix) < 1: + raise RuntimeError("Harbor immutable rule Location has an invalid ID") + + # Harbor persists synchronously, but bounded retries distinguish a stale + # read from accepting an unverified policy. + for attempt in range(1, 6): + matches = [rule for rule in list_rules(client) if targets_hermes_builds(rule)] + if len(matches) == 1 and normalized_rule(matches[0]) == EXPECTED_RULE: + rule_id = matches[0].get("id") + if rule_id == int(suffix): + return rule_id + if attempt < 5: + time.sleep(attempt) + raise RuntimeError("created Harbor immutable rule did not verify exactly") + + +def main() -> int: + """Load the runtime-only admin credential and enforce the tracked policy.""" + origin = os.environ.get("HARBOR_API_ORIGIN", "") + password_file = Path(os.environ.get("HARBOR_ADMIN_PASSWORD_FILE", "")) + password = password_file.read_text(encoding="utf-8").strip() + if not password: + raise RuntimeError("Harbor admin password is empty") + client = HarborClient(origin, "admin", password) + for attempt in range(1, 13): + try: + rule_id = ensure_rule(client) + robot_id = ensure_publisher_can_read_rule(client) + break + except HarborUnavailable: + if attempt == 12: + raise + time.sleep(min(attempt * 2, 15)) + print( + "Hermes agent immutable build-tag rule is active " + f"(id={rule_id}); publisher preflight access is active (robot={robot_id})" + ) + return 0 + + +if __name__ == "__main__": # pragma: no cover - exercised through main() + raise SystemExit(main()) diff --git a/services/hermes-observer-bindings/kustomization.yaml b/services/hermes-observer-bindings/kustomization.yaml new file mode 100644 index 00000000..4dbf36b5 --- /dev/null +++ b/services/hermes-observer-bindings/kustomization.yaml @@ -0,0 +1,5 @@ +# services/hermes-observer-bindings/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - rolebindings.yaml diff --git a/services/hermes-observer-bindings/rolebindings.yaml b/services/hermes-observer-bindings/rolebindings.yaml new file mode 100644 index 00000000..403cd39a --- /dev/null +++ b/services/hermes-observer-bindings/rolebindings.yaml @@ -0,0 +1,204 @@ +# services/hermes-observer-bindings/rolebindings.yaml +apiVersion: v1 +kind: List +items: + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: ai} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: bstein-dev-home} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: cassandra} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: cert-manager} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: climate} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: comms} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: crypto} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: default} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: finance} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: flux-system} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: game-stream} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: gitea} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: harbor} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: health} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: hermes} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: hermes-chat} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: hermes-triage-demo} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: jellyfin} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: jenkins} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: kube-node-lease} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: kube-public} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: kube-system} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: logging} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: longhorn-system} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: mailu-mailserver} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: maintenance} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: metallb-system} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: monitoring} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: nextcloud} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: openclaw} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: outline} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: planka} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: postgres} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: quality} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: sso} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: sui-metrics} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: traefik} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: vault} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: vaultwarden} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: veles} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} diff --git a/services/hermes-observer-rbac/kustomization.yaml b/services/hermes-observer-rbac/kustomization.yaml new file mode 100644 index 00000000..cff6f4b9 --- /dev/null +++ b/services/hermes-observer-rbac/kustomization.yaml @@ -0,0 +1,5 @@ +# services/hermes-observer-rbac/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - rbac.yaml diff --git a/services/hermes-observer-rbac/rbac.yaml b/services/hermes-observer-rbac/rbac.yaml new file mode 100644 index 00000000..737ac4dd --- /dev/null +++ b/services/hermes-observer-rbac/rbac.yaml @@ -0,0 +1,112 @@ +# services/hermes-observer-rbac/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: hermes-agent-cluster-observer-v2 + labels: + app.kubernetes.io/name: hermes-agent + app.kubernetes.io/part-of: hermes +rules: + - apiGroups: [""] + resources: + - namespaces + - nodes + - persistentvolumes + verbs: ["get", "list", "watch"] + - apiGroups: ["networking.k8s.io"] + resources: ["ingressclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: + - csidrivers + - csinodes + - storageclasses + - volumeattachments + verbs: ["get", "list", "watch"] + - apiGroups: ["metrics.k8s.io"] + resources: ["nodes"] + verbs: ["get", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: hermes-agent-namespaced-observer-v2 + labels: + app.kubernetes.io/name: hermes-agent + app.kubernetes.io/part-of: hermes +rules: + - apiGroups: [""] + resources: + - configmaps + - endpoints + - events + - persistentvolumeclaims + - pods + - pods/log + - replicationcontrollers + - resourcequotas + - services + verbs: ["get", "list", "watch"] + - apiGroups: ["apps"] + resources: + - controllerrevisions + - daemonsets + - deployments + - replicasets + - statefulsets + verbs: ["get", "list", "watch"] + - apiGroups: ["batch"] + resources: ["cronjobs", "jobs"] + verbs: ["get", "list", "watch"] + - apiGroups: ["autoscaling"] + resources: ["horizontalpodautoscalers"] + verbs: ["get", "list", "watch"] + - apiGroups: ["networking.k8s.io"] + resources: ["ingresses", "networkpolicies"] + verbs: ["get", "list", "watch"] + - apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["get", "list", "watch"] + - apiGroups: ["policy"] + resources: ["poddisruptionbudgets"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csistoragecapacities"] + verbs: ["get", "list", "watch"] + - apiGroups: ["metrics.k8s.io"] + resources: ["pods"] + verbs: ["get", "list"] + - apiGroups: + - helm.toolkit.fluxcd.io + - image.toolkit.fluxcd.io + - kustomize.toolkit.fluxcd.io + - notification.toolkit.fluxcd.io + - source.toolkit.fluxcd.io + resources: ["*"] + verbs: ["get", "list", "watch"] + - apiGroups: ["longhorn.io"] + resources: + - backingimages + - engines + - instancemanagers + - nodes + - replicas + - settings + - volumes + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: hermes-agent-cluster-observer-v2 + labels: + app.kubernetes.io/name: hermes-agent + app.kubernetes.io/part-of: hermes +subjects: + - kind: ServiceAccount + name: hermes-agent + namespace: hermes +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: hermes-agent-cluster-observer-v2 diff --git a/services/hermes-scm-broker/deployment.yaml b/services/hermes-scm-broker/deployment.yaml new file mode 100644 index 00000000..649ada1d --- /dev/null +++ b/services/hermes-scm-broker/deployment.yaml @@ -0,0 +1,98 @@ +# services/hermes-scm-broker/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hermes-scm-broker + namespace: hermes-scm + labels: + app: hermes-scm-broker +spec: + replicas: 1 + revisionHistoryLimit: 2 + selector: + matchLabels: + app: hermes-scm-broker + template: + metadata: + labels: + app: hermes-scm-broker + annotations: + vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/role: hermes-scm-broker + vault.hashicorp.com/agent-inject-secret-gitea-token: kv/data/atlas/hermes/developer-gitea + vault.hashicorp.com/agent-inject-template-gitea-token: | + {{- with secret "kv/data/atlas/hermes/developer-gitea" -}} + {{ .Data.data.token }} + {{- end }} + vault.hashicorp.com/agent-pre-populate-only: "true" + vault.hashicorp.com/agent-init-first: "true" + vault.hashicorp.com/agent-requests-cpu: 10m + vault.hashicorp.com/agent-requests-mem: 32Mi + vault.hashicorp.com/agent-limits-cpu: 50m + vault.hashicorp.com/agent-limits-mem: 64Mi + spec: + serviceAccountName: hermes-scm-broker + automountServiceAccountToken: true + securityContext: + fsGroup: 10000 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: + type: RuntimeDefault + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: [arm64] + - key: node-role.kubernetes.io/worker + operator: In + values: ["true"] + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: hardware + operator: In + values: [rpi5] + containers: + - name: broker + image: registry.bstein.dev/bstein/hermes-agent + imagePullPolicy: IfNotPresent + command: + - /opt/hermes/.venv/bin/python + - /opt/broker/scm_broker.py + ports: + - {name: http, containerPort: 9081, protocol: TCP} + readinessProbe: + httpGet: {path: /healthz, port: http} + initialDelaySeconds: 3 + periodSeconds: 10 + livenessProbe: + httpGet: {path: /healthz, port: http} + initialDelaySeconds: 15 + periodSeconds: 30 + failureThreshold: 10 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 10000 + runAsGroup: 10000 + volumeMounts: + - {name: broker-code, mountPath: /opt/broker, readOnly: true} + - {name: tmp, mountPath: /tmp} + resources: + requests: {cpu: 50m, memory: 128Mi} + limits: {cpu: "1", memory: 768Mi} + volumes: + - name: broker-code + configMap: + name: hermes-scm-boundary + defaultMode: 0555 + - name: tmp + emptyDir: + sizeLimit: 1Gi diff --git a/services/hermes-scm-broker/kustomization.yaml b/services/hermes-scm-broker/kustomization.yaml new file mode 100644 index 00000000..5dac3620 --- /dev/null +++ b/services/hermes-scm-broker/kustomization.yaml @@ -0,0 +1,11 @@ +# services/hermes-scm-broker/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: hermes-scm +resources: + - service.yaml + - deployment.yaml + - networkpolicy.yaml +images: + - name: registry.bstein.dev/bstein/hermes-agent + digest: sha256:37ebf720c783ae908a602916ffccf88d43d205a157957f5dc4b487867aee45e7 diff --git a/services/hermes-scm-broker/networkpolicy.yaml b/services/hermes-scm-broker/networkpolicy.yaml new file mode 100644 index 00000000..8187e333 --- /dev/null +++ b/services/hermes-scm-broker/networkpolicy.yaml @@ -0,0 +1,46 @@ +# services/hermes-scm-broker/networkpolicy.yaml +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: hermes-scm-broker-isolation + namespace: hermes-scm +spec: + podSelector: + matchLabels: + app: hermes-scm-broker + policyTypes: [Ingress, Egress] + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: hermes + podSelector: + matchLabels: + app: hermes-agent + ports: + - {protocol: TCP, port: 9081} + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - {protocol: UDP, port: 53} + - {protocol: TCP, port: 53} + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: vault + podSelector: + matchLabels: + app: vault + ports: + - {protocol: TCP, port: 8200} + - to: + - ipBlock: + cidr: 192.168.22.9/32 + ports: + - {protocol: TCP, port: 443} diff --git a/services/hermes-scm-broker/service.yaml b/services/hermes-scm-broker/service.yaml new file mode 100644 index 00000000..0b08c7f6 --- /dev/null +++ b/services/hermes-scm-broker/service.yaml @@ -0,0 +1,13 @@ +# services/hermes-scm-broker/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: hermes-scm-broker + namespace: hermes-scm +spec: + selector: + app: hermes-scm-broker + ports: + - name: http + port: 9081 + targetPort: http diff --git a/services/hermes-scm-namespace/kustomization.yaml b/services/hermes-scm-namespace/kustomization.yaml new file mode 100644 index 00000000..a9e89c7b --- /dev/null +++ b/services/hermes-scm-namespace/kustomization.yaml @@ -0,0 +1,5 @@ +# services/hermes-scm-namespace/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - namespace.yaml diff --git a/services/hermes-scm-namespace/namespace.yaml b/services/hermes-scm-namespace/namespace.yaml new file mode 100644 index 00000000..542f9602 --- /dev/null +++ b/services/hermes-scm-namespace/namespace.yaml @@ -0,0 +1,14 @@ +# services/hermes-scm-namespace/namespace.yaml +apiVersion: v1 +kind: Namespace +metadata: + name: hermes-scm + labels: + app.kubernetes.io/name: hermes-scm-broker +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: hermes-scm-broker + namespace: hermes-scm +automountServiceAccountToken: true diff --git a/services/hermes/NOTES.md b/services/hermes/NOTES.md index c70e528a..aec53d0c 100644 --- a/services/hermes/NOTES.md +++ b/services/hermes/NOTES.md @@ -102,6 +102,110 @@ console tails and named artifact contents in its deterministic bundle. A report must say `retained Ariadne evidence` when that fallback is used; it must not pretend direct Jenkins access succeeded. +## Publishing an agent image after review + +The `hermes-agent-image` Jenkins job is the only supported agent image builder. +It runs daemonless Kaniko v1.23.2 under the unbound `hermes-image-builder` +ServiceAccount without a service-account token, host socket, +privileged container, or writable Git credential. It accepts only an exact +40-character revision that is both the checked-out commit and current +`atlas/titan-iac` `main`, so a human must merge the source PR first. + +The build pod requires an ARM64 worker and gives `hardware=rpi5` the maximum +scheduler preference. Healthy rpi4 workers remain valid fallback capacity when +the rpi5 pool cannot fit the full request. Cordoned, unhealthy, amd64, and +reserved last-resort nodes remain excluded by the required placement boundary. + +Kaniko runs as UID 0 because it must unpack an image root filesystem and enter +that filesystem for Dockerfile `RUN` instructions. Its capability set is the +minimum proven by an exact ARM64 no-push build of this Dockerfile: +`CHOWN`, `FOWNER`, `DAC_OVERRIDE`, `SETGID`, and `SETUID`. `SETGID` is required +because pinned Kaniko applies the base image's supplementary group list before +each Dockerfile `RUN`; `SETUID` lets apt drop privileges to its `_apt` account +while downloading package indexes. It still has no privilege +escalation, service-account token, host path, daemon socket, or Docker/BuildKit +TCP endpoint and uses the runtime-default seccomp profile. This is residual +root-in-the-build-pod risk, bounded to disposable `emptyDir` storage and a +human-reviewed `main` revision; Dockerfile changes require the same scrutiny as +executable cluster code. + +Pinned Kaniko v1.23.2 parses Dockerfile `RUN` heredocs but does not materialize +their inline files when executing a command. The Jenkins lane therefore enables +a bounded compatibility replay after copying the reviewed Dockerfile and runner +from the same checked-out commit. Before every replay, the runner inventories +the entire Dockerfile as logical instructions, including split opcodes, +operators, and either Dockerfile escape character. It rejects every heredoc +form outside the exact nine Node/Python blocks before launching an interpreter. +Each accepted body is then replayed immediately after its corresponding Kaniko +`RUN`, preserving source order before any dependent regression. Docker and +BuildKit keep their native behavior because the compatibility argument defaults +off. The final TypeScript build, Python compile, and source assertions remain +mandatory, so an omitted or drifted replay fails before any image can pass +release verification. + +From agent.hermes, trigger that one fixed job with: + +```sh +jenkins_image_build_trigger.py '' +``` + +The helper has no general Jenkins credential or caller-selectable job name. Its +Vault-projected token is bound by Jenkins only to `hermes-agent-image`. The job +also requires its fixed publish confirmation. Each run claims the unique tag +`git--build-` and refuses to overwrite an +existing tag. It cross-checks Kaniko's digest and tagged-image evidence, then +independently reads the pushed tag from Harbor before archiving a JSON record +and Flux digest patch. A successful post-condition then revalidates the exact +six-file evidence set and archives those six explicit paths with empty archives +forbidden. Apply that patch on a new branch and submit it for human review; the +build never changes Git, reconciles Flux, or deploys by itself. +The helper posts only to the fixed HTTPS Build Token Root endpoint and accepts +only its real queue responses: HTTP 201 or a non-followed HTTP 303 with an exact +same-origin `/queue/item//` location. An unauthenticated request is +denied, and the token cannot select, configure, read, or administer another +Jenkins job. The pipeline independently rejects any commit that is not the +current `origin/main`, preserving the human merge/review boundary. + +Harbor independently enforces an enabled immutable-tag rule scoped to only the +`bstein/hermes-agent` repository and `git-*-build-*` tags. A second manifest PUT, +retag, or deletion is rejected by Harbor even if a caller bypasses the Jenkins +preflight. A revisioned, Flux-tracked policy Job creates or verifies that exact +rule with a runtime-only Vault credential; it refuses to alter a conflicting +rule and retries only explicit transport/readiness failures while Harbor +starts. The same Job grants the existing Jenkins publisher only +`immutable-tag:list` on `bstein`, preserving its other project scopes and never +calling Harbor's separate secret-rotation endpoint. The pipeline uses that +read-only permission to require the exact enabled rule before Kaniko starts. +That check also makes an already-running Jenkins controller fail closed during +rollout; Jenkins cannot become Ready on the new revision before the policy Job +succeeds. + +The tracked rollout order is deliberate: the revisioned Vault role Job must +complete before the Vault Kustomization becomes Ready. A separate Flux +Kustomization then runs the seed Job under the dedicated +`hermes-jenkins-token-seed` identity, which can only create/read the exact +`kv/data/atlas/hermes/developer-jenkins` path and request random bytes. Jenkins +depends on both that seed and the Harbor policy Job; Hermes depends on Jenkins. +The seeder uses KV-v2 CAS create-only semantics. It never changes an existing +token, never fills a missing field in an existing secret, and fails closed when +it cannot distinguish absence from a read error. The recurring Vault +configuration job reconciles the same narrow role and policy. + +Rotate this fixed-job token only as a coordinated operator action: + +1. Stop new `hermes-agent-image` triggers and wait for its queue and executor to + drain. +2. Read the current KV-v2 metadata version for + `kv/atlas/hermes/developer-jenkins`. +3. Generate a fresh value from Vault and patch only `build_token` with + `vault kv patch -cas=`. Keep the value in a mode-0600 + temporary file or standard input, never a command argument or log. +4. Roll Jenkins first and wait until it is Ready, then roll `hermes-agent` and + wait until it is Ready. Do not resume triggers between those two consumers. +5. Run one reviewed-main canary. If rollback is required, repeat the CAS patch + with the previous value as another coordinated rotation; never delete the + secret to make the seeder recreate it. + ## The actual supervised triage algorithm 1. Classify the request as test/build triage, service health, or alert tuning. diff --git a/services/hermes/agent-configmap.yaml b/services/hermes/agent-configmap.yaml index cc49951c..91da82f6 100644 --- a/services/hermes/agent-configmap.yaml +++ b/services/hermes/agent-configmap.yaml @@ -197,9 +197,9 @@ data: not replace these coordinator-wide rules. Never call `kanban_show` without a known, non-empty task ID. Ad-hoc inspection and acceptance checks do not need a synthetic Kanban lookup, and must load a skill only when its workflow - materially applies. Atlas HTTPS Git authentication is already supplied by - the runtime-only `GIT_ASKPASS`; use it without reading or exposing the - credential. Coordinator guidance lives at + materially applies. Atlas Git access is supplied by the isolated SCM + broker; no repository token is present in this pod. Use only configured + broker remotes and clients. Coordinator guidance lives at `/opt/data/workspace/AGENTS.md` when more detail is needed. The Jetson classifier is mandatory for AUTO selection. Switchyard may use @@ -303,70 +303,75 @@ data: objective's difficulty warrants it; otherwise synthesize at the original effort. - This owner-only pod has cluster-admin access across Atlas, including logs, - Secrets, exec, port-forwarding, rollout operations, and Flux reconciliation. - Prefer the titan-iac Git/Flux workflow for every durable cluster change; - direct operations are available for explicit operator requests, incident - recovery, and verification, and must be followed by a matching source-of- - truth change when they alter desired state. Never expose credentials in - chat or logs. Triage belongs at triage.hermes.bstein.dev. + This owner-only pod has scoped read-only Kubernetes diagnostics across + Atlas, including resource status, events, logs, and Flux/Helm evidence. It + cannot read Secrets, exec or attach to pods, create service-account tokens, + mutate workloads or RBAC, or reconcile Flux. Put every durable cluster + change on a reviewed titan-iac branch. Never expose credentials in chat or + logs. Triage belongs at triage.hermes.bstein.dev. ## Atlas engineering access The Atlas organization has private visibility. Repository visibility is preserved per project and may be public or private; do not infer a repository's visibility from the organization setting. Repositories are - canonical at `https://scm.bstein.dev/atlas/.git`. HTTPS Git - authentication is already supplied through `GIT_ASKPASS`. Verify the remote + canonical at `https://scm.bstein.dev/atlas/.git`; worker Git traffic + uses `http://hermes-scm-broker.hermes-scm.svc.cluster.local:9081/git/atlas/.git`. + Verify the broker remote and cleanly separate pre-existing changes, create a task branch, run the repository's tests, use `git push --dry-run` when proving access, and push a real branch only when the requested implementation is review-ready. Never force-push. - `scm.bstein.dev` is Gitea, not GitHub. Never load or follow a GitHub/`gh` - skill for an Atlas remote, and do not interpret an unauthenticated Gitea - HTTP 404 as a missing private repository. Use authenticated Git for clone, - fetch, and push. For pull-request metadata, comments, and merges, call - `/opt/coordinator/gitea_api.py METHOD /api/v1/...`; it injects the runtime - Vault token without exposing it to the command line, environment, output, - or transcript. Before a consequential merge, independently verify the - exact base/head ancestry and diff, run the relevant tests, and confirm the - candidate Jenkins result. Prefer the internal endpoint in - `JENKINS_BASE_URL`; when Jenkins API authorization prevents a read, use the - existing cluster access to inspect the controller's job/build files and - logs rather than guessing a public hostname. The preferred read-only path - is `/opt/coordinator/jenkins_build_evidence.py JOB [--branch BRANCH] - [--commit SHA] --wait`; it distinguishes a genuinely terminal build from - nested Jenkins execution metadata and returns bounded JSON/log evidence. - With a branch, the helper also tries the conventional `JOB-branches` - multibranch name. For JSON Gitea mutations, prefer repeatable shell-safe - `--field KEY=VALUE` arguments over hand-quoted JSON. - After merging, observe the default-branch build to a terminal result and - report exact commit, build, and test evidence. + `scm.bstein.dev` is Forgejo/Gitea, not GitHub. Never load or follow a + GitHub/`gh` skill for an Atlas remote, and do not interpret an + unauthenticated Gitea HTTP 404 as a missing private repository. Use + brokered Git for clone, fetch, and creation of a new namespaced feature + branch. Existing-ref updates, protected refs, deletion, and force-push are + rejected. The broker inflates and scans every pushed object, so thin + packs are rejected; always push with `git push --no-thin` so the pack + is self-contained. For bounded + repository/pull-request evidence or to create a review-ready draft PR, load + `$manage-atlas-pull-requests` and use `/opt/scm/gitea_api.py`. The + client carries no repository credential; a separate least-authority + workload performs the allowed operation. It permits only Atlas-scoped + metadata reads and verified same-repository draft creation. Updates, merge, + approve, close, delete, comments, repository administration, and force-push + are unavailable; never bypass the client with raw HTTP. Leave every PR + unmerged for Brad's review. Independently + verify the exact base/head ancestry and diff, run the relevant tests, and + confirm the candidate Jenkins result before presenting a handoff. Prefer + the internal endpoint in + `JENKINS_BASE_URL`; when Jenkins API authorization prevents a read, use + Kubernetes pod logs and status evidence without exec or secret access. + Do not bypass the boundary to inspect controller files. If Brad separately + reports that he merged the PR, observe + the default-branch build to a terminal result and report exact commit, + build, and test evidence. The workspace already has a non-secret Hermes Git author identity. Do not use `git reset --hard`, even in a fresh clone; use a detached worktree or a clean branch switch when comparing revisions, and preserve any unexpected - file as user state. Use the Gitea pull-request merge endpoint for an open - PR so both Git history and PR state remain auditable. If an authorized - manual merge already reached the base branch, reconcile the open PR with - `Do=manually-merged` and its exact merge commit instead of leaving stale - review state. + file as user state. PR publication ends at the verified open draft; Brad + owns review and merge authority. The terminal PATH contains the pinned operator tools. Start cluster work with `kubectl config current-context`, read-only status/events/logs, and the - relevant `titan-iac` manifests. Put durable desired-state changes on a - reviewable `titan-iac` branch, validate Kustomize and client dry-run, then - use Flux reconciliation after the tracked change is published. Direct - `kubectl` mutations are for explicit incident recovery or ephemeral - verification, not ordinary delivery. + relevant `titan-iac` manifests. The broker namespace is deliberately + excluded from agent RBAC. Kubernetes cannot express a deny on one namespace + inside an all-namespace list, so enumerate namespaces and run namespaced + diagnostics instead of relying on `kubectl ... --all-namespaces`. Put + durable desired-state changes on a + reviewable `titan-iac` branch and validate Kustomize and client dry-run. + Brad owns merge and Flux reconciliation. Direct `kubectl` mutation is not + available to the agent; hand an explicit incident mutation back to Brad. - Node SSH uses a dedicated audited Hermes identity: `ssh titan-04` (or any - current Kubernetes node name). Host keys are pinned and password fallback - is disabled. Use SSH only for host-level evidence or repairs that cannot be - performed through Kubernetes. Read first, identify the exact node and - impact, avoid fleet-wide destructive commands, and reflect persistent host - configuration in the appropriate tracked provisioning source. + Node SSH uses the dedicated locked-password `hermes-agent` OS account: + `ssh titan-04` (or any current Kubernetes node name). Host keys are pinned, + password fallback is disabled, and the account has no sudo, disk, runtime, + or Kubernetes-storage authority. Use it for unprivileged host evidence; + hand privileged host repair back to Brad and reflect persistent changes in + the tracked provisioning source. Treat every credential, SSH identity, host-trust record, and credential-bearing access-client state as runtime-only Vault data. Keep @@ -403,7 +408,7 @@ data: [model]` for a persistent override. The first native Codex worker requires one device-code login; refreshed provider credentials persist through Vault while client caches remain disposable. The owner - workspace includes cluster-admin Kubernetes - access plus `kubectl`, `flux`, `helm`, `kustomize`, `vault`, `sops`, `age`, + workspace includes scoped read-only Kubernetes diagnostics plus `kubectl`, + `flux`, `helm`, `kustomize`, `vault`, `sops`, `age`, `terraform`, `k9s`, `jq`, `yq`, `gh`, Git, SSH, Python, Node, the browser/computer tools, and the native provider CLIs. diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml index 97ec0e81..26a025ef 100644 --- a/services/hermes/agent-deployment.yaml +++ b/services/hermes/agent-deployment.yaml @@ -46,15 +46,10 @@ spec: {{- with secret "kv/data/atlas/hermes/agent-tokens" -}} {{ .Data.data.codex_auth_json }} {{- end }} - vault.hashicorp.com/agent-inject-secret-gitea-token: kv/data/atlas/hermes/developer-gitea - vault.hashicorp.com/agent-inject-template-gitea-token: | - {{- with secret "kv/data/atlas/hermes/developer-gitea" -}} - {{ .Data.data.token }} - {{- end }} - vault.hashicorp.com/agent-inject-secret-gitea-username: kv/data/atlas/hermes/developer-gitea - vault.hashicorp.com/agent-inject-template-gitea-username: | - {{- with secret "kv/data/atlas/hermes/developer-gitea" -}} - {{ .Data.data.username }} + vault.hashicorp.com/agent-inject-secret-jenkins-image-build-token: kv/data/atlas/hermes/developer-jenkins + vault.hashicorp.com/agent-inject-template-jenkins-image-build-token: | + {{- with secret "kv/data/atlas/hermes/developer-jenkins" -}} + {{ .Data.data.build_token }} {{- end }} vault.hashicorp.com/agent-inject-secret-node-ssh-private-key: kv/data/atlas/hermes/developer-ssh vault.hashicorp.com/agent-inject-template-node-ssh-private-key: | @@ -186,9 +181,8 @@ spec: chmod 0600 "${profile_env}" chown 10000:10000 "${profile_env}" done - upsert_env GIT_ASKPASS /opt/coordinator/gitea_askpass.sh upsert_env GIT_TERMINAL_PROMPT 0 - upsert_env GITEA_BASE_URL https://scm.bstein.dev + upsert_env HERMES_SCM_BROKER_URL http://hermes-scm-broker.hermes-scm.svc.cluster.local:9081 upsert_env JENKINS_BASE_URL http://jenkins.jenkins.svc.cluster.local:8080 upsert_env ARIADNE_BASE_URL http://ariadne.maintenance.svc.cluster.local upsert_env VICTORIA_METRICS_URL http://victoria-metrics-single-server.monitoring.svc.cluster.local:8428 @@ -663,6 +657,8 @@ spec: - {name: subprocess-secret-patch, mountPath: /opt/hermes/tools/process_registry.py, subPath: process_registry.py} - {name: kubeconfig, mountPath: /opt/data/home/.kube/config, subPath: config, readOnly: true} - {name: auto-router-plugin, mountPath: /opt/data/plugins/auto-router, readOnly: true} + - {name: atlas-pr-skill, mountPath: /opt/data/workspace/skills/manage-atlas-pull-requests, readOnly: true} + - {name: scm-boundary, mountPath: /opt/scm, readOnly: true} - {name: routing-catalog, mountPath: /routing-catalog, readOnly: true} - {name: tmp, mountPath: /tmp} startupProbe: @@ -826,6 +822,8 @@ spec: - {name: subprocess-secret-patch, mountPath: /opt/hermes/tools/process_registry.py, subPath: process_registry.py} - {name: kubeconfig, mountPath: /opt/data/home/.kube/config, subPath: config, readOnly: true} - {name: auto-router-plugin, mountPath: /opt/data/plugins/auto-router, readOnly: true} + - {name: atlas-pr-skill, mountPath: /opt/data/workspace/skills/manage-atlas-pull-requests, readOnly: true} + - {name: scm-boundary, mountPath: /opt/scm, readOnly: true} - {name: routing-catalog, mountPath: /routing-catalog, readOnly: true} - {name: tmp, mountPath: /tmp} - {name: ttyd-index, mountPath: /ttyd-index, readOnly: true} @@ -899,6 +897,8 @@ spec: - {name: auth-patch, mountPath: /opt/hermes/hermes_cli/auth.py, subPath: auth.py} - {name: kubeconfig, mountPath: /opt/data/home/.kube/config, subPath: config, readOnly: true} - {name: auto-router-plugin, mountPath: /opt/data/plugins/auto-router, readOnly: true} + - {name: atlas-pr-skill, mountPath: /opt/data/workspace/skills/manage-atlas-pull-requests, readOnly: true} + - {name: scm-boundary, mountPath: /opt/scm, readOnly: true} - {name: tmp, mountPath: /tmp} readinessProbe: exec: @@ -907,8 +907,8 @@ spec: - /opt/coordinator/cli_lane_capabilities.py initialDelaySeconds: 2 periodSeconds: 5 - timeoutSeconds: 2 - failureThreshold: 1 + timeoutSeconds: 5 + failureThreshold: 3 resources: requests: {cpu: 100m, memory: 256Mi} limits: {cpu: "2", memory: 6Gi} @@ -1202,6 +1202,16 @@ spec: - {key: dashboard-api.py, path: dashboard/plugin_api.py} - {key: dashboard-index.js, path: dashboard/dist/index.js} - {key: dashboard-style.css, path: dashboard/dist/style.css} + - name: atlas-pr-skill + configMap: + name: hermes-atlas-pr-skill + items: + - {key: SKILL.md, path: SKILL.md} + - {key: openai.yaml, path: agents/openai.yaml} + - name: scm-boundary + configMap: + name: hermes-scm-boundary + defaultMode: 0555 - name: image-policy configMap: name: hermes-image-policy diff --git a/services/hermes/agent-kubeconfig.yaml b/services/hermes/agent-kubeconfig.yaml index 15b38da9..9ad9242f 100644 --- a/services/hermes/agent-kubeconfig.yaml +++ b/services/hermes/agent-kubeconfig.yaml @@ -11,9 +11,9 @@ users: user: tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token contexts: - - name: atlas-owner + - name: atlas-observer context: cluster: atlas user: hermes-agent namespace: default -current-context: atlas-owner +current-context: atlas-observer diff --git a/services/hermes/agent-rbac.yaml b/services/hermes/agent-rbac.yaml deleted file mode 100644 index 9dbff9e3..00000000 --- a/services/hermes/agent-rbac.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# services/hermes/agent-rbac.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: hermes-agent-cluster-admin - labels: - app.kubernetes.io/name: hermes-agent - app.kubernetes.io/part-of: hermes -subjects: - - kind: ServiceAccount - name: hermes-agent - namespace: hermes -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cluster-admin diff --git a/services/hermes/kustomization.yaml b/services/hermes/kustomization.yaml index 9c8df595..978d2941 100644 --- a/services/hermes/kustomization.yaml +++ b/services/hermes/kustomization.yaml @@ -7,13 +7,13 @@ images: digest: sha256:37ebf720c783ae908a602916ffccf88d43d205a157957f5dc4b487867aee45e7 resources: - namespace.yaml + - scm-common - vault-serviceaccount.yaml - configmap.yaml - agent-configmap.yaml - chat-configmap.yaml - switchyard-configmap.yaml - rbac.yaml - - agent-rbac.yaml - node-ssh-access.yaml - pvc.yaml - switchyard-pvc.yaml @@ -84,14 +84,13 @@ configMapGenerator: - classifier_broker.py=scripts/classifier_broker.py - claude_oauth_broker.py=scripts/claude_oauth_broker.py - worker_route_broker.py=scripts/worker_route_broker.py - - gitea_api.py=scripts/gitea_api.py - - gitea_askpass.sh=scripts/gitea_askpass.sh - hermes_coordinator.py=scripts/hermes_coordinator.py - hermes_model_routing.py=scripts/hermes_model_routing.py - hermes_stt_client.py=scripts/hermes_stt_client.py - image_broker.py=scripts/image_broker.py - install_agent_tools.sh=scripts/install_agent_tools.sh - jenkins_build_evidence.py=scripts/jenkins_build_evidence.py + - jenkins_image_build_trigger.py=scripts/jenkins_image_build_trigger.py - kanban_status_recovery.py=scripts/kanban_status_recovery.py - migrate_herdr_state.py=scripts/migrate_herdr_state.py - migrate_api_session_lineage.py=scripts/migrate_api_session_lineage.py @@ -119,6 +118,15 @@ configMapGenerator: - config=agent-kubeconfig.yaml options: disableNameSuffixHash: true + - name: hermes-node-account-hardener + namespace: hermes + files: + - node_account_hardening.py=scripts/node_account_hardening.py + - node_account_audit.py=scripts/node_account_audit.py + - node_account_io.py=scripts/node_account_io.py + - node_polkit_audit.py=scripts/node_polkit_audit.py + options: + disableNameSuffixHash: true - name: hermes-auto-router-plugin namespace: hermes files: @@ -173,3 +181,10 @@ configMapGenerator: - alert-review.md=skills/tune-atlas-alerts/references/alert-review.md options: disableNameSuffixHash: true + - name: hermes-atlas-pr-skill + namespace: hermes + files: + - SKILL.md=skills/manage-atlas-pull-requests/SKILL.md + - openai.yaml=skills/manage-atlas-pull-requests/agents/openai.yaml + options: + disableNameSuffixHash: true diff --git a/services/hermes/networkpolicy.yaml b/services/hermes/networkpolicy.yaml index 3205230e..461a7973 100644 --- a/services/hermes/networkpolicy.yaml +++ b/services/hermes/networkpolicy.yaml @@ -120,11 +120,65 @@ spec: ports: - {protocol: TCP, port: 9010} - {protocol: TCP, port: 9011} - # agent.hermes.bstein.dev is an owner-only engineering workstation. The - # browser boundary remains OAuth-protected, while its workers need to reach - # every cluster namespace, Atlas LAN service, and hosted provider endpoint. egress: - - {} + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - {protocol: UDP, port: 53} + - {protocol: TCP, port: 53} + - to: + - namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: NotIn + values: [gitea, hermes-scm] + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: hermes-scm + podSelector: + matchLabels: + app: hermes-scm-broker + ports: + - {protocol: TCP, port: 9081} + - to: + - ipBlock: + cidr: 10.43.0.1/32 + ports: + - {protocol: TCP, port: 443} + - to: + - ipBlock: + cidr: 192.168.0.0/16 + except: + - 192.168.22.9/32 + - to: + - ipBlock: + cidr: 0.0.0.0/0 + except: + - 10.0.0.0/8 + - 100.64.0.0/10 + - 127.0.0.0/8 + - 169.254.0.0/16 + - 172.16.0.0/12 + - 192.168.0.0/16 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: hermes-node-ssh-access-isolation + namespace: hermes +spec: + podSelector: + matchLabels: + app: hermes-node-ssh-access + policyTypes: [Ingress, Egress] + ingress: [] + egress: [] --- apiVersion: networking.k8s.io/v1 kind: NetworkPolicy diff --git a/services/hermes/node-ssh-access.yaml b/services/hermes/node-ssh-access.yaml index d9c6b223..04b0c78b 100644 --- a/services/hermes/node-ssh-access.yaml +++ b/services/hermes/node-ssh-access.yaml @@ -45,32 +45,16 @@ spec: - operator: Exists containers: - name: key-reconciler - image: busybox:1.37 + # The pinned index supports the cluster's amd64 and arm64 nodes. + image: python@sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df imagePullPolicy: IfNotPresent command: [/bin/sh, -ec] args: - | - reconcile() { - key="$(cat /vault/secrets/node-ssh-public-key)" - found=0 - for user in atlas oceanus; do - home="/host-home/${user}" - [ -d "${home}" ] || continue - found=1 - identity="$(awk -F: -v name="${user}" '$1 == name {print $3 ":" $4; exit}' /host-etc/passwd)" - [ -n "${identity}" ] || identity="$(stat -c %u:%g "${home}")" - uid="${identity%%:*}" - gid="${identity##*:}" - install -d -m 0700 -o "${uid}" -g "${gid}" "${home}/.ssh" - touch "${home}/.ssh/authorized_keys" - grep -qxF "${key}" "${home}/.ssh/authorized_keys" || printf '%s\n' "${key}" >> "${home}/.ssh/authorized_keys" - chown "${uid}:${gid}" "${home}/.ssh/authorized_keys" - chmod 0600 "${home}/.ssh/authorized_keys" - done - [ "${found}" = 1 ] || { echo "no supported node SSH account found" >&2; return 1; } - } while true; do - reconcile + /usr/local/bin/python \ + /opt/node-hardener/node_account_hardening.py \ + --public-key-file /vault/secrets/node-ssh-public-key sleep 300 done securityContext: @@ -83,8 +67,21 @@ spec: volumeMounts: - name: host-home mountPath: /host-home - - name: host-passwd - mountPath: /host-etc/passwd + - name: host-etc + mountPath: /host-etc + - name: host-polkit-share + mountPath: /host-polkit-share + readOnly: true + - name: host-k3s + mountPath: /host-k3s + - name: host-kubelet + mountPath: /host-kubelet + - name: host-run-k3s + mountPath: /host-run-k3s + - name: host-run-containerd + mountPath: /host-run-containerd + - name: coordinator + mountPath: /opt/node-hardener readOnly: true - name: vault-secrets mountPath: /vault/secrets @@ -94,19 +91,43 @@ spec: resources: requests: cpu: 5m - memory: 8Mi + memory: 32Mi limits: cpu: 50m - memory: 32Mi + memory: 128Mi volumes: - name: host-home hostPath: path: /home type: Directory - - name: host-passwd + - name: host-etc hostPath: - path: /etc/passwd - type: File + path: /etc + type: Directory + - name: host-polkit-share + hostPath: + path: /usr/share/polkit-1 + type: Directory + - name: host-k3s + hostPath: + path: /var/lib/rancher/k3s + type: Directory + - name: host-kubelet + hostPath: + path: /var/lib/kubelet + type: Directory + - name: host-run-k3s + hostPath: + path: /run/k3s + type: Directory + - name: host-run-containerd + hostPath: + path: /run/containerd + type: Directory + - name: coordinator + configMap: + name: hermes-node-account-hardener + defaultMode: 0555 - name: vault-secrets csi: driver: secrets-store.csi.k8s.io diff --git a/services/hermes/oauth2-proxy.yaml b/services/hermes/oauth2-proxy.yaml index 24383b3a..52dea326 100644 --- a/services/hermes/oauth2-proxy.yaml +++ b/services/hermes/oauth2-proxy.yaml @@ -16,6 +16,9 @@ metadata: spec: selector: app: hermes-agent + # The agent dashboard and terminal must stay reachable while the + # cli-lane-runner readiness probe holds the shared pod NotReady. + publishNotReadyAddresses: true ports: - {name: http, port: 80, targetPort: auth-http} --- diff --git a/services/hermes/scm-common/kustomization.yaml b/services/hermes/scm-common/kustomization.yaml new file mode 100644 index 00000000..a4756ccd --- /dev/null +++ b/services/hermes/scm-common/kustomization.yaml @@ -0,0 +1,17 @@ +# services/hermes/scm-common/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +configMapGenerator: + - name: hermes-scm-boundary + files: + - deadline_http.py=scripts/deadline_http.py + - git_pack_objects.py=scripts/git_pack_objects.py + - gitea_api.py=scripts/gitea_api.py + - gitea_api_policy.py=scripts/gitea_api_policy.py + - receive_pack_scan.py=scripts/receive_pack_scan.py + - scm_broker.py=scripts/scm_broker.py + - scm_broker_io.py=scripts/scm_broker_io.py + - scm_broker_server.py=scripts/scm_broker_server.py + - scm_broker_client.py=scripts/scm_broker_client.py + options: + disableNameSuffixHash: true diff --git a/services/hermes/scm-common/scripts/deadline_http.py b/services/hermes/scm-common/scripts/deadline_http.py new file mode 100644 index 00000000..e513a446 --- /dev/null +++ b/services/hermes/scm-common/scripts/deadline_http.py @@ -0,0 +1,344 @@ +#!/usr/bin/env python3 +"""Absolute wall-clock deadlines for every outbound Hermes HTTP exchange. + +Small control exchanges run inside a killable helper process, so connect, +send, read, and DNS all share one hard bound that no blocking socket state +can outlive. Large streaming Git exchanges cannot cross a process boundary; +they instead register their connection with a watchdog that force-closes the +socket when the same kind of absolute deadline expires. +""" + +from __future__ import annotations + +import base64 +import contextlib +import email.message +import http.client +import io +import json +import os +import signal +import socket +import subprocess +import sys +import threading +import time +import urllib.error +import urllib.request +from dataclasses import dataclass + +from gitea_api_policy import PolicyError + +MAX_CONTROL_BYTES = 128 * 1024 +MAX_PROTOCOL_BYTES = 4 * 1024 * 1024 +MAX_STREAM_SECONDS = 900.0 +READ_CHUNK = 64 * 1024 + + +class _RejectRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + raise urllib.error.URLError("redirect rejected") + + +@dataclass(frozen=True) +class Result: + """Minimal HTTP evidence returned across the process boundary.""" + + status: int + content_type: str + body: bytes + + +def _read_response(response, maximum: int, deadline: float) -> bytes: + chunks: list[bytes] = [] + total = 0 + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError + raw = getattr(getattr(response, "fp", None), "raw", None) + sock = getattr(raw, "_sock", raw) + if hasattr(sock, "settimeout"): + sock.settimeout(remaining) + chunk = response.read(min(READ_CHUNK, maximum + 1 - total)) + if not chunk: + break + chunks.append(chunk) + total += len(chunk) + if total > maximum: + raise ValueError("response too large") + return b"".join(chunks) + + +def _child_exchange(control: dict[str, object]) -> Result: + url = control.get("url") + method = control.get("method") + headers = control.get("headers") + encoded = control.get("body") + maximum = control.get("maximum") + timeout = control.get("timeout") + if ( + not isinstance(url, str) + or not isinstance(method, str) + or not isinstance(headers, dict) + or not all(isinstance(key, str) and isinstance(value, str) for key, value in headers.items()) + or not isinstance(encoded, str) + or not isinstance(maximum, int) + or not 0 <= maximum <= 2 * 1024 * 1024 + or not isinstance(timeout, (int, float)) + or not 0 < timeout <= 120 + ): + raise ValueError("invalid control") + body = base64.b64decode(encoded, validate=True) if encoded else None + if body is not None and len(body) > MAX_CONTROL_BYTES: + raise ValueError("request too large") + deadline = time.monotonic() + float(timeout) + request = urllib.request.Request( + url, data=body, method=method, headers=headers # type: ignore[arg-type] + ) + opener = urllib.request.build_opener(_RejectRedirect()) + try: + response = opener.open(request, timeout=max(deadline - time.monotonic(), 0.001)) + except urllib.error.HTTPError as error: + response = error + with response: + status = getattr(response, "status", response.getcode()) + content_type = response.headers.get_content_type() + value = _read_response(response, maximum, deadline) + return Result(int(status), str(content_type), value) + + +def _child_main() -> int: + try: + raw = sys.stdin.buffer.read(MAX_CONTROL_BYTES + 1) + if len(raw) > MAX_CONTROL_BYTES: + raise ValueError("control too large") + control = json.loads(raw) + if not isinstance(control, dict): + raise ValueError("invalid control") + result = _child_exchange(control) + output = { + "ok": True, + "status": result.status, + "content_type": result.content_type, + "body": base64.b64encode(result.body).decode("ascii"), + } + except Exception: + output = {"ok": False} + sys.stdout.write(json.dumps(output, separators=(",", ":"))) + return 0 if output["ok"] else 1 + + +def _terminate(process: subprocess.Popen[bytes]) -> None: + with contextlib.suppress(ProcessLookupError): + os.killpg(process.pid, signal.SIGKILL) + process.communicate() + + +def exchange( + request: urllib.request.Request, + *, + maximum: int, + timeout: float, + popen=subprocess.Popen, +) -> Result: + """Execute a whole request under one monotonic deadline and size limit.""" + if not 0 < timeout <= 120 or not 0 <= maximum <= 2 * 1024 * 1024: + raise PolicyError("HTTP control request bounds are invalid") + body = request.data or b"" + if not isinstance(body, bytes) or len(body) > MAX_CONTROL_BYTES: + raise PolicyError("HTTP control request exceeds the safe size limit") + control = json.dumps( + { + "url": request.full_url, + "method": request.get_method(), + "headers": dict(request.header_items()), + "body": base64.b64encode(body).decode("ascii"), + "maximum": maximum, + "timeout": timeout, + }, + separators=(",", ":"), + ).encode() + deadline = time.monotonic() + timeout + process = popen( + [sys.executable, os.path.realpath(__file__), "--child"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + start_new_session=True, + env={"LANG": "C.UTF-8", "PATH": "/usr/bin:/bin"}, + ) + try: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise subprocess.TimeoutExpired(process.args, timeout) + output, _error = process.communicate(control, timeout=remaining) + except subprocess.TimeoutExpired as exc: + _terminate(process) + raise PolicyError("HTTP control request deadline exceeded") from exc + if process.returncode != 0 or len(output) > MAX_PROTOCOL_BYTES: + raise PolicyError("HTTP control request failed") + try: + result = json.loads(output) + if not isinstance(result, dict) or result.get("ok") is not True: + raise ValueError + status = result["status"] + content_type = result["content_type"] + encoded = result["body"] + if not isinstance(status, int) or not isinstance(content_type, str) or not isinstance(encoded, str): + raise ValueError + value = base64.b64decode(encoded, validate=True) + except (KeyError, ValueError, TypeError, json.JSONDecodeError) as exc: + raise PolicyError("HTTP control helper returned invalid evidence") from exc + if len(value) > maximum: + raise PolicyError("HTTP control response exceeds the safe size limit") + return Result(status, content_type, value) + + +class _AdaptedResponse: + """Present one finished exchange through the urlopen reader interface.""" + + def __init__(self, result: Result): + self.status = result.status + self.headers = email.message.Message() + self.headers["Content-Type"] = result.content_type + self._stream = io.BytesIO(result.body) + + def read(self, limit: int = -1) -> bytes: + return self._stream.read(limit) + + def __enter__(self): + return self + + def __exit__(self, *_excinfo: object) -> bool: + return False + + +def open_bounded( + request: urllib.request.Request, + *, + maximum: int, + timeout: float, + popen=subprocess.Popen, +) -> _AdaptedResponse: + """Open one control exchange whose whole lifetime shares one deadline.""" + return _AdaptedResponse( + exchange(request, maximum=maximum, timeout=timeout, popen=popen) + ) + + +def _resolve_within(host: str, port: int, timeout: float) -> None: + """Resolve one host under a hard timeout the socket layer never covers. + + ``getaddrinfo`` ignores socket timeouts and runs before any socket + exists, so a slow or hostile resolver would otherwise outlive the whole + stream deadline. Running it in a joinable worker bounds resolution by the + same wall clock; a stuck lookup leaks only one daemon thread. + """ + outcome: dict[str, object] = {} + + def run() -> None: + try: + outcome["value"] = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM) + except OSError as exc: + outcome["error"] = exc + + worker = threading.Thread(target=run, daemon=True) + worker.start() + worker.join(timeout) + if worker.is_alive(): + raise PolicyError("HTTP stream DNS resolution deadline exceeded") + if "error" in outcome: + raise outcome["error"] # type: ignore[misc] + + +class StreamDeadline: + """Force-close tracked connections once an absolute deadline passes.""" + + def __init__(self, timeout: float, *, timer=threading.Timer): + if not 0 < timeout <= MAX_STREAM_SECONDS: + raise PolicyError("HTTP stream deadline bounds are invalid") + self._deadline = time.monotonic() + timeout + self._lock = threading.Lock() + self._connections: list[http.client.HTTPConnection] = [] + self.expired = False + self._timer = timer(timeout, self._expire) + self._timer.daemon = True + self._timer.start() + + def remaining(self) -> float: + value = self._deadline - time.monotonic() + if value <= 0 or self.expired: + raise PolicyError("HTTP stream deadline exceeded") + return value + + def _expire(self) -> None: + with self._lock: + self.expired = True + connections = list(self._connections) + for connection in connections: + # Shut the raw socket down first so a blocked send or recv wakes + # immediately; close alone leaves peers waiting on the old fd. + sock = getattr(connection, "sock", None) + if sock is not None: + with contextlib.suppress(OSError): + sock.shutdown(socket.SHUT_RDWR) + connection.close() + + def _track(self, connection: http.client.HTTPConnection): + with self._lock: + expired = self.expired + if not expired: + self._connections.append(connection) + if expired: + connection.close() + raise PolicyError("HTTP stream deadline exceeded") + return connection + + def _bounded_connection(self, connection: http.client.HTTPConnection): + """Register one connection and bound its DNS phase by the deadline. + + The watchdog can only close a live socket, but ``connect`` resolves + the host before any socket exists. Wrapping ``connect`` runs that + resolution under the same absolute deadline first, then hands off to + the real connect for the socket phases the watchdog already covers. + """ + original_connect = connection.connect + + def connect() -> None: + _resolve_within(connection.host, connection.port, self.remaining()) + original_connect() + + connection.connect = connect # type: ignore[method-assign] + return self._track(connection) + + def cancel(self) -> None: + self._timer.cancel() + + def handlers(self) -> tuple[urllib.request.BaseHandler, ...]: + """Build urllib handlers whose connections obey this deadline.""" + deadline = self + + class GuardedHTTPHandler(urllib.request.HTTPHandler): + def http_open(self, req): + return self.do_open( + lambda host, **kwargs: deadline._bounded_connection( + http.client.HTTPConnection(host, **kwargs) + ), + req, + ) + + class GuardedHTTPSHandler(urllib.request.HTTPSHandler): + def https_open(self, req): + return self.do_open( + lambda host, **kwargs: deadline._bounded_connection( + http.client.HTTPSConnection(host, **kwargs) + ), + req, + ) + + return (GuardedHTTPHandler(), GuardedHTTPSHandler()) + + +if __name__ == "__main__": + raise SystemExit(_child_main() if sys.argv[1:] == ["--child"] else 2) diff --git a/services/hermes/scm-common/scripts/git_pack_objects.py b/services/hermes/scm-common/scripts/git_pack_objects.py new file mode 100644 index 00000000..d9a3e715 --- /dev/null +++ b/services/hermes/scm-common/scripts/git_pack_objects.py @@ -0,0 +1,308 @@ +#!/usr/bin/env python3 +"""Bounded Git pack inflation and delta resolution for quarantine scanning. + +Pushed objects arrive zlib-compressed inside a packfile, so any content +screening must inflate the real payloads first. This module unpacks one +self-contained pack under explicit object, size, and checksum bounds and +resolves every delta against bases inside the same pack. Thin packs are +rejected outright: a delta against a base the broker cannot see would leave +part of the pushed content unscanned. +""" + +from __future__ import annotations + +import hashlib +import tempfile +import zlib +from typing import BinaryIO + +from gitea_api_policy import PolicyError + +OBJECT_TYPE_NAMES = {1: b"commit", 2: b"tree", 3: b"blob", 4: b"tag"} +OFS_DELTA = 6 +REF_DELTA = 7 +MAX_PACK_OBJECTS = 50_000 +MAX_OBJECT_BYTES = 32 * 1024 * 1024 +MAX_TOTAL_BYTES = 256 * 1024 * 1024 +MAX_RESOLVE_PASSES = 64 +SPOOL_MEMORY_LIMIT = 256 * 1024 +READ_CHUNK = 64 * 1024 + + +def _read_exact(stream: BinaryIO, size: int) -> bytes: + value = stream.read(size) + if len(value) != size: + raise PolicyError("Git pack data ended early") + return value + + +def _object_header(stream: BinaryIO) -> tuple[int, int]: + """Parse one object's type and declared inflated size.""" + byte = _read_exact(stream, 1)[0] + type_code = (byte >> 4) & 0x7 + size = byte & 0x0F + shift = 4 + while byte & 0x80: + if shift > 60: + raise PolicyError("Git object header is invalid") + byte = _read_exact(stream, 1)[0] + size |= (byte & 0x7F) << shift + shift += 7 + return type_code, size + + +def _delta_distance(stream: BinaryIO) -> int: + """Parse one ofs-delta base distance.""" + byte = _read_exact(stream, 1)[0] + value = byte & 0x7F + while byte & 0x80: + byte = _read_exact(stream, 1)[0] + value = ((value + 1) << 7) | (byte & 0x7F) + if value > 1 << 48: + raise PolicyError("Git delta base distance is invalid") + return value + + +def _inflate(stream: BinaryIO, declared: int) -> BinaryIO: + """Inflate one object into a quarantine spool, bounded by its header.""" + spool = tempfile.SpooledTemporaryFile( # noqa: SIM115 - caller owns returned spool + max_size=SPOOL_MEMORY_LIMIT, dir="/tmp" + ) + decompressor = zlib.decompressobj() + produced = 0 + try: + while not decompressor.eof: + data = stream.read(READ_CHUNK) + if not data: + raise PolicyError("Git pack object data ended early") + while True: + value = decompressor.decompress(data, READ_CHUNK) + produced += len(value) + if produced > declared: + raise PolicyError("Git object size does not match its header") + spool.write(value) + data = decompressor.unconsumed_tail + if decompressor.eof or not data: + break + if decompressor.unused_data: + stream.seek(-len(decompressor.unused_data), 1) + if produced != declared: + raise PolicyError("Git object size does not match its header") + spool.seek(0) + return spool + except zlib.error as exc: + spool.close() + raise PolicyError("Git pack object data is corrupt") from exc + except Exception: + spool.close() + raise + + +def _delta_size(delta: bytes, position: int) -> tuple[int, int]: + size = 0 + shift = 0 + while True: + if position >= len(delta) or shift > 60: + raise PolicyError("Git delta header is invalid") + byte = delta[position] + position += 1 + size |= (byte & 0x7F) << shift + shift += 7 + if not byte & 0x80: + return size, position + + +def _apply_delta(base: bytes, delta: bytes) -> bytes: + """Apply one Git delta and require exact declared sizes throughout.""" + source_size, position = _delta_size(delta, 0) + target_size, position = _delta_size(delta, position) + if source_size != len(base) or target_size > MAX_OBJECT_BYTES: + raise PolicyError("Git delta sizes are invalid") + parts: list[bytes] = [] + produced = 0 + while position < len(delta): + opcode = delta[position] + position += 1 + if opcode & 0x80: + operands = [bit for bit in (1, 2, 4, 8, 16, 32, 64) if opcode & bit] + if position + len(operands) > len(delta): + raise PolicyError("Git delta copy is truncated") + offset = 0 + size = 0 + for shift, bit in enumerate((1, 2, 4, 8)): + if opcode & bit: + offset |= delta[position] << (8 * shift) + position += 1 + for shift, bit in enumerate((16, 32, 64)): + if opcode & bit: + size |= delta[position] << (8 * shift) + position += 1 + size = size or 0x10000 + if offset + size > len(base): + raise PolicyError("Git delta copy is out of bounds") + parts.append(base[offset : offset + size]) + produced += size + elif opcode: + if position + opcode > len(delta): + raise PolicyError("Git delta insert is truncated") + parts.append(delta[position : position + opcode]) + position += opcode + produced += opcode + else: + raise PolicyError("Git delta instruction is invalid") + if produced > target_size: + raise PolicyError("Git delta result exceeds its declared size") + if produced != target_size: + raise PolicyError("Git delta result size is invalid") + return b"".join(parts) + + +def _payload_sha(type_code: int, payload: BinaryIO, size: int) -> bytes: + digest = hashlib.sha1(OBJECT_TYPE_NAMES[type_code] + b" %d\x00" % size) + payload.seek(0) + while True: + chunk = payload.read(READ_CHUNK) + if not chunk: + break + digest.update(chunk) + payload.seek(0) + return digest.digest() + + +def _verify_checksum(stream: BinaryIO, start: int, end: int) -> None: + trailer = _read_exact(stream, 20) + stream.seek(start) + digest = hashlib.sha1() + remaining = end - start + while remaining: + chunk = stream.read(min(READ_CHUNK, remaining)) + digest.update(chunk) + remaining -= len(chunk) + if digest.digest() != trailer: + raise PolicyError("Git pack checksum is invalid") + stream.seek(end + 20) + + +class _Entry: + """One pack entry: either resolved content or an unresolved delta.""" + + def __init__(self, type_code: int, payload: BinaryIO, size: int, base): + self.type_code = type_code + self.payload = payload + self.size = size + self.base = base + + +def _resolve(entries: dict[int, _Entry], total: list[int]) -> None: + """Resolve every delta against in-pack bases only, to a fixed point.""" + by_sha: dict[bytes, int] = {} + for offset, entry in entries.items(): + if entry.base is None: + by_sha.setdefault( + _payload_sha(entry.type_code, entry.payload, entry.size), offset + ) + for _ in range(MAX_RESOLVE_PASSES): + pending = [item for item in entries.items() if item[1].base is not None] + if not pending: + return + progressed = False + for offset, entry in pending: + base_key = entry.base + base_offset = base_key if isinstance(base_key, int) else by_sha.get(base_key) + base = entries.get(base_offset) if base_offset is not None else None + if base is None or base.base is not None: + if isinstance(base_key, int) and base is None: + raise PolicyError("Git delta base offset is invalid") + continue + base.payload.seek(0) + entry.payload.seek(0) + value = _apply_delta(base.payload.read(), entry.payload.read()) + total[0] += len(value) + if total[0] > MAX_TOTAL_BYTES: + raise PolicyError("Git pack contents exceed the safe size limit") + entry.payload.close() + spool = tempfile.SpooledTemporaryFile( # noqa: SIM115 - entry owns spool + max_size=SPOOL_MEMORY_LIMIT, dir="/tmp" + ) + spool.write(value) + spool.seek(0) + entry.type_code = base.type_code + entry.payload = spool + entry.size = len(value) + entry.base = None + by_sha.setdefault(_payload_sha(entry.type_code, spool, entry.size), offset) + progressed = True + if not progressed: + raise PolicyError("Git delta base is outside the pack; push full packs") + raise PolicyError("Git delta chains exceed the safe resolution limit") + + +def unpack_objects(stream: BinaryIO) -> list[tuple[int, BinaryIO]]: + """Inflate one self-contained pack and return every resolved payload.""" + start = stream.tell() + header = _read_exact(stream, 12) + version = int.from_bytes(header[4:8], "big") + count = int.from_bytes(header[8:12], "big") + if header[:4] != b"PACK" or version not in {2, 3}: + raise PolicyError("Git pack header is invalid") + if count > MAX_PACK_OBJECTS: + raise PolicyError("Git pack contains too many objects") + entries: dict[int, _Entry] = {} + total = [0] + try: + for _ in range(count): + offset = stream.tell() - start + type_code, declared = _object_header(stream) + if declared > MAX_OBJECT_BYTES: + raise PolicyError("Git object exceeds the safe size limit") + total[0] += declared + if total[0] > MAX_TOTAL_BYTES: + raise PolicyError("Git pack contents exceed the safe size limit") + base: int | bytes | None = None + if type_code == OFS_DELTA: + distance = _delta_distance(stream) + base = offset - distance + if base < 0: + raise PolicyError("Git delta base offset is invalid") + elif type_code == REF_DELTA: + base = _read_exact(stream, 20) + elif type_code not in OBJECT_TYPE_NAMES: + raise PolicyError("Git pack object type is invalid") + entries[offset] = _Entry(type_code, _inflate(stream, declared), declared, base) + _verify_checksum(stream, start, stream.tell()) + _resolve(entries, total) + except Exception: + for entry in entries.values(): + entry.payload.close() + raise + for entry in entries.values(): + entry.payload.seek(0) + return [ + (entries[offset].type_code, entries[offset].payload) + for offset in sorted(entries) + ] + + +def parse_loose_object(value: bytes) -> tuple[int, bytes]: + """Inflate one quarantined loose object and return its typed payload.""" + if len(value) > MAX_OBJECT_BYTES: + raise PolicyError("Git object exceeds the safe size limit") + decompressor = zlib.decompressobj() + try: + inflated = decompressor.decompress(value, MAX_OBJECT_BYTES + 1) + except zlib.error as exc: + raise PolicyError("Git loose object data is corrupt") from exc + if not decompressor.eof or decompressor.unused_data or len(inflated) > MAX_OBJECT_BYTES: + raise PolicyError("Git loose object framing is invalid") + kind, separator, remainder = inflated.partition(b"\x00") + name, space, size = kind.partition(b" ") + types = {label: code for code, label in OBJECT_TYPE_NAMES.items()} + if ( + not separator + or not space + or name not in types + or not size.isdigit() + or int(size) != len(remainder) + ): + raise PolicyError("Git loose object header is invalid") + return types[name], remainder diff --git a/services/hermes/scm-common/scripts/gitea_api.py b/services/hermes/scm-common/scripts/gitea_api.py new file mode 100644 index 00000000..3a7de241 --- /dev/null +++ b/services/hermes/scm-common/scripts/gitea_api.py @@ -0,0 +1,492 @@ +#!/usr/bin/env python3 +"""Use the private Atlas Forgejo API through a least-authority PR boundary.""" + +from __future__ import annotations + +import argparse +import base64 +import json +import os +import re +import sys +import urllib.error +import urllib.parse +import urllib.request +from collections.abc import Callable +from pathlib import Path + +import deadline_http +from gitea_api_policy import ( + PolicyError, + _draft_title, + _reject_forbidden, + _validate_body, + _validate_pr_number, + _validate_pr_number_segment, + _validate_query, + _validate_ref, + _validate_ref_bounds, + _validate_repo, + _validate_sha, +) + +CANONICAL_BASE_URL = "https://scm.bstein.dev" +ALLOWED_OWNER = "atlas" +DEFAULT_TOKEN_FILE = Path("/vault/secrets/gitea-token") +CREATE_PATH_RE = re.compile(r"/api/v1/repos/atlas/([^/]+)/pulls\Z") +MAX_ERROR_BYTES = 8192 +MAX_RESPONSE_BYTES = 2 * 1024 * 1024 +MAX_API_TARGET_LENGTH = 768 +MAX_API_PATH_LENGTH = 512 +RAW_API_TARGET_RE = re.compile(r"[A-Za-z0-9/_.?&=-]+\Z") + + +def _safe_urlopen(request: urllib.request.Request, timeout: int): + """Open one exchange under a killable absolute wall-clock deadline. + + The helper process rejects redirects before urllib can copy + authentication headers, and its whole connect/send/read lifetime + shares one hard deadline. + """ + return deadline_http.open_bounded( + request, maximum=MAX_RESPONSE_BYTES, timeout=timeout + ) + + +def read_token(path: Path = DEFAULT_TOKEN_FILE) -> str: + """Read the token from the pod-lifetime Vault projection.""" + token = path.read_text(encoding="utf-8").strip() + if not token: + raise ValueError(f"runtime credential is empty: {path}") + return token + + +def configured_base_url() -> str: + """Reject attempts to redirect credentials away from the canonical origin.""" + configured = os.environ.get("GITEA_BASE_URL", CANONICAL_BASE_URL) + if configured != CANONICAL_BASE_URL: + raise PolicyError("Forgejo origin is fixed to the private Atlas SCM service") + return configured + + +def _split_api_path(path: str) -> urllib.parse.SplitResult: + if not isinstance(path, str) or len(path) > MAX_API_TARGET_LENGTH: + raise PolicyError("API target exceeds the safe size limit") + if ( + not path.isascii() + or any(ord(character) < 32 or ord(character) == 127 for character in path) + or "\\" in path + or not RAW_API_TARGET_RE.fullmatch(path) + ): + raise PolicyError("API target contains non-canonical raw characters") + target = urllib.parse.urlsplit(path) + canonical = urllib.parse.urlunsplit(("", "", target.path, target.query, "")) + if canonical != path: + raise PolicyError("API target is not in exact canonical form") + if target.scheme or target.netloc or target.fragment: + raise PolicyError("API path must be relative to the Atlas SCM origin") + if not target.path.startswith("/api/v1/"): + raise PolicyError("API path must start with /api/v1/") + if len(target.path) > MAX_API_PATH_LENGTH: + raise PolicyError("API path exceeds the safe size limit") + segments = target.path.split("/") + if "//" in target.path or any(segment in {".", ".."} for segment in segments): + raise PolicyError("encoded or non-canonical API paths are not allowed") + return target + + +def _authorize_read( + target: urllib.parse.SplitResult, *, forbidden: tuple[str, ...] = () +) -> str: + """Allow only repository, PR, branch, commit, and status metadata reads.""" + prefix = "/api/v1/repos/atlas/" + remainder = target.path.removeprefix(prefix) + if remainder == target.path: + raise PolicyError("reads are limited to explicit Atlas repository metadata") + repo, separator, suffix = remainder.partition("/") + _reject_forbidden(repo, "repository", forbidden) + _validate_repo(repo) + + if not separator: + _validate_query(target, set()) + return "repository" + if suffix == "pulls": + _validate_query(target, {"page", "limit", "state"}) + return "pull-list" + content_match = re.fullmatch(r"pulls/([^/]+)\.(?:patch|diff)", suffix) + if content_match: + _validate_pr_number_segment(content_match.group(1)) + raise PolicyError("repository API route is outside the metadata read allowlist") + pull_match = re.fullmatch(r"pulls/([^/]+)", suffix) + if pull_match: + _validate_pr_number_segment(pull_match.group(1)) + _validate_query(target, set()) + return "pull" + evidence_match = re.fullmatch(r"pulls/([^/]+)/(?:commits|files)", suffix) + if evidence_match: + _validate_pr_number_segment(evidence_match.group(1)) + _validate_query(target, {"page", "limit"}) + return "pull-evidence" + if suffix == "branches": + _validate_query(target, {"page", "limit"}) + return "branch-list" + branch_match = re.fullmatch(r"branches/([^/]+)", suffix) + if branch_match: + _validate_ref(branch_match.group(1), "branch") + _validate_query(target, set()) + return "branch" + if suffix == "commits": + _validate_query(target, {"page", "limit"}) + return "commit-list" + if re.fullmatch(r"git/commits/[0-9a-fA-F]{40}", suffix): + _validate_query(target, set()) + return "commit" + if re.fullmatch( + r"(?:commits/[0-9a-fA-F]{40}/status|commits/[0-9a-fA-F]{40}/statuses|statuses/[0-9a-fA-F]{40})", + suffix, + ): + _validate_query(target, {"page", "limit"}) + return "status" + raise PolicyError("repository API route is outside the metadata read allowlist") + + +def api_url(base_url: str, path: str) -> str: + """Return a canonical same-origin URL without forwarding credentials.""" + if base_url != CANONICAL_BASE_URL: + raise PolicyError("Forgejo origin is fixed to the private Atlas SCM service") + target = _split_api_path(path) + return urllib.parse.urlunsplit( + ("https", "scm.bstein.dev", target.path, target.query, "") + ) + + +def authorize_request( + method: str, + path: str, + data: object | None, + *, + forbidden: tuple[str, ...] = (), +) -> str: + """Validate one request and return its bounded operation name.""" + normalized_method = method.upper() + target = _split_api_path(path) + if normalized_method == "GET": + if data is not None: + raise PolicyError("read operations cannot include a request body") + return _authorize_read(target, forbidden=forbidden) + if target.query: + raise PolicyError("mutating operations cannot include query parameters") + if normalized_method != "POST": + raise PolicyError("only read and create-draft operations are available") + match = CREATE_PATH_RE.fullmatch(target.path) + if not match: + raise PolicyError("POST is limited to creating an Atlas draft pull request") + _reject_forbidden(match.group(1), "repository", forbidden) + _validate_repo(match.group(1)) + if not isinstance(data, dict) or set(data) != {"base", "body", "head", "title"}: + raise PolicyError("draft creation accepts only base, body, head, and title") + if data["title"] != _draft_title(data["title"], forbidden=forbidden): + raise PolicyError("new pull requests must use the Gitea draft-title prefix") + _validate_body(data["body"], forbidden=forbidden) + _validate_ref_bounds(data["base"], "base", forbidden=forbidden) + _validate_ref_bounds(data["head"], "head", forbidden=forbidden) + _validate_ref(data["base"], "base", forbidden=forbidden) + _validate_ref(data["head"], "head", forbidden=forbidden) + return "create-draft" + + +def build_request( + method: str, + path: str, + *, + base_url: str, + token: str, + data: object | None = None, +) -> urllib.request.Request: + """Build an authorized request without putting the token in its URL or body.""" + authorize_request(method, path, data, forbidden=(token,)) + return _build_request(method, path, base_url=base_url, token=token, data=data) + + +def _build_request( + method: str, + path: str, + *, + base_url: str, + token: str, + data: object | None = None, +) -> urllib.request.Request: + """Build one already-authorized upstream request.""" + payload = None + if data is not None: + payload = json.dumps(data, separators=(",", ":")).encode("utf-8") + return urllib.request.Request( + api_url(base_url, path), + data=payload, + method=method.upper(), + headers={ + "Accept": "application/json", + "Authorization": f"token {token}", + "Content-Type": "application/json", + "User-Agent": "hermes-atlas-pr-client/2", + }, + ) + + +def redact_bytes(value: bytes, token: str) -> bytes: + """Remove exact runtime-token and authorization-header values from output.""" + redacted = value.replace(token.encode("utf-8"), b"[REDACTED]") + return re.sub( + rb"(?i)(authorization\s*[:=]\s*)(?:token|bearer)?\s*[^\s,;\"}]+", + rb"\1[REDACTED]", + redacted, + ) + + +def _reject_response_credential(value: bytes, token: str) -> None: + """Fail closed if an upstream body reflects any ordinary token encoding.""" + raw = token.encode("utf-8") + forms = { + raw, + base64.b64encode(raw), + base64.b64encode(b"hermes-automation:" + raw), + urllib.parse.quote(token, safe="").encode("ascii"), + } + if any(form and form in value for form in forms): + raise PolicyError("Forgejo response contains runtime credential material") + + +def _request( + method: str, + path: str, + data: object | None, + *, + token: str, + opener: Callable[..., object] = _safe_urlopen, + authorized: bool = False, + expected_status: int, +) -> bytes: + builder = _build_request if authorized else build_request + request = builder(method, path, base_url=configured_base_url(), token=token, data=data) + with opener(request, timeout=30) as response: # type: ignore[attr-defined] + status = getattr(response, "status", None) + if status is None and hasattr(response, "getcode"): + status = response.getcode() # type: ignore[attr-defined] + if status != expected_status: + raise PolicyError("Forgejo returned an unexpected HTTP status") + if response.headers.get_content_type() != "application/json": # type: ignore[attr-defined] + raise PolicyError("Forgejo returned an unexpected response type") + body = response.read(MAX_RESPONSE_BYTES + 1) # type: ignore[attr-defined] + if len(body) > MAX_RESPONSE_BYTES: + raise PolicyError("Forgejo response exceeds the safe size limit") + _reject_response_credential(body, token) + return redact_bytes(body, token) + + +def read( + path: str, *, token: str, opener: Callable[..., object] = _safe_urlopen +) -> bytes: + """Read one explicitly allowed Atlas repository metadata resource.""" + return _request( + "GET", path, None, token=token, opener=opener, expected_status=200 + ) + + +def _nested(document: dict[str, object], *keys: str) -> object: + value: object = document + for key in keys: + if not isinstance(value, dict) or key not in value: + raise PolicyError("Forgejo omitted required pull-request evidence") + value = value[key] + return value + + +def _require_create_response( + value: bytes, + *, + repo: str, + base: str, + head: str, + head_sha: str, + title: str, + body: str, +) -> dict[str, object]: + """Require exact server evidence for the intended open draft handoff.""" + try: + document = json.loads(value) + except json.JSONDecodeError as exc: + raise PolicyError("Forgejo returned invalid pull-request metadata") from exc + if not isinstance(document, dict): + raise PolicyError("Forgejo returned invalid pull-request metadata") + number = _validate_pr_number(document.get("number")) + full_name = f"{ALLOWED_OWNER}/{repo}" + expected_html = f"{CANONICAL_BASE_URL}/{full_name}/pulls/{number}" + checks = ( + document.get("state") == "open", + document.get("draft") is True, + document.get("merged") is False, + document.get("html_url") == expected_html, + document.get("url") == expected_html, + document.get("title") == title, + document.get("body") == body, + _nested(document, "base", "ref") == base, + _nested(document, "base", "repo", "full_name") == full_name, + _nested(document, "head", "ref") == head, + _nested(document, "head", "sha") == head_sha, + _nested(document, "head", "repo", "full_name") == full_name, + ) + if not all(checks): + raise PolicyError("Forgejo did not confirm the exact draft pull-request state") + return document + + +def create_draft( + repo: str, + *, + base: str, + head: str, + head_sha: str, + title: str, + body: str, + token: str, + opener: Callable[..., object] = _safe_urlopen, +) -> bytes: + """Create and verify a same-repository draft pull request for human review.""" + _reject_forbidden(repo, "repository", (token,)) + repo = _validate_repo(repo) + title = _draft_title(title, forbidden=(token,)) + body = _validate_body(body, forbidden=(token,)) + head_sha = _validate_sha(head_sha) + _validate_ref_bounds(base, "base", forbidden=(token,)) + _validate_ref_bounds(head, "head", forbidden=(token,)) + base = _validate_ref(base, "base", forbidden=(token,)) + head = _validate_ref(head, "head", forbidden=(token,)) + data = {"base": base, "body": body, "head": head, "title": title} + result = _request( + "POST", + f"/api/v1/repos/{ALLOWED_OWNER}/{repo}/pulls", + data, + token=token, + opener=opener, + authorized=True, + expected_status=201, + ) + _require_create_response( + result, + repo=repo, + base=base, + head=head, + head_sha=head_sha, + title=title, + body=body, + ) + return result + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + """Parse the intentionally narrow read and create-draft operations.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--dry-run", + action="store_true", + help="validate and describe the request without reading a token or using the network", + ) + subparsers = parser.add_subparsers(dest="operation", required=True) + read_parser = subparsers.add_parser("read", help="read allowed Atlas metadata") + read_parser.add_argument("path") + create_parser = subparsers.add_parser( + "create-draft", help="create a same-repository draft pull request" + ) + create_parser.add_argument("repo") + create_parser.add_argument("--base", required=True) + create_parser.add_argument("--head", required=True) + create_parser.add_argument("--head-sha", required=True) + create_parser.add_argument("--title", required=True) + create_parser.add_argument("--body", default="") + return parser.parse_args(argv) + + +def _operation( + args: argparse.Namespace, *, validate_refs: bool = True +) -> tuple[str, str, object | None]: + if args.operation == "read": + return "GET", args.path, None + repo = _validate_repo(args.repo) + _validate_sha(args.head_sha) + body = _validate_body(args.body) + title = _draft_title(args.title) + _validate_ref_bounds(args.base, "base") + _validate_ref_bounds(args.head, "head") + base = _validate_ref(args.base, "base") if validate_refs else args.base + head = _validate_ref(args.head, "head") if validate_refs else args.head + data = { + "base": base, + "body": body, + "head": head, + "title": title, + } + return "POST", f"/api/v1/repos/{ALLOWED_OWNER}/{repo}/pulls", data + + +def _write_body(body: bytes) -> None: + if body: + sys.stdout.buffer.write(body) + if not body.endswith(b"\n"): + sys.stdout.buffer.write(b"\n") + + +def main(argv: list[str] | None = None) -> int: + """Validate, optionally describe, and execute one safe Forgejo operation.""" + try: + args = parse_args(argv) + if args.dry_run: + method, path, data = _operation(args, validate_refs=False) + operation = authorize_request(method, path, data) + print( + json.dumps( + { + "dry_run": True, + "host": "scm.bstein.dev", + "operation": operation, + "owner": ALLOWED_OWNER, + "path": path, + "request_fields": sorted(data) + if isinstance(data, dict) + else [], + }, + sort_keys=True, + ) + ) + return 0 + method, path, data = _operation(args, validate_refs=False) + from scm_broker_client import create_draft as broker_create_draft + from scm_broker_client import read as broker_read + + if args.operation == "read": + body = broker_read(path) + else: + assert isinstance(data, dict) + body = broker_create_draft( + args.repo, + base=str(data["base"]), + head=str(data["head"]), + head_sha=args.head_sha, + title=str(data["title"]), + body=str(data["body"]), + ) + _write_body(body) + return 0 + except urllib.error.HTTPError as exc: + exc.read(MAX_ERROR_BYTES) + print(f"SCM broker request failed with HTTP {exc.code}", file=sys.stderr) + return 1 + except (OSError, PolicyError, ValueError, json.JSONDecodeError): + print( + "Forgejo request rejected or unavailable; no credential was disclosed", + file=sys.stderr, + ) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/hermes/scm-common/scripts/gitea_api_policy.py b/services/hermes/scm-common/scripts/gitea_api_policy.py new file mode 100644 index 00000000..db8833cf --- /dev/null +++ b/services/hermes/scm-common/scripts/gitea_api_policy.py @@ -0,0 +1,490 @@ +"""Validation policy for Hermes' least-authority Atlas Forgejo client. + +The screening catches structured credential assignments, known token formats, +and long high-entropy values. It is a fail-closed accident barrier, not proof +that text is secret-free: a short or multiword secret under an innocuous key is +not reliably distinguishable from prose and must never be supplied by callers. +JSON depth/nodes/documents and multiline gaps are deliberately bounded; large +or unusual configuration blobs are not a supported pull-request field format. +""" + +# ruff: noqa: SIM905 + +from __future__ import annotations + +import json +import re +import subprocess +import urllib.parse +from collections import Counter +from math import log2 + +REPO_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,99}\Z") +SHA_RE = re.compile(r"[0-9a-fA-F]{40}\Z") +DRAFT_TITLE_PREFIX = "WIP: " +GIT_BIN = "/usr/bin/git" +MAX_QUERY_LENGTH = 128 +MAX_QUERY_ITEMS = 3 +MAX_QUERY_KEY_LENGTH = 16 +MAX_QUERY_VALUE_LENGTH = 16 +MAX_PAGE = 10_000 +MAX_LIMIT = 50 +MAX_PR_NUMBER = 2_147_483_647 +MAX_PR_NUMBER_DIGITS = 10 +MAX_REF_CHARACTERS = 200 +MAX_REF_UTF8_BYTES = 255 +MAX_TITLE_UTF8_BYTES = 512 +MAX_BODY_UTF8_BYTES = 32_768 +MAX_JSON_DOCUMENTS = 32 +MAX_JSON_NODES = 2_048 +MAX_JSON_DEPTH = 32 +CANONICAL_QUERY_RE = re.compile(r"[A-Za-z0-9_=&-]*\Z") +ASSIGNMENT_RE = re.compile( + r"""(?mx) + (?[\"']?) + (?P\.?[A-Za-z][A-Za-z0-9_.-]{0,127}) + (?P=key_quote) + [ \t]*(?P[:=])[ \t]* + (?P + \"(?:\\.|[^\"\\\r\n])*\" | + '(?:\\.|[^'\\\r\n])*' | + [^\r\n,;}&]{0,2048} + ) + """ +) +JSON_KEY_RE = re.compile(r'"(?P[^"]{1,512})"(?P[\x00-\x20\x7f]{0,256}):') +MULTILINE_ASSIGNMENT_RE = re.compile( + r"""(?mx) + (?["']?) + (?P\.?[A-Za-z][A-Za-z0-9_.-]{0,127}) + (?P=key_quote) + [ \t]{0,64}(?:\r?\n[ \t]{0,64}){0,4} + (?P[:=]) + [ \t]{0,64}(?:\r?\n[ \t]{0,64}){1,4} + (?P[^\r\n,;}&]{1,2048}) + """ +) +CAMEL_BOUNDARY_RE = re.compile(r"(?<=[a-z0-9])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])") +CREDENTIAL_SCHEME_RE = re.compile( + r"(?i)\b(?:bearer|basic|token)\s+" + r"(?=[A-Za-z0-9+/_=.:-]{8,}\b)(?=[A-Za-z0-9+/_=.:-]*[0-9+/_=.-])" + r"[A-Za-z0-9+/_=.:-]{8,}" +) +STANDALONE_SECRET_PATTERNS = tuple( + re.compile(pattern, re.IGNORECASE) + for pattern in ( + r"(? str: + if not REPO_RE.fullmatch(repo) or repo in {".", ".."}: + raise PolicyError("repository name is outside the Atlas allowlist") + return repo + + +def _reject_forbidden(value: object, name: str, forbidden: tuple[str, ...]) -> None: + """Reject exact runtime credentials before invoking helpers or the network.""" + if not isinstance(value, str) or any(secret and secret in value for secret in forbidden): + raise PolicyError(f"{name} contains runtime credential material") + + +def _validate_ref_bounds( + value: object, name: str, *, forbidden: tuple[str, ...] = () +) -> str: + """Reject oversized or non-encodable refs before invoking Git.""" + _reject_forbidden(value, name, forbidden) + if not value or len(value) > MAX_REF_CHARACTERS: + raise PolicyError(f"{name} exceeds the safe branch-name limit") + try: + encoded = value.encode("utf-8", errors="strict") + except UnicodeEncodeError as exc: + raise PolicyError(f"{name} is not valid UTF-8 text") from exc + if len(encoded) > MAX_REF_UTF8_BYTES: + raise PolicyError(f"{name} exceeds the safe UTF-8 branch-name limit") + return value + + +def _validate_ref( + value: object, name: str, *, forbidden: tuple[str, ...] = () +) -> str: + """Validate the complete Git ref grammar using Git itself.""" + value = _validate_ref_bounds(value, name, forbidden=forbidden) + if value.startswith("-"): + raise PolicyError(f"{name} must be a same-repository branch name") + if any(ord(character) < 32 or ord(character) == 127 for character in value): + raise PolicyError(f"{name} must be a safe same-repository branch name") + result = subprocess.run( + [GIT_BIN, "check-ref-format", f"refs/heads/{value}"], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + if result.returncode != 0: + raise PolicyError(f"{name} must be a safe same-repository branch name") + return value + + +def _validate_sha(value: object, name: str = "head SHA") -> str: + if not isinstance(value, str) or not SHA_RE.fullmatch(value): + raise PolicyError(f"{name} must be a full Git SHA-1") + return value.lower() + + +def _validate_pr_number_segment(value: object) -> int: + """Validate a canonical bounded pull-request number from an API path.""" + if ( + not isinstance(value, str) + or len(value) > MAX_PR_NUMBER_DIGITS + or not re.fullmatch(r"[1-9][0-9]*", value) + ): + raise PolicyError("pull-request number must use bounded canonical ASCII digits") + number = int(value) + if number > MAX_PR_NUMBER: + raise PolicyError("pull-request number is outside its allowed range") + return number + + +def _validate_pr_number(value: object) -> int: + """Validate a bounded pull-request number returned by Forgejo.""" + if ( + not isinstance(value, int) + or isinstance(value, bool) + or not 1 <= value <= MAX_PR_NUMBER + ): + raise PolicyError("Forgejo omitted a valid pull-request number") + return value + + +def _validate_text( + value: object, + name: str, + maximum: int, + maximum_bytes: int, + *, + required: bool, +) -> str: + if not isinstance(value, str): + raise PolicyError(f"{name} must be text") + if required and not value.strip(): + raise PolicyError(f"{name} must not be empty") + try: + encoded = value.encode("utf-8", errors="strict") + except UnicodeEncodeError as exc: + raise PolicyError(f"{name} is not valid UTF-8 text") from exc + if len(value) > maximum or len(encoded) > maximum_bytes or "\x00" in value: + raise PolicyError(f"{name} exceeds the safe request limit") + return value + + +def _normalized_key(value: str) -> tuple[tuple[str, ...], str]: + """Split camelCase and separator-based assignment keys into semantics.""" + separated = CAMEL_BOUNDARY_RE.sub(" ", value.strip(".\"'")) + words = tuple(re.findall(r"[A-Za-z0-9]+", separated.lower())) + return words, "".join(words) + + +def _is_sensitive_key(value: str) -> bool: + words, compact = _normalized_key(value) + word_set = set(words) + if word_set & SENSITIVE_KEY_WORDS: + return True + if compact in SENSITIVE_COMPACT_KEYS: + return True + if compact.endswith(SENSITIVE_COMPACT_SUFFIXES): + return True + if "key" in word_set and word_set & KEY_MODIFIER_WORDS: + return True + if {"client", "email"} <= word_set or {"client", "id"} <= word_set: + return True + if {"access", "id"} <= word_set or {"connection", "string"} <= word_set: + return True + return compact.endswith("configjson") + + +def _strip_assignment_value(value: str) -> tuple[str, bool]: + stripped = value.strip() + quoted = ( + len(stripped) >= 2 and stripped[0] in {'"', "'"} and stripped[-1] == stripped[0] + ) + if quoted: + stripped = stripped[1:-1].strip() + return stripped, quoted + + +def _looks_like_prose(value: str) -> bool: + words = re.findall(r"[A-Za-z]+(?:[-'][A-Za-z]+)?", value) + if len(words) < 2: + return False + if re.search(r"[$`{}\[\]\\@/:=+_]", value): + return False + return words[0].lower() in PROSE_LEAD_WORDS + + +def _looks_sensitive_assignment_value(value: str) -> bool: + stripped, quoted = _strip_assignment_value(value) + if not stripped: + return False + if quoted or stripped.startswith(("{", "[", "|", ">", "!!", "&", "*")): + return True + if CREDENTIAL_SCHEME_RE.search(stripped): + return True + if any(pattern.search(stripped) for pattern in STANDALONE_SECRET_PATTERNS): + return True + if re.search(r"(?:https?://|[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+)", stripped): + return True + if re.search(r"(?:\$\{|\$\(|\\[nrt]|[A-Za-z0-9+/]{16,}={0,3}\Z)", stripped): + return True + return not _looks_like_prose(stripped) + + +def _is_structural_credential_assignment(key: str, value: str) -> bool: + words, compact_key = _normalized_key(key) + stripped, _quoted = _strip_assignment_value(value) + _value_words, compact_value = _normalized_key(stripped) + return ( + compact_key == "type" and compact_value in {"serviceaccount", "credential"} + ) or ("service" in words and "account" in words and bool(stripped)) + + +def _decode_json_key(raw: str) -> tuple[str, bool]: + """Decode one bounded JSON key without silently accepting bad escapes.""" + try: + decoded = json.loads(f'"{raw}"') + except (json.JSONDecodeError, UnicodeError): + approximate = re.sub(r"[^A-Za-z0-9]+", "_", raw.replace("\\", "")) + return approximate, False + return decoded if isinstance(decoded, str) else "", True + + +def _json_value_has_sensitive_assignment( + value: object, *, depth: int = 0, nodes: list[int] | None = None +) -> bool: + """Walk one decoded JSON value with explicit depth and node ceilings.""" + if nodes is None: + nodes = [0] + nodes[0] += 1 + if nodes[0] > MAX_JSON_NODES or depth > MAX_JSON_DEPTH: + raise PolicyError("structured pull-request text exceeds the scan limit") + if isinstance(value, dict): + for key, assigned in value.items(): + if not isinstance(key, str): + continue + if _is_sensitive_key(key): + return True + assigned_text = assigned if isinstance(assigned, str) else "" + if _is_structural_credential_assignment(key, assigned_text): + return True + if _json_value_has_sensitive_assignment( + assigned, depth=depth + 1, nodes=nodes + ): + return True + elif isinstance(value, list): + return any( + _json_value_has_sensitive_assignment(item, depth=depth + 1, nodes=nodes) + for item in value + ) + return False + + +def _decoded_json_documents(value: str): + """Yield bounded embedded JSON object or array fragments.""" + decoder = json.JSONDecoder() + index = 0 + attempts = 0 + while attempts < MAX_JSON_DOCUMENTS: + positions = [ + position for token in "{[" if (position := value.find(token, index)) >= 0 + ] + if not positions: + return + start = min(positions) + attempts += 1 + try: + document, end = decoder.raw_decode(value, start) + except (json.JSONDecodeError, RecursionError, ValueError): + index = start + 1 + continue + yield document + index = max(end, start + 1) + if "{" in value[index:] or "[" in value[index:]: + raise PolicyError("structured pull-request text exceeds the scan limit") + + +def _has_structured_sensitive_assignment(value: str) -> bool: + """Detect bounded JSON and multiline YAML/env credential assignments.""" + for candidate in JSON_KEY_RE.finditer(value): + decoded, valid = _decode_json_key(candidate.group("key")) + if _is_sensitive_key(decoded): + return True + if not valid and _is_sensitive_key(candidate.group("key").replace("\\", "")): + return True + for candidate in MULTILINE_ASSIGNMENT_RE.finditer(value): + key = candidate.group("key") + assigned = candidate.group("value") + if _is_structural_credential_assignment(key, assigned) or ( + _is_sensitive_key(key) and _looks_sensitive_assignment_value(assigned) + ): + return True + return any( + _json_value_has_sensitive_assignment(item) + for item in _decoded_json_documents(value) + ) + + +def _has_high_entropy_token(value: str) -> bool: + for match in HIGH_ENTROPY_TOKEN_RE.finditer(value): + token = match.group(0) + if len(token) > 256: + return True + groups = sum( + bool(re.search(pattern, token)) + for pattern in (r"[a-z]", r"[A-Z]", r"[0-9]", r"[+/_=-]") + ) + counts = Counter(token) + entropy = -sum( + (count / len(token)) * log2(count / len(token)) for count in counts.values() + ) + if groups >= 3 and entropy >= 4.0: + return True + return False + + +def _reject_sensitive(value: str, name: str, forbidden: tuple[str, ...]) -> None: + if any(secret and secret in value for secret in forbidden): + raise PolicyError(f"pull-request {name} contains runtime credential material") + if CREDENTIAL_SCHEME_RE.search(value): + raise PolicyError(f"pull-request {name} resembles credential material") + if any(pattern.search(value) for pattern in STANDALONE_SECRET_PATTERNS): + raise PolicyError(f"pull-request {name} resembles credential material") + if _has_structured_sensitive_assignment(value): + raise PolicyError(f"pull-request {name} resembles credential material") + for candidate in ASSIGNMENT_RE.finditer(value): + key = candidate.group("key") + assigned = candidate.group("value") + if _is_structural_credential_assignment(key, assigned) or ( + _is_sensitive_key(key) and _looks_sensitive_assignment_value(assigned) + ): + raise PolicyError(f"pull-request {name} resembles credential material") + if _has_high_entropy_token(value): + raise PolicyError(f"pull-request {name} resembles credential material") + + +def _validate_body(value: object, *, forbidden: tuple[str, ...] = ()) -> str: + body = _validate_text(value, "body", 16384, MAX_BODY_UTF8_BYTES, required=False) + _reject_sensitive(body, "body", forbidden) + return body + + +def _draft_title(value: object, *, forbidden: tuple[str, ...] = ()) -> str: + """Return a bounded, secret-screened Gitea draft title.""" + title = _validate_text( + value, "title", 251, MAX_TITLE_UTF8_BYTES, required=True + ).strip() + _reject_sensitive(title, "title", forbidden) + for prefix in ("WIP:", "[WIP]"): + if title.upper().startswith(prefix): + title = title[len(prefix) :].lstrip() + break + if not title: + raise PolicyError("title must contain text after the draft prefix") + return DRAFT_TITLE_PREFIX + title + + +def _validate_query(target: urllib.parse.SplitResult, allowed: set[str]) -> None: + """Accept only short canonical ASCII query strings with bounded pagination.""" + raw = target.query + if ( + len(raw) > MAX_QUERY_LENGTH + or not raw.isascii() + or not CANONICAL_QUERY_RE.fullmatch(raw) + or "%" in raw + ): + raise PolicyError("API query must use short canonical ASCII form") + try: + pairs = urllib.parse.parse_qsl(raw, keep_blank_values=True, strict_parsing=True) + except ValueError as exc: + raise PolicyError("invalid API query") from exc + if len(pairs) > MAX_QUERY_ITEMS: + raise PolicyError("too many API query parameters") + if len({key for key, _ in pairs}) != len(pairs): + raise PolicyError("duplicate API query parameters are not allowed") + if any( + not key + or len(key) > MAX_QUERY_KEY_LENGTH + or len(value) > MAX_QUERY_VALUE_LENGTH + for key, value in pairs + ): + raise PolicyError("API query key or value exceeds its safe limit") + if any(key not in allowed for key, _ in pairs): + raise PolicyError("API query parameter is outside the read allowlist") + values = dict(pairs) + for name in ("page", "limit"): + if name not in values: + continue + if not re.fullmatch(r"[0-9]+", values[name]): + raise PolicyError(f"{name} must use ASCII decimal digits") + number = int(values[name]) + if values[name] != str(number): + raise PolicyError(f"{name} must use canonical ASCII decimal form") + ceiling = MAX_PAGE if name == "page" else MAX_LIMIT + if not 1 <= number <= ceiling: + raise PolicyError(f"{name} is outside its allowed range") + if "state" in values and values["state"] not in {"open", "closed", "all"}: + raise PolicyError("pull-request state is invalid") diff --git a/services/hermes/scm-common/scripts/receive_pack_scan.py b/services/hermes/scm-common/scripts/receive_pack_scan.py new file mode 100644 index 00000000..76980880 --- /dev/null +++ b/services/hermes/scm-common/scripts/receive_pack_scan.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +"""Validate Git receive-pack requests and scan pushed object content. + +Ref commands may only create new namespaced feature branches. Every pushed +object is then inflated in quarantine and its decompressed payload scanned, +so credentials cannot ride through the broker inside compressed pack data. +The screening is a fail-closed accident barrier for credential-shaped +content: exact runtime-token forms, private keys, SSH key material, and +known provider token formats. Entropy heuristics are deliberately excluded +so SOPS ciphertext, pinned digests, and lock files remain pushable; a short +secret disguised as prose is not reliably detectable and must never be +supplied by callers. +""" + +from __future__ import annotations + +import io +import re +from typing import BinaryIO + +import git_pack_objects +from gitea_api_policy import ( + STANDALONE_SECRET_PATTERNS, + PolicyError, + _validate_ref, +) + +ZERO_SHA = b"0" * 40 +MAX_PUSH_COMMANDS = 16 +SCAN_CHUNK = 256 * 1024 +SCAN_CARRY = 8 * 1024 +FEATURE_REF_RE = re.compile( + r"refs/heads/(?:(?:feature|fix|hermes|handoff)/[A-Za-z0-9][A-Za-z0-9._/-]{0,190})\Z" +) +PKT_HEADER_RE = re.compile(rb"[0-9a-f]{4}\Z") +CONTENT_SECRET_PATTERNS = tuple( + re.compile(pattern.pattern.encode("ascii"), re.IGNORECASE) + for pattern in STANDALONE_SECRET_PATTERNS +) + ( + re.compile(rb"(?i)(? bytes | None: + """Read one pkt-line; None marks the flush packet before the pack.""" + header = stream.read(4) + if not header: + raise PolicyError("Git receive-pack request omitted its command terminator") + if not PKT_HEADER_RE.fullmatch(header): + raise PolicyError("Git receive-pack command framing is invalid") + size = int(header, 16) + if size == 0: + return None + if size < 4: + raise PolicyError("Git receive-pack command framing is invalid") + value = stream.read(size - 4) + if len(value) != size - 4: + raise PolicyError("Git receive-pack command framing is invalid") + return value + + +def _validate_command(line: bytes, token: str, forbidden: tuple[bytes, ...]) -> None: + """Permit only creation of one new, namespaced feature branch.""" + if any(form in line for form in forbidden): + raise PolicyError("Git request contains runtime credential material") + command = line.rstrip(b"\n").split(b"\x00", 1)[0] + fields = command.split(b" ") + if len(fields) != 3 or not all( + re.fullmatch(rb"[0-9a-f]{40}", item) for item in fields[:2] + ): + raise PolicyError("Git receive-pack ref command is invalid") + old_sha, new_sha, raw_ref = fields + if old_sha != ZERO_SHA or new_sha == ZERO_SHA: + raise PolicyError("Git broker permits only new feature-branch creation") + try: + ref = raw_ref.decode("ascii") + except UnicodeDecodeError as exc: + raise PolicyError("Git ref must be canonical ASCII") from exc + if not FEATURE_REF_RE.fullmatch(ref): + raise PolicyError("Git push is limited to namespaced feature branches") + _validate_ref(ref.removeprefix("refs/heads/"), "head", forbidden=(token,)) + + +def _scan_payload(payload: BinaryIO, forbidden: tuple[bytes, ...]) -> None: + """Scan one decompressed object payload across chunk boundaries.""" + carry = b"" + while True: + chunk = payload.read(SCAN_CHUNK) + if not chunk: + return + window = carry + chunk + if any(form in window for form in forbidden): + raise PolicyError("Git push contains runtime credential material") + if any(pattern.search(window) for pattern in CONTENT_SECRET_PATTERNS): + raise PolicyError("Git push contains credential-shaped content") + carry = window[-SCAN_CARRY:] + + +def _scan_pack(stream: BinaryIO, forbidden: tuple[bytes, ...]) -> None: + objects = git_pack_objects.unpack_objects(stream) + try: + if stream.read(1): + raise PolicyError("Git request has bytes after its pack") + for _type_code, payload in objects: + _scan_payload(payload, forbidden) + finally: + for _type_code, payload in objects: + payload.close() + + +def validate_receive_pack( + body: bytes | BinaryIO, token: str, forbidden: tuple[bytes, ...] +) -> None: + """Validate one whole receive-pack request: commands, pack, and content.""" + stream = io.BytesIO(body) if isinstance(body, bytes) else body + stream.seek(0) + try: + commands = 0 + while True: + line = _read_pkt_line(stream) + if line is None: + break + _validate_command(line, token, forbidden) + commands += 1 + if commands > MAX_PUSH_COMMANDS: + raise PolicyError("Git push contains too many ref commands") + if commands == 0: + raise PolicyError("Git receive-pack request has no ref command") + _scan_pack(stream, forbidden) + finally: + stream.seek(0) + + +def scan_loose_object(value: bytes, forbidden: tuple[bytes, ...]) -> int: + """Scan one quarantined loose object and return its parsed type code.""" + type_code, payload = git_pack_objects.parse_loose_object(value) + _scan_payload(io.BytesIO(payload), forbidden) + return type_code diff --git a/services/hermes/scm-common/scripts/scm_broker.py b/services/hermes/scm-common/scripts/scm_broker.py new file mode 100644 index 00000000..a9cb3685 --- /dev/null +++ b/services/hermes/scm-common/scripts/scm_broker.py @@ -0,0 +1,455 @@ +#!/usr/bin/env python3 +"""Serve the credential-isolated Atlas API and Git smart-HTTP boundary.""" + +from __future__ import annotations + +import argparse +import base64 +import json +import re +import socket +import tempfile +import time +import urllib.error +import urllib.parse +import urllib.request +from http.server import BaseHTTPRequestHandler +from typing import BinaryIO + +import deadline_http +from gitea_api import ( + CANONICAL_BASE_URL, + PolicyError, + create_draft, + read, + read_token, +) +from gitea_api_policy import _reject_forbidden, _validate_repo +from receive_pack_scan import validate_receive_pack +from scm_broker_io import RejectRedirect, response_status as _status, spool_response +from scm_broker_server import AbsoluteHeaderDeadlineMixin, BoundedThreadingHTTPServer + +BROKER_PORT = 9081 +GIT_USER = "hermes-automation" +MAX_CONTROL_BODY = 64 * 1024 +MAX_GIT_REQUEST = 128 * 1024 * 1024 +MAX_GIT_RESPONSE = 128 * 1024 * 1024 +MAX_HEADERS = 32 +MAX_HEADER_BYTES = 16 * 1024 +SPOOL_MEMORY_LIMIT = 1024 * 1024 +STREAM_CHUNK = 64 * 1024 +INBOUND_HEADER_TIMEOUT = 10.0 +INBOUND_BODY_TIMEOUT = 120.0 +UPSTREAM_DEADLINE_SECONDS = 300.0 +GIT_PATH_RE = re.compile( + r"/git/atlas/(?P[A-Za-z0-9][A-Za-z0-9._-]{0,99})\.git/" + r"(?Pinfo/refs|git-upload-pack|git-receive-pack)\Z" +) + + +def _read_bounded( + stream, + maximum: int, + length: int | None = None, + *, + deadline: float | None = None, + set_timeout=None, +) -> bytes: + if length is not None and not 0 <= length <= maximum: + raise PolicyError("SCM request exceeds the safe size limit") + wanted = maximum + 1 if length is None else length + chunks: list[bytes] = [] + total = 0 + while total < wanted: + if deadline is not None: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise PolicyError("SCM request body deadline exceeded") + if set_timeout is not None: + set_timeout(remaining) + chunk = stream.read(min(STREAM_CHUNK, wanted - total)) + if not chunk: + break + chunks.append(chunk) + total += len(chunk) + value = b"".join(chunks) + if len(value) > maximum or (length is not None and len(value) != length): + raise PolicyError("SCM request exceeds the safe size limit") + return value + + +def _spool_bounded( + stream, + maximum: int, + length: int, + *, + token: str, + context: str, + deadline: float, + set_timeout=None, +) -> tuple[BinaryIO, int]: + """Copy a fixed-length exchange through a bounded-memory disk spool.""" + if not 0 <= length <= maximum: + raise PolicyError("SCM request exceeds the safe size limit") + spool = tempfile.SpooledTemporaryFile( # noqa: SIM115 - caller owns returned spool + max_size=SPOOL_MEMORY_LIMIT, dir="/tmp" + ) + remaining = length + forms = _credential_forms(token) + carry = b"" + try: + while remaining: + remaining_time = deadline - time.monotonic() + if remaining_time <= 0: + raise PolicyError("SCM request body deadline exceeded") + if set_timeout is not None: + set_timeout(remaining_time) + chunk = stream.read(min(STREAM_CHUNK, remaining)) + if not chunk: + raise PolicyError("SCM request body ended early") + candidate = carry + chunk + if any(form in candidate for form in forms): + raise PolicyError(f"{context} contains runtime credential material") + width = max(len(form) for form in forms) - 1 + carry = candidate[-width:] if width else b"" + spool.write(chunk) + remaining -= len(chunk) + spool.seek(0) + return spool, length + except Exception: + spool.close() + raise + + +def _spool_response( + response, maximum: int, token: str, deadline_seconds: float +) -> tuple[BinaryIO, int]: + return spool_response( + response, + maximum, + _credential_forms(token), + memory_limit=SPOOL_MEMORY_LIMIT, + chunk_size=STREAM_CHUNK, + deadline_seconds=deadline_seconds, + ) + + +def _load_json(handler: BaseHTTPRequestHandler) -> dict[str, object]: + if handler.headers.get("Transfer-Encoding"): + raise PolicyError("chunked broker control requests are not allowed") + if handler.headers.get_content_type() != "application/json": + raise PolicyError("broker control request must be JSON") + length = _content_length(handler.headers, MAX_CONTROL_BODY) + deadline = time.monotonic() + INBOUND_BODY_TIMEOUT + value = json.loads( + _read_bounded( + handler.rfile, + MAX_CONTROL_BODY, + length, + deadline=deadline, + set_timeout=handler.connection.settimeout, + ) + ) + if not isinstance(value, dict): + raise PolicyError("broker control request must be an object") + return value + + +def _git_target(raw: str) -> tuple[str, str, str]: + if ( + not raw.isascii() + or any(ord(character) < 32 or ord(character) == 127 for character in raw) + or "%" in raw + or "\\" in raw + or len(raw) > 512 + ): + raise PolicyError("Git target is not canonical ASCII") + target = urllib.parse.urlsplit(raw) + canonical = urllib.parse.urlunsplit(("", "", target.path, target.query, "")) + if canonical != raw: + raise PolicyError("Git target is not in exact canonical form") + if target.scheme or target.netloc or target.fragment: + raise PolicyError("Git target must be relative") + match = GIT_PATH_RE.fullmatch(target.path) + if not match: + raise PolicyError("Git target is outside the Atlas allowlist") + repo = _validate_repo(match.group("repo")) + operation = match.group("operation") + query = target.query + if operation == "info/refs": + if query not in {"service=git-upload-pack", "service=git-receive-pack"}: + raise PolicyError("Git discovery service is outside the allowlist") + service = query.removeprefix("service=") + elif query: + raise PolicyError("Git RPC query parameters are not allowed") + else: + service = operation + return repo, operation, service + + +def _content_length(headers: object, maximum: int) -> int: + """Parse one short canonical bounded Content-Length header.""" + raw = headers.get("Content-Length", "") # type: ignore[attr-defined] + if ( + not isinstance(raw, str) + or not raw.isascii() + or len(raw) > 10 + or not raw.isdigit() + ): + raise PolicyError("SCM request length is invalid") + length = int(raw) + if raw != str(length) or not 0 <= length <= maximum: + raise PolicyError("SCM request length is invalid") + return length + + +def _credential_forms(token: str) -> tuple[bytes, ...]: + basic = base64.b64encode(f"{GIT_USER}:{token}".encode()) + return token.encode("utf-8"), basic, b"Basic " + basic + + +def _validate_receive_pack(body: bytes | BinaryIO, token: str) -> None: + """Permit only vetted new-feature-branch pushes with scanned contents.""" + validate_receive_pack(body, token, _credential_forms(token)) + + +def _guarded_opener(guard: deadline_http.StreamDeadline): + """Build an opener whose connection obeys one absolute stream deadline.""" + return urllib.request.build_opener(RejectRedirect(), *guard.handlers()).open + + +def _upstream_git_request( + target: str, + *, + method: str, + body: bytes | BinaryIO | None, + content_type: str | None, + expected_type: str, + token: str, + body_length: int | None = None, + stream_result: bool = False, + opener=None, +) -> bytes | tuple[BinaryIO, int]: + credentials = base64.b64encode(f"{GIT_USER}:{token}".encode()).decode("ascii") + headers = {"Accept": expected_type, "Authorization": f"Basic {credentials}"} + if content_type: + headers["Content-Type"] = content_type + if body is not None: + if body_length is None: + if not isinstance(body, bytes): + raise PolicyError("Git upstream request length is missing") + body_length = len(body) + headers["Content-Length"] = str(body_length) + request = urllib.request.Request( + CANONICAL_BASE_URL + target, + data=body, + method=method, + headers=headers, + ) + # One wall-clock budget covers connect, send, headers, and body read; the + # watchdog closes the upstream socket if any phase overstays it. + guard = deadline_http.StreamDeadline(UPSTREAM_DEADLINE_SECONDS) + try: + open_upstream = opener if opener is not None else _guarded_opener(guard) + with open_upstream(request, timeout=guard.remaining()) as response: + if _status(response) != 200: + raise PolicyError("Git upstream returned an unexpected HTTP status") + if response.headers.get_content_type() != expected_type: + raise PolicyError("Git upstream returned an unexpected response type") + result, result_length = _spool_response( + response, MAX_GIT_RESPONSE, token, guard.remaining() + ) + finally: + guard.cancel() + if stream_result: + return result, result_length + try: + return result.read() + finally: + result.close() + + +class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler): + """Expose only bounded metadata, draft creation, and smart-HTTP Git.""" + + server_version = "HermesSCMBroker/1" + sys_version = "" + header_deadline_seconds = INBOUND_HEADER_TIMEOUT + + def _validate_headers(self) -> None: + items = list(self.headers.items()) + if len(items) > MAX_HEADERS: + raise PolicyError("SCM request has too many headers") + total = 0 + for name, value in items: + if not name.isascii() or not value.isascii(): + raise PolicyError("SCM request headers must be ASCII") + total += len(name) + len(value) + 4 + if any(ord(character) < 32 and character != "\t" for character in value): + raise PolicyError("SCM request header contains controls") + if total > MAX_HEADER_BYTES: + raise PolicyError("SCM request headers exceed the safe limit") + if len(self.headers.get_all("Content-Length", [])) > 1: + raise PolicyError("SCM request has duplicate Content-Length") + + def _stream( + self, status: int, content_type: str, body: BinaryIO, length: int + ) -> None: + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(length)) + self.end_headers() + deadline = time.monotonic() + INBOUND_BODY_TIMEOUT + while True: + chunk = body.read(STREAM_CHUNK) + if not chunk: + break + remaining = deadline - time.monotonic() + if remaining <= 0: + raise PolicyError("SCM response write deadline exceeded") + self.connection.settimeout(remaining) + self.wfile.write(chunk) + + def log_message(self, _format: str, *_args: object) -> None: + return + + def _json(self, status: int, body: bytes) -> None: + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.send_header("Cache-Control", "no-store") + self.end_headers() + self.wfile.write(body) + + def _reject(self, status: int = 400) -> None: + self._json(status, b'{"error":"request rejected"}') + + def do_GET(self) -> None: + try: + self._validate_headers() + if self.path == "/healthz": + self._json(200, b'{"status":"ok"}') + return + repo, operation, service = _git_target(self.path) + if operation != "info/refs": + raise PolicyError("Git RPC requires POST") + token = read_token() + _reject_forbidden(repo, "repository", (token,)) + expected = f"application/x-{service}-advertisement" + streamed = _upstream_git_request( + f"/atlas/{repo}.git/info/refs?service={service}", + method="GET", + body=None, + content_type=None, + expected_type=expected, + token=token, + stream_result=True, + ) + body, length = streamed + try: + self._stream(200, expected, body, length) + finally: + body.close() + except ( + OSError, + PolicyError, + socket.timeout, + urllib.error.URLError, + ValueError, + ): + self._reject() + + def do_POST(self) -> None: + try: + self._validate_headers() + self.connection.settimeout(INBOUND_BODY_TIMEOUT) + if self.path in {"/v1/metadata", "/v1/drafts"}: + self._control() + else: + self._git_rpc() + except ( + OSError, + PolicyError, + socket.timeout, + urllib.error.URLError, + ValueError, + json.JSONDecodeError, + ): + self._reject() + + def _control(self) -> None: + data = _load_json(self) + token = read_token() + if self.path == "/v1/metadata": + if set(data) != {"path"} or not isinstance(data["path"], str): + raise PolicyError("metadata request fields are invalid") + result = read(data["path"], token=token) + else: + expected = {"base", "body", "head", "head_sha", "repo", "title"} + if set(data) != expected or not all( + isinstance(data[key], str) for key in expected + ): + raise PolicyError("draft request fields are invalid") + result = create_draft(token=token, **data) # type: ignore[arg-type] + if token.encode("utf-8") in result: + raise PolicyError("SCM upstream reflected credential material") + self._json(200, result) + + def _git_rpc(self) -> None: + repo, operation, service = _git_target(self.path) + if ( + operation not in {"git-upload-pack", "git-receive-pack"} + or service != operation + ): + raise PolicyError("Git RPC operation is outside the allowlist") + expected_request = f"application/x-{service}-request" + if self.headers.get_content_type() != expected_request or self.headers.get( + "Transfer-Encoding" + ): + raise PolicyError("Git RPC request type is invalid") + length = _content_length(self.headers, MAX_GIT_REQUEST) + token = read_token() + _reject_forbidden(repo, "repository", (token,)) + body, body_length = _spool_bounded( + self.rfile, + MAX_GIT_REQUEST, + length, + token=token, + context="Git request", + deadline=time.monotonic() + INBOUND_BODY_TIMEOUT, + set_timeout=self.connection.settimeout, + ) + try: + if service == "git-receive-pack": + _validate_receive_pack(body, token) + expected = f"application/x-{service}-result" + streamed = _upstream_git_request( + f"/atlas/{repo}.git/{service}", + method="POST", + body=body, + body_length=body_length, + content_type=expected_request, + expected_type=expected, + token=token, + stream_result=True, + ) + result, result_length = streamed + try: + self._stream(200, expected, result, result_length) + finally: + result.close() + finally: + body.close() + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--listen", default="0.0.0.0") + parser.add_argument("--port", type=int, default=BROKER_PORT) + args = parser.parse_args() + BoundedThreadingHTTPServer((args.listen, args.port), BrokerHandler).serve_forever() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/hermes/scm-common/scripts/scm_broker_client.py b/services/hermes/scm-common/scripts/scm_broker_client.py new file mode 100644 index 00000000..9a5265b6 --- /dev/null +++ b/services/hermes/scm-common/scripts/scm_broker_client.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""Call the credential-isolated Hermes SCM broker over its fixed cluster origin.""" + +from __future__ import annotations + +import json +import urllib.request +from collections.abc import Callable + +import deadline_http +from gitea_api_policy import PolicyError + +BROKER_ORIGIN = "http://hermes-scm-broker.hermes-scm.svc.cluster.local:9081" +MAX_RESPONSE_BYTES = 2 * 1024 * 1024 + + +def _open(request: urllib.request.Request, timeout: int): + """Open one broker exchange under a killable absolute deadline. + + The helper process keeps every request on its fixed in-cluster origin by + rejecting redirects before they are followed. + """ + return deadline_http.open_bounded( + request, maximum=MAX_RESPONSE_BYTES, timeout=timeout + ) + + +def request( + endpoint: str, + payload: dict[str, object], + *, + opener: Callable[..., object] = _open, +) -> bytes: + """Send one bounded broker operation without any repository credential.""" + if endpoint not in {"/v1/metadata", "/v1/drafts"}: + raise PolicyError("SCM broker operation is outside the client allowlist") + body = json.dumps(payload, separators=(",", ":")).encode("utf-8") + if len(body) > 64 * 1024: + raise PolicyError("SCM broker request exceeds the safe size limit") + outgoing = urllib.request.Request( + BROKER_ORIGIN + endpoint, + data=body, + method="POST", + headers={ + "Accept": "application/json", + "Content-Type": "application/json", + "User-Agent": "hermes-scm-broker-client/1", + }, + ) + with opener(outgoing, timeout=30) as response: # type: ignore[attr-defined] + status = getattr(response, "status", None) + if status is None and hasattr(response, "getcode"): + status = response.getcode() # type: ignore[attr-defined] + if status != 200: + raise PolicyError("SCM broker returned an unexpected HTTP status") + content_type = response.headers.get_content_type() # type: ignore[attr-defined] + if content_type != "application/json": + raise PolicyError("SCM broker returned an unexpected response type") + result = response.read(MAX_RESPONSE_BYTES + 1) # type: ignore[attr-defined] + if len(result) > MAX_RESPONSE_BYTES: + raise PolicyError("SCM broker response exceeds the safe size limit") + json.loads(result) + return result + + +def read(path: str, *, opener: Callable[..., object] = _open) -> bytes: + """Read explicitly allowed Atlas metadata through the broker.""" + return request("/v1/metadata", {"path": path}, opener=opener) + + +def create_draft( + repo: str, + *, + base: str, + head: str, + head_sha: str, + title: str, + body: str, + opener: Callable[..., object] = _open, +) -> bytes: + """Create one verified draft through the broker for human review.""" + return request( + "/v1/drafts", + { + "base": base, + "body": body, + "head": head, + "head_sha": head_sha, + "repo": repo, + "title": title, + }, + opener=opener, + ) diff --git a/services/hermes/scm-common/scripts/scm_broker_io.py b/services/hermes/scm-common/scripts/scm_broker_io.py new file mode 100644 index 00000000..96df1fe0 --- /dev/null +++ b/services/hermes/scm-common/scripts/scm_broker_io.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""Bounded-memory stream helpers for the Hermes SCM broker.""" + +from __future__ import annotations + +import tempfile +import time +import urllib.request +from typing import BinaryIO + +from gitea_api_policy import PolicyError + + +class RejectRedirect(urllib.request.HTTPRedirectHandler): + """Reject every upstream redirect before credentials can be forwarded.""" + + def redirect_request(self, req, fp, code, msg, headers, newurl): + raise PolicyError("SCM upstream redirects are not allowed") + + +def response_status(response: object) -> int | None: + """Read an HTTP status from either urllib response interface.""" + value = getattr(response, "status", None) + if value is None and hasattr(response, "getcode"): + value = response.getcode() # type: ignore[attr-defined] + return value + + +def spool_response( + response, + maximum: int, + forbidden: tuple[bytes, ...], + *, + memory_limit: int, + chunk_size: int, + deadline_seconds: float, +) -> tuple[BinaryIO, int]: + """Spool an upstream response to disk while scanning chunk boundaries.""" + spool = tempfile.SpooledTemporaryFile( # noqa: SIM115 - caller owns returned spool + max_size=memory_limit, dir="/tmp" + ) + carry = b"" + total = 0 + deadline = time.monotonic() + deadline_seconds + try: + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise PolicyError("SCM upstream response deadline exceeded") + raw = getattr(getattr(response, "fp", None), "raw", None) + sock = getattr(raw, "_sock", raw) + if hasattr(sock, "settimeout"): + sock.settimeout(remaining) + chunk = response.read(chunk_size) + if not chunk: + break + total += len(chunk) + if total > maximum: + raise PolicyError("SCM upstream response exceeds the safe size limit") + candidate = carry + chunk + if any(value in candidate for value in forbidden): + raise PolicyError( + "Git upstream response contains runtime credential material" + ) + width = max(len(value) for value in forbidden) - 1 + carry = candidate[-width:] if width else b"" + spool.write(chunk) + spool.seek(0) + return spool, total + except Exception: + spool.close() + raise diff --git a/services/hermes/scm-common/scripts/scm_broker_server.py b/services/hermes/scm-common/scripts/scm_broker_server.py new file mode 100644 index 00000000..f29cd314 --- /dev/null +++ b/services/hermes/scm-common/scripts/scm_broker_server.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +"""Concurrency-bounded threaded HTTP server for the Hermes SCM broker.""" + +from __future__ import annotations + +import threading +import time +from http.server import ThreadingHTTPServer + +MAX_CONCURRENT_REQUESTS = 3 + + +class _AbsoluteDeadlineReader: + """Read header lines against one wall-clock deadline, not idle timeouts.""" + + def __init__(self, stream, connection): + self._stream = stream + self._connection = connection + self._deadline: float | None = None + + def begin(self, timeout: float) -> None: + self._deadline = time.monotonic() + timeout + + def end(self) -> None: + self._deadline = None + + def readline(self, limit: int = -1) -> bytes: + if self._deadline is None: + return self._stream.readline(limit) + value = bytearray() + while limit < 0 or len(value) < limit: + remaining = self._deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError("absolute SCM header deadline exceeded") + self._connection.settimeout(remaining) + character = self._stream.read(1) + if not character: + break + value.extend(character) + if character == b"\n": + break + return bytes(value) + + def __getattr__(self, name: str): + return getattr(self._stream, name) + + +class AbsoluteHeaderDeadlineMixin: + """Apply an absolute request-line plus header deadline to HTTP handlers.""" + + header_deadline_seconds = 10.0 + + def setup(self) -> None: + super().setup() + self._header_reader = _AbsoluteDeadlineReader(self.rfile, self.connection) + self.rfile = self._header_reader + + def handle_one_request(self) -> None: + try: + self._header_reader.begin(self.header_deadline_seconds) + try: + self.raw_requestline = self.rfile.readline(65537) + if len(self.raw_requestline) > 65536: + self.requestline = "" + self.request_version = "" + self.command = "" + self.send_error(414) + return + if not self.raw_requestline: + self.close_connection = True + return + if not self.parse_request(): + return + finally: + self._header_reader.end() + method_name = "do_" + self.command + if not hasattr(self, method_name): + self.send_error(501, "Unsupported method") + return + getattr(self, method_name)() + self.wfile.flush() + except TimeoutError: + self.close_connection = True + + +class BoundedThreadingHTTPServer(ThreadingHTTPServer): + """Cap active handlers so large Git exchanges cannot exhaust memory.""" + + daemon_threads = True + block_on_close = True + request_queue_size = 16 + + def __init__(self, *args, **kwargs): + self._slots = threading.BoundedSemaphore(MAX_CONCURRENT_REQUESTS) + super().__init__(*args, **kwargs) + + def process_request(self, request, client_address) -> None: + if not self._slots.acquire(blocking=False): + try: + request.sendall( + b"HTTP/1.1 503 Service Unavailable\r\n" + b"Connection: close\r\nContent-Length: 0\r\n\r\n" + ) + finally: + self.shutdown_request(request) + return + try: + super().process_request(request, client_address) + except Exception: + self._slots.release() + self.shutdown_request(request) + raise + + def process_request_thread(self, request, client_address) -> None: + try: + super().process_request_thread(request, client_address) + finally: + self._slots.release() diff --git a/services/hermes/scripts/gitea_api.py b/services/hermes/scripts/gitea_api.py deleted file mode 100644 index 5544c398..00000000 --- a/services/hermes/scripts/gitea_api.py +++ /dev/null @@ -1,150 +0,0 @@ -#!/usr/bin/env python3 -"""Call the private Atlas Gitea API through a runtime-only token boundary.""" - -from __future__ import annotations - -import argparse -import json -import os -import sys -import urllib.error -import urllib.parse -import urllib.request -from pathlib import Path - - -DEFAULT_BASE_URL = "https://scm.bstein.dev" -DEFAULT_TOKEN_FILE = Path("/runtime-access/gitea-token") -ALLOWED_METHODS = ("GET", "POST", "PUT", "PATCH", "DELETE") - - -def read_token(path: Path = DEFAULT_TOKEN_FILE) -> str: - """Read and validate the token from its in-memory Vault projection.""" - token = path.read_text(encoding="utf-8").strip() - if not token: - raise ValueError(f"runtime credential is empty: {path}") - return token - - -def api_url(base_url: str, path: str) -> str: - """Return a same-origin Gitea API URL for a validated API path.""" - base = urllib.parse.urlsplit(base_url.rstrip("/")) - target = urllib.parse.urlsplit(path) - if base.scheme not in {"http", "https"} or not base.netloc: - raise ValueError("GITEA_BASE_URL must be an absolute HTTP(S) URL") - if target.scheme or target.netloc or target.fragment: - raise ValueError("API path must be relative to the configured Gitea origin") - if not target.path.startswith("/api/v1/"): - raise ValueError("API path must start with /api/v1/") - return urllib.parse.urlunsplit( - (base.scheme, base.netloc, target.path, target.query, "") - ) - - -def build_request( - method: str, - path: str, - *, - base_url: str, - token: str, - data: object | None = None, -) -> urllib.request.Request: - """Build one authenticated request without placing the token in its URL.""" - payload = None - if data is not None: - payload = json.dumps(data, separators=(",", ":")).encode("utf-8") - return urllib.request.Request( - api_url(base_url, path), - data=payload, - method=method, - headers={ - "Accept": "application/json", - "Authorization": f"token {token}", - "Content-Type": "application/json", - "User-Agent": "hermes-atlas-operator/1", - }, - ) - - -def parse_args(argv: list[str] | None = None) -> argparse.Namespace: - """Parse a bounded method, API path, and optional JSON request body.""" - parser = argparse.ArgumentParser( - description="Call the Atlas Gitea API using the runtime Vault token." - ) - parser.add_argument("method", choices=ALLOWED_METHODS) - parser.add_argument("path", help="Gitea path beginning with /api/v1/") - data_group = parser.add_mutually_exclusive_group() - data_group.add_argument("--data-json", help="JSON object/array request body") - data_group.add_argument( - "--data-file", type=Path, help="path to a JSON request body" - ) - data_group.add_argument( - "--field", - action="append", - metavar="KEY=VALUE", - help=( - "repeatable JSON field; bare text remains a string while true, false, " - "null, numbers, objects, and arrays are decoded as JSON" - ), - ) - return parser.parse_args(argv) - - -def parse_fields(values: list[str]) -> dict[str, object]: - """Build a JSON object from shell-safe, repeatable key/value arguments.""" - result: dict[str, object] = {} - for value in values: - key, separator, raw = value.partition("=") - if not separator or not key or any(char.isspace() for char in key): - raise ValueError("each --field must be KEY=VALUE with a non-space key") - try: - result[key] = json.loads(raw) - except json.JSONDecodeError: - result[key] = raw - return result - - -def load_data(args: argparse.Namespace) -> object | None: - """Decode the optional JSON body without involving a shell expansion.""" - if args.data_json is not None: - return json.loads(args.data_json) - if args.data_file is not None: - return json.loads(args.data_file.read_text(encoding="utf-8")) - if args.field is not None: - return parse_fields(args.field) - return None - - -def main(argv: list[str] | None = None) -> int: - """Execute the request, print only its response body, and return HTTP status.""" - args = parse_args(argv) - try: - request = build_request( - args.method, - args.path, - base_url=os.environ.get("GITEA_BASE_URL", DEFAULT_BASE_URL), - token=read_token(), - data=load_data(args), - ) - with urllib.request.urlopen(request, timeout=30) as response: - body = response.read() - if body: - sys.stdout.buffer.write(body) - if not body.endswith(b"\n"): - sys.stdout.buffer.write(b"\n") - return 0 - except urllib.error.HTTPError as exc: - body = exc.read(65536) - print(f"Gitea API returned HTTP {exc.code}", file=sys.stderr) - if body: - sys.stderr.buffer.write(body) - if not body.endswith(b"\n"): - sys.stderr.buffer.write(b"\n") - return 1 - except (OSError, ValueError, json.JSONDecodeError) as exc: - print(f"Gitea API request failed: {exc}", file=sys.stderr) - return 1 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/services/hermes/scripts/gitea_askpass.sh b/services/hermes/scripts/gitea_askpass.sh deleted file mode 100755 index d6146d80..00000000 --- a/services/hermes/scripts/gitea_askpass.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env sh -set -eu - -case "${1:-}" in - *Username*) - if [ -s /runtime-access/gitea-username ]; then - tr -d '\r\n' Path: @@ -139,24 +141,8 @@ def bootstrap_cassandra_state(root: Path) -> dict[str, str]: return {"state": "ready"} -def _gitea_token_path() -> Path: - """Return the runtime-only Gitea credential path without reading it.""" - configured = os.environ.get("HERMES_GITEA_TOKEN_FILE", "").strip() - return Path(configured) if configured else DEFAULT_GITEA_TOKEN_PATH - - -def _gitea_token_available() -> bool: - """Check credential readiness without loading it into process memory.""" - try: - path = _gitea_token_path() - return path.is_file() and path.stat().st_size > 0 - except OSError: - return False - - def sync_cassandra_repo(env_values: dict[str, str]) -> str: - """Clone or fetch Cassandra through the runtime-only askpass credential.""" - token_available = _gitea_token_available() + """Clone or fetch Cassandra through the credential-isolated SCM broker.""" if shutil.which("git") is None: return "git-unavailable" child_env = os.environ.copy() @@ -173,9 +159,7 @@ def sync_cassandra_repo(env_values: dict[str, str]) -> str: } } ) - child_env["GIT_ASKPASS"] = env_values.get( - "GIT_ASKPASS", "/opt/coordinator/gitea_askpass.sh" - ) + child_env.pop("GIT_ASKPASS", None) child_env["GIT_TERMINAL_PROMPT"] = "0" if (CASSANDRA_BASE_PATH / ".git").exists(): try: @@ -232,8 +216,6 @@ def sync_cassandra_repo(env_values: dict[str, str]) -> str: return f"remote-repair-failed-{repaired.returncode}" except (OSError, subprocess.TimeoutExpired): return "remote-repair-failed" - if not token_available: - return "ready; fetch skipped until Gitea token is configured" command = [ "git", "-C", @@ -247,8 +229,6 @@ def sync_cassandra_repo(env_values: dict[str, str]) -> str: CASSANDRA_BASE_PATH.mkdir(parents=True, exist_ok=True) if any(CASSANDRA_BASE_PATH.iterdir()): return "unmanaged-nonempty-directory" - if not token_available: - return "awaiting-gitea-token" command = [ "git", "clone", diff --git a/services/hermes/scripts/jenkins_image_build_trigger.py b/services/hermes/scripts/jenkins_image_build_trigger.py new file mode 100755 index 00000000..6cd91ed8 --- /dev/null +++ b/services/hermes/scripts/jenkins_image_build_trigger.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +"""Trigger only the reviewed-main Hermes agent image release job.""" + +from __future__ import annotations + +import argparse +import json +import re +import urllib.error +import urllib.parse +import urllib.request +from pathlib import Path + + +JENKINS_ORIGIN = "https://ci.bstein.dev" +JENKINS_BUILD_URL = f"{JENKINS_ORIGIN}/buildByToken/buildWithParameters" +JOB_NAME = "hermes-agent-image" +TOKEN_FILE = Path("/runtime-access/jenkins-image-build-token") +REVISION_PATTERN = re.compile(r"^[0-9a-f]{40}$") +QUEUE_PATH_PATTERN = re.compile(r"^/queue/item/[0-9]+/?$") + + +class _NoRedirect(urllib.request.HTTPRedirectHandler): + """Keep a queued-build redirect from becoming an unauthorized job read.""" + + def redirect_request(self, _request, _file, _code, _message, _headers, _url): + return None + + +def _open_without_redirect(request: urllib.request.Request, timeout: int): + """Return the Build Token Root response, including its expected HTTP 303.""" + opener = urllib.request.build_opener(_NoRedirect()) + try: + return opener.open(request, timeout=timeout) + except urllib.error.HTTPError as exc: + if exc.code == 303: + return exc + raise + + +def trigger_build( + revision: str, + *, + token_file: Path = TOKEN_FILE, + opener=_open_without_redirect, +) -> dict[str, str | int]: + """Post the fixed job parameters using its job-scoped build token.""" + revision = revision.strip() + if not REVISION_PATTERN.fullmatch(revision): + raise ValueError("revision must be a lowercase full 40-character commit") + token = token_file.read_text(encoding="utf-8").strip() + if not token: + raise RuntimeError("Jenkins image-build token is empty") + payload = urllib.parse.urlencode( + { + "job": JOB_NAME, + "token": token, + "PUBLISH_IMAGE": "true", + "EXPECTED_SOURCE_REVISION": revision, + "CONFIRM_PUBLISH": "PUBLISH HERMES AGENT", + } + ).encode("utf-8") + request = urllib.request.Request( + JENKINS_BUILD_URL, + data=payload, + headers={"Content-Type": "application/x-www-form-urlencoded"}, + method="POST", + ) + with opener(request, timeout=20) as response: + status = int(response.status) + location = response.headers.get("Location", "") + if status not in {201, 303}: + raise RuntimeError(f"Jenkins trigger returned HTTP {status}") + if not location: + raise RuntimeError("Jenkins trigger omitted the queue Location") + queue_url = urllib.parse.urljoin(f"{JENKINS_ORIGIN}/", location) + parsed_queue = urllib.parse.urlsplit(queue_url) + expected_origin = urllib.parse.urlsplit(JENKINS_ORIGIN) + if ( + parsed_queue.scheme != expected_origin.scheme + or parsed_queue.netloc != expected_origin.netloc + or parsed_queue.query + or parsed_queue.fragment + or not QUEUE_PATH_PATTERN.fullmatch(parsed_queue.path) + ): + raise RuntimeError("Jenkins returned an invalid queue Location") + # Never return the submitted URL: its form body contains the job token. + queue_path = parsed_queue.path + return { + "job": JOB_NAME, + "queue_path": queue_path, + "source_revision": revision, + "status": status, + } + + +def main() -> int: + """Validate one revision, trigger the bounded job, and print safe metadata.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("revision", help="reviewed full commit currently on main") + args = parser.parse_args() + try: + result = trigger_build(args.revision) + except (OSError, ValueError, RuntimeError, urllib.error.URLError) as exc: + print(json.dumps({"error": str(exc)}, sort_keys=True)) + return 1 + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": # pragma: no cover - exercised through main() + raise SystemExit(main()) diff --git a/services/hermes/scripts/node_account_audit.py b/services/hermes/scripts/node_account_audit.py new file mode 100644 index 00000000..fa6245ba --- /dev/null +++ b/services/hermes/scripts/node_account_audit.py @@ -0,0 +1,267 @@ +#!/usr/bin/env python3 +"""Fail-closed supplementary-group and host privilege-policy audits.""" + +from __future__ import annotations + +import re +import stat +import struct +from pathlib import Path + +from node_account_io import HardeningError, read_regular +from node_polkit_audit import audit_polkit_policy + + +def _members(field: str, context: str) -> set[str]: + if not field: + return set() + values = field.split(",") + if any(not value or value.strip() != value for value in values): + raise HardeningError(f"{context} has a malformed member list") + if len(values) != len(set(values)): + raise HardeningError(f"{context} has duplicate members") + return set(values) + + +def audit_membership( + account: str, groups: list[list[str]], gshadow: list[list[str]] +) -> None: + """Reject the dedicated account in any supplementary group/admin list.""" + for record in groups: + if account in _members(record[3], f"group {record[0]}"): + raise HardeningError("dedicated Hermes account has supplementary group access") + for record in gshadow: + principals = _members(record[2], f"gshadow {record[0]} admins") + principals |= _members(record[3], f"gshadow {record[0]} members") + if account in principals: + raise HardeningError("dedicated Hermes account has gshadow group access") + + +def _policy_files(host_etc: Path, polkit_share: Path, root_uid: int) -> list[Path]: + files: list[Path] = [] + sudoers = host_etc / "sudoers" + if sudoers.exists(): + files.append(sudoers) + roots = ( + host_etc / "sudoers.d", + host_etc / "polkit-1/rules.d", + host_etc / "polkit-1/localauthority", + polkit_share / "rules.d", + polkit_share / "localauthority", + ) + for root in roots: + if not root.exists(): + continue + metadata = root.lstat() + if ( + not stat.S_ISDIR(metadata.st_mode) + or stat.S_ISLNK(metadata.st_mode) + or metadata.st_uid != root_uid + or stat.S_IMODE(metadata.st_mode) & 0o022 + ): + raise HardeningError("unsafe sudo/polkit policy directory") + files.extend(path for path in root.rglob("*") if path.is_file()) + if len(files) > 256: + raise HardeningError("too many sudo/polkit policy files") + return sorted(files) + + +def _audit_policy_metadata(snapshot, root_uid: int, account_uid: int) -> None: + if snapshot.uid != root_uid or snapshot.mode & 0o022: + raise HardeningError("sudo/polkit policy metadata permits unsafe mutation") + acl = dict(snapshot.xattrs).get("system.posix_acl_access", b"") + if not acl: + return + header = struct.Struct(" str: + """Remove comments without discarding sudoers' numeric-UID principals.""" + active: list[str] = [] + for line in text.splitlines(): + stripped = line.lstrip() + if not stripped: + continue + # Detect any include spelling before the comment rule below. sudo + # honours both #include and @include; a quoted or space-bearing path + # must never fall through the "#" comment branch and leave a second + # authority file unaudited. Only the exact bare includedir into the + # audited /etc/sudoers.d passes; everything else fails closed. + if re.match(r"(?i)[#@]include(?:dir)?(?=\s|$)", stripped): + include = re.fullmatch( + r"(?i)([#@]include(?:dir)?)\s+([^\s\"'\\]+)", stripped + ) + if include: + directive, target = include.groups() + if ( + directive.lower().endswith("includedir") + and target == "/etc/sudoers.d" + ): + continue + raise HardeningError("sudo policy includes an unaudited authority source") + if stripped.startswith("#") and not re.match(r"#\d+(?:\s|$)", stripped): + continue + active.append(line) + return "\n".join(active) + + +def _mentions_dedicated_identity(text: str, account: str, account_uid: int) -> bool: + account_pattern = rf"(? list[str]: + """Merge sudoers backslash continuations into whole logical lines.""" + lines: list[str] = [] + pending = "" + for line in active.splitlines(): + if line.endswith("\\"): + pending += line[:-1] + " " + continue + lines.append(pending + line) + pending = "" + if pending: + raise HardeningError("sudo policy ends inside a line continuation") + return lines + + +def _sudo_user_aliases(lines: list[str]) -> dict[str, list[str]]: + """Collect User_Alias definitions so grant principals can be expanded.""" + aliases: dict[str, list[str]] = {} + for line in lines: + stripped = line.strip() + if not stripped.startswith("User_Alias"): + continue + for definition in stripped.removeprefix("User_Alias").split(":"): + name, separator, members = definition.partition("=") + name = name.strip() + values = [item.strip() for item in members.split(",")] + if ( + not separator + or not SUDO_ALIAS_NAME_RE.fullmatch(name) + or name in aliases + or not all(values) + ): + raise HardeningError("sudo alias definition is not auditable") + aliases[name] = values + return aliases + + +def _expand_sudo_principal( + principal: str, aliases: dict[str, list[str]], seen: frozenset[str] +) -> None: + """Reject grant principals that reach Hermes through aliases or wildcards.""" + value = principal.strip() + while value.startswith("!"): + value = value[1:].strip() + value = value.strip('"') + if not value: + raise HardeningError("sudo grant principal is malformed") + if value in aliases: + if value in seen: + raise HardeningError("sudo alias expansion is cyclic") + for member in aliases[value]: + _expand_sudo_principal(member, aliases, seen | {value}) + return + if value in {"ALL", "%ALL"}: + raise HardeningError("broad sudo authority includes Hermes") + if value.startswith("+") or value.startswith("%:"): + raise HardeningError("sudo netgroup authority cannot be audited") + if SUDO_ALIAS_NAME_RE.fullmatch(value): + raise HardeningError("sudo alias is undefined") + + +def _audit_sudo_grants(active: str) -> None: + """Expand every grant's user list; only literal non-Hermes principals pass.""" + lines = _joined_sudo_lines(active) + aliases = _sudo_user_aliases(lines) + skip = ("Defaults", "User_Alias", "Runas_Alias", "Host_Alias", "Cmnd_Alias") + for line in lines: + stripped = line.strip() + if not stripped or stripped.startswith(skip): + continue + head, separator, _rest = stripped.partition("=") + fields = head.split() + if not separator or len(fields) < 2: + raise HardeningError("sudo grant line is not auditable") + for principal in " ".join(fields[:-1]).split(","): + _expand_sudo_principal(principal, aliases, frozenset()) + + +def audit_privilege_policies( + account: str, + account_uid: int, + root_uid: int, + host_etc: Path, + polkit_share: Path, +) -> None: + """Reject direct, group, wildcard sudo, or root-equivalent polkit grants.""" + dangerous_polkit = ( + "org.freedesktop.policykit.exec", + "org.freedesktop.systemd1.manage-unit", + "org.freedesktop.udisks2.modify-device", + "org.freedesktop.packagekit", + ) + nsswitch = host_etc / "nsswitch.conf" + if nsswitch.exists(): + try: + nss_snapshot = read_regular(nsswitch, 64 * 1024) + _audit_policy_metadata(nss_snapshot, root_uid, account_uid) + text = nss_snapshot.value.decode("utf-8") + except UnicodeDecodeError as exc: + raise HardeningError("nsswitch policy is not UTF-8") from exc + for line in text.splitlines(): + name, separator, sources = line.partition(":") + if name.strip() in { + "passwd", + "group", + "initgroups", + "shadow", + "sudoers", + } and separator: + active_sources = [item for item in sources.split() if not item.startswith("[")] + if not active_sources or any( + item not in {"files", "systemd"} for item in active_sources + ): + raise HardeningError("external group/account/sudo authority source is unsafe") + total = 0 + for path in _policy_files(host_etc, polkit_share, root_uid): + snapshot = read_regular(path, 256 * 1024) + _audit_policy_metadata(snapshot, root_uid, account_uid) + value = snapshot.value + total += len(value) + if total > 2 * 1024 * 1024: + raise HardeningError("sudo/polkit policy input exceeds safe limit") + try: + text = value.decode("utf-8") + except UnicodeDecodeError as exc: + raise HardeningError("sudo/polkit policy is not UTF-8") from exc + if path.name == "sudoers" or "sudoers.d" in path.parts: + active = _active_sudo_policy(text) + if _mentions_dedicated_identity(active, account, account_uid): + raise HardeningError("dedicated Hermes account has sudo authority") + _audit_sudo_grants(active) + continue + active = "\n".join( + line for line in text.splitlines() if not line.lstrip().startswith("#") + ) + grants = "polkit.Result.YES" in active or re.search( + r"(?im)^\s*Result(?:Any|Inactive|Active)\s*=\s*yes\s*$", active + ) + if grants and _mentions_dedicated_identity(active, account, account_uid): + raise HardeningError("dedicated Hermes account has polkit authority") + broad = "unix-user:*" in active or "Identity=unix-user:*" in active + if broad and grants and any(item in active for item in dangerous_polkit): + raise HardeningError("root-equivalent broad polkit authority includes Hermes") + audit_polkit_policy(path, text) diff --git a/services/hermes/scripts/node_account_hardening.py b/services/hermes/scripts/node_account_hardening.py new file mode 100644 index 00000000..a545af47 --- /dev/null +++ b/services/hermes/scripts/node_account_hardening.py @@ -0,0 +1,496 @@ +#!/usr/bin/env python3 +"""Reconcile a dedicated unprivileged Hermes SSH account on one Atlas node.""" + +from __future__ import annotations + +import argparse +import base64 +import errno +import os +import re +import shlex +import stat +import struct +from contextlib import suppress +from pathlib import Path + +from node_account_audit import audit_membership, audit_privilege_policies +from node_account_io import ( + FileSnapshot, + HardeningError, + account_lock, + assert_unchanged, + atomic_write, + backup_once, + read_regular, +) + +HOST_ETC = Path("/host-etc") +HOST_HOME = Path("/host-home") +HOST_K3S = Path("/host-k3s") +HOST_KUBELET = Path("/host-kubelet") +HOST_RUN_K3S = Path("/host-run-k3s") +HOST_RUN_CONTAINERD = Path("/host-run-containerd") +HOST_POLKIT_SHARE = Path("/host-polkit-share") +ACCOUNT = "hermes-agent" +ACCOUNT_UID = 1200 +ACCOUNT_GID = 1200 +ACCOUNT_HOME = "/home/hermes-agent" +ACCOUNT_SHELL = "/bin/bash" +HOST_ROOT_UID = 0 +HOST_ROOT_GID = 0 +LEGACY_ACCOUNTS = ("atlas", "oceanus") +MAX_ACCOUNT_FILE = 2 * 1024 * 1024 +MAX_AUTHORIZED_KEYS = 1024 * 1024 +ACL_VERSION = 2 +ACL_UNDEFINED_ID = 0xFFFFFFFF +ACL_USER_OBJ = 0x01 +ACL_USER = 0x02 +ACL_GROUP_OBJ = 0x04 +ACL_GROUP = 0x08 +ACL_MASK = 0x10 +ACL_OTHER = 0x20 +ACL_HEADER = struct.Struct(" tuple[bytes, os.stat_result]: + snapshot = read_regular(path, maximum) + return snapshot.value, path.stat(follow_symlinks=False) + + +def _records(value: bytes, fields: int, name: str) -> list[list[str]]: + try: + text = value.decode("utf-8") + except UnicodeDecodeError as exc: + raise HardeningError(f"{name} is not UTF-8") from exc + if not text.endswith("\n"): + raise HardeningError(f"{name} is missing its final newline") + records = [] + names = set() + for line in text.splitlines(): + parts = line.split(":") + if len(parts) != fields or not parts[0] or parts[0] in names: + raise HardeningError(f"{name} has an invalid record") + names.add(parts[0]) + records.append(parts) + return records + + +def _encode(records: list[list[str]]) -> bytes: + return ("\n".join(":".join(record) for record in records) + "\n").encode() + + +def _expected_records() -> dict[str, list[str]]: + return { + "passwd": [ + ACCOUNT, + "x", + str(ACCOUNT_UID), + str(ACCOUNT_GID), + "Hermes Agent", + ACCOUNT_HOME, + ACCOUNT_SHELL, + ], + "group": [ACCOUNT, "x", str(ACCOUNT_GID), ""], + "shadow": [ACCOUNT, "!", "1", "0", "99999", "7", "", "", ""], + "gshadow": [ACCOUNT, "!", "", ""], + } + + +def _reconcile_record( + records: list[list[str]], expected: list[str], *, identity_index: int +) -> list[list[str]]: + for record in records: + same_name = record[0] == expected[0] + same_identity = record[identity_index] == expected[identity_index] + if same_name or same_identity: + if record != expected: + raise HardeningError("dedicated Hermes account identity conflicts") + return records + return [*records, expected] + + +def _reconcile_databases() -> None: + expected = _expected_records() + definitions = ( + ("passwd", 7, 2), + ("group", 4, 2), + ("shadow", 9, 0), + ("gshadow", 4, 0), + ) + planned: list[tuple[Path, FileSnapshot, bytes]] = [] + account_present = False + parsed: dict[str, list[list[str]]] = {} + for name, fields, identity_index in definitions: + path = HOST_ETC / name + snapshot = read_regular(path, MAX_ACCOUNT_FILE) + records = _records(snapshot.value, fields, name) + parsed[name] = records + updated = _reconcile_record( + records, expected[name], identity_index=identity_index + ) + if name == "passwd": + account_present = expected[name] in records + planned.append((path, snapshot, _encode(updated))) + audit_membership(ACCOUNT, parsed["group"], parsed["gshadow"]) + audit_privilege_policies( + ACCOUNT, ACCOUNT_UID, HOST_ROOT_UID, HOST_ETC, HOST_POLKIT_SHARE + ) + try: + (HOST_HOME / ACCOUNT).lstat() + except FileNotFoundError: + pass + else: + if not account_present: + raise HardeningError("dedicated Hermes account home already exists") + for path, snapshot, _updated in planned: + assert_unchanged(path, snapshot, MAX_ACCOUNT_FILE) + for path, snapshot, _updated in planned: + backup_once(path, snapshot, MAX_ACCOUNT_FILE) + for path, snapshot, _updated in planned: + assert_unchanged(path, snapshot, MAX_ACCOUNT_FILE) + written: list[tuple[Path, FileSnapshot, bytes]] = [] + try: + for path, snapshot, updated in planned: + assert_unchanged(path, snapshot, MAX_ACCOUNT_FILE) + atomic_write(path, updated, snapshot) + written.append((path, snapshot, updated)) + for path, _snapshot, _updated in planned: + name = path.name + fields = next(item[1] for item in definitions if item[0] == name) + records = _records(_read_regular(path)[0], fields, name) + if expected[name] not in records: + raise HardeningError("dedicated Hermes account validation failed") + except Exception as error: + for path, snapshot, updated in reversed(written): + current = read_regular(path, MAX_ACCOUNT_FILE) + if ( + current.value != updated + or current.mode != snapshot.mode + or current.uid != snapshot.uid + or current.gid != snapshot.gid + or current.xattrs != snapshot.xattrs + ): + raise HardeningError( + f"concurrent host account change prevents rollback: {path.name}" + ) from error + atomic_write(path, snapshot.value, snapshot) + raise + + +SUPPORTED_KEY_TYPES = {"ssh-ed25519", "ecdsa-sha2-nistp256", "ssh-rsa"} +KEY_TYPE_RE = re.compile(r"(?:sk-)?(?:ssh|ecdsa)-[A-Za-z0-9@._+-]+\Z") + + +def _key_identity(line: bytes) -> tuple[str, bytes] | None: + """Return OpenSSH key type/blob, ignoring options and comments.""" + stripped = line.strip(b"\r\n") + if not stripped or stripped.lstrip().startswith(b"#"): + return None + try: + fields = shlex.split(stripped.decode("ascii"), posix=True) + except (UnicodeDecodeError, ValueError) as exc: + raise HardeningError("authorized key line is malformed") from exc + indexes = [ + index for index, field in enumerate(fields) if KEY_TYPE_RE.fullmatch(field) + ] + if len(indexes) != 1 or indexes[0] + 1 >= len(fields): + raise HardeningError("authorized key line has ambiguous key material") + key_type = fields[indexes[0]] + try: + blob = base64.b64decode(fields[indexes[0] + 1], validate=True) + except ValueError as exc: + raise HardeningError("authorized key payload is invalid") from exc + encoded_type = key_type.encode("ascii") + if len(blob) < 4 or int.from_bytes(blob[:4], "big") != len(encoded_type): + raise HardeningError("authorized key blob is malformed") + if blob[4 : 4 + len(encoded_type)] != encoded_type: + raise HardeningError("authorized key type does not match its blob") + return key_type, blob + + +def _validated_public_key(path: Path) -> tuple[bytes, tuple[str, bytes]]: + value, _ = _read_regular(path, 16 * 1024) + line = value.strip() + if b"\n" in line or b"\r" in line: + raise HardeningError("Hermes public key must contain one line") + identity = _key_identity(line) + fields = line.split() + if ( + identity is None + or identity[0] not in SUPPORTED_KEY_TYPES + or len(fields) not in {2, 3} + or fields[0].decode("ascii") != identity[0] + ): + raise HardeningError("Hermes public key format is unsupported") + return line, identity + + +def _directory(path: Path, *, mode: int, uid: int, gid: int) -> None: + with suppress(FileExistsError): + path.mkdir(mode=mode) + metadata = path.lstat() + if not stat.S_ISDIR(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode): + raise HardeningError(f"unsafe account directory: {path.name}") + if metadata.st_uid not in {0, uid} or metadata.st_gid not in {0, gid}: + raise HardeningError(f"account directory ownership conflicts: {path.name}") + os.chown(path, uid, gid) + path.chmod(mode) + + +def _without_key(value: bytes, identity: tuple[str, bytes]) -> bytes: + kept = [] + for line in value.splitlines(keepends=True): + if _key_identity(line) != identity: + kept.append(line) + return b"".join(kept) + + +def _move_key(public_key: Path) -> None: + key, identity = _validated_public_key(public_key) + target = HOST_HOME / ACCOUNT / ".ssh" / "authorized_keys" + legacy_plan: list[tuple[Path, FileSnapshot, bytes]] = [] + for legacy in LEGACY_ACCOUNTS: + authorized = HOST_HOME / legacy / ".ssh" / "authorized_keys" + try: + snapshot = read_regular(authorized, MAX_AUTHORIZED_KEYS) + except FileNotFoundError: + continue + updated = _without_key(snapshot.value, identity) + if updated != snapshot.value: + legacy_plan.append((authorized, snapshot, updated)) + if legacy_plan and target.exists(): + target_snapshot = read_regular(target, MAX_AUTHORIZED_KEYS) + target_without_key = _without_key(target_snapshot.value, identity) + if target_without_key != target_snapshot.value: + backup_once(target, target_snapshot, MAX_AUTHORIZED_KEYS) + assert_unchanged(target, target_snapshot, MAX_AUTHORIZED_KEYS) + atomic_write(target, target_without_key, target_snapshot) + verified_target = read_regular(target, MAX_AUTHORIZED_KEYS).value + if _without_key(verified_target, identity) != verified_target: + raise HardeningError("preexisting Hermes authorization removal failed") + for authorized, snapshot, _updated in legacy_plan: + assert_unchanged(authorized, snapshot, MAX_AUTHORIZED_KEYS) + backup_once(authorized, snapshot, MAX_AUTHORIZED_KEYS) + for authorized, snapshot, updated in legacy_plan: + assert_unchanged(authorized, snapshot, MAX_AUTHORIZED_KEYS) + atomic_write(authorized, updated, snapshot) + for legacy in LEGACY_ACCOUNTS: + authorized = HOST_HOME / legacy / ".ssh" / "authorized_keys" + try: + value = read_regular(authorized, MAX_AUTHORIZED_KEYS).value + except FileNotFoundError: + continue + if _without_key(value, identity) != value: + raise HardeningError("legacy Hermes authorization key removal failed") + + # Installation happens only after every legacy authorization is absent. A + # crash can therefore remove access temporarily, but can never duplicate + # the machine credential across privileged and unprivileged accounts. + home = HOST_HOME / ACCOUNT + ssh = home / ".ssh" + _directory(home, mode=0o700, uid=ACCOUNT_UID, gid=ACCOUNT_GID) + _directory(ssh, mode=0o700, uid=ACCOUNT_UID, gid=ACCOUNT_GID) + target = ssh / "authorized_keys" + if target.exists(): + current = read_regular(target, MAX_AUTHORIZED_KEYS) + if current.value != key + b"\n": + backup_once(target, current, MAX_AUTHORIZED_KEYS) + else: + current = FileSnapshot( + value=b"", + device=0, + inode=0, + mode=0o600, + uid=ACCOUNT_UID, + gid=ACCOUNT_GID, + size=0, + mtime_ns=0, + ctime_ns=0, + xattrs=(), + ) + atomic_write(target, key + b"\n", current) + if _read_regular(target, MAX_AUTHORIZED_KEYS)[0] != key + b"\n": + raise HardeningError("dedicated Hermes authorized key validation failed") + + +def _decode_acl(value: bytes, mode: int) -> list[tuple[int, int, int]]: + """Decode a bounded POSIX ACL or derive one from ordinary mode bits.""" + if not value: + group = (mode >> 3) & 0o7 + return [ + (ACL_USER_OBJ, (mode >> 6) & 0o7, ACL_UNDEFINED_ID), + (ACL_GROUP_OBJ, group, ACL_UNDEFINED_ID), + (ACL_MASK, group, ACL_UNDEFINED_ID), + (ACL_OTHER, mode & 0o7, ACL_UNDEFINED_ID), + ] + if len(value) < ACL_HEADER.size or (len(value) - ACL_HEADER.size) % ACL_ENTRY.size: + raise HardeningError("sensitive directory ACL is malformed") + if ACL_HEADER.unpack_from(value)[0] != ACL_VERSION: + raise HardeningError("sensitive directory ACL version is unsupported") + entries = [ + ACL_ENTRY.unpack_from(value, offset) + for offset in range(ACL_HEADER.size, len(value), ACL_ENTRY.size) + ] + if any(permission > 0o7 for _tag, permission, _identifier in entries): + raise HardeningError("sensitive directory ACL permission is malformed") + required = {ACL_USER_OBJ, ACL_GROUP_OBJ, ACL_OTHER} + if not required <= {tag for tag, _permission, _identifier in entries}: + raise HardeningError("sensitive directory ACL is incomplete") + return entries + + +def _encode_acl(entries: list[tuple[int, int, int]]) -> bytes: + return ACL_HEADER.pack(ACL_VERSION) + b"".join( + ACL_ENTRY.pack(*entry) for entry in entries + ) + + +def _acl_with_deny(value: bytes, mode: int) -> bytes: + entries = _decode_acl(value, mode) + entries = [ + entry + for entry in entries + if not (entry[0] == ACL_USER and entry[2] == ACCOUNT_UID) + ] + entries.append((ACL_USER, 0, ACCOUNT_UID)) + if not any(tag == ACL_MASK for tag, _permission, _identifier in entries): + entries.append((ACL_MASK, (mode >> 3) & 0o7, ACL_UNDEFINED_ID)) + order = { + ACL_USER_OBJ: 0, + ACL_USER: 1, + ACL_GROUP_OBJ: 2, + ACL_GROUP: 3, + ACL_MASK: 4, + ACL_OTHER: 5, + } + entries.sort(key=lambda entry: (order.get(entry[0], 99), entry[2])) + return _encode_acl(entries) + + +def _read_acl(path: Path) -> bytes: + try: + return os.getxattr(path, ACL_XATTR, follow_symlinks=False) + except OSError as exc: + if exc.errno in {errno.ENODATA, getattr(errno, "ENOATTR", errno.ENODATA)}: + return b"" + raise + + +def _validate_acl_backup(value: bytes) -> None: + if value[:1] not in {b"A", b"N"}: + raise HardeningError("sensitive directory ACL backup is malformed") + if value[:1] == b"A": + if len(value) == 1: + raise HardeningError("sensitive directory ACL backup is malformed") + _decode_acl(value[1:], 0) + elif value != b"N": + raise HardeningError("sensitive directory ACL backup is malformed") + + +def _acl_backup_once(path: Path, value: bytes, backup_name: str | None = None) -> None: + """Persist the original ACL as a private, durable host recovery file.""" + root = HOST_ETC / "hermes-node-boundary" + _directory(root, mode=0o700, uid=HOST_ROOT_UID, gid=HOST_ROOT_GID) + backup = root / f"{backup_name or path.name}.acl" + encoded = b"A" + value if value else b"N" + try: + existing, _metadata = _read_regular(backup, 64 * 1024) + except FileNotFoundError: + pass + else: + _validate_acl_backup(existing) + return + + temporary = root / f".{path.name}.acl.hermes-{os.getpid()}" + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(temporary, flags, 0o600) + try: + if os.write(descriptor, encoded) != len(encoded): + raise HardeningError("short sensitive directory ACL backup write") + os.fchmod(descriptor, 0o600) + os.fchown(descriptor, HOST_ROOT_UID, HOST_ROOT_GID) + os.fsync(descriptor) + except Exception: + temporary.unlink(missing_ok=True) + raise + finally: + os.close(descriptor) + try: + os.link(temporary, backup, follow_symlinks=False) + except FileExistsError: + # Another reconciler won the race; never overwrite the first backup. + pass + finally: + temporary.unlink(missing_ok=True) + stored, _metadata = _read_regular(backup, 64 * 1024) + _validate_acl_backup(stored) + directory = os.open(root, os.O_RDONLY) + try: + os.fsync(directory) + finally: + os.close(directory) + + +def _deny_sensitive_root(path: Path, backup_name: str | None = None) -> None: + metadata = path.lstat() + if not stat.S_ISDIR(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode): + raise HardeningError(f"unsafe sensitive directory: {path.name}") + if metadata.st_uid != HOST_ROOT_UID: + raise HardeningError(f"sensitive directory is not root-owned: {path.name}") + current = _read_acl(path) + _acl_backup_once(path, current, backup_name) + updated = _acl_with_deny(current, stat.S_IMODE(metadata.st_mode)) + try: + os.setxattr(path, ACL_XATTR, updated, follow_symlinks=False) + verified = _decode_acl(_read_acl(path), stat.S_IMODE(metadata.st_mode)) + if (ACL_USER, 0, ACCOUNT_UID) not in verified: + raise HardeningError("sensitive directory ACL validation failed") + except Exception: + if current: + os.setxattr(path, ACL_XATTR, current, follow_symlinks=False) + else: + try: + os.removexattr(path, ACL_XATTR, follow_symlinks=False) + except OSError as exc: + if exc.errno not in { + errno.ENODATA, + getattr(errno, "ENOATTR", errno.ENODATA), + }: + raise + raise + + +def _deny_sensitive_roots() -> None: + for path, backup_name in ( + (HOST_K3S, "var-lib-rancher-k3s"), + (HOST_KUBELET, "var-lib-kubelet"), + (HOST_RUN_K3S, "run-k3s"), + (HOST_RUN_CONTAINERD, "run-containerd"), + ): + _deny_sensitive_root(path, backup_name) + + +def reconcile(public_key: Path) -> None: + """Create the locked account and move only the Hermes authorization key.""" + with account_lock(HOST_ETC, expected_uid=HOST_ROOT_UID): + _reconcile_databases() + _deny_sensitive_roots() + _move_key(public_key) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--public-key-file", type=Path, required=True) + args = parser.parse_args() + reconcile(args.public_key_file) + print("Dedicated Hermes node account reconciled.", flush=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/hermes/scripts/node_account_io.py b/services/hermes/scripts/node_account_io.py new file mode 100644 index 00000000..4fb2ae9e --- /dev/null +++ b/services/hermes/scripts/node_account_io.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +"""Crash-safe, metadata-preserving host account database I/O.""" + +from __future__ import annotations + +import fcntl +import os +import stat +import time +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Iterator + +MAX_XATTRS = 64 +MAX_XATTR_NAME = 255 +MAX_XATTR_VALUE = 256 * 1024 +MAX_XATTR_TOTAL = 1024 * 1024 +LOCK_TIMEOUT_SECONDS = 15.0 + + +class HardeningError(RuntimeError): + """Raised before an unsafe or ambiguous host-account change.""" + + +@dataclass(frozen=True) +class FileSnapshot: + """Bounded file contents and all security-relevant inode metadata.""" + + value: bytes + device: int + inode: int + mode: int + uid: int + gid: int + size: int + mtime_ns: int + ctime_ns: int + xattrs: tuple[tuple[str, bytes], ...] + + +def _stat_identity(item: os.stat_result) -> tuple[int, ...]: + return ( + item.st_dev, + item.st_ino, + item.st_mode, + item.st_uid, + item.st_gid, + item.st_size, + item.st_mtime_ns, + item.st_ctime_ns, + ) + + +def _xattrs(path_or_fd: Path | int) -> tuple[tuple[str, bytes], ...]: + try: + names = sorted(os.listxattr(path_or_fd, follow_symlinks=False)) + except (TypeError, ValueError): + names = sorted(os.listxattr(path_or_fd)) + if len(names) > MAX_XATTRS: + raise HardeningError("host account file has too many extended attributes") + values: list[tuple[str, bytes]] = [] + total = 0 + for name in names: + if len(name.encode("utf-8")) > MAX_XATTR_NAME: + raise HardeningError("host account file xattr name is too long") + try: + value = os.getxattr(path_or_fd, name, follow_symlinks=False) + except (TypeError, ValueError): + value = os.getxattr(path_or_fd, name) + total += len(value) + if len(value) > MAX_XATTR_VALUE or total > MAX_XATTR_TOTAL: + raise HardeningError("host account file xattrs exceed the safe limit") + values.append((name, value)) + return tuple(values) + + +def _restore_xattrs(descriptor: int, expected: tuple[tuple[str, bytes], ...]) -> None: + expected_names = {name for name, _value in expected} + for name, _value in _xattrs(descriptor): + if name not in expected_names: + os.removexattr(descriptor, name) + for name, value in expected: + os.setxattr(descriptor, name, value) + + +def read_regular(path: Path, maximum: int) -> FileSnapshot: + """Read one bounded, non-symlink regular file and capture its metadata.""" + flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(path, flags) + try: + metadata = os.fstat(descriptor) + if not stat.S_ISREG(metadata.st_mode) or metadata.st_size > maximum: + raise HardeningError(f"unsafe regular file: {path.name}") + value = os.read(descriptor, maximum + 1) + attrs = _xattrs(descriptor) + after = os.fstat(descriptor) + finally: + os.close(descriptor) + if len(value) != metadata.st_size or _stat_identity(metadata) != _stat_identity(after): + raise HardeningError(f"short file read: {path.name}") + return FileSnapshot( + value=value, + device=metadata.st_dev, + inode=metadata.st_ino, + mode=stat.S_IMODE(metadata.st_mode), + uid=metadata.st_uid, + gid=metadata.st_gid, + size=metadata.st_size, + mtime_ns=metadata.st_mtime_ns, + ctime_ns=metadata.st_ctime_ns, + xattrs=attrs, + ) + + +def assert_unchanged(path: Path, expected: FileSnapshot, maximum: int) -> None: + """Fail if an account file changed since planning began.""" + current = read_regular(path, maximum) + if current != expected: + raise HardeningError(f"concurrent host account change detected: {path.name}") + + +def atomic_write(path: Path, value: bytes, metadata: FileSnapshot) -> None: + """Atomically replace a file while retaining ACLs, labels, and xattrs.""" + temporary = path.with_name(f".{path.name}.hermes-{os.getpid()}") + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(temporary, flags, metadata.mode) + try: + if os.write(descriptor, value) != len(value): + raise HardeningError(f"short atomic write: {path.name}") + os.fchmod(descriptor, metadata.mode) + os.fchown(descriptor, metadata.uid, metadata.gid) + _restore_xattrs(descriptor, metadata.xattrs) + os.fsync(descriptor) + except Exception: + temporary.unlink(missing_ok=True) + raise + finally: + os.close(descriptor) + os.replace(temporary, path) + directory = os.open(path.parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) + try: + os.fsync(directory) + finally: + os.close(directory) + + +def backup_once(path: Path, snapshot: FileSnapshot, maximum: int) -> Path: + """Create the first durable recovery copy without replacing an older one.""" + backup = path.with_name(path.name + ".hermes-boundary-backup") + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(backup, flags, snapshot.mode) + except FileExistsError: + read_regular(backup, maximum) + return backup + try: + if os.write(descriptor, snapshot.value) != len(snapshot.value): + raise HardeningError(f"short backup write: {path.name}") + os.fchmod(descriptor, snapshot.mode) + os.fchown(descriptor, snapshot.uid, snapshot.gid) + _restore_xattrs(descriptor, snapshot.xattrs) + os.fsync(descriptor) + except Exception: + backup.unlink(missing_ok=True) + raise + finally: + os.close(descriptor) + directory = os.open(path.parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) + try: + os.fsync(directory) + finally: + os.close(directory) + return backup + + +@contextmanager +def account_lock(host_etc: Path, *, expected_uid: int = 0) -> Iterator[None]: + """Hold the standard shadow-utils account lock for the full transaction.""" + path = host_etc / ".pwd.lock" + flags = os.O_WRONLY | os.O_CREAT | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(path, flags, 0o600) + try: + metadata = os.fstat(descriptor) + if ( + not stat.S_ISREG(metadata.st_mode) + or metadata.st_uid != expected_uid + or stat.S_IMODE(metadata.st_mode) != 0o600 + ): + raise HardeningError("unsafe standard account lock") + deadline = time.monotonic() + LOCK_TIMEOUT_SECONDS + while True: + try: + fcntl.lockf(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except BlockingIOError as exc: + if time.monotonic() >= deadline: + raise HardeningError("standard account lock is busy") from exc + time.sleep(0.05) + yield + finally: + try: + fcntl.lockf(descriptor, fcntl.LOCK_UN) + finally: + os.close(descriptor) diff --git a/services/hermes/scripts/node_polkit_audit.py b/services/hermes/scripts/node_polkit_audit.py new file mode 100644 index 00000000..d45a0134 --- /dev/null +++ b/services/hermes/scripts/node_polkit_audit.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +"""Literal-identity proof for polkit policies on hardened Atlas nodes. + +Polkit rules are JavaScript, so a grant can name its subject indirectly: +concatenated strings, escape sequences, bracket lookups, or arithmetic can +compute an identity that plain text matching never sees. Rather than model +the language, every grant must scope itself through exact literal identity +comparisons. Anything the audit cannot prove literal fails closed. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +from node_account_io import HardeningError + +_ALLOWED_ESCAPES = {'"', "'", "\\", "/", "n", "r", "t"} +_STRING_MARK = r"\x00[0-9]+\x00" +_GRANT_RE = re.compile(r"\bResult\s*\.\s*YES\b") +_PKLA_GRANT_RE = re.compile(r"(?im)^\s*Result(?:Any|Inactive|Active)\s*=\s*yes\s*$") +_PKLA_IDENTITY_RE = re.compile(r"unix-(?:user|group):[A-Za-z0-9][A-Za-z0-9._-]{0,99}\Z") +_IDENTITY_USE_RE = re.compile(r"subject\s*(\[|\.\s*(?:user|uid|isInGroup)\b)") +_COMPUTED_TOKEN_RE = re.compile( + r"\b(?:eval|Function|arguments|fromCharCode|fromCodePoint|atob|btoa|unescape" + r"|escape|decodeURI[A-Za-z]*|encodeURI[A-Za-z]*|charCodeAt|codePointAt" + r"|normalize|constructor|prototype|globalThis|Reflect|Proxy)\b" +) +# Logical && and || stay legal; every value-building operator next to a +# string or number literal is rejected so comparisons stay whole literals. +_OPERATOR_ADJACENT_RE = re.compile( + rf"(?:{_STRING_MARK}|[0-9])\s*[+\-*/%<>^~](?![&|=])" + rf"|(?])[+\-*/%<>^~]\s*(?:{_STRING_MARK}|[0-9])" +) +_USER_BEFORE_RE = re.compile(rf"{_STRING_MARK}\s*[=!]==?\s*\Z") +_USER_AFTER_RE = re.compile(rf"\s*[=!]==?\s*{_STRING_MARK}") +_UID_BEFORE_RE = re.compile(r"(? tuple[str, list[str]]: + """Replace string literals with markers and drop JavaScript comments.""" + code: list[str] = [] + strings: list[str] = [] + index = 0 + length = len(text) + while index < length: + character = text[index] + if character == "`": + raise HardeningError("polkit rule uses computed template strings") + if character in {'"', "'"}: + quote = character + index += 1 + value: list[str] = [] + while True: + if index >= length or text[index] in "\r\n": + raise HardeningError("polkit rule string is unterminated") + character = text[index] + if character == "\\": + if index + 1 >= length or text[index + 1] not in _ALLOWED_ESCAPES: + raise HardeningError("polkit rule string uses computed escapes") + escaped = text[index + 1] + value.append({"n": "\n", "r": "\r", "t": "\t"}.get(escaped, escaped)) + index += 2 + continue + index += 1 + if character == quote: + break + value.append(character) + strings.append("".join(value)) + code.append(f"\x00{len(strings) - 1}\x00") + continue + if text.startswith("//", index): + newline = text.find("\n", index) + index = length if newline < 0 else newline + continue + if text.startswith("/*", index): + closing = text.find("*/", index + 2) + if closing < 0: + raise HardeningError("polkit rule comment is unterminated") + code.append(" ") + index = closing + 2 + continue + if character == "/": + # Any remaining slash is a regex literal or a division operator. + # A regex literal may embed a quote character, which would desync + # the string scanner and swallow an unconditional grant. Neither + # regex nor division has a place in an identity-scoped grant, so + # both fail closed before they can hide a Result.YES. + raise HardeningError("polkit rule uses a regex or division operator") + code.append(character) + index += 1 + return "".join(code), strings + + +def _require_literal_identity_use(code: str, match: re.Match[str]) -> None: + """Reject one identity reference unless it is an exact literal comparison.""" + form = match.group(1) + if form.startswith("["): + raise HardeningError("polkit rule reads identity through a computed lookup") + member = form.removeprefix(".").strip() + before = code[max(match.start() - 48, 0) : match.start()] + after = code[match.end() : match.end() + 48] + if member == "user": + literal = _USER_BEFORE_RE.search(before) or _USER_AFTER_RE.match(after) + elif member == "uid": + literal = _UID_BEFORE_RE.search(before) or _UID_AFTER_RE.match(after) + else: + literal = _GROUP_AFTER_RE.match(after) + if not literal: + raise HardeningError("polkit rule identity test is not an exact literal") + + +def _audit_rules_javascript(text: str) -> None: + """Require granting rules to gate on exact literal subject identities. + + With computed tokens, bracket lookups, and subject aliasing rejected in + every rule file, a YES decision can only be written as the literal grant + token, so the grant test below cannot be evaded by construction. + """ + code, _strings = _split_strings(text) + if _COMPUTED_TOKEN_RE.search(code): + raise HardeningError("polkit rule computes values the audit cannot prove") + if _RESULT_USE_RE.search(code): + raise HardeningError("polkit rule reads results through a computed lookup") + for bare in _BARE_SUBJECT_RE.finditer(code): + if not _PARAMETER_BEFORE_RE.search(code[: bare.start()]): + raise HardeningError("polkit rule passes the subject to computed code") + if not _GRANT_RE.search(code): + return + if _OPERATOR_ADJACENT_RE.search(code): + raise HardeningError("polkit grant builds values with operators") + uses = 0 + for match in _IDENTITY_USE_RE.finditer(code): + uses += 1 + _require_literal_identity_use(code, match) + if not uses: + raise HardeningError("polkit grant is not scoped to an explicit identity") + + +def _audit_pkla(text: str) -> None: + """Require granting localauthority entries to name exact literal identities.""" + if not _PKLA_GRANT_RE.search(text): + return + identities: list[str] = [] + for line in text.splitlines(): + name, separator, value = line.strip().partition("=") + if name.strip() != "Identity" or not separator: + continue + entries = [entry.strip() for entry in value.split(";") if entry.strip()] + identities.extend(entries) + if not identities: + raise HardeningError("polkit authority grant lacks an identity scope") + for entry in identities: + if not _PKLA_IDENTITY_RE.fullmatch(entry): + raise HardeningError("polkit authority identity is not an exact literal") + + +def audit_polkit_policy(path: Path, text: str) -> None: + """Prove one polkit policy file cannot grant through a computed identity.""" + if "localauthority" in path.parts or path.suffix == ".pkla": + _audit_pkla(text) + else: + _audit_rules_javascript(text) diff --git a/services/hermes/scripts/stage_runtime_access.py b/services/hermes/scripts/stage_runtime_access.py index f4d73160..9c71bb7e 100644 --- a/services/hermes/scripts/stage_runtime_access.py +++ b/services/hermes/scripts/stage_runtime_access.py @@ -86,6 +86,21 @@ def _write_empty_auth_store() -> None: os.chown(path, OWNER_UID, OWNER_GID) +def _stage_node_ssh_config() -> None: + """Force every Atlas node alias onto the dedicated Hermes OS account.""" + destination = RUNTIME_ROOT / "node-ssh-config" + source = _copy_secret("node-ssh-config", destination) + if "\x00" in source: + destination.unlink(missing_ok=True) + raise RuntimeError("node SSH config contains invalid control data") + destination.write_text( + "Host titan-*\n User hermes-agent\n" + source.rstrip() + "\n", + encoding="utf-8", + ) + destination.chmod(0o600) + os.chown(destination, OWNER_UID, OWNER_GID) + + def stage_agent() -> None: """Stage the owner agent's complete runtime access set.""" for path in (RUNTIME_ROOT, RUNTIME_ROOT / "claude", RUNTIME_ROOT / "codex"): @@ -93,13 +108,12 @@ def stage_agent() -> None: for name in ( "agent-api-key", "chat-relay-key", - "gitea-token", - "gitea-username", + "jenkins-image-build-token", "node-ssh-private-key", - "node-ssh-config", "node-ssh-known-hosts", ): _copy_secret(name, RUNTIME_ROOT / name) + _stage_node_ssh_config() _validated_json( "claude-credentials", RUNTIME_ROOT / "claude" / ".credentials.json", diff --git a/services/hermes/skills/manage-atlas-pull-requests/SKILL.md b/services/hermes/skills/manage-atlas-pull-requests/SKILL.md new file mode 100644 index 00000000..6194f7f9 --- /dev/null +++ b/services/hermes/skills/manage-atlas-pull-requests/SKILL.md @@ -0,0 +1,64 @@ +--- +name: manage-atlas-pull-requests +description: Read bounded private Atlas repository and pull-request metadata or create a verified draft pull request after a tested branch is pushed through Hermes' least-authority Forgejo client. Use for PR handoff in scm.bstein.dev/atlas repositories. Never use it to update, merge, approve, close, delete, or force-push. +--- + +# Manage Atlas pull requests + +Use `/opt/scm/gitea_api.py` for Forgejo API access. The agent pod has no +repository credential. The client calls a separate least-authority broker; +never bypass it with `curl` or direct Gitea HTTP. + +Use the configured broker Git remote for clone, fetch, and creation of a new +namespaced feature branch. Existing-ref updates, protected refs, deletion, +and force-push are rejected by the broker. Never replace that remote with a +credential-bearing URL. + +## Read repository or PR state + +Pass one allowed Atlas metadata API path to the read operation: + +```sh +/opt/scm/gitea_api.py read /api/v1/repos/atlas/REPO/pulls/NUMBER +``` + +The client exposes only repository metadata, PRs and PR evidence, branches, +commits, and commit status. Hooks, Actions secrets/variables, collaborators, +protections, reviews, merge endpoints, releases, and administration are +rejected even as reads. Responses are size-bounded. Treat returned state as +evidence, not authorization to change it. + +## Create a review handoff + +Before opening a PR, verify the remote, branch, clean worktree, diff, tests, and +exact pushed commit. Never force-push. Pass that full 40-character pushed head +SHA explicitly. Keep the short body free of credentials and credential-like +text. The client screens structured assignments, known token formats, and long +high-entropy values as an accidental-disclosure barrier. It cannot prove text +is secret-free: short or multiword secrets under innocuous names may resemble +ordinary prose, so never place credential material in either field. Structured +inspection is deliberately bounded; large, deeply nested, or unusual config +blobs do not belong in these fields and may be rejected. Validate without +reading a credential or using the network: + +```sh +/opt/scm/gitea_api.py --dry-run create-draft REPO \ + --base main --head hermes/TASK --head-sha FULL_PUSHED_SHA \ + --title "Focused change" --body "Tests and review evidence" +``` + +Run the same command without `--dry-run` only when the branch is review-ready. +The client accepts success only when Forgejo returns the exact requested +repository, base/head refs, head SHA, title/body, canonical URLs, positive PR +number, and open/unmerged/draft state. Report its URL and exact head commit to +Brad. Leave it unmerged for human review. PR updates are intentionally absent; +publish a corrected tested commit and ask Brad how to proceed if metadata must +change. + +## Stop at the authority boundary + +Update, merge, approve, close, delete, branch mutation, comments, releases, +repository administration, and force-push are intentionally unavailable. Do +not bypass the client with a raw HTTP request. If one of those actions is +needed, present the tested commit and evidence to Brad and stop for human +review. diff --git a/services/hermes/skills/manage-atlas-pull-requests/agents/openai.yaml b/services/hermes/skills/manage-atlas-pull-requests/agents/openai.yaml new file mode 100644 index 00000000..57cb8496 --- /dev/null +++ b/services/hermes/skills/manage-atlas-pull-requests/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Manage Atlas Pull Requests" + short_description: "Create safe draft PR handoffs for human review" + default_prompt: "Use $manage-atlas-pull-requests to inspect the requested Atlas repository or prepare a tested branch as an unmerged draft pull request. Preserve the human review boundary." diff --git a/services/hermes/switchyard-deployment.yaml b/services/hermes/switchyard-deployment.yaml index 48143682..456d0a1a 100644 --- a/services/hermes/switchyard-deployment.yaml +++ b/services/hermes/switchyard-deployment.yaml @@ -27,7 +27,7 @@ spec: prometheus.io/port: "9005" prometheus.io/path: /metrics vault.hashicorp.com/agent-inject: "true" - vault.hashicorp.com/role: hermes-agent + vault.hashicorp.com/role: hermes-switchyard vault.hashicorp.com/agent-inject-secret-relay-key: kv/data/atlas/hermes/chat-telegram vault.hashicorp.com/agent-inject-template-relay-key: | {{- with secret "kv/data/atlas/hermes/chat-telegram" -}} diff --git a/services/jenkins/configmap-jcasc.yaml b/services/jenkins/configmap-jcasc.yaml index e9aa57b9..2a1e81eb 100644 --- a/services/jenkins/configmap-jcasc.yaml +++ b/services/jenkins/configmap-jcasc.yaml @@ -652,6 +652,25 @@ data: } } } + pipelineJob('hermes-agent-image') { + disabled(false) + description('Human-gated, daemonless Kaniko build for the reviewed atlas/titan-iac main revision. Publishes a content-addressed Hermes agent image and archives a Flux digest patch; it never mutates Git or deploys.') + authenticationToken(System.getenv('HERMES_AGENT_IMAGE_BUILD_TOKEN')) + definition { + cpsScm { + scm { + git { + remote { + url('https://scm.bstein.dev/atlas/titan-iac.git') + credentials('gitea-pat') + } + branches('*/main') + } + } + scriptPath('ci/Jenkinsfile.hermes-agent-image') + } + } + } multibranchPipelineJob('titan-iac-quality-gate') { branchSources { branchSource { diff --git a/services/jenkins/configmap-plugins.yaml b/services/jenkins/configmap-plugins.yaml index 049bc1bd..ea4b450d 100644 --- a/services/jenkins/configmap-plugins.yaml +++ b/services/jenkins/configmap-plugins.yaml @@ -20,6 +20,7 @@ data: gitea:268.v75e47974c01d gitea-checks:603.621.vc708da_fb_371d multibranch-scan-webhook-trigger:1.0.11 + build-token-root:365.v717f8685a_09e # Structured test evidence. Without junit the `junit` step throws # NoSuchMethodError, jenkins.failed_tests is always empty, and triage # has only raw console text to work from. Pinned to the newest release diff --git a/services/jenkins/deployment.yaml b/services/jenkins/deployment.yaml index 1fa22374..46109da7 100644 --- a/services/jenkins/deployment.yaml +++ b/services/jenkins/deployment.yaml @@ -65,6 +65,9 @@ spec: ARIADNE_JENKINS_API_USER={{ .Data.data.username }} ARIADNE_JENKINS_API_TOKEN={{ .Data.data.token }} {{ end }} + {{ with secret "kv/data/atlas/hermes/developer-jenkins" }} + HERMES_AGENT_IMAGE_BUILD_TOKEN={{ .Data.data.build_token }} + {{ end }} bstein.dev/restarted-at: "2026-05-20T09:40:31Z" spec: serviceAccountName: jenkins diff --git a/services/jenkins/hermes-image-builder-serviceaccount.yaml b/services/jenkins/hermes-image-builder-serviceaccount.yaml new file mode 100644 index 00000000..98279417 --- /dev/null +++ b/services/jenkins/hermes-image-builder-serviceaccount.yaml @@ -0,0 +1,7 @@ +# services/jenkins/hermes-image-builder-serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: hermes-image-builder + namespace: jenkins +automountServiceAccountToken: false diff --git a/services/jenkins/kustomization.yaml b/services/jenkins/kustomization.yaml index b69d7642..5f3e0554 100644 --- a/services/jenkins/kustomization.yaml +++ b/services/jenkins/kustomization.yaml @@ -5,6 +5,7 @@ namespace: jenkins resources: - namespace.yaml - serviceaccount.yaml + - hermes-image-builder-serviceaccount.yaml - vault-serviceaccount.yaml - pvc.yaml - cache-pvc.yaml diff --git a/services/jenkins/secretproviderclass.yaml b/services/jenkins/secretproviderclass.yaml index a9d9dd50..148f193f 100644 --- a/services/jenkins/secretproviderclass.yaml +++ b/services/jenkins/secretproviderclass.yaml @@ -8,7 +8,7 @@ spec: provider: vault parameters: vaultAddress: "http://vault.vault.svc.cluster.local:8200" - roleName: "jenkins" + roleName: "jenkins-vault-sync" objects: | - objectName: "harbor-pull__dockerconfigjson" secretPath: "kv/data/atlas/shared/harbor-pull" diff --git a/services/mailu/scripts/mailu_sync.py b/services/mailu/scripts/mailu_sync.py index d04ee25a..5b8837a3 100644 --- a/services/mailu/scripts/mailu_sync.py +++ b/services/mailu/scripts/mailu_sync.py @@ -314,7 +314,7 @@ def main(): conn.close() -if __name__ == "__main__": +if __name__ == "__main__": # pragma: no cover - exercised through main() try: main() except Exception as exc: diff --git a/services/mailu/scripts/mailu_sync_listener.py b/services/mailu/scripts/mailu_sync_listener.py index 48f3d4fe..63c9824f 100644 --- a/services/mailu/scripts/mailu_sync_listener.py +++ b/services/mailu/scripts/mailu_sync_listener.py @@ -100,6 +100,6 @@ class Handler(http.server.BaseHTTPRequestHandler): return -if __name__ == "__main__": +if __name__ == "__main__": # pragma: no cover - server loop server = http.server.ThreadingHTTPServer(("", 8080), Handler) server.serve_forever() diff --git a/services/maintenance/apps/metis-configmap.yaml b/services/maintenance/apps/metis-configmap.yaml index 810d32cc..36979463 100644 --- a/services/maintenance/apps/metis-configmap.yaml +++ b/services/maintenance/apps/metis-configmap.yaml @@ -15,8 +15,8 @@ data: METIS_MAX_DEVICE_BYTES: "1000000000000" METIS_NAMESPACE: maintenance METIS_REMOTE_POD_TIMEOUT_SEC: "14400" - METIS_RUNNER_IMAGE_AMD64: registry.bstein.dev/bstein/metis:0.1.0-296-amd64 # {"$imagepolicy": "maintenance:metis-amd64"} - METIS_RUNNER_IMAGE_ARM64: registry.bstein.dev/bstein/metis:0.1.0-295-arm64 # {"$imagepolicy": "maintenance:metis-arm64"} + METIS_RUNNER_IMAGE_AMD64: registry.bstein.dev/bstein/metis:0.1.0-298-amd64 # {"$imagepolicy": "maintenance:metis-amd64"} + METIS_RUNNER_IMAGE_ARM64: registry.bstein.dev/bstein/metis:0.1.0-298-arm64 # {"$imagepolicy": "maintenance:metis-arm64"} METIS_HARBOR_REGISTRY: registry.bstein.dev METIS_HARBOR_PROJECT: metis METIS_HARBOR_API_BASE: https://registry.bstein.dev/api/v2.0 diff --git a/services/maintenance/kustomization.yaml b/services/maintenance/kustomization.yaml index 5a2b82b0..86d52b56 100644 --- a/services/maintenance/kustomization.yaml +++ b/services/maintenance/kustomization.yaml @@ -13,6 +13,6 @@ images: - name: registry.bstein.dev/bstein/ariadne newTag: 0.1.0-461 # {"$imagepolicy": "maintenance:ariadne:tag"} - name: registry.bstein.dev/bstein/metis - newTag: 0.1.0-295-arm64 # {"$imagepolicy": "maintenance:metis-arm64:tag"} + newTag: 0.1.0-298-arm64 # {"$imagepolicy": "maintenance:metis-arm64:tag"} - name: registry.bstein.dev/bstein/soteria newTag: 0.1.0-120 # {"$imagepolicy": "maintenance:soteria:tag"} diff --git a/services/maintenance/node-ops/metis-sentinel-amd64-daemonset.yaml b/services/maintenance/node-ops/metis-sentinel-amd64-daemonset.yaml index 691ebd94..431af572 100644 --- a/services/maintenance/node-ops/metis-sentinel-amd64-daemonset.yaml +++ b/services/maintenance/node-ops/metis-sentinel-amd64-daemonset.yaml @@ -32,7 +32,7 @@ spec: kubernetes.io/arch: amd64 containers: - name: metis-sentinel - image: registry.bstein.dev/bstein/metis-sentinel:0.1.0-296-amd64 # {"$imagepolicy": "maintenance:metis-sentinel-amd64"} + image: registry.bstein.dev/bstein/metis-sentinel:0.1.0-298-amd64 # {"$imagepolicy": "maintenance:metis-sentinel-amd64"} imagePullPolicy: Always envFrom: - configMapRef: diff --git a/services/maintenance/node-ops/metis-sentinel-arm64-daemonset.yaml b/services/maintenance/node-ops/metis-sentinel-arm64-daemonset.yaml index 21d909a6..6616e07e 100644 --- a/services/maintenance/node-ops/metis-sentinel-arm64-daemonset.yaml +++ b/services/maintenance/node-ops/metis-sentinel-arm64-daemonset.yaml @@ -32,7 +32,7 @@ spec: kubernetes.io/arch: arm64 containers: - name: metis-sentinel - image: registry.bstein.dev/bstein/metis-sentinel:0.1.0-295-arm64 # {"$imagepolicy": "maintenance:metis-sentinel-arm64"} + image: registry.bstein.dev/bstein/metis-sentinel:0.1.0-298-arm64 # {"$imagepolicy": "maintenance:metis-sentinel-arm64"} imagePullPolicy: Always envFrom: - configMapRef: diff --git a/services/vault-hermes-jenkins-token-seed/job.yaml b/services/vault-hermes-jenkins-token-seed/job.yaml new file mode 100644 index 00000000..17f7a4ca --- /dev/null +++ b/services/vault-hermes-jenkins-token-seed/job.yaml @@ -0,0 +1,68 @@ +# services/vault-hermes-jenkins-token-seed/job.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: vault-hermes-jenkins-build-token-seed-2 + namespace: vault +spec: + backoffLimit: 2 + template: + spec: + serviceAccountName: hermes-jenkins-token-seed + enableServiceLinks: false + restartPolicy: Never + nodeSelector: + hardware: rpi5 + kubernetes.io/arch: arm64 + node-role.kubernetes.io/worker: "true" + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: NotIn + values: [titan-04, titan-14, titan-18, titan-19, titan-24] + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: + type: RuntimeDefault + containers: + - name: seed + image: docker.io/hashicorp/vault@sha256:4e33b126a59c0c333b76fb4e894722462659a6bec7c48c9ee8cea56fccfd2569 + imagePullPolicy: IfNotPresent + command: [sh, /scripts/vault_hermes_jenkins_build_token_ensure.sh] + env: + - name: HOME + value: /tmp + - name: VAULT_ADDR + value: http://vault.vault.svc.cluster.local:8200 + - name: VAULT_K8S_ROLE + value: hermes-jenkins-token-seed + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + readOnlyRootFilesystem: true + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 100 + seccompProfile: + type: RuntimeDefault + volumeMounts: + - name: scripts + mountPath: /scripts + readOnly: true + - name: tmp + mountPath: /tmp + resources: + requests: {cpu: 25m, memory: 32Mi} + limits: {cpu: 250m, memory: 128Mi} + volumes: + - name: scripts + configMap: + name: vault-hermes-jenkins-token-seed-script + defaultMode: 0555 + - name: tmp + emptyDir: {} diff --git a/services/vault-hermes-jenkins-token-seed/kustomization.yaml b/services/vault-hermes-jenkins-token-seed/kustomization.yaml new file mode 100644 index 00000000..920caa50 --- /dev/null +++ b/services/vault-hermes-jenkins-token-seed/kustomization.yaml @@ -0,0 +1,13 @@ +# services/vault-hermes-jenkins-token-seed/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: vault +resources: + - serviceaccount.yaml + - job.yaml +generatorOptions: + disableNameSuffixHash: true +configMapGenerator: + - name: vault-hermes-jenkins-token-seed-script + files: + - vault_hermes_jenkins_build_token_ensure.sh=scripts/vault_hermes_jenkins_build_token_ensure.sh diff --git a/services/vault-hermes-jenkins-token-seed/scripts/vault_hermes_jenkins_build_token_ensure.sh b/services/vault-hermes-jenkins-token-seed/scripts/vault_hermes_jenkins_build_token_ensure.sh new file mode 100644 index 00000000..1514b248 --- /dev/null +++ b/services/vault-hermes-jenkins-token-seed/scripts/vault_hermes_jenkins_build_token_ensure.sh @@ -0,0 +1,167 @@ +#!/usr/bin/env sh +set -eu + +secret_path="kv/atlas/hermes/developer-jenkins" +secret_api_path="kv/data/atlas/hermes/developer-jenkins" +jwt_file="${VAULT_K8S_JWT_FILE:-/var/run/secrets/kubernetes.io/serviceaccount/token}" +vault_role="${VAULT_K8S_ROLE:-hermes-jenkins-token-seed}" +payload_file="${TMPDIR:-/tmp}/hermes-jenkins-token.json" + +log() { printf '[hermes-jenkins-token] %s\n' "$*" >&2; } +cleanup() { rm -f "${payload_file}"; } +trap cleanup EXIT HUP INT TERM + +retry_command() { + attempt=1 + while [ "${attempt}" -le 5 ]; do + set +e + command_output="$("$@" 2>&1)" + command_status=$? + set -e + if [ "${command_status}" -eq 0 ]; then + printf '%s' "${command_output}" + return 0 + fi + if [ "${attempt}" -lt 5 ]; then + sleep "${attempt}" + fi + attempt=$((attempt + 1)) + done + return "${command_status}" +} + +ensure_token() { + if [ -n "${VAULT_TOKEN:-}" ]; then + return + fi + jwt="$(cat "${jwt_file}")" + VAULT_TOKEN="$(retry_command vault write -field=token \ + auth/kubernetes/login role="${vault_role}" jwt="${jwt}")" || { + log "Vault Kubernetes login failed after retries" + exit 1 + } + unset jwt + if [ -z "${VAULT_TOKEN}" ]; then + log "Vault Kubernetes login returned an empty token" + exit 1 + fi + export VAULT_TOKEN +} + +# Return 0 for the complete field, 10 when absent, 11 when present but +# incomplete, and 12 for a persistent authorization/transport failure. +read_build_token() { + attempt=1 + while [ "${attempt}" -le 5 ]; do + set +e + secret_json="$(vault read -format=json "${secret_api_path}" 2>&1)" + secret_status=$? + set -e + if [ "${secret_status}" -eq 0 ]; then + unset secret_json + set +e + build_token="$(vault kv get -field=build_token "${secret_path}" 2>&1)" + field_status=$? + set -e + if [ "${field_status}" -eq 0 ]; then + case "${build_token}" in + ''|*[!0-9a-f]*) unset build_token; return 11 ;; + esac + if [ "${#build_token}" -eq 64 ]; then + unset build_token + return 0 + fi + unset build_token + return 11 + fi + if printf '%s' "${build_token}" | grep -q 'No value found'; then + unset build_token + return 11 + fi + unset build_token + elif printf '%s' "${secret_json}" | grep -Eq 'Code: 404|No value found at'; then + unset secret_json + return 10 + fi + if [ "${attempt}" -lt 5 ]; then + sleep "${attempt}" + fi + attempt=$((attempt + 1)) + done + unset secret_json + return 12 +} + +ensure_token + +if read_build_token; then + read_status=0 +else + read_status=$? +fi +case "${read_status}" in + 0) + log "job-scoped token already present; no write performed" + exit 0 + ;; + 11) + log "secret exists without a valid build_token; refusing to overwrite existing fields" + exit 1 + ;; + 12) + log "secret read failed after retries; refusing to seed" + exit 1 + ;; + 10) ;; + *) + log "unexpected secret read state ${read_status}" + exit 1 + ;; +esac + +new_token="$(retry_command vault write -field=random_bytes \ + sys/tools/random/32 format=hex)" || { + log "Vault random token generation failed after retries" + exit 1 + } +case "${new_token}" in + ''|*[!0-9a-f]*) + unset new_token + log "Vault returned an invalid random token" + exit 1 + ;; +esac +if [ "${#new_token}" -ne 64 ]; then + unset new_token + log "Vault returned an invalid random token" + exit 1 +fi + +umask 077 +printf '{"options":{"cas":0},"data":{"build_token":"%s"}}\n' \ + "${new_token}" > "${payload_file}" +unset new_token + +set +e +create_error="$(vault write "${secret_api_path}" @"${payload_file}" 2>&1 >/dev/null)" +create_status=$? +set -e +cleanup +if [ "${create_status}" -eq 0 ]; then + log "job-scoped token created with KV-v2 CAS create-only semantics" + exit 0 +fi + +if printf '%s' "${create_error}" | grep -qi 'check-and-set'; then + unset create_error + if read_build_token; then + log "another seeder won CAS; existing token preserved" + exit 0 + fi + log "CAS race did not produce a readable build_token; refusing to continue" + exit 1 +fi + +unset create_error +log "KV-v2 CAS create failed; refusing to retry a write" +exit 1 diff --git a/services/vault-hermes-jenkins-token-seed/serviceaccount.yaml b/services/vault-hermes-jenkins-token-seed/serviceaccount.yaml new file mode 100644 index 00000000..962471bd --- /dev/null +++ b/services/vault-hermes-jenkins-token-seed/serviceaccount.yaml @@ -0,0 +1,7 @@ +# services/vault-hermes-jenkins-token-seed/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: hermes-jenkins-token-seed + namespace: vault +automountServiceAccountToken: true diff --git a/services/vault/hermes-auth-role-bootstrap-job.yaml b/services/vault/hermes-auth-role-bootstrap-job.yaml index 07c223f0..c383f0e7 100644 --- a/services/vault/hermes-auth-role-bootstrap-job.yaml +++ b/services/vault/hermes-auth-role-bootstrap-job.yaml @@ -3,7 +3,7 @@ apiVersion: batch/v1 kind: Job metadata: - name: vault-k8s-auth-hermes-8 + name: vault-k8s-auth-hermes-10 namespace: vault spec: backoffLimit: 2 @@ -15,16 +15,26 @@ spec: hardware: rpi5 kubernetes.io/arch: arm64 node-role.kubernetes.io/worker: "true" + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: NotIn + values: [titan-04, titan-14, titan-18, titan-19, titan-24] containers: - name: configure-k8s-auth - image: hashicorp/vault:1.21.4 + image: docker.io/hashicorp/vault@sha256:4e33b126a59c0c333b76fb4e894722462659a6bec7c48c9ee8cea56fccfd2569 imagePullPolicy: IfNotPresent command: - sh - /scripts/vault_k8s_auth_configure.sh env: + - name: HOME + value: /tmp - name: VAULT_ADDR - value: http://10.43.57.249:8200 + value: http://vault.vault.svc.cluster.local:8200 - name: VAULT_K8S_ROLE value: vault-admin - name: VAULT_K8S_TOKEN_REVIEWER_JWT_FILE @@ -38,6 +48,18 @@ spec: - name: token-reviewer mountPath: /var/run/secrets/vault-token-reviewer readOnly: true + - name: tmp + mountPath: /tmp + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + readOnlyRootFilesystem: true + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 100 + seccompProfile: + type: RuntimeDefault resources: requests: cpu: 25m @@ -53,3 +75,5 @@ spec: - name: token-reviewer secret: secretName: vault-admin-token-reviewer + - name: tmp + emptyDir: {} diff --git a/services/vault/scripts/vault_k8s_auth_configure.sh b/services/vault/scripts/vault_k8s_auth_configure.sh index 73572e62..89c27121 100644 --- a/services/vault/scripts/vault_k8s_auth_configure.sh +++ b/services/vault/scripts/vault_k8s_auth_configure.sh @@ -235,12 +235,26 @@ write_policy_and_role "mailu-mailserver" "mailu-mailserver" "mailu-vault-sync" \ "mailu/* shared/postmark-relay shared/harbor-pull" "" write_policy_and_role "harbor" "harbor" "harbor-vault-sync" \ "harbor/* shared/harbor-pull" "hermes/developer-harbor" +harbor_policy_bootstrap_policy=' +path "kv/data/atlas/harbor/harbor-core" { + capabilities = ["read"] +} +' +write_raw_policy "harbor-policy-bootstrap" "${harbor_policy_bootstrap_policy}" +log "writing role harbor-policy-bootstrap" +vault_cmd write "auth/kubernetes/role/harbor-policy-bootstrap" \ + bound_service_account_names="harbor-policy-bootstrap" \ + bound_service_account_namespaces="harbor" \ + policies="harbor-policy-bootstrap" \ + ttl="${role_ttl}" write_policy_and_role "nextcloud" "nextcloud" "nextcloud-vault" \ "nextcloud/* shared/keycloak-admin shared/postmark-relay" "" write_policy_and_role "comms" "comms" "comms-vault,atlasbot" \ "comms/* shared/chat-ai-keys-runtime shared/harbor-pull" "" -write_policy_and_role "jenkins" "jenkins" "jenkins,jenkins-vault-sync" \ - "jenkins/* shared/harbor-pull quality/sonarqube-oidc" "hermes/developer-jenkins" +write_policy_and_role "jenkins" "jenkins" "jenkins" \ + "jenkins/* shared/harbor-pull quality/sonarqube-oidc hermes/developer-jenkins" "" +write_policy_and_role "jenkins-vault-sync" "jenkins" "jenkins-vault-sync" \ + "shared/harbor-pull" "" write_policy_and_role "monitoring" "monitoring" "monitoring-vault-sync" \ "monitoring/* shared/postmark-relay shared/harbor-pull" "" write_policy_and_role "logging" "logging" "logging-vault-sync" \ @@ -255,8 +269,30 @@ write_policy_and_role "game-stream" "game-stream" "game-stream-vault" \ "game-stream/*" "" write_policy_and_role "hermes" "hermes" "hermes-vault,hermes-triage" \ "hermes/triage-oidc hermes/agent-tokens hermes/triage-api" "" -write_policy_and_role "hermes-agent" "hermes" "hermes-agent,hermes-switchyard" \ - "hermes/agent-oidc hermes/agent-tokens hermes/chat-telegram hermes/developer-keycloak hermes/developer-gitea hermes/developer-harbor hermes/developer-jenkins hermes/developer-ssh" "" +# The agent keeps its own credentials but no longer holds developer-gitea: +# the least-authority hermes-scm-broker role (below) carries the Gitea token +# so a compromised agent cannot reach the SCM write path directly. +write_policy_and_role "hermes-agent" "hermes" "hermes-agent" \ + "hermes/agent-oidc hermes/agent-tokens hermes/chat-telegram hermes/developer-keycloak hermes/developer-harbor hermes/developer-jenkins hermes/developer-ssh" "" +write_policy_and_role "hermes-switchyard" "hermes" "hermes-switchyard" \ + "hermes/chat-telegram" "" +hermes_jenkins_token_seed_policy=' +path "kv/data/atlas/hermes/developer-jenkins" { + capabilities = ["create", "read"] +} +path "sys/tools/random/32" { + capabilities = ["update"] +} +' +write_raw_policy "hermes-jenkins-token-seed" "${hermes_jenkins_token_seed_policy}" +log "writing role hermes-jenkins-token-seed" +vault_cmd write "auth/kubernetes/role/hermes-jenkins-token-seed" \ + bound_service_account_names="hermes-jenkins-token-seed" \ + bound_service_account_namespaces="vault" \ + policies="hermes-jenkins-token-seed" \ + ttl="${role_ttl}" +write_policy_and_role "hermes-scm-broker" "hermes-scm" "hermes-scm-broker" \ + "hermes/developer-gitea" "" write_policy_and_role "hermes-credential-sync" "hermes" "hermes-agent" \ "" "hermes/agent-tokens" write_policy_and_role "hermes-node-ssh" "hermes" "hermes-node-ssh-access" \ diff --git a/testing/quality_contract.json b/testing/quality_contract.json index 78a4e13d..ea273c9b 100644 --- a/testing/quality_contract.json +++ b/testing/quality_contract.json @@ -14,6 +14,10 @@ } ], "managed_modules": [ + "ci/scripts/hermes_image_release.py", + "dockerfiles/hermes-kaniko-heredoc-runner.py", + "services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py", + "services/hermes/scripts/jenkins_image_build_trigger.py", "ci/scripts/publish_test_metrics.py", "ci/scripts/publish_test_metrics_quality.py", "ci/scripts/semgrep_report.py", @@ -53,9 +57,29 @@ "testing/tests/test_supply_chain_report.py", "testing/tests/test_semgrep_report.py", "testing/tests/test_quality_contract.py", - "testing/tests/test_quality_gate.py" + "testing/tests/test_quality_gate.py", + "services/gitea/scripts/gitea_branch_protection_check.py", + "services/hermes/scm-common/scripts/gitea_api.py", + "services/hermes/scm-common/scripts/gitea_api_policy.py", + "services/hermes/scm-common/scripts/scm_broker.py", + "services/hermes/scm-common/scripts/scm_broker_client.py", + "services/hermes/scm-common/scripts/scm_broker_io.py", + "services/hermes/scm-common/scripts/scm_broker_server.py", + "services/hermes/scripts/hermes_coordinator.py", + "services/hermes/scripts/node_account_audit.py", + "services/hermes/scripts/node_account_hardening.py", + "services/hermes/scripts/node_account_io.py", + "services/hermes/scripts/stage_runtime_access.py", + "services/hermes/scm-common/scripts/deadline_http.py", + "services/hermes/scm-common/scripts/git_pack_objects.py", + "services/hermes/scm-common/scripts/receive_pack_scan.py", + "services/hermes/scripts/node_polkit_audit.py" ], "lint_paths": [ + "ci/scripts/hermes_image_release.py", + "dockerfiles/hermes-kaniko-heredoc-runner.py", + "services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py", + "services/hermes/scripts/jenkins_image_build_trigger.py", "ci/scripts/publish_test_metrics.py", "ci/scripts/publish_test_metrics_quality.py", "ci/scripts/semgrep_report.py", @@ -86,7 +110,15 @@ "services/hermes/scripts/cli_lane_routing.py", "services/hermes/scripts/cli_lane_runner.py", "testing/tests", - "testing" + "testing", + "services/gitea/scripts/gitea_branch_protection_check.py", + "services/hermes/scm-common/scripts", + "services/hermes/scripts/hermes_coordinator.py", + "services/hermes/scripts/node_account_audit.py", + "services/hermes/scripts/node_account_hardening.py", + "services/hermes/scripts/node_account_io.py", + "services/hermes/scripts/stage_runtime_access.py", + "services/hermes/scripts/node_polkit_audit.py" ], "pytest_suites": { "unit": { @@ -99,10 +131,14 @@ "junit": "build/junit-unit.xml", "coverage_sources": [ "ci/scripts", + "dockerfiles", "scripts.render.dashboards_render_atlas", + "services/harbor/scripts", "services/hermes/scripts", "services/mailu/scripts", - "testing" + "testing", + "services/gitea/scripts", + "services/hermes/scm-common/scripts" ], "coverage_xml": "build/coverage-unit.xml" }, @@ -169,7 +205,15 @@ "services/*/scripts/tests/**/*.py", "services/hermes/scripts/cli_lane_*.py", "services/mailu/scripts/mailu_sync.py", - "services/mailu/scripts/mailu_sync_listener.py" + "services/mailu/scripts/mailu_sync_listener.py", + "services/gitea/scripts/gitea_branch_protection_check.py", + "services/hermes/scm-common/scripts/*.py", + "services/hermes/scripts/hermes_coordinator.py", + "services/hermes/scripts/node_account_audit.py", + "services/hermes/scripts/node_account_hardening.py", + "services/hermes/scripts/node_account_io.py", + "services/hermes/scripts/stage_runtime_access.py", + "services/hermes/scripts/node_polkit_audit.py" ], "naming_rules": [ { @@ -212,6 +256,10 @@ "coverage": { "minimum_percent": 95.0, "tracked_files": [ + "ci/scripts/hermes_image_release.py", + "dockerfiles/hermes-kaniko-heredoc-runner.py", + "services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py", + "services/hermes/scripts/jenkins_image_build_trigger.py", "ci/scripts/publish_test_metrics.py", "ci/scripts/publish_test_metrics_quality.py", "ci/scripts/semgrep_report.py", @@ -243,7 +291,23 @@ "testing/quality_docs.py", "testing/quality_hygiene.py", "testing/quality_coverage.py", - "testing/quality_gate.py" + "testing/quality_gate.py", + "services/gitea/scripts/gitea_branch_protection_check.py", + "services/hermes/scm-common/scripts/gitea_api.py", + "services/hermes/scm-common/scripts/gitea_api_policy.py", + "services/hermes/scm-common/scripts/scm_broker.py", + "services/hermes/scm-common/scripts/scm_broker_client.py", + "services/hermes/scm-common/scripts/scm_broker_io.py", + "services/hermes/scm-common/scripts/scm_broker_server.py", + "services/hermes/scripts/hermes_coordinator.py", + "services/hermes/scripts/node_account_audit.py", + "services/hermes/scripts/node_account_hardening.py", + "services/hermes/scripts/node_account_io.py", + "services/hermes/scripts/stage_runtime_access.py", + "services/hermes/scm-common/scripts/deadline_http.py", + "services/hermes/scm-common/scripts/git_pack_objects.py", + "services/hermes/scm-common/scripts/receive_pack_scan.py", + "services/hermes/scripts/node_polkit_audit.py" ] } } diff --git a/testing/quality_coverage.py b/testing/quality_coverage.py index 7339baa1..d99aea03 100644 --- a/testing/quality_coverage.py +++ b/testing/quality_coverage.py @@ -3,42 +3,24 @@ from __future__ import annotations import xml.etree.ElementTree as ET -import math -from dataclasses import dataclass from pathlib import Path from typing import Any -@dataclass(frozen=True) -class CoverageRates: - """Validated line and branch percentages for one source file.""" - - line: float - branch: float - - -def _percentage(class_node: ET.Element, attribute: str) -> float: - """Parse one finite Cobertura rate and fail closed outside its domain.""" - raw = class_node.attrib.get(attribute) - if raw is None: - raise ValueError(f"coverage class missing {attribute}") - value = float(raw) - if not math.isfinite(value) or value < 0.0 or value > 1.0: - raise ValueError(f"coverage class has invalid {attribute}: {raw}") - return value * 100.0 - - -def _load_rates(xml_path: Path, root: Path) -> dict[str, CoverageRates]: - """Load validated per-file line and branch rates from Cobertura XML.""" +def _load_rates(xml_path: Path, root: Path) -> dict[str, dict[str, float | None]]: + """Load per-file line and branch percentages from a Cobertura XML report.""" tree = ET.parse(xml_path) xml_root = tree.getroot() source_roots = [ - Path(node.text) for node in xml_root.findall("./sources/source") if node.text + Path(node.text) + for node in xml_root.findall("./sources/source") + if node.text ] - rates: dict[str, CoverageRates] = {} + rates: dict[str, dict[str, float | None]] = {} for class_node in xml_root.findall(".//class"): filename = class_node.attrib.get("filename") - if not filename: + line_rate = class_node.attrib.get("line-rate") + if not filename or line_rate is None: continue normalized = filename.replace("\\", "/") if normalized.startswith("/"): @@ -50,50 +32,44 @@ def _load_rates(xml_path: Path, root: Path) -> dict[str, CoverageRates]: if candidate.exists(): key = candidate.relative_to(root).as_posix() break - rates[key] = CoverageRates( - line=_percentage(class_node, "line-rate"), - branch=_percentage(class_node, "branch-rate"), - ) + branch_rate = class_node.attrib.get("branch-rate") + rates[key] = { + "line": float(line_rate) * 100.0, + "branch": None if branch_rate is None else float(branch_rate) * 100.0, + } return rates -def _load_percentages(xml_path: Path, root: Path) -> dict[str, float]: - """Load per-file line percentages for backward-compatible callers.""" - return {path: rates.line for path, rates in _load_rates(xml_path, root).items()} - - def run_check(contract: dict[str, Any], root: Path, xml_path: Path) -> list[str]: """Return human-readable issues for tracked files below the coverage floor. - The report is intentionally per-file so a single weak module cannot hide - behind aggregate suite coverage. + Line and branch rates are both enforced per file so a single weak module + cannot hide behind aggregate suite coverage or line-only reporting. """ if not xml_path.exists(): return [f"coverage xml missing: {xml_path.relative_to(root)}"] - try: - rates_by_path = _load_rates(xml_path, root) - except (ET.ParseError, OSError, UnicodeError, ValueError) as error: - return [f"coverage xml invalid: {error}"] + rates = _load_rates(xml_path, root) minimum = float(contract.get("coverage", {}).get("minimum_percent", 95.0)) issues: list[str] = [] for relative_path in contract.get("coverage", {}).get("tracked_files", []): normalized = relative_path.replace("\\", "/") - rates = rates_by_path.get(normalized) - if rates is None: + file_rates = rates.get(normalized) + if file_rates is None: issues.append(f"coverage missing for tracked file: {relative_path}") continue - if rates.line + 1e-9 < minimum: - issues.append( - f"line coverage below {minimum:.1f}%: {relative_path} " - f"({rates.line:.1f}%)" - ) - if rates.branch + 1e-9 < minimum: - issues.append( - f"branch coverage below {minimum:.1f}%: {relative_path} " - f"({rates.branch:.1f}%)" - ) + for metric in ("line", "branch"): + percent = file_rates[metric] + if percent is None: + issues.append( + f"{metric} coverage missing for tracked file: {relative_path}" + ) + elif percent + 1e-9 < minimum: + issues.append( + f"{metric} coverage below {minimum:.1f}%: " + f"{relative_path} ({percent:.1f}%)" + ) return issues @@ -108,16 +84,13 @@ def compute_workspace_line_coverage( if not xml_path.exists(): return 0.0 - try: - percentages = _load_percentages(xml_path, root) - except (ET.ParseError, OSError, UnicodeError, ValueError): - return 0.0 + rates = _load_rates(xml_path, root) samples: list[float] = [] for relative_path in contract.get("coverage", {}).get("tracked_files", []): normalized = relative_path.replace("\\", "/") - percent = percentages.get(normalized) - if percent is not None: - samples.append(percent) + file_rates = rates.get(normalized) + if file_rates is not None: + samples.append(file_rates["line"]) if not samples: return 0.0 return round(sum(samples) / len(samples), 3) diff --git a/testing/quality_gate.py b/testing/quality_gate.py index dc5dd1e4..d436281c 100644 --- a/testing/quality_gate.py +++ b/testing/quality_gate.py @@ -332,7 +332,7 @@ def run_profile( results.append( _result( "coverage", - "Per-file 95% coverage floor for tracked quality-managed modules.", + "Per-file 95% line and branch coverage floor for tracked quality-managed modules.", _status_from_issues(issues), issues=issues, coverage_xml=str(coverage_xml.relative_to(root)), diff --git a/testing/tests/test_hermes_agent_access.py b/testing/tests/test_hermes_agent_access.py deleted file mode 100644 index d4138256..00000000 --- a/testing/tests/test_hermes_agent_access.py +++ /dev/null @@ -1,253 +0,0 @@ -"""Owner and Switchyard access-boundary contracts.""" - -from __future__ import annotations - -from testing.tests.test_hermes_cli_support import ( - HERMES, - SCRIPTS, - _agent_deployment, - yaml, -) - - -def test_owner_agent_has_pinned_dedicated_node_ssh_access(): - deployment = _agent_deployment() - annotations = deployment["spec"]["template"]["metadata"]["annotations"] - assert annotations[ - "vault.hashicorp.com/agent-inject-secret-node-ssh-private-key" - ] == "kv/data/atlas/hermes/developer-ssh" - assert annotations[ - "vault.hashicorp.com/agent-inject-secret-node-ssh-config" - ] == "kv/data/atlas/hermes/developer-ssh" - assert annotations[ - "vault.hashicorp.com/agent-inject-secret-node-ssh-known-hosts" - ] == "kv/data/atlas/hermes/developer-ssh" - - init = next( - item - for item in deployment["spec"]["template"]["spec"]["initContainers"] - if item["name"] == "init-config" - ) - command = init["command"][2] - assert "ln -s /runtime-access/node-ssh-config /opt/data/home/.ssh/config" in command - assert ( - "ln -s /runtime-access/node-ssh-known-hosts /opt/data/home/.ssh/known_hosts" - in command - ) - assert "ln -s home/.ssh /opt/data/.ssh" in command - assert "chmod 0700 /opt/data/home/.ssh" in command - assert ( - "ln -s /runtime-access/node-ssh-private-key " - "/opt/data/home/.ssh/id_ed25519_atlas_nodes" - ) in command - - config = yaml.safe_load((HERMES / "agent-configmap.yaml").read_text())["data"] - assert "ssh_config" not in config - assert "ssh_known_hosts" not in config - assert "ssh-ed25519" not in (HERMES / "agent-configmap.yaml").read_text() - - resources = yaml.safe_load((HERMES / "kustomization.yaml").read_text())[ - "resources" - ] - assert "node-ssh-access.yaml" in resources - access = [ - item - for item in yaml.safe_load_all((HERMES / "node-ssh-access.yaml").read_text()) - if item - ] - service_account = next(item for item in access if item["kind"] == "ServiceAccount") - assert service_account["metadata"]["name"] == "hermes-node-ssh-access" - provider = next(item for item in access if item["kind"] == "SecretProviderClass") - assert provider["metadata"]["name"] == "hermes-node-ssh-access" - assert provider["spec"]["provider"] == "vault" - parameters = provider["spec"]["parameters"] - assert parameters["roleName"] == "hermes-node-ssh" - assert 'secretPath: "kv/data/atlas/hermes/developer-ssh"' in parameters["objects"] - assert 'secretKey: "public_key"' in parameters["objects"] - daemonset = next(item for item in access if item["kind"] == "DaemonSet") - pod = daemonset["spec"]["template"]["spec"] - assert pod["serviceAccountName"] == "hermes-node-ssh-access" - assert pod["automountServiceAccountToken"] is True - host_home = next(item for item in pod["volumes"] if item["name"] == "host-home") - assert host_home["hostPath"] == {"path": "/home", "type": "Directory"} - vault_secrets = next( - item for item in pod["volumes"] if item["name"] == "vault-secrets" - ) - assert vault_secrets["csi"]["driver"] == "secrets-store.csi.k8s.io" - assert vault_secrets["csi"]["volumeAttributes"] == { - "secretProviderClass": "hermes-node-ssh-access" - } - reconciler = pod["containers"][0]["args"][0] - assert "cat /vault/secrets/node-ssh-public-key" in reconciler - assert "grep -qxF" in reconciler - assert "for user in atlas oceanus" in reconciler - assert "/host-etc/passwd" in reconciler - assert "chown \"${uid}:${gid}\"" in reconciler - host_passwd = next( - item for item in pod["volumes"] if item["name"] == "host-passwd" - ) - assert host_passwd["hostPath"] == {"path": "/etc/passwd", "type": "File"} - - -def test_owner_agent_tracks_no_ssh_identity_or_host_key_material(): - """Vault references may be tracked; SSH identities and trust data may not.""" - forbidden = ( - "BEGIN OPENSSH PRIVATE KEY", - "ssh-ed25519 AAAA", - "ssh-rsa AAAA", - "IdentityFile ", - "UserKnownHostsFile ", - "StrictHostKeyChecking ", - "ssh_config:", - "ssh_known_hosts:", - ) - text_suffixes = { - ".conf", - ".json", - ".md", - ".py", - ".sh", - ".toml", - ".yaml", - ".yml", - } - tracked = "\n".join( - path.read_text(encoding="utf-8") - for path in HERMES.rglob("*") - if path.is_file() and path.suffix in text_suffixes - ) - for marker in forbidden: - assert marker not in tracked - - -def test_switchyard_has_a_dedicated_non_owner_identity_and_read_only_catalog(): - """Routing must not inherit the owner agent's cluster-admin capability.""" - service_accounts = [ - item - for item in yaml.safe_load_all( - (HERMES / "vault-serviceaccount.yaml").read_text() - ) - if item - ] - assert any( - item["kind"] == "ServiceAccount" - and item["metadata"]["name"] == "hermes-switchyard" - for item in service_accounts - ) - - switchyard = yaml.safe_load( - (HERMES / "switchyard-deployment.yaml").read_text() - ) - switchyard_pod = switchyard["spec"]["template"]["spec"] - assert switchyard_pod["serviceAccountName"] == "hermes-switchyard" - agent_pod = _agent_deployment()["spec"]["template"]["spec"] - for pod, container_name in ( - (switchyard_pod, "worker-route-broker"), - (agent_pod, "claude-broker"), - ): - container = next(item for item in pod["containers"] if item["name"] == container_name) - catalog = next( - item - for item in container["volumeMounts"] - if item["mountPath"] == "/routing-catalog" - ) - assert catalog["readOnly"] is True - - rbac = [ - item - for item in yaml.safe_load_all((HERMES / "agent-rbac.yaml").read_text()) - if item - ] - binding = next(item for item in rbac if item["kind"] == "ClusterRoleBinding") - assert binding["subjects"] == [ - {"kind": "ServiceAccount", "name": "hermes-agent", "namespace": "hermes"} - ] - - -def test_switchyard_active_state_uses_a_relocatable_rwx_claim(): - """A stale node attachment must not strand the routing authority.""" - claims = [ - item - for item in yaml.safe_load_all((HERMES / "switchyard-pvc.yaml").read_text()) - if item - ] - active_claim = next( - item - for item in claims - if item["metadata"]["name"] == "hermes-switchyard-state-rwx" - ) - assert active_claim["spec"]["accessModes"] == ["ReadWriteMany"] - - deployment = yaml.safe_load((HERMES / "switchyard-deployment.yaml").read_text()) - strategy = deployment["spec"]["strategy"] - assert strategy == { - "type": "RollingUpdate", - "rollingUpdate": {"maxSurge": 1, "maxUnavailable": 0}, - } - pod = deployment["spec"]["template"]["spec"] - state = next(item for item in pod["volumes"] if item["name"] == "state") - assert state["persistentVolumeClaim"]["claimName"] == active_claim["metadata"][ - "name" - ] - - -def test_worker_route_broker_accepts_pod_network_health_checks(): - """Kubelet probes the pod IP, so the broker cannot bind to loopback only.""" - script = (SCRIPTS / "worker_route_broker.py").read_text() - assert 'ThreadingHTTPServer(("0.0.0.0", PORT), Handler)' in script - - -def test_switchyard_network_boundary_allows_vault_bootstrap(): - """The pre-populate init container must reach Vault before routing starts.""" - documents = [ - item - for item in yaml.safe_load_all((HERMES / "networkpolicy.yaml").read_text()) - if item - ] - isolation = next( - item - for item in documents - if item.get("metadata", {}).get("name") == "hermes-switchyard-isolation" - ) - assert any( - rule.get("to") - == [ - { - "namespaceSelector": { - "matchLabels": {"kubernetes.io/metadata.name": "vault"} - }, - "podSelector": {"matchLabels": {"app": "vault"}}, - } - ] - and rule.get("ports") == [{"protocol": "TCP", "port": 8200}] - for rule in isolation["spec"]["egress"] - ) - - -def test_switchyard_network_boundary_allows_metrics_scraping(): - """VictoriaMetrics may scrape Switchyard without widening its API boundary.""" - documents = [ - item - for item in yaml.safe_load_all((HERMES / "networkpolicy.yaml").read_text()) - if item - ] - isolation = next( - item - for item in documents - if item.get("metadata", {}).get("name") == "hermes-switchyard-isolation" - ) - assert any( - rule.get("from") - == [ - { - "namespaceSelector": { - "matchLabels": { - "kubernetes.io/metadata.name": "monitoring" - } - }, - "podSelector": {"matchLabels": {"app": "server"}}, - } - ] - and rule.get("ports") == [{"protocol": "TCP", "port": 9005}] - for rule in isolation["spec"]["ingress"] - ) diff --git a/testing/tests/test_hermes_agent_layout.py b/testing/tests/test_hermes_agent_layout.py index 72ab7f05..36cdff8e 100644 --- a/testing/tests/test_hermes_agent_layout.py +++ b/testing/tests/test_hermes_agent_layout.py @@ -285,6 +285,19 @@ def test_broker_services_survive_sibling_container_readiness_loss(): ): assert services[name]["spec"]["publishNotReadyAddresses"] is True + # The agent web endpoints route through this Service to the same pod the + # cli-lane-runner readiness probe gates; a deferred lane image must not + # take down the dashboard or terminal. + agent_auth = next( + item + for item in yaml.safe_load_all((HERMES / "oauth2-proxy.yaml").read_text()) + if item + and item["kind"] == "Service" + and item["metadata"]["name"] == "oauth2-proxy-hermes-agent" + ) + assert agent_auth["spec"]["selector"] == {"app": "hermes-agent"} + assert agent_auth["spec"]["publishNotReadyAddresses"] is True + def test_cli_lane_domain_modules_are_all_mounted_with_the_runner(): manifest = yaml.safe_load((HERMES / "kustomization.yaml").read_text()) @@ -354,8 +367,8 @@ def test_cli_lane_config_refresh_does_not_restart_active_work(): }, "initialDelaySeconds": 2, "periodSeconds": 5, - "timeoutSeconds": 2, - "failureThreshold": 1, + "timeoutSeconds": 5, + "failureThreshold": 3, } environment = {item["name"]: item["value"] for item in lane["env"]} assert environment["HERMES_CLI_HEALTH_MAX_AGE_SECONDS"] == "60" @@ -374,3 +387,33 @@ def test_agent_dashboard_reconnects_all_transient_websockets(): assert ".then(() => gw.connect())" in dockerfile assert 'api.getSessions(1, 0, profile ?? "")' in dockerfile assert "dashboard token rotated by a server restart" in dockerfile + + +def test_agent_image_runs_execution_safety_patch_and_regressions(): + dockerfiles = HERMES.parents[1] / "dockerfiles" + dockerfile = (dockerfiles / "Dockerfile.hermes-agent").read_text(encoding="utf-8") + dockerignore = (dockerfiles / "Dockerfile.hermes-agent.dockerignore").read_text( + encoding="utf-8" + ) + + for name in ( + "patch-hermes-execution-safety.py", + "hermes-execution-safety-regression.py", + "hermes_execution_patch_support.py", + "patch_hermes_run_safety.py", + "patch_hermes_decomposition_safety.py", + "hermes_execution_regression_support.py", + "hermes_run_safety_regression.py", + "hermes_decomposition_safety_regression.py", + ): + assert f"COPY dockerfiles/{name}" in dockerfile + assert f"!dockerfiles/{name}" in dockerignore + assert "/opt/hermes/.venv/bin/python /tmp/patch-hermes-execution-safety.py" in ( + dockerfile + ) + assert "/opt/hermes/.venv/bin/python /tmp/hermes-execution-safety-regression.py" in ( + dockerfile + ) + assert "COPY services/hermes/scripts/cli_lane_*.py" in dockerfile + assert "!services/hermes/scripts/cli_lane_*.py" in dockerignore + assert "HERMES_CLI_LANE_SOURCE=/tmp/hermes-lane-regression" in dockerfile diff --git a/testing/tests/test_hermes_agent_security.py b/testing/tests/test_hermes_agent_security.py deleted file mode 100644 index 1d713203..00000000 --- a/testing/tests/test_hermes_agent_security.py +++ /dev/null @@ -1,205 +0,0 @@ -"""Agent authentication, networking, and Flux health contracts.""" - -from __future__ import annotations - -from testing.tests.test_hermes_cli_support import ( - FLUX_HERMES, - HERMES, - KEYCLOAK, - _agent_deployment, - yaml, -) - - -def test_agent_image_runs_execution_safety_patch_and_regressions(): - dockerfiles = HERMES.parents[1] / "dockerfiles" - dockerfile = (dockerfiles / "Dockerfile.hermes-agent").read_text(encoding="utf-8") - dockerignore = (dockerfiles / "Dockerfile.hermes-agent.dockerignore").read_text( - encoding="utf-8" - ) - - for name in ( - "patch-hermes-execution-safety.py", - "hermes-execution-safety-regression.py", - "hermes_execution_patch_support.py", - "patch_hermes_run_safety.py", - "patch_hermes_decomposition_safety.py", - "hermes_execution_regression_support.py", - "hermes_run_safety_regression.py", - "hermes_decomposition_safety_regression.py", - ): - assert f"COPY dockerfiles/{name}" in dockerfile - assert f"!dockerfiles/{name}" in dockerignore - assert "/opt/hermes/.venv/bin/python /tmp/patch-hermes-execution-safety.py" in ( - dockerfile - ) - assert "/opt/hermes/.venv/bin/python /tmp/hermes-execution-safety-regression.py" in ( - dockerfile - ) - assert "COPY services/hermes/scripts/cli_lane_*.py" in dockerfile - assert "!services/hermes/scripts/cli_lane_*.py" in dockerignore - assert "HERMES_CLI_LANE_SOURCE=/tmp/hermes-lane-regression" in dockerfile - - -def test_agent_refreshes_routes_after_restoring_cli_logins(): - deployment = _agent_deployment() - init_containers = { - item["name"]: item - for item in deployment["spec"]["template"]["spec"]["initContainers"] - } - configure = init_containers["configure-agent-clients"] - command = configure["command"][-1] - assert "configure_agent_clients.py" in command - assert command.index("configure_agent_clients.py") < command.index( - "hermes_coordinator.py --once" - ) - env = {item["name"]: item["value"] for item in configure["env"]} - assert env["HERMES_AUTH_FILE"] == "/runtime-access/hermes-auth.json" - assert env["PYTHONPATH"] == "/opt/hermes" - for name in ("bootstrap-coordinator", "configure-agent-clients"): - route_env = { - item["name"]: item["value"] - for item in init_containers[name]["env"] - } - assert route_env["CODEX_HOME"] == "/runtime-access/codex" - assert route_env["CLAUDE_CONFIG_DIR"] == "/runtime-access/claude" - assert "/opt/data/tools/bin" in route_env["PATH"] - - assert "patch-web-session-activity" in init_containers - web_patch = init_containers["patch-web-session-activity"] - assert "/opt/coordinator/patch_web_session_activity.py" in web_patch["command"] - - containers = { - item["name"]: item - for item in deployment["spec"]["template"]["spec"]["containers"] - } - steward_env = { - item["name"]: item["value"] - for item in containers["model-steward"]["env"] - } - assert steward_env["CODEX_HOME"] == "/runtime-access/codex" - assert steward_env["CLAUDE_CONFIG_DIR"] == "/runtime-access/claude" - assert "/opt/data/tools/bin" in steward_env["PATH"] - hermes_mounts = { - (item["name"], item["mountPath"], item.get("subPath")) - for item in containers["hermes"]["volumeMounts"] - } - assert ( - "web-server-patch", - "/opt/hermes/hermes_cli/web_server.py", - "web_server.py", - ) in hermes_mounts - - -def test_flux_health_checks_follow_the_owner_oauth_sidecar(): - flux = yaml.safe_load(FLUX_HERMES.read_text()) - checks = { - (item["kind"], item["name"]) - for item in flux["spec"]["healthChecks"] - } - assert ("Deployment", "hermes-agent") in checks - assert ("DaemonSet", "hermes-node-ssh-access") in checks - assert ("Deployment", "oauth2-proxy-hermes-agent") not in checks - - -def test_agent_auth_is_bstein_group_and_email_bounded(): - deployment = _agent_deployment() - oauth = next( - item for item in deployment["spec"]["template"]["spec"]["containers"] - if item["name"] == "oauth2-proxy" - ) - args = oauth["args"] - assert "--user-id-claim=sub" in args - assert "--oidc-groups-claim=groups" in args - assert "--allowed-group=/hermes-owner" in args - assert "--authenticated-emails-file=/etc/oauth2-proxy/allowed-emails" in args - script = (KEYCLOAK / "scripts/hermes_access_oidc_ensure.sh").read_text() - assert 'group_name="hermes-owner"' in script - assert "username=bstein&exact=true" in script - assert '"full.path":"true"' in script - - -def test_agent_network_boundary_allows_only_authenticated_and_metrics_surfaces(): - documents = [ - item - for item in yaml.safe_load_all((HERMES / "networkpolicy.yaml").read_text()) - if item - ] - isolation = next(item for item in documents if item.get("metadata", {}).get("name") == "hermes-agent-isolation") - assert isolation["spec"]["ingress"] == [ - { - "from": [ - { - "namespaceSelector": { - "matchLabels": { - "kubernetes.io/metadata.name": "traefik" - } - }, - "podSelector": { - "matchLabels": {"app.kubernetes.io/name": "traefik"} - }, - } - ], - "ports": [{"protocol": "TCP", "port": 4180}], - }, - { - "from": [ - { - "podSelector": { - "matchLabels": {"app": "hermes-chat-tenant"} - } - } - ], - "ports": [ - {"protocol": "TCP", "port": 9002}, - {"protocol": "TCP", "port": 9003}, - ], - }, - { - "from": [ - { - "podSelector": { - "matchLabels": {"app": "hermes-switchyard"} - } - } - ], - "ports": [ - {"protocol": "TCP", "port": 9003}, - {"protocol": "TCP", "port": 9006}, - ], - }, - { - "from": [ - { - "namespaceSelector": { - "matchLabels": { - "kubernetes.io/metadata.name": "monitoring" - } - }, - "podSelector": {"matchLabels": {"app": "server"}}, - } - ], - "ports": [ - {"protocol": "TCP", "port": 9010}, - {"protocol": "TCP", "port": 9011}, - ], - }, - ] - assert isolation["spec"]["egress"] == [{}] - - -def test_owner_agent_has_cluster_admin_kubernetes_context(): - config = yaml.safe_load((HERMES / "agent-kubeconfig.yaml").read_text()) - assert config["current-context"] == "atlas-owner" - assert config["contexts"][0]["context"]["namespace"] == "default" - rbac_path = HERMES / "agent-rbac.yaml" - documents = [item for item in yaml.safe_load_all(rbac_path.read_text()) if item] - binding = next(item for item in documents if item["kind"] == "ClusterRoleBinding") - assert binding["roleRef"] == { - "apiGroup": "rbac.authorization.k8s.io", - "kind": "ClusterRole", - "name": "cluster-admin", - } - assert binding["subjects"] == [ - {"kind": "ServiceAccount", "name": "hermes-agent", "namespace": "hermes"} - ] diff --git a/testing/tests/test_hermes_chat_codex_broker.py b/testing/tests/test_hermes_chat_codex_broker.py deleted file mode 100644 index 6ff10a23..00000000 --- a/testing/tests/test_hermes_chat_codex_broker.py +++ /dev/null @@ -1,75 +0,0 @@ -"""Hermes chat codex broker contracts.""" - -from __future__ import annotations - -from test_hermes_chat_support import ( - ModuleType, - Path, - _load_broker_module, - base64, - json, - sys, - time, -) - - -def test_codex_broker_refreshes_and_persists_first_party_oauth( - tmp_path: Path, monkeypatch -): - """Expired ChatGPT OAuth refreshes in the canonical Codex CLI store.""" - module = _load_broker_module( - "hermes_codex_refresh_broker", "codex_broker.py", monkeypatch - ) - auth_dir = tmp_path / ".codex" - auth_dir.mkdir() - - def jwt(expires_at: float) -> str: - payload = base64.urlsafe_b64encode( - json.dumps({"exp": expires_at}).encode() - ).decode().rstrip("=") - return f"header.{payload}.signature" - - expired = jwt(time.time() - 60) - live = jwt(time.time() + 3600) - auth_path = auth_dir / "auth.json" - auth_path.write_text( - json.dumps( - { - "auth_mode": "chatgpt", - "tokens": { - "access_token": expired, - "refresh_token": "refresh-old", - }, - } - ) - ) - calls = [] - auth_module = ModuleType("hermes_cli.auth") - - def refresh(access_token, refresh_token, *, timeout_seconds): - calls.append((access_token, refresh_token, timeout_seconds)) - return { - "access_token": live, - "refresh_token": "refresh-new", - "last_refresh": "2026-08-12T20:00:00Z", - } - - auth_module.refresh_codex_oauth_pure = refresh - package = ModuleType("hermes_cli") - package.auth = auth_module - monkeypatch.setitem(sys.modules, "hermes_cli", package) - monkeypatch.setitem(sys.modules, "hermes_cli.auth", auth_module) - monkeypatch.setenv("CODEX_HOME", str(auth_dir)) - - assert module._access_token() == live - persisted = json.loads(auth_path.read_text()) - assert persisted["tokens"]["access_token"] == live - assert persisted["tokens"]["refresh_token"] == "refresh-new" - assert persisted["last_refresh"] == "2026-08-12T20:00:00Z" - assert calls == [(expired, "refresh-old", 30.0)] - assert auth_path.stat().st_mode & 0o777 == 0o600 - - # A healthy token is reused, so repeated routed turns do not spend a - # refresh token or create a second billing/authentication path. - assert module._access_token() == live - assert len(calls) == 1 diff --git a/testing/tests/test_hermes_chat_images.py b/testing/tests/test_hermes_chat_images.py index 19ad8415..48c6c2e5 100644 --- a/testing/tests/test_hermes_chat_images.py +++ b/testing/tests/test_hermes_chat_images.py @@ -1,19 +1,119 @@ -"""Hermes chat images contracts.""" +"""Image generation and sandbox-boundary contracts for Hermes chat.""" from __future__ import annotations -from test_hermes_chat_support import ( +import importlib.util +import sys +import time +from types import SimpleNamespace + +import yaml + +from testing.tests.test_hermes_chat_support import ( HERMES, - Path, - SimpleNamespace, + VAULT, _documents, - importlib, - sys, - time, - yaml, ) +def test_chat_image_generation_uses_private_owner_broker(): + """Family pods get image bytes without receiving the owner's OAuth file.""" + configmap = _documents(HERMES / "chat-configmap.yaml")[0] + assert "shared desktop/Wolf lane" in configmap["data"]["SOUL.md"] + assert "local FLUX waits" in configmap["data"]["SOUL.md"] + assert "Use `image_generate_local`" in configmap["data"]["SOUL.md"] + assert "Use `image_generate_hosted`" in configmap["data"]["SOUL.md"] + assert "ComfyUI endpoint" in configmap["data"]["SOUL.md"] + assert "`MEDIA:` path" in configmap["data"]["SOUL.md"] + assert "`image_edit_latest`" in configmap["data"]["SOUL.md"] + assert ( + "Preserve the most recently selected image lane" in configmap["data"]["SOUL.md"] + ) + config = yaml.safe_load(configmap["data"]["config.yaml"]) + assert config["image_gen"] == { + "provider": "atlas-broker", + "model": "atlas-image-auto-high", + } + assert config["plugins"]["enabled"] == ["atlas-broker", "auto-router"] + + statefulset = _documents(HERMES / "chat-statefulset.yaml")[0] + pod = statefulset["spec"]["template"]["spec"] + hermes = next(item for item in pod["containers"] if item["name"] == "hermes") + mounts = {item["name"]: item for item in hermes["volumeMounts"]} + assert mounts["image-plugin"]["mountPath"] == ( + "/opt/hermes/plugins/image_gen/atlas-broker" + ) + assert mounts["runtime-access"]["mountPath"] == "/runtime-access" + assert "provider-auth" not in mounts + assert not any( + mount["name"] == "home" and "agent" in str(mount) + for mount in hermes["volumeMounts"] + ) + env = {item["name"]: item["value"] for item in hermes["env"]} + assert env["HERMES_IMAGE_BROKER_URL"].startswith("http://hermes-image-broker.") + + plugin = (HERMES / "plugins" / "image-gen-broker" / "__init__.py").read_text() + assert '"local": "flux-2-klein-4b-local"' in plugin + assert '"hosted": "gpt-image-2-high"' in plugin + assert 'name="image_generate_local"' in plugin + assert 'name="image_generate_hosted"' in plugin + assert '"name": "image_edit_latest"' in plugin + assert '"name": "image_edit_latest_local"' in plugin + assert '"name": "image_edit_latest_hosted"' in plugin + assert "def _latest_generated_image" in plugin + assert "newest MEDIA: path from the conversation" in plugin + assert 'candidate.upper().startswith("MEDIA:")' in plugin + assert "override=True" not in plugin + + agent = _documents(HERMES / "agent-deployment.yaml")[0] + containers = agent["spec"]["template"]["spec"]["containers"] + broker = next(item for item in containers if item["name"] == "image-broker") + assert broker["ports"] == [ + {"name": "image-broker", "containerPort": 9002, "protocol": "TCP"} + ] + assert broker["securityContext"]["readOnlyRootFilesystem"] is True + assert broker["securityContext"]["runAsNonRoot"] is True + + services = _documents(HERMES / "service.yaml") + service = next( + item for item in services if item["metadata"]["name"] == "hermes-image-broker" + ) + assert service["spec"]["selector"] == {"app": "hermes-agent"} + + oauth_store = _documents(HERMES / "oauth-session-store.yaml") + redis = next(item for item in oauth_store if item["kind"] == "Deployment") + assert redis["spec"]["strategy"]["type"] == "Recreate" + assert "--appendonly" in redis["spec"]["template"]["spec"]["containers"][0]["args"] + + policies = _documents(HERMES / "networkpolicy.yaml") + agent_policy = next( + item + for item in policies + if item["metadata"]["name"] == "hermes-agent-isolation" + ) + broker_ingress = next( + rule + for rule in agent_policy["spec"]["ingress"] + if {port["port"] for port in rule["ports"]} == {9002, 9003} + ) + assert broker_ingress["from"][0]["podSelector"]["matchLabels"] == { + "app": "hermes-chat-tenant" + } + vault_policy = (VAULT / "scripts" / "vault_k8s_auth_configure.sh").read_text() + agent_role = vault_policy[ + vault_policy.index('write_policy_and_role "hermes-agent"') : vault_policy.index( + "write_policy_and_role", + vault_policy.index('write_policy_and_role "hermes-agent"') + 1, + ) + ] + assert "hermes/developer-gitea" not in agent_role + assert 'write_policy_and_role "hermes-scm-broker" "hermes-scm"' in vault_policy + assert ( + 'write_policy_and_role "hermes-node-ssh" "hermes" ' + '"hermes-node-ssh-access"' in vault_policy + ) + + def test_compact_image_edit_resolves_latest_tenant_artifact(tmp_path, monkeypatch): """Follow-up edits resolve the source server-side and keep tool JSON small.""" provider_module = SimpleNamespace( @@ -63,6 +163,7 @@ def test_compact_image_edit_resolves_latest_tenant_artifact(tmp_path, monkeypatc ) ] + def test_chat_reasoning_uses_switchyard_without_owner_credentials(): """Family pods use AUTO/manual routes without mounting owner credentials.""" configmap = _documents(HERMES / "chat-configmap.yaml")[0] @@ -126,30 +227,31 @@ def test_chat_reasoning_uses_switchyard_without_owner_credentials(): ] statefulset = _documents(HERMES / "chat-statefulset.yaml")[0] - assert statefulset["spec"]["template"]["metadata"]["annotations"][ - "ai.bstein.dev/config-rev" - ] == "20260816-telegram-topics" + assert ( + statefulset["spec"]["template"]["metadata"]["annotations"][ + "ai.bstein.dev/config-rev" + ] + == "20260816-telegram-topics" + ) pod_spec = statefulset["spec"]["template"]["spec"] patch_init = next( - item for item in pod_spec["initContainers"] + item + for item in pod_spec["initContainers"] if item["name"] == "patch-stream-recovery" ) assert patch_init["command"][-2:] == [ "/opt/hermes/agent/conversation_loop.py", "/patched/conversation_loop.py", ] - hermes = next( - item - for item in pod_spec["containers"] - if item["name"] == "hermes" - ) + hermes = next(item for item in pod_spec["containers"] if item["name"] == "hermes") assert { "name": "stream-recovery-patch", "mountPath": "/opt/hermes/agent/conversation_loop.py", "subPath": "conversation_loop.py", } in hermes["volumeMounts"] api_session_init = next( - item for item in pod_spec["initContainers"] + item + for item in pod_spec["initContainers"] if item["name"] == "patch-api-server-sessions" ) assert "patch_api_server_sessions.py" in api_session_init["args"][0] @@ -159,17 +261,16 @@ def test_chat_reasoning_uses_switchyard_without_owner_credentials(): "mountPath": "/opt/hermes/gateway/platforms/api_server.py", "subPath": "api_server.py", } in hermes["volumeMounts"] - assert any( - volume["name"] == "api-server-patch" for volume in pod_spec["volumes"] - ) + assert any(volume["name"] == "api-server-patch" for volume in pod_spec["volumes"]) assert not any( - mount["mountPath"].endswith("/.codex") - for mount in hermes["volumeMounts"] + mount["mountPath"].endswith("/.codex") for mount in hermes["volumeMounts"] ) policies = _documents(HERMES / "networkpolicy.yaml") agent_policy = next( - item for item in policies if item["metadata"]["name"] == "hermes-agent-isolation" + item + for item in policies + if item["metadata"]["name"] == "hermes-agent-isolation" ) broker_ingress = next( rule @@ -179,106 +280,3 @@ def test_chat_reasoning_uses_switchyard_without_owner_credentials(): assert broker_ingress["from"][0]["podSelector"]["matchLabels"] == { "app": "hermes-chat-tenant" } - -def test_image_broker_returns_bytes_and_removes_owner_cache(tmp_path: Path, monkeypatch): - """The broker must not retain a family user's generated image.""" - broker_path = HERMES / "scripts" / "image_broker.py" - spec = importlib.util.spec_from_file_location("hermes_image_broker", broker_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - generated = tmp_path / "generated.png" - generated.write_bytes(b"\x89PNG\r\n\x1a\nprivate-image") - - class Provider: - def generate(self, prompt, aspect, **kwargs): - assert prompt == "paint a blue sphere" - assert aspect == "square" - return { - "success": True, - "image": str(generated), - "model": "gpt-image-2-high", - "quality": "high", - } - - monkeypatch.setattr(module, "_PROVIDER", Provider()) - result = module._generate( - { - "prompt": "paint a blue sphere", - "aspect_ratio": "square", - "model": "gpt-image-2-high", - } - ) - - assert result["success"] is True - assert result["image_b64"] - assert "image" not in result - assert not generated.exists() - -def test_image_broker_auto_falls_back_to_local_and_honors_explicit_routes( - monkeypatch, -): - """AUTO is hosted-first while explicit local never calls the hosted lane.""" - broker_path = HERMES / "scripts" / "image_broker.py" - spec = importlib.util.spec_from_file_location("hermes_image_router", broker_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - calls = [] - - def hosted(payload, model, prompt, aspect): - calls.append(("hosted", model, prompt, aspect)) - return {"success": False, "error": "hosted refusal"} - - def local(payload, timeout=1800.0): - calls.append(("local", payload["model"], timeout)) - return { - "success": True, - "image_b64": "aW1hZ2U=", - "model": "flux-2-klein-4b-local", - "route": "local", - } - - monkeypatch.setattr(module, "_generate_hosted", hosted) - monkeypatch.setattr(module, "_local_request", local) - auto = module._generate( - { - "prompt": "colorize this family photograph", - "aspect_ratio": "portrait", - "model": "atlas-image-auto-high", - } - ) - assert auto["success"] is True - assert auto["route"] == "local" - assert auto["hosted_fallback_reason"] == "hosted refusal" - assert [call[0] for call in calls] == ["hosted", "local"] - - calls.clear() - explicit = module._generate( - { - "prompt": "make a local landscape", - "aspect_ratio": "landscape", - "model": "flux-2-klein-4b-local", - } - ) - assert explicit["route"] == "local" - assert [call[0] for call in calls] == ["local"] - -def test_image_broker_policy_is_narrow_and_operator_extensible(tmp_path: Path, monkeypatch): - """Family-photo restoration stays allowed while the hard boundary remains.""" - broker_path = HERMES / "scripts" / "image_broker.py" - spec = importlib.util.spec_from_file_location("hermes_image_policy", broker_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - policy = tmp_path / "policy.json" - policy.write_text('{"additional_blocked_phrases":["site-specific block"]}') - monkeypatch.setattr(module, "POLICY_PATH", policy) - - assert module._policy_error( - "Colorize my baby photograph with a lighter natural skin tone" - ) is None - assert "minors" in module._policy_error("Create a sexual image of a child") - assert module._policy_error("A site-specific block request") == ( - "request is blocked by the operator image policy" - ) diff --git a/testing/tests/test_hermes_chat_local_compute.py b/testing/tests/test_hermes_chat_local_compute.py deleted file mode 100644 index 8ce39358..00000000 --- a/testing/tests/test_hermes_chat_local_compute.py +++ /dev/null @@ -1,295 +0,0 @@ -"""Hermes chat local compute contracts.""" - -from __future__ import annotations - -from test_hermes_chat_support import ( - HERMES, - Path, - ROOT, - SimpleNamespace, - _documents, - importlib, - json, - tomllib, -) - - -def test_local_flux_runtime_and_gpu_handoff_are_flux_managed(): - """FLUX and Wolf share titan-24 while text stays on titan-20.""" - deployment = _documents(HERMES / "local-image-deployment.yaml")[0] - assert deployment["metadata"]["name"] == "hermes-local-image" - pod = deployment["spec"]["template"]["spec"] - assert pod["serviceAccountName"] == "hermes-gpu-runtime" - local = next(item for item in pod["containers"] if item["name"] == "local-image") - assert len(pod["containers"]) == 1 - assert local["resources"]["requests"]["nvidia.com/gpu.shared"] == 1 - assert local["ports"] == [{"name": "local-image", "containerPort": 9004}] - assert any(mount["mountPath"] == "/models" for mount in local["volumeMounts"]) - model_env = {item["name"]: item["value"] for item in local["env"]} - assert model_env["HERMES_LOCAL_IMAGE_LISTEN_PORT"] == "9004" - assert model_env["HERMES_LOCAL_IMAGE_REVISION"] == ( - "e7b7dc27f91deacad38e78976d1f2b499d76a294" - ) - assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVITY_NODE"] == "titan-24" - assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVE_SM_PERCENT"] == "80" - assert model_env["HERMES_LOCAL_IMAGE_GPU_MAX_EXTERNAL_MEMORY_BYTES"] == ( - "3221225472" - ) - assert model_env["HERMES_LOCAL_IMAGE_OFFLOAD_MODE"] == "sequential" - assert "nvidia-process-exporter-local.monitoring.svc.cluster.local" in model_env[ - "HERMES_LOCAL_IMAGE_GPU_ACTIVITY_URL" - ] - models_volume = next(item for item in pod["volumes"] if item["name"] == "models") - assert models_volume["persistentVolumeClaim"]["claimName"] == ( - "hermes-image-models" - ) - - services = _documents(HERMES / "service.yaml") - image_service = next( - item for item in services if item["metadata"]["name"] == "hermes-local-image" - ) - assert image_service["spec"]["selector"] == {"app": "hermes-local-image"} - - handoff_services = _documents(HERMES / "model-gate-deployment.yaml") - handoff = next( - item - for item in handoff_services - if item["kind"] == "Service" - and item["metadata"]["name"] == "hermes-gpu-handoff" - ) - assert handoff["spec"]["ports"][0]["targetPort"] == "handoff" - - ariadne = _documents( - Path(__file__).parents[2] - / "services/maintenance/apps/ariadne-deployment.yaml" - )[0] - env = { - item["name"]: item["value"] - for item in ariadne["spec"]["template"]["spec"]["containers"][0]["env"] - if "value" in item - } - assert env["GAME_MODE_OLLAMA_URL"] == ( - "http://hermes-gpu-handoff.hermes.svc.cluster.local:11434" - ) - assert env["GAME_MODE_OLLAMA_MODEL"] == "flux-2-klein-4b-local" - - for config_name in ("configmap.yaml", "agent-configmap.yaml", "chat-configmap.yaml"): - config = _documents(HERMES / config_name)[0]["data"]["config.yaml"] - assert "gpt-oss:20b" not in config - assert "atlas-switchyard" in config - switchyard = _documents(HERMES / "switchyard-configmap.yaml")[0]["data"][ - "routes.toml" - ] - assert 'id = "qwen2.5:14b-instruct-q4_0"' in switchyard - assert "qwen2.5:3b-instruct-q4_0" not in switchyard - assert "route/local/qwen2.5-14b/medium" in switchyard - assert "Anthropic and Claude name the same provider" in switchyard - assert "OpenAI and Codex name the same provider" in switchyard - assert "Choose across every configured Codex and Claude family" in switchyard - assert "Claude Fable" in switchyard - assert switchyard.count('Treat "think hard"') == 4 - assert switchyard.count("Never choose below the") >= 5 - switchyard_config = tomllib.loads(switchyard) - routes = switchyard_config["routes"] - configured_targets = switchyard_config["targets"] - for route_name in ("auto_fast", "auto_balanced", "auto_deep", "auto_maximum"): - leading_targets = set(routes[route_name]["targets"][:2]) - assert leading_targets == {"codex_sol_xhigh", "claude_opus_xhigh"} - assert "max_output_tokens" not in routes[route_name] - for route_name in ("auto_fast", "auto_balanced", "auto_deep", "auto_maximum"): - targets = routes[route_name]["targets"] - selector_targets = routes[route_name]["response_schema"] - assert not any(target.startswith("local_") for target in targets) - assert any("fable" in target for target in targets) - assert "local_qwen" not in selector_targets - assert "not eligible for foreground" in routes[route_name]["prompt"] - for route_name in ("auto_deep", "auto_maximum"): - targets = routes[route_name]["targets"] - selector_targets = routes[route_name]["response_schema"] - assert not any(target.endswith("_low") for target in targets) - assert "_low" not in selector_targets - maximum_targets = routes["auto_maximum"]["targets"] - maximum_selector_targets = routes["auto_maximum"]["response_schema"] - assert not any(target.endswith("_medium") for target in maximum_targets) - assert "_medium" not in maximum_selector_targets - assert "absolute high effort floor" in routes["auto_maximum"]["prompt"] - assert "quality mark was missed" in routes["auto_balanced"]["prompt"] - assert "raises the next boundary to xhigh" in routes["auto_maximum"]["prompt"] - assert "Repeated quality misses require xhigh" in routes[ - "worker_auto_maximum" - ]["prompt"] - assert any( - target.startswith("local_") - for target in routes["manual_local_qwen"]["targets"] - ) - for route_name in ( - "manual_codex_luna", - "manual_codex_terra", - "manual_codex_sol", - "manual_claude_haiku", - "manual_claude_fable", - "manual_claude_sonnet", - "manual_claude_opus", - ): - assert not any( - target.startswith("local_") for target in routes[route_name]["targets"] - ) - for provider, families in { - "codex": ("luna", "terra", "sol"), - "claude": ("haiku", "fable", "sonnet", "opus"), - }.items(): - for family in families: - for effort in ("low", "medium", "high", "xhigh"): - route = routes[f"manual_{provider}_{family}_{effort}"] - assert route["id"] == f"atlas/manual/{provider}/{family}/{effort}" - assert route["targets"][0] == f"{provider}_{family}_{effort}" - worker_target = f"worker_{provider}_{family}_{effort}" - assert worker_target in routes["worker_auto_maximum"]["targets"] - assert configured_targets[worker_target]["id"] == ( - f"worker/{provider}/{family}/{effort}" - ) - for route_name in ("auto_fast", "auto_balanced"): - prompt = routes[route_name]["prompt"] - assert "image tool—not the conversational model" in prompt - assert "Do not select a" in prompt - assert "local Qwen or Claude target" in prompt - assert "max_output_tokens" not in routes["worker_auto_maximum"] - model_gate = _documents(HERMES / "model-gate-configmap.yaml")[0]["data"][ - "model_gate.py" - ] - assert "qwen2.5:14b-instruct-q4_0" in model_gate - -def test_titan20_serializes_classifier_and_local_chat_model_residency(): - """Classifier and local chat share one serialized resident Qwen weight.""" - deployment = _documents( - Path(__file__).parents[2] / "services/ai-llm/deployment.yaml" - )[0] - pod = deployment["spec"]["template"]["spec"] - required = pod["affinity"]["nodeAffinity"][ - "requiredDuringSchedulingIgnoredDuringExecution" - ]["nodeSelectorTerms"][0]["matchExpressions"][0] - assert required["values"] == ["titan-20"] - container = pod["containers"][0] - env = {item["name"]: item["value"] for item in container["env"]} - assert env["OLLAMA_MAX_LOADED_MODELS"] == "1" - assert env["OLLAMA_NUM_PARALLEL"] == "1" - assert env["OLLAMA_KEEP_ALIVE"] == "-1" - assert env["OLLAMA_CONTEXT_LENGTH"] == "8192" - warm_command = " ".join(container["command"]) - assert "--keepalive=-1" not in warm_command - models = next(item for item in pod["volumes"] if item["name"] == "models") - assert models["persistentVolumeClaim"]["claimName"] == ( - "ollama-models-titan20" - ) - -def test_local_image_gpu_guard_distinguishes_background_and_saturated_gpu(monkeypatch): - """Lease-idle desktop spikes may coexist, but saturation still blocks FLUX.""" - source = ROOT / "dockerfiles" / "hermes-local-image-server.py" - spec = importlib.util.spec_from_file_location("hermes_local_image_server", source) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - monkeypatch.setattr(module, "GPU_ACTIVITY_NODE", "titan-24") - monkeypatch.setattr(module, "GPU_ACTIVE_SM_PERCENT", 80.0) - monkeypatch.setattr(module, "GPU_MAX_EXTERNAL_MEMORY_BYTES", 3 << 30) - - idle = module._parse_gpu_activity( - '\n'.join( - [ - 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="Xorg"} 0', - 'nvidia_process_gpu_memory_used_bytes{node="titan-24",namespace="host",process="Xorg"} 1900000000', - 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="game-stream",process="wolf"} 3', - 'nvidia_process_gpu_memory_used_bytes{node="titan-24",namespace="hermes",process="python"} 9000000000', - ] - ) - ) - assert idle["interactive_active"] is False - assert idle["external_gpu_memory_bytes"] == 1900000000 - assert idle["external_gpu_sm_percent"] == 3 - - background_spike = module._parse_gpu_activity( - 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="sway"} 41\n' - ) - assert background_spike["interactive_active"] is False - - active = module._parse_gpu_activity( - 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="steam"} 91\n' - ) - assert active["interactive_active"] is True - assert "91%" in active["gpu_guard_reason"] - -def test_local_flux_renderer_uses_a_disposable_cuda_worker(monkeypatch): - """A completed render must not leave its CUDA context in the API process.""" - source = ROOT / "dockerfiles" / "hermes-local-image-server.py" - spec = importlib.util.spec_from_file_location("hermes_local_image_worker", source) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - calls = [] - - def run(command, **kwargs): - calls.append((command, kwargs)) - return SimpleNamespace( - returncode=0, - stdout=b'{"success":true,"route":"local","image_b64":"cG5n"}', - stderr=b"", - ) - - monkeypatch.setattr(module.subprocess, "run", run) - result = module._render({"prompt": "black cat", "aspect_ratio": "square"}) - - assert result["route"] == "local" - command, options = calls[0] - assert command[-1] == "--render-worker" - assert json.loads(options["input"]) == { - "prompt": "black cat", - "aspect_ratio": "square", - } - assert options["timeout"] == module.RENDER_TIMEOUT_SECONDS - assert options["check"] is False - -def test_local_flux_uses_low_vram_offload_without_reducing_resolution(): - """The shared 3080 lane must trade time, not image size, for headroom.""" - source = (ROOT / "dockerfiles" / "hermes-local-image-server.py").read_text() - assert 'OFFLOAD_MODE = os.environ.get(' in source - assert '"HERMES_LOCAL_IMAGE_OFFLOAD_MODE", "sequential"' in source - assert "pipe.enable_sequential_cpu_offload()" in source - assert '"square": (1024, 1024)' in source - -def test_chat_auth_and_relay_are_pod_lifetime_only(): - statefulset = _documents(HERMES / "chat-statefulset.yaml")[0] - pod = statefulset["spec"]["template"]["spec"] - containers = statefulset["spec"]["template"]["spec"]["containers"] - - for name in ("hermes", "webui"): - container = next(item for item in containers if item["name"] == name) - env = {item["name"]: item["value"] for item in container["env"]} - assert env["HERMES_AUTH_FILE"] == "/runtime-access/hermes-auth.json" - mount = next( - item for item in container["volumeMounts"] if item["name"] == "runtime-access" - ) - assert mount["mountPath"] == "/runtime-access" - assert "subPath" not in mount - - hermes_env = { - item["name"]: item["value"] - for item in next(item for item in containers if item["name"] == "hermes")["env"] - } - runtime = next(item for item in pod["volumes"] if item["name"] == "runtime-access") - assert runtime["emptyDir"] == {"medium": "Memory", "sizeLimit": "2Mi"} - assert not any(item["name"] == "provider-auth" for item in pod["volumes"]) - init_command = next( - item for item in pod["initContainers"] if item["name"] == "init-config" - )["command"][2] - for key in ( - "ANTHROPIC_API_KEY", - "API_SERVER_KEY", - "CLAUDE_CODE_OAUTH_TOKEN", - "GITEA_TOKEN", - "HERMES_IMAGE_BROKER_KEY", - "OPENAI_API_KEY", - ): - assert key in init_command - assert "printf 'API_SERVER_KEY=%s" not in init_command - assert hermes_env["AGENT_BROWSER_EXECUTABLE_PATH"].endswith("/chrome-linux/headless_shell") - assert "--no-sandbox" in hermes_env["AGENT_BROWSER_ARGS"] diff --git a/testing/tests/test_hermes_chat_local_runtime.py b/testing/tests/test_hermes_chat_local_runtime.py new file mode 100644 index 00000000..ddb0ef40 --- /dev/null +++ b/testing/tests/test_hermes_chat_local_runtime.py @@ -0,0 +1,172 @@ +"""Local inference and sandbox execution contracts for Hermes chat.""" + +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path +from types import SimpleNamespace + + +from testing.tests.test_hermes_chat_support import ( + HERMES, + ROOT, + _documents, +) + + +def test_titan20_serializes_classifier_and_local_chat_model_residency(): + """Classifier and local chat share one serialized resident Qwen weight.""" + deployment = _documents( + Path(__file__).parents[2] / "services/ai-llm/deployment.yaml" + )[0] + pod = deployment["spec"]["template"]["spec"] + required = pod["affinity"]["nodeAffinity"][ + "requiredDuringSchedulingIgnoredDuringExecution" + ]["nodeSelectorTerms"][0]["matchExpressions"][0] + assert required["values"] == ["titan-20"] + container = pod["containers"][0] + env = {item["name"]: item["value"] for item in container["env"]} + assert env["OLLAMA_MAX_LOADED_MODELS"] == "1" + assert env["OLLAMA_NUM_PARALLEL"] == "1" + assert env["OLLAMA_KEEP_ALIVE"] == "-1" + assert env["OLLAMA_CONTEXT_LENGTH"] == "8192" + warm_command = " ".join(container["command"]) + assert "--keepalive=-1" not in warm_command + models = next(item for item in pod["volumes"] if item["name"] == "models") + assert models["persistentVolumeClaim"]["claimName"] == ("ollama-models-titan20") + + +def test_local_image_gpu_guard_distinguishes_background_and_saturated_gpu(monkeypatch): + """Lease-idle desktop spikes may coexist, but saturation still blocks FLUX.""" + source = ROOT / "dockerfiles" / "hermes-local-image-server.py" + spec = importlib.util.spec_from_file_location("hermes_local_image_server", source) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + monkeypatch.setattr(module, "GPU_ACTIVITY_NODE", "titan-24") + monkeypatch.setattr(module, "GPU_ACTIVE_SM_PERCENT", 80.0) + monkeypatch.setattr(module, "GPU_MAX_EXTERNAL_MEMORY_BYTES", 3 << 30) + + idle = module._parse_gpu_activity( + "\n".join( + [ + 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="Xorg"} 0', + 'nvidia_process_gpu_memory_used_bytes{node="titan-24",namespace="host",process="Xorg"} 1900000000', + 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="game-stream",process="wolf"} 3', + 'nvidia_process_gpu_memory_used_bytes{node="titan-24",namespace="hermes",process="python"} 9000000000', + ] + ) + ) + assert idle["interactive_active"] is False + assert idle["external_gpu_memory_bytes"] == 1900000000 + assert idle["external_gpu_sm_percent"] == 3 + + background_spike = module._parse_gpu_activity( + 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="sway"} 41\n' + ) + assert background_spike["interactive_active"] is False + + active = module._parse_gpu_activity( + 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="steam"} 91\n' + ) + assert active["interactive_active"] is True + assert "91%" in active["gpu_guard_reason"] + + +def test_local_flux_renderer_uses_a_disposable_cuda_worker(monkeypatch): + """A completed render must not leave its CUDA context in the API process.""" + source = ROOT / "dockerfiles" / "hermes-local-image-server.py" + spec = importlib.util.spec_from_file_location("hermes_local_image_worker", source) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + calls = [] + + def run(command, **kwargs): + calls.append((command, kwargs)) + return SimpleNamespace( + returncode=0, + stdout=b'{"success":true,"route":"local","image_b64":"cG5n"}', + stderr=b"", + ) + + monkeypatch.setattr(module.subprocess, "run", run) + result = module._render({"prompt": "black cat", "aspect_ratio": "square"}) + + assert result["route"] == "local" + command, options = calls[0] + assert command[-1] == "--render-worker" + assert json.loads(options["input"]) == { + "prompt": "black cat", + "aspect_ratio": "square", + } + assert options["timeout"] == module.RENDER_TIMEOUT_SECONDS + assert options["check"] is False + + +def test_local_flux_uses_low_vram_offload_without_reducing_resolution(): + """The shared 3080 lane must trade time, not image size, for headroom.""" + source = (ROOT / "dockerfiles" / "hermes-local-image-server.py").read_text() + assert "OFFLOAD_MODE = os.environ.get(" in source + assert '"HERMES_LOCAL_IMAGE_OFFLOAD_MODE", "sequential"' in source + assert "pipe.enable_sequential_cpu_offload()" in source + assert '"square": (1024, 1024)' in source + + +def test_chat_auth_and_relay_are_pod_lifetime_only(): + statefulset = _documents(HERMES / "chat-statefulset.yaml")[0] + pod = statefulset["spec"]["template"]["spec"] + containers = statefulset["spec"]["template"]["spec"]["containers"] + + for name in ("hermes", "webui"): + container = next(item for item in containers if item["name"] == name) + env = {item["name"]: item["value"] for item in container["env"]} + assert env["HERMES_AUTH_FILE"] == "/runtime-access/hermes-auth.json" + mount = next( + item + for item in container["volumeMounts"] + if item["name"] == "runtime-access" + ) + assert mount["mountPath"] == "/runtime-access" + assert "subPath" not in mount + + hermes_env = { + item["name"]: item["value"] + for item in next(item for item in containers if item["name"] == "hermes")["env"] + } + runtime = next(item for item in pod["volumes"] if item["name"] == "runtime-access") + assert runtime["emptyDir"] == {"medium": "Memory", "sizeLimit": "2Mi"} + assert not any(item["name"] == "provider-auth" for item in pod["volumes"]) + init_command = next( + item for item in pod["initContainers"] if item["name"] == "init-config" + )["command"][2] + for key in ( + "ANTHROPIC_API_KEY", + "API_SERVER_KEY", + "CLAUDE_CODE_OAUTH_TOKEN", + "GITEA_TOKEN", + "HERMES_IMAGE_BROKER_KEY", + "OPENAI_API_KEY", + ): + assert key in init_command + assert "printf 'API_SERVER_KEY=%s" not in init_command + assert hermes_env["AGENT_BROWSER_EXECUTABLE_PATH"].endswith( + "/chrome-linux/headless_shell" + ) + assert "--no-sandbox" in hermes_env["AGENT_BROWSER_ARGS"] + + +def test_sandbox_executes_python_with_bounded_output(tmp_path: Path, monkeypatch): + source = ROOT / "dockerfiles" / "hermes-chat-sandbox-server.py" + spec = importlib.util.spec_from_file_location("hermes_chat_sandbox_server", source) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + monkeypatch.setattr(module, "WORKSPACE", tmp_path) + + result = module._execute("import math\nprint(math.comb(10, 3))") + + assert result["success"] is True + assert result["stdout"] == "120\n" + assert result["stderr"] == "" diff --git a/testing/tests/test_hermes_chat_provider_auth.py b/testing/tests/test_hermes_chat_provider_auth.py index 1db93094..8e432131 100644 --- a/testing/tests/test_hermes_chat_provider_auth.py +++ b/testing/tests/test_hermes_chat_provider_auth.py @@ -1,18 +1,458 @@ -"""Hermes chat provider auth contracts.""" +"""Provider subscription and OAuth contracts for Hermes chat.""" from __future__ import annotations -from test_hermes_chat_support import ( - HERMES, - Path, - ROOT, - _documents, +import base64 +import json +import sys +import time +from pathlib import Path +from types import ModuleType + +import pytest + +from testing.tests.test_hermes_chat_support import ( _load_broker_module, - importlib, - json, ) +def test_codex_broker_auth_and_request_contract(tmp_path: Path, monkeypatch): + """The relay is bounded, stateless, and rejects unapproved models.""" + module = _load_broker_module("hermes_codex_broker", "codex_broker.py", monkeypatch) + monkeypatch.setattr(module, "TOKEN", "relay-secret") + + assert module._authorized("Bearer relay-secret") is True + assert module._authorized("Bearer wrong") is False + assert module._real_model("route/codex/gpt-5.6-sol/xhigh") == "gpt-5.6-sol" + payload = module._validate_payload( + { + "model": "gpt-5.6-terra", + "input": "route this chat turn", + "store": True, + "stream": False, + "max_output_tokens": 96, + "max_completion_tokens": 96, + "max_tokens": 96, + "temperature": 0.7, + "top_p": 0.9, + } + ) + assert payload["store"] is False + assert payload["stream"] is True + assert "max_output_tokens" not in payload + assert "max_completion_tokens" not in payload + assert "max_tokens" not in payload + assert "temperature" not in payload + assert "top_p" not in payload + assert payload["input"] == [ + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "route this chat turn"}], + } + ] + response_item = { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "keep this item"}], + } + assert module._validate_payload({"model": "gpt-5.6-terra", "input": response_item})[ + "input" + ] == [response_item] + response_items = [response_item] + assert ( + module._validate_payload({"model": "gpt-5.6-terra", "input": response_items})[ + "input" + ] + is response_items + ) + image_items = [ + { + "type": "message", + "role": "user", + "content": [ + {"type": "input_text", "text": "What color is this?"}, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,cHJpdmF0ZQ==", + "detail": "high", + }, + }, + ], + } + ] + assert module._validate_payload({"model": "gpt-5.6-terra", "input": image_items})[ + "input" + ][0]["content"][1] == { + "type": "input_image", + "image_url": "data:image/png;base64,cHJpdmF0ZQ==", + "detail": "high", + } + switchyard_image_items = [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "What color is this?"}, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,cHJpdmF0ZQ==", + "detail": "auto", + }, + }, + ], + } + ] + assert module._validate_payload( + {"model": "gpt-5.6-terra", "input": switchyard_image_items} + )["input"][0]["content"][1] == { + "type": "input_image", + "image_url": "data:image/png;base64,cHJpdmF0ZQ==", + "detail": "auto", + } + switchyard_base64_items = [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "What color is this?"}, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "cHJpdmF0ZQ==", + }, + }, + ], + } + ] + normalized_base64 = module._validate_payload( + {"model": "gpt-5.6-terra", "input": switchyard_base64_items} + )["input"][0]["content"][1] + assert normalized_base64 == { + "type": "input_image", + "image_url": "data:image/png;base64,cHJpdmF0ZQ==", + } + switchyard_enum_items = [ + { + "role": "user", + "content": [ + { + "type": "input_image", + "image_url": { + "type": "url", + "data": { + "url": "data:image/png;base64,cHJpdmF0ZQ==", + "detail": "high", + }, + }, + } + ], + } + ] + nested_image = module._validate_payload( + {"model": "gpt-5.6-terra", "input": switchyard_enum_items} + )["input"][0]["content"][0] + assert nested_image["image_url"] == "data:image/png;base64,cHJpdmF0ZQ==" + assert nested_image["detail"] == "high" + with pytest.raises(ValueError, match=r"non-empty Responses image URL.*str\[4\]"): + module._validate_payload( + { + "model": "gpt-5.6-terra", + "input": [ + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_image", + "image_url": {"detail": "high"}, + } + ], + } + ], + } + ) + routed = module._validate_payload( + { + "model": "route/codex/gpt-5.6-luna/low", + "input": "use the low route", + "stream": False, + } + ) + assert routed["model"] == "gpt-5.6-luna" + with pytest.raises(ValueError, match="unsupported Codex model"): + module._validate_payload({"model": "unapproved-model", "input": "hello"}) + with pytest.raises(ValueError, match="non-empty Responses input"): + module._validate_payload({"model": "gpt-5.6-terra", "input": ""}) + with pytest.raises(ValueError, match="non-empty Responses input list"): + module._validate_payload({"model": "gpt-5.6-terra", "input": []}) + + completed = { + "id": "resp_test", + "object": "response", + "status": "completed", + "output": [], + } + completed_item = { + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "done"}], + } + assert module._completed_response( + [ + "event: response.created", + 'data: {"type":"response.created","response":{}}', + "event: response.output_item.done", + "data: " + + json.dumps( + { + "type": "response.output_item.done", + "output_index": 0, + "item": completed_item, + } + ), + "event: response.completed", + "data: " + + json.dumps({"type": "response.completed", "response": completed}), + "data: [DONE]", + ] + )["output"] == [completed_item] + raw_stream = ( + "event: response.output_item.done\n" + "data: " + + json.dumps( + { + "type": "response.output_item.done", + "output_index": 0, + "item": completed_item, + } + ) + + "\n\nevent: response.completed\ndata: " + + json.dumps({"type": "response.completed", "response": completed}) + + "\n\n" + ).encode() + normalized = module._normalized_stream( + raw_stream, {**completed, "output": [completed_item]} + ).decode() + terminal_data = next( + line for line in normalized.splitlines() if '"response.completed"' in line + ) + assert json.loads(terminal_data.removeprefix("data: "))["response"]["output"] == [ + completed_item + ] + assert normalized.endswith("\n\n") + streamed_function_item = { + "type": "function_call", + "name": "read_file", + "status": "completed", + "arguments": '{"path":"/tmp"}', + } + streamed_function_body = ( + "event: response.function_call_arguments.delta\n" + 'data: {"type":"response.function_call_arguments.delta",' + '"item_id":"call_1","delta":"{\\"path\\":\\"/tmp\\"}"}\n\n' + "event: response.function_call_arguments.done\n" + 'data: {"type":"response.function_call_arguments.done",' + '"item_id":"call_1","arguments":"{\\"path\\":\\"/tmp\\"}"}\n\n' + "event: response.output_item.done\n" + 'data: {"type":"response.output_item.done","output_index":0,' + '"item":{"type":"function_call","name":"read_file",' + '"arguments":"{\\"path\\":\\"/tmp\\"}"}}\n\n' + "event: response.completed\n" + "data: " + + json.dumps( + { + "type": "response.completed", + "response": {**completed, "output": [streamed_function_item]}, + } + ) + + "\n\n" + ).encode() + normalized_function_stream = module._normalized_stream( + streamed_function_body, {**completed, "output": [streamed_function_item]} + ).decode() + assert "response.function_call_arguments.delta" in normalized_function_stream + assert "response.function_call_arguments.done" not in normalized_function_stream + assert "response.output_item.done" not in normalized_function_stream + normalized_terminal = next( + line + for line in normalized_function_stream.splitlines() + if '"response.completed"' in line + ) + assert ( + json.loads(normalized_terminal.removeprefix("data: "))["response"]["output"] + == [] + ) + with pytest.raises(RuntimeError, match="retryable incomplete response"): + module._completed_response( + [ + "event: response.incomplete", + 'data: {"type":"response.incomplete","response":' + '{"status":"incomplete","incomplete_details":' + '{"reason":"max_output_tokens"}}}', + ] + ) + with pytest.raises(RuntimeError, match="provider unavailable"): + module._completed_response( + [ + "event: error", + 'data: {"type":"error","error":{"message":"provider unavailable"}}', + ] + ) + malformed_tool_item = { + "type": "function_call", + "name": "search_files", + "status": "completed", + "arguments": '{"path":"","offset":', + } + with pytest.raises(RuntimeError, match="malformed function arguments"): + module._completed_response( + [ + "event: response.output_item.done", + "data: " + + json.dumps( + { + "type": "response.output_item.done", + "output_index": 0, + "item": malformed_tool_item, + } + ), + "event: response.completed", + "data: " + + json.dumps({"type": "response.completed", "response": completed}), + ] + ) + valid_tool_item = { + **malformed_tool_item, + "arguments": '{"path":"","offset":0}', + } + assert module._completed_response( + [ + "event: response.output_item.done", + "data: " + + json.dumps( + { + "type": "response.output_item.done", + "output_index": 0, + "item": valid_tool_item, + } + ), + "event: response.completed", + "data: " + + json.dumps({"type": "response.completed", "response": completed}), + ] + )["output"] == [valid_tool_item] + with pytest.raises(RuntimeError, match="malformed function arguments"): + module._completed_response( + [ + "event: response.function_call_arguments.delta", + 'data: {"type":"response.function_call_arguments.delta",' + '"item_id":"call_1","output_index":0,' + '"delta":"{\\"path\\":\\"/tmp\\",\\"offset\\":"}', + "event: response.completed", + "data: " + + json.dumps({"type": "response.completed", "response": completed}), + ] + ) + streamed_tool = module._completed_response( + [ + "event: response.function_call_arguments.delta", + 'data: {"type":"response.function_call_arguments.delta",' + '"item_id":"call_2","output_index":0,' + '"delta":"{\\"path\\":\\"/tmp\\",\\"offset\\":"}', + "event: response.function_call_arguments.done", + 'data: {"type":"response.function_call_arguments.done",' + '"item_id":"call_2","output_index":0,' + '"arguments":"{\\"path\\":\\"/tmp\\",\\"offset\\":0}"}', + "event: response.completed", + "data: " + + json.dumps({"type": "response.completed", "response": completed}), + ] + ) + assert streamed_tool["status"] == "completed" + + auth_dir = tmp_path / ".codex" + auth_dir.mkdir() + # The token payload need only prove the broker reads CODEX_HOME directly. + encoded = ( + base64.urlsafe_b64encode(json.dumps({"exp": time.time() + 3600}).encode()) + .decode() + .rstrip("=") + ) + (auth_dir / "auth.json").write_text( + json.dumps({"tokens": {"access_token": f"header.{encoded}.signature"}}) + ) + monkeypatch.setenv("CODEX_HOME", str(auth_dir)) + assert module._access_token().startswith("header.") + + +def test_codex_broker_refreshes_and_persists_first_party_oauth( + tmp_path: Path, monkeypatch +): + """Expired ChatGPT OAuth refreshes in the canonical Codex CLI store.""" + module = _load_broker_module( + "hermes_codex_refresh_broker", "codex_broker.py", monkeypatch + ) + auth_dir = tmp_path / ".codex" + auth_dir.mkdir() + + def jwt(expires_at: float) -> str: + payload = ( + base64.urlsafe_b64encode(json.dumps({"exp": expires_at}).encode()) + .decode() + .rstrip("=") + ) + return f"header.{payload}.signature" + + expired = jwt(time.time() - 60) + live = jwt(time.time() + 3600) + auth_path = auth_dir / "auth.json" + auth_path.write_text( + json.dumps( + { + "auth_mode": "chatgpt", + "tokens": { + "access_token": expired, + "refresh_token": "refresh-old", + }, + } + ) + ) + calls = [] + auth_module = ModuleType("hermes_cli.auth") + + def refresh(access_token, refresh_token, *, timeout_seconds): + calls.append((access_token, refresh_token, timeout_seconds)) + return { + "access_token": live, + "refresh_token": "refresh-new", + "last_refresh": "2026-08-12T20:00:00Z", + } + + auth_module.refresh_codex_oauth_pure = refresh + package = ModuleType("hermes_cli") + package.auth = auth_module + monkeypatch.setitem(sys.modules, "hermes_cli", package) + monkeypatch.setitem(sys.modules, "hermes_cli.auth", auth_module) + monkeypatch.setenv("CODEX_HOME", str(auth_dir)) + + assert module._access_token() == live + persisted = json.loads(auth_path.read_text()) + assert persisted["tokens"]["access_token"] == live + assert persisted["tokens"]["refresh_token"] == "refresh-new" + assert persisted["last_refresh"] == "2026-08-12T20:00:00Z" + assert calls == [(expired, "refresh-old", 30.0)] + assert auth_path.stat().st_mode & 0o777 == 0o600 + + # A healthy token is reused, so repeated routed turns do not spend a + # refresh token or create a second billing/authentication path. + assert module._access_token() == live + assert len(calls) == 1 + + def test_claude_broker_uses_native_subscription_without_api_billing(monkeypatch): """Claude traffic must use the native first-party CLI subscription lane.""" module = _load_broker_module( @@ -34,159 +474,3 @@ def test_claude_broker_uses_native_subscription_without_api_billing(monkeypatch) assert "ANTHROPIC_API_KEY" not in module._claude_environment() assert "CLAUDE_API_KEY" not in module._claude_environment() assert module.CAPACITY_PATTERN.search("weekly usage limit exhausted") - -def test_codex_native_health_overrides_historical_router_errors( - tmp_path: Path, monkeypatch -): - """Fresh first-party health is authoritative over old Switchyard probes.""" - plugin_path = HERMES / "plugins" / "auto-router" / "provider_status.py" - spec = importlib.util.spec_from_file_location("hermes_provider_status", plugin_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - health_path = tmp_path / "codex.json" - health_path.write_text( - json.dumps( - { - "state": "available", - "authenticated": True, - "transport": "codex-chatgpt-subscription", - } - ) - ) - monkeypatch.setattr(module, "CODEX_HEALTH_PATH", health_path) - monkeypatch.setattr(module, "CLAUDE_HEALTH_PATH", tmp_path / "missing.json") - monkeypatch.setattr( - module, - "_get_json", - lambda url: {"status": "ok"} - if url.endswith("/health") - else { - "models": { - "route/codex/terra/medium": { - "calls": 1, - "errors": 99, - "total_tokens": 12, - } - } - }, - ) - monkeypatch.setattr(module, "_codex_account", lambda: {}) - monkeypatch.setattr(module, "_claude_account", lambda: {}) - - codex = module.provider_status_payload()["providers"]["codex"] - - assert codex["errors"] == 99 - assert codex["state"] == "available" - assert codex["native_health"]["transport"] == "codex-chatgpt-subscription" - -def test_api_session_patch_accepts_parent_lineage(tmp_path: Path): - """API-created workers must persist the originating Hermes session.""" - module_path = HERMES / "scripts" / "patch_api_server_sessions.py" - spec = importlib.util.spec_from_file_location("patch_api_sessions", module_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - source = tmp_path / "api_server.py" - destination = tmp_path / "patched.py" - source.write_text( - "prefix\n" - + module.BEFORE - + "middle\n" - + module.RUNS_BEFORE - + "run body\n" - + module.RUN_CLOSE_BEFORE - + module.RESPONSES_SESSION_BEFORE - + module.EVENT_CALLBACK_SIGNATURE_BEFORE - + "callback docstring and push helper\n" - + module.EVENT_CALLBACK_BODY_BEFORE - + "tool start body\n" - + module.EVENT_CALLBACK_END_BEFORE - + module.EVENT_CALLBACK_CALL_BEFORE - + module.RUN_SWEEP_BEFORE - + "suffix\n", - encoding="utf-8", - ) - - module.patch(source, destination) - patched = destination.read_text(encoding="utf-8") - - assert "X-Hermes-Parent-Session-Id" in patched - assert "parent_session_id=parent_session_id" in patched - assert "Parent session not found" in patched - assert "HERMES_API_DEFAULT_PARENT_MATCH_PREFIXES" in patched - assert "user_message.startswith(default_prefixes)" in patched - assert "session_parent_conflict" in patched - assert "X-Hermes-Conversation-Platform" in patched - assert "X-Hermes-Conversation-Title" in patched - assert 'conversation_platform != "telegram"' in patched - assert "db.record_gateway_session_peer(" in patched - assert 'display_name="Telegram"' in patched - assert "db.reopen_session(session_id)" in patched - assert 'db.end_session(session_id, f"api_run_{terminal_status}")' in patched - assert "def _record_run_activity(" in patched - assert '"_thinking": "Hermes is reasoning"' in patched - assert '"run.started": "Worker started"' in patched - assert '"run.completed": "Worker completed"' in patched - assert '"reasoning.available": "Hermes finished a reasoning step"' in patched - assert '"subagent.progress": "Nested worker progress"' in patched - assert "redact_sensitive_text" in patched - assert 'getattr(os, "O_NOFOLLOW", 0)' in patched - assert "os.fchmod(fd, 0o600)" in patched - assert "session_id=session_id" in patched - assert 'self._record_run_activity(session_id, "run.started")' in patched - assert 'detail = tool_name if event_type in {' in patched - assert 'if event_type == "subagent.tool"' in patched - assert "_RUN_ACTIVITY_HEARTBEAT_SECONDS = 15.0" in patched - assert 'heartbeats.get(session_id, 0.0)' in patched - assert '"subagent.thinking",' in patched - assert "Stream retention and run lifetime are separate" in patched - assert 'terminal_status in {"completed", "failed", "cancelled"}' in patched - assert patched.index("terminal_status = self._run_statuses") < patched.index( - "self._active_run_tasks.pop(run_id, None)" - ) - -def test_switchyard_brokers_and_native_claude_lane_use_the_right_images(): - """Thin brokers stay small while native Claude runs beside owner auth.""" - dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-switchyard-brokers").read_text() - assert "httpx==0.28.1" in dockerfile - assert "worker_route_broker.py" in dockerfile - assert "routing_catalog.py" in dockerfile - - deployment = _documents(HERMES / "switchyard-deployment.yaml")[0] - containers = { - container["name"]: container - for container in deployment["spec"]["template"]["spec"]["containers"] - } - expected = ( - "registry.bstein.dev/bstein/hermes-switchyard-brokers@" - "sha256:ee7e95e060ef8083da505162d7e9030daba15fdd828cc047bbcbe6aa409d2083" - ) - assert containers["worker-route-broker"]["image"] == expected - assert containers["classifier-broker"]["image"] == expected - assert "claude-oauth-broker" not in containers - - agent = _documents(HERMES / "agent-deployment.yaml")[0] - agent_containers = { - container["name"]: container - for container in agent["spec"]["template"]["spec"]["containers"] - } - for container_name in ("hermes", "terminal"): - container = agent_containers[container_name] - environment = {item["name"]: item["value"] for item in container["env"]} - mounts = {item["name"]: item for item in container["volumeMounts"]} - assert environment["HERMES_ROUTING_CATALOG_PATH"] == "/routing-catalog/catalog.json" - assert environment["HERMES_CODEX_HEALTH_PATH"] == "/opt/data/provider-health/codex.json" - assert environment["HERMES_CLAUDE_HEALTH_PATH"] == "/opt/data/provider-health/claude.json" - assert mounts["routing-catalog"]["mountPath"] == "/routing-catalog" - assert mounts["routing-catalog"]["readOnly"] is True - codex = agent_containers["codex-broker"] - codex_environment = {item["name"]: item["value"] for item in codex["env"]} - assert codex_environment["HERMES_CODEX_HEALTH_PATH"] == "/opt/data/provider-health/codex.json" - claude = agent_containers["claude-broker"] - assert claude["image"].startswith("registry.bstein.dev/bstein/hermes-agent@") - assert "unset ANTHROPIC_API_KEY CLAUDE_API_KEY" in claude["args"][0] - assert any( - mount["name"] == "home" and mount["mountPath"] == "/opt/data" - for mount in claude["volumeMounts"] - ) diff --git a/testing/tests/test_hermes_chat_config.py b/testing/tests/test_hermes_chat_quality.py similarity index 58% rename from testing/tests/test_hermes_chat_config.py rename to testing/tests/test_hermes_chat_quality.py index 7dd0e21c..b513b452 100644 --- a/testing/tests/test_hermes_chat_config.py +++ b/testing/tests/test_hermes_chat_quality.py @@ -1,15 +1,19 @@ -"""Hermes chat config contracts.""" +"""Core configuration and workload contracts for Hermes chat.""" from __future__ import annotations -from test_hermes_chat_support import ( +import importlib.util +import sys +from types import SimpleNamespace + +import yaml + +from testing.tests.test_hermes_chat_support import ( HERMES, ROOT, _documents, - yaml, ) - def test_chat_config_enables_real_research_compute_and_delegation(): configmap = _documents(HERMES / "chat-configmap.yaml")[0] config = yaml.safe_load(configmap["data"]["config.yaml"]) @@ -37,6 +41,7 @@ def test_chat_config_enables_real_research_compute_and_delegation(): assert "terminal" not in toolsets assert "code_execution" not in toolsets + def test_agent_config_keeps_delegated_reviewers_from_owning_task_lifecycle(): configmap = _documents(HERMES / "agent-configmap.yaml")[0] instructions = configmap["data"]["AGENTS.md"] @@ -56,22 +61,30 @@ def test_agent_config_keeps_delegated_reviewers_from_owning_task_lifecycle(): assert "Atlas organization has private visibility" in instructions assert "may be public or private" in instructions assert "do not infer a\nrepository's visibility" in instructions - assert "already supplied through `GIT_ASKPASS`" in instructions + assert "brokered Git for clone, fetch" in instructions + assert "client carries no repository credential" in instructions assert "Never call `kanban_show` without a known, non-empty task ID" in instructions assert "bounded ad-hoc inspection and acceptance checks may" in instructions assert "load implementation or TDD skills" in instructions assert "Never call `kanban_show` without\na known, non-empty task ID" in soul assert "must load a skill only when its workflow\nmaterially applies" in soul - assert "runtime-only `GIT_ASKPASS`" in soul - assert "`scm.bstein.dev` is Gitea, not GitHub" in instructions - assert "Never load or follow a GitHub/`gh`" in instructions - assert "/opt/coordinator/gitea_api.py METHOD /api/v1/..." in instructions + assert "no repository token is present in this pod" in soul + assert "`scm.bstein.dev` is Forgejo/Gitea, not GitHub" in instructions + assert "GitHub/`gh` skill for an Atlas remote" in instructions + assert "load\n`$manage-atlas-pull-requests`" in instructions + assert "Updates, merge,\napprove, close, delete, comments" in instructions + assert "Leave every PR\nunmerged for Brad's review" in instructions + assert "Use the Gitea pull-request merge endpoint" not in instructions + assert "reconcile the open PR" not in instructions + assert "use Flux reconciliation after" not in instructions + assert "PR publication ends at the verified open draft" in instructions assert "JENKINS_BASE_URL" in instructions assert "Do not\nuse `git reset --hard`" in instructions rendered = (HERMES / "agent-deployment.yaml").read_text() assert 'git config --global user.name "Hermes Agent"' in rendered assert 'git config --global user.email "hermes@bstein.dev"' in rendered + def test_agent_image_completes_parked_kanban_tasks_atomically(): dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-agent").read_text() @@ -87,6 +100,7 @@ def test_agent_image_completes_parked_kanban_tasks_atomically(): assert "payload.get(\"status\") == \"blocked\"" in dockerfile assert "hermes-kanban-blocked-regression.py" in dockerfile + def test_sandbox_shares_only_the_tenant_workspace_without_credentials(): sandbox_docs = _documents(HERMES / "chat-sandbox.yaml") deployments = [doc for doc in sandbox_docs if doc["kind"] == "Deployment"] @@ -150,6 +164,7 @@ def test_sandbox_shares_only_the_tenant_workspace_without_credentials(): f"hermes-chat-tenant-{ordinal}" ) + def test_gateway_image_honors_ui_model_and_caps_reasoning(): dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-agent").read_text() assert ( @@ -208,6 +223,7 @@ def test_gateway_image_honors_ui_model_and_caps_reasoning(): assert 'params.has("resume")' in dockerfile assert "Open Telegram setup" in dockerfile + def test_telegram_router_local_build_defaults_to_cluster_architecture(): dockerfile = ( ROOT / "dockerfiles" / "Dockerfile.hermes-chat-router" @@ -217,6 +233,7 @@ def test_telegram_router_local_build_defaults_to_cluster_architecture(): assert "ARG TARGETARCH=arm64" in dockerfile assert "GOOS=${TARGETOS} GOARCH=${TARGETARCH}" in dockerfile + def test_chat_oauth_allows_stale_service_worker_retirement(): documents = _documents(HERMES / "oauth2-proxy.yaml") deployment = next( @@ -271,3 +288,169 @@ def test_chat_oauth_allows_stale_service_worker_retirement(): assert agent_template_volume["configMap"]["name"] == ( "hermes-chat-oauth-templates" ) + + +def test_webui_recovers_auth_and_labels_session_scoped_controls(): + dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-webui").read_text() + + assert "res.status===401||res.status===403" in dockerfile + assert "window.location.assign('/oauth2/start?rd='" in dockerfile + assert "childrenExpanded?'▾ ':'▸ '" in dockerfile + assert "'atlas/auto/maximum': 'Automatic · Maximum'" in dockerfile + assert "hermes-webui-telegram-project-patch.py" in dockerfile + telegram_project_patch = ( + ROOT / "dockerfiles" / "hermes-webui-telegram-project-patch.py" + ).read_text() + assert "TELEGRAM_PROJECT_NAME = 'Telegram'" in telegram_project_patch + assert "session_key.startswith('telegram-topic-')" in telegram_project_patch + + router = (ROOT / "dockerfiles" / "hermes-webui-router.js").read_text() + assert "'atlas/auto/fast':'AUTO · Fast'" in router + assert "'atlas/manual/codex/sol':'Codex · SOL'" in router + assert "'atlas/manual/claude/opus':'Claude · Opus'" in router + assert "watchModelOptions('modelSelect')" in router + assert "watchModelOptions('settingsModel')" in router + + +def test_chat_voice_uses_private_jetson_services_and_shared_auto_route(): + statefulset = _documents(HERMES / "chat-statefulset.yaml")[0] + containers = statefulset["spec"]["template"]["spec"]["containers"] + webui = next(item for item in containers if item["name"] == "webui") + env = {item["name"]: item["value"] for item in webui["env"]} + + assert env["HERMES_STT_URL"] == ( + "http://hermes-stt.hermes.svc.cluster.local:9000/v1/audio/transcriptions" + ) + assert env["HERMES_WEBUI_ATLAS_TTS_URL"] == ( + "http://hermes-tts.hermes.svc.cluster.local:9001/v1/audio/speech" + ) + assert "hermes_stt_client.py" in env["HERMES_LOCAL_STT_COMMAND"] + + configmap = _documents(HERMES / "chat-configmap.yaml")[0] + config = yaml.safe_load(configmap["data"]["config.yaml"]) + assert config["stt"] == { + "enabled": True, + "provider": "local_command", + "local": {"model": "small", "language": "auto"}, + } + + dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-webui").read_text() + assert "hermes-webui-atlas-patch.py" in dockerfile + assert "hermes-webui-atlas-voice.js" in dockerfile + voice_script = (ROOT / "dockerfiles" / "hermes-webui-atlas-voice.js").read_text() + assert "/api/transcribe/capability" in voice_script + assert "/api/transcribe" in voice_script + assert "/api/tts" in voice_script + assert "speakResponse(generation)" in voice_script + assert "window._splitForTTS(text,280)" in voice_script + assert "pending=fetchSpeech(chunks[index+1])" in voice_script + assert "restartSoon(token,450)" in voice_script + assert "constraints.voiceIsolation=true" in voice_script + assert "highpass.frequency.value=140" in voice_script + assert "Math.max(0.04,noiseFloor*2.4+0.006)" in voice_script + assert "while(preRoll.length>3) preRoll.shift()" in voice_script + + stt_server = (ROOT / "dockerfiles" / "hermes-jetson-stt-server.py").read_text() + assert "def _repetitive_token" in stt_server + assert "compression_ratio_threshold=2.0" in stt_server + assert "no_speech_threshold=0.5" in stt_server + + +def test_voice_transcript_filter_removes_fan_hallucinations(monkeypatch): + server_path = ROOT / "dockerfiles" / "hermes-jetson-stt-server.py" + spec = importlib.util.spec_from_file_location("hermes_jetson_stt_server", server_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + monkeypatch.setitem(sys.modules, "cgi", SimpleNamespace()) + monkeypatch.setitem( + sys.modules, + "torch", + SimpleNamespace(cuda=SimpleNamespace(is_available=lambda: False)), + ) + monkeypatch.setitem(sys.modules, "whisper", SimpleNamespace()) + spec.loader.exec_module(module) + + result = { + "segments": [ + { + "text": " ththththththththth Testing.", + "no_speech_prob": 0.12, + "avg_logprob": -0.2, + }, + { + "text": " background hum", + "no_speech_prob": 0.82, + "avg_logprob": -0.9, + }, + ] + } + + assert module._clean_transcript(result) == "Testing." + + +def test_voice_models_are_baked_and_runtime_has_no_public_egress(): + stt_dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-jetson-stt").read_text() + tts_dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-jetson-tts").read_text() + assert "ADD --checksum=sha256:aff26ae4" in stt_dockerfile + assert "ADD --checksum=sha256:9ecf7799" in stt_dockerfile + assert "--chmod=0444" in stt_dockerfile + assert "chmod 0555 /opt/models /opt/models/whisper" in stt_dockerfile + assert "HERMES_STT_CACHE=/opt/models/whisper" in stt_dockerfile + assert "ADD --checksum=sha256:4cabf7c3" in tts_dockerfile + assert "ADD --checksum=sha256:db42b97d" in tts_dockerfile + assert tts_dockerfile.count("--chmod=0444") == 6 + assert "chmod 0555 /opt/models /opt/models/piper" in tts_dockerfile + assert "HERMES_TTS_CACHE=/opt/models/piper" in tts_dockerfile + tts_server = (ROOT / "dockerfiles" / "hermes-jetson-tts-server.py").read_text() + assert "download_voice" not in tts_server + assert "baked Piper voice is missing" in tts_server + assert "session_options.intra_op_num_threads = ONNX_THREADS" in tts_server + + policies = _documents(HERMES / "networkpolicy.yaml") + voice_policy = next( + item for item in policies if item["metadata"]["name"] == "hermes-private-voice" + ) + assert voice_policy["spec"]["policyTypes"] == ["Ingress", "Egress"] + assert not any( + "ipBlock" in destination + for rule in voice_policy["spec"]["egress"] + for destination in rule.get("to", []) + ) + + +def test_voice_workloads_have_deliberate_xavier_placement(): + documents = _documents(HERMES / "voice-deployment.yaml") + deployments = { + item["metadata"]["name"]: item + for item in documents + if item["kind"] == "Deployment" + } + stt = deployments["hermes-stt"]["spec"]["template"]["spec"] + tts = deployments["hermes-tts"]["spec"]["template"]["spec"] + + assert "@sha256:" in stt["containers"][0]["image"] + assert "@sha256:" in tts["containers"][0]["image"] + assert stt["nodeSelector"] == {"kubernetes.io/hostname": "titan-21"} + assert tts["nodeSelector"] == {"kubernetes.io/hostname": "titan-21"} + assert stt["automountServiceAccountToken"] is False + assert tts["automountServiceAccountToken"] is False + assert stt["enableServiceLinks"] is False + assert tts["enableServiceLinks"] is False + assert stt["runtimeClassName"] == "nvidia" + assert stt["securityContext"]["supplementalGroups"] == [44] + stt_resources = stt["containers"][0]["resources"] + stt_env = { + item["name"]: item["value"] for item in stt["containers"][0]["env"] + } + assert stt_env["NVIDIA_DRIVER_CAPABILITIES"] == "compute,utility" + assert stt_resources["requests"]["nvidia.com/gpu.shared"] == 1 + assert stt_resources["limits"]["nvidia.com/gpu.shared"] == 1 + assert "nvidia.com/gpu.shared" not in tts["containers"][0]["resources"]["requests"] + tts_env = { + item["name"]: item["value"] for item in tts["containers"][0]["env"] + } + assert tts_env["HERMES_TTS_VOICE"] == "en_US-lessac-medium" + assert tts_env["HERMES_TTS_ONNX_THREADS"] == "2" + assert tts["containers"][0]["resources"]["limits"]["cpu"] == "4" + assert all("hostPath" not in volume for volume in stt["volumes"]) + assert all("hostPath" not in volume for volume in tts["volumes"]) diff --git a/testing/tests/test_hermes_chat_routing_runtime.py b/testing/tests/test_hermes_chat_routing_runtime.py new file mode 100644 index 00000000..6a81d625 --- /dev/null +++ b/testing/tests/test_hermes_chat_routing_runtime.py @@ -0,0 +1,486 @@ +"""Routing and runtime workload contracts for Hermes chat.""" + +from __future__ import annotations + +import importlib.util +import json +import sqlite3 +import sys +import tomllib +from pathlib import Path +from types import SimpleNamespace + + +from testing.tests.test_hermes_chat_support import ( + HERMES, + ROOT, + _documents, +) + + +def test_stale_parent_linked_api_workers_close_on_startup(tmp_path: Path): + """A previous gateway lifetime cannot leave phantom active workers.""" + module_path = HERMES / "scripts" / "migrate_api_session_lineage.py" + spec = importlib.util.spec_from_file_location( + "close_stale_api_workers", module_path + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + database = tmp_path / "state.db" + with sqlite3.connect(database) as connection: + connection.execute( + "CREATE TABLE sessions (id TEXT PRIMARY KEY, source TEXT, " + "parent_session_id TEXT, title TEXT, started_at REAL, ended_at REAL, " + "end_reason TEXT, archived INTEGER DEFAULT 0)" + ) + connection.executemany( + "INSERT INTO sessions " + "(id, source, parent_session_id, started_at, ended_at, end_reason) " + "VALUES (?, ?, ?, 1, ?, ?)", + ( + ("stale", "api_server", "parent", None, None), + ("root", "api_server", None, None, None), + ("finished", "api_server", "parent", 2.0, "api_run_completed"), + ("interactive", "tui", "parent", None, None), + ), + ) + + assert module.migrate(database) == 1 + assert module.migrate(database) == 0 + with sqlite3.connect(database) as connection: + rows = connection.execute( + "SELECT id, ended_at, end_reason FROM sessions ORDER BY id" + ).fetchall() + by_id = {row[0]: row[1:] for row in rows} + assert by_id["stale"][0] is not None + assert by_id["stale"][1] == "api_run_recovered_stale" + assert by_id["root"] == (None, None) + assert by_id["finished"] == (2.0, "api_run_completed") + assert by_id["interactive"] == (None, None) + + +def test_switchyard_brokers_and_native_claude_lane_use_the_right_images(): + """Thin brokers stay small while native Claude runs beside owner auth.""" + dockerfile = ( + ROOT / "dockerfiles" / "Dockerfile.hermes-switchyard-brokers" + ).read_text() + assert "httpx==0.28.1" in dockerfile + assert "worker_route_broker.py" in dockerfile + assert "routing_catalog.py" in dockerfile + + deployment = _documents(HERMES / "switchyard-deployment.yaml")[0] + containers = { + container["name"]: container + for container in deployment["spec"]["template"]["spec"]["containers"] + } + expected = ( + "registry.bstein.dev/bstein/hermes-switchyard-brokers@" + "sha256:ee7e95e060ef8083da505162d7e9030daba15fdd828cc047bbcbe6aa409d2083" + ) + assert containers["worker-route-broker"]["image"] == expected + assert containers["classifier-broker"]["image"] == expected + assert "claude-oauth-broker" not in containers + + agent = _documents(HERMES / "agent-deployment.yaml")[0] + agent_containers = { + container["name"]: container + for container in agent["spec"]["template"]["spec"]["containers"] + } + for container_name in ("hermes", "terminal"): + container = agent_containers[container_name] + environment = {item["name"]: item["value"] for item in container["env"]} + mounts = {item["name"]: item for item in container["volumeMounts"]} + assert ( + environment["HERMES_ROUTING_CATALOG_PATH"] + == "/routing-catalog/catalog.json" + ) + assert ( + environment["HERMES_CODEX_HEALTH_PATH"] + == "/opt/data/provider-health/codex.json" + ) + assert ( + environment["HERMES_CLAUDE_HEALTH_PATH"] + == "/opt/data/provider-health/claude.json" + ) + assert mounts["routing-catalog"]["mountPath"] == "/routing-catalog" + assert mounts["routing-catalog"]["readOnly"] is True + codex = agent_containers["codex-broker"] + codex_environment = {item["name"]: item["value"] for item in codex["env"]} + assert ( + codex_environment["HERMES_CODEX_HEALTH_PATH"] + == "/opt/data/provider-health/codex.json" + ) + claude = agent_containers["claude-broker"] + assert claude["image"].startswith("registry.bstein.dev/bstein/hermes-agent@") + assert "unset ANTHROPIC_API_KEY CLAUDE_API_KEY" in claude["args"][0] + assert any( + mount["name"] == "home" and mount["mountPath"] == "/opt/data" + for mount in claude["volumeMounts"] + ) + + +def test_classifier_broker_bounds_history_without_losing_routing_intent(monkeypatch): + """AUTO classification must fit the Jetson context without losing intent.""" + broker_path = HERMES / "scripts" / "classifier_broker.py" + monkeypatch.setitem(sys.modules, "httpx", SimpleNamespace()) + spec = importlib.util.spec_from_file_location( + "hermes_classifier_broker", broker_path + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + payload = { + "model": "qwen2.5:14b-instruct-q4_0", + "messages": [ + { + "role": "system", + "content": "routing contract\n" + ("candidate policy " * 1000), + }, + { + "role": "user", + "content": "Build and verify the Cassandra release safely.", + }, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_large", + "type": "function", + "function": { + "name": "large_tool", + "arguments": '{"command":"' + ("x" * 5000) + '"}', + }, + } + ], + }, + { + "role": "tool", + "content": "unbounded test output " * 10000, + "tool_call_id": "call_large", + }, + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": "data:image/png;base64,private"}, + }, + {"type": "text", "text": "Turn this cat into a cute clown."}, + ], + }, + ], + "tools": [{"type": "function", "function": {"name": "large_tool"}}], + "tool_choice": "auto", + "parallel_tool_calls": True, + "response_format": {"type": "json_object"}, + } + + compacted = module.compact_payload(payload) + encoded = json.dumps(compacted) + + assert compacted["model"] == payload["model"] + assert compacted["response_format"] == payload["response_format"] + assert "tools" not in compacted + assert "tool_choice" not in compacted + assert "parallel_tool_calls" not in compacted + assert all(message.get("role") != "tool" for message in compacted["messages"]) + assert all("tool_call_id" not in message for message in compacted["messages"]) + assert all("tool_calls" not in message for message in compacted["messages"]) + assert "[tool evidence]" in encoded + assert "[assistant requested an external tool]" in encoded + assert "Build and verify the Cassandra release safely." in encoded + assert "Turn this cat into a cute clown." in encoded + assert "image attachment available to the selected worker" in encoded + assert "data:image/png;base64" not in encoded + assert "unbounded test output " * 100 not in encoded + assert len(encoded) < 18_000 + + +def test_switchyard_classifier_is_bounded_and_fails_open_once(): + """A sick local judge must not hold chat through repeated long retries.""" + config = tomllib.loads( + _documents(HERMES / "switchyard-configmap.yaml")[0]["data"]["routes.toml"] + ) + classifier = config["llm_clients"]["classifier"] + assert classifier["base_url"] == "http://127.0.0.1:9008/v1" + assert classifier["max_retries"] == 0 + for route in ("auto_fast", "auto_balanced"): + assert config["routes"][route]["recent_turn_window"] == 4 + for route in ("auto_deep", "auto_maximum", "worker_auto_maximum"): + assert config["routes"][route]["recent_turn_window"] == 6 + + deployment = _documents(HERMES / "switchyard-deployment.yaml")[0] + containers = { + item["name"]: item + for item in deployment["spec"]["template"]["spec"]["containers"] + } + classifier_container = containers["classifier-broker"] + env = {item["name"]: item["value"] for item in classifier_container["env"]} + assert env["HERMES_CLASSIFIER_BROKER_READ_TIMEOUT"] == "60" + assert classifier_container["readinessProbe"]["httpGet"]["port"] == "classifier" + + +def test_image_broker_returns_bytes_and_removes_owner_cache( + tmp_path: Path, monkeypatch +): + """The broker must not retain a family user's generated image.""" + broker_path = HERMES / "scripts" / "image_broker.py" + spec = importlib.util.spec_from_file_location("hermes_image_broker", broker_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + generated = tmp_path / "generated.png" + generated.write_bytes(b"\x89PNG\r\n\x1a\nprivate-image") + + class Provider: + def generate(self, prompt, aspect, **kwargs): + assert prompt == "paint a blue sphere" + assert aspect == "square" + return { + "success": True, + "image": str(generated), + "model": "gpt-image-2-high", + "quality": "high", + } + + monkeypatch.setattr(module, "_PROVIDER", Provider()) + result = module._generate( + { + "prompt": "paint a blue sphere", + "aspect_ratio": "square", + "model": "gpt-image-2-high", + } + ) + + assert result["success"] is True + assert result["image_b64"] + assert "image" not in result + assert not generated.exists() + + +def test_image_broker_auto_falls_back_to_local_and_honors_explicit_routes( + monkeypatch, +): + """AUTO is hosted-first while explicit local never calls the hosted lane.""" + broker_path = HERMES / "scripts" / "image_broker.py" + spec = importlib.util.spec_from_file_location("hermes_image_router", broker_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + calls = [] + + def hosted(payload, model, prompt, aspect): + calls.append(("hosted", model, prompt, aspect)) + return {"success": False, "error": "hosted refusal"} + + def local(payload, timeout=1800.0): + calls.append(("local", payload["model"], timeout)) + return { + "success": True, + "image_b64": "aW1hZ2U=", + "model": "flux-2-klein-4b-local", + "route": "local", + } + + monkeypatch.setattr(module, "_generate_hosted", hosted) + monkeypatch.setattr(module, "_local_request", local) + auto = module._generate( + { + "prompt": "colorize this family photograph", + "aspect_ratio": "portrait", + "model": "atlas-image-auto-high", + } + ) + assert auto["success"] is True + assert auto["route"] == "local" + assert auto["hosted_fallback_reason"] == "hosted refusal" + assert [call[0] for call in calls] == ["hosted", "local"] + + calls.clear() + explicit = module._generate( + { + "prompt": "make a local landscape", + "aspect_ratio": "landscape", + "model": "flux-2-klein-4b-local", + } + ) + assert explicit["route"] == "local" + assert [call[0] for call in calls] == ["local"] + + +def test_image_broker_policy_is_narrow_and_operator_extensible( + tmp_path: Path, monkeypatch +): + """Family-photo restoration stays allowed while the hard boundary remains.""" + broker_path = HERMES / "scripts" / "image_broker.py" + spec = importlib.util.spec_from_file_location("hermes_image_policy", broker_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + policy = tmp_path / "policy.json" + policy.write_text('{"additional_blocked_phrases":["site-specific block"]}') + monkeypatch.setattr(module, "POLICY_PATH", policy) + + assert ( + module._policy_error( + "Colorize my baby photograph with a lighter natural skin tone" + ) + is None + ) + assert "minors" in module._policy_error("Create a sexual image of a child") + assert module._policy_error("A site-specific block request") == ( + "request is blocked by the operator image policy" + ) + + +def test_local_flux_runtime_and_gpu_handoff_are_flux_managed(): + """FLUX and Wolf share titan-24 while text stays on titan-20.""" + deployment = _documents(HERMES / "local-image-deployment.yaml")[0] + assert deployment["metadata"]["name"] == "hermes-local-image" + pod = deployment["spec"]["template"]["spec"] + assert pod["serviceAccountName"] == "hermes-gpu-runtime" + local = next(item for item in pod["containers"] if item["name"] == "local-image") + assert len(pod["containers"]) == 1 + assert local["resources"]["requests"]["nvidia.com/gpu.shared"] == 1 + assert local["ports"] == [{"name": "local-image", "containerPort": 9004}] + assert any(mount["mountPath"] == "/models" for mount in local["volumeMounts"]) + model_env = {item["name"]: item["value"] for item in local["env"]} + assert model_env["HERMES_LOCAL_IMAGE_LISTEN_PORT"] == "9004" + assert model_env["HERMES_LOCAL_IMAGE_REVISION"] == ( + "e7b7dc27f91deacad38e78976d1f2b499d76a294" + ) + assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVITY_NODE"] == "titan-24" + assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVE_SM_PERCENT"] == "80" + assert model_env["HERMES_LOCAL_IMAGE_GPU_MAX_EXTERNAL_MEMORY_BYTES"] == ( + "3221225472" + ) + assert model_env["HERMES_LOCAL_IMAGE_OFFLOAD_MODE"] == "sequential" + assert ( + "nvidia-process-exporter-local.monitoring.svc.cluster.local" + in model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVITY_URL"] + ) + models_volume = next(item for item in pod["volumes"] if item["name"] == "models") + assert models_volume["persistentVolumeClaim"]["claimName"] == ( + "hermes-image-models" + ) + + services = _documents(HERMES / "service.yaml") + image_service = next( + item for item in services if item["metadata"]["name"] == "hermes-local-image" + ) + assert image_service["spec"]["selector"] == {"app": "hermes-local-image"} + + handoff_services = _documents(HERMES / "model-gate-deployment.yaml") + handoff = next( + item + for item in handoff_services + if item["kind"] == "Service" + and item["metadata"]["name"] == "hermes-gpu-handoff" + ) + assert handoff["spec"]["ports"][0]["targetPort"] == "handoff" + + ariadne = _documents( + Path(__file__).parents[2] / "services/maintenance/apps/ariadne-deployment.yaml" + )[0] + env = { + item["name"]: item["value"] + for item in ariadne["spec"]["template"]["spec"]["containers"][0]["env"] + if "value" in item + } + assert env["GAME_MODE_OLLAMA_URL"] == ( + "http://hermes-gpu-handoff.hermes.svc.cluster.local:11434" + ) + assert env["GAME_MODE_OLLAMA_MODEL"] == "flux-2-klein-4b-local" + + for config_name in ( + "configmap.yaml", + "agent-configmap.yaml", + "chat-configmap.yaml", + ): + config = _documents(HERMES / config_name)[0]["data"]["config.yaml"] + assert "gpt-oss:20b" not in config + assert "atlas-switchyard" in config + switchyard = _documents(HERMES / "switchyard-configmap.yaml")[0]["data"][ + "routes.toml" + ] + assert 'id = "qwen2.5:14b-instruct-q4_0"' in switchyard + assert "qwen2.5:3b-instruct-q4_0" not in switchyard + assert "route/local/qwen2.5-14b/medium" in switchyard + assert "Anthropic and Claude name the same provider" in switchyard + assert "OpenAI and Codex name the same provider" in switchyard + assert "Choose across every configured Codex and Claude family" in switchyard + assert "Claude Fable" in switchyard + assert switchyard.count('Treat "think hard"') == 4 + assert switchyard.count("Never choose below the") >= 5 + switchyard_config = tomllib.loads(switchyard) + routes = switchyard_config["routes"] + configured_targets = switchyard_config["targets"] + for route_name in ("auto_fast", "auto_balanced", "auto_deep", "auto_maximum"): + leading_targets = set(routes[route_name]["targets"][:2]) + assert leading_targets == {"codex_sol_xhigh", "claude_opus_xhigh"} + assert "max_output_tokens" not in routes[route_name] + for route_name in ("auto_fast", "auto_balanced", "auto_deep", "auto_maximum"): + targets = routes[route_name]["targets"] + selector_targets = routes[route_name]["response_schema"] + assert not any(target.startswith("local_") for target in targets) + assert any("fable" in target for target in targets) + assert "local_qwen" not in selector_targets + assert "not eligible for foreground" in routes[route_name]["prompt"] + for route_name in ("auto_deep", "auto_maximum"): + targets = routes[route_name]["targets"] + selector_targets = routes[route_name]["response_schema"] + assert not any(target.endswith("_low") for target in targets) + assert "_low" not in selector_targets + maximum_targets = routes["auto_maximum"]["targets"] + maximum_selector_targets = routes["auto_maximum"]["response_schema"] + assert not any(target.endswith("_medium") for target in maximum_targets) + assert "_medium" not in maximum_selector_targets + assert "absolute high effort floor" in routes["auto_maximum"]["prompt"] + assert "quality mark was missed" in routes["auto_balanced"]["prompt"] + assert "raises the next boundary to xhigh" in routes["auto_maximum"]["prompt"] + assert ( + "Repeated quality misses require xhigh" + in routes["worker_auto_maximum"]["prompt"] + ) + assert any( + target.startswith("local_") for target in routes["manual_local_qwen"]["targets"] + ) + for route_name in ( + "manual_codex_luna", + "manual_codex_terra", + "manual_codex_sol", + "manual_claude_haiku", + "manual_claude_fable", + "manual_claude_sonnet", + "manual_claude_opus", + ): + assert not any( + target.startswith("local_") for target in routes[route_name]["targets"] + ) + for provider, families in { + "codex": ("luna", "terra", "sol"), + "claude": ("haiku", "fable", "sonnet", "opus"), + }.items(): + for family in families: + for effort in ("low", "medium", "high", "xhigh"): + route = routes[f"manual_{provider}_{family}_{effort}"] + assert route["id"] == f"atlas/manual/{provider}/{family}/{effort}" + assert route["targets"][0] == f"{provider}_{family}_{effort}" + worker_target = f"worker_{provider}_{family}_{effort}" + assert worker_target in routes["worker_auto_maximum"]["targets"] + assert configured_targets[worker_target]["id"] == ( + f"worker/{provider}/{family}/{effort}" + ) + for route_name in ("auto_fast", "auto_balanced"): + prompt = routes[route_name]["prompt"] + assert "image tool—not the conversational model" in prompt + assert "Do not select a" in prompt + assert "local Qwen or Claude target" in prompt + assert "max_output_tokens" not in routes["worker_auto_maximum"] + model_gate = _documents(HERMES / "model-gate-configmap.yaml")[0]["data"][ + "model_gate.py" + ] + assert "qwen2.5:14b-instruct-q4_0" in model_gate diff --git a/testing/tests/test_hermes_chat_sandbox.py b/testing/tests/test_hermes_chat_sandbox.py deleted file mode 100644 index fb6856d1..00000000 --- a/testing/tests/test_hermes_chat_sandbox.py +++ /dev/null @@ -1,24 +0,0 @@ -"""Hermes chat sandbox contracts.""" - -from __future__ import annotations - -from test_hermes_chat_support import ( - Path, - ROOT, - importlib, -) - - -def test_sandbox_executes_python_with_bounded_output(tmp_path: Path, monkeypatch): - source = ROOT / "dockerfiles" / "hermes-chat-sandbox-server.py" - spec = importlib.util.spec_from_file_location("hermes_chat_sandbox_server", source) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - monkeypatch.setattr(module, "WORKSPACE", tmp_path) - - result = module._execute("import math\nprint(math.comb(10, 3))") - - assert result["success"] is True - assert result["stdout"] == "120\n" - assert result["stderr"] == "" diff --git a/testing/tests/test_hermes_chat_sessions.py b/testing/tests/test_hermes_chat_session_continuity.py similarity index 72% rename from testing/tests/test_hermes_chat_sessions.py rename to testing/tests/test_hermes_chat_session_continuity.py index c4e2d6f2..0da542cb 100644 --- a/testing/tests/test_hermes_chat_sessions.py +++ b/testing/tests/test_hermes_chat_session_continuity.py @@ -1,21 +1,138 @@ -"""Hermes chat sessions contracts.""" +"""Session lineage and continuity contracts for Hermes chat.""" from __future__ import annotations -from test_hermes_chat_support import ( +import importlib.util +import json +import sqlite3 +from pathlib import Path + +import pytest + +from testing.tests.test_hermes_chat_support import ( HERMES, - Path, - importlib, - json, - pytest, - sqlite3, ) +def test_codex_native_health_overrides_historical_router_errors( + tmp_path: Path, monkeypatch +): + """Fresh first-party health is authoritative over old Switchyard probes.""" + plugin_path = HERMES / "plugins" / "auto-router" / "provider_status.py" + spec = importlib.util.spec_from_file_location("hermes_provider_status", plugin_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + health_path = tmp_path / "codex.json" + health_path.write_text( + json.dumps( + { + "state": "available", + "authenticated": True, + "transport": "codex-chatgpt-subscription", + } + ) + ) + monkeypatch.setattr(module, "CODEX_HEALTH_PATH", health_path) + monkeypatch.setattr(module, "CLAUDE_HEALTH_PATH", tmp_path / "missing.json") + monkeypatch.setattr( + module, + "_get_json", + lambda url: {"status": "ok"} + if url.endswith("/health") + else { + "models": { + "route/codex/terra/medium": { + "calls": 1, + "errors": 99, + "total_tokens": 12, + } + } + }, + ) + monkeypatch.setattr(module, "_codex_account", lambda: {}) + monkeypatch.setattr(module, "_claude_account", lambda: {}) + + codex = module.provider_status_payload()["providers"]["codex"] + + assert codex["errors"] == 99 + assert codex["state"] == "available" + assert codex["native_health"]["transport"] == "codex-chatgpt-subscription" + + +def test_api_session_patch_accepts_parent_lineage(tmp_path: Path): + """API-created workers must persist the originating Hermes session.""" + module_path = HERMES / "scripts" / "patch_api_server_sessions.py" + spec = importlib.util.spec_from_file_location("patch_api_sessions", module_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + source = tmp_path / "api_server.py" + destination = tmp_path / "patched.py" + source.write_text( + "prefix\n" + + module.BEFORE + + "middle\n" + + module.RUNS_BEFORE + + "run body\n" + + module.RUN_CLOSE_BEFORE + + module.RESPONSES_SESSION_BEFORE + + module.EVENT_CALLBACK_SIGNATURE_BEFORE + + "callback docstring and push helper\n" + + module.EVENT_CALLBACK_BODY_BEFORE + + "tool start body\n" + + module.EVENT_CALLBACK_END_BEFORE + + module.EVENT_CALLBACK_CALL_BEFORE + + module.RUN_SWEEP_BEFORE + + "suffix\n", + encoding="utf-8", + ) + + module.patch(source, destination) + patched = destination.read_text(encoding="utf-8") + + assert "X-Hermes-Parent-Session-Id" in patched + assert "parent_session_id=parent_session_id" in patched + assert "Parent session not found" in patched + assert "HERMES_API_DEFAULT_PARENT_MATCH_PREFIXES" in patched + assert "user_message.startswith(default_prefixes)" in patched + assert "session_parent_conflict" in patched + assert "X-Hermes-Conversation-Platform" in patched + assert "X-Hermes-Conversation-Title" in patched + assert 'conversation_platform != "telegram"' in patched + assert "db.record_gateway_session_peer(" in patched + assert 'display_name="Telegram"' in patched + assert "db.reopen_session(session_id)" in patched + assert 'db.end_session(session_id, f"api_run_{terminal_status}")' in patched + assert "def _record_run_activity(" in patched + assert '"_thinking": "Hermes is reasoning"' in patched + assert '"run.started": "Worker started"' in patched + assert '"run.completed": "Worker completed"' in patched + assert '"reasoning.available": "Hermes finished a reasoning step"' in patched + assert '"subagent.progress": "Nested worker progress"' in patched + assert "redact_sensitive_text" in patched + assert 'getattr(os, "O_NOFOLLOW", 0)' in patched + assert "os.fchmod(fd, 0o600)" in patched + assert "session_id=session_id" in patched + assert 'self._record_run_activity(session_id, "run.started")' in patched + assert "detail = tool_name if event_type in {" in patched + assert 'if event_type == "subagent.tool"' in patched + assert "_RUN_ACTIVITY_HEARTBEAT_SECONDS = 15.0" in patched + assert "heartbeats.get(session_id, 0.0)" in patched + assert '"subagent.thinking",' in patched + assert "Stream retention and run lifetime are separate" in patched + assert 'terminal_status in {"completed", "failed", "cancelled"}' in patched + assert patched.index("terminal_status = self._run_statuses") < patched.index( + "self._active_run_tasks.pop(run_id, None)" + ) + + def test_telegram_api_session_migration_is_bounded_and_idempotent(tmp_path: Path): """Only named Telegram conversations receive presentation metadata.""" module_path = HERMES / "scripts" / "migrate_telegram_api_sessions.py" - spec = importlib.util.spec_from_file_location("migrate_telegram_sessions", module_path) + spec = importlib.util.spec_from_file_location( + "migrate_telegram_sessions", module_path + ) assert spec and spec.loader module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) @@ -69,6 +186,7 @@ def test_telegram_api_session_migration_is_bounded_and_idempotent(tmp_path: Path assert telegram[4] == "Telegram · General" assert other == (None, None) + def test_web_session_activity_patch_projects_bounded_events(tmp_path: Path): """The DOM transcript includes events without polluting agent history.""" module_path = HERMES / "scripts" / "patch_web_session_activity.py" @@ -109,6 +227,7 @@ def test_web_session_activity_patch_projects_bounded_events(tmp_path: Path): assert "orphaned children" in patched assert 'item[0].get("ended_at") is None' in patched + def test_web_session_lineage_returns_to_resumed_parent(): """An ended reviewer must not strand the live view away from its parent.""" module_path = HERMES / "scripts" / "patch_web_session_activity.py" @@ -189,6 +308,7 @@ def test_web_session_lineage_returns_to_resumed_parent(): ["root", "accepted-review"], ) + def test_api_activity_patch_coalesces_streaming_heartbeats( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ): @@ -207,8 +327,7 @@ def test_api_activity_patch_coalesces_streaming_heartbeats( "from pathlib import Path\n" "logger = logging.getLogger(__name__)\n" "def redact_sensitive_text(value): return value\n" - "class ActivityRecorder:\n" - + activity_body, + "class ActivityRecorder:\n" + activity_body, namespace, ) recorder = namespace["ActivityRecorder"]() @@ -229,6 +348,7 @@ def test_api_activity_patch_coalesces_streaming_heartbeats( "tool.completed", ] + def test_legacy_api_sessions_are_nested_idempotently(tmp_path: Path): """Known standalone API workers move under Cassandra without data loss.""" module_path = HERMES / "scripts" / "migrate_api_session_lineage.py" @@ -289,10 +409,15 @@ def test_legacy_api_sessions_are_nested_idempotently(tmp_path: Path): for orphan_id in module.LEGACY_ORPHANED_SMOKE_SESSIONS ) -def test_automated_triage_sessions_are_grouped_without_touching_interactive_runs(tmp_path: Path): + +def test_automated_triage_sessions_are_grouped_without_touching_interactive_runs( + tmp_path: Path, +): """Only the stable Ariadne contract moves below the triage parent.""" module_path = HERMES / "scripts" / "migrate_api_session_lineage.py" - spec = importlib.util.spec_from_file_location("migrate_triage_sessions", module_path) + spec = importlib.util.spec_from_file_location( + "migrate_triage_sessions", module_path + ) assert spec and spec.loader module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) @@ -351,42 +476,3 @@ def test_automated_triage_sessions_are_grouped_without_touching_interactive_runs module.TRIAGE_PARENT, "Sonar · soteria/291 · kins-run", ) - -def test_stale_parent_linked_api_workers_close_on_startup(tmp_path: Path): - """A previous gateway lifetime cannot leave phantom active workers.""" - module_path = HERMES / "scripts" / "migrate_api_session_lineage.py" - spec = importlib.util.spec_from_file_location("close_stale_api_workers", module_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - database = tmp_path / "state.db" - with sqlite3.connect(database) as connection: - connection.execute( - "CREATE TABLE sessions (id TEXT PRIMARY KEY, source TEXT, " - "parent_session_id TEXT, title TEXT, started_at REAL, ended_at REAL, " - "end_reason TEXT, archived INTEGER DEFAULT 0)" - ) - connection.executemany( - "INSERT INTO sessions " - "(id, source, parent_session_id, started_at, ended_at, end_reason) " - "VALUES (?, ?, ?, 1, ?, ?)", - ( - ("stale", "api_server", "parent", None, None), - ("root", "api_server", None, None, None), - ("finished", "api_server", "parent", 2.0, "api_run_completed"), - ("interactive", "tui", "parent", None, None), - ), - ) - - assert module.migrate(database) == 1 - assert module.migrate(database) == 0 - with sqlite3.connect(database) as connection: - rows = connection.execute( - "SELECT id, ended_at, end_reason FROM sessions ORDER BY id" - ).fetchall() - by_id = {row[0]: row[1:] for row in rows} - assert by_id["stale"][0] is not None - assert by_id["stale"][1] == "api_run_recovered_stale" - assert by_id["root"] == (None, None) - assert by_id["finished"] == (2.0, "api_run_completed") - assert by_id["interactive"] == (None, None) diff --git a/testing/tests/test_hermes_chat_support.py b/testing/tests/test_hermes_chat_support.py index 7f09512c..bf7d998c 100644 --- a/testing/tests/test_hermes_chat_support.py +++ b/testing/tests/test_hermes_chat_support.py @@ -1,38 +1,23 @@ -"""Contracts for isolated high-quality Hermes chat capabilities.""" +"""Shared paths and loaders for Hermes chat capability contracts.""" from __future__ import annotations -import base64 - import importlib.util - -import json - -import sqlite3 - import sys - -import time - -import tomllib - from pathlib import Path - -from types import ModuleType, SimpleNamespace - -import pytest +from types import SimpleNamespace import yaml ROOT = Path(__file__).parents[2] - HERMES = ROOT / "services" / "hermes" - VAULT = ROOT / "services" / "vault" + def _documents(path: Path) -> list[dict]: return [doc for doc in yaml.safe_load_all(path.read_text()) if doc] + def _load_broker_module(name: str, filename: str, monkeypatch): """Load one broker with its mounted routing-catalog dependency.""" catalog_path = HERMES / "scripts" / "routing_catalog.py" @@ -51,23 +36,3 @@ def _load_broker_module(name: str, filename: str, monkeypatch): module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module - -__all__ = ( - "HERMES", - "ModuleType", - "Path", - "ROOT", - "SimpleNamespace", - "VAULT", - "_documents", - "_load_broker_module", - "base64", - "importlib", - "json", - "pytest", - "sqlite3", - "sys", - "time", - "tomllib", - "yaml", -) diff --git a/testing/tests/test_hermes_chat_switchyard.py b/testing/tests/test_hermes_chat_switchyard.py deleted file mode 100644 index 6f8e3de7..00000000 --- a/testing/tests/test_hermes_chat_switchyard.py +++ /dev/null @@ -1,483 +0,0 @@ -"""Hermes chat switchyard contracts.""" - -from __future__ import annotations - -from test_hermes_chat_support import ( - HERMES, - Path, - SimpleNamespace, - _documents, - _load_broker_module, - base64, - importlib, - json, - pytest, - sys, - time, - tomllib, -) - - -def test_codex_broker_auth_and_request_contract(tmp_path: Path, monkeypatch): - """The relay is bounded, stateless, and rejects unapproved models.""" - module = _load_broker_module( - "hermes_codex_broker", "codex_broker.py", monkeypatch - ) - monkeypatch.setattr(module, "TOKEN", "relay-secret") - - assert module._authorized("Bearer relay-secret") is True - assert module._authorized("Bearer wrong") is False - assert module._real_model("route/codex/gpt-5.6-sol/xhigh") == "gpt-5.6-sol" - payload = module._validate_payload( - { - "model": "gpt-5.6-terra", - "input": "route this chat turn", - "store": True, - "stream": False, - "max_output_tokens": 96, - "max_completion_tokens": 96, - "max_tokens": 96, - "temperature": 0.7, - "top_p": 0.9, - } - ) - assert payload["store"] is False - assert payload["stream"] is True - assert "max_output_tokens" not in payload - assert "max_completion_tokens" not in payload - assert "max_tokens" not in payload - assert "temperature" not in payload - assert "top_p" not in payload - assert payload["input"] == [ - { - "type": "message", - "role": "user", - "content": [{"type": "input_text", "text": "route this chat turn"}], - } - ] - response_item = { - "type": "message", - "role": "user", - "content": [{"type": "input_text", "text": "keep this item"}], - } - assert module._validate_payload( - {"model": "gpt-5.6-terra", "input": response_item} - )["input"] == [response_item] - response_items = [response_item] - assert module._validate_payload( - {"model": "gpt-5.6-terra", "input": response_items} - )["input"] is response_items - image_items = [ - { - "type": "message", - "role": "user", - "content": [ - {"type": "input_text", "text": "What color is this?"}, - { - "type": "image_url", - "image_url": { - "url": "data:image/png;base64,cHJpdmF0ZQ==", - "detail": "high", - }, - }, - ], - } - ] - assert module._validate_payload( - {"model": "gpt-5.6-terra", "input": image_items} - )["input"][0]["content"][1] == { - "type": "input_image", - "image_url": "data:image/png;base64,cHJpdmF0ZQ==", - "detail": "high", - } - switchyard_image_items = [ - { - "role": "user", - "content": [ - {"type": "input_text", "text": "What color is this?"}, - { - "type": "image_url", - "image_url": { - "url": "data:image/png;base64,cHJpdmF0ZQ==", - "detail": "auto", - }, - }, - ], - } - ] - assert module._validate_payload( - {"model": "gpt-5.6-terra", "input": switchyard_image_items} - )["input"][0]["content"][1] == { - "type": "input_image", - "image_url": "data:image/png;base64,cHJpdmF0ZQ==", - "detail": "auto", - } - switchyard_base64_items = [ - { - "role": "user", - "content": [ - {"type": "input_text", "text": "What color is this?"}, - { - "type": "image", - "source": { - "type": "base64", - "media_type": "image/png", - "data": "cHJpdmF0ZQ==", - }, - }, - ], - } - ] - normalized_base64 = module._validate_payload( - {"model": "gpt-5.6-terra", "input": switchyard_base64_items} - )["input"][0]["content"][1] - assert normalized_base64 == { - "type": "input_image", - "image_url": "data:image/png;base64,cHJpdmF0ZQ==", - } - switchyard_enum_items = [ - { - "role": "user", - "content": [ - { - "type": "input_image", - "image_url": { - "type": "url", - "data": { - "url": "data:image/png;base64,cHJpdmF0ZQ==", - "detail": "high", - }, - }, - } - ], - } - ] - nested_image = module._validate_payload( - {"model": "gpt-5.6-terra", "input": switchyard_enum_items} - )["input"][0]["content"][0] - assert nested_image["image_url"] == "data:image/png;base64,cHJpdmF0ZQ==" - assert nested_image["detail"] == "high" - with pytest.raises(ValueError, match=r"non-empty Responses image URL.*str\[4\]"): - module._validate_payload( - { - "model": "gpt-5.6-terra", - "input": [ - { - "type": "message", - "role": "user", - "content": [ - { - "type": "input_image", - "image_url": {"detail": "high"}, - } - ], - } - ], - } - ) - routed = module._validate_payload( - { - "model": "route/codex/gpt-5.6-luna/low", - "input": "use the low route", - "stream": False, - } - ) - assert routed["model"] == "gpt-5.6-luna" - with pytest.raises(ValueError, match="unsupported Codex model"): - module._validate_payload({"model": "unapproved-model", "input": "hello"}) - with pytest.raises(ValueError, match="non-empty Responses input"): - module._validate_payload({"model": "gpt-5.6-terra", "input": ""}) - with pytest.raises(ValueError, match="non-empty Responses input list"): - module._validate_payload({"model": "gpt-5.6-terra", "input": []}) - - completed = { - "id": "resp_test", - "object": "response", - "status": "completed", - "output": [], - } - completed_item = { - "type": "message", - "role": "assistant", - "status": "completed", - "content": [{"type": "output_text", "text": "done"}], - } - assert module._completed_response( - [ - "event: response.created", - 'data: {"type":"response.created","response":{}}', - "event: response.output_item.done", - "data: " - + json.dumps( - { - "type": "response.output_item.done", - "output_index": 0, - "item": completed_item, - } - ), - "event: response.completed", - "data: " - + json.dumps({"type": "response.completed", "response": completed}), - "data: [DONE]", - ] - )["output"] == [completed_item] - raw_stream = ( - "event: response.output_item.done\n" - "data: " - + json.dumps( - { - "type": "response.output_item.done", - "output_index": 0, - "item": completed_item, - } - ) - + "\n\nevent: response.completed\ndata: " - + json.dumps({"type": "response.completed", "response": completed}) - + "\n\n" - ).encode() - normalized = module._normalized_stream( - raw_stream, {**completed, "output": [completed_item]} - ).decode() - terminal_data = next( - line for line in normalized.splitlines() if '"response.completed"' in line - ) - assert json.loads(terminal_data.removeprefix("data: "))["response"][ - "output" - ] == [completed_item] - assert normalized.endswith("\n\n") - streamed_function_item = { - "type": "function_call", - "name": "read_file", - "status": "completed", - "arguments": '{"path":"/tmp"}', - } - streamed_function_body = ( - "event: response.function_call_arguments.delta\n" - 'data: {"type":"response.function_call_arguments.delta",' - '"item_id":"call_1","delta":"{\\"path\\":\\"/tmp\\"}"}\n\n' - "event: response.function_call_arguments.done\n" - 'data: {"type":"response.function_call_arguments.done",' - '"item_id":"call_1","arguments":"{\\"path\\":\\"/tmp\\"}"}\n\n' - "event: response.output_item.done\n" - 'data: {"type":"response.output_item.done","output_index":0,' - '"item":{"type":"function_call","name":"read_file",' - '"arguments":"{\\"path\\":\\"/tmp\\"}"}}\n\n' - "event: response.completed\n" - "data: " - + json.dumps( - { - "type": "response.completed", - "response": {**completed, "output": [streamed_function_item]}, - } - ) - + "\n\n" - ).encode() - normalized_function_stream = module._normalized_stream( - streamed_function_body, {**completed, "output": [streamed_function_item]} - ).decode() - assert "response.function_call_arguments.delta" in normalized_function_stream - assert "response.function_call_arguments.done" not in normalized_function_stream - assert "response.output_item.done" not in normalized_function_stream - normalized_terminal = next( - line - for line in normalized_function_stream.splitlines() - if '"response.completed"' in line - ) - assert json.loads(normalized_terminal.removeprefix("data: "))["response"][ - "output" - ] == [] - with pytest.raises(RuntimeError, match="retryable incomplete response"): - module._completed_response( - [ - "event: response.incomplete", - 'data: {"type":"response.incomplete","response":' - '{"status":"incomplete","incomplete_details":' - '{"reason":"max_output_tokens"}}}', - ] - ) - with pytest.raises(RuntimeError, match="provider unavailable"): - module._completed_response( - [ - "event: error", - 'data: {"type":"error","error":{"message":"provider unavailable"}}', - ] - ) - malformed_tool_item = { - "type": "function_call", - "name": "search_files", - "status": "completed", - "arguments": '{"path":"","offset":', - } - with pytest.raises(RuntimeError, match="malformed function arguments"): - module._completed_response( - [ - "event: response.output_item.done", - "data: " - + json.dumps( - { - "type": "response.output_item.done", - "output_index": 0, - "item": malformed_tool_item, - } - ), - "event: response.completed", - "data: " - + json.dumps({"type": "response.completed", "response": completed}), - ] - ) - valid_tool_item = { - **malformed_tool_item, - "arguments": '{"path":"","offset":0}', - } - assert module._completed_response( - [ - "event: response.output_item.done", - "data: " - + json.dumps( - { - "type": "response.output_item.done", - "output_index": 0, - "item": valid_tool_item, - } - ), - "event: response.completed", - "data: " - + json.dumps({"type": "response.completed", "response": completed}), - ] - )["output"] == [valid_tool_item] - with pytest.raises(RuntimeError, match="malformed function arguments"): - module._completed_response( - [ - "event: response.function_call_arguments.delta", - 'data: {"type":"response.function_call_arguments.delta",' - '"item_id":"call_1","output_index":0,' - '"delta":"{\\"path\\":\\"/tmp\\",\\"offset\\":"}', - "event: response.completed", - "data: " - + json.dumps({"type": "response.completed", "response": completed}), - ] - ) - streamed_tool = module._completed_response( - [ - "event: response.function_call_arguments.delta", - 'data: {"type":"response.function_call_arguments.delta",' - '"item_id":"call_2","output_index":0,' - '"delta":"{\\"path\\":\\"/tmp\\",\\"offset\\":"}', - "event: response.function_call_arguments.done", - 'data: {"type":"response.function_call_arguments.done",' - '"item_id":"call_2","output_index":0,' - '"arguments":"{\\"path\\":\\"/tmp\\",\\"offset\\":0}"}', - "event: response.completed", - "data: " - + json.dumps({"type": "response.completed", "response": completed}), - ] - ) - assert streamed_tool["status"] == "completed" - - auth_dir = tmp_path / ".codex" - auth_dir.mkdir() - # The token payload need only prove the broker reads CODEX_HOME directly. - encoded = base64.urlsafe_b64encode( - json.dumps({"exp": time.time() + 3600}).encode() - ).decode().rstrip("=") - (auth_dir / "auth.json").write_text( - json.dumps({"tokens": {"access_token": f"header.{encoded}.signature"}}) - ) - monkeypatch.setenv("CODEX_HOME", str(auth_dir)) - assert module._access_token().startswith("header.") - -def test_classifier_broker_bounds_history_without_losing_routing_intent(monkeypatch): - """AUTO classification must fit the Jetson context without losing intent.""" - broker_path = HERMES / "scripts" / "classifier_broker.py" - monkeypatch.setitem(sys.modules, "httpx", SimpleNamespace()) - spec = importlib.util.spec_from_file_location("hermes_classifier_broker", broker_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - - payload = { - "model": "qwen2.5:14b-instruct-q4_0", - "messages": [ - { - "role": "system", - "content": "routing contract\n" + ("candidate policy " * 1000), - }, - { - "role": "user", - "content": "Build and verify the Cassandra release safely.", - }, - { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call_large", - "type": "function", - "function": { - "name": "large_tool", - "arguments": '{"command":"' + ("x" * 5000) + '"}', - }, - } - ], - }, - { - "role": "tool", - "content": "unbounded test output " * 10000, - "tool_call_id": "call_large", - }, - { - "role": "user", - "content": [ - {"type": "image_url", "image_url": {"url": "data:image/png;base64,private"}}, - {"type": "text", "text": "Turn this cat into a cute clown."}, - ], - }, - ], - "tools": [{"type": "function", "function": {"name": "large_tool"}}], - "tool_choice": "auto", - "parallel_tool_calls": True, - "response_format": {"type": "json_object"}, - } - - compacted = module.compact_payload(payload) - encoded = json.dumps(compacted) - - assert compacted["model"] == payload["model"] - assert compacted["response_format"] == payload["response_format"] - assert "tools" not in compacted - assert "tool_choice" not in compacted - assert "parallel_tool_calls" not in compacted - assert all(message.get("role") != "tool" for message in compacted["messages"]) - assert all("tool_call_id" not in message for message in compacted["messages"]) - assert all("tool_calls" not in message for message in compacted["messages"]) - assert "[tool evidence]" in encoded - assert "[assistant requested an external tool]" in encoded - assert "Build and verify the Cassandra release safely." in encoded - assert "Turn this cat into a cute clown." in encoded - assert "image attachment available to the selected worker" in encoded - assert "data:image/png;base64" not in encoded - assert "unbounded test output " * 100 not in encoded - assert len(encoded) < 18_000 - -def test_switchyard_classifier_is_bounded_and_fails_open_once(): - """A sick local judge must not hold chat through repeated long retries.""" - config = tomllib.loads( - _documents(HERMES / "switchyard-configmap.yaml")[0]["data"]["routes.toml"] - ) - classifier = config["llm_clients"]["classifier"] - assert classifier["base_url"] == "http://127.0.0.1:9008/v1" - assert classifier["max_retries"] == 0 - for route in ("auto_fast", "auto_balanced"): - assert config["routes"][route]["recent_turn_window"] == 4 - for route in ("auto_deep", "auto_maximum", "worker_auto_maximum"): - assert config["routes"][route]["recent_turn_window"] == 6 - - deployment = _documents(HERMES / "switchyard-deployment.yaml")[0] - containers = { - item["name"]: item - for item in deployment["spec"]["template"]["spec"]["containers"] - } - classifier_container = containers["classifier-broker"] - env = {item["name"]: item["value"] for item in classifier_container["env"]} - assert env["HERMES_CLASSIFIER_BROKER_READ_TIMEOUT"] == "60" - assert classifier_container["readinessProbe"]["httpGet"]["port"] == "classifier" diff --git a/testing/tests/test_hermes_chat_voice.py b/testing/tests/test_hermes_chat_voice.py deleted file mode 100644 index 180a17e4..00000000 --- a/testing/tests/test_hermes_chat_voice.py +++ /dev/null @@ -1,265 +0,0 @@ -"""Hermes chat voice contracts.""" - -from __future__ import annotations - -from test_hermes_chat_support import ( - HERMES, - ROOT, - SimpleNamespace, - VAULT, - _documents, - importlib, - sys, - yaml, -) - - -def test_webui_recovers_auth_and_labels_session_scoped_controls(): - dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-webui").read_text() - - assert "res.status===401||res.status===403" in dockerfile - assert "window.location.assign('/oauth2/start?rd='" in dockerfile - assert "childrenExpanded?'▾ ':'▸ '" in dockerfile - assert "'atlas/auto/maximum': 'Automatic · Maximum'" in dockerfile - assert "hermes-webui-telegram-project-patch.py" in dockerfile - telegram_project_patch = ( - ROOT / "dockerfiles" / "hermes-webui-telegram-project-patch.py" - ).read_text() - assert "TELEGRAM_PROJECT_NAME = 'Telegram'" in telegram_project_patch - assert "session_key.startswith('telegram-topic-')" in telegram_project_patch - - router = (ROOT / "dockerfiles" / "hermes-webui-router.js").read_text() - assert "'atlas/auto/fast':'AUTO · Fast'" in router - assert "'atlas/manual/codex/sol':'Codex · SOL'" in router - assert "'atlas/manual/claude/opus':'Claude · Opus'" in router - assert "watchModelOptions('modelSelect')" in router - assert "watchModelOptions('settingsModel')" in router - -def test_chat_voice_uses_private_jetson_services_and_shared_auto_route(): - statefulset = _documents(HERMES / "chat-statefulset.yaml")[0] - containers = statefulset["spec"]["template"]["spec"]["containers"] - webui = next(item for item in containers if item["name"] == "webui") - env = {item["name"]: item["value"] for item in webui["env"]} - - assert env["HERMES_STT_URL"] == ( - "http://hermes-stt.hermes.svc.cluster.local:9000/v1/audio/transcriptions" - ) - assert env["HERMES_WEBUI_ATLAS_TTS_URL"] == ( - "http://hermes-tts.hermes.svc.cluster.local:9001/v1/audio/speech" - ) - assert "hermes_stt_client.py" in env["HERMES_LOCAL_STT_COMMAND"] - - configmap = _documents(HERMES / "chat-configmap.yaml")[0] - config = yaml.safe_load(configmap["data"]["config.yaml"]) - assert config["stt"] == { - "enabled": True, - "provider": "local_command", - "local": {"model": "small", "language": "auto"}, - } - - dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-webui").read_text() - assert "hermes-webui-atlas-patch.py" in dockerfile - assert "hermes-webui-atlas-voice.js" in dockerfile - voice_script = (ROOT / "dockerfiles" / "hermes-webui-atlas-voice.js").read_text() - assert "/api/transcribe/capability" in voice_script - assert "/api/transcribe" in voice_script - assert "/api/tts" in voice_script - assert "speakResponse(generation)" in voice_script - assert "window._splitForTTS(text,280)" in voice_script - assert "pending=fetchSpeech(chunks[index+1])" in voice_script - assert "restartSoon(token,450)" in voice_script - assert "constraints.voiceIsolation=true" in voice_script - assert "highpass.frequency.value=140" in voice_script - assert "Math.max(0.04,noiseFloor*2.4+0.006)" in voice_script - assert "while(preRoll.length>3) preRoll.shift()" in voice_script - - stt_server = (ROOT / "dockerfiles" / "hermes-jetson-stt-server.py").read_text() - assert "def _repetitive_token" in stt_server - assert "compression_ratio_threshold=2.0" in stt_server - assert "no_speech_threshold=0.5" in stt_server - -def test_voice_transcript_filter_removes_fan_hallucinations(monkeypatch): - server_path = ROOT / "dockerfiles" / "hermes-jetson-stt-server.py" - spec = importlib.util.spec_from_file_location("hermes_jetson_stt_server", server_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - monkeypatch.setitem(sys.modules, "cgi", SimpleNamespace()) - monkeypatch.setitem( - sys.modules, - "torch", - SimpleNamespace(cuda=SimpleNamespace(is_available=lambda: False)), - ) - monkeypatch.setitem(sys.modules, "whisper", SimpleNamespace()) - spec.loader.exec_module(module) - - result = { - "segments": [ - { - "text": " ththththththththth Testing.", - "no_speech_prob": 0.12, - "avg_logprob": -0.2, - }, - { - "text": " background hum", - "no_speech_prob": 0.82, - "avg_logprob": -0.9, - }, - ] - } - - assert module._clean_transcript(result) == "Testing." - -def test_voice_models_are_baked_and_runtime_has_no_public_egress(): - stt_dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-jetson-stt").read_text() - tts_dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-jetson-tts").read_text() - assert "ADD --checksum=sha256:aff26ae4" in stt_dockerfile - assert "ADD --checksum=sha256:9ecf7799" in stt_dockerfile - assert "--chmod=0444" in stt_dockerfile - assert "chmod 0555 /opt/models /opt/models/whisper" in stt_dockerfile - assert "HERMES_STT_CACHE=/opt/models/whisper" in stt_dockerfile - assert "ADD --checksum=sha256:4cabf7c3" in tts_dockerfile - assert "ADD --checksum=sha256:db42b97d" in tts_dockerfile - assert tts_dockerfile.count("--chmod=0444") == 6 - assert "chmod 0555 /opt/models /opt/models/piper" in tts_dockerfile - assert "HERMES_TTS_CACHE=/opt/models/piper" in tts_dockerfile - tts_server = (ROOT / "dockerfiles" / "hermes-jetson-tts-server.py").read_text() - assert "download_voice" not in tts_server - assert "baked Piper voice is missing" in tts_server - assert "session_options.intra_op_num_threads = ONNX_THREADS" in tts_server - - policies = _documents(HERMES / "networkpolicy.yaml") - voice_policy = next( - item for item in policies if item["metadata"]["name"] == "hermes-private-voice" - ) - assert voice_policy["spec"]["policyTypes"] == ["Ingress", "Egress"] - assert not any( - "ipBlock" in destination - for rule in voice_policy["spec"]["egress"] - for destination in rule.get("to", []) - ) - -def test_voice_workloads_have_deliberate_xavier_placement(): - documents = _documents(HERMES / "voice-deployment.yaml") - deployments = { - item["metadata"]["name"]: item - for item in documents - if item["kind"] == "Deployment" - } - stt = deployments["hermes-stt"]["spec"]["template"]["spec"] - tts = deployments["hermes-tts"]["spec"]["template"]["spec"] - - assert "@sha256:" in stt["containers"][0]["image"] - assert "@sha256:" in tts["containers"][0]["image"] - assert stt["nodeSelector"] == {"kubernetes.io/hostname": "titan-21"} - assert tts["nodeSelector"] == {"kubernetes.io/hostname": "titan-21"} - assert stt["automountServiceAccountToken"] is False - assert tts["automountServiceAccountToken"] is False - assert stt["enableServiceLinks"] is False - assert tts["enableServiceLinks"] is False - assert stt["runtimeClassName"] == "nvidia" - assert stt["securityContext"]["supplementalGroups"] == [44] - stt_resources = stt["containers"][0]["resources"] - stt_env = { - item["name"]: item["value"] for item in stt["containers"][0]["env"] - } - assert stt_env["NVIDIA_DRIVER_CAPABILITIES"] == "compute,utility" - assert stt_resources["requests"]["nvidia.com/gpu.shared"] == 1 - assert stt_resources["limits"]["nvidia.com/gpu.shared"] == 1 - assert "nvidia.com/gpu.shared" not in tts["containers"][0]["resources"]["requests"] - tts_env = { - item["name"]: item["value"] for item in tts["containers"][0]["env"] - } - assert tts_env["HERMES_TTS_VOICE"] == "en_US-lessac-medium" - assert tts_env["HERMES_TTS_ONNX_THREADS"] == "2" - assert tts["containers"][0]["resources"]["limits"]["cpu"] == "4" - assert all("hostPath" not in volume for volume in stt["volumes"]) - assert all("hostPath" not in volume for volume in tts["volumes"]) - -def test_chat_image_generation_uses_private_owner_broker(): - """Family pods get image bytes without receiving the owner's OAuth file.""" - configmap = _documents(HERMES / "chat-configmap.yaml")[0] - assert "shared desktop/Wolf lane" in configmap["data"]["SOUL.md"] - assert "local FLUX waits" in configmap["data"]["SOUL.md"] - assert "Use `image_generate_local`" in configmap["data"]["SOUL.md"] - assert "Use `image_generate_hosted`" in configmap["data"]["SOUL.md"] - assert "ComfyUI endpoint" in configmap["data"]["SOUL.md"] - assert "`MEDIA:` path" in configmap["data"]["SOUL.md"] - assert "`image_edit_latest`" in configmap["data"]["SOUL.md"] - assert "Preserve the most recently selected image lane" in configmap["data"]["SOUL.md"] - config = yaml.safe_load(configmap["data"]["config.yaml"]) - assert config["image_gen"] == { - "provider": "atlas-broker", - "model": "atlas-image-auto-high", - } - assert config["plugins"]["enabled"] == ["atlas-broker", "auto-router"] - - statefulset = _documents(HERMES / "chat-statefulset.yaml")[0] - pod = statefulset["spec"]["template"]["spec"] - hermes = next(item for item in pod["containers"] if item["name"] == "hermes") - mounts = {item["name"]: item for item in hermes["volumeMounts"]} - assert mounts["image-plugin"]["mountPath"] == ( - "/opt/hermes/plugins/image_gen/atlas-broker" - ) - assert mounts["runtime-access"]["mountPath"] == "/runtime-access" - assert "provider-auth" not in mounts - assert not any(mount["name"] == "home" and "agent" in str(mount) for mount in hermes["volumeMounts"]) - env = {item["name"]: item["value"] for item in hermes["env"]} - assert env["HERMES_IMAGE_BROKER_URL"].startswith("http://hermes-image-broker.") - - plugin = (HERMES / "plugins" / "image-gen-broker" / "__init__.py").read_text() - assert '"local": "flux-2-klein-4b-local"' in plugin - assert '"hosted": "gpt-image-2-high"' in plugin - assert 'name="image_generate_local"' in plugin - assert 'name="image_generate_hosted"' in plugin - assert '"name": "image_edit_latest"' in plugin - assert '"name": "image_edit_latest_local"' in plugin - assert '"name": "image_edit_latest_hosted"' in plugin - assert "def _latest_generated_image" in plugin - assert "newest MEDIA: path from the conversation" in plugin - assert 'candidate.upper().startswith("MEDIA:")' in plugin - assert "override=True" not in plugin - - agent = _documents(HERMES / "agent-deployment.yaml")[0] - containers = agent["spec"]["template"]["spec"]["containers"] - broker = next(item for item in containers if item["name"] == "image-broker") - assert broker["ports"] == [ - {"name": "image-broker", "containerPort": 9002, "protocol": "TCP"} - ] - assert broker["securityContext"]["readOnlyRootFilesystem"] is True - assert broker["securityContext"]["runAsNonRoot"] is True - - services = _documents(HERMES / "service.yaml") - service = next( - item for item in services if item["metadata"]["name"] == "hermes-image-broker" - ) - assert service["spec"]["selector"] == {"app": "hermes-agent"} - - oauth_store = _documents(HERMES / "oauth-session-store.yaml") - redis = next(item for item in oauth_store if item["kind"] == "Deployment") - assert redis["spec"]["strategy"]["type"] == "Recreate" - assert "--appendonly" in redis["spec"]["template"]["spec"]["containers"][0]["args"] - - policies = _documents(HERMES / "networkpolicy.yaml") - agent_policy = next( - item for item in policies if item["metadata"]["name"] == "hermes-agent-isolation" - ) - broker_ingress = next( - rule - for rule in agent_policy["spec"]["ingress"] - if {port["port"] for port in rule["ports"]} == {9002, 9003} - ) - assert broker_ingress["from"][0]["podSelector"]["matchLabels"] == { - "app": "hermes-chat-tenant" - } - vault_policy = (VAULT / "scripts" / "vault_k8s_auth_configure.sh").read_text() - assert ( - '"hermes/agent-oidc hermes/agent-tokens hermes/chat-telegram ' - 'hermes/developer-keycloak hermes/developer-gitea ' - 'hermes/developer-harbor hermes/developer-jenkins ' - 'hermes/developer-ssh"' - in vault_policy - ) - assert ( - 'write_policy_and_role "hermes-node-ssh" "hermes" ' - '"hermes-node-ssh-access"' in vault_policy - ) diff --git a/testing/tests/test_hermes_cli_lanes_access.py b/testing/tests/test_hermes_cli_lanes_access.py new file mode 100644 index 00000000..9e5a8352 --- /dev/null +++ b/testing/tests/test_hermes_cli_lanes_access.py @@ -0,0 +1,381 @@ +"""Network, OAuth, Kubernetes, and SSH access contracts for Hermes CLI lanes.""" + +from __future__ import annotations + + +import yaml + +from testing.tests.test_hermes_cli_lanes_support import ( + FLUX_HERMES, + HERMES, + KEYCLOAK, + ROOT, + _agent_deployment, + _services, +) + + +def test_broker_services_survive_sibling_container_readiness_loss(): + services = _services() + + for name in ( + "hermes-image-broker", + "hermes-codex-broker", + "hermes-local-image", + "hermes-claude-broker", + ): + assert services[name]["spec"]["publishNotReadyAddresses"] is True + + +def test_agent_dashboard_reconnects_all_transient_websockets(): + dockerfile = (HERMES.parents[1] / "dockerfiles/Dockerfile.hermes-agent").read_text( + encoding="utf-8" + ) + assert "eventsRetryAttempt.current" in dockerfile + assert "if (!unmounting) setVersion((v) => v + 1);" in dockerfile + assert "events feed rejected (${ev.code}) — reload the page" in dockerfile + assert 'url = await api.buildWsUrl("/api/pty", params);' in dockerfile + assert 'url = await buildWsUrl("/api/events", { channel });' in dockerfile + assert dockerfile.count("' await api.getSessions(1, 0,") == 2 + assert ".then(() => gw.connect())" in dockerfile + assert 'api.getSessions(1, 0, profile ?? "")' in dockerfile + assert "dashboard token rotated by a server restart" in dockerfile + + +def test_agent_refreshes_routes_after_restoring_cli_logins(): + deployment = _agent_deployment() + init_containers = { + item["name"]: item + for item in deployment["spec"]["template"]["spec"]["initContainers"] + } + configure = init_containers["configure-agent-clients"] + command = configure["command"][-1] + assert "configure_agent_clients.py" in command + assert command.index("configure_agent_clients.py") < command.index( + "hermes_coordinator.py --once" + ) + env = {item["name"]: item["value"] for item in configure["env"]} + assert env["HERMES_AUTH_FILE"] == "/runtime-access/hermes-auth.json" + assert env["PYTHONPATH"] == "/opt/hermes" + for name in ("bootstrap-coordinator", "configure-agent-clients"): + route_env = { + item["name"]: item["value"] for item in init_containers[name]["env"] + } + assert route_env["CODEX_HOME"] == "/runtime-access/codex" + assert route_env["CLAUDE_CONFIG_DIR"] == "/runtime-access/claude" + assert "/opt/data/tools/bin" in route_env["PATH"] + + assert "patch-web-session-activity" in init_containers + web_patch = init_containers["patch-web-session-activity"] + assert "/opt/coordinator/patch_web_session_activity.py" in web_patch["command"] + + containers = { + item["name"]: item + for item in deployment["spec"]["template"]["spec"]["containers"] + } + steward_env = { + item["name"]: item["value"] for item in containers["model-steward"]["env"] + } + assert steward_env["CODEX_HOME"] == "/runtime-access/codex" + assert steward_env["CLAUDE_CONFIG_DIR"] == "/runtime-access/claude" + assert "/opt/data/tools/bin" in steward_env["PATH"] + hermes_mounts = { + (item["name"], item["mountPath"], item.get("subPath")) + for item in containers["hermes"]["volumeMounts"] + } + assert ( + "web-server-patch", + "/opt/hermes/hermes_cli/web_server.py", + "web_server.py", + ) in hermes_mounts + + +def test_flux_health_checks_follow_the_owner_oauth_sidecar(): + flux = yaml.safe_load(FLUX_HERMES.read_text()) + checks = {(item["kind"], item["name"]) for item in flux["spec"]["healthChecks"]} + assert ("Deployment", "hermes-agent") in checks + assert ("DaemonSet", "hermes-node-ssh-access") in checks + assert ("Deployment", "oauth2-proxy-hermes-agent") not in checks + + +def test_agent_auth_is_bstein_group_and_email_bounded(): + deployment = _agent_deployment() + oauth = next( + item + for item in deployment["spec"]["template"]["spec"]["containers"] + if item["name"] == "oauth2-proxy" + ) + args = oauth["args"] + assert "--user-id-claim=sub" in args + assert "--oidc-groups-claim=groups" in args + assert "--allowed-group=/hermes-owner" in args + assert "--authenticated-emails-file=/etc/oauth2-proxy/allowed-emails" in args + script = (KEYCLOAK / "scripts/hermes_access_oidc_ensure.sh").read_text() + assert 'group_name="hermes-owner"' in script + assert "username=bstein&exact=true" in script + assert '"full.path":"true"' in script + + +def test_agent_network_boundary_allows_only_authenticated_and_metrics_surfaces(): + documents = [ + item + for item in yaml.safe_load_all((HERMES / "networkpolicy.yaml").read_text()) + if item + ] + isolation = next( + item + for item in documents + if item.get("metadata", {}).get("name") == "hermes-agent-isolation" + ) + assert isolation["spec"]["ingress"] == [ + { + "from": [ + { + "namespaceSelector": { + "matchLabels": {"kubernetes.io/metadata.name": "traefik"} + }, + "podSelector": { + "matchLabels": {"app.kubernetes.io/name": "traefik"} + }, + } + ], + "ports": [{"protocol": "TCP", "port": 4180}], + }, + { + "from": [{"podSelector": {"matchLabels": {"app": "hermes-chat-tenant"}}}], + "ports": [ + {"protocol": "TCP", "port": 9002}, + {"protocol": "TCP", "port": 9003}, + ], + }, + { + "from": [{"podSelector": {"matchLabels": {"app": "hermes-switchyard"}}}], + "ports": [ + {"protocol": "TCP", "port": 9003}, + {"protocol": "TCP", "port": 9006}, + ], + }, + { + "from": [ + { + "namespaceSelector": { + "matchLabels": {"kubernetes.io/metadata.name": "monitoring"} + }, + "podSelector": {"matchLabels": {"app": "server"}}, + } + ], + "ports": [ + {"protocol": "TCP", "port": 9010}, + {"protocol": "TCP", "port": 9011}, + ], + }, + ] + egress = isolation["spec"]["egress"] + assert any( + rule.get("ports") == [{"protocol": "TCP", "port": 9081}] for rule in egress + ) + assert "hermes-scm" in yaml.safe_dump(egress) + assert "gitea" in yaml.safe_dump(egress) + assert egress != [{}] + + +def test_owner_agent_has_scoped_read_only_kubernetes_context(): + config = yaml.safe_load((HERMES / "agent-kubeconfig.yaml").read_text()) + assert config["current-context"] == "atlas-observer" + assert config["contexts"][0]["context"]["namespace"] == "default" + rbac_path = ROOT / "services/hermes-observer-rbac/rbac.yaml" + documents = [item for item in yaml.safe_load_all(rbac_path.read_text()) if item] + binding = next(item for item in documents if item["kind"] == "ClusterRoleBinding") + assert binding["roleRef"] == { + "apiGroup": "rbac.authorization.k8s.io", + "kind": "ClusterRole", + "name": "hermes-agent-cluster-observer-v2", + } + assert binding["subjects"] == [ + {"kind": "ServiceAccount", "name": "hermes-agent", "namespace": "hermes"} + ] + + +def test_owner_agent_has_pinned_dedicated_node_ssh_access(): + deployment = _agent_deployment() + annotations = deployment["spec"]["template"]["metadata"]["annotations"] + assert ( + annotations["vault.hashicorp.com/agent-inject-secret-node-ssh-private-key"] + == "kv/data/atlas/hermes/developer-ssh" + ) + assert ( + annotations["vault.hashicorp.com/agent-inject-secret-node-ssh-config"] + == "kv/data/atlas/hermes/developer-ssh" + ) + assert ( + annotations["vault.hashicorp.com/agent-inject-secret-node-ssh-known-hosts"] + == "kv/data/atlas/hermes/developer-ssh" + ) + + init = next( + item + for item in deployment["spec"]["template"]["spec"]["initContainers"] + if item["name"] == "init-config" + ) + command = init["command"][2] + assert "ln -s /runtime-access/node-ssh-config /opt/data/home/.ssh/config" in command + assert ( + "ln -s /runtime-access/node-ssh-known-hosts /opt/data/home/.ssh/known_hosts" + in command + ) + assert "ln -s home/.ssh /opt/data/.ssh" in command + assert "chmod 0700 /opt/data/home/.ssh" in command + assert ( + "ln -s /runtime-access/node-ssh-private-key " + "/opt/data/home/.ssh/id_ed25519_atlas_nodes" + ) in command + + config = yaml.safe_load((HERMES / "agent-configmap.yaml").read_text())["data"] + assert "ssh_config" not in config + assert "ssh_known_hosts" not in config + assert "ssh-ed25519" not in (HERMES / "agent-configmap.yaml").read_text() + + resources = yaml.safe_load((HERMES / "kustomization.yaml").read_text())["resources"] + assert "node-ssh-access.yaml" in resources + access = [ + item + for item in yaml.safe_load_all((HERMES / "node-ssh-access.yaml").read_text()) + if item + ] + service_account = next(item for item in access if item["kind"] == "ServiceAccount") + assert service_account["metadata"]["name"] == "hermes-node-ssh-access" + provider = next(item for item in access if item["kind"] == "SecretProviderClass") + assert provider["metadata"]["name"] == "hermes-node-ssh-access" + assert provider["spec"]["provider"] == "vault" + parameters = provider["spec"]["parameters"] + assert parameters["roleName"] == "hermes-node-ssh" + assert 'secretPath: "kv/data/atlas/hermes/developer-ssh"' in parameters["objects"] + assert 'secretKey: "public_key"' in parameters["objects"] + daemonset = next(item for item in access if item["kind"] == "DaemonSet") + pod = daemonset["spec"]["template"]["spec"] + assert pod["serviceAccountName"] == "hermes-node-ssh-access" + assert pod["automountServiceAccountToken"] is True + host_home = next(item for item in pod["volumes"] if item["name"] == "host-home") + assert host_home["hostPath"] == {"path": "/home", "type": "Directory"} + vault_secrets = next( + item for item in pod["volumes"] if item["name"] == "vault-secrets" + ) + assert vault_secrets["csi"]["driver"] == "secrets-store.csi.k8s.io" + assert vault_secrets["csi"]["volumeAttributes"] == { + "secretProviderClass": "hermes-node-ssh-access" + } + reconciler = pod["containers"][0]["args"][0] + assert "/opt/node-hardener/node_account_hardening.py" in reconciler + assert "--public-key-file /vault/secrets/node-ssh-public-key" in reconciler + assert "sleep 300" in reconciler + host_etc = next(item for item in pod["volumes"] if item["name"] == "host-etc") + assert host_etc["hostPath"] == {"path": "/etc", "type": "Directory"} + hardener = (HERMES / "scripts/node_account_hardening.py").read_text() + assert 'ACCOUNT = "hermes-agent"' in hardener + assert 'LEGACY_ACCOUNTS = ("atlas", "oceanus")' in hardener + assert "ACCOUNT_UID = 1200" in hardener + assert "ACCOUNT_GID = 1200" in hardener + + +def test_owner_agent_tracks_no_ssh_identity_or_host_key_material(): + """Vault references may be tracked; SSH identities and trust data may not.""" + forbidden = ( + "BEGIN OPENSSH PRIVATE KEY", + "ssh-ed25519 AAAA", + "ssh-rsa AAAA", + "IdentityFile ", + "UserKnownHostsFile ", + "StrictHostKeyChecking ", + "ssh_config:", + "ssh_known_hosts:", + ) + text_suffixes = { + ".conf", + ".json", + ".md", + ".py", + ".sh", + ".toml", + ".yaml", + ".yml", + } + tracked = "\n".join( + path.read_text(encoding="utf-8") + for path in HERMES.rglob("*") + if path.is_file() and path.suffix in text_suffixes + ) + for marker in forbidden: + assert marker not in tracked + + +def test_switchyard_has_a_dedicated_non_owner_identity_and_read_only_catalog(): + """Routing must not inherit the owner agent's cluster-admin capability.""" + service_accounts = [ + item + for item in yaml.safe_load_all( + (HERMES / "vault-serviceaccount.yaml").read_text() + ) + if item + ] + assert any( + item["kind"] == "ServiceAccount" + and item["metadata"]["name"] == "hermes-switchyard" + for item in service_accounts + ) + + switchyard = yaml.safe_load((HERMES / "switchyard-deployment.yaml").read_text()) + switchyard_pod = switchyard["spec"]["template"]["spec"] + assert switchyard_pod["serviceAccountName"] == "hermes-switchyard" + agent_pod = _agent_deployment()["spec"]["template"]["spec"] + for pod, container_name in ( + (switchyard_pod, "worker-route-broker"), + (agent_pod, "claude-broker"), + ): + container = next( + item for item in pod["containers"] if item["name"] == container_name + ) + catalog = next( + item + for item in container["volumeMounts"] + if item["mountPath"] == "/routing-catalog" + ) + assert catalog["readOnly"] is True + + rbac = [ + item + for item in yaml.safe_load_all( + (ROOT / "services/hermes-observer-rbac/rbac.yaml").read_text() + ) + if item + ] + binding = next(item for item in rbac if item["kind"] == "ClusterRoleBinding") + assert binding["subjects"] == [ + {"kind": "ServiceAccount", "name": "hermes-agent", "namespace": "hermes"} + ] + + +def test_switchyard_active_state_uses_a_relocatable_rwx_claim(): + """A stale node attachment must not strand the routing authority.""" + claims = [ + item + for item in yaml.safe_load_all((HERMES / "switchyard-pvc.yaml").read_text()) + if item + ] + active_claim = next( + item + for item in claims + if item["metadata"]["name"] == "hermes-switchyard-state-rwx" + ) + assert active_claim["spec"]["accessModes"] == ["ReadWriteMany"] + + deployment = yaml.safe_load((HERMES / "switchyard-deployment.yaml").read_text()) + strategy = deployment["spec"]["strategy"] + assert strategy == { + "type": "RollingUpdate", + "rollingUpdate": {"maxSurge": 1, "maxUnavailable": 0}, + } + pod = deployment["spec"]["template"]["spec"] + state = next(item for item in pod["volumes"] if item["name"] == "state") + assert ( + state["persistentVolumeClaim"]["claimName"] == active_claim["metadata"]["name"] + ) diff --git a/testing/tests/test_hermes_cli_lanes_support.py b/testing/tests/test_hermes_cli_lanes_support.py new file mode 100644 index 00000000..242351b9 --- /dev/null +++ b/testing/tests/test_hermes_cli_lanes_support.py @@ -0,0 +1,79 @@ +"""Shared loaders and manifest helpers for Hermes CLI-lane contracts.""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +import yaml + +ROOT = Path(__file__).parents[2] +SCRIPTS = ROOT / "services/hermes/scripts" +sys.path.insert(0, str(SCRIPTS)) +HERMES = ROOT / "services/hermes" +KEYCLOAK = ROOT / "services/keycloak" +FLUX_HERMES = ROOT / "clusters/atlas/flux-system/applications/hermes/kustomization.yaml" + + +def _load(name: str): + spec = importlib.util.spec_from_file_location(name, SCRIPTS / f"{name}.py") + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +lanes = _load("cli_lane_runner") +policy = _load("claude_command_policy") +migration = _load("migrate_herdr_state") +auth_patch = _load("patch_hermes_auth") +tui_gateway_patch = _load("patch_tui_gateway") +codex_runtime_patch = _load("patch_codex_runtime") +ttyd_patch = _load("patch_ttyd_index") +client_config = _load("configure_agent_clients") + + +def _agent_deployment() -> dict: + return yaml.safe_load((HERMES / "agent-deployment.yaml").read_text()) + + +def _services() -> dict[str, dict]: + return { + item["metadata"]["name"]: item + for item in yaml.safe_load_all((HERMES / "service.yaml").read_text()) + if item + } + + +def _oauth_deployment(name: str) -> dict: + documents = [ + item + for item in yaml.safe_load_all((HERMES / "oauth2-proxy.yaml").read_text()) + if item + ] + return next( + item + for item in documents + if item["kind"] == "Deployment" and item["metadata"]["name"] == name + ) + + +class _SwitchyardResponse: + """Minimal context-managed response used by routing contract tests.""" + + def __init__(self, selected: str, rationale: str = "local classifier vote"): + self.headers = { + "x-model-router-selected-model": selected, + "x-model-router-rationale": rationale, + } + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def read(self): + return b"{}" diff --git a/testing/tests/test_hermes_cli_lanes_toolchain.py b/testing/tests/test_hermes_cli_lanes_toolchain.py new file mode 100644 index 00000000..2ed232cb --- /dev/null +++ b/testing/tests/test_hermes_cli_lanes_toolchain.py @@ -0,0 +1,386 @@ +"""Operator toolchain and runtime-patch contracts for Hermes CLI lanes.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +import yaml + +from testing.tests.test_hermes_cli_lanes_support import ( + HERMES, + SCRIPTS, + _agent_deployment, + auth_patch, + client_config, + codex_runtime_patch, + tui_gateway_patch, + ttyd_patch, +) + + +def test_worker_route_broker_accepts_pod_network_health_checks(): + """Kubelet probes the pod IP, so the broker cannot bind to loopback only.""" + script = (SCRIPTS / "worker_route_broker.py").read_text() + assert 'ThreadingHTTPServer(("0.0.0.0", PORT), Handler)' in script + + +def test_switchyard_network_boundary_allows_vault_bootstrap(): + """The pre-populate init container must reach Vault before routing starts.""" + documents = [ + item + for item in yaml.safe_load_all((HERMES / "networkpolicy.yaml").read_text()) + if item + ] + isolation = next( + item + for item in documents + if item.get("metadata", {}).get("name") == "hermes-switchyard-isolation" + ) + assert any( + rule.get("to") + == [ + { + "namespaceSelector": { + "matchLabels": {"kubernetes.io/metadata.name": "vault"} + }, + "podSelector": {"matchLabels": {"app": "vault"}}, + } + ] + and rule.get("ports") == [{"protocol": "TCP", "port": 8200}] + for rule in isolation["spec"]["egress"] + ) + + +def test_switchyard_network_boundary_allows_metrics_scraping(): + """VictoriaMetrics may scrape Switchyard without widening its API boundary.""" + documents = [ + item + for item in yaml.safe_load_all((HERMES / "networkpolicy.yaml").read_text()) + if item + ] + isolation = next( + item + for item in documents + if item.get("metadata", {}).get("name") == "hermes-switchyard-isolation" + ) + assert any( + rule.get("from") + == [ + { + "namespaceSelector": { + "matchLabels": {"kubernetes.io/metadata.name": "monitoring"} + }, + "podSelector": {"matchLabels": {"app": "server"}}, + } + ] + and rule.get("ports") == [{"protocol": "TCP", "port": 9005}] + for rule in isolation["spec"]["ingress"] + ) + + +def test_owner_agent_installs_the_pinned_operator_toolchain(): + script = (SCRIPTS / "install_agent_tools.sh").read_text() + for value in [ + "flux", + "helm", + "kustomize", + "jq", + "yq", + "gh", + "vault", + "sops", + "age", + "age-keygen", + "k9s", + "terraform", + "go", + "gofmt", + ]: + assert value in script + assert "go1.26.5.linux-arm64.tar.gz" in script + assert "fe4789e92b1f33358680864bbe8704289e7bb5fc207d80623c308935bd696d49" in script + assert script.count("sha256sum -c -") == 1 + + deployment = _agent_deployment() + installer = next( + item + for item in deployment["spec"]["template"]["spec"]["initContainers"] + if item["name"] == "install-agent-tools" + ) + assert "/bin/sh /opt/coordinator/install_agent_tools.sh" in installer["command"][2] + assert any(mount["name"] == "coordinator" for mount in installer["volumeMounts"]) + init_config = next( + item + for item in deployment["spec"]["template"]["spec"]["initContainers"] + if item["name"] == "init-config" + ) + init_command = init_config["command"][2] + assert "# Hermes managed operator PATH." in init_command + assert "/opt/coordinator:/opt/data/tools/bin:/opt/hermes/.venv/bin" in init_command + assert 'chmod 0644 "${profile_file}"' in init_command + + +def test_owner_agent_uses_only_the_canonical_hostname(): + paths = [ + HERMES / "agent-configmap.yaml", + HERMES / "agent-deployment.yaml", + HERMES / "agent-ingress.yaml", + Path(__file__).parents[2] / "scripts/ops/hermes_triage_monitor.py", + ] + for path in paths: + content = path.read_text() + assert "agent.bstein.dev" not in content + assert "agent.hermes.bstein.dev" in content + + +def test_agent_reconnect_retains_complete_history_and_long_tool_budget(): + configmap = yaml.safe_load((HERMES / "agent-configmap.yaml").read_text()) + config = yaml.safe_load(configmap["data"]["config.yaml"]) + display = config["display"] + assert display["resume_exchanges"] >= 10000 + assert display["resume_max_user_chars"] >= 10000000 + assert display["resume_max_assistant_chars"] >= 10000000 + assert config["agent"]["max_turns"] == 180 + assert config["delegation"]["max_iterations"] == 120 + + +def test_auth_patch_honors_explicit_shared_store(tmp_path: Path): + source = tmp_path / "auth.py" + destination = tmp_path / "patched/auth.py" + source.write_text( + 'from pathlib import Path\nimport os\n\ndef _auth_file_path() -> Path:\n path = get_hermes_home() / "auth.json"\n return path\n', + encoding="utf-8", + ) + auth_patch.patch(source, destination) + content = destination.read_text() + assert 'os.environ.get("HERMES_AUTH_FILE"' in content + + +def test_auth_patch_fails_closed_on_upstream_drift(tmp_path: Path): + source = tmp_path / "auth.py" + source.write_text("def changed():\n pass\n", encoding="utf-8") + with pytest.raises(RuntimeError, match="context changed"): + auth_patch.patch(source, tmp_path / "patched.py") + + +def test_codex_runtime_patch_uses_cli_and_forwards_route(tmp_path: Path): + provider = tmp_path / "runtime_provider.py" + provider.write_text(codex_runtime_patch.PROVIDER_BEFORE, encoding="utf-8") + provider_out = tmp_path / "patched/runtime_provider.py" + codex_runtime_patch.patch_provider(provider, provider_out) + assert '"api_mode": "codex_app_server"' in provider_out.read_text() + + session = tmp_path / "codex_app_server_session.py" + session.write_text( + codex_runtime_patch.SESSION_SIGNATURE_BEFORE + + codex_runtime_patch.SESSION_REQUEST_BEFORE, + encoding="utf-8", + ) + session_out = tmp_path / "patched/codex_app_server_session.py" + codex_runtime_patch.patch_session(session, session_out) + session_content = session_out.read_text() + assert 'turn_params["model"] = model' in session_content + assert 'turn_params["effort"] = effort' in session_content + assert '"approvalPolicy": "never"' in session_content + assert '"sandboxPolicy": {"type": "dangerFullAccess"}' in session_content + + turn = tmp_path / "codex_runtime.py" + turn.write_text( + codex_runtime_patch.FALLBACK_CONTEXT_BEFORE + codex_runtime_patch.TURN_BEFORE, + encoding="utf-8", + ) + turn_out = tmp_path / "patched/codex_runtime.py" + codex_runtime_patch.patch_turn(turn, turn_out) + turn_content = turn_out.read_text() + assert "model=str(getattr(agent" in turn_content + assert "build_cross_provider_codex_prompt" in turn_content + + fallback = tmp_path / "chat_completion_helpers.py" + fallback.write_text( + codex_runtime_patch.FALLBACK_RESOLUTION_BEFORE, + encoding="utf-8", + ) + fallback_out = tmp_path / "patched/chat_completion_helpers.py" + codex_runtime_patch.patch_fallback(fallback, fallback_out) + fallback_content = fallback_out.read_text() + assert 'agent.api_mode = "codex_app_server"' in fallback_content + assert "agent._codex_cross_provider_fallback = True" in fallback_content + + loop = tmp_path / "conversation_loop.py" + loop.write_text( + codex_runtime_patch.FALLBACK_DISPATCH_BEFORE + + codex_runtime_patch.RETRY_FALLBACK_DISPATCH_BEFORE + + codex_runtime_patch.STREAM_RECOVERY_BEFORE, + encoding="utf-8", + ) + loop_out = tmp_path / "patched/conversation_loop.py" + codex_runtime_patch.patch_loop(loop, loop_out) + loop_content = loop_out.read_text() + assert loop_content.count('if agent.api_mode == "codex_app_server"') == 2 + assert "build_cross_provider_codex_prompt" in loop_content + retry_dispatch = loop_content.index( + "Fallback activation happens inside this retry loop" + ) + api_kwargs = loop_content.find("agent._build_api_kwargs", retry_dispatch) + assert api_kwargs == -1 or retry_dispatch < api_kwargs + assert "Provider stream ended before a complete response" in loop_content + assert "_is_transport_stub" in loop_content + assert "rerouting ({truncated_tool_call_retries}/4)" in loop_content + + auxiliary = tmp_path / "auxiliary_client.py" + auxiliary.write_text( + codex_runtime_patch.AUXILIARY_TOKEN_BEFORE, + encoding="utf-8", + ) + auxiliary_out = tmp_path / "patched/auxiliary_client.py" + codex_runtime_patch.patch_auxiliary(auxiliary, auxiliary_out) + auxiliary_content = auxiliary_out.read_text() + assert 'os.environ.get("CODEX_HOME"' in auxiliary_content + assert 'Path(codex_home).expanduser() / "auth.json"' in auxiliary_content + assert "never creates a metered API-key lane" in auxiliary_content + + +def test_agent_mounts_codex_auxiliary_runtime_patch(): + deployment = _agent_deployment() + pod = deployment["spec"]["template"]["spec"] + patch_init = next( + item for item in pod["initContainers"] if item["name"] == "patch-codex-runtime" + ) + assert patch_init["command"][-2:] == [ + "/opt/hermes/agent/auxiliary_client.py", + "/patched/auxiliary_client.py", + ] + expected_mount = { + "name": "codex-runtime-patch", + "mountPath": "/opt/hermes/agent/auxiliary_client.py", + "subPath": "auxiliary_client.py", + } + containers = {item["name"]: item for item in pod["containers"]} + for name in ("hermes", "terminal"): + assert expected_mount in containers[name]["volumeMounts"] + + +def test_codex_auxiliary_patch_reads_cli_token_without_copying_it( + tmp_path: Path, + monkeypatch, +): + codex_home = tmp_path / ".codex" + codex_home.mkdir() + (codex_home / "auth.json").write_text( + json.dumps({"tokens": {"access_token": "cli-access-token"}}), + encoding="utf-8", + ) + monkeypatch.setenv("CODEX_HOME", str(codex_home)) + + source = tmp_path / "auxiliary_client.py" + source.write_text( + "import json, logging, os, time\n" + "from pathlib import Path\n" + "logger = logging.getLogger(__name__)\n" + "def read_token():\n" + " try:\n" + " raise RuntimeError('Hermes provider store intentionally empty')\n" + + codex_runtime_patch.AUXILIARY_TOKEN_BEFORE, + encoding="utf-8", + ) + destination = tmp_path / "patched/auxiliary_client.py" + codex_runtime_patch.patch_auxiliary(source, destination) + namespace: dict = {} + exec(compile(destination.read_text(), str(destination), "exec"), namespace) + + assert namespace["read_token"]() == "cli-access-token" + + +def test_codex_runtime_patch_fails_closed_on_upstream_drift(tmp_path: Path): + source = tmp_path / "runtime_provider.py" + source.write_text("def changed():\n pass\n", encoding="utf-8") + with pytest.raises(RuntimeError, match="context changed"): + codex_runtime_patch.patch_provider(source, tmp_path / "patched.py") + + +def test_codex_runtime_migration_uses_owner_unsafe_mode(tmp_path: Path): + config = tmp_path / "config.yaml" + config.write_text("model: {}\n", encoding="utf-8") + codex_home = tmp_path / ".codex" + calls = [] + + class Report: + errors = [] + + @staticmethod + def summary(): + return "configured" + + def migrate(value, **kwargs): + calls.append((value, kwargs)) + return Report() + + client_config.configure_codex_runtime(config, migrate, codex_home) + + assert calls[0][1]["default_permission_profile"] is None + assert calls[0][1]["codex_home"] == codex_home + content = (codex_home / "config.toml").read_text(encoding="utf-8") + assert 'approval_policy = "never"' in content + assert 'sandbox_mode = "danger-full-access"' in content + assert "default_permissions" not in content + + +def test_codex_owner_permissions_replace_stale_profile(tmp_path: Path): + config = tmp_path / "config.toml" + config.write_text( + 'default_permissions = ":danger-no-sandbox"\n\n[features]\nhooks = true\n', + encoding="utf-8", + ) + + client_config.configure_codex_owner_permissions(config) + client_config.configure_codex_owner_permissions(config) + + content = config.read_text(encoding="utf-8") + assert content.count(client_config.OWNER_PERMISSIONS_BEGIN) == 1 + assert content.count('approval_policy = "never"') == 1 + assert content.count('sandbox_mode = "danger-full-access"') == 1 + assert "default_permissions" not in content + assert "[features]\nhooks = true" in content + + +def test_tui_gateway_patch_extends_and_bounds_agent_startup(tmp_path: Path): + source = tmp_path / "server.py" + destination = tmp_path / "patched/server.py" + source.write_text( + "import os\n\n" + tui_gateway_patch.BEFORE + "\ndef unchanged():\n pass\n", + encoding="utf-8", + ) + + tui_gateway_patch.patch(source, destination) + + content = destination.read_text(encoding="utf-8") + assert "HERMES_TUI_AGENT_INIT_TIMEOUT_S" in content + assert "configured = 180.0" in content + assert "return max(30.0, min(configured, 900.0))" in content + assert "timeout: float | None = None" in content + assert "ready.wait(timeout=wait_timeout)" in content + assert "def unchanged():" in content + + +def test_tui_gateway_patch_fails_closed_on_upstream_drift(tmp_path: Path): + source = tmp_path / "server.py" + source.write_text("def changed():\n pass\n", encoding="utf-8") + with pytest.raises(RuntimeError, match="context changed"): + tui_gateway_patch.patch(source, tmp_path / "patched.py") + + +def test_ttyd_clipboard_and_reconnect_patch_remain_enabled(): + source = '' + content = ttyd_patch.patch_html(source) + assert 'id="atlas-ttyd-clipboard"' in content + assert "navigator.clipboard.writeText(text)" in content + assert "class AtlasRecoveringWebSocket" in content + assert "window.location.reload()" in content + assert "event.stopImmediatePropagation()" in content + + +def test_ttyd_patch_fails_closed_on_upstream_drift(): + with pytest.raises(RuntimeError, match="context changed"): + ttyd_patch.patch_html("changed") diff --git a/testing/tests/test_hermes_coordinator_routing.py b/testing/tests/test_hermes_coordinator.py similarity index 88% rename from testing/tests/test_hermes_coordinator_routing.py rename to testing/tests/test_hermes_coordinator.py index ec93322a..683c0201 100644 --- a/testing/tests/test_hermes_coordinator_routing.py +++ b/testing/tests/test_hermes_coordinator.py @@ -1,20 +1,22 @@ -"""Hermes coordinator model routing contracts.""" +"""Unit tests for Hermes coordinator model routing and profile generation.""" from __future__ import annotations -from test_hermes_coordinator_support import ( - Path, +import json +import subprocess +import tomllib +from pathlib import Path + +import yaml + +from testing.tests.test_hermes_coordinator_support import ( SCRIPT, _base_config, catalog_resolver, - json, + coordinator, routing, - subprocess, - tomllib, - yaml, ) - def test_model_version_and_quality_selection_handle_new_and_small_models(): assert routing.model_version("claude-3-5-sonnet-20241022") == (3, 5) assert routing.model_version("gpt-5.7-terra") == (5, 7) @@ -34,10 +36,12 @@ def test_model_version_and_quality_selection_handle_new_and_small_models(): assert routing.choose_claude_for_effort(claude, "medium") == "claude-sonnet-5" assert routing.choose_claude_for_effort(claude, "xhigh") == "claude-opus-4.8" + def test_empty_catalog_retains_current_models(): assert routing.choose_codex_model([], "gpt-5.6-terra") == "gpt-5.6-terra" assert routing.choose_claude_model([], "claude-opus-5") == "claude-opus-5" + def test_dynamic_catalog_resolves_new_models_and_preserves_last_known_good(): """Stable Switchyard aliases follow live releases and survive catalog outages.""" codex = routing.Catalog( @@ -84,6 +88,7 @@ def test_dynamic_catalog_resolves_new_models_and_preserves_last_known_good(): == current["providers"]["claude"]["tiers"] ) + def test_live_catalog_never_routes_to_a_removed_model_tier(): """A live provider catalog must replace a retired tier with a live model.""" previous = { @@ -115,6 +120,7 @@ def test_live_catalog_never_routes_to_a_removed_model_tier(): assert current["providers"]["codex"]["tiers"]["luna"] in codex.models assert current["providers"]["claude"]["tiers"]["opus"] == "claude-sonnet-6" + def test_switchyard_targets_have_unique_upstream_identities(): """Switchyard drops duplicate client/model pairs, so reject them in Git.""" manifest = yaml.safe_load( @@ -133,6 +139,7 @@ def test_switchyard_targets_have_unique_upstream_identities(): if route.get("target"): assert route["target"] in target_names + def test_worker_alias_resolves_before_cli_launch(): document = { "providers": { @@ -147,6 +154,7 @@ def test_worker_alias_resolves_before_cli_launch(): == "worker/codex/gpt-5.8-sol/high" ) + def test_codex_cli_login_counts_as_connected_runtime(monkeypatch): """AUTO routing must recognize the authenticated app-server CLI lane.""" monkeypatch.setattr(routing.shutil, "which", lambda name: "/usr/bin/codex") @@ -163,6 +171,7 @@ def test_codex_cli_login_counts_as_connected_runtime(monkeypatch): assert routing.codex_cli_authenticated() is True + def test_configure_routes_keeps_every_profile_on_switchyard(tmp_path: Path): (tmp_path / "config.yaml").write_text( yaml.safe_dump(_base_config()), encoding="utf-8" @@ -237,6 +246,7 @@ def test_configure_routes_keeps_every_profile_on_switchyard(tmp_path: Path): assert "claude-secret" not in json.dumps(routes) assert (tmp_path / "profiles/codex-high/.env").stat().st_mode & 0o777 == 0o600 + def test_degraded_catalog_does_not_replace_switchyard_authority(tmp_path: Path): base = _base_config() base["model"]["model"] = base["model"]["default"] = "gpt-5.6-sol" @@ -252,6 +262,7 @@ def test_degraded_catalog_does_not_replace_switchyard_authority(tmp_path: Path): assert current["model"]["provider"] == "atlas-switchyard" assert current["fallback_providers"] == [] + def test_degraded_refresh_preserves_switchyard_worker_preference(tmp_path: Path): """A catalog outage must not bypass a managed worker's Switchyard route.""" (tmp_path / "config.yaml").write_text( @@ -276,3 +287,26 @@ def test_degraded_refresh_preserves_switchyard_worker_preference(tmp_path: Path) ) assert worker["model"]["model"] == "atlas/manual/codex/sol" assert worker["model"]["provider"] == "atlas-switchyard" + + +def test_refresh_writes_non_secret_routing_status(tmp_path: Path, monkeypatch): + (tmp_path / "config.yaml").write_text( + yaml.safe_dump(_base_config()), encoding="utf-8" + ) + (tmp_path / ".env").write_text("GITEA_TOKEN=do-not-report\n", encoding="utf-8") + codex = routing.Catalog("openai-codex", ["gpt-5.6-terra"], True, True, "connected") + claude = routing.Catalog("anthropic", ["claude-opus-5"], True, True, "connected") + monkeypatch.setattr(coordinator, "discover_codex_models", lambda: codex) + monkeypatch.setattr(coordinator, "discover_claude_models", lambda: claude) + monkeypatch.setattr( + coordinator, "bootstrap_cassandra_state", lambda root: {"state": "ready"} + ) + monkeypatch.setattr(coordinator, "sync_cassandra_repo", lambda env: "ready") + + status = coordinator.refresh_once(tmp_path) + + status_path = tmp_path / "workspace/coordinator/model-routing.json" + assert status_path.is_file() + assert status["projects"]["cassandra"]["state"] == "ready" + assert status["projects"]["cassandra"]["board_state"] == {"state": "ready"} + assert "do-not-report" not in status_path.read_text(encoding="utf-8") diff --git a/testing/tests/test_hermes_coordinator_boards.py b/testing/tests/test_hermes_coordinator_cassandra.py similarity index 78% rename from testing/tests/test_hermes_coordinator_boards.py rename to testing/tests/test_hermes_coordinator_cassandra.py index aa63943f..646db291 100644 --- a/testing/tests/test_hermes_coordinator_boards.py +++ b/testing/tests/test_hermes_coordinator_cassandra.py @@ -1,39 +1,14 @@ -"""Hermes coordinator board recovery contracts.""" +"""Cassandra workspace and board contracts for the Hermes coordinator.""" from __future__ import annotations -from test_hermes_coordinator_support import ( - Path, - _base_config, - coordinator, - pytest, - routing, - subprocess, - yaml, -) +import subprocess +from pathlib import Path +import pytest -def test_refresh_writes_non_secret_routing_status(tmp_path: Path, monkeypatch): - (tmp_path / "config.yaml").write_text( - yaml.safe_dump(_base_config()), encoding="utf-8" - ) - (tmp_path / ".env").write_text("GITEA_TOKEN=do-not-report\n", encoding="utf-8") - codex = routing.Catalog("openai-codex", ["gpt-5.6-terra"], True, True, "connected") - claude = routing.Catalog("anthropic", ["claude-opus-5"], True, True, "connected") - monkeypatch.setattr(coordinator, "discover_codex_models", lambda: codex) - monkeypatch.setattr(coordinator, "discover_claude_models", lambda: claude) - monkeypatch.setattr( - coordinator, "bootstrap_cassandra_state", lambda root: {"state": "ready"} - ) - monkeypatch.setattr(coordinator, "sync_cassandra_repo", lambda env: "ready") +from testing.tests.test_hermes_coordinator_support import coordinator - status = coordinator.refresh_once(tmp_path) - - status_path = tmp_path / "workspace/coordinator/model-routing.json" - assert status_path.is_file() - assert status["projects"]["cassandra"]["state"] == "ready" - assert status["projects"]["cassandra"]["board_state"] == {"state": "ready"} - assert "do-not-report" not in status_path.read_text(encoding="utf-8") def test_corrupt_cassandra_board_is_preserved_without_blocking_refresh( tmp_path: Path, monkeypatch @@ -64,6 +39,7 @@ def test_corrupt_cassandra_board_is_preserved_without_blocking_refresh( "reason": "row out of order", } + def test_unrelated_cassandra_bootstrap_failure_remains_fatal( tmp_path: Path, monkeypatch ): @@ -87,6 +63,7 @@ def test_unrelated_cassandra_bootstrap_failure_remains_fatal( else: raise AssertionError("unrelated board errors must fail the coordinator refresh") + def test_cassandra_workspace_uses_valid_configured_worktree( tmp_path: Path, monkeypatch ): @@ -101,6 +78,7 @@ def test_cassandra_workspace_uses_valid_configured_worktree( (active / ".git").unlink() assert coordinator.cassandra_workspace() == coordinator.CASSANDRA_BASE_PATH + @pytest.mark.parametrize( ("current_remote", "repair_action"), [ @@ -140,8 +118,14 @@ def test_cassandra_sync_repairs_existing_worktree( assert commands[-1][-4:] == ["fetch", "--quiet", "--prune", "origin"] assert all("GITEA_TOKEN" not in environment for environment in environments) -def test_cassandra_sync_repairs_origin_without_token(tmp_path: Path, monkeypatch): - """Missing credentials skip only the fetch, not the local origin repair.""" + +def test_cassandra_sync_repairs_origin_through_broker_without_token( + tmp_path: Path, monkeypatch +): + """Repository sync uses the credential-isolated broker remote.""" + monkeypatch.setenv( + "HERMES_GITEA_TOKEN_FILE", str(tmp_path / "explicitly-unavailable-token") + ) workspace = tmp_path / "cassandra" (workspace / ".git").mkdir(parents=True) monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", workspace) @@ -156,8 +140,14 @@ def test_cassandra_sync_repairs_origin_without_token(tmp_path: Path, monkeypatch monkeypatch.setattr(coordinator.subprocess, "run", run) state = coordinator.sync_cassandra_repo({}) - assert state == "ready; fetch skipped until Gitea token is configured" - assert [command[-3] for command in commands] == ["remote", "set-url"] + assert state == "ready" + assert any( + "hermes-scm-broker.hermes-scm.svc.cluster.local" in " ".join(cmd) + for cmd in commands + ) + assert all("token" not in " ".join(cmd).lower() for cmd in commands) + assert commands[-1][-4:] == ["fetch", "--quiet", "--prune", "origin"] + def test_migrate_open_cassandra_tasks_preserves_running_and_done_tasks(): """A project switch moves queued work without relocating active evidence.""" diff --git a/testing/tests/test_hermes_coordinator_coverage.py b/testing/tests/test_hermes_coordinator_coverage.py new file mode 100644 index 00000000..fe4f3168 --- /dev/null +++ b/testing/tests/test_hermes_coordinator_coverage.py @@ -0,0 +1,267 @@ +"""Behavioral branch coverage for Hermes project coordination.""" + +from __future__ import annotations + +import json +import subprocess +import sys +import types +from contextlib import nullcontext +from pathlib import Path + +import pytest + +from testing.tests.test_hermes_coordinator_support import coordinator, routing + + +def test_workspace_falls_back_when_unset_or_invalid(tmp_path: Path, monkeypatch): + monkeypatch.delenv("HERMES_CASSANDRA_ACTIVE_WORKTREE", raising=False) + assert coordinator.cassandra_workspace() == coordinator.CASSANDRA_BASE_PATH + monkeypatch.setenv("HERMES_CASSANDRA_ACTIVE_WORKTREE", str(tmp_path / "missing")) + assert coordinator.cassandra_workspace() == coordinator.CASSANDRA_BASE_PATH + + +def test_task_migration_is_noop_for_base_workspace(): + class Board: + @staticmethod + def connect_closing(**_kwargs): + raise AssertionError("base workspace must not access Kanban") + + coordinator._migrate_open_cassandra_tasks(Board, coordinator.CASSANDRA_BASE_PATH) + + +class _Project: + id = "project-id" + + +def _fake_hermes_modules(monkeypatch, *, existing: bool, active: bool): + events = [] + kb = types.ModuleType("hermes_cli.kanban_db") + kb.board_exists = lambda slug: existing + kb.create_board = lambda *args, **kwargs: events.append( + ("create-board", args, kwargs) + ) + kb.set_current_board = lambda slug: events.append(("current", slug)) + kb.connect_closing = lambda **kwargs: nullcontext(object()) + kb.list_tasks = lambda *_a, **_k: [] + + pdb = types.ModuleType("hermes_cli.projects_db") + pdb.connect_closing = lambda: nullcontext(object()) + pdb.get_project = lambda *_a: _Project() if existing else None + pdb.get_active_id = lambda *_a: "active" if active else None + pdb.create_project = ( + lambda *args, **kwargs: events.append(("create-project", kwargs)) or "new-id" + ) + pdb.set_active = lambda *args: events.append(("active", args[-1])) + pdb.update_project = lambda *args, **kwargs: events.append(("update", kwargs)) + pdb.add_folder = lambda *args, **kwargs: events.append(("folder", args[-1], kwargs)) + + package = types.ModuleType("hermes_cli") + package.kanban_db = kb + package.projects_db = pdb + monkeypatch.setitem(sys.modules, "hermes_cli", package) + monkeypatch.setitem(sys.modules, "hermes_cli.kanban_db", kb) + monkeypatch.setitem(sys.modules, "hermes_cli.projects_db", pdb) + return events + + +@pytest.mark.parametrize( + ("existing", "active", "expected"), + [ + (False, False, {"current", "create-project", "active"}), + (False, True, {"current", "create-project"}), + (True, True, {"update", "folder"}), + ], +) +def test_bootstrap_cassandra_creates_or_updates_state( + tmp_path: Path, monkeypatch, existing, active, expected +): + events = _fake_hermes_modules(monkeypatch, existing=existing, active=active) + base = tmp_path / "projects/cassandra" + worktree = tmp_path / "projects/cassandra-worktree" + worktree.mkdir(parents=True) + (worktree / ".git").write_text("gitdir: elsewhere\n", encoding="utf-8") + monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", base) + monkeypatch.setenv("HERMES_CASSANDRA_ACTIVE_WORKTREE", str(worktree)) + coordinator.bootstrap_cassandra(tmp_path / "hermes") + names = {event[0] for event in events} + assert expected <= names + assert "create-board" in names + + +def test_bootstrap_state_ready_path(tmp_path: Path, monkeypatch): + monkeypatch.setattr(coordinator, "bootstrap_cassandra", lambda _root: None) + assert coordinator.bootstrap_cassandra_state(tmp_path) == {"state": "ready"} + + +def test_corruption_type_imports_pinned_exception(monkeypatch): + class Expected(Exception): + pass + + package = types.ModuleType("hermes_cli") + kanban = types.ModuleType("hermes_cli.kanban_db") + kanban.KanbanDbCorruptError = Expected + package.kanban_db = kanban + monkeypatch.setitem(sys.modules, "hermes_cli", package) + monkeypatch.setitem(sys.modules, "hermes_cli.kanban_db", kanban) + assert coordinator._kanban_corruption_type() is Expected + + +def test_repo_sync_reports_missing_git_and_unmanaged_directory( + tmp_path: Path, monkeypatch +): + monkeypatch.setattr(coordinator.shutil, "which", lambda _name: None) + assert coordinator.sync_cassandra_repo({}) == "git-unavailable" + + workspace = tmp_path / "cassandra" + workspace.mkdir() + (workspace / "user-file").write_text("preserve", encoding="utf-8") + monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", workspace) + monkeypatch.setattr(coordinator.shutil, "which", lambda _name: "/usr/bin/git") + assert coordinator.sync_cassandra_repo({}) == "unmanaged-nonempty-directory" + + +@pytest.mark.parametrize( + ("failure", "expected"), + [ + ("get-url-error", "remote-repair-failed"), + ("repair-exit", "remote-repair-failed-9"), + ("repair-error", "remote-repair-failed"), + ("fetch-exit", "sync-failed-8"), + ("fetch-error", "sync-failed"), + ], +) +def test_existing_repo_sync_reports_each_failure( + tmp_path: Path, monkeypatch, failure, expected +): + workspace = tmp_path / "cassandra" + (workspace / ".git").mkdir(parents=True) + monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", workspace) + monkeypatch.setattr(coordinator.shutil, "which", lambda _name: "/usr/bin/git") + + def run(command, **_kwargs): + action = command[-2:] if len(command) >= 2 else [] + if action == ["get-url", "origin"]: + if failure == "get-url-error": + raise OSError("git unavailable") + return subprocess.CompletedProcess(command, 0, stdout="old\n") + if "set-url" in command: + if failure == "repair-error": + raise subprocess.TimeoutExpired(command, 30) + return subprocess.CompletedProcess( + command, 9 if failure == "repair-exit" else 0 + ) + if failure == "fetch-error": + raise OSError("fetch unavailable") + return subprocess.CompletedProcess(command, 8 if failure == "fetch-exit" else 0) + + monkeypatch.setattr(coordinator.subprocess, "run", run) + assert coordinator.sync_cassandra_repo({}) == expected + + +@pytest.mark.parametrize( + ("exit_code", "expected"), [(0, "ready"), (7, "sync-failed-7")] +) +def test_empty_repo_clones_through_broker( + tmp_path: Path, monkeypatch, exit_code, expected +): + workspace = tmp_path / "cassandra" + monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", workspace) + monkeypatch.setattr(coordinator.shutil, "which", lambda _name: "/usr/bin/git") + seen = [] + + def run(command, **kwargs): + seen.append((command, kwargs["env"])) + return subprocess.CompletedProcess(command, exit_code) + + monkeypatch.setattr(coordinator.subprocess, "run", run) + assert ( + coordinator.sync_cassandra_repo( + {"API_SERVER_KEY": "not-forwarded", "SAFE": "yes"} + ) + == expected + ) + assert coordinator.CASSANDRA_REMOTE in seen[0][0] + assert seen[0][1]["SAFE"] == "yes" + assert "API_SERVER_KEY" not in seen[0][1] + + +def _catalog(provider): + return routing.Catalog(provider, ["model"], True, True, "connected") + + +def test_refresh_once_writes_complete_status(tmp_path: Path, monkeypatch): + monkeypatch.setattr(coordinator, "_read_env", lambda _path: {"SAFE": "yes"}) + monkeypatch.setattr(coordinator, "discover_codex_models", lambda: _catalog("codex")) + monkeypatch.setattr( + coordinator, "discover_claude_models", lambda: _catalog("claude") + ) + monkeypatch.setattr( + coordinator, "configure_routes", lambda *_a: {"coordinator": ["auto"]} + ) + monkeypatch.setattr( + coordinator, "bootstrap_cassandra_state", lambda _root: {"state": "ready"} + ) + monkeypatch.setattr(coordinator, "sync_cassandra_repo", lambda _env: "ready") + monkeypatch.setattr( + coordinator, "cassandra_workspace", lambda: tmp_path / "worktree" + ) + written = [] + monkeypatch.setattr( + coordinator, "_atomic_write", lambda path, value: written.append((path, value)) + ) + status = coordinator.refresh_once(tmp_path) + assert status["projects"]["cassandra"]["state"] == "ready" + assert json.loads(written[0][1])["providers"]["codex"]["connected"] is True + + +def test_main_once_success_and_failure(monkeypatch, capsys): + status = {"providers": {"codex": {"state": "ready"}}} + monkeypatch.setattr(sys, "argv", ["coordinator", "--once"]) + monkeypatch.setattr(coordinator, "refresh_once", lambda _root: status) + assert coordinator.main() == 0 + assert "codex=ready" in capsys.readouterr().out + monkeypatch.setattr( + coordinator, + "refresh_once", + lambda _root: (_ for _ in ()).throw(RuntimeError("failed")), + ) + assert coordinator.main() == 1 + assert "RuntimeError" in capsys.readouterr().out + + +@pytest.mark.parametrize(("interval", "expected"), [(1, 300), (400, 400)]) +def test_main_loop_sleeps_with_floor(monkeypatch, interval, expected): + calls = iter(({"providers": {}}, KeyboardInterrupt())) + + def refresh(_root): + value = next(calls) + if isinstance(value, BaseException): + raise value + return value + + sleeps = [] + monkeypatch.setattr(coordinator, "refresh_once", refresh) + monkeypatch.setattr(coordinator.time, "sleep", sleeps.append) + monkeypatch.setattr( + sys, "argv", ["coordinator", "--loop", "--interval", str(interval)] + ) + with pytest.raises(KeyboardInterrupt): + coordinator.main() + assert sleeps == [expected] + + +def test_main_loop_recovers_from_refresh_failure(monkeypatch): + calls = iter((RuntimeError("temporary"), KeyboardInterrupt())) + + def refresh(_root): + error = next(calls) + raise error + + sleeps = [] + monkeypatch.setattr(coordinator, "refresh_once", refresh) + monkeypatch.setattr(coordinator.time, "sleep", sleeps.append) + monkeypatch.setattr(sys, "argv", ["coordinator", "--loop", "--interval", "300"]) + with pytest.raises(KeyboardInterrupt): + coordinator.main() + assert sleeps == [300] diff --git a/testing/tests/test_hermes_coordinator_support.py b/testing/tests/test_hermes_coordinator_support.py index 6f58ea5b..2a41f070 100644 --- a/testing/tests/test_hermes_coordinator_support.py +++ b/testing/tests/test_hermes_coordinator_support.py @@ -1,41 +1,23 @@ -"""Unit tests for Hermes coordinator model routing and profile generation.""" +"""Shared loader and configuration for Hermes coordinator contracts.""" from __future__ import annotations +import importlib import importlib.util - -import json - -import subprocess - import sys - -import tomllib - from pathlib import Path -import pytest - -import yaml - SCRIPT = Path(__file__).parents[2] / "services/hermes/scripts/hermes_coordinator.py" - sys.path.insert(0, str(SCRIPT.parent)) - routing = importlib.import_module("hermes_model_routing") - catalog_resolver = importlib.import_module("routing_catalog") - SPEC = importlib.util.spec_from_file_location("hermes_coordinator", SCRIPT) - assert SPEC and SPEC.loader - coordinator = importlib.util.module_from_spec(SPEC) - sys.modules[SPEC.name] = coordinator - SPEC.loader.exec_module(coordinator) + def _base_config() -> dict: """Return the minimal coordinator configuration used by routing tests.""" return { @@ -51,20 +33,3 @@ def _base_config() -> dict: "toolsets": ["kanban"], "terminal": {"cwd": "/opt/data/workspace"}, } - -__all__ = ( - "Path", - "SCRIPT", - "SPEC", - "_base_config", - "catalog_resolver", - "coordinator", - "importlib", - "json", - "pytest", - "routing", - "subprocess", - "sys", - "tomllib", - "yaml", -) diff --git a/testing/tests/test_hermes_deadline_http.py b/testing/tests/test_hermes_deadline_http.py new file mode 100644 index 00000000..45fd07e4 --- /dev/null +++ b/testing/tests/test_hermes_deadline_http.py @@ -0,0 +1,309 @@ +"""Absolute-deadline contracts for the killable HTTP control helper.""" + +from __future__ import annotations + +import base64 +import io +import json +import subprocess +import threading +import time +import urllib.request +from http.server import BaseHTTPRequestHandler, HTTPServer + +import pytest + +from testing.tests.test_hermes_scm_broker_support import _load + + +class _Handler(BaseHTTPRequestHandler): + def _respond(self): + if self.path == "/slow": + time.sleep(10) + if self.path == "/redirect": + self.send_response(302) + self.send_header("Location", "http://127.0.0.1:1/evil") + self.send_header("Content-Length", "0") + self.end_headers() + return + status = 404 if self.path == "/missing" else 200 + body = b"x" * 64 if self.path == "/large" else b'{"ok":true}' + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + do_GET = _respond + do_POST = _respond + + def log_message(self, *_args): + return + + +@pytest.fixture(scope="module") +def server(): + value = HTTPServer(("127.0.0.1", 0), _Handler) + thread = threading.Thread(target=value.serve_forever, daemon=True) + thread.start() + yield f"http://127.0.0.1:{value.server_port}" + value.shutdown() + + +def _control(url: str, **overrides): + value = { + "url": url, + "method": "GET", + "headers": {"Accept": "application/json"}, + "body": "", + "maximum": 1024, + "timeout": 10, + } + value.update(overrides) + return value + + +def test_child_exchange_round_trips_and_reports_http_errors(server): + module = _load("deadline_http") + result = module._child_exchange(_control(server + "/ok")) + assert (result.status, result.content_type) == (200, "application/json") + assert result.body == b'{"ok":true}' + + posted = module._child_exchange( + _control( + server + "/ok", + method="POST", + body=base64.b64encode(b'{"a":1}').decode(), + ) + ) + assert posted.status == 200 + + missing = module._child_exchange(_control(server + "/missing")) + assert missing.status == 404 + + +def test_child_exchange_rejects_redirects_and_oversized_responses(server): + module = _load("deadline_http") + with pytest.raises(Exception): + module._child_exchange(_control(server + "/redirect")) + with pytest.raises(ValueError, match="too large"): + module._child_exchange(_control(server + "/large", maximum=8)) + + +@pytest.mark.parametrize( + "overrides", + [ + {"url": 1}, + {"method": None}, + {"headers": ["not", "dict"]}, + {"headers": {"Accept": 5}}, + {"body": b"raw"}, + {"maximum": "big"}, + {"maximum": -1}, + {"maximum": 4 * 1024 * 1024}, + {"timeout": "soon"}, + {"timeout": 0}, + {"timeout": 121}, + ], +) +def test_child_exchange_rejects_invalid_control_documents(overrides): + module = _load("deadline_http") + control = _control("http://127.0.0.1:1/") + control.update(overrides) + with pytest.raises(ValueError, match="invalid control"): + module._child_exchange(control) + + +def test_child_exchange_rejects_bad_or_oversized_encoded_bodies(): + module = _load("deadline_http") + with pytest.raises(ValueError): + module._child_exchange(_control("http://127.0.0.1:1/", body="!!not-b64!!")) + huge = base64.b64encode(b"x" * (module.MAX_CONTROL_BYTES + 1)).decode() + with pytest.raises(ValueError, match="request too large"): + module._child_exchange(_control("http://127.0.0.1:1/", body=huge)) + + +def test_read_response_enforces_deadline_and_size(): + module = _load("deadline_http") + + class Reader: + def __init__(self, body: bytes): + self.stream = io.BytesIO(body) + + def read(self, limit: int): + return self.stream.read(limit) + + with pytest.raises(TimeoutError): + module._read_response(Reader(b"abc"), 10, time.monotonic() - 1) + with pytest.raises(ValueError, match="too large"): + module._read_response(Reader(b"abcdef"), 4, time.monotonic() + 5) + value = module._read_response(Reader(b"abc"), 10, time.monotonic() + 5) + assert value == b"abc" + + +def _run_child_main(module, monkeypatch, raw: bytes) -> dict: + stdin = type("Stdin", (), {"buffer": io.BytesIO(raw)})() + stdout = io.StringIO() + monkeypatch.setattr(module.sys, "stdin", stdin) + monkeypatch.setattr(module.sys, "stdout", stdout) + code = module._child_main() + return code, json.loads(stdout.getvalue()) + + +def test_child_main_reports_success_and_failure_documents(monkeypatch, server): + module = _load("deadline_http") + code, output = _run_child_main( + module, monkeypatch, json.dumps(_control(server + "/ok")).encode() + ) + assert code == 0 and output["ok"] is True + assert base64.b64decode(output["body"]) == b'{"ok":true}' + + for raw in ( + b"x" * (module.MAX_CONTROL_BYTES + 1), + b"not json", + b"[1,2]", + ): + code, output = _run_child_main(module, monkeypatch, raw) + assert code == 1 and output == {"ok": False} + + +class _FakeProcess: + def __init__(self, output: bytes, returncode: int = 0): + self.args = ["fake-child"] + self.pid = 0 + self.returncode = returncode + self.output = output + self.stdin_payload = None + + def communicate(self, payload=None, timeout=None): + self.stdin_payload = payload + return self.output, b"" + + +def _fake_popen(output: bytes, returncode: int = 0): + def popen(*_args, **_kwargs): + return _FakeProcess(output, returncode) + + return popen + + +def _success_output(body: bytes, **overrides) -> bytes: + value = { + "ok": True, + "status": 200, + "content_type": "application/json", + "body": base64.b64encode(body).decode("ascii"), + } + value.update(overrides) + return json.dumps(value).encode() + + +def test_exchange_validates_bounds_and_request_body(): + module = _load("deadline_http") + request = urllib.request.Request("https://scm.bstein.dev/api/v1/x") + with pytest.raises(module.PolicyError, match="bounds are invalid"): + module.exchange(request, maximum=1, timeout=0) + with pytest.raises(module.PolicyError, match="bounds are invalid"): + module.exchange(request, maximum=-1, timeout=10) + request.data = "not-bytes" + with pytest.raises(module.PolicyError, match="safe size"): + module.exchange(request, maximum=1, timeout=10) + request.data = b"x" * (module.MAX_CONTROL_BYTES + 1) + with pytest.raises(module.PolicyError, match="safe size"): + module.exchange(request, maximum=1, timeout=10) + + +def test_exchange_returns_result_and_enforces_response_limit(): + module = _load("deadline_http") + request = urllib.request.Request( + "https://scm.bstein.dev/api/v1/x", data=b"{}", method="POST" + ) + result = module.exchange( + request, maximum=64, timeout=10, popen=_fake_popen(_success_output(b"body")) + ) + assert (result.status, result.content_type, result.body) == ( + 200, + "application/json", + b"body", + ) + with pytest.raises(module.PolicyError, match="safe size"): + module.exchange( + request, + maximum=2, + timeout=10, + popen=_fake_popen(_success_output(b"body")), + ) + + +@pytest.mark.parametrize( + ("output", "returncode", "match"), + [ + (b"", 1, "request failed"), + (b"x" * (4 * 1024 * 1024 + 1), 0, "request failed"), + (b"not json", 0, "invalid evidence"), + (b'{"ok":false}', 0, "invalid evidence"), + (b"[1]", 0, "invalid evidence"), + (b'{"ok":true}', 0, "invalid evidence"), + (b'{"ok":true,"status":"200","content_type":"a","body":""}', 0, "invalid evidence"), + (b'{"ok":true,"status":200,"content_type":"a","body":"!!"}', 0, "invalid evidence"), + ], +) +def test_exchange_rejects_invalid_helper_evidence(output, returncode, match): + module = _load("deadline_http") + request = urllib.request.Request("https://scm.bstein.dev/api/v1/x") + with pytest.raises(module.PolicyError, match=match): + module.exchange( + request, maximum=64, timeout=10, popen=_fake_popen(output, returncode) + ) + + +def test_exchange_kills_the_helper_when_the_deadline_expires(server): + module = _load("deadline_http") + request = urllib.request.Request(server + "/slow") + started = time.monotonic() + with pytest.raises(module.PolicyError, match="deadline exceeded"): + module.exchange(request, maximum=64, timeout=1.0) + assert time.monotonic() - started < 5 + + +def test_exchange_fails_before_spawn_wait_when_time_is_exhausted(monkeypatch): + module = _load("deadline_http") + request = urllib.request.Request("https://scm.bstein.dev/api/v1/x") + killed = [] + monkeypatch.setattr(module.os, "killpg", lambda pid, sig: killed.append((pid, sig))) + clock = iter([0.0, 1_000_000.0, 1_000_000.0, 1_000_000.0]) + monkeypatch.setattr(module.time, "monotonic", lambda: next(clock)) + with pytest.raises(module.PolicyError, match="deadline exceeded"): + module.exchange( + request, maximum=64, timeout=10, popen=_fake_popen(_success_output(b"")) + ) + assert killed + + +def test_terminate_tolerates_already_finished_helpers(): + module = _load("deadline_http") + process = subprocess.Popen( + ["/bin/true"], start_new_session=True, stdout=subprocess.DEVNULL + ) + process.wait() + module._terminate(process) + assert process.returncode == 0 + + +def test_child_entrypoint_round_trip_through_real_subprocess(server): + module = _load("deadline_http") + request = urllib.request.Request(server + "/ok") + result = module.exchange(request, maximum=1024, timeout=20) + assert result.status == 200 and result.body == b'{"ok":true}' + + +def test_open_bounded_adapts_result_to_the_urlopen_interface(): + module = _load("deadline_http") + request = urllib.request.Request("https://scm.bstein.dev/api/v1/x") + with module.open_bounded( + request, maximum=64, timeout=10, popen=_fake_popen(_success_output(b"body")) + ) as response: + assert response.status == 200 + assert response.headers.get_content_type() == "application/json" + assert response.read(2) == b"bo" + assert response.read() == b"dy" diff --git a/testing/tests/test_hermes_deadline_stream.py b/testing/tests/test_hermes_deadline_stream.py new file mode 100644 index 00000000..88022591 --- /dev/null +++ b/testing/tests/test_hermes_deadline_stream.py @@ -0,0 +1,304 @@ +"""Watchdog contracts for absolute deadlines on streaming Git exchanges.""" + +from __future__ import annotations + +import threading +import time +import urllib.request +from http.server import BaseHTTPRequestHandler, HTTPServer + +import pytest + +from testing.tests.test_hermes_scm_broker_support import Response, _load + + +def test_stream_deadline_validates_bounds_and_counts_down(): + module = _load("deadline_http") + for timeout in (0, -1, module.MAX_STREAM_SECONDS + 1): + with pytest.raises(module.PolicyError, match="bounds are invalid"): + module.StreamDeadline(timeout) + guard = module.StreamDeadline(30) + try: + first = guard.remaining() + assert 0 < first <= 30 + finally: + guard.cancel() + + +class _FakeTimer: + def __init__(self, timeout, callback): + self.timeout = timeout + self.callback = callback + self.daemon = False + self.started = False + self.cancelled = False + + def start(self): + self.started = True + + def cancel(self): + self.cancelled = True + + +class _FakeSocket: + def __init__(self, fail: bool = False): + self.fail = fail + self.shut = False + + def shutdown(self, _how): + if self.fail: + raise OSError("already gone") + self.shut = True + + +class _FakeConnection: + def __init__(self, sock=None): + self.sock = sock + self.closed = False + + def close(self): + self.closed = True + + +def test_expiry_shuts_down_and_closes_every_tracked_connection(): + module = _load("deadline_http") + guard = module.StreamDeadline(30, timer=_FakeTimer) + assert guard._timer.started and guard._timer.daemon + healthy = _FakeConnection(_FakeSocket()) + broken = _FakeConnection(_FakeSocket(fail=True)) + bare = _FakeConnection() + for connection in (healthy, broken, bare): + assert guard._track(connection) is connection + guard._expire() + assert guard.expired + assert healthy.sock.shut and healthy.closed + assert broken.closed and not broken.sock.shut + assert bare.closed + with pytest.raises(module.PolicyError, match="deadline exceeded"): + guard.remaining() + late = _FakeConnection() + with pytest.raises(module.PolicyError, match="deadline exceeded"): + guard._track(late) + assert late.closed + guard.cancel() + assert guard._timer.cancelled + + +def test_handlers_build_guarded_connections_without_network(monkeypatch): + module = _load("deadline_http") + guard = module.StreamDeadline(30, timer=_FakeTimer) + http_handler, https_handler = guard.handlers() + assert isinstance(http_handler, urllib.request.HTTPHandler) + assert isinstance(https_handler, urllib.request.HTTPSHandler) + for handler, opener_name in ( + (http_handler, "http_open"), + (https_handler, "https_open"), + ): + seen = {} + monkeypatch.setattr( + handler, + "do_open", + lambda factory, req, seen=seen: seen.update(factory=factory, req=req), + ) + getattr(handler, opener_name)("request") + connection = seen["factory"]("127.0.0.1", timeout=1) + assert connection in guard._connections + connection.close() + guard.cancel() + + +def test_dns_resolution_is_bounded_by_the_stream_deadline(monkeypatch): + module = _load("deadline_http") + + def hang(*_args, **_kwargs): + time.sleep(30) + + monkeypatch.setattr(module.socket, "getaddrinfo", hang) + started = time.monotonic() + with pytest.raises(module.PolicyError, match="DNS resolution deadline"): + module._resolve_within("example.invalid", 443, 0.4) + assert time.monotonic() - started < 3 + + +def test_resolve_within_returns_on_success_and_propagates_errors(monkeypatch): + module = _load("deadline_http") + calls = [] + monkeypatch.setattr( + module.socket, "getaddrinfo", lambda *a, **k: calls.append((a, k)) + ) + module._resolve_within("127.0.0.1", 80, 5) + assert calls + + def boom(*_a, **_k): + raise OSError("name resolution failed") + + monkeypatch.setattr(module.socket, "getaddrinfo", boom) + with pytest.raises(OSError, match="name resolution failed"): + module._resolve_within("127.0.0.1", 80, 5) + + +def test_bounded_connection_wraps_connect_to_resolve_first(monkeypatch): + module = _load("deadline_http") + guard = module.StreamDeadline(30, timer=_FakeTimer) + order = [] + monkeypatch.setattr( + module, + "_resolve_within", + lambda host, port, timeout: order.append(("resolve", host, port)), + ) + + class FakeConnection: + host = "scm.bstein.dev" + port = 443 + + def __init__(self): + self.sock = None + + def connect(self): + order.append(("connect", self.host)) + + def close(self): + order.append(("close",)) + + connection = guard._bounded_connection(FakeConnection()) + assert connection in guard._connections + connection.connect() + assert order == [("resolve", "scm.bstein.dev", 443), ("connect", "scm.bstein.dev")] + guard.cancel() + + +def test_bounded_connection_connect_fails_closed_once_expired(monkeypatch): + module = _load("deadline_http") + guard = module.StreamDeadline(30, timer=_FakeTimer) + resolved = [] + monkeypatch.setattr( + module, "_resolve_within", lambda *a, **k: resolved.append(a) + ) + + class FakeConnection: + host = "scm.bstein.dev" + port = 443 + sock = None + + def connect(self): + resolved.append("connected") + + def close(self): + pass + + connection = guard._bounded_connection(FakeConnection()) + guard._expire() + with pytest.raises(module.PolicyError, match="deadline exceeded"): + connection.connect() + assert resolved == [] + guard.cancel() + + +class _TrickleHandler(BaseHTTPRequestHandler): + def do_GET(self): + time.sleep(10) + + def log_message(self, *_args): + return + + +def test_watchdog_interrupts_a_hung_streaming_exchange(): + module = _load("deadline_http") + server = HTTPServer(("127.0.0.1", 0), _TrickleHandler) + threading.Thread(target=server.serve_forever, daemon=True).start() + guard = module.StreamDeadline(1.0) + opener = urllib.request.build_opener(*guard.handlers()) + started = time.monotonic() + try: + with pytest.raises(OSError): + opener.open( + f"http://127.0.0.1:{server.server_port}/slow", timeout=30 + ).read() + assert time.monotonic() - started < 5 + assert guard.expired + finally: + guard.cancel() + server.shutdown() + + +def test_upstream_git_request_runs_under_one_wall_clock_budget(monkeypatch): + broker = _load("scm_broker") + seen = {} + + class RecordingGuard: + def __init__(self, timeout, **_kwargs): + seen["timeout"] = timeout + self.cancelled = False + seen["guard"] = self + + def remaining(self): + return 12.5 + + def cancel(self): + self.cancelled = True + + def handlers(self): + return () + + monkeypatch.setattr(broker.deadline_http, "StreamDeadline", RecordingGuard) + + def opener(request, timeout): + seen["opener_timeout"] = timeout + return Response(b"result", content_type="application/x-git-upload-pack-result") + + result = broker._upstream_git_request( + "/atlas/cassandra.git/git-upload-pack", + method="POST", + body=b"request", + content_type="application/x-git-upload-pack-request", + expected_type="application/x-git-upload-pack-result", + token="sentinel", + opener=opener, + ) + assert result == b"result" + assert seen["timeout"] == broker.UPSTREAM_DEADLINE_SECONDS + assert seen["opener_timeout"] == 12.5 + assert seen["guard"].cancelled + + +def test_upstream_git_request_cancels_guard_when_deadline_already_passed(monkeypatch): + broker = _load("scm_broker") + cancelled = [] + + class ExpiredGuard: + def __init__(self, _timeout, **_kwargs): + pass + + def remaining(self): + raise broker.PolicyError("HTTP stream deadline exceeded") + + def cancel(self): + cancelled.append(True) + + def handlers(self): + return () + + monkeypatch.setattr(broker.deadline_http, "StreamDeadline", ExpiredGuard) + with pytest.raises(broker.PolicyError, match="deadline exceeded"): + broker._upstream_git_request( + "/atlas/cassandra.git/git-upload-pack", + method="POST", + body=b"request", + content_type="application/x-git-upload-pack-request", + expected_type="application/x-git-upload-pack-result", + token="sentinel", + opener=lambda *_a, **_k: None, + ) + assert cancelled == [True] + + +def test_guarded_opener_wires_redirect_rejection_and_deadline_handlers(): + broker = _load("scm_broker") + module = _load("deadline_http") + guard = module.StreamDeadline(30, timer=_FakeTimer) + opener = broker._guarded_opener(guard) + assert callable(opener) + handlers = opener.__self__.handlers + assert any(isinstance(item, broker.RejectRedirect) for item in handlers) + assert any(isinstance(item, urllib.request.HTTPSHandler) for item in handlers) + guard.cancel() diff --git a/testing/tests/test_hermes_git_pack_objects.py b/testing/tests/test_hermes_git_pack_objects.py new file mode 100644 index 00000000..f3a249ec --- /dev/null +++ b/testing/tests/test_hermes_git_pack_objects.py @@ -0,0 +1,288 @@ +"""Bounded pack inflation and delta-resolution contracts for pushed objects.""" + +from __future__ import annotations + +import hashlib +import io +import zlib + +import pytest + +from testing.tests.test_hermes_scm_broker_support import ( + _load, + _object_entry, + _pack_of, +) + + +def _delta_header(type_code: int, size: int, suffix: bytes = b"") -> bytes: + byte = (type_code << 4) | (size & 0x0F) + size >>= 4 + header = bytearray() + while size: + header.append(byte | 0x80) + byte = size & 0x7F + size >>= 7 + header.append(byte) + return bytes(header) + suffix + + +def _size_varint(value: int) -> bytes: + out = bytearray() + while True: + byte = value & 0x7F + value >>= 7 + if value: + out.append(byte | 0x80) + else: + out.append(byte) + return bytes(out) + + +def _ofs_distance(value: int) -> bytes: + out = [value & 0x7F] + value >>= 7 + while value: + value -= 1 + out.insert(0, 0x80 | (value & 0x7F)) + value >>= 7 + return bytes(out) + + +def _delta(source: bytes, target_size: int, instructions: bytes) -> bytes: + return _size_varint(len(source)) + _size_varint(target_size) + instructions + + +def _blob_sha(payload: bytes) -> bytes: + return hashlib.sha1(b"blob %d\x00" % len(payload) + payload).digest() + + +def _read_all(objects): + try: + return [(code, payload.read()) for code, payload in objects] + finally: + for _code, payload in objects: + payload.close() + + +def test_unpack_returns_typed_payloads_for_every_base_object_kind(): + module = _load("git_pack_objects") + entries = [ + _object_entry(1, b"tree 1\n"), + _object_entry(2, b"\x00tree-bytes"), + _object_entry(3, b"blob contents"), + _object_entry(4, b"tag contents"), + ] + result = _read_all(module.unpack_objects(io.BytesIO(_pack_of(entries)))) + assert result == [ + (1, b"tree 1\n"), + (2, b"\x00tree-bytes"), + (3, b"blob contents"), + (4, b"tag contents"), + ] + + +def test_unpack_accepts_the_empty_pack(): + module = _load("git_pack_objects") + assert module.unpack_objects(io.BytesIO(_pack_of([]))) == [] + + +def test_ofs_delta_resolves_against_its_in_pack_base(): + module = _load("git_pack_objects") + base = b"base payload for copying" + base_entry = _object_entry(3, base) + # Copy the first 4 bytes, then insert new bytes. + instructions = bytes([0x80 | 0x10, 4]) + bytes([5]) + b"+tail" + delta = _delta(base, 9, instructions) + delta_entry = _delta_header( + 6, len(delta), _ofs_distance(len(base_entry)) + ) + zlib.compress(delta) + result = _read_all(module.unpack_objects(io.BytesIO(_pack_of([base_entry, delta_entry])))) + assert result == [(3, base), (3, b"base+tail")] + + +def test_ref_delta_resolves_by_sha_and_supports_chains(): + module = _load("git_pack_objects") + base = b"chain base" + first = _delta(base, 4, bytes([4]) + b"one!") + second = _delta(b"one!", 4, bytes([4]) + b"two!") + entries = [ + _object_entry(3, base), + _delta_header(7, len(first), _blob_sha(base)) + zlib.compress(first), + _delta_header(7, len(second), _blob_sha(b"one!")) + zlib.compress(second), + ] + result = _read_all(module.unpack_objects(io.BytesIO(_pack_of(entries)))) + assert [payload for _code, payload in result] == [base, b"one!", b"two!"] + + +def test_thin_pack_ref_delta_with_external_base_is_rejected(): + module = _load("git_pack_objects") + delta = _delta(b"absent", 1, bytes([1]) + b"x") + entry = _delta_header(7, len(delta), b"\xaa" * 20) + zlib.compress(delta) + with pytest.raises(module.PolicyError, match="push full packs"): + module.unpack_objects(io.BytesIO(_pack_of([entry]))) + + +def test_ofs_delta_bad_offsets_fail_closed(): + module = _load("git_pack_objects") + base_entry = _object_entry(3, b"base") + delta = _delta(b"base", 1, bytes([1]) + b"x") + # Distance beyond the pack start. + early = _delta_header(6, len(delta), _ofs_distance(999)) + zlib.compress(delta) + with pytest.raises(module.PolicyError, match="base offset is invalid"): + module.unpack_objects(io.BytesIO(_pack_of([base_entry, early]))) + # Distance landing between object boundaries. + misaligned = _delta_header(6, len(delta), _ofs_distance(3)) + zlib.compress(delta) + with pytest.raises(module.PolicyError, match="base offset is invalid"): + module.unpack_objects(io.BytesIO(_pack_of([base_entry, misaligned]))) + + +@pytest.mark.parametrize( + ("mutate", "match"), + [ + (lambda pack: b"JUNK" + pack[4:], "header is invalid"), + (lambda pack: pack[:4] + (9).to_bytes(4, "big") + pack[8:], "header is invalid"), + (lambda pack: pack[:8] + (2).to_bytes(4, "big") + pack[12:], "Git"), + (lambda pack: pack[:-1], "ended early"), + (lambda pack: pack[:-20] + b"\x00" * 20, "checksum is invalid"), + (lambda pack: pack[:11], "ended early"), + ], +) +def test_malformed_pack_containers_fail_closed(mutate, match): + module = _load("git_pack_objects") + pack = _pack_of([_object_entry(3, b"payload")]) + with pytest.raises(module.PolicyError, match=match): + module.unpack_objects(io.BytesIO(mutate(pack))) + + +def test_object_count_and_type_bounds_fail_closed(monkeypatch): + module = _load("git_pack_objects") + body = b"PACK" + (2).to_bytes(4, "big") + (module.MAX_PACK_OBJECTS + 1).to_bytes( + 4, "big" + ) + with pytest.raises(module.PolicyError, match="too many objects"): + module.unpack_objects(io.BytesIO(body + hashlib.sha1(body).digest())) + reserved = _delta_header(5, 1) + zlib.compress(b"x") + with pytest.raises(module.PolicyError, match="object type is invalid"): + module.unpack_objects(io.BytesIO(_pack_of([reserved]))) + + +def test_declared_size_mismatches_and_zlib_bombs_fail_closed(monkeypatch): + module = _load("git_pack_objects") + longer = _delta_header(3, 2) + zlib.compress(b"longer-than-two") + with pytest.raises(module.PolicyError, match="does not match its header"): + module.unpack_objects(io.BytesIO(_pack_of([longer]))) + shorter = _delta_header(3, 9) + zlib.compress(b"x") + with pytest.raises(module.PolicyError, match="does not match its header"): + module.unpack_objects(io.BytesIO(_pack_of([shorter]))) + corrupt = _delta_header(3, 4) + b"\x00not-zlib\x00" + with pytest.raises(module.PolicyError, match="corrupt"): + module.unpack_objects(io.BytesIO(_pack_of([corrupt]))) + + monkeypatch.setattr(module, "MAX_OBJECT_BYTES", 4) + with pytest.raises(module.PolicyError, match="exceeds the safe size limit"): + module.unpack_objects(io.BytesIO(_pack_of([_object_entry(3, b"12345")]))) + monkeypatch.setattr(module, "MAX_OBJECT_BYTES", 1024) + monkeypatch.setattr(module, "MAX_TOTAL_BYTES", 8) + with pytest.raises(module.PolicyError, match="contents exceed"): + module.unpack_objects( + io.BytesIO(_pack_of([_object_entry(3, b"12345"), _object_entry(3, b"12345")])) + ) + + +def test_object_and_distance_varints_are_bounded(): + module = _load("git_pack_objects") + with pytest.raises(module.PolicyError, match="object header is invalid"): + module._object_header(io.BytesIO(b"\xff" * 12)) + with pytest.raises(module.PolicyError, match="ended early"): + module._object_header(io.BytesIO(b"\x80")) + with pytest.raises(module.PolicyError, match="distance is invalid"): + module._delta_distance(io.BytesIO(b"\xff" * 12)) + + +@pytest.mark.parametrize( + ("base", "delta", "match"), + [ + (b"base", b"", "delta header is invalid"), + (b"base", _size_varint(4), "delta header is invalid"), + (b"base", _size_varint(9) + _size_varint(1) + bytes([1]) + b"x", "sizes are invalid"), + (b"base", _size_varint(4) + _size_varint(2) + bytes([0]), "instruction is invalid"), + (b"base", _size_varint(4) + _size_varint(2) + bytes([5]) + b"ab", "insert is truncated"), + (b"base", _size_varint(4) + _size_varint(2) + bytes([0x80 | 0x10]), "copy is truncated"), + ( + b"base", + _size_varint(4) + _size_varint(2) + bytes([0x80 | 0x01 | 0x10, 3, 9]), + "copy is out of bounds", + ), + (b"base", _size_varint(4) + _size_varint(1) + bytes([2]) + b"ab", "exceeds its declared size"), + (b"base", _size_varint(4) + _size_varint(3) + bytes([2]) + b"ab", "result size is invalid"), + ], +) +def test_delta_application_rejects_malformed_instructions(base, delta, match): + module = _load("git_pack_objects") + with pytest.raises(module.PolicyError, match=match): + module._apply_delta(base, delta) + + +def test_delta_copy_with_zero_size_field_means_the_git_default(): + module = _load("git_pack_objects") + base = b"z" * 0x10000 + delta = _delta(base, 0x10000, bytes([0x80 | 0x10 | 0x20, 0, 0])) + assert module._apply_delta(base, delta) == base + + +def test_delta_target_size_cap_applies(monkeypatch): + module = _load("git_pack_objects") + monkeypatch.setattr(module, "MAX_OBJECT_BYTES", 4) + delta = _delta(b"base", 5, bytes([5]) + b"abcde") + with pytest.raises(module.PolicyError, match="sizes are invalid"): + module._apply_delta(b"base", delta) + + +def test_resolution_pass_and_total_caps_apply(monkeypatch): + module = _load("git_pack_objects") + base = b"pass base" + first = _delta(base, 4, bytes([4]) + b"one!") + second = _delta(b"one!", 4, bytes([4]) + b"two!") + entries = [ + _delta_header(7, len(second), _blob_sha(b"one!")) + zlib.compress(second), + _object_entry(3, base), + _delta_header(7, len(first), _blob_sha(base)) + zlib.compress(first), + ] + pack = _pack_of(entries) + monkeypatch.setattr(module, "MAX_RESOLVE_PASSES", 1) + with pytest.raises(module.PolicyError, match="resolution limit"): + module.unpack_objects(io.BytesIO(pack)) + monkeypatch.setattr(module, "MAX_RESOLVE_PASSES", 64) + result = _read_all(module.unpack_objects(io.BytesIO(pack))) + assert [payload for _code, payload in result] == [b"two!", base, b"one!"] + + monkeypatch.setattr(module, "MAX_TOTAL_BYTES", len(base) + len(first) + len(second) + 1) + with pytest.raises(module.PolicyError, match="contents exceed"): + module.unpack_objects(io.BytesIO(pack)) + + +def test_parse_loose_object_round_trip_and_bounds(monkeypatch): + module = _load("git_pack_objects") + payload = b"loose blob" + value = zlib.compress(b"blob %d\x00" % len(payload) + payload) + assert module.parse_loose_object(value) == (3, payload) + + with pytest.raises(module.PolicyError, match="corrupt"): + module.parse_loose_object(b"\x00junk") + with pytest.raises(module.PolicyError, match="framing is invalid"): + module.parse_loose_object(value + b"trailing") + with pytest.raises(module.PolicyError, match="framing is invalid"): + module.parse_loose_object(value[:-4]) + for header in ( + b"noodle 3\x00abc", + b"blob3\x00abc", + b"blob x\x00abc", + b"blob 4\x00abc", + b"blob 3", + ): + with pytest.raises(module.PolicyError, match="header is invalid"): + module.parse_loose_object(zlib.compress(header)) + monkeypatch.setattr(module, "MAX_OBJECT_BYTES", 4) + with pytest.raises(module.PolicyError, match="exceeds the safe size limit"): + module.parse_loose_object(b"12345") \ No newline at end of file diff --git a/testing/tests/test_hermes_gitea_branch_coverage.py b/testing/tests/test_hermes_gitea_branch_coverage.py new file mode 100644 index 00000000..48421f01 --- /dev/null +++ b/testing/tests/test_hermes_gitea_branch_coverage.py @@ -0,0 +1,209 @@ +"""Behavioral branch coverage for the Gitea protection policy checker.""" + +from __future__ import annotations + +import json +import os +import stat +import sys +from pathlib import Path + +import pytest + +from testing.tests.test_hermes_scm_broker_support import ROOT, _load_path + + +def _module(name: str = "branch_protection_coverage"): + return _load_path( + name, + ROOT / "services/gitea/scripts/gitea_branch_protection_check.py", + ) + + +def _rule(module, name: str = "main", **updates): + value = { + "rule_name": name, + "priority": 1, + "created_at": "2026-01-01T00:00:00Z", + **module._required("bstein"), + } + value.update(updates) + return value + + +def test_read_bounded_accepts_regular_file_and_closes_descriptor( + tmp_path: Path, monkeypatch +): + module = _module("branch_read_regular") + source = tmp_path / "rules.json" + source.write_bytes(b"[]") + closed: list[int] = [] + real_close = os.close + + def close(descriptor: int) -> None: + closed.append(descriptor) + real_close(descriptor) + + monkeypatch.setattr(module.os, "close", close) + assert module._read_bounded(source) == b"[]" + assert len(closed) == 1 + + +@pytest.mark.parametrize("kind", ["directory", "oversized", "short"]) +def test_read_bounded_rejects_unsafe_or_changed_input( + tmp_path: Path, monkeypatch, kind +): + module = _module(f"branch_read_{kind}") + source = tmp_path / "rules" + source.write_bytes(b"[]") + real_fstat = module.os.fstat + real_read = module.os.read + + if kind == "directory": + monkeypatch.setattr( + module.os, + "fstat", + lambda descriptor: os.stat_result( + (stat.S_IFDIR | 0o700, 0, 0, 0, 0, 0, 2, 0, 0, 0) + ), + ) + elif kind == "oversized": + monkeypatch.setattr( + module.os, + "fstat", + lambda descriptor: os.stat_result( + (stat.S_IFREG | 0o600, 0, 0, 0, 0, 0, module.MAX_INPUT + 1, 0, 0, 0) + ), + ) + else: + monkeypatch.setattr(module.os, "fstat", real_fstat) + monkeypatch.setattr(module.os, "read", lambda descriptor, maximum: b"[") + with pytest.raises(module.PolicyError): + module._read_bounded(source) + monkeypatch.setattr(module.os, "read", real_read) + + +@pytest.mark.parametrize( + "pattern", + [ + "{main}", + "{main,master}", + "m?in", + "m[ai]in", + "m[!z]in", + "m[a-z]in", + "m\\ain", + "m\\", + "m[", + "m[]", + "m[!]", + "m[[a]", + "m[\\a]", + "m[-a]", + "m[a-]", + "m[a-b-c]", + "m[z-a]", + "m[aa-b]", + "m{ain}", + "m{ain,}", + "m{ain,aster", + ], +) +def test_every_special_or_malformed_glob_fails_closed(pattern): + module = _module("branch_glob_invalid") + with pytest.raises(module.PolicyError, match="glob branch protection"): + module._literal_rule(pattern) + + +@pytest.mark.parametrize("pattern", ["", "x" * 256, "máin"]) +def test_rule_names_are_ascii_and_bounded(pattern): + module = _module("branch_name_bounds") + with pytest.raises(module.PolicyError, match="rule name"): + module._literal_rule(pattern) + + +def test_plain_rule_matching_is_exact_and_case_sensitive(): + module = _module("branch_literal_case") + assert module._literal_rule("main") == "main" + assert module._literal_rule("MAIN") == "MAIN" + assert module.evaluate( + json.dumps([_rule(module, name="MAIN")]).encode(), "main", "bstein" + ) == "ABSENT" + + +@pytest.mark.parametrize("value", [None, "x" * 65, "invalid", "2026-01-01T00:00:00"]) +def test_created_at_requires_short_timezone_aware_iso(value): + module = _module("branch_created_bounds") + with pytest.raises(module.PolicyError, match="creation time"): + module._created(value) + + +@pytest.mark.parametrize( + "value", + [b"x" * (1024 * 1024 + 1), b"{}", json.dumps([None]).encode()], +) +def test_evaluate_rejects_invalid_top_level_inputs(value): + module = _module("branch_evaluate_top") + with pytest.raises((module.PolicyError, json.JSONDecodeError)): + module.evaluate(value, "main", "bstein") + + +def test_evaluate_rejects_unknown_branch_and_excess_rules(): + module = _module("branch_evaluate_bounds") + with pytest.raises(module.PolicyError, match="input"): + module.evaluate(b"[]", "release", "bstein") + with pytest.raises(module.PolicyError, match="response"): + module.evaluate(json.dumps([{}] * 101).encode(), "main", "bstein") + + +@pytest.mark.parametrize("priority", [True, -1, 1_000_001, "1"]) +def test_evaluate_rejects_ambiguous_priority_types(priority): + module = _module("branch_priority_bounds") + rule = _rule(module, priority=priority) + with pytest.raises(module.PolicyError, match="priority"): + module.evaluate(json.dumps([rule]).encode(), "main", "bstein") + + +def test_legacy_zero_priority_is_accepted_but_duplicate_ties_fail_closed(): + module = _module("branch_priority_zero") + assert module.evaluate( + json.dumps([_rule(module, priority=0)]).encode(), "main", "bstein" + ) == "PRESENT" + + first = _rule(module, priority=0, created_at="2026-01-01T00:00:00Z") + second = _rule( + module, + priority=0, + created_at="2026-01-01T00:00:00Z", + enable_force_push=True, + ) + with pytest.raises(module.PolicyError, match="duplicate primary-branch"): + module.evaluate(json.dumps([first, second]).encode(), "main", "bstein") + + +def test_any_special_rule_blocks_absent_create_and_present_readback(): + module = _module("branch_special_readback") + for rules in ( + [_rule(module, name="feature/*")], + [_rule(module), _rule(module, name="{main}", priority=0)], + ): + with pytest.raises(module.PolicyError, match="human review"): + module.evaluate(json.dumps(rules).encode(), "main", "bstein") + + +def test_main_reports_success_and_failures(tmp_path: Path, monkeypatch, capsys): + module = _module("branch_main_paths") + source = tmp_path / "rules.json" + source.write_text(json.dumps([_rule(module)]), encoding="utf-8") + monkeypatch.setattr(sys, "argv", ["checker", str(source), "main", "bstein"]) + assert module.main() == 0 + assert capsys.readouterr().out.strip() == "PRESENT" + + source.write_text("not-json", encoding="utf-8") + assert module.main() == 1 + assert "branch protection check failed" in capsys.readouterr().err + + monkeypatch.setattr( + module, "_read_bounded", lambda _path: (_ for _ in ()).throw(OSError()) + ) + assert module.main() == 1 diff --git a/testing/tests/test_hermes_gitea_internal_coverage.py b/testing/tests/test_hermes_gitea_internal_coverage.py new file mode 100644 index 00000000..da720878 --- /dev/null +++ b/testing/tests/test_hermes_gitea_internal_coverage.py @@ -0,0 +1,380 @@ +"""Behavioral coverage for internal safe-Gitea client and policy paths.""" + +from __future__ import annotations + +import json +import subprocess +import sys +import types +import urllib.error +import urllib.parse +from email.message import Message +from pathlib import Path + +import pytest + +from testing.tests.test_hermes_gitea_support import CLIENT_PATH, Response, _load +from testing.tests.test_hermes_scm_broker_support import _load_path + + +def _policy(name: str = "gitea_policy_coverage"): + return _load_path(name, CLIENT_PATH.parent / "gitea_api_policy.py") + + +def test_runtime_token_reader_and_fixed_origin(tmp_path: Path, monkeypatch): + module = _load() + token = tmp_path / "token" + token.write_text(" synthetic-runtime-value \n", encoding="utf-8") + assert module.read_token(token) == "synthetic-runtime-value" + token.write_text(" \n", encoding="utf-8") + with pytest.raises(ValueError, match="empty"): + module.read_token(token) + + monkeypatch.setenv("GITEA_BASE_URL", module.CANONICAL_BASE_URL) + assert module.configured_base_url() == module.CANONICAL_BASE_URL + monkeypatch.setenv("GITEA_BASE_URL", "https://example.invalid") + with pytest.raises(module.PolicyError, match="fixed"): + module.configured_base_url() + + +def test_safe_urlopen_delegates_only_to_deadline_bounded_exchange(monkeypatch): + module = _load() + sentinel = object() + monkeypatch.setattr( + module.deadline_http, + "open_bounded", + lambda request, *, maximum, timeout: (request, maximum, timeout, sentinel), + ) + request = object() + assert module._safe_urlopen(request, 7) == ( + request, + module.MAX_RESPONSE_BYTES, + 7, + sentinel, + ) + + +def test_api_target_and_request_body_edge_paths(monkeypatch): + module = _load() + with pytest.raises(module.PolicyError, match="path exceeds"): + module._split_api_path("/api/v1/" + "x" * 510) + with pytest.raises(module.PolicyError, match="request body"): + module.authorize_request( + "GET", "/api/v1/repos/atlas/cassandra", {"unexpected": True} + ) + with pytest.raises(module.PolicyError, match="query"): + module.authorize_request( + "POST", + "/api/v1/repos/atlas/cassandra/pulls?page=1", + {}, + ) + with pytest.raises(module.PolicyError, match="only read"): + module.authorize_request("TRACE", "/api/v1/repos/atlas/cassandra/pulls", None) + with pytest.raises(module.PolicyError, match="accepts only"): + module.authorize_request("POST", "/api/v1/repos/atlas/cassandra/pulls", []) + + +def test_response_status_fallback_and_nested_fail_closed(): + module = _load() + + class GetCodeResponse(Response): + def __init__(self): + super().__init__({"name": "cassandra"}, status=200) + del self.status + + def getcode(self): + return 200 + + assert ( + module.read( + "/api/v1/repos/atlas/cassandra", + token="synthetic", + opener=lambda *_a, **_k: GetCodeResponse(), + ) + == b'{"name": "cassandra"}' + ) + with pytest.raises(module.PolicyError, match="omitted required"): + module._nested({"base": None}, "base", "ref") + + +@pytest.mark.parametrize("body", [b"not-json", b"[]"]) +def test_create_response_requires_json_object(body: bytes): + module = _load() + with pytest.raises(module.PolicyError, match="invalid pull-request metadata"): + module._require_create_response( + body, + repo="cassandra", + base="main", + head="feature/test", + head_sha="a" * 40, + title="WIP: Test", + body="Evidence", + ) + + +def test_write_body_handles_empty_newline_and_missing_newline(monkeypatch): + module = _load() + + class Buffer: + value = bytearray() + + @classmethod + def write(cls, value): + cls.value.extend(value) + + monkeypatch.setattr(module.sys, "stdout", types.SimpleNamespace(buffer=Buffer)) + module._write_body(b"") + module._write_body(b"one\n") + module._write_body(b"two") + assert bytes(Buffer.value) == b"one\ntwo\n" + + +def test_main_executes_broker_read_and_create_paths(monkeypatch): + module = _load() + outputs: list[bytes] = [] + client = types.ModuleType("scm_broker_client") + client.read = lambda path: json.dumps({"path": path}).encode() + client.create_draft = lambda repo, **data: json.dumps( + {"repo": repo, **data}, sort_keys=True + ).encode() + monkeypatch.setitem(sys.modules, "scm_broker_client", client) + monkeypatch.setattr(module, "_write_body", outputs.append) + + assert module.main(["read", "/api/v1/repos/atlas/cassandra"]) == 0 + assert json.loads(outputs.pop()) == {"path": "/api/v1/repos/atlas/cassandra"} + assert ( + module.main( + [ + "create-draft", + "cassandra", + "--base", + "main", + "--head", + "feature/coverage", + "--head-sha", + "a" * 40, + "--title", + "Coverage repair", + "--body", + "Review the focused tests", + ] + ) + == 0 + ) + assert json.loads(outputs.pop())["repo"] == "cassandra" + + +def test_main_handles_http_and_policy_failures(monkeypatch, capsys): + module = _load() + client = types.ModuleType("scm_broker_client") + + def http_failure(_path): + headers = Message() + raise urllib.error.HTTPError("url", 503, "unavailable", headers, None) + + client.read = http_failure + client.create_draft = lambda *_a, **_k: b"{}" + monkeypatch.setitem(sys.modules, "scm_broker_client", client) + assert module.main(["read", "/api/v1/repos/atlas/cassandra"]) == 1 + assert "HTTP 503" in capsys.readouterr().err + client.read = lambda _path: (_ for _ in ()).throw(module.PolicyError("rejected")) + assert module.main(["read", "/not-allowed"]) == 1 + assert "no credential" in capsys.readouterr().err + + +@pytest.mark.parametrize( + ("function", "value"), + [ + ("_validate_repo", "."), + ("_validate_repo", ".."), + ("_validate_sha", "short"), + ("_validate_pr_number", True), + ("_validate_pr_number", 0), + ("_validate_pr_number", 2_147_483_648), + ], +) +def test_policy_scalar_validators_reject_ambiguous_values(function, value): + module = _policy("gitea_policy_scalar") + with pytest.raises(module.PolicyError): + getattr(module, function)(value) + + +def test_ref_bounds_cover_utf8_controls_git_failure_and_success(monkeypatch): + module = _policy("gitea_policy_refs") + with pytest.raises(module.PolicyError, match="valid UTF-8"): + module._validate_ref_bounds("\ud800", "head") + with pytest.raises(module.PolicyError, match="UTF-8"): + module._validate_ref_bounds("🧪" * 64, "head") + with pytest.raises(module.PolicyError, match="safe same"): + module._validate_ref("bad\nref", "head") + monkeypatch.setattr( + module.subprocess, + "run", + lambda *_a, **_k: subprocess.CompletedProcess([], 1), + ) + with pytest.raises(module.PolicyError, match="same-repository"): + module._validate_ref("invalid-ref", "head") + monkeypatch.setattr( + module.subprocess, + "run", + lambda *_a, **_k: subprocess.CompletedProcess([], 0), + ) + assert module._validate_ref("feature/valid", "head") == "feature/valid" + + +@pytest.mark.parametrize( + ("value", "required", "match"), + [ + (None, False, "must be text"), + (" ", True, "must not be empty"), + ("\ud800", False, "valid UTF-8"), + ("x\x00y", False, "safe request limit"), + ], +) +def test_text_validation_rejects_nontext_empty_invalid_and_controls( + value, required, match +): + module = _policy("gitea_policy_text") + with pytest.raises(module.PolicyError, match=match): + module._validate_text(value, "field", 20, 20, required=required) + + +@pytest.mark.parametrize( + ("key", "expected"), + [ + ("password", True), + ("accountKey", True), + ("registry-key", True), + ("clientEmail", True), + ("clientId", True), + ("accessId", True), + ("connectionString", True), + ("dockerConfigJson", True), + ("release_note", False), + ], +) +def test_sensitive_key_semantics_cover_generic_forms(key, expected): + module = _policy("gitea_policy_keys") + assert module._is_sensitive_key(key) is expected + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("", False), + ('"synthetic"', True), + ("{encoded}", True), + ("Bearer value-12345678", True), + ("ghp_" + "a" * 24, True), + ("https://example.invalid/value", True), + ("name@example.invalid", True), + ("${RUNTIME_VALUE}", True), + ("AbCdEf0123456789", True), + ("reject empty values", False), + ("ordinary", True), + ], +) +def test_assignment_value_shape_distinguishes_prose_from_credentials(value, expected): + module = _policy("gitea_policy_assignment_values") + assert module._looks_sensitive_assignment_value(value) is expected + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("single", False), + ("add regression coverage", True), + ("add regression_coverage", False), + ("ordinary engineering prose", False), + ], +) +def test_prose_classifier_covers_length_punctuation_and_lead_words(value, expected): + module = _policy("gitea_policy_prose") + assert module._looks_like_prose(value) is expected + + +def test_structured_json_walk_covers_lists_nonstring_keys_and_limits(): + module = _policy("gitea_policy_json_walk") + assert module._json_value_has_sensitive_assignment({1: "ignored"}) is False + assert module._json_value_has_sensitive_assignment([{"release": "safe"}]) is False + assert ( + module._json_value_has_sensitive_assignment({"client_secret": "value"}) is True + ) + assert ( + module._json_value_has_sensitive_assignment( + {"outer": {"client_secret": "value"}} + ) + is True + ) + assert ( + module._json_value_has_sensitive_assignment({"type": "service-account"}) is True + ) + with pytest.raises(module.PolicyError, match="scan limit"): + module._json_value_has_sensitive_assignment([], depth=33) + with pytest.raises(module.PolicyError, match="scan limit"): + module._json_value_has_sensitive_assignment([], nodes=[2048]) + + +def test_json_key_decoder_and_embedded_document_scanner_cover_failures(): + module = _policy("gitea_policy_json_decoder") + assert module._decode_json_key("client\\u005fsecret") == ("client_secret", True) + decoded, valid = module._decode_json_key("client\\qsecret") + assert valid is False and "client" in decoded + assert list(module._decoded_json_documents("prose only")) == [] + assert list(module._decoded_json_documents("x {bad y [1, 2]")) == [[1, 2]] + with pytest.raises(module.PolicyError, match="scan limit"): + list(module._decoded_json_documents("{" * 33)) + assert ( + module._has_structured_sensitive_assignment('"client\\qsecret": value') is True + ) + assert ( + module._has_structured_sensitive_assignment("type:\n service-account") is True + ) + + +def test_high_entropy_detector_covers_long_mixed_and_low_entropy_values(): + module = _policy("gitea_policy_entropy") + assert module._has_high_entropy_token("A1_" * 100) is True + assert ( + module._has_high_entropy_token( + "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789+/_=" + ) + is True + ) + assert module._has_high_entropy_token("a" * 64) is False + assert module._has_high_entropy_token("short prose") is False + + +@pytest.mark.parametrize( + "query", + [ + "bad", + "a=1&b=2&c=3&d=4", + "page=1&page=2", + "=value", + "unexpected=1", + "page=abc", + "page=01", + "page=10001", + "limit=51", + "state=merged", + ], +) +def test_query_validator_covers_each_rejection_class(query): + module = _policy("gitea_policy_query") + target = urllib.parse.urlsplit("/api/v1/repos/atlas/cassandra/pulls?" + query) + with pytest.raises(module.PolicyError): + module._validate_query(target, {"page", "limit", "state"}) + + +def test_draft_title_rejects_prefix_without_content(): + module = _policy("gitea_policy_empty_draft") + with pytest.raises(module.PolicyError, match="after the draft prefix"): + module._draft_title("WIP:") + + +def test_query_validator_rejects_noncanonical_raw_form(): + module = _policy("gitea_policy_raw_query") + target = urllib.parse.SplitResult("", "", "/api/v1/repos/atlas/cassandra", "%", "") + with pytest.raises(module.PolicyError, match="canonical ASCII"): + module._validate_query(target, set()) diff --git a/testing/tests/test_hermes_gitea_pr_client.py b/testing/tests/test_hermes_gitea_pr_client.py new file mode 100644 index 00000000..4ddb9830 --- /dev/null +++ b/testing/tests/test_hermes_gitea_pr_client.py @@ -0,0 +1,296 @@ +"""Route and request-boundary contracts for the Atlas Gitea client.""" + +from __future__ import annotations + +import urllib.error +import urllib.request + +import pytest + +from testing.tests.test_hermes_gitea_support import HEAD_SHA, Response, _load + +def test_default_opener_rejects_redirects_under_an_absolute_deadline(monkeypatch): + client = _load() + source = urllib.request.Request( + "https://scm.bstein.dev/api/v1/repos/atlas/cassandra", + headers={"Authorization": "token redirect-sentinel"}, + ) + + with pytest.raises(urllib.error.URLError) as exc: + client.deadline_http._RejectRedirect().redirect_request( + source, + None, + 302, + "Found", + {}, + "https://evil.example/collect", + ) + + assert "redirect-sentinel" not in str(exc.value) + seen = [] + monkeypatch.setattr( + client.deadline_http, + "open_bounded", + lambda request, *, maximum, timeout: seen.append( + (request, maximum, timeout) + ), + ) + client._safe_urlopen(source, 30) + assert seen == [(source, client.MAX_RESPONSE_BYTES, 30)] + + +@pytest.mark.parametrize( + ("base_url", "path"), + [ + ("https://evil.example", "/api/v1/repos/atlas/cassandra"), + ("http://scm.bstein.dev", "/api/v1/repos/atlas/cassandra"), + ("https://scm.bstein.dev:443", "/api/v1/repos/atlas/cassandra"), + ("https://scm.bstein.dev/", "/api/v1/repos/atlas/cassandra"), + ("HTTPS://scm.bstein.dev", "/api/v1/repos/atlas/cassandra"), + ("https://SCM.bstein.dev", "/api/v1/repos/atlas/cassandra"), + ("https://user@scm.bstein.dev", "/api/v1/repos/atlas/cassandra"), + ("https://scm.bstein.dev", "https://evil.example/api/v1/repos/atlas/cassandra"), + ("https://scm.bstein.dev", "/api/v1/repos/evil/cassandra"), + ("https://scm.bstein.dev", "/api/v1/repos/%61tlas/cassandra"), + ("https://scm.bstein.dev", "/api/v1/repos/atlas/../admin"), + ], +) +def test_host_owner_and_path_escape_attempts_are_rejected(base_url: str, path: str): + client = _load() + + with pytest.raises(client.PolicyError): + client.build_request("GET", path, base_url=base_url, token="secret") + + +@pytest.mark.parametrize( + "path", + [ + "/api/v1/repos/atlas/cassandra/pulls/1\nHost: evil.example", + "/api/v1/repos/atlas/cassandra/pulls/1\r\nX-Test: value", + "/api/v1/repos/atlas/cassandra/pulls/1\tignored", + "/api/v1/repos/atlas/cassandra/pulls/1\x00ignored", + "/api/v1/repos/atlas/cassandra/pulls/1\x1fignored", + "/api/v1/repos/atlas/cassandra/pulls/1\x7fignored", + "/api/v1/repos/atlas/cassandra\\pulls\\1", + "/api/v1/repos/atlas/cassandra/pulls/%31", + "/api/v1/repos/atlas/cassandra/pulls/１", + " https://scm.bstein.dev/api/v1/repos/atlas/cassandra", + "https://scm.bstein.dev/api/v1/repos/atlas/cassandra", + ], +) +def test_raw_noncanonical_target_is_rejected_before_urlsplit_and_opener( + path: str, monkeypatch +): + client = _load() + split_called = False + opener_called = False + original_urlsplit = client.urllib.parse.urlsplit + + def urlsplit(*args, **kwargs): + nonlocal split_called + split_called = True + return original_urlsplit(*args, **kwargs) + + def opener(*_args, **_kwargs): + nonlocal opener_called + opener_called = True + return Response(b"{}") + + monkeypatch.setattr(client.urllib.parse, "urlsplit", urlsplit) + with pytest.raises(client.PolicyError): + client.read(path, token="runtime", opener=opener) + assert split_called is False + assert opener_called is False + + +@pytest.mark.parametrize( + "path", + [ + "/api/v1/repos/atlas/cassandra/pulls/1?", + "//scm.bstein.dev/api/v1/repos/atlas/cassandra", + "/api/v1/repos/atlas/cassandra/./pulls/1", + "/api/v1/repos/atlas/cassandra/../admin", + "/api/v1/repos/atlas/cassandra//pulls/1", + "/api/v1/repos/atlas/cassandra/pulls/1?limit=01", + ], +) +def test_noncanonical_round_trip_or_segments_never_reach_opener(path: str): + client = _load() + opener_called = False + + def opener(*_args, **_kwargs): + nonlocal opener_called + opener_called = True + return Response(b"{}") + + with pytest.raises(client.PolicyError): + client.read(path, token="runtime", opener=opener) + assert opener_called is False + + +@pytest.mark.parametrize( + "path", + [ + "/api/v1/repos/atlas/cassandra", + "/api/v1/repos/atlas/cassandra/pulls?state=open&limit=20&page=1", + "/api/v1/repos/atlas/cassandra/pulls/7", + "/api/v1/repos/atlas/cassandra/pulls/7/commits?limit=20", + "/api/v1/repos/atlas/cassandra/pulls/7/files?page=1", + "/api/v1/repos/atlas/cassandra/branches", + "/api/v1/repos/atlas/cassandra/branches/main", + "/api/v1/repos/atlas/cassandra/commits?limit=10", + f"/api/v1/repos/atlas/cassandra/git/commits/{HEAD_SHA}", + f"/api/v1/repos/atlas/cassandra/commits/{HEAD_SHA}/status", + f"/api/v1/repos/atlas/cassandra/commits/{HEAD_SHA}/statuses?limit=10", + f"/api/v1/repos/atlas/cassandra/statuses/{HEAD_SHA}?page=1", + ], +) +def test_explicit_read_allowlist_accepts_only_engineering_metadata(path: str): + client = _load() + + request = client.build_request( + "GET", path, base_url=client.CANONICAL_BASE_URL, token="runtime" + ) + assert request.method == "GET" + + +@pytest.mark.parametrize( + "path", + [ + "/api/v1/repos/atlas/cassandra/hooks", + "/api/v1/repos/atlas/cassandra/actions/secrets", + "/api/v1/repos/atlas/cassandra/actions/variables", + "/api/v1/repos/atlas/cassandra/collaborators", + "/api/v1/repos/atlas/cassandra/branch_protections", + "/api/v1/repos/atlas/cassandra/keys", + "/api/v1/repos/atlas/cassandra/pulls/7/reviews", + "/api/v1/repos/atlas/cassandra/pulls/7/merge", + "/api/v1/repos/atlas/cassandra/pulls/7.diff", + "/api/v1/repos/atlas/cassandra/releases", + ], +) +def test_privileged_or_content_routes_are_denied_even_for_get(path: str): + client = _load() + + with pytest.raises(client.PolicyError, match="outside the metadata read allowlist"): + client.authorize_request("GET", path, None) + + +@pytest.mark.parametrize( + "path", + [ + "/api/v1/repos/atlas/cassandra/pulls?limit=51", + "/api/v1/repos/atlas/cassandra/pulls?state=merged", + "/api/v1/repos/atlas/cassandra/pulls?private=true", + "/api/v1/repos/atlas/cassandra/pulls?limit=1&limit=2", + "/api/v1/repos/atlas/cassandra?p=1", + ], +) +def test_read_query_is_bounded(path: str): + client = _load() + + with pytest.raises(client.PolicyError): + client.authorize_request("GET", path, None) + + +@pytest.mark.parametrize( + "query", + [ + "page=0", + "page=10001", + "page=01", + "limit=0", + "limit=51", + "limit=01", + "page=" + "9" * 4000, + "page=０", + "page=%EF%BC%90", + "state=%6fpen", + "p%61ge=1", + "page=1&page=2", + "page=1&" + "x" * 17 + "=1", + "state=" + "x" * 17, + "page=1&limit=2&state=open&extra=3", + ], +) +def test_read_query_requires_canonical_bounded_ascii(query: str): + client = _load() + + with pytest.raises(client.PolicyError): + client.authorize_request( + "GET", f"/api/v1/repos/atlas/cassandra/pulls?{query}", None + ) + + +@pytest.mark.parametrize( + "suffix", + [ + "pulls/{number}", + "pulls/{number}.patch", + "pulls/{number}.diff", + "pulls/{number}/commits", + "pulls/{number}/files", + "commits/{number}/status", + "commits/{number}/statuses", + "statuses/{number}", + "branches/{number}", + ], +) +def test_oversized_numeric_or_captured_path_never_reaches_opener(suffix: str): + client = _load() + called = False + + def opener(*_args, **_kwargs): + nonlocal called + called = True + return Response(b"{}") + + path = "/api/v1/repos/atlas/cassandra/" + suffix.format(number="9" * 4000) + with pytest.raises(client.PolicyError): + client.read(path, token="runtime", opener=opener) + assert called is False + + +@pytest.mark.parametrize( + "number", + ["0", "01", "2147483648", "１２", "%31", "12345678901"], +) +@pytest.mark.parametrize("tail", ["", ".patch", ".diff", "/commits", "/files"]) +def test_noncanonical_or_out_of_range_pr_number_never_reaches_opener( + number: str, tail: str +): + client = _load() + called = False + + def opener(*_args, **_kwargs): + nonlocal called + called = True + return Response(b"{}") + + with pytest.raises(client.PolicyError): + client.read( + f"/api/v1/repos/atlas/cassandra/pulls/{number}{tail}", + token="runtime", + opener=opener, + ) + assert called is False + + +def test_maximum_bounded_pr_number_is_readable(): + client = _load() + called = False + + def opener(*_args, **_kwargs): + nonlocal called + called = True + return Response(b"{}") + + assert ( + client.read( + "/api/v1/repos/atlas/cassandra/pulls/2147483647", + token="runtime", + opener=opener, + ) + == b"{}" + ) + assert called is True diff --git a/testing/tests/test_hermes_gitea_pr_creation.py b/testing/tests/test_hermes_gitea_pr_creation.py new file mode 100644 index 00000000..ad2967ed --- /dev/null +++ b/testing/tests/test_hermes_gitea_pr_creation.py @@ -0,0 +1,203 @@ +"""Draft-creation input contracts for the Atlas Gitea client.""" + +from __future__ import annotations + +import pytest + +from testing.tests.test_hermes_gitea_support import ( + HEAD_SHA, + Response, + _draft_payload, + _draft_response, + _load, +) + + +@pytest.mark.parametrize( + ("method", "path", "data"), + [ + ("DELETE", "/api/v1/repos/atlas/cassandra/pulls/4", None), + ("POST", "/api/v1/repos/atlas/cassandra/pulls/4/merge", {}), + ( + "POST", + "/api/v1/repos/atlas/cassandra/pulls/4/reviews", + {"event": "APPROVED"}, + ), + ("PATCH", "/api/v1/repos/atlas/cassandra/pulls/4", {"title": "WIP: x"}), + ("PUT", "/api/v1/repos/atlas/cassandra/branches/main", {}), + ], +) +def test_merge_approve_close_delete_update_and_other_mutations_are_rejected( + method: str, path: str, data: object +): + client = _load() + + with pytest.raises(client.PolicyError): + client.build_request( + method, path, base_url=client.CANONICAL_BASE_URL, token="secret", data=data + ) + + +@pytest.mark.parametrize( + "ref", + [ + "foo/.bar", + "foo/bar.lock/baz", + "foo..bar", + "foo@{bar", + "foo//bar", + "-danger", + "danger.", + "danger~one", + "danger^one", + "danger:one", + "danger one", + ], +) +def test_complete_git_ref_validation_rejects_invalid_names(ref: str): + client = _load() + + with pytest.raises(client.PolicyError): + client._validate_ref(ref, "head") + + +def test_git_ref_validation_uses_fixed_trusted_binary(): + client = _load() + + assert client._validate_ref.__globals__["GIT_BIN"] == "/usr/bin/git" + assert client._validate_ref("hermes/valid-fix", "head") == "hermes/valid-fix" + + +@pytest.mark.parametrize("field", ["base", "head"]) +@pytest.mark.parametrize("oversized", ["r" * 100_000, "🧪" * 128]) +def test_oversized_ref_never_invokes_git_request_or_opener( + field: str, oversized: str, monkeypatch +): + client = _load() + git_called = False + request_built = False + opener_called = False + original_build_request = client.build_request + + def git_run(*_args, **_kwargs): + nonlocal git_called + git_called = True + raise AssertionError("Git must not receive an oversized ref") + + def build_request(*args, **kwargs): + nonlocal request_built + request_built = True + return original_build_request(*args, **kwargs) + + def opener(*_args, **_kwargs): + nonlocal opener_called + opener_called = True + return Response(_draft_response()) + + monkeypatch.setattr(client._validate_ref.__globals__["subprocess"], "run", git_run) + client.build_request = build_request + refs = {"base": "main", "head": "hermes/fix"} + refs[field] = oversized + with pytest.raises(client.PolicyError, match="branch-name limit"): + client.create_draft( + "cassandra", + base=refs["base"], + head=refs["head"], + head_sha=HEAD_SHA, + title="Focused fix", + body="Review evidence", + token="runtime", + opener=opener, + ) + assert git_called is False + assert request_built is False + assert opener_called is False + + +@pytest.mark.parametrize( + ("field", "oversized"), + [("repo", "r" * 101), ("title", "🧪" * 200), ("body", "🧪" * 9_000)], +) +def test_other_text_bounds_fail_before_git_request_or_opener( + field: str, oversized: str, monkeypatch +): + client = _load() + git_called = False + request_built = False + opener_called = False + original_build_request = client.build_request + + def git_run(*_args, **_kwargs): + nonlocal git_called + git_called = True + raise AssertionError("Git must not run before cheap input bounds") + + def build_request(*args, **kwargs): + nonlocal request_built + request_built = True + return original_build_request(*args, **kwargs) + + def opener(*_args, **_kwargs): + nonlocal opener_called + opener_called = True + return Response(_draft_response()) + + monkeypatch.setattr(client._validate_ref.__globals__["subprocess"], "run", git_run) + client.build_request = build_request + values = { + "repo": "cassandra", + "title": "Focused fix", + "body": "Review evidence", + } + values[field] = oversized + with pytest.raises(client.PolicyError): + client.create_draft( + values["repo"], + base="main", + head="hermes/fix", + head_sha=HEAD_SHA, + title=values["title"], + body=values["body"], + token="runtime", + opener=opener, + ) + assert git_called is False + assert request_built is False + assert opener_called is False + + +def test_create_forces_draft_title_and_same_repository_branch_names(): + client = _load() + assert ( + client.authorize_request( + "POST", "/api/v1/repos/atlas/cassandra/pulls", _draft_payload() + ) + == "create-draft" + ) + with pytest.raises(client.PolicyError, match="draft-title prefix"): + client.authorize_request( + "POST", + "/api/v1/repos/atlas/cassandra/pulls", + _draft_payload(title="Not a draft"), + ) + with pytest.raises(client.PolicyError): + client.authorize_request( + "POST", + "/api/v1/repos/atlas/cassandra/pulls", + _draft_payload(head="someone:branch"), + ) + + +def test_runtime_token_is_only_an_authorization_header(): + client = _load() + request = client.build_request( + "POST", + "/api/v1/repos/atlas/cassandra/pulls", + base_url=client.CANONICAL_BASE_URL, + token="do-not-leak", + data=_draft_payload(), + ) + + assert "do-not-leak" not in request.full_url + assert b"do-not-leak" not in request.data + assert request.get_header("Authorization") == "token do-not-leak" diff --git a/testing/tests/test_hermes_gitea_pr_dlp.py b/testing/tests/test_hermes_gitea_pr_dlp.py new file mode 100644 index 00000000..4a2b1d27 --- /dev/null +++ b/testing/tests/test_hermes_gitea_pr_dlp.py @@ -0,0 +1,317 @@ +"""Credential-loss-prevention contracts for Atlas draft creation.""" + +from __future__ import annotations + +import pytest + +from testing.tests.test_hermes_gitea_support import ( + HEAD_SHA, + Response, + _draft_response, + _load, +) + + +@pytest.mark.parametrize( + "sensitive", + [ + "client_" + "secret=not-a-real-value", + '{"client_' + 'secret":\n"synthetic-value"}', + '{"client_' + 'secret"\n:\n"synthetic-value"}', + '{"client\\u005f' + 'secret":"synthetic-value"}', + "client_" + "secret:\n synthetic-value", + '{"client\\q' + 'secret":"synthetic-value"}', + '{"client\n' + 'secret":"synthetic-value"}', + '"client_' + 'secret"\x0b:\n"synthetic-value"', + "ACCESS_" + "TOKEN = 'not-a-real-value'", + '{"refresh_' + 'token": "not-a-real-value"}', + "private_" + "key: not-a-real-value", + "AWS_SECRET_ACCESS_" + "KEY=not-a-real-value", + "aws_access_key_" + "id: not-a-real-value", + "AWS_SESSION_" + "TOKEN = 'not-a-real-value'", + '{"AccessKey' + 'Id":"not-a-real-value"}', + '{"SecretAccess' + 'Key":"not-a-real-value"}', + '{"Session' + 'Token":"not-a-real-value"}', + "aws-security-" + "token: not-a-real-value", + "Account" + "Key=not-a-real-value", + "SharedAccess" + "Signature: not-a-real-value", + "AZURE_STORAGE_CONNECTION_" + "STRING='not-a-real-value'", + "DefaultEndpointsProtocol=https;AccountName=fake;Account" + + "Key=not-a-real-value;EndpointSuffix=example", + "?sv=2024-11-04&ss=b&srt=sco&sp=rwdlac&se=2099-01-01&sig=" + "not-a-real-value", + "DOCKER_AUTH_" + "CONFIG='not-a-real-value'", + '{"auths":{"registry.example":{"auth":"bm90LXJlYWw="}}}', + '{"identity' + 'token":"not-a-real-value"}', + '{"type":"service_' + 'account","client_email":"fake@example.test"}', + '{"private_key_' + 'id":"not-a-real-value"}', + '{"client_' + 'email":"fake@example.test"}', + "GOOGLE_CREDENTIALS" + "=not-a-real-value", + "personal_access_" + "token: not-a-real-value", + "GITEA_" + "TOKEN=not-a-real-value", + "gitlab-token" + ": not-a-real-value", + "pat" + "=not-a-real-value", + "Authorization: " + "Bearer not-a-real-credential-value", + "authorization = " + '"Basic not-a-real-credential-value"', + "Bearer" + "=not-a-real-credential-value", + "Basic" + ": not-a-real-credential-value", + "pass" + "word=not-a-real-credential", + "ghp_" + "notarealcredentialvalue123456", + "github_pat_" + "notarealcredentialvalue123456", + "glpat-" + "notarealcredentialvalue123456", + "xoxb-" + "not-a-real-credential-value-123456", + "sk-ant-" + "notarealcredentialvalue123456", + "sk-proj-" + "notarealcredentialvalue123456", + "sk_live_" + "notarealcredentialvalue123456", + "ya29." + "notarealcredentialvalue123456", + "gta_" + "notarealcredentialvalue123456", + "whsec_" + "notarealcredentialvalue123456", + "npm_" + "notarealcredentialvalue123456", + "pypi-" + "notarealcredentialvalue123456789012345", + "hf_" + "notarealcredentialvalue123456", + "SG." + "notarealvalue1234" + ".notarealcredentialvalue123456", + "SK" + "a" * 32, + "https://hooks.slack.com/services/" + "T000/B000/notarealvalue123456", + "https://discord.com/api/webhooks/123456789/" + "notarealcredentialvalue123456", + "https://fake.webhook.office.com/" + "notarealcredentialvalue123456", + "webhook_" + "url=https://example.test/not-real", + "FutureCloudSigning" + "Credential=not-a-real-value", + "future-client-signing-" + "key: not-a-real-value", + "future_client_signing_" + "key='not-a-real-value'", + '{"serviceAccountPrivate' + 'Key":"not-a-real-value"}', + "CONTAINER_REGISTRY_" + "CREDENTIAL=not-a-real-value", + "someWebhookSigning" + "Secret: not-a-real-value", + "client" + "Key=not-a-real-value", + "session" + "Key: not-a-real-value", + "access" + "Id=not-a-real-value", + "credentials" + ": {user: fake}", + "private" + "Key: |", + "nuget_api_" + "key=not-a-real-value", + "oy2" + "a" * 44, + "sk_test_" + "notarealcredentialvalue123456", + "A1b2C3d4E5f6G7h8I9j0K_l-M+n/O=pQ2rS3tU4vW5xY6zZ7aB8cC9d", + "AKIA" + "A" * 16, + "AIza" + "a" * 35, + "-----BEGIN OPENSSH " + "PRIVATE KEY-----", + "ssh-ed25519 " + "bm90YXJlYWxjcmVkZW50aWFsdmFsdWU=", + "eyJnotarealheader." + "notarealpayloadvalue." + "notarealsignature", + ], +) +@pytest.mark.parametrize("field", ["title", "body"]) +def test_create_rejects_expanded_credential_shapes_before_network( + sensitive: str, field: str +): + client = _load() + called = False + request_built = False + + original_build_request = client.build_request + + def build_request(*args, **kwargs): + nonlocal request_built + request_built = True + return original_build_request(*args, **kwargs) + + def opener(*_args, **_kwargs): + nonlocal called + called = True + return Response(_draft_response()) + + client.build_request = build_request + values = {"title": "Focused fix", "body": "Review evidence"} + values[field] = sensitive + + with pytest.raises(client.PolicyError, match="credential material"): + client.create_draft( + "cassandra", + base="main", + head="hermes/fix", + head_sha=HEAD_SHA, + title=values["title"], + body=values["body"], + token="runtime-sentinel", + opener=opener, + ) + assert request_built is False + assert called is False + + +def test_very_long_compact_body_is_rejected_before_request_or_network(): + client = _load() + request_built = False + opener_called = False + original_build_request = client.build_request + + def build_request(*args, **kwargs): + nonlocal request_built + request_built = True + return original_build_request(*args, **kwargs) + + def opener(*_args, **_kwargs): + nonlocal opener_called + opener_called = True + return Response(_draft_response()) + + client.build_request = build_request + with pytest.raises(client.PolicyError, match="credential material"): + client.create_draft( + "cassandra", + base="main", + head="hermes/fix", + head_sha=HEAD_SHA, + title="Focused fix", + body="a" * 300, + token="runtime-sentinel", + opener=opener, + ) + assert request_built is False + assert opener_called is False + + +@pytest.mark.parametrize( + "safe_text", + [ + "AWS_SECRET_ACCESS_KEY is injected at runtime", + "Token: reject empty values", + "Authorization = preserve header behavior", + "Password: add regression", + "Review the Authorization header behavior", + "Bearer authentication is required for this route", + "Document Docker auths payload rejection", + "The client_email field belongs to service accounts", + "AccountKey assignments must be blocked", + "This patch changes token validation without including a value", + "FutureCloudSigningCredential handling needs a regression test", + "The clientKey name is documented without an assigned value", + "A SHA-256 digest 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef is evidence", + ], +) +def test_credential_policy_keeps_normal_engineering_prose_usable(safe_text: str): + client = _load() + + assert client._validate_body(safe_text) == safe_text + assert client._draft_title(safe_text) == f"WIP: {safe_text}" + + +@pytest.mark.parametrize("field", ["title", "body"]) +def test_create_rejects_exact_runtime_token_before_network(field: str): + client = _load() + called = False + request_built = False + + original_build_request = client.build_request + + def build_request(*args, **kwargs): + nonlocal request_built + request_built = True + return original_build_request(*args, **kwargs) + + def opener(*_args, **_kwargs): + nonlocal called + called = True + return Response(_draft_response()) + + client.build_request = build_request + runtime_token = "exact-random-runtime-sentinel-7b73ac61" + values = {"title": "Focused fix", "body": "Review evidence"} + values[field] = f"Accidental {runtime_token} value" + with pytest.raises(client.PolicyError, match="runtime credential"): + client.create_draft( + "cassandra", + base="main", + head="hermes/fix", + head_sha=HEAD_SHA, + title=values["title"], + body=values["body"], + token=runtime_token, + opener=opener, + ) + assert request_built is False + assert called is False + + +@pytest.mark.parametrize("field", ["repo", "base", "head", "title", "body"]) +def test_every_public_field_rejects_exact_runtime_token_before_git_or_network( + field: str, monkeypatch +): + client = _load() + git_called = False + request_built = False + opener_called = False + runtime_token = "runtime-sentinel" + values = { + "repo": "cassandra", + "base": "main", + "head": "hermes/fix", + "title": "Focused fix", + "body": "Review evidence", + } + values[field] = runtime_token + + def git_run(*_args, **_kwargs): + nonlocal git_called + git_called = True + raise AssertionError("Git must not run for a credential-bearing field") + + def build_request(*_args, **_kwargs): + nonlocal request_built + request_built = True + raise AssertionError("a request must not be built") + + def opener(*_args, **_kwargs): + nonlocal opener_called + opener_called = True + raise AssertionError("the opener must not be called") + + monkeypatch.setattr(client._validate_ref.__globals__["subprocess"], "run", git_run) + monkeypatch.setattr(client, "build_request", build_request) + with pytest.raises(client.PolicyError, match="runtime credential"): + client.create_draft( + values["repo"], + base=values["base"], + head=values["head"], + head_sha=HEAD_SHA, + title=values["title"], + body=values["body"], + token=runtime_token, + opener=opener, + ) + assert git_called is False + assert request_built is False + assert opener_called is False + + +@pytest.mark.parametrize( + "sensitive", + [ + " ".join(["{}"] * 32) + ' {"client_secret":"synthetic-value"}', + "prefix_" + "ghp_" + "notarealcredentialvalue123456_suffix", + "client_secret: correct horse battery staple", + "'client_secret':\n synthetic-value", + ], +) +@pytest.mark.parametrize("field", ["title", "body"]) +def test_structured_scanner_closes_bounded_and_wrapped_token_bypasses( + sensitive: str, field: str +): + client = _load() + values = {"title": "Focused fix", "body": "Review evidence"} + values[field] = sensitive + called = False + + def opener(*_args, **_kwargs): + nonlocal called + called = True + return Response(_draft_response()) + + with pytest.raises(client.PolicyError, match="credential material"): + client.create_draft( + "cassandra", + base="main", + head="hermes/fix", + head_sha=HEAD_SHA, + title=values["title"], + body=values["body"], + token="runtime-sentinel", + opener=opener, + ) + assert called is False diff --git a/testing/tests/test_hermes_gitea_pr_integration.py b/testing/tests/test_hermes_gitea_pr_integration.py new file mode 100644 index 00000000..ee482cfe --- /dev/null +++ b/testing/tests/test_hermes_gitea_pr_integration.py @@ -0,0 +1,174 @@ +"""Runtime and Flux integration contracts for the safe Atlas PR client.""" + +from __future__ import annotations + +import importlib.util +import io +import json +import sys +import urllib.error +from pathlib import Path + +import pytest +import yaml + +ROOT = Path(__file__).parents[2] +CLIENT_PATH = ROOT / "services/hermes/scm-common/scripts/gitea_api.py" +HEAD_SHA = "465cf9146b05c174a2a8d310aff6c64be58277b6" +if str(CLIENT_PATH.parent) not in sys.path: + sys.path.insert(0, str(CLIENT_PATH.parent)) + + +def _load(): + spec = importlib.util.spec_from_file_location( + "safe_gitea_api_integration", CLIENT_PATH + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def test_dry_run_has_no_file_input_and_uses_no_token_or_network(monkeypatch, capsys): + client = _load() + monkeypatch.setattr(client, "read_token", lambda: pytest.fail("read token")) + monkeypatch.setattr( + client, "_safe_urlopen", lambda *_a, **_k: pytest.fail("network") + ) + + assert ( + client.main( + [ + "--dry-run", + "create-draft", + "cassandra", + "--base", + "main", + "--head", + "hermes/fix", + "--head-sha", + HEAD_SHA, + "--title", + "Repair", + "--body", + "Evidence only", + ] + ) + == 0 + ) + output = json.loads(capsys.readouterr().out) + assert output["operation"] == "create-draft" + assert output["owner"] == "atlas" + assert "Evidence only" not in json.dumps(output) + with pytest.raises(SystemExit): + client.parse_args( + [ + "create-draft", + "cassandra", + "--base", + "main", + "--head", + "hermes/fix", + "--head-sha", + HEAD_SHA, + "--title", + "Repair", + "--body-file", + "/runtime-access/gitea-token", + ] + ) + + +def test_http_error_path_redacts_token(monkeypatch, capsys): + client = _load() + monkeypatch.setattr(client, "read_token", lambda: "do-not-leak") + + def fail(*_args, **_kwargs): + raise urllib.error.HTTPError( + "https://scm.bstein.dev/api/v1/repos/atlas/cassandra", + 403, + "forbidden", + {}, + io.BytesIO(b"Authorization: token do-not-leak"), + ) + + monkeypatch.setattr(client, "read", lambda *_a, **_k: fail()) + + assert client.main(["read", "/api/v1/repos/atlas/cassandra"]) == 1 + captured = capsys.readouterr() + assert "do-not-leak" not in captured.err + assert "credential was disclosed" in captured.err + + +def test_flux_manifest_isolates_vault_token_in_separate_broker_only(): + client_source = CLIENT_PATH.read_text(encoding="utf-8") + assert "scm_broker_client" in client_source + assert "GITEA_TOKEN" not in client_source + + deployment = yaml.safe_load( + (ROOT / "services/hermes/agent-deployment.yaml").read_text(encoding="utf-8") + ) + template = deployment["spec"]["template"] + annotations = template["metadata"]["annotations"] + assert not any("gitea" in key.lower() for key in annotations) + runtime = next( + volume + for volume in template["spec"]["volumes"] + if volume["name"] == "runtime-access" + ) + assert runtime["emptyDir"]["medium"] == "Memory" + + expected = {"hermes", "terminal", "cli-lane-runner"} + mounted = { + container["name"] + for container in template["spec"]["containers"] + if any( + mount["name"] == "atlas-pr-skill" + and mount["mountPath"] + == "/opt/data/workspace/skills/manage-atlas-pull-requests" + and mount.get("readOnly") is True + for mount in container.get("volumeMounts", []) + ) + } + assert mounted == expected + + kustomization = yaml.safe_load( + (ROOT / "services/hermes/kustomization.yaml").read_text(encoding="utf-8") + ) + common = yaml.safe_load( + (ROOT / "services/hermes/scm-common/kustomization.yaml").read_text( + encoding="utf-8" + ) + ) + boundary = common["configMapGenerator"][0] + assert boundary["name"] == "hermes-scm-boundary" + assert "gitea_api.py=scripts/gitea_api.py" in boundary["files"] + assert "gitea_api_policy.py=scripts/gitea_api_policy.py" in boundary["files"] + assert "scm_broker_client.py=scripts/scm_broker_client.py" in boundary["files"] + assert not any("gitea_askpass" in item for item in boundary["files"]) + generator = next( + item + for item in kustomization["configMapGenerator"] + if item["name"] == "hermes-atlas-pr-skill" + ) + assert generator["files"] == [ + "SKILL.md=skills/manage-atlas-pull-requests/SKILL.md", + "openai.yaml=skills/manage-atlas-pull-requests/agents/openai.yaml", + ] + + broker = yaml.safe_load( + (ROOT / "services/hermes-scm-broker/deployment.yaml").read_text( + encoding="utf-8" + ) + ) + assert broker["metadata"]["namespace"] == "hermes-scm" + pod = broker["spec"]["template"] + assert pod["spec"]["serviceAccountName"] == "hermes-scm-broker" + assert pod["metadata"]["annotations"][ + "vault.hashicorp.com/agent-inject-secret-gitea-token" + ] == "kv/data/atlas/hermes/developer-gitea" + assert not any( + volume.get("hostPath") or volume.get("persistentVolumeClaim") + for volume in pod["spec"]["volumes"] + ) diff --git a/testing/tests/test_hermes_gitea_pr_postconditions.py b/testing/tests/test_hermes_gitea_pr_postconditions.py new file mode 100644 index 00000000..c06a7784 --- /dev/null +++ b/testing/tests/test_hermes_gitea_pr_postconditions.py @@ -0,0 +1,199 @@ +"""Response and postcondition contracts for Atlas draft creation.""" + +from __future__ import annotations + +import base64 +import copy +import json + +import pytest + +from testing.tests.test_hermes_gitea_support import ( + HEAD_SHA, + Response, + _draft_response, + _load, +) + +@pytest.mark.parametrize("status", [200, 202, 204, 206]) +def test_create_accepts_only_http_201(status: int): + client = _load() + + with pytest.raises(client.PolicyError, match="unexpected HTTP status"): + client.create_draft( + "cassandra", + base="main", + head="hermes/fix", + head_sha=HEAD_SHA, + title="Focused fix", + body="Review evidence", + token="runtime", + opener=lambda *_a, **_k: Response(_draft_response(), status=status), + ) + + +@pytest.mark.parametrize("status", [201, 202, 204, 206]) +def test_read_accepts_only_http_200(status: int): + client = _load() + + with pytest.raises(client.PolicyError, match="unexpected HTTP status"): + client.read( + "/api/v1/repos/atlas/cassandra", + token="runtime", + opener=lambda *_a, **_k: Response(b"{}", status=status), + ) + + +def test_successful_create_validates_each_ref_once(monkeypatch): + client = _load() + calls = [] + + def git_run(command, **_kwargs): + calls.append(command) + return type("Result", (), {"returncode": 0})() + + monkeypatch.setattr(client._validate_ref.__globals__["subprocess"], "run", git_run) + client.create_draft( + "cassandra", + base="main", + head="hermes/fix", + head_sha=HEAD_SHA, + title="Focused fix", + body="Review evidence", + token="runtime", + opener=lambda *_a, **_k: Response(_draft_response()), + ) + + assert [command[-1] for command in calls] == [ + "refs/heads/main", + "refs/heads/hermes/fix", + ] + + +def test_create_verifies_every_server_postcondition(): + client = _load() + calls = [] + + def opener(request, timeout): + calls.append((request, timeout)) + return Response(_draft_response()) + + result = client.create_draft( + "cassandra", + base="main", + head="hermes/fix", + head_sha=HEAD_SHA, + title="Focused fix", + body="Review evidence", + token="runtime", + opener=opener, + ) + + assert json.loads(result) == _draft_response() + payload = json.loads(calls[0][0].data) + assert payload == { + "base": "main", + "body": "Review evidence", + "head": "hermes/fix", + "title": "WIP: Focused fix", + } + assert calls[0][1] == 30 + + +def test_create_postcondition_rejects_every_material_mismatch(): + client = _load() + mutations = [ + ("number", 0), + ("number", 2_147_483_648), + ("state", "closed"), + ("draft", False), + ("merged", True), + ("html_url", "https://evil.example/pulls/3"), + ("url", "https://evil.example/api/pulls/3"), + ("title", "Focused fix"), + ("body", "different"), + ] + documents = [] + for key, value in mutations: + document = _draft_response() + document[key] = value + documents.append(document) + for path, value in [ + (("base", "ref"), "master"), + (("base", "repo", "full_name"), "evil/cassandra"), + (("head", "ref"), "other"), + (("head", "sha"), "0" * 40), + (("head", "repo", "full_name"), "evil/cassandra"), + ]: + document = copy.deepcopy(_draft_response()) + target = document + for key in path[:-1]: + target = target[key] + target[path[-1]] = value + documents.append(document) + + for document in documents: + with pytest.raises(client.PolicyError): + client._require_create_response( + json.dumps(document).encode(), + repo="cassandra", + base="main", + head="hermes/fix", + head_sha=HEAD_SHA, + title="WIP: Focused fix", + body="Review evidence", + ) + + +def test_read_response_is_bounded(): + client = _load() + + with pytest.raises(client.PolicyError, match="safe size limit"): + client.read( + "/api/v1/repos/atlas/cassandra", + token="runtime", + opener=lambda *_a, **_k: Response(b"x" * (client.MAX_RESPONSE_BYTES + 1)), + ) + + +def test_direct_api_rejects_unexpected_success_content_type(): + client = _load() + response = Response(b"{}") + response.headers.replace_header("Content-Type", "text/html") + + with pytest.raises(client.PolicyError, match="unexpected response type"): + client.read( + "/api/v1/repos/atlas/cassandra", + token="runtime", + opener=lambda *_a, **_k: response, + ) + + +def test_output_redaction_covers_exact_token_and_authorization_header(): + client = _load() + raw = b'{"message":"do-not-leak","debug":"Authorization: token do-not-leak"}' + redacted = client.redact_bytes(raw, "do-not-leak") + + assert b"do-not-leak" not in redacted + assert redacted.count(b"[REDACTED]") >= 1 + + +@pytest.mark.parametrize( + "reflected", + [ + b"runtime-sentinel", + base64.b64encode(b"runtime-sentinel"), + base64.b64encode(b"hermes-automation:runtime-sentinel"), + ], +) +def test_direct_api_rejects_credential_reflection(reflected: bytes): + client = _load() + + with pytest.raises(client.PolicyError, match="credential material"): + client.read( + "/api/v1/repos/atlas/cassandra", + token="runtime-sentinel", + opener=lambda *_a, **_k: Response( + b'{"unexpected":"' + reflected + b'"}' + ), + ) diff --git a/testing/tests/test_hermes_gitea_support.py b/testing/tests/test_hermes_gitea_support.py new file mode 100644 index 00000000..8b00e784 --- /dev/null +++ b/testing/tests/test_hermes_gitea_support.py @@ -0,0 +1,75 @@ +"""Shared fixtures for the Atlas-only Gitea client contracts.""" + +from __future__ import annotations + +import importlib.util +import json +import sys +from email.message import Message +from pathlib import Path + +ROOT = Path(__file__).parents[2] +CLIENT_PATH = ROOT / "services/hermes/scm-common/scripts/gitea_api.py" +HEAD_SHA = "465cf9146b05c174a2a8d310aff6c64be58277b6" +if str(CLIENT_PATH.parent) not in sys.path: + sys.path.insert(0, str(CLIENT_PATH.parent)) + + +def _load(): + spec = importlib.util.spec_from_file_location("safe_gitea_api", CLIENT_PATH) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _draft_payload(**updates): + payload = { + "base": "main", + "body": "Review evidence", + "head": "hermes/review-fix", + "title": "WIP: Repair review findings", + } + payload.update(updates) + return payload + + +def _draft_response(**updates): + response = { + "number": 3, + "state": "open", + "draft": True, + "merged": False, + "html_url": "https://scm.bstein.dev/atlas/cassandra/pulls/3", + "url": "https://scm.bstein.dev/atlas/cassandra/pulls/3", + "title": "WIP: Focused fix", + "body": "Review evidence", + "base": {"ref": "main", "repo": {"full_name": "atlas/cassandra"}}, + "head": { + "ref": "hermes/fix", + "sha": HEAD_SHA, + "repo": {"full_name": "atlas/cassandra"}, + }, + } + response.update(updates) + return response + + +class Response: + def __init__(self, body: object, status: int | None = None): + self.body = body if isinstance(body, bytes) else json.dumps(body).encode() + self.status = ( + status if status is not None else (201 if isinstance(body, dict) else 200) + ) + self.headers = Message() + self.headers["Content-Type"] = "application/json" + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def read(self, limit=-1): + return self.body if limit < 0 else self.body[:limit] diff --git a/testing/tests/test_hermes_image_builder.py b/testing/tests/test_hermes_image_builder.py new file mode 100644 index 00000000..8d7418d3 --- /dev/null +++ b/testing/tests/test_hermes_image_builder.py @@ -0,0 +1,473 @@ +"""Safety and artifact contracts for the Hermes agent image release lane.""" + +from __future__ import annotations + +import importlib.util +import io +import json +import re +import urllib.parse +from pathlib import Path + +import pytest +import yaml + + +REPO_ROOT = Path(__file__).resolve().parents[2] +PIPELINE_PATH = REPO_ROOT / "ci/Jenkinsfile.hermes-agent-image" +RELEASE_SCRIPT = REPO_ROOT / "ci/scripts/hermes_image_release.py" +TRIGGER_SCRIPT = REPO_ROOT / "services/hermes/scripts/jenkins_image_build_trigger.py" +HEREDOC_RUNNER = REPO_ROOT / "dockerfiles/hermes-kaniko-heredoc-runner.py" + + +def _load_release_module(): + spec = importlib.util.spec_from_file_location( + "hermes_image_release", RELEASE_SCRIPT + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _load_trigger_module(): + spec = importlib.util.spec_from_file_location( + "jenkins_image_build_trigger", TRIGGER_SCRIPT + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _load_heredoc_runner(): + spec = importlib.util.spec_from_file_location( + "hermes_kaniko_heredoc_runner", HEREDOC_RUNNER + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _pod_spec() -> dict: + source = PIPELINE_PATH.read_text(encoding="utf-8") + pod_yaml = source.split('yaml """', 1)[1].split('"""', 1)[0] + return yaml.safe_load(pod_yaml)["spec"] + + +def test_builder_pod_is_daemonless_and_kernel_bounded() -> None: + """The builder gets only proven build caps, never host, daemon, or K8s access.""" + source = PIPELINE_PATH.read_text(encoding="utf-8") + spec = _pod_spec() + assert spec["serviceAccountName"] == "hermes-image-builder" + assert spec["automountServiceAccountToken"] is False + assert spec["enableServiceLinks"] is False + assert "hostPath" not in source + assert "docker.sock" not in source + assert "tcp://" not in source + assert "buildkitd" not in source.lower() + assert "dind" not in source.lower() + assert "serviceAccountToken" not in source + + containers = {item["name"]: item for item in spec["containers"]} + kaniko = containers["kaniko"] + assert kaniko["image"] == ( + "gcr.io/kaniko-project/executor@sha256:" + "c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e" + ) + assert kaniko["securityContext"]["capabilities"]["add"] == [ + "CHOWN", + "FOWNER", + "DAC_OVERRIDE", + "SETGID", + "SETUID", + ] + for container in containers.values(): + security = container["securityContext"] + assert security["allowPrivilegeEscalation"] is False + assert security["capabilities"]["drop"] == ["ALL"] + assert security["seccompProfile"]["type"] == "RuntimeDefault" + assert security.get("privileged", False) is False + assert "add" not in containers["jnlp"]["securityContext"]["capabilities"] + assert "add" not in containers["python"]["securityContext"]["capabilities"] + + +def test_pipeline_requires_reviewed_main_and_runtime_credentials() -> None: + """Publishing requires explicit confirmation and a reviewed main commit.""" + source = PIPELINE_PATH.read_text(encoding="utf-8") + assert 'test "${PUBLISH_IMAGE}" = "true"' in source + assert 'test "${CONFIRM_PUBLISH}" = "PUBLISH HERMES AGENT"' in source + assert "git rev-parse origin/main" in source + assert "credentialsId: 'harbor-robot'" in source + assert "set +x" in source + assert "umask 077" in source + assert "unset HARBOR_USER HARBOR_PASSWORD auth" in source + assert "/busybox/rm -f /kaniko/.docker/config.json" in source + assert "--digest-file=" in source + assert "--image-name-tag-with-digest-file=" in source + assert "--destination=" in source + assert "assert-absent" in source + assert "git-${actual_revision}-build-${BUILD_NUMBER}" in source + assert source.count("--build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1") == 1 + + +def test_kaniko_replays_only_the_exact_reviewed_heredoc_contract() -> None: + """Pinned Kaniko's ignored inline files are replayed in exact source order.""" + module = _load_heredoc_runner() + dockerfile = (REPO_ROOT / "dockerfiles/Dockerfile.hermes-agent").read_text() + blocks = module.extract_blocks(dockerfile) + assert tuple(command[0] for command, _body in blocks) == ( + "node", + "python", + "python", + "python", + "python", + "python", + "python", + "python", + "node", + ) + assert "session message total" in blocks[-1][1] + compat = dockerfile.split("ARG HERMES_KANIKO_HEREDOC_COMPAT=0", 1)[1] + assert compat.count("python /tmp/hermes-kaniko-heredoc-runner.py") == 9 + assert [int(value) for value in re.findall(r"--block-index (\d+)", compat)] == list( + range(1, 10) + ) + ignored = ( + REPO_ROOT / "dockerfiles/Dockerfile.hermes-agent.dockerignore" + ).read_text() + assert "!dockerfiles/Dockerfile.hermes-agent" in ignored + assert "!dockerfiles/hermes-kaniko-heredoc-runner.py" in ignored + + +def test_kaniko_heredoc_runner_rejects_drift_and_executes_separately( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """Missing blocks fail closed and each reviewed body gets a fresh process.""" + module = _load_heredoc_runner() + source = (REPO_ROOT / "dockerfiles/Dockerfile.hermes-agent").read_text() + with pytest.raises(ValueError, match="contract changed"): + module.extract_blocks(source.replace("RUN node <<'NODE'", "RUN node", 1)) + + calls = [] + + def run(command, **kwargs): + calls.append((command, kwargs)) + + monkeypatch.setattr(module.subprocess, "run", run) + dockerfile = tmp_path / "Dockerfile" + dockerfile.write_text(source, encoding="utf-8") + for block_index in range(1, 10): + module.replay(dockerfile, block_index) + assert len(calls) == 9 + assert all(call[1]["check"] is True for call in calls) + assert all(call[1]["text"] is True for call in calls) + assert all(call[1]["input"].endswith("\n") for call in calls) + + +def test_jenkins_job_is_manual_and_reads_pipeline_from_main() -> None: + """JCasC must not publish unreviewed branch contents or poll automatically.""" + config = yaml.safe_load( + (REPO_ROOT / "services/jenkins/configmap-jcasc.yaml").read_text( + encoding="utf-8" + ) + ) + jobs = config["data"]["jobs.yaml"] + block = jobs.split("pipelineJob('hermes-agent-image')", 1)[1].split( + "pipelineJob(", 1 + )[0] + assert "branches('*/main')" in block + assert "scriptPath('ci/Jenkinsfile.hermes-agent-image')" in block + assert ( + "authenticationToken(System.getenv('HERMES_AGENT_IMAGE_BUILD_TOKEN'))" in block + ) + assert "pipelineTriggers" not in block + assert "scmTrigger" not in block + + +def test_agent_trigger_is_limited_to_the_image_job(tmp_path: Path) -> None: + """Agent Hermes gets one job token, fixed parameters, and no Jenkins admin API.""" + module = _load_trigger_module() + revision = "a" * 40 + token_path = tmp_path / "token" + token_path.write_text("private-job-token\n", encoding="utf-8") + captured = {} + + class Response(io.BytesIO): + status = 201 + headers = {"Location": "https://ci.bstein.dev/queue/item/42/"} + + def __enter__(self): + return self + + def __exit__(self, *_args): + self.close() + + def opener(request, timeout): + captured["request"] = request + captured["timeout"] = timeout + return Response(b"") + + result = module.trigger_build(revision, token_file=token_path, opener=opener) + request = captured["request"] + fields = urllib.parse.parse_qs(request.data.decode("utf-8")) + assert request.full_url == module.JENKINS_BUILD_URL + assert fields == { + "CONFIRM_PUBLISH": ["PUBLISH HERMES AGENT"], + "EXPECTED_SOURCE_REVISION": [revision], + "PUBLISH_IMAGE": ["true"], + "job": ["hermes-agent-image"], + "token": ["private-job-token"], + } + assert captured["timeout"] == 20 + assert "private-job-token" not in json.dumps(result) + assert result["source_revision"] == revision + + +def test_agent_trigger_rejects_unsafe_revision_and_empty_token(tmp_path: Path) -> None: + """No user-controlled job, URL, or abbreviated revision reaches Jenkins.""" + module = _load_trigger_module() + token_path = tmp_path / "token" + token_path.write_text("token\n", encoding="utf-8") + with pytest.raises(ValueError): + module.trigger_build("main", token_file=token_path) + token_path.write_text("\n", encoding="utf-8") + with pytest.raises(RuntimeError, match="empty"): + module.trigger_build("a" * 40, token_file=token_path) + + +def test_agent_trigger_accepts_existing_queue_redirect(tmp_path: Path) -> None: + """HTTP 303 means the exact release is already queued, not a trigger failure.""" + module = _load_trigger_module() + token_path = tmp_path / "token" + token_path.write_text("token\n", encoding="utf-8") + + class Response(io.BytesIO): + status = 303 + headers = {"Location": "https://ci.bstein.dev/queue/item/7/"} + + def __enter__(self): + return self + + def __exit__(self, *_args): + self.close() + + result = module.trigger_build( + "e" * 40, token_file=token_path, opener=lambda *_args, **_kwargs: Response() + ) + assert result["status"] == 303 + assert result["queue_path"] == "/queue/item/7/" + + +def test_job_token_is_generated_and_injected_only_at_runtime() -> None: + """The fixed-job credential stays in Vault and pod-lifetime memory.""" + plugins = (REPO_ROOT / "services/jenkins/configmap-plugins.yaml").read_text( + encoding="utf-8" + ) + vault = ( + REPO_ROOT / "services/vault/scripts/vault_k8s_auth_configure.sh" + ).read_text(encoding="utf-8") + seeder = ( + REPO_ROOT + / "services/vault-hermes-jenkins-token-seed/scripts/vault_hermes_jenkins_build_token_ensure.sh" + ).read_text(encoding="utf-8") + jenkins = (REPO_ROOT / "services/jenkins/deployment.yaml").read_text( + encoding="utf-8" + ) + agent = (REPO_ROOT / "services/hermes/agent-deployment.yaml").read_text( + encoding="utf-8" + ) + stage = (REPO_ROOT / "services/hermes/scripts/stage_runtime_access.py").read_text( + encoding="utf-8" + ) + assert "build-token-root:365.v717f8685a_09e" in plugins + assert 'write_raw_policy "hermes-jenkins-token-seed"' in vault + assert "sys/tools/random/32 format=hex" in seeder + assert '"options":{"cas":0}' in seeder + assert "kv/data/atlas/hermes/developer-jenkins" in seeder + assert "HERMES_AGENT_IMAGE_BUILD_TOKEN={{ .Data.data.build_token }}" in jenkins + assert "agent-inject-secret-jenkins-image-build-token" in agent + assert '"jenkins-image-build-token"' in stage + + +def test_release_renderer_preserves_manifest_and_emits_safe_artifacts( + tmp_path: Path, +) -> None: + """The release artifact is exact, reviewable, and contains no credentials.""" + module = _load_release_module() + old_digest = "sha256:" + "1" * 64 + new_digest = "sha256:" + "2" * 64 + revision = "a" * 40 + build_number = "17" + manifest = tmp_path / "kustomization.yaml" + manifest.write_text( + "apiVersion: kustomize.config.k8s.io/v1beta1\n" + "kind: Kustomization\n" + "images:\n" + f" - name: {module.DEFAULT_IMAGE}\n" + f" digest: {old_digest}\n", + encoding="utf-8", + ) + output = tmp_path / "out" + metadata = module.write_release_artifacts( + digest=f"{new_digest}\n", + source_revision=revision, + build_number=build_number, + destination=f"{module.DEFAULT_IMAGE}:git-{revision}-build-{build_number}", + kustomization=manifest, + output_dir=output, + ) + + assert manifest.read_text(encoding="utf-8").endswith(f"{old_digest}\n") + assert ( + (output / "hermes-kustomization.yaml") + .read_text(encoding="utf-8") + .endswith(f"{new_digest}\n") + ) + patch = (output / "hermes-image-update.patch").read_text(encoding="utf-8") + assert f"- digest: {old_digest}" in patch + assert f"+ digest: {new_digest}" in patch + assert ( + json.loads((output / "hermes-agent-image.json").read_text(encoding="utf-8")) + == metadata + ) + assert set(metadata) == { + "digest", + "build_number", + "flux_image", + "image", + "published_tag", + "source_revision", + } + + +@pytest.mark.parametrize( + ("digest", "revision", "destination"), + [ + ( + "latest", + "a" * 40, + "registry.bstein.dev/bstein/hermes-agent:git-" + "a" * 40 + "-build-1", + ), + ( + "sha256:" + "1" * 64, + "short", + "registry.bstein.dev/bstein/hermes-agent:git-short-build-1", + ), + ( + "sha256:" + "1" * 64, + "a" * 40, + "registry.bstein.dev/bstein/hermes-agent:latest", + ), + ], +) +def test_release_renderer_rejects_unpinned_inputs( + tmp_path: Path, digest: str, revision: str, destination: str +) -> None: + """Only exact digests and immutable source-derived tags are accepted.""" + module = _load_release_module() + manifest = tmp_path / "kustomization.yaml" + manifest.write_text( + "images:\n" + f" - name: {module.DEFAULT_IMAGE}\n" + " digest: sha256:" + "0" * 64 + "\n", + encoding="utf-8", + ) + with pytest.raises(ValueError): + module.write_release_artifacts( + digest=digest, + source_revision=revision, + build_number="1", + destination=destination, + kustomization=manifest, + output_dir=tmp_path / "out", + ) + + +def test_release_renderer_fails_closed_on_manifest_drift(tmp_path: Path) -> None: + """Missing, duplicate, or already-current image entries require human review.""" + module = _load_release_module() + digest = "sha256:" + "f" * 64 + missing = "images:\n - name: example.invalid/other\n digest: " + digest + "\n" + with pytest.raises(ValueError, match="found 0"): + module.render_kustomization(missing, digest) + + duplicate = ( + "images:\n" + f" - name: {module.DEFAULT_IMAGE}\n digest: {digest}\n" + f" - name: {module.DEFAULT_IMAGE}\n digest: {digest}\n" + ) + with pytest.raises(ValueError, match="found 2"): + module.render_kustomization(duplicate, digest) + + manifest = tmp_path / "kustomization.yaml" + manifest.write_text( + f"images:\n - name: {module.DEFAULT_IMAGE}\n digest: {digest}\n", + encoding="utf-8", + ) + revision = "b" * 40 + with pytest.raises(ValueError, match="already matches"): + module.write_release_artifacts( + digest=digest, + source_revision=revision, + build_number="1", + destination=f"{module.DEFAULT_IMAGE}:git-{revision}-build-1", + kustomization=manifest, + output_dir=tmp_path / "out", + ) + + +def test_release_cli_reads_digest_file( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The pipeline CLI uses the Kaniko digest file as its sole digest input.""" + module = _load_release_module() + digest = "sha256:" + "c" * 64 + revision = "d" * 40 + manifest = tmp_path / "kustomization.yaml" + manifest.write_text( + f"images:\n - name: {module.DEFAULT_IMAGE}\n digest: sha256:" + + "0" * 64 + + "\n", + encoding="utf-8", + ) + digest_file = tmp_path / "digest" + digest_file.write_text(digest + "\n", encoding="utf-8") + destination = f"{module.DEFAULT_IMAGE}:git-{revision}-build-9" + image_file = tmp_path / "image" + image_file.write_text(f"{destination}@{digest}\n", encoding="utf-8") + output = tmp_path / "out" + monkeypatch.setenv("HARBOR_USER", "robot") + monkeypatch.setenv("HARBOR_PASSWORD", "secret") + monkeypatch.setattr( + module, "verify_registry_digest", lambda *_args, **_kwargs: None + ) + monkeypatch.setattr( + "sys.argv", + [ + "hermes_image_release.py", + "render", + "--digest-file", + str(digest_file), + "--image-file", + str(image_file), + "--source-revision", + revision, + "--build-number", + "9", + "--destination", + destination, + "--kustomization", + str(manifest), + "--output-dir", + str(output), + ], + ) + assert module.main() == 0 + assert ( + json.loads((output / "hermes-agent-image.json").read_text(encoding="utf-8"))[ + "digest" + ] + == digest + ) diff --git a/testing/tests/test_hermes_image_builder_adversarial.py b/testing/tests/test_hermes_image_builder_adversarial.py new file mode 100644 index 00000000..be65e1de --- /dev/null +++ b/testing/tests/test_hermes_image_builder_adversarial.py @@ -0,0 +1,469 @@ +"""Adversarial boundaries for the Hermes image publisher and trigger.""" + +from __future__ import annotations + +import importlib.util +import io +import json +from pathlib import Path + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parents[2] +RELEASE_SCRIPT = REPO_ROOT / "ci/scripts/hermes_image_release.py" +TRIGGER_SCRIPT = REPO_ROOT / "services/hermes/scripts/jenkins_image_build_trigger.py" + + +def _load(path: Path, name: str): + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class Response(io.BytesIO): + """Small context-managed HTTP response fixture.""" + + def __init__( + self, + status: int, + headers: dict[str, str] | None = None, + body: bytes = b"", + ): + super().__init__(body) + self.status = status + self.headers = headers or {} + + +def test_kaniko_evidence_binds_digest_destination_and_unique_build() -> None: + """Both Kaniko artifacts must describe one exact non-replayable tag.""" + module = _load(RELEASE_SCRIPT, "hermes_image_release_evidence") + revision = "a" * 40 + digest = "sha256:" + "b" * 64 + destination = f"{module.DEFAULT_IMAGE}:git-{revision}-build-42" + assert module.validate_destination(destination, revision, "42") == ( + revision, + "42", + ) + assert ( + module.validate_kaniko_evidence( + digest_text=f"{digest}\n", + image_text=f"{destination}@{digest}\n", + destination=destination, + ) + == digest + ) + + +@pytest.mark.parametrize( + ("digest_text", "image_template"), + [ + ("sha256:" + "a" * 64 + "\nextra\n", "{destination}@{digest}"), + ("sha256:" + "a" * 64, "{destination}@sha256:" + "b" * 64), + ("sha256:" + "a" * 64, "registry.invalid/x:y@{digest}"), + ("sha256:" + "a" * 64, "{destination}@{digest}\nextra"), + ], +) +def test_kaniko_evidence_rejects_cross_artifact_mismatch( + digest_text: str, image_template: str +) -> None: + """A digest, tag, repository, or cardinality mismatch stops rendering.""" + module = _load(RELEASE_SCRIPT, "hermes_image_release_mismatch") + revision = "c" * 40 + digest = "sha256:" + "a" * 64 + destination = f"{module.DEFAULT_IMAGE}:git-{revision}-build-8" + image_text = image_template.format(destination=destination, digest=digest) + with pytest.raises(ValueError): + module.validate_kaniko_evidence( + digest_text=digest_text, + image_text=image_text, + destination=destination, + ) + + +def test_registry_preflight_rejects_existing_tag_and_auth_failures() -> None: + """Only an authenticated 404 permits Kaniko to claim the unique tag.""" + module = _load(RELEASE_SCRIPT, "hermes_image_release_preflight") + destination = f"{module.DEFAULT_IMAGE}:git-{'d' * 40}-build-3" + captured = {} + + def missing(request, timeout): + captured["request"] = request + captured["timeout"] = timeout + return Response(404) + + module.assert_tag_absent( + destination, username="robot", password="private", opener=missing + ) + request = captured["request"] + assert request.method == "GET" + assert request.full_url.startswith( + "https://registry.bstein.dev/api/v2.0/projects/bstein/repositories/" + "hermes-agent/artifacts/" + ) + assert "private" not in request.full_url + assert captured["timeout"] == 20 + + for status, message in ( + (200, "already exists"), + (401, "HTTP 401"), + (503, "HTTP 503"), + ): + with pytest.raises(RuntimeError, match=message): + module.assert_tag_absent( + destination, + username="robot", + password="private", + opener=lambda *_args, code=status: Response(code), + ) + + +def test_registry_preflight_requires_exact_server_immutable_policy() -> None: + """The already-running controller cannot publish before Flux installs the rule.""" + module = _load(RELEASE_SCRIPT, "hermes_image_release_policy_preflight") + expected = { + "disabled": False, + "action": "immutable", + "template": "immutable_template", + "tag_selectors": [ + { + "kind": "doublestar", + "decoration": "matches", + "pattern": "git-*-build-*", + } + ], + "scope_selectors": { + "repository": [ + { + "kind": "doublestar", + "decoration": "repoMatches", + "pattern": "hermes-agent", + } + ] + }, + } + captured = {} + + def exact(request, timeout): + captured["url"] = request.full_url + captured["timeout"] = timeout + return Response( + 200, + {"X-Total-Count": "1"}, + body=json.dumps([{"id": 9, **expected}]).encode(), + ) + + module.verify_immutable_policy(username="robot", password="private", opener=exact) + assert captured["url"].endswith( + "/projects/bstein/immutabletagrules?page=1&page_size=100" + ) + assert captured["timeout"] == 20 + + for status, body, headers, message in ( + (403, b"", {}, "HTTP 403"), + (200, b"[]", {"X-Total-Count": "0"}, "absent"), + ( + 200, + json.dumps([{**expected, "disabled": True}]).encode(), + {"X-Total-Count": "1"}, + "not exact", + ), + (200, b"not-json", {}, "invalid"), + ( + 200, + json.dumps([expected]).encode(), + {"X-Total-Count": "2"}, + "truncated", + ), + ): + with pytest.raises(RuntimeError, match=message): + module.verify_immutable_policy( + username="robot", + password="private", + opener=lambda *_args, code=status, value=body, evidence=headers: Response( + code, evidence, body=value + ), + ) + + +def test_harbor_client_fails_closed_on_input_size_auth_and_json( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Malformed inputs, oversized bodies, missing auth, and bad JSON all fail.""" + module = _load(RELEASE_SCRIPT, "hermes_image_release_harbor_errors") + destination = f"{module.DEFAULT_IMAGE}:git-{'1' * 40}-build-6" + with pytest.raises(ValueError, match="destination"): + module.assert_tag_absent( + "registry.invalid/x:tag", username="robot", password="private" + ) + with pytest.raises(RuntimeError, match="credentials"): + module.assert_tag_absent(destination, username="", password="private") + with pytest.raises(RuntimeError, match="size limit"): + module.assert_tag_absent( + destination, + username="robot", + password="private", + opener=lambda *_args: Response(200, body=b"x" * 1_048_577), + ) + with pytest.raises(RuntimeError, match="invalid artifact JSON"): + module.verify_registry_digest( + destination, + "sha256:" + "2" * 64, + username="robot", + password="private", + opener=lambda *_args: Response(200, body=b"\xff"), + ) + monkeypatch.delenv("HARBOR_USER", raising=False) + monkeypatch.delenv("HARBOR_PASSWORD", raising=False) + with pytest.raises(RuntimeError, match="unavailable"): + module._credentials() + + +def test_default_harbor_opener_returns_http_responses( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The real wrapper returns both normal and non-redirect HTTP responses.""" + module = _load(RELEASE_SCRIPT, "hermes_image_release_opener") + request = module.urllib.request.Request("https://registry.bstein.dev/test") + + class SuccessOpener: + def open(self, _request, timeout): + assert timeout == 7 + return Response(204) + + monkeypatch.setattr( + module.urllib.request, "build_opener", lambda *_handlers: SuccessOpener() + ) + assert module._registry_request(request, 7).status == 204 + + class ErrorOpener: + def open(self, _request, timeout): + assert timeout == 8 + raise module.urllib.error.HTTPError( + _request.full_url, 404, "missing", {}, None + ) + + monkeypatch.setattr( + module.urllib.request, "build_opener", lambda *_handlers: ErrorOpener() + ) + assert module._registry_request(request, 8).code == 404 + + +@pytest.mark.parametrize( + ("status", "artifact", "message"), + [ + (404, {}, "HTTP 404"), + (200, {}, "omitted"), + (200, {"digest": "sha256:" + "2" * 64}, "does not match"), + (200, {"digest": "latest"}, "omitted"), + ], +) +def test_registry_verification_rejects_missing_or_mismatched_digest( + status: int, artifact: dict[str, str], message: str +) -> None: + """Rendering requires an independent exact Harbor digest response.""" + module = _load(RELEASE_SCRIPT, "hermes_image_release_registry") + digest = "sha256:" + "1" * 64 + destination = f"{module.DEFAULT_IMAGE}:git-{'e' * 40}-build-4" + body = json.dumps(artifact).encode("utf-8") + with pytest.raises(RuntimeError, match=message): + module.verify_registry_digest( + destination, + digest, + username="robot", + password="private", + opener=lambda *_args: Response(status, body=body), + ) + + +def test_registry_verification_accepts_exact_pushed_digest() -> None: + """An exact authenticated Harbor digest allows artifact rendering.""" + module = _load(RELEASE_SCRIPT, "hermes_image_release_registry_ok") + digest = "sha256:" + "4" * 64 + destination = f"{module.DEFAULT_IMAGE}:git-{'f' * 40}-build-5" + with pytest.raises(RuntimeError, match="expected tag"): + module.verify_registry_digest( + destination, + digest, + username="robot", + password="private", + opener=lambda *_args: Response( + 200, + body=json.dumps( + {"digest": digest, "tags": [{"name": "different-tag"}]} + ).encode("utf-8"), + ), + ) + module.verify_registry_digest( + destination, + digest, + username="robot", + password="private", + opener=lambda *_args: Response( + 200, + body=json.dumps( + { + "digest": digest, + "tags": [ + { + "name": destination.rsplit(":", 1)[1], + "immutable": True, + } + ], + } + ).encode("utf-8"), + ), + ) + + +def test_registry_verification_rejects_tag_without_server_immutability() -> None: + """Digest evidence is insufficient unless Harbor reports the build tag immutable.""" + module = _load(RELEASE_SCRIPT, "hermes_image_release_registry_mutable") + digest = "sha256:" + "7" * 64 + destination = f"{module.DEFAULT_IMAGE}:git-{'8' * 40}-build-19" + captured = {} + + def opener(request, _timeout): + captured["url"] = request.full_url + return Response( + 200, + body=json.dumps( + { + "digest": digest, + "tags": [ + { + "name": destination.rsplit(":", 1)[1], + "immutable": False, + } + ], + } + ).encode(), + ) + + with pytest.raises(RuntimeError, match="immutable"): + module.verify_registry_digest( + destination, + digest, + username="robot", + password="private", + opener=opener, + ) + assert captured["url"].endswith("?with_immutable_status=true") + + +def test_release_main_preflight_uses_fixed_credentials_and_destination( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The preflight CLI validates and checks exactly the requested build tag.""" + module = _load(RELEASE_SCRIPT, "hermes_image_release_main_absent") + revision = "3" * 40 + destination = f"{module.DEFAULT_IMAGE}:git-{revision}-build-12" + captured = {} + monkeypatch.setenv("HARBOR_USER", "robot") + monkeypatch.setenv("HARBOR_PASSWORD", "private") + + def assert_absent(value, **credentials): + captured["destination"] = value + captured["credentials"] = credentials + + def verify_policy(**credentials): + captured["policy_credentials"] = credentials + + monkeypatch.setattr(module, "assert_tag_absent", assert_absent) + monkeypatch.setattr(module, "verify_immutable_policy", verify_policy) + monkeypatch.setattr( + "sys.argv", + [ + "hermes_image_release.py", + "assert-absent", + "--source-revision", + revision, + "--build-number", + "12", + "--destination", + destination, + ], + ) + assert module.main() == 0 + assert captured == { + "destination": destination, + "credentials": {"username": "robot", "password": "private"}, + "policy_credentials": {"username": "robot", "password": "private"}, + } + + +def test_renderer_skips_a_matching_entry_without_a_digest() -> None: + """Only the one complete image entry is changed when an earlier entry drifts.""" + module = _load(RELEASE_SCRIPT, "hermes_image_release_manifest_scan") + digest = "sha256:" + "5" * 64 + source = ( + "images:\n" + f" - name: {module.DEFAULT_IMAGE}\n" + " newTag: ignored\n" + " - name: example.invalid/other\n" + " newTag: stable\n" + f" - name: {module.DEFAULT_IMAGE}\n" + " digest: sha256:" + "0" * 64 + "\n" + ) + assert module.render_kustomization(source, digest).endswith(f"{digest}\n") + + +@pytest.mark.parametrize("status", [200, 202, 204, 301, 302, 307, 308]) +def test_trigger_rejects_non_plugin_success_status(tmp_path: Path, status: int) -> None: + """Generic proxy successes and redirects are not proof of a queued build.""" + module = _load(TRIGGER_SCRIPT, f"jenkins_trigger_status_{status}") + token = tmp_path / "token" + token.write_text("private\n", encoding="utf-8") + response = lambda *_args, **_kwargs: Response( # noqa: E731 + status, {"Location": "https://ci.bstein.dev/queue/item/9/"} + ) + with pytest.raises(RuntimeError, match=f"HTTP {status}"): + module.trigger_build("a" * 40, token_file=token, opener=response) + + +@pytest.mark.parametrize( + "location", + [ + "", + "https://evil.invalid/queue/item/9/", + "http://ci.bstein.dev/queue/item/9/", + "https://ci.bstein.dev/job/hermes-agent-image/9/", + "https://ci.bstein.dev/queue/item/9/?token=leak", + "https://ci.bstein.dev/queue/item/9/#fragment", + "https://ci.bstein.dev/queue/item/not-a-number/", + ], +) +def test_trigger_rejects_missing_malformed_or_cross_origin_queue_location( + tmp_path: Path, location: str +) -> None: + """A real accepted status still needs the exact same-origin queue resource.""" + module = _load(TRIGGER_SCRIPT, "jenkins_trigger_location") + token = tmp_path / "token" + token.write_text("private\n", encoding="utf-8") + response = lambda *_args, **_kwargs: Response( # noqa: E731 + 201, {"Location": location} + ) + with pytest.raises(RuntimeError, match="Location"): + module.trigger_build("b" * 40, token_file=token, opener=response) + + +def test_trigger_uses_https_and_accepts_exact_relative_queue_path( + tmp_path: Path, +) -> None: + """A relative plugin Location is resolved only against the fixed HTTPS origin.""" + module = _load(TRIGGER_SCRIPT, "jenkins_trigger_origin") + token = tmp_path / "token" + token.write_text("private\n", encoding="utf-8") + captured = {} + + def opener(request, timeout): + captured["url"] = request.full_url + captured["timeout"] = timeout + return Response(201, {"Location": "/queue/item/11/"}) + + result = module.trigger_build("c" * 40, token_file=token, opener=opener) + assert captured["url"].startswith("https://ci.bstein.dev/") + assert captured["timeout"] == 20 + assert result["queue_path"] == "/queue/item/11/" diff --git a/testing/tests/test_hermes_image_builder_branch_edges.py b/testing/tests/test_hermes_image_builder_branch_edges.py new file mode 100644 index 00000000..8ada5cbb --- /dev/null +++ b/testing/tests/test_hermes_image_builder_branch_edges.py @@ -0,0 +1,227 @@ +"""Branch-edge coverage for the Hermes release and Harbor policy scripts.""" + +from __future__ import annotations + +import importlib.util +import io +import json +from pathlib import Path + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parents[2] +RELEASE = REPO_ROOT / "ci/scripts/hermes_image_release.py" +HARBOR = ( + REPO_ROOT + / "services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py" +) + + +def _load(path: Path, name: str): + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class Response(io.BytesIO): + """Context-managed urllib response used without network access.""" + + def __init__( + self, + status: int, + body: bytes = b"", + headers: dict[str, str] | None = None, + ) -> None: + super().__init__(body) + self.status = status + self.headers = headers or {} + + +class FakeClient: + """Ordered Harbor response fake with the pinned API origin.""" + + origin = "https://registry.bstein.dev/api/v2.0" + + def __init__(self, responses) -> None: + self.responses = list(responses) + self.calls = [] + + def request(self, method, path, payload=None): + self.calls.append((method, path, payload)) + return self.responses.pop(0) + + +def _count(value: int) -> dict[str, str]: + return {"X-Total-Count": str(value)} + + +def _publisher(module, *, immutable: bool = False, with_duration: bool = True) -> dict: + access = [ + {"resource": "repository", "action": "pull"}, + {"resource": "repository", "action": "push"}, + ] + if immutable: + access.append({"resource": "immutable-tag", "action": "list"}) + robot = { + "id": 41, + "name": module.PUBLISH_ROBOT, + "description": "publisher", + "level": "system", + "editable": True, + "disable": False, + "permissions": [ + {"kind": "project", "namespace": module.PROJECT, "access": access} + ], + } + if with_duration: + robot["duration"] = -1 + return robot + + +def test_release_redirect_handler_never_forwards_credentials() -> None: + """The registry opener refuses to follow any redirect target.""" + module = _load(RELEASE, "release_redirect_edges") + handler = module._NoRedirect() + assert handler.redirect_request(None, None, 0, None, None, None) is None + + +def test_release_policy_read_requires_credentials_and_bounded_body() -> None: + """The policy preflight rejects empty credentials and oversized responses.""" + module = _load(RELEASE, "release_policy_read_edges") + with pytest.raises(RuntimeError, match="credentials are empty"): + module._immutable_rules_response(username="", password="private") + with pytest.raises(RuntimeError, match="size limit"): + module._immutable_rules_response( + username="robot", + password="private", + opener=lambda *_args: Response(200, b"x" * 1_048_577), + ) + + +@pytest.mark.parametrize("headers", [{}, {"X-Total-Count": "many"}]) +def test_release_policy_rejects_missing_or_invalid_count(headers: dict) -> None: + """A proxy that strips or mangles count evidence cannot prove completeness.""" + module = _load(RELEASE, "release_policy_count_edges") + with pytest.raises(RuntimeError, match="total count"): + module.verify_immutable_policy( + username="robot", + password="private", + opener=lambda *_args, evidence=headers: Response(200, b"[]", evidence), + ) + + +@pytest.mark.parametrize("body", [b"{}", b"[1]"]) +def test_release_policy_rejects_non_rule_list_shapes(body: bytes) -> None: + """Valid JSON that is not a list of rule objects fails the preflight.""" + module = _load(RELEASE, "release_policy_shape_edges") + with pytest.raises(RuntimeError, match="invalid shape"): + module.verify_immutable_policy( + username="robot", + password="private", + opener=lambda *_args, value=body: Response(200, value, _count(1)), + ) + + +def test_renderer_ignores_trailing_entry_without_digest_at_eof() -> None: + """A dangling final image entry cannot shadow the one complete digest.""" + module = _load(RELEASE, "release_render_eof_edges") + digest = "sha256:" + "6" * 64 + source = ( + "images:\n" + f" - name: {module.DEFAULT_IMAGE}\n" + " digest: sha256:" + "0" * 64 + "\n" + f" - name: {module.DEFAULT_IMAGE}\n" + " newTag: dangling\n" + ) + rendered = module.render_kustomization(source, digest) + assert f"digest: {digest}\n" in rendered + assert rendered.endswith("newTag: dangling\n") + + +def test_harbor_verified_publisher_rejects_exact_identity_with_bad_scope() -> None: + """An exact robot identity still fails verification on a malformed scope.""" + module = _load(HARBOR, "harbor_verified_scope_edges") + robot = _publisher(module) + robot["permissions"] = "bad" + assert module._verified_publisher(robot) is False + + +def test_harbor_publisher_update_omits_absent_duration(monkeypatch) -> None: + """A robot without a configured duration is preserved without inventing one.""" + module = _load(HARBOR, "harbor_duration_edges") + monkeypatch.setattr(module.time, "sleep", lambda _seconds: None) + verified = _publisher(module, immutable=True) + client = FakeClient( + [ + (200, json.dumps([{"id": 41, "name": module.PUBLISH_ROBOT}]).encode(), _count(1)), + (200, json.dumps(_publisher(module, with_duration=False)).encode(), {}), + (200, b"", {}), + (200, json.dumps(verified).encode(), {}), + ] + ) + assert module.ensure_publisher_can_read_rule(client) == 41 + method, path, payload = client.calls[2] + assert (method, path) == ("PUT", "/robots/41") + assert "duration" not in payload + + +def test_harbor_publisher_verify_retries_past_invalid_json(monkeypatch) -> None: + """One stale unreadable reread retries instead of passing or failing outright.""" + module = _load(HARBOR, "harbor_verify_retry_edges") + sleeps = [] + monkeypatch.setattr(module.time, "sleep", sleeps.append) + verified = _publisher(module, immutable=True) + client = FakeClient( + [ + (200, json.dumps([{"id": 41, "name": module.PUBLISH_ROBOT}]).encode(), _count(1)), + (200, json.dumps(_publisher(module)).encode(), {}), + (200, b"", {}), + (200, b"not-json", {}), + (200, json.dumps(verified).encode(), {}), + ] + ) + assert module.ensure_publisher_can_read_rule(client) == 41 + assert sleeps == [1] + + +def test_harbor_rule_verify_requires_the_created_rule_id(monkeypatch) -> None: + """A matching rule under a different ID is a stale read, not creation proof.""" + module = _load(HARBOR, "harbor_rule_id_retry_edges") + sleeps = [] + monkeypatch.setattr(module.time, "sleep", sleeps.append) + location = "/api/v2.0/projects/bstein/immutabletagrules/23" + client = FakeClient( + [ + (200, b"[]", _count(0)), + (201, b"", {"Location": location}), + (200, json.dumps([{"id": 99, **module.EXPECTED_RULE}]).encode(), _count(1)), + (200, json.dumps([{"id": 23, **module.EXPECTED_RULE}]).encode(), _count(1)), + ] + ) + assert module.ensure_rule(client) == 23 + assert sleeps == [1] + + +def test_harbor_main_reraises_after_bounded_unavailable_retries( + monkeypatch, tmp_path: Path +) -> None: + """Persistent Harbor unavailability surfaces instead of looping forever.""" + module = _load(HARBOR, "harbor_main_retry_edges") + password_file = tmp_path / "password" + password_file.write_text("private\n", encoding="utf-8") + monkeypatch.setenv("HARBOR_API_ORIGIN", module.EXPECTED_ORIGIN) + monkeypatch.setenv("HARBOR_ADMIN_PASSWORD_FILE", str(password_file)) + monkeypatch.setattr(module, "HarborClient", lambda *_args: object()) + + def never_ready(_client): + raise module.HarborUnavailable("still warming") + + monkeypatch.setattr(module, "ensure_rule", never_ready) + sleeps = [] + monkeypatch.setattr(module.time, "sleep", sleeps.append) + with pytest.raises(module.HarborUnavailable, match="still warming"): + module.main() + assert len(sleeps) == 11 diff --git a/testing/tests/test_hermes_image_builder_coverage.py b/testing/tests/test_hermes_image_builder_coverage.py new file mode 100644 index 00000000..13a33177 --- /dev/null +++ b/testing/tests/test_hermes_image_builder_coverage.py @@ -0,0 +1,456 @@ +"""Boundary coverage for the three security-critical image-lane helpers.""" + +from __future__ import annotations + +import importlib.util +import io +import json +import sys +import urllib.error +from pathlib import Path + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parents[2] +RUNNER = REPO_ROOT / "dockerfiles/hermes-kaniko-heredoc-runner.py" +DOCKERFILE = REPO_ROOT / "dockerfiles/Dockerfile.hermes-agent" +HARBOR = ( + REPO_ROOT + / "services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py" +) +TRIGGER = REPO_ROOT / "services/hermes/scripts/jenkins_image_build_trigger.py" + + +def _load(path: Path, name: str): + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class Response(io.BytesIO): + """Context-managed urllib response used without network access.""" + + def __init__( + self, + status: int, + body: bytes = b"", + headers: dict[str, str] | None = None, + ) -> None: + super().__init__(body) + self.status = status + self.headers = headers or {} + + def __enter__(self): + return self + + def __exit__(self, *_args): + self.close() + + +class FakeClient: + """Ordered Harbor response fake with the pinned API origin.""" + + origin = "https://registry.bstein.dev/api/v2.0" + + def __init__(self, responses) -> None: + self.responses = list(responses) + self.calls = [] + + def request(self, method, path, payload=None): + self.calls.append((method, path, payload)) + return self.responses.pop(0) + + +def _count(value: int) -> dict[str, str]: + return {"X-Total-Count": str(value)} + + +def _robot(module, *, immutable: bool = False, duration=-1) -> dict: + access = [ + {"resource": "repository", "action": "pull"}, + {"resource": "repository", "action": "push"}, + ] + if immutable: + access.append({"resource": "immutable-tag", "action": "list"}) + return { + "id": 41, + "name": module.PUBLISH_ROBOT, + "description": "publisher", + "level": "system", + "duration": duration, + "editable": True, + "disable": False, + "permissions": [ + { + "kind": "project", + "namespace": module.PROJECT, + "access": access, + } + ], + } + + +def _robot_list(module, *, robot_id=41, name=None): + return json.dumps( + [{"id": robot_id, "name": name or module.PUBLISH_ROBOT}] + ).encode() + + +def test_runner_rejects_directive_and_heredoc_boundaries(tmp_path: Path) -> None: + """Malformed parser state, bodies, sizes, and indices all fail closed.""" + module = _load(RUNNER, "runner_boundary_coverage") + assert module._escape_character([""]) == "\\" + with pytest.raises(ValueError, match="multiple"): + module._escape_character(["# escape=\\", "# escape=\\"]) + with pytest.raises(ValueError, match="unsupported"): + module._escape_character(["# escape=^"]) + with pytest.raises(ValueError, match="unterminated.*continuation"): + module._logical_instruction(["R\\"], 0, "\\") + with pytest.raises(ValueError, match="unterminated node"): + module.extract_blocks("RUN node <<'NODE'\nbody") + + source = DOCKERFILE.read_text(encoding="utf-8") + marker = "RUN node <<'NODE'\n" + body_start = source.index(marker) + len(marker) + body_end = source.index("\nNODE", body_start) + empty = source[:body_start] + source[body_end + 1 :] + with pytest.raises(ValueError, match="empty node"): + module.extract_blocks(empty) + + dockerfile = tmp_path / "Dockerfile" + dockerfile.write_text("", encoding="utf-8") + with pytest.raises(ValueError, match="size"): + module.replay(dockerfile, 1) + with dockerfile.open("wb") as stream: + stream.truncate(module.MAX_DOCKERFILE_BYTES + 1) + with pytest.raises(ValueError, match="size"): + module.replay(dockerfile, 1) + dockerfile.write_text(source, encoding="utf-8") + for index in (0, 10): + with pytest.raises(ValueError, match="index"): + module.replay(dockerfile, index) + + +def test_runner_main_dispatches_the_exact_path_and_index( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """The command-line wrapper cannot omit either bounded input.""" + module = _load(RUNNER, "runner_main_coverage") + dockerfile = tmp_path / "Dockerfile" + seen = [] + monkeypatch.setattr(module, "replay", lambda path, index: seen.append((path, index))) + monkeypatch.setattr( + sys, + "argv", + ["runner", "--dockerfile", str(dockerfile), "--block-index", "4"], + ) + assert module.main() == 0 + assert seen == [(dockerfile, 4)] + + +def test_trigger_redirect_wrapper_accepts_only_plugin_303( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The low-level opener returns a real response or the one expected redirect.""" + module = _load(TRIGGER, "trigger_opener_coverage") + assert module._NoRedirect().redirect_request(None, None, 0, None, None, None) is None + request = module.urllib.request.Request(module.JENKINS_BUILD_URL) + + class Opener: + def __init__(self, result) -> None: + self.result = result + + def open(self, _request, timeout): + assert timeout == 9 + if isinstance(self.result, BaseException): + raise self.result + return self.result + + success = Response(201) + monkeypatch.setattr( + module.urllib.request, "build_opener", lambda *_args: Opener(success) + ) + assert module._open_without_redirect(request, 9) is success + redirect = module.urllib.error.HTTPError( + request.full_url, 303, "queued", {"Location": "/queue/item/1/"}, None + ) + monkeypatch.setattr( + module.urllib.request, "build_opener", lambda *_args: Opener(redirect) + ) + assert module._open_without_redirect(request, 9) is redirect + denied = module.urllib.error.HTTPError(request.full_url, 403, "denied", {}, None) + monkeypatch.setattr( + module.urllib.request, "build_opener", lambda *_args: Opener(denied) + ) + with pytest.raises(urllib.error.HTTPError): + module._open_without_redirect(request, 9) + + +def test_trigger_main_reports_safe_success_and_errors( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """CLI output contains safe metadata and converts expected failures to JSON.""" + module = _load(TRIGGER, "trigger_main_coverage") + revision = "a" * 40 + monkeypatch.setattr(sys, "argv", ["trigger", revision]) + monkeypatch.setattr( + module, + "trigger_build", + lambda value: {"job": module.JOB_NAME, "source_revision": value, "status": 201}, + ) + assert module.main() == 0 + assert json.loads(capsys.readouterr().out)["source_revision"] == revision + monkeypatch.setattr( + module, "trigger_build", lambda _value: (_ for _ in ()).throw(OSError("closed")) + ) + assert module.main() == 1 + assert json.loads(capsys.readouterr().out) == {"error": "closed"} + + +def test_harbor_http_client_bounds_requests_and_failures( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Credentials stay same-origin and all response/error paths remain bounded.""" + module = _load(HARBOR, "harbor_client_coverage") + assert module.NoRedirect().redirect_request(None, None, 0, None, None, None) is None + + class Opener: + result = Response(200, b"ok", {"X-Test": "yes"}) + + def open(self, request, timeout): + self.request = request + assert timeout == 20 + if isinstance(self.result, BaseException): + raise self.result + return self.result + + opener = Opener() + monkeypatch.setattr(module.urllib.request, "build_opener", lambda *_args: opener) + client = module.HarborClient(module.EXPECTED_ORIGIN + "/", "admin", "private") + status, body, headers = client.request("POST", "/rules", {"value": 1}) + assert (status, body, headers["X-Test"]) == (200, b"ok", "yes") + assert opener.request.get_header("Content-type") == "application/json" + + opener.result = module.urllib.error.HTTPError( + module.EXPECTED_ORIGIN, 404, "missing", {}, io.BytesIO(b"missing") + ) + assert client.request("GET", "/missing")[:2] == (404, b"missing") + opener.result = module.urllib.error.URLError("offline") + with pytest.raises(module.HarborUnavailable): + client.request("GET", "/rules") + opener.result = Response(200, b"x" * (module.MAX_RESPONSE + 1)) + with pytest.raises(RuntimeError, match="size"): + client.request("GET", "/rules") + + +@pytest.mark.parametrize("body", [b"\xff", b"not-json", b"{}", b"[1]"]) +def test_harbor_json_lists_reject_invalid_shapes(body: bytes) -> None: + """Malformed encodings, JSON, objects, and scalar members are never lists.""" + module = _load(HARBOR, f"harbor_json_{body!r}") + with pytest.raises(RuntimeError, match="invalid"): + module._json_list(body, "fixture") + + +@pytest.mark.parametrize( + "robot,match", + [ + ({"permissions": "bad"}, "permissions"), + ({"permissions": [1]}, "permissions"), + ({"permissions": []}, "one existing"), + ( + { + "permissions": [ + {"kind": "project", "namespace": "bstein", "access": []}, + {"kind": "project", "namespace": "bstein", "access": []}, + ] + }, + "one existing", + ), + ( + { + "permissions": [ + {"kind": "project", "namespace": "bstein", "access": "bad"} + ] + }, + "access", + ), + ( + { + "permissions": [ + {"kind": "project", "namespace": "bstein", "access": []} + ] + }, + "pull/push", + ), + ], +) +def test_publisher_scope_rejects_malformed_or_incomplete_permissions( + robot: dict, match: str +) -> None: + """Robot mutations require one structurally exact existing push scope.""" + module = _load(HARBOR, f"harbor_scope_{match}") + with pytest.raises(RuntimeError, match=match): + module._publisher_scope(robot) + assert module._verified_publisher(robot) is False + + +@pytest.mark.parametrize( + "responses,match,unavailable", + [ + ([(503, b"", {})], "503", True), + ([(403, b"", {})], "403", False), + ([(200, b"[]", _count(0))], "exactly one", False), + ([(200, b"[]", _count(0))], "exactly one", False), + ([(200, b'[{\"name\":\"robot$jenkins-pipelines\",\"id\":0}]', _count(1))], "valid ID", False), + ], +) +def test_publisher_list_rejects_status_count_and_id( + responses, match: str, unavailable: bool +) -> None: + """Publisher discovery distinguishes readiness from permanent policy errors.""" + module = _load(HARBOR, f"harbor_publisher_list_{match}_{unavailable}") + error = module.HarborUnavailable if unavailable else RuntimeError + with pytest.raises(error, match=match): + module.ensure_publisher_can_read_rule(FakeClient(responses)) + + +@pytest.mark.parametrize( + "second,match,unavailable", + [ + ((503, b"", {}), "503", True), + ((403, b"", {}), "403", False), + ((200, b"not-json", {}), "invalid robot JSON", False), + ((200, b"[]", {}), "invalid shape", False), + ((200, b'{}', {}), "not active and exact", False), + ], +) +def test_publisher_read_rejects_status_and_identity( + second, match: str, unavailable: bool +) -> None: + """The detailed robot reread must be available, valid, and exact.""" + module = _load(HARBOR, f"harbor_publisher_read_{match}_{unavailable}") + responses = [(200, _robot_list(module), _count(1)), second] + error = module.HarborUnavailable if unavailable else RuntimeError + with pytest.raises(error, match=match): + module.ensure_publisher_can_read_rule(FakeClient(responses)) + + +def test_publisher_update_rejects_duration_reference_and_status( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Only the preserved scope and a successful policy PUT can be accepted.""" + module = _load(HARBOR, "harbor_publisher_update_coverage") + for duration in ("forever",): + responses = [ + (200, _robot_list(module), _count(1)), + (200, json.dumps(_robot(module, duration=duration)).encode(), {}), + ] + with pytest.raises(RuntimeError, match="duration"): + module.ensure_publisher_can_read_rule(FakeClient(responses)) + + robot = _robot(module) + monkeypatch.setattr(module, "_publisher_scope", lambda _robot: ({}, [])) + responses = [ + (200, _robot_list(module), _count(1)), + (200, json.dumps(robot).encode(), {}), + ] + with pytest.raises(RuntimeError, match="normalization"): + module.ensure_publisher_can_read_rule(FakeClient(responses)) + + monkeypatch.undo() + for status, error in ((503, module.HarborUnavailable), (409, RuntimeError)): + responses = [ + (200, _robot_list(module), _count(1)), + (200, json.dumps(_robot(module)).encode(), {}), + (status, b"", {}), + ] + with pytest.raises(error): + module.ensure_publisher_can_read_rule(FakeClient(responses)) + + +def test_publisher_and_rule_verification_retries_are_bounded( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Stale reads may retry, but never silently become successful evidence.""" + module = _load(HARBOR, "harbor_retry_coverage") + monkeypatch.setattr(module.time, "sleep", lambda _seconds: None) + responses = [ + (200, _robot_list(module), _count(1)), + (200, json.dumps(_robot(module)).encode(), {}), + (200, b"", {}), + *((500, b"", {}) for _ in range(5)), + ] + with pytest.raises(RuntimeError, match="did not verify"): + module.ensure_publisher_can_read_rule(FakeClient(responses)) + + rule_path = "/api/v2.0/projects/bstein/immutabletagrules/23" + responses = [ + (200, b"[]", _count(0)), + (201, b"", {"Location": rule_path}), + *((200, b"[]", _count(0)) for _ in range(5)), + ] + with pytest.raises(RuntimeError, match="did not verify"): + module.ensure_rule(FakeClient(responses)) + + +def test_rule_rejects_ids_create_status_and_location_suffix() -> None: + """Neither existing nor newly created rules can omit their exact positive ID.""" + module = _load(HARBOR, "harbor_rule_id_coverage") + bad_rule = {"id": 0, **module.EXPECTED_RULE} + with pytest.raises(RuntimeError, match="valid ID"): + module.ensure_rule( + FakeClient([(200, json.dumps([bad_rule]).encode(), _count(1))]) + ) + with pytest.raises(module.HarborUnavailable): + module.ensure_rule( + FakeClient([(200, b"[]", _count(0)), (503, b"", {})]) + ) + with pytest.raises(RuntimeError, match="invalid ID"): + module.ensure_rule( + FakeClient( + [ + (200, b"[]", _count(0)), + ( + 201, + b"", + { + "location": "/api/v2.0/projects/bstein/immutabletagrules/nope" + }, + ), + ] + ) + ) + + +def test_harbor_main_retries_then_reports_exact_ids( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + """Runtime credentials are required and transient startup failures are bounded.""" + module = _load(HARBOR, "harbor_main_coverage") + password_file = tmp_path / "password" + password_file.write_text("private\n", encoding="utf-8") + monkeypatch.setenv("HARBOR_API_ORIGIN", module.EXPECTED_ORIGIN) + monkeypatch.setenv("HARBOR_ADMIN_PASSWORD_FILE", str(password_file)) + monkeypatch.setattr(module, "HarborClient", lambda *_args: object()) + attempts = iter([module.HarborUnavailable("warming"), 17]) + + def ensure_rule(_client): + result = next(attempts) + if isinstance(result, BaseException): + raise result + return result + + monkeypatch.setattr(module, "ensure_rule", ensure_rule) + monkeypatch.setattr(module, "ensure_publisher_can_read_rule", lambda _client: 41) + monkeypatch.setattr(module.time, "sleep", lambda seconds: None) + assert module.main() == 0 + assert "id=17" in capsys.readouterr().out + + password_file.write_text("\n", encoding="utf-8") + with pytest.raises(RuntimeError, match="empty"): + module.main() diff --git a/testing/tests/test_hermes_image_builder_fresh_review.py b/testing/tests/test_hermes_image_builder_fresh_review.py new file mode 100644 index 00000000..27ccc9b3 --- /dev/null +++ b/testing/tests/test_hermes_image_builder_fresh_review.py @@ -0,0 +1,335 @@ +"""Fresh fail-closed review cases for the Hermes image builder.""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +import pytest +import yaml + + +REPO_ROOT = Path(__file__).resolve().parents[2] +PIPELINE = REPO_ROOT / "ci/Jenkinsfile.hermes-agent-image" +DOCKERFILE = REPO_ROOT / "dockerfiles/Dockerfile.hermes-agent" +RUNNER = REPO_ROOT / "dockerfiles/hermes-kaniko-heredoc-runner.py" +RELEASE = REPO_ROOT / "ci/scripts/hermes_image_release.py" +BUILDER_SA = REPO_ROOT / "services/jenkins/hermes-image-builder-serviceaccount.yaml" + + +def _load(path: Path, name: str): + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _pod_spec() -> dict: + source = PIPELINE.read_text(encoding="utf-8") + pod_yaml = source.split('yaml """', 1)[1].split('"""', 1)[0] + return yaml.safe_load(pod_yaml)["spec"] + + +def test_builder_prefers_rpi5_with_healthy_arm64_worker_fallback() -> None: + """Disposable builds prefer rpi5 without excluding schedulable rpi4 workers.""" + spec = _pod_spec() + assert spec["nodeSelector"] == { + "kubernetes.io/arch": "arm64", + "node-role.kubernetes.io/worker": "true", + } + affinity = spec["affinity"]["nodeAffinity"] + assert affinity["preferredDuringSchedulingIgnoredDuringExecution"] == [ + { + "weight": 100, + "preference": { + "matchExpressions": [ + {"key": "hardware", "operator": "In", "values": ["rpi5"]} + ] + }, + } + ] + expressions = affinity["requiredDuringSchedulingIgnoredDuringExecution"][ + "nodeSelectorTerms" + ][0]["matchExpressions"] + host_rule = next( + rule for rule in expressions if rule["key"] == "kubernetes.io/hostname" + ) + assert host_rule["operator"] == "NotIn" + assert set(host_rule["values"]) >= { + "titan-04", + "titan-14", + "titan-18", + "titan-19", + "titan-22", + "titan-24", + } + + +@pytest.mark.parametrize( + "unsupported", + [ + "RUN cat < None: + """Any RUN heredoc outside the reviewed nine is rejected before replay.""" + module = _load(RUNNER, f"heredoc_reject_{abs(hash(unsupported))}") + dockerfile = tmp_path / "Dockerfile" + dockerfile.write_text( + DOCKERFILE.read_text(encoding="utf-8") + "\n" + unsupported + "\n", + encoding="utf-8", + ) + calls = [] + monkeypatch.setattr(module.subprocess, "run", lambda *_args, **_kwargs: calls.append(1)) + with pytest.raises(ValueError, match="unsupported RUN heredoc"): + module.replay(dockerfile, 1) + assert calls == [] + + +def test_exact_reviewed_heredocs_remain_buildkit_native_by_default() -> None: + """Current source inventories cleanly and enables replay only for Kaniko.""" + module = _load(RUNNER, "heredoc_positive_contract") + source = DOCKERFILE.read_text(encoding="utf-8") + assert len(module.extract_blocks(source)) == 9 + assert source.count("ARG HERMES_KANIKO_HEREDOC_COMPAT=0") == 1 + assert "Kaniko v1.23.2" in RUNNER.read_text(encoding="utf-8") + pipeline = PIPELINE.read_text(encoding="utf-8") + assert pipeline.count("--build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1") == 1 + + +def test_appended_tenth_reviewed_form_rejects_before_execution( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """Even an otherwise supported marker cannot expand the nine-block set.""" + module = _load(RUNNER, "heredoc_tenth_block") + dockerfile = tmp_path / "Dockerfile" + dockerfile.write_text( + DOCKERFILE.read_text(encoding="utf-8") + + "\nRUN node <<'NODE'\nconsole.log('tenth');\nNODE\n", + encoding="utf-8", + ) + calls = [] + monkeypatch.setattr(module.subprocess, "run", lambda *_args, **_kwargs: calls.append(1)) + with pytest.raises(ValueError, match="contract changed"): + module.replay(dockerfile, 1) + assert calls == [] + + +@pytest.mark.parametrize( + ("prefix", "unsupported"), + [ + ("", "R\\\nUN cat < None: + """Split RUN/opcode operators and backtick escapes cannot evade inventory.""" + module = _load(RUNNER, f"heredoc_logical_{abs(hash((prefix, unsupported)))}") + dockerfile = tmp_path / "Dockerfile" + dockerfile.write_text( + prefix + DOCKERFILE.read_text(encoding="utf-8") + "\n" + unsupported + "\n", + encoding="utf-8", + ) + calls = [] + monkeypatch.setattr(module.subprocess, "run", lambda *_args, **_kwargs: calls.append(1)) + with pytest.raises(ValueError, match="unsupported RUN heredoc"): + module.replay(dockerfile, 1) + assert calls == [] + + +def test_merged_main_replays_each_block_before_dependent_work() -> None: + """PR13's blocked-task regression runs only after its source patch replay.""" + source = DOCKERFILE.read_text(encoding="utf-8") + replay_positions = [ + source.index(f"--block-index {index}") for index in range(1, 10) + ] + # Locate repeated markers rather than trusting one synthetic occurrence. + block_positions = [] + for marker in ("RUN node <<'NODE'", "RUN python - <<'PY'"): + start = 0 + while True: + position = source.find(marker, start) + if position < 0: + break + block_positions.append(position) + start = position + len(marker) + block_positions.sort() + assert len(block_positions) == len(replay_positions) == 9 + for index, (block, replay) in enumerate(zip(block_positions, replay_positions)): + assert block < replay + if index + 1 < len(block_positions): + assert replay < block_positions[index + 1] + regression = source.index( + "RUN /opt/hermes/.venv/bin/python /tmp/hermes-kanban-blocked-regression.py" + ) + assert replay_positions[4] < regression < replay_positions[5] + + +def test_builder_service_account_is_explicit_tokenless_and_unbound() -> None: + """The build Pod selects one tokenless SA that no tracked RBAC grants bind.""" + account = yaml.safe_load(BUILDER_SA.read_text(encoding="utf-8")) + assert account == { + "apiVersion": "v1", + "kind": "ServiceAccount", + "metadata": {"name": "hermes-image-builder", "namespace": "jenkins"}, + "automountServiceAccountToken": False, + } + kustomization = yaml.safe_load( + (REPO_ROOT / "services/jenkins/kustomization.yaml").read_text(encoding="utf-8") + ) + assert "hermes-image-builder-serviceaccount.yaml" in kustomization["resources"] + + spec = _pod_spec() + assert spec["serviceAccountName"] == "hermes-image-builder" + assert spec["automountServiceAccountToken"] is False + for volume in spec.get("volumes", []): + projected = volume.get("projected", {}) + assert all("serviceAccountToken" not in item for item in projected.get("sources", [])) + + for manifest in (REPO_ROOT / "services/jenkins").glob("*.yaml"): + for document in yaml.safe_load_all(manifest.read_text(encoding="utf-8")): + if not isinstance(document, dict) or document.get("kind") not in { + "RoleBinding", + "ClusterRoleBinding", + }: + continue + subjects = document.get("subjects") or [] + assert not any( + item.get("kind") == "ServiceAccount" + and item.get("name") == "hermes-image-builder" + for item in subjects + ) + + +def _release_fixture(tmp_path: Path): + module = _load(RELEASE, f"release_evidence_{tmp_path.name}") + digest = "sha256:" + "7" * 64 + revision = "8" * 40 + build = "23" + destination = f"{module.DEFAULT_IMAGE}:git-{revision}-build-{build}" + digest_file = tmp_path / "hermes-agent.digest" + image_file = tmp_path / "hermes-agent.image" + manifest = tmp_path / "kustomization.yaml" + output = tmp_path / "release" + digest_file.write_text(digest + "\n", encoding="utf-8") + image_file.write_text(f"{destination}@{digest}\n", encoding="utf-8") + manifest.write_text( + "images:\n" + f" - name: {module.DEFAULT_IMAGE}\n" + " digest: sha256:" + "0" * 64 + "\n", + encoding="utf-8", + ) + module.write_release_artifacts( + digest=digest, + source_revision=revision, + build_number=build, + destination=destination, + kustomization=manifest, + output_dir=output, + ) + kwargs = { + "digest_file": digest_file, + "image_file": image_file, + "source_revision": revision, + "build_number": build, + "destination": destination, + "kustomization": manifest, + "output_dir": output, + } + return module, kwargs + + +def test_success_evidence_revalidation_accepts_only_exact_complete_set( + tmp_path: Path, +) -> None: + """Exact evidence passes; an extra, missing, or altered file fails closed.""" + module, kwargs = _release_fixture(tmp_path) + module.validate_release_artifacts(**kwargs) + + extra = kwargs["output_dir"] / "unexpected" + extra.write_text("surprise\n", encoding="utf-8") + with pytest.raises(ValueError, match="exactly three"): + module.validate_release_artifacts(**kwargs) + extra.unlink() + + metadata = kwargs["output_dir"] / "hermes-agent-image.json" + original = metadata.read_text(encoding="utf-8") + metadata.write_text(original.replace('"build_number": "23"', '"build_number": "24"')) + with pytest.raises(ValueError, match="incomplete or mismatched"): + module.validate_release_artifacts(**kwargs) + metadata.write_text(original, encoding="utf-8") + (kwargs["output_dir"] / "hermes-image-update.patch").unlink() + with pytest.raises(ValueError, match="exactly three"): + module.validate_release_artifacts(**kwargs) + + +def test_verify_evidence_cli_needs_no_runtime_registry_credential( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Post-success validation is deterministic and cannot mask missing creds.""" + module, kwargs = _release_fixture(tmp_path) + monkeypatch.delenv("HARBOR_USER", raising=False) + monkeypatch.delenv("HARBOR_PASSWORD", raising=False) + monkeypatch.setattr( + sys, + "argv", + [ + "hermes_image_release.py", + "verify-evidence", + "--digest-file", + str(kwargs["digest_file"]), + "--image-file", + str(kwargs["image_file"]), + "--source-revision", + kwargs["source_revision"], + "--build-number", + kwargs["build_number"], + "--destination", + kwargs["destination"], + "--kustomization", + str(kwargs["kustomization"]), + "--output-dir", + str(kwargs["output_dir"]), + ], + ) + assert module.main() == 0 + + +def test_pipeline_success_post_requires_and_archives_exact_six_files() -> None: + """Missing or partial post-success evidence must change the build to failed.""" + source = PIPELINE.read_text(encoding="utf-8") + post = source.split(" post {", 1)[1] + assert "success {" in post + assert "always {" not in post + assert "verify-evidence" in post + assert 'allowEmptyArchive: false' in post + archive = post.split("artifacts: '", 1)[1].split("'", 1)[0] + paths = archive.split(",") + assert len(paths) == 6 + assert len(set(paths)) == 6 + assert all("*" not in path for path in paths) + assert "find build -type f" in post diff --git a/testing/tests/test_hermes_image_builder_harbor.py b/testing/tests/test_hermes_image_builder_harbor.py new file mode 100644 index 00000000..9a64e24a --- /dev/null +++ b/testing/tests/test_hermes_image_builder_harbor.py @@ -0,0 +1,325 @@ +"""Server-side Harbor immutability contracts for Hermes agent releases.""" + +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path + +import pytest +import yaml + + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ( + REPO_ROOT + / "services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py" +) + + +def _load_module(): + spec = importlib.util.spec_from_file_location("harbor_immutability", SCRIPT) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _rule(module, *, rule_id: int = 17, disabled: bool = False) -> dict: + return {"id": rule_id, **module.EXPECTED_RULE, "disabled": disabled} + + +def _robot(module, *, immutable: bool = False, extra_immutable: bool = False) -> dict: + access = [ + {"resource": "repository", "action": "pull", "effect": "allow"}, + {"resource": "repository", "action": "push", "effect": "allow"}, + ] + if immutable: + access.append({"resource": "immutable-tag", "action": "list"}) + if extra_immutable: + access.append({"resource": "immutable-tag", "action": "delete"}) + return { + "id": 41, + "name": module.PUBLISH_ROBOT, + "description": "Jenkins publisher", + "level": "system", + "duration": -1, + "editable": True, + "disable": False, + "permissions": [ + { + "kind": "project", + "namespace": "other", + "access": [{"resource": "repository", "action": "pull"}], + }, + {"kind": "project", "namespace": "bstein", "access": access}, + ], + } + + +def _count(value: int) -> dict[str, str]: + return {"X-Total-Count": str(value)} + + +class FakeClient: + """Small deterministic Harbor API fake.""" + + def __init__(self, responses): + self.origin = "https://registry.bstein.dev/api/v2.0" + self.responses = list(responses) + self.calls = [] + + def request(self, method, path, payload=None): + self.calls.append((method, path, payload)) + return self.responses.pop(0) + + +def test_flux_tracks_exact_immutable_rule_before_jenkins() -> None: + """The Harbor rule is a reviewed prerequisite, not a Jenkins preflight only.""" + harbor = yaml.safe_load( + ( + REPO_ROOT + / "clusters/atlas/flux-system/applications/harbor/kustomization.yaml" + ).read_text() + ) + jenkins = yaml.safe_load( + ( + REPO_ROOT + / "clusters/atlas/flux-system/applications/jenkins/kustomization.yaml" + ).read_text() + ) + check = { + "apiVersion": "batch/v1", + "kind": "Job", + "name": "harbor-hermes-agent-immutability-ensure-1", + "namespace": "harbor", + } + assert check in harbor["spec"]["healthChecks"] + assert "harbor" in {item["name"] for item in jenkins["spec"]["dependsOn"]} + + +def test_policy_job_uses_runtime_vault_secret_and_hardened_pinned_image() -> None: + """No Harbor credential is committed or retained in a mutable workload.""" + job = yaml.safe_load( + (REPO_ROOT / "services/harbor/hermes-agent-immutability-job.yaml").read_text() + ) + template = job["spec"]["template"] + annotations = template["metadata"]["annotations"] + assert annotations["vault.hashicorp.com/role"] == "harbor-policy-bootstrap" + assert ( + annotations["vault.hashicorp.com/agent-inject-secret-harbor-admin-password"] + == "kv/data/atlas/harbor/harbor-core" + ) + pod = template["spec"] + assert pod["serviceAccountName"] == "harbor-policy-bootstrap" + assert pod["enableServiceLinks"] is False + container = pod["containers"][0] + assert "@sha256:" in container["image"] + security = container["securityContext"] + assert security["runAsNonRoot"] is True + assert security["readOnlyRootFilesystem"] is True + assert security["allowPrivilegeEscalation"] is False + assert security["capabilities"]["drop"] == ["ALL"] + + vault = ( + REPO_ROOT / "services/vault/scripts/vault_k8s_auth_configure.sh" + ).read_text() + policy = vault.split("harbor_policy_bootstrap_policy='", 1)[1].split("'", 1)[0] + assert 'path "kv/data/atlas/harbor/harbor-core"' in policy + assert 'capabilities = ["read"]' in policy + assert "*" not in policy + assert 'bound_service_account_names="harbor-policy-bootstrap"' in vault + script = SCRIPT.read_text() + assert 'PUBLISH_ROBOT = "robot$jenkins-pipelines"' in script + assert '{"resource": "immutable-tag", "action": "list"}' in script + assert 'client.request(\n "PUT", f"/robots/{robot_id}", payload' in script + assert '"secret"' not in script.split("payload = {", 1)[1].split("}", 1)[0] + + +def test_rule_contract_is_exact_repository_and_unique_build_tags() -> None: + """Unrelated bstein repositories and ordinary Hermes tags stay mutable.""" + module = _load_module() + assert module.EXPECTED_RULE == { + "disabled": False, + "action": "immutable", + "template": "immutable_template", + "tag_selectors": [ + { + "kind": "doublestar", + "decoration": "matches", + "pattern": "git-*-build-*", + } + ], + "scope_selectors": { + "repository": [ + { + "kind": "doublestar", + "decoration": "repoMatches", + "pattern": "hermes-agent", + } + ] + }, + } + + +def test_existing_exact_rule_is_idempotent_without_mutation() -> None: + """A rerun only validates the exact enabled rule.""" + module = _load_module() + body = json.dumps([_rule(module)]).encode() + client = FakeClient([(200, body, _count(1))]) + assert module.ensure_rule(client) == 17 + assert [call[0] for call in client.calls] == ["GET"] + + +def test_create_requires_201_exact_location_and_verified_reread() -> None: + """Policy bootstrap fails closed until Harbor returns the exact persisted rule.""" + module = _load_module() + body = json.dumps([_rule(module, rule_id=23)]).encode() + client = FakeClient( + [ + (200, b"[]", _count(0)), + ( + 201, + b"", + {"Location": "/api/v2.0/projects/bstein/immutabletagrules/23"}, + ), + (200, body, _count(1)), + ] + ) + assert module.ensure_rule(client) == 23 + assert client.calls[1] == ( + "POST", + "/projects/bstein/immutabletagrules", + module.EXPECTED_RULE, + ) + + +@pytest.mark.parametrize( + "responses,match", + [ + ([(200, b"[]", _count(0)), (200, b"", {})], "HTTP 200"), + ( + [ + (200, b"[]", _count(0)), + (201, b"", {"Location": "https://evil.invalid/1"}), + ], + "Location", + ), + ], +) +def test_create_rejects_noncanonical_responses(responses, match: str) -> None: + """Proxy success pages and foreign locations can never count as enforcement.""" + module = _load_module() + with pytest.raises(RuntimeError, match=match): + module.ensure_rule(FakeClient(responses)) + + +def test_readiness_failures_are_distinct_from_policy_rejections() -> None: + """The tracked Job may wait for Harbor without retrying an auth denial.""" + module = _load_module() + with pytest.raises(module.HarborUnavailable): + module.ensure_rule(FakeClient([(503, b"", {})])) + with pytest.raises(RuntimeError, match="HTTP 403") as exc: + module.ensure_rule(FakeClient([(403, b"", {})])) + assert not isinstance(exc.value, module.HarborUnavailable) + + +def test_disabled_or_duplicate_exact_scope_fails_closed() -> None: + """Bootstrap never silently edits a conflicting security policy.""" + module = _load_module() + disabled = json.dumps([_rule(module, disabled=True)]).encode() + with pytest.raises(RuntimeError, match="not enabled"): + module.ensure_rule(FakeClient([(200, disabled, _count(1))])) + + duplicate = json.dumps([_rule(module), _rule(module, rule_id=18)]).encode() + with pytest.raises(RuntimeError, match="multiple"): + module.ensure_rule(FakeClient([(200, duplicate, _count(2))])) + + +def test_policy_lists_reject_missing_or_truncated_count_evidence() -> None: + """A hidden second page can never produce a duplicate rule or robot update.""" + module = _load_module() + body = json.dumps([_rule(module)]).encode() + for headers in ({}, _count(2)): + with pytest.raises(RuntimeError, match="count|truncated"): + module.ensure_rule(FakeClient([(200, body, headers)])) + robot_body = json.dumps( + [{"id": 41, "name": module.PUBLISH_ROBOT}] + ).encode() + with pytest.raises(RuntimeError, match="truncated"): + module.ensure_publisher_can_read_rule( + FakeClient([(200, robot_body, _count(2))]) + ) + + +def test_publisher_policy_adds_only_read_access_and_preserves_robot() -> None: + """Bootstrap preserves every existing scope and never touches robot secret state.""" + module = _load_module() + original = _robot(module) + persisted = _robot(module, immutable=True) + client = FakeClient( + [ + ( + 200, + json.dumps([{"id": 41, "name": module.PUBLISH_ROBOT}]).encode(), + _count(1), + ), + (200, json.dumps(original).encode(), {}), + (200, b"", {}), + (200, json.dumps(persisted).encode(), {}), + ] + ) + assert module.ensure_publisher_can_read_rule(client) == 41 + method, path, payload = client.calls[2] + assert (method, path) == ("PUT", "/robots/41") + assert "secret" not in payload + assert payload["permissions"][0] == original["permissions"][0] + assert payload["permissions"][1]["access"][-1] == { + "resource": "immutable-tag", + "action": "list", + } + + +def test_publisher_policy_is_idempotent_and_rejects_broad_access() -> None: + """An exact read policy is stable; mutation-capable immutable access is blocked.""" + module = _load_module() + exact = _robot(module, immutable=True) + client = FakeClient( + [ + ( + 200, + json.dumps([{"id": 41, "name": module.PUBLISH_ROBOT}]).encode(), + _count(1), + ), + (200, json.dumps(exact).encode(), {}), + ] + ) + assert module.ensure_publisher_can_read_rule(client) == 41 + assert [call[0] for call in client.calls] == ["GET", "GET"] + + broad = _robot(module, immutable=True, extra_immutable=True) + client = FakeClient( + [ + ( + 200, + json.dumps([{"id": 41, "name": module.PUBLISH_ROBOT}]).encode(), + _count(1), + ), + (200, json.dumps(broad).encode(), {}), + ] + ) + with pytest.raises(RuntimeError, match="broader"): + module.ensure_publisher_can_read_rule(client) + + +def test_harbor_client_rejects_plaintext_or_ambiguous_origins() -> None: + """Runtime admin credentials are never sent over HTTP or a URL with query state.""" + module = _load_module() + for origin in ( + "http://registry.bstein.dev/api/v2.0", + "https://registry.bstein.dev/api/v2.0?next=evil", + "https://other.invalid/api/v2.0", + "", + ): + with pytest.raises(ValueError): + module.HarborClient(origin, "admin", "secret") diff --git a/testing/tests/test_hermes_image_builder_vault.py b/testing/tests/test_hermes_image_builder_vault.py new file mode 100644 index 00000000..6dadf10c --- /dev/null +++ b/testing/tests/test_hermes_image_builder_vault.py @@ -0,0 +1,329 @@ +"""Vault least-privilege and fail-closed seeding tests for image releases.""" + +from __future__ import annotations + +import os +import stat +import subprocess +from pathlib import Path + +import pytest +import yaml + + +REPO_ROOT = Path(__file__).resolve().parents[2] +VAULT_CONFIG = REPO_ROOT / "services/vault/scripts/vault_k8s_auth_configure.sh" +SEEDER = ( + REPO_ROOT + / "services/vault-hermes-jenkins-token-seed/scripts/vault_hermes_jenkins_build_token_ensure.sh" +) + + +def test_developer_jenkins_access_is_read_only_and_bound_to_two_consumers() -> None: + """Only the Jenkins controller and Hermes agent can read the fixed-job token.""" + source = VAULT_CONFIG.read_text(encoding="utf-8") + assert 'write_policy_and_role "jenkins" "jenkins" "jenkins"' in source + assert ( + '"jenkins/* shared/harbor-pull quality/sonarqube-oidc ' + 'hermes/developer-jenkins" ""' + ) in source + assert 'write_policy_and_role "hermes-agent" "hermes" "hermes-agent"' in source + assert 'hermes/developer-jenkins hermes/developer-ssh" ""' in source + assert 'write_policy_and_role "hermes-switchyard" "hermes"' in source + assert '"hermes/chat-telegram" ""' in source + assert '"hermes/developer-jenkins"' not in source + + jenkins_spc = yaml.safe_load( + (REPO_ROOT / "services/jenkins/secretproviderclass.yaml").read_text() + ) + assert jenkins_spc["spec"]["parameters"]["roleName"] == "jenkins-vault-sync" + switchyard = yaml.safe_load( + (REPO_ROOT / "services/hermes/switchyard-deployment.yaml").read_text() + ) + annotations = switchyard["spec"]["template"]["metadata"]["annotations"] + assert annotations["vault.hashicorp.com/role"] == "hermes-switchyard" + + +def test_flux_orders_seed_then_jenkins_then_hermes() -> None: + """The fixed token must be Ready before either consumer is rolled.""" + app_root = REPO_ROOT / "clusters/atlas/flux-system/applications" + vault = yaml.safe_load((app_root / "vault/kustomization.yaml").read_text()) + seed = yaml.safe_load( + (app_root / "vault-hermes-jenkins-token-seed/kustomization.yaml").read_text() + ) + jenkins = yaml.safe_load((app_root / "jenkins/kustomization.yaml").read_text()) + hermes = yaml.safe_load((app_root / "hermes/kustomization.yaml").read_text()) + role_check = { + "apiVersion": "batch/v1", + "kind": "Job", + "name": "vault-k8s-auth-hermes-10", + "namespace": "vault", + } + seed_check = { + "apiVersion": "batch/v1", + "kind": "Job", + "name": "vault-hermes-jenkins-build-token-seed-2", + "namespace": "vault", + } + assert role_check in vault["spec"]["healthChecks"] + assert {item["name"] for item in seed["spec"]["dependsOn"]} == {"vault"} + assert seed_check in seed["spec"]["healthChecks"] + assert jenkins["spec"]["suspend"] is False + assert {item["name"] for item in jenkins["spec"]["dependsOn"]} >= { + "helm", + "harbor", + "vault-hermes-jenkins-token-seed", + } + assert "jenkins" in {item["name"] for item in hermes["spec"]["dependsOn"]} + + +def test_seed_job_is_revisioned_bounded_and_tracks_fail_closed_script() -> None: + """The prerequisite is a tracked one-shot on healthy non-reserved capacity.""" + job = yaml.safe_load( + ( + REPO_ROOT / "services/vault-hermes-jenkins-token-seed/job.yaml" + ).read_text() + ) + assert job["metadata"]["name"] == "vault-hermes-jenkins-build-token-seed-2" + pod = job["spec"]["template"]["spec"] + assert pod["serviceAccountName"] == "hermes-jenkins-token-seed" + assert pod["enableServiceLinks"] is False + assert pod["restartPolicy"] == "Never" + assert pod["nodeSelector"]["hardware"] == "rpi5" + expression = pod["affinity"]["nodeAffinity"][ + "requiredDuringSchedulingIgnoredDuringExecution" + ]["nodeSelectorTerms"][0]["matchExpressions"][0] + assert set(expression["values"]) >= { + "titan-04", + "titan-14", + "titan-18", + "titan-19", + "titan-24", + } + container = pod["containers"][0] + assert "@sha256:" in container["image"] + security = container["securityContext"] + assert security["runAsNonRoot"] is True + assert security["readOnlyRootFilesystem"] is True + assert security["allowPrivilegeEscalation"] is False + assert security["capabilities"]["drop"] == ["ALL"] + source = SEEDER.read_text(encoding="utf-8") + assert '"options":{"cas":0}' in source + assert 'vault kv get -field=build_token "${secret_path}"' in source + assert "kv/data/atlas/hermes/developer-jenkins" in source + assert '"build_token"[[:space:]]' not in source + assert "kv patch" not in source + assert "kv delete" not in source + assert SEEDER.name not in VAULT_CONFIG.read_text(encoding="utf-8") + + +def test_seed_vault_policy_is_exact_create_read_and_rng_only() -> None: + """The seed identity cannot rotate, delete, list, or touch unrelated KV.""" + source = VAULT_CONFIG.read_text(encoding="utf-8") + policy = source.split("hermes_jenkins_token_seed_policy='", 1)[1].split("'", 1)[0] + assert 'path "kv/data/atlas/hermes/developer-jenkins"' in policy + assert 'capabilities = ["create", "read"]' in policy + assert 'path "sys/tools/random/32"' in policy + assert 'capabilities = ["update"]' in policy + for forbidden in ("delete", "patch", "list", 'kv/data/atlas/hermes/*'): + assert forbidden not in policy + assert 'bound_service_account_names="hermes-jenkins-token-seed"' in source + assert 'bound_service_account_namespaces="vault"' in source + + +def _fake_vault_tools(tmp_path: Path) -> tuple[Path, Path, Path]: + """Create deterministic Vault/sleep fakes and return their evidence paths.""" + binary_dir = tmp_path / "bin" + binary_dir.mkdir(parents=True) + calls = tmp_path / "calls" + stdin_capture = tmp_path / "stdin" + vault = binary_dir / "vault" + vault.write_text( + """#!/bin/sh +set -eu +printf '%s\n' "$*" >> "$FAKE_CALL_LOG" +scenario="$FAKE_SCENARIO" +if [ "$1" = "write" ] && [ "$2" = "-field=random_bytes" ] && [ "$3" = "sys/tools/random/32" ]; then + printf '%064d\n' 0 + exit 0 +fi +if [ "$1" = "kv" ] && [ "$2" = "get" ] && [ "$3" = "-field=build_token" ]; then + case "$scenario" in + existing) + printf '%064d\n' 1 + exit 0 + ;; + cas-race) + printf '%064d\n' 2 + exit 0 + ;; + field-prefix) + printf 'prefix%064d\n' 3 + exit 0 + ;; + field-suffix) + printf '%064dsuffix\n' 4 + exit 0 + ;; + field-multiline) + printf '%064d\nextra\n' 5 + exit 0 + ;; + missing-field|metadata-only|custom-metadata) + echo 'No value found at build_token' >&2 + exit 2 + ;; + esac +fi +if [ "$1" = "write" ] && [ "$2" = "kv/data/atlas/hermes/developer-jenkins" ]; then + payload="${3#@}" + cp "$payload" "$FAKE_STDIN_CAPTURE" + if [ "$scenario" = "cas-race" ]; then + : > "$FAKE_WINNER" + echo 'check-and-set parameter did not match' >&2 + exit 2 + fi + exit 0 +fi +if [ "$1" = "read" ] && [ "$2" = "-format=json" ]; then + case "$scenario" in + existing) + printf '{"data":{"data":{"build_token":"%064d"},"metadata":{"build_token":"decoy"}}}\n' 1 + exit 0 + ;; + missing-field) + printf '{"data":{"data":{"other":"preserve-me"}}}\n' + exit 0 + ;; + metadata-only) + printf '{"data":{"data":{"other":"preserve-me"},"metadata":{"build_token":"%064d"}}}\n' 6 + exit 0 + ;; + custom-metadata) + printf '{"data":{"data":{"other":"preserve-me"},"metadata":{"custom_metadata":{"build_token":"%064d"}}}}\n' 7 + exit 0 + ;; + field-prefix|field-suffix|field-multiline) + printf '{"data":{"data":{"build_token":"present"}}}\n' + exit 0 + ;; + absent) + echo 'Code: 404' >&2 + exit 2 + ;; + cas-race) + if [ -f "$FAKE_WINNER" ]; then + printf '{"data":{"data":{"build_token":"%064d"}}}\n' 2 + exit 0 + fi + echo 'Code: 404' >&2 + exit 2 + ;; + transient) + count=0 + if [ -f "$FAKE_COUNT" ]; then count="$(cat "$FAKE_COUNT")"; fi + count=$((count + 1)) + printf '%s' "$count" > "$FAKE_COUNT" + if [ "$count" -lt 3 ]; then echo 'temporary upstream error' >&2; exit 2; fi + echo 'Code: 404' >&2 + exit 2 + ;; + read-error) + echo 'permission denied' >&2 + exit 2 + ;; + esac +fi +echo "unexpected fake Vault call: $*" >&2 +exit 99 +""", + encoding="utf-8", + ) + sleep = binary_dir / "sleep" + sleep.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + for executable in (vault, sleep): + executable.chmod(executable.stat().st_mode | stat.S_IXUSR) + return binary_dir, calls, stdin_capture + + +def _run_seeder( + tmp_path: Path, scenario: str +) -> tuple[subprocess.CompletedProcess, str]: + binary_dir, calls, stdin_capture = _fake_vault_tools(tmp_path) + env = { + **os.environ, + "PATH": f"{binary_dir}:{os.environ['PATH']}", + "VAULT_TOKEN": "test-token", + "FAKE_SCENARIO": scenario, + "FAKE_CALL_LOG": str(calls), + "FAKE_STDIN_CAPTURE": str(stdin_capture), + "FAKE_WINNER": str(tmp_path / "winner"), + "FAKE_COUNT": str(tmp_path / "count"), + "TMPDIR": str(tmp_path), + } + result = subprocess.run( + ["sh", str(SEEDER)], + env=env, + text=True, + capture_output=True, + check=False, + ) + call_text = calls.read_text(encoding="utf-8") if calls.exists() else "" + return result, call_text + + +@pytest.mark.parametrize( + "scenario", + ["existing", "missing-field", "metadata-only", "custom-metadata", "read-error"], +) +def test_seeder_never_overwrites_existing_or_ambiguous_state( + tmp_path: Path, scenario: str +) -> None: + """Existing fields and persistent read failures can never become a put.""" + result, calls = _run_seeder(tmp_path, scenario) + assert (result.returncode == 0) is (scenario == "existing") + assert "write kv/data/atlas/hermes/developer-jenkins" not in calls + assert "sys/tools/random" not in calls + + +@pytest.mark.parametrize("scenario", ["field-prefix", "field-suffix", "field-multiline"]) +def test_seeder_requires_the_entire_exact_data_field( + tmp_path: Path, scenario: str +) -> None: + """Prefix, suffix, or multiline values cannot satisfy readiness.""" + result, calls = _run_seeder(tmp_path, scenario) + assert result.returncode != 0 + assert "kv get -field=build_token kv/atlas/hermes/developer-jenkins" in calls + assert "write kv/data/atlas/hermes/developer-jenkins" not in calls + assert "sys/tools/random" not in calls + + +def test_seeder_uses_single_cas_create_and_stdin_for_absent_secret( + tmp_path: Path, +) -> None: + """A confirmed 404 produces one create-only write without a token argument.""" + result, calls = _run_seeder(tmp_path, "absent") + assert result.returncode == 0, result.stderr + assert calls.count("write kv/data/atlas/hermes/developer-jenkins") == 1 + assert "@" in calls + assert "0000000000000000000000000000000000000000000000000000000000000000" not in calls + payload = (tmp_path / "stdin").read_text(encoding="utf-8") + assert '"options":{"cas":0}' in payload + assert '"build_token":"' in payload + assert not (tmp_path / "hermes-jenkins-token.json").exists() + + +def test_seeder_retries_reads_and_accepts_only_a_verified_cas_winner( + tmp_path: Path, +) -> None: + """Transient reads retry; a competing create is accepted only after reread.""" + transient, transient_calls = _run_seeder(tmp_path / "transient", "transient") + assert transient.returncode == 0, transient.stderr + assert transient_calls.count("read -format=json kv/data/atlas/hermes/developer-jenkins") == 3 + assert transient_calls.count("write kv/data/atlas/hermes/developer-jenkins") == 1 + + race, race_calls = _run_seeder(tmp_path / "race", "cas-race") + assert race.returncode == 0, race.stderr + assert race_calls.count("write kv/data/atlas/hermes/developer-jenkins") == 1 + assert race_calls.count("read -format=json kv/data/atlas/hermes/developer-jenkins") == 2 + assert "another seeder won CAS" in race.stderr diff --git a/testing/tests/test_hermes_node_account_hardening.py b/testing/tests/test_hermes_node_account_hardening.py new file mode 100644 index 00000000..9cf2bb3a --- /dev/null +++ b/testing/tests/test_hermes_node_account_hardening.py @@ -0,0 +1,252 @@ +"""Core contracts for the dedicated Hermes node SSH identity.""" + +from __future__ import annotations + +import os +from pathlib import Path + +import pytest +import yaml + +from testing.tests.test_hermes_node_account_support import ( + ROOT, + SCRIPT, + _fixture, + _key, + _load, +) + +def test_reconciler_creates_locked_groupless_account_and_moves_only_exact_key( + tmp_path: Path, monkeypatch +): + module, originals, key, other, public_key = _fixture(tmp_path, monkeypatch) + + module.reconcile(public_key) + first = { + name: (module.HOST_ETC / name).read_text(encoding="utf-8") + for name in originals + } + module.reconcile(public_key) + second = { + name: (module.HOST_ETC / name).read_text(encoding="utf-8") + for name in originals + } + + assert first == second + expected = module._expected_records() + for name, original in originals.items(): + assert first[name] == original + ":".join(expected[name]) + "\n" + backup = module.HOST_ETC / f"{name}.hermes-boundary-backup" + assert backup.read_text(encoding="utf-8") == original + assert expected["shadow"][1] == "!" + assert expected["group"][-1] == "" + assert "disk:x:6:atlas\n" in first["group"] + assert "sudo:x:27:oceanus\n" in first["group"] + + hermes_keys = ( + module.HOST_HOME / module.ACCOUNT / ".ssh/authorized_keys" + ).read_text(encoding="utf-8") + assert hermes_keys == key + "\n" + for user in module.LEGACY_ACCOUNTS: + legacy = ( + module.HOST_HOME / user / ".ssh/authorized_keys" + ).read_text(encoding="utf-8") + assert legacy == f"{other} {user}\n" + backup = module.HOST_HOME / user / ".ssh/authorized_keys.hermes-boundary-backup" + assert backup.read_text(encoding="utf-8") == f"{other} {user}\n{key}\n" + + +def test_identity_conflict_fails_closed_without_editing_account_databases( + tmp_path: Path, monkeypatch +): + module, originals, _key, _other, public_key = _fixture(tmp_path, monkeypatch) + conflict = originals["passwd"] + ( + f"unrelated:x:{module.ACCOUNT_UID}:{module.ACCOUNT_GID}:Other:/home/other:/bin/bash\n" + ) + (module.HOST_ETC / "passwd").write_text(conflict, encoding="utf-8") + + with pytest.raises(module.HardeningError, match="identity conflicts"): + module.reconcile(public_key) + + assert (module.HOST_ETC / "passwd").read_text(encoding="utf-8") == conflict + for name in ("group", "shadow", "gshadow"): + assert (module.HOST_ETC / name).read_text(encoding="utf-8") == originals[name] + assert not (module.HOST_HOME / module.ACCOUNT).exists() + + +def test_malformed_account_file_fails_closed(tmp_path: Path, monkeypatch): + module, originals, _key, _other, public_key = _fixture(tmp_path, monkeypatch) + (module.HOST_ETC / "group").write_text("malformed\n", encoding="utf-8") + + with pytest.raises(module.HardeningError, match="invalid record"): + module.reconcile(public_key) + + assert (module.HOST_ETC / "passwd").read_text(encoding="utf-8") == originals[ + "passwd" + ] + assert (module.HOST_ETC / "group").read_text(encoding="utf-8") == "malformed\n" + + +def test_preexisting_home_fails_before_account_database_or_key_changes( + tmp_path: Path, monkeypatch +): + module, originals, _key, _other, public_key = _fixture(tmp_path, monkeypatch) + home = module.HOST_HOME / module.ACCOUNT + home.mkdir() + (home / "unrelated").write_text("preserve\n", encoding="utf-8") + + with pytest.raises(module.HardeningError, match="home already exists"): + module.reconcile(public_key) + + for name, original in originals.items(): + assert (module.HOST_ETC / name).read_text(encoding="utf-8") == original + assert not (module.HOST_ETC / f"{name}.hermes-boundary-backup").exists() + assert (home / "unrelated").read_text(encoding="utf-8") == "preserve\n" + + +def test_flux_daemonset_reconciles_every_node_without_mutating_human_groups(): + documents = list( + yaml.safe_load_all( + (ROOT / "services/hermes/node-ssh-access.yaml").read_text(encoding="utf-8") + ) + ) + daemonset = next(item for item in documents if item["kind"] == "DaemonSet") + spec = daemonset["spec"]["template"]["spec"] + assert spec["tolerations"] == [{"operator": "Exists"}] + command = spec["containers"][0]["args"][0] + assert "/opt/node-hardener/node_account_hardening.py" in command + assert "sleep 300" in command + mounts = {item["name"]: item for item in spec["volumes"]} + assert mounts["host-home"]["hostPath"]["path"] == "/home" + assert mounts["host-etc"]["hostPath"]["path"] == "/etc" + assert mounts["host-k3s"]["hostPath"]["path"] == "/var/lib/rancher/k3s" + assert mounts["host-kubelet"]["hostPath"]["path"] == "/var/lib/kubelet" + assert mounts["host-run-k3s"]["hostPath"]["path"] == "/run/k3s" + assert mounts["host-run-containerd"]["hostPath"]["path"] == "/run/containerd" + assert "usermod" not in command + assert "groupmod" not in command + + policies = list( + yaml.safe_load_all( + (ROOT / "services/hermes/networkpolicy.yaml").read_text(encoding="utf-8") + ) + ) + isolation = next( + item + for item in policies + if item["metadata"]["name"] == "hermes-node-ssh-access-isolation" + ) + assert isolation["spec"]["ingress"] == [] + assert isolation["spec"]["egress"] == [] + + source = SCRIPT.read_text(encoding="utf-8") + assert "ACCOUNT = \"hermes-agent\"" in source + assert "ACCOUNT_UID = 1200" in source + assert "ACCOUNT_GID = 1200" in source + assert 'LEGACY_ACCOUNTS = ("atlas", "oceanus")' in source + assert "audit_membership" in source + image = spec["containers"][0]["image"] + assert image == ( + "python@sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df" + ) + assert mounts["host-polkit-share"]["hostPath"]["path"] == "/usr/share/polkit-1" + + +def test_sensitive_roots_get_explicit_zero_permission_acl_for_hermes( + tmp_path: Path, monkeypatch +): + module = _load() + sensitive = tmp_path / "k3s" + sensitive.mkdir(mode=0o755) + host_etc = tmp_path / "etc" + host_etc.mkdir() + monkeypatch.setattr(module, "ACCOUNT_UID", 1200) + monkeypatch.setattr(module, "HOST_ETC", host_etc) + monkeypatch.setattr(module, "HOST_ROOT_UID", os.getuid()) + monkeypatch.setattr(module, "HOST_ROOT_GID", os.getgid()) + # Production requires uid 0. The test process owns its fixture but runs the + # real xattr/ACL implementation using the test owner's uid as that boundary. + + module._deny_sensitive_root(sensitive) + first = module._read_acl(sensitive) + module._deny_sensitive_root(sensitive) + second = module._read_acl(sensitive) + + assert first == second + entries = module._decode_acl(first, 0o755) + assert (module.ACL_USER, 0, 1200) in entries + assert (module.ACL_USER_OBJ, 0o7, module.ACL_UNDEFINED_ID) in entries + assert (module.ACL_GROUP_OBJ, 0o5, module.ACL_UNDEFINED_ID) in entries + assert (module.ACL_OTHER, 0o5, module.ACL_UNDEFINED_ID) in entries + backup = host_etc / "hermes-node-boundary/k3s.acl" + assert backup.read_bytes() == b"N" + assert backup.stat().st_mode & 0o777 == 0o600 + + +def test_sensitive_root_rejects_non_directory_and_non_root_owner( + tmp_path: Path, monkeypatch +): + module = _load() + regular = tmp_path / "not-a-directory" + regular.write_text("preserve", encoding="utf-8") + with pytest.raises(module.HardeningError, match="unsafe sensitive directory"): + module._deny_sensitive_root(regular) + + directory = tmp_path / "not-root-owned" + directory.mkdir() + if directory.stat().st_uid == 0: + pytest.skip("cannot construct a non-root-owned fixture as root") + with pytest.raises(module.HardeningError, match="not root-owned"): + module._deny_sensitive_root(directory) + + +def test_acl_parser_rejects_malformed_or_incomplete_values(): + module = _load() + with pytest.raises(module.HardeningError, match="malformed"): + module._decode_acl(b"too short", 0o755) + incomplete = module._encode_acl( + [(module.ACL_USER_OBJ, 0o7, module.ACL_UNDEFINED_ID)] + ) + with pytest.raises(module.HardeningError, match="incomplete"): + module._decode_acl(incomplete, 0o755) + + +def test_acl_failure_stops_before_installing_the_ssh_key(tmp_path: Path, monkeypatch): + module = _load() + host_etc = tmp_path / "etc" + host_etc.mkdir() + monkeypatch.setattr(module, "HOST_ETC", host_etc) + monkeypatch.setattr(module, "HOST_ROOT_UID", os.getuid()) + public_key = tmp_path / "public-key" + public_key.write_text( + _key(b"synthetic-key") + "\n", + encoding="utf-8", + ) + moved = False + monkeypatch.setattr(module, "_reconcile_databases", lambda: None) + + def fail_acl(): + raise module.HardeningError("ACL unavailable") + + def move_key(_path): + nonlocal moved + moved = True + + monkeypatch.setattr(module, "_deny_sensitive_roots", fail_acl) + monkeypatch.setattr(module, "_move_key", move_key) + + with pytest.raises(module.HardeningError, match="ACL unavailable"): + module.reconcile(public_key) + assert moved is False + + +def test_runtime_ssh_config_forces_dedicated_account_for_every_titan(): + stage = ( + ROOT / "services/hermes/scripts/stage_runtime_access.py" + ).read_text(encoding="utf-8") + assert '"Host titan-*\\n User hermes-agent\\n"' in stage + agent = yaml.safe_load( + (ROOT / "services/hermes/agent-deployment.yaml").read_text(encoding="utf-8") + ) + assert "User atlas" not in yaml.safe_dump(agent) + assert "User oceanus" not in yaml.safe_dump(agent) diff --git a/testing/tests/test_hermes_node_account_io.py b/testing/tests/test_hermes_node_account_io.py new file mode 100644 index 00000000..6b05cc0a --- /dev/null +++ b/testing/tests/test_hermes_node_account_io.py @@ -0,0 +1,222 @@ +"""Crash-safety and metadata contracts for Hermes node-account writes.""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path + +import pytest +import yaml + +from testing.tests.test_hermes_node_account_support import ( + ROOT, + _fixture, + _load, + _typed_key, +) + +def test_key_identity_ignores_options_and_comments_and_removes_legacy_first( + tmp_path: Path, monkeypatch +): + module, _originals, key, other, public_key = _fixture(tmp_path, monkeypatch) + key_fields = key.split() + decorated = f'restrict,command="echo denied" {key_fields[0]} {key_fields[1]} old-comment' + for user in module.LEGACY_ACCOUNTS: + path = module.HOST_HOME / user / ".ssh/authorized_keys" + path.write_text(f"{other} {user}\n{decorated}\n", encoding="utf-8") + writes: list[Path] = [] + real_write = module.atomic_write + + def recording_write(path, value, metadata): + writes.append(path) + real_write(path, value, metadata) + + monkeypatch.setattr(module, "atomic_write", recording_write) + module.reconcile(public_key) + + key_writes = [path for path in writes if path.name == "authorized_keys"] + assert key_writes[-1].parts[-3:] == (module.ACCOUNT, ".ssh", "authorized_keys") + assert all(module.ACCOUNT not in path.parts for path in key_writes[:-1]) + + +def test_unrelated_modern_human_key_type_is_preserved(tmp_path: Path, monkeypatch): + module, _originals, key, _other, public_key = _fixture(tmp_path, monkeypatch) + modern = _typed_key("sk-ssh-ed25519@openssh.com", b"human-security-key", "human") + for user in module.LEGACY_ACCOUNTS: + path = module.HOST_HOME / user / ".ssh/authorized_keys" + path.write_text(f"{modern}\n{key}\n", encoding="utf-8") + + module.reconcile(public_key) + + for user in module.LEGACY_ACCOUNTS: + path = module.HOST_HOME / user / ".ssh/authorized_keys" + assert path.read_text() == modern + "\n" + + +def test_crash_during_legacy_key_removal_never_installs_duplicate_key( + tmp_path: Path, monkeypatch +): + module, _originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + real_write = module.atomic_write + key_writes = 0 + + def crash_on_second_key(path, value, metadata): + nonlocal key_writes + if path.name == "authorized_keys": + key_writes += 1 + if key_writes == 2: + raise module.HardeningError("synthetic crash") + real_write(path, value, metadata) + + monkeypatch.setattr(module, "atomic_write", crash_on_second_key) + with pytest.raises(module.HardeningError, match="synthetic crash"): + module.reconcile(public_key) + assert not (module.HOST_HOME / module.ACCOUNT / ".ssh/authorized_keys").exists() + + +def test_preexisting_target_key_is_removed_before_legacy_cleanup_and_reinstalled( + tmp_path: Path, monkeypatch +): + module, _originals, key, _other, public_key = _fixture(tmp_path, monkeypatch) + target = module.HOST_HOME / module.ACCOUNT / ".ssh/authorized_keys" + target.parent.mkdir(parents=True) + target.write_text(key + " existing\n") + writes: list[Path] = [] + real_write = module.atomic_write + + def recording_write(path, value, metadata): + if path.name == "authorized_keys": + writes.append(path) + real_write(path, value, metadata) + + monkeypatch.setattr(module, "atomic_write", recording_write) + module._move_key(public_key) + + assert writes[0] == target + assert writes[-1] == target + assert all(path != target for path in writes[1:-1]) + assert target.read_text() == key + "\n" + + +def test_account_database_write_preserves_mode_owner_and_xattrs( + tmp_path: Path, monkeypatch +): + module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + passwd = module.HOST_ETC / "passwd" + passwd.chmod(0o640) + os.setxattr(passwd, "user.hermes-test", b"preserve") + before = passwd.stat() + + module.reconcile(public_key) + + after = passwd.stat() + assert after.st_mode & 0o777 == 0o640 + assert (after.st_uid, after.st_gid) == (before.st_uid, before.st_gid) + assert os.getxattr(passwd, "user.hermes-test") == b"preserve" + backup = module.HOST_ETC / "passwd.hermes-boundary-backup" + assert os.getxattr(backup, "user.hermes-test") == b"preserve" + assert backup.read_text() == originals["passwd"] + + +def test_atomic_write_restores_acl_and_security_label_xattrs(tmp_path: Path, monkeypatch): + module = _load() + io_module = sys.modules[module.atomic_write.__module__] + target = tmp_path / "passwd" + target.write_bytes(b"old\n") + metadata = io_module.FileSnapshot( + value=b"old\n", + device=target.stat().st_dev, + inode=target.stat().st_ino, + mode=0o600, + uid=os.getuid(), + gid=os.getgid(), + size=4, + mtime_ns=target.stat().st_mtime_ns, + ctime_ns=target.stat().st_ctime_ns, + xattrs=( + ("system.posix_acl_access", b"synthetic-acl"), + ("security.selinux", b"synthetic-label"), + ("user.audit", b"synthetic-xattr"), + ), + ) + restored: list[tuple[str, bytes]] = [] + monkeypatch.setattr( + io_module.os, + "setxattr", + lambda _fd, name, value: restored.append((name, value)), + ) + + io_module.atomic_write(target, b"new\n", metadata) + + assert target.read_bytes() == b"new\n" + assert restored == list(metadata.xattrs) + + +def test_concurrent_database_change_is_detected_before_any_account_write( + tmp_path: Path, monkeypatch +): + module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + real_backup = module.backup_once + backup_count = 0 + writes: list[Path] = [] + + def racing_backup(path, snapshot, maximum): + nonlocal backup_count + result = real_backup(path, snapshot, maximum) + backup_count += 1 + if backup_count == 4: + group = module.HOST_ETC / "group" + group.write_text(originals["group"] + "race:x:4000:\n") + return result + + monkeypatch.setattr(module, "backup_once", racing_backup) + monkeypatch.setattr(module, "atomic_write", lambda path, *_args: writes.append(path)) + with pytest.raises(module.HardeningError, match="concurrent host account change"): + module.reconcile(public_key) + assert writes == [] + + +def test_flux_orders_observer_rbac_before_hermes_prunes_old_authority(): + observer = yaml.safe_load( + (ROOT / "clusters/atlas/flux-system/applications/hermes-observer-rbac/kustomization.yaml").read_text() + ) + hermes = yaml.safe_load( + (ROOT / "clusters/atlas/flux-system/applications/hermes/kustomization.yaml").read_text() + ) + bindings = yaml.safe_load( + ( + ROOT + / "clusters/atlas/flux-system/applications/hermes-observer-bindings/kustomization.yaml" + ).read_text() + ) + assert "dependsOn" not in observer["spec"] + assert {item["name"] for item in hermes["spec"]["dependsOn"]} >= { + "hermes-observer-rbac", + "hermes-scm-broker", + } + assert {item["name"] for item in bindings["spec"]["dependsOn"]} == { + "hermes-observer-rbac", + "hermes", + } + assert "hermes-observer-bindings" not in { + item["name"] for item in hermes["spec"]["dependsOn"] + } + + +def test_sensitive_root_acl_backups_have_unique_host_path_names(monkeypatch): + module = _load() + seen = [] + monkeypatch.setattr( + module, + "_deny_sensitive_root", + lambda path, backup_name=None: seen.append((path, backup_name)), + ) + module._deny_sensitive_roots() + assert [name for _path, name in seen] == [ + "var-lib-rancher-k3s", + "var-lib-kubelet", + "run-k3s", + "run-containerd", + ] + assert len({name for _path, name in seen}) == 4 diff --git a/testing/tests/test_hermes_node_account_privilege_audit.py b/testing/tests/test_hermes_node_account_privilege_audit.py new file mode 100644 index 00000000..3370fe4f --- /dev/null +++ b/testing/tests/test_hermes_node_account_privilege_audit.py @@ -0,0 +1,196 @@ +"""Host privilege-policy audits for the Hermes node SSH identity.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from testing.tests.test_hermes_node_account_support import _fixture + + +@pytest.mark.parametrize( + "database,record", + [ + ("group", "disk:x:6:atlas,hermes-agent\n"), + ("gshadow", "disk:!:hermes-agent:atlas\n"), + ("gshadow", "disk:!::atlas,hermes-agent\n"), + ], +) +def test_unexpected_group_or_gshadow_membership_fails_before_writes( + tmp_path: Path, monkeypatch, database: str, record: str +): + module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + path = module.HOST_ETC / database + path.write_text( + "\n".join( + line + for line in originals[database].splitlines() + if not line.startswith("disk:") + ) + + "\n" + + record, + encoding="utf-8", + ) + before = {name: (module.HOST_ETC / name).read_bytes() for name in originals} + + with pytest.raises(module.HardeningError, match="group access"): + module.reconcile(public_key) + + assert {name: (module.HOST_ETC / name).read_bytes() for name in originals} == before + + +@pytest.mark.parametrize( + "relative,value", + [ + ("sudoers.d/hermes", "hermes-agent ALL=(ALL:ALL) NOPASSWD: ALL\n"), + ("sudoers", "ALL ALL=(ALL:ALL) ALL\n"), + ( + "polkit-1/rules.d/90-root.rules", + 'if (subject.user == "hermes-agent") return polkit.Result.YES;\n', + ), + ( + "polkit-1/rules.d/90-root.rules", + "unix-user:* org.freedesktop.policykit.exec polkit.Result.YES\n", + ), + ], +) +def test_sudo_and_root_equivalent_polkit_authority_fail_closed( + tmp_path: Path, monkeypatch, relative: str, value: str +): + module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + policy = module.HOST_ETC / relative + policy.parent.mkdir(parents=True, exist_ok=True) + policy.write_text(value, encoding="utf-8") + + with pytest.raises(module.HardeningError, match="sudo|polkit"): + module.reconcile(public_key) + + assert all( + (module.HOST_ETC / name).read_text() == original + for name, original in originals.items() + ) + + +def test_external_group_authority_source_is_rejected(tmp_path: Path, monkeypatch): + module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + (module.HOST_ETC / "nsswitch.conf").write_text("passwd: files\ngroup: files ldap\n") + with pytest.raises(module.HardeningError, match="external group"): + module.reconcile(public_key) + assert all( + (module.HOST_ETC / name).read_text() == original + for name, original in originals.items() + ) + + +@pytest.mark.parametrize("database", ["passwd", "group", "initgroups", "shadow"]) +def test_external_identity_authority_sources_fail_closed( + tmp_path: Path, monkeypatch, database: str +): + module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + (module.HOST_ETC / "nsswitch.conf").write_text( + f"passwd: files\n{database}: files ldap\n", encoding="utf-8" + ) + with pytest.raises(module.HardeningError, match="external group/account"): + module.reconcile(public_key) + assert all( + (module.HOST_ETC / name).read_text() == original + for name, original in originals.items() + ) + + +@pytest.mark.parametrize( + "value", + [ + "#1200 ALL=(ALL:ALL) NOPASSWD: ALL\n", + "User_Alias HERMES_IDS = #1200\nHERMES_IDS ALL=(ALL:ALL) NOPASSWD: ALL\n", + "User_Alias HERMES_GROUP = %#1200\nHERMES_GROUP ALL=(ALL:ALL) NOPASSWD: ALL\n", + ], +) +def test_visudo_valid_numeric_and_alias_grants_fail_closed( + tmp_path: Path, monkeypatch, value: str +): + module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + monkeypatch.setattr(module, "ACCOUNT_UID", 1200) + policy = module.HOST_ETC / "sudoers" + policy.write_text(value, encoding="utf-8") + validation = subprocess.run( + ["/usr/bin/visudo", "-c", "-f", str(policy)], + check=False, + capture_output=True, + text=True, + ) + assert validation.returncode == 0, validation.stderr + + with pytest.raises(module.HardeningError, match="sudo authority"): + module.reconcile(public_key) + + assert all( + (module.HOST_ETC / name).read_text() == original + for name, original in originals.items() + ) + + +def test_sudo_include_is_bounded_to_the_audited_standard_directory( + tmp_path: Path, monkeypatch +): + module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + sudoers = module.HOST_ETC / "sudoers" + sudoers.write_text("#includedir /opt/external-sudoers\n", encoding="utf-8") + with pytest.raises(module.HardeningError, match="unaudited authority source"): + module.reconcile(public_key) + + sudoers.write_text("#includedir /etc/sudoers.d\n", encoding="utf-8") + module.reconcile(public_key) + assert all( + (module.HOST_ETC / name).read_text().startswith(original) + for name, original in originals.items() + ) + + +@pytest.mark.parametrize( + "relative,value", + [ + ( + "polkit-1/rules.d/90-hermes.rules", + "polkit.addRule(function(action, subject) {\n" + " if (subject.uid == 1200) return polkit.Result.YES;\n" + "});\n", + ), + ( + "polkit-1/localauthority/50-local.d/hermes.pkla", + "[Hermes]\nIdentity=unix-user:1200\n" + "Action=org.freedesktop.policykit.exec\nResultActive=yes\n", + ), + ], +) +def test_numeric_polkit_grants_fail_closed( + tmp_path: Path, monkeypatch, relative: str, value: str +): + module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + monkeypatch.setattr(module, "ACCOUNT_UID", 1200) + policy = module.HOST_ETC / relative + policy.parent.mkdir(parents=True, exist_ok=True) + policy.write_text(value, encoding="utf-8") + + with pytest.raises(module.HardeningError, match="polkit authority"): + module.reconcile(public_key) + + assert all( + (module.HOST_ETC / name).read_text() == original + for name, original in originals.items() + ) + + +def test_writable_privilege_policy_is_rejected(tmp_path: Path, monkeypatch): + module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + sudoers = module.HOST_ETC / "sudoers" + sudoers.write_text("root ALL=(ALL:ALL) ALL\n") + sudoers.chmod(0o666) + with pytest.raises(module.HardeningError, match="unsafe mutation"): + module.reconcile(public_key) + assert all( + (module.HOST_ETC / name).read_text() == original + for name, original in originals.items() + ) diff --git a/testing/tests/test_hermes_node_account_support.py b/testing/tests/test_hermes_node_account_support.py new file mode 100644 index 00000000..2d74d124 --- /dev/null +++ b/testing/tests/test_hermes_node_account_support.py @@ -0,0 +1,90 @@ +"""Shared fixtures for dedicated Hermes node-account contracts.""" + +from __future__ import annotations + +import base64 +import importlib.util +import os +import sys +from pathlib import Path + +ROOT = Path(__file__).parents[2] +SCRIPT = ROOT / "services/hermes/scripts/node_account_hardening.py" +sys.path.insert(0, str(SCRIPT.parent)) + + +def _key(material: bytes, comment: str = "") -> str: + key_type = b"ssh-ed25519" + blob = len(key_type).to_bytes(4, "big") + key_type + material + suffix = f" {comment}" if comment else "" + return "ssh-ed25519 " + base64.b64encode(blob).decode() + suffix + + +def _typed_key(key_type: str, material: bytes, comment: str = "") -> str: + encoded_type = key_type.encode() + blob = len(encoded_type).to_bytes(4, "big") + encoded_type + material + suffix = f" {comment}" if comment else "" + return f"{key_type} {base64.b64encode(blob).decode()}{suffix}" + + +def _load(): + spec = importlib.util.spec_from_file_location("node_account_hardening_test", SCRIPT) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _fixture(tmp_path: Path, monkeypatch): + module = _load() + host_etc = tmp_path / "etc" + host_home = tmp_path / "home" + host_etc.mkdir() + host_home.mkdir() + originals = { + "passwd": ( + "root:x:0:0:root:/root:/bin/bash\n" + "atlas:x:2000:2000:Atlas:/home/atlas:/bin/bash\n" + "oceanus:x:2001:2001:Oceanus:/home/oceanus:/bin/bash\n" + ), + "group": ( + "root:x:0:\n" + "atlas:x:2000:\n" + "oceanus:x:2001:\n" + "disk:x:6:atlas\n" + "sudo:x:27:oceanus\n" + ), + "shadow": ( + "root:!:1:0:99999:7:::\n" + "atlas:!:1:0:99999:7:::\n" + "oceanus:!:1:0:99999:7:::\n" + ), + "gshadow": ( + "root:!::\n" + "atlas:!::\n" + "oceanus:!::\n" + "disk:!::atlas\n" + "sudo:!::oceanus\n" + ), + } + for name, value in originals.items(): + (host_etc / name).write_text(value, encoding="utf-8") + key = _key(b"synthetic-hermes-key") + other = _key(b"human-operator-key") + for user in ("atlas", "oceanus"): + ssh = host_home / user / ".ssh" + ssh.mkdir(parents=True) + (ssh / "authorized_keys").write_text( + f"{other} {user}\n{key}\n", encoding="utf-8" + ) + public_key = tmp_path / "public-key" + public_key.write_text(key + "\n", encoding="utf-8") + monkeypatch.setattr(module, "HOST_ETC", host_etc) + monkeypatch.setattr(module, "HOST_HOME", host_home) + monkeypatch.setattr(module, "ACCOUNT_UID", os.getuid()) + monkeypatch.setattr(module, "ACCOUNT_GID", os.getgid()) + monkeypatch.setattr(module, "HOST_ROOT_UID", os.getuid()) + monkeypatch.setattr(module, "HOST_POLKIT_SHARE", tmp_path / "missing-polkit") + monkeypatch.setattr(module, "_deny_sensitive_roots", lambda: None) + return module, originals, key, other, public_key diff --git a/testing/tests/test_hermes_node_acl_coverage.py b/testing/tests/test_hermes_node_acl_coverage.py new file mode 100644 index 00000000..572fd0d2 --- /dev/null +++ b/testing/tests/test_hermes_node_acl_coverage.py @@ -0,0 +1,206 @@ +"""Behavioral branch coverage for node-storage ACL hardening.""" + +from __future__ import annotations + +import errno +import os +import pytest + +from testing.tests.test_hermes_node_account_support import _fixture, _load + + +def test_acl_decoder_rejects_version_permissions_and_adds_missing_mask(): + module = _load() + valid = [ + (module.ACL_USER_OBJ, 7, module.ACL_UNDEFINED_ID), + (module.ACL_GROUP_OBJ, 5, module.ACL_UNDEFINED_ID), + (module.ACL_OTHER, 5, module.ACL_UNDEFINED_ID), + ] + wrong_version = module.ACL_HEADER.pack(99) + b"".join( + module.ACL_ENTRY.pack(*entry) for entry in valid + ) + with pytest.raises(module.HardeningError, match="version"): + module._decode_acl(wrong_version, 0o755) + bad_permission = module._encode_acl([*valid, (module.ACL_USER, 8, 1200)]) + with pytest.raises(module.HardeningError, match="permission"): + module._decode_acl(bad_permission, 0o755) + updated = module._decode_acl( + module._acl_with_deny(module._encode_acl(valid), 0o755), 0o755 + ) + assert (module.ACL_MASK, 5, module.ACL_UNDEFINED_ID) in updated + assert (module.ACL_USER, 0, module.ACCOUNT_UID) in updated + + +def test_read_acl_handles_absence_and_propagates_other_errors(monkeypatch, tmp_path): + module = _load() + path = tmp_path / "root" + path.mkdir() + + def missing(*_args, **_kwargs): + raise OSError(errno.ENODATA, "missing") + + monkeypatch.setattr(module.os, "getxattr", missing) + assert module._read_acl(path) == b"" + + def denied(*_args, **_kwargs): + raise OSError(errno.EPERM, "denied") + + monkeypatch.setattr(module.os, "getxattr", denied) + with pytest.raises(OSError): + module._read_acl(path) + + +@pytest.mark.parametrize("value", [b"", b"X", b"A", b"Nextra"]) +def test_acl_backup_validator_rejects_malformed_encodings(value): + module = _load() + with pytest.raises(module.HardeningError, match="backup is malformed"): + module._validate_acl_backup(value) + module._validate_acl_backup(b"N") + valid = module._encode_acl( + [ + (module.ACL_USER_OBJ, 7, module.ACL_UNDEFINED_ID), + (module.ACL_GROUP_OBJ, 5, module.ACL_UNDEFINED_ID), + (module.ACL_OTHER, 5, module.ACL_UNDEFINED_ID), + ] + ) + module._validate_acl_backup(b"A" + valid) + + +def test_acl_backup_reuses_valid_existing_copy(tmp_path, monkeypatch): + module = _load() + host_etc = tmp_path / "etc" + root = host_etc / "hermes-node-boundary" + root.mkdir(parents=True) + backup = root / "k3s.acl" + backup.write_bytes(b"N") + monkeypatch.setattr(module, "HOST_ETC", host_etc) + monkeypatch.setattr(module, "HOST_ROOT_UID", os.getuid()) + monkeypatch.setattr(module, "HOST_ROOT_GID", os.getgid()) + module._acl_backup_once(tmp_path / "k3s", b"") + assert backup.read_bytes() == b"N" + + +def test_acl_backup_short_write_cleans_temporary(tmp_path, monkeypatch): + module = _load() + host_etc = tmp_path / "etc" + host_etc.mkdir() + monkeypatch.setattr(module, "HOST_ETC", host_etc) + monkeypatch.setattr(module, "HOST_ROOT_UID", os.getuid()) + monkeypatch.setattr(module, "HOST_ROOT_GID", os.getgid()) + monkeypatch.setattr(module.os, "write", lambda _fd, _value: 0) + with pytest.raises(module.HardeningError, match="short sensitive"): + module._acl_backup_once(tmp_path / "k3s", b"") + assert not list((host_etc / "hermes-node-boundary").glob(".*.hermes-*")) + + +def test_acl_backup_tolerates_concurrent_first_writer(tmp_path, monkeypatch): + module = _load() + host_etc = tmp_path / "etc" + host_etc.mkdir() + monkeypatch.setattr(module, "HOST_ETC", host_etc) + monkeypatch.setattr(module, "HOST_ROOT_UID", os.getuid()) + monkeypatch.setattr(module, "HOST_ROOT_GID", os.getgid()) + real_link = os.link + + def race(source, target, **kwargs): + real_link(source, target, **kwargs) + raise FileExistsError() + + monkeypatch.setattr(module.os, "link", race) + module._acl_backup_once(tmp_path / "k3s", b"") + assert (host_etc / "hermes-node-boundary/k3s.acl").read_bytes() == b"N" + + +def test_sensitive_root_failure_restores_existing_acl(tmp_path, monkeypatch): + module = _load() + root = tmp_path / "k3s" + root.mkdir() + monkeypatch.setattr(module, "HOST_ROOT_UID", os.getuid()) + current = module._encode_acl( + [ + (module.ACL_USER_OBJ, 7, module.ACL_UNDEFINED_ID), + (module.ACL_GROUP_OBJ, 5, module.ACL_UNDEFINED_ID), + (module.ACL_MASK, 5, module.ACL_UNDEFINED_ID), + (module.ACL_OTHER, 5, module.ACL_UNDEFINED_ID), + ] + ) + reads = iter((current, current)) + writes = [] + monkeypatch.setattr(module, "_read_acl", lambda _path: next(reads)) + monkeypatch.setattr(module, "_acl_backup_once", lambda *_a, **_k: None) + monkeypatch.setattr( + module.os, "setxattr", lambda *args, **kwargs: writes.append(args[2]) + ) + with pytest.raises(module.HardeningError, match="validation failed"): + module._deny_sensitive_root(root) + assert writes[-1] == current + + +@pytest.mark.parametrize("remove_errno", [errno.ENODATA, errno.EPERM]) +def test_sensitive_root_failure_removes_new_acl_or_propagates_cleanup_error( + tmp_path, monkeypatch, remove_errno +): + module = _load() + root = tmp_path / "k3s" + root.mkdir() + monkeypatch.setattr(module, "HOST_ROOT_UID", os.getuid()) + monkeypatch.setattr(module, "_read_acl", lambda _path: b"") + monkeypatch.setattr(module, "_acl_backup_once", lambda *_a, **_k: None) + monkeypatch.setattr( + module.os, + "setxattr", + lambda *_a, **_k: (_ for _ in ()).throw(RuntimeError("set failed")), + ) + monkeypatch.setattr( + module.os, + "removexattr", + lambda *_a, **_k: (_ for _ in ()).throw(OSError(remove_errno, "remove")), + ) + expected = OSError if remove_errno == errno.EPERM else RuntimeError + with pytest.raises(expected): + module._deny_sensitive_root(root) + + +def test_database_validation_failure_rolls_back_all_writes(tmp_path, monkeypatch): + module, originals, _key_value, _other, _public = _fixture(tmp_path, monkeypatch) + real_read = module._read_regular + calls = 0 + + def missing_expected(path, maximum=module.MAX_ACCOUNT_FILE): + nonlocal calls + calls += 1 + if calls == 1: + return originals["passwd"].encode(), path.stat(follow_symlinks=False) + return real_read(path, maximum) + + monkeypatch.setattr(module, "_read_regular", missing_expected) + with pytest.raises(module.HardeningError, match="validation failed"): + module._reconcile_databases() + for name, value in originals.items(): + assert (module.HOST_ETC / name).read_text() == value + + +def test_preexisting_target_key_removal_is_verified(tmp_path, monkeypatch): + module, _originals, key, _other, public = _fixture(tmp_path, monkeypatch) + target = module.HOST_HOME / module.ACCOUNT / ".ssh/authorized_keys" + target.parent.mkdir(parents=True) + target.write_text(key + "\n", encoding="utf-8") + real_write = module.atomic_write + + def ignore_preinstall_target(path, value, metadata): + if path == target and value == b"": + return + real_write(path, value, metadata) + + monkeypatch.setattr(module, "atomic_write", ignore_preinstall_target) + with pytest.raises(module.HardeningError, match="preexisting Hermes authorization"): + module._move_key(public) + + +def test_existing_target_without_key_does_not_need_precleanup(tmp_path, monkeypatch): + module, _originals, _key_value, other, public = _fixture(tmp_path, monkeypatch) + target = module.HOST_HOME / module.ACCOUNT / ".ssh/authorized_keys" + target.parent.mkdir(parents=True) + target.write_text(other + "\n", encoding="utf-8") + module._move_key(public) + assert target.read_text().endswith("\n") diff --git a/testing/tests/test_hermes_node_audit_coverage.py b/testing/tests/test_hermes_node_audit_coverage.py new file mode 100644 index 00000000..20114ada --- /dev/null +++ b/testing/tests/test_hermes_node_audit_coverage.py @@ -0,0 +1,265 @@ +"""Behavioral branch coverage for dedicated node-account privilege audits.""" + +from __future__ import annotations + +import os +import struct +import sys +from types import SimpleNamespace + +import pytest + +from testing.tests.test_hermes_node_account_support import _load + + +def _audit_module(): + hardening = _load() + return sys.modules[hardening.audit_membership.__module__] + + +def test_member_parser_handles_empty_valid_malformed_and_duplicates(): + module = _audit_module() + assert module._members("", "group") == set() + assert module._members("one,two", "group") == {"one", "two"} + for value in ("one,", " one", "one,one"): + with pytest.raises(module.HardeningError): + module._members(value, "group") + + +def test_membership_audit_accepts_unrelated_records_and_rejects_account(): + module = _audit_module() + module.audit_membership( + "hermes-agent", [["disk", "x", "6", "atlas"]], [["disk", "!", "atlas", ""]] + ) + with pytest.raises(module.HardeningError, match="supplementary"): + module.audit_membership( + "hermes-agent", [["disk", "x", "6", "hermes-agent"]], [] + ) + with pytest.raises(module.HardeningError, match="gshadow"): + module.audit_membership("hermes-agent", [], [["disk", "!", "hermes-agent", ""]]) + + +@pytest.mark.parametrize("kind", ["mode", "symlink"]) +def test_policy_files_reject_unsafe_directories(tmp_path, kind): + module = _audit_module() + host_etc = tmp_path / "etc" + host_etc.mkdir() + root = host_etc / "sudoers.d" + if kind == "mode": + root.mkdir(mode=0o777) + root.chmod(0o777) + else: + target = tmp_path / "target" + target.mkdir() + root.symlink_to(target) + with pytest.raises(module.HardeningError, match="unsafe sudo/polkit"): + module._policy_files(host_etc, tmp_path / "share", os.getuid()) + + +def test_policy_file_count_is_bounded(tmp_path): + module = _audit_module() + root = tmp_path / "etc/sudoers.d" + root.mkdir(parents=True) + for index in range(257): + (root / str(index)).write_text("# safe\n", encoding="utf-8") + with pytest.raises(module.HardeningError, match="too many"): + module._policy_files(tmp_path / "etc", tmp_path / "share", os.getuid()) + + +def _acl(*entries): + return struct.pack("= 1000", "exact literal"), + ("subject.uid == -1", "operators"), + ("subject.user == name", "exact literal"), + ("subject.user.indexOf(\"h\") == 0", "exact literal"), + ("subject.isInGroup(group)", "exact literal"), + ('subject["user"] == "atlas"', "computed lookup"), + ], +) +def test_computed_identity_grants_fail_closed(snippet: str, match: str): + module = _load() + text = ( + "polkit.addRule(function(action, subject) {\n" + f" if ({snippet}) {{ return polkit.Result.YES; }}\n" + "});\n" + ) + with pytest.raises(module.HardeningError, match=match): + _rules(module, text) + + +@pytest.mark.parametrize( + ("text", "match"), + [ + ("var x = `hermes`;\n", "template strings"), + ('var x = "unterminated\n', "unterminated"), + ('var x = "bad\\x2descape";\n', "computed escapes"), + ('var x = "dangling\\', "computed escapes"), + ("/* never closed\n", "comment is unterminated"), + ("var f = eval;\n", "cannot prove"), + ('var s = String.fromCharCode(104);\n', "cannot prove"), + ("var a = arguments;\n", "cannot prove"), + ('var r = polkit.Result["YES"];\n', "computed lookup"), + ("var r = polkit.Result;\n", "computed lookup"), + ("var s = subject;\n", "computed code"), + ("check(subject);\n", "computed code"), + ], +) +def test_unprovable_rule_constructs_fail_closed(text: str, match: str): + module = _load() + with pytest.raises(module.HardeningError, match=match): + _rules(module, text) + + +def test_regex_literal_tokenizer_desync_grant_is_rejected(): + """A regex literal embedding a quote must not open a phantom string. + + Without regex awareness the two /'/ literals fool the string scanner into + swallowing the unconditional Result.YES between their quotes; the grant + then reads as unscoped and would slip past every later check. + """ + module = _load() + exploit = ( + "polkit.addRule(function(action, subject) { " + "var a = /'/; " + 'if (action.id == "org.freedesktop.policykit.exec") ' + "{ return polkit.Result.YES; } " + "var b = /'/; });\n" + ) + with pytest.raises(module.HardeningError, match="regex or division"): + _rules(module, exploit) + + +@pytest.mark.parametrize( + "snippet", + [ + "var a = /'/;", + 'var a = /"/;', + "var a = /abc/;", + "var a = /[/'\"]/;", + "var a = /x\\/y/;", + "var a = subject.user / 2;", + "// a comment / with a slash then\nvar a = /'/;", + '/* block */ var a = /"/;', + "return polkit.Result.YES / 1;", + ], +) +def test_every_slash_form_outside_a_comment_fails_closed(snippet: str): + module = _load() + text = f"polkit.addRule(function(action, subject) {{ {snippet} }});\n" + with pytest.raises(module.HardeningError, match="regex or division"): + _rules(module, text) + + +def test_slashes_confined_to_strings_and_comments_stay_legal(): + module = _load() + # Slashes inside string literals and // or /* */ comments are inert. + _rules( + module, + "// path-like /usr/bin comment\n" + "/* another /etc/ comment */\n" + "polkit.addRule(function(action, subject) {\n" + ' if (action.id == "org.freedesktop.systemd1.manage-units" &&\n' + ' subject.user == "systemd-network") {\n' + " return polkit.Result.YES;\n" + " }\n" + "});\n", + ) + + +def test_string_and_comment_handling_keeps_literal_rules_auditable(): + module = _load() + code, strings = module._split_strings( + 'var a = "with // not a comment"; // trailing\n' + "/* block */ var b = 'quo\\'te\\n';\n" + ) + assert strings == ["with // not a comment", "quo'te\n"] + assert "//" not in code and "block" not in code + assert "\x000\x00" in code and "\x001\x00" in code + + +def test_grant_detection_survives_spacing_and_strings(): + module = _load() + with pytest.raises(module.HardeningError, match="explicit identity"): + _rules(module, "polkit.addRule(function() { return polkit.Result . YES; });\n") + # A grant token inside a string is inert and needs no identity scope. + _rules(module, 'var label = "polkit.Result.YES";\n') + + +@pytest.mark.parametrize( + ("text", "match"), + [ + ( + "[All]\nIdentity=unix-user:*\nAction=org.example\nResultAny=yes\n", + "exact literal", + ), + ( + "[N]\nIdentity=unix-netgroup:ops\nAction=org.example\nResultActive=yes\n", + "exact literal", + ), + ("[N]\nAction=org.example\nResultInactive=yes\n", "identity scope"), + ("[N]\nIdentity=\nAction=org.example\nResultAny=yes\n", "identity scope"), + ], +) +def test_pkla_grants_require_exact_literal_identities(text: str, match: str): + module = _load() + with pytest.raises(module.HardeningError, match=match): + module.audit_polkit_policy(Path("localauthority/50-local.d/a.pkla"), text) + + +def test_pkla_literal_grants_and_deny_entries_pass(): + module = _load() + module.audit_polkit_policy( + Path("localauthority/50-local.d/a.pkla"), + "[Mount]\nIdentity=unix-user:atlas;unix-group:operators\n" + "Action=org.freedesktop.udisks2.filesystem-mount\nResultAny=yes\n", + ) + module.audit_polkit_policy( + Path("etc/b.pkla"), + "[Deny]\nIdentity=unix-user:*\nAction=org.example\nResultAny=no\n", + ) + + +def test_integration_rejects_previously_invisible_grants(tmp_path: Path): + spec = importlib.util.spec_from_file_location( + "node_account_audit_polkit_test", SCRIPTS / "node_account_audit.py" + ) + assert spec and spec.loader + audit = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = audit + spec.loader.exec_module(audit) + + host_etc = tmp_path / "etc" + rules = host_etc / "polkit-1/rules.d" + rules.mkdir(parents=True) + (rules / "90-open.rules").write_text( + "polkit.addRule(function() { return polkit.Result.YES; });\n", + encoding="utf-8", + ) + with pytest.raises(audit.HardeningError, match="explicit identity"): + audit.audit_privilege_policies( + "hermes-agent", 1200, os.getuid(), host_etc, tmp_path / "missing" + ) + + (rules / "90-open.rules").unlink() + local = host_etc / "polkit-1/localauthority/50-local.d" + local.mkdir(parents=True) + (local / "wide.pkla").write_text( + "[W]\nIdentity=unix-user:*\nAction=org.example.benign\nResultAny=yes\n", + encoding="utf-8", + ) + with pytest.raises(audit.HardeningError, match="exact literal"): + audit.audit_privilege_policies( + "hermes-agent", 1200, os.getuid(), host_etc, tmp_path / "missing" + ) \ No newline at end of file diff --git a/testing/tests/test_hermes_node_sudo_alias_audit.py b/testing/tests/test_hermes_node_sudo_alias_audit.py new file mode 100644 index 00000000..e62b82bf --- /dev/null +++ b/testing/tests/test_hermes_node_sudo_alias_audit.py @@ -0,0 +1,134 @@ +"""Sudo alias and computed-identity denial proofs for the Hermes account.""" + +from __future__ import annotations + +import importlib.util +import os +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).parents[2] +SCRIPTS = ROOT / "services/hermes/scripts" +sys.path.insert(0, str(SCRIPTS)) + + +def _load(): + spec = importlib.util.spec_from_file_location( + "node_account_audit_sudo_test", SCRIPTS / "node_account_audit.py" + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _audit(tmp_path: Path, sudoers: str): + module = _load() + host_etc = tmp_path / "etc" + (host_etc / "sudoers.d").mkdir(parents=True, exist_ok=True) + (host_etc / "sudoers.d/atlas").write_text(sudoers, encoding="utf-8") + module.audit_privilege_policies( + "hermes-agent", 1200, os.getuid(), host_etc, tmp_path / "missing-share" + ) + + +def test_benign_distribution_policy_passes(tmp_path: Path): + _audit( + tmp_path, + "Defaults secure_path=/usr/bin\n" + "root ALL=(ALL:ALL) ALL\n" + "%sudo ALL=(ALL:ALL) ALL\n" + "#999 ALL=(root) /usr/bin/uptime\n" + "alice, bob ALL=(root) NOPASSWD: /usr/bin/systemctl status\n", + ) + + +def test_alias_reaching_hermes_directly_is_denied(tmp_path: Path): + module = _load() + with pytest.raises(module.HardeningError, match="sudo authority"): + _audit( + tmp_path, + "User_Alias OPERATORS = atlas, hermes-agent\n" + "OPERATORS ALL=(ALL) ALL\n", + ) + + +def test_alias_reaching_hermes_by_numeric_uid_is_denied(tmp_path: Path): + module = _load() + with pytest.raises(module.HardeningError, match="sudo authority"): + _audit(tmp_path, "User_Alias OPERATORS = #1200\nOPERATORS ALL=(ALL) ALL\n") + + +def test_alias_expanding_to_all_is_denied(tmp_path: Path): + module = _load() + with pytest.raises(module.HardeningError, match="broad sudo authority"): + _audit(tmp_path, "User_Alias ADMINS = ALL\nADMINS ALL=(ALL) ALL\n") + + +def test_nested_alias_expanding_to_all_via_continuation_is_denied(tmp_path: Path): + module = _load() + with pytest.raises(module.HardeningError, match="broad sudo authority"): + _audit( + tmp_path, + "User_Alias INNER = atlas, \\\n ALL\n" + "User_Alias OUTER = INNER\n" + "OUTER ALL=(ALL) ALL\n", + ) + + +@pytest.mark.parametrize( + ("sudoers", "match"), + [ + ("+operators ALL=(ALL) ALL\n", "netgroup"), + ("%:S-1-5-32 ALL=(ALL) ALL\n", "netgroup"), + ("GHOSTS ALL=(ALL) ALL\n", "alias is undefined"), + ("User_Alias A = B\nUser_Alias B = A\nA ALL=(ALL) ALL\n", "cyclic"), + ("User_Alias broken\nroot ALL=(ALL) ALL\n", "not auditable"), + ("User_Alias lower = atlas\n", "not auditable"), + ("User_Alias A = atlas\nUser_Alias A = bob\nA ALL=(ALL) ALL\n", "not auditable"), + ("User_Alias A = atlas,,bob\nA ALL=(ALL) ALL\n", "not auditable"), + ("stray-line-without-equals\n", "not auditable"), + ("=orphan (ALL) ALL\n", "not auditable"), + ("! ALL=(ALL) ALL\n", "malformed"), + ("User_Alias A = atlas : B = ALL\nB ALL=(ALL) ALL\n", "broad sudo"), + ], +) +def test_unauditable_or_broad_sudo_policies_fail_closed( + tmp_path: Path, sudoers: str, match: str +): + module = _load() + with pytest.raises(module.HardeningError, match=match): + _audit(tmp_path, sudoers) + + +def test_negated_and_quoted_principals_are_expanded_before_judging(tmp_path: Path): + module = _load() + with pytest.raises(module.HardeningError, match="broad sudo authority"): + _audit(tmp_path, '!"ALL" ALL=(ALL) ALL\n') + _audit(tmp_path, "!alice ALL=(ALL) ALL\n") + + +def test_continuation_join_requires_a_complete_final_line(): + module = _load() + with pytest.raises(module.HardeningError, match="line continuation"): + module._joined_sudo_lines("root ALL=(ALL) ALL \\") + assert module._joined_sudo_lines("a \\\nb\nc") == ["a b", "c"] + + +def test_alias_definitions_parse_multiple_groups_per_line(): + module = _load() + aliases = module._sudo_user_aliases( + ["User_Alias A = atlas, bob : B = carol", "Runas_Alias R = root"] + ) + assert aliases == {"A": ["atlas", "bob"], "B": ["carol"]} + + +def test_expansion_accepts_nested_literal_users(): + module = _load() + aliases = {"A": ["atlas", "B"], "B": ["carol"]} + module._expand_sudo_principal("A", aliases, frozenset()) + module._expand_sudo_principal("%wheel", aliases, frozenset()) + module._expand_sudo_principal("#999", aliases, frozenset()) \ No newline at end of file diff --git a/testing/tests/test_hermes_receive_pack_scan.py b/testing/tests/test_hermes_receive_pack_scan.py new file mode 100644 index 00000000..7d17fa6f --- /dev/null +++ b/testing/tests/test_hermes_receive_pack_scan.py @@ -0,0 +1,217 @@ +"""Quarantine content-scanning contracts for brokered Git pushes.""" + +from __future__ import annotations + +import hashlib +import io +import zlib + +import pytest + +from testing.tests.test_hermes_scm_broker_support import ( + _blob_pack, + _load, + _object_entry, + _pack_of, + _receive_command, +) + +ZERO = b"0" * 40 +COMMIT = b"1" * 40 +TOKEN = "runtime-sentinel" +FORMS = (b"runtime-sentinel",) + + +def _request(pack: bytes, ref: bytes = b"refs/heads/hermes/scan") -> bytes: + return _receive_command(ZERO, COMMIT, ref, pack) + + +def test_clean_push_with_real_content_passes(): + scan = _load("receive_pack_scan") + pack = _blob_pack( + b"# services/example/deployment.yaml\napiVersion: apps/v1\n", + b"tree 5b3a\nparent none\n", + ) + scan.validate_receive_pack(_request(pack), TOKEN, FORMS) + + +@pytest.mark.parametrize( + "payload", + [ + b"-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaA==\n", + b"-----BEGIN RSA PRIVATE KEY-----\nMIIE\n", + b"key = ghp_" + b"a1" * 12, + b"aws_access_key_id = AKIAABCDEFGHIJKLMNOP", + b"jwt eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxIn0.c2lnbmF0dXJlLXNlZ21lbnQ", + b"hook https://hooks.slack.com/services/T000/B000/tokentokentokentoken", + b"AGE-SECRET-KEY-1QQPCXR7QQPCXR7QQPCXR7QQPCXR7", + b"PuTTY-User-Key-File-3: ssh-ed25519", + b"Authorization: Bearer abc123-tokenvalue", + b"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF2c hermes@atlas", + b"gitea gta_" + b"z9" * 12, + ], +) +def test_credential_shaped_object_content_is_rejected(payload): + scan = _load("receive_pack_scan") + with pytest.raises(scan.PolicyError, match="credential-shaped"): + scan.validate_receive_pack(_request(_blob_pack(payload)), TOKEN, FORMS) + + +def test_runtime_token_hidden_by_compression_is_still_caught(): + scan = _load("receive_pack_scan") + pack = _blob_pack(b"config value: runtime-sentinel\n") + # The compressed wire bytes do not contain the token; only the + # decompressed payload does. + assert b"runtime-sentinel" not in pack + with pytest.raises(scan.PolicyError, match="runtime credential material"): + scan.validate_receive_pack(_request(pack), TOKEN, FORMS) + + +def _size_varint(value: int) -> bytes: + out = bytearray() + while True: + byte = value & 0x7F + value >>= 7 + if value: + out.append(byte | 0x80) + else: + out.append(byte) + return bytes(out) + + +def _typed_header(type_code: int, size: int) -> bytes: + byte = (type_code << 4) | (size & 0x0F) + size >>= 4 + header = bytearray() + while size: + header.append(byte | 0x80) + byte = size & 0x7F + size >>= 7 + header.append(byte) + return bytes(header) + + +def test_secret_assembled_only_by_delta_resolution_is_caught(): + scan = _load("receive_pack_scan") + base = b"docs: AKIAABCD" + completion = b"EFGHIJKLMNOP" + delta = ( + _size_varint(len(base)) + + _size_varint(len(base) + len(completion)) + + bytes([0x80 | 0x10, len(base)]) + + bytes([len(completion)]) + + completion + ) + base_sha = hashlib.sha1(b"blob %d\x00" % len(base) + base).digest() + delta_entry = _typed_header(7, len(delta)) + base_sha + zlib.compress(delta) + pack = _pack_of([_object_entry(3, base), delta_entry]) + # Neither the base nor the delta insert alone matches; the resolved + # payload is the only place the credential exists. + with pytest.raises(scan.PolicyError, match="runtime credential|credential-shaped"): + scan.validate_receive_pack(_request(pack), TOKEN, FORMS) + + +def test_thin_pack_deltas_are_rejected_outright(): + scan = _load("receive_pack_scan") + delta = _size_varint(4) + _size_varint(1) + bytes([1]) + b"x" + entry = _typed_header(7, len(delta)) + b"\xbb" * 20 + zlib.compress(delta) + with pytest.raises(scan.PolicyError, match="push full packs"): + scan.validate_receive_pack(_request(_pack_of([entry])), TOKEN, FORMS) + + +def test_trailing_bytes_after_the_pack_are_rejected(): + scan = _load("receive_pack_scan") + body = _request(_pack_of([])) + b"extra" + with pytest.raises(scan.PolicyError, match="bytes after its pack"): + scan.validate_receive_pack(body, TOKEN, FORMS) + + +@pytest.mark.parametrize( + ("line", "match"), + [ + (b"shallow " + COMMIT, "ref command is invalid"), + (COMMIT + b" " + COMMIT + b" refs/heads/hermes/x", "feature-branch creation"), + (ZERO + b" " + ZERO + b" refs/heads/hermes/x", "feature-branch creation"), + (ZERO + b" " + COMMIT + b" refs/heads/main", "namespaced feature branches"), + (ZERO + b" " + COMMIT + b" refs/tags/v1", "namespaced feature branches"), + (ZERO + b" " + COMMIT + b" refs/heads/hermes/\xff", "canonical ASCII"), + (b"zz" * 20 + b" " + COMMIT + b" refs/heads/hermes/x", "ref command is invalid"), + ], +) +def test_push_commands_outside_the_narrow_grant_are_rejected(line, match): + scan = _load("receive_pack_scan") + framed = f"{len(line) + 5:04x}".encode() + line + b"\n" + b"0000" + with pytest.raises(scan.PolicyError, match=match): + scan.validate_receive_pack(framed + _pack_of([]), TOKEN, FORMS) + + +def test_command_stream_bounds_are_enforced(): + scan = _load("receive_pack_scan") + with pytest.raises(scan.PolicyError, match="no ref command"): + scan.validate_receive_pack(b"0000" + _pack_of([]), TOKEN, FORMS) + with pytest.raises(scan.PolicyError, match="omitted its command terminator"): + scan.validate_receive_pack(b"", TOKEN, FORMS) + line = ZERO + b" " + COMMIT + b" refs/heads/hermes/x\n" + framed = f"{len(line) + 4:04x}".encode() + line + many = framed * (scan.MAX_PUSH_COMMANDS + 1) + b"0000" + _pack_of([]) + with pytest.raises(scan.PolicyError, match="too many ref commands"): + scan.validate_receive_pack(many, TOKEN, FORMS) + with pytest.raises(scan.PolicyError, match="framing is invalid"): + scan.validate_receive_pack(b"zzzz", TOKEN, FORMS) + with pytest.raises(scan.PolicyError, match="framing is invalid"): + scan.validate_receive_pack(b"0003", TOKEN, FORMS) + with pytest.raises(scan.PolicyError, match="framing is invalid"): + scan.validate_receive_pack(b"0009abc", TOKEN, FORMS) + + +def test_credential_forms_in_command_lines_are_rejected(): + scan = _load("receive_pack_scan") + line = ZERO + b" " + COMMIT + b" refs/heads/hermes/x runtime-sentinel\n" + framed = f"{len(line) + 4:04x}".encode() + line + b"0000" + with pytest.raises(scan.PolicyError, match="runtime credential material"): + scan.validate_receive_pack(framed + _pack_of([]), TOKEN, FORMS) + + +def test_scan_crosses_chunk_boundaries_with_carry(monkeypatch): + scan = _load("receive_pack_scan") + monkeypatch.setattr(scan, "SCAN_CHUNK", 8) + monkeypatch.setattr(scan, "SCAN_CARRY", 48) + payload = b"x" * 20 + b" AKIAABCDEFGHIJKLMNOP " + b"y" * 20 + with pytest.raises(scan.PolicyError, match="credential-shaped"): + scan._scan_payload(io.BytesIO(payload), FORMS) + scan._scan_payload(io.BytesIO(b"clean " * 40), FORMS) + scan._scan_payload(io.BytesIO(b""), FORMS) + + +def test_scan_loose_object_reports_type_and_scans_content(): + scan = _load("receive_pack_scan") + clean = zlib.compress(b"blob 5\x00clean") + assert scan.scan_loose_object(clean, FORMS) == 3 + secret = b"-----BEGIN EC PRIVATE KEY-----" + dirty = zlib.compress(b"blob %d\x00" % len(secret) + secret) + with pytest.raises(scan.PolicyError, match="credential-shaped"): + scan.scan_loose_object(dirty, FORMS) + leaked = b"the runtime-sentinel value" + with pytest.raises(scan.PolicyError, match="runtime credential material"): + scan.scan_loose_object( + zlib.compress(b"blob %d\x00" % len(leaked) + leaked), FORMS + ) + + +def test_content_patterns_extend_the_shared_standalone_corpus(): + scan = _load("receive_pack_scan") + policy = _load("gitea_api_policy") + assert len(scan.CONTENT_SECRET_PATTERNS) == len(policy.STANDALONE_SECRET_PATTERNS) + 3 + assert all( + isinstance(pattern.pattern, bytes) for pattern in scan.CONTENT_SECRET_PATTERNS + ) + + +def test_validation_rewinds_the_stream_for_upstream_forwarding(): + scan = _load("receive_pack_scan") + body = _request(_blob_pack(b"safe content")) + stream = io.BytesIO(body) + scan.validate_receive_pack(stream, TOKEN, FORMS) + assert stream.tell() == 0 + with pytest.raises(scan.PolicyError): + scan.validate_receive_pack(io.BytesIO(b"zzzz"), TOKEN, FORMS) \ No newline at end of file diff --git a/testing/tests/test_hermes_runtime_access.py b/testing/tests/test_hermes_runtime_access.py index 7e455eb6..83bf11f2 100644 --- a/testing/tests/test_hermes_runtime_access.py +++ b/testing/tests/test_hermes_runtime_access.py @@ -11,14 +11,19 @@ from pathlib import Path import pytest import yaml - ROOT = Path(__file__).parents[2] HERMES = ROOT / "services" / "hermes" SCRIPTS = HERMES / "scripts" +SCM_SCRIPTS = HERMES / "scm-common" / "scripts" +if str(SCRIPTS) not in sys.path: + sys.path.insert(0, str(SCRIPTS)) +if str(SCM_SCRIPTS) not in sys.path: + sys.path.insert(0, str(SCM_SCRIPTS)) def _load(name: str): - spec = importlib.util.spec_from_file_location(name, SCRIPTS / f"{name}.py") + root = SCM_SCRIPTS if name == "gitea_api" else SCRIPTS + spec = importlib.util.spec_from_file_location(name, root / f"{name}.py") assert spec and spec.loader module = importlib.util.module_from_spec(spec) sys.modules[spec.name] = module @@ -31,19 +36,23 @@ def test_gitea_api_builds_runtime_authenticated_same_origin_requests(): request = gitea_api.build_request( "POST", - "/api/v1/repos/atlas/cassandra/issues/1/comments", + "/api/v1/repos/atlas/cassandra/pulls", base_url="https://scm.bstein.dev", token="runtime-only-value", - data={"body": "reviewed"}, + data={ + "base": "main", + "body": "Ready for review.", + "head": "hermes/repair", + "title": "WIP: Repair semantic review findings", + }, ) assert isinstance(request, urllib.request.Request) - assert request.full_url == ( - "https://scm.bstein.dev/api/v1/repos/atlas/cassandra/issues/1/comments" - ) + assert request.full_url == "https://scm.bstein.dev/api/v1/repos/atlas/cassandra/pulls" assert request.method == "POST" assert request.get_header("Authorization") == "token runtime-only-value" - assert json.loads(request.data) == {"body": "reviewed"} + assert json.loads(request.data)["title"].startswith("WIP: ") + assert "draft" not in json.loads(request.data) @pytest.mark.parametrize( @@ -61,25 +70,14 @@ def test_gitea_api_rejects_foreign_or_non_api_targets(path: str): gitea_api.api_url("https://scm.bstein.dev", path) -def test_gitea_api_parses_shell_safe_fields_as_json_values(): +@pytest.mark.parametrize("method", ["DELETE", "PUT", "OPTIONS"]) +def test_gitea_api_rejects_unavailable_methods(method: str): gitea_api = _load("gitea_api") - assert gitea_api.parse_fields( - ["Do=manually-merged", "MergeCommitID=abc123", "enabled=true", "count=2"] - ) == { - "Do": "manually-merged", - "MergeCommitID": "abc123", - "enabled": True, - "count": 2, - } - - -@pytest.mark.parametrize("value", ["missing-separator", "=missing-key", "bad key=x"]) -def test_gitea_api_rejects_invalid_shell_safe_fields(value: str): - gitea_api = _load("gitea_api") - - with pytest.raises(ValueError): - gitea_api.parse_fields([value]) + with pytest.raises(gitea_api.PolicyError): + gitea_api.authorize_request( + method, "/api/v1/repos/atlas/cassandra/pulls/1", None + ) def test_jenkins_evidence_ignores_nested_execution_result(): @@ -175,8 +173,12 @@ def test_agent_runtime_stage_keeps_credentials_in_memory(tmp_path: Path, monkeyp values = { "agent-api-key": "agent-key", "chat-relay-key": "relay-key", + # gitea-token/username are present in Vault but the agent staging must + # deliberately not copy them (the least-authority broker holds Gitea + # access); the assertions below prove they never reach the runtime. "gitea-token": "gitea-key", "gitea-username": "hermes-automation", + "jenkins-image-build-token": "job-scoped-token", "node-ssh-private-key": "private-key", "node-ssh-config": "host-config", "node-ssh-known-hosts": "known-hosts", @@ -196,10 +198,15 @@ def test_agent_runtime_stage_keeps_credentials_in_memory(tmp_path: Path, monkeyp assert (runtime / "claude/.credentials.json").stat().st_mode & 0o777 == 0o600 assert (runtime / "codex/auth.json").stat().st_mode & 0o777 == 0o600 + assert (runtime / "jenkins-image-build-token").stat().st_mode & 0o777 == 0o600 assert (runtime / "claude/settings.json").is_symlink() assert (runtime / "codex/skills").is_symlink() assert not (runtime / "claude/backups").exists() assert not (runtime / "codex/sessions").exists() + assert not (runtime / "gitea-token").exists() + assert not (runtime / "gitea-username").exists() + ssh_config = (runtime / "node-ssh-config").read_text(encoding="utf-8") + assert ssh_config.startswith("Host titan-*\n User hermes-agent\n") auth = json.loads((runtime / "hermes-auth.json").read_text(encoding="utf-8")) assert auth == {"version": 1, "providers": {}, "credential_pool": {}} @@ -363,12 +370,12 @@ def test_manifests_never_seed_access_material_into_persistent_env(): assert triage_annotations[ "vault.hashicorp.com/agent-inject-secret-triage-api-key" ] == "kv/data/atlas/hermes/triage-api" - askpass = (SCRIPTS / "gitea_askpass.sh").read_text(encoding="utf-8") - assert "/runtime-access/gitea-token" in askpass - assert "GITEA_TOKEN" not in askpass - gitea_api = (SCRIPTS / "gitea_api.py").read_text(encoding="utf-8") - assert "/runtime-access/gitea-token" in gitea_api + gitea_api = ( + ROOT / "services/hermes/scm-common/scripts/gitea_api.py" + ).read_text(encoding="utf-8") + assert "scm_broker_client" in gitea_api assert "GITEA_TOKEN" not in gitea_api + assert "gitea-token" not in annotations def test_chat_media_reads_one_raw_runtime_secret(tmp_path: Path): diff --git a/testing/tests/test_hermes_runtime_stage_coverage.py b/testing/tests/test_hermes_runtime_stage_coverage.py new file mode 100644 index 00000000..004853c2 --- /dev/null +++ b/testing/tests/test_hermes_runtime_stage_coverage.py @@ -0,0 +1,137 @@ +"""Behavioral branch coverage for Vault-to-memory runtime access staging.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import pytest + +from testing.tests.test_hermes_runtime_access import _load + + +def _stage(tmp_path: Path, monkeypatch): + module = _load("stage_runtime_access") + vault = tmp_path / "vault" + runtime = tmp_path / "runtime" + home = tmp_path / "home" + vault.mkdir() + home.mkdir() + monkeypatch.setattr(module, "VAULT_ROOT", vault) + monkeypatch.setattr(module, "RUNTIME_ROOT", runtime) + monkeypatch.setattr(module, "PERSISTENT_HOME", home) + monkeypatch.setattr(module.os, "chown", lambda *_args: None) + return module, vault, runtime, home + + +def test_private_directory_and_secret_copy_validate_mode_and_content( + tmp_path, monkeypatch +): + module, vault, runtime, _home = _stage(tmp_path, monkeypatch) + module._owned_directory(runtime) + assert runtime.stat().st_mode & 0o777 == 0o700 + (vault / "empty").write_text(" \n", encoding="utf-8") + with pytest.raises(RuntimeError, match="is empty"): + module._copy_secret("empty", runtime / "empty") + (vault / "value").write_text(" staged \n", encoding="utf-8") + assert module._copy_secret("value", runtime / "value") == "staged" + assert (runtime / "value").read_text(encoding="utf-8") == "staged\n" + + +@pytest.mark.parametrize( + ("document", "required", "match"), + [ + ({"outer": {}}, ("outer", "token"), "invalid shape"), + ({"outer": {"token": ""}}, ("outer", "token"), "empty credential"), + ({"outer": {"token": 7}}, ("outer", "token"), "empty credential"), + ], +) +def test_json_staging_rejects_missing_nontext_and_empty_credentials( + tmp_path, monkeypatch, document, required, match +): + module, vault, runtime, _home = _stage(tmp_path, monkeypatch) + runtime.mkdir() + (vault / "credential").write_text(json.dumps(document), encoding="utf-8") + destination = runtime / "credential.json" + with pytest.raises(RuntimeError, match=match): + module._validated_json("credential", destination, required) + assert not destination.exists() + + +def test_noncredential_linker_skips_absent_symlink_and_existing_entries( + tmp_path, monkeypatch +): + module, _vault, runtime, home = _stage(tmp_path, monkeypatch) + runtime.mkdir() + source = home / "state" + destination = runtime / "state" + destination.mkdir() + module._link_noncredential_state(source, destination, ("allowed",)) + + source.mkdir() + (source / "allowed").write_text("safe", encoding="utf-8") + (source / "existing").write_text("new", encoding="utf-8") + (source / "target-link").write_text("new", encoding="utf-8") + (source / "linked").symlink_to(source / "allowed") + (destination / "existing").write_text("preserve", encoding="utf-8") + (destination / "target-link").symlink_to(destination / "existing") + module._link_noncredential_state( + source, + destination, + ("missing", "linked", "existing", "target-link", "allowed"), + ) + assert (destination / "allowed").is_symlink() + assert (destination / "existing").read_text(encoding="utf-8") == "preserve" + + +def test_empty_auth_store_is_private_and_noncredential(tmp_path, monkeypatch): + module, _vault, runtime, _home = _stage(tmp_path, monkeypatch) + runtime.mkdir() + module._write_empty_auth_store() + path = runtime / "hermes-auth.json" + assert json.loads(path.read_text(encoding="utf-8")) == { + "version": 1, + "providers": {}, + "credential_pool": {}, + } + assert path.stat().st_mode & 0o777 == 0o600 + + +def test_node_ssh_config_rejects_nul_and_removes_partial_file(tmp_path, monkeypatch): + module, vault, runtime, _home = _stage(tmp_path, monkeypatch) + runtime.mkdir() + (vault / "node-ssh-config").write_bytes(b"Host titan-01\x00\n") + with pytest.raises(RuntimeError, match="control data"): + module._stage_node_ssh_config() + assert not (runtime / "node-ssh-config").exists() + + +@pytest.mark.parametrize( + ("mode", "secret_name"), + [("chat", "chat-relay-key"), ("triage", "triage-api-key")], +) +def test_isolated_tenant_staging_exposes_only_required_secret( + tmp_path, monkeypatch, mode, secret_name +): + module, vault, runtime, _home = _stage(tmp_path, monkeypatch) + (vault / secret_name).write_text("synthetic-value\n", encoding="utf-8") + getattr(module, f"stage_{mode}")() + assert (runtime / secret_name).read_text(encoding="utf-8") == "synthetic-value\n" + assert (runtime / "hermes-auth.json").exists() + assert sorted(path.name for path in runtime.iterdir()) == sorted( + ["hermes-auth.json", secret_name] + ) + + +@pytest.mark.parametrize("mode", ["agent", "chat", "triage"]) +def test_main_dispatches_each_mode(mode, monkeypatch, capsys): + module = _load("stage_runtime_access") + calls = [] + monkeypatch.setattr(module, "stage_agent", lambda: calls.append("agent")) + monkeypatch.setattr(module, "stage_chat", lambda: calls.append("chat")) + monkeypatch.setattr(module, "stage_triage", lambda: calls.append("triage")) + monkeypatch.setattr(sys, "argv", ["stage-runtime-access", mode]) + assert module.main() == 0 + assert calls == [mode] + assert "staged from Vault" in capsys.readouterr().out diff --git a/testing/tests/test_hermes_scm_broker.py b/testing/tests/test_hermes_scm_broker.py new file mode 100644 index 00000000..2a0fad44 --- /dev/null +++ b/testing/tests/test_hermes_scm_broker.py @@ -0,0 +1,146 @@ +"""Atlas route and Git protocol contracts for the Hermes SCM broker.""" + +from __future__ import annotations + +import pytest + +from testing.tests.test_hermes_scm_broker_support import ( + Response, + _load, + _receive_command, +) + +@pytest.mark.parametrize( + "target", + [ + "/api/v1/repos/atlas/cassandra/pulls/1/merge", + "/api/v1/repos/atlas/cassandra/pulls/1/reviews", + "/api/v1/admin/users", + "/git/evil/cassandra.git/info/refs?service=git-upload-pack", + "/git/atlas/cassandra.git/hooks", + "/git/atlas/cassandra.git/git-receive-pack?force=true", + "/git/atlas/cassandra.git/info/refs?service=git-upload-pack\nignored", + "/git/atlas/cassandra.git/info/refs?service=git-upload-pack\tignored", + "/git/atlas/cassandra.git/info/refs?service=git-upload-pack#fragment", + ], +) +def test_broker_rejects_privileged_or_non_atlas_routes(target: str): + broker = _load("scm_broker") + + with pytest.raises(broker.PolicyError): + broker._git_target(target) + + +def test_broker_metadata_boundary_reuses_explicit_read_allowlist(): + api = _load("gitea_api") + + assert ( + api.authorize_request("GET", "/api/v1/repos/atlas/cassandra/pulls/1", None) + == "pull" + ) + for path in ( + "/api/v1/repos/atlas/cassandra/pulls/1/merge", + "/api/v1/repos/atlas/cassandra/pulls/1/reviews", + "/api/v1/repos/atlas/cassandra/hooks", + "/api/v1/admin/users", + ): + with pytest.raises(api.PolicyError): + api.authorize_request("GET", path, None) + + +def test_receive_pack_allows_only_new_namespaced_feature_branch(): + broker = _load("scm_broker") + zero = b"0" * 40 + commit = b"1" * 40 + + broker._validate_receive_pack( + _receive_command(zero, commit, b"refs/heads/hermes/focused-fix"), + "runtime-sentinel", + ) + for old, new, ref in ( + (commit, b"2" * 40, b"refs/heads/hermes/focused-fix"), + (commit, zero, b"refs/heads/hermes/focused-fix"), + (zero, commit, b"refs/heads/main"), + (zero, commit, b"refs/heads/master"), + (zero, commit, b"refs/tags/release"), + ): + with pytest.raises(broker.PolicyError): + broker._validate_receive_pack( + _receive_command(old, new, ref), "runtime-sentinel" + ) + + +def test_git_proxy_uses_fixed_origin_and_never_reflects_credential(): + broker = _load("scm_broker") + seen = [] + + def opener(request, timeout): + seen.append((request, timeout)) + return Response( + b"git-result", + content_type="application/x-git-upload-pack-result", + ) + + result = broker._upstream_git_request( + "/atlas/cassandra.git/git-upload-pack", + method="POST", + body=b"request", + content_type="application/x-git-upload-pack-request", + expected_type="application/x-git-upload-pack-result", + token="runtime-sentinel", + opener=opener, + ) + + assert result == b"git-result" + request = seen[0][0] + assert request.full_url == ( + "https://scm.bstein.dev/atlas/cassandra.git/git-upload-pack" + ) + assert b"runtime-sentinel" not in request.data + assert "runtime-sentinel" not in request.full_url + + with pytest.raises(broker.PolicyError, match="credential material"): + broker._upstream_git_request( + "/atlas/cassandra.git/git-upload-pack", + method="POST", + body=b"request", + content_type="application/x-git-upload-pack-request", + expected_type="application/x-git-upload-pack-result", + token="runtime-sentinel", + opener=lambda *_a, **_k: Response( + b"runtime-sentinel", + content_type="application/x-git-upload-pack-result", + ), + ) + encoded = broker._credential_forms("runtime-sentinel")[1] + with pytest.raises(broker.PolicyError, match="credential material"): + broker._upstream_git_request( + "/atlas/cassandra.git/git-upload-pack", + method="POST", + body=b"request", + content_type="application/x-git-upload-pack-request", + expected_type="application/x-git-upload-pack-result", + token="runtime-sentinel", + opener=lambda *_a, **_k: Response( + b"Authorization: Basic " + encoded, + content_type="application/x-git-upload-pack-result", + ), + ) +@pytest.mark.parametrize("status", [201, 202, 204, 206, 301, 302, 307, 308]) +def test_git_proxy_requires_exact_http_200(status: int): + broker = _load("scm_broker") + + with pytest.raises(broker.PolicyError, match="unexpected HTTP status"): + broker._upstream_git_request( + "/atlas/cassandra.git/git-upload-pack", + method="POST", + body=b"request", + content_type="application/x-git-upload-pack-request", + expected_type="application/x-git-upload-pack-result", + token="runtime-sentinel", + opener=lambda *_a, **_k: Response( + b"result", + status=status, + content_type="application/x-git-upload-pack-result", + ), + ) diff --git a/testing/tests/test_hermes_scm_broker_client_coverage.py b/testing/tests/test_hermes_scm_broker_client_coverage.py new file mode 100644 index 00000000..9fa65fa6 --- /dev/null +++ b/testing/tests/test_hermes_scm_broker_client_coverage.py @@ -0,0 +1,160 @@ +"""Behavioral coverage for the credential-free SCM broker client and I/O.""" + +from __future__ import annotations + +import io +import json +import pytest + +from testing.tests.test_hermes_scm_broker_support import Response, _load + + +def test_client_default_opener_uses_deadline_bounded_exchange(monkeypatch): + client = _load("scm_broker_client") + sentinel = object() + monkeypatch.setattr( + client.deadline_http, + "open_bounded", + lambda request, *, maximum, timeout: (request, maximum, timeout, sentinel), + ) + assert client._open("request", 4) == ( + "request", + client.MAX_RESPONSE_BYTES, + 4, + sentinel, + ) + + +@pytest.mark.parametrize( + ("endpoint", "payload", "match"), + [ + ("/v1/admin", {}, "outside"), + ("/v1/metadata", {"path": "x" * (64 * 1024)}, "safe size"), + ], +) +def test_client_rejects_unknown_or_oversized_operations(endpoint, payload, match): + client = _load("scm_broker_client") + with pytest.raises(client.PolicyError, match=match): + client.request(endpoint, payload, opener=lambda *_a, **_k: None) + + +class _GetCodeResponse(Response): + def __init__(self, body: bytes, *, content_type="application/json", code=200): + super().__init__(body, content_type=content_type) + del self.status + self._code = code + + def getcode(self): + return self._code + + +def test_client_accepts_getcode_fallback_and_create_payload(): + client = _load("scm_broker_client") + seen = [] + + def opener(request, timeout): + seen.append((request, timeout)) + return _GetCodeResponse(b"{}") + + result = client.create_draft( + "cassandra", + base="main", + head="feature/coverage", + head_sha="a" * 40, + title="WIP: Coverage", + body="Review evidence", + opener=opener, + ) + assert result == b"{}" + assert json.loads(seen[0][0].data)["repo"] == "cassandra" + + +@pytest.mark.parametrize( + ("response", "match"), + [ + (_GetCodeResponse(b"{}", code=201), "HTTP status"), + (_GetCodeResponse(b"{}", content_type="text/plain"), "response type"), + ( + Response(b"x" * (2 * 1024 * 1024 + 1), content_type="application/json"), + "safe size", + ), + (Response(b"not-json", content_type="application/json"), "Expecting value"), + ], +) +def test_client_rejects_bad_status_type_size_and_json(response, match): + client = _load("scm_broker_client") + expected = ( + json.JSONDecodeError if match == "Expecting value" else client.PolicyError + ) + with pytest.raises(expected, match=match): + client.request( + "/v1/metadata", + {"path": "/api/v1/repos/atlas/cassandra"}, + opener=lambda *_a, **_k: response, + ) + + +class _Socket: + def __init__(self): + self.timeouts = [] + + def settimeout(self, value): + self.timeouts.append(value) + + +class _Stream: + def __init__(self, body: bytes, socket_shape: str = "direct"): + self.body = io.BytesIO(body) + self.socket = _Socket() + if socket_shape == "nested": + self.fp = type( + "FP", (), {"raw": type("Raw", (), {"_sock": self.socket})()} + )() + elif socket_shape == "direct": + self.fp = type("FP", (), {"raw": self.socket})() + else: + self.fp = None + + def read(self, size): + return self.body.read(size) + + +def test_response_spool_handles_direct_and_absent_socket_shapes(): + module = _load("scm_broker_io") + for shape in ("direct", "absent"): + stream = _Stream(b"safe", shape) + spool, length = module.spool_response( + stream, + 16, + (b"forbidden",), + memory_limit=8, + chunk_size=2, + deadline_seconds=5, + ) + try: + assert length == 4 and spool.read() == b"safe" + finally: + spool.close() + + +@pytest.mark.parametrize("failure", ["deadline", "size", "credential", "read"]) +def test_response_spool_closes_and_rejects_each_failure(monkeypatch, failure): + module = _load("scm_broker_io") + stream = _Stream(b"safe-forbidden-value") + kwargs = { + "maximum": 128, + "forbidden": (b"forbidden",), + "memory_limit": 8, + "chunk_size": 5, + "deadline_seconds": 5, + } + if failure == "deadline": + ticks = iter((10.0, 16.0)) + monkeypatch.setattr(module.time, "monotonic", lambda: next(ticks)) + elif failure == "size": + kwargs["maximum"] = 2 + elif failure == "read": + stream.read = lambda _size: (_ for _ in ()).throw(OSError("read failed")) + expected = module.PolicyError if failure != "read" else OSError + with pytest.raises(expected): + module.spool_response(stream, **kwargs) diff --git a/testing/tests/test_hermes_scm_broker_handler_coverage.py b/testing/tests/test_hermes_scm_broker_handler_coverage.py new file mode 100644 index 00000000..57245a6a --- /dev/null +++ b/testing/tests/test_hermes_scm_broker_handler_coverage.py @@ -0,0 +1,265 @@ +"""Behavioral branch coverage for SCM broker request handling.""" + +from __future__ import annotations + +import io +import json +import sys +from email.message import Message + +import pytest + +from testing.tests.test_hermes_scm_broker_support import _load, _receive_command + + +class _Connection: + def __init__(self): + self.timeouts = [] + + def settimeout(self, value): + self.timeouts.append(value) + + +def _headers(*, content_type="application/json", body=b""): + value = Message() + value["Content-Type"] = content_type + value["Content-Length"] = str(len(body)) + return value + + +def _handler(broker, *, path="/healthz", body=b"", content_type="application/json"): + handler = object.__new__(broker.BrokerHandler) + handler.path = path + handler.headers = _headers(content_type=content_type, body=body) + handler.rfile = io.BytesIO(body) + handler.wfile = io.BytesIO() + handler.connection = _Connection() + handler.events = [] + handler.send_response = lambda status: handler.events.append(("status", status)) + handler.send_header = lambda name, value: handler.events.append((name, value)) + handler.end_headers = lambda: handler.events.append(("headers", "done")) + return handler + + +def test_header_validation_accepts_safe_ascii_and_rejects_controls(): + broker = _load("scm_broker") + handler = _handler(broker) + handler._validate_headers() + + class HeaderBag: + def items(self): + return [("X-Test", "bad\x01value")] + + def get_all(self, _name, default): + return default + + handler.headers = HeaderBag() + with pytest.raises(broker.PolicyError, match="controls"): + handler._validate_headers() + + class UnicodeHeader(HeaderBag): + def items(self): + return [("X-Test", "máin")] + + handler.headers = UnicodeHeader() + with pytest.raises(broker.PolicyError, match="ASCII"): + handler._validate_headers() + + +def test_json_reject_and_stream_response_methods(monkeypatch): + broker = _load("scm_broker") + handler = _handler(broker) + handler._json(202, b'{"ok":true}') + assert ("status", 202) in handler.events + assert handler.wfile.getvalue() == b'{"ok":true}' + + handler = _handler(broker) + handler._reject(403) + assert ("status", 403) in handler.events + assert json.loads(handler.wfile.getvalue()) == {"error": "request rejected"} + + handler = _handler(broker) + handler._stream(200, "application/test", io.BytesIO(b"streamed"), 8) + assert handler.wfile.getvalue() == b"streamed" + assert handler.connection.timeouts + handler.log_message("ignored %s", "value") + + handler = _handler(broker) + ticks = iter((0.0, 121.0)) + monkeypatch.setattr(broker.time, "monotonic", lambda: next(ticks)) + with pytest.raises(broker.PolicyError, match="write deadline"): + handler._stream(200, "application/test", io.BytesIO(b"x"), 1) + + +def test_get_health_and_git_discovery_paths(monkeypatch): + broker = _load("scm_broker") + handler = _handler(broker) + handler.do_GET() + assert json.loads(handler.wfile.getvalue()) == {"status": "ok"} + + closed = [] + + class Body(io.BytesIO): + def close(self): + closed.append(True) + super().close() + + handler = _handler( + broker, + path="/git/atlas/cassandra.git/info/refs?service=git-upload-pack", + ) + monkeypatch.setattr(broker, "read_token", lambda: "sentinel") + monkeypatch.setattr( + broker, + "_upstream_git_request", + lambda *args, **kwargs: (Body(b"advertisement"), 13), + ) + handler.do_GET() + assert handler.wfile.getvalue() == b"advertisement" + assert closed == [True] + + handler = _handler(broker, path="/git/atlas/cassandra.git/git-upload-pack") + handler.do_GET() + assert json.loads(handler.wfile.getvalue()) == {"error": "request rejected"} + + +def test_post_dispatches_control_and_git_or_rejects(monkeypatch): + broker = _load("scm_broker") + control = _handler(broker, path="/v1/metadata", body=b"{}") + git = _handler(broker, path="/git/atlas/cassandra.git/git-upload-pack", body=b"") + calls = [] + control._control = lambda: calls.append("control") + control._git_rpc = lambda: calls.append("git") + git._control = lambda: calls.append("control") + git._git_rpc = lambda: calls.append("git") + control.do_POST() + git.do_POST() + assert calls == ["control", "git"] + + rejected = _handler(broker, path="/v1/metadata", body=b"{}") + rejected._control = lambda: (_ for _ in ()).throw(broker.PolicyError("no")) + rejected.do_POST() + assert json.loads(rejected.wfile.getvalue()) == {"error": "request rejected"} + + +def test_control_metadata_and_draft_fields(monkeypatch): + broker = _load("scm_broker") + monkeypatch.setattr(broker, "read_token", lambda: "sentinel") + monkeypatch.setattr( + broker, "read", lambda path, token: json.dumps({"path": path}).encode() + ) + monkeypatch.setattr( + broker, + "create_draft", + lambda token, **data: json.dumps({"token_used": bool(token), **data}).encode(), + ) + + metadata_body = b'{"path":"/api/v1/repos/atlas/cassandra"}' + metadata = _handler(broker, path="/v1/metadata", body=metadata_body) + metadata._control() + assert json.loads(metadata.wfile.getvalue())["path"].endswith("cassandra") + + draft_data = { + "base": "main", + "body": "Review evidence", + "head": "feature/coverage", + "head_sha": "a" * 40, + "repo": "cassandra", + "title": "WIP: Coverage", + } + draft_body = json.dumps(draft_data).encode() + draft = _handler(broker, path="/v1/drafts", body=draft_body) + draft._control() + assert json.loads(draft.wfile.getvalue())["repo"] == "cassandra" + + for path, body, match in ( + ("/v1/metadata", b'{"wrong":"field"}', "fields"), + ("/v1/drafts", b'{"repo":1}', "fields"), + ): + with pytest.raises(broker.PolicyError, match=match): + _handler(broker, path=path, body=body)._control() + + monkeypatch.setattr(broker, "read", lambda *_a, **_k: b"sentinel") + with pytest.raises(broker.PolicyError, match="reflected"): + _handler(broker, path="/v1/metadata", body=metadata_body)._control() + + +@pytest.mark.parametrize("service", ["git-upload-pack", "git-receive-pack"]) +def test_git_rpc_streams_allowed_upload_and_receive(monkeypatch, service): + broker = _load("scm_broker") + if service == "git-receive-pack": + body = _receive_command(b"0" * 40, b"1" * 40, b"refs/heads/hermes/coverage") + else: + body = b"upload-request" + handler = _handler( + broker, + path=f"/git/atlas/cassandra.git/{service}", + body=body, + content_type=f"application/x-{service}-request", + ) + monkeypatch.setattr(broker, "read_token", lambda: "sentinel") + seen = [] + + def upstream(target, **kwargs): + seen.append((target, kwargs)) + return io.BytesIO(b"result"), 6 + + monkeypatch.setattr(broker, "_upstream_git_request", upstream) + handler._git_rpc() + assert handler.wfile.getvalue() == b"result" + assert seen[0][0].endswith(service) + assert seen[0][1]["body_length"] == len(body) + + +@pytest.mark.parametrize( + ("path", "content_type", "transfer", "match"), + [ + ( + "/git/atlas/cassandra.git/info/refs?service=git-upload-pack", + "application/x-git-upload-pack-request", + None, + "operation", + ), + ( + "/git/atlas/cassandra.git/git-upload-pack", + "text/plain", + None, + "request type", + ), + ( + "/git/atlas/cassandra.git/git-upload-pack", + "application/x-git-upload-pack-request", + "chunked", + "request type", + ), + ], +) +def test_git_rpc_rejects_discovery_wrong_type_and_chunking( + path, content_type, transfer, match +): + broker = _load("scm_broker") + handler = _handler(broker, path=path, content_type=content_type) + if transfer: + handler.headers["Transfer-Encoding"] = transfer + with pytest.raises(broker.PolicyError, match=match): + handler._git_rpc() + + +def test_broker_main_constructs_bounded_server(monkeypatch): + broker = _load("scm_broker") + seen = [] + + class Server: + def __init__(self, address, handler): + seen.append((address, handler)) + + def serve_forever(self): + seen.append("served") + + monkeypatch.setattr(broker, "BoundedThreadingHTTPServer", Server) + monkeypatch.setattr( + sys, "argv", ["broker", "--listen", "127.0.0.1", "--port", "9191"] + ) + assert broker.main() == 0 + assert seen[0] == (("127.0.0.1", 9191), broker.BrokerHandler) + assert seen[1] == "served" diff --git a/testing/tests/test_hermes_scm_broker_internal_coverage.py b/testing/tests/test_hermes_scm_broker_internal_coverage.py new file mode 100644 index 00000000..b38702ac --- /dev/null +++ b/testing/tests/test_hermes_scm_broker_internal_coverage.py @@ -0,0 +1,272 @@ +"""Behavioral branch coverage for SCM broker protocol helpers.""" + +from __future__ import annotations + +import io +from email.message import Message + +import pytest + +from testing.tests.test_hermes_scm_broker_support import ( + Response, + _load, + _receive_command, +) + + +def test_broker_redirect_status_and_bounded_read_helpers(monkeypatch): + broker = _load("scm_broker") + with pytest.raises(broker.PolicyError, match="redirects"): + broker.RejectRedirect().redirect_request( + object(), None, 302, "redirect", {}, "https://elsewhere.invalid" + ) + + class CodeOnly: + def getcode(self): + return 204 + + assert broker._status(type("Status", (), {"status": 200})()) == 200 + assert broker._status(CodeOnly()) == 204 + assert broker._status(object()) is None + + with pytest.raises(broker.PolicyError, match="safe size"): + broker._read_bounded(io.BytesIO(b"x"), 4, -1) + with pytest.raises(broker.PolicyError, match="safe size"): + broker._read_bounded(io.BytesIO(b"xxxxx"), 4) + with pytest.raises(broker.PolicyError, match="safe size"): + broker._read_bounded(io.BytesIO(b"x"), 4, 2) + with pytest.raises(broker.PolicyError, match="deadline"): + broker._read_bounded(io.BytesIO(b"x"), 4, 1, deadline=0) + + timeouts = [] + assert ( + broker._read_bounded( + io.BytesIO(b"xy"), + 4, + 2, + deadline=broker.time.monotonic() + 5, + set_timeout=timeouts.append, + ) + == b"xy" + ) + assert timeouts + + +def test_spool_bounded_rejects_bounds_early_eof_and_closes_on_read_error(): + broker = _load("scm_broker") + for length in (-1, 5): + with pytest.raises(broker.PolicyError, match="safe size"): + broker._spool_bounded( + io.BytesIO(b"x"), + 4, + length, + token="sentinel", + context="request", + deadline=999999999, + ) + with pytest.raises(broker.PolicyError, match="ended early"): + broker._spool_bounded( + io.BytesIO(b"x"), + 4, + 2, + token="sentinel", + context="request", + deadline=999999999, + ) + + class Broken: + def read(self, _size): + raise OSError("broken stream") + + with pytest.raises(OSError, match="broken"): + broker._spool_bounded( + Broken(), 4, 1, token="sentinel", context="request", deadline=999999999 + ) + + +class _HandlerInput: + def __init__(self, body: bytes, *, content_type="application/json", length=None): + self.headers = Message() + self.headers["Content-Type"] = content_type + if length is None: + length = len(body) + self.headers["Content-Length"] = str(length) + self.rfile = io.BytesIO(body) + self.connection = type( + "Connection", (), {"settimeout": lambda self, value: None} + )() + + +def test_control_json_loader_requires_fixed_json_object(): + broker = _load("scm_broker") + handler = _HandlerInput(b"{}") + handler.headers["Transfer-Encoding"] = "chunked" + with pytest.raises(broker.PolicyError, match="chunked"): + broker._load_json(handler) + with pytest.raises(broker.PolicyError, match="must be JSON"): + broker._load_json(_HandlerInput(b"{}", content_type="text/plain")) + with pytest.raises(broker.PolicyError, match="must be an object"): + broker._load_json(_HandlerInput(b"[]")) + assert broker._load_json(_HandlerInput(b'{"path":"safe"}')) == {"path": "safe"} + + +@pytest.mark.parametrize( + ("target", "expected"), + [ + ( + "/git/atlas/cassandra.git/info/refs?service=git-upload-pack", + ("cassandra", "info/refs", "git-upload-pack"), + ), + ( + "/git/atlas/cassandra.git/info/refs?service=git-receive-pack", + ("cassandra", "info/refs", "git-receive-pack"), + ), + ( + "/git/atlas/cassandra.git/git-upload-pack", + ("cassandra", "git-upload-pack", "git-upload-pack"), + ), + ( + "/git/atlas/cassandra.git/git-receive-pack", + ("cassandra", "git-receive-pack", "git-receive-pack"), + ), + ], +) +def test_git_target_accepts_each_exact_smart_http_route(target, expected): + broker = _load("scm_broker") + assert broker._git_target(target) == expected + + +def test_content_length_and_credential_helpers_cover_safe_and_rejected_values(): + broker = _load("scm_broker") + headers = Message() + headers["Content-Length"] = "0" + assert broker._content_length(headers, 1) == 0 + forms = broker._credential_forms("sentinel") + assert len(forms) == 3 and forms[0] == b"sentinel" + zero = b"0" * 40 + commit = b"1" * 40 + line = zero + b" " + commit + b" refs/heads/hermes/x " + forms[2] + b"\n" + framed = f"{len(line) + 4:04x}".encode() + line + b"0000" + with pytest.raises(broker.PolicyError, match="credential material"): + broker._validate_receive_pack(framed, "sentinel") + + +def test_receive_pack_validation_rewinds_stream_and_accepts_bytes(): + broker = _load("scm_broker") + body = _receive_command(b"0" * 40, b"1" * 40, b"refs/heads/hermes/rewind") + stream = io.BytesIO(body) + stream.seek(9) + broker._validate_receive_pack(stream, "sentinel") + assert stream.tell() == 0 + broker._validate_receive_pack(body, "sentinel") + + +@pytest.mark.parametrize( + ("body", "match"), + [ + (b"bad!", "framing"), + (b"0000", "no ref command"), + (b"0003", "framing"), + (b"0008abc", "framing"), + (b"0008a b\n0000", "ref command"), + ], +) +def test_receive_pack_rejects_bad_framing_and_commands(body, match): + broker = _load("scm_broker") + with pytest.raises(broker.PolicyError, match=match): + broker._validate_receive_pack(body, "sentinel") + + +def _packet(old: bytes, new: bytes, ref: bytes, *, terminator=True) -> bytes: + command = old + b" " + new + b" " + ref + b"\n" + framed = f"{len(command) + 4:04x}".encode() + command + return framed + (b"0000" if terminator else b"") + + +def test_receive_pack_rejects_nonascii_many_commands_and_missing_terminator( + monkeypatch, +): + broker = _load("scm_broker") + zero = b"0" * 40 + commit = b"1" * 40 + with pytest.raises(broker.PolicyError, match="canonical ASCII"): + broker._validate_receive_pack( + _packet(zero, commit, b"refs/heads/hermes/\xff"), "sentinel" + ) + command = _packet(zero, commit, b"refs/heads/hermes/fix", terminator=False) + with pytest.raises(broker.PolicyError, match="terminator"): + broker._validate_receive_pack(command, "sentinel") + limit = _load("receive_pack_scan").MAX_PUSH_COMMANDS + many = command * (limit + 1) + b"0000" + with pytest.raises(broker.PolicyError, match="too many"): + broker._validate_receive_pack(many, "sentinel") + + +def test_upstream_request_infers_byte_length_and_requires_stream_length(): + broker = _load("scm_broker") + seen = [] + + def opener(request, timeout): + seen.append((request, timeout)) + return Response(b"result", content_type="application/x-git-upload-pack-result") + + assert ( + broker._upstream_git_request( + "/atlas/cassandra.git/git-upload-pack", + method="POST", + body=b"request", + content_type=None, + expected_type="application/x-git-upload-pack-result", + token="sentinel", + opener=opener, + ) + == b"result" + ) + assert seen[0][0].get_header("Content-length") == "7" + with pytest.raises(broker.PolicyError, match="length is missing"): + broker._upstream_git_request( + "/atlas/cassandra.git/git-upload-pack", + method="POST", + body=io.BytesIO(b"request"), + content_type=None, + expected_type="application/x-git-upload-pack-result", + token="sentinel", + opener=opener, + ) + + +def test_upstream_request_rejects_wrong_type_and_supports_getcode_status(): + broker = _load("scm_broker") + + class CodeOnly(Response): + def __init__(self, body, content_type): + super().__init__(body, content_type=content_type) + del self.status + + def getcode(self): + return 200 + + assert ( + broker._upstream_git_request( + "/atlas/cassandra.git/info/refs?service=git-upload-pack", + method="GET", + body=None, + content_type=None, + expected_type="application/x-git-upload-pack-advertisement", + token="sentinel", + opener=lambda *_a, **_k: CodeOnly( + b"advertisement", "application/x-git-upload-pack-advertisement" + ), + ) + == b"advertisement" + ) + with pytest.raises(broker.PolicyError, match="response type"): + broker._upstream_git_request( + "/atlas/cassandra.git/info/refs?service=git-upload-pack", + method="GET", + body=None, + content_type=None, + expected_type="application/x-git-upload-pack-advertisement", + token="sentinel", + opener=lambda *_a, **_k: Response(b"bad", content_type="text/plain"), + ) diff --git a/testing/tests/test_hermes_scm_broker_policy.py b/testing/tests/test_hermes_scm_broker_policy.py new file mode 100644 index 00000000..163cbc91 --- /dev/null +++ b/testing/tests/test_hermes_scm_broker_policy.py @@ -0,0 +1,285 @@ +"""RBAC, Flux, and human-review contracts for the Hermes SCM broker.""" + +from __future__ import annotations + +import json + +import pytest +import yaml + +from testing.tests.test_hermes_scm_broker_support import ROOT, _load_path + +def _can_i(role: dict, api_group: str, resource: str, verb: str) -> bool: + return any( + (api_group in rule["apiGroups"] or "*" in rule["apiGroups"]) + and (resource in rule["resources"] or "*" in rule["resources"]) + and (verb in rule["verbs"] or "*" in rule["verbs"]) + for rule in role["rules"] + ) + + +def test_agent_rbac_preserves_read_diagnostics_without_administrator_authority(): + documents = list( + yaml.safe_load_all( + (ROOT / "services/hermes-observer-rbac/rbac.yaml").read_text( + encoding="utf-8" + ) + ) + ) + cluster_role, namespaced_role, binding = documents + assert binding["roleRef"]["name"] == "hermes-agent-cluster-observer-v2" + assert _can_i(cluster_role, "", "nodes", "get") + assert _can_i(namespaced_role, "", "pods", "get") + assert _can_i(namespaced_role, "", "pods/log", "get") + assert _can_i( + namespaced_role, + "kustomize.toolkit.fluxcd.io", + "kustomizations", + "list", + ) + + denied = ( + ("", "secrets", "get"), + ("", "serviceaccounts/token", "create"), + ("", "pods", "create"), + ("", "pods/exec", "create"), + ("", "pods/attach", "create"), + ("", "pods/portforward", "create"), + ("rbac.authorization.k8s.io", "clusterrolebindings", "create"), + ("authorization.k8s.io", "selfsubjectaccessreviews", "create"), + ("apps", "deployments", "patch"), + ) + assert all( + not _can_i(role, *request) + for request in denied + for role in (cluster_role, namespaced_role) + ) + + bindings = yaml.safe_load( + (ROOT / "services/hermes-observer-bindings/rolebindings.yaml").read_text( + encoding="utf-8" + ) + )["items"] + namespaces = {item["metadata"]["namespace"] for item in bindings} + assert "hermes-scm" not in namespaces + assert {"cassandra", "flux-system", "hermes", "kube-system"} <= namespaces + assert all(item["roleRef"]["name"].endswith("namespaced-observer-v2") for item in bindings) + + +def test_flux_boundary_keeps_broker_secret_and_network_separate_from_agent(): + agent = yaml.safe_load( + (ROOT / "services/hermes/agent-deployment.yaml").read_text(encoding="utf-8") + ) + annotations = agent["spec"]["template"]["metadata"]["annotations"] + assert not any("gitea" in key.lower() for key in annotations) + assert "GIT_ASKPASS" not in json.dumps(agent) + + broker = yaml.safe_load( + (ROOT / "services/hermes-scm-broker/deployment.yaml").read_text( + encoding="utf-8" + ) + ) + assert broker["metadata"]["namespace"] == "hermes-scm" + assert broker["spec"]["template"]["spec"]["serviceAccountName"] == ( + "hermes-scm-broker" + ) + assert "gitea-token" in json.dumps(broker) + container = broker["spec"]["template"]["spec"]["containers"][0] + assert container["resources"]["limits"]["memory"] == "768Mi" + assert container["livenessProbe"]["failureThreshold"] == 10 + tmp = next( + item for item in broker["spec"]["template"]["spec"]["volumes"] if item["name"] == "tmp" + ) + assert tmp["emptyDir"]["sizeLimit"] == "1Gi" + + policy = yaml.safe_load( + (ROOT / "services/hermes-scm-broker/networkpolicy.yaml").read_text( + encoding="utf-8" + ) + ) + ingress = policy["spec"]["ingress"] + assert ingress[0]["from"][0]["namespaceSelector"]["matchLabels"] == { + "kubernetes.io/metadata.name": "hermes" + } + assert ingress[0]["from"][0]["podSelector"]["matchLabels"] == { + "app": "hermes-agent" + } + + vault = ( + ROOT / "services/vault/scripts/vault_k8s_auth_configure.sh" + ).read_text(encoding="utf-8") + assert 'write_policy_and_role "hermes-scm-broker" "hermes-scm"' in vault + assert '"hermes/developer-gitea" ""' in vault + agent_start = vault.index('write_policy_and_role "hermes-agent"') + agent_end = vault.index("write_policy_and_role", agent_start + 1) + assert "developer-gitea" not in vault[agent_start:agent_end] + + +def test_flux_bootstrap_has_no_agent_namespace_dependency_cycle(): + hermes_resources = yaml.safe_load( + (ROOT / "services/hermes/kustomization.yaml").read_text(encoding="utf-8") + )["resources"] + assert "namespace.yaml" in hermes_resources + assert "scm-common" in hermes_resources + + observer = yaml.safe_load( + ( + ROOT + / "clusters/atlas/flux-system/applications/hermes-observer-rbac/kustomization.yaml" + ).read_text(encoding="utf-8") + ) + assert "dependsOn" not in observer["spec"] + observer_objects = [ + item + for item in yaml.safe_load_all( + (ROOT / "services/hermes-observer-rbac/rbac.yaml").read_text( + encoding="utf-8" + ) + ) + if item + ] + assert {item["kind"] for item in observer_objects} <= { + "ClusterRole", + "ClusterRoleBinding", + } + + hermes_flux = yaml.safe_load( + ( + ROOT / "clusters/atlas/flux-system/applications/hermes/kustomization.yaml" + ).read_text(encoding="utf-8") + ) + assert "hermes-observer-rbac" in { + item["name"] for item in hermes_flux["spec"]["dependsOn"] + } + bindings_flux = yaml.safe_load( + ( + ROOT + / "clusters/atlas/flux-system/applications/hermes-observer-bindings/kustomization.yaml" + ).read_text(encoding="utf-8") + ) + assert {item["name"] for item in bindings_flux["spec"]["dependsOn"]} == { + "hermes", + "hermes-observer-rbac", + } + assert bindings_flux["metadata"]["name"] not in { + item["name"] for item in hermes_flux["spec"]["dependsOn"] + } + + applications = ( + ROOT / "clusters/atlas/flux-system/applications/kustomization.yaml" + ).read_text(encoding="utf-8") + assert "hermes-scm-agent-code" not in applications + + broker_code = yaml.safe_load( + ( + ROOT + / "clusters/atlas/flux-system/applications/hermes-scm-broker-code/kustomization.yaml" + ).read_text(encoding="utf-8") + ) + assert broker_code["spec"]["path"] == "./services/hermes/scm-common" + assert broker_code["spec"]["targetNamespace"] == "hermes-scm" + assert broker_code["spec"]["dependsOn"] == [{"name": "hermes-scm-namespace"}] + + +def test_gitea_bootstrap_enforces_human_review_without_overwriting_drift(): + script = ( + ROOT / "services/gitea/scripts/gitea_atlas_identity_ensure.sh" + ).read_text(encoding="utf-8") + + assert 'required_approvals\\\":1' in script + assert 'enable_push_whitelist\\\":true' in script + assert 'push_whitelist_usernames\\\":[\\\"${protected_reviewer}\\\"]' in script + assert 'enable_merge_whitelist\\\":true' in script + assert 'merge_whitelist_usernames\\\":[\\\"${protected_reviewer}\\\"]' in script + assert "protection differs from the human-review policy" in script + assert "api_request PATCH" not in script + job = yaml.safe_load( + (ROOT / "services/gitea/atlas-identity-bootstrap-job.yaml").read_text() + ) + init = job["spec"]["template"]["spec"]["initContainers"][0] + assert init["image"] == ( + "python@sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df" + ) + assert init["securityContext"]["runAsNonRoot"] is True + assert init["securityContext"]["capabilities"]["drop"] == ["ALL"] + assert "/opt/python/bin/python3" in script + + +def _protection(rule_name: str, priority: int, helper, **overrides): + value = { + "rule_name": rule_name, + "priority": priority, + "created_at": "2026-01-01T00:00:00Z", + **helper._required("bstein"), + } + value.update(overrides) + return value + + +def test_branch_protection_requires_exact_literal_rules_only(): + helper = _load_path( + "branch_protection_test", + ROOT / "services/gitea/scripts/gitea_branch_protection_check.py", + ) + exact = _protection("main", 2, helper) + assert ( + helper.evaluate(json.dumps([exact]).encode(), "main", "bstein") == "PRESENT" + ) + + drifted = _protection("main", 2, helper, required_approvals=0) + with pytest.raises(helper.PolicyError, match="effective main protection differs"): + helper.evaluate(json.dumps([drifted]).encode(), "main", "bstein") + + duplicate = _protection("main", 1, helper) + with pytest.raises(helper.PolicyError, match="duplicate primary-branch"): + helper.evaluate(json.dumps([exact, duplicate]).encode(), "main", "bstein") + + +@pytest.mark.parametrize( + "rule_name", + [ + "release/*", + "release/**/v1.17", + "*", + "**", + "M*", + "m{ain,aster}", + r"m\ain", + "m?in", + "m[ai]in", + ], +) +def test_branch_glob_rules_fail_closed_pending_human_review(rule_name: str): + helper = _load_path( + "branch_protection_glob_parity_test", + ROOT / "services/gitea/scripts/gitea_branch_protection_check.py", + ) + rules = [_protection(rule_name, 1, helper)] + with pytest.raises(helper.PolicyError, match="human review"): + helper.evaluate(json.dumps(rules).encode(), "main", "bstein") + + +@pytest.mark.parametrize( + "rules", + [ + [{"rule_name": "main", "priority": 0, "created_at": "2026-01-01T00:00:00Z"}], + [{"rule_name": "main", "priority": 1, "created_at": "not-a-time"}], + [{"rule_name": "main"}], + ], +) +def test_branch_protection_rejects_ambiguous_rules_before_creation(rules): + helper = _load_path( + "branch_protection_adversarial_test", + ROOT / "services/gitea/scripts/gitea_branch_protection_check.py", + ) + with pytest.raises(helper.PolicyError): + helper.evaluate(json.dumps(rules).encode(), "main", "bstein") + + +def test_branch_protection_reports_absent_only_when_no_rule_matches(): + helper = _load_path( + "branch_protection_absent_test", + ROOT / "services/gitea/scripts/gitea_branch_protection_check.py", + ) + rules = [_protection("develop", 1, helper)] + assert helper.evaluate(json.dumps(rules).encode(), "master", "bstein") == "ABSENT" diff --git a/testing/tests/test_hermes_scm_broker_streaming.py b/testing/tests/test_hermes_scm_broker_streaming.py new file mode 100644 index 00000000..78a63f27 --- /dev/null +++ b/testing/tests/test_hermes_scm_broker_streaming.py @@ -0,0 +1,237 @@ +"""Bounded I/O and concurrency contracts for the Hermes SCM broker.""" + +from __future__ import annotations + +import io +import json +from email.message import Message + +import pytest + +from testing.tests.test_hermes_scm_broker_support import Response, _load + + +def test_agent_broker_client_sends_no_credential_or_authorization_header(): + client = _load("scm_broker_client") + seen = [] + + def opener(request, timeout): + seen.append((request, timeout)) + return Response(b"{}", content_type="application/json") + + assert client.read("/api/v1/repos/atlas/cassandra", opener=opener) == b"{}" + request = seen[0][0] + assert request.full_url == client.BROKER_ORIGIN + "/v1/metadata" + assert request.get_header("Authorization") is None + assert json.loads(request.data) == {"path": "/api/v1/repos/atlas/cassandra"} + + +@pytest.mark.parametrize("raw", ["", "01", "9" * 4000, "134217729", "１２"]) +def test_broker_content_length_is_canonical_and_bounded(raw: str): + broker = _load("scm_broker") + headers = Message() + headers["Content-Length"] = raw + + with pytest.raises(broker.PolicyError, match="request length"): + broker._content_length(headers, broker.MAX_GIT_REQUEST) + + +def test_large_git_exchange_rolls_to_disk_and_scans_chunk_boundaries(): + broker = _load("scm_broker") + token = "runtime-sentinel" + safe = b"x" * (broker.SPOOL_MEMORY_LIMIT + 1) + inbound_timeouts = [] + spool, length = broker._spool_bounded( + io.BytesIO(safe), + broker.MAX_GIT_REQUEST, + len(safe), + token=token, + context="Git request", + deadline=broker.time.monotonic() + 30, + set_timeout=inbound_timeouts.append, + ) + try: + assert length == len(safe) + assert spool._rolled is True + assert spool.read(4) == b"xxxx" + assert inbound_timeouts + assert all(0 < value <= 30 for value in inbound_timeouts) + finally: + spool.close() + + crossing = b"x" * (broker.STREAM_CHUNK - 5) + token.encode() + b"tail" + with pytest.raises(broker.PolicyError, match="credential material"): + broker._spool_bounded( + io.BytesIO(crossing), + broker.MAX_GIT_REQUEST, + len(crossing), + token=token, + context="Git request", + deadline=broker.time.monotonic() + 30, + ) + + +def test_large_upstream_response_is_streamed_through_bounded_spool(): + broker = _load("scm_broker") + body = b"z" * (broker.SPOOL_MEMORY_LIMIT + 1) + seen = [] + + def opener(request, **_kwargs): + seen.append(request) + assert request.get_header("Content-length") == "7" + assert request.data.read() == b"request" + return Response(body, content_type="application/x-git-upload-pack-result") + + streamed = broker._upstream_git_request( + "/atlas/cassandra.git/git-upload-pack", + method="POST", + body=io.BytesIO(b"request"), + body_length=7, + content_type="application/x-git-upload-pack-request", + expected_type="application/x-git-upload-pack-result", + token="runtime-sentinel", + stream_result=True, + opener=opener, + ) + spool, length = streamed + try: + assert length == len(body) + assert spool._rolled is True + assert spool.read(3) == b"zzz" + assert len(seen) == 1 + finally: + spool.close() + + +def test_body_deadline_fails_closed_before_reading(): + broker = _load("scm_broker") + with pytest.raises(broker.PolicyError, match="deadline"): + broker._spool_bounded( + io.BytesIO(b"body"), + broker.MAX_GIT_REQUEST, + 4, + token="runtime-sentinel", + context="Git request", + deadline=broker.time.monotonic() - 1, + ) + + +def test_upstream_stream_applies_absolute_socket_deadline(): + broker = _load("scm_broker") + + class DeadlineSocket: + def __init__(self): + self.values = [] + + def settimeout(self, value): + self.values.append(value) + + response = Response(b"safe", content_type="application/x-git-upload-pack-result") + sock = DeadlineSocket() + response.fp = type("FP", (), {"raw": type("Raw", (), {"_sock": sock})()})() + spool, length = broker._spool_response( + response, broker.MAX_GIT_RESPONSE, "sentinel", broker.INBOUND_BODY_TIMEOUT + ) + try: + assert length == 4 + assert sock.values + assert all(0 < value <= broker.INBOUND_BODY_TIMEOUT for value in sock.values) + finally: + spool.close() + + +def test_broker_rejects_work_when_concurrency_slots_are_exhausted(): + broker = _load("scm_broker") + + class FakeSocket: + def __init__(self): + self.value = b"" + + def sendall(self, value): + self.value += value + + def shutdown(self, _how): + return None + + def close(self): + return None + + server = broker.BoundedThreadingHTTPServer(("127.0.0.1", 0), broker.BrokerHandler) + try: + assert broker.BoundedThreadingHTTPServer.__module__ == "scm_broker_server" + assert server._slots.acquire(blocking=False) + assert server._slots.acquire(blocking=False) + assert server._slots.acquire(blocking=False) + request = FakeSocket() + server.process_request(request, ("127.0.0.1", 1)) + assert b"503 Service Unavailable" in request.value + finally: + server._slots.release() + server._slots.release() + server._slots.release() + server.server_close() + + +def test_three_simultaneous_broker_calls_have_bounded_capacity(): + server_module = _load("scm_broker_server") + semaphore = server_module.threading.BoundedSemaphore( + server_module.MAX_CONCURRENT_REQUESTS + ) + assert server_module.MAX_CONCURRENT_REQUESTS >= 3 + assert [semaphore.acquire(blocking=False) for _ in range(3)] == [True, True, True] + assert semaphore.acquire(blocking=False) is False + + +def test_header_deadline_is_absolute_against_slow_trickle(monkeypatch): + server_module = _load("scm_broker_server") + + class Stream: + def __init__(self): + self.value = io.BytesIO(b"GET /healthz HTTP/1.1\r\n") + + def read(self, length): + return self.value.read(length) + + class Connection: + def __init__(self): + self.timeouts = [] + + def settimeout(self, value): + self.timeouts.append(value) + + ticks = iter((0.0, 0.02, 0.04, 0.06, 0.08, 0.11)) + monkeypatch.setattr(server_module.time, "monotonic", lambda: next(ticks)) + connection = Connection() + reader = server_module._AbsoluteDeadlineReader(Stream(), connection) + reader.begin(0.1) + + with pytest.raises(TimeoutError, match="absolute SCM header deadline"): + reader.readline(65537) + + assert connection.timeouts == pytest.approx([0.08, 0.06, 0.04, 0.02]) + + +def test_broker_bounds_header_count_size_and_duplicate_lengths(): + broker = _load("scm_broker") + + def validate(headers): + handler = object.__new__(broker.BrokerHandler) + handler.headers = headers + handler._validate_headers() + + too_many = Message() + for index in range(broker.MAX_HEADERS + 1): + too_many[f"X-Test-{index}"] = "safe" + with pytest.raises(broker.PolicyError, match="too many headers"): + validate(too_many) + + too_large = Message() + too_large["X-Test"] = "x" * broker.MAX_HEADER_BYTES + with pytest.raises(broker.PolicyError, match="safe limit"): + validate(too_large) + + duplicate = Message() + duplicate["Content-Length"] = "1" + duplicate["Content-Length"] = "1" + with pytest.raises(broker.PolicyError, match="duplicate Content-Length"): + validate(duplicate) diff --git a/testing/tests/test_hermes_scm_broker_support.py b/testing/tests/test_hermes_scm_broker_support.py new file mode 100644 index 00000000..e3682494 --- /dev/null +++ b/testing/tests/test_hermes_scm_broker_support.py @@ -0,0 +1,84 @@ +"""Shared fixtures for credential-isolated Hermes SCM broker tests.""" + +from __future__ import annotations + +import hashlib +import importlib.util +import io +import sys +import zlib +from email.message import Message +from pathlib import Path + +ROOT = Path(__file__).parents[2] +SCRIPTS = ROOT / "services/hermes/scm-common/scripts" +sys.path.insert(0, str(SCRIPTS)) + + +def _load(name: str): + path = SCRIPTS / f"{name}.py" + spec = importlib.util.spec_from_file_location(f"test_{name}", path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _load_path(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +class Response: + def __init__(self, body: bytes, *, status: int = 200, content_type: str): + self.body = body + self.stream = io.BytesIO(body) + self.status = status + self.headers = Message() + self.headers["Content-Type"] = content_type + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def read(self, limit=-1): + return self.stream.read(limit) + + +def _object_entry(type_code: int, payload: bytes) -> bytes: + """Encode one non-delta pack entry with a real header and zlib body.""" + size = len(payload) + byte = (type_code << 4) | (size & 0x0F) + size >>= 4 + header = bytearray() + while size: + header.append(byte | 0x80) + byte = size & 0x7F + size >>= 7 + header.append(byte) + return bytes(header) + zlib.compress(payload) + + +def _pack_of(entries: list[bytes]) -> bytes: + """Assemble encoded entries into a checksummed version-2 pack.""" + body = b"PACK" + (2).to_bytes(4, "big") + len(entries).to_bytes(4, "big") + for entry in entries: + body += entry + return body + hashlib.sha1(body).digest() + + +def _blob_pack(*payloads: bytes) -> bytes: + return _pack_of([_object_entry(3, payload) for payload in payloads]) + + +def _receive_command(old: bytes, new: bytes, ref: bytes, pack: bytes = b"") -> bytes: + command = old + b" " + new + b" " + ref + b"\x00report-status\n" + framed = f"{len(command) + 4:04x}".encode() + command + b"0000" + return framed + (pack or _pack_of([])) diff --git a/testing/tests/test_hermes_scm_server_coverage.py b/testing/tests/test_hermes_scm_server_coverage.py new file mode 100644 index 00000000..7a358640 --- /dev/null +++ b/testing/tests/test_hermes_scm_server_coverage.py @@ -0,0 +1,180 @@ +"""Behavioral branch coverage for the SCM broker HTTP server primitives.""" + +from __future__ import annotations + +import io + +import pytest + +from testing.tests.test_hermes_scm_broker_support import _load + + +class _Connection: + def __init__(self): + self.timeouts = [] + + def settimeout(self, value): + self.timeouts.append(value) + + +class _FlushBuffer(io.BytesIO): + def __init__(self): + super().__init__() + self.flushes = 0 + + def flush(self): + self.flushes += 1 + + +def test_absolute_reader_supports_idle_limit_newline_eof_and_attribute_delegation(): + module = _load("scm_broker_server") + stream = io.BytesIO(b"first\nsecond") + reader = module._AbsoluteDeadlineReader(stream, _Connection()) + assert reader.readline(3) == b"fir" + reader.begin(5) + assert reader.readline() == b"st\n" + assert reader.readline() == b"second" + assert reader.readline() == b"" + assert reader.closed is False + reader.end() + assert reader.readline() == b"" + + +def _handler_type(module): + class StubHandler: + request_version = "HTTP/1.1" + command = "GET" + + def setup(self): + self.connection = _Connection() + self.rfile = io.BytesIO(b"") + self.wfile = _FlushBuffer() + + def parse_request(self): + return self.parse_result + + def send_error(self, code, *_args): + self.errors.append(code) + + def do_GET(self): + self.calls.append("GET") + + return type( + "DeadlineHandler", (module.AbsoluteHeaderDeadlineMixin, StubHandler), {} + ) + + +def _handler(module, raw: bytes, *, parse=True): + handler_type = _handler_type(module) + handler = object.__new__(handler_type) + handler.setup() + handler._header_reader._stream = io.BytesIO(raw) + handler.parse_result = parse + handler.errors = [] + handler.calls = [] + return handler + + +def test_header_mixin_setup_and_known_request_dispatch(): + module = _load("scm_broker_server") + handler = _handler(module, b"GET /healthz HTTP/1.1\r\n") + handler.handle_one_request() + assert handler.calls == ["GET"] + assert handler.wfile.flushes == 1 + assert handler._header_reader._deadline is None + + +@pytest.mark.parametrize( + ("raw", "parse", "command", "expected_error", "closed"), + [ + (b"x" * 65537, True, "GET", 414, False), + (b"", True, "GET", None, True), + (b"GET / HTTP/1.1\r\n", False, "GET", None, False), + (b"TRACE / HTTP/1.1\r\n", True, "TRACE", 501, False), + ], +) +def test_header_mixin_rejects_long_empty_unparsed_and_unknown_requests( + raw, parse, command, expected_error, closed +): + module = _load("scm_broker_server") + handler = _handler(module, raw, parse=parse) + handler.command = command + handler.close_connection = False + handler.handle_one_request() + assert handler.errors == ([] if expected_error is None else [expected_error]) + assert handler.close_connection is closed + + +def test_header_mixin_closes_on_absolute_timeout(monkeypatch): + module = _load("scm_broker_server") + handler = _handler(module, b"GET / HTTP/1.1\r\n") + handler.close_connection = False + monkeypatch.setattr( + handler._header_reader, + "readline", + lambda *_a, **_k: (_ for _ in ()).throw(TimeoutError()), + ) + handler.handle_one_request() + assert handler.close_connection is True + + +class _Slots: + def __init__(self, available=True): + self.available = available + self.releases = 0 + + def acquire(self, *, blocking): + assert blocking is False + return self.available + + def release(self): + self.releases += 1 + + +def _server_shell(module, available=True): + server = object.__new__(module.BoundedThreadingHTTPServer) + server._slots = _Slots(available) + server.shutdowns = [] + server.shutdown_request = server.shutdowns.append + return server + + +def test_bounded_server_delegates_accepted_request(monkeypatch): + module = _load("scm_broker_server") + server = _server_shell(module) + calls = [] + monkeypatch.setattr( + module.ThreadingHTTPServer, + "process_request", + lambda self, request, address: calls.append((request, address)), + ) + server.process_request("socket", ("127.0.0.1", 1)) + assert calls == [("socket", ("127.0.0.1", 1))] + assert server._slots.releases == 0 + + +def test_bounded_server_releases_on_dispatch_failure(monkeypatch): + module = _load("scm_broker_server") + server = _server_shell(module) + monkeypatch.setattr( + module.ThreadingHTTPServer, + "process_request", + lambda *_a, **_k: (_ for _ in ()).throw(RuntimeError("dispatch")), + ) + with pytest.raises(RuntimeError, match="dispatch"): + server.process_request("socket", ("127.0.0.1", 1)) + assert server._slots.releases == 1 + assert server.shutdowns == ["socket"] + + +def test_bounded_server_thread_always_releases(monkeypatch): + module = _load("scm_broker_server") + server = _server_shell(module) + monkeypatch.setattr( + module.ThreadingHTTPServer, + "process_request_thread", + lambda *_a, **_k: (_ for _ in ()).throw(RuntimeError("handler")), + ) + with pytest.raises(RuntimeError, match="handler"): + server.process_request_thread("socket", ("127.0.0.1", 1)) + assert server._slots.releases == 1 diff --git a/testing/tests/test_quality_contract.py b/testing/tests/test_quality_contract.py index 59417eff..f7bf38b7 100644 --- a/testing/tests/test_quality_contract.py +++ b/testing/tests/test_quality_contract.py @@ -100,6 +100,7 @@ def test_coverage_check_enforces_per_file_floor(tmp_path: Path): + @@ -112,13 +113,14 @@ def test_coverage_check_enforces_per_file_floor(tmp_path: Path): contract = { "coverage": { "minimum_percent": 95.0, - "tracked_files": ["ok.py", "low.py", "missing.py"], + "tracked_files": ["ok.py", "low.py", "weak_branches.py", "missing.py"], } } issues = run_coverage_check(contract, tmp_path, coverage_xml) assert "line coverage below 95.0%: low.py (90.0%)" in issues + assert "branch coverage below 95.0%: weak_branches.py (90.0%)" in issues assert "coverage missing for tracked file: missing.py" in issues @@ -144,6 +146,7 @@ def test_coverage_check_handles_missing_xml_and_source_root_mapping(tmp_path: Pa + diff --git a/testing/tests/test_quality_coverage_helpers.py b/testing/tests/test_quality_coverage_helpers.py index beae7d0a..7e6d90f3 100644 --- a/testing/tests/test_quality_coverage_helpers.py +++ b/testing/tests/test_quality_coverage_helpers.py @@ -5,8 +5,6 @@ from __future__ import annotations import textwrap from pathlib import Path -import pytest - from testing.quality_coverage import compute_workspace_line_coverage, run_check @@ -14,15 +12,10 @@ def test_compute_workspace_line_coverage_handles_missing_xml(tmp_path: Path) -> """Missing coverage XML should produce a zero workspace coverage score.""" contract = {"coverage": {"tracked_files": ["managed.py"]}} - assert ( - compute_workspace_line_coverage(contract, tmp_path, tmp_path / "missing.xml") - == 0.0 - ) + assert compute_workspace_line_coverage(contract, tmp_path, tmp_path / "missing.xml") == 0.0 -def test_compute_workspace_line_coverage_averages_present_tracked_files( - tmp_path: Path, -) -> None: +def test_compute_workspace_line_coverage_averages_present_tracked_files(tmp_path: Path) -> None: """Workspace coverage should average only tracked files that appear in the report.""" coverage_xml = tmp_path / "coverage.xml" @@ -33,8 +26,8 @@ def test_compute_workspace_line_coverage_averages_present_tracked_files( - - + + @@ -48,9 +41,7 @@ def test_compute_workspace_line_coverage_averages_present_tracked_files( assert compute_workspace_line_coverage(contract, tmp_path, coverage_xml) == 75.0 -def test_run_check_keeps_relative_names_when_source_roots_do_not_match( - tmp_path: Path, -) -> None: +def test_run_check_keeps_relative_names_when_source_roots_do_not_match(tmp_path: Path) -> None: """Relative filenames should remain relative when no declared source root contains them.""" coverage_xml = tmp_path / "coverage.xml" @@ -85,65 +76,62 @@ def test_run_check_keeps_relative_names_when_source_roots_do_not_match( assert issues == ["line coverage below 95.0%: relative.py (80.0%)"] -@pytest.mark.parametrize("branch_rate", [None, "nan", "inf", "-0.1", "1.1", "broken"]) -def test_run_check_rejects_missing_or_invalid_branch_rates( - tmp_path: Path, - branch_rate: str | None, -) -> None: - """Tracked files must carry finite Cobertura branch evidence.""" - attribute = "" if branch_rate is None else f' branch-rate="{branch_rate}"' +def test_run_check_fails_a_file_below_the_branch_floor(tmp_path: Path) -> None: + """Full line coverage must never hide a file whose branch coverage is weak.""" + coverage_xml = tmp_path / "coverage.xml" coverage_xml.write_text( - f'', + textwrap.dedent( + """\ + + + + + + + + + + + """ + ), encoding="utf-8", ) issues = run_check( - {"coverage": {"tracked_files": ["managed.py"]}}, + {"coverage": {"minimum_percent": 95.0, "tracked_files": ["weak.py", "solid.py"]}}, tmp_path, coverage_xml, ) - assert len(issues) == 1 - assert issues[0].startswith("coverage xml invalid:") - assert ( - compute_workspace_line_coverage( - {"coverage": {"tracked_files": ["managed.py"]}}, - tmp_path, - coverage_xml, - ) - == 0.0 - ) + assert issues == ["branch coverage below 95.0%: weak.py (90.0%)"] -def test_run_check_enforces_line_and_branch_rates_independently(tmp_path: Path) -> None: - """A strong line score must not hide weak branch coverage.""" +def test_run_check_fails_closed_when_branch_evidence_is_absent(tmp_path: Path) -> None: + """A report generated without branch measurement cannot satisfy the gate.""" + coverage_xml = tmp_path / "coverage.xml" coverage_xml.write_text( - '', + textwrap.dedent( + """\ + + + + + + + + + + """ + ), encoding="utf-8", ) - assert run_check( - {"coverage": {"minimum_percent": 95, "tracked_files": ["managed.py"]}}, + issues = run_check( + {"coverage": {"minimum_percent": 95.0, "tracked_files": ["unmeasured.py"]}}, tmp_path, coverage_xml, - ) == ["branch coverage below 95.0%: managed.py (50.0%)"] - - -def test_run_check_accepts_cobertura_branchless_file_semantics(tmp_path: Path) -> None: - """Coverage.py reports branchless files with branch-rate one.""" - coverage_xml = tmp_path / "coverage.xml" - coverage_xml.write_text( - '', - encoding="utf-8", ) - assert ( - run_check( - {"coverage": {"minimum_percent": 95, "tracked_files": ["managed.py"]}}, - tmp_path, - coverage_xml, - ) - == [] - ) + assert issues == ["branch coverage missing for tracked file: unmeasured.py"] diff --git a/testing/tests/test_quality_gate.py b/testing/tests/test_quality_gate.py index 256a78e6..fd334d85 100644 --- a/testing/tests/test_quality_gate.py +++ b/testing/tests/test_quality_gate.py @@ -60,6 +60,32 @@ def test_run_profile_aggregates_internal_and_pytest_results(tmp_path: Path, monk assert any(result.get("junit") == "build/junit-unit.xml" for result in summary["results"]) +def test_run_profile_fails_synthetic_file_below_branch_floor(tmp_path: Path): + """The gate itself must fail a tracked file with full lines but weak branches.""" + build_dir = tmp_path / "build" + build_dir.mkdir() + (build_dir / "coverage-unit.xml").write_text( + "" + '' + "", + encoding="utf-8", + ) + + contract = { + "profiles": {"local": ["coverage"]}, + "pytest_suites": {"unit": {"coverage_xml": "build/coverage-unit.xml"}}, + "coverage": {"minimum_percent": 95.0, "tracked_files": ["weak.py"]}, + } + + summary = quality_gate.run_profile(contract, tmp_path, "local", build_dir) + + assert summary["status"] == "failed" + coverage_result = summary["results"][0] + assert coverage_result["name"] == "coverage" + assert coverage_result["issues"] == ["branch coverage below 95.0%: weak.py (90.0%)"] + assert summary["workspace_line_coverage_percent"] == 100.0 + + def test_main_writes_summary_file(tmp_path: Path, monkeypatch): summary = {"status": "ok", "profile": "local", "results": [], "manual_scripts": []} monkeypatch.chdir(tmp_path) diff --git a/testing/tests/test_quality_hygiene_helpers.py b/testing/tests/test_quality_hygiene_helpers.py index 9902f445..9a96c86f 100644 --- a/testing/tests/test_quality_hygiene_helpers.py +++ b/testing/tests/test_quality_hygiene_helpers.py @@ -7,6 +7,32 @@ from pathlib import Path from testing.quality_hygiene import count_files_over_line_limit, run_check +def test_run_check_skips_conftest_files_in_naming_rules(tmp_path: Path) -> None: + """A conftest.py is pytest plumbing, never a naming-rule violation.""" + + tests_dir = tmp_path / "tests" + tests_dir.mkdir() + (tests_dir / "conftest.py").write_text("fixtures = True\n", encoding="utf-8") + (tests_dir / "helper.py").write_text("value = 1\n", encoding="utf-8") + + contract = { + "hygiene": { + "naming_rules": [ + { + "glob": "tests/*.py", + "pattern": "^test_[a-z0-9_]+\\.py$", + "description": "Pytest files use test_*.py names.", + } + ] + } + } + + issues = run_check(contract, tmp_path) + + assert len(issues) == 1 + assert "helper.py" in issues[0] + + def test_count_files_over_line_limit_counts_only_long_matches(tmp_path: Path) -> None: """Only files beyond the configured limit should contribute to the LOC total.""" @@ -24,28 +50,3 @@ def test_count_files_over_line_limit_counts_only_long_matches(tmp_path: Path) -> } assert count_files_over_line_limit(contract, tmp_path) == 1 - - -def test_run_check_exempts_conftest_from_naming_rules(tmp_path: Path) -> None: - """Shared conftest.py fixtures never violate test naming rules.""" - - tests_dir = tmp_path / "tests" - tests_dir.mkdir() - (tests_dir / "conftest.py").write_text("fixtures = True\n", encoding="utf-8") - (tests_dir / "test_named.py").write_text("checked = True\n", encoding="utf-8") - - contract = { - "hygiene": { - "max_lines": 3, - "line_limit_globs": [], - "naming_rules": [ - { - "glob": "tests/*.py", - "pattern": "^test_[a-z0-9_]+\\.py$", - "description": "pytest files use test_*.py names", - } - ], - } - } - - assert run_check(contract, tmp_path) == [] diff --git a/testing/tests/test_semgrep_report.py b/testing/tests/test_semgrep_report.py index 89a15251..7c5e16ab 100644 --- a/testing/tests/test_semgrep_report.py +++ b/testing/tests/test_semgrep_report.py @@ -100,6 +100,29 @@ def test_build_sonar_issues_uses_safe_defaults_for_sparse_findings() -> None: assert issue["primaryLocation"]["textRange"] == {"startLine": 1, "endLine": 1} +def test_build_sonar_issues_clamps_inverted_line_ranges() -> None: + """An end line before the start line should collapse to a valid range.""" + + issues = semgrep_report.build_sonar_issues( + { + "results": [ + { + "check_id": "python.lang.correctness.range", + "path": "app/main.py", + "start": {"line": 9}, + "end": {"line": 3}, + "extra": {"severity": "WARNING", "message": "inverted"}, + } + ] + } + ) + + assert issues["issues"][0]["primaryLocation"]["textRange"] == { + "startLine": 9, + "endLine": 9, + } + + def test_read_json_handles_invalid_json_and_non_object(tmp_path: Path) -> None: """Report loading should fail closed for bad JSON and wrong top-level shapes."""