From 762784da6b558eedce5ec3e3993fdf42e79a97ec Mon Sep 17 00:00:00 2001 From: jenkins Date: Sun, 16 Aug 2026 19:54:41 -0300 Subject: [PATCH 01/12] hermes: bound Atlas pull request client --- services/hermes/agent-configmap.yaml | 30 +- services/hermes/agent-deployment.yaml | 9 + services/hermes/kustomization.yaml | 7 + services/hermes/scripts/gitea_api.py | 422 ++++++++++++++---- .../manage-atlas-pull-requests/SKILL.md | 58 +++ .../agents/openai.yaml | 4 + testing/tests/test_hermes_chat_quality.py | 8 +- testing/tests/test_hermes_gitea_pr_client.py | 322 +++++++++++++ testing/tests/test_hermes_runtime_access.py | 40 +- 9 files changed, 784 insertions(+), 116 deletions(-) mode change 100644 => 100755 services/hermes/scripts/gitea_api.py create mode 100644 services/hermes/skills/manage-atlas-pull-requests/SKILL.md create mode 100644 services/hermes/skills/manage-atlas-pull-requests/agents/openai.yaml create mode 100644 testing/tests/test_hermes_gitea_pr_client.py diff --git a/services/hermes/agent-configmap.yaml b/services/hermes/agent-configmap.yaml index 0b81379d..b35da59a 100644 --- a/services/hermes/agent-configmap.yaml +++ b/services/hermes/agent-configmap.yaml @@ -315,15 +315,21 @@ data: real branch only when the requested implementation is review-ready. Never force-push. - `scm.bstein.dev` is Gitea, not GitHub. Never load or follow a GitHub/`gh` - skill for an Atlas remote, and do not interpret an unauthenticated Gitea - HTTP 404 as a missing private repository. Use authenticated Git for clone, - fetch, and push. For pull-request metadata, comments, and merges, call - `/opt/coordinator/gitea_api.py METHOD /api/v1/...`; it injects the runtime - Vault token without exposing it to the command line, environment, output, - or transcript. Before a consequential merge, independently verify the - exact base/head ancestry and diff, run the relevant tests, and confirm the - candidate Jenkins result. Prefer the internal endpoint in + `scm.bstein.dev` is Forgejo/Gitea, not GitHub. Never load or follow a + GitHub/`gh` skill for an Atlas remote, and do not interpret an + unauthenticated Gitea HTTP 404 as a missing private repository. Use + authenticated Git for clone, fetch, and non-force push. For repository and + pull-request reads, or to create/update a review-ready draft PR, load + `$manage-atlas-pull-requests` and use `/opt/coordinator/gitea_api.py`. The + client injects the runtime Vault token without exposing it to the command + line, environment, output, or transcript. It permits only Atlas-scoped + reads, same-repository draft creation, and title/body updates while the PR + remains draft. Merge, approve, close, delete, comments, repository + administration, and force-push are unavailable; never bypass the client + with raw HTTP. Leave every PR unmerged for Brad's review. Independently + verify the exact base/head ancestry and diff, run the relevant tests, and + confirm the candidate Jenkins result before presenting a handoff. Prefer + the internal endpoint in `JENKINS_BASE_URL`; when Jenkins API authorization prevents a read, use the existing cluster access to inspect the controller's job/build files and logs rather than guessing a public hostname. The preferred read-only path @@ -331,10 +337,8 @@ data: [--commit SHA] --wait`; it distinguishes a genuinely terminal build from nested Jenkins execution metadata and returns bounded JSON/log evidence. With a branch, the helper also tries the conventional `JOB-branches` - multibranch name. For JSON Gitea mutations, prefer repeatable shell-safe - `--field KEY=VALUE` arguments over hand-quoted JSON. - After merging, observe the default-branch build to a terminal result and - report exact commit, build, and test evidence. + multibranch name. After Brad merges, observe the default-branch build to a + terminal result and report exact commit, build, and test evidence. The workspace already has a non-secret Hermes Git author identity. Do not use `git reset --hard`, even in a fresh clone; use a detached worktree or a diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml index 5ed758cf..4b93f92d 100644 --- a/services/hermes/agent-deployment.yaml +++ b/services/hermes/agent-deployment.yaml @@ -663,6 +663,7 @@ spec: - {name: subprocess-secret-patch, mountPath: /opt/hermes/tools/process_registry.py, subPath: process_registry.py} - {name: kubeconfig, mountPath: /opt/data/home/.kube/config, subPath: config, readOnly: true} - {name: auto-router-plugin, mountPath: /opt/data/plugins/auto-router, readOnly: true} + - {name: atlas-pr-skill, mountPath: /opt/data/workspace/skills/manage-atlas-pull-requests, readOnly: true} - {name: routing-catalog, mountPath: /routing-catalog, readOnly: true} - {name: tmp, mountPath: /tmp} startupProbe: @@ -826,6 +827,7 @@ spec: - {name: subprocess-secret-patch, mountPath: /opt/hermes/tools/process_registry.py, subPath: process_registry.py} - {name: kubeconfig, mountPath: /opt/data/home/.kube/config, subPath: config, readOnly: true} - {name: auto-router-plugin, mountPath: /opt/data/plugins/auto-router, readOnly: true} + - {name: atlas-pr-skill, mountPath: /opt/data/workspace/skills/manage-atlas-pull-requests, readOnly: true} - {name: routing-catalog, mountPath: /routing-catalog, readOnly: true} - {name: tmp, mountPath: /tmp} - {name: ttyd-index, mountPath: /ttyd-index, readOnly: true} @@ -894,6 +896,7 @@ spec: - {name: auth-patch, mountPath: /opt/hermes/hermes_cli/auth.py, subPath: auth.py} - {name: kubeconfig, mountPath: /opt/data/home/.kube/config, subPath: config, readOnly: true} - {name: auto-router-plugin, mountPath: /opt/data/plugins/auto-router, readOnly: true} + - {name: atlas-pr-skill, mountPath: /opt/data/workspace/skills/manage-atlas-pull-requests, readOnly: true} - {name: tmp, mountPath: /tmp} resources: requests: {cpu: 100m, memory: 256Mi} @@ -1188,6 +1191,12 @@ spec: - {key: dashboard-api.py, path: dashboard/plugin_api.py} - {key: dashboard-index.js, path: dashboard/dist/index.js} - {key: dashboard-style.css, path: dashboard/dist/style.css} + - name: atlas-pr-skill + configMap: + name: hermes-atlas-pr-skill + items: + - {key: SKILL.md, path: SKILL.md} + - {key: openai.yaml, path: agents/openai.yaml} - name: image-policy configMap: name: hermes-image-policy diff --git a/services/hermes/kustomization.yaml b/services/hermes/kustomization.yaml index 3b9c2345..87984530 100644 --- a/services/hermes/kustomization.yaml +++ b/services/hermes/kustomization.yaml @@ -158,3 +158,10 @@ configMapGenerator: - alert-review.md=skills/tune-atlas-alerts/references/alert-review.md options: disableNameSuffixHash: true + - name: hermes-atlas-pr-skill + namespace: hermes + files: + - SKILL.md=skills/manage-atlas-pull-requests/SKILL.md + - openai.yaml=skills/manage-atlas-pull-requests/agents/openai.yaml + options: + disableNameSuffixHash: true diff --git a/services/hermes/scripts/gitea_api.py b/services/hermes/scripts/gitea_api.py old mode 100644 new mode 100755 index 5544c398..73a2bffd --- a/services/hermes/scripts/gitea_api.py +++ b/services/hermes/scripts/gitea_api.py @@ -1,43 +1,158 @@ #!/usr/bin/env python3 -"""Call the private Atlas Gitea API through a runtime-only token boundary.""" +"""Use the private Atlas Forgejo API through a least-authority PR boundary.""" from __future__ import annotations import argparse import json import os +import re import sys import urllib.error import urllib.parse import urllib.request +from collections.abc import Callable from pathlib import Path - -DEFAULT_BASE_URL = "https://scm.bstein.dev" +CANONICAL_BASE_URL = "https://scm.bstein.dev" +ALLOWED_OWNER = "atlas" DEFAULT_TOKEN_FILE = Path("/runtime-access/gitea-token") -ALLOWED_METHODS = ("GET", "POST", "PUT", "PATCH", "DELETE") +REPO_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,99}\Z") +PR_PATH_RE = re.compile(r"/api/v1/repos/atlas/([^/]+)/pulls/([1-9][0-9]*)\Z") +CREATE_PATH_RE = re.compile(r"/api/v1/repos/atlas/([^/]+)/pulls\Z") +READ_PATH_RE = re.compile(r"/api/v1/repos/atlas/([^/]+)(?:/.*)?\Z") +SAFE_REF_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._/-]{0,199}\Z") +MAX_ERROR_BYTES = 8192 +DRAFT_TITLE_PREFIX = "WIP: " + + +class PolicyError(ValueError): + """Raised when a requested Forgejo operation exceeds the safe boundary.""" def read_token(path: Path = DEFAULT_TOKEN_FILE) -> str: - """Read and validate the token from its in-memory Vault projection.""" + """Read the token from the pod-lifetime Vault projection.""" token = path.read_text(encoding="utf-8").strip() if not token: raise ValueError(f"runtime credential is empty: {path}") return token -def api_url(base_url: str, path: str) -> str: - """Return a same-origin Gitea API URL for a validated API path.""" - base = urllib.parse.urlsplit(base_url.rstrip("/")) +def configured_base_url() -> str: + """Reject attempts to redirect credentials away from the canonical origin.""" + configured = os.environ.get("GITEA_BASE_URL", CANONICAL_BASE_URL).rstrip("/") + if configured != CANONICAL_BASE_URL: + raise PolicyError("Forgejo origin is fixed to the private Atlas SCM service") + return configured + + +def _validate_repo(repo: str) -> str: + if not REPO_RE.fullmatch(repo) or repo in {".", ".."}: + raise PolicyError("repository name is outside the Atlas allowlist") + return repo + + +def _validate_ref(value: object, name: str) -> str: + if not isinstance(value, str) or not SAFE_REF_RE.fullmatch(value): + raise PolicyError(f"{name} must be a same-repository branch name") + if ( + value.startswith(".") + or value.endswith(("/", ".", ".lock")) + or "//" in value + or ".." in value + or "@{" in value + ): + raise PolicyError(f"{name} must be a safe same-repository branch name") + return value + + +def _validate_text(value: object, name: str, maximum: int, *, required: bool) -> str: + if not isinstance(value, str): + raise PolicyError(f"{name} must be text") + if required and not value.strip(): + raise PolicyError(f"{name} must not be empty") + if len(value) > maximum or "\x00" in value: + raise PolicyError(f"{name} exceeds the safe request limit") + return value + + +def _draft_title(value: object) -> str: + """Return a bounded title using Gitea's configured default draft prefix.""" + title = _validate_text(value, "title", 251, required=True).strip() + for prefix in ("WIP:", "[WIP]"): + if title.upper().startswith(prefix): + title = title[len(prefix) :].lstrip() + break + if not title: + raise PolicyError("title must contain text after the draft prefix") + return DRAFT_TITLE_PREFIX + title + + +def _split_api_path(path: str) -> urllib.parse.SplitResult: target = urllib.parse.urlsplit(path) - if base.scheme not in {"http", "https"} or not base.netloc: - raise ValueError("GITEA_BASE_URL must be an absolute HTTP(S) URL") if target.scheme or target.netloc or target.fragment: - raise ValueError("API path must be relative to the configured Gitea origin") + raise PolicyError("API path must be relative to the Atlas SCM origin") if not target.path.startswith("/api/v1/"): - raise ValueError("API path must start with /api/v1/") + raise PolicyError("API path must start with /api/v1/") + if "%" in target.path or "//" in target.path or "/../" in f"{target.path}/": + raise PolicyError("encoded or non-canonical API paths are not allowed") + return target + + +def api_url(base_url: str, path: str) -> str: + """Return a canonical same-origin URL without forwarding credentials.""" + if base_url.rstrip("/") != CANONICAL_BASE_URL: + raise PolicyError("Forgejo origin is fixed to the private Atlas SCM service") + target = _split_api_path(path) return urllib.parse.urlunsplit( - (base.scheme, base.netloc, target.path, target.query, "") + ("https", "scm.bstein.dev", target.path, target.query, "") + ) + + +def authorize_request(method: str, path: str, data: object | None) -> str: + """Validate one request and return its bounded operation name.""" + normalized_method = method.upper() + target = _split_api_path(path) + if normalized_method == "GET": + if data is not None: + raise PolicyError("read operations cannot include a request body") + match = READ_PATH_RE.fullmatch(target.path) + if not match: + raise PolicyError("reads are limited to repositories owned by atlas") + _validate_repo(match.group(1)) + return "read" + + if target.query: + raise PolicyError("mutating operations cannot include query parameters") + if normalized_method == "POST": + match = CREATE_PATH_RE.fullmatch(target.path) + if not match: + raise PolicyError("POST is limited to creating an Atlas draft pull request") + _validate_repo(match.group(1)) + if not isinstance(data, dict) or set(data) != {"base", "body", "head", "title"}: + raise PolicyError("draft creation accepts only base, body, head, and title") + _validate_ref(data["base"], "base") + _validate_ref(data["head"], "head") + if data["title"] != _draft_title(data["title"]): + raise PolicyError("new pull requests must use the Gitea draft-title prefix") + _validate_text(data["body"], "body", 65536, required=False) + return "create-draft" + + if normalized_method == "PATCH": + match = PR_PATH_RE.fullmatch(target.path) + if not match: + raise PolicyError("PATCH is limited to Atlas draft pull-request metadata") + _validate_repo(match.group(1)) + if not isinstance(data, dict) or not data or not set(data) <= {"body", "title"}: + raise PolicyError("draft updates accept only title and body") + if "title" in data and data["title"] != _draft_title(data["title"]): + raise PolicyError("updated pull-request titles must preserve draft state") + if "body" in data: + _validate_text(data["body"], "body", 65536, required=False) + return "update-draft" + + raise PolicyError( + "method is not available: merge, approve, close, delete, and force-push are outside this client" ) @@ -49,100 +164,255 @@ def build_request( token: str, data: object | None = None, ) -> urllib.request.Request: - """Build one authenticated request without placing the token in its URL.""" + """Build an authorized request without putting the token in its URL or body.""" + authorize_request(method, path, data) payload = None if data is not None: payload = json.dumps(data, separators=(",", ":")).encode("utf-8") return urllib.request.Request( api_url(base_url, path), data=payload, - method=method, + method=method.upper(), headers={ "Accept": "application/json", "Authorization": f"token {token}", "Content-Type": "application/json", - "User-Agent": "hermes-atlas-operator/1", + "User-Agent": "hermes-atlas-pr-client/1", }, ) -def parse_args(argv: list[str] | None = None) -> argparse.Namespace: - """Parse a bounded method, API path, and optional JSON request body.""" - parser = argparse.ArgumentParser( - description="Call the Atlas Gitea API using the runtime Vault token." +def redact_bytes(value: bytes, token: str) -> bytes: + """Remove exact runtime-token and authorization-header values from output.""" + redacted = value.replace(token.encode("utf-8"), b"[REDACTED]") + return re.sub( + rb"(?i)(authorization\s*[:=]\s*)(?:token|bearer)?\s*[^\s,;\"}]+", + rb"\1[REDACTED]", + redacted, ) - parser.add_argument("method", choices=ALLOWED_METHODS) - parser.add_argument("path", help="Gitea path beginning with /api/v1/") - data_group = parser.add_mutually_exclusive_group() - data_group.add_argument("--data-json", help="JSON object/array request body") - data_group.add_argument( - "--data-file", type=Path, help="path to a JSON request body" - ) - data_group.add_argument( - "--field", - action="append", - metavar="KEY=VALUE", - help=( - "repeatable JSON field; bare text remains a string while true, false, " - "null, numbers, objects, and arrays are decoded as JSON" - ), - ) - return parser.parse_args(argv) -def parse_fields(values: list[str]) -> dict[str, object]: - """Build a JSON object from shell-safe, repeatable key/value arguments.""" - result: dict[str, object] = {} - for value in values: - key, separator, raw = value.partition("=") - if not separator or not key or any(char.isspace() for char in key): - raise ValueError("each --field must be KEY=VALUE with a non-space key") - try: - result[key] = json.loads(raw) - except json.JSONDecodeError: - result[key] = raw +def _request( + method: str, + path: str, + data: object | None, + *, + token: str, + opener: Callable[..., object] = urllib.request.urlopen, +) -> bytes: + request = build_request( + method, + path, + base_url=configured_base_url(), + token=token, + data=data, + ) + with opener(request, timeout=30) as response: # type: ignore[attr-defined] + return redact_bytes(response.read(), token) # type: ignore[attr-defined] + + +def read( + path: str, *, token: str, opener: Callable[..., object] = urllib.request.urlopen +) -> bytes: + """Read one Atlas repository API resource.""" + return _request("GET", path, None, token=token, opener=opener) + + +def create_draft( + repo: str, + *, + base: str, + head: str, + title: str, + body: str, + token: str, + opener: Callable[..., object] = urllib.request.urlopen, +) -> bytes: + """Create a same-repository draft pull request for human review.""" + data = { + "base": base, + "body": body, + "head": head, + "title": _draft_title(title), + } + result = _request( + "POST", + f"/api/v1/repos/{ALLOWED_OWNER}/{_validate_repo(repo)}/pulls", + data, + token=token, + opener=opener, + ) + _require_draft_response(result) return result -def load_data(args: argparse.Namespace) -> object | None: - """Decode the optional JSON body without involving a shell expansion.""" - if args.data_json is not None: - return json.loads(args.data_json) - if args.data_file is not None: - return json.loads(args.data_file.read_text(encoding="utf-8")) - if args.field is not None: - return parse_fields(args.field) - return None +def _require_draft_response(value: bytes) -> dict[str, object]: + """Fail closed unless Forgejo confirms the resulting PR remains a draft.""" + try: + document = json.loads(value) + except json.JSONDecodeError as exc: + raise PolicyError("Forgejo returned invalid pull-request metadata") from exc + if not isinstance(document, dict) or document.get("draft") is not True: + raise PolicyError( + "Forgejo did not confirm draft state; human review is required" + ) + return document + + +def update_draft( + repo: str, + number: int, + updates: dict[str, str], + *, + token: str, + opener: Callable[..., object] = urllib.request.urlopen, +) -> bytes: + """Update title/body only after Forgejo confirms the PR is still a draft.""" + if not isinstance(number, int) or isinstance(number, bool) or number < 1: + raise PolicyError("pull-request number must be positive") + path = f"/api/v1/repos/{ALLOWED_OWNER}/{_validate_repo(repo)}/pulls/{number}" + current_raw = _request("GET", path, None, token=token, opener=opener) + try: + current = json.loads(current_raw) + except json.JSONDecodeError as exc: + raise PolicyError("Forgejo returned invalid pull-request metadata") from exc + if not isinstance(current, dict) or current.get("draft") is not True: + raise PolicyError("pull request is not a draft; human review now owns it") + safe_updates = dict(updates) + if "title" in safe_updates: + safe_updates["title"] = _draft_title(safe_updates["title"]) + result = _request("PATCH", path, safe_updates, token=token, opener=opener) + _require_draft_response(result) + return result + + +def _body(args: argparse.Namespace) -> str: + if args.body_file is None: + return "" + return args.body_file.read_text(encoding="utf-8") + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + """Parse the three intentionally narrow client operations.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--dry-run", + action="store_true", + help="validate and describe the request without reading a token or using the network", + ) + subparsers = parser.add_subparsers(dest="operation", required=True) + + read_parser = subparsers.add_parser( + "read", help="read an Atlas repository API path" + ) + read_parser.add_argument("path") + + create_parser = subparsers.add_parser( + "create-draft", help="create a same-repository draft pull request" + ) + create_parser.add_argument("repo") + create_parser.add_argument("--base", required=True) + create_parser.add_argument("--head", required=True) + create_parser.add_argument("--title", required=True) + create_parser.add_argument("--body-file", type=Path) + + update_parser = subparsers.add_parser( + "update-draft", help="update title/body of an existing draft pull request" + ) + update_parser.add_argument("repo") + update_parser.add_argument("number", type=int) + update_parser.add_argument("--title") + update_parser.add_argument("--body-file", type=Path) + return parser.parse_args(argv) + + +def _operation(args: argparse.Namespace) -> tuple[str, str, object | None]: + if args.operation == "read": + return "GET", args.path, None + repo = _validate_repo(args.repo) + if args.operation == "create-draft": + data = { + "base": args.base, + "body": _body(args), + "head": args.head, + "title": _draft_title(args.title), + } + return "POST", f"/api/v1/repos/{ALLOWED_OWNER}/{repo}/pulls", data + if args.number < 1: + raise PolicyError("pull-request number must be positive") + data = {} + if args.title is not None: + data["title"] = _draft_title(args.title) + if args.body_file is not None: + data["body"] = _body(args) + return "PATCH", f"/api/v1/repos/{ALLOWED_OWNER}/{repo}/pulls/{args.number}", data + + +def _write_body(body: bytes) -> None: + if body: + sys.stdout.buffer.write(body) + if not body.endswith(b"\n"): + sys.stdout.buffer.write(b"\n") def main(argv: list[str] | None = None) -> int: - """Execute the request, print only its response body, and return HTTP status.""" - args = parse_args(argv) + """Validate, optionally describe, and execute one safe Forgejo operation.""" + token = "" try: - request = build_request( - args.method, - args.path, - base_url=os.environ.get("GITEA_BASE_URL", DEFAULT_BASE_URL), - token=read_token(), - data=load_data(args), - ) - with urllib.request.urlopen(request, timeout=30) as response: - body = response.read() - if body: - sys.stdout.buffer.write(body) - if not body.endswith(b"\n"): - sys.stdout.buffer.write(b"\n") + args = parse_args(argv) + method, path, data = _operation(args) + operation = authorize_request(method, path, data) + if args.dry_run: + print( + json.dumps( + { + "dry_run": True, + "host": "scm.bstein.dev", + "operation": operation, + "owner": ALLOWED_OWNER, + "path": path, + "request_fields": sorted(data) + if isinstance(data, dict) + else [], + "requires_draft_preflight": operation == "update-draft", + }, + sort_keys=True, + ) + ) + return 0 + token = read_token() + if args.operation == "read": + body = read(path, token=token) + elif args.operation == "create-draft": + assert isinstance(data, dict) + body = create_draft( + args.repo, + base=str(data["base"]), + head=str(data["head"]), + title=str(data["title"]), + body=str(data["body"]), + token=token, + ) + else: + assert isinstance(data, dict) + body = update_draft(args.repo, args.number, data, token=token) + _write_body(body) return 0 except urllib.error.HTTPError as exc: - body = exc.read(65536) - print(f"Gitea API returned HTTP {exc.code}", file=sys.stderr) + body = redact_bytes(exc.read(MAX_ERROR_BYTES), token) if token else b"" + print(f"Forgejo request failed with HTTP {exc.code}", file=sys.stderr) if body: sys.stderr.buffer.write(body) if not body.endswith(b"\n"): sys.stderr.buffer.write(b"\n") return 1 - except (OSError, ValueError, json.JSONDecodeError) as exc: - print(f"Gitea API request failed: {exc}", file=sys.stderr) + except (OSError, PolicyError, ValueError, json.JSONDecodeError): + # Deliberately omit exception details. File contents, HTTP request + # objects, and provider errors can carry credential material. + print( + "Forgejo request rejected or unavailable; no credential was disclosed", + file=sys.stderr, + ) return 1 diff --git a/services/hermes/skills/manage-atlas-pull-requests/SKILL.md b/services/hermes/skills/manage-atlas-pull-requests/SKILL.md new file mode 100644 index 00000000..4e73a3cf --- /dev/null +++ b/services/hermes/skills/manage-atlas-pull-requests/SKILL.md @@ -0,0 +1,58 @@ +--- +name: manage-atlas-pull-requests +description: Read private Atlas repository and pull-request metadata, create a draft pull request after a tested branch is pushed, or update the title/body of an existing draft through Hermes' least-authority Forgejo client. Use for PR handoff in scm.bstein.dev/atlas repositories. Never use it to merge, approve, close, delete, or force-push. +--- + +# Manage Atlas pull requests + +Use `/opt/coordinator/gitea_api.py` for Forgejo API access. It reads its token +from the pod-lifetime Vault projection; never read that file, copy the token, +put it in an argument or environment variable, or replace this client with +`curl`. + +## Read repository or PR state + +Pass an Atlas repository API path to the read operation: + +```sh +/opt/coordinator/gitea_api.py read /api/v1/repos/atlas/REPO/pulls/NUMBER +``` + +Reads outside `scm.bstein.dev/atlas` are rejected. Treat returned state as +evidence, not authorization to change it. + +## Create a review handoff + +Before opening a PR, verify the remote, branch, clean worktree, diff, tests, and +exact pushed commit. Never force-push. Write the PR description to a workspace +file so multiline Markdown is not shell-quoted, then validate without reading a +credential or using the network: + +```sh +/opt/coordinator/gitea_api.py --dry-run create-draft REPO \ + --base main --head hermes/TASK --title "Focused change" \ + --body-file /tmp/pr-body.md +``` + +Run the same command without `--dry-run` only when the branch is review-ready. +The client always creates a draft in the same repository. Report its URL and +exact head commit to Brad. Leave it unmerged for human review. + +## Update an existing draft + +Use only a title, a body file, or both: + +```sh +/opt/coordinator/gitea_api.py update-draft REPO NUMBER \ + --body-file /tmp/pr-body.md +``` + +The client first reads the PR from Forgejo and refuses the update unless it is +still a draft. It never changes head/base, review state, or merge state. + +## Stop at the authority boundary + +Merge, approve, close, delete, branch mutation, comments, releases, repository +administration, and force-push are intentionally unavailable. Do not bypass the +client with a raw HTTP request. If one of those actions is needed, present the +tested commit and evidence to Brad and stop for human review. diff --git a/services/hermes/skills/manage-atlas-pull-requests/agents/openai.yaml b/services/hermes/skills/manage-atlas-pull-requests/agents/openai.yaml new file mode 100644 index 00000000..57cb8496 --- /dev/null +++ b/services/hermes/skills/manage-atlas-pull-requests/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Manage Atlas Pull Requests" + short_description: "Create safe draft PR handoffs for human review" + default_prompt: "Use $manage-atlas-pull-requests to inspect the requested Atlas repository or prepare a tested branch as an unmerged draft pull request. Preserve the human review boundary." diff --git a/testing/tests/test_hermes_chat_quality.py b/testing/tests/test_hermes_chat_quality.py index 818f7679..09f6b00c 100644 --- a/testing/tests/test_hermes_chat_quality.py +++ b/testing/tests/test_hermes_chat_quality.py @@ -99,9 +99,11 @@ def test_agent_config_keeps_delegated_reviewers_from_owning_task_lifecycle(): assert "Never call `kanban_show` without\na known, non-empty task ID" in soul assert "must load a skill only when its workflow\nmaterially applies" in soul assert "runtime-only `GIT_ASKPASS`" in soul - assert "`scm.bstein.dev` is Gitea, not GitHub" in instructions - assert "Never load or follow a GitHub/`gh`" in instructions - assert "/opt/coordinator/gitea_api.py METHOD /api/v1/..." in instructions + assert "`scm.bstein.dev` is Forgejo/Gitea, not GitHub" in instructions + assert "GitHub/`gh` skill for an Atlas remote" in instructions + assert "load\n`$manage-atlas-pull-requests`" in instructions + assert "Merge, approve, close, delete, comments" in instructions + assert "Leave every PR unmerged for Brad's review" in instructions assert "JENKINS_BASE_URL" in instructions assert "Do not\nuse `git reset --hard`" in instructions rendered = (HERMES / "agent-deployment.yaml").read_text() diff --git a/testing/tests/test_hermes_gitea_pr_client.py b/testing/tests/test_hermes_gitea_pr_client.py new file mode 100644 index 00000000..658af189 --- /dev/null +++ b/testing/tests/test_hermes_gitea_pr_client.py @@ -0,0 +1,322 @@ +"""Adversarial contracts for the Atlas-only draft pull-request client.""" + +from __future__ import annotations + +import importlib.util +import io +import json +import sys +import urllib.error +from pathlib import Path + +import pytest +import yaml + +ROOT = Path(__file__).parents[2] +CLIENT_PATH = ROOT / "services/hermes/scripts/gitea_api.py" + + +def _load(): + spec = importlib.util.spec_from_file_location("safe_gitea_api", CLIENT_PATH) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _draft_payload(**updates): + payload = { + "base": "main", + "body": "Review evidence", + "head": "hermes/review-fix", + "title": "WIP: Repair review findings", + } + payload.update(updates) + return payload + + +class Response: + def __init__(self, body: object): + self.body = body if isinstance(body, bytes) else json.dumps(body).encode() + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def read(self): + return self.body + + +@pytest.mark.parametrize( + ("base_url", "path"), + [ + ("https://evil.example", "/api/v1/repos/atlas/cassandra"), + ("http://scm.bstein.dev", "/api/v1/repos/atlas/cassandra"), + ("https://scm.bstein.dev:443", "/api/v1/repos/atlas/cassandra"), + ("https://scm.bstein.dev", "https://evil.example/api/v1/repos/atlas/cassandra"), + ("https://scm.bstein.dev", "/api/v1/repos/evil/cassandra"), + ("https://scm.bstein.dev", "/api/v1/repos/%61tlas/cassandra"), + ("https://scm.bstein.dev", "/api/v1/repos/atlas/../admin"), + ], +) +def test_host_owner_and_path_escape_attempts_are_rejected(base_url: str, path: str): + client = _load() + + with pytest.raises(client.PolicyError): + client.build_request("GET", path, base_url=base_url, token="secret") + + +@pytest.mark.parametrize( + ("method", "path", "data"), + [ + ("DELETE", "/api/v1/repos/atlas/cassandra/pulls/4", None), + ("POST", "/api/v1/repos/atlas/cassandra/pulls/4/merge", {}), + ( + "POST", + "/api/v1/repos/atlas/cassandra/pulls/4/reviews", + {"event": "APPROVED"}, + ), + ("PATCH", "/api/v1/repos/atlas/cassandra/pulls/4", {"state": "closed"}), + ("PATCH", "/api/v1/repos/atlas/cassandra/pulls/4", {"draft": False}), + ("PUT", "/api/v1/repos/atlas/cassandra/branches/main", {}), + ], +) +def test_merge_approve_close_delete_and_other_mutations_are_rejected( + method: str, path: str, data: object +): + client = _load() + + with pytest.raises(client.PolicyError): + client.build_request( + method, path, base_url=client.CANONICAL_BASE_URL, token="secret", data=data + ) + + +def test_create_forces_draft_title_and_same_repository_branch_names(): + client = _load() + assert ( + client.authorize_request( + "POST", "/api/v1/repos/atlas/cassandra/pulls", _draft_payload() + ) + == "create-draft" + ) + + with pytest.raises(client.PolicyError, match="draft-title prefix"): + client.authorize_request( + "POST", + "/api/v1/repos/atlas/cassandra/pulls", + _draft_payload(title="Not a draft"), + ) + with pytest.raises(client.PolicyError): + client.authorize_request( + "POST", + "/api/v1/repos/atlas/cassandra/pulls", + _draft_payload(head="someone:branch"), + ) + + +def test_update_checks_server_draft_state_before_patching(): + client = _load() + calls = [] + responses = iter( + [ + Response({"number": 7, "draft": True}), + Response({"number": 7, "draft": True}), + ] + ) + + def opener(request, timeout): + calls.append((request.method, request.full_url, request.data, timeout)) + return next(responses) + + result = client.update_draft( + "cassandra", 7, {"title": "Narrowed repair"}, token="runtime", opener=opener + ) + + assert json.loads(result) == {"number": 7, "draft": True} + assert [call[0] for call in calls] == ["GET", "PATCH"] + assert json.loads(calls[1][2]) == {"title": "WIP: Narrowed repair"} + + +def test_update_rejects_non_draft_without_sending_patch(): + client = _load() + calls = [] + + def opener(request, timeout): + calls.append((request.method, timeout)) + return Response({"number": 7, "draft": False}) + + with pytest.raises(client.PolicyError, match="human review now owns"): + client.update_draft( + "cassandra", 7, {"body": "new"}, token="runtime", opener=opener + ) + assert calls == [("GET", 30)] + + +def test_runtime_token_is_only_an_authorization_header(): + client = _load() + request = client.build_request( + "POST", + "/api/v1/repos/atlas/cassandra/pulls", + base_url=client.CANONICAL_BASE_URL, + token="do-not-leak", + data=_draft_payload(), + ) + + assert "do-not-leak" not in request.full_url + assert b"do-not-leak" not in request.data + assert request.get_header("Authorization") == "token do-not-leak" + + +def test_create_uses_live_gitea_schema_and_verifies_draft_response(): + client = _load() + calls = [] + + def opener(request, timeout): + calls.append((json.loads(request.data), timeout)) + return Response({"number": 3, "draft": True}) + + result = client.create_draft( + "cassandra", + base="main", + head="hermes/fix", + title="Focused fix", + body="Evidence", + token="runtime", + opener=opener, + ) + + assert json.loads(result)["draft"] is True + assert calls[0][0]["title"] == "WIP: Focused fix" + assert "draft" not in calls[0][0] + + +def test_create_fails_closed_when_server_does_not_confirm_draft(): + client = _load() + + with pytest.raises(client.PolicyError, match="did not confirm draft"): + client.create_draft( + "cassandra", + base="main", + head="hermes/fix", + title="Focused fix", + body="Evidence", + token="runtime", + opener=lambda *_a, **_k: Response({"number": 3, "draft": False}), + ) + + +def test_output_redaction_covers_exact_token_and_authorization_header(): + client = _load() + raw = b'{"message":"do-not-leak","debug":"Authorization: token do-not-leak"}' + redacted = client.redact_bytes(raw, "do-not-leak") + + assert b"do-not-leak" not in redacted + assert redacted.count(b"[REDACTED]") >= 1 + + +def test_dry_run_does_not_read_token_or_use_network( + tmp_path: Path, monkeypatch, capsys +): + client = _load() + body = tmp_path / "body.md" + body.write_text("Evidence only\n", encoding="utf-8") + monkeypatch.setattr(client, "read_token", lambda: pytest.fail("read token")) + monkeypatch.setattr( + client.urllib.request, "urlopen", lambda *_a, **_k: pytest.fail("network") + ) + + assert ( + client.main( + [ + "--dry-run", + "create-draft", + "cassandra", + "--base", + "main", + "--head", + "hermes/fix", + "--title", + "Repair", + "--body-file", + str(body), + ] + ) + == 0 + ) + output = json.loads(capsys.readouterr().out) + assert output["operation"] == "create-draft" + assert output["owner"] == "atlas" + assert "Evidence only" not in json.dumps(output) + + +def test_http_error_path_redacts_token(monkeypatch, capsys): + client = _load() + monkeypatch.setattr(client, "read_token", lambda: "do-not-leak") + + def fail(*_args, **_kwargs): + raise urllib.error.HTTPError( + "https://scm.bstein.dev/api/v1/repos/atlas/cassandra", + 403, + "forbidden", + {}, + io.BytesIO(b"Authorization: token do-not-leak"), + ) + + monkeypatch.setattr(client, "read", lambda *_a, **_k: fail()) + + assert client.main(["read", "/api/v1/repos/atlas/cassandra"]) == 1 + captured = capsys.readouterr() + assert "do-not-leak" not in captured.err + assert "HTTP 403" in captured.err + + +def test_flux_manifest_projects_runtime_vault_token_and_skill_only(): + client_source = CLIENT_PATH.read_text(encoding="utf-8") + assert "/runtime-access/gitea-token" in client_source + assert "GITEA_TOKEN" not in client_source + + deployment = yaml.safe_load( + (ROOT / "services/hermes/agent-deployment.yaml").read_text(encoding="utf-8") + ) + template = deployment["spec"]["template"] + annotations = template["metadata"]["annotations"] + assert annotations["vault.hashicorp.com/agent-inject-secret-gitea-token"] == ( + "kv/data/atlas/hermes/developer-gitea" + ) + runtime = next( + volume + for volume in template["spec"]["volumes"] + if volume["name"] == "runtime-access" + ) + assert runtime["emptyDir"]["medium"] == "Memory" + + expected = {"hermes", "terminal", "cli-lane-runner"} + mounted = { + container["name"] + for container in template["spec"]["containers"] + if any( + mount["name"] == "atlas-pr-skill" + and mount["mountPath"] + == "/opt/data/workspace/skills/manage-atlas-pull-requests" + and mount.get("readOnly") is True + for mount in container.get("volumeMounts", []) + ) + } + assert mounted == expected + + kustomization = yaml.safe_load( + (ROOT / "services/hermes/kustomization.yaml").read_text(encoding="utf-8") + ) + generator = next( + item + for item in kustomization["configMapGenerator"] + if item["name"] == "hermes-atlas-pr-skill" + ) + assert generator["files"] == [ + "SKILL.md=skills/manage-atlas-pull-requests/SKILL.md", + "openai.yaml=skills/manage-atlas-pull-requests/agents/openai.yaml", + ] diff --git a/testing/tests/test_hermes_runtime_access.py b/testing/tests/test_hermes_runtime_access.py index 7e455eb6..7ad5087a 100644 --- a/testing/tests/test_hermes_runtime_access.py +++ b/testing/tests/test_hermes_runtime_access.py @@ -11,7 +11,6 @@ from pathlib import Path import pytest import yaml - ROOT = Path(__file__).parents[2] HERMES = ROOT / "services" / "hermes" SCRIPTS = HERMES / "scripts" @@ -31,19 +30,23 @@ def test_gitea_api_builds_runtime_authenticated_same_origin_requests(): request = gitea_api.build_request( "POST", - "/api/v1/repos/atlas/cassandra/issues/1/comments", + "/api/v1/repos/atlas/cassandra/pulls", base_url="https://scm.bstein.dev", token="runtime-only-value", - data={"body": "reviewed"}, + data={ + "base": "main", + "body": "Ready for review.", + "head": "hermes/repair", + "title": "WIP: Repair semantic review findings", + }, ) assert isinstance(request, urllib.request.Request) - assert request.full_url == ( - "https://scm.bstein.dev/api/v1/repos/atlas/cassandra/issues/1/comments" - ) + assert request.full_url == "https://scm.bstein.dev/api/v1/repos/atlas/cassandra/pulls" assert request.method == "POST" assert request.get_header("Authorization") == "token runtime-only-value" - assert json.loads(request.data) == {"body": "reviewed"} + assert json.loads(request.data)["title"].startswith("WIP: ") + assert "draft" not in json.loads(request.data) @pytest.mark.parametrize( @@ -61,25 +64,14 @@ def test_gitea_api_rejects_foreign_or_non_api_targets(path: str): gitea_api.api_url("https://scm.bstein.dev", path) -def test_gitea_api_parses_shell_safe_fields_as_json_values(): +@pytest.mark.parametrize("method", ["DELETE", "PUT", "OPTIONS"]) +def test_gitea_api_rejects_unavailable_methods(method: str): gitea_api = _load("gitea_api") - assert gitea_api.parse_fields( - ["Do=manually-merged", "MergeCommitID=abc123", "enabled=true", "count=2"] - ) == { - "Do": "manually-merged", - "MergeCommitID": "abc123", - "enabled": True, - "count": 2, - } - - -@pytest.mark.parametrize("value", ["missing-separator", "=missing-key", "bad key=x"]) -def test_gitea_api_rejects_invalid_shell_safe_fields(value: str): - gitea_api = _load("gitea_api") - - with pytest.raises(ValueError): - gitea_api.parse_fields([value]) + with pytest.raises(gitea_api.PolicyError): + gitea_api.authorize_request( + method, "/api/v1/repos/atlas/cassandra/pulls/1", None + ) def test_jenkins_evidence_ignores_nested_execution_result(): -- 2.47.2 From cf3e46c2b684270ff931d82fcadcee6777fb9543 Mon Sep 17 00:00:00 2001 From: jenkins Date: Sun, 16 Aug 2026 20:18:49 -0300 Subject: [PATCH 02/12] hermes: harden Forgejo client boundary --- services/hermes/agent-configmap.yaml | 33 +- services/hermes/scripts/gitea_api.py | 381 +++++++++------- .../manage-atlas-pull-requests/SKILL.md | 48 +- testing/tests/test_hermes_chat_quality.py | 8 +- testing/tests/test_hermes_gitea_pr_client.py | 417 +++++++++++------- .../tests/test_hermes_gitea_pr_integration.py | 147 ++++++ 6 files changed, 671 insertions(+), 363 deletions(-) create mode 100644 testing/tests/test_hermes_gitea_pr_integration.py diff --git a/services/hermes/agent-configmap.yaml b/services/hermes/agent-configmap.yaml index b35da59a..782876cf 100644 --- a/services/hermes/agent-configmap.yaml +++ b/services/hermes/agent-configmap.yaml @@ -318,15 +318,15 @@ data: `scm.bstein.dev` is Forgejo/Gitea, not GitHub. Never load or follow a GitHub/`gh` skill for an Atlas remote, and do not interpret an unauthenticated Gitea HTTP 404 as a missing private repository. Use - authenticated Git for clone, fetch, and non-force push. For repository and - pull-request reads, or to create/update a review-ready draft PR, load + authenticated Git for clone, fetch, and non-force push. For bounded + repository/pull-request evidence or to create a review-ready draft PR, load `$manage-atlas-pull-requests` and use `/opt/coordinator/gitea_api.py`. The client injects the runtime Vault token without exposing it to the command line, environment, output, or transcript. It permits only Atlas-scoped - reads, same-repository draft creation, and title/body updates while the PR - remains draft. Merge, approve, close, delete, comments, repository - administration, and force-push are unavailable; never bypass the client - with raw HTTP. Leave every PR unmerged for Brad's review. Independently + metadata reads and verified same-repository draft creation. Updates, merge, + approve, close, delete, comments, repository administration, and force-push + are unavailable; never bypass the client with raw HTTP. Leave every PR + unmerged for Brad's review. Independently verify the exact base/head ancestry and diff, run the relevant tests, and confirm the candidate Jenkins result before presenting a handoff. Prefer the internal endpoint in @@ -337,25 +337,24 @@ data: [--commit SHA] --wait`; it distinguishes a genuinely terminal build from nested Jenkins execution metadata and returns bounded JSON/log evidence. With a branch, the helper also tries the conventional `JOB-branches` - multibranch name. After Brad merges, observe the default-branch build to a - terminal result and report exact commit, build, and test evidence. + multibranch name. If Brad separately reports that he merged the PR, observe + the default-branch build to a terminal result and report exact commit, + build, and test evidence. The workspace already has a non-secret Hermes Git author identity. Do not use `git reset --hard`, even in a fresh clone; use a detached worktree or a clean branch switch when comparing revisions, and preserve any unexpected - file as user state. Use the Gitea pull-request merge endpoint for an open - PR so both Git history and PR state remain auditable. If an authorized - manual merge already reached the base branch, reconcile the open PR with - `Do=manually-merged` and its exact merge commit instead of leaving stale - review state. + file as user state. PR publication ends at the verified open draft; Brad + owns review and merge authority. The terminal PATH contains the pinned operator tools. Start cluster work with `kubectl config current-context`, read-only status/events/logs, and the relevant `titan-iac` manifests. Put durable desired-state changes on a - reviewable `titan-iac` branch, validate Kustomize and client dry-run, then - use Flux reconciliation after the tracked change is published. Direct - `kubectl` mutations are for explicit incident recovery or ephemeral - verification, not ordinary delivery. + reviewable `titan-iac` branch and validate Kustomize and client dry-run. + Brad owns merge and routine Flux reconciliation; perform either only under + a separate explicit operator request. Direct `kubectl` mutations are for + explicit incident recovery or ephemeral verification, not ordinary + delivery. Node SSH uses a dedicated audited Hermes identity: `ssh titan-04` (or any current Kubernetes node name). Host keys are pinned and password fallback diff --git a/services/hermes/scripts/gitea_api.py b/services/hermes/scripts/gitea_api.py index 73a2bffd..057a79a1 100755 --- a/services/hermes/scripts/gitea_api.py +++ b/services/hermes/scripts/gitea_api.py @@ -7,6 +7,7 @@ import argparse import json import os import re +import subprocess import sys import urllib.error import urllib.parse @@ -18,18 +19,41 @@ CANONICAL_BASE_URL = "https://scm.bstein.dev" ALLOWED_OWNER = "atlas" DEFAULT_TOKEN_FILE = Path("/runtime-access/gitea-token") REPO_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,99}\Z") -PR_PATH_RE = re.compile(r"/api/v1/repos/atlas/([^/]+)/pulls/([1-9][0-9]*)\Z") +SHA_RE = re.compile(r"[0-9a-fA-F]{40}\Z") CREATE_PATH_RE = re.compile(r"/api/v1/repos/atlas/([^/]+)/pulls\Z") -READ_PATH_RE = re.compile(r"/api/v1/repos/atlas/([^/]+)(?:/.*)?\Z") -SAFE_REF_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._/-]{0,199}\Z") MAX_ERROR_BYTES = 8192 +MAX_RESPONSE_BYTES = 2 * 1024 * 1024 DRAFT_TITLE_PREFIX = "WIP: " +GIT_BIN = "/usr/bin/git" +SENSITIVE_BODY_PATTERNS = ( + re.compile(r"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----"), + re.compile(r"(?i)\b(?:password|passwd|secret|token|api[_-]?key)\s*[:=]"), + re.compile(r"(?i)\bauthorization\s*:\s*(?:bearer|token|basic)\s+\S+"), + re.compile(r"\b(?:gh[pousr]_[A-Za-z0-9]{20,}|sk-[A-Za-z0-9]{20,})\b"), + re.compile(r"\b(?:AKIA[0-9A-Z]{16}|AIza[0-9A-Za-z_-]{35})\b"), + re.compile(r"\beyJ[A-Za-z0-9_-]{12,}\.[A-Za-z0-9_-]{12,}\.[A-Za-z0-9_-]{8,}\b"), +) class PolicyError(ValueError): """Raised when a requested Forgejo operation exceeds the safe boundary.""" +class RejectRedirectHandler(urllib.request.HTTPRedirectHandler): + """Reject every redirect before urllib can copy authentication headers.""" + + def redirect_request(self, req, fp, code, msg, headers, newurl): + raise PolicyError("Forgejo redirects are not allowed") + + +_SAFE_OPENER = urllib.request.build_opener(RejectRedirectHandler()) + + +def _safe_urlopen(request: urllib.request.Request, timeout: int): + """Open one request using a handler that never follows redirects.""" + return _SAFE_OPENER.open(request, timeout=timeout) + + def read_token(path: Path = DEFAULT_TOKEN_FILE) -> str: """Read the token from the pod-lifetime Vault projection.""" token = path.read_text(encoding="utf-8").strip() @@ -53,19 +77,29 @@ def _validate_repo(repo: str) -> str: def _validate_ref(value: object, name: str) -> str: - if not isinstance(value, str) or not SAFE_REF_RE.fullmatch(value): + """Validate the complete Git ref grammar using Git itself.""" + if not isinstance(value, str) or not value or value.startswith("-"): raise PolicyError(f"{name} must be a same-repository branch name") - if ( - value.startswith(".") - or value.endswith(("/", ".", ".lock")) - or "//" in value - or ".." in value - or "@{" in value - ): + if any(ord(character) < 32 or ord(character) == 127 for character in value): + raise PolicyError(f"{name} must be a safe same-repository branch name") + result = subprocess.run( + [GIT_BIN, "check-ref-format", f"refs/heads/{value}"], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + if result.returncode != 0: raise PolicyError(f"{name} must be a safe same-repository branch name") return value +def _validate_sha(value: object, name: str = "head SHA") -> str: + if not isinstance(value, str) or not SHA_RE.fullmatch(value): + raise PolicyError(f"{name} must be a full Git SHA-1") + return value.lower() + + def _validate_text(value: object, name: str, maximum: int, *, required: bool) -> str: if not isinstance(value, str): raise PolicyError(f"{name} must be text") @@ -76,6 +110,15 @@ def _validate_text(value: object, name: str, maximum: int, *, required: bool) -> return value +def _validate_body(value: object, *, forbidden: tuple[str, ...] = ()) -> str: + body = _validate_text(value, "body", 16384, required=False) + if any(secret and secret in body for secret in forbidden): + raise PolicyError("pull-request body contains runtime credential material") + if any(pattern.search(body) for pattern in SENSITIVE_BODY_PATTERNS): + raise PolicyError("pull-request body resembles credential material") + return body + + def _draft_title(value: object) -> str: """Return a bounded title using Gitea's configured default draft prefix.""" title = _validate_text(value, "title", 251, required=True).strip() @@ -99,6 +142,73 @@ def _split_api_path(path: str) -> urllib.parse.SplitResult: return target +def _validate_query(target: urllib.parse.SplitResult, allowed: set[str]) -> None: + try: + pairs = urllib.parse.parse_qsl( + target.query, keep_blank_values=True, strict_parsing=True + ) + except ValueError as exc: + raise PolicyError("invalid API query") from exc + if len({key for key, _ in pairs}) != len(pairs): + raise PolicyError("duplicate API query parameters are not allowed") + if any(key not in allowed for key, _ in pairs): + raise PolicyError("API query parameter is outside the read allowlist") + values = dict(pairs) + for name in ("page", "limit"): + if name not in values: + continue + if not values[name].isdigit() or int(values[name]) < 1: + raise PolicyError(f"{name} must be a positive integer") + if "limit" in values and int(values["limit"]) > 50: + raise PolicyError("read limit cannot exceed 50") + if "state" in values and values["state"] not in {"open", "closed", "all"}: + raise PolicyError("pull-request state is invalid") + + +def _authorize_read(target: urllib.parse.SplitResult) -> str: + """Allow only repository, PR, branch, commit, and status metadata reads.""" + prefix = "/api/v1/repos/atlas/" + remainder = target.path.removeprefix(prefix) + if remainder == target.path: + raise PolicyError("reads are limited to explicit Atlas repository metadata") + repo, separator, suffix = remainder.partition("/") + _validate_repo(repo) + + if not separator: + _validate_query(target, set()) + return "repository" + if suffix == "pulls": + _validate_query(target, {"page", "limit", "state"}) + return "pull-list" + if re.fullmatch(r"pulls/[1-9][0-9]*", suffix): + _validate_query(target, set()) + return "pull" + if re.fullmatch(r"pulls/[1-9][0-9]*/(?:commits|files)", suffix): + _validate_query(target, {"page", "limit"}) + return "pull-evidence" + if suffix == "branches": + _validate_query(target, {"page", "limit"}) + return "branch-list" + branch_match = re.fullmatch(r"branches/([^/]+)", suffix) + if branch_match: + _validate_ref(branch_match.group(1), "branch") + _validate_query(target, set()) + return "branch" + if suffix == "commits": + _validate_query(target, {"page", "limit"}) + return "commit-list" + if re.fullmatch(r"git/commits/[0-9a-fA-F]{40}", suffix): + _validate_query(target, set()) + return "commit" + if re.fullmatch( + r"(?:commits/[0-9a-fA-F]{40}/status|commits/[0-9a-fA-F]{40}/statuses|statuses/[0-9a-fA-F]{40})", + suffix, + ): + _validate_query(target, {"page", "limit"}) + return "status" + raise PolicyError("repository API route is outside the metadata read allowlist") + + def api_url(base_url: str, path: str) -> str: """Return a canonical same-origin URL without forwarding credentials.""" if base_url.rstrip("/") != CANONICAL_BASE_URL: @@ -116,44 +226,23 @@ def authorize_request(method: str, path: str, data: object | None) -> str: if normalized_method == "GET": if data is not None: raise PolicyError("read operations cannot include a request body") - match = READ_PATH_RE.fullmatch(target.path) - if not match: - raise PolicyError("reads are limited to repositories owned by atlas") - _validate_repo(match.group(1)) - return "read" - + return _authorize_read(target) if target.query: raise PolicyError("mutating operations cannot include query parameters") - if normalized_method == "POST": - match = CREATE_PATH_RE.fullmatch(target.path) - if not match: - raise PolicyError("POST is limited to creating an Atlas draft pull request") - _validate_repo(match.group(1)) - if not isinstance(data, dict) or set(data) != {"base", "body", "head", "title"}: - raise PolicyError("draft creation accepts only base, body, head, and title") - _validate_ref(data["base"], "base") - _validate_ref(data["head"], "head") - if data["title"] != _draft_title(data["title"]): - raise PolicyError("new pull requests must use the Gitea draft-title prefix") - _validate_text(data["body"], "body", 65536, required=False) - return "create-draft" - - if normalized_method == "PATCH": - match = PR_PATH_RE.fullmatch(target.path) - if not match: - raise PolicyError("PATCH is limited to Atlas draft pull-request metadata") - _validate_repo(match.group(1)) - if not isinstance(data, dict) or not data or not set(data) <= {"body", "title"}: - raise PolicyError("draft updates accept only title and body") - if "title" in data and data["title"] != _draft_title(data["title"]): - raise PolicyError("updated pull-request titles must preserve draft state") - if "body" in data: - _validate_text(data["body"], "body", 65536, required=False) - return "update-draft" - - raise PolicyError( - "method is not available: merge, approve, close, delete, and force-push are outside this client" - ) + if normalized_method != "POST": + raise PolicyError("only read and create-draft operations are available") + match = CREATE_PATH_RE.fullmatch(target.path) + if not match: + raise PolicyError("POST is limited to creating an Atlas draft pull request") + _validate_repo(match.group(1)) + if not isinstance(data, dict) or set(data) != {"base", "body", "head", "title"}: + raise PolicyError("draft creation accepts only base, body, head, and title") + _validate_ref(data["base"], "base") + _validate_ref(data["head"], "head") + if data["title"] != _draft_title(data["title"]): + raise PolicyError("new pull requests must use the Gitea draft-title prefix") + _validate_body(data["body"]) + return "create-draft" def build_request( @@ -177,7 +266,7 @@ def build_request( "Accept": "application/json", "Authorization": f"token {token}", "Content-Type": "application/json", - "User-Agent": "hermes-atlas-pr-client/1", + "User-Agent": "hermes-atlas-pr-client/2", }, ) @@ -198,102 +287,116 @@ def _request( data: object | None, *, token: str, - opener: Callable[..., object] = urllib.request.urlopen, + opener: Callable[..., object] = _safe_urlopen, ) -> bytes: request = build_request( - method, - path, - base_url=configured_base_url(), - token=token, - data=data, + method, path, base_url=configured_base_url(), token=token, data=data ) with opener(request, timeout=30) as response: # type: ignore[attr-defined] - return redact_bytes(response.read(), token) # type: ignore[attr-defined] + body = response.read(MAX_RESPONSE_BYTES + 1) # type: ignore[attr-defined] + if len(body) > MAX_RESPONSE_BYTES: + raise PolicyError("Forgejo response exceeds the safe size limit") + return redact_bytes(body, token) def read( - path: str, *, token: str, opener: Callable[..., object] = urllib.request.urlopen + path: str, *, token: str, opener: Callable[..., object] = _safe_urlopen ) -> bytes: - """Read one Atlas repository API resource.""" + """Read one explicitly allowed Atlas repository metadata resource.""" return _request("GET", path, None, token=token, opener=opener) +def _nested(document: dict[str, object], *keys: str) -> object: + value: object = document + for key in keys: + if not isinstance(value, dict) or key not in value: + raise PolicyError("Forgejo omitted required pull-request evidence") + value = value[key] + return value + + +def _require_create_response( + value: bytes, + *, + repo: str, + base: str, + head: str, + head_sha: str, + title: str, + body: str, +) -> dict[str, object]: + """Require exact server evidence for the intended open draft handoff.""" + try: + document = json.loads(value) + except json.JSONDecodeError as exc: + raise PolicyError("Forgejo returned invalid pull-request metadata") from exc + if not isinstance(document, dict): + raise PolicyError("Forgejo returned invalid pull-request metadata") + number = document.get("number") + if not isinstance(number, int) or isinstance(number, bool) or number < 1: + raise PolicyError("Forgejo omitted a valid pull-request number") + full_name = f"{ALLOWED_OWNER}/{repo}" + expected_html = f"{CANONICAL_BASE_URL}/{full_name}/pulls/{number}" + expected_api = f"{CANONICAL_BASE_URL}/api/v1/repos/{full_name}/pulls/{number}" + checks = ( + document.get("state") == "open", + document.get("draft") is True, + document.get("merged") is False, + document.get("html_url") == expected_html, + document.get("url") == expected_api, + document.get("title") == title, + document.get("body") == body, + _nested(document, "base", "ref") == base, + _nested(document, "base", "repo", "full_name") == full_name, + _nested(document, "head", "ref") == head, + _nested(document, "head", "sha") == head_sha, + _nested(document, "head", "repo", "full_name") == full_name, + ) + if not all(checks): + raise PolicyError("Forgejo did not confirm the exact draft pull-request state") + return document + + def create_draft( repo: str, *, base: str, head: str, + head_sha: str, title: str, body: str, token: str, - opener: Callable[..., object] = urllib.request.urlopen, + opener: Callable[..., object] = _safe_urlopen, ) -> bytes: - """Create a same-repository draft pull request for human review.""" - data = { - "base": base, - "body": body, - "head": head, - "title": _draft_title(title), - } + """Create and verify a same-repository draft pull request for human review.""" + repo = _validate_repo(repo) + base = _validate_ref(base, "base") + head = _validate_ref(head, "head") + head_sha = _validate_sha(head_sha) + title = _draft_title(title) + body = _validate_body(body, forbidden=(token,)) + data = {"base": base, "body": body, "head": head, "title": title} result = _request( "POST", - f"/api/v1/repos/{ALLOWED_OWNER}/{_validate_repo(repo)}/pulls", + f"/api/v1/repos/{ALLOWED_OWNER}/{repo}/pulls", data, token=token, opener=opener, ) - _require_draft_response(result) + _require_create_response( + result, + repo=repo, + base=base, + head=head, + head_sha=head_sha, + title=title, + body=body, + ) return result -def _require_draft_response(value: bytes) -> dict[str, object]: - """Fail closed unless Forgejo confirms the resulting PR remains a draft.""" - try: - document = json.loads(value) - except json.JSONDecodeError as exc: - raise PolicyError("Forgejo returned invalid pull-request metadata") from exc - if not isinstance(document, dict) or document.get("draft") is not True: - raise PolicyError( - "Forgejo did not confirm draft state; human review is required" - ) - return document - - -def update_draft( - repo: str, - number: int, - updates: dict[str, str], - *, - token: str, - opener: Callable[..., object] = urllib.request.urlopen, -) -> bytes: - """Update title/body only after Forgejo confirms the PR is still a draft.""" - if not isinstance(number, int) or isinstance(number, bool) or number < 1: - raise PolicyError("pull-request number must be positive") - path = f"/api/v1/repos/{ALLOWED_OWNER}/{_validate_repo(repo)}/pulls/{number}" - current_raw = _request("GET", path, None, token=token, opener=opener) - try: - current = json.loads(current_raw) - except json.JSONDecodeError as exc: - raise PolicyError("Forgejo returned invalid pull-request metadata") from exc - if not isinstance(current, dict) or current.get("draft") is not True: - raise PolicyError("pull request is not a draft; human review now owns it") - safe_updates = dict(updates) - if "title" in safe_updates: - safe_updates["title"] = _draft_title(safe_updates["title"]) - result = _request("PATCH", path, safe_updates, token=token, opener=opener) - _require_draft_response(result) - return result - - -def _body(args: argparse.Namespace) -> str: - if args.body_file is None: - return "" - return args.body_file.read_text(encoding="utf-8") - - def parse_args(argv: list[str] | None = None) -> argparse.Namespace: - """Parse the three intentionally narrow client operations.""" + """Parse the intentionally narrow read and create-draft operations.""" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--dry-run", @@ -301,28 +404,17 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: help="validate and describe the request without reading a token or using the network", ) subparsers = parser.add_subparsers(dest="operation", required=True) - - read_parser = subparsers.add_parser( - "read", help="read an Atlas repository API path" - ) + read_parser = subparsers.add_parser("read", help="read allowed Atlas metadata") read_parser.add_argument("path") - create_parser = subparsers.add_parser( "create-draft", help="create a same-repository draft pull request" ) create_parser.add_argument("repo") create_parser.add_argument("--base", required=True) create_parser.add_argument("--head", required=True) + create_parser.add_argument("--head-sha", required=True) create_parser.add_argument("--title", required=True) - create_parser.add_argument("--body-file", type=Path) - - update_parser = subparsers.add_parser( - "update-draft", help="update title/body of an existing draft pull request" - ) - update_parser.add_argument("repo") - update_parser.add_argument("number", type=int) - update_parser.add_argument("--title") - update_parser.add_argument("--body-file", type=Path) + create_parser.add_argument("--body", default="") return parser.parse_args(argv) @@ -330,22 +422,14 @@ def _operation(args: argparse.Namespace) -> tuple[str, str, object | None]: if args.operation == "read": return "GET", args.path, None repo = _validate_repo(args.repo) - if args.operation == "create-draft": - data = { - "base": args.base, - "body": _body(args), - "head": args.head, - "title": _draft_title(args.title), - } - return "POST", f"/api/v1/repos/{ALLOWED_OWNER}/{repo}/pulls", data - if args.number < 1: - raise PolicyError("pull-request number must be positive") - data = {} - if args.title is not None: - data["title"] = _draft_title(args.title) - if args.body_file is not None: - data["body"] = _body(args) - return "PATCH", f"/api/v1/repos/{ALLOWED_OWNER}/{repo}/pulls/{args.number}", data + _validate_sha(args.head_sha) + data = { + "base": _validate_ref(args.base, "base"), + "body": _validate_body(args.body), + "head": _validate_ref(args.head, "head"), + "title": _draft_title(args.title), + } + return "POST", f"/api/v1/repos/{ALLOWED_OWNER}/{repo}/pulls", data def _write_body(body: bytes) -> None: @@ -374,7 +458,6 @@ def main(argv: list[str] | None = None) -> int: "request_fields": sorted(data) if isinstance(data, dict) else [], - "requires_draft_preflight": operation == "update-draft", }, sort_keys=True, ) @@ -383,19 +466,17 @@ def main(argv: list[str] | None = None) -> int: token = read_token() if args.operation == "read": body = read(path, token=token) - elif args.operation == "create-draft": + else: assert isinstance(data, dict) body = create_draft( args.repo, base=str(data["base"]), head=str(data["head"]), + head_sha=args.head_sha, title=str(data["title"]), body=str(data["body"]), token=token, ) - else: - assert isinstance(data, dict) - body = update_draft(args.repo, args.number, data, token=token) _write_body(body) return 0 except urllib.error.HTTPError as exc: @@ -407,8 +488,6 @@ def main(argv: list[str] | None = None) -> int: sys.stderr.buffer.write(b"\n") return 1 except (OSError, PolicyError, ValueError, json.JSONDecodeError): - # Deliberately omit exception details. File contents, HTTP request - # objects, and provider errors can carry credential material. print( "Forgejo request rejected or unavailable; no credential was disclosed", file=sys.stderr, diff --git a/services/hermes/skills/manage-atlas-pull-requests/SKILL.md b/services/hermes/skills/manage-atlas-pull-requests/SKILL.md index 4e73a3cf..32fd1ccf 100644 --- a/services/hermes/skills/manage-atlas-pull-requests/SKILL.md +++ b/services/hermes/skills/manage-atlas-pull-requests/SKILL.md @@ -1,6 +1,6 @@ --- name: manage-atlas-pull-requests -description: Read private Atlas repository and pull-request metadata, create a draft pull request after a tested branch is pushed, or update the title/body of an existing draft through Hermes' least-authority Forgejo client. Use for PR handoff in scm.bstein.dev/atlas repositories. Never use it to merge, approve, close, delete, or force-push. +description: Read bounded private Atlas repository and pull-request metadata or create a verified draft pull request after a tested branch is pushed through Hermes' least-authority Forgejo client. Use for PR handoff in scm.bstein.dev/atlas repositories. Never use it to update, merge, approve, close, delete, or force-push. --- # Manage Atlas pull requests @@ -12,47 +12,43 @@ put it in an argument or environment variable, or replace this client with ## Read repository or PR state -Pass an Atlas repository API path to the read operation: +Pass one allowed Atlas metadata API path to the read operation: ```sh /opt/coordinator/gitea_api.py read /api/v1/repos/atlas/REPO/pulls/NUMBER ``` -Reads outside `scm.bstein.dev/atlas` are rejected. Treat returned state as +The client exposes only repository metadata, PRs and PR evidence, branches, +commits, and commit status. Hooks, Actions secrets/variables, collaborators, +protections, reviews, merge endpoints, releases, and administration are +rejected even as reads. Responses are size-bounded. Treat returned state as evidence, not authorization to change it. ## Create a review handoff Before opening a PR, verify the remote, branch, clean worktree, diff, tests, and -exact pushed commit. Never force-push. Write the PR description to a workspace -file so multiline Markdown is not shell-quoted, then validate without reading a -credential or using the network: +exact pushed commit. Never force-push. Pass that full 40-character pushed head +SHA explicitly. Keep the short body free of credentials and credential-like +text. Validate without reading a credential or using the network: ```sh /opt/coordinator/gitea_api.py --dry-run create-draft REPO \ - --base main --head hermes/TASK --title "Focused change" \ - --body-file /tmp/pr-body.md + --base main --head hermes/TASK --head-sha FULL_PUSHED_SHA \ + --title "Focused change" --body "Tests and review evidence" ``` Run the same command without `--dry-run` only when the branch is review-ready. -The client always creates a draft in the same repository. Report its URL and -exact head commit to Brad. Leave it unmerged for human review. - -## Update an existing draft - -Use only a title, a body file, or both: - -```sh -/opt/coordinator/gitea_api.py update-draft REPO NUMBER \ - --body-file /tmp/pr-body.md -``` - -The client first reads the PR from Forgejo and refuses the update unless it is -still a draft. It never changes head/base, review state, or merge state. +The client accepts success only when Forgejo returns the exact requested +repository, base/head refs, head SHA, title/body, canonical URLs, positive PR +number, and open/unmerged/draft state. Report its URL and exact head commit to +Brad. Leave it unmerged for human review. PR updates are intentionally absent; +publish a corrected tested commit and ask Brad how to proceed if metadata must +change. ## Stop at the authority boundary -Merge, approve, close, delete, branch mutation, comments, releases, repository -administration, and force-push are intentionally unavailable. Do not bypass the -client with a raw HTTP request. If one of those actions is needed, present the -tested commit and evidence to Brad and stop for human review. +Update, merge, approve, close, delete, branch mutation, comments, releases, +repository administration, and force-push are intentionally unavailable. Do +not bypass the client with a raw HTTP request. If one of those actions is +needed, present the tested commit and evidence to Brad and stop for human +review. diff --git a/testing/tests/test_hermes_chat_quality.py b/testing/tests/test_hermes_chat_quality.py index 09f6b00c..025f3bf7 100644 --- a/testing/tests/test_hermes_chat_quality.py +++ b/testing/tests/test_hermes_chat_quality.py @@ -102,8 +102,12 @@ def test_agent_config_keeps_delegated_reviewers_from_owning_task_lifecycle(): assert "`scm.bstein.dev` is Forgejo/Gitea, not GitHub" in instructions assert "GitHub/`gh` skill for an Atlas remote" in instructions assert "load\n`$manage-atlas-pull-requests`" in instructions - assert "Merge, approve, close, delete, comments" in instructions - assert "Leave every PR unmerged for Brad's review" in instructions + assert "Updates, merge,\napprove, close, delete, comments" in instructions + assert "Leave every PR\nunmerged for Brad's review" in instructions + assert "Use the Gitea pull-request merge endpoint" not in instructions + assert "reconcile the open PR" not in instructions + assert "use Flux reconciliation after" not in instructions + assert "PR publication ends at the verified open draft" in instructions assert "JENKINS_BASE_URL" in instructions assert "Do not\nuse `git reset --hard`" in instructions rendered = (HERMES / "agent-deployment.yaml").read_text() diff --git a/testing/tests/test_hermes_gitea_pr_client.py b/testing/tests/test_hermes_gitea_pr_client.py index 658af189..91475bcf 100644 --- a/testing/tests/test_hermes_gitea_pr_client.py +++ b/testing/tests/test_hermes_gitea_pr_client.py @@ -2,18 +2,18 @@ from __future__ import annotations +import copy import importlib.util -import io import json import sys -import urllib.error +import urllib.request from pathlib import Path import pytest -import yaml ROOT = Path(__file__).parents[2] CLIENT_PATH = ROOT / "services/hermes/scripts/gitea_api.py" +HEAD_SHA = "465cf9146b05c174a2a8d310aff6c64be58277b6" def _load(): @@ -36,6 +36,27 @@ def _draft_payload(**updates): return payload +def _draft_response(**updates): + response = { + "number": 3, + "state": "open", + "draft": True, + "merged": False, + "html_url": "https://scm.bstein.dev/atlas/cassandra/pulls/3", + "url": "https://scm.bstein.dev/api/v1/repos/atlas/cassandra/pulls/3", + "title": "WIP: Focused fix", + "body": "Review evidence", + "base": {"ref": "main", "repo": {"full_name": "atlas/cassandra"}}, + "head": { + "ref": "hermes/fix", + "sha": HEAD_SHA, + "repo": {"full_name": "atlas/cassandra"}, + }, + } + response.update(updates) + return response + + class Response: def __init__(self, body: object): self.body = body if isinstance(body, bytes) else json.dumps(body).encode() @@ -46,8 +67,32 @@ class Response: def __exit__(self, *_args): return False - def read(self): - return self.body + def read(self, limit=-1): + return self.body if limit < 0 else self.body[:limit] + + +def test_redirect_handler_rejects_cross_origin_with_sentinel_authorization(): + client = _load() + source = urllib.request.Request( + "https://scm.bstein.dev/api/v1/repos/atlas/cassandra", + headers={"Authorization": "token redirect-sentinel"}, + ) + + with pytest.raises(client.PolicyError, match="redirects are not allowed") as exc: + client.RejectRedirectHandler().redirect_request( + source, + None, + 302, + "Found", + {}, + "https://evil.example/collect", + ) + + assert "redirect-sentinel" not in str(exc.value) + assert any( + isinstance(handler, client.RejectRedirectHandler) + for handler in client._SAFE_OPENER.handlers + ) @pytest.mark.parametrize( @@ -69,6 +114,71 @@ def test_host_owner_and_path_escape_attempts_are_rejected(base_url: str, path: s client.build_request("GET", path, base_url=base_url, token="secret") +@pytest.mark.parametrize( + "path", + [ + "/api/v1/repos/atlas/cassandra", + "/api/v1/repos/atlas/cassandra/pulls?state=open&limit=20&page=1", + "/api/v1/repos/atlas/cassandra/pulls/7", + "/api/v1/repos/atlas/cassandra/pulls/7/commits?limit=20", + "/api/v1/repos/atlas/cassandra/pulls/7/files?page=1", + "/api/v1/repos/atlas/cassandra/branches", + "/api/v1/repos/atlas/cassandra/branches/main", + "/api/v1/repos/atlas/cassandra/commits?limit=10", + f"/api/v1/repos/atlas/cassandra/git/commits/{HEAD_SHA}", + f"/api/v1/repos/atlas/cassandra/commits/{HEAD_SHA}/status", + f"/api/v1/repos/atlas/cassandra/commits/{HEAD_SHA}/statuses?limit=10", + f"/api/v1/repos/atlas/cassandra/statuses/{HEAD_SHA}?page=1", + ], +) +def test_explicit_read_allowlist_accepts_only_engineering_metadata(path: str): + client = _load() + + request = client.build_request( + "GET", path, base_url=client.CANONICAL_BASE_URL, token="runtime" + ) + assert request.method == "GET" + + +@pytest.mark.parametrize( + "path", + [ + "/api/v1/repos/atlas/cassandra/hooks", + "/api/v1/repos/atlas/cassandra/actions/secrets", + "/api/v1/repos/atlas/cassandra/actions/variables", + "/api/v1/repos/atlas/cassandra/collaborators", + "/api/v1/repos/atlas/cassandra/branch_protections", + "/api/v1/repos/atlas/cassandra/keys", + "/api/v1/repos/atlas/cassandra/pulls/7/reviews", + "/api/v1/repos/atlas/cassandra/pulls/7/merge", + "/api/v1/repos/atlas/cassandra/pulls/7.diff", + "/api/v1/repos/atlas/cassandra/releases", + ], +) +def test_privileged_or_content_routes_are_denied_even_for_get(path: str): + client = _load() + + with pytest.raises(client.PolicyError, match="outside the metadata read allowlist"): + client.authorize_request("GET", path, None) + + +@pytest.mark.parametrize( + "path", + [ + "/api/v1/repos/atlas/cassandra/pulls?limit=51", + "/api/v1/repos/atlas/cassandra/pulls?state=merged", + "/api/v1/repos/atlas/cassandra/pulls?private=true", + "/api/v1/repos/atlas/cassandra/pulls?limit=1&limit=2", + "/api/v1/repos/atlas/cassandra?p=1", + ], +) +def test_read_query_is_bounded(path: str): + client = _load() + + with pytest.raises(client.PolicyError): + client.authorize_request("GET", path, None) + + @pytest.mark.parametrize( ("method", "path", "data"), [ @@ -79,12 +189,11 @@ def test_host_owner_and_path_escape_attempts_are_rejected(base_url: str, path: s "/api/v1/repos/atlas/cassandra/pulls/4/reviews", {"event": "APPROVED"}, ), - ("PATCH", "/api/v1/repos/atlas/cassandra/pulls/4", {"state": "closed"}), - ("PATCH", "/api/v1/repos/atlas/cassandra/pulls/4", {"draft": False}), + ("PATCH", "/api/v1/repos/atlas/cassandra/pulls/4", {"title": "WIP: x"}), ("PUT", "/api/v1/repos/atlas/cassandra/branches/main", {}), ], ) -def test_merge_approve_close_delete_and_other_mutations_are_rejected( +def test_merge_approve_close_delete_update_and_other_mutations_are_rejected( method: str, path: str, data: object ): client = _load() @@ -95,6 +204,36 @@ def test_merge_approve_close_delete_and_other_mutations_are_rejected( ) +@pytest.mark.parametrize( + "ref", + [ + "foo/.bar", + "foo/bar.lock/baz", + "foo..bar", + "foo@{bar", + "foo//bar", + "-danger", + "danger.", + "danger~one", + "danger^one", + "danger:one", + "danger one", + ], +) +def test_complete_git_ref_validation_rejects_invalid_names(ref: str): + client = _load() + + with pytest.raises(client.PolicyError): + client._validate_ref(ref, "head") + + +def test_git_ref_validation_uses_fixed_trusted_binary(): + client = _load() + + assert client.GIT_BIN == "/usr/bin/git" + assert client._validate_ref("hermes/valid-fix", "head") == "hermes/valid-fix" + + def test_create_forces_draft_title_and_same_repository_branch_names(): client = _load() assert ( @@ -103,7 +242,6 @@ def test_create_forces_draft_title_and_same_repository_branch_names(): ) == "create-draft" ) - with pytest.raises(client.PolicyError, match="draft-title prefix"): client.authorize_request( "POST", @@ -118,44 +256,6 @@ def test_create_forces_draft_title_and_same_repository_branch_names(): ) -def test_update_checks_server_draft_state_before_patching(): - client = _load() - calls = [] - responses = iter( - [ - Response({"number": 7, "draft": True}), - Response({"number": 7, "draft": True}), - ] - ) - - def opener(request, timeout): - calls.append((request.method, request.full_url, request.data, timeout)) - return next(responses) - - result = client.update_draft( - "cassandra", 7, {"title": "Narrowed repair"}, token="runtime", opener=opener - ) - - assert json.loads(result) == {"number": 7, "draft": True} - assert [call[0] for call in calls] == ["GET", "PATCH"] - assert json.loads(calls[1][2]) == {"title": "WIP: Narrowed repair"} - - -def test_update_rejects_non_draft_without_sending_patch(): - client = _load() - calls = [] - - def opener(request, timeout): - calls.append((request.method, timeout)) - return Response({"number": 7, "draft": False}) - - with pytest.raises(client.PolicyError, match="human review now owns"): - client.update_draft( - "cassandra", 7, {"body": "new"}, token="runtime", opener=opener - ) - assert calls == [("GET", 30)] - - def test_runtime_token_is_only_an_authorization_header(): client = _load() request = client.build_request( @@ -171,41 +271,128 @@ def test_runtime_token_is_only_an_authorization_header(): assert request.get_header("Authorization") == "token do-not-leak" -def test_create_uses_live_gitea_schema_and_verifies_draft_response(): +@pytest.mark.parametrize( + "body", + [ + "pass" + "word=not-a-real-credential", + "Authorization: " + "Bearer not-a-real-credential-value", + "token: " + "ghp_" + "notarealcredentialvalue123456", + "-----BEGIN OPENSSH " + "PRIVATE KEY-----", + "eyJnotarealheader." + "notarealpayloadvalue." + "notarealsignature", + ], +) +def test_body_rejects_common_credential_shapes(body: str): + client = _load() + + with pytest.raises(client.PolicyError, match="credential material"): + client._validate_body(body) + + +def test_create_rejects_exact_runtime_token_before_network(): + client = _load() + called = False + + def opener(*_args, **_kwargs): + nonlocal called + called = True + return Response(_draft_response()) + + with pytest.raises(client.PolicyError, match="runtime credential"): + client.create_draft( + "cassandra", + base="main", + head="hermes/fix", + head_sha=HEAD_SHA, + title="Focused fix", + body="accidental runtime-sentinel value", + token="runtime-sentinel", + opener=opener, + ) + assert called is False + + +def test_create_verifies_every_server_postcondition(): client = _load() calls = [] def opener(request, timeout): - calls.append((json.loads(request.data), timeout)) - return Response({"number": 3, "draft": True}) + calls.append((request, timeout)) + return Response(_draft_response()) result = client.create_draft( "cassandra", base="main", head="hermes/fix", + head_sha=HEAD_SHA, title="Focused fix", - body="Evidence", + body="Review evidence", token="runtime", opener=opener, ) - assert json.loads(result)["draft"] is True - assert calls[0][0]["title"] == "WIP: Focused fix" - assert "draft" not in calls[0][0] + assert json.loads(result) == _draft_response() + payload = json.loads(calls[0][0].data) + assert payload == { + "base": "main", + "body": "Review evidence", + "head": "hermes/fix", + "title": "WIP: Focused fix", + } + assert calls[0][1] == 30 -def test_create_fails_closed_when_server_does_not_confirm_draft(): +def test_create_postcondition_rejects_every_material_mismatch(): + client = _load() + mutations = [ + ("number", 0), + ("state", "closed"), + ("draft", False), + ("merged", True), + ("html_url", "https://evil.example/pulls/3"), + ("url", "https://evil.example/api/pulls/3"), + ("title", "Focused fix"), + ("body", "different"), + ] + documents = [] + for key, value in mutations: + document = _draft_response() + document[key] = value + documents.append(document) + for path, value in [ + (("base", "ref"), "master"), + (("base", "repo", "full_name"), "evil/cassandra"), + (("head", "ref"), "other"), + (("head", "sha"), "0" * 40), + (("head", "repo", "full_name"), "evil/cassandra"), + ]: + document = copy.deepcopy(_draft_response()) + target = document + for key in path[:-1]: + target = target[key] + target[path[-1]] = value + documents.append(document) + + for document in documents: + with pytest.raises(client.PolicyError): + client._require_create_response( + json.dumps(document).encode(), + repo="cassandra", + base="main", + head="hermes/fix", + head_sha=HEAD_SHA, + title="WIP: Focused fix", + body="Review evidence", + ) + + +def test_read_response_is_bounded(): client = _load() - with pytest.raises(client.PolicyError, match="did not confirm draft"): - client.create_draft( - "cassandra", - base="main", - head="hermes/fix", - title="Focused fix", - body="Evidence", + with pytest.raises(client.PolicyError, match="safe size limit"): + client.read( + "/api/v1/repos/atlas/cassandra", token="runtime", - opener=lambda *_a, **_k: Response({"number": 3, "draft": False}), + opener=lambda *_a, **_k: Response(b"x" * (client.MAX_RESPONSE_BYTES + 1)), ) @@ -216,107 +403,3 @@ def test_output_redaction_covers_exact_token_and_authorization_header(): assert b"do-not-leak" not in redacted assert redacted.count(b"[REDACTED]") >= 1 - - -def test_dry_run_does_not_read_token_or_use_network( - tmp_path: Path, monkeypatch, capsys -): - client = _load() - body = tmp_path / "body.md" - body.write_text("Evidence only\n", encoding="utf-8") - monkeypatch.setattr(client, "read_token", lambda: pytest.fail("read token")) - monkeypatch.setattr( - client.urllib.request, "urlopen", lambda *_a, **_k: pytest.fail("network") - ) - - assert ( - client.main( - [ - "--dry-run", - "create-draft", - "cassandra", - "--base", - "main", - "--head", - "hermes/fix", - "--title", - "Repair", - "--body-file", - str(body), - ] - ) - == 0 - ) - output = json.loads(capsys.readouterr().out) - assert output["operation"] == "create-draft" - assert output["owner"] == "atlas" - assert "Evidence only" not in json.dumps(output) - - -def test_http_error_path_redacts_token(monkeypatch, capsys): - client = _load() - monkeypatch.setattr(client, "read_token", lambda: "do-not-leak") - - def fail(*_args, **_kwargs): - raise urllib.error.HTTPError( - "https://scm.bstein.dev/api/v1/repos/atlas/cassandra", - 403, - "forbidden", - {}, - io.BytesIO(b"Authorization: token do-not-leak"), - ) - - monkeypatch.setattr(client, "read", lambda *_a, **_k: fail()) - - assert client.main(["read", "/api/v1/repos/atlas/cassandra"]) == 1 - captured = capsys.readouterr() - assert "do-not-leak" not in captured.err - assert "HTTP 403" in captured.err - - -def test_flux_manifest_projects_runtime_vault_token_and_skill_only(): - client_source = CLIENT_PATH.read_text(encoding="utf-8") - assert "/runtime-access/gitea-token" in client_source - assert "GITEA_TOKEN" not in client_source - - deployment = yaml.safe_load( - (ROOT / "services/hermes/agent-deployment.yaml").read_text(encoding="utf-8") - ) - template = deployment["spec"]["template"] - annotations = template["metadata"]["annotations"] - assert annotations["vault.hashicorp.com/agent-inject-secret-gitea-token"] == ( - "kv/data/atlas/hermes/developer-gitea" - ) - runtime = next( - volume - for volume in template["spec"]["volumes"] - if volume["name"] == "runtime-access" - ) - assert runtime["emptyDir"]["medium"] == "Memory" - - expected = {"hermes", "terminal", "cli-lane-runner"} - mounted = { - container["name"] - for container in template["spec"]["containers"] - if any( - mount["name"] == "atlas-pr-skill" - and mount["mountPath"] - == "/opt/data/workspace/skills/manage-atlas-pull-requests" - and mount.get("readOnly") is True - for mount in container.get("volumeMounts", []) - ) - } - assert mounted == expected - - kustomization = yaml.safe_load( - (ROOT / "services/hermes/kustomization.yaml").read_text(encoding="utf-8") - ) - generator = next( - item - for item in kustomization["configMapGenerator"] - if item["name"] == "hermes-atlas-pr-skill" - ) - assert generator["files"] == [ - "SKILL.md=skills/manage-atlas-pull-requests/SKILL.md", - "openai.yaml=skills/manage-atlas-pull-requests/agents/openai.yaml", - ] diff --git a/testing/tests/test_hermes_gitea_pr_integration.py b/testing/tests/test_hermes_gitea_pr_integration.py new file mode 100644 index 00000000..d3b47eef --- /dev/null +++ b/testing/tests/test_hermes_gitea_pr_integration.py @@ -0,0 +1,147 @@ +"""Runtime and Flux integration contracts for the safe Atlas PR client.""" + +from __future__ import annotations + +import importlib.util +import io +import json +import sys +import urllib.error +from pathlib import Path + +import pytest +import yaml + +ROOT = Path(__file__).parents[2] +CLIENT_PATH = ROOT / "services/hermes/scripts/gitea_api.py" +HEAD_SHA = "465cf9146b05c174a2a8d310aff6c64be58277b6" + + +def _load(): + spec = importlib.util.spec_from_file_location( + "safe_gitea_api_integration", CLIENT_PATH + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def test_dry_run_has_no_file_input_and_uses_no_token_or_network(monkeypatch, capsys): + client = _load() + monkeypatch.setattr(client, "read_token", lambda: pytest.fail("read token")) + monkeypatch.setattr( + client, "_safe_urlopen", lambda *_a, **_k: pytest.fail("network") + ) + + assert ( + client.main( + [ + "--dry-run", + "create-draft", + "cassandra", + "--base", + "main", + "--head", + "hermes/fix", + "--head-sha", + HEAD_SHA, + "--title", + "Repair", + "--body", + "Evidence only", + ] + ) + == 0 + ) + output = json.loads(capsys.readouterr().out) + assert output["operation"] == "create-draft" + assert output["owner"] == "atlas" + assert "Evidence only" not in json.dumps(output) + with pytest.raises(SystemExit): + client.parse_args( + [ + "create-draft", + "cassandra", + "--base", + "main", + "--head", + "hermes/fix", + "--head-sha", + HEAD_SHA, + "--title", + "Repair", + "--body-file", + "/runtime-access/gitea-token", + ] + ) + + +def test_http_error_path_redacts_token(monkeypatch, capsys): + client = _load() + monkeypatch.setattr(client, "read_token", lambda: "do-not-leak") + + def fail(*_args, **_kwargs): + raise urllib.error.HTTPError( + "https://scm.bstein.dev/api/v1/repos/atlas/cassandra", + 403, + "forbidden", + {}, + io.BytesIO(b"Authorization: token do-not-leak"), + ) + + monkeypatch.setattr(client, "read", lambda *_a, **_k: fail()) + + assert client.main(["read", "/api/v1/repos/atlas/cassandra"]) == 1 + captured = capsys.readouterr() + assert "do-not-leak" not in captured.err + assert "HTTP 403" in captured.err + + +def test_flux_manifest_projects_runtime_vault_token_and_skill_only(): + client_source = CLIENT_PATH.read_text(encoding="utf-8") + assert "/runtime-access/gitea-token" in client_source + assert "GITEA_TOKEN" not in client_source + + deployment = yaml.safe_load( + (ROOT / "services/hermes/agent-deployment.yaml").read_text(encoding="utf-8") + ) + template = deployment["spec"]["template"] + annotations = template["metadata"]["annotations"] + assert annotations["vault.hashicorp.com/agent-inject-secret-gitea-token"] == ( + "kv/data/atlas/hermes/developer-gitea" + ) + runtime = next( + volume + for volume in template["spec"]["volumes"] + if volume["name"] == "runtime-access" + ) + assert runtime["emptyDir"]["medium"] == "Memory" + + expected = {"hermes", "terminal", "cli-lane-runner"} + mounted = { + container["name"] + for container in template["spec"]["containers"] + if any( + mount["name"] == "atlas-pr-skill" + and mount["mountPath"] + == "/opt/data/workspace/skills/manage-atlas-pull-requests" + and mount.get("readOnly") is True + for mount in container.get("volumeMounts", []) + ) + } + assert mounted == expected + + kustomization = yaml.safe_load( + (ROOT / "services/hermes/kustomization.yaml").read_text(encoding="utf-8") + ) + generator = next( + item + for item in kustomization["configMapGenerator"] + if item["name"] == "hermes-atlas-pr-skill" + ) + assert generator["files"] == [ + "SKILL.md=skills/manage-atlas-pull-requests/SKILL.md", + "openai.yaml=skills/manage-atlas-pull-requests/agents/openai.yaml", + ] -- 2.47.2 From 2c1d97e551e5b61a3aa1e821e412ef4948bbcc8d Mon Sep 17 00:00:00 2001 From: jenkins Date: Sun, 16 Aug 2026 20:21:04 -0300 Subject: [PATCH 03/12] hermes: match canonical Forgejo PR URL --- services/hermes/scripts/gitea_api.py | 3 +-- testing/tests/test_hermes_gitea_pr_client.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/services/hermes/scripts/gitea_api.py b/services/hermes/scripts/gitea_api.py index 057a79a1..b4a98eca 100755 --- a/services/hermes/scripts/gitea_api.py +++ b/services/hermes/scripts/gitea_api.py @@ -337,13 +337,12 @@ def _require_create_response( raise PolicyError("Forgejo omitted a valid pull-request number") full_name = f"{ALLOWED_OWNER}/{repo}" expected_html = f"{CANONICAL_BASE_URL}/{full_name}/pulls/{number}" - expected_api = f"{CANONICAL_BASE_URL}/api/v1/repos/{full_name}/pulls/{number}" checks = ( document.get("state") == "open", document.get("draft") is True, document.get("merged") is False, document.get("html_url") == expected_html, - document.get("url") == expected_api, + document.get("url") == expected_html, document.get("title") == title, document.get("body") == body, _nested(document, "base", "ref") == base, diff --git a/testing/tests/test_hermes_gitea_pr_client.py b/testing/tests/test_hermes_gitea_pr_client.py index 91475bcf..447f3067 100644 --- a/testing/tests/test_hermes_gitea_pr_client.py +++ b/testing/tests/test_hermes_gitea_pr_client.py @@ -43,7 +43,7 @@ def _draft_response(**updates): "draft": True, "merged": False, "html_url": "https://scm.bstein.dev/atlas/cassandra/pulls/3", - "url": "https://scm.bstein.dev/api/v1/repos/atlas/cassandra/pulls/3", + "url": "https://scm.bstein.dev/atlas/cassandra/pulls/3", "title": "WIP: Focused fix", "body": "Review evidence", "base": {"ref": "main", "repo": {"full_name": "atlas/cassandra"}}, -- 2.47.2 From 8c6e3acdacb223f68b1716f0a74bd543e18890e6 Mon Sep 17 00:00:00 2001 From: jenkins Date: Sun, 16 Aug 2026 20:41:14 -0300 Subject: [PATCH 04/12] hermes: isolate Atlas SCM write authority --- .../kustomization.yaml | 19 + .../hermes-observer-rbac/kustomization.yaml | 16 + .../hermes-scm-broker-code/kustomization.yaml | 19 + .../hermes-scm-broker/kustomization.yaml | 27 + .../hermes-scm-namespace/kustomization.yaml | 16 + .../applications/hermes/kustomization.yaml | 2 + .../applications/kustomization.yaml | 5 + scripts/tests/test_dashboards_render_atlas.py | 98 - ...test_dashboards_render_atlas_drilldowns.py | 127 ++ .../gitea/atlas-identity-bootstrap-job.yaml | 20 +- services/gitea/kustomization.yaml | 1 + .../scripts/gitea_atlas_identity_ensure.sh | 45 + .../scripts/gitea_branch_protection_check.py | 207 ++ .../kustomization.yaml | 5 + .../rolebindings.yaml | 204 ++ .../hermes-observer-rbac/kustomization.yaml | 5 + services/hermes-observer-rbac/rbac.yaml | 112 + services/hermes-scm-broker/deployment.yaml | 98 + services/hermes-scm-broker/kustomization.yaml | 11 + services/hermes-scm-broker/networkpolicy.yaml | 46 + services/hermes-scm-broker/service.yaml | 13 + .../hermes-scm-namespace/kustomization.yaml | 5 + services/hermes-scm-namespace/namespace.yaml | 14 + services/hermes/agent-configmap.yaml | 74 +- services/hermes/agent-deployment.yaml | 20 +- services/hermes/agent-kubeconfig.yaml | 4 +- services/hermes/agent-rbac.yaml | 16 - services/hermes/kustomization.yaml | 12 +- services/hermes/networkpolicy.yaml | 62 +- services/hermes/node-ssh-access.yaml | 77 +- services/hermes/scm-common/kustomization.yaml | 14 + .../{ => scm-common}/scripts/gitea_api.py | 284 ++- .../scm-common/scripts/gitea_api_policy.py | 490 +++++ .../hermes/scm-common/scripts/scm_broker.py | 496 +++++ .../scm-common/scripts/scm_broker_client.py | 95 + .../scm-common/scripts/scm_broker_io.py | 72 + .../scm-common/scripts/scm_broker_server.py | 118 ++ services/hermes/scripts/gitea_askpass.sh | 19 - services/hermes/scripts/hermes_coordinator.py | 32 +- services/hermes/scripts/node_account_audit.py | 178 ++ .../hermes/scripts/node_account_hardening.py | 496 +++++ services/hermes/scripts/node_account_io.py | 205 ++ .../hermes/scripts/stage_runtime_access.py | 19 +- .../manage-atlas-pull-requests/SKILL.md | 24 +- .../vault/hermes-auth-role-bootstrap-job.yaml | 2 +- .../vault/scripts/vault_k8s_auth_configure.sh | 4 +- testing/quality_contract.json | 57 + testing/quality_coverage.py | 61 +- testing/tests/test_hermes_chat_images.py | 282 +++ .../tests/test_hermes_chat_local_runtime.py | 172 ++ .../tests/test_hermes_chat_provider_auth.py | 476 +++++ testing/tests/test_hermes_chat_quality.py | 1812 +---------------- .../tests/test_hermes_chat_routing_runtime.py | 486 +++++ .../test_hermes_chat_session_continuity.py | 478 +++++ testing/tests/test_hermes_chat_support.py | 38 + testing/tests/test_hermes_cli_lanes.py | 1591 +-------------- testing/tests/test_hermes_cli_lanes_access.py | 378 ++++ .../test_hermes_cli_lanes_configuration.py | 444 ++++ testing/tests/test_hermes_cli_lanes_kanban.py | 392 ++++ .../tests/test_hermes_cli_lanes_support.py | 79 + .../tests/test_hermes_cli_lanes_toolchain.py | 386 ++++ testing/tests/test_hermes_coordinator.py | 212 +- .../test_hermes_coordinator_cassandra.py | 194 ++ .../tests/test_hermes_coordinator_coverage.py | 267 +++ .../tests/test_hermes_coordinator_support.py | 35 + .../test_hermes_gitea_branch_coverage.py | 183 ++ .../test_hermes_gitea_internal_coverage.py | 375 ++++ testing/tests/test_hermes_gitea_pr_client.py | 398 ++-- .../tests/test_hermes_gitea_pr_creation.py | 203 ++ testing/tests/test_hermes_gitea_pr_dlp.py | 317 +++ .../tests/test_hermes_gitea_pr_integration.py | 41 +- .../test_hermes_gitea_pr_postconditions.py | 199 ++ testing/tests/test_hermes_gitea_support.py | 75 + .../test_hermes_node_account_hardening.py | 252 +++ testing/tests/test_hermes_node_account_io.py | 222 ++ ...est_hermes_node_account_privilege_audit.py | 196 ++ .../tests/test_hermes_node_account_support.py | 90 + .../tests/test_hermes_node_acl_coverage.py | 206 ++ .../tests/test_hermes_node_audit_coverage.py | 221 ++ .../test_hermes_node_hardening_coverage.py | 211 ++ testing/tests/test_hermes_node_io_coverage.py | 189 ++ testing/tests/test_hermes_runtime_access.py | 24 +- .../test_hermes_runtime_stage_coverage.py | 137 ++ testing/tests/test_hermes_scm_broker.py | 146 ++ .../test_hermes_scm_broker_client_coverage.py | 157 ++ ...test_hermes_scm_broker_handler_coverage.py | 265 +++ ...est_hermes_scm_broker_internal_coverage.py | 263 +++ .../tests/test_hermes_scm_broker_policy.py | 321 +++ .../tests/test_hermes_scm_broker_streaming.py | 237 +++ .../tests/test_hermes_scm_broker_support.py | 55 + .../tests/test_hermes_scm_server_coverage.py | 180 ++ .../tests/test_quality_coverage_helpers.py | 87 +- 92 files changed, 12731 insertions(+), 4287 deletions(-) create mode 100644 clusters/atlas/flux-system/applications/hermes-observer-bindings/kustomization.yaml create mode 100644 clusters/atlas/flux-system/applications/hermes-observer-rbac/kustomization.yaml create mode 100644 clusters/atlas/flux-system/applications/hermes-scm-broker-code/kustomization.yaml create mode 100644 clusters/atlas/flux-system/applications/hermes-scm-broker/kustomization.yaml create mode 100644 clusters/atlas/flux-system/applications/hermes-scm-namespace/kustomization.yaml create mode 100644 scripts/tests/test_dashboards_render_atlas_drilldowns.py create mode 100644 services/gitea/scripts/gitea_branch_protection_check.py create mode 100644 services/hermes-observer-bindings/kustomization.yaml create mode 100644 services/hermes-observer-bindings/rolebindings.yaml create mode 100644 services/hermes-observer-rbac/kustomization.yaml create mode 100644 services/hermes-observer-rbac/rbac.yaml create mode 100644 services/hermes-scm-broker/deployment.yaml create mode 100644 services/hermes-scm-broker/kustomization.yaml create mode 100644 services/hermes-scm-broker/networkpolicy.yaml create mode 100644 services/hermes-scm-broker/service.yaml create mode 100644 services/hermes-scm-namespace/kustomization.yaml create mode 100644 services/hermes-scm-namespace/namespace.yaml delete mode 100644 services/hermes/agent-rbac.yaml create mode 100644 services/hermes/scm-common/kustomization.yaml rename services/hermes/{ => scm-common}/scripts/gitea_api.py (65%) mode change 100755 => 100644 create mode 100644 services/hermes/scm-common/scripts/gitea_api_policy.py create mode 100644 services/hermes/scm-common/scripts/scm_broker.py create mode 100644 services/hermes/scm-common/scripts/scm_broker_client.py create mode 100644 services/hermes/scm-common/scripts/scm_broker_io.py create mode 100644 services/hermes/scm-common/scripts/scm_broker_server.py delete mode 100755 services/hermes/scripts/gitea_askpass.sh create mode 100644 services/hermes/scripts/node_account_audit.py create mode 100644 services/hermes/scripts/node_account_hardening.py create mode 100644 services/hermes/scripts/node_account_io.py create mode 100644 testing/tests/test_hermes_chat_images.py create mode 100644 testing/tests/test_hermes_chat_local_runtime.py create mode 100644 testing/tests/test_hermes_chat_provider_auth.py create mode 100644 testing/tests/test_hermes_chat_routing_runtime.py create mode 100644 testing/tests/test_hermes_chat_session_continuity.py create mode 100644 testing/tests/test_hermes_chat_support.py create mode 100644 testing/tests/test_hermes_cli_lanes_access.py create mode 100644 testing/tests/test_hermes_cli_lanes_configuration.py create mode 100644 testing/tests/test_hermes_cli_lanes_kanban.py create mode 100644 testing/tests/test_hermes_cli_lanes_support.py create mode 100644 testing/tests/test_hermes_cli_lanes_toolchain.py create mode 100644 testing/tests/test_hermes_coordinator_cassandra.py create mode 100644 testing/tests/test_hermes_coordinator_coverage.py create mode 100644 testing/tests/test_hermes_coordinator_support.py create mode 100644 testing/tests/test_hermes_gitea_branch_coverage.py create mode 100644 testing/tests/test_hermes_gitea_internal_coverage.py create mode 100644 testing/tests/test_hermes_gitea_pr_creation.py create mode 100644 testing/tests/test_hermes_gitea_pr_dlp.py create mode 100644 testing/tests/test_hermes_gitea_pr_postconditions.py create mode 100644 testing/tests/test_hermes_gitea_support.py create mode 100644 testing/tests/test_hermes_node_account_hardening.py create mode 100644 testing/tests/test_hermes_node_account_io.py create mode 100644 testing/tests/test_hermes_node_account_privilege_audit.py create mode 100644 testing/tests/test_hermes_node_account_support.py create mode 100644 testing/tests/test_hermes_node_acl_coverage.py create mode 100644 testing/tests/test_hermes_node_audit_coverage.py create mode 100644 testing/tests/test_hermes_node_hardening_coverage.py create mode 100644 testing/tests/test_hermes_node_io_coverage.py create mode 100644 testing/tests/test_hermes_runtime_stage_coverage.py create mode 100644 testing/tests/test_hermes_scm_broker.py create mode 100644 testing/tests/test_hermes_scm_broker_client_coverage.py create mode 100644 testing/tests/test_hermes_scm_broker_handler_coverage.py create mode 100644 testing/tests/test_hermes_scm_broker_internal_coverage.py create mode 100644 testing/tests/test_hermes_scm_broker_policy.py create mode 100644 testing/tests/test_hermes_scm_broker_streaming.py create mode 100644 testing/tests/test_hermes_scm_broker_support.py create mode 100644 testing/tests/test_hermes_scm_server_coverage.py diff --git a/clusters/atlas/flux-system/applications/hermes-observer-bindings/kustomization.yaml b/clusters/atlas/flux-system/applications/hermes-observer-bindings/kustomization.yaml new file mode 100644 index 00000000..b64913d5 --- /dev/null +++ b/clusters/atlas/flux-system/applications/hermes-observer-bindings/kustomization.yaml @@ -0,0 +1,19 @@ +# clusters/atlas/flux-system/applications/hermes-observer-bindings/kustomization.yaml +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: hermes-observer-bindings + namespace: flux-system +spec: + interval: 10m + path: ./services/hermes-observer-bindings + prune: true + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + wait: true + timeout: 5m + dependsOn: + - name: hermes + - name: hermes-observer-rbac diff --git a/clusters/atlas/flux-system/applications/hermes-observer-rbac/kustomization.yaml b/clusters/atlas/flux-system/applications/hermes-observer-rbac/kustomization.yaml new file mode 100644 index 00000000..c2749739 --- /dev/null +++ b/clusters/atlas/flux-system/applications/hermes-observer-rbac/kustomization.yaml @@ -0,0 +1,16 @@ +# clusters/atlas/flux-system/applications/hermes-observer-rbac/kustomization.yaml +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: hermes-observer-rbac + namespace: flux-system +spec: + interval: 10m + path: ./services/hermes-observer-rbac + prune: true + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + wait: true + timeout: 5m diff --git a/clusters/atlas/flux-system/applications/hermes-scm-broker-code/kustomization.yaml b/clusters/atlas/flux-system/applications/hermes-scm-broker-code/kustomization.yaml new file mode 100644 index 00000000..20ae82e2 --- /dev/null +++ b/clusters/atlas/flux-system/applications/hermes-scm-broker-code/kustomization.yaml @@ -0,0 +1,19 @@ +# clusters/atlas/flux-system/applications/hermes-scm-broker-code/kustomization.yaml +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: hermes-scm-broker-code + namespace: flux-system +spec: + interval: 10m + path: ./services/hermes/scm-common + targetNamespace: hermes-scm + prune: true + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + wait: true + timeout: 5m + dependsOn: + - name: hermes-scm-namespace diff --git a/clusters/atlas/flux-system/applications/hermes-scm-broker/kustomization.yaml b/clusters/atlas/flux-system/applications/hermes-scm-broker/kustomization.yaml new file mode 100644 index 00000000..70b3989e --- /dev/null +++ b/clusters/atlas/flux-system/applications/hermes-scm-broker/kustomization.yaml @@ -0,0 +1,27 @@ +# clusters/atlas/flux-system/applications/hermes-scm-broker/kustomization.yaml +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: hermes-scm-broker + namespace: flux-system +spec: + interval: 10m + path: ./services/hermes-scm-broker + targetNamespace: hermes-scm + prune: true + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + wait: true + timeout: 10m + healthChecks: + - apiVersion: apps/v1 + kind: Deployment + name: hermes-scm-broker + namespace: hermes-scm + dependsOn: + - name: vault + - name: gitea + - name: hermes-scm-namespace + - name: hermes-scm-broker-code diff --git a/clusters/atlas/flux-system/applications/hermes-scm-namespace/kustomization.yaml b/clusters/atlas/flux-system/applications/hermes-scm-namespace/kustomization.yaml new file mode 100644 index 00000000..9f066709 --- /dev/null +++ b/clusters/atlas/flux-system/applications/hermes-scm-namespace/kustomization.yaml @@ -0,0 +1,16 @@ +# clusters/atlas/flux-system/applications/hermes-scm-namespace/kustomization.yaml +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: hermes-scm-namespace + namespace: flux-system +spec: + interval: 10m + path: ./services/hermes-scm-namespace + prune: true + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + wait: true + timeout: 5m diff --git a/clusters/atlas/flux-system/applications/hermes/kustomization.yaml b/clusters/atlas/flux-system/applications/hermes/kustomization.yaml index 3949282b..df7a5560 100644 --- a/clusters/atlas/flux-system/applications/hermes/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/hermes/kustomization.yaml @@ -60,3 +60,5 @@ spec: - name: keycloak - name: longhorn - name: vault + - name: hermes-scm-broker + - name: hermes-observer-rbac diff --git a/clusters/atlas/flux-system/applications/kustomization.yaml b/clusters/atlas/flux-system/applications/kustomization.yaml index eb8bc86a..39a859d0 100644 --- a/clusters/atlas/flux-system/applications/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/kustomization.yaml @@ -27,7 +27,12 @@ resources: - jenkins/kustomization.yaml - ai-llm/kustomization.yaml - openclaw/kustomization.yaml + - hermes-scm-namespace/kustomization.yaml + - hermes-scm-broker-code/kustomization.yaml - hermes/kustomization.yaml + - hermes-observer-rbac/kustomization.yaml + - hermes-observer-bindings/kustomization.yaml + - hermes-scm-broker/kustomization.yaml - hermes-chat/kustomization.yaml - hermes-triage-demo/kustomization.yaml - game-stream/kustomization.yaml diff --git a/scripts/tests/test_dashboards_render_atlas.py b/scripts/tests/test_dashboards_render_atlas.py index bba16cdf..a522f849 100644 --- a/scripts/tests/test_dashboards_render_atlas.py +++ b/scripts/tests/test_dashboards_render_atlas.py @@ -416,101 +416,3 @@ def test_jobs_dashboard_bar_gauges_use_solid_threshold_colors(): assert {"color": "dark-yellow", "value": 93} in threshold_steps assert {"color": "dark-blue", "value": 100} in threshold_steps - - -def test_jobs_dashboard_collapses_heavy_drilldowns_for_light_first_paint(): - mod = load_module() - dashboard = mod.build_jobs_dashboard() - panels = dashboard["panels"] - rows = [panel for panel in panels if panel["type"] == "row"] - visible_query_panels = [panel for panel in panels if panel["type"] != "row"] - nested_panels_by_title = { - child["title"]: child - for row in rows - for child in row.get("panels", []) - } - - assert len(panels) == 18 - assert len(visible_query_panels) == 12 - assert sum(len(panel.get("targets", [])) for panel in visible_query_panels) == 12 - assert all( - panel["title"] != "Coverage Gap to 95% by Suite" - for panel in visible_query_panels - ) - assert [row["title"] for row in rows] == [ - "CI Runs And Test Result History", - "Check Failure Rates By Suite", - "Check Healthy Rates By Suite", - "Test Drilldowns And Problem Tests", - "Telemetry Completeness And Branches", - "SonarQube Project Health", - ] - assert all(row["collapsed"] for row in rows) - - assert "Coverage Failure Rate" in nested_panels_by_title - assert "Supply Chain Healthy Rate" in nested_panels_by_title - assert "Test Category Health History" in nested_panels_by_title - assert "Selected Test Pass Rate History" in nested_panels_by_title - assert "Coverage Metrics Present by Suite" in nested_panels_by_title - assert "SonarQube API Up" in nested_panels_by_title - - failure_rate_panel = nested_panels_by_title["Coverage Failure Rate"] - assert failure_rate_panel["type"] == "state-timeline" - assert failure_rate_panel["fieldConfig"]["defaults"]["unit"] == "percent" - assert failure_rate_panel["fieldConfig"]["defaults"]["max"] == 100 - assert failure_rate_panel["fieldConfig"]["defaults"]["thresholds"]["steps"][0]["color"] == "dark-blue" - assert "increase(" not in failure_rate_panel["targets"][0]["expr"] - assert "platform_quality:check_failed_flag:present_1h" in failure_rate_panel["targets"][0]["expr"] - assert "platform_quality:check_seen_flag:present_1h" in failure_rate_panel["targets"][0]["expr"] - assert "platform_quality:check_status:present_1h" not in failure_rate_panel["targets"][0]["expr"] - assert '.*_quality_gate_checks_total' not in failure_rate_panel["targets"][0]["expr"] - assert "last_over_time" not in failure_rate_panel["targets"][0]["expr"] - assert 'label_replace' not in failure_rate_panel["targets"][0]["expr"] - assert "0 *" in failure_rate_panel["targets"][0]["expr"] - assert "and on(suite)" not in failure_rate_panel["targets"][0]["expr"] - - pass_rate_panel = nested_panels_by_title["Selected Test Pass Rate History"] - assert pass_rate_panel["type"] == "state-timeline" - assert "platform_quality:test_case_pass_rate:percent_1h" in pass_rate_panel["targets"][0]["expr"] - assert "platform_quality_gate_test_case_result" not in pass_rate_panel["targets"][0]["expr"] - - pass_fail_panel = nested_panels_by_title["Selected Test Pass/Fail History"] - assert pass_fail_panel["fieldConfig"]["defaults"]["custom"]["drawStyle"] == "bars" - assert all( - "platform_quality:test_case_status:count_1h" in target["expr"] - for target in pass_fail_panel["targets"] - ) - - problematic_panel = nested_panels_by_title["Problematic Tests Over Time (Top failures)"] - assert problematic_panel["type"] == "state-timeline" - assert problematic_panel["gridPos"]["w"] == 24 - assert 'test!=""' in problematic_panel["targets"][0]["expr"] - assert "vector(0)" not in problematic_panel["targets"][0]["expr"] - assert problematic_panel["fieldConfig"]["defaults"]["thresholds"]["steps"] == [ - {"color": "dark-blue", "value": None}, - {"color": "dark-green", "value": 2}, - {"color": "dark-yellow", "value": 3}, - {"color": "dark-orange", "value": 5}, - {"color": "dark-red", "value": 8}, - ] - assert "rolling 24h failure count" in problematic_panel["description"] - assert "at least two recent failures" in problematic_panel["description"] - - sonar_mix_panel = nested_panels_by_title["Sonar Gate Status Mix (Selected)"] - sonar_health_panel = nested_panels_by_title["Sonar Gate Health by Project"] - assert sonar_mix_panel["gridPos"]["w"] == 4 - assert sonar_health_panel["gridPos"]["w"] == 8 - assert sonar_health_panel["type"] == "state-timeline" - assert "platform_quality:sonar_gate_health_percent:latest_1h" in sonar_health_panel["targets"][0]["expr"] - assert "sonarqube_project_quality_gate_pass" not in sonar_health_panel["targets"][0]["expr"] - - branch_panel = nested_panels_by_title["Primary Branch Clean by Suite (7d)"] - recent_branch_panel = nested_panels_by_title["Recent Branch Evidence by Suite (7d)"] - assert branch_panel["gridPos"]["x"] == 12 - assert recent_branch_panel["gridPos"]["x"] == 18 - assert "[7d:1h]" in recent_branch_panel["targets"][0]["expr"] - assert "[7d:1h]" in branch_panel["targets"][0]["expr"] - assert branch_panel["fieldConfig"]["defaults"]["unit"] == "percent" - assert "unless on(suite)" in branch_panel["targets"][0]["expr"] - assert "> bool 0" in branch_panel["targets"][0]["expr"] - assert branch_panel["targets"][0]["expr"].startswith("sort(") diff --git a/scripts/tests/test_dashboards_render_atlas_drilldowns.py b/scripts/tests/test_dashboards_render_atlas_drilldowns.py new file mode 100644 index 00000000..08871b71 --- /dev/null +++ b/scripts/tests/test_dashboards_render_atlas_drilldowns.py @@ -0,0 +1,127 @@ +"""Detailed Atlas Jobs dashboard collapse and drilldown contracts.""" + +from scripts.tests.test_dashboards_render_atlas import load_module + + +def test_jobs_dashboard_collapses_heavy_drilldowns_for_light_first_paint(): + mod = load_module() + dashboard = mod.build_jobs_dashboard() + panels = dashboard["panels"] + rows = [panel for panel in panels if panel["type"] == "row"] + visible_query_panels = [panel for panel in panels if panel["type"] != "row"] + nested_panels_by_title = { + child["title"]: child for row in rows for child in row.get("panels", []) + } + + assert len(panels) == 18 + assert len(visible_query_panels) == 12 + assert sum(len(panel.get("targets", [])) for panel in visible_query_panels) == 12 + assert all( + panel["title"] != "Coverage Gap to 95% by Suite" + for panel in visible_query_panels + ) + assert [row["title"] for row in rows] == [ + "CI Runs And Test Result History", + "Check Failure Rates By Suite", + "Check Healthy Rates By Suite", + "Test Drilldowns And Problem Tests", + "Telemetry Completeness And Branches", + "SonarQube Project Health", + ] + assert all(row["collapsed"] for row in rows) + + assert "Coverage Failure Rate" in nested_panels_by_title + assert "Supply Chain Healthy Rate" in nested_panels_by_title + assert "Test Category Health History" in nested_panels_by_title + assert "Selected Test Pass Rate History" in nested_panels_by_title + assert "Coverage Metrics Present by Suite" in nested_panels_by_title + assert "SonarQube API Up" in nested_panels_by_title + + failure_rate_panel = nested_panels_by_title["Coverage Failure Rate"] + assert failure_rate_panel["type"] == "state-timeline" + assert failure_rate_panel["fieldConfig"]["defaults"]["unit"] == "percent" + assert failure_rate_panel["fieldConfig"]["defaults"]["max"] == 100 + assert ( + failure_rate_panel["fieldConfig"]["defaults"]["thresholds"]["steps"][0]["color"] + == "dark-blue" + ) + assert "increase(" not in failure_rate_panel["targets"][0]["expr"] + assert ( + "platform_quality:check_failed_flag:present_1h" + in failure_rate_panel["targets"][0]["expr"] + ) + assert ( + "platform_quality:check_seen_flag:present_1h" + in failure_rate_panel["targets"][0]["expr"] + ) + assert ( + "platform_quality:check_status:present_1h" + not in failure_rate_panel["targets"][0]["expr"] + ) + assert ( + ".*_quality_gate_checks_total" not in failure_rate_panel["targets"][0]["expr"] + ) + assert "last_over_time" not in failure_rate_panel["targets"][0]["expr"] + assert "label_replace" not in failure_rate_panel["targets"][0]["expr"] + assert "0 *" in failure_rate_panel["targets"][0]["expr"] + assert "and on(suite)" not in failure_rate_panel["targets"][0]["expr"] + + pass_rate_panel = nested_panels_by_title["Selected Test Pass Rate History"] + assert pass_rate_panel["type"] == "state-timeline" + assert ( + "platform_quality:test_case_pass_rate:percent_1h" + in pass_rate_panel["targets"][0]["expr"] + ) + assert ( + "platform_quality_gate_test_case_result" + not in pass_rate_panel["targets"][0]["expr"] + ) + + pass_fail_panel = nested_panels_by_title["Selected Test Pass/Fail History"] + assert pass_fail_panel["fieldConfig"]["defaults"]["custom"]["drawStyle"] == "bars" + assert all( + "platform_quality:test_case_status:count_1h" in target["expr"] + for target in pass_fail_panel["targets"] + ) + + problematic_panel = nested_panels_by_title[ + "Problematic Tests Over Time (Top failures)" + ] + assert problematic_panel["type"] == "state-timeline" + assert problematic_panel["gridPos"]["w"] == 24 + assert 'test!=""' in problematic_panel["targets"][0]["expr"] + assert "vector(0)" not in problematic_panel["targets"][0]["expr"] + assert problematic_panel["fieldConfig"]["defaults"]["thresholds"]["steps"] == [ + {"color": "dark-blue", "value": None}, + {"color": "dark-green", "value": 2}, + {"color": "dark-yellow", "value": 3}, + {"color": "dark-orange", "value": 5}, + {"color": "dark-red", "value": 8}, + ] + assert "rolling 24h failure count" in problematic_panel["description"] + assert "at least two recent failures" in problematic_panel["description"] + + sonar_mix_panel = nested_panels_by_title["Sonar Gate Status Mix (Selected)"] + sonar_health_panel = nested_panels_by_title["Sonar Gate Health by Project"] + assert sonar_mix_panel["gridPos"]["w"] == 4 + assert sonar_health_panel["gridPos"]["w"] == 8 + assert sonar_health_panel["type"] == "state-timeline" + assert ( + "platform_quality:sonar_gate_health_percent:latest_1h" + in sonar_health_panel["targets"][0]["expr"] + ) + assert ( + "sonarqube_project_quality_gate_pass" + not in sonar_health_panel["targets"][0]["expr"] + ) + + branch_panel = nested_panels_by_title["Primary Branch Clean by Suite (7d)"] + recent_branch_panel = nested_panels_by_title["Recent Branch Evidence by Suite (7d)"] + assert branch_panel["gridPos"]["x"] == 12 + assert recent_branch_panel["gridPos"]["x"] == 18 + assert "[7d:1h]" in recent_branch_panel["targets"][0]["expr"] + assert "[7d:1h]" in branch_panel["targets"][0]["expr"] + assert branch_panel["fieldConfig"]["defaults"]["unit"] == "percent" + assert "unless on(suite)" in branch_panel["targets"][0]["expr"] + assert "> bool 0" in branch_panel["targets"][0]["expr"] + assert branch_panel["targets"][0]["expr"].startswith("sort(") diff --git a/services/gitea/atlas-identity-bootstrap-job.yaml b/services/gitea/atlas-identity-bootstrap-job.yaml index c7e2071f..bacecd01 100644 --- a/services/gitea/atlas-identity-bootstrap-job.yaml +++ b/services/gitea/atlas-identity-bootstrap-job.yaml @@ -2,7 +2,7 @@ apiVersion: batch/v1 kind: Job metadata: - name: gitea-atlas-identity-bootstrap-3 + name: gitea-atlas-identity-bootstrap-5 namespace: gitea labels: app.kubernetes.io/name: gitea @@ -34,6 +34,20 @@ spec: nodeSelector: node-role.kubernetes.io/worker: "true" hardware: rpi5 + initContainers: + - name: python-runtime + image: python@sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df + command: [/bin/sh, -ec] + args: ["cp -R /usr/local/. /python/"] + securityContext: + allowPrivilegeEscalation: false + capabilities: {drop: [ALL]} + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + volumeMounts: + - {name: python-runtime, mountPath: /python} containers: - name: bootstrap image: gitea/gitea:1.23 @@ -80,6 +94,7 @@ spec: - name: bootstrap mountPath: /opt/bootstrap readOnly: true + - {name: python-runtime, mountPath: /opt/python, readOnly: true} volumes: - name: data persistentVolumeClaim: @@ -88,3 +103,6 @@ spec: configMap: name: gitea-atlas-identity-bootstrap defaultMode: 0555 + - name: python-runtime + emptyDir: + sizeLimit: 128Mi diff --git a/services/gitea/kustomization.yaml b/services/gitea/kustomization.yaml index 8c473a38..6511418a 100644 --- a/services/gitea/kustomization.yaml +++ b/services/gitea/kustomization.yaml @@ -15,6 +15,7 @@ configMapGenerator: - name: gitea-atlas-identity-bootstrap files: - gitea_atlas_identity_ensure.sh=scripts/gitea_atlas_identity_ensure.sh + - gitea_branch_protection_check.py=scripts/gitea_branch_protection_check.py generatorOptions: disableNameSuffixHash: true diff --git a/services/gitea/scripts/gitea_atlas_identity_ensure.sh b/services/gitea/scripts/gitea_atlas_identity_ensure.sh index f98f08f5..6d5dbd90 100644 --- a/services/gitea/scripts/gitea_atlas_identity_ensure.sh +++ b/services/gitea/scripts/gitea_atlas_identity_ensure.sh @@ -19,6 +19,7 @@ source_owner="${GITEA_SOURCE_OWNER:-bstein}" managed_repositories="${GITEA_MANAGED_REPOSITORIES:-hermes-code-demo cassandra soteria pegasus metis ananke ariadne typhon titan-iac}" contributor_team=Contributors contributor_units='["repo.actions","repo.packages","repo.code","repo.issues","repo.wiki","repo.pulls","repo.releases","repo.projects"]' +protected_reviewer=bstein die() { echo "Gitea Atlas identity bootstrap failed: $*" >&2 @@ -272,6 +273,46 @@ transfer_managed_repositories() { trap - EXIT HUP INT TERM } +ensure_default_branch_protection() { + repository=$1 + repo_file=$(mktemp) + protections_file=$(mktemp) + response_file=$(mktemp) + + status=$(api_request GET "/repos/${organization}/${repository}" '' "${repo_file}") + expect_status "${status}" 200 "read ${repository} metadata for branch protection" + default_branch=$(sed -n 's/.*"default_branch":"\([^"]*\)".*/\1/p' "${repo_file}") + case "${default_branch}" in + main|master) ;; + *) die "${repository} has unsupported default branch ${default_branch:-missing}" ;; + esac + + status=$(api_request GET "/repos/${organization}/${repository}/branch_protections" '' "${protections_file}") + expect_status "${status}" 200 "list ${repository} branch protections" + if ! protection_state=$( + /opt/python/bin/python3 /opt/bootstrap/gitea_branch_protection_check.py \ + "${protections_file}" "${default_branch}" "${protected_reviewer}" + ); then + die "${repository} ${default_branch} protection differs from the human-review policy" + fi + + if [ "${protection_state}" = ABSENT ]; then + payload="{\"rule_name\":\"${default_branch}\",\"enable_push\":true,\"enable_push_whitelist\":true,\"push_whitelist_usernames\":[\"${protected_reviewer}\"],\"push_whitelist_teams\":[],\"push_whitelist_deploy_keys\":false,\"enable_force_push\":false,\"enable_force_push_allowlist\":false,\"force_push_allowlist_usernames\":[],\"force_push_allowlist_teams\":[],\"force_push_allowlist_deploy_keys\":false,\"enable_merge_whitelist\":true,\"merge_whitelist_usernames\":[\"${protected_reviewer}\"],\"merge_whitelist_teams\":[],\"enable_approvals_whitelist\":true,\"approvals_whitelist_username\":[\"${protected_reviewer}\"],\"approvals_whitelist_teams\":[],\"required_approvals\":1,\"block_on_rejected_reviews\":true,\"block_on_outdated_branch\":true,\"dismiss_stale_approvals\":true,\"block_admin_merge_override\":true}" + status=$(api_request POST "/repos/${organization}/${repository}/branch_protections" "${payload}" "${response_file}") + expect_status "${status}" 201 "protect ${repository} ${default_branch}" + status=$(api_request GET "/repos/${organization}/${repository}/branch_protections" '' "${protections_file}") + expect_status "${status}" 200 "read back ${repository} branch protections" + protection_state=$( + /opt/python/bin/python3 /opt/bootstrap/gitea_branch_protection_check.py \ + "${protections_file}" "${default_branch}" "${protected_reviewer}" + ) || die "${repository} ${default_branch} protection differs from the human-review policy" + fi + [ "${protection_state}" = PRESENT ] || \ + die "${repository} ${default_branch} protection did not become effective" + + rm -f "${repo_file}" "${protections_file}" "${response_file}" +} + wait_for_gitea ensure_user "${reconciler_user}" "${reconciler_email}" true ensure_user "${hermes_user}" "${hermes_email}" false @@ -318,6 +359,10 @@ expect_status "${status}" 204 "add Hermes to Atlas contributors" transfer_managed_repositories +for repository in ${managed_repositories}; do + ensure_default_branch_protection "${repository}" +done + vault_write gitea/atlas-reconciler "{\"data\":{\"username\":\"${reconciler_user}\",\"token\":\"${reconciler_token}\",\"base_url\":\"${public_url}\"}}" vault_write hermes/developer-gitea "{\"data\":{\"username\":\"${hermes_user}\",\"email\":\"${hermes_email}\",\"token\":\"${hermes_token}\",\"base_url\":\"${public_url}\"}}" diff --git a/services/gitea/scripts/gitea_branch_protection_check.py b/services/gitea/scripts/gitea_branch_protection_check.py new file mode 100644 index 00000000..e3f112c9 --- /dev/null +++ b/services/gitea/scripts/gitea_branch_protection_check.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python3 +"""Validate the first effective Gitea protection for an Atlas primary branch.""" + +from __future__ import annotations + +import argparse +import datetime +import json +import os +import re +import stat +from pathlib import Path + +MAX_INPUT = 1024 * 1024 +MAX_RULES = 100 +MAX_RULE_NAME = 255 +SPECIAL = frozenset("*?\\[]{}") + + +class PolicyError(RuntimeError): + """The protection response is ambiguous or violates human review policy.""" + + +def _read_bounded(path: Path) -> bytes: + descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + try: + metadata = os.fstat(descriptor) + if not stat.S_ISREG(metadata.st_mode) or metadata.st_size > MAX_INPUT: + raise PolicyError("branch protection response exceeds the safe limit") + value = os.read(descriptor, MAX_INPUT + 1) + finally: + os.close(descriptor) + if len(value) != metadata.st_size: + raise PolicyError("branch protection response changed while reading") + return value + + +def _glob_regex(pattern: str, position: int = 0, terminators: str = "") -> tuple[str, int]: + """Compile the gobwas/glob syntax Gitea v1.23.8 uses.""" + pieces: list[str] = [] + while position < len(pattern): + character = pattern[position] + if character in terminators: + break + if character == "\\": + position += 1 + if position >= len(pattern): + raise ValueError("trailing escape") + pieces.append(re.escape(pattern[position])) + elif character == "*": + if position + 1 < len(pattern) and pattern[position + 1] == "*": + pieces.append(".*") + position += 1 + else: + pieces.append("[^/]*") + elif character == "?": + pieces.append("[^/]") + elif character == "[": + end = pattern.find("]", position + 1) + if end < 0: + raise ValueError("unterminated range") + value = pattern[position + 1 : end] + if not value: + raise ValueError("empty range") + negate = value.startswith("!") + value = value[1:] if negate else value + if not value or "[" in value or "\\" in value: + raise ValueError("invalid range") + if "-" in value: + if value.count("-") != 1 or value.startswith("-") or value.endswith("-"): + raise ValueError("invalid range") + low, high = value.split("-") + if len(low) != 1 or len(high) != 1 or ord(high) < ord(low): + raise ValueError("invalid range") + content = re.escape(low) + "-" + re.escape(high) + else: + content = re.escape(value) + pieces.append("[" + ("^" if negate else "") + content + "]") + position = end + elif character == "{": + alternatives: list[str] = [] + position += 1 + while True: + alternative, position = _glob_regex(pattern, position, ",}") + alternatives.append(alternative) + if position >= len(pattern): + raise ValueError("unterminated alternatives") + if pattern[position] == "}": + break + position += 1 + if len(alternatives) < 2 or any(not item for item in alternatives): + raise ValueError("invalid alternatives") + pieces.append("(?:" + "|".join(alternatives) + ")") + elif character in "]}": + # A closing delimiter outside its grammar context is literal in + # gobwas/glob's lexer. + pieces.append(re.escape(character)) + else: + pieces.append(re.escape(character)) + position += 1 + return "".join(pieces), position + + +def _is_plain(pattern: str) -> bool: + return not any(character in SPECIAL for character in pattern) + + +def _matches(pattern: str, branch: str) -> bool: + if not pattern.isascii() or not 1 <= len(pattern) <= MAX_RULE_NAME: + raise PolicyError("branch protection rule name is invalid") + if _is_plain(pattern): + return pattern.casefold() == branch.casefold() + try: + expression, position = _glob_regex(pattern) + if position != len(pattern): + raise ValueError("incomplete glob") + except (re.error, ValueError): + # Gitea quotes an invalid special pattern and matches it literally. + # A literal containing a special byte cannot equal main or master. + return False + return re.fullmatch(expression, branch) is not None + + +def _created(value: object) -> datetime.datetime: + if not isinstance(value, str) or len(value) > 64: + raise PolicyError("branch protection creation time is invalid") + try: + result = datetime.datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as exc: + raise PolicyError("branch protection creation time is invalid") from exc + if result.tzinfo is None: + raise PolicyError("branch protection creation time is invalid") + return result + + +def _required(reviewer: str) -> dict[str, object]: + return { + "enable_push": True, + "enable_push_whitelist": True, + "push_whitelist_usernames": [reviewer], + "push_whitelist_deploy_keys": False, + "enable_force_push": False, + "enable_merge_whitelist": True, + "merge_whitelist_usernames": [reviewer], + "enable_approvals_whitelist": True, + "approvals_whitelist_username": [reviewer], + "required_approvals": 1, + "block_on_rejected_reviews": True, + "block_on_outdated_branch": True, + "dismiss_stale_approvals": True, + "block_admin_merge_override": True, + } + + +def evaluate(value: bytes, branch: str, reviewer: str) -> str: + """Return PRESENT/ABSENT or fail on ambiguous/effective policy drift.""" + if len(value) > MAX_INPUT or branch not in {"main", "master"}: + raise PolicyError("branch protection input is invalid") + data = json.loads(value) + if not isinstance(data, list) or len(data) > MAX_RULES: + raise PolicyError("branch protection response is invalid") + ordered: list[tuple[int, bool, datetime.datetime, int, dict[str, object]]] = [] + for index, raw in enumerate(data): + if not isinstance(raw, dict): + raise PolicyError("branch protection entry is invalid") + priority = raw.get("priority") + rule_name = raw.get("rule_name") + if ( + not isinstance(priority, int) + or isinstance(priority, bool) + or not 1 <= priority <= 1_000_000 + or not isinstance(rule_name, str) + ): + raise PolicyError("branch protection priority is ambiguous") + created = _created(raw.get("created_at")) + if _matches(rule_name, branch): + # Gitea v1.23.8 sorts by priority, then puts plain names before + # globs, then uses creation time. The API exposes every component. + ordered.append((priority, not _is_plain(rule_name), created, index, raw)) + if not ordered: + return "ABSENT" + effective = min(ordered)[4] + for key, expected in _required(reviewer).items(): + if effective.get(key) != expected: + raise PolicyError( + f"effective {branch} protection differs from human-review policy" + ) + return "PRESENT" + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("response", type=Path) + parser.add_argument("branch", choices=("main", "master")) + parser.add_argument("reviewer") + args = parser.parse_args() + try: + value = _read_bounded(args.response) + print(evaluate(value, args.branch, args.reviewer)) + except (OSError, json.JSONDecodeError, PolicyError) as exc: + print(f"branch protection check failed: {exc}", file=__import__("sys").stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/hermes-observer-bindings/kustomization.yaml b/services/hermes-observer-bindings/kustomization.yaml new file mode 100644 index 00000000..4dbf36b5 --- /dev/null +++ b/services/hermes-observer-bindings/kustomization.yaml @@ -0,0 +1,5 @@ +# services/hermes-observer-bindings/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - rolebindings.yaml diff --git a/services/hermes-observer-bindings/rolebindings.yaml b/services/hermes-observer-bindings/rolebindings.yaml new file mode 100644 index 00000000..403cd39a --- /dev/null +++ b/services/hermes-observer-bindings/rolebindings.yaml @@ -0,0 +1,204 @@ +# services/hermes-observer-bindings/rolebindings.yaml +apiVersion: v1 +kind: List +items: + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: ai} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: bstein-dev-home} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: cassandra} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: cert-manager} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: climate} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: comms} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: crypto} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: default} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: finance} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: flux-system} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: game-stream} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: gitea} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: harbor} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: health} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: hermes} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: hermes-chat} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: hermes-triage-demo} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: jellyfin} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: jenkins} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: kube-node-lease} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: kube-public} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: kube-system} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: logging} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: longhorn-system} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: mailu-mailserver} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: maintenance} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: metallb-system} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: monitoring} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: nextcloud} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: openclaw} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: outline} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: planka} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: postgres} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: quality} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: sso} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: sui-metrics} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: traefik} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: vault} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: vaultwarden} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: veles} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} diff --git a/services/hermes-observer-rbac/kustomization.yaml b/services/hermes-observer-rbac/kustomization.yaml new file mode 100644 index 00000000..cff6f4b9 --- /dev/null +++ b/services/hermes-observer-rbac/kustomization.yaml @@ -0,0 +1,5 @@ +# services/hermes-observer-rbac/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - rbac.yaml diff --git a/services/hermes-observer-rbac/rbac.yaml b/services/hermes-observer-rbac/rbac.yaml new file mode 100644 index 00000000..737ac4dd --- /dev/null +++ b/services/hermes-observer-rbac/rbac.yaml @@ -0,0 +1,112 @@ +# services/hermes-observer-rbac/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: hermes-agent-cluster-observer-v2 + labels: + app.kubernetes.io/name: hermes-agent + app.kubernetes.io/part-of: hermes +rules: + - apiGroups: [""] + resources: + - namespaces + - nodes + - persistentvolumes + verbs: ["get", "list", "watch"] + - apiGroups: ["networking.k8s.io"] + resources: ["ingressclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: + - csidrivers + - csinodes + - storageclasses + - volumeattachments + verbs: ["get", "list", "watch"] + - apiGroups: ["metrics.k8s.io"] + resources: ["nodes"] + verbs: ["get", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: hermes-agent-namespaced-observer-v2 + labels: + app.kubernetes.io/name: hermes-agent + app.kubernetes.io/part-of: hermes +rules: + - apiGroups: [""] + resources: + - configmaps + - endpoints + - events + - persistentvolumeclaims + - pods + - pods/log + - replicationcontrollers + - resourcequotas + - services + verbs: ["get", "list", "watch"] + - apiGroups: ["apps"] + resources: + - controllerrevisions + - daemonsets + - deployments + - replicasets + - statefulsets + verbs: ["get", "list", "watch"] + - apiGroups: ["batch"] + resources: ["cronjobs", "jobs"] + verbs: ["get", "list", "watch"] + - apiGroups: ["autoscaling"] + resources: ["horizontalpodautoscalers"] + verbs: ["get", "list", "watch"] + - apiGroups: ["networking.k8s.io"] + resources: ["ingresses", "networkpolicies"] + verbs: ["get", "list", "watch"] + - apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["get", "list", "watch"] + - apiGroups: ["policy"] + resources: ["poddisruptionbudgets"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csistoragecapacities"] + verbs: ["get", "list", "watch"] + - apiGroups: ["metrics.k8s.io"] + resources: ["pods"] + verbs: ["get", "list"] + - apiGroups: + - helm.toolkit.fluxcd.io + - image.toolkit.fluxcd.io + - kustomize.toolkit.fluxcd.io + - notification.toolkit.fluxcd.io + - source.toolkit.fluxcd.io + resources: ["*"] + verbs: ["get", "list", "watch"] + - apiGroups: ["longhorn.io"] + resources: + - backingimages + - engines + - instancemanagers + - nodes + - replicas + - settings + - volumes + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: hermes-agent-cluster-observer-v2 + labels: + app.kubernetes.io/name: hermes-agent + app.kubernetes.io/part-of: hermes +subjects: + - kind: ServiceAccount + name: hermes-agent + namespace: hermes +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: hermes-agent-cluster-observer-v2 diff --git a/services/hermes-scm-broker/deployment.yaml b/services/hermes-scm-broker/deployment.yaml new file mode 100644 index 00000000..649ada1d --- /dev/null +++ b/services/hermes-scm-broker/deployment.yaml @@ -0,0 +1,98 @@ +# services/hermes-scm-broker/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hermes-scm-broker + namespace: hermes-scm + labels: + app: hermes-scm-broker +spec: + replicas: 1 + revisionHistoryLimit: 2 + selector: + matchLabels: + app: hermes-scm-broker + template: + metadata: + labels: + app: hermes-scm-broker + annotations: + vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/role: hermes-scm-broker + vault.hashicorp.com/agent-inject-secret-gitea-token: kv/data/atlas/hermes/developer-gitea + vault.hashicorp.com/agent-inject-template-gitea-token: | + {{- with secret "kv/data/atlas/hermes/developer-gitea" -}} + {{ .Data.data.token }} + {{- end }} + vault.hashicorp.com/agent-pre-populate-only: "true" + vault.hashicorp.com/agent-init-first: "true" + vault.hashicorp.com/agent-requests-cpu: 10m + vault.hashicorp.com/agent-requests-mem: 32Mi + vault.hashicorp.com/agent-limits-cpu: 50m + vault.hashicorp.com/agent-limits-mem: 64Mi + spec: + serviceAccountName: hermes-scm-broker + automountServiceAccountToken: true + securityContext: + fsGroup: 10000 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: + type: RuntimeDefault + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: [arm64] + - key: node-role.kubernetes.io/worker + operator: In + values: ["true"] + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: hardware + operator: In + values: [rpi5] + containers: + - name: broker + image: registry.bstein.dev/bstein/hermes-agent + imagePullPolicy: IfNotPresent + command: + - /opt/hermes/.venv/bin/python + - /opt/broker/scm_broker.py + ports: + - {name: http, containerPort: 9081, protocol: TCP} + readinessProbe: + httpGet: {path: /healthz, port: http} + initialDelaySeconds: 3 + periodSeconds: 10 + livenessProbe: + httpGet: {path: /healthz, port: http} + initialDelaySeconds: 15 + periodSeconds: 30 + failureThreshold: 10 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 10000 + runAsGroup: 10000 + volumeMounts: + - {name: broker-code, mountPath: /opt/broker, readOnly: true} + - {name: tmp, mountPath: /tmp} + resources: + requests: {cpu: 50m, memory: 128Mi} + limits: {cpu: "1", memory: 768Mi} + volumes: + - name: broker-code + configMap: + name: hermes-scm-boundary + defaultMode: 0555 + - name: tmp + emptyDir: + sizeLimit: 1Gi diff --git a/services/hermes-scm-broker/kustomization.yaml b/services/hermes-scm-broker/kustomization.yaml new file mode 100644 index 00000000..5dac3620 --- /dev/null +++ b/services/hermes-scm-broker/kustomization.yaml @@ -0,0 +1,11 @@ +# services/hermes-scm-broker/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: hermes-scm +resources: + - service.yaml + - deployment.yaml + - networkpolicy.yaml +images: + - name: registry.bstein.dev/bstein/hermes-agent + digest: sha256:37ebf720c783ae908a602916ffccf88d43d205a157957f5dc4b487867aee45e7 diff --git a/services/hermes-scm-broker/networkpolicy.yaml b/services/hermes-scm-broker/networkpolicy.yaml new file mode 100644 index 00000000..8187e333 --- /dev/null +++ b/services/hermes-scm-broker/networkpolicy.yaml @@ -0,0 +1,46 @@ +# services/hermes-scm-broker/networkpolicy.yaml +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: hermes-scm-broker-isolation + namespace: hermes-scm +spec: + podSelector: + matchLabels: + app: hermes-scm-broker + policyTypes: [Ingress, Egress] + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: hermes + podSelector: + matchLabels: + app: hermes-agent + ports: + - {protocol: TCP, port: 9081} + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - {protocol: UDP, port: 53} + - {protocol: TCP, port: 53} + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: vault + podSelector: + matchLabels: + app: vault + ports: + - {protocol: TCP, port: 8200} + - to: + - ipBlock: + cidr: 192.168.22.9/32 + ports: + - {protocol: TCP, port: 443} diff --git a/services/hermes-scm-broker/service.yaml b/services/hermes-scm-broker/service.yaml new file mode 100644 index 00000000..0b08c7f6 --- /dev/null +++ b/services/hermes-scm-broker/service.yaml @@ -0,0 +1,13 @@ +# services/hermes-scm-broker/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: hermes-scm-broker + namespace: hermes-scm +spec: + selector: + app: hermes-scm-broker + ports: + - name: http + port: 9081 + targetPort: http diff --git a/services/hermes-scm-namespace/kustomization.yaml b/services/hermes-scm-namespace/kustomization.yaml new file mode 100644 index 00000000..a9e89c7b --- /dev/null +++ b/services/hermes-scm-namespace/kustomization.yaml @@ -0,0 +1,5 @@ +# services/hermes-scm-namespace/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - namespace.yaml diff --git a/services/hermes-scm-namespace/namespace.yaml b/services/hermes-scm-namespace/namespace.yaml new file mode 100644 index 00000000..542f9602 --- /dev/null +++ b/services/hermes-scm-namespace/namespace.yaml @@ -0,0 +1,14 @@ +# services/hermes-scm-namespace/namespace.yaml +apiVersion: v1 +kind: Namespace +metadata: + name: hermes-scm + labels: + app.kubernetes.io/name: hermes-scm-broker +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: hermes-scm-broker + namespace: hermes-scm +automountServiceAccountToken: true diff --git a/services/hermes/agent-configmap.yaml b/services/hermes/agent-configmap.yaml index 782876cf..b53b2c92 100644 --- a/services/hermes/agent-configmap.yaml +++ b/services/hermes/agent-configmap.yaml @@ -189,9 +189,9 @@ data: not replace these coordinator-wide rules. Never call `kanban_show` without a known, non-empty task ID. Ad-hoc inspection and acceptance checks do not need a synthetic Kanban lookup, and must load a skill only when its workflow - materially applies. Atlas HTTPS Git authentication is already supplied by - the runtime-only `GIT_ASKPASS`; use it without reading or exposing the - credential. Coordinator guidance lives at + materially applies. Atlas Git access is supplied by the isolated SCM + broker; no repository token is present in this pod. Use only configured + broker remotes and clients. Coordinator guidance lives at `/opt/data/workspace/AGENTS.md` when more detail is needed. The Jetson classifier is mandatory for AUTO selection. Switchyard may use @@ -295,21 +295,21 @@ data: objective's difficulty warrants it; otherwise synthesize at the original effort. - This owner-only pod has cluster-admin access across Atlas, including logs, - Secrets, exec, port-forwarding, rollout operations, and Flux reconciliation. - Prefer the titan-iac Git/Flux workflow for every durable cluster change; - direct operations are available for explicit operator requests, incident - recovery, and verification, and must be followed by a matching source-of- - truth change when they alter desired state. Never expose credentials in - chat or logs. Triage belongs at triage.hermes.bstein.dev. + This owner-only pod has scoped read-only Kubernetes diagnostics across + Atlas, including resource status, events, logs, and Flux/Helm evidence. It + cannot read Secrets, exec or attach to pods, create service-account tokens, + mutate workloads or RBAC, or reconcile Flux. Put every durable cluster + change on a reviewed titan-iac branch. Never expose credentials in chat or + logs. Triage belongs at triage.hermes.bstein.dev. ## Atlas engineering access The Atlas organization has private visibility. Repository visibility is preserved per project and may be public or private; do not infer a repository's visibility from the organization setting. Repositories are - canonical at `https://scm.bstein.dev/atlas/.git`. HTTPS Git - authentication is already supplied through `GIT_ASKPASS`. Verify the remote + canonical at `https://scm.bstein.dev/atlas/.git`; worker Git traffic + uses `http://hermes-scm-broker.hermes-scm.svc.cluster.local:9081/git/atlas/.git`. + Verify the broker remote and cleanly separate pre-existing changes, create a task branch, run the repository's tests, use `git push --dry-run` when proving access, and push a real branch only when the requested implementation is review-ready. Never @@ -318,11 +318,13 @@ data: `scm.bstein.dev` is Forgejo/Gitea, not GitHub. Never load or follow a GitHub/`gh` skill for an Atlas remote, and do not interpret an unauthenticated Gitea HTTP 404 as a missing private repository. Use - authenticated Git for clone, fetch, and non-force push. For bounded + brokered Git for clone, fetch, and creation of a new namespaced feature + branch. Existing-ref updates, protected refs, deletion, and force-push are + rejected. For bounded repository/pull-request evidence or to create a review-ready draft PR, load - `$manage-atlas-pull-requests` and use `/opt/coordinator/gitea_api.py`. The - client injects the runtime Vault token without exposing it to the command - line, environment, output, or transcript. It permits only Atlas-scoped + `$manage-atlas-pull-requests` and use `/opt/scm/gitea_api.py`. The + client carries no repository credential; a separate least-authority + workload performs the allowed operation. It permits only Atlas-scoped metadata reads and verified same-repository draft creation. Updates, merge, approve, close, delete, comments, repository administration, and force-push are unavailable; never bypass the client with raw HTTP. Leave every PR @@ -330,14 +332,10 @@ data: verify the exact base/head ancestry and diff, run the relevant tests, and confirm the candidate Jenkins result before presenting a handoff. Prefer the internal endpoint in - `JENKINS_BASE_URL`; when Jenkins API authorization prevents a read, use the - existing cluster access to inspect the controller's job/build files and - logs rather than guessing a public hostname. The preferred read-only path - is `/opt/coordinator/jenkins_build_evidence.py JOB [--branch BRANCH] - [--commit SHA] --wait`; it distinguishes a genuinely terminal build from - nested Jenkins execution metadata and returns bounded JSON/log evidence. - With a branch, the helper also tries the conventional `JOB-branches` - multibranch name. If Brad separately reports that he merged the PR, observe + `JENKINS_BASE_URL`; when Jenkins API authorization prevents a read, use + Kubernetes pod logs and status evidence without exec or secret access. + Do not bypass the boundary to inspect controller files. If Brad separately + reports that he merged the PR, observe the default-branch build to a terminal result and report exact commit, build, and test evidence. @@ -349,19 +347,21 @@ data: The terminal PATH contains the pinned operator tools. Start cluster work with `kubectl config current-context`, read-only status/events/logs, and the - relevant `titan-iac` manifests. Put durable desired-state changes on a + relevant `titan-iac` manifests. The broker namespace is deliberately + excluded from agent RBAC. Kubernetes cannot express a deny on one namespace + inside an all-namespace list, so enumerate namespaces and run namespaced + diagnostics instead of relying on `kubectl ... --all-namespaces`. Put + durable desired-state changes on a reviewable `titan-iac` branch and validate Kustomize and client dry-run. - Brad owns merge and routine Flux reconciliation; perform either only under - a separate explicit operator request. Direct `kubectl` mutations are for - explicit incident recovery or ephemeral verification, not ordinary - delivery. + Brad owns merge and Flux reconciliation. Direct `kubectl` mutation is not + available to the agent; hand an explicit incident mutation back to Brad. - Node SSH uses a dedicated audited Hermes identity: `ssh titan-04` (or any - current Kubernetes node name). Host keys are pinned and password fallback - is disabled. Use SSH only for host-level evidence or repairs that cannot be - performed through Kubernetes. Read first, identify the exact node and - impact, avoid fleet-wide destructive commands, and reflect persistent host - configuration in the appropriate tracked provisioning source. + Node SSH uses the dedicated locked-password `hermes-agent` OS account: + `ssh titan-04` (or any current Kubernetes node name). Host keys are pinned, + password fallback is disabled, and the account has no sudo, disk, runtime, + or Kubernetes-storage authority. Use it for unprivileged host evidence; + hand privileged host repair back to Brad and reflect persistent changes in + the tracked provisioning source. Treat every credential, SSH identity, host-trust record, and credential-bearing access-client state as runtime-only Vault data. Keep @@ -398,7 +398,7 @@ data: [model]` for a persistent override. The first native Codex worker requires one device-code login; refreshed provider credentials persist through Vault while client caches remain disposable. The owner - workspace includes cluster-admin Kubernetes - access plus `kubectl`, `flux`, `helm`, `kustomize`, `vault`, `sops`, `age`, + workspace includes scoped read-only Kubernetes diagnostics plus `kubectl`, + `flux`, `helm`, `kustomize`, `vault`, `sops`, `age`, `terraform`, `k9s`, `jq`, `yq`, `gh`, Git, SSH, Python, Node, the browser/computer tools, and the native provider CLIs. diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml index 4b93f92d..ccbc36ea 100644 --- a/services/hermes/agent-deployment.yaml +++ b/services/hermes/agent-deployment.yaml @@ -46,16 +46,6 @@ spec: {{- with secret "kv/data/atlas/hermes/agent-tokens" -}} {{ .Data.data.codex_auth_json }} {{- end }} - vault.hashicorp.com/agent-inject-secret-gitea-token: kv/data/atlas/hermes/developer-gitea - vault.hashicorp.com/agent-inject-template-gitea-token: | - {{- with secret "kv/data/atlas/hermes/developer-gitea" -}} - {{ .Data.data.token }} - {{- end }} - vault.hashicorp.com/agent-inject-secret-gitea-username: kv/data/atlas/hermes/developer-gitea - vault.hashicorp.com/agent-inject-template-gitea-username: | - {{- with secret "kv/data/atlas/hermes/developer-gitea" -}} - {{ .Data.data.username }} - {{- end }} vault.hashicorp.com/agent-inject-secret-node-ssh-private-key: kv/data/atlas/hermes/developer-ssh vault.hashicorp.com/agent-inject-template-node-ssh-private-key: | {{- with secret "kv/data/atlas/hermes/developer-ssh" -}} @@ -186,9 +176,8 @@ spec: chmod 0600 "${profile_env}" chown 10000:10000 "${profile_env}" done - upsert_env GIT_ASKPASS /opt/coordinator/gitea_askpass.sh upsert_env GIT_TERMINAL_PROMPT 0 - upsert_env GITEA_BASE_URL https://scm.bstein.dev + upsert_env HERMES_SCM_BROKER_URL http://hermes-scm-broker.hermes-scm.svc.cluster.local:9081 upsert_env JENKINS_BASE_URL http://jenkins.jenkins.svc.cluster.local:8080 upsert_env ARIADNE_BASE_URL http://ariadne.maintenance.svc.cluster.local upsert_env VICTORIA_METRICS_URL http://victoria-metrics-single-server.monitoring.svc.cluster.local:8428 @@ -664,6 +653,7 @@ spec: - {name: kubeconfig, mountPath: /opt/data/home/.kube/config, subPath: config, readOnly: true} - {name: auto-router-plugin, mountPath: /opt/data/plugins/auto-router, readOnly: true} - {name: atlas-pr-skill, mountPath: /opt/data/workspace/skills/manage-atlas-pull-requests, readOnly: true} + - {name: scm-boundary, mountPath: /opt/scm, readOnly: true} - {name: routing-catalog, mountPath: /routing-catalog, readOnly: true} - {name: tmp, mountPath: /tmp} startupProbe: @@ -828,6 +818,7 @@ spec: - {name: kubeconfig, mountPath: /opt/data/home/.kube/config, subPath: config, readOnly: true} - {name: auto-router-plugin, mountPath: /opt/data/plugins/auto-router, readOnly: true} - {name: atlas-pr-skill, mountPath: /opt/data/workspace/skills/manage-atlas-pull-requests, readOnly: true} + - {name: scm-boundary, mountPath: /opt/scm, readOnly: true} - {name: routing-catalog, mountPath: /routing-catalog, readOnly: true} - {name: tmp, mountPath: /tmp} - {name: ttyd-index, mountPath: /ttyd-index, readOnly: true} @@ -897,6 +888,7 @@ spec: - {name: kubeconfig, mountPath: /opt/data/home/.kube/config, subPath: config, readOnly: true} - {name: auto-router-plugin, mountPath: /opt/data/plugins/auto-router, readOnly: true} - {name: atlas-pr-skill, mountPath: /opt/data/workspace/skills/manage-atlas-pull-requests, readOnly: true} + - {name: scm-boundary, mountPath: /opt/scm, readOnly: true} - {name: tmp, mountPath: /tmp} resources: requests: {cpu: 100m, memory: 256Mi} @@ -1197,6 +1189,10 @@ spec: items: - {key: SKILL.md, path: SKILL.md} - {key: openai.yaml, path: agents/openai.yaml} + - name: scm-boundary + configMap: + name: hermes-scm-boundary + defaultMode: 0555 - name: image-policy configMap: name: hermes-image-policy diff --git a/services/hermes/agent-kubeconfig.yaml b/services/hermes/agent-kubeconfig.yaml index 15b38da9..9ad9242f 100644 --- a/services/hermes/agent-kubeconfig.yaml +++ b/services/hermes/agent-kubeconfig.yaml @@ -11,9 +11,9 @@ users: user: tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token contexts: - - name: atlas-owner + - name: atlas-observer context: cluster: atlas user: hermes-agent namespace: default -current-context: atlas-owner +current-context: atlas-observer diff --git a/services/hermes/agent-rbac.yaml b/services/hermes/agent-rbac.yaml deleted file mode 100644 index 9dbff9e3..00000000 --- a/services/hermes/agent-rbac.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# services/hermes/agent-rbac.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: hermes-agent-cluster-admin - labels: - app.kubernetes.io/name: hermes-agent - app.kubernetes.io/part-of: hermes -subjects: - - kind: ServiceAccount - name: hermes-agent - namespace: hermes -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cluster-admin diff --git a/services/hermes/kustomization.yaml b/services/hermes/kustomization.yaml index 87984530..fe60bef9 100644 --- a/services/hermes/kustomization.yaml +++ b/services/hermes/kustomization.yaml @@ -7,13 +7,13 @@ images: digest: sha256:37ebf720c783ae908a602916ffccf88d43d205a157957f5dc4b487867aee45e7 resources: - namespace.yaml + - scm-common - vault-serviceaccount.yaml - configmap.yaml - agent-configmap.yaml - chat-configmap.yaml - switchyard-configmap.yaml - rbac.yaml - - agent-rbac.yaml - node-ssh-access.yaml - pvc.yaml - switchyard-pvc.yaml @@ -69,8 +69,6 @@ configMapGenerator: - classifier_broker.py=scripts/classifier_broker.py - claude_oauth_broker.py=scripts/claude_oauth_broker.py - worker_route_broker.py=scripts/worker_route_broker.py - - gitea_api.py=scripts/gitea_api.py - - gitea_askpass.sh=scripts/gitea_askpass.sh - hermes_coordinator.py=scripts/hermes_coordinator.py - hermes_model_routing.py=scripts/hermes_model_routing.py - hermes_stt_client.py=scripts/hermes_stt_client.py @@ -104,6 +102,14 @@ configMapGenerator: - config=agent-kubeconfig.yaml options: disableNameSuffixHash: true + - name: hermes-node-account-hardener + namespace: hermes + files: + - node_account_hardening.py=scripts/node_account_hardening.py + - node_account_audit.py=scripts/node_account_audit.py + - node_account_io.py=scripts/node_account_io.py + options: + disableNameSuffixHash: true - name: hermes-auto-router-plugin namespace: hermes files: diff --git a/services/hermes/networkpolicy.yaml b/services/hermes/networkpolicy.yaml index d972889e..f9c3b09c 100644 --- a/services/hermes/networkpolicy.yaml +++ b/services/hermes/networkpolicy.yaml @@ -119,11 +119,65 @@ spec: app: server ports: - {protocol: TCP, port: 9010} - # agent.hermes.bstein.dev is an owner-only engineering workstation. The - # browser boundary remains OAuth-protected, while its workers need to reach - # every cluster namespace, Atlas LAN service, and hosted provider endpoint. egress: - - {} + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - {protocol: UDP, port: 53} + - {protocol: TCP, port: 53} + - to: + - namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: NotIn + values: [gitea, hermes-scm] + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: hermes-scm + podSelector: + matchLabels: + app: hermes-scm-broker + ports: + - {protocol: TCP, port: 9081} + - to: + - ipBlock: + cidr: 10.43.0.1/32 + ports: + - {protocol: TCP, port: 443} + - to: + - ipBlock: + cidr: 192.168.0.0/16 + except: + - 192.168.22.9/32 + - to: + - ipBlock: + cidr: 0.0.0.0/0 + except: + - 10.0.0.0/8 + - 100.64.0.0/10 + - 127.0.0.0/8 + - 169.254.0.0/16 + - 172.16.0.0/12 + - 192.168.0.0/16 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: hermes-node-ssh-access-isolation + namespace: hermes +spec: + podSelector: + matchLabels: + app: hermes-node-ssh-access + policyTypes: [Ingress, Egress] + ingress: [] + egress: [] --- apiVersion: networking.k8s.io/v1 kind: NetworkPolicy diff --git a/services/hermes/node-ssh-access.yaml b/services/hermes/node-ssh-access.yaml index d9c6b223..04b0c78b 100644 --- a/services/hermes/node-ssh-access.yaml +++ b/services/hermes/node-ssh-access.yaml @@ -45,32 +45,16 @@ spec: - operator: Exists containers: - name: key-reconciler - image: busybox:1.37 + # The pinned index supports the cluster's amd64 and arm64 nodes. + image: python@sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df imagePullPolicy: IfNotPresent command: [/bin/sh, -ec] args: - | - reconcile() { - key="$(cat /vault/secrets/node-ssh-public-key)" - found=0 - for user in atlas oceanus; do - home="/host-home/${user}" - [ -d "${home}" ] || continue - found=1 - identity="$(awk -F: -v name="${user}" '$1 == name {print $3 ":" $4; exit}' /host-etc/passwd)" - [ -n "${identity}" ] || identity="$(stat -c %u:%g "${home}")" - uid="${identity%%:*}" - gid="${identity##*:}" - install -d -m 0700 -o "${uid}" -g "${gid}" "${home}/.ssh" - touch "${home}/.ssh/authorized_keys" - grep -qxF "${key}" "${home}/.ssh/authorized_keys" || printf '%s\n' "${key}" >> "${home}/.ssh/authorized_keys" - chown "${uid}:${gid}" "${home}/.ssh/authorized_keys" - chmod 0600 "${home}/.ssh/authorized_keys" - done - [ "${found}" = 1 ] || { echo "no supported node SSH account found" >&2; return 1; } - } while true; do - reconcile + /usr/local/bin/python \ + /opt/node-hardener/node_account_hardening.py \ + --public-key-file /vault/secrets/node-ssh-public-key sleep 300 done securityContext: @@ -83,8 +67,21 @@ spec: volumeMounts: - name: host-home mountPath: /host-home - - name: host-passwd - mountPath: /host-etc/passwd + - name: host-etc + mountPath: /host-etc + - name: host-polkit-share + mountPath: /host-polkit-share + readOnly: true + - name: host-k3s + mountPath: /host-k3s + - name: host-kubelet + mountPath: /host-kubelet + - name: host-run-k3s + mountPath: /host-run-k3s + - name: host-run-containerd + mountPath: /host-run-containerd + - name: coordinator + mountPath: /opt/node-hardener readOnly: true - name: vault-secrets mountPath: /vault/secrets @@ -94,19 +91,43 @@ spec: resources: requests: cpu: 5m - memory: 8Mi + memory: 32Mi limits: cpu: 50m - memory: 32Mi + memory: 128Mi volumes: - name: host-home hostPath: path: /home type: Directory - - name: host-passwd + - name: host-etc hostPath: - path: /etc/passwd - type: File + path: /etc + type: Directory + - name: host-polkit-share + hostPath: + path: /usr/share/polkit-1 + type: Directory + - name: host-k3s + hostPath: + path: /var/lib/rancher/k3s + type: Directory + - name: host-kubelet + hostPath: + path: /var/lib/kubelet + type: Directory + - name: host-run-k3s + hostPath: + path: /run/k3s + type: Directory + - name: host-run-containerd + hostPath: + path: /run/containerd + type: Directory + - name: coordinator + configMap: + name: hermes-node-account-hardener + defaultMode: 0555 - name: vault-secrets csi: driver: secrets-store.csi.k8s.io diff --git a/services/hermes/scm-common/kustomization.yaml b/services/hermes/scm-common/kustomization.yaml new file mode 100644 index 00000000..17dc1da6 --- /dev/null +++ b/services/hermes/scm-common/kustomization.yaml @@ -0,0 +1,14 @@ +# services/hermes/scm-common/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +configMapGenerator: + - name: hermes-scm-boundary + files: + - gitea_api.py=scripts/gitea_api.py + - gitea_api_policy.py=scripts/gitea_api_policy.py + - scm_broker.py=scripts/scm_broker.py + - scm_broker_io.py=scripts/scm_broker_io.py + - scm_broker_server.py=scripts/scm_broker_server.py + - scm_broker_client.py=scripts/scm_broker_client.py + options: + disableNameSuffixHash: true diff --git a/services/hermes/scripts/gitea_api.py b/services/hermes/scm-common/scripts/gitea_api.py old mode 100755 new mode 100644 similarity index 65% rename from services/hermes/scripts/gitea_api.py rename to services/hermes/scm-common/scripts/gitea_api.py index b4a98eca..d207cc63 --- a/services/hermes/scripts/gitea_api.py +++ b/services/hermes/scm-common/scripts/gitea_api.py @@ -4,10 +4,10 @@ from __future__ import annotations import argparse +import base64 import json import os import re -import subprocess import sys import urllib.error import urllib.parse @@ -15,28 +15,29 @@ import urllib.request from collections.abc import Callable from pathlib import Path +from gitea_api_policy import ( + PolicyError, + _draft_title, + _reject_forbidden, + _validate_body, + _validate_pr_number, + _validate_pr_number_segment, + _validate_query, + _validate_ref, + _validate_ref_bounds, + _validate_repo, + _validate_sha, +) + CANONICAL_BASE_URL = "https://scm.bstein.dev" ALLOWED_OWNER = "atlas" -DEFAULT_TOKEN_FILE = Path("/runtime-access/gitea-token") -REPO_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,99}\Z") -SHA_RE = re.compile(r"[0-9a-fA-F]{40}\Z") +DEFAULT_TOKEN_FILE = Path("/vault/secrets/gitea-token") CREATE_PATH_RE = re.compile(r"/api/v1/repos/atlas/([^/]+)/pulls\Z") MAX_ERROR_BYTES = 8192 MAX_RESPONSE_BYTES = 2 * 1024 * 1024 -DRAFT_TITLE_PREFIX = "WIP: " -GIT_BIN = "/usr/bin/git" -SENSITIVE_BODY_PATTERNS = ( - re.compile(r"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----"), - re.compile(r"(?i)\b(?:password|passwd|secret|token|api[_-]?key)\s*[:=]"), - re.compile(r"(?i)\bauthorization\s*:\s*(?:bearer|token|basic)\s+\S+"), - re.compile(r"\b(?:gh[pousr]_[A-Za-z0-9]{20,}|sk-[A-Za-z0-9]{20,})\b"), - re.compile(r"\b(?:AKIA[0-9A-Z]{16}|AIza[0-9A-Za-z_-]{35})\b"), - re.compile(r"\beyJ[A-Za-z0-9_-]{12,}\.[A-Za-z0-9_-]{12,}\.[A-Za-z0-9_-]{8,}\b"), -) - - -class PolicyError(ValueError): - """Raised when a requested Forgejo operation exceeds the safe boundary.""" +MAX_API_TARGET_LENGTH = 768 +MAX_API_PATH_LENGTH = 512 +RAW_API_TARGET_RE = re.compile(r"[A-Za-z0-9/_.?&=-]+\Z") class RejectRedirectHandler(urllib.request.HTTPRedirectHandler): @@ -64,114 +65,48 @@ def read_token(path: Path = DEFAULT_TOKEN_FILE) -> str: def configured_base_url() -> str: """Reject attempts to redirect credentials away from the canonical origin.""" - configured = os.environ.get("GITEA_BASE_URL", CANONICAL_BASE_URL).rstrip("/") + configured = os.environ.get("GITEA_BASE_URL", CANONICAL_BASE_URL) if configured != CANONICAL_BASE_URL: raise PolicyError("Forgejo origin is fixed to the private Atlas SCM service") return configured -def _validate_repo(repo: str) -> str: - if not REPO_RE.fullmatch(repo) or repo in {".", ".."}: - raise PolicyError("repository name is outside the Atlas allowlist") - return repo - - -def _validate_ref(value: object, name: str) -> str: - """Validate the complete Git ref grammar using Git itself.""" - if not isinstance(value, str) or not value or value.startswith("-"): - raise PolicyError(f"{name} must be a same-repository branch name") - if any(ord(character) < 32 or ord(character) == 127 for character in value): - raise PolicyError(f"{name} must be a safe same-repository branch name") - result = subprocess.run( - [GIT_BIN, "check-ref-format", f"refs/heads/{value}"], - stdin=subprocess.DEVNULL, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - check=False, - ) - if result.returncode != 0: - raise PolicyError(f"{name} must be a safe same-repository branch name") - return value - - -def _validate_sha(value: object, name: str = "head SHA") -> str: - if not isinstance(value, str) or not SHA_RE.fullmatch(value): - raise PolicyError(f"{name} must be a full Git SHA-1") - return value.lower() - - -def _validate_text(value: object, name: str, maximum: int, *, required: bool) -> str: - if not isinstance(value, str): - raise PolicyError(f"{name} must be text") - if required and not value.strip(): - raise PolicyError(f"{name} must not be empty") - if len(value) > maximum or "\x00" in value: - raise PolicyError(f"{name} exceeds the safe request limit") - return value - - -def _validate_body(value: object, *, forbidden: tuple[str, ...] = ()) -> str: - body = _validate_text(value, "body", 16384, required=False) - if any(secret and secret in body for secret in forbidden): - raise PolicyError("pull-request body contains runtime credential material") - if any(pattern.search(body) for pattern in SENSITIVE_BODY_PATTERNS): - raise PolicyError("pull-request body resembles credential material") - return body - - -def _draft_title(value: object) -> str: - """Return a bounded title using Gitea's configured default draft prefix.""" - title = _validate_text(value, "title", 251, required=True).strip() - for prefix in ("WIP:", "[WIP]"): - if title.upper().startswith(prefix): - title = title[len(prefix) :].lstrip() - break - if not title: - raise PolicyError("title must contain text after the draft prefix") - return DRAFT_TITLE_PREFIX + title - - def _split_api_path(path: str) -> urllib.parse.SplitResult: + if not isinstance(path, str) or len(path) > MAX_API_TARGET_LENGTH: + raise PolicyError("API target exceeds the safe size limit") + if ( + not path.isascii() + or any(ord(character) < 32 or ord(character) == 127 for character in path) + or "\\" in path + or not RAW_API_TARGET_RE.fullmatch(path) + ): + raise PolicyError("API target contains non-canonical raw characters") target = urllib.parse.urlsplit(path) + canonical = urllib.parse.urlunsplit(("", "", target.path, target.query, "")) + if canonical != path: + raise PolicyError("API target is not in exact canonical form") if target.scheme or target.netloc or target.fragment: raise PolicyError("API path must be relative to the Atlas SCM origin") if not target.path.startswith("/api/v1/"): raise PolicyError("API path must start with /api/v1/") - if "%" in target.path or "//" in target.path or "/../" in f"{target.path}/": + if len(target.path) > MAX_API_PATH_LENGTH: + raise PolicyError("API path exceeds the safe size limit") + segments = target.path.split("/") + if "//" in target.path or any(segment in {".", ".."} for segment in segments): raise PolicyError("encoded or non-canonical API paths are not allowed") return target -def _validate_query(target: urllib.parse.SplitResult, allowed: set[str]) -> None: - try: - pairs = urllib.parse.parse_qsl( - target.query, keep_blank_values=True, strict_parsing=True - ) - except ValueError as exc: - raise PolicyError("invalid API query") from exc - if len({key for key, _ in pairs}) != len(pairs): - raise PolicyError("duplicate API query parameters are not allowed") - if any(key not in allowed for key, _ in pairs): - raise PolicyError("API query parameter is outside the read allowlist") - values = dict(pairs) - for name in ("page", "limit"): - if name not in values: - continue - if not values[name].isdigit() or int(values[name]) < 1: - raise PolicyError(f"{name} must be a positive integer") - if "limit" in values and int(values["limit"]) > 50: - raise PolicyError("read limit cannot exceed 50") - if "state" in values and values["state"] not in {"open", "closed", "all"}: - raise PolicyError("pull-request state is invalid") - - -def _authorize_read(target: urllib.parse.SplitResult) -> str: +def _authorize_read( + target: urllib.parse.SplitResult, *, forbidden: tuple[str, ...] = () +) -> str: """Allow only repository, PR, branch, commit, and status metadata reads.""" prefix = "/api/v1/repos/atlas/" remainder = target.path.removeprefix(prefix) if remainder == target.path: raise PolicyError("reads are limited to explicit Atlas repository metadata") repo, separator, suffix = remainder.partition("/") + _reject_forbidden(repo, "repository", forbidden) _validate_repo(repo) if not separator: @@ -180,10 +115,18 @@ def _authorize_read(target: urllib.parse.SplitResult) -> str: if suffix == "pulls": _validate_query(target, {"page", "limit", "state"}) return "pull-list" - if re.fullmatch(r"pulls/[1-9][0-9]*", suffix): + content_match = re.fullmatch(r"pulls/([^/]+)\.(?:patch|diff)", suffix) + if content_match: + _validate_pr_number_segment(content_match.group(1)) + raise PolicyError("repository API route is outside the metadata read allowlist") + pull_match = re.fullmatch(r"pulls/([^/]+)", suffix) + if pull_match: + _validate_pr_number_segment(pull_match.group(1)) _validate_query(target, set()) return "pull" - if re.fullmatch(r"pulls/[1-9][0-9]*/(?:commits|files)", suffix): + evidence_match = re.fullmatch(r"pulls/([^/]+)/(?:commits|files)", suffix) + if evidence_match: + _validate_pr_number_segment(evidence_match.group(1)) _validate_query(target, {"page", "limit"}) return "pull-evidence" if suffix == "branches": @@ -211,7 +154,7 @@ def _authorize_read(target: urllib.parse.SplitResult) -> str: def api_url(base_url: str, path: str) -> str: """Return a canonical same-origin URL without forwarding credentials.""" - if base_url.rstrip("/") != CANONICAL_BASE_URL: + if base_url != CANONICAL_BASE_URL: raise PolicyError("Forgejo origin is fixed to the private Atlas SCM service") target = _split_api_path(path) return urllib.parse.urlunsplit( @@ -219,14 +162,20 @@ def api_url(base_url: str, path: str) -> str: ) -def authorize_request(method: str, path: str, data: object | None) -> str: +def authorize_request( + method: str, + path: str, + data: object | None, + *, + forbidden: tuple[str, ...] = (), +) -> str: """Validate one request and return its bounded operation name.""" normalized_method = method.upper() target = _split_api_path(path) if normalized_method == "GET": if data is not None: raise PolicyError("read operations cannot include a request body") - return _authorize_read(target) + return _authorize_read(target, forbidden=forbidden) if target.query: raise PolicyError("mutating operations cannot include query parameters") if normalized_method != "POST": @@ -234,14 +183,17 @@ def authorize_request(method: str, path: str, data: object | None) -> str: match = CREATE_PATH_RE.fullmatch(target.path) if not match: raise PolicyError("POST is limited to creating an Atlas draft pull request") + _reject_forbidden(match.group(1), "repository", forbidden) _validate_repo(match.group(1)) if not isinstance(data, dict) or set(data) != {"base", "body", "head", "title"}: raise PolicyError("draft creation accepts only base, body, head, and title") - _validate_ref(data["base"], "base") - _validate_ref(data["head"], "head") - if data["title"] != _draft_title(data["title"]): + if data["title"] != _draft_title(data["title"], forbidden=forbidden): raise PolicyError("new pull requests must use the Gitea draft-title prefix") - _validate_body(data["body"]) + _validate_body(data["body"], forbidden=forbidden) + _validate_ref_bounds(data["base"], "base", forbidden=forbidden) + _validate_ref_bounds(data["head"], "head", forbidden=forbidden) + _validate_ref(data["base"], "base", forbidden=forbidden) + _validate_ref(data["head"], "head", forbidden=forbidden) return "create-draft" @@ -254,7 +206,19 @@ def build_request( data: object | None = None, ) -> urllib.request.Request: """Build an authorized request without putting the token in its URL or body.""" - authorize_request(method, path, data) + authorize_request(method, path, data, forbidden=(token,)) + return _build_request(method, path, base_url=base_url, token=token, data=data) + + +def _build_request( + method: str, + path: str, + *, + base_url: str, + token: str, + data: object | None = None, +) -> urllib.request.Request: + """Build one already-authorized upstream request.""" payload = None if data is not None: payload = json.dumps(data, separators=(",", ":")).encode("utf-8") @@ -281,6 +245,19 @@ def redact_bytes(value: bytes, token: str) -> bytes: ) +def _reject_response_credential(value: bytes, token: str) -> None: + """Fail closed if an upstream body reflects any ordinary token encoding.""" + raw = token.encode("utf-8") + forms = { + raw, + base64.b64encode(raw), + base64.b64encode(b"hermes-automation:" + raw), + urllib.parse.quote(token, safe="").encode("ascii"), + } + if any(form and form in value for form in forms): + raise PolicyError("Forgejo response contains runtime credential material") + + def _request( method: str, path: str, @@ -288,14 +265,23 @@ def _request( *, token: str, opener: Callable[..., object] = _safe_urlopen, + authorized: bool = False, + expected_status: int, ) -> bytes: - request = build_request( - method, path, base_url=configured_base_url(), token=token, data=data - ) + builder = _build_request if authorized else build_request + request = builder(method, path, base_url=configured_base_url(), token=token, data=data) with opener(request, timeout=30) as response: # type: ignore[attr-defined] + status = getattr(response, "status", None) + if status is None and hasattr(response, "getcode"): + status = response.getcode() # type: ignore[attr-defined] + if status != expected_status: + raise PolicyError("Forgejo returned an unexpected HTTP status") + if response.headers.get_content_type() != "application/json": # type: ignore[attr-defined] + raise PolicyError("Forgejo returned an unexpected response type") body = response.read(MAX_RESPONSE_BYTES + 1) # type: ignore[attr-defined] if len(body) > MAX_RESPONSE_BYTES: raise PolicyError("Forgejo response exceeds the safe size limit") + _reject_response_credential(body, token) return redact_bytes(body, token) @@ -303,7 +289,9 @@ def read( path: str, *, token: str, opener: Callable[..., object] = _safe_urlopen ) -> bytes: """Read one explicitly allowed Atlas repository metadata resource.""" - return _request("GET", path, None, token=token, opener=opener) + return _request( + "GET", path, None, token=token, opener=opener, expected_status=200 + ) def _nested(document: dict[str, object], *keys: str) -> object: @@ -332,9 +320,7 @@ def _require_create_response( raise PolicyError("Forgejo returned invalid pull-request metadata") from exc if not isinstance(document, dict): raise PolicyError("Forgejo returned invalid pull-request metadata") - number = document.get("number") - if not isinstance(number, int) or isinstance(number, bool) or number < 1: - raise PolicyError("Forgejo omitted a valid pull-request number") + number = _validate_pr_number(document.get("number")) full_name = f"{ALLOWED_OWNER}/{repo}" expected_html = f"{CANONICAL_BASE_URL}/{full_name}/pulls/{number}" checks = ( @@ -368,12 +354,15 @@ def create_draft( opener: Callable[..., object] = _safe_urlopen, ) -> bytes: """Create and verify a same-repository draft pull request for human review.""" + _reject_forbidden(repo, "repository", (token,)) repo = _validate_repo(repo) - base = _validate_ref(base, "base") - head = _validate_ref(head, "head") - head_sha = _validate_sha(head_sha) - title = _draft_title(title) + title = _draft_title(title, forbidden=(token,)) body = _validate_body(body, forbidden=(token,)) + head_sha = _validate_sha(head_sha) + _validate_ref_bounds(base, "base", forbidden=(token,)) + _validate_ref_bounds(head, "head", forbidden=(token,)) + base = _validate_ref(base, "base", forbidden=(token,)) + head = _validate_ref(head, "head", forbidden=(token,)) data = {"base": base, "body": body, "head": head, "title": title} result = _request( "POST", @@ -381,6 +370,8 @@ def create_draft( data, token=token, opener=opener, + authorized=True, + expected_status=201, ) _require_create_response( result, @@ -417,16 +408,24 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: return parser.parse_args(argv) -def _operation(args: argparse.Namespace) -> tuple[str, str, object | None]: +def _operation( + args: argparse.Namespace, *, validate_refs: bool = True +) -> tuple[str, str, object | None]: if args.operation == "read": return "GET", args.path, None repo = _validate_repo(args.repo) _validate_sha(args.head_sha) + body = _validate_body(args.body) + title = _draft_title(args.title) + _validate_ref_bounds(args.base, "base") + _validate_ref_bounds(args.head, "head") + base = _validate_ref(args.base, "base") if validate_refs else args.base + head = _validate_ref(args.head, "head") if validate_refs else args.head data = { - "base": _validate_ref(args.base, "base"), - "body": _validate_body(args.body), - "head": _validate_ref(args.head, "head"), - "title": _draft_title(args.title), + "base": base, + "body": body, + "head": head, + "title": title, } return "POST", f"/api/v1/repos/{ALLOWED_OWNER}/{repo}/pulls", data @@ -440,12 +439,11 @@ def _write_body(body: bytes) -> None: def main(argv: list[str] | None = None) -> int: """Validate, optionally describe, and execute one safe Forgejo operation.""" - token = "" try: args = parse_args(argv) - method, path, data = _operation(args) - operation = authorize_request(method, path, data) if args.dry_run: + method, path, data = _operation(args, validate_refs=False) + operation = authorize_request(method, path, data) print( json.dumps( { @@ -462,29 +460,27 @@ def main(argv: list[str] | None = None) -> int: ) ) return 0 - token = read_token() + method, path, data = _operation(args, validate_refs=False) + from scm_broker_client import create_draft as broker_create_draft + from scm_broker_client import read as broker_read + if args.operation == "read": - body = read(path, token=token) + body = broker_read(path) else: assert isinstance(data, dict) - body = create_draft( + body = broker_create_draft( args.repo, base=str(data["base"]), head=str(data["head"]), head_sha=args.head_sha, title=str(data["title"]), body=str(data["body"]), - token=token, ) _write_body(body) return 0 except urllib.error.HTTPError as exc: - body = redact_bytes(exc.read(MAX_ERROR_BYTES), token) if token else b"" - print(f"Forgejo request failed with HTTP {exc.code}", file=sys.stderr) - if body: - sys.stderr.buffer.write(body) - if not body.endswith(b"\n"): - sys.stderr.buffer.write(b"\n") + exc.read(MAX_ERROR_BYTES) + print(f"SCM broker request failed with HTTP {exc.code}", file=sys.stderr) return 1 except (OSError, PolicyError, ValueError, json.JSONDecodeError): print( diff --git a/services/hermes/scm-common/scripts/gitea_api_policy.py b/services/hermes/scm-common/scripts/gitea_api_policy.py new file mode 100644 index 00000000..db8833cf --- /dev/null +++ b/services/hermes/scm-common/scripts/gitea_api_policy.py @@ -0,0 +1,490 @@ +"""Validation policy for Hermes' least-authority Atlas Forgejo client. + +The screening catches structured credential assignments, known token formats, +and long high-entropy values. It is a fail-closed accident barrier, not proof +that text is secret-free: a short or multiword secret under an innocuous key is +not reliably distinguishable from prose and must never be supplied by callers. +JSON depth/nodes/documents and multiline gaps are deliberately bounded; large +or unusual configuration blobs are not a supported pull-request field format. +""" + +# ruff: noqa: SIM905 + +from __future__ import annotations + +import json +import re +import subprocess +import urllib.parse +from collections import Counter +from math import log2 + +REPO_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,99}\Z") +SHA_RE = re.compile(r"[0-9a-fA-F]{40}\Z") +DRAFT_TITLE_PREFIX = "WIP: " +GIT_BIN = "/usr/bin/git" +MAX_QUERY_LENGTH = 128 +MAX_QUERY_ITEMS = 3 +MAX_QUERY_KEY_LENGTH = 16 +MAX_QUERY_VALUE_LENGTH = 16 +MAX_PAGE = 10_000 +MAX_LIMIT = 50 +MAX_PR_NUMBER = 2_147_483_647 +MAX_PR_NUMBER_DIGITS = 10 +MAX_REF_CHARACTERS = 200 +MAX_REF_UTF8_BYTES = 255 +MAX_TITLE_UTF8_BYTES = 512 +MAX_BODY_UTF8_BYTES = 32_768 +MAX_JSON_DOCUMENTS = 32 +MAX_JSON_NODES = 2_048 +MAX_JSON_DEPTH = 32 +CANONICAL_QUERY_RE = re.compile(r"[A-Za-z0-9_=&-]*\Z") +ASSIGNMENT_RE = re.compile( + r"""(?mx) + (?[\"']?) + (?P\.?[A-Za-z][A-Za-z0-9_.-]{0,127}) + (?P=key_quote) + [ \t]*(?P[:=])[ \t]* + (?P + \"(?:\\.|[^\"\\\r\n])*\" | + '(?:\\.|[^'\\\r\n])*' | + [^\r\n,;}&]{0,2048} + ) + """ +) +JSON_KEY_RE = re.compile(r'"(?P[^"]{1,512})"(?P[\x00-\x20\x7f]{0,256}):') +MULTILINE_ASSIGNMENT_RE = re.compile( + r"""(?mx) + (?["']?) + (?P\.?[A-Za-z][A-Za-z0-9_.-]{0,127}) + (?P=key_quote) + [ \t]{0,64}(?:\r?\n[ \t]{0,64}){0,4} + (?P[:=]) + [ \t]{0,64}(?:\r?\n[ \t]{0,64}){1,4} + (?P[^\r\n,;}&]{1,2048}) + """ +) +CAMEL_BOUNDARY_RE = re.compile(r"(?<=[a-z0-9])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])") +CREDENTIAL_SCHEME_RE = re.compile( + r"(?i)\b(?:bearer|basic|token)\s+" + r"(?=[A-Za-z0-9+/_=.:-]{8,}\b)(?=[A-Za-z0-9+/_=.:-]*[0-9+/_=.-])" + r"[A-Za-z0-9+/_=.:-]{8,}" +) +STANDALONE_SECRET_PATTERNS = tuple( + re.compile(pattern, re.IGNORECASE) + for pattern in ( + r"(? str: + if not REPO_RE.fullmatch(repo) or repo in {".", ".."}: + raise PolicyError("repository name is outside the Atlas allowlist") + return repo + + +def _reject_forbidden(value: object, name: str, forbidden: tuple[str, ...]) -> None: + """Reject exact runtime credentials before invoking helpers or the network.""" + if not isinstance(value, str) or any(secret and secret in value for secret in forbidden): + raise PolicyError(f"{name} contains runtime credential material") + + +def _validate_ref_bounds( + value: object, name: str, *, forbidden: tuple[str, ...] = () +) -> str: + """Reject oversized or non-encodable refs before invoking Git.""" + _reject_forbidden(value, name, forbidden) + if not value or len(value) > MAX_REF_CHARACTERS: + raise PolicyError(f"{name} exceeds the safe branch-name limit") + try: + encoded = value.encode("utf-8", errors="strict") + except UnicodeEncodeError as exc: + raise PolicyError(f"{name} is not valid UTF-8 text") from exc + if len(encoded) > MAX_REF_UTF8_BYTES: + raise PolicyError(f"{name} exceeds the safe UTF-8 branch-name limit") + return value + + +def _validate_ref( + value: object, name: str, *, forbidden: tuple[str, ...] = () +) -> str: + """Validate the complete Git ref grammar using Git itself.""" + value = _validate_ref_bounds(value, name, forbidden=forbidden) + if value.startswith("-"): + raise PolicyError(f"{name} must be a same-repository branch name") + if any(ord(character) < 32 or ord(character) == 127 for character in value): + raise PolicyError(f"{name} must be a safe same-repository branch name") + result = subprocess.run( + [GIT_BIN, "check-ref-format", f"refs/heads/{value}"], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + if result.returncode != 0: + raise PolicyError(f"{name} must be a safe same-repository branch name") + return value + + +def _validate_sha(value: object, name: str = "head SHA") -> str: + if not isinstance(value, str) or not SHA_RE.fullmatch(value): + raise PolicyError(f"{name} must be a full Git SHA-1") + return value.lower() + + +def _validate_pr_number_segment(value: object) -> int: + """Validate a canonical bounded pull-request number from an API path.""" + if ( + not isinstance(value, str) + or len(value) > MAX_PR_NUMBER_DIGITS + or not re.fullmatch(r"[1-9][0-9]*", value) + ): + raise PolicyError("pull-request number must use bounded canonical ASCII digits") + number = int(value) + if number > MAX_PR_NUMBER: + raise PolicyError("pull-request number is outside its allowed range") + return number + + +def _validate_pr_number(value: object) -> int: + """Validate a bounded pull-request number returned by Forgejo.""" + if ( + not isinstance(value, int) + or isinstance(value, bool) + or not 1 <= value <= MAX_PR_NUMBER + ): + raise PolicyError("Forgejo omitted a valid pull-request number") + return value + + +def _validate_text( + value: object, + name: str, + maximum: int, + maximum_bytes: int, + *, + required: bool, +) -> str: + if not isinstance(value, str): + raise PolicyError(f"{name} must be text") + if required and not value.strip(): + raise PolicyError(f"{name} must not be empty") + try: + encoded = value.encode("utf-8", errors="strict") + except UnicodeEncodeError as exc: + raise PolicyError(f"{name} is not valid UTF-8 text") from exc + if len(value) > maximum or len(encoded) > maximum_bytes or "\x00" in value: + raise PolicyError(f"{name} exceeds the safe request limit") + return value + + +def _normalized_key(value: str) -> tuple[tuple[str, ...], str]: + """Split camelCase and separator-based assignment keys into semantics.""" + separated = CAMEL_BOUNDARY_RE.sub(" ", value.strip(".\"'")) + words = tuple(re.findall(r"[A-Za-z0-9]+", separated.lower())) + return words, "".join(words) + + +def _is_sensitive_key(value: str) -> bool: + words, compact = _normalized_key(value) + word_set = set(words) + if word_set & SENSITIVE_KEY_WORDS: + return True + if compact in SENSITIVE_COMPACT_KEYS: + return True + if compact.endswith(SENSITIVE_COMPACT_SUFFIXES): + return True + if "key" in word_set and word_set & KEY_MODIFIER_WORDS: + return True + if {"client", "email"} <= word_set or {"client", "id"} <= word_set: + return True + if {"access", "id"} <= word_set or {"connection", "string"} <= word_set: + return True + return compact.endswith("configjson") + + +def _strip_assignment_value(value: str) -> tuple[str, bool]: + stripped = value.strip() + quoted = ( + len(stripped) >= 2 and stripped[0] in {'"', "'"} and stripped[-1] == stripped[0] + ) + if quoted: + stripped = stripped[1:-1].strip() + return stripped, quoted + + +def _looks_like_prose(value: str) -> bool: + words = re.findall(r"[A-Za-z]+(?:[-'][A-Za-z]+)?", value) + if len(words) < 2: + return False + if re.search(r"[$`{}\[\]\\@/:=+_]", value): + return False + return words[0].lower() in PROSE_LEAD_WORDS + + +def _looks_sensitive_assignment_value(value: str) -> bool: + stripped, quoted = _strip_assignment_value(value) + if not stripped: + return False + if quoted or stripped.startswith(("{", "[", "|", ">", "!!", "&", "*")): + return True + if CREDENTIAL_SCHEME_RE.search(stripped): + return True + if any(pattern.search(stripped) for pattern in STANDALONE_SECRET_PATTERNS): + return True + if re.search(r"(?:https?://|[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+)", stripped): + return True + if re.search(r"(?:\$\{|\$\(|\\[nrt]|[A-Za-z0-9+/]{16,}={0,3}\Z)", stripped): + return True + return not _looks_like_prose(stripped) + + +def _is_structural_credential_assignment(key: str, value: str) -> bool: + words, compact_key = _normalized_key(key) + stripped, _quoted = _strip_assignment_value(value) + _value_words, compact_value = _normalized_key(stripped) + return ( + compact_key == "type" and compact_value in {"serviceaccount", "credential"} + ) or ("service" in words and "account" in words and bool(stripped)) + + +def _decode_json_key(raw: str) -> tuple[str, bool]: + """Decode one bounded JSON key without silently accepting bad escapes.""" + try: + decoded = json.loads(f'"{raw}"') + except (json.JSONDecodeError, UnicodeError): + approximate = re.sub(r"[^A-Za-z0-9]+", "_", raw.replace("\\", "")) + return approximate, False + return decoded if isinstance(decoded, str) else "", True + + +def _json_value_has_sensitive_assignment( + value: object, *, depth: int = 0, nodes: list[int] | None = None +) -> bool: + """Walk one decoded JSON value with explicit depth and node ceilings.""" + if nodes is None: + nodes = [0] + nodes[0] += 1 + if nodes[0] > MAX_JSON_NODES or depth > MAX_JSON_DEPTH: + raise PolicyError("structured pull-request text exceeds the scan limit") + if isinstance(value, dict): + for key, assigned in value.items(): + if not isinstance(key, str): + continue + if _is_sensitive_key(key): + return True + assigned_text = assigned if isinstance(assigned, str) else "" + if _is_structural_credential_assignment(key, assigned_text): + return True + if _json_value_has_sensitive_assignment( + assigned, depth=depth + 1, nodes=nodes + ): + return True + elif isinstance(value, list): + return any( + _json_value_has_sensitive_assignment(item, depth=depth + 1, nodes=nodes) + for item in value + ) + return False + + +def _decoded_json_documents(value: str): + """Yield bounded embedded JSON object or array fragments.""" + decoder = json.JSONDecoder() + index = 0 + attempts = 0 + while attempts < MAX_JSON_DOCUMENTS: + positions = [ + position for token in "{[" if (position := value.find(token, index)) >= 0 + ] + if not positions: + return + start = min(positions) + attempts += 1 + try: + document, end = decoder.raw_decode(value, start) + except (json.JSONDecodeError, RecursionError, ValueError): + index = start + 1 + continue + yield document + index = max(end, start + 1) + if "{" in value[index:] or "[" in value[index:]: + raise PolicyError("structured pull-request text exceeds the scan limit") + + +def _has_structured_sensitive_assignment(value: str) -> bool: + """Detect bounded JSON and multiline YAML/env credential assignments.""" + for candidate in JSON_KEY_RE.finditer(value): + decoded, valid = _decode_json_key(candidate.group("key")) + if _is_sensitive_key(decoded): + return True + if not valid and _is_sensitive_key(candidate.group("key").replace("\\", "")): + return True + for candidate in MULTILINE_ASSIGNMENT_RE.finditer(value): + key = candidate.group("key") + assigned = candidate.group("value") + if _is_structural_credential_assignment(key, assigned) or ( + _is_sensitive_key(key) and _looks_sensitive_assignment_value(assigned) + ): + return True + return any( + _json_value_has_sensitive_assignment(item) + for item in _decoded_json_documents(value) + ) + + +def _has_high_entropy_token(value: str) -> bool: + for match in HIGH_ENTROPY_TOKEN_RE.finditer(value): + token = match.group(0) + if len(token) > 256: + return True + groups = sum( + bool(re.search(pattern, token)) + for pattern in (r"[a-z]", r"[A-Z]", r"[0-9]", r"[+/_=-]") + ) + counts = Counter(token) + entropy = -sum( + (count / len(token)) * log2(count / len(token)) for count in counts.values() + ) + if groups >= 3 and entropy >= 4.0: + return True + return False + + +def _reject_sensitive(value: str, name: str, forbidden: tuple[str, ...]) -> None: + if any(secret and secret in value for secret in forbidden): + raise PolicyError(f"pull-request {name} contains runtime credential material") + if CREDENTIAL_SCHEME_RE.search(value): + raise PolicyError(f"pull-request {name} resembles credential material") + if any(pattern.search(value) for pattern in STANDALONE_SECRET_PATTERNS): + raise PolicyError(f"pull-request {name} resembles credential material") + if _has_structured_sensitive_assignment(value): + raise PolicyError(f"pull-request {name} resembles credential material") + for candidate in ASSIGNMENT_RE.finditer(value): + key = candidate.group("key") + assigned = candidate.group("value") + if _is_structural_credential_assignment(key, assigned) or ( + _is_sensitive_key(key) and _looks_sensitive_assignment_value(assigned) + ): + raise PolicyError(f"pull-request {name} resembles credential material") + if _has_high_entropy_token(value): + raise PolicyError(f"pull-request {name} resembles credential material") + + +def _validate_body(value: object, *, forbidden: tuple[str, ...] = ()) -> str: + body = _validate_text(value, "body", 16384, MAX_BODY_UTF8_BYTES, required=False) + _reject_sensitive(body, "body", forbidden) + return body + + +def _draft_title(value: object, *, forbidden: tuple[str, ...] = ()) -> str: + """Return a bounded, secret-screened Gitea draft title.""" + title = _validate_text( + value, "title", 251, MAX_TITLE_UTF8_BYTES, required=True + ).strip() + _reject_sensitive(title, "title", forbidden) + for prefix in ("WIP:", "[WIP]"): + if title.upper().startswith(prefix): + title = title[len(prefix) :].lstrip() + break + if not title: + raise PolicyError("title must contain text after the draft prefix") + return DRAFT_TITLE_PREFIX + title + + +def _validate_query(target: urllib.parse.SplitResult, allowed: set[str]) -> None: + """Accept only short canonical ASCII query strings with bounded pagination.""" + raw = target.query + if ( + len(raw) > MAX_QUERY_LENGTH + or not raw.isascii() + or not CANONICAL_QUERY_RE.fullmatch(raw) + or "%" in raw + ): + raise PolicyError("API query must use short canonical ASCII form") + try: + pairs = urllib.parse.parse_qsl(raw, keep_blank_values=True, strict_parsing=True) + except ValueError as exc: + raise PolicyError("invalid API query") from exc + if len(pairs) > MAX_QUERY_ITEMS: + raise PolicyError("too many API query parameters") + if len({key for key, _ in pairs}) != len(pairs): + raise PolicyError("duplicate API query parameters are not allowed") + if any( + not key + or len(key) > MAX_QUERY_KEY_LENGTH + or len(value) > MAX_QUERY_VALUE_LENGTH + for key, value in pairs + ): + raise PolicyError("API query key or value exceeds its safe limit") + if any(key not in allowed for key, _ in pairs): + raise PolicyError("API query parameter is outside the read allowlist") + values = dict(pairs) + for name in ("page", "limit"): + if name not in values: + continue + if not re.fullmatch(r"[0-9]+", values[name]): + raise PolicyError(f"{name} must use ASCII decimal digits") + number = int(values[name]) + if values[name] != str(number): + raise PolicyError(f"{name} must use canonical ASCII decimal form") + ceiling = MAX_PAGE if name == "page" else MAX_LIMIT + if not 1 <= number <= ceiling: + raise PolicyError(f"{name} is outside its allowed range") + if "state" in values and values["state"] not in {"open", "closed", "all"}: + raise PolicyError("pull-request state is invalid") diff --git a/services/hermes/scm-common/scripts/scm_broker.py b/services/hermes/scm-common/scripts/scm_broker.py new file mode 100644 index 00000000..a0c4043c --- /dev/null +++ b/services/hermes/scm-common/scripts/scm_broker.py @@ -0,0 +1,496 @@ +#!/usr/bin/env python3 +"""Serve the credential-isolated Atlas API and Git smart-HTTP boundary.""" + +from __future__ import annotations + +import argparse +import base64 +import json +import re +import socket +import tempfile +import time +import urllib.error +import urllib.parse +import urllib.request +from http.server import BaseHTTPRequestHandler +from typing import BinaryIO + +from gitea_api import ( + CANONICAL_BASE_URL, + PolicyError, + create_draft, + read, + read_token, +) +from gitea_api_policy import _reject_forbidden, _validate_ref, _validate_repo +from scm_broker_io import RejectRedirect, response_status as _status, spool_response +from scm_broker_server import AbsoluteHeaderDeadlineMixin, BoundedThreadingHTTPServer + +BROKER_PORT = 9081 +GIT_USER = "hermes-automation" +MAX_CONTROL_BODY = 64 * 1024 +MAX_GIT_REQUEST = 128 * 1024 * 1024 +MAX_GIT_RESPONSE = 128 * 1024 * 1024 +MAX_PUSH_COMMANDS = 16 +MAX_HEADERS = 32 +MAX_HEADER_BYTES = 16 * 1024 +SPOOL_MEMORY_LIMIT = 1024 * 1024 +STREAM_CHUNK = 64 * 1024 +INBOUND_HEADER_TIMEOUT = 10.0 +INBOUND_BODY_TIMEOUT = 120.0 +ZERO_SHA = b"0" * 40 +GIT_PATH_RE = re.compile( + r"/git/atlas/(?P[A-Za-z0-9][A-Za-z0-9._-]{0,99})\.git/" + r"(?Pinfo/refs|git-upload-pack|git-receive-pack)\Z" +) +FEATURE_REF_RE = re.compile( + r"refs/heads/(?:(?:feature|fix|hermes|handoff)/[A-Za-z0-9][A-Za-z0-9._/-]{0,190})\Z" +) + + +UPSTREAM_OPENER = urllib.request.build_opener(RejectRedirect()) + + +def _read_bounded( + stream, + maximum: int, + length: int | None = None, + *, + deadline: float | None = None, + set_timeout=None, +) -> bytes: + if length is not None and not 0 <= length <= maximum: + raise PolicyError("SCM request exceeds the safe size limit") + wanted = maximum + 1 if length is None else length + chunks: list[bytes] = [] + total = 0 + while total < wanted: + if deadline is not None: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise PolicyError("SCM request body deadline exceeded") + if set_timeout is not None: + set_timeout(remaining) + chunk = stream.read(min(STREAM_CHUNK, wanted - total)) + if not chunk: + break + chunks.append(chunk) + total += len(chunk) + value = b"".join(chunks) + if len(value) > maximum or (length is not None and len(value) != length): + raise PolicyError("SCM request exceeds the safe size limit") + return value + + +def _spool_bounded( + stream, + maximum: int, + length: int, + *, + token: str, + context: str, + deadline: float, + set_timeout=None, +) -> tuple[BinaryIO, int]: + """Copy a fixed-length exchange through a bounded-memory disk spool.""" + if not 0 <= length <= maximum: + raise PolicyError("SCM request exceeds the safe size limit") + spool = tempfile.SpooledTemporaryFile( # noqa: SIM115 - caller owns returned spool + max_size=SPOOL_MEMORY_LIMIT, dir="/tmp" + ) + remaining = length + forms = _credential_forms(token) + carry = b"" + try: + while remaining: + remaining_time = deadline - time.monotonic() + if remaining_time <= 0: + raise PolicyError("SCM request body deadline exceeded") + if set_timeout is not None: + set_timeout(remaining_time) + chunk = stream.read(min(STREAM_CHUNK, remaining)) + if not chunk: + raise PolicyError("SCM request body ended early") + candidate = carry + chunk + if any(form in candidate for form in forms): + raise PolicyError(f"{context} contains runtime credential material") + width = max(len(form) for form in forms) - 1 + carry = candidate[-width:] if width else b"" + spool.write(chunk) + remaining -= len(chunk) + spool.seek(0) + return spool, length + except Exception: + spool.close() + raise + + +def _spool_response(response, maximum: int, token: str) -> tuple[BinaryIO, int]: + return spool_response( + response, + maximum, + _credential_forms(token), + memory_limit=SPOOL_MEMORY_LIMIT, + chunk_size=STREAM_CHUNK, + deadline_seconds=INBOUND_BODY_TIMEOUT, + ) + + +def _load_json(handler: BaseHTTPRequestHandler) -> dict[str, object]: + if handler.headers.get("Transfer-Encoding"): + raise PolicyError("chunked broker control requests are not allowed") + if handler.headers.get_content_type() != "application/json": + raise PolicyError("broker control request must be JSON") + length = _content_length(handler.headers, MAX_CONTROL_BODY) + deadline = time.monotonic() + INBOUND_BODY_TIMEOUT + value = json.loads( + _read_bounded( + handler.rfile, + MAX_CONTROL_BODY, + length, + deadline=deadline, + set_timeout=handler.connection.settimeout, + ) + ) + if not isinstance(value, dict): + raise PolicyError("broker control request must be an object") + return value + + +def _git_target(raw: str) -> tuple[str, str, str]: + if ( + not raw.isascii() + or any(ord(character) < 32 or ord(character) == 127 for character in raw) + or "%" in raw + or "\\" in raw + or len(raw) > 512 + ): + raise PolicyError("Git target is not canonical ASCII") + target = urllib.parse.urlsplit(raw) + canonical = urllib.parse.urlunsplit(("", "", target.path, target.query, "")) + if canonical != raw: + raise PolicyError("Git target is not in exact canonical form") + if target.scheme or target.netloc or target.fragment: + raise PolicyError("Git target must be relative") + match = GIT_PATH_RE.fullmatch(target.path) + if not match: + raise PolicyError("Git target is outside the Atlas allowlist") + repo = _validate_repo(match.group("repo")) + operation = match.group("operation") + query = target.query + if operation == "info/refs": + if query not in {"service=git-upload-pack", "service=git-receive-pack"}: + raise PolicyError("Git discovery service is outside the allowlist") + service = query.removeprefix("service=") + elif query: + raise PolicyError("Git RPC query parameters are not allowed") + else: + service = operation + return repo, operation, service + + +def _content_length(headers: object, maximum: int) -> int: + """Parse one short canonical bounded Content-Length header.""" + raw = headers.get("Content-Length", "") # type: ignore[attr-defined] + if ( + not isinstance(raw, str) + or not raw.isascii() + or len(raw) > 10 + or not raw.isdigit() + ): + raise PolicyError("SCM request length is invalid") + length = int(raw) + if raw != str(length) or not 0 <= length <= maximum: + raise PolicyError("SCM request length is invalid") + return length + + +def _credential_forms(token: str) -> tuple[bytes, ...]: + basic = base64.b64encode(f"{GIT_USER}:{token}".encode()) + return token.encode("utf-8"), basic, b"Basic " + basic + + +def _reject_credential_bytes(value: bytes, token: str, context: str) -> None: + if any(form in value for form in _credential_forms(token)): + raise PolicyError(f"{context} contains runtime credential material") + + +def _receive_prefix(body: bytes | BinaryIO) -> bytes: + if isinstance(body, bytes): + return body + position = body.tell() + try: + body.seek(0) + return body.read(64 * 1024) + finally: + body.seek(position) + + +def _validate_receive_pack(body: bytes | BinaryIO, token: str) -> None: + """Permit only creation of new, namespaced feature branches.""" + body = _receive_prefix(body) + _reject_credential_bytes(body, token, "Git request") + position = 0 + commands = 0 + while position + 4 <= len(body): + header = body[position : position + 4] + if not re.fullmatch(rb"[0-9a-f]{4}", header): + raise PolicyError("Git receive-pack command framing is invalid") + size = int(header, 16) + if size == 0: + if commands == 0: + raise PolicyError("Git receive-pack request has no ref command") + return + if size < 4 or position + size > len(body): + raise PolicyError("Git receive-pack command framing is invalid") + command = body[position + 4 : position + size].rstrip(b"\n") + command = command.split(b"\x00", 1)[0] + fields = command.split(b" ") + if len(fields) != 3 or not all( + re.fullmatch(rb"[0-9a-f]{40}", item) for item in fields[:2] + ): + raise PolicyError("Git receive-pack ref command is invalid") + old_sha, new_sha, raw_ref = fields + if old_sha != ZERO_SHA or new_sha == ZERO_SHA: + raise PolicyError("Git broker permits only new feature-branch creation") + try: + ref = raw_ref.decode("ascii") + except UnicodeDecodeError as exc: + raise PolicyError("Git ref must be canonical ASCII") from exc + if not FEATURE_REF_RE.fullmatch(ref): + raise PolicyError("Git push is limited to namespaced feature branches") + _validate_ref(ref.removeprefix("refs/heads/"), "head", forbidden=(token,)) + commands += 1 + if commands > MAX_PUSH_COMMANDS: + raise PolicyError("Git push contains too many ref commands") + position += size + raise PolicyError("Git receive-pack request omitted its command terminator") + + +def _upstream_git_request( + target: str, + *, + method: str, + body: bytes | BinaryIO | None, + content_type: str | None, + expected_type: str, + token: str, + body_length: int | None = None, + stream_result: bool = False, + opener=UPSTREAM_OPENER.open, +) -> bytes | tuple[BinaryIO, int]: + credentials = base64.b64encode(f"{GIT_USER}:{token}".encode()).decode("ascii") + headers = {"Accept": expected_type, "Authorization": f"Basic {credentials}"} + if content_type: + headers["Content-Type"] = content_type + if body is not None: + if body_length is None: + if not isinstance(body, bytes): + raise PolicyError("Git upstream request length is missing") + body_length = len(body) + headers["Content-Length"] = str(body_length) + request = urllib.request.Request( + CANONICAL_BASE_URL + target, + data=body, + method=method, + headers=headers, + ) + with opener(request, timeout=120) as response: + if _status(response) != 200: + raise PolicyError("Git upstream returned an unexpected HTTP status") + if response.headers.get_content_type() != expected_type: + raise PolicyError("Git upstream returned an unexpected response type") + result, result_length = _spool_response(response, MAX_GIT_RESPONSE, token) + if stream_result: + return result, result_length + try: + return result.read() + finally: + result.close() + + +class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler): + """Expose only bounded metadata, draft creation, and smart-HTTP Git.""" + + server_version = "HermesSCMBroker/1" + sys_version = "" + header_deadline_seconds = INBOUND_HEADER_TIMEOUT + + def _validate_headers(self) -> None: + items = list(self.headers.items()) + if len(items) > MAX_HEADERS: + raise PolicyError("SCM request has too many headers") + total = 0 + for name, value in items: + if not name.isascii() or not value.isascii(): + raise PolicyError("SCM request headers must be ASCII") + total += len(name) + len(value) + 4 + if any(ord(character) < 32 and character != "\t" for character in value): + raise PolicyError("SCM request header contains controls") + if total > MAX_HEADER_BYTES: + raise PolicyError("SCM request headers exceed the safe limit") + if len(self.headers.get_all("Content-Length", [])) > 1: + raise PolicyError("SCM request has duplicate Content-Length") + + def _stream( + self, status: int, content_type: str, body: BinaryIO, length: int + ) -> None: + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(length)) + self.end_headers() + deadline = time.monotonic() + INBOUND_BODY_TIMEOUT + while True: + chunk = body.read(STREAM_CHUNK) + if not chunk: + break + remaining = deadline - time.monotonic() + if remaining <= 0: + raise PolicyError("SCM response write deadline exceeded") + self.connection.settimeout(remaining) + self.wfile.write(chunk) + + def log_message(self, _format: str, *_args: object) -> None: + return + + def _json(self, status: int, body: bytes) -> None: + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.send_header("Cache-Control", "no-store") + self.end_headers() + self.wfile.write(body) + + def _reject(self, status: int = 400) -> None: + self._json(status, b'{"error":"request rejected"}') + + def do_GET(self) -> None: + try: + self._validate_headers() + if self.path == "/healthz": + self._json(200, b'{"status":"ok"}') + return + repo, operation, service = _git_target(self.path) + if operation != "info/refs": + raise PolicyError("Git RPC requires POST") + token = read_token() + _reject_forbidden(repo, "repository", (token,)) + expected = f"application/x-{service}-advertisement" + streamed = _upstream_git_request( + f"/atlas/{repo}.git/info/refs?service={service}", + method="GET", + body=None, + content_type=None, + expected_type=expected, + token=token, + stream_result=True, + ) + body, length = streamed + try: + self._stream(200, expected, body, length) + finally: + body.close() + except ( + OSError, + PolicyError, + socket.timeout, + urllib.error.URLError, + ValueError, + ): + self._reject() + + def do_POST(self) -> None: + try: + self._validate_headers() + self.connection.settimeout(INBOUND_BODY_TIMEOUT) + if self.path in {"/v1/metadata", "/v1/drafts"}: + self._control() + else: + self._git_rpc() + except ( + OSError, + PolicyError, + socket.timeout, + urllib.error.URLError, + ValueError, + json.JSONDecodeError, + ): + self._reject() + + def _control(self) -> None: + data = _load_json(self) + token = read_token() + if self.path == "/v1/metadata": + if set(data) != {"path"} or not isinstance(data["path"], str): + raise PolicyError("metadata request fields are invalid") + result = read(data["path"], token=token) + else: + expected = {"base", "body", "head", "head_sha", "repo", "title"} + if set(data) != expected or not all( + isinstance(data[key], str) for key in expected + ): + raise PolicyError("draft request fields are invalid") + result = create_draft(token=token, **data) # type: ignore[arg-type] + if token.encode("utf-8") in result: + raise PolicyError("SCM upstream reflected credential material") + self._json(200, result) + + def _git_rpc(self) -> None: + repo, operation, service = _git_target(self.path) + if ( + operation not in {"git-upload-pack", "git-receive-pack"} + or service != operation + ): + raise PolicyError("Git RPC operation is outside the allowlist") + expected_request = f"application/x-{service}-request" + if self.headers.get_content_type() != expected_request or self.headers.get( + "Transfer-Encoding" + ): + raise PolicyError("Git RPC request type is invalid") + length = _content_length(self.headers, MAX_GIT_REQUEST) + token = read_token() + _reject_forbidden(repo, "repository", (token,)) + body, body_length = _spool_bounded( + self.rfile, + MAX_GIT_REQUEST, + length, + token=token, + context="Git request", + deadline=time.monotonic() + INBOUND_BODY_TIMEOUT, + set_timeout=self.connection.settimeout, + ) + try: + if service == "git-receive-pack": + _validate_receive_pack(body, token) + expected = f"application/x-{service}-result" + streamed = _upstream_git_request( + f"/atlas/{repo}.git/{service}", + method="POST", + body=body, + body_length=body_length, + content_type=expected_request, + expected_type=expected, + token=token, + stream_result=True, + ) + result, result_length = streamed + try: + self._stream(200, expected, result, result_length) + finally: + result.close() + finally: + body.close() + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--listen", default="0.0.0.0") + parser.add_argument("--port", type=int, default=BROKER_PORT) + args = parser.parse_args() + BoundedThreadingHTTPServer((args.listen, args.port), BrokerHandler).serve_forever() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/hermes/scm-common/scripts/scm_broker_client.py b/services/hermes/scm-common/scripts/scm_broker_client.py new file mode 100644 index 00000000..6c57ddfd --- /dev/null +++ b/services/hermes/scm-common/scripts/scm_broker_client.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Call the credential-isolated Hermes SCM broker over its fixed cluster origin.""" + +from __future__ import annotations + +import json +import urllib.request +from collections.abc import Callable + +from gitea_api_policy import PolicyError + +BROKER_ORIGIN = "http://hermes-scm-broker.hermes-scm.svc.cluster.local:9081" +MAX_RESPONSE_BYTES = 2 * 1024 * 1024 + + +class RejectRedirectHandler(urllib.request.HTTPRedirectHandler): + """Keep every broker request on its fixed in-cluster origin.""" + + def redirect_request(self, req, fp, code, msg, headers, newurl): + raise PolicyError("SCM broker redirects are not allowed") + + +_OPENER = urllib.request.build_opener(RejectRedirectHandler()) + + +def _open(request: urllib.request.Request, timeout: int): + return _OPENER.open(request, timeout=timeout) + + +def request( + endpoint: str, + payload: dict[str, object], + *, + opener: Callable[..., object] = _open, +) -> bytes: + """Send one bounded broker operation without any repository credential.""" + if endpoint not in {"/v1/metadata", "/v1/drafts"}: + raise PolicyError("SCM broker operation is outside the client allowlist") + body = json.dumps(payload, separators=(",", ":")).encode("utf-8") + if len(body) > 64 * 1024: + raise PolicyError("SCM broker request exceeds the safe size limit") + outgoing = urllib.request.Request( + BROKER_ORIGIN + endpoint, + data=body, + method="POST", + headers={ + "Accept": "application/json", + "Content-Type": "application/json", + "User-Agent": "hermes-scm-broker-client/1", + }, + ) + with opener(outgoing, timeout=30) as response: # type: ignore[attr-defined] + status = getattr(response, "status", None) + if status is None and hasattr(response, "getcode"): + status = response.getcode() # type: ignore[attr-defined] + if status != 200: + raise PolicyError("SCM broker returned an unexpected HTTP status") + content_type = response.headers.get_content_type() # type: ignore[attr-defined] + if content_type != "application/json": + raise PolicyError("SCM broker returned an unexpected response type") + result = response.read(MAX_RESPONSE_BYTES + 1) # type: ignore[attr-defined] + if len(result) > MAX_RESPONSE_BYTES: + raise PolicyError("SCM broker response exceeds the safe size limit") + json.loads(result) + return result + + +def read(path: str, *, opener: Callable[..., object] = _open) -> bytes: + """Read explicitly allowed Atlas metadata through the broker.""" + return request("/v1/metadata", {"path": path}, opener=opener) + + +def create_draft( + repo: str, + *, + base: str, + head: str, + head_sha: str, + title: str, + body: str, + opener: Callable[..., object] = _open, +) -> bytes: + """Create one verified draft through the broker for human review.""" + return request( + "/v1/drafts", + { + "base": base, + "body": body, + "head": head, + "head_sha": head_sha, + "repo": repo, + "title": title, + }, + opener=opener, + ) diff --git a/services/hermes/scm-common/scripts/scm_broker_io.py b/services/hermes/scm-common/scripts/scm_broker_io.py new file mode 100644 index 00000000..96df1fe0 --- /dev/null +++ b/services/hermes/scm-common/scripts/scm_broker_io.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""Bounded-memory stream helpers for the Hermes SCM broker.""" + +from __future__ import annotations + +import tempfile +import time +import urllib.request +from typing import BinaryIO + +from gitea_api_policy import PolicyError + + +class RejectRedirect(urllib.request.HTTPRedirectHandler): + """Reject every upstream redirect before credentials can be forwarded.""" + + def redirect_request(self, req, fp, code, msg, headers, newurl): + raise PolicyError("SCM upstream redirects are not allowed") + + +def response_status(response: object) -> int | None: + """Read an HTTP status from either urllib response interface.""" + value = getattr(response, "status", None) + if value is None and hasattr(response, "getcode"): + value = response.getcode() # type: ignore[attr-defined] + return value + + +def spool_response( + response, + maximum: int, + forbidden: tuple[bytes, ...], + *, + memory_limit: int, + chunk_size: int, + deadline_seconds: float, +) -> tuple[BinaryIO, int]: + """Spool an upstream response to disk while scanning chunk boundaries.""" + spool = tempfile.SpooledTemporaryFile( # noqa: SIM115 - caller owns returned spool + max_size=memory_limit, dir="/tmp" + ) + carry = b"" + total = 0 + deadline = time.monotonic() + deadline_seconds + try: + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise PolicyError("SCM upstream response deadline exceeded") + raw = getattr(getattr(response, "fp", None), "raw", None) + sock = getattr(raw, "_sock", raw) + if hasattr(sock, "settimeout"): + sock.settimeout(remaining) + chunk = response.read(chunk_size) + if not chunk: + break + total += len(chunk) + if total > maximum: + raise PolicyError("SCM upstream response exceeds the safe size limit") + candidate = carry + chunk + if any(value in candidate for value in forbidden): + raise PolicyError( + "Git upstream response contains runtime credential material" + ) + width = max(len(value) for value in forbidden) - 1 + carry = candidate[-width:] if width else b"" + spool.write(chunk) + spool.seek(0) + return spool, total + except Exception: + spool.close() + raise diff --git a/services/hermes/scm-common/scripts/scm_broker_server.py b/services/hermes/scm-common/scripts/scm_broker_server.py new file mode 100644 index 00000000..f29cd314 --- /dev/null +++ b/services/hermes/scm-common/scripts/scm_broker_server.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +"""Concurrency-bounded threaded HTTP server for the Hermes SCM broker.""" + +from __future__ import annotations + +import threading +import time +from http.server import ThreadingHTTPServer + +MAX_CONCURRENT_REQUESTS = 3 + + +class _AbsoluteDeadlineReader: + """Read header lines against one wall-clock deadline, not idle timeouts.""" + + def __init__(self, stream, connection): + self._stream = stream + self._connection = connection + self._deadline: float | None = None + + def begin(self, timeout: float) -> None: + self._deadline = time.monotonic() + timeout + + def end(self) -> None: + self._deadline = None + + def readline(self, limit: int = -1) -> bytes: + if self._deadline is None: + return self._stream.readline(limit) + value = bytearray() + while limit < 0 or len(value) < limit: + remaining = self._deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError("absolute SCM header deadline exceeded") + self._connection.settimeout(remaining) + character = self._stream.read(1) + if not character: + break + value.extend(character) + if character == b"\n": + break + return bytes(value) + + def __getattr__(self, name: str): + return getattr(self._stream, name) + + +class AbsoluteHeaderDeadlineMixin: + """Apply an absolute request-line plus header deadline to HTTP handlers.""" + + header_deadline_seconds = 10.0 + + def setup(self) -> None: + super().setup() + self._header_reader = _AbsoluteDeadlineReader(self.rfile, self.connection) + self.rfile = self._header_reader + + def handle_one_request(self) -> None: + try: + self._header_reader.begin(self.header_deadline_seconds) + try: + self.raw_requestline = self.rfile.readline(65537) + if len(self.raw_requestline) > 65536: + self.requestline = "" + self.request_version = "" + self.command = "" + self.send_error(414) + return + if not self.raw_requestline: + self.close_connection = True + return + if not self.parse_request(): + return + finally: + self._header_reader.end() + method_name = "do_" + self.command + if not hasattr(self, method_name): + self.send_error(501, "Unsupported method") + return + getattr(self, method_name)() + self.wfile.flush() + except TimeoutError: + self.close_connection = True + + +class BoundedThreadingHTTPServer(ThreadingHTTPServer): + """Cap active handlers so large Git exchanges cannot exhaust memory.""" + + daemon_threads = True + block_on_close = True + request_queue_size = 16 + + def __init__(self, *args, **kwargs): + self._slots = threading.BoundedSemaphore(MAX_CONCURRENT_REQUESTS) + super().__init__(*args, **kwargs) + + def process_request(self, request, client_address) -> None: + if not self._slots.acquire(blocking=False): + try: + request.sendall( + b"HTTP/1.1 503 Service Unavailable\r\n" + b"Connection: close\r\nContent-Length: 0\r\n\r\n" + ) + finally: + self.shutdown_request(request) + return + try: + super().process_request(request, client_address) + except Exception: + self._slots.release() + self.shutdown_request(request) + raise + + def process_request_thread(self, request, client_address) -> None: + try: + super().process_request_thread(request, client_address) + finally: + self._slots.release() diff --git a/services/hermes/scripts/gitea_askpass.sh b/services/hermes/scripts/gitea_askpass.sh deleted file mode 100755 index d6146d80..00000000 --- a/services/hermes/scripts/gitea_askpass.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env sh -set -eu - -case "${1:-}" in - *Username*) - if [ -s /runtime-access/gitea-username ]; then - tr -d '\r\n' Path: @@ -139,24 +141,8 @@ def bootstrap_cassandra_state(root: Path) -> dict[str, str]: return {"state": "ready"} -def _gitea_token_path() -> Path: - """Return the runtime-only Gitea credential path without reading it.""" - configured = os.environ.get("HERMES_GITEA_TOKEN_FILE", "").strip() - return Path(configured) if configured else DEFAULT_GITEA_TOKEN_PATH - - -def _gitea_token_available() -> bool: - """Check credential readiness without loading it into process memory.""" - try: - path = _gitea_token_path() - return path.is_file() and path.stat().st_size > 0 - except OSError: - return False - - def sync_cassandra_repo(env_values: dict[str, str]) -> str: - """Clone or fetch Cassandra through the runtime-only askpass credential.""" - token_available = _gitea_token_available() + """Clone or fetch Cassandra through the credential-isolated SCM broker.""" if shutil.which("git") is None: return "git-unavailable" child_env = os.environ.copy() @@ -173,9 +159,7 @@ def sync_cassandra_repo(env_values: dict[str, str]) -> str: } } ) - child_env["GIT_ASKPASS"] = env_values.get( - "GIT_ASKPASS", "/opt/coordinator/gitea_askpass.sh" - ) + child_env.pop("GIT_ASKPASS", None) child_env["GIT_TERMINAL_PROMPT"] = "0" if (CASSANDRA_BASE_PATH / ".git").exists(): try: @@ -232,8 +216,6 @@ def sync_cassandra_repo(env_values: dict[str, str]) -> str: return f"remote-repair-failed-{repaired.returncode}" except (OSError, subprocess.TimeoutExpired): return "remote-repair-failed" - if not token_available: - return "ready; fetch skipped until Gitea token is configured" command = [ "git", "-C", @@ -247,8 +229,6 @@ def sync_cassandra_repo(env_values: dict[str, str]) -> str: CASSANDRA_BASE_PATH.mkdir(parents=True, exist_ok=True) if any(CASSANDRA_BASE_PATH.iterdir()): return "unmanaged-nonempty-directory" - if not token_available: - return "awaiting-gitea-token" command = [ "git", "clone", diff --git a/services/hermes/scripts/node_account_audit.py b/services/hermes/scripts/node_account_audit.py new file mode 100644 index 00000000..b1795e99 --- /dev/null +++ b/services/hermes/scripts/node_account_audit.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +"""Fail-closed supplementary-group and host privilege-policy audits.""" + +from __future__ import annotations + +import re +import stat +import struct +from pathlib import Path + +from node_account_io import HardeningError, read_regular + + +def _members(field: str, context: str) -> set[str]: + if not field: + return set() + values = field.split(",") + if any(not value or value.strip() != value for value in values): + raise HardeningError(f"{context} has a malformed member list") + if len(values) != len(set(values)): + raise HardeningError(f"{context} has duplicate members") + return set(values) + + +def audit_membership( + account: str, groups: list[list[str]], gshadow: list[list[str]] +) -> None: + """Reject the dedicated account in any supplementary group/admin list.""" + for record in groups: + if account in _members(record[3], f"group {record[0]}"): + raise HardeningError("dedicated Hermes account has supplementary group access") + for record in gshadow: + principals = _members(record[2], f"gshadow {record[0]} admins") + principals |= _members(record[3], f"gshadow {record[0]} members") + if account in principals: + raise HardeningError("dedicated Hermes account has gshadow group access") + + +def _policy_files(host_etc: Path, polkit_share: Path, root_uid: int) -> list[Path]: + files: list[Path] = [] + sudoers = host_etc / "sudoers" + if sudoers.exists(): + files.append(sudoers) + roots = ( + host_etc / "sudoers.d", + host_etc / "polkit-1/rules.d", + host_etc / "polkit-1/localauthority", + polkit_share / "rules.d", + polkit_share / "localauthority", + ) + for root in roots: + if not root.exists(): + continue + metadata = root.lstat() + if ( + not stat.S_ISDIR(metadata.st_mode) + or stat.S_ISLNK(metadata.st_mode) + or metadata.st_uid != root_uid + or stat.S_IMODE(metadata.st_mode) & 0o022 + ): + raise HardeningError("unsafe sudo/polkit policy directory") + files.extend(path for path in root.rglob("*") if path.is_file()) + if len(files) > 256: + raise HardeningError("too many sudo/polkit policy files") + return sorted(files) + + +def _audit_policy_metadata(snapshot, root_uid: int, account_uid: int) -> None: + if snapshot.uid != root_uid or snapshot.mode & 0o022: + raise HardeningError("sudo/polkit policy metadata permits unsafe mutation") + acl = dict(snapshot.xattrs).get("system.posix_acl_access", b"") + if not acl: + return + header = struct.Struct(" str: + """Remove comments without discarding sudoers' numeric-UID principals.""" + active: list[str] = [] + for line in text.splitlines(): + stripped = line.lstrip() + if not stripped: + continue + include = re.fullmatch( + r"(?i)([#@]include|[#@]includedir)\s+([^\s]+)", stripped + ) + if include: + directive, target = include.groups() + if directive.lower().endswith("includedir") and target == "/etc/sudoers.d": + continue + raise HardeningError("sudo policy includes an unaudited authority source") + if stripped.startswith("#") and not re.match(r"#\d+(?:\s|$)", stripped): + continue + active.append(line) + return "\n".join(active) + + +def _mentions_dedicated_identity(text: str, account: str, account_uid: int) -> bool: + account_pattern = rf"(? None: + """Reject direct, group, wildcard sudo, or root-equivalent polkit grants.""" + dangerous_polkit = ( + "org.freedesktop.policykit.exec", + "org.freedesktop.systemd1.manage-unit", + "org.freedesktop.udisks2.modify-device", + "org.freedesktop.packagekit", + ) + nsswitch = host_etc / "nsswitch.conf" + if nsswitch.exists(): + try: + nss_snapshot = read_regular(nsswitch, 64 * 1024) + _audit_policy_metadata(nss_snapshot, root_uid, account_uid) + text = nss_snapshot.value.decode("utf-8") + except UnicodeDecodeError as exc: + raise HardeningError("nsswitch policy is not UTF-8") from exc + for line in text.splitlines(): + name, separator, sources = line.partition(":") + if name.strip() in { + "passwd", + "group", + "initgroups", + "shadow", + "sudoers", + } and separator: + active_sources = [item for item in sources.split() if not item.startswith("[")] + if not active_sources or any( + item not in {"files", "systemd"} for item in active_sources + ): + raise HardeningError("external group/account/sudo authority source is unsafe") + total = 0 + for path in _policy_files(host_etc, polkit_share, root_uid): + snapshot = read_regular(path, 256 * 1024) + _audit_policy_metadata(snapshot, root_uid, account_uid) + value = snapshot.value + total += len(value) + if total > 2 * 1024 * 1024: + raise HardeningError("sudo/polkit policy input exceeds safe limit") + try: + text = value.decode("utf-8") + except UnicodeDecodeError as exc: + raise HardeningError("sudo/polkit policy is not UTF-8") from exc + if path.name == "sudoers" or "sudoers.d" in path.parts: + active = _active_sudo_policy(text) + if _mentions_dedicated_identity(active, account, account_uid): + raise HardeningError("dedicated Hermes account has sudo authority") + for line in active.splitlines(): + fields = line.split() + if fields and fields[0] in {"ALL", "%ALL"} and "=" in line: + raise HardeningError("broad sudo authority includes Hermes") + continue + active = "\n".join( + line for line in text.splitlines() if not line.lstrip().startswith("#") + ) + grants = "polkit.Result.YES" in active or re.search( + r"(?im)^\s*Result(?:Any|Inactive|Active)\s*=\s*yes\s*$", active + ) + if grants and _mentions_dedicated_identity(active, account, account_uid): + raise HardeningError("dedicated Hermes account has polkit authority") + broad = "unix-user:*" in active or "Identity=unix-user:*" in active + if broad and grants and any(item in active for item in dangerous_polkit): + raise HardeningError("root-equivalent broad polkit authority includes Hermes") diff --git a/services/hermes/scripts/node_account_hardening.py b/services/hermes/scripts/node_account_hardening.py new file mode 100644 index 00000000..a545af47 --- /dev/null +++ b/services/hermes/scripts/node_account_hardening.py @@ -0,0 +1,496 @@ +#!/usr/bin/env python3 +"""Reconcile a dedicated unprivileged Hermes SSH account on one Atlas node.""" + +from __future__ import annotations + +import argparse +import base64 +import errno +import os +import re +import shlex +import stat +import struct +from contextlib import suppress +from pathlib import Path + +from node_account_audit import audit_membership, audit_privilege_policies +from node_account_io import ( + FileSnapshot, + HardeningError, + account_lock, + assert_unchanged, + atomic_write, + backup_once, + read_regular, +) + +HOST_ETC = Path("/host-etc") +HOST_HOME = Path("/host-home") +HOST_K3S = Path("/host-k3s") +HOST_KUBELET = Path("/host-kubelet") +HOST_RUN_K3S = Path("/host-run-k3s") +HOST_RUN_CONTAINERD = Path("/host-run-containerd") +HOST_POLKIT_SHARE = Path("/host-polkit-share") +ACCOUNT = "hermes-agent" +ACCOUNT_UID = 1200 +ACCOUNT_GID = 1200 +ACCOUNT_HOME = "/home/hermes-agent" +ACCOUNT_SHELL = "/bin/bash" +HOST_ROOT_UID = 0 +HOST_ROOT_GID = 0 +LEGACY_ACCOUNTS = ("atlas", "oceanus") +MAX_ACCOUNT_FILE = 2 * 1024 * 1024 +MAX_AUTHORIZED_KEYS = 1024 * 1024 +ACL_VERSION = 2 +ACL_UNDEFINED_ID = 0xFFFFFFFF +ACL_USER_OBJ = 0x01 +ACL_USER = 0x02 +ACL_GROUP_OBJ = 0x04 +ACL_GROUP = 0x08 +ACL_MASK = 0x10 +ACL_OTHER = 0x20 +ACL_HEADER = struct.Struct(" tuple[bytes, os.stat_result]: + snapshot = read_regular(path, maximum) + return snapshot.value, path.stat(follow_symlinks=False) + + +def _records(value: bytes, fields: int, name: str) -> list[list[str]]: + try: + text = value.decode("utf-8") + except UnicodeDecodeError as exc: + raise HardeningError(f"{name} is not UTF-8") from exc + if not text.endswith("\n"): + raise HardeningError(f"{name} is missing its final newline") + records = [] + names = set() + for line in text.splitlines(): + parts = line.split(":") + if len(parts) != fields or not parts[0] or parts[0] in names: + raise HardeningError(f"{name} has an invalid record") + names.add(parts[0]) + records.append(parts) + return records + + +def _encode(records: list[list[str]]) -> bytes: + return ("\n".join(":".join(record) for record in records) + "\n").encode() + + +def _expected_records() -> dict[str, list[str]]: + return { + "passwd": [ + ACCOUNT, + "x", + str(ACCOUNT_UID), + str(ACCOUNT_GID), + "Hermes Agent", + ACCOUNT_HOME, + ACCOUNT_SHELL, + ], + "group": [ACCOUNT, "x", str(ACCOUNT_GID), ""], + "shadow": [ACCOUNT, "!", "1", "0", "99999", "7", "", "", ""], + "gshadow": [ACCOUNT, "!", "", ""], + } + + +def _reconcile_record( + records: list[list[str]], expected: list[str], *, identity_index: int +) -> list[list[str]]: + for record in records: + same_name = record[0] == expected[0] + same_identity = record[identity_index] == expected[identity_index] + if same_name or same_identity: + if record != expected: + raise HardeningError("dedicated Hermes account identity conflicts") + return records + return [*records, expected] + + +def _reconcile_databases() -> None: + expected = _expected_records() + definitions = ( + ("passwd", 7, 2), + ("group", 4, 2), + ("shadow", 9, 0), + ("gshadow", 4, 0), + ) + planned: list[tuple[Path, FileSnapshot, bytes]] = [] + account_present = False + parsed: dict[str, list[list[str]]] = {} + for name, fields, identity_index in definitions: + path = HOST_ETC / name + snapshot = read_regular(path, MAX_ACCOUNT_FILE) + records = _records(snapshot.value, fields, name) + parsed[name] = records + updated = _reconcile_record( + records, expected[name], identity_index=identity_index + ) + if name == "passwd": + account_present = expected[name] in records + planned.append((path, snapshot, _encode(updated))) + audit_membership(ACCOUNT, parsed["group"], parsed["gshadow"]) + audit_privilege_policies( + ACCOUNT, ACCOUNT_UID, HOST_ROOT_UID, HOST_ETC, HOST_POLKIT_SHARE + ) + try: + (HOST_HOME / ACCOUNT).lstat() + except FileNotFoundError: + pass + else: + if not account_present: + raise HardeningError("dedicated Hermes account home already exists") + for path, snapshot, _updated in planned: + assert_unchanged(path, snapshot, MAX_ACCOUNT_FILE) + for path, snapshot, _updated in planned: + backup_once(path, snapshot, MAX_ACCOUNT_FILE) + for path, snapshot, _updated in planned: + assert_unchanged(path, snapshot, MAX_ACCOUNT_FILE) + written: list[tuple[Path, FileSnapshot, bytes]] = [] + try: + for path, snapshot, updated in planned: + assert_unchanged(path, snapshot, MAX_ACCOUNT_FILE) + atomic_write(path, updated, snapshot) + written.append((path, snapshot, updated)) + for path, _snapshot, _updated in planned: + name = path.name + fields = next(item[1] for item in definitions if item[0] == name) + records = _records(_read_regular(path)[0], fields, name) + if expected[name] not in records: + raise HardeningError("dedicated Hermes account validation failed") + except Exception as error: + for path, snapshot, updated in reversed(written): + current = read_regular(path, MAX_ACCOUNT_FILE) + if ( + current.value != updated + or current.mode != snapshot.mode + or current.uid != snapshot.uid + or current.gid != snapshot.gid + or current.xattrs != snapshot.xattrs + ): + raise HardeningError( + f"concurrent host account change prevents rollback: {path.name}" + ) from error + atomic_write(path, snapshot.value, snapshot) + raise + + +SUPPORTED_KEY_TYPES = {"ssh-ed25519", "ecdsa-sha2-nistp256", "ssh-rsa"} +KEY_TYPE_RE = re.compile(r"(?:sk-)?(?:ssh|ecdsa)-[A-Za-z0-9@._+-]+\Z") + + +def _key_identity(line: bytes) -> tuple[str, bytes] | None: + """Return OpenSSH key type/blob, ignoring options and comments.""" + stripped = line.strip(b"\r\n") + if not stripped or stripped.lstrip().startswith(b"#"): + return None + try: + fields = shlex.split(stripped.decode("ascii"), posix=True) + except (UnicodeDecodeError, ValueError) as exc: + raise HardeningError("authorized key line is malformed") from exc + indexes = [ + index for index, field in enumerate(fields) if KEY_TYPE_RE.fullmatch(field) + ] + if len(indexes) != 1 or indexes[0] + 1 >= len(fields): + raise HardeningError("authorized key line has ambiguous key material") + key_type = fields[indexes[0]] + try: + blob = base64.b64decode(fields[indexes[0] + 1], validate=True) + except ValueError as exc: + raise HardeningError("authorized key payload is invalid") from exc + encoded_type = key_type.encode("ascii") + if len(blob) < 4 or int.from_bytes(blob[:4], "big") != len(encoded_type): + raise HardeningError("authorized key blob is malformed") + if blob[4 : 4 + len(encoded_type)] != encoded_type: + raise HardeningError("authorized key type does not match its blob") + return key_type, blob + + +def _validated_public_key(path: Path) -> tuple[bytes, tuple[str, bytes]]: + value, _ = _read_regular(path, 16 * 1024) + line = value.strip() + if b"\n" in line or b"\r" in line: + raise HardeningError("Hermes public key must contain one line") + identity = _key_identity(line) + fields = line.split() + if ( + identity is None + or identity[0] not in SUPPORTED_KEY_TYPES + or len(fields) not in {2, 3} + or fields[0].decode("ascii") != identity[0] + ): + raise HardeningError("Hermes public key format is unsupported") + return line, identity + + +def _directory(path: Path, *, mode: int, uid: int, gid: int) -> None: + with suppress(FileExistsError): + path.mkdir(mode=mode) + metadata = path.lstat() + if not stat.S_ISDIR(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode): + raise HardeningError(f"unsafe account directory: {path.name}") + if metadata.st_uid not in {0, uid} or metadata.st_gid not in {0, gid}: + raise HardeningError(f"account directory ownership conflicts: {path.name}") + os.chown(path, uid, gid) + path.chmod(mode) + + +def _without_key(value: bytes, identity: tuple[str, bytes]) -> bytes: + kept = [] + for line in value.splitlines(keepends=True): + if _key_identity(line) != identity: + kept.append(line) + return b"".join(kept) + + +def _move_key(public_key: Path) -> None: + key, identity = _validated_public_key(public_key) + target = HOST_HOME / ACCOUNT / ".ssh" / "authorized_keys" + legacy_plan: list[tuple[Path, FileSnapshot, bytes]] = [] + for legacy in LEGACY_ACCOUNTS: + authorized = HOST_HOME / legacy / ".ssh" / "authorized_keys" + try: + snapshot = read_regular(authorized, MAX_AUTHORIZED_KEYS) + except FileNotFoundError: + continue + updated = _without_key(snapshot.value, identity) + if updated != snapshot.value: + legacy_plan.append((authorized, snapshot, updated)) + if legacy_plan and target.exists(): + target_snapshot = read_regular(target, MAX_AUTHORIZED_KEYS) + target_without_key = _without_key(target_snapshot.value, identity) + if target_without_key != target_snapshot.value: + backup_once(target, target_snapshot, MAX_AUTHORIZED_KEYS) + assert_unchanged(target, target_snapshot, MAX_AUTHORIZED_KEYS) + atomic_write(target, target_without_key, target_snapshot) + verified_target = read_regular(target, MAX_AUTHORIZED_KEYS).value + if _without_key(verified_target, identity) != verified_target: + raise HardeningError("preexisting Hermes authorization removal failed") + for authorized, snapshot, _updated in legacy_plan: + assert_unchanged(authorized, snapshot, MAX_AUTHORIZED_KEYS) + backup_once(authorized, snapshot, MAX_AUTHORIZED_KEYS) + for authorized, snapshot, updated in legacy_plan: + assert_unchanged(authorized, snapshot, MAX_AUTHORIZED_KEYS) + atomic_write(authorized, updated, snapshot) + for legacy in LEGACY_ACCOUNTS: + authorized = HOST_HOME / legacy / ".ssh" / "authorized_keys" + try: + value = read_regular(authorized, MAX_AUTHORIZED_KEYS).value + except FileNotFoundError: + continue + if _without_key(value, identity) != value: + raise HardeningError("legacy Hermes authorization key removal failed") + + # Installation happens only after every legacy authorization is absent. A + # crash can therefore remove access temporarily, but can never duplicate + # the machine credential across privileged and unprivileged accounts. + home = HOST_HOME / ACCOUNT + ssh = home / ".ssh" + _directory(home, mode=0o700, uid=ACCOUNT_UID, gid=ACCOUNT_GID) + _directory(ssh, mode=0o700, uid=ACCOUNT_UID, gid=ACCOUNT_GID) + target = ssh / "authorized_keys" + if target.exists(): + current = read_regular(target, MAX_AUTHORIZED_KEYS) + if current.value != key + b"\n": + backup_once(target, current, MAX_AUTHORIZED_KEYS) + else: + current = FileSnapshot( + value=b"", + device=0, + inode=0, + mode=0o600, + uid=ACCOUNT_UID, + gid=ACCOUNT_GID, + size=0, + mtime_ns=0, + ctime_ns=0, + xattrs=(), + ) + atomic_write(target, key + b"\n", current) + if _read_regular(target, MAX_AUTHORIZED_KEYS)[0] != key + b"\n": + raise HardeningError("dedicated Hermes authorized key validation failed") + + +def _decode_acl(value: bytes, mode: int) -> list[tuple[int, int, int]]: + """Decode a bounded POSIX ACL or derive one from ordinary mode bits.""" + if not value: + group = (mode >> 3) & 0o7 + return [ + (ACL_USER_OBJ, (mode >> 6) & 0o7, ACL_UNDEFINED_ID), + (ACL_GROUP_OBJ, group, ACL_UNDEFINED_ID), + (ACL_MASK, group, ACL_UNDEFINED_ID), + (ACL_OTHER, mode & 0o7, ACL_UNDEFINED_ID), + ] + if len(value) < ACL_HEADER.size or (len(value) - ACL_HEADER.size) % ACL_ENTRY.size: + raise HardeningError("sensitive directory ACL is malformed") + if ACL_HEADER.unpack_from(value)[0] != ACL_VERSION: + raise HardeningError("sensitive directory ACL version is unsupported") + entries = [ + ACL_ENTRY.unpack_from(value, offset) + for offset in range(ACL_HEADER.size, len(value), ACL_ENTRY.size) + ] + if any(permission > 0o7 for _tag, permission, _identifier in entries): + raise HardeningError("sensitive directory ACL permission is malformed") + required = {ACL_USER_OBJ, ACL_GROUP_OBJ, ACL_OTHER} + if not required <= {tag for tag, _permission, _identifier in entries}: + raise HardeningError("sensitive directory ACL is incomplete") + return entries + + +def _encode_acl(entries: list[tuple[int, int, int]]) -> bytes: + return ACL_HEADER.pack(ACL_VERSION) + b"".join( + ACL_ENTRY.pack(*entry) for entry in entries + ) + + +def _acl_with_deny(value: bytes, mode: int) -> bytes: + entries = _decode_acl(value, mode) + entries = [ + entry + for entry in entries + if not (entry[0] == ACL_USER and entry[2] == ACCOUNT_UID) + ] + entries.append((ACL_USER, 0, ACCOUNT_UID)) + if not any(tag == ACL_MASK for tag, _permission, _identifier in entries): + entries.append((ACL_MASK, (mode >> 3) & 0o7, ACL_UNDEFINED_ID)) + order = { + ACL_USER_OBJ: 0, + ACL_USER: 1, + ACL_GROUP_OBJ: 2, + ACL_GROUP: 3, + ACL_MASK: 4, + ACL_OTHER: 5, + } + entries.sort(key=lambda entry: (order.get(entry[0], 99), entry[2])) + return _encode_acl(entries) + + +def _read_acl(path: Path) -> bytes: + try: + return os.getxattr(path, ACL_XATTR, follow_symlinks=False) + except OSError as exc: + if exc.errno in {errno.ENODATA, getattr(errno, "ENOATTR", errno.ENODATA)}: + return b"" + raise + + +def _validate_acl_backup(value: bytes) -> None: + if value[:1] not in {b"A", b"N"}: + raise HardeningError("sensitive directory ACL backup is malformed") + if value[:1] == b"A": + if len(value) == 1: + raise HardeningError("sensitive directory ACL backup is malformed") + _decode_acl(value[1:], 0) + elif value != b"N": + raise HardeningError("sensitive directory ACL backup is malformed") + + +def _acl_backup_once(path: Path, value: bytes, backup_name: str | None = None) -> None: + """Persist the original ACL as a private, durable host recovery file.""" + root = HOST_ETC / "hermes-node-boundary" + _directory(root, mode=0o700, uid=HOST_ROOT_UID, gid=HOST_ROOT_GID) + backup = root / f"{backup_name or path.name}.acl" + encoded = b"A" + value if value else b"N" + try: + existing, _metadata = _read_regular(backup, 64 * 1024) + except FileNotFoundError: + pass + else: + _validate_acl_backup(existing) + return + + temporary = root / f".{path.name}.acl.hermes-{os.getpid()}" + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(temporary, flags, 0o600) + try: + if os.write(descriptor, encoded) != len(encoded): + raise HardeningError("short sensitive directory ACL backup write") + os.fchmod(descriptor, 0o600) + os.fchown(descriptor, HOST_ROOT_UID, HOST_ROOT_GID) + os.fsync(descriptor) + except Exception: + temporary.unlink(missing_ok=True) + raise + finally: + os.close(descriptor) + try: + os.link(temporary, backup, follow_symlinks=False) + except FileExistsError: + # Another reconciler won the race; never overwrite the first backup. + pass + finally: + temporary.unlink(missing_ok=True) + stored, _metadata = _read_regular(backup, 64 * 1024) + _validate_acl_backup(stored) + directory = os.open(root, os.O_RDONLY) + try: + os.fsync(directory) + finally: + os.close(directory) + + +def _deny_sensitive_root(path: Path, backup_name: str | None = None) -> None: + metadata = path.lstat() + if not stat.S_ISDIR(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode): + raise HardeningError(f"unsafe sensitive directory: {path.name}") + if metadata.st_uid != HOST_ROOT_UID: + raise HardeningError(f"sensitive directory is not root-owned: {path.name}") + current = _read_acl(path) + _acl_backup_once(path, current, backup_name) + updated = _acl_with_deny(current, stat.S_IMODE(metadata.st_mode)) + try: + os.setxattr(path, ACL_XATTR, updated, follow_symlinks=False) + verified = _decode_acl(_read_acl(path), stat.S_IMODE(metadata.st_mode)) + if (ACL_USER, 0, ACCOUNT_UID) not in verified: + raise HardeningError("sensitive directory ACL validation failed") + except Exception: + if current: + os.setxattr(path, ACL_XATTR, current, follow_symlinks=False) + else: + try: + os.removexattr(path, ACL_XATTR, follow_symlinks=False) + except OSError as exc: + if exc.errno not in { + errno.ENODATA, + getattr(errno, "ENOATTR", errno.ENODATA), + }: + raise + raise + + +def _deny_sensitive_roots() -> None: + for path, backup_name in ( + (HOST_K3S, "var-lib-rancher-k3s"), + (HOST_KUBELET, "var-lib-kubelet"), + (HOST_RUN_K3S, "run-k3s"), + (HOST_RUN_CONTAINERD, "run-containerd"), + ): + _deny_sensitive_root(path, backup_name) + + +def reconcile(public_key: Path) -> None: + """Create the locked account and move only the Hermes authorization key.""" + with account_lock(HOST_ETC, expected_uid=HOST_ROOT_UID): + _reconcile_databases() + _deny_sensitive_roots() + _move_key(public_key) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--public-key-file", type=Path, required=True) + args = parser.parse_args() + reconcile(args.public_key_file) + print("Dedicated Hermes node account reconciled.", flush=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/hermes/scripts/node_account_io.py b/services/hermes/scripts/node_account_io.py new file mode 100644 index 00000000..4fb2ae9e --- /dev/null +++ b/services/hermes/scripts/node_account_io.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +"""Crash-safe, metadata-preserving host account database I/O.""" + +from __future__ import annotations + +import fcntl +import os +import stat +import time +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Iterator + +MAX_XATTRS = 64 +MAX_XATTR_NAME = 255 +MAX_XATTR_VALUE = 256 * 1024 +MAX_XATTR_TOTAL = 1024 * 1024 +LOCK_TIMEOUT_SECONDS = 15.0 + + +class HardeningError(RuntimeError): + """Raised before an unsafe or ambiguous host-account change.""" + + +@dataclass(frozen=True) +class FileSnapshot: + """Bounded file contents and all security-relevant inode metadata.""" + + value: bytes + device: int + inode: int + mode: int + uid: int + gid: int + size: int + mtime_ns: int + ctime_ns: int + xattrs: tuple[tuple[str, bytes], ...] + + +def _stat_identity(item: os.stat_result) -> tuple[int, ...]: + return ( + item.st_dev, + item.st_ino, + item.st_mode, + item.st_uid, + item.st_gid, + item.st_size, + item.st_mtime_ns, + item.st_ctime_ns, + ) + + +def _xattrs(path_or_fd: Path | int) -> tuple[tuple[str, bytes], ...]: + try: + names = sorted(os.listxattr(path_or_fd, follow_symlinks=False)) + except (TypeError, ValueError): + names = sorted(os.listxattr(path_or_fd)) + if len(names) > MAX_XATTRS: + raise HardeningError("host account file has too many extended attributes") + values: list[tuple[str, bytes]] = [] + total = 0 + for name in names: + if len(name.encode("utf-8")) > MAX_XATTR_NAME: + raise HardeningError("host account file xattr name is too long") + try: + value = os.getxattr(path_or_fd, name, follow_symlinks=False) + except (TypeError, ValueError): + value = os.getxattr(path_or_fd, name) + total += len(value) + if len(value) > MAX_XATTR_VALUE or total > MAX_XATTR_TOTAL: + raise HardeningError("host account file xattrs exceed the safe limit") + values.append((name, value)) + return tuple(values) + + +def _restore_xattrs(descriptor: int, expected: tuple[tuple[str, bytes], ...]) -> None: + expected_names = {name for name, _value in expected} + for name, _value in _xattrs(descriptor): + if name not in expected_names: + os.removexattr(descriptor, name) + for name, value in expected: + os.setxattr(descriptor, name, value) + + +def read_regular(path: Path, maximum: int) -> FileSnapshot: + """Read one bounded, non-symlink regular file and capture its metadata.""" + flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(path, flags) + try: + metadata = os.fstat(descriptor) + if not stat.S_ISREG(metadata.st_mode) or metadata.st_size > maximum: + raise HardeningError(f"unsafe regular file: {path.name}") + value = os.read(descriptor, maximum + 1) + attrs = _xattrs(descriptor) + after = os.fstat(descriptor) + finally: + os.close(descriptor) + if len(value) != metadata.st_size or _stat_identity(metadata) != _stat_identity(after): + raise HardeningError(f"short file read: {path.name}") + return FileSnapshot( + value=value, + device=metadata.st_dev, + inode=metadata.st_ino, + mode=stat.S_IMODE(metadata.st_mode), + uid=metadata.st_uid, + gid=metadata.st_gid, + size=metadata.st_size, + mtime_ns=metadata.st_mtime_ns, + ctime_ns=metadata.st_ctime_ns, + xattrs=attrs, + ) + + +def assert_unchanged(path: Path, expected: FileSnapshot, maximum: int) -> None: + """Fail if an account file changed since planning began.""" + current = read_regular(path, maximum) + if current != expected: + raise HardeningError(f"concurrent host account change detected: {path.name}") + + +def atomic_write(path: Path, value: bytes, metadata: FileSnapshot) -> None: + """Atomically replace a file while retaining ACLs, labels, and xattrs.""" + temporary = path.with_name(f".{path.name}.hermes-{os.getpid()}") + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(temporary, flags, metadata.mode) + try: + if os.write(descriptor, value) != len(value): + raise HardeningError(f"short atomic write: {path.name}") + os.fchmod(descriptor, metadata.mode) + os.fchown(descriptor, metadata.uid, metadata.gid) + _restore_xattrs(descriptor, metadata.xattrs) + os.fsync(descriptor) + except Exception: + temporary.unlink(missing_ok=True) + raise + finally: + os.close(descriptor) + os.replace(temporary, path) + directory = os.open(path.parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) + try: + os.fsync(directory) + finally: + os.close(directory) + + +def backup_once(path: Path, snapshot: FileSnapshot, maximum: int) -> Path: + """Create the first durable recovery copy without replacing an older one.""" + backup = path.with_name(path.name + ".hermes-boundary-backup") + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(backup, flags, snapshot.mode) + except FileExistsError: + read_regular(backup, maximum) + return backup + try: + if os.write(descriptor, snapshot.value) != len(snapshot.value): + raise HardeningError(f"short backup write: {path.name}") + os.fchmod(descriptor, snapshot.mode) + os.fchown(descriptor, snapshot.uid, snapshot.gid) + _restore_xattrs(descriptor, snapshot.xattrs) + os.fsync(descriptor) + except Exception: + backup.unlink(missing_ok=True) + raise + finally: + os.close(descriptor) + directory = os.open(path.parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) + try: + os.fsync(directory) + finally: + os.close(directory) + return backup + + +@contextmanager +def account_lock(host_etc: Path, *, expected_uid: int = 0) -> Iterator[None]: + """Hold the standard shadow-utils account lock for the full transaction.""" + path = host_etc / ".pwd.lock" + flags = os.O_WRONLY | os.O_CREAT | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(path, flags, 0o600) + try: + metadata = os.fstat(descriptor) + if ( + not stat.S_ISREG(metadata.st_mode) + or metadata.st_uid != expected_uid + or stat.S_IMODE(metadata.st_mode) != 0o600 + ): + raise HardeningError("unsafe standard account lock") + deadline = time.monotonic() + LOCK_TIMEOUT_SECONDS + while True: + try: + fcntl.lockf(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except BlockingIOError as exc: + if time.monotonic() >= deadline: + raise HardeningError("standard account lock is busy") from exc + time.sleep(0.05) + yield + finally: + try: + fcntl.lockf(descriptor, fcntl.LOCK_UN) + finally: + os.close(descriptor) diff --git a/services/hermes/scripts/stage_runtime_access.py b/services/hermes/scripts/stage_runtime_access.py index f4d73160..07423d70 100644 --- a/services/hermes/scripts/stage_runtime_access.py +++ b/services/hermes/scripts/stage_runtime_access.py @@ -86,6 +86,21 @@ def _write_empty_auth_store() -> None: os.chown(path, OWNER_UID, OWNER_GID) +def _stage_node_ssh_config() -> None: + """Force every Atlas node alias onto the dedicated Hermes OS account.""" + destination = RUNTIME_ROOT / "node-ssh-config" + source = _copy_secret("node-ssh-config", destination) + if "\x00" in source: + destination.unlink(missing_ok=True) + raise RuntimeError("node SSH config contains invalid control data") + destination.write_text( + "Host titan-*\n User hermes-agent\n" + source.rstrip() + "\n", + encoding="utf-8", + ) + destination.chmod(0o600) + os.chown(destination, OWNER_UID, OWNER_GID) + + def stage_agent() -> None: """Stage the owner agent's complete runtime access set.""" for path in (RUNTIME_ROOT, RUNTIME_ROOT / "claude", RUNTIME_ROOT / "codex"): @@ -93,13 +108,11 @@ def stage_agent() -> None: for name in ( "agent-api-key", "chat-relay-key", - "gitea-token", - "gitea-username", "node-ssh-private-key", - "node-ssh-config", "node-ssh-known-hosts", ): _copy_secret(name, RUNTIME_ROOT / name) + _stage_node_ssh_config() _validated_json( "claude-credentials", RUNTIME_ROOT / "claude" / ".credentials.json", diff --git a/services/hermes/skills/manage-atlas-pull-requests/SKILL.md b/services/hermes/skills/manage-atlas-pull-requests/SKILL.md index 32fd1ccf..6194f7f9 100644 --- a/services/hermes/skills/manage-atlas-pull-requests/SKILL.md +++ b/services/hermes/skills/manage-atlas-pull-requests/SKILL.md @@ -5,17 +5,21 @@ description: Read bounded private Atlas repository and pull-request metadata or # Manage Atlas pull requests -Use `/opt/coordinator/gitea_api.py` for Forgejo API access. It reads its token -from the pod-lifetime Vault projection; never read that file, copy the token, -put it in an argument or environment variable, or replace this client with -`curl`. +Use `/opt/scm/gitea_api.py` for Forgejo API access. The agent pod has no +repository credential. The client calls a separate least-authority broker; +never bypass it with `curl` or direct Gitea HTTP. + +Use the configured broker Git remote for clone, fetch, and creation of a new +namespaced feature branch. Existing-ref updates, protected refs, deletion, +and force-push are rejected by the broker. Never replace that remote with a +credential-bearing URL. ## Read repository or PR state Pass one allowed Atlas metadata API path to the read operation: ```sh -/opt/coordinator/gitea_api.py read /api/v1/repos/atlas/REPO/pulls/NUMBER +/opt/scm/gitea_api.py read /api/v1/repos/atlas/REPO/pulls/NUMBER ``` The client exposes only repository metadata, PRs and PR evidence, branches, @@ -29,10 +33,16 @@ evidence, not authorization to change it. Before opening a PR, verify the remote, branch, clean worktree, diff, tests, and exact pushed commit. Never force-push. Pass that full 40-character pushed head SHA explicitly. Keep the short body free of credentials and credential-like -text. Validate without reading a credential or using the network: +text. The client screens structured assignments, known token formats, and long +high-entropy values as an accidental-disclosure barrier. It cannot prove text +is secret-free: short or multiword secrets under innocuous names may resemble +ordinary prose, so never place credential material in either field. Structured +inspection is deliberately bounded; large, deeply nested, or unusual config +blobs do not belong in these fields and may be rejected. Validate without +reading a credential or using the network: ```sh -/opt/coordinator/gitea_api.py --dry-run create-draft REPO \ +/opt/scm/gitea_api.py --dry-run create-draft REPO \ --base main --head hermes/TASK --head-sha FULL_PUSHED_SHA \ --title "Focused change" --body "Tests and review evidence" ``` diff --git a/services/vault/hermes-auth-role-bootstrap-job.yaml b/services/vault/hermes-auth-role-bootstrap-job.yaml index 07c223f0..7804a9f3 100644 --- a/services/vault/hermes-auth-role-bootstrap-job.yaml +++ b/services/vault/hermes-auth-role-bootstrap-job.yaml @@ -3,7 +3,7 @@ apiVersion: batch/v1 kind: Job metadata: - name: vault-k8s-auth-hermes-8 + name: vault-k8s-auth-hermes-9 namespace: vault spec: backoffLimit: 2 diff --git a/services/vault/scripts/vault_k8s_auth_configure.sh b/services/vault/scripts/vault_k8s_auth_configure.sh index 73572e62..612391c9 100644 --- a/services/vault/scripts/vault_k8s_auth_configure.sh +++ b/services/vault/scripts/vault_k8s_auth_configure.sh @@ -256,7 +256,9 @@ write_policy_and_role "game-stream" "game-stream" "game-stream-vault" \ write_policy_and_role "hermes" "hermes" "hermes-vault,hermes-triage" \ "hermes/triage-oidc hermes/agent-tokens hermes/triage-api" "" write_policy_and_role "hermes-agent" "hermes" "hermes-agent,hermes-switchyard" \ - "hermes/agent-oidc hermes/agent-tokens hermes/chat-telegram hermes/developer-keycloak hermes/developer-gitea hermes/developer-harbor hermes/developer-jenkins hermes/developer-ssh" "" + "hermes/agent-oidc hermes/agent-tokens hermes/chat-telegram hermes/developer-keycloak hermes/developer-harbor hermes/developer-jenkins hermes/developer-ssh" "" +write_policy_and_role "hermes-scm-broker" "hermes-scm" "hermes-scm-broker" \ + "hermes/developer-gitea" "" write_policy_and_role "hermes-credential-sync" "hermes" "hermes-agent" \ "" "hermes/agent-tokens" write_policy_and_role "hermes-node-ssh" "hermes" "hermes-node-ssh-access" \ diff --git a/testing/quality_contract.json b/testing/quality_contract.json index e8793d84..805857d9 100644 --- a/testing/quality_contract.json +++ b/testing/quality_contract.json @@ -20,6 +20,18 @@ "ci/scripts/supply_chain_report.py", "services/mailu/scripts/mailu_sync.py", "services/mailu/scripts/mailu_sync_listener.py", + "services/gitea/scripts/gitea_branch_protection_check.py", + "services/hermes/scm-common/scripts/gitea_api.py", + "services/hermes/scm-common/scripts/gitea_api_policy.py", + "services/hermes/scm-common/scripts/scm_broker.py", + "services/hermes/scm-common/scripts/scm_broker_client.py", + "services/hermes/scm-common/scripts/scm_broker_io.py", + "services/hermes/scm-common/scripts/scm_broker_server.py", + "services/hermes/scripts/hermes_coordinator.py", + "services/hermes/scripts/node_account_audit.py", + "services/hermes/scripts/node_account_hardening.py", + "services/hermes/scripts/node_account_io.py", + "services/hermes/scripts/stage_runtime_access.py", "testing/__init__.py", "testing/quality_contract.py", "testing/quality_docs.py", @@ -43,6 +55,13 @@ "services/comms/scripts/tests", "services/mailu/scripts/mailu_sync.py", "services/mailu/scripts/mailu_sync_listener.py", + "services/gitea/scripts/gitea_branch_protection_check.py", + "services/hermes/scm-common/scripts", + "services/hermes/scripts/hermes_coordinator.py", + "services/hermes/scripts/node_account_audit.py", + "services/hermes/scripts/node_account_hardening.py", + "services/hermes/scripts/node_account_io.py", + "services/hermes/scripts/stage_runtime_access.py", "testing/tests", "testing" ], @@ -58,6 +77,9 @@ "coverage_sources": [ "ci/scripts", "scripts.render.dashboards_render_atlas", + "services/gitea/scripts", + "services/hermes/scm-common/scripts", + "services/hermes/scripts", "services/mailu/scripts", "testing" ], @@ -124,6 +146,13 @@ "ci/tests/**/*.py", "scripts/tests/**/*.py", "services/*/scripts/tests/**/*.py", + "services/gitea/scripts/gitea_branch_protection_check.py", + "services/hermes/scm-common/scripts/*.py", + "services/hermes/scripts/hermes_coordinator.py", + "services/hermes/scripts/node_account_audit.py", + "services/hermes/scripts/node_account_hardening.py", + "services/hermes/scripts/node_account_io.py", + "services/hermes/scripts/stage_runtime_access.py", "services/mailu/scripts/mailu_sync.py", "services/mailu/scripts/mailu_sync_listener.py" ], @@ -167,6 +196,22 @@ }, "coverage": { "minimum_percent": 95.0, + "minimum_branch_percent": 95.0, + "branch_tracked_files": [ + "services/gitea/scripts/gitea_branch_protection_check.py", + "services/hermes/scm-common/scripts/gitea_api.py", + "services/hermes/scm-common/scripts/gitea_api_policy.py", + "services/hermes/scm-common/scripts/scm_broker.py", + "services/hermes/scm-common/scripts/scm_broker_client.py", + "services/hermes/scm-common/scripts/scm_broker_io.py", + "services/hermes/scm-common/scripts/scm_broker_server.py", + "services/hermes/scripts/hermes_coordinator.py", + "services/hermes/scripts/node_account_audit.py", + "services/hermes/scripts/node_account_hardening.py", + "services/hermes/scripts/node_account_io.py", + "services/hermes/scripts/stage_runtime_access.py", + "testing/quality_coverage.py" + ], "tracked_files": [ "ci/scripts/publish_test_metrics.py", "ci/scripts/publish_test_metrics_quality.py", @@ -174,6 +219,18 @@ "ci/scripts/supply_chain_report.py", "services/mailu/scripts/mailu_sync.py", "services/mailu/scripts/mailu_sync_listener.py", + "services/gitea/scripts/gitea_branch_protection_check.py", + "services/hermes/scm-common/scripts/gitea_api.py", + "services/hermes/scm-common/scripts/gitea_api_policy.py", + "services/hermes/scm-common/scripts/scm_broker.py", + "services/hermes/scm-common/scripts/scm_broker_client.py", + "services/hermes/scm-common/scripts/scm_broker_io.py", + "services/hermes/scm-common/scripts/scm_broker_server.py", + "services/hermes/scripts/hermes_coordinator.py", + "services/hermes/scripts/node_account_audit.py", + "services/hermes/scripts/node_account_hardening.py", + "services/hermes/scripts/node_account_io.py", + "services/hermes/scripts/stage_runtime_access.py", "testing/quality_contract.py", "testing/quality_docs.py", "testing/quality_hygiene.py", diff --git a/testing/quality_coverage.py b/testing/quality_coverage.py index 78d6649f..98cc65d5 100644 --- a/testing/quality_coverage.py +++ b/testing/quality_coverage.py @@ -3,23 +3,31 @@ from __future__ import annotations import xml.etree.ElementTree as ET +from dataclasses import dataclass from pathlib import Path from typing import Any -def _load_percentages(xml_path: Path, root: Path) -> dict[str, float]: - """Load per-file line-rate percentages from a Cobertura XML report.""" +@dataclass(frozen=True) +class CoverageRates: + """Line and branch percentages reported for one source file.""" + + line: float + branch: float | None + + +def _load_percentages(xml_path: Path, root: Path) -> dict[str, CoverageRates]: + """Load per-file line and branch percentages from a Cobertura report.""" tree = ET.parse(xml_path) xml_root = tree.getroot() source_roots = [ - Path(node.text) - for node in xml_root.findall("./sources/source") - if node.text + Path(node.text) for node in xml_root.findall("./sources/source") if node.text ] - percentages: dict[str, float] = {} + percentages: dict[str, CoverageRates] = {} for class_node in xml_root.findall(".//class"): filename = class_node.attrib.get("filename") line_rate = class_node.attrib.get("line-rate") + branch_rate = class_node.attrib.get("branch-rate") if not filename or line_rate is None: continue normalized = filename.replace("\\", "/") @@ -32,7 +40,10 @@ def _load_percentages(xml_path: Path, root: Path) -> dict[str, float]: if candidate.exists(): key = candidate.relative_to(root).as_posix() break - percentages[key] = float(line_rate) * 100.0 + percentages[key] = CoverageRates( + line=float(line_rate) * 100.0, + branch=float(branch_rate) * 100.0 if branch_rate is not None else None, + ) return percentages @@ -47,17 +58,39 @@ def run_check(contract: dict[str, Any], root: Path, xml_path: Path) -> list[str] percentages = _load_percentages(xml_path, root) minimum = float(contract.get("coverage", {}).get("minimum_percent", 95.0)) + coverage_contract = contract.get("coverage", {}) + branch_minimum = coverage_contract.get("minimum_branch_percent") + if branch_minimum is not None: + branch_minimum = float(branch_minimum) + branch_tracked = { + path.replace("\\", "/") + for path in coverage_contract.get( + "branch_tracked_files", + coverage_contract.get("tracked_files", []) + if branch_minimum is not None + else [], + ) + } issues: list[str] = [] for relative_path in contract.get("coverage", {}).get("tracked_files", []): normalized = relative_path.replace("\\", "/") - percent = percentages.get(normalized) - if percent is None: + rates = percentages.get(normalized) + if rates is None: issues.append(f"coverage missing for tracked file: {relative_path}") continue - if percent + 1e-9 < minimum: + if rates.line + 1e-9 < minimum: issues.append( - f"coverage below {minimum:.1f}%: {relative_path} ({percent:.1f}%)" + f"coverage below {minimum:.1f}%: {relative_path} ({rates.line:.1f}%)" + ) + if branch_minimum is None or normalized not in branch_tracked: + continue + if rates.branch is None: + issues.append(f"branch coverage missing for tracked file: {relative_path}") + elif rates.branch + 1e-9 < branch_minimum: + issues.append( + "branch coverage below " + f"{branch_minimum:.1f}%: {relative_path} ({rates.branch:.1f}%)" ) return issues @@ -77,9 +110,9 @@ def compute_workspace_line_coverage( samples: list[float] = [] for relative_path in contract.get("coverage", {}).get("tracked_files", []): normalized = relative_path.replace("\\", "/") - percent = percentages.get(normalized) - if percent is not None: - samples.append(percent) + rates = percentages.get(normalized) + if rates is not None: + samples.append(rates.line) if not samples: return 0.0 return round(sum(samples) / len(samples), 3) diff --git a/testing/tests/test_hermes_chat_images.py b/testing/tests/test_hermes_chat_images.py new file mode 100644 index 00000000..48c6c2e5 --- /dev/null +++ b/testing/tests/test_hermes_chat_images.py @@ -0,0 +1,282 @@ +"""Image generation and sandbox-boundary contracts for Hermes chat.""" + +from __future__ import annotations + +import importlib.util +import sys +import time +from types import SimpleNamespace + +import yaml + +from testing.tests.test_hermes_chat_support import ( + HERMES, + VAULT, + _documents, +) + + +def test_chat_image_generation_uses_private_owner_broker(): + """Family pods get image bytes without receiving the owner's OAuth file.""" + configmap = _documents(HERMES / "chat-configmap.yaml")[0] + assert "shared desktop/Wolf lane" in configmap["data"]["SOUL.md"] + assert "local FLUX waits" in configmap["data"]["SOUL.md"] + assert "Use `image_generate_local`" in configmap["data"]["SOUL.md"] + assert "Use `image_generate_hosted`" in configmap["data"]["SOUL.md"] + assert "ComfyUI endpoint" in configmap["data"]["SOUL.md"] + assert "`MEDIA:` path" in configmap["data"]["SOUL.md"] + assert "`image_edit_latest`" in configmap["data"]["SOUL.md"] + assert ( + "Preserve the most recently selected image lane" in configmap["data"]["SOUL.md"] + ) + config = yaml.safe_load(configmap["data"]["config.yaml"]) + assert config["image_gen"] == { + "provider": "atlas-broker", + "model": "atlas-image-auto-high", + } + assert config["plugins"]["enabled"] == ["atlas-broker", "auto-router"] + + statefulset = _documents(HERMES / "chat-statefulset.yaml")[0] + pod = statefulset["spec"]["template"]["spec"] + hermes = next(item for item in pod["containers"] if item["name"] == "hermes") + mounts = {item["name"]: item for item in hermes["volumeMounts"]} + assert mounts["image-plugin"]["mountPath"] == ( + "/opt/hermes/plugins/image_gen/atlas-broker" + ) + assert mounts["runtime-access"]["mountPath"] == "/runtime-access" + assert "provider-auth" not in mounts + assert not any( + mount["name"] == "home" and "agent" in str(mount) + for mount in hermes["volumeMounts"] + ) + env = {item["name"]: item["value"] for item in hermes["env"]} + assert env["HERMES_IMAGE_BROKER_URL"].startswith("http://hermes-image-broker.") + + plugin = (HERMES / "plugins" / "image-gen-broker" / "__init__.py").read_text() + assert '"local": "flux-2-klein-4b-local"' in plugin + assert '"hosted": "gpt-image-2-high"' in plugin + assert 'name="image_generate_local"' in plugin + assert 'name="image_generate_hosted"' in plugin + assert '"name": "image_edit_latest"' in plugin + assert '"name": "image_edit_latest_local"' in plugin + assert '"name": "image_edit_latest_hosted"' in plugin + assert "def _latest_generated_image" in plugin + assert "newest MEDIA: path from the conversation" in plugin + assert 'candidate.upper().startswith("MEDIA:")' in plugin + assert "override=True" not in plugin + + agent = _documents(HERMES / "agent-deployment.yaml")[0] + containers = agent["spec"]["template"]["spec"]["containers"] + broker = next(item for item in containers if item["name"] == "image-broker") + assert broker["ports"] == [ + {"name": "image-broker", "containerPort": 9002, "protocol": "TCP"} + ] + assert broker["securityContext"]["readOnlyRootFilesystem"] is True + assert broker["securityContext"]["runAsNonRoot"] is True + + services = _documents(HERMES / "service.yaml") + service = next( + item for item in services if item["metadata"]["name"] == "hermes-image-broker" + ) + assert service["spec"]["selector"] == {"app": "hermes-agent"} + + oauth_store = _documents(HERMES / "oauth-session-store.yaml") + redis = next(item for item in oauth_store if item["kind"] == "Deployment") + assert redis["spec"]["strategy"]["type"] == "Recreate" + assert "--appendonly" in redis["spec"]["template"]["spec"]["containers"][0]["args"] + + policies = _documents(HERMES / "networkpolicy.yaml") + agent_policy = next( + item + for item in policies + if item["metadata"]["name"] == "hermes-agent-isolation" + ) + broker_ingress = next( + rule + for rule in agent_policy["spec"]["ingress"] + if {port["port"] for port in rule["ports"]} == {9002, 9003} + ) + assert broker_ingress["from"][0]["podSelector"]["matchLabels"] == { + "app": "hermes-chat-tenant" + } + vault_policy = (VAULT / "scripts" / "vault_k8s_auth_configure.sh").read_text() + agent_role = vault_policy[ + vault_policy.index('write_policy_and_role "hermes-agent"') : vault_policy.index( + "write_policy_and_role", + vault_policy.index('write_policy_and_role "hermes-agent"') + 1, + ) + ] + assert "hermes/developer-gitea" not in agent_role + assert 'write_policy_and_role "hermes-scm-broker" "hermes-scm"' in vault_policy + assert ( + 'write_policy_and_role "hermes-node-ssh" "hermes" ' + '"hermes-node-ssh-access"' in vault_policy + ) + + +def test_compact_image_edit_resolves_latest_tenant_artifact(tmp_path, monkeypatch): + """Follow-up edits resolve the source server-side and keep tool JSON small.""" + provider_module = SimpleNamespace( + DEFAULT_ASPECT_RATIO="square", + ImageGenProvider=object, + error_response=lambda **value: value, + normalize_reference_images=lambda value: value, + resolve_aspect_ratio=lambda value: value, + save_b64_image=lambda *_args, **_kwargs: tmp_path / "saved.png", + success_response=lambda **value: value, + ) + monkeypatch.setitem(sys.modules, "agent", SimpleNamespace()) + monkeypatch.setitem(sys.modules, "agent.image_gen_provider", provider_module) + spec = importlib.util.spec_from_file_location( + "hermes_image_plugin", + HERMES / "plugins" / "image-gen-broker" / "__init__.py", + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + image_dir = tmp_path / "cache" / "images" + image_dir.mkdir(parents=True) + older = image_dir / "atlas_flux-old.png" + newest = image_dir / "atlas_gpt-image-new.png" + older.write_bytes(b"older") + newest.write_bytes(b"newest") + older.touch() + time.sleep(0.001) + newest.touch() + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + + calls = [] + monkeypatch.setattr( + module, + "_handle_image_generate", + lambda args, route: calls.append((args, route)) or "ok", + ) + assert module._handle_hosted_edit({"prompt": "make it a clown"}) == "ok" + assert calls == [ + ( + { + "prompt": "make it a clown", + "image_url": str(newest.resolve()), + }, + "hosted", + ) + ] + + +def test_chat_reasoning_uses_switchyard_without_owner_credentials(): + """Family pods use AUTO/manual routes without mounting owner credentials.""" + configmap = _documents(HERMES / "chat-configmap.yaml")[0] + config = yaml.safe_load(configmap["data"]["config.yaml"]) + assert config["model"] == { + "provider": "atlas-switchyard", + "default": "atlas/auto/fast", + "model": "atlas/auto/fast", + } + assert config["providers"]["atlas-switchyard"] == { + "name": "Automatic Router", + "api": "http://hermes-switchyard.hermes.svc.cluster.local:9005/v1", + "api_key": "atlas-switchyard", + "default_model": "atlas/auto/fast", + "transport": "chat_completions", + } + assert config["platforms"]["api_server"]["extra"]["model_routes"] == { + route: {"provider": "atlas-switchyard", "model": route} + for route in [ + "atlas/auto/fast", + "atlas/auto/balanced", + "atlas/auto/deep", + "atlas/auto/maximum", + "atlas/manual/codex/luna", + "atlas/manual/codex/terra", + "atlas/manual/codex/sol", + "atlas/manual/claude/haiku", + "atlas/manual/claude/fable", + "atlas/manual/claude/sonnet", + "atlas/manual/claude/opus", + "atlas/manual/local/qwen-14b", + ] + } + + agent = _documents(HERMES / "agent-deployment.yaml")[0] + containers = agent["spec"]["template"]["spec"]["containers"] + broker = next(item for item in containers if item["name"] == "codex-broker") + assert broker["ports"] == [ + {"name": "codex-broker", "containerPort": 9003, "protocol": "TCP"} + ] + assert broker["securityContext"]["readOnlyRootFilesystem"] is True + assert broker["securityContext"]["runAsNonRoot"] is True + assert {item["name"]: item["value"] for item in broker["env"]}.items() >= { + "PYTHONPATH": "/opt/hermes", + "HERMES_CODEX_BROKER_LISTEN_PORT": "9003", + "HERMES_ROUTING_CATALOG_PATH": "/routing-catalog/catalog.json", + }.items() + + services = _documents(HERMES / "service.yaml") + service = next( + item for item in services if item["metadata"]["name"] == "hermes-codex-broker" + ) + assert service["spec"]["selector"] == {"app": "hermes-agent"} + assert service["spec"]["ports"] == [ + { + "name": "http", + "port": 9003, + "targetPort": "codex-broker", + "protocol": "TCP", + } + ] + + statefulset = _documents(HERMES / "chat-statefulset.yaml")[0] + assert ( + statefulset["spec"]["template"]["metadata"]["annotations"][ + "ai.bstein.dev/config-rev" + ] + == "20260816-telegram-topics" + ) + pod_spec = statefulset["spec"]["template"]["spec"] + patch_init = next( + item + for item in pod_spec["initContainers"] + if item["name"] == "patch-stream-recovery" + ) + assert patch_init["command"][-2:] == [ + "/opt/hermes/agent/conversation_loop.py", + "/patched/conversation_loop.py", + ] + hermes = next(item for item in pod_spec["containers"] if item["name"] == "hermes") + assert { + "name": "stream-recovery-patch", + "mountPath": "/opt/hermes/agent/conversation_loop.py", + "subPath": "conversation_loop.py", + } in hermes["volumeMounts"] + api_session_init = next( + item + for item in pod_spec["initContainers"] + if item["name"] == "patch-api-server-sessions" + ) + assert "patch_api_server_sessions.py" in api_session_init["args"][0] + assert "migrate_telegram_api_sessions.py" in api_session_init["args"][0] + assert { + "name": "api-server-patch", + "mountPath": "/opt/hermes/gateway/platforms/api_server.py", + "subPath": "api_server.py", + } in hermes["volumeMounts"] + assert any(volume["name"] == "api-server-patch" for volume in pod_spec["volumes"]) + assert not any( + mount["mountPath"].endswith("/.codex") for mount in hermes["volumeMounts"] + ) + + policies = _documents(HERMES / "networkpolicy.yaml") + agent_policy = next( + item + for item in policies + if item["metadata"]["name"] == "hermes-agent-isolation" + ) + broker_ingress = next( + rule + for rule in agent_policy["spec"]["ingress"] + if {port["port"] for port in rule["ports"]} == {9002, 9003} + ) + assert broker_ingress["from"][0]["podSelector"]["matchLabels"] == { + "app": "hermes-chat-tenant" + } diff --git a/testing/tests/test_hermes_chat_local_runtime.py b/testing/tests/test_hermes_chat_local_runtime.py new file mode 100644 index 00000000..ddb0ef40 --- /dev/null +++ b/testing/tests/test_hermes_chat_local_runtime.py @@ -0,0 +1,172 @@ +"""Local inference and sandbox execution contracts for Hermes chat.""" + +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path +from types import SimpleNamespace + + +from testing.tests.test_hermes_chat_support import ( + HERMES, + ROOT, + _documents, +) + + +def test_titan20_serializes_classifier_and_local_chat_model_residency(): + """Classifier and local chat share one serialized resident Qwen weight.""" + deployment = _documents( + Path(__file__).parents[2] / "services/ai-llm/deployment.yaml" + )[0] + pod = deployment["spec"]["template"]["spec"] + required = pod["affinity"]["nodeAffinity"][ + "requiredDuringSchedulingIgnoredDuringExecution" + ]["nodeSelectorTerms"][0]["matchExpressions"][0] + assert required["values"] == ["titan-20"] + container = pod["containers"][0] + env = {item["name"]: item["value"] for item in container["env"]} + assert env["OLLAMA_MAX_LOADED_MODELS"] == "1" + assert env["OLLAMA_NUM_PARALLEL"] == "1" + assert env["OLLAMA_KEEP_ALIVE"] == "-1" + assert env["OLLAMA_CONTEXT_LENGTH"] == "8192" + warm_command = " ".join(container["command"]) + assert "--keepalive=-1" not in warm_command + models = next(item for item in pod["volumes"] if item["name"] == "models") + assert models["persistentVolumeClaim"]["claimName"] == ("ollama-models-titan20") + + +def test_local_image_gpu_guard_distinguishes_background_and_saturated_gpu(monkeypatch): + """Lease-idle desktop spikes may coexist, but saturation still blocks FLUX.""" + source = ROOT / "dockerfiles" / "hermes-local-image-server.py" + spec = importlib.util.spec_from_file_location("hermes_local_image_server", source) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + monkeypatch.setattr(module, "GPU_ACTIVITY_NODE", "titan-24") + monkeypatch.setattr(module, "GPU_ACTIVE_SM_PERCENT", 80.0) + monkeypatch.setattr(module, "GPU_MAX_EXTERNAL_MEMORY_BYTES", 3 << 30) + + idle = module._parse_gpu_activity( + "\n".join( + [ + 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="Xorg"} 0', + 'nvidia_process_gpu_memory_used_bytes{node="titan-24",namespace="host",process="Xorg"} 1900000000', + 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="game-stream",process="wolf"} 3', + 'nvidia_process_gpu_memory_used_bytes{node="titan-24",namespace="hermes",process="python"} 9000000000', + ] + ) + ) + assert idle["interactive_active"] is False + assert idle["external_gpu_memory_bytes"] == 1900000000 + assert idle["external_gpu_sm_percent"] == 3 + + background_spike = module._parse_gpu_activity( + 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="sway"} 41\n' + ) + assert background_spike["interactive_active"] is False + + active = module._parse_gpu_activity( + 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="steam"} 91\n' + ) + assert active["interactive_active"] is True + assert "91%" in active["gpu_guard_reason"] + + +def test_local_flux_renderer_uses_a_disposable_cuda_worker(monkeypatch): + """A completed render must not leave its CUDA context in the API process.""" + source = ROOT / "dockerfiles" / "hermes-local-image-server.py" + spec = importlib.util.spec_from_file_location("hermes_local_image_worker", source) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + calls = [] + + def run(command, **kwargs): + calls.append((command, kwargs)) + return SimpleNamespace( + returncode=0, + stdout=b'{"success":true,"route":"local","image_b64":"cG5n"}', + stderr=b"", + ) + + monkeypatch.setattr(module.subprocess, "run", run) + result = module._render({"prompt": "black cat", "aspect_ratio": "square"}) + + assert result["route"] == "local" + command, options = calls[0] + assert command[-1] == "--render-worker" + assert json.loads(options["input"]) == { + "prompt": "black cat", + "aspect_ratio": "square", + } + assert options["timeout"] == module.RENDER_TIMEOUT_SECONDS + assert options["check"] is False + + +def test_local_flux_uses_low_vram_offload_without_reducing_resolution(): + """The shared 3080 lane must trade time, not image size, for headroom.""" + source = (ROOT / "dockerfiles" / "hermes-local-image-server.py").read_text() + assert "OFFLOAD_MODE = os.environ.get(" in source + assert '"HERMES_LOCAL_IMAGE_OFFLOAD_MODE", "sequential"' in source + assert "pipe.enable_sequential_cpu_offload()" in source + assert '"square": (1024, 1024)' in source + + +def test_chat_auth_and_relay_are_pod_lifetime_only(): + statefulset = _documents(HERMES / "chat-statefulset.yaml")[0] + pod = statefulset["spec"]["template"]["spec"] + containers = statefulset["spec"]["template"]["spec"]["containers"] + + for name in ("hermes", "webui"): + container = next(item for item in containers if item["name"] == name) + env = {item["name"]: item["value"] for item in container["env"]} + assert env["HERMES_AUTH_FILE"] == "/runtime-access/hermes-auth.json" + mount = next( + item + for item in container["volumeMounts"] + if item["name"] == "runtime-access" + ) + assert mount["mountPath"] == "/runtime-access" + assert "subPath" not in mount + + hermes_env = { + item["name"]: item["value"] + for item in next(item for item in containers if item["name"] == "hermes")["env"] + } + runtime = next(item for item in pod["volumes"] if item["name"] == "runtime-access") + assert runtime["emptyDir"] == {"medium": "Memory", "sizeLimit": "2Mi"} + assert not any(item["name"] == "provider-auth" for item in pod["volumes"]) + init_command = next( + item for item in pod["initContainers"] if item["name"] == "init-config" + )["command"][2] + for key in ( + "ANTHROPIC_API_KEY", + "API_SERVER_KEY", + "CLAUDE_CODE_OAUTH_TOKEN", + "GITEA_TOKEN", + "HERMES_IMAGE_BROKER_KEY", + "OPENAI_API_KEY", + ): + assert key in init_command + assert "printf 'API_SERVER_KEY=%s" not in init_command + assert hermes_env["AGENT_BROWSER_EXECUTABLE_PATH"].endswith( + "/chrome-linux/headless_shell" + ) + assert "--no-sandbox" in hermes_env["AGENT_BROWSER_ARGS"] + + +def test_sandbox_executes_python_with_bounded_output(tmp_path: Path, monkeypatch): + source = ROOT / "dockerfiles" / "hermes-chat-sandbox-server.py" + spec = importlib.util.spec_from_file_location("hermes_chat_sandbox_server", source) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + monkeypatch.setattr(module, "WORKSPACE", tmp_path) + + result = module._execute("import math\nprint(math.comb(10, 3))") + + assert result["success"] is True + assert result["stdout"] == "120\n" + assert result["stderr"] == "" diff --git a/testing/tests/test_hermes_chat_provider_auth.py b/testing/tests/test_hermes_chat_provider_auth.py new file mode 100644 index 00000000..8e432131 --- /dev/null +++ b/testing/tests/test_hermes_chat_provider_auth.py @@ -0,0 +1,476 @@ +"""Provider subscription and OAuth contracts for Hermes chat.""" + +from __future__ import annotations + +import base64 +import json +import sys +import time +from pathlib import Path +from types import ModuleType + +import pytest + +from testing.tests.test_hermes_chat_support import ( + _load_broker_module, +) + + +def test_codex_broker_auth_and_request_contract(tmp_path: Path, monkeypatch): + """The relay is bounded, stateless, and rejects unapproved models.""" + module = _load_broker_module("hermes_codex_broker", "codex_broker.py", monkeypatch) + monkeypatch.setattr(module, "TOKEN", "relay-secret") + + assert module._authorized("Bearer relay-secret") is True + assert module._authorized("Bearer wrong") is False + assert module._real_model("route/codex/gpt-5.6-sol/xhigh") == "gpt-5.6-sol" + payload = module._validate_payload( + { + "model": "gpt-5.6-terra", + "input": "route this chat turn", + "store": True, + "stream": False, + "max_output_tokens": 96, + "max_completion_tokens": 96, + "max_tokens": 96, + "temperature": 0.7, + "top_p": 0.9, + } + ) + assert payload["store"] is False + assert payload["stream"] is True + assert "max_output_tokens" not in payload + assert "max_completion_tokens" not in payload + assert "max_tokens" not in payload + assert "temperature" not in payload + assert "top_p" not in payload + assert payload["input"] == [ + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "route this chat turn"}], + } + ] + response_item = { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "keep this item"}], + } + assert module._validate_payload({"model": "gpt-5.6-terra", "input": response_item})[ + "input" + ] == [response_item] + response_items = [response_item] + assert ( + module._validate_payload({"model": "gpt-5.6-terra", "input": response_items})[ + "input" + ] + is response_items + ) + image_items = [ + { + "type": "message", + "role": "user", + "content": [ + {"type": "input_text", "text": "What color is this?"}, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,cHJpdmF0ZQ==", + "detail": "high", + }, + }, + ], + } + ] + assert module._validate_payload({"model": "gpt-5.6-terra", "input": image_items})[ + "input" + ][0]["content"][1] == { + "type": "input_image", + "image_url": "data:image/png;base64,cHJpdmF0ZQ==", + "detail": "high", + } + switchyard_image_items = [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "What color is this?"}, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,cHJpdmF0ZQ==", + "detail": "auto", + }, + }, + ], + } + ] + assert module._validate_payload( + {"model": "gpt-5.6-terra", "input": switchyard_image_items} + )["input"][0]["content"][1] == { + "type": "input_image", + "image_url": "data:image/png;base64,cHJpdmF0ZQ==", + "detail": "auto", + } + switchyard_base64_items = [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "What color is this?"}, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "cHJpdmF0ZQ==", + }, + }, + ], + } + ] + normalized_base64 = module._validate_payload( + {"model": "gpt-5.6-terra", "input": switchyard_base64_items} + )["input"][0]["content"][1] + assert normalized_base64 == { + "type": "input_image", + "image_url": "data:image/png;base64,cHJpdmF0ZQ==", + } + switchyard_enum_items = [ + { + "role": "user", + "content": [ + { + "type": "input_image", + "image_url": { + "type": "url", + "data": { + "url": "data:image/png;base64,cHJpdmF0ZQ==", + "detail": "high", + }, + }, + } + ], + } + ] + nested_image = module._validate_payload( + {"model": "gpt-5.6-terra", "input": switchyard_enum_items} + )["input"][0]["content"][0] + assert nested_image["image_url"] == "data:image/png;base64,cHJpdmF0ZQ==" + assert nested_image["detail"] == "high" + with pytest.raises(ValueError, match=r"non-empty Responses image URL.*str\[4\]"): + module._validate_payload( + { + "model": "gpt-5.6-terra", + "input": [ + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_image", + "image_url": {"detail": "high"}, + } + ], + } + ], + } + ) + routed = module._validate_payload( + { + "model": "route/codex/gpt-5.6-luna/low", + "input": "use the low route", + "stream": False, + } + ) + assert routed["model"] == "gpt-5.6-luna" + with pytest.raises(ValueError, match="unsupported Codex model"): + module._validate_payload({"model": "unapproved-model", "input": "hello"}) + with pytest.raises(ValueError, match="non-empty Responses input"): + module._validate_payload({"model": "gpt-5.6-terra", "input": ""}) + with pytest.raises(ValueError, match="non-empty Responses input list"): + module._validate_payload({"model": "gpt-5.6-terra", "input": []}) + + completed = { + "id": "resp_test", + "object": "response", + "status": "completed", + "output": [], + } + completed_item = { + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "done"}], + } + assert module._completed_response( + [ + "event: response.created", + 'data: {"type":"response.created","response":{}}', + "event: response.output_item.done", + "data: " + + json.dumps( + { + "type": "response.output_item.done", + "output_index": 0, + "item": completed_item, + } + ), + "event: response.completed", + "data: " + + json.dumps({"type": "response.completed", "response": completed}), + "data: [DONE]", + ] + )["output"] == [completed_item] + raw_stream = ( + "event: response.output_item.done\n" + "data: " + + json.dumps( + { + "type": "response.output_item.done", + "output_index": 0, + "item": completed_item, + } + ) + + "\n\nevent: response.completed\ndata: " + + json.dumps({"type": "response.completed", "response": completed}) + + "\n\n" + ).encode() + normalized = module._normalized_stream( + raw_stream, {**completed, "output": [completed_item]} + ).decode() + terminal_data = next( + line for line in normalized.splitlines() if '"response.completed"' in line + ) + assert json.loads(terminal_data.removeprefix("data: "))["response"]["output"] == [ + completed_item + ] + assert normalized.endswith("\n\n") + streamed_function_item = { + "type": "function_call", + "name": "read_file", + "status": "completed", + "arguments": '{"path":"/tmp"}', + } + streamed_function_body = ( + "event: response.function_call_arguments.delta\n" + 'data: {"type":"response.function_call_arguments.delta",' + '"item_id":"call_1","delta":"{\\"path\\":\\"/tmp\\"}"}\n\n' + "event: response.function_call_arguments.done\n" + 'data: {"type":"response.function_call_arguments.done",' + '"item_id":"call_1","arguments":"{\\"path\\":\\"/tmp\\"}"}\n\n' + "event: response.output_item.done\n" + 'data: {"type":"response.output_item.done","output_index":0,' + '"item":{"type":"function_call","name":"read_file",' + '"arguments":"{\\"path\\":\\"/tmp\\"}"}}\n\n' + "event: response.completed\n" + "data: " + + json.dumps( + { + "type": "response.completed", + "response": {**completed, "output": [streamed_function_item]}, + } + ) + + "\n\n" + ).encode() + normalized_function_stream = module._normalized_stream( + streamed_function_body, {**completed, "output": [streamed_function_item]} + ).decode() + assert "response.function_call_arguments.delta" in normalized_function_stream + assert "response.function_call_arguments.done" not in normalized_function_stream + assert "response.output_item.done" not in normalized_function_stream + normalized_terminal = next( + line + for line in normalized_function_stream.splitlines() + if '"response.completed"' in line + ) + assert ( + json.loads(normalized_terminal.removeprefix("data: "))["response"]["output"] + == [] + ) + with pytest.raises(RuntimeError, match="retryable incomplete response"): + module._completed_response( + [ + "event: response.incomplete", + 'data: {"type":"response.incomplete","response":' + '{"status":"incomplete","incomplete_details":' + '{"reason":"max_output_tokens"}}}', + ] + ) + with pytest.raises(RuntimeError, match="provider unavailable"): + module._completed_response( + [ + "event: error", + 'data: {"type":"error","error":{"message":"provider unavailable"}}', + ] + ) + malformed_tool_item = { + "type": "function_call", + "name": "search_files", + "status": "completed", + "arguments": '{"path":"","offset":', + } + with pytest.raises(RuntimeError, match="malformed function arguments"): + module._completed_response( + [ + "event: response.output_item.done", + "data: " + + json.dumps( + { + "type": "response.output_item.done", + "output_index": 0, + "item": malformed_tool_item, + } + ), + "event: response.completed", + "data: " + + json.dumps({"type": "response.completed", "response": completed}), + ] + ) + valid_tool_item = { + **malformed_tool_item, + "arguments": '{"path":"","offset":0}', + } + assert module._completed_response( + [ + "event: response.output_item.done", + "data: " + + json.dumps( + { + "type": "response.output_item.done", + "output_index": 0, + "item": valid_tool_item, + } + ), + "event: response.completed", + "data: " + + json.dumps({"type": "response.completed", "response": completed}), + ] + )["output"] == [valid_tool_item] + with pytest.raises(RuntimeError, match="malformed function arguments"): + module._completed_response( + [ + "event: response.function_call_arguments.delta", + 'data: {"type":"response.function_call_arguments.delta",' + '"item_id":"call_1","output_index":0,' + '"delta":"{\\"path\\":\\"/tmp\\",\\"offset\\":"}', + "event: response.completed", + "data: " + + json.dumps({"type": "response.completed", "response": completed}), + ] + ) + streamed_tool = module._completed_response( + [ + "event: response.function_call_arguments.delta", + 'data: {"type":"response.function_call_arguments.delta",' + '"item_id":"call_2","output_index":0,' + '"delta":"{\\"path\\":\\"/tmp\\",\\"offset\\":"}', + "event: response.function_call_arguments.done", + 'data: {"type":"response.function_call_arguments.done",' + '"item_id":"call_2","output_index":0,' + '"arguments":"{\\"path\\":\\"/tmp\\",\\"offset\\":0}"}', + "event: response.completed", + "data: " + + json.dumps({"type": "response.completed", "response": completed}), + ] + ) + assert streamed_tool["status"] == "completed" + + auth_dir = tmp_path / ".codex" + auth_dir.mkdir() + # The token payload need only prove the broker reads CODEX_HOME directly. + encoded = ( + base64.urlsafe_b64encode(json.dumps({"exp": time.time() + 3600}).encode()) + .decode() + .rstrip("=") + ) + (auth_dir / "auth.json").write_text( + json.dumps({"tokens": {"access_token": f"header.{encoded}.signature"}}) + ) + monkeypatch.setenv("CODEX_HOME", str(auth_dir)) + assert module._access_token().startswith("header.") + + +def test_codex_broker_refreshes_and_persists_first_party_oauth( + tmp_path: Path, monkeypatch +): + """Expired ChatGPT OAuth refreshes in the canonical Codex CLI store.""" + module = _load_broker_module( + "hermes_codex_refresh_broker", "codex_broker.py", monkeypatch + ) + auth_dir = tmp_path / ".codex" + auth_dir.mkdir() + + def jwt(expires_at: float) -> str: + payload = ( + base64.urlsafe_b64encode(json.dumps({"exp": expires_at}).encode()) + .decode() + .rstrip("=") + ) + return f"header.{payload}.signature" + + expired = jwt(time.time() - 60) + live = jwt(time.time() + 3600) + auth_path = auth_dir / "auth.json" + auth_path.write_text( + json.dumps( + { + "auth_mode": "chatgpt", + "tokens": { + "access_token": expired, + "refresh_token": "refresh-old", + }, + } + ) + ) + calls = [] + auth_module = ModuleType("hermes_cli.auth") + + def refresh(access_token, refresh_token, *, timeout_seconds): + calls.append((access_token, refresh_token, timeout_seconds)) + return { + "access_token": live, + "refresh_token": "refresh-new", + "last_refresh": "2026-08-12T20:00:00Z", + } + + auth_module.refresh_codex_oauth_pure = refresh + package = ModuleType("hermes_cli") + package.auth = auth_module + monkeypatch.setitem(sys.modules, "hermes_cli", package) + monkeypatch.setitem(sys.modules, "hermes_cli.auth", auth_module) + monkeypatch.setenv("CODEX_HOME", str(auth_dir)) + + assert module._access_token() == live + persisted = json.loads(auth_path.read_text()) + assert persisted["tokens"]["access_token"] == live + assert persisted["tokens"]["refresh_token"] == "refresh-new" + assert persisted["last_refresh"] == "2026-08-12T20:00:00Z" + assert calls == [(expired, "refresh-old", 30.0)] + assert auth_path.stat().st_mode & 0o777 == 0o600 + + # A healthy token is reused, so repeated routed turns do not spend a + # refresh token or create a second billing/authentication path. + assert module._access_token() == live + assert len(calls) == 1 + + +def test_claude_broker_uses_native_subscription_without_api_billing(monkeypatch): + """Claude traffic must use the native first-party CLI subscription lane.""" + module = _load_broker_module( + "hermes_claude_broker", "claude_oauth_broker.py", monkeypatch + ) + monkeypatch.setenv("ANTHROPIC_API_KEY", "must-not-leak") + monkeypatch.setenv("CLAUDE_API_KEY", "must-not-leak") + monkeypatch.setattr( + module, + "resolve_route", + lambda route: "claude-fable-5" if "/fable/" in route else route, + ) + + model, effort = module._route( + "route/claude/fable/xhigh", {"output_config": {"effort": "xhigh"}} + ) + + assert (model, effort) == ("claude-fable-5", "xhigh") + assert "ANTHROPIC_API_KEY" not in module._claude_environment() + assert "CLAUDE_API_KEY" not in module._claude_environment() + assert module.CAPACITY_PATTERN.search("weekly usage limit exhausted") diff --git a/testing/tests/test_hermes_chat_quality.py b/testing/tests/test_hermes_chat_quality.py index 025f3bf7..b513b452 100644 --- a/testing/tests/test_hermes_chat_quality.py +++ b/testing/tests/test_hermes_chat_quality.py @@ -1,49 +1,18 @@ -"""Contracts for isolated high-quality Hermes chat capabilities.""" +"""Core configuration and workload contracts for Hermes chat.""" from __future__ import annotations -import base64 import importlib.util -import json -import sqlite3 import sys -import time -import tomllib -from pathlib import Path -from types import ModuleType, SimpleNamespace +from types import SimpleNamespace -import pytest import yaml - -ROOT = Path(__file__).parents[2] -HERMES = ROOT / "services" / "hermes" -VAULT = ROOT / "services" / "vault" - - -def _documents(path: Path) -> list[dict]: - return [doc for doc in yaml.safe_load_all(path.read_text()) if doc] - - -def _load_broker_module(name: str, filename: str, monkeypatch): - """Load one broker with its mounted routing-catalog dependency.""" - catalog_path = HERMES / "scripts" / "routing_catalog.py" - catalog_spec = importlib.util.spec_from_file_location( - "routing_catalog", catalog_path - ) - assert catalog_spec and catalog_spec.loader - catalog = importlib.util.module_from_spec(catalog_spec) - catalog_spec.loader.exec_module(catalog) - monkeypatch.setitem(sys.modules, "routing_catalog", catalog) - monkeypatch.setitem(sys.modules, "httpx", SimpleNamespace()) - - broker_path = HERMES / "scripts" / filename - spec = importlib.util.spec_from_file_location(name, broker_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - +from testing.tests.test_hermes_chat_support import ( + HERMES, + ROOT, + _documents, +) def test_chat_config_enables_real_research_compute_and_delegation(): configmap = _documents(HERMES / "chat-configmap.yaml")[0] @@ -92,13 +61,14 @@ def test_agent_config_keeps_delegated_reviewers_from_owning_task_lifecycle(): assert "Atlas organization has private visibility" in instructions assert "may be public or private" in instructions assert "do not infer a\nrepository's visibility" in instructions - assert "already supplied through `GIT_ASKPASS`" in instructions + assert "brokered Git for clone, fetch" in instructions + assert "client carries no repository credential" in instructions assert "Never call `kanban_show` without a known, non-empty task ID" in instructions assert "bounded ad-hoc inspection and acceptance checks may" in instructions assert "load implementation or TDD skills" in instructions assert "Never call `kanban_show` without\na known, non-empty task ID" in soul assert "must load a skill only when its workflow\nmaterially applies" in soul - assert "runtime-only `GIT_ASKPASS`" in soul + assert "no repository token is present in this pod" in soul assert "`scm.bstein.dev` is Forgejo/Gitea, not GitHub" in instructions assert "GitHub/`gh` skill for an Atlas remote" in instructions assert "load\n`$manage-atlas-pull-requests`" in instructions @@ -484,1765 +454,3 @@ def test_voice_workloads_have_deliberate_xavier_placement(): assert tts["containers"][0]["resources"]["limits"]["cpu"] == "4" assert all("hostPath" not in volume for volume in stt["volumes"]) assert all("hostPath" not in volume for volume in tts["volumes"]) - - -def test_chat_image_generation_uses_private_owner_broker(): - """Family pods get image bytes without receiving the owner's OAuth file.""" - configmap = _documents(HERMES / "chat-configmap.yaml")[0] - assert "shared desktop/Wolf lane" in configmap["data"]["SOUL.md"] - assert "local FLUX waits" in configmap["data"]["SOUL.md"] - assert "Use `image_generate_local`" in configmap["data"]["SOUL.md"] - assert "Use `image_generate_hosted`" in configmap["data"]["SOUL.md"] - assert "ComfyUI endpoint" in configmap["data"]["SOUL.md"] - assert "`MEDIA:` path" in configmap["data"]["SOUL.md"] - assert "`image_edit_latest`" in configmap["data"]["SOUL.md"] - assert "Preserve the most recently selected image lane" in configmap["data"]["SOUL.md"] - config = yaml.safe_load(configmap["data"]["config.yaml"]) - assert config["image_gen"] == { - "provider": "atlas-broker", - "model": "atlas-image-auto-high", - } - assert config["plugins"]["enabled"] == ["atlas-broker", "auto-router"] - - statefulset = _documents(HERMES / "chat-statefulset.yaml")[0] - pod = statefulset["spec"]["template"]["spec"] - hermes = next(item for item in pod["containers"] if item["name"] == "hermes") - mounts = {item["name"]: item for item in hermes["volumeMounts"]} - assert mounts["image-plugin"]["mountPath"] == ( - "/opt/hermes/plugins/image_gen/atlas-broker" - ) - assert mounts["runtime-access"]["mountPath"] == "/runtime-access" - assert "provider-auth" not in mounts - assert not any(mount["name"] == "home" and "agent" in str(mount) for mount in hermes["volumeMounts"]) - env = {item["name"]: item["value"] for item in hermes["env"]} - assert env["HERMES_IMAGE_BROKER_URL"].startswith("http://hermes-image-broker.") - - plugin = (HERMES / "plugins" / "image-gen-broker" / "__init__.py").read_text() - assert '"local": "flux-2-klein-4b-local"' in plugin - assert '"hosted": "gpt-image-2-high"' in plugin - assert 'name="image_generate_local"' in plugin - assert 'name="image_generate_hosted"' in plugin - assert '"name": "image_edit_latest"' in plugin - assert '"name": "image_edit_latest_local"' in plugin - assert '"name": "image_edit_latest_hosted"' in plugin - assert "def _latest_generated_image" in plugin - assert "newest MEDIA: path from the conversation" in plugin - assert 'candidate.upper().startswith("MEDIA:")' in plugin - assert "override=True" not in plugin - - agent = _documents(HERMES / "agent-deployment.yaml")[0] - containers = agent["spec"]["template"]["spec"]["containers"] - broker = next(item for item in containers if item["name"] == "image-broker") - assert broker["ports"] == [ - {"name": "image-broker", "containerPort": 9002, "protocol": "TCP"} - ] - assert broker["securityContext"]["readOnlyRootFilesystem"] is True - assert broker["securityContext"]["runAsNonRoot"] is True - - services = _documents(HERMES / "service.yaml") - service = next( - item for item in services if item["metadata"]["name"] == "hermes-image-broker" - ) - assert service["spec"]["selector"] == {"app": "hermes-agent"} - - oauth_store = _documents(HERMES / "oauth-session-store.yaml") - redis = next(item for item in oauth_store if item["kind"] == "Deployment") - assert redis["spec"]["strategy"]["type"] == "Recreate" - assert "--appendonly" in redis["spec"]["template"]["spec"]["containers"][0]["args"] - - policies = _documents(HERMES / "networkpolicy.yaml") - agent_policy = next( - item for item in policies if item["metadata"]["name"] == "hermes-agent-isolation" - ) - broker_ingress = next( - rule - for rule in agent_policy["spec"]["ingress"] - if {port["port"] for port in rule["ports"]} == {9002, 9003} - ) - assert broker_ingress["from"][0]["podSelector"]["matchLabels"] == { - "app": "hermes-chat-tenant" - } - vault_policy = (VAULT / "scripts" / "vault_k8s_auth_configure.sh").read_text() - assert ( - '"hermes/agent-oidc hermes/agent-tokens hermes/chat-telegram ' - 'hermes/developer-keycloak hermes/developer-gitea ' - 'hermes/developer-harbor hermes/developer-jenkins ' - 'hermes/developer-ssh"' - in vault_policy - ) - assert ( - 'write_policy_and_role "hermes-node-ssh" "hermes" ' - '"hermes-node-ssh-access"' in vault_policy - ) - - -def test_compact_image_edit_resolves_latest_tenant_artifact(tmp_path, monkeypatch): - """Follow-up edits resolve the source server-side and keep tool JSON small.""" - provider_module = SimpleNamespace( - DEFAULT_ASPECT_RATIO="square", - ImageGenProvider=object, - error_response=lambda **value: value, - normalize_reference_images=lambda value: value, - resolve_aspect_ratio=lambda value: value, - save_b64_image=lambda *_args, **_kwargs: tmp_path / "saved.png", - success_response=lambda **value: value, - ) - monkeypatch.setitem(sys.modules, "agent", SimpleNamespace()) - monkeypatch.setitem(sys.modules, "agent.image_gen_provider", provider_module) - spec = importlib.util.spec_from_file_location( - "hermes_image_plugin", - HERMES / "plugins" / "image-gen-broker" / "__init__.py", - ) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - - image_dir = tmp_path / "cache" / "images" - image_dir.mkdir(parents=True) - older = image_dir / "atlas_flux-old.png" - newest = image_dir / "atlas_gpt-image-new.png" - older.write_bytes(b"older") - newest.write_bytes(b"newest") - older.touch() - time.sleep(0.001) - newest.touch() - monkeypatch.setenv("HERMES_HOME", str(tmp_path)) - - calls = [] - monkeypatch.setattr( - module, - "_handle_image_generate", - lambda args, route: calls.append((args, route)) or "ok", - ) - assert module._handle_hosted_edit({"prompt": "make it a clown"}) == "ok" - assert calls == [ - ( - { - "prompt": "make it a clown", - "image_url": str(newest.resolve()), - }, - "hosted", - ) - ] - - -def test_chat_reasoning_uses_switchyard_without_owner_credentials(): - """Family pods use AUTO/manual routes without mounting owner credentials.""" - configmap = _documents(HERMES / "chat-configmap.yaml")[0] - config = yaml.safe_load(configmap["data"]["config.yaml"]) - assert config["model"] == { - "provider": "atlas-switchyard", - "default": "atlas/auto/fast", - "model": "atlas/auto/fast", - } - assert config["providers"]["atlas-switchyard"] == { - "name": "Automatic Router", - "api": "http://hermes-switchyard.hermes.svc.cluster.local:9005/v1", - "api_key": "atlas-switchyard", - "default_model": "atlas/auto/fast", - "transport": "chat_completions", - } - assert config["platforms"]["api_server"]["extra"]["model_routes"] == { - route: {"provider": "atlas-switchyard", "model": route} - for route in [ - "atlas/auto/fast", - "atlas/auto/balanced", - "atlas/auto/deep", - "atlas/auto/maximum", - "atlas/manual/codex/luna", - "atlas/manual/codex/terra", - "atlas/manual/codex/sol", - "atlas/manual/claude/haiku", - "atlas/manual/claude/fable", - "atlas/manual/claude/sonnet", - "atlas/manual/claude/opus", - "atlas/manual/local/qwen-14b", - ] - } - - agent = _documents(HERMES / "agent-deployment.yaml")[0] - containers = agent["spec"]["template"]["spec"]["containers"] - broker = next(item for item in containers if item["name"] == "codex-broker") - assert broker["ports"] == [ - {"name": "codex-broker", "containerPort": 9003, "protocol": "TCP"} - ] - assert broker["securityContext"]["readOnlyRootFilesystem"] is True - assert broker["securityContext"]["runAsNonRoot"] is True - assert {item["name"]: item["value"] for item in broker["env"]}.items() >= { - "PYTHONPATH": "/opt/hermes", - "HERMES_CODEX_BROKER_LISTEN_PORT": "9003", - "HERMES_ROUTING_CATALOG_PATH": "/routing-catalog/catalog.json", - }.items() - - services = _documents(HERMES / "service.yaml") - service = next( - item for item in services if item["metadata"]["name"] == "hermes-codex-broker" - ) - assert service["spec"]["selector"] == {"app": "hermes-agent"} - assert service["spec"]["ports"] == [ - { - "name": "http", - "port": 9003, - "targetPort": "codex-broker", - "protocol": "TCP", - } - ] - - statefulset = _documents(HERMES / "chat-statefulset.yaml")[0] - assert statefulset["spec"]["template"]["metadata"]["annotations"][ - "ai.bstein.dev/config-rev" - ] == "20260816-telegram-topics" - pod_spec = statefulset["spec"]["template"]["spec"] - patch_init = next( - item for item in pod_spec["initContainers"] - if item["name"] == "patch-stream-recovery" - ) - assert patch_init["command"][-2:] == [ - "/opt/hermes/agent/conversation_loop.py", - "/patched/conversation_loop.py", - ] - hermes = next( - item - for item in pod_spec["containers"] - if item["name"] == "hermes" - ) - assert { - "name": "stream-recovery-patch", - "mountPath": "/opt/hermes/agent/conversation_loop.py", - "subPath": "conversation_loop.py", - } in hermes["volumeMounts"] - api_session_init = next( - item for item in pod_spec["initContainers"] - if item["name"] == "patch-api-server-sessions" - ) - assert "patch_api_server_sessions.py" in api_session_init["args"][0] - assert "migrate_telegram_api_sessions.py" in api_session_init["args"][0] - assert { - "name": "api-server-patch", - "mountPath": "/opt/hermes/gateway/platforms/api_server.py", - "subPath": "api_server.py", - } in hermes["volumeMounts"] - assert any( - volume["name"] == "api-server-patch" for volume in pod_spec["volumes"] - ) - assert not any( - mount["mountPath"].endswith("/.codex") - for mount in hermes["volumeMounts"] - ) - - policies = _documents(HERMES / "networkpolicy.yaml") - agent_policy = next( - item for item in policies if item["metadata"]["name"] == "hermes-agent-isolation" - ) - broker_ingress = next( - rule - for rule in agent_policy["spec"]["ingress"] - if {port["port"] for port in rule["ports"]} == {9002, 9003} - ) - assert broker_ingress["from"][0]["podSelector"]["matchLabels"] == { - "app": "hermes-chat-tenant" - } - - -def test_codex_broker_auth_and_request_contract(tmp_path: Path, monkeypatch): - """The relay is bounded, stateless, and rejects unapproved models.""" - module = _load_broker_module( - "hermes_codex_broker", "codex_broker.py", monkeypatch - ) - monkeypatch.setattr(module, "TOKEN", "relay-secret") - - assert module._authorized("Bearer relay-secret") is True - assert module._authorized("Bearer wrong") is False - assert module._real_model("route/codex/gpt-5.6-sol/xhigh") == "gpt-5.6-sol" - payload = module._validate_payload( - { - "model": "gpt-5.6-terra", - "input": "route this chat turn", - "store": True, - "stream": False, - "max_output_tokens": 96, - "max_completion_tokens": 96, - "max_tokens": 96, - "temperature": 0.7, - "top_p": 0.9, - } - ) - assert payload["store"] is False - assert payload["stream"] is True - assert "max_output_tokens" not in payload - assert "max_completion_tokens" not in payload - assert "max_tokens" not in payload - assert "temperature" not in payload - assert "top_p" not in payload - assert payload["input"] == [ - { - "type": "message", - "role": "user", - "content": [{"type": "input_text", "text": "route this chat turn"}], - } - ] - response_item = { - "type": "message", - "role": "user", - "content": [{"type": "input_text", "text": "keep this item"}], - } - assert module._validate_payload( - {"model": "gpt-5.6-terra", "input": response_item} - )["input"] == [response_item] - response_items = [response_item] - assert module._validate_payload( - {"model": "gpt-5.6-terra", "input": response_items} - )["input"] is response_items - image_items = [ - { - "type": "message", - "role": "user", - "content": [ - {"type": "input_text", "text": "What color is this?"}, - { - "type": "image_url", - "image_url": { - "url": "data:image/png;base64,cHJpdmF0ZQ==", - "detail": "high", - }, - }, - ], - } - ] - assert module._validate_payload( - {"model": "gpt-5.6-terra", "input": image_items} - )["input"][0]["content"][1] == { - "type": "input_image", - "image_url": "data:image/png;base64,cHJpdmF0ZQ==", - "detail": "high", - } - switchyard_image_items = [ - { - "role": "user", - "content": [ - {"type": "input_text", "text": "What color is this?"}, - { - "type": "image_url", - "image_url": { - "url": "data:image/png;base64,cHJpdmF0ZQ==", - "detail": "auto", - }, - }, - ], - } - ] - assert module._validate_payload( - {"model": "gpt-5.6-terra", "input": switchyard_image_items} - )["input"][0]["content"][1] == { - "type": "input_image", - "image_url": "data:image/png;base64,cHJpdmF0ZQ==", - "detail": "auto", - } - switchyard_base64_items = [ - { - "role": "user", - "content": [ - {"type": "input_text", "text": "What color is this?"}, - { - "type": "image", - "source": { - "type": "base64", - "media_type": "image/png", - "data": "cHJpdmF0ZQ==", - }, - }, - ], - } - ] - normalized_base64 = module._validate_payload( - {"model": "gpt-5.6-terra", "input": switchyard_base64_items} - )["input"][0]["content"][1] - assert normalized_base64 == { - "type": "input_image", - "image_url": "data:image/png;base64,cHJpdmF0ZQ==", - } - switchyard_enum_items = [ - { - "role": "user", - "content": [ - { - "type": "input_image", - "image_url": { - "type": "url", - "data": { - "url": "data:image/png;base64,cHJpdmF0ZQ==", - "detail": "high", - }, - }, - } - ], - } - ] - nested_image = module._validate_payload( - {"model": "gpt-5.6-terra", "input": switchyard_enum_items} - )["input"][0]["content"][0] - assert nested_image["image_url"] == "data:image/png;base64,cHJpdmF0ZQ==" - assert nested_image["detail"] == "high" - with pytest.raises(ValueError, match=r"non-empty Responses image URL.*str\[4\]"): - module._validate_payload( - { - "model": "gpt-5.6-terra", - "input": [ - { - "type": "message", - "role": "user", - "content": [ - { - "type": "input_image", - "image_url": {"detail": "high"}, - } - ], - } - ], - } - ) - routed = module._validate_payload( - { - "model": "route/codex/gpt-5.6-luna/low", - "input": "use the low route", - "stream": False, - } - ) - assert routed["model"] == "gpt-5.6-luna" - with pytest.raises(ValueError, match="unsupported Codex model"): - module._validate_payload({"model": "unapproved-model", "input": "hello"}) - with pytest.raises(ValueError, match="non-empty Responses input"): - module._validate_payload({"model": "gpt-5.6-terra", "input": ""}) - with pytest.raises(ValueError, match="non-empty Responses input list"): - module._validate_payload({"model": "gpt-5.6-terra", "input": []}) - - completed = { - "id": "resp_test", - "object": "response", - "status": "completed", - "output": [], - } - completed_item = { - "type": "message", - "role": "assistant", - "status": "completed", - "content": [{"type": "output_text", "text": "done"}], - } - assert module._completed_response( - [ - "event: response.created", - 'data: {"type":"response.created","response":{}}', - "event: response.output_item.done", - "data: " - + json.dumps( - { - "type": "response.output_item.done", - "output_index": 0, - "item": completed_item, - } - ), - "event: response.completed", - "data: " - + json.dumps({"type": "response.completed", "response": completed}), - "data: [DONE]", - ] - )["output"] == [completed_item] - raw_stream = ( - "event: response.output_item.done\n" - "data: " - + json.dumps( - { - "type": "response.output_item.done", - "output_index": 0, - "item": completed_item, - } - ) - + "\n\nevent: response.completed\ndata: " - + json.dumps({"type": "response.completed", "response": completed}) - + "\n\n" - ).encode() - normalized = module._normalized_stream( - raw_stream, {**completed, "output": [completed_item]} - ).decode() - terminal_data = next( - line for line in normalized.splitlines() if '"response.completed"' in line - ) - assert json.loads(terminal_data.removeprefix("data: "))["response"][ - "output" - ] == [completed_item] - assert normalized.endswith("\n\n") - streamed_function_item = { - "type": "function_call", - "name": "read_file", - "status": "completed", - "arguments": '{"path":"/tmp"}', - } - streamed_function_body = ( - "event: response.function_call_arguments.delta\n" - 'data: {"type":"response.function_call_arguments.delta",' - '"item_id":"call_1","delta":"{\\"path\\":\\"/tmp\\"}"}\n\n' - "event: response.function_call_arguments.done\n" - 'data: {"type":"response.function_call_arguments.done",' - '"item_id":"call_1","arguments":"{\\"path\\":\\"/tmp\\"}"}\n\n' - "event: response.output_item.done\n" - 'data: {"type":"response.output_item.done","output_index":0,' - '"item":{"type":"function_call","name":"read_file",' - '"arguments":"{\\"path\\":\\"/tmp\\"}"}}\n\n' - "event: response.completed\n" - "data: " - + json.dumps( - { - "type": "response.completed", - "response": {**completed, "output": [streamed_function_item]}, - } - ) - + "\n\n" - ).encode() - normalized_function_stream = module._normalized_stream( - streamed_function_body, {**completed, "output": [streamed_function_item]} - ).decode() - assert "response.function_call_arguments.delta" in normalized_function_stream - assert "response.function_call_arguments.done" not in normalized_function_stream - assert "response.output_item.done" not in normalized_function_stream - normalized_terminal = next( - line - for line in normalized_function_stream.splitlines() - if '"response.completed"' in line - ) - assert json.loads(normalized_terminal.removeprefix("data: "))["response"][ - "output" - ] == [] - with pytest.raises(RuntimeError, match="retryable incomplete response"): - module._completed_response( - [ - "event: response.incomplete", - 'data: {"type":"response.incomplete","response":' - '{"status":"incomplete","incomplete_details":' - '{"reason":"max_output_tokens"}}}', - ] - ) - with pytest.raises(RuntimeError, match="provider unavailable"): - module._completed_response( - [ - "event: error", - 'data: {"type":"error","error":{"message":"provider unavailable"}}', - ] - ) - malformed_tool_item = { - "type": "function_call", - "name": "search_files", - "status": "completed", - "arguments": '{"path":"","offset":', - } - with pytest.raises(RuntimeError, match="malformed function arguments"): - module._completed_response( - [ - "event: response.output_item.done", - "data: " - + json.dumps( - { - "type": "response.output_item.done", - "output_index": 0, - "item": malformed_tool_item, - } - ), - "event: response.completed", - "data: " - + json.dumps({"type": "response.completed", "response": completed}), - ] - ) - valid_tool_item = { - **malformed_tool_item, - "arguments": '{"path":"","offset":0}', - } - assert module._completed_response( - [ - "event: response.output_item.done", - "data: " - + json.dumps( - { - "type": "response.output_item.done", - "output_index": 0, - "item": valid_tool_item, - } - ), - "event: response.completed", - "data: " - + json.dumps({"type": "response.completed", "response": completed}), - ] - )["output"] == [valid_tool_item] - with pytest.raises(RuntimeError, match="malformed function arguments"): - module._completed_response( - [ - "event: response.function_call_arguments.delta", - 'data: {"type":"response.function_call_arguments.delta",' - '"item_id":"call_1","output_index":0,' - '"delta":"{\\"path\\":\\"/tmp\\",\\"offset\\":"}', - "event: response.completed", - "data: " - + json.dumps({"type": "response.completed", "response": completed}), - ] - ) - streamed_tool = module._completed_response( - [ - "event: response.function_call_arguments.delta", - 'data: {"type":"response.function_call_arguments.delta",' - '"item_id":"call_2","output_index":0,' - '"delta":"{\\"path\\":\\"/tmp\\",\\"offset\\":"}', - "event: response.function_call_arguments.done", - 'data: {"type":"response.function_call_arguments.done",' - '"item_id":"call_2","output_index":0,' - '"arguments":"{\\"path\\":\\"/tmp\\",\\"offset\\":0}"}', - "event: response.completed", - "data: " - + json.dumps({"type": "response.completed", "response": completed}), - ] - ) - assert streamed_tool["status"] == "completed" - - auth_dir = tmp_path / ".codex" - auth_dir.mkdir() - # The token payload need only prove the broker reads CODEX_HOME directly. - encoded = base64.urlsafe_b64encode( - json.dumps({"exp": time.time() + 3600}).encode() - ).decode().rstrip("=") - (auth_dir / "auth.json").write_text( - json.dumps({"tokens": {"access_token": f"header.{encoded}.signature"}}) - ) - monkeypatch.setenv("CODEX_HOME", str(auth_dir)) - assert module._access_token().startswith("header.") - - -def test_codex_broker_refreshes_and_persists_first_party_oauth( - tmp_path: Path, monkeypatch -): - """Expired ChatGPT OAuth refreshes in the canonical Codex CLI store.""" - module = _load_broker_module( - "hermes_codex_refresh_broker", "codex_broker.py", monkeypatch - ) - auth_dir = tmp_path / ".codex" - auth_dir.mkdir() - - def jwt(expires_at: float) -> str: - payload = base64.urlsafe_b64encode( - json.dumps({"exp": expires_at}).encode() - ).decode().rstrip("=") - return f"header.{payload}.signature" - - expired = jwt(time.time() - 60) - live = jwt(time.time() + 3600) - auth_path = auth_dir / "auth.json" - auth_path.write_text( - json.dumps( - { - "auth_mode": "chatgpt", - "tokens": { - "access_token": expired, - "refresh_token": "refresh-old", - }, - } - ) - ) - calls = [] - auth_module = ModuleType("hermes_cli.auth") - - def refresh(access_token, refresh_token, *, timeout_seconds): - calls.append((access_token, refresh_token, timeout_seconds)) - return { - "access_token": live, - "refresh_token": "refresh-new", - "last_refresh": "2026-08-12T20:00:00Z", - } - - auth_module.refresh_codex_oauth_pure = refresh - package = ModuleType("hermes_cli") - package.auth = auth_module - monkeypatch.setitem(sys.modules, "hermes_cli", package) - monkeypatch.setitem(sys.modules, "hermes_cli.auth", auth_module) - monkeypatch.setenv("CODEX_HOME", str(auth_dir)) - - assert module._access_token() == live - persisted = json.loads(auth_path.read_text()) - assert persisted["tokens"]["access_token"] == live - assert persisted["tokens"]["refresh_token"] == "refresh-new" - assert persisted["last_refresh"] == "2026-08-12T20:00:00Z" - assert calls == [(expired, "refresh-old", 30.0)] - assert auth_path.stat().st_mode & 0o777 == 0o600 - - # A healthy token is reused, so repeated routed turns do not spend a - # refresh token or create a second billing/authentication path. - assert module._access_token() == live - assert len(calls) == 1 - - -def test_claude_broker_uses_native_subscription_without_api_billing(monkeypatch): - """Claude traffic must use the native first-party CLI subscription lane.""" - module = _load_broker_module( - "hermes_claude_broker", "claude_oauth_broker.py", monkeypatch - ) - monkeypatch.setenv("ANTHROPIC_API_KEY", "must-not-leak") - monkeypatch.setenv("CLAUDE_API_KEY", "must-not-leak") - monkeypatch.setattr( - module, - "resolve_route", - lambda route: "claude-fable-5" if "/fable/" in route else route, - ) - - model, effort = module._route( - "route/claude/fable/xhigh", {"output_config": {"effort": "xhigh"}} - ) - - assert (model, effort) == ("claude-fable-5", "xhigh") - assert "ANTHROPIC_API_KEY" not in module._claude_environment() - assert "CLAUDE_API_KEY" not in module._claude_environment() - assert module.CAPACITY_PATTERN.search("weekly usage limit exhausted") - - -def test_codex_native_health_overrides_historical_router_errors( - tmp_path: Path, monkeypatch -): - """Fresh first-party health is authoritative over old Switchyard probes.""" - plugin_path = HERMES / "plugins" / "auto-router" / "provider_status.py" - spec = importlib.util.spec_from_file_location("hermes_provider_status", plugin_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - health_path = tmp_path / "codex.json" - health_path.write_text( - json.dumps( - { - "state": "available", - "authenticated": True, - "transport": "codex-chatgpt-subscription", - } - ) - ) - monkeypatch.setattr(module, "CODEX_HEALTH_PATH", health_path) - monkeypatch.setattr(module, "CLAUDE_HEALTH_PATH", tmp_path / "missing.json") - monkeypatch.setattr( - module, - "_get_json", - lambda url: {"status": "ok"} - if url.endswith("/health") - else { - "models": { - "route/codex/terra/medium": { - "calls": 1, - "errors": 99, - "total_tokens": 12, - } - } - }, - ) - monkeypatch.setattr(module, "_codex_account", lambda: {}) - monkeypatch.setattr(module, "_claude_account", lambda: {}) - - codex = module.provider_status_payload()["providers"]["codex"] - - assert codex["errors"] == 99 - assert codex["state"] == "available" - assert codex["native_health"]["transport"] == "codex-chatgpt-subscription" - - -def test_api_session_patch_accepts_parent_lineage(tmp_path: Path): - """API-created workers must persist the originating Hermes session.""" - module_path = HERMES / "scripts" / "patch_api_server_sessions.py" - spec = importlib.util.spec_from_file_location("patch_api_sessions", module_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - source = tmp_path / "api_server.py" - destination = tmp_path / "patched.py" - source.write_text( - "prefix\n" - + module.BEFORE - + "middle\n" - + module.RUNS_BEFORE - + "run body\n" - + module.RUN_CLOSE_BEFORE - + module.RESPONSES_SESSION_BEFORE - + module.EVENT_CALLBACK_SIGNATURE_BEFORE - + "callback docstring and push helper\n" - + module.EVENT_CALLBACK_BODY_BEFORE - + "tool start body\n" - + module.EVENT_CALLBACK_END_BEFORE - + module.EVENT_CALLBACK_CALL_BEFORE - + module.RUN_SWEEP_BEFORE - + "suffix\n", - encoding="utf-8", - ) - - module.patch(source, destination) - patched = destination.read_text(encoding="utf-8") - - assert "X-Hermes-Parent-Session-Id" in patched - assert "parent_session_id=parent_session_id" in patched - assert "Parent session not found" in patched - assert "HERMES_API_DEFAULT_PARENT_MATCH_PREFIXES" in patched - assert "user_message.startswith(default_prefixes)" in patched - assert "session_parent_conflict" in patched - assert "X-Hermes-Conversation-Platform" in patched - assert "X-Hermes-Conversation-Title" in patched - assert 'conversation_platform != "telegram"' in patched - assert "db.record_gateway_session_peer(" in patched - assert 'display_name="Telegram"' in patched - assert "db.reopen_session(session_id)" in patched - assert 'db.end_session(session_id, f"api_run_{terminal_status}")' in patched - assert "def _record_run_activity(" in patched - assert '"_thinking": "Hermes is reasoning"' in patched - assert '"run.started": "Worker started"' in patched - assert '"run.completed": "Worker completed"' in patched - assert '"reasoning.available": "Hermes finished a reasoning step"' in patched - assert '"subagent.progress": "Nested worker progress"' in patched - assert "redact_sensitive_text" in patched - assert 'getattr(os, "O_NOFOLLOW", 0)' in patched - assert "os.fchmod(fd, 0o600)" in patched - assert "session_id=session_id" in patched - assert 'self._record_run_activity(session_id, "run.started")' in patched - assert 'detail = tool_name if event_type in {' in patched - assert 'if event_type == "subagent.tool"' in patched - assert "_RUN_ACTIVITY_HEARTBEAT_SECONDS = 15.0" in patched - assert 'heartbeats.get(session_id, 0.0)' in patched - assert '"subagent.thinking",' in patched - assert "Stream retention and run lifetime are separate" in patched - assert 'terminal_status in {"completed", "failed", "cancelled"}' in patched - assert patched.index("terminal_status = self._run_statuses") < patched.index( - "self._active_run_tasks.pop(run_id, None)" - ) - - -def test_telegram_api_session_migration_is_bounded_and_idempotent(tmp_path: Path): - """Only named Telegram conversations receive presentation metadata.""" - module_path = HERMES / "scripts" / "migrate_telegram_api_sessions.py" - spec = importlib.util.spec_from_file_location("migrate_telegram_sessions", module_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - state = tmp_path / "state.db" - responses = tmp_path / "response_store.db" - with sqlite3.connect(state) as connection: - connection.execute( - """CREATE TABLE sessions ( - id TEXT PRIMARY KEY, source TEXT, session_key TEXT, chat_type TEXT, - display_name TEXT, origin_json TEXT, title TEXT - )""" - ) - connection.executemany( - "INSERT INTO sessions (id, source) VALUES (?, ?)", - (("telegram-session", "api_server"), ("other-session", "api_server")), - ) - with sqlite3.connect(responses) as connection: - connection.execute( - "CREATE TABLE conversations (name TEXT PRIMARY KEY, response_id TEXT NOT NULL)" - ) - connection.execute( - "CREATE TABLE responses (response_id TEXT PRIMARY KEY, data TEXT NOT NULL, accessed_at REAL NOT NULL)" - ) - connection.executemany( - "INSERT INTO responses VALUES (?, ?, 0)", - ( - ("telegram-response", json.dumps({"session_id": "telegram-session"})), - ("other-response", json.dumps({"session_id": "other-session"})), - ), - ) - connection.executemany( - "INSERT INTO conversations VALUES (?, ?)", - (("telegram", "telegram-response"), ("unrelated", "other-response")), - ) - - assert module.migrate(state, responses) == 1 - assert module.migrate(state, responses) == 0 - with sqlite3.connect(state) as connection: - telegram = connection.execute( - "SELECT session_key, chat_type, display_name, origin_json, title " - "FROM sessions WHERE id = 'telegram-session'" - ).fetchone() - other = connection.execute( - "SELECT session_key, title FROM sessions WHERE id = 'other-session'" - ).fetchone() - assert telegram[:3] == ("telegram", "private", "Telegram") - assert json.loads(telegram[3]) == { - "platform": "telegram", - "session_key": "telegram", - } - assert telegram[4] == "Telegram · General" - assert other == (None, None) - - -def test_web_session_activity_patch_projects_bounded_events(tmp_path: Path): - """The DOM transcript includes events without polluting agent history.""" - module_path = HERMES / "scripts" / "patch_web_session_activity.py" - spec = importlib.util.spec_from_file_location("patch_web_activity", module_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - source = tmp_path / "web_server.py" - destination = tmp_path / "patched.py" - source.write_text( - "prefix\n" - + module.LATEST_ROWS_BEFORE - + module.LATEST_SELECTION_BEFORE - + module.HELPER_MARKER - + " db = object()\n" - + module.MESSAGES_BEFORE - + "suffix\n", - encoding="utf-8", - ) - - module.patch(source, destination) - patched = destination.read_text(encoding="utf-8") - - assert "def _run_activity_messages(" in patched - assert 'root = get_hermes_home() / "run-activity"' in patched - assert "path.stat().st_size > 600_000" in patched - assert "entries[-1_000:]" in patched - assert "*_run_activity_messages(sid)" in patched - assert "messages.sort(" in patched - assert "limit: Optional[int] = None" in patched - assert "total_messages = len(messages)" in patched - assert "min(int(limit), 10_000)" in patched - assert '"total_messages": total_messages' in patched - assert "SELECT id, parent_session_id, started_at, ended_at" in patched - assert "newest still-open member of the lineage" in patched - assert "immediate objective parent" in patched - assert "mixes unrelated workstreams" in patched - assert "orphaned children" in patched - assert 'item[0].get("ended_at") is None' in patched - - -def test_web_session_lineage_returns_to_resumed_parent(): - """An ended reviewer must not strand the live view away from its parent.""" - module_path = HERMES / "scripts" / "patch_web_session_activity.py" - spec = importlib.util.spec_from_file_location("patch_web_lineage", module_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - - namespace: dict[str, object] = {} - exec( - "def select_active(sid, db, rows):\n" + module.LATEST_SELECTION_AFTER, - namespace, - ) - - class FakeDB: - def get_session(self, session_id): - return next((row for row in rows if row["id"] == session_id), None) - - rows = [ - { - "id": "umbrella", - "parent_session_id": None, - "started_at": 0.0, - "ended_at": None, - }, - { - "id": "root", - "parent_session_id": "umbrella", - "started_at": 1.0, - "ended_at": None, - }, - { - "id": "review-1", - "parent_session_id": "root", - "started_at": 2.0, - "ended_at": 3.0, - }, - { - "id": "review-2", - "parent_session_id": "root", - "started_at": 4.0, - "ended_at": None, - }, - ] - select_active = namespace["select_active"] - - assert select_active("root", FakeDB(), rows) == ( - "review-2", - ["root", "review-2"], - ) - assert select_active("review-1", FakeDB(), rows) == ( - "review-2", - ["root", "review-2"], - ) - rows[2]["ended_at"] = 5.0 - rows[3]["ended_at"] = 5.0 - assert select_active("root", FakeDB(), rows) == ("root", ["root"]) - - rows[1]["ended_at"] = 9.0 - rows.append( - { - "id": "orphaned-review", - "parent_session_id": "root", - "started_at": 6.0, - "ended_at": None, - } - ) - rows.append( - { - "id": "accepted-review", - "parent_session_id": "root", - "started_at": 7.0, - "ended_at": 8.0, - } - ) - assert select_active("review-1", FakeDB(), rows) == ( - "accepted-review", - ["root", "accepted-review"], - ) - - -def test_api_activity_patch_coalesces_streaming_heartbeats( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -): - """Token callbacks stay bounded while every tool transition is retained.""" - module_path = HERMES / "scripts" / "patch_api_server_sessions.py" - spec = importlib.util.spec_from_file_location("patch_api_activity", module_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - activity_body = module.EVENT_CALLBACK_SIGNATURE_AFTER.split( - " def _make_run_event_callback(", 1 - )[0] - namespace: dict[str, object] = {} - exec( - "import hashlib, json, logging, os, time\n" - "from pathlib import Path\n" - "logger = logging.getLogger(__name__)\n" - "def redact_sensitive_text(value): return value\n" - "class ActivityRecorder:\n" - + activity_body, - namespace, - ) - recorder = namespace["ActivityRecorder"]() - monkeypatch.setenv("HERMES_HOME", str(tmp_path)) - - recorder._record_run_activity("session", "_thinking") - recorder._record_run_activity("session", "reasoning.available") - recorder._record_run_activity("session", "subagent.thinking") - recorder._record_run_activity("session", "tool.started", tool_name="terminal") - recorder._record_run_activity("session", "tool.completed", tool_name="terminal") - - journals = list((tmp_path / "run-activity").glob("*.jsonl")) - assert len(journals) == 1 - entries = [json.loads(line) for line in journals[0].read_text().splitlines()] - assert [entry["activity_event"] for entry in entries] == [ - "_thinking", - "tool.started", - "tool.completed", - ] - - -def test_legacy_api_sessions_are_nested_idempotently(tmp_path: Path): - """Known standalone API workers move under Cassandra without data loss.""" - module_path = HERMES / "scripts" / "migrate_api_session_lineage.py" - spec = importlib.util.spec_from_file_location("migrate_api_sessions", module_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - database = tmp_path / "state.db" - with sqlite3.connect(database) as connection: - connection.execute( - "CREATE TABLE sessions (id TEXT PRIMARY KEY, source TEXT, " - "parent_session_id TEXT, title TEXT, transcript TEXT, archived INTEGER DEFAULT 0)" - ) - connection.execute( - "INSERT INTO sessions (id, source, parent_session_id, title, transcript) " - "VALUES (?, 'tui', NULL, 'Cassandra', 'parent-data')", - (module.LEGACY_CASSANDRA_PARENT,), - ) - worker_id = next(iter(module.LEGACY_CASSANDRA_WORKERS)) - connection.execute( - "INSERT INTO sessions (id, source, parent_session_id, title, transcript) " - "VALUES (?, 'api_server', NULL, 'old', 'keep-me')", - (worker_id,), - ) - connection.executemany( - "INSERT INTO sessions (id, source, parent_session_id, title, transcript) " - "VALUES (?, 'api_server', NULL, 'old smoke', 'keep-smoke')", - ((orphan_id,) for orphan_id in module.LEGACY_ORPHANED_SMOKE_SESSIONS), - ) - - assert module.migrate(database) == 1 + len(module.LEGACY_ORPHANED_SMOKE_SESSIONS) - assert module.migrate(database) == 0 - with sqlite3.connect(database) as connection: - row = connection.execute( - "SELECT parent_session_id, title, transcript FROM sessions WHERE id = ?", - (worker_id,), - ).fetchone() - assert row == ( - module.LEGACY_CASSANDRA_PARENT, - module.LEGACY_CASSANDRA_WORKERS[worker_id], - "keep-me", - ) - with sqlite3.connect(database) as connection: - orphans = connection.execute( - "SELECT id, archived, title, transcript FROM sessions " - "WHERE id IN ({}) ORDER BY id".format( - ",".join("?" for _ in module.LEGACY_ORPHANED_SMOKE_SESSIONS) - ), - tuple(module.LEGACY_ORPHANED_SMOKE_SESSIONS), - ).fetchall() - assert orphans == sorted( - ( - orphan_id, - 1, - module.LEGACY_ORPHANED_SMOKE_SESSIONS[orphan_id], - "keep-smoke", - ) - for orphan_id in module.LEGACY_ORPHANED_SMOKE_SESSIONS - ) - - -def test_automated_triage_sessions_are_grouped_without_touching_interactive_runs(tmp_path: Path): - """Only the stable Ariadne contract moves below the triage parent.""" - module_path = HERMES / "scripts" / "migrate_api_session_lineage.py" - spec = importlib.util.spec_from_file_location("migrate_triage_sessions", module_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - database = tmp_path / "state.db" - with sqlite3.connect(database) as connection: - connection.execute( - "CREATE TABLE sessions (id TEXT PRIMARY KEY, source TEXT, " - "parent_session_id TEXT, title TEXT, started_at REAL, archived INTEGER DEFAULT 0)" - ) - connection.execute( - "CREATE TABLE messages (session_id TEXT, role TEXT, content TEXT)" - ) - connection.executemany( - "INSERT INTO sessions (id, source, started_at) VALUES (?, 'api_server', ?)", - (("triage-run", 1.0), ("jenkins-run", 1.5), ("interactive-run", 2.0)), - ) - connection.executemany( - "INSERT INTO messages (session_id, role, content) VALUES (?, 'user', ?)", - ( - ( - "triage-run", - module.TRIAGE_MESSAGE_PREFIXES[0] - + " Fix for incident sonar/bstein_home/python:S2208/finding-key.", - ), - ( - "jenkins-run", - module.TRIAGE_MESSAGE_PREFIXES[1] - + "\nAnalyze incident soteria/291 for the Jenkins job soteria.", - ), - ("interactive-run", "Please explain this alert to me."), - ), - ) - - assert module.migrate(database, group_triage=True) == 2 - assert module.migrate(database, group_triage=True) == 0 - with sqlite3.connect(database) as connection: - parent = connection.execute( - "SELECT title FROM sessions WHERE id = ?", (module.TRIAGE_PARENT,) - ).fetchone() - triage = connection.execute( - "SELECT parent_session_id, title FROM sessions WHERE id = 'triage-run'" - ).fetchone() - interactive = connection.execute( - "SELECT parent_session_id FROM sessions WHERE id = 'interactive-run'" - ).fetchone() - jenkins = connection.execute( - "SELECT parent_session_id, title FROM sessions WHERE id = 'jenkins-run'" - ).fetchone() - assert parent == (module.TRIAGE_PARENT_TITLE,) - assert triage == ( - module.TRIAGE_PARENT, - "Sonar · bstein_home · python:S2208 · iage-run", - ) - assert interactive == (None,) - assert jenkins == ( - module.TRIAGE_PARENT, - "Sonar · soteria/291 · kins-run", - ) - - -def test_stale_parent_linked_api_workers_close_on_startup(tmp_path: Path): - """A previous gateway lifetime cannot leave phantom active workers.""" - module_path = HERMES / "scripts" / "migrate_api_session_lineage.py" - spec = importlib.util.spec_from_file_location("close_stale_api_workers", module_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - database = tmp_path / "state.db" - with sqlite3.connect(database) as connection: - connection.execute( - "CREATE TABLE sessions (id TEXT PRIMARY KEY, source TEXT, " - "parent_session_id TEXT, title TEXT, started_at REAL, ended_at REAL, " - "end_reason TEXT, archived INTEGER DEFAULT 0)" - ) - connection.executemany( - "INSERT INTO sessions " - "(id, source, parent_session_id, started_at, ended_at, end_reason) " - "VALUES (?, ?, ?, 1, ?, ?)", - ( - ("stale", "api_server", "parent", None, None), - ("root", "api_server", None, None, None), - ("finished", "api_server", "parent", 2.0, "api_run_completed"), - ("interactive", "tui", "parent", None, None), - ), - ) - - assert module.migrate(database) == 1 - assert module.migrate(database) == 0 - with sqlite3.connect(database) as connection: - rows = connection.execute( - "SELECT id, ended_at, end_reason FROM sessions ORDER BY id" - ).fetchall() - by_id = {row[0]: row[1:] for row in rows} - assert by_id["stale"][0] is not None - assert by_id["stale"][1] == "api_run_recovered_stale" - assert by_id["root"] == (None, None) - assert by_id["finished"] == (2.0, "api_run_completed") - assert by_id["interactive"] == (None, None) - - -def test_switchyard_brokers_and_native_claude_lane_use_the_right_images(): - """Thin brokers stay small while native Claude runs beside owner auth.""" - dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-switchyard-brokers").read_text() - assert "httpx==0.28.1" in dockerfile - assert "worker_route_broker.py" in dockerfile - assert "routing_catalog.py" in dockerfile - - deployment = _documents(HERMES / "switchyard-deployment.yaml")[0] - containers = { - container["name"]: container - for container in deployment["spec"]["template"]["spec"]["containers"] - } - expected = ( - "registry.bstein.dev/bstein/hermes-switchyard-brokers@" - "sha256:ee7e95e060ef8083da505162d7e9030daba15fdd828cc047bbcbe6aa409d2083" - ) - assert containers["worker-route-broker"]["image"] == expected - assert containers["classifier-broker"]["image"] == expected - assert "claude-oauth-broker" not in containers - - agent = _documents(HERMES / "agent-deployment.yaml")[0] - agent_containers = { - container["name"]: container - for container in agent["spec"]["template"]["spec"]["containers"] - } - for container_name in ("hermes", "terminal"): - container = agent_containers[container_name] - environment = {item["name"]: item["value"] for item in container["env"]} - mounts = {item["name"]: item for item in container["volumeMounts"]} - assert environment["HERMES_ROUTING_CATALOG_PATH"] == "/routing-catalog/catalog.json" - assert environment["HERMES_CODEX_HEALTH_PATH"] == "/opt/data/provider-health/codex.json" - assert environment["HERMES_CLAUDE_HEALTH_PATH"] == "/opt/data/provider-health/claude.json" - assert mounts["routing-catalog"]["mountPath"] == "/routing-catalog" - assert mounts["routing-catalog"]["readOnly"] is True - codex = agent_containers["codex-broker"] - codex_environment = {item["name"]: item["value"] for item in codex["env"]} - assert codex_environment["HERMES_CODEX_HEALTH_PATH"] == "/opt/data/provider-health/codex.json" - claude = agent_containers["claude-broker"] - assert claude["image"].startswith("registry.bstein.dev/bstein/hermes-agent@") - assert "unset ANTHROPIC_API_KEY CLAUDE_API_KEY" in claude["args"][0] - assert any( - mount["name"] == "home" and mount["mountPath"] == "/opt/data" - for mount in claude["volumeMounts"] - ) - - -def test_classifier_broker_bounds_history_without_losing_routing_intent(monkeypatch): - """AUTO classification must fit the Jetson context without losing intent.""" - broker_path = HERMES / "scripts" / "classifier_broker.py" - monkeypatch.setitem(sys.modules, "httpx", SimpleNamespace()) - spec = importlib.util.spec_from_file_location("hermes_classifier_broker", broker_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - - payload = { - "model": "qwen2.5:14b-instruct-q4_0", - "messages": [ - { - "role": "system", - "content": "routing contract\n" + ("candidate policy " * 1000), - }, - { - "role": "user", - "content": "Build and verify the Cassandra release safely.", - }, - { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call_large", - "type": "function", - "function": { - "name": "large_tool", - "arguments": '{"command":"' + ("x" * 5000) + '"}', - }, - } - ], - }, - { - "role": "tool", - "content": "unbounded test output " * 10000, - "tool_call_id": "call_large", - }, - { - "role": "user", - "content": [ - {"type": "image_url", "image_url": {"url": "data:image/png;base64,private"}}, - {"type": "text", "text": "Turn this cat into a cute clown."}, - ], - }, - ], - "tools": [{"type": "function", "function": {"name": "large_tool"}}], - "tool_choice": "auto", - "parallel_tool_calls": True, - "response_format": {"type": "json_object"}, - } - - compacted = module.compact_payload(payload) - encoded = json.dumps(compacted) - - assert compacted["model"] == payload["model"] - assert compacted["response_format"] == payload["response_format"] - assert "tools" not in compacted - assert "tool_choice" not in compacted - assert "parallel_tool_calls" not in compacted - assert all(message.get("role") != "tool" for message in compacted["messages"]) - assert all("tool_call_id" not in message for message in compacted["messages"]) - assert all("tool_calls" not in message for message in compacted["messages"]) - assert "[tool evidence]" in encoded - assert "[assistant requested an external tool]" in encoded - assert "Build and verify the Cassandra release safely." in encoded - assert "Turn this cat into a cute clown." in encoded - assert "image attachment available to the selected worker" in encoded - assert "data:image/png;base64" not in encoded - assert "unbounded test output " * 100 not in encoded - assert len(encoded) < 18_000 - - -def test_switchyard_classifier_is_bounded_and_fails_open_once(): - """A sick local judge must not hold chat through repeated long retries.""" - config = tomllib.loads( - _documents(HERMES / "switchyard-configmap.yaml")[0]["data"]["routes.toml"] - ) - classifier = config["llm_clients"]["classifier"] - assert classifier["base_url"] == "http://127.0.0.1:9008/v1" - assert classifier["max_retries"] == 0 - for route in ("auto_fast", "auto_balanced"): - assert config["routes"][route]["recent_turn_window"] == 4 - for route in ("auto_deep", "auto_maximum", "worker_auto_maximum"): - assert config["routes"][route]["recent_turn_window"] == 6 - - deployment = _documents(HERMES / "switchyard-deployment.yaml")[0] - containers = { - item["name"]: item - for item in deployment["spec"]["template"]["spec"]["containers"] - } - classifier_container = containers["classifier-broker"] - env = {item["name"]: item["value"] for item in classifier_container["env"]} - assert env["HERMES_CLASSIFIER_BROKER_READ_TIMEOUT"] == "60" - assert classifier_container["readinessProbe"]["httpGet"]["port"] == "classifier" - - -def test_image_broker_returns_bytes_and_removes_owner_cache(tmp_path: Path, monkeypatch): - """The broker must not retain a family user's generated image.""" - broker_path = HERMES / "scripts" / "image_broker.py" - spec = importlib.util.spec_from_file_location("hermes_image_broker", broker_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - generated = tmp_path / "generated.png" - generated.write_bytes(b"\x89PNG\r\n\x1a\nprivate-image") - - class Provider: - def generate(self, prompt, aspect, **kwargs): - assert prompt == "paint a blue sphere" - assert aspect == "square" - return { - "success": True, - "image": str(generated), - "model": "gpt-image-2-high", - "quality": "high", - } - - monkeypatch.setattr(module, "_PROVIDER", Provider()) - result = module._generate( - { - "prompt": "paint a blue sphere", - "aspect_ratio": "square", - "model": "gpt-image-2-high", - } - ) - - assert result["success"] is True - assert result["image_b64"] - assert "image" not in result - assert not generated.exists() - - -def test_image_broker_auto_falls_back_to_local_and_honors_explicit_routes( - monkeypatch, -): - """AUTO is hosted-first while explicit local never calls the hosted lane.""" - broker_path = HERMES / "scripts" / "image_broker.py" - spec = importlib.util.spec_from_file_location("hermes_image_router", broker_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - calls = [] - - def hosted(payload, model, prompt, aspect): - calls.append(("hosted", model, prompt, aspect)) - return {"success": False, "error": "hosted refusal"} - - def local(payload, timeout=1800.0): - calls.append(("local", payload["model"], timeout)) - return { - "success": True, - "image_b64": "aW1hZ2U=", - "model": "flux-2-klein-4b-local", - "route": "local", - } - - monkeypatch.setattr(module, "_generate_hosted", hosted) - monkeypatch.setattr(module, "_local_request", local) - auto = module._generate( - { - "prompt": "colorize this family photograph", - "aspect_ratio": "portrait", - "model": "atlas-image-auto-high", - } - ) - assert auto["success"] is True - assert auto["route"] == "local" - assert auto["hosted_fallback_reason"] == "hosted refusal" - assert [call[0] for call in calls] == ["hosted", "local"] - - calls.clear() - explicit = module._generate( - { - "prompt": "make a local landscape", - "aspect_ratio": "landscape", - "model": "flux-2-klein-4b-local", - } - ) - assert explicit["route"] == "local" - assert [call[0] for call in calls] == ["local"] - - -def test_image_broker_policy_is_narrow_and_operator_extensible(tmp_path: Path, monkeypatch): - """Family-photo restoration stays allowed while the hard boundary remains.""" - broker_path = HERMES / "scripts" / "image_broker.py" - spec = importlib.util.spec_from_file_location("hermes_image_policy", broker_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - policy = tmp_path / "policy.json" - policy.write_text('{"additional_blocked_phrases":["site-specific block"]}') - monkeypatch.setattr(module, "POLICY_PATH", policy) - - assert module._policy_error( - "Colorize my baby photograph with a lighter natural skin tone" - ) is None - assert "minors" in module._policy_error("Create a sexual image of a child") - assert module._policy_error("A site-specific block request") == ( - "request is blocked by the operator image policy" - ) - - -def test_local_flux_runtime_and_gpu_handoff_are_flux_managed(): - """FLUX and Wolf share titan-24 while text stays on titan-20.""" - deployment = _documents(HERMES / "local-image-deployment.yaml")[0] - assert deployment["metadata"]["name"] == "hermes-local-image" - pod = deployment["spec"]["template"]["spec"] - assert pod["serviceAccountName"] == "hermes-gpu-runtime" - local = next(item for item in pod["containers"] if item["name"] == "local-image") - assert len(pod["containers"]) == 1 - assert local["resources"]["requests"]["nvidia.com/gpu.shared"] == 1 - assert local["ports"] == [{"name": "local-image", "containerPort": 9004}] - assert any(mount["mountPath"] == "/models" for mount in local["volumeMounts"]) - model_env = {item["name"]: item["value"] for item in local["env"]} - assert model_env["HERMES_LOCAL_IMAGE_LISTEN_PORT"] == "9004" - assert model_env["HERMES_LOCAL_IMAGE_REVISION"] == ( - "e7b7dc27f91deacad38e78976d1f2b499d76a294" - ) - assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVITY_NODE"] == "titan-24" - assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVE_SM_PERCENT"] == "80" - assert model_env["HERMES_LOCAL_IMAGE_GPU_MAX_EXTERNAL_MEMORY_BYTES"] == ( - "3221225472" - ) - assert model_env["HERMES_LOCAL_IMAGE_OFFLOAD_MODE"] == "sequential" - assert "nvidia-process-exporter-local.monitoring.svc.cluster.local" in model_env[ - "HERMES_LOCAL_IMAGE_GPU_ACTIVITY_URL" - ] - models_volume = next(item for item in pod["volumes"] if item["name"] == "models") - assert models_volume["persistentVolumeClaim"]["claimName"] == ( - "hermes-image-models" - ) - - services = _documents(HERMES / "service.yaml") - image_service = next( - item for item in services if item["metadata"]["name"] == "hermes-local-image" - ) - assert image_service["spec"]["selector"] == {"app": "hermes-local-image"} - - handoff_services = _documents(HERMES / "model-gate-deployment.yaml") - handoff = next( - item - for item in handoff_services - if item["kind"] == "Service" - and item["metadata"]["name"] == "hermes-gpu-handoff" - ) - assert handoff["spec"]["ports"][0]["targetPort"] == "handoff" - - ariadne = _documents( - Path(__file__).parents[2] - / "services/maintenance/apps/ariadne-deployment.yaml" - )[0] - env = { - item["name"]: item["value"] - for item in ariadne["spec"]["template"]["spec"]["containers"][0]["env"] - if "value" in item - } - assert env["GAME_MODE_OLLAMA_URL"] == ( - "http://hermes-gpu-handoff.hermes.svc.cluster.local:11434" - ) - assert env["GAME_MODE_OLLAMA_MODEL"] == "flux-2-klein-4b-local" - - for config_name in ("configmap.yaml", "agent-configmap.yaml", "chat-configmap.yaml"): - config = _documents(HERMES / config_name)[0]["data"]["config.yaml"] - assert "gpt-oss:20b" not in config - assert "atlas-switchyard" in config - switchyard = _documents(HERMES / "switchyard-configmap.yaml")[0]["data"][ - "routes.toml" - ] - assert 'id = "qwen2.5:14b-instruct-q4_0"' in switchyard - assert "qwen2.5:3b-instruct-q4_0" not in switchyard - assert "route/local/qwen2.5-14b/medium" in switchyard - assert "Anthropic and Claude name the same provider" in switchyard - assert "OpenAI and Codex name the same provider" in switchyard - assert "Choose across every configured Codex and Claude family" in switchyard - assert "Claude Fable" in switchyard - assert switchyard.count('Treat "think hard"') == 4 - assert switchyard.count("Never choose below the") >= 5 - switchyard_config = tomllib.loads(switchyard) - routes = switchyard_config["routes"] - configured_targets = switchyard_config["targets"] - for route_name in ("auto_fast", "auto_balanced", "auto_deep", "auto_maximum"): - leading_targets = set(routes[route_name]["targets"][:2]) - assert leading_targets == {"codex_sol_xhigh", "claude_opus_xhigh"} - assert "max_output_tokens" not in routes[route_name] - for route_name in ("auto_fast", "auto_balanced", "auto_deep", "auto_maximum"): - targets = routes[route_name]["targets"] - selector_targets = routes[route_name]["response_schema"] - assert not any(target.startswith("local_") for target in targets) - assert any("fable" in target for target in targets) - assert "local_qwen" not in selector_targets - assert "not eligible for foreground" in routes[route_name]["prompt"] - for route_name in ("auto_deep", "auto_maximum"): - targets = routes[route_name]["targets"] - selector_targets = routes[route_name]["response_schema"] - assert not any(target.endswith("_low") for target in targets) - assert "_low" not in selector_targets - maximum_targets = routes["auto_maximum"]["targets"] - maximum_selector_targets = routes["auto_maximum"]["response_schema"] - assert not any(target.endswith("_medium") for target in maximum_targets) - assert "_medium" not in maximum_selector_targets - assert "absolute high effort floor" in routes["auto_maximum"]["prompt"] - assert "quality mark was missed" in routes["auto_balanced"]["prompt"] - assert "raises the next boundary to xhigh" in routes["auto_maximum"]["prompt"] - assert "Repeated quality misses require xhigh" in routes[ - "worker_auto_maximum" - ]["prompt"] - assert any( - target.startswith("local_") - for target in routes["manual_local_qwen"]["targets"] - ) - for route_name in ( - "manual_codex_luna", - "manual_codex_terra", - "manual_codex_sol", - "manual_claude_haiku", - "manual_claude_fable", - "manual_claude_sonnet", - "manual_claude_opus", - ): - assert not any( - target.startswith("local_") for target in routes[route_name]["targets"] - ) - for provider, families in { - "codex": ("luna", "terra", "sol"), - "claude": ("haiku", "fable", "sonnet", "opus"), - }.items(): - for family in families: - for effort in ("low", "medium", "high", "xhigh"): - route = routes[f"manual_{provider}_{family}_{effort}"] - assert route["id"] == f"atlas/manual/{provider}/{family}/{effort}" - assert route["targets"][0] == f"{provider}_{family}_{effort}" - worker_target = f"worker_{provider}_{family}_{effort}" - assert worker_target in routes["worker_auto_maximum"]["targets"] - assert configured_targets[worker_target]["id"] == ( - f"worker/{provider}/{family}/{effort}" - ) - for route_name in ("auto_fast", "auto_balanced"): - prompt = routes[route_name]["prompt"] - assert "image tool—not the conversational model" in prompt - assert "Do not select a" in prompt - assert "local Qwen or Claude target" in prompt - assert "max_output_tokens" not in routes["worker_auto_maximum"] - model_gate = _documents(HERMES / "model-gate-configmap.yaml")[0]["data"][ - "model_gate.py" - ] - assert "qwen2.5:14b-instruct-q4_0" in model_gate - - -def test_titan20_serializes_classifier_and_local_chat_model_residency(): - """Classifier and local chat share one serialized resident Qwen weight.""" - deployment = _documents( - Path(__file__).parents[2] / "services/ai-llm/deployment.yaml" - )[0] - pod = deployment["spec"]["template"]["spec"] - required = pod["affinity"]["nodeAffinity"][ - "requiredDuringSchedulingIgnoredDuringExecution" - ]["nodeSelectorTerms"][0]["matchExpressions"][0] - assert required["values"] == ["titan-20"] - container = pod["containers"][0] - env = {item["name"]: item["value"] for item in container["env"]} - assert env["OLLAMA_MAX_LOADED_MODELS"] == "1" - assert env["OLLAMA_NUM_PARALLEL"] == "1" - assert env["OLLAMA_KEEP_ALIVE"] == "-1" - assert env["OLLAMA_CONTEXT_LENGTH"] == "8192" - warm_command = " ".join(container["command"]) - assert "--keepalive=-1" not in warm_command - models = next(item for item in pod["volumes"] if item["name"] == "models") - assert models["persistentVolumeClaim"]["claimName"] == ( - "ollama-models-titan20" - ) - - -def test_local_image_gpu_guard_distinguishes_background_and_saturated_gpu(monkeypatch): - """Lease-idle desktop spikes may coexist, but saturation still blocks FLUX.""" - source = ROOT / "dockerfiles" / "hermes-local-image-server.py" - spec = importlib.util.spec_from_file_location("hermes_local_image_server", source) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - monkeypatch.setattr(module, "GPU_ACTIVITY_NODE", "titan-24") - monkeypatch.setattr(module, "GPU_ACTIVE_SM_PERCENT", 80.0) - monkeypatch.setattr(module, "GPU_MAX_EXTERNAL_MEMORY_BYTES", 3 << 30) - - idle = module._parse_gpu_activity( - '\n'.join( - [ - 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="Xorg"} 0', - 'nvidia_process_gpu_memory_used_bytes{node="titan-24",namespace="host",process="Xorg"} 1900000000', - 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="game-stream",process="wolf"} 3', - 'nvidia_process_gpu_memory_used_bytes{node="titan-24",namespace="hermes",process="python"} 9000000000', - ] - ) - ) - assert idle["interactive_active"] is False - assert idle["external_gpu_memory_bytes"] == 1900000000 - assert idle["external_gpu_sm_percent"] == 3 - - background_spike = module._parse_gpu_activity( - 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="sway"} 41\n' - ) - assert background_spike["interactive_active"] is False - - active = module._parse_gpu_activity( - 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="steam"} 91\n' - ) - assert active["interactive_active"] is True - assert "91%" in active["gpu_guard_reason"] - - -def test_local_flux_renderer_uses_a_disposable_cuda_worker(monkeypatch): - """A completed render must not leave its CUDA context in the API process.""" - source = ROOT / "dockerfiles" / "hermes-local-image-server.py" - spec = importlib.util.spec_from_file_location("hermes_local_image_worker", source) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - calls = [] - - def run(command, **kwargs): - calls.append((command, kwargs)) - return SimpleNamespace( - returncode=0, - stdout=b'{"success":true,"route":"local","image_b64":"cG5n"}', - stderr=b"", - ) - - monkeypatch.setattr(module.subprocess, "run", run) - result = module._render({"prompt": "black cat", "aspect_ratio": "square"}) - - assert result["route"] == "local" - command, options = calls[0] - assert command[-1] == "--render-worker" - assert json.loads(options["input"]) == { - "prompt": "black cat", - "aspect_ratio": "square", - } - assert options["timeout"] == module.RENDER_TIMEOUT_SECONDS - assert options["check"] is False - - -def test_local_flux_uses_low_vram_offload_without_reducing_resolution(): - """The shared 3080 lane must trade time, not image size, for headroom.""" - source = (ROOT / "dockerfiles" / "hermes-local-image-server.py").read_text() - assert 'OFFLOAD_MODE = os.environ.get(' in source - assert '"HERMES_LOCAL_IMAGE_OFFLOAD_MODE", "sequential"' in source - assert "pipe.enable_sequential_cpu_offload()" in source - assert '"square": (1024, 1024)' in source - - -def test_chat_auth_and_relay_are_pod_lifetime_only(): - statefulset = _documents(HERMES / "chat-statefulset.yaml")[0] - pod = statefulset["spec"]["template"]["spec"] - containers = statefulset["spec"]["template"]["spec"]["containers"] - - for name in ("hermes", "webui"): - container = next(item for item in containers if item["name"] == name) - env = {item["name"]: item["value"] for item in container["env"]} - assert env["HERMES_AUTH_FILE"] == "/runtime-access/hermes-auth.json" - mount = next( - item for item in container["volumeMounts"] if item["name"] == "runtime-access" - ) - assert mount["mountPath"] == "/runtime-access" - assert "subPath" not in mount - - hermes_env = { - item["name"]: item["value"] - for item in next(item for item in containers if item["name"] == "hermes")["env"] - } - runtime = next(item for item in pod["volumes"] if item["name"] == "runtime-access") - assert runtime["emptyDir"] == {"medium": "Memory", "sizeLimit": "2Mi"} - assert not any(item["name"] == "provider-auth" for item in pod["volumes"]) - init_command = next( - item for item in pod["initContainers"] if item["name"] == "init-config" - )["command"][2] - for key in ( - "ANTHROPIC_API_KEY", - "API_SERVER_KEY", - "CLAUDE_CODE_OAUTH_TOKEN", - "GITEA_TOKEN", - "HERMES_IMAGE_BROKER_KEY", - "OPENAI_API_KEY", - ): - assert key in init_command - assert "printf 'API_SERVER_KEY=%s" not in init_command - assert hermes_env["AGENT_BROWSER_EXECUTABLE_PATH"].endswith("/chrome-linux/headless_shell") - assert "--no-sandbox" in hermes_env["AGENT_BROWSER_ARGS"] - - -def test_sandbox_executes_python_with_bounded_output(tmp_path: Path, monkeypatch): - source = ROOT / "dockerfiles" / "hermes-chat-sandbox-server.py" - spec = importlib.util.spec_from_file_location("hermes_chat_sandbox_server", source) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - monkeypatch.setattr(module, "WORKSPACE", tmp_path) - - result = module._execute("import math\nprint(math.comb(10, 3))") - - assert result["success"] is True - assert result["stdout"] == "120\n" - assert result["stderr"] == "" diff --git a/testing/tests/test_hermes_chat_routing_runtime.py b/testing/tests/test_hermes_chat_routing_runtime.py new file mode 100644 index 00000000..6a81d625 --- /dev/null +++ b/testing/tests/test_hermes_chat_routing_runtime.py @@ -0,0 +1,486 @@ +"""Routing and runtime workload contracts for Hermes chat.""" + +from __future__ import annotations + +import importlib.util +import json +import sqlite3 +import sys +import tomllib +from pathlib import Path +from types import SimpleNamespace + + +from testing.tests.test_hermes_chat_support import ( + HERMES, + ROOT, + _documents, +) + + +def test_stale_parent_linked_api_workers_close_on_startup(tmp_path: Path): + """A previous gateway lifetime cannot leave phantom active workers.""" + module_path = HERMES / "scripts" / "migrate_api_session_lineage.py" + spec = importlib.util.spec_from_file_location( + "close_stale_api_workers", module_path + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + database = tmp_path / "state.db" + with sqlite3.connect(database) as connection: + connection.execute( + "CREATE TABLE sessions (id TEXT PRIMARY KEY, source TEXT, " + "parent_session_id TEXT, title TEXT, started_at REAL, ended_at REAL, " + "end_reason TEXT, archived INTEGER DEFAULT 0)" + ) + connection.executemany( + "INSERT INTO sessions " + "(id, source, parent_session_id, started_at, ended_at, end_reason) " + "VALUES (?, ?, ?, 1, ?, ?)", + ( + ("stale", "api_server", "parent", None, None), + ("root", "api_server", None, None, None), + ("finished", "api_server", "parent", 2.0, "api_run_completed"), + ("interactive", "tui", "parent", None, None), + ), + ) + + assert module.migrate(database) == 1 + assert module.migrate(database) == 0 + with sqlite3.connect(database) as connection: + rows = connection.execute( + "SELECT id, ended_at, end_reason FROM sessions ORDER BY id" + ).fetchall() + by_id = {row[0]: row[1:] for row in rows} + assert by_id["stale"][0] is not None + assert by_id["stale"][1] == "api_run_recovered_stale" + assert by_id["root"] == (None, None) + assert by_id["finished"] == (2.0, "api_run_completed") + assert by_id["interactive"] == (None, None) + + +def test_switchyard_brokers_and_native_claude_lane_use_the_right_images(): + """Thin brokers stay small while native Claude runs beside owner auth.""" + dockerfile = ( + ROOT / "dockerfiles" / "Dockerfile.hermes-switchyard-brokers" + ).read_text() + assert "httpx==0.28.1" in dockerfile + assert "worker_route_broker.py" in dockerfile + assert "routing_catalog.py" in dockerfile + + deployment = _documents(HERMES / "switchyard-deployment.yaml")[0] + containers = { + container["name"]: container + for container in deployment["spec"]["template"]["spec"]["containers"] + } + expected = ( + "registry.bstein.dev/bstein/hermes-switchyard-brokers@" + "sha256:ee7e95e060ef8083da505162d7e9030daba15fdd828cc047bbcbe6aa409d2083" + ) + assert containers["worker-route-broker"]["image"] == expected + assert containers["classifier-broker"]["image"] == expected + assert "claude-oauth-broker" not in containers + + agent = _documents(HERMES / "agent-deployment.yaml")[0] + agent_containers = { + container["name"]: container + for container in agent["spec"]["template"]["spec"]["containers"] + } + for container_name in ("hermes", "terminal"): + container = agent_containers[container_name] + environment = {item["name"]: item["value"] for item in container["env"]} + mounts = {item["name"]: item for item in container["volumeMounts"]} + assert ( + environment["HERMES_ROUTING_CATALOG_PATH"] + == "/routing-catalog/catalog.json" + ) + assert ( + environment["HERMES_CODEX_HEALTH_PATH"] + == "/opt/data/provider-health/codex.json" + ) + assert ( + environment["HERMES_CLAUDE_HEALTH_PATH"] + == "/opt/data/provider-health/claude.json" + ) + assert mounts["routing-catalog"]["mountPath"] == "/routing-catalog" + assert mounts["routing-catalog"]["readOnly"] is True + codex = agent_containers["codex-broker"] + codex_environment = {item["name"]: item["value"] for item in codex["env"]} + assert ( + codex_environment["HERMES_CODEX_HEALTH_PATH"] + == "/opt/data/provider-health/codex.json" + ) + claude = agent_containers["claude-broker"] + assert claude["image"].startswith("registry.bstein.dev/bstein/hermes-agent@") + assert "unset ANTHROPIC_API_KEY CLAUDE_API_KEY" in claude["args"][0] + assert any( + mount["name"] == "home" and mount["mountPath"] == "/opt/data" + for mount in claude["volumeMounts"] + ) + + +def test_classifier_broker_bounds_history_without_losing_routing_intent(monkeypatch): + """AUTO classification must fit the Jetson context without losing intent.""" + broker_path = HERMES / "scripts" / "classifier_broker.py" + monkeypatch.setitem(sys.modules, "httpx", SimpleNamespace()) + spec = importlib.util.spec_from_file_location( + "hermes_classifier_broker", broker_path + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + payload = { + "model": "qwen2.5:14b-instruct-q4_0", + "messages": [ + { + "role": "system", + "content": "routing contract\n" + ("candidate policy " * 1000), + }, + { + "role": "user", + "content": "Build and verify the Cassandra release safely.", + }, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_large", + "type": "function", + "function": { + "name": "large_tool", + "arguments": '{"command":"' + ("x" * 5000) + '"}', + }, + } + ], + }, + { + "role": "tool", + "content": "unbounded test output " * 10000, + "tool_call_id": "call_large", + }, + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": "data:image/png;base64,private"}, + }, + {"type": "text", "text": "Turn this cat into a cute clown."}, + ], + }, + ], + "tools": [{"type": "function", "function": {"name": "large_tool"}}], + "tool_choice": "auto", + "parallel_tool_calls": True, + "response_format": {"type": "json_object"}, + } + + compacted = module.compact_payload(payload) + encoded = json.dumps(compacted) + + assert compacted["model"] == payload["model"] + assert compacted["response_format"] == payload["response_format"] + assert "tools" not in compacted + assert "tool_choice" not in compacted + assert "parallel_tool_calls" not in compacted + assert all(message.get("role") != "tool" for message in compacted["messages"]) + assert all("tool_call_id" not in message for message in compacted["messages"]) + assert all("tool_calls" not in message for message in compacted["messages"]) + assert "[tool evidence]" in encoded + assert "[assistant requested an external tool]" in encoded + assert "Build and verify the Cassandra release safely." in encoded + assert "Turn this cat into a cute clown." in encoded + assert "image attachment available to the selected worker" in encoded + assert "data:image/png;base64" not in encoded + assert "unbounded test output " * 100 not in encoded + assert len(encoded) < 18_000 + + +def test_switchyard_classifier_is_bounded_and_fails_open_once(): + """A sick local judge must not hold chat through repeated long retries.""" + config = tomllib.loads( + _documents(HERMES / "switchyard-configmap.yaml")[0]["data"]["routes.toml"] + ) + classifier = config["llm_clients"]["classifier"] + assert classifier["base_url"] == "http://127.0.0.1:9008/v1" + assert classifier["max_retries"] == 0 + for route in ("auto_fast", "auto_balanced"): + assert config["routes"][route]["recent_turn_window"] == 4 + for route in ("auto_deep", "auto_maximum", "worker_auto_maximum"): + assert config["routes"][route]["recent_turn_window"] == 6 + + deployment = _documents(HERMES / "switchyard-deployment.yaml")[0] + containers = { + item["name"]: item + for item in deployment["spec"]["template"]["spec"]["containers"] + } + classifier_container = containers["classifier-broker"] + env = {item["name"]: item["value"] for item in classifier_container["env"]} + assert env["HERMES_CLASSIFIER_BROKER_READ_TIMEOUT"] == "60" + assert classifier_container["readinessProbe"]["httpGet"]["port"] == "classifier" + + +def test_image_broker_returns_bytes_and_removes_owner_cache( + tmp_path: Path, monkeypatch +): + """The broker must not retain a family user's generated image.""" + broker_path = HERMES / "scripts" / "image_broker.py" + spec = importlib.util.spec_from_file_location("hermes_image_broker", broker_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + generated = tmp_path / "generated.png" + generated.write_bytes(b"\x89PNG\r\n\x1a\nprivate-image") + + class Provider: + def generate(self, prompt, aspect, **kwargs): + assert prompt == "paint a blue sphere" + assert aspect == "square" + return { + "success": True, + "image": str(generated), + "model": "gpt-image-2-high", + "quality": "high", + } + + monkeypatch.setattr(module, "_PROVIDER", Provider()) + result = module._generate( + { + "prompt": "paint a blue sphere", + "aspect_ratio": "square", + "model": "gpt-image-2-high", + } + ) + + assert result["success"] is True + assert result["image_b64"] + assert "image" not in result + assert not generated.exists() + + +def test_image_broker_auto_falls_back_to_local_and_honors_explicit_routes( + monkeypatch, +): + """AUTO is hosted-first while explicit local never calls the hosted lane.""" + broker_path = HERMES / "scripts" / "image_broker.py" + spec = importlib.util.spec_from_file_location("hermes_image_router", broker_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + calls = [] + + def hosted(payload, model, prompt, aspect): + calls.append(("hosted", model, prompt, aspect)) + return {"success": False, "error": "hosted refusal"} + + def local(payload, timeout=1800.0): + calls.append(("local", payload["model"], timeout)) + return { + "success": True, + "image_b64": "aW1hZ2U=", + "model": "flux-2-klein-4b-local", + "route": "local", + } + + monkeypatch.setattr(module, "_generate_hosted", hosted) + monkeypatch.setattr(module, "_local_request", local) + auto = module._generate( + { + "prompt": "colorize this family photograph", + "aspect_ratio": "portrait", + "model": "atlas-image-auto-high", + } + ) + assert auto["success"] is True + assert auto["route"] == "local" + assert auto["hosted_fallback_reason"] == "hosted refusal" + assert [call[0] for call in calls] == ["hosted", "local"] + + calls.clear() + explicit = module._generate( + { + "prompt": "make a local landscape", + "aspect_ratio": "landscape", + "model": "flux-2-klein-4b-local", + } + ) + assert explicit["route"] == "local" + assert [call[0] for call in calls] == ["local"] + + +def test_image_broker_policy_is_narrow_and_operator_extensible( + tmp_path: Path, monkeypatch +): + """Family-photo restoration stays allowed while the hard boundary remains.""" + broker_path = HERMES / "scripts" / "image_broker.py" + spec = importlib.util.spec_from_file_location("hermes_image_policy", broker_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + policy = tmp_path / "policy.json" + policy.write_text('{"additional_blocked_phrases":["site-specific block"]}') + monkeypatch.setattr(module, "POLICY_PATH", policy) + + assert ( + module._policy_error( + "Colorize my baby photograph with a lighter natural skin tone" + ) + is None + ) + assert "minors" in module._policy_error("Create a sexual image of a child") + assert module._policy_error("A site-specific block request") == ( + "request is blocked by the operator image policy" + ) + + +def test_local_flux_runtime_and_gpu_handoff_are_flux_managed(): + """FLUX and Wolf share titan-24 while text stays on titan-20.""" + deployment = _documents(HERMES / "local-image-deployment.yaml")[0] + assert deployment["metadata"]["name"] == "hermes-local-image" + pod = deployment["spec"]["template"]["spec"] + assert pod["serviceAccountName"] == "hermes-gpu-runtime" + local = next(item for item in pod["containers"] if item["name"] == "local-image") + assert len(pod["containers"]) == 1 + assert local["resources"]["requests"]["nvidia.com/gpu.shared"] == 1 + assert local["ports"] == [{"name": "local-image", "containerPort": 9004}] + assert any(mount["mountPath"] == "/models" for mount in local["volumeMounts"]) + model_env = {item["name"]: item["value"] for item in local["env"]} + assert model_env["HERMES_LOCAL_IMAGE_LISTEN_PORT"] == "9004" + assert model_env["HERMES_LOCAL_IMAGE_REVISION"] == ( + "e7b7dc27f91deacad38e78976d1f2b499d76a294" + ) + assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVITY_NODE"] == "titan-24" + assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVE_SM_PERCENT"] == "80" + assert model_env["HERMES_LOCAL_IMAGE_GPU_MAX_EXTERNAL_MEMORY_BYTES"] == ( + "3221225472" + ) + assert model_env["HERMES_LOCAL_IMAGE_OFFLOAD_MODE"] == "sequential" + assert ( + "nvidia-process-exporter-local.monitoring.svc.cluster.local" + in model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVITY_URL"] + ) + models_volume = next(item for item in pod["volumes"] if item["name"] == "models") + assert models_volume["persistentVolumeClaim"]["claimName"] == ( + "hermes-image-models" + ) + + services = _documents(HERMES / "service.yaml") + image_service = next( + item for item in services if item["metadata"]["name"] == "hermes-local-image" + ) + assert image_service["spec"]["selector"] == {"app": "hermes-local-image"} + + handoff_services = _documents(HERMES / "model-gate-deployment.yaml") + handoff = next( + item + for item in handoff_services + if item["kind"] == "Service" + and item["metadata"]["name"] == "hermes-gpu-handoff" + ) + assert handoff["spec"]["ports"][0]["targetPort"] == "handoff" + + ariadne = _documents( + Path(__file__).parents[2] / "services/maintenance/apps/ariadne-deployment.yaml" + )[0] + env = { + item["name"]: item["value"] + for item in ariadne["spec"]["template"]["spec"]["containers"][0]["env"] + if "value" in item + } + assert env["GAME_MODE_OLLAMA_URL"] == ( + "http://hermes-gpu-handoff.hermes.svc.cluster.local:11434" + ) + assert env["GAME_MODE_OLLAMA_MODEL"] == "flux-2-klein-4b-local" + + for config_name in ( + "configmap.yaml", + "agent-configmap.yaml", + "chat-configmap.yaml", + ): + config = _documents(HERMES / config_name)[0]["data"]["config.yaml"] + assert "gpt-oss:20b" not in config + assert "atlas-switchyard" in config + switchyard = _documents(HERMES / "switchyard-configmap.yaml")[0]["data"][ + "routes.toml" + ] + assert 'id = "qwen2.5:14b-instruct-q4_0"' in switchyard + assert "qwen2.5:3b-instruct-q4_0" not in switchyard + assert "route/local/qwen2.5-14b/medium" in switchyard + assert "Anthropic and Claude name the same provider" in switchyard + assert "OpenAI and Codex name the same provider" in switchyard + assert "Choose across every configured Codex and Claude family" in switchyard + assert "Claude Fable" in switchyard + assert switchyard.count('Treat "think hard"') == 4 + assert switchyard.count("Never choose below the") >= 5 + switchyard_config = tomllib.loads(switchyard) + routes = switchyard_config["routes"] + configured_targets = switchyard_config["targets"] + for route_name in ("auto_fast", "auto_balanced", "auto_deep", "auto_maximum"): + leading_targets = set(routes[route_name]["targets"][:2]) + assert leading_targets == {"codex_sol_xhigh", "claude_opus_xhigh"} + assert "max_output_tokens" not in routes[route_name] + for route_name in ("auto_fast", "auto_balanced", "auto_deep", "auto_maximum"): + targets = routes[route_name]["targets"] + selector_targets = routes[route_name]["response_schema"] + assert not any(target.startswith("local_") for target in targets) + assert any("fable" in target for target in targets) + assert "local_qwen" not in selector_targets + assert "not eligible for foreground" in routes[route_name]["prompt"] + for route_name in ("auto_deep", "auto_maximum"): + targets = routes[route_name]["targets"] + selector_targets = routes[route_name]["response_schema"] + assert not any(target.endswith("_low") for target in targets) + assert "_low" not in selector_targets + maximum_targets = routes["auto_maximum"]["targets"] + maximum_selector_targets = routes["auto_maximum"]["response_schema"] + assert not any(target.endswith("_medium") for target in maximum_targets) + assert "_medium" not in maximum_selector_targets + assert "absolute high effort floor" in routes["auto_maximum"]["prompt"] + assert "quality mark was missed" in routes["auto_balanced"]["prompt"] + assert "raises the next boundary to xhigh" in routes["auto_maximum"]["prompt"] + assert ( + "Repeated quality misses require xhigh" + in routes["worker_auto_maximum"]["prompt"] + ) + assert any( + target.startswith("local_") for target in routes["manual_local_qwen"]["targets"] + ) + for route_name in ( + "manual_codex_luna", + "manual_codex_terra", + "manual_codex_sol", + "manual_claude_haiku", + "manual_claude_fable", + "manual_claude_sonnet", + "manual_claude_opus", + ): + assert not any( + target.startswith("local_") for target in routes[route_name]["targets"] + ) + for provider, families in { + "codex": ("luna", "terra", "sol"), + "claude": ("haiku", "fable", "sonnet", "opus"), + }.items(): + for family in families: + for effort in ("low", "medium", "high", "xhigh"): + route = routes[f"manual_{provider}_{family}_{effort}"] + assert route["id"] == f"atlas/manual/{provider}/{family}/{effort}" + assert route["targets"][0] == f"{provider}_{family}_{effort}" + worker_target = f"worker_{provider}_{family}_{effort}" + assert worker_target in routes["worker_auto_maximum"]["targets"] + assert configured_targets[worker_target]["id"] == ( + f"worker/{provider}/{family}/{effort}" + ) + for route_name in ("auto_fast", "auto_balanced"): + prompt = routes[route_name]["prompt"] + assert "image tool—not the conversational model" in prompt + assert "Do not select a" in prompt + assert "local Qwen or Claude target" in prompt + assert "max_output_tokens" not in routes["worker_auto_maximum"] + model_gate = _documents(HERMES / "model-gate-configmap.yaml")[0]["data"][ + "model_gate.py" + ] + assert "qwen2.5:14b-instruct-q4_0" in model_gate diff --git a/testing/tests/test_hermes_chat_session_continuity.py b/testing/tests/test_hermes_chat_session_continuity.py new file mode 100644 index 00000000..0da542cb --- /dev/null +++ b/testing/tests/test_hermes_chat_session_continuity.py @@ -0,0 +1,478 @@ +"""Session lineage and continuity contracts for Hermes chat.""" + +from __future__ import annotations + +import importlib.util +import json +import sqlite3 +from pathlib import Path + +import pytest + +from testing.tests.test_hermes_chat_support import ( + HERMES, +) + + +def test_codex_native_health_overrides_historical_router_errors( + tmp_path: Path, monkeypatch +): + """Fresh first-party health is authoritative over old Switchyard probes.""" + plugin_path = HERMES / "plugins" / "auto-router" / "provider_status.py" + spec = importlib.util.spec_from_file_location("hermes_provider_status", plugin_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + health_path = tmp_path / "codex.json" + health_path.write_text( + json.dumps( + { + "state": "available", + "authenticated": True, + "transport": "codex-chatgpt-subscription", + } + ) + ) + monkeypatch.setattr(module, "CODEX_HEALTH_PATH", health_path) + monkeypatch.setattr(module, "CLAUDE_HEALTH_PATH", tmp_path / "missing.json") + monkeypatch.setattr( + module, + "_get_json", + lambda url: {"status": "ok"} + if url.endswith("/health") + else { + "models": { + "route/codex/terra/medium": { + "calls": 1, + "errors": 99, + "total_tokens": 12, + } + } + }, + ) + monkeypatch.setattr(module, "_codex_account", lambda: {}) + monkeypatch.setattr(module, "_claude_account", lambda: {}) + + codex = module.provider_status_payload()["providers"]["codex"] + + assert codex["errors"] == 99 + assert codex["state"] == "available" + assert codex["native_health"]["transport"] == "codex-chatgpt-subscription" + + +def test_api_session_patch_accepts_parent_lineage(tmp_path: Path): + """API-created workers must persist the originating Hermes session.""" + module_path = HERMES / "scripts" / "patch_api_server_sessions.py" + spec = importlib.util.spec_from_file_location("patch_api_sessions", module_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + source = tmp_path / "api_server.py" + destination = tmp_path / "patched.py" + source.write_text( + "prefix\n" + + module.BEFORE + + "middle\n" + + module.RUNS_BEFORE + + "run body\n" + + module.RUN_CLOSE_BEFORE + + module.RESPONSES_SESSION_BEFORE + + module.EVENT_CALLBACK_SIGNATURE_BEFORE + + "callback docstring and push helper\n" + + module.EVENT_CALLBACK_BODY_BEFORE + + "tool start body\n" + + module.EVENT_CALLBACK_END_BEFORE + + module.EVENT_CALLBACK_CALL_BEFORE + + module.RUN_SWEEP_BEFORE + + "suffix\n", + encoding="utf-8", + ) + + module.patch(source, destination) + patched = destination.read_text(encoding="utf-8") + + assert "X-Hermes-Parent-Session-Id" in patched + assert "parent_session_id=parent_session_id" in patched + assert "Parent session not found" in patched + assert "HERMES_API_DEFAULT_PARENT_MATCH_PREFIXES" in patched + assert "user_message.startswith(default_prefixes)" in patched + assert "session_parent_conflict" in patched + assert "X-Hermes-Conversation-Platform" in patched + assert "X-Hermes-Conversation-Title" in patched + assert 'conversation_platform != "telegram"' in patched + assert "db.record_gateway_session_peer(" in patched + assert 'display_name="Telegram"' in patched + assert "db.reopen_session(session_id)" in patched + assert 'db.end_session(session_id, f"api_run_{terminal_status}")' in patched + assert "def _record_run_activity(" in patched + assert '"_thinking": "Hermes is reasoning"' in patched + assert '"run.started": "Worker started"' in patched + assert '"run.completed": "Worker completed"' in patched + assert '"reasoning.available": "Hermes finished a reasoning step"' in patched + assert '"subagent.progress": "Nested worker progress"' in patched + assert "redact_sensitive_text" in patched + assert 'getattr(os, "O_NOFOLLOW", 0)' in patched + assert "os.fchmod(fd, 0o600)" in patched + assert "session_id=session_id" in patched + assert 'self._record_run_activity(session_id, "run.started")' in patched + assert "detail = tool_name if event_type in {" in patched + assert 'if event_type == "subagent.tool"' in patched + assert "_RUN_ACTIVITY_HEARTBEAT_SECONDS = 15.0" in patched + assert "heartbeats.get(session_id, 0.0)" in patched + assert '"subagent.thinking",' in patched + assert "Stream retention and run lifetime are separate" in patched + assert 'terminal_status in {"completed", "failed", "cancelled"}' in patched + assert patched.index("terminal_status = self._run_statuses") < patched.index( + "self._active_run_tasks.pop(run_id, None)" + ) + + +def test_telegram_api_session_migration_is_bounded_and_idempotent(tmp_path: Path): + """Only named Telegram conversations receive presentation metadata.""" + module_path = HERMES / "scripts" / "migrate_telegram_api_sessions.py" + spec = importlib.util.spec_from_file_location( + "migrate_telegram_sessions", module_path + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + state = tmp_path / "state.db" + responses = tmp_path / "response_store.db" + with sqlite3.connect(state) as connection: + connection.execute( + """CREATE TABLE sessions ( + id TEXT PRIMARY KEY, source TEXT, session_key TEXT, chat_type TEXT, + display_name TEXT, origin_json TEXT, title TEXT + )""" + ) + connection.executemany( + "INSERT INTO sessions (id, source) VALUES (?, ?)", + (("telegram-session", "api_server"), ("other-session", "api_server")), + ) + with sqlite3.connect(responses) as connection: + connection.execute( + "CREATE TABLE conversations (name TEXT PRIMARY KEY, response_id TEXT NOT NULL)" + ) + connection.execute( + "CREATE TABLE responses (response_id TEXT PRIMARY KEY, data TEXT NOT NULL, accessed_at REAL NOT NULL)" + ) + connection.executemany( + "INSERT INTO responses VALUES (?, ?, 0)", + ( + ("telegram-response", json.dumps({"session_id": "telegram-session"})), + ("other-response", json.dumps({"session_id": "other-session"})), + ), + ) + connection.executemany( + "INSERT INTO conversations VALUES (?, ?)", + (("telegram", "telegram-response"), ("unrelated", "other-response")), + ) + + assert module.migrate(state, responses) == 1 + assert module.migrate(state, responses) == 0 + with sqlite3.connect(state) as connection: + telegram = connection.execute( + "SELECT session_key, chat_type, display_name, origin_json, title " + "FROM sessions WHERE id = 'telegram-session'" + ).fetchone() + other = connection.execute( + "SELECT session_key, title FROM sessions WHERE id = 'other-session'" + ).fetchone() + assert telegram[:3] == ("telegram", "private", "Telegram") + assert json.loads(telegram[3]) == { + "platform": "telegram", + "session_key": "telegram", + } + assert telegram[4] == "Telegram · General" + assert other == (None, None) + + +def test_web_session_activity_patch_projects_bounded_events(tmp_path: Path): + """The DOM transcript includes events without polluting agent history.""" + module_path = HERMES / "scripts" / "patch_web_session_activity.py" + spec = importlib.util.spec_from_file_location("patch_web_activity", module_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + source = tmp_path / "web_server.py" + destination = tmp_path / "patched.py" + source.write_text( + "prefix\n" + + module.LATEST_ROWS_BEFORE + + module.LATEST_SELECTION_BEFORE + + module.HELPER_MARKER + + " db = object()\n" + + module.MESSAGES_BEFORE + + "suffix\n", + encoding="utf-8", + ) + + module.patch(source, destination) + patched = destination.read_text(encoding="utf-8") + + assert "def _run_activity_messages(" in patched + assert 'root = get_hermes_home() / "run-activity"' in patched + assert "path.stat().st_size > 600_000" in patched + assert "entries[-1_000:]" in patched + assert "*_run_activity_messages(sid)" in patched + assert "messages.sort(" in patched + assert "limit: Optional[int] = None" in patched + assert "total_messages = len(messages)" in patched + assert "min(int(limit), 10_000)" in patched + assert '"total_messages": total_messages' in patched + assert "SELECT id, parent_session_id, started_at, ended_at" in patched + assert "newest still-open member of the lineage" in patched + assert "immediate objective parent" in patched + assert "mixes unrelated workstreams" in patched + assert "orphaned children" in patched + assert 'item[0].get("ended_at") is None' in patched + + +def test_web_session_lineage_returns_to_resumed_parent(): + """An ended reviewer must not strand the live view away from its parent.""" + module_path = HERMES / "scripts" / "patch_web_session_activity.py" + spec = importlib.util.spec_from_file_location("patch_web_lineage", module_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + namespace: dict[str, object] = {} + exec( + "def select_active(sid, db, rows):\n" + module.LATEST_SELECTION_AFTER, + namespace, + ) + + class FakeDB: + def get_session(self, session_id): + return next((row for row in rows if row["id"] == session_id), None) + + rows = [ + { + "id": "umbrella", + "parent_session_id": None, + "started_at": 0.0, + "ended_at": None, + }, + { + "id": "root", + "parent_session_id": "umbrella", + "started_at": 1.0, + "ended_at": None, + }, + { + "id": "review-1", + "parent_session_id": "root", + "started_at": 2.0, + "ended_at": 3.0, + }, + { + "id": "review-2", + "parent_session_id": "root", + "started_at": 4.0, + "ended_at": None, + }, + ] + select_active = namespace["select_active"] + + assert select_active("root", FakeDB(), rows) == ( + "review-2", + ["root", "review-2"], + ) + assert select_active("review-1", FakeDB(), rows) == ( + "review-2", + ["root", "review-2"], + ) + rows[2]["ended_at"] = 5.0 + rows[3]["ended_at"] = 5.0 + assert select_active("root", FakeDB(), rows) == ("root", ["root"]) + + rows[1]["ended_at"] = 9.0 + rows.append( + { + "id": "orphaned-review", + "parent_session_id": "root", + "started_at": 6.0, + "ended_at": None, + } + ) + rows.append( + { + "id": "accepted-review", + "parent_session_id": "root", + "started_at": 7.0, + "ended_at": 8.0, + } + ) + assert select_active("review-1", FakeDB(), rows) == ( + "accepted-review", + ["root", "accepted-review"], + ) + + +def test_api_activity_patch_coalesces_streaming_heartbeats( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +): + """Token callbacks stay bounded while every tool transition is retained.""" + module_path = HERMES / "scripts" / "patch_api_server_sessions.py" + spec = importlib.util.spec_from_file_location("patch_api_activity", module_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + activity_body = module.EVENT_CALLBACK_SIGNATURE_AFTER.split( + " def _make_run_event_callback(", 1 + )[0] + namespace: dict[str, object] = {} + exec( + "import hashlib, json, logging, os, time\n" + "from pathlib import Path\n" + "logger = logging.getLogger(__name__)\n" + "def redact_sensitive_text(value): return value\n" + "class ActivityRecorder:\n" + activity_body, + namespace, + ) + recorder = namespace["ActivityRecorder"]() + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + + recorder._record_run_activity("session", "_thinking") + recorder._record_run_activity("session", "reasoning.available") + recorder._record_run_activity("session", "subagent.thinking") + recorder._record_run_activity("session", "tool.started", tool_name="terminal") + recorder._record_run_activity("session", "tool.completed", tool_name="terminal") + + journals = list((tmp_path / "run-activity").glob("*.jsonl")) + assert len(journals) == 1 + entries = [json.loads(line) for line in journals[0].read_text().splitlines()] + assert [entry["activity_event"] for entry in entries] == [ + "_thinking", + "tool.started", + "tool.completed", + ] + + +def test_legacy_api_sessions_are_nested_idempotently(tmp_path: Path): + """Known standalone API workers move under Cassandra without data loss.""" + module_path = HERMES / "scripts" / "migrate_api_session_lineage.py" + spec = importlib.util.spec_from_file_location("migrate_api_sessions", module_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + database = tmp_path / "state.db" + with sqlite3.connect(database) as connection: + connection.execute( + "CREATE TABLE sessions (id TEXT PRIMARY KEY, source TEXT, " + "parent_session_id TEXT, title TEXT, transcript TEXT, archived INTEGER DEFAULT 0)" + ) + connection.execute( + "INSERT INTO sessions (id, source, parent_session_id, title, transcript) " + "VALUES (?, 'tui', NULL, 'Cassandra', 'parent-data')", + (module.LEGACY_CASSANDRA_PARENT,), + ) + worker_id = next(iter(module.LEGACY_CASSANDRA_WORKERS)) + connection.execute( + "INSERT INTO sessions (id, source, parent_session_id, title, transcript) " + "VALUES (?, 'api_server', NULL, 'old', 'keep-me')", + (worker_id,), + ) + connection.executemany( + "INSERT INTO sessions (id, source, parent_session_id, title, transcript) " + "VALUES (?, 'api_server', NULL, 'old smoke', 'keep-smoke')", + ((orphan_id,) for orphan_id in module.LEGACY_ORPHANED_SMOKE_SESSIONS), + ) + + assert module.migrate(database) == 1 + len(module.LEGACY_ORPHANED_SMOKE_SESSIONS) + assert module.migrate(database) == 0 + with sqlite3.connect(database) as connection: + row = connection.execute( + "SELECT parent_session_id, title, transcript FROM sessions WHERE id = ?", + (worker_id,), + ).fetchone() + assert row == ( + module.LEGACY_CASSANDRA_PARENT, + module.LEGACY_CASSANDRA_WORKERS[worker_id], + "keep-me", + ) + with sqlite3.connect(database) as connection: + orphans = connection.execute( + "SELECT id, archived, title, transcript FROM sessions " + "WHERE id IN ({}) ORDER BY id".format( + ",".join("?" for _ in module.LEGACY_ORPHANED_SMOKE_SESSIONS) + ), + tuple(module.LEGACY_ORPHANED_SMOKE_SESSIONS), + ).fetchall() + assert orphans == sorted( + ( + orphan_id, + 1, + module.LEGACY_ORPHANED_SMOKE_SESSIONS[orphan_id], + "keep-smoke", + ) + for orphan_id in module.LEGACY_ORPHANED_SMOKE_SESSIONS + ) + + +def test_automated_triage_sessions_are_grouped_without_touching_interactive_runs( + tmp_path: Path, +): + """Only the stable Ariadne contract moves below the triage parent.""" + module_path = HERMES / "scripts" / "migrate_api_session_lineage.py" + spec = importlib.util.spec_from_file_location( + "migrate_triage_sessions", module_path + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + database = tmp_path / "state.db" + with sqlite3.connect(database) as connection: + connection.execute( + "CREATE TABLE sessions (id TEXT PRIMARY KEY, source TEXT, " + "parent_session_id TEXT, title TEXT, started_at REAL, archived INTEGER DEFAULT 0)" + ) + connection.execute( + "CREATE TABLE messages (session_id TEXT, role TEXT, content TEXT)" + ) + connection.executemany( + "INSERT INTO sessions (id, source, started_at) VALUES (?, 'api_server', ?)", + (("triage-run", 1.0), ("jenkins-run", 1.5), ("interactive-run", 2.0)), + ) + connection.executemany( + "INSERT INTO messages (session_id, role, content) VALUES (?, 'user', ?)", + ( + ( + "triage-run", + module.TRIAGE_MESSAGE_PREFIXES[0] + + " Fix for incident sonar/bstein_home/python:S2208/finding-key.", + ), + ( + "jenkins-run", + module.TRIAGE_MESSAGE_PREFIXES[1] + + "\nAnalyze incident soteria/291 for the Jenkins job soteria.", + ), + ("interactive-run", "Please explain this alert to me."), + ), + ) + + assert module.migrate(database, group_triage=True) == 2 + assert module.migrate(database, group_triage=True) == 0 + with sqlite3.connect(database) as connection: + parent = connection.execute( + "SELECT title FROM sessions WHERE id = ?", (module.TRIAGE_PARENT,) + ).fetchone() + triage = connection.execute( + "SELECT parent_session_id, title FROM sessions WHERE id = 'triage-run'" + ).fetchone() + interactive = connection.execute( + "SELECT parent_session_id FROM sessions WHERE id = 'interactive-run'" + ).fetchone() + jenkins = connection.execute( + "SELECT parent_session_id, title FROM sessions WHERE id = 'jenkins-run'" + ).fetchone() + assert parent == (module.TRIAGE_PARENT_TITLE,) + assert triage == ( + module.TRIAGE_PARENT, + "Sonar · bstein_home · python:S2208 · iage-run", + ) + assert interactive == (None,) + assert jenkins == ( + module.TRIAGE_PARENT, + "Sonar · soteria/291 · kins-run", + ) diff --git a/testing/tests/test_hermes_chat_support.py b/testing/tests/test_hermes_chat_support.py new file mode 100644 index 00000000..bf7d998c --- /dev/null +++ b/testing/tests/test_hermes_chat_support.py @@ -0,0 +1,38 @@ +"""Shared paths and loaders for Hermes chat capability contracts.""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path +from types import SimpleNamespace + +import yaml + +ROOT = Path(__file__).parents[2] +HERMES = ROOT / "services" / "hermes" +VAULT = ROOT / "services" / "vault" + + +def _documents(path: Path) -> list[dict]: + return [doc for doc in yaml.safe_load_all(path.read_text()) if doc] + + +def _load_broker_module(name: str, filename: str, monkeypatch): + """Load one broker with its mounted routing-catalog dependency.""" + catalog_path = HERMES / "scripts" / "routing_catalog.py" + catalog_spec = importlib.util.spec_from_file_location( + "routing_catalog", catalog_path + ) + assert catalog_spec and catalog_spec.loader + catalog = importlib.util.module_from_spec(catalog_spec) + catalog_spec.loader.exec_module(catalog) + monkeypatch.setitem(sys.modules, "routing_catalog", catalog) + monkeypatch.setitem(sys.modules, "httpx", SimpleNamespace()) + + broker_path = HERMES / "scripts" / filename + spec = importlib.util.spec_from_file_location(name, broker_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module diff --git a/testing/tests/test_hermes_cli_lanes.py b/testing/tests/test_hermes_cli_lanes.py index fe5272da..c40740aa 100644 --- a/testing/tests/test_hermes_cli_lanes.py +++ b/testing/tests/test_hermes_cli_lanes.py @@ -1,93 +1,19 @@ -"""Focused tests for Agent Hermes' direct Codex and Claude Kanban lanes.""" +"""Provider routing and process-isolation contracts for Hermes CLI lanes.""" from __future__ import annotations -import importlib.util import json import os import signal import sys -from contextlib import nullcontext from pathlib import Path -from types import SimpleNamespace - -import pytest -import yaml -SCRIPTS = Path(__file__).parents[2] / "services/hermes/scripts" -sys.path.insert(0, str(SCRIPTS)) -HERMES = Path(__file__).parents[2] / "services/hermes" -KEYCLOAK = Path(__file__).parents[2] / "services/keycloak" -FLUX_HERMES = ( - Path(__file__).parents[2] - / "clusters/atlas/flux-system/applications/hermes/kustomization.yaml" +from testing.tests.test_hermes_cli_lanes_support import ( + _SwitchyardResponse, + lanes, ) - -def _load(name: str): - spec = importlib.util.spec_from_file_location(name, SCRIPTS / f"{name}.py") - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - sys.modules[spec.name] = module - spec.loader.exec_module(module) - return module - - -lanes = _load("cli_lane_runner") -policy = _load("claude_command_policy") -migration = _load("migrate_herdr_state") -auth_patch = _load("patch_hermes_auth") -tui_gateway_patch = _load("patch_tui_gateway") -codex_runtime_patch = _load("patch_codex_runtime") -ttyd_patch = _load("patch_ttyd_index") -client_config = _load("configure_agent_clients") - - -def _agent_deployment() -> dict: - return yaml.safe_load((HERMES / "agent-deployment.yaml").read_text()) - - -def _services() -> dict[str, dict]: - return { - item["metadata"]["name"]: item - for item in yaml.safe_load_all((HERMES / "service.yaml").read_text()) - if item - } - - -def _oauth_deployment(name: str) -> dict: - documents = [ - item - for item in yaml.safe_load_all((HERMES / "oauth2-proxy.yaml").read_text()) - if item - ] - return next( - item - for item in documents - if item["kind"] == "Deployment" and item["metadata"]["name"] == name - ) - - -class _SwitchyardResponse: - """Minimal context-managed response used by routing contract tests.""" - - def __init__(self, selected: str, rationale: str = "local classifier vote"): - self.headers = { - "x-model-router-selected-model": selected, - "x-model-router-rationale": rationale, - } - - def __enter__(self): - return self - - def __exit__(self, *_args): - return False - - def read(self): - return b"{}" - - def test_auto_lane_uses_switchyard_worker_decision(): observed = {} @@ -460,1512 +386,3 @@ def test_worker_terminal_descendants_in_separate_groups_are_killed(monkeypatch): (5000, signal.SIGKILL), } assert process_signals == [(5000, signal.SIGTERM), (5000, signal.SIGKILL)] - - -def test_unassigned_ready_task_is_persistently_routed_to_auto_lane(monkeypatch): - task = SimpleNamespace(id="t_auto", assignee=None, status="ready") - assigned = [] - - class Connection: - def close(self): - return None - - def assign_task(_conn, task_id, profile): - assigned.append((task_id, profile)) - task.assignee = profile - return True - - fake_db = SimpleNamespace( - list_boards=lambda include_archived=False: [{"slug": "cassandra"}], - scoped_current_board=lambda _board: nullcontext(), - connect=lambda board: Connection(), - recompute_ready=lambda _conn: None, - list_tasks=lambda _conn: [task], - assign_task=assign_task, - get_task=lambda _conn, _task_id: task, - claim_task=lambda _conn, _task_id, **_kwargs: task, - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - - assert lanes.claim_ready(set(), 1) == [("cassandra", "t_auto")] - assert assigned == [("t_auto", "cli-auto")] - - -def test_corrupt_board_is_quarantined_without_stopping_healthy_lanes(monkeypatch, capsys): - class CorruptBoardError(Exception): - pass - - task = SimpleNamespace(id="t_healthy", assignee="cli-auto", status="ready") - - class Connection: - def close(self): - return None - - def connect(*, board): - if board == "cassandra": - raise CorruptBoardError("integrity_check failed") - return Connection() - - fake_db = SimpleNamespace( - KanbanDbCorruptError=CorruptBoardError, - list_boards=lambda include_archived=False: [ - {"slug": "cassandra"}, - {"slug": "healthy"}, - ], - scoped_current_board=lambda _board: nullcontext(), - connect=connect, - recompute_ready=lambda _conn: None, - list_tasks=lambda _conn: [task], - claim_task=lambda _conn, _task_id, **_kwargs: task, - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - lanes.BOARD_CORRUPTION_ERRORS.clear() - - assert lanes.claim_ready(set(), 1) == [("healthy", "t_healthy")] - assert "temporarily skipping Kanban board 'cassandra'" in capsys.readouterr().err - - -def test_transient_board_scan_failure_does_not_stop_healthy_lanes(monkeypatch, capsys): - task = SimpleNamespace(id="t_healthy", assignee="cli-auto", status="ready") - - class Connection: - def __init__(self, board): - self.board = board - - def close(self): - return None - - def recompute_ready(connection): - if connection.board == "cassandra": - raise lanes.sqlite3.OperationalError("disk I/O error") - - fake_db = SimpleNamespace( - list_boards=lambda include_archived=False: [ - {"slug": "cassandra"}, - {"slug": "healthy"}, - ], - scoped_current_board=lambda _board: nullcontext(), - connect=lambda board: Connection(board), - recompute_ready=recompute_ready, - list_tasks=lambda _conn: [task], - claim_task=lambda _conn, _task_id, **_kwargs: task, - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - lanes.BOARD_CORRUPTION_ERRORS.clear() - - assert lanes.claim_ready(set(), 1) == [("healthy", "t_healthy")] - error = capsys.readouterr().err - assert "temporarily skipping Kanban board 'cassandra'" in error - assert "storage OperationalError: disk I/O error" in error - - -def test_board_call_retries_storage_faults_on_fresh_connections(): - connections = [] - - class Connection: - def __init__(self): - self.closed = False - - def close(self): - self.closed = True - - def connect(*, board): - assert board == "cassandra" - connection = Connection() - connections.append(connection) - return connection - - attempts = [] - - def operation(_connection): - attempts.append(1) - if len(attempts) < 3: - raise lanes.sqlite3.OperationalError("disk I/O error") - return "healthy" - - fake_db = SimpleNamespace( - scoped_current_board=lambda _board: nullcontext(), - connect=connect, - ) - lanes.BOARD_CORRUPTION_ERRORS.clear() - - assert lanes._board_call(fake_db, "cassandra", operation) == "healthy" - assert len(connections) == 3 - assert all(connection.closed for connection in connections) - - -@pytest.mark.parametrize( - ("result", "expected_action"), - [ - (lanes.ProcessResult(0, "plain text only", None, False), "block"), - ( - lanes.ProcessResult( - 0, - "", - { - "status": "completed", - "summary": "done", - "changed_files": ["src/a.py"], - "tests_run": ["pytest -q"], - "artifacts": ["reports/result.json"], - "blockers": [], - }, - False, - ), - "complete", - ), - ( - lanes.ProcessResult( - 0, - "", - { - "status": "completed", - "summary": "The full test suite is still running.", - "changed_files": ["src/a.py"], - "tests_run": ["pytest -q — in progress"], - "artifacts": [], - "blockers": [], - }, - False, - ), - "block", - ), - ], -) -def test_claim_requires_structured_evidence_and_surfaces_artifacts( - tmp_path: Path, - monkeypatch, - result, - expected_action, -): - task = SimpleNamespace( - id="t_worker", - current_run_id=4, - assignee="cli-auto", - max_runtime_seconds=60, - ) - calls = [] - heartbeats = [] - connections = [] - artifact = tmp_path / "reports/result.json" - artifact.parent.mkdir() - artifact.write_text("{}\n", encoding="utf-8") - - class Connection: - def __init__(self): - self.closed = False - - def close(self): - self.closed = True - - def connect(*, board): - assert board == "cassandra" - connection = Connection() - connections.append(connection) - return connection - - fake_db = SimpleNamespace( - scoped_current_board=lambda _board: nullcontext(), - connect=connect, - get_task=lambda _conn, _task_id: task, - worker_log_path=lambda _task_id, board: tmp_path / "worker.log", - _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_worker"), - set_branch_name=lambda *_args: None, - set_workspace_path=lambda *_args: None, - build_worker_context=lambda *_args: "bounded objective", - heartbeat_worker=lambda _conn, _task_id, *, note, expected_run_id: ( - heartbeats.append((note, expected_run_id)) or True - ), - add_comment=lambda *_args: None, - complete_task=lambda *_args, **kwargs: calls.append(("complete", kwargs)), - block_task=lambda *_args, **kwargs: calls.append(("block", kwargs)), - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - monkeypatch.setattr(lanes, "state_path", lambda _board, _task_id: tmp_path / "state.json") - monkeypatch.setattr( - lanes, - "select_route", - lambda *_args, **_kwargs: lanes.Route( - "codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, () - ), - ) - def run_provider(*args, **_kwargs): - assert connections[0].closed - before_heartbeat = len(connections) - assert args[6]("working") is True - assert len(connections) == before_heartbeat + 1 - assert connections[-1].closed - return result - - monkeypatch.setattr(lanes, "run_provider", run_provider) - - lanes.execute_claim("cassandra", "t_worker") - - assert calls[0][0] == expected_action - assert heartbeats == [("working", 4)] - if expected_action == "complete": - assert calls[0][1]["metadata"]["artifacts"] == [str(artifact)] - assert calls[0][1]["metadata"]["tests_run"] == ["pytest -q"] - else: - assert calls[0][1]["kind"] == "capability" - assert all(connection.closed for connection in connections) - - -def test_goal_card_continues_after_local_judge_rejects_progress( - tmp_path: Path, - monkeypatch, -): - task = SimpleNamespace( - id="t_goal", - current_run_id=12, - assignee="cli-auto", - max_runtime_seconds=300, - goal_mode=True, - goal_max_turns=3, - ) - calls = [] - comments = [] - - class Connection: - def close(self): - return None - - fake_db = SimpleNamespace( - scoped_current_board=lambda _board: nullcontext(), - connect=lambda board: Connection(), - get_task=lambda _conn, _task_id: task, - worker_log_path=lambda _task_id, board: tmp_path / "worker.log", - _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_goal"), - set_branch_name=lambda *_args: None, - set_workspace_path=lambda *_args: None, - build_worker_context=lambda *_args: "Run tests, commit, push, and verify remote HEAD.", - heartbeat_worker=lambda *_args, **_kwargs: True, - add_comment=lambda _conn, _task_id, _author, body: comments.append(body), - complete_task=lambda *_args, **kwargs: calls.append(("complete", kwargs)), - block_task=lambda *_args, **kwargs: calls.append(("block", kwargs)), - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - monkeypatch.setattr( - lanes, - "state_path", - lambda _board, _task_id: tmp_path / "state.json", - ) - claude_low = lanes.Route( - "claude", "claude-fable-5", "low", "claude-low", "jetson", "vote", 1, () - ) - codex_low = lanes.Route( - "codex", "gpt-5.6-luna", "low", "codex-low", "manual", "fallback", 1, () - ) - codex_xhigh = lanes.Route( - "codex", "gpt-5.6-sol", "xhigh", "codex-xhigh", "jetson", "escalated", 1, () - ) - route_calls = [] - - def select_route(_prompt, assignee, **kwargs): - route_calls.append((assignee, kwargs)) - if assignee == "cli-codex-low": - return codex_low - if len(route_calls) == 1: - return claude_low - return codex_xhigh - - monkeypatch.setattr(lanes, "select_route", select_route) - monkeypatch.setattr(lanes, "fresh_unavailable_provider", lambda: None) - reports = [ - lanes.ProcessResult(1, "authentication expired", None, True), - lanes.ProcessResult( - 0, - "first turn", - { - "status": "completed", - "summary": "Focused tests passed.", - "changed_files": ["src/a.py"], - "tests_run": ["pytest focused: passed"], - "artifacts": [], - "blockers": [], - }, - False, - ), - lanes.ProcessResult( - 0, - "second turn", - { - "status": "completed", - "summary": "Full tests passed; commit pushed and remote HEAD verified.", - "changed_files": ["src/a.py"], - "tests_run": ["pytest full: passed"], - "artifacts": [], - "blockers": [], - }, - False, - ), - ] - monkeypatch.setattr(lanes, "run_provider", lambda *_args, **_kwargs: reports.pop(0)) - verdicts = iter( - [ - (False, "commit, push, and remote verification are missing"), - (True, "all explicit acceptance criteria have evidence"), - ] - ) - judge_contexts = [] - - def judge_goal_completion(objective, *_args, **_kwargs): - judge_contexts.append(objective) - return next(verdicts) - - monkeypatch.setattr( - lanes.cli_lane_goal, - "judge_goal_completion", - judge_goal_completion, - ) - - lanes.execute_claim("cassandra", "t_goal") - - assert calls[0][0] == "complete" - assert calls[0][1]["metadata"]["goal_turn"] == 2 - assert any("Goal completion rejected; continuing turn 2/3" in item for item in comments) - assert any("Goal route 2/3: codex/gpt-5.6-sol at xhigh" in item for item in comments) - assert route_calls[2][1]["exclude_provider"] == "claude" - assert "prior rejected reports" in judge_contexts[1] - assert "commit, push, and remote verification are missing" in judge_contexts[1] - assert reports == [] - - -def test_workspace_preparation_failure_durably_blocks_the_claim(tmp_path: Path, monkeypatch): - task = SimpleNamespace(id="t_bad_worktree", current_run_id=7, assignee="cli-auto") - calls = [] - - class Connection: - def close(self): - return None - - fake_db = SimpleNamespace( - scoped_current_board=lambda _board: nullcontext(), - connect=lambda board: Connection(), - get_task=lambda _conn, _task_id: task, - worker_log_path=lambda _task_id, board: tmp_path / "worker.log", - _resolve_worktree_workspace=lambda _task, board: (_ for _ in ()).throw( - ValueError("not a Git repository") - ), - block_task=lambda *_args, **kwargs: calls.append(kwargs), - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - monkeypatch.setattr(lanes, "state_path", lambda _board, _task_id: tmp_path / "state.json") - - lanes.execute_claim("cassandra", "t_bad_worktree") - - assert calls[0]["kind"] == "capability" - assert calls[0]["expected_run_id"] == 7 - assert "not a Git repository" in calls[0]["reason"] - - -def test_artifacts_cannot_escape_the_task_worktree(tmp_path: Path): - workspace = tmp_path / "workspace" - workspace.mkdir() - inside = workspace / "report.json" - outside = tmp_path / "auth.json" - inside.write_text("{}\n", encoding="utf-8") - outside.write_text("secret\n", encoding="utf-8") - - assert lanes.workspace_artifacts( - workspace, - ["report.json", str(outside), "missing.json"], - ) == [str(inside)] - - -def test_restart_provider_change_includes_explicit_workspace_handoff(tmp_path: Path, monkeypatch): - task = SimpleNamespace( - id="t_resume", - current_run_id=9, - assignee="cli-auto", - max_runtime_seconds=60, - ) - state_file = tmp_path / "state.json" - state_file.write_text( - json.dumps({"current_route": {"provider": "claude"}}), - encoding="utf-8", - ) - (tmp_path / "worker.log").write_text("prior provider evidence", encoding="utf-8") - prompts = [] - - class Connection: - def close(self): - return None - - fake_db = SimpleNamespace( - scoped_current_board=lambda _board: nullcontext(), - connect=lambda board: Connection(), - get_task=lambda _conn, _task_id: task, - worker_log_path=lambda _task_id, board: tmp_path / "worker.log", - _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_resume"), - set_branch_name=lambda *_args: None, - set_workspace_path=lambda *_args: None, - build_worker_context=lambda *_args: "resume objective", - add_comment=lambda *_args: None, - complete_task=lambda *_args, **_kwargs: None, - block_task=lambda *_args, **_kwargs: None, - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - monkeypatch.setattr(lanes, "state_path", lambda _board, _task_id: state_file) - monkeypatch.setattr( - lanes, - "select_route", - lambda *_args, **_kwargs: lanes.Route( - "codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, () - ), - ) - monkeypatch.setattr( - lanes, - "git_handoff", - lambda _workspace, output: f"HANDOFF:{output}", - ) - monkeypatch.setattr( - lanes, - "run_provider", - lambda _route, prompt, *_args, **_kwargs: ( - prompts.append(prompt) - or lanes.ProcessResult( - 0, - "", - { - "status": "completed", - "summary": "done", - "changed_files": [], - "tests_run": [], - "artifacts": [], - "blockers": [], - }, - False, - ) - ), - ) - - lanes.execute_claim("cassandra", "t_resume") - - assert "HANDOFF:prior provider evidence" in prompts[0] - - -def test_provider_commands_are_structured_unattended_and_capped(tmp_path: Path): - route = lanes.Route("codex", "gpt-5.6-sol", "xhigh", "codex-xhigh", "jetson", "vote", 1, ()) - command = lanes._codex_command(route, "Work.", tmp_path, {}, tmp_path / "result.json") - assert "--dangerously-bypass-approvals-and-sandbox" in command - assert "--json" in command - assert "--output-schema" in command - assert 'model_reasoning_effort="xhigh"' in command - - claude_state = {"claude_session_id": "13864642-2985-4f91-bef5-53f145f878e8"} - claude = lanes._claude_command( - lanes.Route("claude", "claude-opus-5", "xhigh", "claude-xhigh", "jetson", "vote", 1, ()), - "Review.", - claude_state, - False, - ) - assert "--dangerously-skip-permissions" in claude - assert "--output-format" in claude and "stream-json" in claude - assert "--json-schema" in claude - assert "--disallowedTools" in claude - assert "Bash(kubectl apply *)" not in claude - assert "Bash(flux reconcile *)" not in claude - assert "max" not in claude - - -def test_worker_contract_separates_review_findings_from_task_blockers(tmp_path: Path): - prompt = lanes.build_prompt("Review the change.", tmp_path) - - assert "put defects and risks in findings" in prompt - assert "blockers array must be empty whenever status is completed" in prompt - assert "findings" in lanes.RESULT_SCHEMA["properties"] - assert set(lanes.RESULT_SCHEMA["required"]) == set( - lanes.RESULT_SCHEMA["properties"] - ) - assert "assigned task itself" in lanes.RESULT_SCHEMA["properties"]["blockers"]["description"] - - -@pytest.mark.parametrize( - "command", - [ - "git push --force origin main", - "git reset --hard HEAD~1", - "git clean -fd", - ], -) -def test_claude_pretool_hook_blocks_hard_denies(command: str): - assert policy.denial_reason(command) - - -def test_claude_pretool_hook_allows_normal_engineering(): - assert policy.denial_reason("pytest -q testing/tests") is None - assert policy.denial_reason("git push origin feature/hermes") is None - assert policy.denial_reason("kubectl delete pod -n cassandra stuck-worker") is None - assert policy.denial_reason("flux reconcile kustomization hermes") is None - assert policy.denial_reason("vault kv get kv/atlas/hermes") is None - - -def test_claude_settings_preserve_state_and_install_three_guardrail_layers(tmp_path: Path): - state = tmp_path / ".claude.json" - settings = tmp_path / "settings.json" - state.write_text('{"promptQueueUseCount": 4}\n', encoding="utf-8") - settings.write_text( - json.dumps( - { - "theme": "dark", - "permissions": { - "deny": [ - "Bash(kubectl apply *)", - "Bash(flux reconcile *)", - "Bash(vault kv *)", - "Bash(custom-owner-rule *)", - ] - }, - } - ) - + "\n", - encoding="utf-8", - ) - - client_config.configure_claude_state(state) - client_config.configure_claude_settings(settings) - - state_value = json.loads(state.read_text()) - settings_value = json.loads(settings.read_text()) - assert state_value["promptQueueUseCount"] == 4 - assert state_value["bypassPermissionsModeAccepted"] is True - assert settings_value["theme"] == "dark" - assert "Bash(git reset --hard *)" in settings_value["permissions"]["deny"] - assert "Bash(custom-owner-rule *)" in settings_value["permissions"]["deny"] - assert "Bash(kubectl apply *)" not in settings_value["permissions"]["deny"] - assert "Bash(flux reconcile *)" not in settings_value["permissions"]["deny"] - assert "Bash(vault kv *)" not in settings_value["permissions"]["deny"] - hook = settings_value["hooks"]["PreToolUse"][0]["hooks"][0] - assert "claude_command_policy.py" in hook["command"] - - -def test_legacy_state_is_archived_without_removing_provider_transcripts(tmp_path: Path): - session = tmp_path / "home/.config/herdr/session.json" - session.parent.mkdir(parents=True) - session.write_text('{"agents":[{"agent":"claude","session_id":"abc"}]}', encoding="utf-8") - binary = tmp_path / "tools/bin/herdr" - binary.parent.mkdir(parents=True) - binary.write_text("legacy", encoding="utf-8") - (tmp_path / "home/.claude").mkdir() - (tmp_path / "home/.codex").mkdir() - - archive = migration.archive_legacy_state(tmp_path) - - value = json.loads(archive.read_text()) - assert value["legacy_session"]["agents"][0]["session_id"] == "abc" - assert (tmp_path / "home/.claude").is_dir() - assert (tmp_path / "home/.codex").is_dir() - assert not binary.exists() - assert not (tmp_path / "home/.config/herdr").exists() - - -def test_agent_uses_one_native_kanban_control_plane(): - configmap = yaml.safe_load((HERMES / "agent-configmap.yaml").read_text()) - config = yaml.safe_load(configmap["data"]["config.yaml"]) - assert config["model"] == { - "provider": "atlas-switchyard", - "default": "atlas/auto/balanced", - "model": "atlas/auto/balanced", - } - assert config["kanban"]["dispatch_in_gateway"] is True - assert config["kanban"]["default_assignee"] == "cli-auto" - assert config["plugins"]["enabled"] == ["auto-router"] - - deployment = _agent_deployment() - pod = deployment["spec"]["template"]["spec"] - assert pod["enableServiceLinks"] is False - names = {item["name"] for item in pod["containers"]} - assert "cli-lane-runner" in names - assert "terminal" in names - assert not any("herdr" in name for name in names) - rendered = (HERMES / "agent-deployment.yaml").read_text() - assert "herdr server" not in rendered - assert "herdr-dispatch" not in rendered - - -def test_cli_lane_reserves_cpu_headroom_for_ui_and_auth(): - deployment = _agent_deployment() - containers = { - item["name"]: item - for item in deployment["spec"]["template"]["spec"]["containers"] - } - lane = containers["cli-lane-runner"] - environment = {item["name"]: item["value"] for item in lane["env"]} - - assert environment["HERMES_CLI_LANE_CONCURRENCY"] == "2" - assert lane["resources"] == { - "requests": {"cpu": "100m", "memory": "256Mi"}, - "limits": {"cpu": "2", "memory": "6Gi"}, - } - - -def test_agent_avoids_unhealthy_nodes_and_fits_its_remaining_capacity(): - """Placement correction: keep the agent off nodes that cannot hold it. - - titan-04 is cordoned after repeated kernel undervoltage and kubelet - failure, and titan-19 was probe/Longhorn unstable under worker load, so - both must join the existing hard exclusions. That leaves titan-05 as the - healthy candidate, which is tight enough on requested CPU that the main - container has to give back 50m to schedule there. - """ - pod = _agent_deployment()["spec"]["template"]["spec"] - hostnames = next( - item - for item in pod["affinity"]["nodeAffinity"][ - "requiredDuringSchedulingIgnoredDuringExecution" - ]["nodeSelectorTerms"][0]["matchExpressions"] - if item["key"] == "kubernetes.io/hostname" - ) - - assert hostnames["operator"] == "NotIn" - assert set(hostnames["values"]) >= {"titan-04", "titan-19"} - - hermes = next( - item for item in pod["containers"] if item["name"] == "hermes" - ) - assert hermes["resources"]["requests"]["cpu"] == "300m" - - -def test_agent_root_is_stock_dashboard_and_terminal_is_a_separate_path(): - deployment = _agent_deployment() - pod = deployment["spec"]["template"]["spec"] - containers = {item["name"]: item for item in pod["containers"]} - assert "webui" not in containers - - assert "dashboard" not in containers - hermes = containers["hermes"] - hermes_env = {item["name"]: item["value"] for item in hermes["env"]} - assert hermes_env["HERMES_STREAM_STALE_TIMEOUT"] == "600" - assert hermes_env["HERMES_API_CALL_STALE_TIMEOUT"] == "600" - assert hermes["command"] == ["/bin/sh", "-ec"] - startup = hermes["args"][0] - assert ". /opt/data/.env" in startup - assert "exec /init /opt/hermes/docker/main-wrapper.sh gateway run" in startup - hermes_env = {item["name"]: item["value"] for item in hermes["env"]} - assert hermes_env["HERMES_DASHBOARD"] == "1" - assert hermes_env["HERMES_DASHBOARD_HOST"] == "127.0.0.1" - assert hermes_env["HERMES_DASHBOARD_PORT"] == "9119" - assert hermes_env["HERMES_TUI_AGENT_INIT_TIMEOUT_S"] == "180" - assert hermes["securityContext"]["runAsUser"] == 0 - assert hermes["securityContext"]["runAsGroup"] == 0 - for probe_name in ("startupProbe", "readinessProbe", "livenessProbe"): - probe = hermes[probe_name] - assert probe["exec"]["command"] == [ - "curl", - "-fsS", - "http://127.0.0.1:9119/api/status", - ] - - terminal = containers["terminal"] - command = terminal["args"][0] - assert "--base-path /terminal" in command - assert "--check-origin" not in command - assert "/usr/bin/tmux new-session -A" in command - assert "--continue" in command - assert "--yolo" in command - terminal_env = {item["name"]: item["value"] for item in terminal["env"]} - assert terminal_env["HERMES_TUI_AGENT_INIT_TIMEOUT_S"] == "180" - - claude_broker = containers["claude-broker"] - claude_env = {item["name"]: item["value"] for item in claude_broker["env"]} - assert claude_env["HERMES_CLAUDE_BROKER_CONCURRENCY"] == "2" - assert claude_broker["readinessProbe"]["tcpSocket"] == {"port": "claude-broker"} - assert claude_broker["livenessProbe"]["tcpSocket"] == {"port": "claude-broker"} - - args = containers["oauth2-proxy"]["args"] - terminal_upstream = "--upstream=http://127.0.0.1:7681/terminal/" - dashboard_upstream = "--upstream=http://127.0.0.1:9119/" - assert terminal_upstream in args - assert dashboard_upstream in args - assert args.index(terminal_upstream) < args.index(dashboard_upstream) - assert "--pass-host-header=false" in args - assert "--cookie-refresh=19m" in args - assert "--session-store-type=redis" in args - assert any( - arg.startswith("--redis-connection-url=redis://hermes-oauth-sessions.") - for arg in args - ) - - patch_init = next( - item for item in pod["initContainers"] - if item["name"] == "patch-tui-gateway" - ) - assert patch_init["command"][-1] == "/patched/server.py" - for name in ("hermes", "terminal"): - mounts = containers[name]["volumeMounts"] - assert { - "name": "tui-gateway-patch", - "mountPath": "/opt/hermes/tui_gateway/server.py", - "subPath": "server.py", - } in mounts - - ingress_documents = [ - item - for item in yaml.safe_load_all((HERMES / "agent-ingress.yaml").read_text()) - if item - ] - middlewares = { - item["metadata"]["name"]: item - for item in ingress_documents - if item["kind"] == "Middleware" - } - assert middlewares["hermes-agent-terminal-slash"]["spec"]["redirectRegex"][ - "replacement" - ].endswith("/terminal/") - assert middlewares["hermes-agent-stock-dashboard-headers"]["spec"]["headers"][ - "customRequestHeaders" - ]["Origin"] == "http://127.0.0.1:9119" - ingresses = { - item["metadata"]["name"]: item - for item in ingress_documents - if item["kind"] == "Ingress" - } - assert ingresses["hermes-agent-dashboard"]["metadata"]["annotations"][ - "traefik.ingress.kubernetes.io/router.middlewares" - ] == "hermes-hermes-agent-stock-dashboard-headers@kubernetescrd" - assert ingresses["hermes-agent-terminal"]["metadata"]["annotations"][ - "traefik.ingress.kubernetes.io/router.middlewares" - ] == "hermes-hermes-agent-terminal-slash@kubernetescrd" - - -def test_broker_services_survive_sibling_container_readiness_loss(): - services = _services() - - for name in ( - "hermes-image-broker", - "hermes-codex-broker", - "hermes-local-image", - "hermes-claude-broker", - ): - assert services[name]["spec"]["publishNotReadyAddresses"] is True - - -def test_agent_dashboard_reconnects_all_transient_websockets(): - dockerfile = ( - HERMES.parents[1] / "dockerfiles/Dockerfile.hermes-agent" - ).read_text(encoding="utf-8") - assert "eventsRetryAttempt.current" in dockerfile - assert "if (!unmounting) setVersion((v) => v + 1);" in dockerfile - assert "events feed rejected (${ev.code}) — reload the page" in dockerfile - assert 'url = await api.buildWsUrl("/api/pty", params);' in dockerfile - assert 'url = await buildWsUrl("/api/events", { channel });' in dockerfile - assert dockerfile.count("' await api.getSessions(1, 0,") == 2 - assert ".then(() => gw.connect())" in dockerfile - assert 'api.getSessions(1, 0, profile ?? "")' in dockerfile - assert "dashboard token rotated by a server restart" in dockerfile - - -def test_agent_refreshes_routes_after_restoring_cli_logins(): - deployment = _agent_deployment() - init_containers = { - item["name"]: item - for item in deployment["spec"]["template"]["spec"]["initContainers"] - } - configure = init_containers["configure-agent-clients"] - command = configure["command"][-1] - assert "configure_agent_clients.py" in command - assert command.index("configure_agent_clients.py") < command.index( - "hermes_coordinator.py --once" - ) - env = {item["name"]: item["value"] for item in configure["env"]} - assert env["HERMES_AUTH_FILE"] == "/runtime-access/hermes-auth.json" - assert env["PYTHONPATH"] == "/opt/hermes" - for name in ("bootstrap-coordinator", "configure-agent-clients"): - route_env = { - item["name"]: item["value"] - for item in init_containers[name]["env"] - } - assert route_env["CODEX_HOME"] == "/runtime-access/codex" - assert route_env["CLAUDE_CONFIG_DIR"] == "/runtime-access/claude" - assert "/opt/data/tools/bin" in route_env["PATH"] - - assert "patch-web-session-activity" in init_containers - web_patch = init_containers["patch-web-session-activity"] - assert "/opt/coordinator/patch_web_session_activity.py" in web_patch["command"] - - containers = { - item["name"]: item - for item in deployment["spec"]["template"]["spec"]["containers"] - } - steward_env = { - item["name"]: item["value"] - for item in containers["model-steward"]["env"] - } - assert steward_env["CODEX_HOME"] == "/runtime-access/codex" - assert steward_env["CLAUDE_CONFIG_DIR"] == "/runtime-access/claude" - assert "/opt/data/tools/bin" in steward_env["PATH"] - hermes_mounts = { - (item["name"], item["mountPath"], item.get("subPath")) - for item in containers["hermes"]["volumeMounts"] - } - assert ( - "web-server-patch", - "/opt/hermes/hermes_cli/web_server.py", - "web_server.py", - ) in hermes_mounts - - -def test_flux_health_checks_follow_the_owner_oauth_sidecar(): - flux = yaml.safe_load(FLUX_HERMES.read_text()) - checks = { - (item["kind"], item["name"]) - for item in flux["spec"]["healthChecks"] - } - assert ("Deployment", "hermes-agent") in checks - assert ("DaemonSet", "hermes-node-ssh-access") in checks - assert ("Deployment", "oauth2-proxy-hermes-agent") not in checks - - -def test_agent_auth_is_bstein_group_and_email_bounded(): - deployment = _agent_deployment() - oauth = next( - item for item in deployment["spec"]["template"]["spec"]["containers"] - if item["name"] == "oauth2-proxy" - ) - args = oauth["args"] - assert "--user-id-claim=sub" in args - assert "--oidc-groups-claim=groups" in args - assert "--allowed-group=/hermes-owner" in args - assert "--authenticated-emails-file=/etc/oauth2-proxy/allowed-emails" in args - script = (KEYCLOAK / "scripts/hermes_access_oidc_ensure.sh").read_text() - assert 'group_name="hermes-owner"' in script - assert "username=bstein&exact=true" in script - assert '"full.path":"true"' in script - - -def test_agent_network_boundary_allows_only_authenticated_and_metrics_surfaces(): - documents = [ - item - for item in yaml.safe_load_all((HERMES / "networkpolicy.yaml").read_text()) - if item - ] - isolation = next(item for item in documents if item.get("metadata", {}).get("name") == "hermes-agent-isolation") - assert isolation["spec"]["ingress"] == [ - { - "from": [ - { - "namespaceSelector": { - "matchLabels": { - "kubernetes.io/metadata.name": "traefik" - } - }, - "podSelector": { - "matchLabels": {"app.kubernetes.io/name": "traefik"} - }, - } - ], - "ports": [{"protocol": "TCP", "port": 4180}], - }, - { - "from": [ - { - "podSelector": { - "matchLabels": {"app": "hermes-chat-tenant"} - } - } - ], - "ports": [ - {"protocol": "TCP", "port": 9002}, - {"protocol": "TCP", "port": 9003}, - ], - }, - { - "from": [ - { - "podSelector": { - "matchLabels": {"app": "hermes-switchyard"} - } - } - ], - "ports": [ - {"protocol": "TCP", "port": 9003}, - {"protocol": "TCP", "port": 9006}, - ], - }, - { - "from": [ - { - "namespaceSelector": { - "matchLabels": { - "kubernetes.io/metadata.name": "monitoring" - } - }, - "podSelector": {"matchLabels": {"app": "server"}}, - } - ], - "ports": [{"protocol": "TCP", "port": 9010}], - }, - ] - assert isolation["spec"]["egress"] == [{}] - - -def test_owner_agent_has_cluster_admin_kubernetes_context(): - config = yaml.safe_load((HERMES / "agent-kubeconfig.yaml").read_text()) - assert config["current-context"] == "atlas-owner" - assert config["contexts"][0]["context"]["namespace"] == "default" - rbac_path = HERMES / "agent-rbac.yaml" - documents = [item for item in yaml.safe_load_all(rbac_path.read_text()) if item] - binding = next(item for item in documents if item["kind"] == "ClusterRoleBinding") - assert binding["roleRef"] == { - "apiGroup": "rbac.authorization.k8s.io", - "kind": "ClusterRole", - "name": "cluster-admin", - } - assert binding["subjects"] == [ - {"kind": "ServiceAccount", "name": "hermes-agent", "namespace": "hermes"} - ] - - -def test_owner_agent_has_pinned_dedicated_node_ssh_access(): - deployment = _agent_deployment() - annotations = deployment["spec"]["template"]["metadata"]["annotations"] - assert annotations[ - "vault.hashicorp.com/agent-inject-secret-node-ssh-private-key" - ] == "kv/data/atlas/hermes/developer-ssh" - assert annotations[ - "vault.hashicorp.com/agent-inject-secret-node-ssh-config" - ] == "kv/data/atlas/hermes/developer-ssh" - assert annotations[ - "vault.hashicorp.com/agent-inject-secret-node-ssh-known-hosts" - ] == "kv/data/atlas/hermes/developer-ssh" - - init = next( - item - for item in deployment["spec"]["template"]["spec"]["initContainers"] - if item["name"] == "init-config" - ) - command = init["command"][2] - assert "ln -s /runtime-access/node-ssh-config /opt/data/home/.ssh/config" in command - assert ( - "ln -s /runtime-access/node-ssh-known-hosts /opt/data/home/.ssh/known_hosts" - in command - ) - assert "ln -s home/.ssh /opt/data/.ssh" in command - assert "chmod 0700 /opt/data/home/.ssh" in command - assert ( - "ln -s /runtime-access/node-ssh-private-key " - "/opt/data/home/.ssh/id_ed25519_atlas_nodes" - ) in command - - config = yaml.safe_load((HERMES / "agent-configmap.yaml").read_text())["data"] - assert "ssh_config" not in config - assert "ssh_known_hosts" not in config - assert "ssh-ed25519" not in (HERMES / "agent-configmap.yaml").read_text() - - resources = yaml.safe_load((HERMES / "kustomization.yaml").read_text())[ - "resources" - ] - assert "node-ssh-access.yaml" in resources - access = [ - item - for item in yaml.safe_load_all((HERMES / "node-ssh-access.yaml").read_text()) - if item - ] - service_account = next(item for item in access if item["kind"] == "ServiceAccount") - assert service_account["metadata"]["name"] == "hermes-node-ssh-access" - provider = next(item for item in access if item["kind"] == "SecretProviderClass") - assert provider["metadata"]["name"] == "hermes-node-ssh-access" - assert provider["spec"]["provider"] == "vault" - parameters = provider["spec"]["parameters"] - assert parameters["roleName"] == "hermes-node-ssh" - assert 'secretPath: "kv/data/atlas/hermes/developer-ssh"' in parameters["objects"] - assert 'secretKey: "public_key"' in parameters["objects"] - daemonset = next(item for item in access if item["kind"] == "DaemonSet") - pod = daemonset["spec"]["template"]["spec"] - assert pod["serviceAccountName"] == "hermes-node-ssh-access" - assert pod["automountServiceAccountToken"] is True - host_home = next(item for item in pod["volumes"] if item["name"] == "host-home") - assert host_home["hostPath"] == {"path": "/home", "type": "Directory"} - vault_secrets = next( - item for item in pod["volumes"] if item["name"] == "vault-secrets" - ) - assert vault_secrets["csi"]["driver"] == "secrets-store.csi.k8s.io" - assert vault_secrets["csi"]["volumeAttributes"] == { - "secretProviderClass": "hermes-node-ssh-access" - } - reconciler = pod["containers"][0]["args"][0] - assert "cat /vault/secrets/node-ssh-public-key" in reconciler - assert "grep -qxF" in reconciler - assert "for user in atlas oceanus" in reconciler - assert "/host-etc/passwd" in reconciler - assert "chown \"${uid}:${gid}\"" in reconciler - host_passwd = next( - item for item in pod["volumes"] if item["name"] == "host-passwd" - ) - assert host_passwd["hostPath"] == {"path": "/etc/passwd", "type": "File"} - - -def test_owner_agent_tracks_no_ssh_identity_or_host_key_material(): - """Vault references may be tracked; SSH identities and trust data may not.""" - forbidden = ( - "BEGIN OPENSSH PRIVATE KEY", - "ssh-ed25519 AAAA", - "ssh-rsa AAAA", - "IdentityFile ", - "UserKnownHostsFile ", - "StrictHostKeyChecking ", - "ssh_config:", - "ssh_known_hosts:", - ) - text_suffixes = { - ".conf", - ".json", - ".md", - ".py", - ".sh", - ".toml", - ".yaml", - ".yml", - } - tracked = "\n".join( - path.read_text(encoding="utf-8") - for path in HERMES.rglob("*") - if path.is_file() and path.suffix in text_suffixes - ) - for marker in forbidden: - assert marker not in tracked - - -def test_switchyard_has_a_dedicated_non_owner_identity_and_read_only_catalog(): - """Routing must not inherit the owner agent's cluster-admin capability.""" - service_accounts = [ - item - for item in yaml.safe_load_all( - (HERMES / "vault-serviceaccount.yaml").read_text() - ) - if item - ] - assert any( - item["kind"] == "ServiceAccount" - and item["metadata"]["name"] == "hermes-switchyard" - for item in service_accounts - ) - - switchyard = yaml.safe_load( - (HERMES / "switchyard-deployment.yaml").read_text() - ) - switchyard_pod = switchyard["spec"]["template"]["spec"] - assert switchyard_pod["serviceAccountName"] == "hermes-switchyard" - agent_pod = _agent_deployment()["spec"]["template"]["spec"] - for pod, container_name in ( - (switchyard_pod, "worker-route-broker"), - (agent_pod, "claude-broker"), - ): - container = next(item for item in pod["containers"] if item["name"] == container_name) - catalog = next( - item - for item in container["volumeMounts"] - if item["mountPath"] == "/routing-catalog" - ) - assert catalog["readOnly"] is True - - rbac = [ - item - for item in yaml.safe_load_all((HERMES / "agent-rbac.yaml").read_text()) - if item - ] - binding = next(item for item in rbac if item["kind"] == "ClusterRoleBinding") - assert binding["subjects"] == [ - {"kind": "ServiceAccount", "name": "hermes-agent", "namespace": "hermes"} - ] - - -def test_switchyard_active_state_uses_a_relocatable_rwx_claim(): - """A stale node attachment must not strand the routing authority.""" - claims = [ - item - for item in yaml.safe_load_all((HERMES / "switchyard-pvc.yaml").read_text()) - if item - ] - active_claim = next( - item - for item in claims - if item["metadata"]["name"] == "hermes-switchyard-state-rwx" - ) - assert active_claim["spec"]["accessModes"] == ["ReadWriteMany"] - - deployment = yaml.safe_load((HERMES / "switchyard-deployment.yaml").read_text()) - strategy = deployment["spec"]["strategy"] - assert strategy == { - "type": "RollingUpdate", - "rollingUpdate": {"maxSurge": 1, "maxUnavailable": 0}, - } - pod = deployment["spec"]["template"]["spec"] - state = next(item for item in pod["volumes"] if item["name"] == "state") - assert state["persistentVolumeClaim"]["claimName"] == active_claim["metadata"][ - "name" - ] - - -def test_worker_route_broker_accepts_pod_network_health_checks(): - """Kubelet probes the pod IP, so the broker cannot bind to loopback only.""" - script = (SCRIPTS / "worker_route_broker.py").read_text() - assert 'ThreadingHTTPServer(("0.0.0.0", PORT), Handler)' in script - - -def test_switchyard_network_boundary_allows_vault_bootstrap(): - """The pre-populate init container must reach Vault before routing starts.""" - documents = [ - item - for item in yaml.safe_load_all((HERMES / "networkpolicy.yaml").read_text()) - if item - ] - isolation = next( - item - for item in documents - if item.get("metadata", {}).get("name") == "hermes-switchyard-isolation" - ) - assert any( - rule.get("to") - == [ - { - "namespaceSelector": { - "matchLabels": {"kubernetes.io/metadata.name": "vault"} - }, - "podSelector": {"matchLabels": {"app": "vault"}}, - } - ] - and rule.get("ports") == [{"protocol": "TCP", "port": 8200}] - for rule in isolation["spec"]["egress"] - ) - - -def test_switchyard_network_boundary_allows_metrics_scraping(): - """VictoriaMetrics may scrape Switchyard without widening its API boundary.""" - documents = [ - item - for item in yaml.safe_load_all((HERMES / "networkpolicy.yaml").read_text()) - if item - ] - isolation = next( - item - for item in documents - if item.get("metadata", {}).get("name") == "hermes-switchyard-isolation" - ) - assert any( - rule.get("from") - == [ - { - "namespaceSelector": { - "matchLabels": { - "kubernetes.io/metadata.name": "monitoring" - } - }, - "podSelector": {"matchLabels": {"app": "server"}}, - } - ] - and rule.get("ports") == [{"protocol": "TCP", "port": 9005}] - for rule in isolation["spec"]["ingress"] - ) - - -def test_owner_agent_installs_the_pinned_operator_toolchain(): - script = (SCRIPTS / "install_agent_tools.sh").read_text() - for value in [ - "flux", - "helm", - "kustomize", - "jq", - "yq", - "gh", - "vault", - "sops", - "age", - "age-keygen", - "k9s", - "terraform", - "go", - "gofmt", - ]: - assert value in script - assert "go1.26.5.linux-arm64.tar.gz" in script - assert ( - "fe4789e92b1f33358680864bbe8704289e7bb5fc207d80623c308935bd696d49" - in script - ) - assert script.count("sha256sum -c -") == 1 - - deployment = _agent_deployment() - installer = next( - item - for item in deployment["spec"]["template"]["spec"]["initContainers"] - if item["name"] == "install-agent-tools" - ) - assert "/bin/sh /opt/coordinator/install_agent_tools.sh" in installer["command"][2] - assert any(mount["name"] == "coordinator" for mount in installer["volumeMounts"]) - init_config = next( - item - for item in deployment["spec"]["template"]["spec"]["initContainers"] - if item["name"] == "init-config" - ) - init_command = init_config["command"][2] - assert "# Hermes managed operator PATH." in init_command - assert "/opt/coordinator:/opt/data/tools/bin:/opt/hermes/.venv/bin" in init_command - assert 'chmod 0644 "${profile_file}"' in init_command - - -def test_owner_agent_uses_only_the_canonical_hostname(): - paths = [ - HERMES / "agent-configmap.yaml", - HERMES / "agent-deployment.yaml", - HERMES / "agent-ingress.yaml", - Path(__file__).parents[2] / "scripts/ops/hermes_triage_monitor.py", - ] - for path in paths: - content = path.read_text() - assert "agent.bstein.dev" not in content - assert "agent.hermes.bstein.dev" in content - - -def test_agent_reconnect_retains_complete_history_and_long_tool_budget(): - configmap = yaml.safe_load((HERMES / "agent-configmap.yaml").read_text()) - config = yaml.safe_load(configmap["data"]["config.yaml"]) - display = config["display"] - assert display["resume_exchanges"] >= 10000 - assert display["resume_max_user_chars"] >= 10000000 - assert display["resume_max_assistant_chars"] >= 10000000 - assert config["agent"]["max_turns"] == 180 - assert config["delegation"]["max_iterations"] == 120 - - -def test_auth_patch_honors_explicit_shared_store(tmp_path: Path): - source = tmp_path / "auth.py" - destination = tmp_path / "patched/auth.py" - source.write_text( - 'from pathlib import Path\nimport os\n\ndef _auth_file_path() -> Path:\n path = get_hermes_home() / "auth.json"\n return path\n', - encoding="utf-8", - ) - auth_patch.patch(source, destination) - content = destination.read_text() - assert 'os.environ.get("HERMES_AUTH_FILE"' in content - - -def test_auth_patch_fails_closed_on_upstream_drift(tmp_path: Path): - source = tmp_path / "auth.py" - source.write_text("def changed():\n pass\n", encoding="utf-8") - with pytest.raises(RuntimeError, match="context changed"): - auth_patch.patch(source, tmp_path / "patched.py") - - -def test_codex_runtime_patch_uses_cli_and_forwards_route(tmp_path: Path): - provider = tmp_path / "runtime_provider.py" - provider.write_text(codex_runtime_patch.PROVIDER_BEFORE, encoding="utf-8") - provider_out = tmp_path / "patched/runtime_provider.py" - codex_runtime_patch.patch_provider(provider, provider_out) - assert '"api_mode": "codex_app_server"' in provider_out.read_text() - - session = tmp_path / "codex_app_server_session.py" - session.write_text( - codex_runtime_patch.SESSION_SIGNATURE_BEFORE - + codex_runtime_patch.SESSION_REQUEST_BEFORE, - encoding="utf-8", - ) - session_out = tmp_path / "patched/codex_app_server_session.py" - codex_runtime_patch.patch_session(session, session_out) - session_content = session_out.read_text() - assert 'turn_params["model"] = model' in session_content - assert 'turn_params["effort"] = effort' in session_content - assert '"approvalPolicy": "never"' in session_content - assert '"sandboxPolicy": {"type": "dangerFullAccess"}' in session_content - - turn = tmp_path / "codex_runtime.py" - turn.write_text( - codex_runtime_patch.FALLBACK_CONTEXT_BEFORE - + codex_runtime_patch.TURN_BEFORE, - encoding="utf-8", - ) - turn_out = tmp_path / "patched/codex_runtime.py" - codex_runtime_patch.patch_turn(turn, turn_out) - turn_content = turn_out.read_text() - assert "model=str(getattr(agent" in turn_content - assert "build_cross_provider_codex_prompt" in turn_content - - fallback = tmp_path / "chat_completion_helpers.py" - fallback.write_text( - codex_runtime_patch.FALLBACK_RESOLUTION_BEFORE, - encoding="utf-8", - ) - fallback_out = tmp_path / "patched/chat_completion_helpers.py" - codex_runtime_patch.patch_fallback(fallback, fallback_out) - fallback_content = fallback_out.read_text() - assert 'agent.api_mode = "codex_app_server"' in fallback_content - assert "agent._codex_cross_provider_fallback = True" in fallback_content - - loop = tmp_path / "conversation_loop.py" - loop.write_text( - codex_runtime_patch.FALLBACK_DISPATCH_BEFORE - + codex_runtime_patch.RETRY_FALLBACK_DISPATCH_BEFORE - + codex_runtime_patch.STREAM_RECOVERY_BEFORE, - encoding="utf-8", - ) - loop_out = tmp_path / "patched/conversation_loop.py" - codex_runtime_patch.patch_loop(loop, loop_out) - loop_content = loop_out.read_text() - assert loop_content.count('if agent.api_mode == "codex_app_server"') == 2 - assert "build_cross_provider_codex_prompt" in loop_content - retry_dispatch = loop_content.index( - "Fallback activation happens inside this retry loop" - ) - api_kwargs = loop_content.find("agent._build_api_kwargs", retry_dispatch) - assert api_kwargs == -1 or retry_dispatch < api_kwargs - assert "Provider stream ended before a complete response" in loop_content - assert "_is_transport_stub" in loop_content - assert "rerouting ({truncated_tool_call_retries}/4)" in loop_content - - auxiliary = tmp_path / "auxiliary_client.py" - auxiliary.write_text( - codex_runtime_patch.AUXILIARY_TOKEN_BEFORE, - encoding="utf-8", - ) - auxiliary_out = tmp_path / "patched/auxiliary_client.py" - codex_runtime_patch.patch_auxiliary(auxiliary, auxiliary_out) - auxiliary_content = auxiliary_out.read_text() - assert 'os.environ.get("CODEX_HOME"' in auxiliary_content - assert 'Path(codex_home).expanduser() / "auth.json"' in auxiliary_content - assert "never creates a metered API-key lane" in auxiliary_content - - -def test_agent_mounts_codex_auxiliary_runtime_patch(): - deployment = _agent_deployment() - pod = deployment["spec"]["template"]["spec"] - patch_init = next( - item for item in pod["initContainers"] - if item["name"] == "patch-codex-runtime" - ) - assert patch_init["command"][-2:] == [ - "/opt/hermes/agent/auxiliary_client.py", - "/patched/auxiliary_client.py", - ] - expected_mount = { - "name": "codex-runtime-patch", - "mountPath": "/opt/hermes/agent/auxiliary_client.py", - "subPath": "auxiliary_client.py", - } - containers = {item["name"]: item for item in pod["containers"]} - for name in ("hermes", "terminal"): - assert expected_mount in containers[name]["volumeMounts"] - - -def test_codex_auxiliary_patch_reads_cli_token_without_copying_it( - tmp_path: Path, - monkeypatch, -): - codex_home = tmp_path / ".codex" - codex_home.mkdir() - (codex_home / "auth.json").write_text( - json.dumps({"tokens": {"access_token": "cli-access-token"}}), - encoding="utf-8", - ) - monkeypatch.setenv("CODEX_HOME", str(codex_home)) - - source = tmp_path / "auxiliary_client.py" - source.write_text( - "import json, logging, os, time\n" - "from pathlib import Path\n" - "logger = logging.getLogger(__name__)\n" - "def read_token():\n" - " try:\n" - " raise RuntimeError('Hermes provider store intentionally empty')\n" - + codex_runtime_patch.AUXILIARY_TOKEN_BEFORE, - encoding="utf-8", - ) - destination = tmp_path / "patched/auxiliary_client.py" - codex_runtime_patch.patch_auxiliary(source, destination) - namespace: dict = {} - exec(compile(destination.read_text(), str(destination), "exec"), namespace) - - assert namespace["read_token"]() == "cli-access-token" - - -def test_codex_runtime_patch_fails_closed_on_upstream_drift(tmp_path: Path): - source = tmp_path / "runtime_provider.py" - source.write_text("def changed():\n pass\n", encoding="utf-8") - with pytest.raises(RuntimeError, match="context changed"): - codex_runtime_patch.patch_provider(source, tmp_path / "patched.py") - - -def test_codex_runtime_migration_uses_owner_unsafe_mode(tmp_path: Path): - config = tmp_path / "config.yaml" - config.write_text("model: {}\n", encoding="utf-8") - codex_home = tmp_path / ".codex" - calls = [] - - class Report: - errors = [] - - @staticmethod - def summary(): - return "configured" - - def migrate(value, **kwargs): - calls.append((value, kwargs)) - return Report() - - client_config.configure_codex_runtime(config, migrate, codex_home) - - assert calls[0][1]["default_permission_profile"] is None - assert calls[0][1]["codex_home"] == codex_home - content = (codex_home / "config.toml").read_text(encoding="utf-8") - assert 'approval_policy = "never"' in content - assert 'sandbox_mode = "danger-full-access"' in content - assert "default_permissions" not in content - - -def test_codex_owner_permissions_replace_stale_profile(tmp_path: Path): - config = tmp_path / "config.toml" - config.write_text( - 'default_permissions = ":danger-no-sandbox"\n\n[features]\nhooks = true\n', - encoding="utf-8", - ) - - client_config.configure_codex_owner_permissions(config) - client_config.configure_codex_owner_permissions(config) - - content = config.read_text(encoding="utf-8") - assert content.count(client_config.OWNER_PERMISSIONS_BEGIN) == 1 - assert content.count('approval_policy = "never"') == 1 - assert content.count('sandbox_mode = "danger-full-access"') == 1 - assert "default_permissions" not in content - assert "[features]\nhooks = true" in content - - -def test_tui_gateway_patch_extends_and_bounds_agent_startup(tmp_path: Path): - source = tmp_path / "server.py" - destination = tmp_path / "patched/server.py" - source.write_text( - "import os\n\n" + tui_gateway_patch.BEFORE + "\ndef unchanged():\n pass\n", - encoding="utf-8", - ) - - tui_gateway_patch.patch(source, destination) - - content = destination.read_text(encoding="utf-8") - assert "HERMES_TUI_AGENT_INIT_TIMEOUT_S" in content - assert 'configured = 180.0' in content - assert "return max(30.0, min(configured, 900.0))" in content - assert "timeout: float | None = None" in content - assert "ready.wait(timeout=wait_timeout)" in content - assert "def unchanged():" in content - - -def test_tui_gateway_patch_fails_closed_on_upstream_drift(tmp_path: Path): - source = tmp_path / "server.py" - source.write_text("def changed():\n pass\n", encoding="utf-8") - with pytest.raises(RuntimeError, match="context changed"): - tui_gateway_patch.patch(source, tmp_path / "patched.py") - - -def test_ttyd_clipboard_and_reconnect_patch_remain_enabled(): - source = '' - content = ttyd_patch.patch_html(source) - assert 'id="atlas-ttyd-clipboard"' in content - assert "navigator.clipboard.writeText(text)" in content - assert "class AtlasRecoveringWebSocket" in content - assert "window.location.reload()" in content - assert "event.stopImmediatePropagation()" in content - - -def test_ttyd_patch_fails_closed_on_upstream_drift(): - with pytest.raises(RuntimeError, match="context changed"): - ttyd_patch.patch_html("changed") diff --git a/testing/tests/test_hermes_cli_lanes_access.py b/testing/tests/test_hermes_cli_lanes_access.py new file mode 100644 index 00000000..01d9f4bf --- /dev/null +++ b/testing/tests/test_hermes_cli_lanes_access.py @@ -0,0 +1,378 @@ +"""Network, OAuth, Kubernetes, and SSH access contracts for Hermes CLI lanes.""" + +from __future__ import annotations + + +import yaml + +from testing.tests.test_hermes_cli_lanes_support import ( + FLUX_HERMES, + HERMES, + KEYCLOAK, + ROOT, + _agent_deployment, + _services, +) + + +def test_broker_services_survive_sibling_container_readiness_loss(): + services = _services() + + for name in ( + "hermes-image-broker", + "hermes-codex-broker", + "hermes-local-image", + "hermes-claude-broker", + ): + assert services[name]["spec"]["publishNotReadyAddresses"] is True + + +def test_agent_dashboard_reconnects_all_transient_websockets(): + dockerfile = (HERMES.parents[1] / "dockerfiles/Dockerfile.hermes-agent").read_text( + encoding="utf-8" + ) + assert "eventsRetryAttempt.current" in dockerfile + assert "if (!unmounting) setVersion((v) => v + 1);" in dockerfile + assert "events feed rejected (${ev.code}) — reload the page" in dockerfile + assert 'url = await api.buildWsUrl("/api/pty", params);' in dockerfile + assert 'url = await buildWsUrl("/api/events", { channel });' in dockerfile + assert dockerfile.count("' await api.getSessions(1, 0,") == 2 + assert ".then(() => gw.connect())" in dockerfile + assert 'api.getSessions(1, 0, profile ?? "")' in dockerfile + assert "dashboard token rotated by a server restart" in dockerfile + + +def test_agent_refreshes_routes_after_restoring_cli_logins(): + deployment = _agent_deployment() + init_containers = { + item["name"]: item + for item in deployment["spec"]["template"]["spec"]["initContainers"] + } + configure = init_containers["configure-agent-clients"] + command = configure["command"][-1] + assert "configure_agent_clients.py" in command + assert command.index("configure_agent_clients.py") < command.index( + "hermes_coordinator.py --once" + ) + env = {item["name"]: item["value"] for item in configure["env"]} + assert env["HERMES_AUTH_FILE"] == "/runtime-access/hermes-auth.json" + assert env["PYTHONPATH"] == "/opt/hermes" + for name in ("bootstrap-coordinator", "configure-agent-clients"): + route_env = { + item["name"]: item["value"] for item in init_containers[name]["env"] + } + assert route_env["CODEX_HOME"] == "/runtime-access/codex" + assert route_env["CLAUDE_CONFIG_DIR"] == "/runtime-access/claude" + assert "/opt/data/tools/bin" in route_env["PATH"] + + assert "patch-web-session-activity" in init_containers + web_patch = init_containers["patch-web-session-activity"] + assert "/opt/coordinator/patch_web_session_activity.py" in web_patch["command"] + + containers = { + item["name"]: item + for item in deployment["spec"]["template"]["spec"]["containers"] + } + steward_env = { + item["name"]: item["value"] for item in containers["model-steward"]["env"] + } + assert steward_env["CODEX_HOME"] == "/runtime-access/codex" + assert steward_env["CLAUDE_CONFIG_DIR"] == "/runtime-access/claude" + assert "/opt/data/tools/bin" in steward_env["PATH"] + hermes_mounts = { + (item["name"], item["mountPath"], item.get("subPath")) + for item in containers["hermes"]["volumeMounts"] + } + assert ( + "web-server-patch", + "/opt/hermes/hermes_cli/web_server.py", + "web_server.py", + ) in hermes_mounts + + +def test_flux_health_checks_follow_the_owner_oauth_sidecar(): + flux = yaml.safe_load(FLUX_HERMES.read_text()) + checks = {(item["kind"], item["name"]) for item in flux["spec"]["healthChecks"]} + assert ("Deployment", "hermes-agent") in checks + assert ("DaemonSet", "hermes-node-ssh-access") in checks + assert ("Deployment", "oauth2-proxy-hermes-agent") not in checks + + +def test_agent_auth_is_bstein_group_and_email_bounded(): + deployment = _agent_deployment() + oauth = next( + item + for item in deployment["spec"]["template"]["spec"]["containers"] + if item["name"] == "oauth2-proxy" + ) + args = oauth["args"] + assert "--user-id-claim=sub" in args + assert "--oidc-groups-claim=groups" in args + assert "--allowed-group=/hermes-owner" in args + assert "--authenticated-emails-file=/etc/oauth2-proxy/allowed-emails" in args + script = (KEYCLOAK / "scripts/hermes_access_oidc_ensure.sh").read_text() + assert 'group_name="hermes-owner"' in script + assert "username=bstein&exact=true" in script + assert '"full.path":"true"' in script + + +def test_agent_network_boundary_allows_only_authenticated_and_metrics_surfaces(): + documents = [ + item + for item in yaml.safe_load_all((HERMES / "networkpolicy.yaml").read_text()) + if item + ] + isolation = next( + item + for item in documents + if item.get("metadata", {}).get("name") == "hermes-agent-isolation" + ) + assert isolation["spec"]["ingress"] == [ + { + "from": [ + { + "namespaceSelector": { + "matchLabels": {"kubernetes.io/metadata.name": "traefik"} + }, + "podSelector": { + "matchLabels": {"app.kubernetes.io/name": "traefik"} + }, + } + ], + "ports": [{"protocol": "TCP", "port": 4180}], + }, + { + "from": [{"podSelector": {"matchLabels": {"app": "hermes-chat-tenant"}}}], + "ports": [ + {"protocol": "TCP", "port": 9002}, + {"protocol": "TCP", "port": 9003}, + ], + }, + { + "from": [{"podSelector": {"matchLabels": {"app": "hermes-switchyard"}}}], + "ports": [ + {"protocol": "TCP", "port": 9003}, + {"protocol": "TCP", "port": 9006}, + ], + }, + { + "from": [ + { + "namespaceSelector": { + "matchLabels": {"kubernetes.io/metadata.name": "monitoring"} + }, + "podSelector": {"matchLabels": {"app": "server"}}, + } + ], + "ports": [{"protocol": "TCP", "port": 9010}], + }, + ] + egress = isolation["spec"]["egress"] + assert any( + rule.get("ports") == [{"protocol": "TCP", "port": 9081}] for rule in egress + ) + assert "hermes-scm" in yaml.safe_dump(egress) + assert "gitea" in yaml.safe_dump(egress) + assert egress != [{}] + + +def test_owner_agent_has_scoped_read_only_kubernetes_context(): + config = yaml.safe_load((HERMES / "agent-kubeconfig.yaml").read_text()) + assert config["current-context"] == "atlas-observer" + assert config["contexts"][0]["context"]["namespace"] == "default" + rbac_path = ROOT / "services/hermes-observer-rbac/rbac.yaml" + documents = [item for item in yaml.safe_load_all(rbac_path.read_text()) if item] + binding = next(item for item in documents if item["kind"] == "ClusterRoleBinding") + assert binding["roleRef"] == { + "apiGroup": "rbac.authorization.k8s.io", + "kind": "ClusterRole", + "name": "hermes-agent-cluster-observer-v2", + } + assert binding["subjects"] == [ + {"kind": "ServiceAccount", "name": "hermes-agent", "namespace": "hermes"} + ] + + +def test_owner_agent_has_pinned_dedicated_node_ssh_access(): + deployment = _agent_deployment() + annotations = deployment["spec"]["template"]["metadata"]["annotations"] + assert ( + annotations["vault.hashicorp.com/agent-inject-secret-node-ssh-private-key"] + == "kv/data/atlas/hermes/developer-ssh" + ) + assert ( + annotations["vault.hashicorp.com/agent-inject-secret-node-ssh-config"] + == "kv/data/atlas/hermes/developer-ssh" + ) + assert ( + annotations["vault.hashicorp.com/agent-inject-secret-node-ssh-known-hosts"] + == "kv/data/atlas/hermes/developer-ssh" + ) + + init = next( + item + for item in deployment["spec"]["template"]["spec"]["initContainers"] + if item["name"] == "init-config" + ) + command = init["command"][2] + assert "ln -s /runtime-access/node-ssh-config /opt/data/home/.ssh/config" in command + assert ( + "ln -s /runtime-access/node-ssh-known-hosts /opt/data/home/.ssh/known_hosts" + in command + ) + assert "ln -s home/.ssh /opt/data/.ssh" in command + assert "chmod 0700 /opt/data/home/.ssh" in command + assert ( + "ln -s /runtime-access/node-ssh-private-key " + "/opt/data/home/.ssh/id_ed25519_atlas_nodes" + ) in command + + config = yaml.safe_load((HERMES / "agent-configmap.yaml").read_text())["data"] + assert "ssh_config" not in config + assert "ssh_known_hosts" not in config + assert "ssh-ed25519" not in (HERMES / "agent-configmap.yaml").read_text() + + resources = yaml.safe_load((HERMES / "kustomization.yaml").read_text())["resources"] + assert "node-ssh-access.yaml" in resources + access = [ + item + for item in yaml.safe_load_all((HERMES / "node-ssh-access.yaml").read_text()) + if item + ] + service_account = next(item for item in access if item["kind"] == "ServiceAccount") + assert service_account["metadata"]["name"] == "hermes-node-ssh-access" + provider = next(item for item in access if item["kind"] == "SecretProviderClass") + assert provider["metadata"]["name"] == "hermes-node-ssh-access" + assert provider["spec"]["provider"] == "vault" + parameters = provider["spec"]["parameters"] + assert parameters["roleName"] == "hermes-node-ssh" + assert 'secretPath: "kv/data/atlas/hermes/developer-ssh"' in parameters["objects"] + assert 'secretKey: "public_key"' in parameters["objects"] + daemonset = next(item for item in access if item["kind"] == "DaemonSet") + pod = daemonset["spec"]["template"]["spec"] + assert pod["serviceAccountName"] == "hermes-node-ssh-access" + assert pod["automountServiceAccountToken"] is True + host_home = next(item for item in pod["volumes"] if item["name"] == "host-home") + assert host_home["hostPath"] == {"path": "/home", "type": "Directory"} + vault_secrets = next( + item for item in pod["volumes"] if item["name"] == "vault-secrets" + ) + assert vault_secrets["csi"]["driver"] == "secrets-store.csi.k8s.io" + assert vault_secrets["csi"]["volumeAttributes"] == { + "secretProviderClass": "hermes-node-ssh-access" + } + reconciler = pod["containers"][0]["args"][0] + assert "/opt/node-hardener/node_account_hardening.py" in reconciler + assert "--public-key-file /vault/secrets/node-ssh-public-key" in reconciler + assert "sleep 300" in reconciler + host_etc = next(item for item in pod["volumes"] if item["name"] == "host-etc") + assert host_etc["hostPath"] == {"path": "/etc", "type": "Directory"} + hardener = (HERMES / "scripts/node_account_hardening.py").read_text() + assert 'ACCOUNT = "hermes-agent"' in hardener + assert 'LEGACY_ACCOUNTS = ("atlas", "oceanus")' in hardener + assert "ACCOUNT_UID = 1200" in hardener + assert "ACCOUNT_GID = 1200" in hardener + + +def test_owner_agent_tracks_no_ssh_identity_or_host_key_material(): + """Vault references may be tracked; SSH identities and trust data may not.""" + forbidden = ( + "BEGIN OPENSSH PRIVATE KEY", + "ssh-ed25519 AAAA", + "ssh-rsa AAAA", + "IdentityFile ", + "UserKnownHostsFile ", + "StrictHostKeyChecking ", + "ssh_config:", + "ssh_known_hosts:", + ) + text_suffixes = { + ".conf", + ".json", + ".md", + ".py", + ".sh", + ".toml", + ".yaml", + ".yml", + } + tracked = "\n".join( + path.read_text(encoding="utf-8") + for path in HERMES.rglob("*") + if path.is_file() and path.suffix in text_suffixes + ) + for marker in forbidden: + assert marker not in tracked + + +def test_switchyard_has_a_dedicated_non_owner_identity_and_read_only_catalog(): + """Routing must not inherit the owner agent's cluster-admin capability.""" + service_accounts = [ + item + for item in yaml.safe_load_all( + (HERMES / "vault-serviceaccount.yaml").read_text() + ) + if item + ] + assert any( + item["kind"] == "ServiceAccount" + and item["metadata"]["name"] == "hermes-switchyard" + for item in service_accounts + ) + + switchyard = yaml.safe_load((HERMES / "switchyard-deployment.yaml").read_text()) + switchyard_pod = switchyard["spec"]["template"]["spec"] + assert switchyard_pod["serviceAccountName"] == "hermes-switchyard" + agent_pod = _agent_deployment()["spec"]["template"]["spec"] + for pod, container_name in ( + (switchyard_pod, "worker-route-broker"), + (agent_pod, "claude-broker"), + ): + container = next( + item for item in pod["containers"] if item["name"] == container_name + ) + catalog = next( + item + for item in container["volumeMounts"] + if item["mountPath"] == "/routing-catalog" + ) + assert catalog["readOnly"] is True + + rbac = [ + item + for item in yaml.safe_load_all( + (ROOT / "services/hermes-observer-rbac/rbac.yaml").read_text() + ) + if item + ] + binding = next(item for item in rbac if item["kind"] == "ClusterRoleBinding") + assert binding["subjects"] == [ + {"kind": "ServiceAccount", "name": "hermes-agent", "namespace": "hermes"} + ] + + +def test_switchyard_active_state_uses_a_relocatable_rwx_claim(): + """A stale node attachment must not strand the routing authority.""" + claims = [ + item + for item in yaml.safe_load_all((HERMES / "switchyard-pvc.yaml").read_text()) + if item + ] + active_claim = next( + item + for item in claims + if item["metadata"]["name"] == "hermes-switchyard-state-rwx" + ) + assert active_claim["spec"]["accessModes"] == ["ReadWriteMany"] + + deployment = yaml.safe_load((HERMES / "switchyard-deployment.yaml").read_text()) + strategy = deployment["spec"]["strategy"] + assert strategy == { + "type": "RollingUpdate", + "rollingUpdate": {"maxSurge": 1, "maxUnavailable": 0}, + } + pod = deployment["spec"]["template"]["spec"] + state = next(item for item in pod["volumes"] if item["name"] == "state") + assert ( + state["persistentVolumeClaim"]["claimName"] == active_claim["metadata"]["name"] + ) diff --git a/testing/tests/test_hermes_cli_lanes_configuration.py b/testing/tests/test_hermes_cli_lanes_configuration.py new file mode 100644 index 00000000..bf1807f4 --- /dev/null +++ b/testing/tests/test_hermes_cli_lanes_configuration.py @@ -0,0 +1,444 @@ +"""Agent configuration and dashboard contracts for Hermes CLI lanes.""" + +from __future__ import annotations + +import json +import sys +from contextlib import nullcontext +from pathlib import Path +from types import SimpleNamespace + +import pytest +import yaml + +from testing.tests.test_hermes_cli_lanes_support import ( + HERMES, + _agent_deployment, + client_config, + lanes, + migration, + policy, +) + + +def test_workspace_preparation_failure_durably_blocks_the_claim( + tmp_path: Path, monkeypatch +): + task = SimpleNamespace(id="t_bad_worktree", current_run_id=7, assignee="cli-auto") + calls = [] + + class Connection: + def close(self): + return None + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + get_task=lambda _conn, _task_id: task, + worker_log_path=lambda _task_id, board: tmp_path / "worker.log", + _resolve_worktree_workspace=lambda _task, board: (_ for _ in ()).throw( + ValueError("not a Git repository") + ), + block_task=lambda *_args, **kwargs: calls.append(kwargs), + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + monkeypatch.setattr( + lanes, "state_path", lambda _board, _task_id: tmp_path / "state.json" + ) + + lanes.execute_claim("cassandra", "t_bad_worktree") + + assert calls[0]["kind"] == "capability" + assert calls[0]["expected_run_id"] == 7 + assert "not a Git repository" in calls[0]["reason"] + + +def test_artifacts_cannot_escape_the_task_worktree(tmp_path: Path): + workspace = tmp_path / "workspace" + workspace.mkdir() + inside = workspace / "report.json" + outside = tmp_path / "auth.json" + inside.write_text("{}\n", encoding="utf-8") + outside.write_text("secret\n", encoding="utf-8") + + assert lanes.workspace_artifacts( + workspace, + ["report.json", str(outside), "missing.json"], + ) == [str(inside)] + + +def test_restart_provider_change_includes_explicit_workspace_handoff( + tmp_path: Path, monkeypatch +): + task = SimpleNamespace( + id="t_resume", + current_run_id=9, + assignee="cli-auto", + max_runtime_seconds=60, + ) + state_file = tmp_path / "state.json" + state_file.write_text( + json.dumps({"current_route": {"provider": "claude"}}), + encoding="utf-8", + ) + (tmp_path / "worker.log").write_text("prior provider evidence", encoding="utf-8") + prompts = [] + + class Connection: + def close(self): + return None + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + get_task=lambda _conn, _task_id: task, + worker_log_path=lambda _task_id, board: tmp_path / "worker.log", + _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_resume"), + set_branch_name=lambda *_args: None, + set_workspace_path=lambda *_args: None, + build_worker_context=lambda *_args: "resume objective", + add_comment=lambda *_args: None, + complete_task=lambda *_args, **_kwargs: None, + block_task=lambda *_args, **_kwargs: None, + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + monkeypatch.setattr(lanes, "state_path", lambda _board, _task_id: state_file) + monkeypatch.setattr( + lanes, + "select_route", + lambda *_args, **_kwargs: lanes.Route( + "codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, () + ), + ) + monkeypatch.setattr( + lanes, + "git_handoff", + lambda _workspace, output: f"HANDOFF:{output}", + ) + monkeypatch.setattr( + lanes, + "run_provider", + lambda _route, prompt, *_args, **_kwargs: ( + prompts.append(prompt) + or lanes.ProcessResult( + 0, + "", + { + "status": "completed", + "summary": "done", + "changed_files": [], + "tests_run": [], + "artifacts": [], + "blockers": [], + }, + False, + ) + ), + ) + + lanes.execute_claim("cassandra", "t_resume") + + assert "HANDOFF:prior provider evidence" in prompts[0] + + +def test_provider_commands_are_structured_unattended_and_capped(tmp_path: Path): + route = lanes.Route( + "codex", "gpt-5.6-sol", "xhigh", "codex-xhigh", "jetson", "vote", 1, () + ) + command = lanes._codex_command( + route, "Work.", tmp_path, {}, tmp_path / "result.json" + ) + assert "--dangerously-bypass-approvals-and-sandbox" in command + assert "--json" in command + assert "--output-schema" in command + assert 'model_reasoning_effort="xhigh"' in command + + claude_state = {"claude_session_id": "13864642-2985-4f91-bef5-53f145f878e8"} + claude = lanes._claude_command( + lanes.Route( + "claude", "claude-opus-5", "xhigh", "claude-xhigh", "jetson", "vote", 1, () + ), + "Review.", + claude_state, + False, + ) + assert "--dangerously-skip-permissions" in claude + assert "--output-format" in claude and "stream-json" in claude + assert "--json-schema" in claude + assert "--disallowedTools" in claude + assert "Bash(kubectl apply *)" not in claude + assert "Bash(flux reconcile *)" not in claude + assert "max" not in claude + + +def test_worker_contract_separates_review_findings_from_task_blockers(tmp_path: Path): + prompt = lanes.build_prompt("Review the change.", tmp_path) + + assert "put defects and risks in findings" in prompt + assert "blockers array must be empty whenever status is completed" in prompt + assert "findings" in lanes.RESULT_SCHEMA["properties"] + assert set(lanes.RESULT_SCHEMA["required"]) == set( + lanes.RESULT_SCHEMA["properties"] + ) + assert ( + "assigned task itself" + in lanes.RESULT_SCHEMA["properties"]["blockers"]["description"] + ) + + +@pytest.mark.parametrize( + "command", + [ + "git push --force origin main", + "git reset --hard HEAD~1", + "git clean -fd", + ], +) +def test_claude_pretool_hook_blocks_hard_denies(command: str): + assert policy.denial_reason(command) + + +def test_claude_pretool_hook_allows_normal_engineering(): + assert policy.denial_reason("pytest -q testing/tests") is None + assert policy.denial_reason("git push origin feature/hermes") is None + assert policy.denial_reason("kubectl delete pod -n cassandra stuck-worker") is None + assert policy.denial_reason("flux reconcile kustomization hermes") is None + assert policy.denial_reason("vault kv get kv/atlas/hermes") is None + + +def test_claude_settings_preserve_state_and_install_three_guardrail_layers( + tmp_path: Path, +): + state = tmp_path / ".claude.json" + settings = tmp_path / "settings.json" + state.write_text('{"promptQueueUseCount": 4}\n', encoding="utf-8") + settings.write_text( + json.dumps( + { + "theme": "dark", + "permissions": { + "deny": [ + "Bash(kubectl apply *)", + "Bash(flux reconcile *)", + "Bash(vault kv *)", + "Bash(custom-owner-rule *)", + ] + }, + } + ) + + "\n", + encoding="utf-8", + ) + + client_config.configure_claude_state(state) + client_config.configure_claude_settings(settings) + + state_value = json.loads(state.read_text()) + settings_value = json.loads(settings.read_text()) + assert state_value["promptQueueUseCount"] == 4 + assert state_value["bypassPermissionsModeAccepted"] is True + assert settings_value["theme"] == "dark" + assert "Bash(git reset --hard *)" in settings_value["permissions"]["deny"] + assert "Bash(custom-owner-rule *)" in settings_value["permissions"]["deny"] + assert "Bash(kubectl apply *)" not in settings_value["permissions"]["deny"] + assert "Bash(flux reconcile *)" not in settings_value["permissions"]["deny"] + assert "Bash(vault kv *)" not in settings_value["permissions"]["deny"] + hook = settings_value["hooks"]["PreToolUse"][0]["hooks"][0] + assert "claude_command_policy.py" in hook["command"] + + +def test_legacy_state_is_archived_without_removing_provider_transcripts(tmp_path: Path): + session = tmp_path / "home/.config/herdr/session.json" + session.parent.mkdir(parents=True) + session.write_text( + '{"agents":[{"agent":"claude","session_id":"abc"}]}', encoding="utf-8" + ) + binary = tmp_path / "tools/bin/herdr" + binary.parent.mkdir(parents=True) + binary.write_text("legacy", encoding="utf-8") + (tmp_path / "home/.claude").mkdir() + (tmp_path / "home/.codex").mkdir() + + archive = migration.archive_legacy_state(tmp_path) + + value = json.loads(archive.read_text()) + assert value["legacy_session"]["agents"][0]["session_id"] == "abc" + assert (tmp_path / "home/.claude").is_dir() + assert (tmp_path / "home/.codex").is_dir() + assert not binary.exists() + assert not (tmp_path / "home/.config/herdr").exists() + + +def test_agent_uses_one_native_kanban_control_plane(): + configmap = yaml.safe_load((HERMES / "agent-configmap.yaml").read_text()) + config = yaml.safe_load(configmap["data"]["config.yaml"]) + assert config["model"] == { + "provider": "atlas-switchyard", + "default": "atlas/auto/balanced", + "model": "atlas/auto/balanced", + } + assert config["kanban"]["dispatch_in_gateway"] is True + assert config["kanban"]["default_assignee"] == "cli-auto" + assert config["plugins"]["enabled"] == ["auto-router"] + + deployment = _agent_deployment() + pod = deployment["spec"]["template"]["spec"] + assert pod["enableServiceLinks"] is False + names = {item["name"] for item in pod["containers"]} + assert "cli-lane-runner" in names + assert "terminal" in names + assert not any("herdr" in name for name in names) + rendered = (HERMES / "agent-deployment.yaml").read_text() + assert "herdr server" not in rendered + assert "herdr-dispatch" not in rendered + + +def test_cli_lane_reserves_cpu_headroom_for_ui_and_auth(): + deployment = _agent_deployment() + containers = { + item["name"]: item + for item in deployment["spec"]["template"]["spec"]["containers"] + } + lane = containers["cli-lane-runner"] + environment = {item["name"]: item["value"] for item in lane["env"]} + + assert environment["HERMES_CLI_LANE_CONCURRENCY"] == "2" + assert lane["resources"] == { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "2", "memory": "6Gi"}, + } + + +def test_agent_avoids_unhealthy_nodes_and_fits_its_remaining_capacity(): + """Placement correction: keep the agent off nodes that cannot hold it. + + titan-04 is cordoned after repeated kernel undervoltage and kubelet + failure, and titan-19 was probe/Longhorn unstable under worker load, so + both must join the existing hard exclusions. That leaves titan-05 as the + healthy candidate, which is tight enough on requested CPU that the main + container has to give back 50m to schedule there. + """ + pod = _agent_deployment()["spec"]["template"]["spec"] + hostnames = next( + item + for item in pod["affinity"]["nodeAffinity"][ + "requiredDuringSchedulingIgnoredDuringExecution" + ]["nodeSelectorTerms"][0]["matchExpressions"] + if item["key"] == "kubernetes.io/hostname" + ) + + assert hostnames["operator"] == "NotIn" + assert set(hostnames["values"]) >= {"titan-04", "titan-19"} + + hermes = next(item for item in pod["containers"] if item["name"] == "hermes") + assert hermes["resources"]["requests"]["cpu"] == "300m" + + +def test_agent_root_is_stock_dashboard_and_terminal_is_a_separate_path(): + deployment = _agent_deployment() + pod = deployment["spec"]["template"]["spec"] + containers = {item["name"]: item for item in pod["containers"]} + assert "webui" not in containers + + assert "dashboard" not in containers + hermes = containers["hermes"] + hermes_env = {item["name"]: item["value"] for item in hermes["env"]} + assert hermes_env["HERMES_STREAM_STALE_TIMEOUT"] == "600" + assert hermes_env["HERMES_API_CALL_STALE_TIMEOUT"] == "600" + assert hermes["command"] == ["/bin/sh", "-ec"] + startup = hermes["args"][0] + assert ". /opt/data/.env" in startup + assert "exec /init /opt/hermes/docker/main-wrapper.sh gateway run" in startup + hermes_env = {item["name"]: item["value"] for item in hermes["env"]} + assert hermes_env["HERMES_DASHBOARD"] == "1" + assert hermes_env["HERMES_DASHBOARD_HOST"] == "127.0.0.1" + assert hermes_env["HERMES_DASHBOARD_PORT"] == "9119" + assert hermes_env["HERMES_TUI_AGENT_INIT_TIMEOUT_S"] == "180" + assert hermes["securityContext"]["runAsUser"] == 0 + assert hermes["securityContext"]["runAsGroup"] == 0 + for probe_name in ("startupProbe", "readinessProbe", "livenessProbe"): + probe = hermes[probe_name] + assert probe["exec"]["command"] == [ + "curl", + "-fsS", + "http://127.0.0.1:9119/api/status", + ] + + terminal = containers["terminal"] + command = terminal["args"][0] + assert "--base-path /terminal" in command + assert "--check-origin" not in command + assert "/usr/bin/tmux new-session -A" in command + assert "--continue" in command + assert "--yolo" in command + terminal_env = {item["name"]: item["value"] for item in terminal["env"]} + assert terminal_env["HERMES_TUI_AGENT_INIT_TIMEOUT_S"] == "180" + + claude_broker = containers["claude-broker"] + claude_env = {item["name"]: item["value"] for item in claude_broker["env"]} + assert claude_env["HERMES_CLAUDE_BROKER_CONCURRENCY"] == "2" + assert claude_broker["readinessProbe"]["tcpSocket"] == {"port": "claude-broker"} + assert claude_broker["livenessProbe"]["tcpSocket"] == {"port": "claude-broker"} + + args = containers["oauth2-proxy"]["args"] + terminal_upstream = "--upstream=http://127.0.0.1:7681/terminal/" + dashboard_upstream = "--upstream=http://127.0.0.1:9119/" + assert terminal_upstream in args + assert dashboard_upstream in args + assert args.index(terminal_upstream) < args.index(dashboard_upstream) + assert "--pass-host-header=false" in args + assert "--cookie-refresh=19m" in args + assert "--session-store-type=redis" in args + assert any( + arg.startswith("--redis-connection-url=redis://hermes-oauth-sessions.") + for arg in args + ) + + patch_init = next( + item for item in pod["initContainers"] if item["name"] == "patch-tui-gateway" + ) + assert patch_init["command"][-1] == "/patched/server.py" + for name in ("hermes", "terminal"): + mounts = containers[name]["volumeMounts"] + assert { + "name": "tui-gateway-patch", + "mountPath": "/opt/hermes/tui_gateway/server.py", + "subPath": "server.py", + } in mounts + + ingress_documents = [ + item + for item in yaml.safe_load_all((HERMES / "agent-ingress.yaml").read_text()) + if item + ] + middlewares = { + item["metadata"]["name"]: item + for item in ingress_documents + if item["kind"] == "Middleware" + } + assert middlewares["hermes-agent-terminal-slash"]["spec"]["redirectRegex"][ + "replacement" + ].endswith("/terminal/") + assert ( + middlewares["hermes-agent-stock-dashboard-headers"]["spec"]["headers"][ + "customRequestHeaders" + ]["Origin"] + == "http://127.0.0.1:9119" + ) + ingresses = { + item["metadata"]["name"]: item + for item in ingress_documents + if item["kind"] == "Ingress" + } + assert ( + ingresses["hermes-agent-dashboard"]["metadata"]["annotations"][ + "traefik.ingress.kubernetes.io/router.middlewares" + ] + == "hermes-hermes-agent-stock-dashboard-headers@kubernetescrd" + ) + assert ( + ingresses["hermes-agent-terminal"]["metadata"]["annotations"][ + "traefik.ingress.kubernetes.io/router.middlewares" + ] + == "hermes-hermes-agent-terminal-slash@kubernetescrd" + ) diff --git a/testing/tests/test_hermes_cli_lanes_kanban.py b/testing/tests/test_hermes_cli_lanes_kanban.py new file mode 100644 index 00000000..279b74c5 --- /dev/null +++ b/testing/tests/test_hermes_cli_lanes_kanban.py @@ -0,0 +1,392 @@ +"""Kanban claim and lifecycle contracts for Hermes CLI lanes.""" + +from __future__ import annotations + +import sys +from contextlib import nullcontext +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from testing.tests.test_hermes_cli_lanes_support import ( + lanes, +) + + +def test_unassigned_ready_task_is_persistently_routed_to_auto_lane(monkeypatch): + task = SimpleNamespace(id="t_auto", assignee=None, status="ready") + assigned = [] + + class Connection: + def close(self): + return None + + def assign_task(_conn, task_id, profile): + assigned.append((task_id, profile)) + task.assignee = profile + return True + + fake_db = SimpleNamespace( + list_boards=lambda include_archived=False: [{"slug": "cassandra"}], + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + recompute_ready=lambda _conn: None, + list_tasks=lambda _conn: [task], + assign_task=assign_task, + get_task=lambda _conn, _task_id: task, + claim_task=lambda _conn, _task_id, **_kwargs: task, + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + + assert lanes.claim_ready(set(), 1) == [("cassandra", "t_auto")] + assert assigned == [("t_auto", "cli-auto")] + + +def test_corrupt_board_is_quarantined_without_stopping_healthy_lanes( + monkeypatch, capsys +): + class CorruptBoardError(Exception): + pass + + task = SimpleNamespace(id="t_healthy", assignee="cli-auto", status="ready") + + class Connection: + def close(self): + return None + + def connect(*, board): + if board == "cassandra": + raise CorruptBoardError("integrity_check failed") + return Connection() + + fake_db = SimpleNamespace( + KanbanDbCorruptError=CorruptBoardError, + list_boards=lambda include_archived=False: [ + {"slug": "cassandra"}, + {"slug": "healthy"}, + ], + scoped_current_board=lambda _board: nullcontext(), + connect=connect, + recompute_ready=lambda _conn: None, + list_tasks=lambda _conn: [task], + claim_task=lambda _conn, _task_id, **_kwargs: task, + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + lanes.BOARD_CORRUPTION_ERRORS.clear() + + assert lanes.claim_ready(set(), 1) == [("healthy", "t_healthy")] + assert "temporarily skipping Kanban board 'cassandra'" in capsys.readouterr().err + + +def test_transient_board_scan_failure_does_not_stop_healthy_lanes(monkeypatch, capsys): + task = SimpleNamespace(id="t_healthy", assignee="cli-auto", status="ready") + + class Connection: + def __init__(self, board): + self.board = board + + def close(self): + return None + + def recompute_ready(connection): + if connection.board == "cassandra": + raise lanes.sqlite3.OperationalError("disk I/O error") + + fake_db = SimpleNamespace( + list_boards=lambda include_archived=False: [ + {"slug": "cassandra"}, + {"slug": "healthy"}, + ], + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(board), + recompute_ready=recompute_ready, + list_tasks=lambda _conn: [task], + claim_task=lambda _conn, _task_id, **_kwargs: task, + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + lanes.BOARD_CORRUPTION_ERRORS.clear() + + assert lanes.claim_ready(set(), 1) == [("healthy", "t_healthy")] + error = capsys.readouterr().err + assert "temporarily skipping Kanban board 'cassandra'" in error + assert "storage OperationalError: disk I/O error" in error + + +def test_board_call_retries_storage_faults_on_fresh_connections(): + connections = [] + + class Connection: + def __init__(self): + self.closed = False + + def close(self): + self.closed = True + + def connect(*, board): + assert board == "cassandra" + connection = Connection() + connections.append(connection) + return connection + + attempts = [] + + def operation(_connection): + attempts.append(1) + if len(attempts) < 3: + raise lanes.sqlite3.OperationalError("disk I/O error") + return "healthy" + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=connect, + ) + lanes.BOARD_CORRUPTION_ERRORS.clear() + + assert lanes._board_call(fake_db, "cassandra", operation) == "healthy" + assert len(connections) == 3 + assert all(connection.closed for connection in connections) + + +@pytest.mark.parametrize( + ("result", "expected_action"), + [ + (lanes.ProcessResult(0, "plain text only", None, False), "block"), + ( + lanes.ProcessResult( + 0, + "", + { + "status": "completed", + "summary": "done", + "changed_files": ["src/a.py"], + "tests_run": ["pytest -q"], + "artifacts": ["reports/result.json"], + "blockers": [], + }, + False, + ), + "complete", + ), + ( + lanes.ProcessResult( + 0, + "", + { + "status": "completed", + "summary": "The full test suite is still running.", + "changed_files": ["src/a.py"], + "tests_run": ["pytest -q — in progress"], + "artifacts": [], + "blockers": [], + }, + False, + ), + "block", + ), + ], +) +def test_claim_requires_structured_evidence_and_surfaces_artifacts( + tmp_path: Path, + monkeypatch, + result, + expected_action, +): + task = SimpleNamespace( + id="t_worker", + current_run_id=4, + assignee="cli-auto", + max_runtime_seconds=60, + ) + calls = [] + heartbeats = [] + connections = [] + artifact = tmp_path / "reports/result.json" + artifact.parent.mkdir() + artifact.write_text("{}\n", encoding="utf-8") + + class Connection: + def __init__(self): + self.closed = False + + def close(self): + self.closed = True + + def connect(*, board): + assert board == "cassandra" + connection = Connection() + connections.append(connection) + return connection + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=connect, + get_task=lambda _conn, _task_id: task, + worker_log_path=lambda _task_id, board: tmp_path / "worker.log", + _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_worker"), + set_branch_name=lambda *_args: None, + set_workspace_path=lambda *_args: None, + build_worker_context=lambda *_args: "bounded objective", + heartbeat_worker=lambda _conn, _task_id, *, note, expected_run_id: ( + heartbeats.append((note, expected_run_id)) or True + ), + add_comment=lambda *_args: None, + complete_task=lambda *_args, **kwargs: calls.append(("complete", kwargs)), + block_task=lambda *_args, **kwargs: calls.append(("block", kwargs)), + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + monkeypatch.setattr( + lanes, "state_path", lambda _board, _task_id: tmp_path / "state.json" + ) + monkeypatch.setattr( + lanes, + "select_route", + lambda *_args, **_kwargs: lanes.Route( + "codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, () + ), + ) + + def run_provider(*args, **_kwargs): + assert connections[0].closed + before_heartbeat = len(connections) + assert args[6]("working") is True + assert len(connections) == before_heartbeat + 1 + assert connections[-1].closed + return result + + monkeypatch.setattr(lanes, "run_provider", run_provider) + + lanes.execute_claim("cassandra", "t_worker") + + assert calls[0][0] == expected_action + assert heartbeats == [("working", 4)] + if expected_action == "complete": + assert calls[0][1]["metadata"]["artifacts"] == [str(artifact)] + assert calls[0][1]["metadata"]["tests_run"] == ["pytest -q"] + else: + assert calls[0][1]["kind"] == "capability" + assert all(connection.closed for connection in connections) + + +def test_goal_card_continues_after_local_judge_rejects_progress( + tmp_path: Path, + monkeypatch, +): + task = SimpleNamespace( + id="t_goal", + current_run_id=12, + assignee="cli-auto", + max_runtime_seconds=300, + goal_mode=True, + goal_max_turns=3, + ) + calls = [] + comments = [] + + class Connection: + def close(self): + return None + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + get_task=lambda _conn, _task_id: task, + worker_log_path=lambda _task_id, board: tmp_path / "worker.log", + _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_goal"), + set_branch_name=lambda *_args: None, + set_workspace_path=lambda *_args: None, + build_worker_context=lambda *_args: "Run tests, commit, push, and verify remote HEAD.", + heartbeat_worker=lambda *_args, **_kwargs: True, + add_comment=lambda _conn, _task_id, _author, body: comments.append(body), + complete_task=lambda *_args, **kwargs: calls.append(("complete", kwargs)), + block_task=lambda *_args, **kwargs: calls.append(("block", kwargs)), + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + monkeypatch.setattr( + lanes, + "state_path", + lambda _board, _task_id: tmp_path / "state.json", + ) + claude_low = lanes.Route( + "claude", "claude-fable-5", "low", "claude-low", "jetson", "vote", 1, () + ) + codex_low = lanes.Route( + "codex", "gpt-5.6-luna", "low", "codex-low", "manual", "fallback", 1, () + ) + codex_xhigh = lanes.Route( + "codex", "gpt-5.6-sol", "xhigh", "codex-xhigh", "jetson", "escalated", 1, () + ) + route_calls = [] + + def select_route(_prompt, assignee, **kwargs): + route_calls.append((assignee, kwargs)) + if assignee == "cli-codex-low": + return codex_low + if len(route_calls) == 1: + return claude_low + return codex_xhigh + + monkeypatch.setattr(lanes, "select_route", select_route) + monkeypatch.setattr(lanes, "fresh_unavailable_provider", lambda: None) + reports = [ + lanes.ProcessResult(1, "authentication expired", None, True), + lanes.ProcessResult( + 0, + "first turn", + { + "status": "completed", + "summary": "Focused tests passed.", + "changed_files": ["src/a.py"], + "tests_run": ["pytest focused: passed"], + "artifacts": [], + "blockers": [], + }, + False, + ), + lanes.ProcessResult( + 0, + "second turn", + { + "status": "completed", + "summary": "Full tests passed; commit pushed and remote HEAD verified.", + "changed_files": ["src/a.py"], + "tests_run": ["pytest full: passed"], + "artifacts": [], + "blockers": [], + }, + False, + ), + ] + monkeypatch.setattr(lanes, "run_provider", lambda *_args, **_kwargs: reports.pop(0)) + verdicts = iter( + [ + (False, "commit, push, and remote verification are missing"), + (True, "all explicit acceptance criteria have evidence"), + ] + ) + judge_contexts = [] + + def judge_goal_completion(objective, *_args, **_kwargs): + judge_contexts.append(objective) + return next(verdicts) + + monkeypatch.setattr( + lanes.cli_lane_goal, + "judge_goal_completion", + judge_goal_completion, + ) + + lanes.execute_claim("cassandra", "t_goal") + + assert calls[0][0] == "complete" + assert calls[0][1]["metadata"]["goal_turn"] == 2 + assert any( + "Goal completion rejected; continuing turn 2/3" in item for item in comments + ) + assert any( + "Goal route 2/3: codex/gpt-5.6-sol at xhigh" in item for item in comments + ) + assert route_calls[2][1]["exclude_provider"] == "claude" + assert "prior rejected reports" in judge_contexts[1] + assert "commit, push, and remote verification are missing" in judge_contexts[1] + assert reports == [] diff --git a/testing/tests/test_hermes_cli_lanes_support.py b/testing/tests/test_hermes_cli_lanes_support.py new file mode 100644 index 00000000..242351b9 --- /dev/null +++ b/testing/tests/test_hermes_cli_lanes_support.py @@ -0,0 +1,79 @@ +"""Shared loaders and manifest helpers for Hermes CLI-lane contracts.""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +import yaml + +ROOT = Path(__file__).parents[2] +SCRIPTS = ROOT / "services/hermes/scripts" +sys.path.insert(0, str(SCRIPTS)) +HERMES = ROOT / "services/hermes" +KEYCLOAK = ROOT / "services/keycloak" +FLUX_HERMES = ROOT / "clusters/atlas/flux-system/applications/hermes/kustomization.yaml" + + +def _load(name: str): + spec = importlib.util.spec_from_file_location(name, SCRIPTS / f"{name}.py") + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +lanes = _load("cli_lane_runner") +policy = _load("claude_command_policy") +migration = _load("migrate_herdr_state") +auth_patch = _load("patch_hermes_auth") +tui_gateway_patch = _load("patch_tui_gateway") +codex_runtime_patch = _load("patch_codex_runtime") +ttyd_patch = _load("patch_ttyd_index") +client_config = _load("configure_agent_clients") + + +def _agent_deployment() -> dict: + return yaml.safe_load((HERMES / "agent-deployment.yaml").read_text()) + + +def _services() -> dict[str, dict]: + return { + item["metadata"]["name"]: item + for item in yaml.safe_load_all((HERMES / "service.yaml").read_text()) + if item + } + + +def _oauth_deployment(name: str) -> dict: + documents = [ + item + for item in yaml.safe_load_all((HERMES / "oauth2-proxy.yaml").read_text()) + if item + ] + return next( + item + for item in documents + if item["kind"] == "Deployment" and item["metadata"]["name"] == name + ) + + +class _SwitchyardResponse: + """Minimal context-managed response used by routing contract tests.""" + + def __init__(self, selected: str, rationale: str = "local classifier vote"): + self.headers = { + "x-model-router-selected-model": selected, + "x-model-router-rationale": rationale, + } + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def read(self): + return b"{}" diff --git a/testing/tests/test_hermes_cli_lanes_toolchain.py b/testing/tests/test_hermes_cli_lanes_toolchain.py new file mode 100644 index 00000000..2ed232cb --- /dev/null +++ b/testing/tests/test_hermes_cli_lanes_toolchain.py @@ -0,0 +1,386 @@ +"""Operator toolchain and runtime-patch contracts for Hermes CLI lanes.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +import yaml + +from testing.tests.test_hermes_cli_lanes_support import ( + HERMES, + SCRIPTS, + _agent_deployment, + auth_patch, + client_config, + codex_runtime_patch, + tui_gateway_patch, + ttyd_patch, +) + + +def test_worker_route_broker_accepts_pod_network_health_checks(): + """Kubelet probes the pod IP, so the broker cannot bind to loopback only.""" + script = (SCRIPTS / "worker_route_broker.py").read_text() + assert 'ThreadingHTTPServer(("0.0.0.0", PORT), Handler)' in script + + +def test_switchyard_network_boundary_allows_vault_bootstrap(): + """The pre-populate init container must reach Vault before routing starts.""" + documents = [ + item + for item in yaml.safe_load_all((HERMES / "networkpolicy.yaml").read_text()) + if item + ] + isolation = next( + item + for item in documents + if item.get("metadata", {}).get("name") == "hermes-switchyard-isolation" + ) + assert any( + rule.get("to") + == [ + { + "namespaceSelector": { + "matchLabels": {"kubernetes.io/metadata.name": "vault"} + }, + "podSelector": {"matchLabels": {"app": "vault"}}, + } + ] + and rule.get("ports") == [{"protocol": "TCP", "port": 8200}] + for rule in isolation["spec"]["egress"] + ) + + +def test_switchyard_network_boundary_allows_metrics_scraping(): + """VictoriaMetrics may scrape Switchyard without widening its API boundary.""" + documents = [ + item + for item in yaml.safe_load_all((HERMES / "networkpolicy.yaml").read_text()) + if item + ] + isolation = next( + item + for item in documents + if item.get("metadata", {}).get("name") == "hermes-switchyard-isolation" + ) + assert any( + rule.get("from") + == [ + { + "namespaceSelector": { + "matchLabels": {"kubernetes.io/metadata.name": "monitoring"} + }, + "podSelector": {"matchLabels": {"app": "server"}}, + } + ] + and rule.get("ports") == [{"protocol": "TCP", "port": 9005}] + for rule in isolation["spec"]["ingress"] + ) + + +def test_owner_agent_installs_the_pinned_operator_toolchain(): + script = (SCRIPTS / "install_agent_tools.sh").read_text() + for value in [ + "flux", + "helm", + "kustomize", + "jq", + "yq", + "gh", + "vault", + "sops", + "age", + "age-keygen", + "k9s", + "terraform", + "go", + "gofmt", + ]: + assert value in script + assert "go1.26.5.linux-arm64.tar.gz" in script + assert "fe4789e92b1f33358680864bbe8704289e7bb5fc207d80623c308935bd696d49" in script + assert script.count("sha256sum -c -") == 1 + + deployment = _agent_deployment() + installer = next( + item + for item in deployment["spec"]["template"]["spec"]["initContainers"] + if item["name"] == "install-agent-tools" + ) + assert "/bin/sh /opt/coordinator/install_agent_tools.sh" in installer["command"][2] + assert any(mount["name"] == "coordinator" for mount in installer["volumeMounts"]) + init_config = next( + item + for item in deployment["spec"]["template"]["spec"]["initContainers"] + if item["name"] == "init-config" + ) + init_command = init_config["command"][2] + assert "# Hermes managed operator PATH." in init_command + assert "/opt/coordinator:/opt/data/tools/bin:/opt/hermes/.venv/bin" in init_command + assert 'chmod 0644 "${profile_file}"' in init_command + + +def test_owner_agent_uses_only_the_canonical_hostname(): + paths = [ + HERMES / "agent-configmap.yaml", + HERMES / "agent-deployment.yaml", + HERMES / "agent-ingress.yaml", + Path(__file__).parents[2] / "scripts/ops/hermes_triage_monitor.py", + ] + for path in paths: + content = path.read_text() + assert "agent.bstein.dev" not in content + assert "agent.hermes.bstein.dev" in content + + +def test_agent_reconnect_retains_complete_history_and_long_tool_budget(): + configmap = yaml.safe_load((HERMES / "agent-configmap.yaml").read_text()) + config = yaml.safe_load(configmap["data"]["config.yaml"]) + display = config["display"] + assert display["resume_exchanges"] >= 10000 + assert display["resume_max_user_chars"] >= 10000000 + assert display["resume_max_assistant_chars"] >= 10000000 + assert config["agent"]["max_turns"] == 180 + assert config["delegation"]["max_iterations"] == 120 + + +def test_auth_patch_honors_explicit_shared_store(tmp_path: Path): + source = tmp_path / "auth.py" + destination = tmp_path / "patched/auth.py" + source.write_text( + 'from pathlib import Path\nimport os\n\ndef _auth_file_path() -> Path:\n path = get_hermes_home() / "auth.json"\n return path\n', + encoding="utf-8", + ) + auth_patch.patch(source, destination) + content = destination.read_text() + assert 'os.environ.get("HERMES_AUTH_FILE"' in content + + +def test_auth_patch_fails_closed_on_upstream_drift(tmp_path: Path): + source = tmp_path / "auth.py" + source.write_text("def changed():\n pass\n", encoding="utf-8") + with pytest.raises(RuntimeError, match="context changed"): + auth_patch.patch(source, tmp_path / "patched.py") + + +def test_codex_runtime_patch_uses_cli_and_forwards_route(tmp_path: Path): + provider = tmp_path / "runtime_provider.py" + provider.write_text(codex_runtime_patch.PROVIDER_BEFORE, encoding="utf-8") + provider_out = tmp_path / "patched/runtime_provider.py" + codex_runtime_patch.patch_provider(provider, provider_out) + assert '"api_mode": "codex_app_server"' in provider_out.read_text() + + session = tmp_path / "codex_app_server_session.py" + session.write_text( + codex_runtime_patch.SESSION_SIGNATURE_BEFORE + + codex_runtime_patch.SESSION_REQUEST_BEFORE, + encoding="utf-8", + ) + session_out = tmp_path / "patched/codex_app_server_session.py" + codex_runtime_patch.patch_session(session, session_out) + session_content = session_out.read_text() + assert 'turn_params["model"] = model' in session_content + assert 'turn_params["effort"] = effort' in session_content + assert '"approvalPolicy": "never"' in session_content + assert '"sandboxPolicy": {"type": "dangerFullAccess"}' in session_content + + turn = tmp_path / "codex_runtime.py" + turn.write_text( + codex_runtime_patch.FALLBACK_CONTEXT_BEFORE + codex_runtime_patch.TURN_BEFORE, + encoding="utf-8", + ) + turn_out = tmp_path / "patched/codex_runtime.py" + codex_runtime_patch.patch_turn(turn, turn_out) + turn_content = turn_out.read_text() + assert "model=str(getattr(agent" in turn_content + assert "build_cross_provider_codex_prompt" in turn_content + + fallback = tmp_path / "chat_completion_helpers.py" + fallback.write_text( + codex_runtime_patch.FALLBACK_RESOLUTION_BEFORE, + encoding="utf-8", + ) + fallback_out = tmp_path / "patched/chat_completion_helpers.py" + codex_runtime_patch.patch_fallback(fallback, fallback_out) + fallback_content = fallback_out.read_text() + assert 'agent.api_mode = "codex_app_server"' in fallback_content + assert "agent._codex_cross_provider_fallback = True" in fallback_content + + loop = tmp_path / "conversation_loop.py" + loop.write_text( + codex_runtime_patch.FALLBACK_DISPATCH_BEFORE + + codex_runtime_patch.RETRY_FALLBACK_DISPATCH_BEFORE + + codex_runtime_patch.STREAM_RECOVERY_BEFORE, + encoding="utf-8", + ) + loop_out = tmp_path / "patched/conversation_loop.py" + codex_runtime_patch.patch_loop(loop, loop_out) + loop_content = loop_out.read_text() + assert loop_content.count('if agent.api_mode == "codex_app_server"') == 2 + assert "build_cross_provider_codex_prompt" in loop_content + retry_dispatch = loop_content.index( + "Fallback activation happens inside this retry loop" + ) + api_kwargs = loop_content.find("agent._build_api_kwargs", retry_dispatch) + assert api_kwargs == -1 or retry_dispatch < api_kwargs + assert "Provider stream ended before a complete response" in loop_content + assert "_is_transport_stub" in loop_content + assert "rerouting ({truncated_tool_call_retries}/4)" in loop_content + + auxiliary = tmp_path / "auxiliary_client.py" + auxiliary.write_text( + codex_runtime_patch.AUXILIARY_TOKEN_BEFORE, + encoding="utf-8", + ) + auxiliary_out = tmp_path / "patched/auxiliary_client.py" + codex_runtime_patch.patch_auxiliary(auxiliary, auxiliary_out) + auxiliary_content = auxiliary_out.read_text() + assert 'os.environ.get("CODEX_HOME"' in auxiliary_content + assert 'Path(codex_home).expanduser() / "auth.json"' in auxiliary_content + assert "never creates a metered API-key lane" in auxiliary_content + + +def test_agent_mounts_codex_auxiliary_runtime_patch(): + deployment = _agent_deployment() + pod = deployment["spec"]["template"]["spec"] + patch_init = next( + item for item in pod["initContainers"] if item["name"] == "patch-codex-runtime" + ) + assert patch_init["command"][-2:] == [ + "/opt/hermes/agent/auxiliary_client.py", + "/patched/auxiliary_client.py", + ] + expected_mount = { + "name": "codex-runtime-patch", + "mountPath": "/opt/hermes/agent/auxiliary_client.py", + "subPath": "auxiliary_client.py", + } + containers = {item["name"]: item for item in pod["containers"]} + for name in ("hermes", "terminal"): + assert expected_mount in containers[name]["volumeMounts"] + + +def test_codex_auxiliary_patch_reads_cli_token_without_copying_it( + tmp_path: Path, + monkeypatch, +): + codex_home = tmp_path / ".codex" + codex_home.mkdir() + (codex_home / "auth.json").write_text( + json.dumps({"tokens": {"access_token": "cli-access-token"}}), + encoding="utf-8", + ) + monkeypatch.setenv("CODEX_HOME", str(codex_home)) + + source = tmp_path / "auxiliary_client.py" + source.write_text( + "import json, logging, os, time\n" + "from pathlib import Path\n" + "logger = logging.getLogger(__name__)\n" + "def read_token():\n" + " try:\n" + " raise RuntimeError('Hermes provider store intentionally empty')\n" + + codex_runtime_patch.AUXILIARY_TOKEN_BEFORE, + encoding="utf-8", + ) + destination = tmp_path / "patched/auxiliary_client.py" + codex_runtime_patch.patch_auxiliary(source, destination) + namespace: dict = {} + exec(compile(destination.read_text(), str(destination), "exec"), namespace) + + assert namespace["read_token"]() == "cli-access-token" + + +def test_codex_runtime_patch_fails_closed_on_upstream_drift(tmp_path: Path): + source = tmp_path / "runtime_provider.py" + source.write_text("def changed():\n pass\n", encoding="utf-8") + with pytest.raises(RuntimeError, match="context changed"): + codex_runtime_patch.patch_provider(source, tmp_path / "patched.py") + + +def test_codex_runtime_migration_uses_owner_unsafe_mode(tmp_path: Path): + config = tmp_path / "config.yaml" + config.write_text("model: {}\n", encoding="utf-8") + codex_home = tmp_path / ".codex" + calls = [] + + class Report: + errors = [] + + @staticmethod + def summary(): + return "configured" + + def migrate(value, **kwargs): + calls.append((value, kwargs)) + return Report() + + client_config.configure_codex_runtime(config, migrate, codex_home) + + assert calls[0][1]["default_permission_profile"] is None + assert calls[0][1]["codex_home"] == codex_home + content = (codex_home / "config.toml").read_text(encoding="utf-8") + assert 'approval_policy = "never"' in content + assert 'sandbox_mode = "danger-full-access"' in content + assert "default_permissions" not in content + + +def test_codex_owner_permissions_replace_stale_profile(tmp_path: Path): + config = tmp_path / "config.toml" + config.write_text( + 'default_permissions = ":danger-no-sandbox"\n\n[features]\nhooks = true\n', + encoding="utf-8", + ) + + client_config.configure_codex_owner_permissions(config) + client_config.configure_codex_owner_permissions(config) + + content = config.read_text(encoding="utf-8") + assert content.count(client_config.OWNER_PERMISSIONS_BEGIN) == 1 + assert content.count('approval_policy = "never"') == 1 + assert content.count('sandbox_mode = "danger-full-access"') == 1 + assert "default_permissions" not in content + assert "[features]\nhooks = true" in content + + +def test_tui_gateway_patch_extends_and_bounds_agent_startup(tmp_path: Path): + source = tmp_path / "server.py" + destination = tmp_path / "patched/server.py" + source.write_text( + "import os\n\n" + tui_gateway_patch.BEFORE + "\ndef unchanged():\n pass\n", + encoding="utf-8", + ) + + tui_gateway_patch.patch(source, destination) + + content = destination.read_text(encoding="utf-8") + assert "HERMES_TUI_AGENT_INIT_TIMEOUT_S" in content + assert "configured = 180.0" in content + assert "return max(30.0, min(configured, 900.0))" in content + assert "timeout: float | None = None" in content + assert "ready.wait(timeout=wait_timeout)" in content + assert "def unchanged():" in content + + +def test_tui_gateway_patch_fails_closed_on_upstream_drift(tmp_path: Path): + source = tmp_path / "server.py" + source.write_text("def changed():\n pass\n", encoding="utf-8") + with pytest.raises(RuntimeError, match="context changed"): + tui_gateway_patch.patch(source, tmp_path / "patched.py") + + +def test_ttyd_clipboard_and_reconnect_patch_remain_enabled(): + source = '' + content = ttyd_patch.patch_html(source) + assert 'id="atlas-ttyd-clipboard"' in content + assert "navigator.clipboard.writeText(text)" in content + assert "class AtlasRecoveringWebSocket" in content + assert "window.location.reload()" in content + assert "event.stopImmediatePropagation()" in content + + +def test_ttyd_patch_fails_closed_on_upstream_drift(): + with pytest.raises(RuntimeError, match="context changed"): + ttyd_patch.patch_html("changed") diff --git a/testing/tests/test_hermes_coordinator.py b/testing/tests/test_hermes_coordinator.py index bc2d5b5f..683c0201 100644 --- a/testing/tests/test_hermes_coordinator.py +++ b/testing/tests/test_hermes_coordinator.py @@ -2,44 +2,20 @@ from __future__ import annotations -import importlib.util import json import subprocess -import sys import tomllib from pathlib import Path -import pytest import yaml - -SCRIPT = Path(__file__).parents[2] / "services/hermes/scripts/hermes_coordinator.py" -sys.path.insert(0, str(SCRIPT.parent)) -routing = importlib.import_module("hermes_model_routing") -catalog_resolver = importlib.import_module("routing_catalog") -SPEC = importlib.util.spec_from_file_location("hermes_coordinator", SCRIPT) -assert SPEC and SPEC.loader -coordinator = importlib.util.module_from_spec(SPEC) -sys.modules[SPEC.name] = coordinator -SPEC.loader.exec_module(coordinator) - - -def _base_config() -> dict: - """Return the minimal coordinator configuration used by routing tests.""" - return { - "model": { - "provider": "openai-codex", - "default": "gpt-5.6-terra", - "model": "gpt-5.6-terra", - }, - "fallback_providers": [ - {"provider": "anthropic", "model": "claude-opus-5"}, - dict(routing.LOCAL_FALLBACK), - ], - "toolsets": ["kanban"], - "terminal": {"cwd": "/opt/data/workspace"}, - } - +from testing.tests.test_hermes_coordinator_support import ( + SCRIPT, + _base_config, + catalog_resolver, + coordinator, + routing, +) def test_model_version_and_quality_selection_handle_new_and_small_models(): assert routing.model_version("claude-3-5-sonnet-20241022") == (3, 5) @@ -334,177 +310,3 @@ def test_refresh_writes_non_secret_routing_status(tmp_path: Path, monkeypatch): assert status["projects"]["cassandra"]["state"] == "ready" assert status["projects"]["cassandra"]["board_state"] == {"state": "ready"} assert "do-not-report" not in status_path.read_text(encoding="utf-8") - - -def test_corrupt_cassandra_board_is_preserved_without_blocking_refresh( - tmp_path: Path, monkeypatch -): - """A damaged project board degrades Kanban instead of stopping all services.""" - - class FakeKanbanDbCorruptError(Exception): - def __init__(self): - super().__init__("integrity check failed") - self.db_path = tmp_path / "kanban.db" - self.backup_path = tmp_path / "kanban.db.corrupt.backup" - self.reason = "row out of order" - - def raise_corruption(_root: Path) -> None: - raise FakeKanbanDbCorruptError() - - monkeypatch.setattr(coordinator, "bootstrap_cassandra", raise_corruption) - monkeypatch.setattr( - coordinator, "_kanban_corruption_type", lambda: FakeKanbanDbCorruptError - ) - - status = coordinator.bootstrap_cassandra_state(tmp_path) - - assert status == { - "state": "corrupt-preserved", - "database": str(tmp_path / "kanban.db"), - "backup": str(tmp_path / "kanban.db.corrupt.backup"), - "reason": "row out of order", - } - - -def test_unrelated_cassandra_bootstrap_failure_remains_fatal( - tmp_path: Path, monkeypatch -): - """Only the known fail-closed corruption state may be degraded.""" - - class FakeKanbanDbCorruptError(Exception): - pass - - def raise_permission_error(_root: Path) -> None: - raise PermissionError("cannot access board") - - monkeypatch.setattr(coordinator, "bootstrap_cassandra", raise_permission_error) - monkeypatch.setattr( - coordinator, "_kanban_corruption_type", lambda: FakeKanbanDbCorruptError - ) - - try: - coordinator.bootstrap_cassandra_state(tmp_path) - except PermissionError as error: - assert str(error) == "cannot access board" - else: - raise AssertionError("unrelated board errors must fail the coordinator refresh") - - -def test_cassandra_workspace_uses_valid_configured_worktree( - tmp_path: Path, monkeypatch -): - """The coordinator selects an explicit Git worktree but rejects stale paths.""" - active = tmp_path / "cassandra-v69" - active.mkdir() - (active / ".git").write_text("gitdir: elsewhere\n", encoding="utf-8") - monkeypatch.setenv("HERMES_CASSANDRA_ACTIVE_WORKTREE", str(active)) - - assert coordinator.cassandra_workspace() == active - - (active / ".git").unlink() - assert coordinator.cassandra_workspace() == coordinator.CASSANDRA_BASE_PATH - - -@pytest.mark.parametrize( - ("current_remote", "repair_action"), - [ - ("https://scm.bstein.dev/bstein/cassandra.git", "set-url"), - (None, "add"), - (coordinator.CASSANDRA_REMOTE, None), - ], - ids=["old-origin", "missing-origin", "canonical-origin"], -) -def test_cassandra_sync_repairs_existing_worktree( - tmp_path: Path, monkeypatch, current_remote: str | None, repair_action: str | None -): - """Existing worktrees use the canonical Atlas origin before fetching.""" - workspace = tmp_path / "cassandra" - (workspace / ".git").mkdir(parents=True) - monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", workspace) - monkeypatch.setattr(coordinator.shutil, "which", lambda _name: "/usr/bin/git") - token_path = tmp_path / "gitea-token" - token_path.write_text("configured\n", encoding="utf-8") - monkeypatch.setenv("HERMES_GITEA_TOKEN_FILE", str(token_path)) - commands: list[list[str]] = [] - environments: list[dict[str, str]] = [] - - def run(command, **kwargs): - commands.append(command) - environments.append(kwargs.get("env", {})) - if command[-2:] == ["get-url", "origin"]: - return subprocess.CompletedProcess( - command, 0 if current_remote else 2, stdout=f"{current_remote or ''}\n" - ) - return subprocess.CompletedProcess(command, 0) - - monkeypatch.setattr(coordinator.subprocess, "run", run) - assert coordinator.sync_cassandra_repo({"GITEA_TOKEN": "must-not-pass"}) == "ready" - repair_commands = [command for command in commands if repair_action in command] - assert bool(repair_commands) is bool(repair_action) - assert commands[-1][-4:] == ["fetch", "--quiet", "--prune", "origin"] - assert all("GITEA_TOKEN" not in environment for environment in environments) - - -def test_cassandra_sync_repairs_origin_without_token(tmp_path: Path, monkeypatch): - """Missing credentials skip only the fetch, not the local origin repair.""" - workspace = tmp_path / "cassandra" - (workspace / ".git").mkdir(parents=True) - monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", workspace) - monkeypatch.setattr(coordinator.shutil, "which", lambda _name: "/usr/bin/git") - commands: list[list[str]] = [] - - def run(command, **_kwargs): - commands.append(command) - stdout = "old-origin\n" if command[-2:] == ["get-url", "origin"] else None - return subprocess.CompletedProcess(command, 0, stdout=stdout) - - monkeypatch.setattr(coordinator.subprocess, "run", run) - state = coordinator.sync_cassandra_repo({}) - - assert state == "ready; fetch skipped until Gitea token is configured" - assert [command[-3] for command in commands] == ["remote", "set-url"] - - -def test_migrate_open_cassandra_tasks_preserves_running_and_done_tasks(): - """A project switch moves queued work without relocating active evidence.""" - - class Task: - def __init__(self, task_id: str, status: str): - self.id = task_id - self.status = status - self.workspace_kind = "dir" - self.workspace_path = str(coordinator.CASSANDRA_BASE_PATH) - - class Closing: - def __enter__(self): - return object() - - def __exit__(self, *_args): - return None - - class FakeKanban: - tasks = [ - Task("queued", "todo"), - Task("active", "running"), - Task("done", "done"), - ] - moved: list[tuple[str, str]] = [] - - @staticmethod - def connect_closing(*, board: str): - assert board == "cassandra" - return Closing() - - @classmethod - def list_tasks(cls, _connection, *, include_archived: bool): - assert include_archived is False - return cls.tasks - - @classmethod - def set_workspace_path(cls, _connection, task_id: str, path: Path): - cls.moved.append((task_id, str(path))) - - active = Path("/opt/data/workspace/projects/cassandra-hermes-v69") - coordinator._migrate_open_cassandra_tasks(FakeKanban, active) - - assert FakeKanban.moved == [("queued", str(active))] diff --git a/testing/tests/test_hermes_coordinator_cassandra.py b/testing/tests/test_hermes_coordinator_cassandra.py new file mode 100644 index 00000000..646db291 --- /dev/null +++ b/testing/tests/test_hermes_coordinator_cassandra.py @@ -0,0 +1,194 @@ +"""Cassandra workspace and board contracts for the Hermes coordinator.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from testing.tests.test_hermes_coordinator_support import coordinator + + +def test_corrupt_cassandra_board_is_preserved_without_blocking_refresh( + tmp_path: Path, monkeypatch +): + """A damaged project board degrades Kanban instead of stopping all services.""" + + class FakeKanbanDbCorruptError(Exception): + def __init__(self): + super().__init__("integrity check failed") + self.db_path = tmp_path / "kanban.db" + self.backup_path = tmp_path / "kanban.db.corrupt.backup" + self.reason = "row out of order" + + def raise_corruption(_root: Path) -> None: + raise FakeKanbanDbCorruptError() + + monkeypatch.setattr(coordinator, "bootstrap_cassandra", raise_corruption) + monkeypatch.setattr( + coordinator, "_kanban_corruption_type", lambda: FakeKanbanDbCorruptError + ) + + status = coordinator.bootstrap_cassandra_state(tmp_path) + + assert status == { + "state": "corrupt-preserved", + "database": str(tmp_path / "kanban.db"), + "backup": str(tmp_path / "kanban.db.corrupt.backup"), + "reason": "row out of order", + } + + +def test_unrelated_cassandra_bootstrap_failure_remains_fatal( + tmp_path: Path, monkeypatch +): + """Only the known fail-closed corruption state may be degraded.""" + + class FakeKanbanDbCorruptError(Exception): + pass + + def raise_permission_error(_root: Path) -> None: + raise PermissionError("cannot access board") + + monkeypatch.setattr(coordinator, "bootstrap_cassandra", raise_permission_error) + monkeypatch.setattr( + coordinator, "_kanban_corruption_type", lambda: FakeKanbanDbCorruptError + ) + + try: + coordinator.bootstrap_cassandra_state(tmp_path) + except PermissionError as error: + assert str(error) == "cannot access board" + else: + raise AssertionError("unrelated board errors must fail the coordinator refresh") + + +def test_cassandra_workspace_uses_valid_configured_worktree( + tmp_path: Path, monkeypatch +): + """The coordinator selects an explicit Git worktree but rejects stale paths.""" + active = tmp_path / "cassandra-v69" + active.mkdir() + (active / ".git").write_text("gitdir: elsewhere\n", encoding="utf-8") + monkeypatch.setenv("HERMES_CASSANDRA_ACTIVE_WORKTREE", str(active)) + + assert coordinator.cassandra_workspace() == active + + (active / ".git").unlink() + assert coordinator.cassandra_workspace() == coordinator.CASSANDRA_BASE_PATH + + +@pytest.mark.parametrize( + ("current_remote", "repair_action"), + [ + ("https://scm.bstein.dev/bstein/cassandra.git", "set-url"), + (None, "add"), + (coordinator.CASSANDRA_REMOTE, None), + ], + ids=["old-origin", "missing-origin", "canonical-origin"], +) +def test_cassandra_sync_repairs_existing_worktree( + tmp_path: Path, monkeypatch, current_remote: str | None, repair_action: str | None +): + """Existing worktrees use the canonical Atlas origin before fetching.""" + workspace = tmp_path / "cassandra" + (workspace / ".git").mkdir(parents=True) + monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", workspace) + monkeypatch.setattr(coordinator.shutil, "which", lambda _name: "/usr/bin/git") + token_path = tmp_path / "gitea-token" + token_path.write_text("configured\n", encoding="utf-8") + monkeypatch.setenv("HERMES_GITEA_TOKEN_FILE", str(token_path)) + commands: list[list[str]] = [] + environments: list[dict[str, str]] = [] + + def run(command, **kwargs): + commands.append(command) + environments.append(kwargs.get("env", {})) + if command[-2:] == ["get-url", "origin"]: + return subprocess.CompletedProcess( + command, 0 if current_remote else 2, stdout=f"{current_remote or ''}\n" + ) + return subprocess.CompletedProcess(command, 0) + + monkeypatch.setattr(coordinator.subprocess, "run", run) + assert coordinator.sync_cassandra_repo({"GITEA_TOKEN": "must-not-pass"}) == "ready" + repair_commands = [command for command in commands if repair_action in command] + assert bool(repair_commands) is bool(repair_action) + assert commands[-1][-4:] == ["fetch", "--quiet", "--prune", "origin"] + assert all("GITEA_TOKEN" not in environment for environment in environments) + + +def test_cassandra_sync_repairs_origin_through_broker_without_token( + tmp_path: Path, monkeypatch +): + """Repository sync uses the credential-isolated broker remote.""" + monkeypatch.setenv( + "HERMES_GITEA_TOKEN_FILE", str(tmp_path / "explicitly-unavailable-token") + ) + workspace = tmp_path / "cassandra" + (workspace / ".git").mkdir(parents=True) + monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", workspace) + monkeypatch.setattr(coordinator.shutil, "which", lambda _name: "/usr/bin/git") + commands: list[list[str]] = [] + + def run(command, **_kwargs): + commands.append(command) + stdout = "old-origin\n" if command[-2:] == ["get-url", "origin"] else None + return subprocess.CompletedProcess(command, 0, stdout=stdout) + + monkeypatch.setattr(coordinator.subprocess, "run", run) + state = coordinator.sync_cassandra_repo({}) + + assert state == "ready" + assert any( + "hermes-scm-broker.hermes-scm.svc.cluster.local" in " ".join(cmd) + for cmd in commands + ) + assert all("token" not in " ".join(cmd).lower() for cmd in commands) + assert commands[-1][-4:] == ["fetch", "--quiet", "--prune", "origin"] + + +def test_migrate_open_cassandra_tasks_preserves_running_and_done_tasks(): + """A project switch moves queued work without relocating active evidence.""" + + class Task: + def __init__(self, task_id: str, status: str): + self.id = task_id + self.status = status + self.workspace_kind = "dir" + self.workspace_path = str(coordinator.CASSANDRA_BASE_PATH) + + class Closing: + def __enter__(self): + return object() + + def __exit__(self, *_args): + return None + + class FakeKanban: + tasks = [ + Task("queued", "todo"), + Task("active", "running"), + Task("done", "done"), + ] + moved: list[tuple[str, str]] = [] + + @staticmethod + def connect_closing(*, board: str): + assert board == "cassandra" + return Closing() + + @classmethod + def list_tasks(cls, _connection, *, include_archived: bool): + assert include_archived is False + return cls.tasks + + @classmethod + def set_workspace_path(cls, _connection, task_id: str, path: Path): + cls.moved.append((task_id, str(path))) + + active = Path("/opt/data/workspace/projects/cassandra-hermes-v69") + coordinator._migrate_open_cassandra_tasks(FakeKanban, active) + + assert FakeKanban.moved == [("queued", str(active))] diff --git a/testing/tests/test_hermes_coordinator_coverage.py b/testing/tests/test_hermes_coordinator_coverage.py new file mode 100644 index 00000000..fe4f3168 --- /dev/null +++ b/testing/tests/test_hermes_coordinator_coverage.py @@ -0,0 +1,267 @@ +"""Behavioral branch coverage for Hermes project coordination.""" + +from __future__ import annotations + +import json +import subprocess +import sys +import types +from contextlib import nullcontext +from pathlib import Path + +import pytest + +from testing.tests.test_hermes_coordinator_support import coordinator, routing + + +def test_workspace_falls_back_when_unset_or_invalid(tmp_path: Path, monkeypatch): + monkeypatch.delenv("HERMES_CASSANDRA_ACTIVE_WORKTREE", raising=False) + assert coordinator.cassandra_workspace() == coordinator.CASSANDRA_BASE_PATH + monkeypatch.setenv("HERMES_CASSANDRA_ACTIVE_WORKTREE", str(tmp_path / "missing")) + assert coordinator.cassandra_workspace() == coordinator.CASSANDRA_BASE_PATH + + +def test_task_migration_is_noop_for_base_workspace(): + class Board: + @staticmethod + def connect_closing(**_kwargs): + raise AssertionError("base workspace must not access Kanban") + + coordinator._migrate_open_cassandra_tasks(Board, coordinator.CASSANDRA_BASE_PATH) + + +class _Project: + id = "project-id" + + +def _fake_hermes_modules(monkeypatch, *, existing: bool, active: bool): + events = [] + kb = types.ModuleType("hermes_cli.kanban_db") + kb.board_exists = lambda slug: existing + kb.create_board = lambda *args, **kwargs: events.append( + ("create-board", args, kwargs) + ) + kb.set_current_board = lambda slug: events.append(("current", slug)) + kb.connect_closing = lambda **kwargs: nullcontext(object()) + kb.list_tasks = lambda *_a, **_k: [] + + pdb = types.ModuleType("hermes_cli.projects_db") + pdb.connect_closing = lambda: nullcontext(object()) + pdb.get_project = lambda *_a: _Project() if existing else None + pdb.get_active_id = lambda *_a: "active" if active else None + pdb.create_project = ( + lambda *args, **kwargs: events.append(("create-project", kwargs)) or "new-id" + ) + pdb.set_active = lambda *args: events.append(("active", args[-1])) + pdb.update_project = lambda *args, **kwargs: events.append(("update", kwargs)) + pdb.add_folder = lambda *args, **kwargs: events.append(("folder", args[-1], kwargs)) + + package = types.ModuleType("hermes_cli") + package.kanban_db = kb + package.projects_db = pdb + monkeypatch.setitem(sys.modules, "hermes_cli", package) + monkeypatch.setitem(sys.modules, "hermes_cli.kanban_db", kb) + monkeypatch.setitem(sys.modules, "hermes_cli.projects_db", pdb) + return events + + +@pytest.mark.parametrize( + ("existing", "active", "expected"), + [ + (False, False, {"current", "create-project", "active"}), + (False, True, {"current", "create-project"}), + (True, True, {"update", "folder"}), + ], +) +def test_bootstrap_cassandra_creates_or_updates_state( + tmp_path: Path, monkeypatch, existing, active, expected +): + events = _fake_hermes_modules(monkeypatch, existing=existing, active=active) + base = tmp_path / "projects/cassandra" + worktree = tmp_path / "projects/cassandra-worktree" + worktree.mkdir(parents=True) + (worktree / ".git").write_text("gitdir: elsewhere\n", encoding="utf-8") + monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", base) + monkeypatch.setenv("HERMES_CASSANDRA_ACTIVE_WORKTREE", str(worktree)) + coordinator.bootstrap_cassandra(tmp_path / "hermes") + names = {event[0] for event in events} + assert expected <= names + assert "create-board" in names + + +def test_bootstrap_state_ready_path(tmp_path: Path, monkeypatch): + monkeypatch.setattr(coordinator, "bootstrap_cassandra", lambda _root: None) + assert coordinator.bootstrap_cassandra_state(tmp_path) == {"state": "ready"} + + +def test_corruption_type_imports_pinned_exception(monkeypatch): + class Expected(Exception): + pass + + package = types.ModuleType("hermes_cli") + kanban = types.ModuleType("hermes_cli.kanban_db") + kanban.KanbanDbCorruptError = Expected + package.kanban_db = kanban + monkeypatch.setitem(sys.modules, "hermes_cli", package) + monkeypatch.setitem(sys.modules, "hermes_cli.kanban_db", kanban) + assert coordinator._kanban_corruption_type() is Expected + + +def test_repo_sync_reports_missing_git_and_unmanaged_directory( + tmp_path: Path, monkeypatch +): + monkeypatch.setattr(coordinator.shutil, "which", lambda _name: None) + assert coordinator.sync_cassandra_repo({}) == "git-unavailable" + + workspace = tmp_path / "cassandra" + workspace.mkdir() + (workspace / "user-file").write_text("preserve", encoding="utf-8") + monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", workspace) + monkeypatch.setattr(coordinator.shutil, "which", lambda _name: "/usr/bin/git") + assert coordinator.sync_cassandra_repo({}) == "unmanaged-nonempty-directory" + + +@pytest.mark.parametrize( + ("failure", "expected"), + [ + ("get-url-error", "remote-repair-failed"), + ("repair-exit", "remote-repair-failed-9"), + ("repair-error", "remote-repair-failed"), + ("fetch-exit", "sync-failed-8"), + ("fetch-error", "sync-failed"), + ], +) +def test_existing_repo_sync_reports_each_failure( + tmp_path: Path, monkeypatch, failure, expected +): + workspace = tmp_path / "cassandra" + (workspace / ".git").mkdir(parents=True) + monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", workspace) + monkeypatch.setattr(coordinator.shutil, "which", lambda _name: "/usr/bin/git") + + def run(command, **_kwargs): + action = command[-2:] if len(command) >= 2 else [] + if action == ["get-url", "origin"]: + if failure == "get-url-error": + raise OSError("git unavailable") + return subprocess.CompletedProcess(command, 0, stdout="old\n") + if "set-url" in command: + if failure == "repair-error": + raise subprocess.TimeoutExpired(command, 30) + return subprocess.CompletedProcess( + command, 9 if failure == "repair-exit" else 0 + ) + if failure == "fetch-error": + raise OSError("fetch unavailable") + return subprocess.CompletedProcess(command, 8 if failure == "fetch-exit" else 0) + + monkeypatch.setattr(coordinator.subprocess, "run", run) + assert coordinator.sync_cassandra_repo({}) == expected + + +@pytest.mark.parametrize( + ("exit_code", "expected"), [(0, "ready"), (7, "sync-failed-7")] +) +def test_empty_repo_clones_through_broker( + tmp_path: Path, monkeypatch, exit_code, expected +): + workspace = tmp_path / "cassandra" + monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", workspace) + monkeypatch.setattr(coordinator.shutil, "which", lambda _name: "/usr/bin/git") + seen = [] + + def run(command, **kwargs): + seen.append((command, kwargs["env"])) + return subprocess.CompletedProcess(command, exit_code) + + monkeypatch.setattr(coordinator.subprocess, "run", run) + assert ( + coordinator.sync_cassandra_repo( + {"API_SERVER_KEY": "not-forwarded", "SAFE": "yes"} + ) + == expected + ) + assert coordinator.CASSANDRA_REMOTE in seen[0][0] + assert seen[0][1]["SAFE"] == "yes" + assert "API_SERVER_KEY" not in seen[0][1] + + +def _catalog(provider): + return routing.Catalog(provider, ["model"], True, True, "connected") + + +def test_refresh_once_writes_complete_status(tmp_path: Path, monkeypatch): + monkeypatch.setattr(coordinator, "_read_env", lambda _path: {"SAFE": "yes"}) + monkeypatch.setattr(coordinator, "discover_codex_models", lambda: _catalog("codex")) + monkeypatch.setattr( + coordinator, "discover_claude_models", lambda: _catalog("claude") + ) + monkeypatch.setattr( + coordinator, "configure_routes", lambda *_a: {"coordinator": ["auto"]} + ) + monkeypatch.setattr( + coordinator, "bootstrap_cassandra_state", lambda _root: {"state": "ready"} + ) + monkeypatch.setattr(coordinator, "sync_cassandra_repo", lambda _env: "ready") + monkeypatch.setattr( + coordinator, "cassandra_workspace", lambda: tmp_path / "worktree" + ) + written = [] + monkeypatch.setattr( + coordinator, "_atomic_write", lambda path, value: written.append((path, value)) + ) + status = coordinator.refresh_once(tmp_path) + assert status["projects"]["cassandra"]["state"] == "ready" + assert json.loads(written[0][1])["providers"]["codex"]["connected"] is True + + +def test_main_once_success_and_failure(monkeypatch, capsys): + status = {"providers": {"codex": {"state": "ready"}}} + monkeypatch.setattr(sys, "argv", ["coordinator", "--once"]) + monkeypatch.setattr(coordinator, "refresh_once", lambda _root: status) + assert coordinator.main() == 0 + assert "codex=ready" in capsys.readouterr().out + monkeypatch.setattr( + coordinator, + "refresh_once", + lambda _root: (_ for _ in ()).throw(RuntimeError("failed")), + ) + assert coordinator.main() == 1 + assert "RuntimeError" in capsys.readouterr().out + + +@pytest.mark.parametrize(("interval", "expected"), [(1, 300), (400, 400)]) +def test_main_loop_sleeps_with_floor(monkeypatch, interval, expected): + calls = iter(({"providers": {}}, KeyboardInterrupt())) + + def refresh(_root): + value = next(calls) + if isinstance(value, BaseException): + raise value + return value + + sleeps = [] + monkeypatch.setattr(coordinator, "refresh_once", refresh) + monkeypatch.setattr(coordinator.time, "sleep", sleeps.append) + monkeypatch.setattr( + sys, "argv", ["coordinator", "--loop", "--interval", str(interval)] + ) + with pytest.raises(KeyboardInterrupt): + coordinator.main() + assert sleeps == [expected] + + +def test_main_loop_recovers_from_refresh_failure(monkeypatch): + calls = iter((RuntimeError("temporary"), KeyboardInterrupt())) + + def refresh(_root): + error = next(calls) + raise error + + sleeps = [] + monkeypatch.setattr(coordinator, "refresh_once", refresh) + monkeypatch.setattr(coordinator.time, "sleep", sleeps.append) + monkeypatch.setattr(sys, "argv", ["coordinator", "--loop", "--interval", "300"]) + with pytest.raises(KeyboardInterrupt): + coordinator.main() + assert sleeps == [300] diff --git a/testing/tests/test_hermes_coordinator_support.py b/testing/tests/test_hermes_coordinator_support.py new file mode 100644 index 00000000..2a41f070 --- /dev/null +++ b/testing/tests/test_hermes_coordinator_support.py @@ -0,0 +1,35 @@ +"""Shared loader and configuration for Hermes coordinator contracts.""" + +from __future__ import annotations + +import importlib +import importlib.util +import sys +from pathlib import Path + +SCRIPT = Path(__file__).parents[2] / "services/hermes/scripts/hermes_coordinator.py" +sys.path.insert(0, str(SCRIPT.parent)) +routing = importlib.import_module("hermes_model_routing") +catalog_resolver = importlib.import_module("routing_catalog") +SPEC = importlib.util.spec_from_file_location("hermes_coordinator", SCRIPT) +assert SPEC and SPEC.loader +coordinator = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = coordinator +SPEC.loader.exec_module(coordinator) + + +def _base_config() -> dict: + """Return the minimal coordinator configuration used by routing tests.""" + return { + "model": { + "provider": "openai-codex", + "default": "gpt-5.6-terra", + "model": "gpt-5.6-terra", + }, + "fallback_providers": [ + {"provider": "anthropic", "model": "claude-opus-5"}, + dict(routing.LOCAL_FALLBACK), + ], + "toolsets": ["kanban"], + "terminal": {"cwd": "/opt/data/workspace"}, + } diff --git a/testing/tests/test_hermes_gitea_branch_coverage.py b/testing/tests/test_hermes_gitea_branch_coverage.py new file mode 100644 index 00000000..362e8726 --- /dev/null +++ b/testing/tests/test_hermes_gitea_branch_coverage.py @@ -0,0 +1,183 @@ +"""Behavioral branch coverage for the Gitea protection policy checker.""" + +from __future__ import annotations + +import json +import os +import stat +import sys +from pathlib import Path + +import pytest + +from testing.tests.test_hermes_scm_broker_support import ROOT, _load_path + + +def _module(name: str = "branch_protection_coverage"): + return _load_path( + name, + ROOT / "services/gitea/scripts/gitea_branch_protection_check.py", + ) + + +def _rule(module, name: str = "main", **updates): + value = { + "rule_name": name, + "priority": 1, + "created_at": "2026-01-01T00:00:00Z", + **module._required("bstein"), + } + value.update(updates) + return value + + +def test_read_bounded_accepts_regular_file_and_closes_descriptor( + tmp_path: Path, monkeypatch +): + module = _module("branch_read_regular") + source = tmp_path / "rules.json" + source.write_bytes(b"[]") + closed: list[int] = [] + real_close = os.close + + def close(descriptor: int) -> None: + closed.append(descriptor) + real_close(descriptor) + + monkeypatch.setattr(module.os, "close", close) + assert module._read_bounded(source) == b"[]" + assert len(closed) == 1 + + +@pytest.mark.parametrize("kind", ["directory", "oversized", "short"]) +def test_read_bounded_rejects_unsafe_or_changed_input( + tmp_path: Path, monkeypatch, kind +): + module = _module(f"branch_read_{kind}") + source = tmp_path / "rules" + source.write_bytes(b"[]") + real_fstat = module.os.fstat + real_read = module.os.read + + if kind == "directory": + monkeypatch.setattr( + module.os, + "fstat", + lambda descriptor: os.stat_result( + (stat.S_IFDIR | 0o700, 0, 0, 0, 0, 0, 2, 0, 0, 0) + ), + ) + elif kind == "oversized": + monkeypatch.setattr( + module.os, + "fstat", + lambda descriptor: os.stat_result( + (stat.S_IFREG | 0o600, 0, 0, 0, 0, 0, module.MAX_INPUT + 1, 0, 0, 0) + ), + ) + else: + monkeypatch.setattr(module.os, "fstat", real_fstat) + monkeypatch.setattr(module.os, "read", lambda descriptor, maximum: b"[") + with pytest.raises(module.PolicyError): + module._read_bounded(source) + monkeypatch.setattr(module.os, "read", real_read) + + +@pytest.mark.parametrize( + ("pattern", "branch", "expected"), + [ + ("m?in", "main", True), + ("m[ai]in", "main", True), + ("m[!z]in", "main", True), + ("m]ain", "m]ain", True), + ("m}ain", "m}ain", True), + ("m[a-z]in", "main", True), + ("m{ain,aster}", "main", True), + ("m\\ain", "main", True), + ], +) +def test_glob_grammar_accepts_supported_constructs(pattern, branch, expected): + module = _module("branch_glob_supported") + assert module._matches(pattern, branch) is expected + + +@pytest.mark.parametrize( + "pattern", + [ + "m\\", + "m[", + "m[]", + "m[!]", + "m[[a]", + "m[\\a]", + "m[-a]", + "m[a-]", + "m[a-b-c]", + "m[z-a]", + "m[aa-b]", + "m{ain}", + "m{ain,}", + "m{ain,aster", + ], +) +def test_invalid_globs_are_literal_nonmatches(pattern): + module = _module("branch_glob_invalid") + assert module._matches(pattern, "main") is False + + +@pytest.mark.parametrize("pattern", ["", "x" * 256, "máin"]) +def test_rule_names_are_ascii_and_bounded(pattern): + module = _module("branch_name_bounds") + with pytest.raises(module.PolicyError, match="rule name"): + module._matches(pattern, "main") + + +@pytest.mark.parametrize("value", [None, "x" * 65, "invalid", "2026-01-01T00:00:00"]) +def test_created_at_requires_short_timezone_aware_iso(value): + module = _module("branch_created_bounds") + with pytest.raises(module.PolicyError, match="creation time"): + module._created(value) + + +@pytest.mark.parametrize( + "value", + [b"x" * (1024 * 1024 + 1), b"{}", json.dumps([None]).encode()], +) +def test_evaluate_rejects_invalid_top_level_inputs(value): + module = _module("branch_evaluate_top") + with pytest.raises((module.PolicyError, json.JSONDecodeError)): + module.evaluate(value, "main", "bstein") + + +def test_evaluate_rejects_unknown_branch_and_excess_rules(): + module = _module("branch_evaluate_bounds") + with pytest.raises(module.PolicyError, match="input"): + module.evaluate(b"[]", "release", "bstein") + with pytest.raises(module.PolicyError, match="response"): + module.evaluate(json.dumps([{}] * 101).encode(), "main", "bstein") + + +@pytest.mark.parametrize("priority", [True, 0, 1_000_001, "1"]) +def test_evaluate_rejects_ambiguous_priority_types(priority): + module = _module("branch_priority_bounds") + rule = _rule(module, priority=priority) + with pytest.raises(module.PolicyError, match="priority"): + module.evaluate(json.dumps([rule]).encode(), "main", "bstein") + + +def test_main_reports_success_and_failures(tmp_path: Path, monkeypatch, capsys): + module = _module("branch_main_paths") + source = tmp_path / "rules.json" + source.write_text(json.dumps([_rule(module)]), encoding="utf-8") + monkeypatch.setattr(sys, "argv", ["checker", str(source), "main", "bstein"]) + assert module.main() == 0 + assert capsys.readouterr().out.strip() == "PRESENT" + + source.write_text("not-json", encoding="utf-8") + assert module.main() == 1 + assert "branch protection check failed" in capsys.readouterr().err + + monkeypatch.setattr( + module, "_read_bounded", lambda _path: (_ for _ in ()).throw(OSError()) + ) + assert module.main() == 1 diff --git a/testing/tests/test_hermes_gitea_internal_coverage.py b/testing/tests/test_hermes_gitea_internal_coverage.py new file mode 100644 index 00000000..f811827c --- /dev/null +++ b/testing/tests/test_hermes_gitea_internal_coverage.py @@ -0,0 +1,375 @@ +"""Behavioral coverage for internal safe-Gitea client and policy paths.""" + +from __future__ import annotations + +import json +import subprocess +import sys +import types +import urllib.error +import urllib.parse +from email.message import Message +from pathlib import Path + +import pytest + +from testing.tests.test_hermes_gitea_support import CLIENT_PATH, Response, _load +from testing.tests.test_hermes_scm_broker_support import _load_path + + +def _policy(name: str = "gitea_policy_coverage"): + return _load_path(name, CLIENT_PATH.parent / "gitea_api_policy.py") + + +def test_runtime_token_reader_and_fixed_origin(tmp_path: Path, monkeypatch): + module = _load() + token = tmp_path / "token" + token.write_text(" synthetic-runtime-value \n", encoding="utf-8") + assert module.read_token(token) == "synthetic-runtime-value" + token.write_text(" \n", encoding="utf-8") + with pytest.raises(ValueError, match="empty"): + module.read_token(token) + + monkeypatch.setenv("GITEA_BASE_URL", module.CANONICAL_BASE_URL) + assert module.configured_base_url() == module.CANONICAL_BASE_URL + monkeypatch.setenv("GITEA_BASE_URL", "https://example.invalid") + with pytest.raises(module.PolicyError, match="fixed"): + module.configured_base_url() + + +def test_safe_urlopen_delegates_only_to_redirect_rejecting_opener(monkeypatch): + module = _load() + sentinel = object() + monkeypatch.setattr( + module._SAFE_OPENER, + "open", + lambda request, timeout: (request, timeout, sentinel), + ) + request = object() + assert module._safe_urlopen(request, 7) == (request, 7, sentinel) + + +def test_api_target_and_request_body_edge_paths(monkeypatch): + module = _load() + with pytest.raises(module.PolicyError, match="path exceeds"): + module._split_api_path("/api/v1/" + "x" * 510) + with pytest.raises(module.PolicyError, match="request body"): + module.authorize_request( + "GET", "/api/v1/repos/atlas/cassandra", {"unexpected": True} + ) + with pytest.raises(module.PolicyError, match="query"): + module.authorize_request( + "POST", + "/api/v1/repos/atlas/cassandra/pulls?page=1", + {}, + ) + with pytest.raises(module.PolicyError, match="only read"): + module.authorize_request("TRACE", "/api/v1/repos/atlas/cassandra/pulls", None) + with pytest.raises(module.PolicyError, match="accepts only"): + module.authorize_request("POST", "/api/v1/repos/atlas/cassandra/pulls", []) + + +def test_response_status_fallback_and_nested_fail_closed(): + module = _load() + + class GetCodeResponse(Response): + def __init__(self): + super().__init__({"name": "cassandra"}, status=200) + del self.status + + def getcode(self): + return 200 + + assert ( + module.read( + "/api/v1/repos/atlas/cassandra", + token="synthetic", + opener=lambda *_a, **_k: GetCodeResponse(), + ) + == b'{"name": "cassandra"}' + ) + with pytest.raises(module.PolicyError, match="omitted required"): + module._nested({"base": None}, "base", "ref") + + +@pytest.mark.parametrize("body", [b"not-json", b"[]"]) +def test_create_response_requires_json_object(body: bytes): + module = _load() + with pytest.raises(module.PolicyError, match="invalid pull-request metadata"): + module._require_create_response( + body, + repo="cassandra", + base="main", + head="feature/test", + head_sha="a" * 40, + title="WIP: Test", + body="Evidence", + ) + + +def test_write_body_handles_empty_newline_and_missing_newline(monkeypatch): + module = _load() + + class Buffer: + value = bytearray() + + @classmethod + def write(cls, value): + cls.value.extend(value) + + monkeypatch.setattr(module.sys, "stdout", types.SimpleNamespace(buffer=Buffer)) + module._write_body(b"") + module._write_body(b"one\n") + module._write_body(b"two") + assert bytes(Buffer.value) == b"one\ntwo\n" + + +def test_main_executes_broker_read_and_create_paths(monkeypatch): + module = _load() + outputs: list[bytes] = [] + client = types.ModuleType("scm_broker_client") + client.read = lambda path: json.dumps({"path": path}).encode() + client.create_draft = lambda repo, **data: json.dumps( + {"repo": repo, **data}, sort_keys=True + ).encode() + monkeypatch.setitem(sys.modules, "scm_broker_client", client) + monkeypatch.setattr(module, "_write_body", outputs.append) + + assert module.main(["read", "/api/v1/repos/atlas/cassandra"]) == 0 + assert json.loads(outputs.pop()) == {"path": "/api/v1/repos/atlas/cassandra"} + assert ( + module.main( + [ + "create-draft", + "cassandra", + "--base", + "main", + "--head", + "feature/coverage", + "--head-sha", + "a" * 40, + "--title", + "Coverage repair", + "--body", + "Review the focused tests", + ] + ) + == 0 + ) + assert json.loads(outputs.pop())["repo"] == "cassandra" + + +def test_main_handles_http_and_policy_failures(monkeypatch, capsys): + module = _load() + client = types.ModuleType("scm_broker_client") + + def http_failure(_path): + headers = Message() + raise urllib.error.HTTPError("url", 503, "unavailable", headers, None) + + client.read = http_failure + client.create_draft = lambda *_a, **_k: b"{}" + monkeypatch.setitem(sys.modules, "scm_broker_client", client) + assert module.main(["read", "/api/v1/repos/atlas/cassandra"]) == 1 + assert "HTTP 503" in capsys.readouterr().err + client.read = lambda _path: (_ for _ in ()).throw(module.PolicyError("rejected")) + assert module.main(["read", "/not-allowed"]) == 1 + assert "no credential" in capsys.readouterr().err + + +@pytest.mark.parametrize( + ("function", "value"), + [ + ("_validate_repo", "."), + ("_validate_repo", ".."), + ("_validate_sha", "short"), + ("_validate_pr_number", True), + ("_validate_pr_number", 0), + ("_validate_pr_number", 2_147_483_648), + ], +) +def test_policy_scalar_validators_reject_ambiguous_values(function, value): + module = _policy("gitea_policy_scalar") + with pytest.raises(module.PolicyError): + getattr(module, function)(value) + + +def test_ref_bounds_cover_utf8_controls_git_failure_and_success(monkeypatch): + module = _policy("gitea_policy_refs") + with pytest.raises(module.PolicyError, match="valid UTF-8"): + module._validate_ref_bounds("\ud800", "head") + with pytest.raises(module.PolicyError, match="UTF-8"): + module._validate_ref_bounds("🧪" * 64, "head") + with pytest.raises(module.PolicyError, match="safe same"): + module._validate_ref("bad\nref", "head") + monkeypatch.setattr( + module.subprocess, + "run", + lambda *_a, **_k: subprocess.CompletedProcess([], 1), + ) + with pytest.raises(module.PolicyError, match="same-repository"): + module._validate_ref("invalid-ref", "head") + monkeypatch.setattr( + module.subprocess, + "run", + lambda *_a, **_k: subprocess.CompletedProcess([], 0), + ) + assert module._validate_ref("feature/valid", "head") == "feature/valid" + + +@pytest.mark.parametrize( + ("value", "required", "match"), + [ + (None, False, "must be text"), + (" ", True, "must not be empty"), + ("\ud800", False, "valid UTF-8"), + ("x\x00y", False, "safe request limit"), + ], +) +def test_text_validation_rejects_nontext_empty_invalid_and_controls( + value, required, match +): + module = _policy("gitea_policy_text") + with pytest.raises(module.PolicyError, match=match): + module._validate_text(value, "field", 20, 20, required=required) + + +@pytest.mark.parametrize( + ("key", "expected"), + [ + ("password", True), + ("accountKey", True), + ("registry-key", True), + ("clientEmail", True), + ("clientId", True), + ("accessId", True), + ("connectionString", True), + ("dockerConfigJson", True), + ("release_note", False), + ], +) +def test_sensitive_key_semantics_cover_generic_forms(key, expected): + module = _policy("gitea_policy_keys") + assert module._is_sensitive_key(key) is expected + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("", False), + ('"synthetic"', True), + ("{encoded}", True), + ("Bearer value-12345678", True), + ("ghp_" + "a" * 24, True), + ("https://example.invalid/value", True), + ("name@example.invalid", True), + ("${RUNTIME_VALUE}", True), + ("AbCdEf0123456789", True), + ("reject empty values", False), + ("ordinary", True), + ], +) +def test_assignment_value_shape_distinguishes_prose_from_credentials(value, expected): + module = _policy("gitea_policy_assignment_values") + assert module._looks_sensitive_assignment_value(value) is expected + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("single", False), + ("add regression coverage", True), + ("add regression_coverage", False), + ("ordinary engineering prose", False), + ], +) +def test_prose_classifier_covers_length_punctuation_and_lead_words(value, expected): + module = _policy("gitea_policy_prose") + assert module._looks_like_prose(value) is expected + + +def test_structured_json_walk_covers_lists_nonstring_keys_and_limits(): + module = _policy("gitea_policy_json_walk") + assert module._json_value_has_sensitive_assignment({1: "ignored"}) is False + assert module._json_value_has_sensitive_assignment([{"release": "safe"}]) is False + assert ( + module._json_value_has_sensitive_assignment({"client_secret": "value"}) is True + ) + assert ( + module._json_value_has_sensitive_assignment( + {"outer": {"client_secret": "value"}} + ) + is True + ) + assert ( + module._json_value_has_sensitive_assignment({"type": "service-account"}) is True + ) + with pytest.raises(module.PolicyError, match="scan limit"): + module._json_value_has_sensitive_assignment([], depth=33) + with pytest.raises(module.PolicyError, match="scan limit"): + module._json_value_has_sensitive_assignment([], nodes=[2048]) + + +def test_json_key_decoder_and_embedded_document_scanner_cover_failures(): + module = _policy("gitea_policy_json_decoder") + assert module._decode_json_key("client\\u005fsecret") == ("client_secret", True) + decoded, valid = module._decode_json_key("client\\qsecret") + assert valid is False and "client" in decoded + assert list(module._decoded_json_documents("prose only")) == [] + assert list(module._decoded_json_documents("x {bad y [1, 2]")) == [[1, 2]] + with pytest.raises(module.PolicyError, match="scan limit"): + list(module._decoded_json_documents("{" * 33)) + assert ( + module._has_structured_sensitive_assignment('"client\\qsecret": value') is True + ) + assert ( + module._has_structured_sensitive_assignment("type:\n service-account") is True + ) + + +def test_high_entropy_detector_covers_long_mixed_and_low_entropy_values(): + module = _policy("gitea_policy_entropy") + assert module._has_high_entropy_token("A1_" * 100) is True + assert ( + module._has_high_entropy_token( + "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789+/_=" + ) + is True + ) + assert module._has_high_entropy_token("a" * 64) is False + assert module._has_high_entropy_token("short prose") is False + + +@pytest.mark.parametrize( + "query", + [ + "bad", + "a=1&b=2&c=3&d=4", + "page=1&page=2", + "=value", + "unexpected=1", + "page=abc", + "page=01", + "page=10001", + "limit=51", + "state=merged", + ], +) +def test_query_validator_covers_each_rejection_class(query): + module = _policy("gitea_policy_query") + target = urllib.parse.urlsplit("/api/v1/repos/atlas/cassandra/pulls?" + query) + with pytest.raises(module.PolicyError): + module._validate_query(target, {"page", "limit", "state"}) + + +def test_draft_title_rejects_prefix_without_content(): + module = _policy("gitea_policy_empty_draft") + with pytest.raises(module.PolicyError, match="after the draft prefix"): + module._draft_title("WIP:") + + +def test_query_validator_rejects_noncanonical_raw_form(): + module = _policy("gitea_policy_raw_query") + target = urllib.parse.SplitResult("", "", "/api/v1/repos/atlas/cassandra", "%", "") + with pytest.raises(module.PolicyError, match="canonical ASCII"): + module._validate_query(target, set()) diff --git a/testing/tests/test_hermes_gitea_pr_client.py b/testing/tests/test_hermes_gitea_pr_client.py index 447f3067..ae419e10 100644 --- a/testing/tests/test_hermes_gitea_pr_client.py +++ b/testing/tests/test_hermes_gitea_pr_client.py @@ -1,75 +1,12 @@ -"""Adversarial contracts for the Atlas-only draft pull-request client.""" +"""Route and request-boundary contracts for the Atlas Gitea client.""" from __future__ import annotations -import copy -import importlib.util -import json -import sys import urllib.request -from pathlib import Path import pytest -ROOT = Path(__file__).parents[2] -CLIENT_PATH = ROOT / "services/hermes/scripts/gitea_api.py" -HEAD_SHA = "465cf9146b05c174a2a8d310aff6c64be58277b6" - - -def _load(): - spec = importlib.util.spec_from_file_location("safe_gitea_api", CLIENT_PATH) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - sys.modules[spec.name] = module - spec.loader.exec_module(module) - return module - - -def _draft_payload(**updates): - payload = { - "base": "main", - "body": "Review evidence", - "head": "hermes/review-fix", - "title": "WIP: Repair review findings", - } - payload.update(updates) - return payload - - -def _draft_response(**updates): - response = { - "number": 3, - "state": "open", - "draft": True, - "merged": False, - "html_url": "https://scm.bstein.dev/atlas/cassandra/pulls/3", - "url": "https://scm.bstein.dev/atlas/cassandra/pulls/3", - "title": "WIP: Focused fix", - "body": "Review evidence", - "base": {"ref": "main", "repo": {"full_name": "atlas/cassandra"}}, - "head": { - "ref": "hermes/fix", - "sha": HEAD_SHA, - "repo": {"full_name": "atlas/cassandra"}, - }, - } - response.update(updates) - return response - - -class Response: - def __init__(self, body: object): - self.body = body if isinstance(body, bytes) else json.dumps(body).encode() - - def __enter__(self): - return self - - def __exit__(self, *_args): - return False - - def read(self, limit=-1): - return self.body if limit < 0 else self.body[:limit] - +from testing.tests.test_hermes_gitea_support import HEAD_SHA, Response, _load def test_redirect_handler_rejects_cross_origin_with_sentinel_authorization(): client = _load() @@ -101,6 +38,10 @@ def test_redirect_handler_rejects_cross_origin_with_sentinel_authorization(): ("https://evil.example", "/api/v1/repos/atlas/cassandra"), ("http://scm.bstein.dev", "/api/v1/repos/atlas/cassandra"), ("https://scm.bstein.dev:443", "/api/v1/repos/atlas/cassandra"), + ("https://scm.bstein.dev/", "/api/v1/repos/atlas/cassandra"), + ("HTTPS://scm.bstein.dev", "/api/v1/repos/atlas/cassandra"), + ("https://SCM.bstein.dev", "/api/v1/repos/atlas/cassandra"), + ("https://user@scm.bstein.dev", "/api/v1/repos/atlas/cassandra"), ("https://scm.bstein.dev", "https://evil.example/api/v1/repos/atlas/cassandra"), ("https://scm.bstein.dev", "/api/v1/repos/evil/cassandra"), ("https://scm.bstein.dev", "/api/v1/repos/%61tlas/cassandra"), @@ -114,6 +55,72 @@ def test_host_owner_and_path_escape_attempts_are_rejected(base_url: str, path: s client.build_request("GET", path, base_url=base_url, token="secret") +@pytest.mark.parametrize( + "path", + [ + "/api/v1/repos/atlas/cassandra/pulls/1\nHost: evil.example", + "/api/v1/repos/atlas/cassandra/pulls/1\r\nX-Test: value", + "/api/v1/repos/atlas/cassandra/pulls/1\tignored", + "/api/v1/repos/atlas/cassandra/pulls/1\x00ignored", + "/api/v1/repos/atlas/cassandra/pulls/1\x1fignored", + "/api/v1/repos/atlas/cassandra/pulls/1\x7fignored", + "/api/v1/repos/atlas/cassandra\\pulls\\1", + "/api/v1/repos/atlas/cassandra/pulls/%31", + "/api/v1/repos/atlas/cassandra/pulls/１", + " https://scm.bstein.dev/api/v1/repos/atlas/cassandra", + "https://scm.bstein.dev/api/v1/repos/atlas/cassandra", + ], +) +def test_raw_noncanonical_target_is_rejected_before_urlsplit_and_opener( + path: str, monkeypatch +): + client = _load() + split_called = False + opener_called = False + original_urlsplit = client.urllib.parse.urlsplit + + def urlsplit(*args, **kwargs): + nonlocal split_called + split_called = True + return original_urlsplit(*args, **kwargs) + + def opener(*_args, **_kwargs): + nonlocal opener_called + opener_called = True + return Response(b"{}") + + monkeypatch.setattr(client.urllib.parse, "urlsplit", urlsplit) + with pytest.raises(client.PolicyError): + client.read(path, token="runtime", opener=opener) + assert split_called is False + assert opener_called is False + + +@pytest.mark.parametrize( + "path", + [ + "/api/v1/repos/atlas/cassandra/pulls/1?", + "//scm.bstein.dev/api/v1/repos/atlas/cassandra", + "/api/v1/repos/atlas/cassandra/./pulls/1", + "/api/v1/repos/atlas/cassandra/../admin", + "/api/v1/repos/atlas/cassandra//pulls/1", + "/api/v1/repos/atlas/cassandra/pulls/1?limit=01", + ], +) +def test_noncanonical_round_trip_or_segments_never_reach_opener(path: str): + client = _load() + opener_called = False + + def opener(*_args, **_kwargs): + nonlocal opener_called + opener_called = True + return Response(b"{}") + + with pytest.raises(client.PolicyError): + client.read(path, token="runtime", opener=opener) + assert opener_called is False + + @pytest.mark.parametrize( "path", [ @@ -180,226 +187,103 @@ def test_read_query_is_bounded(path: str): @pytest.mark.parametrize( - ("method", "path", "data"), + "query", [ - ("DELETE", "/api/v1/repos/atlas/cassandra/pulls/4", None), - ("POST", "/api/v1/repos/atlas/cassandra/pulls/4/merge", {}), - ( - "POST", - "/api/v1/repos/atlas/cassandra/pulls/4/reviews", - {"event": "APPROVED"}, - ), - ("PATCH", "/api/v1/repos/atlas/cassandra/pulls/4", {"title": "WIP: x"}), - ("PUT", "/api/v1/repos/atlas/cassandra/branches/main", {}), + "page=0", + "page=10001", + "page=01", + "limit=0", + "limit=51", + "limit=01", + "page=" + "9" * 4000, + "page=０", + "page=%EF%BC%90", + "state=%6fpen", + "p%61ge=1", + "page=1&page=2", + "page=1&" + "x" * 17 + "=1", + "state=" + "x" * 17, + "page=1&limit=2&state=open&extra=3", ], ) -def test_merge_approve_close_delete_update_and_other_mutations_are_rejected( - method: str, path: str, data: object -): +def test_read_query_requires_canonical_bounded_ascii(query: str): client = _load() with pytest.raises(client.PolicyError): - client.build_request( - method, path, base_url=client.CANONICAL_BASE_URL, token="secret", data=data + client.authorize_request( + "GET", f"/api/v1/repos/atlas/cassandra/pulls?{query}", None ) @pytest.mark.parametrize( - "ref", + "suffix", [ - "foo/.bar", - "foo/bar.lock/baz", - "foo..bar", - "foo@{bar", - "foo//bar", - "-danger", - "danger.", - "danger~one", - "danger^one", - "danger:one", - "danger one", + "pulls/{number}", + "pulls/{number}.patch", + "pulls/{number}.diff", + "pulls/{number}/commits", + "pulls/{number}/files", + "commits/{number}/status", + "commits/{number}/statuses", + "statuses/{number}", + "branches/{number}", ], ) -def test_complete_git_ref_validation_rejects_invalid_names(ref: str): - client = _load() - - with pytest.raises(client.PolicyError): - client._validate_ref(ref, "head") - - -def test_git_ref_validation_uses_fixed_trusted_binary(): - client = _load() - - assert client.GIT_BIN == "/usr/bin/git" - assert client._validate_ref("hermes/valid-fix", "head") == "hermes/valid-fix" - - -def test_create_forces_draft_title_and_same_repository_branch_names(): - client = _load() - assert ( - client.authorize_request( - "POST", "/api/v1/repos/atlas/cassandra/pulls", _draft_payload() - ) - == "create-draft" - ) - with pytest.raises(client.PolicyError, match="draft-title prefix"): - client.authorize_request( - "POST", - "/api/v1/repos/atlas/cassandra/pulls", - _draft_payload(title="Not a draft"), - ) - with pytest.raises(client.PolicyError): - client.authorize_request( - "POST", - "/api/v1/repos/atlas/cassandra/pulls", - _draft_payload(head="someone:branch"), - ) - - -def test_runtime_token_is_only_an_authorization_header(): - client = _load() - request = client.build_request( - "POST", - "/api/v1/repos/atlas/cassandra/pulls", - base_url=client.CANONICAL_BASE_URL, - token="do-not-leak", - data=_draft_payload(), - ) - - assert "do-not-leak" not in request.full_url - assert b"do-not-leak" not in request.data - assert request.get_header("Authorization") == "token do-not-leak" - - -@pytest.mark.parametrize( - "body", - [ - "pass" + "word=not-a-real-credential", - "Authorization: " + "Bearer not-a-real-credential-value", - "token: " + "ghp_" + "notarealcredentialvalue123456", - "-----BEGIN OPENSSH " + "PRIVATE KEY-----", - "eyJnotarealheader." + "notarealpayloadvalue." + "notarealsignature", - ], -) -def test_body_rejects_common_credential_shapes(body: str): - client = _load() - - with pytest.raises(client.PolicyError, match="credential material"): - client._validate_body(body) - - -def test_create_rejects_exact_runtime_token_before_network(): +def test_oversized_numeric_or_captured_path_never_reaches_opener(suffix: str): client = _load() called = False def opener(*_args, **_kwargs): nonlocal called called = True - return Response(_draft_response()) + return Response(b"{}") - with pytest.raises(client.PolicyError, match="runtime credential"): - client.create_draft( - "cassandra", - base="main", - head="hermes/fix", - head_sha=HEAD_SHA, - title="Focused fix", - body="accidental runtime-sentinel value", - token="runtime-sentinel", + path = "/api/v1/repos/atlas/cassandra/" + suffix.format(number="9" * 4000) + with pytest.raises(client.PolicyError): + client.read(path, token="runtime", opener=opener) + assert called is False + + +@pytest.mark.parametrize( + "number", + ["0", "01", "2147483648", "１２", "%31", "12345678901"], +) +@pytest.mark.parametrize("tail", ["", ".patch", ".diff", "/commits", "/files"]) +def test_noncanonical_or_out_of_range_pr_number_never_reaches_opener( + number: str, tail: str +): + client = _load() + called = False + + def opener(*_args, **_kwargs): + nonlocal called + called = True + return Response(b"{}") + + with pytest.raises(client.PolicyError): + client.read( + f"/api/v1/repos/atlas/cassandra/pulls/{number}{tail}", + token="runtime", opener=opener, ) assert called is False -def test_create_verifies_every_server_postcondition(): +def test_maximum_bounded_pr_number_is_readable(): client = _load() - calls = [] + called = False - def opener(request, timeout): - calls.append((request, timeout)) - return Response(_draft_response()) + def opener(*_args, **_kwargs): + nonlocal called + called = True + return Response(b"{}") - result = client.create_draft( - "cassandra", - base="main", - head="hermes/fix", - head_sha=HEAD_SHA, - title="Focused fix", - body="Review evidence", - token="runtime", - opener=opener, - ) - - assert json.loads(result) == _draft_response() - payload = json.loads(calls[0][0].data) - assert payload == { - "base": "main", - "body": "Review evidence", - "head": "hermes/fix", - "title": "WIP: Focused fix", - } - assert calls[0][1] == 30 - - -def test_create_postcondition_rejects_every_material_mismatch(): - client = _load() - mutations = [ - ("number", 0), - ("state", "closed"), - ("draft", False), - ("merged", True), - ("html_url", "https://evil.example/pulls/3"), - ("url", "https://evil.example/api/pulls/3"), - ("title", "Focused fix"), - ("body", "different"), - ] - documents = [] - for key, value in mutations: - document = _draft_response() - document[key] = value - documents.append(document) - for path, value in [ - (("base", "ref"), "master"), - (("base", "repo", "full_name"), "evil/cassandra"), - (("head", "ref"), "other"), - (("head", "sha"), "0" * 40), - (("head", "repo", "full_name"), "evil/cassandra"), - ]: - document = copy.deepcopy(_draft_response()) - target = document - for key in path[:-1]: - target = target[key] - target[path[-1]] = value - documents.append(document) - - for document in documents: - with pytest.raises(client.PolicyError): - client._require_create_response( - json.dumps(document).encode(), - repo="cassandra", - base="main", - head="hermes/fix", - head_sha=HEAD_SHA, - title="WIP: Focused fix", - body="Review evidence", - ) - - -def test_read_response_is_bounded(): - client = _load() - - with pytest.raises(client.PolicyError, match="safe size limit"): + assert ( client.read( - "/api/v1/repos/atlas/cassandra", + "/api/v1/repos/atlas/cassandra/pulls/2147483647", token="runtime", - opener=lambda *_a, **_k: Response(b"x" * (client.MAX_RESPONSE_BYTES + 1)), + opener=opener, ) - - -def test_output_redaction_covers_exact_token_and_authorization_header(): - client = _load() - raw = b'{"message":"do-not-leak","debug":"Authorization: token do-not-leak"}' - redacted = client.redact_bytes(raw, "do-not-leak") - - assert b"do-not-leak" not in redacted - assert redacted.count(b"[REDACTED]") >= 1 + == b"{}" + ) + assert called is True diff --git a/testing/tests/test_hermes_gitea_pr_creation.py b/testing/tests/test_hermes_gitea_pr_creation.py new file mode 100644 index 00000000..ad2967ed --- /dev/null +++ b/testing/tests/test_hermes_gitea_pr_creation.py @@ -0,0 +1,203 @@ +"""Draft-creation input contracts for the Atlas Gitea client.""" + +from __future__ import annotations + +import pytest + +from testing.tests.test_hermes_gitea_support import ( + HEAD_SHA, + Response, + _draft_payload, + _draft_response, + _load, +) + + +@pytest.mark.parametrize( + ("method", "path", "data"), + [ + ("DELETE", "/api/v1/repos/atlas/cassandra/pulls/4", None), + ("POST", "/api/v1/repos/atlas/cassandra/pulls/4/merge", {}), + ( + "POST", + "/api/v1/repos/atlas/cassandra/pulls/4/reviews", + {"event": "APPROVED"}, + ), + ("PATCH", "/api/v1/repos/atlas/cassandra/pulls/4", {"title": "WIP: x"}), + ("PUT", "/api/v1/repos/atlas/cassandra/branches/main", {}), + ], +) +def test_merge_approve_close_delete_update_and_other_mutations_are_rejected( + method: str, path: str, data: object +): + client = _load() + + with pytest.raises(client.PolicyError): + client.build_request( + method, path, base_url=client.CANONICAL_BASE_URL, token="secret", data=data + ) + + +@pytest.mark.parametrize( + "ref", + [ + "foo/.bar", + "foo/bar.lock/baz", + "foo..bar", + "foo@{bar", + "foo//bar", + "-danger", + "danger.", + "danger~one", + "danger^one", + "danger:one", + "danger one", + ], +) +def test_complete_git_ref_validation_rejects_invalid_names(ref: str): + client = _load() + + with pytest.raises(client.PolicyError): + client._validate_ref(ref, "head") + + +def test_git_ref_validation_uses_fixed_trusted_binary(): + client = _load() + + assert client._validate_ref.__globals__["GIT_BIN"] == "/usr/bin/git" + assert client._validate_ref("hermes/valid-fix", "head") == "hermes/valid-fix" + + +@pytest.mark.parametrize("field", ["base", "head"]) +@pytest.mark.parametrize("oversized", ["r" * 100_000, "🧪" * 128]) +def test_oversized_ref_never_invokes_git_request_or_opener( + field: str, oversized: str, monkeypatch +): + client = _load() + git_called = False + request_built = False + opener_called = False + original_build_request = client.build_request + + def git_run(*_args, **_kwargs): + nonlocal git_called + git_called = True + raise AssertionError("Git must not receive an oversized ref") + + def build_request(*args, **kwargs): + nonlocal request_built + request_built = True + return original_build_request(*args, **kwargs) + + def opener(*_args, **_kwargs): + nonlocal opener_called + opener_called = True + return Response(_draft_response()) + + monkeypatch.setattr(client._validate_ref.__globals__["subprocess"], "run", git_run) + client.build_request = build_request + refs = {"base": "main", "head": "hermes/fix"} + refs[field] = oversized + with pytest.raises(client.PolicyError, match="branch-name limit"): + client.create_draft( + "cassandra", + base=refs["base"], + head=refs["head"], + head_sha=HEAD_SHA, + title="Focused fix", + body="Review evidence", + token="runtime", + opener=opener, + ) + assert git_called is False + assert request_built is False + assert opener_called is False + + +@pytest.mark.parametrize( + ("field", "oversized"), + [("repo", "r" * 101), ("title", "🧪" * 200), ("body", "🧪" * 9_000)], +) +def test_other_text_bounds_fail_before_git_request_or_opener( + field: str, oversized: str, monkeypatch +): + client = _load() + git_called = False + request_built = False + opener_called = False + original_build_request = client.build_request + + def git_run(*_args, **_kwargs): + nonlocal git_called + git_called = True + raise AssertionError("Git must not run before cheap input bounds") + + def build_request(*args, **kwargs): + nonlocal request_built + request_built = True + return original_build_request(*args, **kwargs) + + def opener(*_args, **_kwargs): + nonlocal opener_called + opener_called = True + return Response(_draft_response()) + + monkeypatch.setattr(client._validate_ref.__globals__["subprocess"], "run", git_run) + client.build_request = build_request + values = { + "repo": "cassandra", + "title": "Focused fix", + "body": "Review evidence", + } + values[field] = oversized + with pytest.raises(client.PolicyError): + client.create_draft( + values["repo"], + base="main", + head="hermes/fix", + head_sha=HEAD_SHA, + title=values["title"], + body=values["body"], + token="runtime", + opener=opener, + ) + assert git_called is False + assert request_built is False + assert opener_called is False + + +def test_create_forces_draft_title_and_same_repository_branch_names(): + client = _load() + assert ( + client.authorize_request( + "POST", "/api/v1/repos/atlas/cassandra/pulls", _draft_payload() + ) + == "create-draft" + ) + with pytest.raises(client.PolicyError, match="draft-title prefix"): + client.authorize_request( + "POST", + "/api/v1/repos/atlas/cassandra/pulls", + _draft_payload(title="Not a draft"), + ) + with pytest.raises(client.PolicyError): + client.authorize_request( + "POST", + "/api/v1/repos/atlas/cassandra/pulls", + _draft_payload(head="someone:branch"), + ) + + +def test_runtime_token_is_only_an_authorization_header(): + client = _load() + request = client.build_request( + "POST", + "/api/v1/repos/atlas/cassandra/pulls", + base_url=client.CANONICAL_BASE_URL, + token="do-not-leak", + data=_draft_payload(), + ) + + assert "do-not-leak" not in request.full_url + assert b"do-not-leak" not in request.data + assert request.get_header("Authorization") == "token do-not-leak" diff --git a/testing/tests/test_hermes_gitea_pr_dlp.py b/testing/tests/test_hermes_gitea_pr_dlp.py new file mode 100644 index 00000000..4a2b1d27 --- /dev/null +++ b/testing/tests/test_hermes_gitea_pr_dlp.py @@ -0,0 +1,317 @@ +"""Credential-loss-prevention contracts for Atlas draft creation.""" + +from __future__ import annotations + +import pytest + +from testing.tests.test_hermes_gitea_support import ( + HEAD_SHA, + Response, + _draft_response, + _load, +) + + +@pytest.mark.parametrize( + "sensitive", + [ + "client_" + "secret=not-a-real-value", + '{"client_' + 'secret":\n"synthetic-value"}', + '{"client_' + 'secret"\n:\n"synthetic-value"}', + '{"client\\u005f' + 'secret":"synthetic-value"}', + "client_" + "secret:\n synthetic-value", + '{"client\\q' + 'secret":"synthetic-value"}', + '{"client\n' + 'secret":"synthetic-value"}', + '"client_' + 'secret"\x0b:\n"synthetic-value"', + "ACCESS_" + "TOKEN = 'not-a-real-value'", + '{"refresh_' + 'token": "not-a-real-value"}', + "private_" + "key: not-a-real-value", + "AWS_SECRET_ACCESS_" + "KEY=not-a-real-value", + "aws_access_key_" + "id: not-a-real-value", + "AWS_SESSION_" + "TOKEN = 'not-a-real-value'", + '{"AccessKey' + 'Id":"not-a-real-value"}', + '{"SecretAccess' + 'Key":"not-a-real-value"}', + '{"Session' + 'Token":"not-a-real-value"}', + "aws-security-" + "token: not-a-real-value", + "Account" + "Key=not-a-real-value", + "SharedAccess" + "Signature: not-a-real-value", + "AZURE_STORAGE_CONNECTION_" + "STRING='not-a-real-value'", + "DefaultEndpointsProtocol=https;AccountName=fake;Account" + + "Key=not-a-real-value;EndpointSuffix=example", + "?sv=2024-11-04&ss=b&srt=sco&sp=rwdlac&se=2099-01-01&sig=" + "not-a-real-value", + "DOCKER_AUTH_" + "CONFIG='not-a-real-value'", + '{"auths":{"registry.example":{"auth":"bm90LXJlYWw="}}}', + '{"identity' + 'token":"not-a-real-value"}', + '{"type":"service_' + 'account","client_email":"fake@example.test"}', + '{"private_key_' + 'id":"not-a-real-value"}', + '{"client_' + 'email":"fake@example.test"}', + "GOOGLE_CREDENTIALS" + "=not-a-real-value", + "personal_access_" + "token: not-a-real-value", + "GITEA_" + "TOKEN=not-a-real-value", + "gitlab-token" + ": not-a-real-value", + "pat" + "=not-a-real-value", + "Authorization: " + "Bearer not-a-real-credential-value", + "authorization = " + '"Basic not-a-real-credential-value"', + "Bearer" + "=not-a-real-credential-value", + "Basic" + ": not-a-real-credential-value", + "pass" + "word=not-a-real-credential", + "ghp_" + "notarealcredentialvalue123456", + "github_pat_" + "notarealcredentialvalue123456", + "glpat-" + "notarealcredentialvalue123456", + "xoxb-" + "not-a-real-credential-value-123456", + "sk-ant-" + "notarealcredentialvalue123456", + "sk-proj-" + "notarealcredentialvalue123456", + "sk_live_" + "notarealcredentialvalue123456", + "ya29." + "notarealcredentialvalue123456", + "gta_" + "notarealcredentialvalue123456", + "whsec_" + "notarealcredentialvalue123456", + "npm_" + "notarealcredentialvalue123456", + "pypi-" + "notarealcredentialvalue123456789012345", + "hf_" + "notarealcredentialvalue123456", + "SG." + "notarealvalue1234" + ".notarealcredentialvalue123456", + "SK" + "a" * 32, + "https://hooks.slack.com/services/" + "T000/B000/notarealvalue123456", + "https://discord.com/api/webhooks/123456789/" + "notarealcredentialvalue123456", + "https://fake.webhook.office.com/" + "notarealcredentialvalue123456", + "webhook_" + "url=https://example.test/not-real", + "FutureCloudSigning" + "Credential=not-a-real-value", + "future-client-signing-" + "key: not-a-real-value", + "future_client_signing_" + "key='not-a-real-value'", + '{"serviceAccountPrivate' + 'Key":"not-a-real-value"}', + "CONTAINER_REGISTRY_" + "CREDENTIAL=not-a-real-value", + "someWebhookSigning" + "Secret: not-a-real-value", + "client" + "Key=not-a-real-value", + "session" + "Key: not-a-real-value", + "access" + "Id=not-a-real-value", + "credentials" + ": {user: fake}", + "private" + "Key: |", + "nuget_api_" + "key=not-a-real-value", + "oy2" + "a" * 44, + "sk_test_" + "notarealcredentialvalue123456", + "A1b2C3d4E5f6G7h8I9j0K_l-M+n/O=pQ2rS3tU4vW5xY6zZ7aB8cC9d", + "AKIA" + "A" * 16, + "AIza" + "a" * 35, + "-----BEGIN OPENSSH " + "PRIVATE KEY-----", + "ssh-ed25519 " + "bm90YXJlYWxjcmVkZW50aWFsdmFsdWU=", + "eyJnotarealheader." + "notarealpayloadvalue." + "notarealsignature", + ], +) +@pytest.mark.parametrize("field", ["title", "body"]) +def test_create_rejects_expanded_credential_shapes_before_network( + sensitive: str, field: str +): + client = _load() + called = False + request_built = False + + original_build_request = client.build_request + + def build_request(*args, **kwargs): + nonlocal request_built + request_built = True + return original_build_request(*args, **kwargs) + + def opener(*_args, **_kwargs): + nonlocal called + called = True + return Response(_draft_response()) + + client.build_request = build_request + values = {"title": "Focused fix", "body": "Review evidence"} + values[field] = sensitive + + with pytest.raises(client.PolicyError, match="credential material"): + client.create_draft( + "cassandra", + base="main", + head="hermes/fix", + head_sha=HEAD_SHA, + title=values["title"], + body=values["body"], + token="runtime-sentinel", + opener=opener, + ) + assert request_built is False + assert called is False + + +def test_very_long_compact_body_is_rejected_before_request_or_network(): + client = _load() + request_built = False + opener_called = False + original_build_request = client.build_request + + def build_request(*args, **kwargs): + nonlocal request_built + request_built = True + return original_build_request(*args, **kwargs) + + def opener(*_args, **_kwargs): + nonlocal opener_called + opener_called = True + return Response(_draft_response()) + + client.build_request = build_request + with pytest.raises(client.PolicyError, match="credential material"): + client.create_draft( + "cassandra", + base="main", + head="hermes/fix", + head_sha=HEAD_SHA, + title="Focused fix", + body="a" * 300, + token="runtime-sentinel", + opener=opener, + ) + assert request_built is False + assert opener_called is False + + +@pytest.mark.parametrize( + "safe_text", + [ + "AWS_SECRET_ACCESS_KEY is injected at runtime", + "Token: reject empty values", + "Authorization = preserve header behavior", + "Password: add regression", + "Review the Authorization header behavior", + "Bearer authentication is required for this route", + "Document Docker auths payload rejection", + "The client_email field belongs to service accounts", + "AccountKey assignments must be blocked", + "This patch changes token validation without including a value", + "FutureCloudSigningCredential handling needs a regression test", + "The clientKey name is documented without an assigned value", + "A SHA-256 digest 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef is evidence", + ], +) +def test_credential_policy_keeps_normal_engineering_prose_usable(safe_text: str): + client = _load() + + assert client._validate_body(safe_text) == safe_text + assert client._draft_title(safe_text) == f"WIP: {safe_text}" + + +@pytest.mark.parametrize("field", ["title", "body"]) +def test_create_rejects_exact_runtime_token_before_network(field: str): + client = _load() + called = False + request_built = False + + original_build_request = client.build_request + + def build_request(*args, **kwargs): + nonlocal request_built + request_built = True + return original_build_request(*args, **kwargs) + + def opener(*_args, **_kwargs): + nonlocal called + called = True + return Response(_draft_response()) + + client.build_request = build_request + runtime_token = "exact-random-runtime-sentinel-7b73ac61" + values = {"title": "Focused fix", "body": "Review evidence"} + values[field] = f"Accidental {runtime_token} value" + with pytest.raises(client.PolicyError, match="runtime credential"): + client.create_draft( + "cassandra", + base="main", + head="hermes/fix", + head_sha=HEAD_SHA, + title=values["title"], + body=values["body"], + token=runtime_token, + opener=opener, + ) + assert request_built is False + assert called is False + + +@pytest.mark.parametrize("field", ["repo", "base", "head", "title", "body"]) +def test_every_public_field_rejects_exact_runtime_token_before_git_or_network( + field: str, monkeypatch +): + client = _load() + git_called = False + request_built = False + opener_called = False + runtime_token = "runtime-sentinel" + values = { + "repo": "cassandra", + "base": "main", + "head": "hermes/fix", + "title": "Focused fix", + "body": "Review evidence", + } + values[field] = runtime_token + + def git_run(*_args, **_kwargs): + nonlocal git_called + git_called = True + raise AssertionError("Git must not run for a credential-bearing field") + + def build_request(*_args, **_kwargs): + nonlocal request_built + request_built = True + raise AssertionError("a request must not be built") + + def opener(*_args, **_kwargs): + nonlocal opener_called + opener_called = True + raise AssertionError("the opener must not be called") + + monkeypatch.setattr(client._validate_ref.__globals__["subprocess"], "run", git_run) + monkeypatch.setattr(client, "build_request", build_request) + with pytest.raises(client.PolicyError, match="runtime credential"): + client.create_draft( + values["repo"], + base=values["base"], + head=values["head"], + head_sha=HEAD_SHA, + title=values["title"], + body=values["body"], + token=runtime_token, + opener=opener, + ) + assert git_called is False + assert request_built is False + assert opener_called is False + + +@pytest.mark.parametrize( + "sensitive", + [ + " ".join(["{}"] * 32) + ' {"client_secret":"synthetic-value"}', + "prefix_" + "ghp_" + "notarealcredentialvalue123456_suffix", + "client_secret: correct horse battery staple", + "'client_secret':\n synthetic-value", + ], +) +@pytest.mark.parametrize("field", ["title", "body"]) +def test_structured_scanner_closes_bounded_and_wrapped_token_bypasses( + sensitive: str, field: str +): + client = _load() + values = {"title": "Focused fix", "body": "Review evidence"} + values[field] = sensitive + called = False + + def opener(*_args, **_kwargs): + nonlocal called + called = True + return Response(_draft_response()) + + with pytest.raises(client.PolicyError, match="credential material"): + client.create_draft( + "cassandra", + base="main", + head="hermes/fix", + head_sha=HEAD_SHA, + title=values["title"], + body=values["body"], + token="runtime-sentinel", + opener=opener, + ) + assert called is False diff --git a/testing/tests/test_hermes_gitea_pr_integration.py b/testing/tests/test_hermes_gitea_pr_integration.py index d3b47eef..ee482cfe 100644 --- a/testing/tests/test_hermes_gitea_pr_integration.py +++ b/testing/tests/test_hermes_gitea_pr_integration.py @@ -13,8 +13,10 @@ import pytest import yaml ROOT = Path(__file__).parents[2] -CLIENT_PATH = ROOT / "services/hermes/scripts/gitea_api.py" +CLIENT_PATH = ROOT / "services/hermes/scm-common/scripts/gitea_api.py" HEAD_SHA = "465cf9146b05c174a2a8d310aff6c64be58277b6" +if str(CLIENT_PATH.parent) not in sys.path: + sys.path.insert(0, str(CLIENT_PATH.parent)) def _load(): @@ -96,12 +98,12 @@ def test_http_error_path_redacts_token(monkeypatch, capsys): assert client.main(["read", "/api/v1/repos/atlas/cassandra"]) == 1 captured = capsys.readouterr() assert "do-not-leak" not in captured.err - assert "HTTP 403" in captured.err + assert "credential was disclosed" in captured.err -def test_flux_manifest_projects_runtime_vault_token_and_skill_only(): +def test_flux_manifest_isolates_vault_token_in_separate_broker_only(): client_source = CLIENT_PATH.read_text(encoding="utf-8") - assert "/runtime-access/gitea-token" in client_source + assert "scm_broker_client" in client_source assert "GITEA_TOKEN" not in client_source deployment = yaml.safe_load( @@ -109,9 +111,7 @@ def test_flux_manifest_projects_runtime_vault_token_and_skill_only(): ) template = deployment["spec"]["template"] annotations = template["metadata"]["annotations"] - assert annotations["vault.hashicorp.com/agent-inject-secret-gitea-token"] == ( - "kv/data/atlas/hermes/developer-gitea" - ) + assert not any("gitea" in key.lower() for key in annotations) runtime = next( volume for volume in template["spec"]["volumes"] @@ -136,6 +136,17 @@ def test_flux_manifest_projects_runtime_vault_token_and_skill_only(): kustomization = yaml.safe_load( (ROOT / "services/hermes/kustomization.yaml").read_text(encoding="utf-8") ) + common = yaml.safe_load( + (ROOT / "services/hermes/scm-common/kustomization.yaml").read_text( + encoding="utf-8" + ) + ) + boundary = common["configMapGenerator"][0] + assert boundary["name"] == "hermes-scm-boundary" + assert "gitea_api.py=scripts/gitea_api.py" in boundary["files"] + assert "gitea_api_policy.py=scripts/gitea_api_policy.py" in boundary["files"] + assert "scm_broker_client.py=scripts/scm_broker_client.py" in boundary["files"] + assert not any("gitea_askpass" in item for item in boundary["files"]) generator = next( item for item in kustomization["configMapGenerator"] @@ -145,3 +156,19 @@ def test_flux_manifest_projects_runtime_vault_token_and_skill_only(): "SKILL.md=skills/manage-atlas-pull-requests/SKILL.md", "openai.yaml=skills/manage-atlas-pull-requests/agents/openai.yaml", ] + + broker = yaml.safe_load( + (ROOT / "services/hermes-scm-broker/deployment.yaml").read_text( + encoding="utf-8" + ) + ) + assert broker["metadata"]["namespace"] == "hermes-scm" + pod = broker["spec"]["template"] + assert pod["spec"]["serviceAccountName"] == "hermes-scm-broker" + assert pod["metadata"]["annotations"][ + "vault.hashicorp.com/agent-inject-secret-gitea-token" + ] == "kv/data/atlas/hermes/developer-gitea" + assert not any( + volume.get("hostPath") or volume.get("persistentVolumeClaim") + for volume in pod["spec"]["volumes"] + ) diff --git a/testing/tests/test_hermes_gitea_pr_postconditions.py b/testing/tests/test_hermes_gitea_pr_postconditions.py new file mode 100644 index 00000000..c06a7784 --- /dev/null +++ b/testing/tests/test_hermes_gitea_pr_postconditions.py @@ -0,0 +1,199 @@ +"""Response and postcondition contracts for Atlas draft creation.""" + +from __future__ import annotations + +import base64 +import copy +import json + +import pytest + +from testing.tests.test_hermes_gitea_support import ( + HEAD_SHA, + Response, + _draft_response, + _load, +) + +@pytest.mark.parametrize("status", [200, 202, 204, 206]) +def test_create_accepts_only_http_201(status: int): + client = _load() + + with pytest.raises(client.PolicyError, match="unexpected HTTP status"): + client.create_draft( + "cassandra", + base="main", + head="hermes/fix", + head_sha=HEAD_SHA, + title="Focused fix", + body="Review evidence", + token="runtime", + opener=lambda *_a, **_k: Response(_draft_response(), status=status), + ) + + +@pytest.mark.parametrize("status", [201, 202, 204, 206]) +def test_read_accepts_only_http_200(status: int): + client = _load() + + with pytest.raises(client.PolicyError, match="unexpected HTTP status"): + client.read( + "/api/v1/repos/atlas/cassandra", + token="runtime", + opener=lambda *_a, **_k: Response(b"{}", status=status), + ) + + +def test_successful_create_validates_each_ref_once(monkeypatch): + client = _load() + calls = [] + + def git_run(command, **_kwargs): + calls.append(command) + return type("Result", (), {"returncode": 0})() + + monkeypatch.setattr(client._validate_ref.__globals__["subprocess"], "run", git_run) + client.create_draft( + "cassandra", + base="main", + head="hermes/fix", + head_sha=HEAD_SHA, + title="Focused fix", + body="Review evidence", + token="runtime", + opener=lambda *_a, **_k: Response(_draft_response()), + ) + + assert [command[-1] for command in calls] == [ + "refs/heads/main", + "refs/heads/hermes/fix", + ] + + +def test_create_verifies_every_server_postcondition(): + client = _load() + calls = [] + + def opener(request, timeout): + calls.append((request, timeout)) + return Response(_draft_response()) + + result = client.create_draft( + "cassandra", + base="main", + head="hermes/fix", + head_sha=HEAD_SHA, + title="Focused fix", + body="Review evidence", + token="runtime", + opener=opener, + ) + + assert json.loads(result) == _draft_response() + payload = json.loads(calls[0][0].data) + assert payload == { + "base": "main", + "body": "Review evidence", + "head": "hermes/fix", + "title": "WIP: Focused fix", + } + assert calls[0][1] == 30 + + +def test_create_postcondition_rejects_every_material_mismatch(): + client = _load() + mutations = [ + ("number", 0), + ("number", 2_147_483_648), + ("state", "closed"), + ("draft", False), + ("merged", True), + ("html_url", "https://evil.example/pulls/3"), + ("url", "https://evil.example/api/pulls/3"), + ("title", "Focused fix"), + ("body", "different"), + ] + documents = [] + for key, value in mutations: + document = _draft_response() + document[key] = value + documents.append(document) + for path, value in [ + (("base", "ref"), "master"), + (("base", "repo", "full_name"), "evil/cassandra"), + (("head", "ref"), "other"), + (("head", "sha"), "0" * 40), + (("head", "repo", "full_name"), "evil/cassandra"), + ]: + document = copy.deepcopy(_draft_response()) + target = document + for key in path[:-1]: + target = target[key] + target[path[-1]] = value + documents.append(document) + + for document in documents: + with pytest.raises(client.PolicyError): + client._require_create_response( + json.dumps(document).encode(), + repo="cassandra", + base="main", + head="hermes/fix", + head_sha=HEAD_SHA, + title="WIP: Focused fix", + body="Review evidence", + ) + + +def test_read_response_is_bounded(): + client = _load() + + with pytest.raises(client.PolicyError, match="safe size limit"): + client.read( + "/api/v1/repos/atlas/cassandra", + token="runtime", + opener=lambda *_a, **_k: Response(b"x" * (client.MAX_RESPONSE_BYTES + 1)), + ) + + +def test_direct_api_rejects_unexpected_success_content_type(): + client = _load() + response = Response(b"{}") + response.headers.replace_header("Content-Type", "text/html") + + with pytest.raises(client.PolicyError, match="unexpected response type"): + client.read( + "/api/v1/repos/atlas/cassandra", + token="runtime", + opener=lambda *_a, **_k: response, + ) + + +def test_output_redaction_covers_exact_token_and_authorization_header(): + client = _load() + raw = b'{"message":"do-not-leak","debug":"Authorization: token do-not-leak"}' + redacted = client.redact_bytes(raw, "do-not-leak") + + assert b"do-not-leak" not in redacted + assert redacted.count(b"[REDACTED]") >= 1 + + +@pytest.mark.parametrize( + "reflected", + [ + b"runtime-sentinel", + base64.b64encode(b"runtime-sentinel"), + base64.b64encode(b"hermes-automation:runtime-sentinel"), + ], +) +def test_direct_api_rejects_credential_reflection(reflected: bytes): + client = _load() + + with pytest.raises(client.PolicyError, match="credential material"): + client.read( + "/api/v1/repos/atlas/cassandra", + token="runtime-sentinel", + opener=lambda *_a, **_k: Response( + b'{"unexpected":"' + reflected + b'"}' + ), + ) diff --git a/testing/tests/test_hermes_gitea_support.py b/testing/tests/test_hermes_gitea_support.py new file mode 100644 index 00000000..8b00e784 --- /dev/null +++ b/testing/tests/test_hermes_gitea_support.py @@ -0,0 +1,75 @@ +"""Shared fixtures for the Atlas-only Gitea client contracts.""" + +from __future__ import annotations + +import importlib.util +import json +import sys +from email.message import Message +from pathlib import Path + +ROOT = Path(__file__).parents[2] +CLIENT_PATH = ROOT / "services/hermes/scm-common/scripts/gitea_api.py" +HEAD_SHA = "465cf9146b05c174a2a8d310aff6c64be58277b6" +if str(CLIENT_PATH.parent) not in sys.path: + sys.path.insert(0, str(CLIENT_PATH.parent)) + + +def _load(): + spec = importlib.util.spec_from_file_location("safe_gitea_api", CLIENT_PATH) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _draft_payload(**updates): + payload = { + "base": "main", + "body": "Review evidence", + "head": "hermes/review-fix", + "title": "WIP: Repair review findings", + } + payload.update(updates) + return payload + + +def _draft_response(**updates): + response = { + "number": 3, + "state": "open", + "draft": True, + "merged": False, + "html_url": "https://scm.bstein.dev/atlas/cassandra/pulls/3", + "url": "https://scm.bstein.dev/atlas/cassandra/pulls/3", + "title": "WIP: Focused fix", + "body": "Review evidence", + "base": {"ref": "main", "repo": {"full_name": "atlas/cassandra"}}, + "head": { + "ref": "hermes/fix", + "sha": HEAD_SHA, + "repo": {"full_name": "atlas/cassandra"}, + }, + } + response.update(updates) + return response + + +class Response: + def __init__(self, body: object, status: int | None = None): + self.body = body if isinstance(body, bytes) else json.dumps(body).encode() + self.status = ( + status if status is not None else (201 if isinstance(body, dict) else 200) + ) + self.headers = Message() + self.headers["Content-Type"] = "application/json" + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def read(self, limit=-1): + return self.body if limit < 0 else self.body[:limit] diff --git a/testing/tests/test_hermes_node_account_hardening.py b/testing/tests/test_hermes_node_account_hardening.py new file mode 100644 index 00000000..9cf2bb3a --- /dev/null +++ b/testing/tests/test_hermes_node_account_hardening.py @@ -0,0 +1,252 @@ +"""Core contracts for the dedicated Hermes node SSH identity.""" + +from __future__ import annotations + +import os +from pathlib import Path + +import pytest +import yaml + +from testing.tests.test_hermes_node_account_support import ( + ROOT, + SCRIPT, + _fixture, + _key, + _load, +) + +def test_reconciler_creates_locked_groupless_account_and_moves_only_exact_key( + tmp_path: Path, monkeypatch +): + module, originals, key, other, public_key = _fixture(tmp_path, monkeypatch) + + module.reconcile(public_key) + first = { + name: (module.HOST_ETC / name).read_text(encoding="utf-8") + for name in originals + } + module.reconcile(public_key) + second = { + name: (module.HOST_ETC / name).read_text(encoding="utf-8") + for name in originals + } + + assert first == second + expected = module._expected_records() + for name, original in originals.items(): + assert first[name] == original + ":".join(expected[name]) + "\n" + backup = module.HOST_ETC / f"{name}.hermes-boundary-backup" + assert backup.read_text(encoding="utf-8") == original + assert expected["shadow"][1] == "!" + assert expected["group"][-1] == "" + assert "disk:x:6:atlas\n" in first["group"] + assert "sudo:x:27:oceanus\n" in first["group"] + + hermes_keys = ( + module.HOST_HOME / module.ACCOUNT / ".ssh/authorized_keys" + ).read_text(encoding="utf-8") + assert hermes_keys == key + "\n" + for user in module.LEGACY_ACCOUNTS: + legacy = ( + module.HOST_HOME / user / ".ssh/authorized_keys" + ).read_text(encoding="utf-8") + assert legacy == f"{other} {user}\n" + backup = module.HOST_HOME / user / ".ssh/authorized_keys.hermes-boundary-backup" + assert backup.read_text(encoding="utf-8") == f"{other} {user}\n{key}\n" + + +def test_identity_conflict_fails_closed_without_editing_account_databases( + tmp_path: Path, monkeypatch +): + module, originals, _key, _other, public_key = _fixture(tmp_path, monkeypatch) + conflict = originals["passwd"] + ( + f"unrelated:x:{module.ACCOUNT_UID}:{module.ACCOUNT_GID}:Other:/home/other:/bin/bash\n" + ) + (module.HOST_ETC / "passwd").write_text(conflict, encoding="utf-8") + + with pytest.raises(module.HardeningError, match="identity conflicts"): + module.reconcile(public_key) + + assert (module.HOST_ETC / "passwd").read_text(encoding="utf-8") == conflict + for name in ("group", "shadow", "gshadow"): + assert (module.HOST_ETC / name).read_text(encoding="utf-8") == originals[name] + assert not (module.HOST_HOME / module.ACCOUNT).exists() + + +def test_malformed_account_file_fails_closed(tmp_path: Path, monkeypatch): + module, originals, _key, _other, public_key = _fixture(tmp_path, monkeypatch) + (module.HOST_ETC / "group").write_text("malformed\n", encoding="utf-8") + + with pytest.raises(module.HardeningError, match="invalid record"): + module.reconcile(public_key) + + assert (module.HOST_ETC / "passwd").read_text(encoding="utf-8") == originals[ + "passwd" + ] + assert (module.HOST_ETC / "group").read_text(encoding="utf-8") == "malformed\n" + + +def test_preexisting_home_fails_before_account_database_or_key_changes( + tmp_path: Path, monkeypatch +): + module, originals, _key, _other, public_key = _fixture(tmp_path, monkeypatch) + home = module.HOST_HOME / module.ACCOUNT + home.mkdir() + (home / "unrelated").write_text("preserve\n", encoding="utf-8") + + with pytest.raises(module.HardeningError, match="home already exists"): + module.reconcile(public_key) + + for name, original in originals.items(): + assert (module.HOST_ETC / name).read_text(encoding="utf-8") == original + assert not (module.HOST_ETC / f"{name}.hermes-boundary-backup").exists() + assert (home / "unrelated").read_text(encoding="utf-8") == "preserve\n" + + +def test_flux_daemonset_reconciles_every_node_without_mutating_human_groups(): + documents = list( + yaml.safe_load_all( + (ROOT / "services/hermes/node-ssh-access.yaml").read_text(encoding="utf-8") + ) + ) + daemonset = next(item for item in documents if item["kind"] == "DaemonSet") + spec = daemonset["spec"]["template"]["spec"] + assert spec["tolerations"] == [{"operator": "Exists"}] + command = spec["containers"][0]["args"][0] + assert "/opt/node-hardener/node_account_hardening.py" in command + assert "sleep 300" in command + mounts = {item["name"]: item for item in spec["volumes"]} + assert mounts["host-home"]["hostPath"]["path"] == "/home" + assert mounts["host-etc"]["hostPath"]["path"] == "/etc" + assert mounts["host-k3s"]["hostPath"]["path"] == "/var/lib/rancher/k3s" + assert mounts["host-kubelet"]["hostPath"]["path"] == "/var/lib/kubelet" + assert mounts["host-run-k3s"]["hostPath"]["path"] == "/run/k3s" + assert mounts["host-run-containerd"]["hostPath"]["path"] == "/run/containerd" + assert "usermod" not in command + assert "groupmod" not in command + + policies = list( + yaml.safe_load_all( + (ROOT / "services/hermes/networkpolicy.yaml").read_text(encoding="utf-8") + ) + ) + isolation = next( + item + for item in policies + if item["metadata"]["name"] == "hermes-node-ssh-access-isolation" + ) + assert isolation["spec"]["ingress"] == [] + assert isolation["spec"]["egress"] == [] + + source = SCRIPT.read_text(encoding="utf-8") + assert "ACCOUNT = \"hermes-agent\"" in source + assert "ACCOUNT_UID = 1200" in source + assert "ACCOUNT_GID = 1200" in source + assert 'LEGACY_ACCOUNTS = ("atlas", "oceanus")' in source + assert "audit_membership" in source + image = spec["containers"][0]["image"] + assert image == ( + "python@sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df" + ) + assert mounts["host-polkit-share"]["hostPath"]["path"] == "/usr/share/polkit-1" + + +def test_sensitive_roots_get_explicit_zero_permission_acl_for_hermes( + tmp_path: Path, monkeypatch +): + module = _load() + sensitive = tmp_path / "k3s" + sensitive.mkdir(mode=0o755) + host_etc = tmp_path / "etc" + host_etc.mkdir() + monkeypatch.setattr(module, "ACCOUNT_UID", 1200) + monkeypatch.setattr(module, "HOST_ETC", host_etc) + monkeypatch.setattr(module, "HOST_ROOT_UID", os.getuid()) + monkeypatch.setattr(module, "HOST_ROOT_GID", os.getgid()) + # Production requires uid 0. The test process owns its fixture but runs the + # real xattr/ACL implementation using the test owner's uid as that boundary. + + module._deny_sensitive_root(sensitive) + first = module._read_acl(sensitive) + module._deny_sensitive_root(sensitive) + second = module._read_acl(sensitive) + + assert first == second + entries = module._decode_acl(first, 0o755) + assert (module.ACL_USER, 0, 1200) in entries + assert (module.ACL_USER_OBJ, 0o7, module.ACL_UNDEFINED_ID) in entries + assert (module.ACL_GROUP_OBJ, 0o5, module.ACL_UNDEFINED_ID) in entries + assert (module.ACL_OTHER, 0o5, module.ACL_UNDEFINED_ID) in entries + backup = host_etc / "hermes-node-boundary/k3s.acl" + assert backup.read_bytes() == b"N" + assert backup.stat().st_mode & 0o777 == 0o600 + + +def test_sensitive_root_rejects_non_directory_and_non_root_owner( + tmp_path: Path, monkeypatch +): + module = _load() + regular = tmp_path / "not-a-directory" + regular.write_text("preserve", encoding="utf-8") + with pytest.raises(module.HardeningError, match="unsafe sensitive directory"): + module._deny_sensitive_root(regular) + + directory = tmp_path / "not-root-owned" + directory.mkdir() + if directory.stat().st_uid == 0: + pytest.skip("cannot construct a non-root-owned fixture as root") + with pytest.raises(module.HardeningError, match="not root-owned"): + module._deny_sensitive_root(directory) + + +def test_acl_parser_rejects_malformed_or_incomplete_values(): + module = _load() + with pytest.raises(module.HardeningError, match="malformed"): + module._decode_acl(b"too short", 0o755) + incomplete = module._encode_acl( + [(module.ACL_USER_OBJ, 0o7, module.ACL_UNDEFINED_ID)] + ) + with pytest.raises(module.HardeningError, match="incomplete"): + module._decode_acl(incomplete, 0o755) + + +def test_acl_failure_stops_before_installing_the_ssh_key(tmp_path: Path, monkeypatch): + module = _load() + host_etc = tmp_path / "etc" + host_etc.mkdir() + monkeypatch.setattr(module, "HOST_ETC", host_etc) + monkeypatch.setattr(module, "HOST_ROOT_UID", os.getuid()) + public_key = tmp_path / "public-key" + public_key.write_text( + _key(b"synthetic-key") + "\n", + encoding="utf-8", + ) + moved = False + monkeypatch.setattr(module, "_reconcile_databases", lambda: None) + + def fail_acl(): + raise module.HardeningError("ACL unavailable") + + def move_key(_path): + nonlocal moved + moved = True + + monkeypatch.setattr(module, "_deny_sensitive_roots", fail_acl) + monkeypatch.setattr(module, "_move_key", move_key) + + with pytest.raises(module.HardeningError, match="ACL unavailable"): + module.reconcile(public_key) + assert moved is False + + +def test_runtime_ssh_config_forces_dedicated_account_for_every_titan(): + stage = ( + ROOT / "services/hermes/scripts/stage_runtime_access.py" + ).read_text(encoding="utf-8") + assert '"Host titan-*\\n User hermes-agent\\n"' in stage + agent = yaml.safe_load( + (ROOT / "services/hermes/agent-deployment.yaml").read_text(encoding="utf-8") + ) + assert "User atlas" not in yaml.safe_dump(agent) + assert "User oceanus" not in yaml.safe_dump(agent) diff --git a/testing/tests/test_hermes_node_account_io.py b/testing/tests/test_hermes_node_account_io.py new file mode 100644 index 00000000..6b05cc0a --- /dev/null +++ b/testing/tests/test_hermes_node_account_io.py @@ -0,0 +1,222 @@ +"""Crash-safety and metadata contracts for Hermes node-account writes.""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path + +import pytest +import yaml + +from testing.tests.test_hermes_node_account_support import ( + ROOT, + _fixture, + _load, + _typed_key, +) + +def test_key_identity_ignores_options_and_comments_and_removes_legacy_first( + tmp_path: Path, monkeypatch +): + module, _originals, key, other, public_key = _fixture(tmp_path, monkeypatch) + key_fields = key.split() + decorated = f'restrict,command="echo denied" {key_fields[0]} {key_fields[1]} old-comment' + for user in module.LEGACY_ACCOUNTS: + path = module.HOST_HOME / user / ".ssh/authorized_keys" + path.write_text(f"{other} {user}\n{decorated}\n", encoding="utf-8") + writes: list[Path] = [] + real_write = module.atomic_write + + def recording_write(path, value, metadata): + writes.append(path) + real_write(path, value, metadata) + + monkeypatch.setattr(module, "atomic_write", recording_write) + module.reconcile(public_key) + + key_writes = [path for path in writes if path.name == "authorized_keys"] + assert key_writes[-1].parts[-3:] == (module.ACCOUNT, ".ssh", "authorized_keys") + assert all(module.ACCOUNT not in path.parts for path in key_writes[:-1]) + + +def test_unrelated_modern_human_key_type_is_preserved(tmp_path: Path, monkeypatch): + module, _originals, key, _other, public_key = _fixture(tmp_path, monkeypatch) + modern = _typed_key("sk-ssh-ed25519@openssh.com", b"human-security-key", "human") + for user in module.LEGACY_ACCOUNTS: + path = module.HOST_HOME / user / ".ssh/authorized_keys" + path.write_text(f"{modern}\n{key}\n", encoding="utf-8") + + module.reconcile(public_key) + + for user in module.LEGACY_ACCOUNTS: + path = module.HOST_HOME / user / ".ssh/authorized_keys" + assert path.read_text() == modern + "\n" + + +def test_crash_during_legacy_key_removal_never_installs_duplicate_key( + tmp_path: Path, monkeypatch +): + module, _originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + real_write = module.atomic_write + key_writes = 0 + + def crash_on_second_key(path, value, metadata): + nonlocal key_writes + if path.name == "authorized_keys": + key_writes += 1 + if key_writes == 2: + raise module.HardeningError("synthetic crash") + real_write(path, value, metadata) + + monkeypatch.setattr(module, "atomic_write", crash_on_second_key) + with pytest.raises(module.HardeningError, match="synthetic crash"): + module.reconcile(public_key) + assert not (module.HOST_HOME / module.ACCOUNT / ".ssh/authorized_keys").exists() + + +def test_preexisting_target_key_is_removed_before_legacy_cleanup_and_reinstalled( + tmp_path: Path, monkeypatch +): + module, _originals, key, _other, public_key = _fixture(tmp_path, monkeypatch) + target = module.HOST_HOME / module.ACCOUNT / ".ssh/authorized_keys" + target.parent.mkdir(parents=True) + target.write_text(key + " existing\n") + writes: list[Path] = [] + real_write = module.atomic_write + + def recording_write(path, value, metadata): + if path.name == "authorized_keys": + writes.append(path) + real_write(path, value, metadata) + + monkeypatch.setattr(module, "atomic_write", recording_write) + module._move_key(public_key) + + assert writes[0] == target + assert writes[-1] == target + assert all(path != target for path in writes[1:-1]) + assert target.read_text() == key + "\n" + + +def test_account_database_write_preserves_mode_owner_and_xattrs( + tmp_path: Path, monkeypatch +): + module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + passwd = module.HOST_ETC / "passwd" + passwd.chmod(0o640) + os.setxattr(passwd, "user.hermes-test", b"preserve") + before = passwd.stat() + + module.reconcile(public_key) + + after = passwd.stat() + assert after.st_mode & 0o777 == 0o640 + assert (after.st_uid, after.st_gid) == (before.st_uid, before.st_gid) + assert os.getxattr(passwd, "user.hermes-test") == b"preserve" + backup = module.HOST_ETC / "passwd.hermes-boundary-backup" + assert os.getxattr(backup, "user.hermes-test") == b"preserve" + assert backup.read_text() == originals["passwd"] + + +def test_atomic_write_restores_acl_and_security_label_xattrs(tmp_path: Path, monkeypatch): + module = _load() + io_module = sys.modules[module.atomic_write.__module__] + target = tmp_path / "passwd" + target.write_bytes(b"old\n") + metadata = io_module.FileSnapshot( + value=b"old\n", + device=target.stat().st_dev, + inode=target.stat().st_ino, + mode=0o600, + uid=os.getuid(), + gid=os.getgid(), + size=4, + mtime_ns=target.stat().st_mtime_ns, + ctime_ns=target.stat().st_ctime_ns, + xattrs=( + ("system.posix_acl_access", b"synthetic-acl"), + ("security.selinux", b"synthetic-label"), + ("user.audit", b"synthetic-xattr"), + ), + ) + restored: list[tuple[str, bytes]] = [] + monkeypatch.setattr( + io_module.os, + "setxattr", + lambda _fd, name, value: restored.append((name, value)), + ) + + io_module.atomic_write(target, b"new\n", metadata) + + assert target.read_bytes() == b"new\n" + assert restored == list(metadata.xattrs) + + +def test_concurrent_database_change_is_detected_before_any_account_write( + tmp_path: Path, monkeypatch +): + module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + real_backup = module.backup_once + backup_count = 0 + writes: list[Path] = [] + + def racing_backup(path, snapshot, maximum): + nonlocal backup_count + result = real_backup(path, snapshot, maximum) + backup_count += 1 + if backup_count == 4: + group = module.HOST_ETC / "group" + group.write_text(originals["group"] + "race:x:4000:\n") + return result + + monkeypatch.setattr(module, "backup_once", racing_backup) + monkeypatch.setattr(module, "atomic_write", lambda path, *_args: writes.append(path)) + with pytest.raises(module.HardeningError, match="concurrent host account change"): + module.reconcile(public_key) + assert writes == [] + + +def test_flux_orders_observer_rbac_before_hermes_prunes_old_authority(): + observer = yaml.safe_load( + (ROOT / "clusters/atlas/flux-system/applications/hermes-observer-rbac/kustomization.yaml").read_text() + ) + hermes = yaml.safe_load( + (ROOT / "clusters/atlas/flux-system/applications/hermes/kustomization.yaml").read_text() + ) + bindings = yaml.safe_load( + ( + ROOT + / "clusters/atlas/flux-system/applications/hermes-observer-bindings/kustomization.yaml" + ).read_text() + ) + assert "dependsOn" not in observer["spec"] + assert {item["name"] for item in hermes["spec"]["dependsOn"]} >= { + "hermes-observer-rbac", + "hermes-scm-broker", + } + assert {item["name"] for item in bindings["spec"]["dependsOn"]} == { + "hermes-observer-rbac", + "hermes", + } + assert "hermes-observer-bindings" not in { + item["name"] for item in hermes["spec"]["dependsOn"] + } + + +def test_sensitive_root_acl_backups_have_unique_host_path_names(monkeypatch): + module = _load() + seen = [] + monkeypatch.setattr( + module, + "_deny_sensitive_root", + lambda path, backup_name=None: seen.append((path, backup_name)), + ) + module._deny_sensitive_roots() + assert [name for _path, name in seen] == [ + "var-lib-rancher-k3s", + "var-lib-kubelet", + "run-k3s", + "run-containerd", + ] + assert len({name for _path, name in seen}) == 4 diff --git a/testing/tests/test_hermes_node_account_privilege_audit.py b/testing/tests/test_hermes_node_account_privilege_audit.py new file mode 100644 index 00000000..3370fe4f --- /dev/null +++ b/testing/tests/test_hermes_node_account_privilege_audit.py @@ -0,0 +1,196 @@ +"""Host privilege-policy audits for the Hermes node SSH identity.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from testing.tests.test_hermes_node_account_support import _fixture + + +@pytest.mark.parametrize( + "database,record", + [ + ("group", "disk:x:6:atlas,hermes-agent\n"), + ("gshadow", "disk:!:hermes-agent:atlas\n"), + ("gshadow", "disk:!::atlas,hermes-agent\n"), + ], +) +def test_unexpected_group_or_gshadow_membership_fails_before_writes( + tmp_path: Path, monkeypatch, database: str, record: str +): + module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + path = module.HOST_ETC / database + path.write_text( + "\n".join( + line + for line in originals[database].splitlines() + if not line.startswith("disk:") + ) + + "\n" + + record, + encoding="utf-8", + ) + before = {name: (module.HOST_ETC / name).read_bytes() for name in originals} + + with pytest.raises(module.HardeningError, match="group access"): + module.reconcile(public_key) + + assert {name: (module.HOST_ETC / name).read_bytes() for name in originals} == before + + +@pytest.mark.parametrize( + "relative,value", + [ + ("sudoers.d/hermes", "hermes-agent ALL=(ALL:ALL) NOPASSWD: ALL\n"), + ("sudoers", "ALL ALL=(ALL:ALL) ALL\n"), + ( + "polkit-1/rules.d/90-root.rules", + 'if (subject.user == "hermes-agent") return polkit.Result.YES;\n', + ), + ( + "polkit-1/rules.d/90-root.rules", + "unix-user:* org.freedesktop.policykit.exec polkit.Result.YES\n", + ), + ], +) +def test_sudo_and_root_equivalent_polkit_authority_fail_closed( + tmp_path: Path, monkeypatch, relative: str, value: str +): + module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + policy = module.HOST_ETC / relative + policy.parent.mkdir(parents=True, exist_ok=True) + policy.write_text(value, encoding="utf-8") + + with pytest.raises(module.HardeningError, match="sudo|polkit"): + module.reconcile(public_key) + + assert all( + (module.HOST_ETC / name).read_text() == original + for name, original in originals.items() + ) + + +def test_external_group_authority_source_is_rejected(tmp_path: Path, monkeypatch): + module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + (module.HOST_ETC / "nsswitch.conf").write_text("passwd: files\ngroup: files ldap\n") + with pytest.raises(module.HardeningError, match="external group"): + module.reconcile(public_key) + assert all( + (module.HOST_ETC / name).read_text() == original + for name, original in originals.items() + ) + + +@pytest.mark.parametrize("database", ["passwd", "group", "initgroups", "shadow"]) +def test_external_identity_authority_sources_fail_closed( + tmp_path: Path, monkeypatch, database: str +): + module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + (module.HOST_ETC / "nsswitch.conf").write_text( + f"passwd: files\n{database}: files ldap\n", encoding="utf-8" + ) + with pytest.raises(module.HardeningError, match="external group/account"): + module.reconcile(public_key) + assert all( + (module.HOST_ETC / name).read_text() == original + for name, original in originals.items() + ) + + +@pytest.mark.parametrize( + "value", + [ + "#1200 ALL=(ALL:ALL) NOPASSWD: ALL\n", + "User_Alias HERMES_IDS = #1200\nHERMES_IDS ALL=(ALL:ALL) NOPASSWD: ALL\n", + "User_Alias HERMES_GROUP = %#1200\nHERMES_GROUP ALL=(ALL:ALL) NOPASSWD: ALL\n", + ], +) +def test_visudo_valid_numeric_and_alias_grants_fail_closed( + tmp_path: Path, monkeypatch, value: str +): + module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + monkeypatch.setattr(module, "ACCOUNT_UID", 1200) + policy = module.HOST_ETC / "sudoers" + policy.write_text(value, encoding="utf-8") + validation = subprocess.run( + ["/usr/bin/visudo", "-c", "-f", str(policy)], + check=False, + capture_output=True, + text=True, + ) + assert validation.returncode == 0, validation.stderr + + with pytest.raises(module.HardeningError, match="sudo authority"): + module.reconcile(public_key) + + assert all( + (module.HOST_ETC / name).read_text() == original + for name, original in originals.items() + ) + + +def test_sudo_include_is_bounded_to_the_audited_standard_directory( + tmp_path: Path, monkeypatch +): + module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + sudoers = module.HOST_ETC / "sudoers" + sudoers.write_text("#includedir /opt/external-sudoers\n", encoding="utf-8") + with pytest.raises(module.HardeningError, match="unaudited authority source"): + module.reconcile(public_key) + + sudoers.write_text("#includedir /etc/sudoers.d\n", encoding="utf-8") + module.reconcile(public_key) + assert all( + (module.HOST_ETC / name).read_text().startswith(original) + for name, original in originals.items() + ) + + +@pytest.mark.parametrize( + "relative,value", + [ + ( + "polkit-1/rules.d/90-hermes.rules", + "polkit.addRule(function(action, subject) {\n" + " if (subject.uid == 1200) return polkit.Result.YES;\n" + "});\n", + ), + ( + "polkit-1/localauthority/50-local.d/hermes.pkla", + "[Hermes]\nIdentity=unix-user:1200\n" + "Action=org.freedesktop.policykit.exec\nResultActive=yes\n", + ), + ], +) +def test_numeric_polkit_grants_fail_closed( + tmp_path: Path, monkeypatch, relative: str, value: str +): + module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + monkeypatch.setattr(module, "ACCOUNT_UID", 1200) + policy = module.HOST_ETC / relative + policy.parent.mkdir(parents=True, exist_ok=True) + policy.write_text(value, encoding="utf-8") + + with pytest.raises(module.HardeningError, match="polkit authority"): + module.reconcile(public_key) + + assert all( + (module.HOST_ETC / name).read_text() == original + for name, original in originals.items() + ) + + +def test_writable_privilege_policy_is_rejected(tmp_path: Path, monkeypatch): + module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + sudoers = module.HOST_ETC / "sudoers" + sudoers.write_text("root ALL=(ALL:ALL) ALL\n") + sudoers.chmod(0o666) + with pytest.raises(module.HardeningError, match="unsafe mutation"): + module.reconcile(public_key) + assert all( + (module.HOST_ETC / name).read_text() == original + for name, original in originals.items() + ) diff --git a/testing/tests/test_hermes_node_account_support.py b/testing/tests/test_hermes_node_account_support.py new file mode 100644 index 00000000..2d74d124 --- /dev/null +++ b/testing/tests/test_hermes_node_account_support.py @@ -0,0 +1,90 @@ +"""Shared fixtures for dedicated Hermes node-account contracts.""" + +from __future__ import annotations + +import base64 +import importlib.util +import os +import sys +from pathlib import Path + +ROOT = Path(__file__).parents[2] +SCRIPT = ROOT / "services/hermes/scripts/node_account_hardening.py" +sys.path.insert(0, str(SCRIPT.parent)) + + +def _key(material: bytes, comment: str = "") -> str: + key_type = b"ssh-ed25519" + blob = len(key_type).to_bytes(4, "big") + key_type + material + suffix = f" {comment}" if comment else "" + return "ssh-ed25519 " + base64.b64encode(blob).decode() + suffix + + +def _typed_key(key_type: str, material: bytes, comment: str = "") -> str: + encoded_type = key_type.encode() + blob = len(encoded_type).to_bytes(4, "big") + encoded_type + material + suffix = f" {comment}" if comment else "" + return f"{key_type} {base64.b64encode(blob).decode()}{suffix}" + + +def _load(): + spec = importlib.util.spec_from_file_location("node_account_hardening_test", SCRIPT) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _fixture(tmp_path: Path, monkeypatch): + module = _load() + host_etc = tmp_path / "etc" + host_home = tmp_path / "home" + host_etc.mkdir() + host_home.mkdir() + originals = { + "passwd": ( + "root:x:0:0:root:/root:/bin/bash\n" + "atlas:x:2000:2000:Atlas:/home/atlas:/bin/bash\n" + "oceanus:x:2001:2001:Oceanus:/home/oceanus:/bin/bash\n" + ), + "group": ( + "root:x:0:\n" + "atlas:x:2000:\n" + "oceanus:x:2001:\n" + "disk:x:6:atlas\n" + "sudo:x:27:oceanus\n" + ), + "shadow": ( + "root:!:1:0:99999:7:::\n" + "atlas:!:1:0:99999:7:::\n" + "oceanus:!:1:0:99999:7:::\n" + ), + "gshadow": ( + "root:!::\n" + "atlas:!::\n" + "oceanus:!::\n" + "disk:!::atlas\n" + "sudo:!::oceanus\n" + ), + } + for name, value in originals.items(): + (host_etc / name).write_text(value, encoding="utf-8") + key = _key(b"synthetic-hermes-key") + other = _key(b"human-operator-key") + for user in ("atlas", "oceanus"): + ssh = host_home / user / ".ssh" + ssh.mkdir(parents=True) + (ssh / "authorized_keys").write_text( + f"{other} {user}\n{key}\n", encoding="utf-8" + ) + public_key = tmp_path / "public-key" + public_key.write_text(key + "\n", encoding="utf-8") + monkeypatch.setattr(module, "HOST_ETC", host_etc) + monkeypatch.setattr(module, "HOST_HOME", host_home) + monkeypatch.setattr(module, "ACCOUNT_UID", os.getuid()) + monkeypatch.setattr(module, "ACCOUNT_GID", os.getgid()) + monkeypatch.setattr(module, "HOST_ROOT_UID", os.getuid()) + monkeypatch.setattr(module, "HOST_POLKIT_SHARE", tmp_path / "missing-polkit") + monkeypatch.setattr(module, "_deny_sensitive_roots", lambda: None) + return module, originals, key, other, public_key diff --git a/testing/tests/test_hermes_node_acl_coverage.py b/testing/tests/test_hermes_node_acl_coverage.py new file mode 100644 index 00000000..572fd0d2 --- /dev/null +++ b/testing/tests/test_hermes_node_acl_coverage.py @@ -0,0 +1,206 @@ +"""Behavioral branch coverage for node-storage ACL hardening.""" + +from __future__ import annotations + +import errno +import os +import pytest + +from testing.tests.test_hermes_node_account_support import _fixture, _load + + +def test_acl_decoder_rejects_version_permissions_and_adds_missing_mask(): + module = _load() + valid = [ + (module.ACL_USER_OBJ, 7, module.ACL_UNDEFINED_ID), + (module.ACL_GROUP_OBJ, 5, module.ACL_UNDEFINED_ID), + (module.ACL_OTHER, 5, module.ACL_UNDEFINED_ID), + ] + wrong_version = module.ACL_HEADER.pack(99) + b"".join( + module.ACL_ENTRY.pack(*entry) for entry in valid + ) + with pytest.raises(module.HardeningError, match="version"): + module._decode_acl(wrong_version, 0o755) + bad_permission = module._encode_acl([*valid, (module.ACL_USER, 8, 1200)]) + with pytest.raises(module.HardeningError, match="permission"): + module._decode_acl(bad_permission, 0o755) + updated = module._decode_acl( + module._acl_with_deny(module._encode_acl(valid), 0o755), 0o755 + ) + assert (module.ACL_MASK, 5, module.ACL_UNDEFINED_ID) in updated + assert (module.ACL_USER, 0, module.ACCOUNT_UID) in updated + + +def test_read_acl_handles_absence_and_propagates_other_errors(monkeypatch, tmp_path): + module = _load() + path = tmp_path / "root" + path.mkdir() + + def missing(*_args, **_kwargs): + raise OSError(errno.ENODATA, "missing") + + monkeypatch.setattr(module.os, "getxattr", missing) + assert module._read_acl(path) == b"" + + def denied(*_args, **_kwargs): + raise OSError(errno.EPERM, "denied") + + monkeypatch.setattr(module.os, "getxattr", denied) + with pytest.raises(OSError): + module._read_acl(path) + + +@pytest.mark.parametrize("value", [b"", b"X", b"A", b"Nextra"]) +def test_acl_backup_validator_rejects_malformed_encodings(value): + module = _load() + with pytest.raises(module.HardeningError, match="backup is malformed"): + module._validate_acl_backup(value) + module._validate_acl_backup(b"N") + valid = module._encode_acl( + [ + (module.ACL_USER_OBJ, 7, module.ACL_UNDEFINED_ID), + (module.ACL_GROUP_OBJ, 5, module.ACL_UNDEFINED_ID), + (module.ACL_OTHER, 5, module.ACL_UNDEFINED_ID), + ] + ) + module._validate_acl_backup(b"A" + valid) + + +def test_acl_backup_reuses_valid_existing_copy(tmp_path, monkeypatch): + module = _load() + host_etc = tmp_path / "etc" + root = host_etc / "hermes-node-boundary" + root.mkdir(parents=True) + backup = root / "k3s.acl" + backup.write_bytes(b"N") + monkeypatch.setattr(module, "HOST_ETC", host_etc) + monkeypatch.setattr(module, "HOST_ROOT_UID", os.getuid()) + monkeypatch.setattr(module, "HOST_ROOT_GID", os.getgid()) + module._acl_backup_once(tmp_path / "k3s", b"") + assert backup.read_bytes() == b"N" + + +def test_acl_backup_short_write_cleans_temporary(tmp_path, monkeypatch): + module = _load() + host_etc = tmp_path / "etc" + host_etc.mkdir() + monkeypatch.setattr(module, "HOST_ETC", host_etc) + monkeypatch.setattr(module, "HOST_ROOT_UID", os.getuid()) + monkeypatch.setattr(module, "HOST_ROOT_GID", os.getgid()) + monkeypatch.setattr(module.os, "write", lambda _fd, _value: 0) + with pytest.raises(module.HardeningError, match="short sensitive"): + module._acl_backup_once(tmp_path / "k3s", b"") + assert not list((host_etc / "hermes-node-boundary").glob(".*.hermes-*")) + + +def test_acl_backup_tolerates_concurrent_first_writer(tmp_path, monkeypatch): + module = _load() + host_etc = tmp_path / "etc" + host_etc.mkdir() + monkeypatch.setattr(module, "HOST_ETC", host_etc) + monkeypatch.setattr(module, "HOST_ROOT_UID", os.getuid()) + monkeypatch.setattr(module, "HOST_ROOT_GID", os.getgid()) + real_link = os.link + + def race(source, target, **kwargs): + real_link(source, target, **kwargs) + raise FileExistsError() + + monkeypatch.setattr(module.os, "link", race) + module._acl_backup_once(tmp_path / "k3s", b"") + assert (host_etc / "hermes-node-boundary/k3s.acl").read_bytes() == b"N" + + +def test_sensitive_root_failure_restores_existing_acl(tmp_path, monkeypatch): + module = _load() + root = tmp_path / "k3s" + root.mkdir() + monkeypatch.setattr(module, "HOST_ROOT_UID", os.getuid()) + current = module._encode_acl( + [ + (module.ACL_USER_OBJ, 7, module.ACL_UNDEFINED_ID), + (module.ACL_GROUP_OBJ, 5, module.ACL_UNDEFINED_ID), + (module.ACL_MASK, 5, module.ACL_UNDEFINED_ID), + (module.ACL_OTHER, 5, module.ACL_UNDEFINED_ID), + ] + ) + reads = iter((current, current)) + writes = [] + monkeypatch.setattr(module, "_read_acl", lambda _path: next(reads)) + monkeypatch.setattr(module, "_acl_backup_once", lambda *_a, **_k: None) + monkeypatch.setattr( + module.os, "setxattr", lambda *args, **kwargs: writes.append(args[2]) + ) + with pytest.raises(module.HardeningError, match="validation failed"): + module._deny_sensitive_root(root) + assert writes[-1] == current + + +@pytest.mark.parametrize("remove_errno", [errno.ENODATA, errno.EPERM]) +def test_sensitive_root_failure_removes_new_acl_or_propagates_cleanup_error( + tmp_path, monkeypatch, remove_errno +): + module = _load() + root = tmp_path / "k3s" + root.mkdir() + monkeypatch.setattr(module, "HOST_ROOT_UID", os.getuid()) + monkeypatch.setattr(module, "_read_acl", lambda _path: b"") + monkeypatch.setattr(module, "_acl_backup_once", lambda *_a, **_k: None) + monkeypatch.setattr( + module.os, + "setxattr", + lambda *_a, **_k: (_ for _ in ()).throw(RuntimeError("set failed")), + ) + monkeypatch.setattr( + module.os, + "removexattr", + lambda *_a, **_k: (_ for _ in ()).throw(OSError(remove_errno, "remove")), + ) + expected = OSError if remove_errno == errno.EPERM else RuntimeError + with pytest.raises(expected): + module._deny_sensitive_root(root) + + +def test_database_validation_failure_rolls_back_all_writes(tmp_path, monkeypatch): + module, originals, _key_value, _other, _public = _fixture(tmp_path, monkeypatch) + real_read = module._read_regular + calls = 0 + + def missing_expected(path, maximum=module.MAX_ACCOUNT_FILE): + nonlocal calls + calls += 1 + if calls == 1: + return originals["passwd"].encode(), path.stat(follow_symlinks=False) + return real_read(path, maximum) + + monkeypatch.setattr(module, "_read_regular", missing_expected) + with pytest.raises(module.HardeningError, match="validation failed"): + module._reconcile_databases() + for name, value in originals.items(): + assert (module.HOST_ETC / name).read_text() == value + + +def test_preexisting_target_key_removal_is_verified(tmp_path, monkeypatch): + module, _originals, key, _other, public = _fixture(tmp_path, monkeypatch) + target = module.HOST_HOME / module.ACCOUNT / ".ssh/authorized_keys" + target.parent.mkdir(parents=True) + target.write_text(key + "\n", encoding="utf-8") + real_write = module.atomic_write + + def ignore_preinstall_target(path, value, metadata): + if path == target and value == b"": + return + real_write(path, value, metadata) + + monkeypatch.setattr(module, "atomic_write", ignore_preinstall_target) + with pytest.raises(module.HardeningError, match="preexisting Hermes authorization"): + module._move_key(public) + + +def test_existing_target_without_key_does_not_need_precleanup(tmp_path, monkeypatch): + module, _originals, _key_value, other, public = _fixture(tmp_path, monkeypatch) + target = module.HOST_HOME / module.ACCOUNT / ".ssh/authorized_keys" + target.parent.mkdir(parents=True) + target.write_text(other + "\n", encoding="utf-8") + module._move_key(public) + assert target.read_text().endswith("\n") diff --git a/testing/tests/test_hermes_node_audit_coverage.py b/testing/tests/test_hermes_node_audit_coverage.py new file mode 100644 index 00000000..8c011b5b --- /dev/null +++ b/testing/tests/test_hermes_node_audit_coverage.py @@ -0,0 +1,221 @@ +"""Behavioral branch coverage for dedicated node-account privilege audits.""" + +from __future__ import annotations + +import os +import struct +import sys +from types import SimpleNamespace + +import pytest + +from testing.tests.test_hermes_node_account_support import _load + + +def _audit_module(): + hardening = _load() + return sys.modules[hardening.audit_membership.__module__] + + +def test_member_parser_handles_empty_valid_malformed_and_duplicates(): + module = _audit_module() + assert module._members("", "group") == set() + assert module._members("one,two", "group") == {"one", "two"} + for value in ("one,", " one", "one,one"): + with pytest.raises(module.HardeningError): + module._members(value, "group") + + +def test_membership_audit_accepts_unrelated_records_and_rejects_account(): + module = _audit_module() + module.audit_membership( + "hermes-agent", [["disk", "x", "6", "atlas"]], [["disk", "!", "atlas", ""]] + ) + with pytest.raises(module.HardeningError, match="supplementary"): + module.audit_membership( + "hermes-agent", [["disk", "x", "6", "hermes-agent"]], [] + ) + with pytest.raises(module.HardeningError, match="gshadow"): + module.audit_membership("hermes-agent", [], [["disk", "!", "hermes-agent", ""]]) + + +@pytest.mark.parametrize("kind", ["mode", "symlink"]) +def test_policy_files_reject_unsafe_directories(tmp_path, kind): + module = _audit_module() + host_etc = tmp_path / "etc" + host_etc.mkdir() + root = host_etc / "sudoers.d" + if kind == "mode": + root.mkdir(mode=0o777) + root.chmod(0o777) + else: + target = tmp_path / "target" + target.mkdir() + root.symlink_to(target) + with pytest.raises(module.HardeningError, match="unsafe sudo/polkit"): + module._policy_files(host_etc, tmp_path / "share", os.getuid()) + + +def test_policy_file_count_is_bounded(tmp_path): + module = _audit_module() + root = tmp_path / "etc/sudoers.d" + root.mkdir(parents=True) + for index in range(257): + (root / str(index)).write_text("# safe\n", encoding="utf-8") + with pytest.raises(module.HardeningError, match="too many"): + module._policy_files(tmp_path / "etc", tmp_path / "share", os.getuid()) + + +def _acl(*entries): + return struct.pack(" bytes: + command = old + b" " + new + b" " + ref + b"\n" + framed = f"{len(command) + 4:04x}".encode() + command + return framed + (b"0000" if terminator else b"") + + +def test_receive_pack_rejects_nonascii_many_commands_and_missing_terminator( + monkeypatch, +): + broker = _load("scm_broker") + zero = b"0" * 40 + commit = b"1" * 40 + with pytest.raises(broker.PolicyError, match="canonical ASCII"): + broker._validate_receive_pack( + _packet(zero, commit, b"refs/heads/hermes/\xff"), "sentinel" + ) + command = _packet(zero, commit, b"refs/heads/hermes/fix", terminator=False) + with pytest.raises(broker.PolicyError, match="terminator"): + broker._validate_receive_pack(command, "sentinel") + many = command * (broker.MAX_PUSH_COMMANDS + 1) + b"0000" + with pytest.raises(broker.PolicyError, match="too many"): + broker._validate_receive_pack(many, "sentinel") + + +def test_upstream_request_infers_byte_length_and_requires_stream_length(): + broker = _load("scm_broker") + seen = [] + + def opener(request, timeout): + seen.append((request, timeout)) + return Response(b"result", content_type="application/x-git-upload-pack-result") + + assert ( + broker._upstream_git_request( + "/atlas/cassandra.git/git-upload-pack", + method="POST", + body=b"request", + content_type=None, + expected_type="application/x-git-upload-pack-result", + token="sentinel", + opener=opener, + ) + == b"result" + ) + assert seen[0][0].get_header("Content-length") == "7" + with pytest.raises(broker.PolicyError, match="length is missing"): + broker._upstream_git_request( + "/atlas/cassandra.git/git-upload-pack", + method="POST", + body=io.BytesIO(b"request"), + content_type=None, + expected_type="application/x-git-upload-pack-result", + token="sentinel", + opener=opener, + ) + + +def test_upstream_request_rejects_wrong_type_and_supports_getcode_status(): + broker = _load("scm_broker") + + class CodeOnly(Response): + def __init__(self, body, content_type): + super().__init__(body, content_type=content_type) + del self.status + + def getcode(self): + return 200 + + assert ( + broker._upstream_git_request( + "/atlas/cassandra.git/info/refs?service=git-upload-pack", + method="GET", + body=None, + content_type=None, + expected_type="application/x-git-upload-pack-advertisement", + token="sentinel", + opener=lambda *_a, **_k: CodeOnly( + b"advertisement", "application/x-git-upload-pack-advertisement" + ), + ) + == b"advertisement" + ) + with pytest.raises(broker.PolicyError, match="response type"): + broker._upstream_git_request( + "/atlas/cassandra.git/info/refs?service=git-upload-pack", + method="GET", + body=None, + content_type=None, + expected_type="application/x-git-upload-pack-advertisement", + token="sentinel", + opener=lambda *_a, **_k: Response(b"bad", content_type="text/plain"), + ) diff --git a/testing/tests/test_hermes_scm_broker_policy.py b/testing/tests/test_hermes_scm_broker_policy.py new file mode 100644 index 00000000..25d94fa0 --- /dev/null +++ b/testing/tests/test_hermes_scm_broker_policy.py @@ -0,0 +1,321 @@ +"""RBAC, Flux, and human-review contracts for the Hermes SCM broker.""" + +from __future__ import annotations + +import json + +import pytest +import yaml + +from testing.tests.test_hermes_scm_broker_support import ROOT, _load_path + +def _can_i(role: dict, api_group: str, resource: str, verb: str) -> bool: + return any( + (api_group in rule["apiGroups"] or "*" in rule["apiGroups"]) + and (resource in rule["resources"] or "*" in rule["resources"]) + and (verb in rule["verbs"] or "*" in rule["verbs"]) + for rule in role["rules"] + ) + + +def test_agent_rbac_preserves_read_diagnostics_without_administrator_authority(): + documents = list( + yaml.safe_load_all( + (ROOT / "services/hermes-observer-rbac/rbac.yaml").read_text( + encoding="utf-8" + ) + ) + ) + cluster_role, namespaced_role, binding = documents + assert binding["roleRef"]["name"] == "hermes-agent-cluster-observer-v2" + assert _can_i(cluster_role, "", "nodes", "get") + assert _can_i(namespaced_role, "", "pods", "get") + assert _can_i(namespaced_role, "", "pods/log", "get") + assert _can_i( + namespaced_role, + "kustomize.toolkit.fluxcd.io", + "kustomizations", + "list", + ) + + denied = ( + ("", "secrets", "get"), + ("", "serviceaccounts/token", "create"), + ("", "pods", "create"), + ("", "pods/exec", "create"), + ("", "pods/attach", "create"), + ("", "pods/portforward", "create"), + ("rbac.authorization.k8s.io", "clusterrolebindings", "create"), + ("authorization.k8s.io", "selfsubjectaccessreviews", "create"), + ("apps", "deployments", "patch"), + ) + assert all( + not _can_i(role, *request) + for request in denied + for role in (cluster_role, namespaced_role) + ) + + bindings = yaml.safe_load( + (ROOT / "services/hermes-observer-bindings/rolebindings.yaml").read_text( + encoding="utf-8" + ) + )["items"] + namespaces = {item["metadata"]["namespace"] for item in bindings} + assert "hermes-scm" not in namespaces + assert {"cassandra", "flux-system", "hermes", "kube-system"} <= namespaces + assert all(item["roleRef"]["name"].endswith("namespaced-observer-v2") for item in bindings) + + +def test_flux_boundary_keeps_broker_secret_and_network_separate_from_agent(): + agent = yaml.safe_load( + (ROOT / "services/hermes/agent-deployment.yaml").read_text(encoding="utf-8") + ) + annotations = agent["spec"]["template"]["metadata"]["annotations"] + assert not any("gitea" in key.lower() for key in annotations) + assert "GIT_ASKPASS" not in json.dumps(agent) + + broker = yaml.safe_load( + (ROOT / "services/hermes-scm-broker/deployment.yaml").read_text( + encoding="utf-8" + ) + ) + assert broker["metadata"]["namespace"] == "hermes-scm" + assert broker["spec"]["template"]["spec"]["serviceAccountName"] == ( + "hermes-scm-broker" + ) + assert "gitea-token" in json.dumps(broker) + container = broker["spec"]["template"]["spec"]["containers"][0] + assert container["resources"]["limits"]["memory"] == "768Mi" + assert container["livenessProbe"]["failureThreshold"] == 10 + tmp = next( + item for item in broker["spec"]["template"]["spec"]["volumes"] if item["name"] == "tmp" + ) + assert tmp["emptyDir"]["sizeLimit"] == "1Gi" + + policy = yaml.safe_load( + (ROOT / "services/hermes-scm-broker/networkpolicy.yaml").read_text( + encoding="utf-8" + ) + ) + ingress = policy["spec"]["ingress"] + assert ingress[0]["from"][0]["namespaceSelector"]["matchLabels"] == { + "kubernetes.io/metadata.name": "hermes" + } + assert ingress[0]["from"][0]["podSelector"]["matchLabels"] == { + "app": "hermes-agent" + } + + vault = ( + ROOT / "services/vault/scripts/vault_k8s_auth_configure.sh" + ).read_text(encoding="utf-8") + assert 'write_policy_and_role "hermes-scm-broker" "hermes-scm"' in vault + assert '"hermes/developer-gitea" ""' in vault + agent_start = vault.index('write_policy_and_role "hermes-agent"') + agent_end = vault.index("write_policy_and_role", agent_start + 1) + assert "developer-gitea" not in vault[agent_start:agent_end] + + +def test_flux_bootstrap_has_no_agent_namespace_dependency_cycle(): + hermes_resources = yaml.safe_load( + (ROOT / "services/hermes/kustomization.yaml").read_text(encoding="utf-8") + )["resources"] + assert "namespace.yaml" in hermes_resources + assert "scm-common" in hermes_resources + + observer = yaml.safe_load( + ( + ROOT + / "clusters/atlas/flux-system/applications/hermes-observer-rbac/kustomization.yaml" + ).read_text(encoding="utf-8") + ) + assert "dependsOn" not in observer["spec"] + observer_objects = [ + item + for item in yaml.safe_load_all( + (ROOT / "services/hermes-observer-rbac/rbac.yaml").read_text( + encoding="utf-8" + ) + ) + if item + ] + assert {item["kind"] for item in observer_objects} <= { + "ClusterRole", + "ClusterRoleBinding", + } + + hermes_flux = yaml.safe_load( + ( + ROOT / "clusters/atlas/flux-system/applications/hermes/kustomization.yaml" + ).read_text(encoding="utf-8") + ) + assert "hermes-observer-rbac" in { + item["name"] for item in hermes_flux["spec"]["dependsOn"] + } + bindings_flux = yaml.safe_load( + ( + ROOT + / "clusters/atlas/flux-system/applications/hermes-observer-bindings/kustomization.yaml" + ).read_text(encoding="utf-8") + ) + assert {item["name"] for item in bindings_flux["spec"]["dependsOn"]} == { + "hermes", + "hermes-observer-rbac", + } + assert bindings_flux["metadata"]["name"] not in { + item["name"] for item in hermes_flux["spec"]["dependsOn"] + } + + applications = ( + ROOT / "clusters/atlas/flux-system/applications/kustomization.yaml" + ).read_text(encoding="utf-8") + assert "hermes-scm-agent-code" not in applications + + broker_code = yaml.safe_load( + ( + ROOT + / "clusters/atlas/flux-system/applications/hermes-scm-broker-code/kustomization.yaml" + ).read_text(encoding="utf-8") + ) + assert broker_code["spec"]["path"] == "./services/hermes/scm-common" + assert broker_code["spec"]["targetNamespace"] == "hermes-scm" + assert broker_code["spec"]["dependsOn"] == [{"name": "hermes-scm-namespace"}] + + +def test_gitea_bootstrap_enforces_human_review_without_overwriting_drift(): + script = ( + ROOT / "services/gitea/scripts/gitea_atlas_identity_ensure.sh" + ).read_text(encoding="utf-8") + + assert 'required_approvals\\\":1' in script + assert 'enable_push_whitelist\\\":true' in script + assert 'push_whitelist_usernames\\\":[\\\"${protected_reviewer}\\\"]' in script + assert 'enable_merge_whitelist\\\":true' in script + assert 'merge_whitelist_usernames\\\":[\\\"${protected_reviewer}\\\"]' in script + assert "protection differs from the human-review policy" in script + assert "api_request PATCH" not in script + job = yaml.safe_load( + (ROOT / "services/gitea/atlas-identity-bootstrap-job.yaml").read_text() + ) + init = job["spec"]["template"]["spec"]["initContainers"][0] + assert init["image"] == ( + "python@sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df" + ) + assert init["securityContext"]["runAsNonRoot"] is True + assert init["securityContext"]["capabilities"]["drop"] == ["ALL"] + assert "/opt/python/bin/python3" in script + + +def _protection(rule_name: str, priority: int, helper, **overrides): + value = { + "rule_name": rule_name, + "priority": priority, + "created_at": "2026-01-01T00:00:00Z", + **helper._required("bstein"), + } + value.update(overrides) + return value + + +def test_branch_protection_uses_first_effective_glob_by_priority(): + helper = _load_path( + "branch_protection_test", + ROOT / "services/gitea/scripts/gitea_branch_protection_check.py", + ) + earlier_drift = _protection("m*", 1, helper, required_approvals=0) + later_exact = _protection("main", 2, helper) + + with pytest.raises(helper.PolicyError, match="effective main protection differs"): + helper.evaluate(json.dumps([later_exact, earlier_drift]).encode(), "main", "bstein") + + earlier_drift["required_approvals"] = 1 + assert ( + helper.evaluate(json.dumps([later_exact, earlier_drift]).encode(), "main", "bstein") + == "PRESENT" + ) + brace_drift = _protection("m{ain,aster}", 1, helper, required_approvals=0) + with pytest.raises(helper.PolicyError, match="effective master protection differs"): + helper.evaluate(json.dumps([brace_drift]).encode(), "master", "bstein") + + upper_glob = _protection("M*", 1, helper, required_approvals=0) + assert ( + helper.evaluate(json.dumps([upper_glob, later_exact]).encode(), "main", "bstein") + == "PRESENT" + ) + + +def test_branch_protection_matches_gitea_v1238_tie_order(): + helper = _load_path( + "branch_protection_order_test", + ROOT / "services/gitea/scripts/gitea_branch_protection_check.py", + ) + older_glob = _protection( + "m*", 1, helper, created_at="2025-01-01T00:00:00Z", required_approvals=0 + ) + newer_plain = _protection( + "MAIN", 1, helper, created_at="2026-01-01T00:00:00Z" + ) + assert ( + helper.evaluate(json.dumps([older_glob, newer_plain]).encode(), "main", "bstein") + == "PRESENT" + ) + + older_glob["created_at"] = "2026-01-02T00:00:00Z" + earlier_glob = _protection( + "m{ain,aster}", + 1, + helper, + created_at="2026-01-01T00:00:00Z", + required_approvals=0, + ) + with pytest.raises(helper.PolicyError, match="effective main protection differs"): + helper.evaluate(json.dumps([older_glob, earlier_glob]).encode(), "main", "bstein") + + +@pytest.mark.parametrize( + ("rule_name", "branch", "expected"), + [ + ("release/*", "release/v1.17", True), + ("release/**/v1.17", "release/test/1/v1.17", True), + ("release/*/v1.17", "release/test/1/v1.17", False), + ("*", "release/v1.16", False), + ("**", "release/v1.16", True), + ("MAIN", "main", True), + ("M*", "main", False), + ("m{ain,aster}", "master", True), + (r"m\ain", "main", True), + ], +) +def test_branch_glob_matches_gitea_v1238_source_corpus( + rule_name: str, branch: str, expected: bool +): + helper = _load_path( + "branch_protection_glob_parity_test", + ROOT / "services/gitea/scripts/gitea_branch_protection_check.py", + ) + assert helper._matches(rule_name, branch) is expected + + +@pytest.mark.parametrize( + "rules", + [ + [{"rule_name": "main", "priority": 0, "created_at": "2026-01-01T00:00:00Z"}], + [{"rule_name": "main", "priority": 1, "created_at": "not-a-time"}], + [{"rule_name": "main"}], + ], +) +def test_branch_protection_rejects_ambiguous_rules_before_creation(rules): + helper = _load_path( + "branch_protection_adversarial_test", + ROOT / "services/gitea/scripts/gitea_branch_protection_check.py", + ) + with pytest.raises(helper.PolicyError): + helper.evaluate(json.dumps(rules).encode(), "main", "bstein") + + +def test_branch_protection_reports_absent_only_when_no_rule_matches(): + helper = _load_path( + "branch_protection_absent_test", + ROOT / "services/gitea/scripts/gitea_branch_protection_check.py", + ) + rules = [_protection("release/*", 1, helper)] + assert helper.evaluate(json.dumps(rules).encode(), "master", "bstein") == "ABSENT" diff --git a/testing/tests/test_hermes_scm_broker_streaming.py b/testing/tests/test_hermes_scm_broker_streaming.py new file mode 100644 index 00000000..2fa9683f --- /dev/null +++ b/testing/tests/test_hermes_scm_broker_streaming.py @@ -0,0 +1,237 @@ +"""Bounded I/O and concurrency contracts for the Hermes SCM broker.""" + +from __future__ import annotations + +import io +import json +from email.message import Message + +import pytest + +from testing.tests.test_hermes_scm_broker_support import Response, _load + + +def test_agent_broker_client_sends_no_credential_or_authorization_header(): + client = _load("scm_broker_client") + seen = [] + + def opener(request, timeout): + seen.append((request, timeout)) + return Response(b"{}", content_type="application/json") + + assert client.read("/api/v1/repos/atlas/cassandra", opener=opener) == b"{}" + request = seen[0][0] + assert request.full_url == client.BROKER_ORIGIN + "/v1/metadata" + assert request.get_header("Authorization") is None + assert json.loads(request.data) == {"path": "/api/v1/repos/atlas/cassandra"} + + +@pytest.mark.parametrize("raw", ["", "01", "9" * 4000, "134217729", "１２"]) +def test_broker_content_length_is_canonical_and_bounded(raw: str): + broker = _load("scm_broker") + headers = Message() + headers["Content-Length"] = raw + + with pytest.raises(broker.PolicyError, match="request length"): + broker._content_length(headers, broker.MAX_GIT_REQUEST) + + +def test_large_git_exchange_rolls_to_disk_and_scans_chunk_boundaries(): + broker = _load("scm_broker") + token = "runtime-sentinel" + safe = b"x" * (broker.SPOOL_MEMORY_LIMIT + 1) + inbound_timeouts = [] + spool, length = broker._spool_bounded( + io.BytesIO(safe), + broker.MAX_GIT_REQUEST, + len(safe), + token=token, + context="Git request", + deadline=broker.time.monotonic() + 30, + set_timeout=inbound_timeouts.append, + ) + try: + assert length == len(safe) + assert spool._rolled is True + assert spool.read(4) == b"xxxx" + assert inbound_timeouts + assert all(0 < value <= 30 for value in inbound_timeouts) + finally: + spool.close() + + crossing = b"x" * (broker.STREAM_CHUNK - 5) + token.encode() + b"tail" + with pytest.raises(broker.PolicyError, match="credential material"): + broker._spool_bounded( + io.BytesIO(crossing), + broker.MAX_GIT_REQUEST, + len(crossing), + token=token, + context="Git request", + deadline=broker.time.monotonic() + 30, + ) + + +def test_large_upstream_response_is_streamed_through_bounded_spool(): + broker = _load("scm_broker") + body = b"z" * (broker.SPOOL_MEMORY_LIMIT + 1) + seen = [] + + def opener(request, **_kwargs): + seen.append(request) + assert request.get_header("Content-length") == "7" + assert request.data.read() == b"request" + return Response(body, content_type="application/x-git-upload-pack-result") + + streamed = broker._upstream_git_request( + "/atlas/cassandra.git/git-upload-pack", + method="POST", + body=io.BytesIO(b"request"), + body_length=7, + content_type="application/x-git-upload-pack-request", + expected_type="application/x-git-upload-pack-result", + token="runtime-sentinel", + stream_result=True, + opener=opener, + ) + spool, length = streamed + try: + assert length == len(body) + assert spool._rolled is True + assert spool.read(3) == b"zzz" + assert len(seen) == 1 + finally: + spool.close() + + +def test_body_deadline_fails_closed_before_reading(): + broker = _load("scm_broker") + with pytest.raises(broker.PolicyError, match="deadline"): + broker._spool_bounded( + io.BytesIO(b"body"), + broker.MAX_GIT_REQUEST, + 4, + token="runtime-sentinel", + context="Git request", + deadline=broker.time.monotonic() - 1, + ) + + +def test_upstream_stream_applies_absolute_socket_deadline(): + broker = _load("scm_broker") + + class DeadlineSocket: + def __init__(self): + self.values = [] + + def settimeout(self, value): + self.values.append(value) + + response = Response(b"safe", content_type="application/x-git-upload-pack-result") + sock = DeadlineSocket() + response.fp = type("FP", (), {"raw": type("Raw", (), {"_sock": sock})()})() + spool, length = broker._spool_response( + response, broker.MAX_GIT_RESPONSE, "sentinel" + ) + try: + assert length == 4 + assert sock.values + assert all(0 < value <= broker.INBOUND_BODY_TIMEOUT for value in sock.values) + finally: + spool.close() + + +def test_broker_rejects_work_when_concurrency_slots_are_exhausted(): + broker = _load("scm_broker") + + class FakeSocket: + def __init__(self): + self.value = b"" + + def sendall(self, value): + self.value += value + + def shutdown(self, _how): + return None + + def close(self): + return None + + server = broker.BoundedThreadingHTTPServer(("127.0.0.1", 0), broker.BrokerHandler) + try: + assert broker.BoundedThreadingHTTPServer.__module__ == "scm_broker_server" + assert server._slots.acquire(blocking=False) + assert server._slots.acquire(blocking=False) + assert server._slots.acquire(blocking=False) + request = FakeSocket() + server.process_request(request, ("127.0.0.1", 1)) + assert b"503 Service Unavailable" in request.value + finally: + server._slots.release() + server._slots.release() + server._slots.release() + server.server_close() + + +def test_three_simultaneous_broker_calls_have_bounded_capacity(): + server_module = _load("scm_broker_server") + semaphore = server_module.threading.BoundedSemaphore( + server_module.MAX_CONCURRENT_REQUESTS + ) + assert server_module.MAX_CONCURRENT_REQUESTS >= 3 + assert [semaphore.acquire(blocking=False) for _ in range(3)] == [True, True, True] + assert semaphore.acquire(blocking=False) is False + + +def test_header_deadline_is_absolute_against_slow_trickle(monkeypatch): + server_module = _load("scm_broker_server") + + class Stream: + def __init__(self): + self.value = io.BytesIO(b"GET /healthz HTTP/1.1\r\n") + + def read(self, length): + return self.value.read(length) + + class Connection: + def __init__(self): + self.timeouts = [] + + def settimeout(self, value): + self.timeouts.append(value) + + ticks = iter((0.0, 0.02, 0.04, 0.06, 0.08, 0.11)) + monkeypatch.setattr(server_module.time, "monotonic", lambda: next(ticks)) + connection = Connection() + reader = server_module._AbsoluteDeadlineReader(Stream(), connection) + reader.begin(0.1) + + with pytest.raises(TimeoutError, match="absolute SCM header deadline"): + reader.readline(65537) + + assert connection.timeouts == pytest.approx([0.08, 0.06, 0.04, 0.02]) + + +def test_broker_bounds_header_count_size_and_duplicate_lengths(): + broker = _load("scm_broker") + + def validate(headers): + handler = object.__new__(broker.BrokerHandler) + handler.headers = headers + handler._validate_headers() + + too_many = Message() + for index in range(broker.MAX_HEADERS + 1): + too_many[f"X-Test-{index}"] = "safe" + with pytest.raises(broker.PolicyError, match="too many headers"): + validate(too_many) + + too_large = Message() + too_large["X-Test"] = "x" * broker.MAX_HEADER_BYTES + with pytest.raises(broker.PolicyError, match="safe limit"): + validate(too_large) + + duplicate = Message() + duplicate["Content-Length"] = "1" + duplicate["Content-Length"] = "1" + with pytest.raises(broker.PolicyError, match="duplicate Content-Length"): + validate(duplicate) diff --git a/testing/tests/test_hermes_scm_broker_support.py b/testing/tests/test_hermes_scm_broker_support.py new file mode 100644 index 00000000..6e2184d9 --- /dev/null +++ b/testing/tests/test_hermes_scm_broker_support.py @@ -0,0 +1,55 @@ +"""Shared fixtures for credential-isolated Hermes SCM broker tests.""" + +from __future__ import annotations + +import importlib.util +import io +import sys +from email.message import Message +from pathlib import Path + +ROOT = Path(__file__).parents[2] +SCRIPTS = ROOT / "services/hermes/scm-common/scripts" +sys.path.insert(0, str(SCRIPTS)) + + +def _load(name: str): + path = SCRIPTS / f"{name}.py" + spec = importlib.util.spec_from_file_location(f"test_{name}", path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _load_path(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +class Response: + def __init__(self, body: bytes, *, status: int = 200, content_type: str): + self.body = body + self.stream = io.BytesIO(body) + self.status = status + self.headers = Message() + self.headers["Content-Type"] = content_type + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def read(self, limit=-1): + return self.stream.read(limit) + + +def _receive_command(old: bytes, new: bytes, ref: bytes) -> bytes: + command = old + b" " + new + b" " + ref + b"\x00report-status\n" + return f"{len(command) + 4:04x}".encode() + command + b"0000PACK" diff --git a/testing/tests/test_hermes_scm_server_coverage.py b/testing/tests/test_hermes_scm_server_coverage.py new file mode 100644 index 00000000..7a358640 --- /dev/null +++ b/testing/tests/test_hermes_scm_server_coverage.py @@ -0,0 +1,180 @@ +"""Behavioral branch coverage for the SCM broker HTTP server primitives.""" + +from __future__ import annotations + +import io + +import pytest + +from testing.tests.test_hermes_scm_broker_support import _load + + +class _Connection: + def __init__(self): + self.timeouts = [] + + def settimeout(self, value): + self.timeouts.append(value) + + +class _FlushBuffer(io.BytesIO): + def __init__(self): + super().__init__() + self.flushes = 0 + + def flush(self): + self.flushes += 1 + + +def test_absolute_reader_supports_idle_limit_newline_eof_and_attribute_delegation(): + module = _load("scm_broker_server") + stream = io.BytesIO(b"first\nsecond") + reader = module._AbsoluteDeadlineReader(stream, _Connection()) + assert reader.readline(3) == b"fir" + reader.begin(5) + assert reader.readline() == b"st\n" + assert reader.readline() == b"second" + assert reader.readline() == b"" + assert reader.closed is False + reader.end() + assert reader.readline() == b"" + + +def _handler_type(module): + class StubHandler: + request_version = "HTTP/1.1" + command = "GET" + + def setup(self): + self.connection = _Connection() + self.rfile = io.BytesIO(b"") + self.wfile = _FlushBuffer() + + def parse_request(self): + return self.parse_result + + def send_error(self, code, *_args): + self.errors.append(code) + + def do_GET(self): + self.calls.append("GET") + + return type( + "DeadlineHandler", (module.AbsoluteHeaderDeadlineMixin, StubHandler), {} + ) + + +def _handler(module, raw: bytes, *, parse=True): + handler_type = _handler_type(module) + handler = object.__new__(handler_type) + handler.setup() + handler._header_reader._stream = io.BytesIO(raw) + handler.parse_result = parse + handler.errors = [] + handler.calls = [] + return handler + + +def test_header_mixin_setup_and_known_request_dispatch(): + module = _load("scm_broker_server") + handler = _handler(module, b"GET /healthz HTTP/1.1\r\n") + handler.handle_one_request() + assert handler.calls == ["GET"] + assert handler.wfile.flushes == 1 + assert handler._header_reader._deadline is None + + +@pytest.mark.parametrize( + ("raw", "parse", "command", "expected_error", "closed"), + [ + (b"x" * 65537, True, "GET", 414, False), + (b"", True, "GET", None, True), + (b"GET / HTTP/1.1\r\n", False, "GET", None, False), + (b"TRACE / HTTP/1.1\r\n", True, "TRACE", 501, False), + ], +) +def test_header_mixin_rejects_long_empty_unparsed_and_unknown_requests( + raw, parse, command, expected_error, closed +): + module = _load("scm_broker_server") + handler = _handler(module, raw, parse=parse) + handler.command = command + handler.close_connection = False + handler.handle_one_request() + assert handler.errors == ([] if expected_error is None else [expected_error]) + assert handler.close_connection is closed + + +def test_header_mixin_closes_on_absolute_timeout(monkeypatch): + module = _load("scm_broker_server") + handler = _handler(module, b"GET / HTTP/1.1\r\n") + handler.close_connection = False + monkeypatch.setattr( + handler._header_reader, + "readline", + lambda *_a, **_k: (_ for _ in ()).throw(TimeoutError()), + ) + handler.handle_one_request() + assert handler.close_connection is True + + +class _Slots: + def __init__(self, available=True): + self.available = available + self.releases = 0 + + def acquire(self, *, blocking): + assert blocking is False + return self.available + + def release(self): + self.releases += 1 + + +def _server_shell(module, available=True): + server = object.__new__(module.BoundedThreadingHTTPServer) + server._slots = _Slots(available) + server.shutdowns = [] + server.shutdown_request = server.shutdowns.append + return server + + +def test_bounded_server_delegates_accepted_request(monkeypatch): + module = _load("scm_broker_server") + server = _server_shell(module) + calls = [] + monkeypatch.setattr( + module.ThreadingHTTPServer, + "process_request", + lambda self, request, address: calls.append((request, address)), + ) + server.process_request("socket", ("127.0.0.1", 1)) + assert calls == [("socket", ("127.0.0.1", 1))] + assert server._slots.releases == 0 + + +def test_bounded_server_releases_on_dispatch_failure(monkeypatch): + module = _load("scm_broker_server") + server = _server_shell(module) + monkeypatch.setattr( + module.ThreadingHTTPServer, + "process_request", + lambda *_a, **_k: (_ for _ in ()).throw(RuntimeError("dispatch")), + ) + with pytest.raises(RuntimeError, match="dispatch"): + server.process_request("socket", ("127.0.0.1", 1)) + assert server._slots.releases == 1 + assert server.shutdowns == ["socket"] + + +def test_bounded_server_thread_always_releases(monkeypatch): + module = _load("scm_broker_server") + server = _server_shell(module) + monkeypatch.setattr( + module.ThreadingHTTPServer, + "process_request_thread", + lambda *_a, **_k: (_ for _ in ()).throw(RuntimeError("handler")), + ) + with pytest.raises(RuntimeError, match="handler"): + server.process_request_thread("socket", ("127.0.0.1", 1)) + assert server._slots.releases == 1 diff --git a/testing/tests/test_quality_coverage_helpers.py b/testing/tests/test_quality_coverage_helpers.py index 5eaa624e..fc22a8ab 100644 --- a/testing/tests/test_quality_coverage_helpers.py +++ b/testing/tests/test_quality_coverage_helpers.py @@ -12,10 +12,15 @@ def test_compute_workspace_line_coverage_handles_missing_xml(tmp_path: Path) -> """Missing coverage XML should produce a zero workspace coverage score.""" contract = {"coverage": {"tracked_files": ["managed.py"]}} - assert compute_workspace_line_coverage(contract, tmp_path, tmp_path / "missing.xml") == 0.0 + assert ( + compute_workspace_line_coverage(contract, tmp_path, tmp_path / "missing.xml") + == 0.0 + ) -def test_compute_workspace_line_coverage_averages_present_tracked_files(tmp_path: Path) -> None: +def test_compute_workspace_line_coverage_averages_present_tracked_files( + tmp_path: Path, +) -> None: """Workspace coverage should average only tracked files that appear in the report.""" coverage_xml = tmp_path / "coverage.xml" @@ -26,8 +31,8 @@ def test_compute_workspace_line_coverage_averages_present_tracked_files(tmp_path - - + + @@ -41,7 +46,9 @@ def test_compute_workspace_line_coverage_averages_present_tracked_files(tmp_path assert compute_workspace_line_coverage(contract, tmp_path, coverage_xml) == 75.0 -def test_run_check_keeps_relative_names_when_source_roots_do_not_match(tmp_path: Path) -> None: +def test_run_check_keeps_relative_names_when_source_roots_do_not_match( + tmp_path: Path, +) -> None: """Relative filenames should remain relative when no declared source root contains them.""" coverage_xml = tmp_path / "coverage.xml" @@ -74,3 +81,73 @@ def test_run_check_keeps_relative_names_when_source_roots_do_not_match(tmp_path: ) assert issues == ["coverage below 95.0%: relative.py (80.0%)"] + + +def test_run_check_enforces_branch_floor_and_requires_branch_data( + tmp_path: Path, +) -> None: + """Tracked modules must report and meet the configured branch floor.""" + + coverage_xml = tmp_path / "coverage.xml" + coverage_xml.write_text( + textwrap.dedent( + """\ + + + + + + + + + + + """ + ), + encoding="utf-8", + ) + + issues = run_check( + { + "coverage": { + "minimum_percent": 95.0, + "minimum_branch_percent": 95.0, + "tracked_files": ["low.py", "missing.py"], + } + }, + tmp_path, + coverage_xml, + ) + + assert issues == [ + "branch coverage below 95.0%: low.py (90.0%)", + "branch coverage missing for tracked file: missing.py", + ] + + +def test_run_check_limits_branch_floor_to_explicit_branch_tracked_files( + tmp_path: Path, +) -> None: + """A branch rollout can cover selected security modules without hiding lines.""" + + coverage_xml = tmp_path / "coverage.xml" + coverage_xml.write_text( + """ + + + """, + encoding="utf-8", + ) + issues = run_check( + { + "coverage": { + "minimum_percent": 95, + "minimum_branch_percent": 95, + "tracked_files": ["new.py", "legacy.py"], + "branch_tracked_files": ["new.py"], + } + }, + tmp_path, + coverage_xml, + ) + assert issues == [] -- 2.47.2 From 2019b082760b2b2840243c7dcdc124e7adc0601e Mon Sep 17 00:00:00 2001 From: jenkins Date: Mon, 17 Aug 2026 15:15:35 -0300 Subject: [PATCH 05/12] gitea: require literal branch protection rules Replace the reimplemented gobwas/glob matcher with exact-literal rule matching that fails closed on any special or malformed pattern, accepts legacy zero priorities, and rejects duplicate primary-branch rules. Co-Authored-By: Claude Fable 5 --- .../scripts/gitea_branch_protection_check.py | 111 ++++-------------- .../test_hermes_gitea_branch_coverage.py | 70 +++++++---- .../tests/test_hermes_scm_broker_policy.py | 86 ++++---------- 3 files changed, 98 insertions(+), 169 deletions(-) diff --git a/services/gitea/scripts/gitea_branch_protection_check.py b/services/gitea/scripts/gitea_branch_protection_check.py index e3f112c9..5cb9717d 100644 --- a/services/gitea/scripts/gitea_branch_protection_check.py +++ b/services/gitea/scripts/gitea_branch_protection_check.py @@ -7,7 +7,6 @@ import argparse import datetime import json import os -import re import stat from pathlib import Path @@ -35,90 +34,25 @@ def _read_bounded(path: Path) -> bytes: return value -def _glob_regex(pattern: str, position: int = 0, terminators: str = "") -> tuple[str, int]: - """Compile the gobwas/glob syntax Gitea v1.23.8 uses.""" - pieces: list[str] = [] - while position < len(pattern): - character = pattern[position] - if character in terminators: - break - if character == "\\": - position += 1 - if position >= len(pattern): - raise ValueError("trailing escape") - pieces.append(re.escape(pattern[position])) - elif character == "*": - if position + 1 < len(pattern) and pattern[position + 1] == "*": - pieces.append(".*") - position += 1 - else: - pieces.append("[^/]*") - elif character == "?": - pieces.append("[^/]") - elif character == "[": - end = pattern.find("]", position + 1) - if end < 0: - raise ValueError("unterminated range") - value = pattern[position + 1 : end] - if not value: - raise ValueError("empty range") - negate = value.startswith("!") - value = value[1:] if negate else value - if not value or "[" in value or "\\" in value: - raise ValueError("invalid range") - if "-" in value: - if value.count("-") != 1 or value.startswith("-") or value.endswith("-"): - raise ValueError("invalid range") - low, high = value.split("-") - if len(low) != 1 or len(high) != 1 or ord(high) < ord(low): - raise ValueError("invalid range") - content = re.escape(low) + "-" + re.escape(high) - else: - content = re.escape(value) - pieces.append("[" + ("^" if negate else "") + content + "]") - position = end - elif character == "{": - alternatives: list[str] = [] - position += 1 - while True: - alternative, position = _glob_regex(pattern, position, ",}") - alternatives.append(alternative) - if position >= len(pattern): - raise ValueError("unterminated alternatives") - if pattern[position] == "}": - break - position += 1 - if len(alternatives) < 2 or any(not item for item in alternatives): - raise ValueError("invalid alternatives") - pieces.append("(?:" + "|".join(alternatives) + ")") - elif character in "]}": - # A closing delimiter outside its grammar context is literal in - # gobwas/glob's lexer. - pieces.append(re.escape(character)) - else: - pieces.append(re.escape(character)) - position += 1 - return "".join(pieces), position - - def _is_plain(pattern: str) -> bool: return not any(character in SPECIAL for character in pattern) -def _matches(pattern: str, branch: str) -> bool: +def _literal_rule(pattern: str) -> str: + """Return one exact rule name or reject Gitea glob interpretation. + + Gitea 1.23.8 delegates matching to gobwas/glob. Reimplementing that + grammar and its error quoting rules here would create a second policy + engine. The reconciler instead accepts only plain ASCII literals. This + deliberately rejects braces, ranges, escapes, malformed globs, and even + special rules that appear unrelated: an operator must remove ambiguity + before this job may attest the primary-branch boundary. + """ if not pattern.isascii() or not 1 <= len(pattern) <= MAX_RULE_NAME: raise PolicyError("branch protection rule name is invalid") - if _is_plain(pattern): - return pattern.casefold() == branch.casefold() - try: - expression, position = _glob_regex(pattern) - if position != len(pattern): - raise ValueError("incomplete glob") - except (re.error, ValueError): - # Gitea quotes an invalid special pattern and matches it literally. - # A literal containing a special byte cannot equal main or master. - return False - return re.fullmatch(expression, branch) is not None + if not _is_plain(pattern): + raise PolicyError("glob branch protection rules require human review") + return pattern def _created(value: object) -> datetime.datetime: @@ -159,7 +93,7 @@ def evaluate(value: bytes, branch: str, reviewer: str) -> str: data = json.loads(value) if not isinstance(data, list) or len(data) > MAX_RULES: raise PolicyError("branch protection response is invalid") - ordered: list[tuple[int, bool, datetime.datetime, int, dict[str, object]]] = [] + ordered: list[tuple[int, datetime.datetime, int, dict[str, object]]] = [] for index, raw in enumerate(data): if not isinstance(raw, dict): raise PolicyError("branch protection entry is invalid") @@ -168,18 +102,23 @@ def evaluate(value: bytes, branch: str, reviewer: str) -> str: if ( not isinstance(priority, int) or isinstance(priority, bool) - or not 1 <= priority <= 1_000_000 + or not 0 <= priority <= 1_000_000 or not isinstance(rule_name, str) ): raise PolicyError("branch protection priority is ambiguous") created = _created(raw.get("created_at")) - if _matches(rule_name, branch): - # Gitea v1.23.8 sorts by priority, then puts plain names before - # globs, then uses creation time. The API exposes every component. - ordered.append((priority, not _is_plain(rule_name), created, index, raw)) + literal = _literal_rule(rule_name) + if literal == branch: + # With every rule constrained to a plain literal, Gitea's glob + # comparator cannot affect this result. Preserve API ordering: + # priority first (including legacy zero), creation time, then the + # stable response order for an otherwise exact tie. + ordered.append((priority, created, index, raw)) if not ordered: return "ABSENT" - effective = min(ordered)[4] + if len(ordered) != 1: + raise PolicyError("duplicate primary-branch protections are ambiguous") + effective = ordered[0][3] for key, expected in _required(reviewer).items(): if effective.get(key) != expected: raise PolicyError( diff --git a/testing/tests/test_hermes_gitea_branch_coverage.py b/testing/tests/test_hermes_gitea_branch_coverage.py index 362e8726..48421f01 100644 --- a/testing/tests/test_hermes_gitea_branch_coverage.py +++ b/testing/tests/test_hermes_gitea_branch_coverage.py @@ -83,27 +83,16 @@ def test_read_bounded_rejects_unsafe_or_changed_input( monkeypatch.setattr(module.os, "read", real_read) -@pytest.mark.parametrize( - ("pattern", "branch", "expected"), - [ - ("m?in", "main", True), - ("m[ai]in", "main", True), - ("m[!z]in", "main", True), - ("m]ain", "m]ain", True), - ("m}ain", "m}ain", True), - ("m[a-z]in", "main", True), - ("m{ain,aster}", "main", True), - ("m\\ain", "main", True), - ], -) -def test_glob_grammar_accepts_supported_constructs(pattern, branch, expected): - module = _module("branch_glob_supported") - assert module._matches(pattern, branch) is expected - - @pytest.mark.parametrize( "pattern", [ + "{main}", + "{main,master}", + "m?in", + "m[ai]in", + "m[!z]in", + "m[a-z]in", + "m\\ain", "m\\", "m[", "m[]", @@ -120,16 +109,26 @@ def test_glob_grammar_accepts_supported_constructs(pattern, branch, expected): "m{ain,aster", ], ) -def test_invalid_globs_are_literal_nonmatches(pattern): +def test_every_special_or_malformed_glob_fails_closed(pattern): module = _module("branch_glob_invalid") - assert module._matches(pattern, "main") is False + with pytest.raises(module.PolicyError, match="glob branch protection"): + module._literal_rule(pattern) @pytest.mark.parametrize("pattern", ["", "x" * 256, "máin"]) def test_rule_names_are_ascii_and_bounded(pattern): module = _module("branch_name_bounds") with pytest.raises(module.PolicyError, match="rule name"): - module._matches(pattern, "main") + module._literal_rule(pattern) + + +def test_plain_rule_matching_is_exact_and_case_sensitive(): + module = _module("branch_literal_case") + assert module._literal_rule("main") == "main" + assert module._literal_rule("MAIN") == "MAIN" + assert module.evaluate( + json.dumps([_rule(module, name="MAIN")]).encode(), "main", "bstein" + ) == "ABSENT" @pytest.mark.parametrize("value", [None, "x" * 65, "invalid", "2026-01-01T00:00:00"]) @@ -157,7 +156,7 @@ def test_evaluate_rejects_unknown_branch_and_excess_rules(): module.evaluate(json.dumps([{}] * 101).encode(), "main", "bstein") -@pytest.mark.parametrize("priority", [True, 0, 1_000_001, "1"]) +@pytest.mark.parametrize("priority", [True, -1, 1_000_001, "1"]) def test_evaluate_rejects_ambiguous_priority_types(priority): module = _module("branch_priority_bounds") rule = _rule(module, priority=priority) @@ -165,6 +164,33 @@ def test_evaluate_rejects_ambiguous_priority_types(priority): module.evaluate(json.dumps([rule]).encode(), "main", "bstein") +def test_legacy_zero_priority_is_accepted_but_duplicate_ties_fail_closed(): + module = _module("branch_priority_zero") + assert module.evaluate( + json.dumps([_rule(module, priority=0)]).encode(), "main", "bstein" + ) == "PRESENT" + + first = _rule(module, priority=0, created_at="2026-01-01T00:00:00Z") + second = _rule( + module, + priority=0, + created_at="2026-01-01T00:00:00Z", + enable_force_push=True, + ) + with pytest.raises(module.PolicyError, match="duplicate primary-branch"): + module.evaluate(json.dumps([first, second]).encode(), "main", "bstein") + + +def test_any_special_rule_blocks_absent_create_and_present_readback(): + module = _module("branch_special_readback") + for rules in ( + [_rule(module, name="feature/*")], + [_rule(module), _rule(module, name="{main}", priority=0)], + ): + with pytest.raises(module.PolicyError, match="human review"): + module.evaluate(json.dumps(rules).encode(), "main", "bstein") + + def test_main_reports_success_and_failures(tmp_path: Path, monkeypatch, capsys): module = _module("branch_main_paths") source = tmp_path / "rules.json" diff --git a/testing/tests/test_hermes_scm_broker_policy.py b/testing/tests/test_hermes_scm_broker_policy.py index 25d94fa0..163cbc91 100644 --- a/testing/tests/test_hermes_scm_broker_policy.py +++ b/testing/tests/test_hermes_scm_broker_policy.py @@ -216,83 +216,47 @@ def _protection(rule_name: str, priority: int, helper, **overrides): return value -def test_branch_protection_uses_first_effective_glob_by_priority(): +def test_branch_protection_requires_exact_literal_rules_only(): helper = _load_path( "branch_protection_test", ROOT / "services/gitea/scripts/gitea_branch_protection_check.py", ) - earlier_drift = _protection("m*", 1, helper, required_approvals=0) - later_exact = _protection("main", 2, helper) + exact = _protection("main", 2, helper) + assert ( + helper.evaluate(json.dumps([exact]).encode(), "main", "bstein") == "PRESENT" + ) + drifted = _protection("main", 2, helper, required_approvals=0) with pytest.raises(helper.PolicyError, match="effective main protection differs"): - helper.evaluate(json.dumps([later_exact, earlier_drift]).encode(), "main", "bstein") + helper.evaluate(json.dumps([drifted]).encode(), "main", "bstein") - earlier_drift["required_approvals"] = 1 - assert ( - helper.evaluate(json.dumps([later_exact, earlier_drift]).encode(), "main", "bstein") - == "PRESENT" - ) - brace_drift = _protection("m{ain,aster}", 1, helper, required_approvals=0) - with pytest.raises(helper.PolicyError, match="effective master protection differs"): - helper.evaluate(json.dumps([brace_drift]).encode(), "master", "bstein") - - upper_glob = _protection("M*", 1, helper, required_approvals=0) - assert ( - helper.evaluate(json.dumps([upper_glob, later_exact]).encode(), "main", "bstein") - == "PRESENT" - ) - - -def test_branch_protection_matches_gitea_v1238_tie_order(): - helper = _load_path( - "branch_protection_order_test", - ROOT / "services/gitea/scripts/gitea_branch_protection_check.py", - ) - older_glob = _protection( - "m*", 1, helper, created_at="2025-01-01T00:00:00Z", required_approvals=0 - ) - newer_plain = _protection( - "MAIN", 1, helper, created_at="2026-01-01T00:00:00Z" - ) - assert ( - helper.evaluate(json.dumps([older_glob, newer_plain]).encode(), "main", "bstein") - == "PRESENT" - ) - - older_glob["created_at"] = "2026-01-02T00:00:00Z" - earlier_glob = _protection( - "m{ain,aster}", - 1, - helper, - created_at="2026-01-01T00:00:00Z", - required_approvals=0, - ) - with pytest.raises(helper.PolicyError, match="effective main protection differs"): - helper.evaluate(json.dumps([older_glob, earlier_glob]).encode(), "main", "bstein") + duplicate = _protection("main", 1, helper) + with pytest.raises(helper.PolicyError, match="duplicate primary-branch"): + helper.evaluate(json.dumps([exact, duplicate]).encode(), "main", "bstein") @pytest.mark.parametrize( - ("rule_name", "branch", "expected"), + "rule_name", [ - ("release/*", "release/v1.17", True), - ("release/**/v1.17", "release/test/1/v1.17", True), - ("release/*/v1.17", "release/test/1/v1.17", False), - ("*", "release/v1.16", False), - ("**", "release/v1.16", True), - ("MAIN", "main", True), - ("M*", "main", False), - ("m{ain,aster}", "master", True), - (r"m\ain", "main", True), + "release/*", + "release/**/v1.17", + "*", + "**", + "M*", + "m{ain,aster}", + r"m\ain", + "m?in", + "m[ai]in", ], ) -def test_branch_glob_matches_gitea_v1238_source_corpus( - rule_name: str, branch: str, expected: bool -): +def test_branch_glob_rules_fail_closed_pending_human_review(rule_name: str): helper = _load_path( "branch_protection_glob_parity_test", ROOT / "services/gitea/scripts/gitea_branch_protection_check.py", ) - assert helper._matches(rule_name, branch) is expected + rules = [_protection(rule_name, 1, helper)] + with pytest.raises(helper.PolicyError, match="human review"): + helper.evaluate(json.dumps(rules).encode(), "main", "bstein") @pytest.mark.parametrize( @@ -317,5 +281,5 @@ def test_branch_protection_reports_absent_only_when_no_rule_matches(): "branch_protection_absent_test", ROOT / "services/gitea/scripts/gitea_branch_protection_check.py", ) - rules = [_protection("release/*", 1, helper)] + rules = [_protection("develop", 1, helper)] assert helper.evaluate(json.dumps(rules).encode(), "master", "bstein") == "ABSENT" -- 2.47.2 From 578239a49651f4e9ba5bb348fc7a78a3f11e5f30 Mon Sep 17 00:00:00 2001 From: jenkins Date: Mon, 17 Aug 2026 15:15:35 -0300 Subject: [PATCH 06/12] hermes: bound SCM control calls with hard deadlines Run every Gitea API and broker control exchange inside a killable helper process whose connect, send, and read share one absolute wall-clock deadline, and add a watchdog that force-closes streaming connections at expiry. Redirects are rejected before authentication headers can move. Co-Authored-By: Claude Fable 5 --- .../scm-common/scripts/deadline_http.py | 302 +++++++++++++++++ .../hermes/scm-common/scripts/gitea_api.py | 22 +- .../scm-common/scripts/scm_broker_client.py | 20 +- testing/tests/test_hermes_deadline_http.py | 309 ++++++++++++++++++ .../test_hermes_gitea_internal_coverage.py | 15 +- testing/tests/test_hermes_gitea_pr_client.py | 19 +- .../test_hermes_scm_broker_client_coverage.py | 17 +- 7 files changed, 663 insertions(+), 41 deletions(-) create mode 100644 services/hermes/scm-common/scripts/deadline_http.py create mode 100644 testing/tests/test_hermes_deadline_http.py diff --git a/services/hermes/scm-common/scripts/deadline_http.py b/services/hermes/scm-common/scripts/deadline_http.py new file mode 100644 index 00000000..a19abccc --- /dev/null +++ b/services/hermes/scm-common/scripts/deadline_http.py @@ -0,0 +1,302 @@ +#!/usr/bin/env python3 +"""Absolute wall-clock deadlines for every outbound Hermes HTTP exchange. + +Small control exchanges run inside a killable helper process, so connect, +send, read, and DNS all share one hard bound that no blocking socket state +can outlive. Large streaming Git exchanges cannot cross a process boundary; +they instead register their connection with a watchdog that force-closes the +socket when the same kind of absolute deadline expires. +""" + +from __future__ import annotations + +import base64 +import contextlib +import email.message +import http.client +import io +import json +import os +import signal +import socket +import subprocess +import sys +import threading +import time +import urllib.error +import urllib.request +from dataclasses import dataclass + +from gitea_api_policy import PolicyError + +MAX_CONTROL_BYTES = 128 * 1024 +MAX_PROTOCOL_BYTES = 4 * 1024 * 1024 +MAX_STREAM_SECONDS = 900.0 +READ_CHUNK = 64 * 1024 + + +class _RejectRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + raise urllib.error.URLError("redirect rejected") + + +@dataclass(frozen=True) +class Result: + """Minimal HTTP evidence returned across the process boundary.""" + + status: int + content_type: str + body: bytes + + +def _read_response(response, maximum: int, deadline: float) -> bytes: + chunks: list[bytes] = [] + total = 0 + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError + raw = getattr(getattr(response, "fp", None), "raw", None) + sock = getattr(raw, "_sock", raw) + if hasattr(sock, "settimeout"): + sock.settimeout(remaining) + chunk = response.read(min(READ_CHUNK, maximum + 1 - total)) + if not chunk: + break + chunks.append(chunk) + total += len(chunk) + if total > maximum: + raise ValueError("response too large") + return b"".join(chunks) + + +def _child_exchange(control: dict[str, object]) -> Result: + url = control.get("url") + method = control.get("method") + headers = control.get("headers") + encoded = control.get("body") + maximum = control.get("maximum") + timeout = control.get("timeout") + if ( + not isinstance(url, str) + or not isinstance(method, str) + or not isinstance(headers, dict) + or not all(isinstance(key, str) and isinstance(value, str) for key, value in headers.items()) + or not isinstance(encoded, str) + or not isinstance(maximum, int) + or not 0 <= maximum <= 2 * 1024 * 1024 + or not isinstance(timeout, (int, float)) + or not 0 < timeout <= 120 + ): + raise ValueError("invalid control") + body = base64.b64decode(encoded, validate=True) if encoded else None + if body is not None and len(body) > MAX_CONTROL_BYTES: + raise ValueError("request too large") + deadline = time.monotonic() + float(timeout) + request = urllib.request.Request( + url, data=body, method=method, headers=headers # type: ignore[arg-type] + ) + opener = urllib.request.build_opener(_RejectRedirect()) + try: + response = opener.open(request, timeout=max(deadline - time.monotonic(), 0.001)) + except urllib.error.HTTPError as error: + response = error + with response: + status = getattr(response, "status", response.getcode()) + content_type = response.headers.get_content_type() + value = _read_response(response, maximum, deadline) + return Result(int(status), str(content_type), value) + + +def _child_main() -> int: + try: + raw = sys.stdin.buffer.read(MAX_CONTROL_BYTES + 1) + if len(raw) > MAX_CONTROL_BYTES: + raise ValueError("control too large") + control = json.loads(raw) + if not isinstance(control, dict): + raise ValueError("invalid control") + result = _child_exchange(control) + output = { + "ok": True, + "status": result.status, + "content_type": result.content_type, + "body": base64.b64encode(result.body).decode("ascii"), + } + except Exception: + output = {"ok": False} + sys.stdout.write(json.dumps(output, separators=(",", ":"))) + return 0 if output["ok"] else 1 + + +def _terminate(process: subprocess.Popen[bytes]) -> None: + with contextlib.suppress(ProcessLookupError): + os.killpg(process.pid, signal.SIGKILL) + process.communicate() + + +def exchange( + request: urllib.request.Request, + *, + maximum: int, + timeout: float, + popen=subprocess.Popen, +) -> Result: + """Execute a whole request under one monotonic deadline and size limit.""" + if not 0 < timeout <= 120 or not 0 <= maximum <= 2 * 1024 * 1024: + raise PolicyError("HTTP control request bounds are invalid") + body = request.data or b"" + if not isinstance(body, bytes) or len(body) > MAX_CONTROL_BYTES: + raise PolicyError("HTTP control request exceeds the safe size limit") + control = json.dumps( + { + "url": request.full_url, + "method": request.get_method(), + "headers": dict(request.header_items()), + "body": base64.b64encode(body).decode("ascii"), + "maximum": maximum, + "timeout": timeout, + }, + separators=(",", ":"), + ).encode() + deadline = time.monotonic() + timeout + process = popen( + [sys.executable, os.path.realpath(__file__), "--child"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + start_new_session=True, + env={"LANG": "C.UTF-8", "PATH": "/usr/bin:/bin"}, + ) + try: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise subprocess.TimeoutExpired(process.args, timeout) + output, _error = process.communicate(control, timeout=remaining) + except subprocess.TimeoutExpired as exc: + _terminate(process) + raise PolicyError("HTTP control request deadline exceeded") from exc + if process.returncode != 0 or len(output) > MAX_PROTOCOL_BYTES: + raise PolicyError("HTTP control request failed") + try: + result = json.loads(output) + if not isinstance(result, dict) or result.get("ok") is not True: + raise ValueError + status = result["status"] + content_type = result["content_type"] + encoded = result["body"] + if not isinstance(status, int) or not isinstance(content_type, str) or not isinstance(encoded, str): + raise ValueError + value = base64.b64decode(encoded, validate=True) + except (KeyError, ValueError, TypeError, json.JSONDecodeError) as exc: + raise PolicyError("HTTP control helper returned invalid evidence") from exc + if len(value) > maximum: + raise PolicyError("HTTP control response exceeds the safe size limit") + return Result(status, content_type, value) + + +class _AdaptedResponse: + """Present one finished exchange through the urlopen reader interface.""" + + def __init__(self, result: Result): + self.status = result.status + self.headers = email.message.Message() + self.headers["Content-Type"] = result.content_type + self._stream = io.BytesIO(result.body) + + def read(self, limit: int = -1) -> bytes: + return self._stream.read(limit) + + def __enter__(self): + return self + + def __exit__(self, *_excinfo: object) -> bool: + return False + + +def open_bounded( + request: urllib.request.Request, + *, + maximum: int, + timeout: float, + popen=subprocess.Popen, +) -> _AdaptedResponse: + """Open one control exchange whose whole lifetime shares one deadline.""" + return _AdaptedResponse( + exchange(request, maximum=maximum, timeout=timeout, popen=popen) + ) + + +class StreamDeadline: + """Force-close tracked connections once an absolute deadline passes.""" + + def __init__(self, timeout: float, *, timer=threading.Timer): + if not 0 < timeout <= MAX_STREAM_SECONDS: + raise PolicyError("HTTP stream deadline bounds are invalid") + self._deadline = time.monotonic() + timeout + self._lock = threading.Lock() + self._connections: list[http.client.HTTPConnection] = [] + self.expired = False + self._timer = timer(timeout, self._expire) + self._timer.daemon = True + self._timer.start() + + def remaining(self) -> float: + value = self._deadline - time.monotonic() + if value <= 0 or self.expired: + raise PolicyError("HTTP stream deadline exceeded") + return value + + def _expire(self) -> None: + with self._lock: + self.expired = True + connections = list(self._connections) + for connection in connections: + # Shut the raw socket down first so a blocked send or recv wakes + # immediately; close alone leaves peers waiting on the old fd. + sock = getattr(connection, "sock", None) + if sock is not None: + with contextlib.suppress(OSError): + sock.shutdown(socket.SHUT_RDWR) + connection.close() + + def _track(self, connection: http.client.HTTPConnection): + with self._lock: + expired = self.expired + if not expired: + self._connections.append(connection) + if expired: + connection.close() + raise PolicyError("HTTP stream deadline exceeded") + return connection + + def cancel(self) -> None: + self._timer.cancel() + + def handlers(self) -> tuple[urllib.request.BaseHandler, ...]: + """Build urllib handlers whose connections obey this deadline.""" + deadline = self + + class GuardedHTTPHandler(urllib.request.HTTPHandler): + def http_open(self, req): + return self.do_open( + lambda host, **kwargs: deadline._track( + http.client.HTTPConnection(host, **kwargs) + ), + req, + ) + + class GuardedHTTPSHandler(urllib.request.HTTPSHandler): + def https_open(self, req): + return self.do_open( + lambda host, **kwargs: deadline._track( + http.client.HTTPSConnection(host, **kwargs) + ), + req, + ) + + return (GuardedHTTPHandler(), GuardedHTTPSHandler()) + + +if __name__ == "__main__": + raise SystemExit(_child_main() if sys.argv[1:] == ["--child"] else 2) diff --git a/services/hermes/scm-common/scripts/gitea_api.py b/services/hermes/scm-common/scripts/gitea_api.py index d207cc63..3a7de241 100644 --- a/services/hermes/scm-common/scripts/gitea_api.py +++ b/services/hermes/scm-common/scripts/gitea_api.py @@ -15,6 +15,7 @@ import urllib.request from collections.abc import Callable from pathlib import Path +import deadline_http from gitea_api_policy import ( PolicyError, _draft_title, @@ -40,19 +41,16 @@ MAX_API_PATH_LENGTH = 512 RAW_API_TARGET_RE = re.compile(r"[A-Za-z0-9/_.?&=-]+\Z") -class RejectRedirectHandler(urllib.request.HTTPRedirectHandler): - """Reject every redirect before urllib can copy authentication headers.""" - - def redirect_request(self, req, fp, code, msg, headers, newurl): - raise PolicyError("Forgejo redirects are not allowed") - - -_SAFE_OPENER = urllib.request.build_opener(RejectRedirectHandler()) - - def _safe_urlopen(request: urllib.request.Request, timeout: int): - """Open one request using a handler that never follows redirects.""" - return _SAFE_OPENER.open(request, timeout=timeout) + """Open one exchange under a killable absolute wall-clock deadline. + + The helper process rejects redirects before urllib can copy + authentication headers, and its whole connect/send/read lifetime + shares one hard deadline. + """ + return deadline_http.open_bounded( + request, maximum=MAX_RESPONSE_BYTES, timeout=timeout + ) def read_token(path: Path = DEFAULT_TOKEN_FILE) -> str: diff --git a/services/hermes/scm-common/scripts/scm_broker_client.py b/services/hermes/scm-common/scripts/scm_broker_client.py index 6c57ddfd..9a5265b6 100644 --- a/services/hermes/scm-common/scripts/scm_broker_client.py +++ b/services/hermes/scm-common/scripts/scm_broker_client.py @@ -7,24 +7,22 @@ import json import urllib.request from collections.abc import Callable +import deadline_http from gitea_api_policy import PolicyError BROKER_ORIGIN = "http://hermes-scm-broker.hermes-scm.svc.cluster.local:9081" MAX_RESPONSE_BYTES = 2 * 1024 * 1024 -class RejectRedirectHandler(urllib.request.HTTPRedirectHandler): - """Keep every broker request on its fixed in-cluster origin.""" - - def redirect_request(self, req, fp, code, msg, headers, newurl): - raise PolicyError("SCM broker redirects are not allowed") - - -_OPENER = urllib.request.build_opener(RejectRedirectHandler()) - - def _open(request: urllib.request.Request, timeout: int): - return _OPENER.open(request, timeout=timeout) + """Open one broker exchange under a killable absolute deadline. + + The helper process keeps every request on its fixed in-cluster origin by + rejecting redirects before they are followed. + """ + return deadline_http.open_bounded( + request, maximum=MAX_RESPONSE_BYTES, timeout=timeout + ) def request( diff --git a/testing/tests/test_hermes_deadline_http.py b/testing/tests/test_hermes_deadline_http.py new file mode 100644 index 00000000..45fd07e4 --- /dev/null +++ b/testing/tests/test_hermes_deadline_http.py @@ -0,0 +1,309 @@ +"""Absolute-deadline contracts for the killable HTTP control helper.""" + +from __future__ import annotations + +import base64 +import io +import json +import subprocess +import threading +import time +import urllib.request +from http.server import BaseHTTPRequestHandler, HTTPServer + +import pytest + +from testing.tests.test_hermes_scm_broker_support import _load + + +class _Handler(BaseHTTPRequestHandler): + def _respond(self): + if self.path == "/slow": + time.sleep(10) + if self.path == "/redirect": + self.send_response(302) + self.send_header("Location", "http://127.0.0.1:1/evil") + self.send_header("Content-Length", "0") + self.end_headers() + return + status = 404 if self.path == "/missing" else 200 + body = b"x" * 64 if self.path == "/large" else b'{"ok":true}' + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + do_GET = _respond + do_POST = _respond + + def log_message(self, *_args): + return + + +@pytest.fixture(scope="module") +def server(): + value = HTTPServer(("127.0.0.1", 0), _Handler) + thread = threading.Thread(target=value.serve_forever, daemon=True) + thread.start() + yield f"http://127.0.0.1:{value.server_port}" + value.shutdown() + + +def _control(url: str, **overrides): + value = { + "url": url, + "method": "GET", + "headers": {"Accept": "application/json"}, + "body": "", + "maximum": 1024, + "timeout": 10, + } + value.update(overrides) + return value + + +def test_child_exchange_round_trips_and_reports_http_errors(server): + module = _load("deadline_http") + result = module._child_exchange(_control(server + "/ok")) + assert (result.status, result.content_type) == (200, "application/json") + assert result.body == b'{"ok":true}' + + posted = module._child_exchange( + _control( + server + "/ok", + method="POST", + body=base64.b64encode(b'{"a":1}').decode(), + ) + ) + assert posted.status == 200 + + missing = module._child_exchange(_control(server + "/missing")) + assert missing.status == 404 + + +def test_child_exchange_rejects_redirects_and_oversized_responses(server): + module = _load("deadline_http") + with pytest.raises(Exception): + module._child_exchange(_control(server + "/redirect")) + with pytest.raises(ValueError, match="too large"): + module._child_exchange(_control(server + "/large", maximum=8)) + + +@pytest.mark.parametrize( + "overrides", + [ + {"url": 1}, + {"method": None}, + {"headers": ["not", "dict"]}, + {"headers": {"Accept": 5}}, + {"body": b"raw"}, + {"maximum": "big"}, + {"maximum": -1}, + {"maximum": 4 * 1024 * 1024}, + {"timeout": "soon"}, + {"timeout": 0}, + {"timeout": 121}, + ], +) +def test_child_exchange_rejects_invalid_control_documents(overrides): + module = _load("deadline_http") + control = _control("http://127.0.0.1:1/") + control.update(overrides) + with pytest.raises(ValueError, match="invalid control"): + module._child_exchange(control) + + +def test_child_exchange_rejects_bad_or_oversized_encoded_bodies(): + module = _load("deadline_http") + with pytest.raises(ValueError): + module._child_exchange(_control("http://127.0.0.1:1/", body="!!not-b64!!")) + huge = base64.b64encode(b"x" * (module.MAX_CONTROL_BYTES + 1)).decode() + with pytest.raises(ValueError, match="request too large"): + module._child_exchange(_control("http://127.0.0.1:1/", body=huge)) + + +def test_read_response_enforces_deadline_and_size(): + module = _load("deadline_http") + + class Reader: + def __init__(self, body: bytes): + self.stream = io.BytesIO(body) + + def read(self, limit: int): + return self.stream.read(limit) + + with pytest.raises(TimeoutError): + module._read_response(Reader(b"abc"), 10, time.monotonic() - 1) + with pytest.raises(ValueError, match="too large"): + module._read_response(Reader(b"abcdef"), 4, time.monotonic() + 5) + value = module._read_response(Reader(b"abc"), 10, time.monotonic() + 5) + assert value == b"abc" + + +def _run_child_main(module, monkeypatch, raw: bytes) -> dict: + stdin = type("Stdin", (), {"buffer": io.BytesIO(raw)})() + stdout = io.StringIO() + monkeypatch.setattr(module.sys, "stdin", stdin) + monkeypatch.setattr(module.sys, "stdout", stdout) + code = module._child_main() + return code, json.loads(stdout.getvalue()) + + +def test_child_main_reports_success_and_failure_documents(monkeypatch, server): + module = _load("deadline_http") + code, output = _run_child_main( + module, monkeypatch, json.dumps(_control(server + "/ok")).encode() + ) + assert code == 0 and output["ok"] is True + assert base64.b64decode(output["body"]) == b'{"ok":true}' + + for raw in ( + b"x" * (module.MAX_CONTROL_BYTES + 1), + b"not json", + b"[1,2]", + ): + code, output = _run_child_main(module, monkeypatch, raw) + assert code == 1 and output == {"ok": False} + + +class _FakeProcess: + def __init__(self, output: bytes, returncode: int = 0): + self.args = ["fake-child"] + self.pid = 0 + self.returncode = returncode + self.output = output + self.stdin_payload = None + + def communicate(self, payload=None, timeout=None): + self.stdin_payload = payload + return self.output, b"" + + +def _fake_popen(output: bytes, returncode: int = 0): + def popen(*_args, **_kwargs): + return _FakeProcess(output, returncode) + + return popen + + +def _success_output(body: bytes, **overrides) -> bytes: + value = { + "ok": True, + "status": 200, + "content_type": "application/json", + "body": base64.b64encode(body).decode("ascii"), + } + value.update(overrides) + return json.dumps(value).encode() + + +def test_exchange_validates_bounds_and_request_body(): + module = _load("deadline_http") + request = urllib.request.Request("https://scm.bstein.dev/api/v1/x") + with pytest.raises(module.PolicyError, match="bounds are invalid"): + module.exchange(request, maximum=1, timeout=0) + with pytest.raises(module.PolicyError, match="bounds are invalid"): + module.exchange(request, maximum=-1, timeout=10) + request.data = "not-bytes" + with pytest.raises(module.PolicyError, match="safe size"): + module.exchange(request, maximum=1, timeout=10) + request.data = b"x" * (module.MAX_CONTROL_BYTES + 1) + with pytest.raises(module.PolicyError, match="safe size"): + module.exchange(request, maximum=1, timeout=10) + + +def test_exchange_returns_result_and_enforces_response_limit(): + module = _load("deadline_http") + request = urllib.request.Request( + "https://scm.bstein.dev/api/v1/x", data=b"{}", method="POST" + ) + result = module.exchange( + request, maximum=64, timeout=10, popen=_fake_popen(_success_output(b"body")) + ) + assert (result.status, result.content_type, result.body) == ( + 200, + "application/json", + b"body", + ) + with pytest.raises(module.PolicyError, match="safe size"): + module.exchange( + request, + maximum=2, + timeout=10, + popen=_fake_popen(_success_output(b"body")), + ) + + +@pytest.mark.parametrize( + ("output", "returncode", "match"), + [ + (b"", 1, "request failed"), + (b"x" * (4 * 1024 * 1024 + 1), 0, "request failed"), + (b"not json", 0, "invalid evidence"), + (b'{"ok":false}', 0, "invalid evidence"), + (b"[1]", 0, "invalid evidence"), + (b'{"ok":true}', 0, "invalid evidence"), + (b'{"ok":true,"status":"200","content_type":"a","body":""}', 0, "invalid evidence"), + (b'{"ok":true,"status":200,"content_type":"a","body":"!!"}', 0, "invalid evidence"), + ], +) +def test_exchange_rejects_invalid_helper_evidence(output, returncode, match): + module = _load("deadline_http") + request = urllib.request.Request("https://scm.bstein.dev/api/v1/x") + with pytest.raises(module.PolicyError, match=match): + module.exchange( + request, maximum=64, timeout=10, popen=_fake_popen(output, returncode) + ) + + +def test_exchange_kills_the_helper_when_the_deadline_expires(server): + module = _load("deadline_http") + request = urllib.request.Request(server + "/slow") + started = time.monotonic() + with pytest.raises(module.PolicyError, match="deadline exceeded"): + module.exchange(request, maximum=64, timeout=1.0) + assert time.monotonic() - started < 5 + + +def test_exchange_fails_before_spawn_wait_when_time_is_exhausted(monkeypatch): + module = _load("deadline_http") + request = urllib.request.Request("https://scm.bstein.dev/api/v1/x") + killed = [] + monkeypatch.setattr(module.os, "killpg", lambda pid, sig: killed.append((pid, sig))) + clock = iter([0.0, 1_000_000.0, 1_000_000.0, 1_000_000.0]) + monkeypatch.setattr(module.time, "monotonic", lambda: next(clock)) + with pytest.raises(module.PolicyError, match="deadline exceeded"): + module.exchange( + request, maximum=64, timeout=10, popen=_fake_popen(_success_output(b"")) + ) + assert killed + + +def test_terminate_tolerates_already_finished_helpers(): + module = _load("deadline_http") + process = subprocess.Popen( + ["/bin/true"], start_new_session=True, stdout=subprocess.DEVNULL + ) + process.wait() + module._terminate(process) + assert process.returncode == 0 + + +def test_child_entrypoint_round_trip_through_real_subprocess(server): + module = _load("deadline_http") + request = urllib.request.Request(server + "/ok") + result = module.exchange(request, maximum=1024, timeout=20) + assert result.status == 200 and result.body == b'{"ok":true}' + + +def test_open_bounded_adapts_result_to_the_urlopen_interface(): + module = _load("deadline_http") + request = urllib.request.Request("https://scm.bstein.dev/api/v1/x") + with module.open_bounded( + request, maximum=64, timeout=10, popen=_fake_popen(_success_output(b"body")) + ) as response: + assert response.status == 200 + assert response.headers.get_content_type() == "application/json" + assert response.read(2) == b"bo" + assert response.read() == b"dy" diff --git a/testing/tests/test_hermes_gitea_internal_coverage.py b/testing/tests/test_hermes_gitea_internal_coverage.py index f811827c..da720878 100644 --- a/testing/tests/test_hermes_gitea_internal_coverage.py +++ b/testing/tests/test_hermes_gitea_internal_coverage.py @@ -37,16 +37,21 @@ def test_runtime_token_reader_and_fixed_origin(tmp_path: Path, monkeypatch): module.configured_base_url() -def test_safe_urlopen_delegates_only_to_redirect_rejecting_opener(monkeypatch): +def test_safe_urlopen_delegates_only_to_deadline_bounded_exchange(monkeypatch): module = _load() sentinel = object() monkeypatch.setattr( - module._SAFE_OPENER, - "open", - lambda request, timeout: (request, timeout, sentinel), + module.deadline_http, + "open_bounded", + lambda request, *, maximum, timeout: (request, maximum, timeout, sentinel), ) request = object() - assert module._safe_urlopen(request, 7) == (request, 7, sentinel) + assert module._safe_urlopen(request, 7) == ( + request, + module.MAX_RESPONSE_BYTES, + 7, + sentinel, + ) def test_api_target_and_request_body_edge_paths(monkeypatch): diff --git a/testing/tests/test_hermes_gitea_pr_client.py b/testing/tests/test_hermes_gitea_pr_client.py index ae419e10..4ddb9830 100644 --- a/testing/tests/test_hermes_gitea_pr_client.py +++ b/testing/tests/test_hermes_gitea_pr_client.py @@ -2,21 +2,22 @@ from __future__ import annotations +import urllib.error import urllib.request import pytest from testing.tests.test_hermes_gitea_support import HEAD_SHA, Response, _load -def test_redirect_handler_rejects_cross_origin_with_sentinel_authorization(): +def test_default_opener_rejects_redirects_under_an_absolute_deadline(monkeypatch): client = _load() source = urllib.request.Request( "https://scm.bstein.dev/api/v1/repos/atlas/cassandra", headers={"Authorization": "token redirect-sentinel"}, ) - with pytest.raises(client.PolicyError, match="redirects are not allowed") as exc: - client.RejectRedirectHandler().redirect_request( + with pytest.raises(urllib.error.URLError) as exc: + client.deadline_http._RejectRedirect().redirect_request( source, None, 302, @@ -26,10 +27,16 @@ def test_redirect_handler_rejects_cross_origin_with_sentinel_authorization(): ) assert "redirect-sentinel" not in str(exc.value) - assert any( - isinstance(handler, client.RejectRedirectHandler) - for handler in client._SAFE_OPENER.handlers + seen = [] + monkeypatch.setattr( + client.deadline_http, + "open_bounded", + lambda request, *, maximum, timeout: seen.append( + (request, maximum, timeout) + ), ) + client._safe_urlopen(source, 30) + assert seen == [(source, client.MAX_RESPONSE_BYTES, 30)] @pytest.mark.parametrize( diff --git a/testing/tests/test_hermes_scm_broker_client_coverage.py b/testing/tests/test_hermes_scm_broker_client_coverage.py index ca6a357d..9fa65fa6 100644 --- a/testing/tests/test_hermes_scm_broker_client_coverage.py +++ b/testing/tests/test_hermes_scm_broker_client_coverage.py @@ -9,17 +9,20 @@ import pytest from testing.tests.test_hermes_scm_broker_support import Response, _load -def test_client_redirect_and_default_opener_are_fail_closed(monkeypatch): +def test_client_default_opener_uses_deadline_bounded_exchange(monkeypatch): client = _load("scm_broker_client") - with pytest.raises(client.PolicyError, match="redirects"): - client.RejectRedirectHandler().redirect_request( - object(), None, 302, "redirect", {}, "http://elsewhere.invalid" - ) sentinel = object() monkeypatch.setattr( - client._OPENER, "open", lambda request, timeout: (request, timeout, sentinel) + client.deadline_http, + "open_bounded", + lambda request, *, maximum, timeout: (request, maximum, timeout, sentinel), + ) + assert client._open("request", 4) == ( + "request", + client.MAX_RESPONSE_BYTES, + 4, + sentinel, ) - assert client._open("request", 4) == ("request", 4, sentinel) @pytest.mark.parametrize( -- 2.47.2 From 340ec58b68b3e8e1f46a1337b4855acedc1bc6ac Mon Sep 17 00:00:00 2001 From: jenkins Date: Mon, 17 Aug 2026 15:15:47 -0300 Subject: [PATCH 07/12] hermes: quarantine-scan pushed git objects Inflate every receive-pack object under strict pack, size, and checksum bounds, resolve deltas against in-pack bases only, and scan the real decompressed payloads for runtime-token forms, private keys, SSH key material, and known provider token formats. Thin packs are rejected so no pushed content escapes the scan, and upstream Git exchanges now run under one absolute stream deadline. Co-Authored-By: Claude Fable 5 --- services/hermes/agent-configmap.yaml | 4 +- services/hermes/scm-common/kustomization.yaml | 3 + .../scm-common/scripts/git_pack_objects.py | 308 ++++++++++++++++++ .../scm-common/scripts/receive_pack_scan.py | 143 ++++++++ .../hermes/scm-common/scripts/scm_broker.py | 103 ++---- testing/tests/test_hermes_deadline_stream.py | 217 ++++++++++++ testing/tests/test_hermes_git_pack_objects.py | 288 ++++++++++++++++ .../tests/test_hermes_receive_pack_scan.py | 217 ++++++++++++ ...est_hermes_scm_broker_internal_coverage.py | 29 +- .../tests/test_hermes_scm_broker_streaming.py | 2 +- .../tests/test_hermes_scm_broker_support.py | 33 +- 11 files changed, 1261 insertions(+), 86 deletions(-) create mode 100644 services/hermes/scm-common/scripts/git_pack_objects.py create mode 100644 services/hermes/scm-common/scripts/receive_pack_scan.py create mode 100644 testing/tests/test_hermes_deadline_stream.py create mode 100644 testing/tests/test_hermes_git_pack_objects.py create mode 100644 testing/tests/test_hermes_receive_pack_scan.py diff --git a/services/hermes/agent-configmap.yaml b/services/hermes/agent-configmap.yaml index b53b2c92..28d520fb 100644 --- a/services/hermes/agent-configmap.yaml +++ b/services/hermes/agent-configmap.yaml @@ -320,7 +320,9 @@ data: unauthenticated Gitea HTTP 404 as a missing private repository. Use brokered Git for clone, fetch, and creation of a new namespaced feature branch. Existing-ref updates, protected refs, deletion, and force-push are - rejected. For bounded + rejected. The broker inflates and scans every pushed object, so thin + packs are rejected; always push with `git push --no-thin` so the pack + is self-contained. For bounded repository/pull-request evidence or to create a review-ready draft PR, load `$manage-atlas-pull-requests` and use `/opt/scm/gitea_api.py`. The client carries no repository credential; a separate least-authority diff --git a/services/hermes/scm-common/kustomization.yaml b/services/hermes/scm-common/kustomization.yaml index 17dc1da6..a4756ccd 100644 --- a/services/hermes/scm-common/kustomization.yaml +++ b/services/hermes/scm-common/kustomization.yaml @@ -4,8 +4,11 @@ kind: Kustomization configMapGenerator: - name: hermes-scm-boundary files: + - deadline_http.py=scripts/deadline_http.py + - git_pack_objects.py=scripts/git_pack_objects.py - gitea_api.py=scripts/gitea_api.py - gitea_api_policy.py=scripts/gitea_api_policy.py + - receive_pack_scan.py=scripts/receive_pack_scan.py - scm_broker.py=scripts/scm_broker.py - scm_broker_io.py=scripts/scm_broker_io.py - scm_broker_server.py=scripts/scm_broker_server.py diff --git a/services/hermes/scm-common/scripts/git_pack_objects.py b/services/hermes/scm-common/scripts/git_pack_objects.py new file mode 100644 index 00000000..d9a3e715 --- /dev/null +++ b/services/hermes/scm-common/scripts/git_pack_objects.py @@ -0,0 +1,308 @@ +#!/usr/bin/env python3 +"""Bounded Git pack inflation and delta resolution for quarantine scanning. + +Pushed objects arrive zlib-compressed inside a packfile, so any content +screening must inflate the real payloads first. This module unpacks one +self-contained pack under explicit object, size, and checksum bounds and +resolves every delta against bases inside the same pack. Thin packs are +rejected outright: a delta against a base the broker cannot see would leave +part of the pushed content unscanned. +""" + +from __future__ import annotations + +import hashlib +import tempfile +import zlib +from typing import BinaryIO + +from gitea_api_policy import PolicyError + +OBJECT_TYPE_NAMES = {1: b"commit", 2: b"tree", 3: b"blob", 4: b"tag"} +OFS_DELTA = 6 +REF_DELTA = 7 +MAX_PACK_OBJECTS = 50_000 +MAX_OBJECT_BYTES = 32 * 1024 * 1024 +MAX_TOTAL_BYTES = 256 * 1024 * 1024 +MAX_RESOLVE_PASSES = 64 +SPOOL_MEMORY_LIMIT = 256 * 1024 +READ_CHUNK = 64 * 1024 + + +def _read_exact(stream: BinaryIO, size: int) -> bytes: + value = stream.read(size) + if len(value) != size: + raise PolicyError("Git pack data ended early") + return value + + +def _object_header(stream: BinaryIO) -> tuple[int, int]: + """Parse one object's type and declared inflated size.""" + byte = _read_exact(stream, 1)[0] + type_code = (byte >> 4) & 0x7 + size = byte & 0x0F + shift = 4 + while byte & 0x80: + if shift > 60: + raise PolicyError("Git object header is invalid") + byte = _read_exact(stream, 1)[0] + size |= (byte & 0x7F) << shift + shift += 7 + return type_code, size + + +def _delta_distance(stream: BinaryIO) -> int: + """Parse one ofs-delta base distance.""" + byte = _read_exact(stream, 1)[0] + value = byte & 0x7F + while byte & 0x80: + byte = _read_exact(stream, 1)[0] + value = ((value + 1) << 7) | (byte & 0x7F) + if value > 1 << 48: + raise PolicyError("Git delta base distance is invalid") + return value + + +def _inflate(stream: BinaryIO, declared: int) -> BinaryIO: + """Inflate one object into a quarantine spool, bounded by its header.""" + spool = tempfile.SpooledTemporaryFile( # noqa: SIM115 - caller owns returned spool + max_size=SPOOL_MEMORY_LIMIT, dir="/tmp" + ) + decompressor = zlib.decompressobj() + produced = 0 + try: + while not decompressor.eof: + data = stream.read(READ_CHUNK) + if not data: + raise PolicyError("Git pack object data ended early") + while True: + value = decompressor.decompress(data, READ_CHUNK) + produced += len(value) + if produced > declared: + raise PolicyError("Git object size does not match its header") + spool.write(value) + data = decompressor.unconsumed_tail + if decompressor.eof or not data: + break + if decompressor.unused_data: + stream.seek(-len(decompressor.unused_data), 1) + if produced != declared: + raise PolicyError("Git object size does not match its header") + spool.seek(0) + return spool + except zlib.error as exc: + spool.close() + raise PolicyError("Git pack object data is corrupt") from exc + except Exception: + spool.close() + raise + + +def _delta_size(delta: bytes, position: int) -> tuple[int, int]: + size = 0 + shift = 0 + while True: + if position >= len(delta) or shift > 60: + raise PolicyError("Git delta header is invalid") + byte = delta[position] + position += 1 + size |= (byte & 0x7F) << shift + shift += 7 + if not byte & 0x80: + return size, position + + +def _apply_delta(base: bytes, delta: bytes) -> bytes: + """Apply one Git delta and require exact declared sizes throughout.""" + source_size, position = _delta_size(delta, 0) + target_size, position = _delta_size(delta, position) + if source_size != len(base) or target_size > MAX_OBJECT_BYTES: + raise PolicyError("Git delta sizes are invalid") + parts: list[bytes] = [] + produced = 0 + while position < len(delta): + opcode = delta[position] + position += 1 + if opcode & 0x80: + operands = [bit for bit in (1, 2, 4, 8, 16, 32, 64) if opcode & bit] + if position + len(operands) > len(delta): + raise PolicyError("Git delta copy is truncated") + offset = 0 + size = 0 + for shift, bit in enumerate((1, 2, 4, 8)): + if opcode & bit: + offset |= delta[position] << (8 * shift) + position += 1 + for shift, bit in enumerate((16, 32, 64)): + if opcode & bit: + size |= delta[position] << (8 * shift) + position += 1 + size = size or 0x10000 + if offset + size > len(base): + raise PolicyError("Git delta copy is out of bounds") + parts.append(base[offset : offset + size]) + produced += size + elif opcode: + if position + opcode > len(delta): + raise PolicyError("Git delta insert is truncated") + parts.append(delta[position : position + opcode]) + position += opcode + produced += opcode + else: + raise PolicyError("Git delta instruction is invalid") + if produced > target_size: + raise PolicyError("Git delta result exceeds its declared size") + if produced != target_size: + raise PolicyError("Git delta result size is invalid") + return b"".join(parts) + + +def _payload_sha(type_code: int, payload: BinaryIO, size: int) -> bytes: + digest = hashlib.sha1(OBJECT_TYPE_NAMES[type_code] + b" %d\x00" % size) + payload.seek(0) + while True: + chunk = payload.read(READ_CHUNK) + if not chunk: + break + digest.update(chunk) + payload.seek(0) + return digest.digest() + + +def _verify_checksum(stream: BinaryIO, start: int, end: int) -> None: + trailer = _read_exact(stream, 20) + stream.seek(start) + digest = hashlib.sha1() + remaining = end - start + while remaining: + chunk = stream.read(min(READ_CHUNK, remaining)) + digest.update(chunk) + remaining -= len(chunk) + if digest.digest() != trailer: + raise PolicyError("Git pack checksum is invalid") + stream.seek(end + 20) + + +class _Entry: + """One pack entry: either resolved content or an unresolved delta.""" + + def __init__(self, type_code: int, payload: BinaryIO, size: int, base): + self.type_code = type_code + self.payload = payload + self.size = size + self.base = base + + +def _resolve(entries: dict[int, _Entry], total: list[int]) -> None: + """Resolve every delta against in-pack bases only, to a fixed point.""" + by_sha: dict[bytes, int] = {} + for offset, entry in entries.items(): + if entry.base is None: + by_sha.setdefault( + _payload_sha(entry.type_code, entry.payload, entry.size), offset + ) + for _ in range(MAX_RESOLVE_PASSES): + pending = [item for item in entries.items() if item[1].base is not None] + if not pending: + return + progressed = False + for offset, entry in pending: + base_key = entry.base + base_offset = base_key if isinstance(base_key, int) else by_sha.get(base_key) + base = entries.get(base_offset) if base_offset is not None else None + if base is None or base.base is not None: + if isinstance(base_key, int) and base is None: + raise PolicyError("Git delta base offset is invalid") + continue + base.payload.seek(0) + entry.payload.seek(0) + value = _apply_delta(base.payload.read(), entry.payload.read()) + total[0] += len(value) + if total[0] > MAX_TOTAL_BYTES: + raise PolicyError("Git pack contents exceed the safe size limit") + entry.payload.close() + spool = tempfile.SpooledTemporaryFile( # noqa: SIM115 - entry owns spool + max_size=SPOOL_MEMORY_LIMIT, dir="/tmp" + ) + spool.write(value) + spool.seek(0) + entry.type_code = base.type_code + entry.payload = spool + entry.size = len(value) + entry.base = None + by_sha.setdefault(_payload_sha(entry.type_code, spool, entry.size), offset) + progressed = True + if not progressed: + raise PolicyError("Git delta base is outside the pack; push full packs") + raise PolicyError("Git delta chains exceed the safe resolution limit") + + +def unpack_objects(stream: BinaryIO) -> list[tuple[int, BinaryIO]]: + """Inflate one self-contained pack and return every resolved payload.""" + start = stream.tell() + header = _read_exact(stream, 12) + version = int.from_bytes(header[4:8], "big") + count = int.from_bytes(header[8:12], "big") + if header[:4] != b"PACK" or version not in {2, 3}: + raise PolicyError("Git pack header is invalid") + if count > MAX_PACK_OBJECTS: + raise PolicyError("Git pack contains too many objects") + entries: dict[int, _Entry] = {} + total = [0] + try: + for _ in range(count): + offset = stream.tell() - start + type_code, declared = _object_header(stream) + if declared > MAX_OBJECT_BYTES: + raise PolicyError("Git object exceeds the safe size limit") + total[0] += declared + if total[0] > MAX_TOTAL_BYTES: + raise PolicyError("Git pack contents exceed the safe size limit") + base: int | bytes | None = None + if type_code == OFS_DELTA: + distance = _delta_distance(stream) + base = offset - distance + if base < 0: + raise PolicyError("Git delta base offset is invalid") + elif type_code == REF_DELTA: + base = _read_exact(stream, 20) + elif type_code not in OBJECT_TYPE_NAMES: + raise PolicyError("Git pack object type is invalid") + entries[offset] = _Entry(type_code, _inflate(stream, declared), declared, base) + _verify_checksum(stream, start, stream.tell()) + _resolve(entries, total) + except Exception: + for entry in entries.values(): + entry.payload.close() + raise + for entry in entries.values(): + entry.payload.seek(0) + return [ + (entries[offset].type_code, entries[offset].payload) + for offset in sorted(entries) + ] + + +def parse_loose_object(value: bytes) -> tuple[int, bytes]: + """Inflate one quarantined loose object and return its typed payload.""" + if len(value) > MAX_OBJECT_BYTES: + raise PolicyError("Git object exceeds the safe size limit") + decompressor = zlib.decompressobj() + try: + inflated = decompressor.decompress(value, MAX_OBJECT_BYTES + 1) + except zlib.error as exc: + raise PolicyError("Git loose object data is corrupt") from exc + if not decompressor.eof or decompressor.unused_data or len(inflated) > MAX_OBJECT_BYTES: + raise PolicyError("Git loose object framing is invalid") + kind, separator, remainder = inflated.partition(b"\x00") + name, space, size = kind.partition(b" ") + types = {label: code for code, label in OBJECT_TYPE_NAMES.items()} + if ( + not separator + or not space + or name not in types + or not size.isdigit() + or int(size) != len(remainder) + ): + raise PolicyError("Git loose object header is invalid") + return types[name], remainder diff --git a/services/hermes/scm-common/scripts/receive_pack_scan.py b/services/hermes/scm-common/scripts/receive_pack_scan.py new file mode 100644 index 00000000..76980880 --- /dev/null +++ b/services/hermes/scm-common/scripts/receive_pack_scan.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +"""Validate Git receive-pack requests and scan pushed object content. + +Ref commands may only create new namespaced feature branches. Every pushed +object is then inflated in quarantine and its decompressed payload scanned, +so credentials cannot ride through the broker inside compressed pack data. +The screening is a fail-closed accident barrier for credential-shaped +content: exact runtime-token forms, private keys, SSH key material, and +known provider token formats. Entropy heuristics are deliberately excluded +so SOPS ciphertext, pinned digests, and lock files remain pushable; a short +secret disguised as prose is not reliably detectable and must never be +supplied by callers. +""" + +from __future__ import annotations + +import io +import re +from typing import BinaryIO + +import git_pack_objects +from gitea_api_policy import ( + STANDALONE_SECRET_PATTERNS, + PolicyError, + _validate_ref, +) + +ZERO_SHA = b"0" * 40 +MAX_PUSH_COMMANDS = 16 +SCAN_CHUNK = 256 * 1024 +SCAN_CARRY = 8 * 1024 +FEATURE_REF_RE = re.compile( + r"refs/heads/(?:(?:feature|fix|hermes|handoff)/[A-Za-z0-9][A-Za-z0-9._/-]{0,190})\Z" +) +PKT_HEADER_RE = re.compile(rb"[0-9a-f]{4}\Z") +CONTENT_SECRET_PATTERNS = tuple( + re.compile(pattern.pattern.encode("ascii"), re.IGNORECASE) + for pattern in STANDALONE_SECRET_PATTERNS +) + ( + re.compile(rb"(?i)(? bytes | None: + """Read one pkt-line; None marks the flush packet before the pack.""" + header = stream.read(4) + if not header: + raise PolicyError("Git receive-pack request omitted its command terminator") + if not PKT_HEADER_RE.fullmatch(header): + raise PolicyError("Git receive-pack command framing is invalid") + size = int(header, 16) + if size == 0: + return None + if size < 4: + raise PolicyError("Git receive-pack command framing is invalid") + value = stream.read(size - 4) + if len(value) != size - 4: + raise PolicyError("Git receive-pack command framing is invalid") + return value + + +def _validate_command(line: bytes, token: str, forbidden: tuple[bytes, ...]) -> None: + """Permit only creation of one new, namespaced feature branch.""" + if any(form in line for form in forbidden): + raise PolicyError("Git request contains runtime credential material") + command = line.rstrip(b"\n").split(b"\x00", 1)[0] + fields = command.split(b" ") + if len(fields) != 3 or not all( + re.fullmatch(rb"[0-9a-f]{40}", item) for item in fields[:2] + ): + raise PolicyError("Git receive-pack ref command is invalid") + old_sha, new_sha, raw_ref = fields + if old_sha != ZERO_SHA or new_sha == ZERO_SHA: + raise PolicyError("Git broker permits only new feature-branch creation") + try: + ref = raw_ref.decode("ascii") + except UnicodeDecodeError as exc: + raise PolicyError("Git ref must be canonical ASCII") from exc + if not FEATURE_REF_RE.fullmatch(ref): + raise PolicyError("Git push is limited to namespaced feature branches") + _validate_ref(ref.removeprefix("refs/heads/"), "head", forbidden=(token,)) + + +def _scan_payload(payload: BinaryIO, forbidden: tuple[bytes, ...]) -> None: + """Scan one decompressed object payload across chunk boundaries.""" + carry = b"" + while True: + chunk = payload.read(SCAN_CHUNK) + if not chunk: + return + window = carry + chunk + if any(form in window for form in forbidden): + raise PolicyError("Git push contains runtime credential material") + if any(pattern.search(window) for pattern in CONTENT_SECRET_PATTERNS): + raise PolicyError("Git push contains credential-shaped content") + carry = window[-SCAN_CARRY:] + + +def _scan_pack(stream: BinaryIO, forbidden: tuple[bytes, ...]) -> None: + objects = git_pack_objects.unpack_objects(stream) + try: + if stream.read(1): + raise PolicyError("Git request has bytes after its pack") + for _type_code, payload in objects: + _scan_payload(payload, forbidden) + finally: + for _type_code, payload in objects: + payload.close() + + +def validate_receive_pack( + body: bytes | BinaryIO, token: str, forbidden: tuple[bytes, ...] +) -> None: + """Validate one whole receive-pack request: commands, pack, and content.""" + stream = io.BytesIO(body) if isinstance(body, bytes) else body + stream.seek(0) + try: + commands = 0 + while True: + line = _read_pkt_line(stream) + if line is None: + break + _validate_command(line, token, forbidden) + commands += 1 + if commands > MAX_PUSH_COMMANDS: + raise PolicyError("Git push contains too many ref commands") + if commands == 0: + raise PolicyError("Git receive-pack request has no ref command") + _scan_pack(stream, forbidden) + finally: + stream.seek(0) + + +def scan_loose_object(value: bytes, forbidden: tuple[bytes, ...]) -> int: + """Scan one quarantined loose object and return its parsed type code.""" + type_code, payload = git_pack_objects.parse_loose_object(value) + _scan_payload(io.BytesIO(payload), forbidden) + return type_code diff --git a/services/hermes/scm-common/scripts/scm_broker.py b/services/hermes/scm-common/scripts/scm_broker.py index a0c4043c..a9cb3685 100644 --- a/services/hermes/scm-common/scripts/scm_broker.py +++ b/services/hermes/scm-common/scripts/scm_broker.py @@ -16,6 +16,7 @@ import urllib.request from http.server import BaseHTTPRequestHandler from typing import BinaryIO +import deadline_http from gitea_api import ( CANONICAL_BASE_URL, PolicyError, @@ -23,7 +24,8 @@ from gitea_api import ( read, read_token, ) -from gitea_api_policy import _reject_forbidden, _validate_ref, _validate_repo +from gitea_api_policy import _reject_forbidden, _validate_repo +from receive_pack_scan import validate_receive_pack from scm_broker_io import RejectRedirect, response_status as _status, spool_response from scm_broker_server import AbsoluteHeaderDeadlineMixin, BoundedThreadingHTTPServer @@ -32,24 +34,17 @@ GIT_USER = "hermes-automation" MAX_CONTROL_BODY = 64 * 1024 MAX_GIT_REQUEST = 128 * 1024 * 1024 MAX_GIT_RESPONSE = 128 * 1024 * 1024 -MAX_PUSH_COMMANDS = 16 MAX_HEADERS = 32 MAX_HEADER_BYTES = 16 * 1024 SPOOL_MEMORY_LIMIT = 1024 * 1024 STREAM_CHUNK = 64 * 1024 INBOUND_HEADER_TIMEOUT = 10.0 INBOUND_BODY_TIMEOUT = 120.0 -ZERO_SHA = b"0" * 40 +UPSTREAM_DEADLINE_SECONDS = 300.0 GIT_PATH_RE = re.compile( r"/git/atlas/(?P[A-Za-z0-9][A-Za-z0-9._-]{0,99})\.git/" r"(?Pinfo/refs|git-upload-pack|git-receive-pack)\Z" ) -FEATURE_REF_RE = re.compile( - r"refs/heads/(?:(?:feature|fix|hermes|handoff)/[A-Za-z0-9][A-Za-z0-9._/-]{0,190})\Z" -) - - -UPSTREAM_OPENER = urllib.request.build_opener(RejectRedirect()) def _read_bounded( @@ -126,14 +121,16 @@ def _spool_bounded( raise -def _spool_response(response, maximum: int, token: str) -> tuple[BinaryIO, int]: +def _spool_response( + response, maximum: int, token: str, deadline_seconds: float +) -> tuple[BinaryIO, int]: return spool_response( response, maximum, _credential_forms(token), memory_limit=SPOOL_MEMORY_LIMIT, chunk_size=STREAM_CHUNK, - deadline_seconds=INBOUND_BODY_TIMEOUT, + deadline_seconds=deadline_seconds, ) @@ -211,61 +208,14 @@ def _credential_forms(token: str) -> tuple[bytes, ...]: return token.encode("utf-8"), basic, b"Basic " + basic -def _reject_credential_bytes(value: bytes, token: str, context: str) -> None: - if any(form in value for form in _credential_forms(token)): - raise PolicyError(f"{context} contains runtime credential material") - - -def _receive_prefix(body: bytes | BinaryIO) -> bytes: - if isinstance(body, bytes): - return body - position = body.tell() - try: - body.seek(0) - return body.read(64 * 1024) - finally: - body.seek(position) - - def _validate_receive_pack(body: bytes | BinaryIO, token: str) -> None: - """Permit only creation of new, namespaced feature branches.""" - body = _receive_prefix(body) - _reject_credential_bytes(body, token, "Git request") - position = 0 - commands = 0 - while position + 4 <= len(body): - header = body[position : position + 4] - if not re.fullmatch(rb"[0-9a-f]{4}", header): - raise PolicyError("Git receive-pack command framing is invalid") - size = int(header, 16) - if size == 0: - if commands == 0: - raise PolicyError("Git receive-pack request has no ref command") - return - if size < 4 or position + size > len(body): - raise PolicyError("Git receive-pack command framing is invalid") - command = body[position + 4 : position + size].rstrip(b"\n") - command = command.split(b"\x00", 1)[0] - fields = command.split(b" ") - if len(fields) != 3 or not all( - re.fullmatch(rb"[0-9a-f]{40}", item) for item in fields[:2] - ): - raise PolicyError("Git receive-pack ref command is invalid") - old_sha, new_sha, raw_ref = fields - if old_sha != ZERO_SHA or new_sha == ZERO_SHA: - raise PolicyError("Git broker permits only new feature-branch creation") - try: - ref = raw_ref.decode("ascii") - except UnicodeDecodeError as exc: - raise PolicyError("Git ref must be canonical ASCII") from exc - if not FEATURE_REF_RE.fullmatch(ref): - raise PolicyError("Git push is limited to namespaced feature branches") - _validate_ref(ref.removeprefix("refs/heads/"), "head", forbidden=(token,)) - commands += 1 - if commands > MAX_PUSH_COMMANDS: - raise PolicyError("Git push contains too many ref commands") - position += size - raise PolicyError("Git receive-pack request omitted its command terminator") + """Permit only vetted new-feature-branch pushes with scanned contents.""" + validate_receive_pack(body, token, _credential_forms(token)) + + +def _guarded_opener(guard: deadline_http.StreamDeadline): + """Build an opener whose connection obeys one absolute stream deadline.""" + return urllib.request.build_opener(RejectRedirect(), *guard.handlers()).open def _upstream_git_request( @@ -278,7 +228,7 @@ def _upstream_git_request( token: str, body_length: int | None = None, stream_result: bool = False, - opener=UPSTREAM_OPENER.open, + opener=None, ) -> bytes | tuple[BinaryIO, int]: credentials = base64.b64encode(f"{GIT_USER}:{token}".encode()).decode("ascii") headers = {"Accept": expected_type, "Authorization": f"Basic {credentials}"} @@ -296,12 +246,21 @@ def _upstream_git_request( method=method, headers=headers, ) - with opener(request, timeout=120) as response: - if _status(response) != 200: - raise PolicyError("Git upstream returned an unexpected HTTP status") - if response.headers.get_content_type() != expected_type: - raise PolicyError("Git upstream returned an unexpected response type") - result, result_length = _spool_response(response, MAX_GIT_RESPONSE, token) + # One wall-clock budget covers connect, send, headers, and body read; the + # watchdog closes the upstream socket if any phase overstays it. + guard = deadline_http.StreamDeadline(UPSTREAM_DEADLINE_SECONDS) + try: + open_upstream = opener if opener is not None else _guarded_opener(guard) + with open_upstream(request, timeout=guard.remaining()) as response: + if _status(response) != 200: + raise PolicyError("Git upstream returned an unexpected HTTP status") + if response.headers.get_content_type() != expected_type: + raise PolicyError("Git upstream returned an unexpected response type") + result, result_length = _spool_response( + response, MAX_GIT_RESPONSE, token, guard.remaining() + ) + finally: + guard.cancel() if stream_result: return result, result_length try: diff --git a/testing/tests/test_hermes_deadline_stream.py b/testing/tests/test_hermes_deadline_stream.py new file mode 100644 index 00000000..d4f97efe --- /dev/null +++ b/testing/tests/test_hermes_deadline_stream.py @@ -0,0 +1,217 @@ +"""Watchdog contracts for absolute deadlines on streaming Git exchanges.""" + +from __future__ import annotations + +import threading +import time +import urllib.request +from http.server import BaseHTTPRequestHandler, HTTPServer + +import pytest + +from testing.tests.test_hermes_scm_broker_support import Response, _load + + +def test_stream_deadline_validates_bounds_and_counts_down(): + module = _load("deadline_http") + for timeout in (0, -1, module.MAX_STREAM_SECONDS + 1): + with pytest.raises(module.PolicyError, match="bounds are invalid"): + module.StreamDeadline(timeout) + guard = module.StreamDeadline(30) + try: + first = guard.remaining() + assert 0 < first <= 30 + finally: + guard.cancel() + + +class _FakeTimer: + def __init__(self, timeout, callback): + self.timeout = timeout + self.callback = callback + self.daemon = False + self.started = False + self.cancelled = False + + def start(self): + self.started = True + + def cancel(self): + self.cancelled = True + + +class _FakeSocket: + def __init__(self, fail: bool = False): + self.fail = fail + self.shut = False + + def shutdown(self, _how): + if self.fail: + raise OSError("already gone") + self.shut = True + + +class _FakeConnection: + def __init__(self, sock=None): + self.sock = sock + self.closed = False + + def close(self): + self.closed = True + + +def test_expiry_shuts_down_and_closes_every_tracked_connection(): + module = _load("deadline_http") + guard = module.StreamDeadline(30, timer=_FakeTimer) + assert guard._timer.started and guard._timer.daemon + healthy = _FakeConnection(_FakeSocket()) + broken = _FakeConnection(_FakeSocket(fail=True)) + bare = _FakeConnection() + for connection in (healthy, broken, bare): + assert guard._track(connection) is connection + guard._expire() + assert guard.expired + assert healthy.sock.shut and healthy.closed + assert broken.closed and not broken.sock.shut + assert bare.closed + with pytest.raises(module.PolicyError, match="deadline exceeded"): + guard.remaining() + late = _FakeConnection() + with pytest.raises(module.PolicyError, match="deadline exceeded"): + guard._track(late) + assert late.closed + guard.cancel() + assert guard._timer.cancelled + + +def test_handlers_build_guarded_connections_without_network(monkeypatch): + module = _load("deadline_http") + guard = module.StreamDeadline(30, timer=_FakeTimer) + http_handler, https_handler = guard.handlers() + assert isinstance(http_handler, urllib.request.HTTPHandler) + assert isinstance(https_handler, urllib.request.HTTPSHandler) + for handler, opener_name in ( + (http_handler, "http_open"), + (https_handler, "https_open"), + ): + seen = {} + monkeypatch.setattr( + handler, + "do_open", + lambda factory, req, seen=seen: seen.update(factory=factory, req=req), + ) + getattr(handler, opener_name)("request") + connection = seen["factory"]("127.0.0.1", timeout=1) + assert connection in guard._connections + connection.close() + guard.cancel() + + +class _TrickleHandler(BaseHTTPRequestHandler): + def do_GET(self): + time.sleep(10) + + def log_message(self, *_args): + return + + +def test_watchdog_interrupts_a_hung_streaming_exchange(): + module = _load("deadline_http") + server = HTTPServer(("127.0.0.1", 0), _TrickleHandler) + threading.Thread(target=server.serve_forever, daemon=True).start() + guard = module.StreamDeadline(1.0) + opener = urllib.request.build_opener(*guard.handlers()) + started = time.monotonic() + try: + with pytest.raises(OSError): + opener.open( + f"http://127.0.0.1:{server.server_port}/slow", timeout=30 + ).read() + assert time.monotonic() - started < 5 + assert guard.expired + finally: + guard.cancel() + server.shutdown() + + +def test_upstream_git_request_runs_under_one_wall_clock_budget(monkeypatch): + broker = _load("scm_broker") + seen = {} + + class RecordingGuard: + def __init__(self, timeout, **_kwargs): + seen["timeout"] = timeout + self.cancelled = False + seen["guard"] = self + + def remaining(self): + return 12.5 + + def cancel(self): + self.cancelled = True + + def handlers(self): + return () + + monkeypatch.setattr(broker.deadline_http, "StreamDeadline", RecordingGuard) + + def opener(request, timeout): + seen["opener_timeout"] = timeout + return Response(b"result", content_type="application/x-git-upload-pack-result") + + result = broker._upstream_git_request( + "/atlas/cassandra.git/git-upload-pack", + method="POST", + body=b"request", + content_type="application/x-git-upload-pack-request", + expected_type="application/x-git-upload-pack-result", + token="sentinel", + opener=opener, + ) + assert result == b"result" + assert seen["timeout"] == broker.UPSTREAM_DEADLINE_SECONDS + assert seen["opener_timeout"] == 12.5 + assert seen["guard"].cancelled + + +def test_upstream_git_request_cancels_guard_when_deadline_already_passed(monkeypatch): + broker = _load("scm_broker") + cancelled = [] + + class ExpiredGuard: + def __init__(self, _timeout, **_kwargs): + pass + + def remaining(self): + raise broker.PolicyError("HTTP stream deadline exceeded") + + def cancel(self): + cancelled.append(True) + + def handlers(self): + return () + + monkeypatch.setattr(broker.deadline_http, "StreamDeadline", ExpiredGuard) + with pytest.raises(broker.PolicyError, match="deadline exceeded"): + broker._upstream_git_request( + "/atlas/cassandra.git/git-upload-pack", + method="POST", + body=b"request", + content_type="application/x-git-upload-pack-request", + expected_type="application/x-git-upload-pack-result", + token="sentinel", + opener=lambda *_a, **_k: None, + ) + assert cancelled == [True] + + +def test_guarded_opener_wires_redirect_rejection_and_deadline_handlers(): + broker = _load("scm_broker") + module = _load("deadline_http") + guard = module.StreamDeadline(30, timer=_FakeTimer) + opener = broker._guarded_opener(guard) + assert callable(opener) + handlers = opener.__self__.handlers + assert any(isinstance(item, broker.RejectRedirect) for item in handlers) + assert any(isinstance(item, urllib.request.HTTPSHandler) for item in handlers) + guard.cancel() diff --git a/testing/tests/test_hermes_git_pack_objects.py b/testing/tests/test_hermes_git_pack_objects.py new file mode 100644 index 00000000..f3a249ec --- /dev/null +++ b/testing/tests/test_hermes_git_pack_objects.py @@ -0,0 +1,288 @@ +"""Bounded pack inflation and delta-resolution contracts for pushed objects.""" + +from __future__ import annotations + +import hashlib +import io +import zlib + +import pytest + +from testing.tests.test_hermes_scm_broker_support import ( + _load, + _object_entry, + _pack_of, +) + + +def _delta_header(type_code: int, size: int, suffix: bytes = b"") -> bytes: + byte = (type_code << 4) | (size & 0x0F) + size >>= 4 + header = bytearray() + while size: + header.append(byte | 0x80) + byte = size & 0x7F + size >>= 7 + header.append(byte) + return bytes(header) + suffix + + +def _size_varint(value: int) -> bytes: + out = bytearray() + while True: + byte = value & 0x7F + value >>= 7 + if value: + out.append(byte | 0x80) + else: + out.append(byte) + return bytes(out) + + +def _ofs_distance(value: int) -> bytes: + out = [value & 0x7F] + value >>= 7 + while value: + value -= 1 + out.insert(0, 0x80 | (value & 0x7F)) + value >>= 7 + return bytes(out) + + +def _delta(source: bytes, target_size: int, instructions: bytes) -> bytes: + return _size_varint(len(source)) + _size_varint(target_size) + instructions + + +def _blob_sha(payload: bytes) -> bytes: + return hashlib.sha1(b"blob %d\x00" % len(payload) + payload).digest() + + +def _read_all(objects): + try: + return [(code, payload.read()) for code, payload in objects] + finally: + for _code, payload in objects: + payload.close() + + +def test_unpack_returns_typed_payloads_for_every_base_object_kind(): + module = _load("git_pack_objects") + entries = [ + _object_entry(1, b"tree 1\n"), + _object_entry(2, b"\x00tree-bytes"), + _object_entry(3, b"blob contents"), + _object_entry(4, b"tag contents"), + ] + result = _read_all(module.unpack_objects(io.BytesIO(_pack_of(entries)))) + assert result == [ + (1, b"tree 1\n"), + (2, b"\x00tree-bytes"), + (3, b"blob contents"), + (4, b"tag contents"), + ] + + +def test_unpack_accepts_the_empty_pack(): + module = _load("git_pack_objects") + assert module.unpack_objects(io.BytesIO(_pack_of([]))) == [] + + +def test_ofs_delta_resolves_against_its_in_pack_base(): + module = _load("git_pack_objects") + base = b"base payload for copying" + base_entry = _object_entry(3, base) + # Copy the first 4 bytes, then insert new bytes. + instructions = bytes([0x80 | 0x10, 4]) + bytes([5]) + b"+tail" + delta = _delta(base, 9, instructions) + delta_entry = _delta_header( + 6, len(delta), _ofs_distance(len(base_entry)) + ) + zlib.compress(delta) + result = _read_all(module.unpack_objects(io.BytesIO(_pack_of([base_entry, delta_entry])))) + assert result == [(3, base), (3, b"base+tail")] + + +def test_ref_delta_resolves_by_sha_and_supports_chains(): + module = _load("git_pack_objects") + base = b"chain base" + first = _delta(base, 4, bytes([4]) + b"one!") + second = _delta(b"one!", 4, bytes([4]) + b"two!") + entries = [ + _object_entry(3, base), + _delta_header(7, len(first), _blob_sha(base)) + zlib.compress(first), + _delta_header(7, len(second), _blob_sha(b"one!")) + zlib.compress(second), + ] + result = _read_all(module.unpack_objects(io.BytesIO(_pack_of(entries)))) + assert [payload for _code, payload in result] == [base, b"one!", b"two!"] + + +def test_thin_pack_ref_delta_with_external_base_is_rejected(): + module = _load("git_pack_objects") + delta = _delta(b"absent", 1, bytes([1]) + b"x") + entry = _delta_header(7, len(delta), b"\xaa" * 20) + zlib.compress(delta) + with pytest.raises(module.PolicyError, match="push full packs"): + module.unpack_objects(io.BytesIO(_pack_of([entry]))) + + +def test_ofs_delta_bad_offsets_fail_closed(): + module = _load("git_pack_objects") + base_entry = _object_entry(3, b"base") + delta = _delta(b"base", 1, bytes([1]) + b"x") + # Distance beyond the pack start. + early = _delta_header(6, len(delta), _ofs_distance(999)) + zlib.compress(delta) + with pytest.raises(module.PolicyError, match="base offset is invalid"): + module.unpack_objects(io.BytesIO(_pack_of([base_entry, early]))) + # Distance landing between object boundaries. + misaligned = _delta_header(6, len(delta), _ofs_distance(3)) + zlib.compress(delta) + with pytest.raises(module.PolicyError, match="base offset is invalid"): + module.unpack_objects(io.BytesIO(_pack_of([base_entry, misaligned]))) + + +@pytest.mark.parametrize( + ("mutate", "match"), + [ + (lambda pack: b"JUNK" + pack[4:], "header is invalid"), + (lambda pack: pack[:4] + (9).to_bytes(4, "big") + pack[8:], "header is invalid"), + (lambda pack: pack[:8] + (2).to_bytes(4, "big") + pack[12:], "Git"), + (lambda pack: pack[:-1], "ended early"), + (lambda pack: pack[:-20] + b"\x00" * 20, "checksum is invalid"), + (lambda pack: pack[:11], "ended early"), + ], +) +def test_malformed_pack_containers_fail_closed(mutate, match): + module = _load("git_pack_objects") + pack = _pack_of([_object_entry(3, b"payload")]) + with pytest.raises(module.PolicyError, match=match): + module.unpack_objects(io.BytesIO(mutate(pack))) + + +def test_object_count_and_type_bounds_fail_closed(monkeypatch): + module = _load("git_pack_objects") + body = b"PACK" + (2).to_bytes(4, "big") + (module.MAX_PACK_OBJECTS + 1).to_bytes( + 4, "big" + ) + with pytest.raises(module.PolicyError, match="too many objects"): + module.unpack_objects(io.BytesIO(body + hashlib.sha1(body).digest())) + reserved = _delta_header(5, 1) + zlib.compress(b"x") + with pytest.raises(module.PolicyError, match="object type is invalid"): + module.unpack_objects(io.BytesIO(_pack_of([reserved]))) + + +def test_declared_size_mismatches_and_zlib_bombs_fail_closed(monkeypatch): + module = _load("git_pack_objects") + longer = _delta_header(3, 2) + zlib.compress(b"longer-than-two") + with pytest.raises(module.PolicyError, match="does not match its header"): + module.unpack_objects(io.BytesIO(_pack_of([longer]))) + shorter = _delta_header(3, 9) + zlib.compress(b"x") + with pytest.raises(module.PolicyError, match="does not match its header"): + module.unpack_objects(io.BytesIO(_pack_of([shorter]))) + corrupt = _delta_header(3, 4) + b"\x00not-zlib\x00" + with pytest.raises(module.PolicyError, match="corrupt"): + module.unpack_objects(io.BytesIO(_pack_of([corrupt]))) + + monkeypatch.setattr(module, "MAX_OBJECT_BYTES", 4) + with pytest.raises(module.PolicyError, match="exceeds the safe size limit"): + module.unpack_objects(io.BytesIO(_pack_of([_object_entry(3, b"12345")]))) + monkeypatch.setattr(module, "MAX_OBJECT_BYTES", 1024) + monkeypatch.setattr(module, "MAX_TOTAL_BYTES", 8) + with pytest.raises(module.PolicyError, match="contents exceed"): + module.unpack_objects( + io.BytesIO(_pack_of([_object_entry(3, b"12345"), _object_entry(3, b"12345")])) + ) + + +def test_object_and_distance_varints_are_bounded(): + module = _load("git_pack_objects") + with pytest.raises(module.PolicyError, match="object header is invalid"): + module._object_header(io.BytesIO(b"\xff" * 12)) + with pytest.raises(module.PolicyError, match="ended early"): + module._object_header(io.BytesIO(b"\x80")) + with pytest.raises(module.PolicyError, match="distance is invalid"): + module._delta_distance(io.BytesIO(b"\xff" * 12)) + + +@pytest.mark.parametrize( + ("base", "delta", "match"), + [ + (b"base", b"", "delta header is invalid"), + (b"base", _size_varint(4), "delta header is invalid"), + (b"base", _size_varint(9) + _size_varint(1) + bytes([1]) + b"x", "sizes are invalid"), + (b"base", _size_varint(4) + _size_varint(2) + bytes([0]), "instruction is invalid"), + (b"base", _size_varint(4) + _size_varint(2) + bytes([5]) + b"ab", "insert is truncated"), + (b"base", _size_varint(4) + _size_varint(2) + bytes([0x80 | 0x10]), "copy is truncated"), + ( + b"base", + _size_varint(4) + _size_varint(2) + bytes([0x80 | 0x01 | 0x10, 3, 9]), + "copy is out of bounds", + ), + (b"base", _size_varint(4) + _size_varint(1) + bytes([2]) + b"ab", "exceeds its declared size"), + (b"base", _size_varint(4) + _size_varint(3) + bytes([2]) + b"ab", "result size is invalid"), + ], +) +def test_delta_application_rejects_malformed_instructions(base, delta, match): + module = _load("git_pack_objects") + with pytest.raises(module.PolicyError, match=match): + module._apply_delta(base, delta) + + +def test_delta_copy_with_zero_size_field_means_the_git_default(): + module = _load("git_pack_objects") + base = b"z" * 0x10000 + delta = _delta(base, 0x10000, bytes([0x80 | 0x10 | 0x20, 0, 0])) + assert module._apply_delta(base, delta) == base + + +def test_delta_target_size_cap_applies(monkeypatch): + module = _load("git_pack_objects") + monkeypatch.setattr(module, "MAX_OBJECT_BYTES", 4) + delta = _delta(b"base", 5, bytes([5]) + b"abcde") + with pytest.raises(module.PolicyError, match="sizes are invalid"): + module._apply_delta(b"base", delta) + + +def test_resolution_pass_and_total_caps_apply(monkeypatch): + module = _load("git_pack_objects") + base = b"pass base" + first = _delta(base, 4, bytes([4]) + b"one!") + second = _delta(b"one!", 4, bytes([4]) + b"two!") + entries = [ + _delta_header(7, len(second), _blob_sha(b"one!")) + zlib.compress(second), + _object_entry(3, base), + _delta_header(7, len(first), _blob_sha(base)) + zlib.compress(first), + ] + pack = _pack_of(entries) + monkeypatch.setattr(module, "MAX_RESOLVE_PASSES", 1) + with pytest.raises(module.PolicyError, match="resolution limit"): + module.unpack_objects(io.BytesIO(pack)) + monkeypatch.setattr(module, "MAX_RESOLVE_PASSES", 64) + result = _read_all(module.unpack_objects(io.BytesIO(pack))) + assert [payload for _code, payload in result] == [b"two!", base, b"one!"] + + monkeypatch.setattr(module, "MAX_TOTAL_BYTES", len(base) + len(first) + len(second) + 1) + with pytest.raises(module.PolicyError, match="contents exceed"): + module.unpack_objects(io.BytesIO(pack)) + + +def test_parse_loose_object_round_trip_and_bounds(monkeypatch): + module = _load("git_pack_objects") + payload = b"loose blob" + value = zlib.compress(b"blob %d\x00" % len(payload) + payload) + assert module.parse_loose_object(value) == (3, payload) + + with pytest.raises(module.PolicyError, match="corrupt"): + module.parse_loose_object(b"\x00junk") + with pytest.raises(module.PolicyError, match="framing is invalid"): + module.parse_loose_object(value + b"trailing") + with pytest.raises(module.PolicyError, match="framing is invalid"): + module.parse_loose_object(value[:-4]) + for header in ( + b"noodle 3\x00abc", + b"blob3\x00abc", + b"blob x\x00abc", + b"blob 4\x00abc", + b"blob 3", + ): + with pytest.raises(module.PolicyError, match="header is invalid"): + module.parse_loose_object(zlib.compress(header)) + monkeypatch.setattr(module, "MAX_OBJECT_BYTES", 4) + with pytest.raises(module.PolicyError, match="exceeds the safe size limit"): + module.parse_loose_object(b"12345") \ No newline at end of file diff --git a/testing/tests/test_hermes_receive_pack_scan.py b/testing/tests/test_hermes_receive_pack_scan.py new file mode 100644 index 00000000..7d17fa6f --- /dev/null +++ b/testing/tests/test_hermes_receive_pack_scan.py @@ -0,0 +1,217 @@ +"""Quarantine content-scanning contracts for brokered Git pushes.""" + +from __future__ import annotations + +import hashlib +import io +import zlib + +import pytest + +from testing.tests.test_hermes_scm_broker_support import ( + _blob_pack, + _load, + _object_entry, + _pack_of, + _receive_command, +) + +ZERO = b"0" * 40 +COMMIT = b"1" * 40 +TOKEN = "runtime-sentinel" +FORMS = (b"runtime-sentinel",) + + +def _request(pack: bytes, ref: bytes = b"refs/heads/hermes/scan") -> bytes: + return _receive_command(ZERO, COMMIT, ref, pack) + + +def test_clean_push_with_real_content_passes(): + scan = _load("receive_pack_scan") + pack = _blob_pack( + b"# services/example/deployment.yaml\napiVersion: apps/v1\n", + b"tree 5b3a\nparent none\n", + ) + scan.validate_receive_pack(_request(pack), TOKEN, FORMS) + + +@pytest.mark.parametrize( + "payload", + [ + b"-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaA==\n", + b"-----BEGIN RSA PRIVATE KEY-----\nMIIE\n", + b"key = ghp_" + b"a1" * 12, + b"aws_access_key_id = AKIAABCDEFGHIJKLMNOP", + b"jwt eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxIn0.c2lnbmF0dXJlLXNlZ21lbnQ", + b"hook https://hooks.slack.com/services/T000/B000/tokentokentokentoken", + b"AGE-SECRET-KEY-1QQPCXR7QQPCXR7QQPCXR7QQPCXR7", + b"PuTTY-User-Key-File-3: ssh-ed25519", + b"Authorization: Bearer abc123-tokenvalue", + b"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF2c hermes@atlas", + b"gitea gta_" + b"z9" * 12, + ], +) +def test_credential_shaped_object_content_is_rejected(payload): + scan = _load("receive_pack_scan") + with pytest.raises(scan.PolicyError, match="credential-shaped"): + scan.validate_receive_pack(_request(_blob_pack(payload)), TOKEN, FORMS) + + +def test_runtime_token_hidden_by_compression_is_still_caught(): + scan = _load("receive_pack_scan") + pack = _blob_pack(b"config value: runtime-sentinel\n") + # The compressed wire bytes do not contain the token; only the + # decompressed payload does. + assert b"runtime-sentinel" not in pack + with pytest.raises(scan.PolicyError, match="runtime credential material"): + scan.validate_receive_pack(_request(pack), TOKEN, FORMS) + + +def _size_varint(value: int) -> bytes: + out = bytearray() + while True: + byte = value & 0x7F + value >>= 7 + if value: + out.append(byte | 0x80) + else: + out.append(byte) + return bytes(out) + + +def _typed_header(type_code: int, size: int) -> bytes: + byte = (type_code << 4) | (size & 0x0F) + size >>= 4 + header = bytearray() + while size: + header.append(byte | 0x80) + byte = size & 0x7F + size >>= 7 + header.append(byte) + return bytes(header) + + +def test_secret_assembled_only_by_delta_resolution_is_caught(): + scan = _load("receive_pack_scan") + base = b"docs: AKIAABCD" + completion = b"EFGHIJKLMNOP" + delta = ( + _size_varint(len(base)) + + _size_varint(len(base) + len(completion)) + + bytes([0x80 | 0x10, len(base)]) + + bytes([len(completion)]) + + completion + ) + base_sha = hashlib.sha1(b"blob %d\x00" % len(base) + base).digest() + delta_entry = _typed_header(7, len(delta)) + base_sha + zlib.compress(delta) + pack = _pack_of([_object_entry(3, base), delta_entry]) + # Neither the base nor the delta insert alone matches; the resolved + # payload is the only place the credential exists. + with pytest.raises(scan.PolicyError, match="runtime credential|credential-shaped"): + scan.validate_receive_pack(_request(pack), TOKEN, FORMS) + + +def test_thin_pack_deltas_are_rejected_outright(): + scan = _load("receive_pack_scan") + delta = _size_varint(4) + _size_varint(1) + bytes([1]) + b"x" + entry = _typed_header(7, len(delta)) + b"\xbb" * 20 + zlib.compress(delta) + with pytest.raises(scan.PolicyError, match="push full packs"): + scan.validate_receive_pack(_request(_pack_of([entry])), TOKEN, FORMS) + + +def test_trailing_bytes_after_the_pack_are_rejected(): + scan = _load("receive_pack_scan") + body = _request(_pack_of([])) + b"extra" + with pytest.raises(scan.PolicyError, match="bytes after its pack"): + scan.validate_receive_pack(body, TOKEN, FORMS) + + +@pytest.mark.parametrize( + ("line", "match"), + [ + (b"shallow " + COMMIT, "ref command is invalid"), + (COMMIT + b" " + COMMIT + b" refs/heads/hermes/x", "feature-branch creation"), + (ZERO + b" " + ZERO + b" refs/heads/hermes/x", "feature-branch creation"), + (ZERO + b" " + COMMIT + b" refs/heads/main", "namespaced feature branches"), + (ZERO + b" " + COMMIT + b" refs/tags/v1", "namespaced feature branches"), + (ZERO + b" " + COMMIT + b" refs/heads/hermes/\xff", "canonical ASCII"), + (b"zz" * 20 + b" " + COMMIT + b" refs/heads/hermes/x", "ref command is invalid"), + ], +) +def test_push_commands_outside_the_narrow_grant_are_rejected(line, match): + scan = _load("receive_pack_scan") + framed = f"{len(line) + 5:04x}".encode() + line + b"\n" + b"0000" + with pytest.raises(scan.PolicyError, match=match): + scan.validate_receive_pack(framed + _pack_of([]), TOKEN, FORMS) + + +def test_command_stream_bounds_are_enforced(): + scan = _load("receive_pack_scan") + with pytest.raises(scan.PolicyError, match="no ref command"): + scan.validate_receive_pack(b"0000" + _pack_of([]), TOKEN, FORMS) + with pytest.raises(scan.PolicyError, match="omitted its command terminator"): + scan.validate_receive_pack(b"", TOKEN, FORMS) + line = ZERO + b" " + COMMIT + b" refs/heads/hermes/x\n" + framed = f"{len(line) + 4:04x}".encode() + line + many = framed * (scan.MAX_PUSH_COMMANDS + 1) + b"0000" + _pack_of([]) + with pytest.raises(scan.PolicyError, match="too many ref commands"): + scan.validate_receive_pack(many, TOKEN, FORMS) + with pytest.raises(scan.PolicyError, match="framing is invalid"): + scan.validate_receive_pack(b"zzzz", TOKEN, FORMS) + with pytest.raises(scan.PolicyError, match="framing is invalid"): + scan.validate_receive_pack(b"0003", TOKEN, FORMS) + with pytest.raises(scan.PolicyError, match="framing is invalid"): + scan.validate_receive_pack(b"0009abc", TOKEN, FORMS) + + +def test_credential_forms_in_command_lines_are_rejected(): + scan = _load("receive_pack_scan") + line = ZERO + b" " + COMMIT + b" refs/heads/hermes/x runtime-sentinel\n" + framed = f"{len(line) + 4:04x}".encode() + line + b"0000" + with pytest.raises(scan.PolicyError, match="runtime credential material"): + scan.validate_receive_pack(framed + _pack_of([]), TOKEN, FORMS) + + +def test_scan_crosses_chunk_boundaries_with_carry(monkeypatch): + scan = _load("receive_pack_scan") + monkeypatch.setattr(scan, "SCAN_CHUNK", 8) + monkeypatch.setattr(scan, "SCAN_CARRY", 48) + payload = b"x" * 20 + b" AKIAABCDEFGHIJKLMNOP " + b"y" * 20 + with pytest.raises(scan.PolicyError, match="credential-shaped"): + scan._scan_payload(io.BytesIO(payload), FORMS) + scan._scan_payload(io.BytesIO(b"clean " * 40), FORMS) + scan._scan_payload(io.BytesIO(b""), FORMS) + + +def test_scan_loose_object_reports_type_and_scans_content(): + scan = _load("receive_pack_scan") + clean = zlib.compress(b"blob 5\x00clean") + assert scan.scan_loose_object(clean, FORMS) == 3 + secret = b"-----BEGIN EC PRIVATE KEY-----" + dirty = zlib.compress(b"blob %d\x00" % len(secret) + secret) + with pytest.raises(scan.PolicyError, match="credential-shaped"): + scan.scan_loose_object(dirty, FORMS) + leaked = b"the runtime-sentinel value" + with pytest.raises(scan.PolicyError, match="runtime credential material"): + scan.scan_loose_object( + zlib.compress(b"blob %d\x00" % len(leaked) + leaked), FORMS + ) + + +def test_content_patterns_extend_the_shared_standalone_corpus(): + scan = _load("receive_pack_scan") + policy = _load("gitea_api_policy") + assert len(scan.CONTENT_SECRET_PATTERNS) == len(policy.STANDALONE_SECRET_PATTERNS) + 3 + assert all( + isinstance(pattern.pattern, bytes) for pattern in scan.CONTENT_SECRET_PATTERNS + ) + + +def test_validation_rewinds_the_stream_for_upstream_forwarding(): + scan = _load("receive_pack_scan") + body = _request(_blob_pack(b"safe content")) + stream = io.BytesIO(body) + scan.validate_receive_pack(stream, TOKEN, FORMS) + assert stream.tell() == 0 + with pytest.raises(scan.PolicyError): + scan.validate_receive_pack(io.BytesIO(b"zzzz"), TOKEN, FORMS) \ No newline at end of file diff --git a/testing/tests/test_hermes_scm_broker_internal_coverage.py b/testing/tests/test_hermes_scm_broker_internal_coverage.py index abab8406..b38702ac 100644 --- a/testing/tests/test_hermes_scm_broker_internal_coverage.py +++ b/testing/tests/test_hermes_scm_broker_internal_coverage.py @@ -7,7 +7,11 @@ from email.message import Message import pytest -from testing.tests.test_hermes_scm_broker_support import Response, _load +from testing.tests.test_hermes_scm_broker_support import ( + Response, + _load, + _receive_command, +) def test_broker_redirect_status_and_bounded_read_helpers(monkeypatch): @@ -139,18 +143,22 @@ def test_content_length_and_credential_helpers_cover_safe_and_rejected_values(): assert broker._content_length(headers, 1) == 0 forms = broker._credential_forms("sentinel") assert len(forms) == 3 and forms[0] == b"sentinel" - broker._reject_credential_bytes(b"safe", "sentinel", "response") + zero = b"0" * 40 + commit = b"1" * 40 + line = zero + b" " + commit + b" refs/heads/hermes/x " + forms[2] + b"\n" + framed = f"{len(line) + 4:04x}".encode() + line + b"0000" with pytest.raises(broker.PolicyError, match="credential material"): - broker._reject_credential_bytes(b"prefix " + forms[2], "sentinel", "response") + broker._validate_receive_pack(framed, "sentinel") -def test_receive_prefix_preserves_stream_position_and_accepts_bytes(): +def test_receive_pack_validation_rewinds_stream_and_accepts_bytes(): broker = _load("scm_broker") - assert broker._receive_prefix(b"bytes") == b"bytes" - stream = io.BytesIO(b"prefix") - stream.seek(3) - assert broker._receive_prefix(stream) == b"prefix" - assert stream.tell() == 3 + body = _receive_command(b"0" * 40, b"1" * 40, b"refs/heads/hermes/rewind") + stream = io.BytesIO(body) + stream.seek(9) + broker._validate_receive_pack(stream, "sentinel") + assert stream.tell() == 0 + broker._validate_receive_pack(body, "sentinel") @pytest.mark.parametrize( @@ -188,7 +196,8 @@ def test_receive_pack_rejects_nonascii_many_commands_and_missing_terminator( command = _packet(zero, commit, b"refs/heads/hermes/fix", terminator=False) with pytest.raises(broker.PolicyError, match="terminator"): broker._validate_receive_pack(command, "sentinel") - many = command * (broker.MAX_PUSH_COMMANDS + 1) + b"0000" + limit = _load("receive_pack_scan").MAX_PUSH_COMMANDS + many = command * (limit + 1) + b"0000" with pytest.raises(broker.PolicyError, match="too many"): broker._validate_receive_pack(many, "sentinel") diff --git a/testing/tests/test_hermes_scm_broker_streaming.py b/testing/tests/test_hermes_scm_broker_streaming.py index 2fa9683f..78a63f27 100644 --- a/testing/tests/test_hermes_scm_broker_streaming.py +++ b/testing/tests/test_hermes_scm_broker_streaming.py @@ -130,7 +130,7 @@ def test_upstream_stream_applies_absolute_socket_deadline(): sock = DeadlineSocket() response.fp = type("FP", (), {"raw": type("Raw", (), {"_sock": sock})()})() spool, length = broker._spool_response( - response, broker.MAX_GIT_RESPONSE, "sentinel" + response, broker.MAX_GIT_RESPONSE, "sentinel", broker.INBOUND_BODY_TIMEOUT ) try: assert length == 4 diff --git a/testing/tests/test_hermes_scm_broker_support.py b/testing/tests/test_hermes_scm_broker_support.py index 6e2184d9..e3682494 100644 --- a/testing/tests/test_hermes_scm_broker_support.py +++ b/testing/tests/test_hermes_scm_broker_support.py @@ -2,9 +2,11 @@ from __future__ import annotations +import hashlib import importlib.util import io import sys +import zlib from email.message import Message from pathlib import Path @@ -50,6 +52,33 @@ class Response: return self.stream.read(limit) -def _receive_command(old: bytes, new: bytes, ref: bytes) -> bytes: +def _object_entry(type_code: int, payload: bytes) -> bytes: + """Encode one non-delta pack entry with a real header and zlib body.""" + size = len(payload) + byte = (type_code << 4) | (size & 0x0F) + size >>= 4 + header = bytearray() + while size: + header.append(byte | 0x80) + byte = size & 0x7F + size >>= 7 + header.append(byte) + return bytes(header) + zlib.compress(payload) + + +def _pack_of(entries: list[bytes]) -> bytes: + """Assemble encoded entries into a checksummed version-2 pack.""" + body = b"PACK" + (2).to_bytes(4, "big") + len(entries).to_bytes(4, "big") + for entry in entries: + body += entry + return body + hashlib.sha1(body).digest() + + +def _blob_pack(*payloads: bytes) -> bytes: + return _pack_of([_object_entry(3, payload) for payload in payloads]) + + +def _receive_command(old: bytes, new: bytes, ref: bytes, pack: bytes = b"") -> bytes: command = old + b" " + new + b" " + ref + b"\x00report-status\n" - return f"{len(command) + 4:04x}".encode() + command + b"0000PACK" + framed = f"{len(command) + 4:04x}".encode() + command + b"0000" + return framed + (pack or _pack_of([])) -- 2.47.2 From b8f2163af0da2eda5cef68825244a8bad88a3109 Mon Sep 17 00:00:00 2001 From: jenkins Date: Mon, 17 Aug 2026 15:15:47 -0300 Subject: [PATCH 08/12] hermes: prove sudo and polkit denial closed Expand sudoers User_Alias chains so aliases, wildcards, netgroups, and undefined names cannot smuggle authority to the Hermes account, and require polkit grants to scope through exact literal identity comparisons: computed strings, bracket lookups, subject aliasing, operator-built values, and unconditional or wildcard grants fail closed. Co-Authored-By: Claude Fable 5 --- services/hermes/kustomization.yaml | 1 + services/hermes/scripts/node_account_audit.py | 88 +++++++- services/hermes/scripts/node_polkit_audit.py | 161 +++++++++++++ .../tests/test_hermes_node_polkit_identity.py | 211 ++++++++++++++++++ .../test_hermes_node_sudo_alias_audit.py | 134 +++++++++++ 5 files changed, 591 insertions(+), 4 deletions(-) create mode 100644 services/hermes/scripts/node_polkit_audit.py create mode 100644 testing/tests/test_hermes_node_polkit_identity.py create mode 100644 testing/tests/test_hermes_node_sudo_alias_audit.py diff --git a/services/hermes/kustomization.yaml b/services/hermes/kustomization.yaml index fe60bef9..fc39cc66 100644 --- a/services/hermes/kustomization.yaml +++ b/services/hermes/kustomization.yaml @@ -108,6 +108,7 @@ configMapGenerator: - node_account_hardening.py=scripts/node_account_hardening.py - node_account_audit.py=scripts/node_account_audit.py - node_account_io.py=scripts/node_account_io.py + - node_polkit_audit.py=scripts/node_polkit_audit.py options: disableNameSuffixHash: true - name: hermes-auto-router-plugin diff --git a/services/hermes/scripts/node_account_audit.py b/services/hermes/scripts/node_account_audit.py index b1795e99..31b54302 100644 --- a/services/hermes/scripts/node_account_audit.py +++ b/services/hermes/scripts/node_account_audit.py @@ -9,6 +9,7 @@ import struct from pathlib import Path from node_account_io import HardeningError, read_regular +from node_polkit_audit import audit_polkit_policy def _members(field: str, context: str) -> set[str]: @@ -108,6 +109,87 @@ def _mentions_dedicated_identity(text: str, account: str, account_uid: int) -> b return bool(re.search(account_pattern, text) or re.search(numeric_pattern, text)) +SUDO_ALIAS_NAME_RE = re.compile(r"[A-Z][A-Z0-9_]*\Z") + + +def _joined_sudo_lines(active: str) -> list[str]: + """Merge sudoers backslash continuations into whole logical lines.""" + lines: list[str] = [] + pending = "" + for line in active.splitlines(): + if line.endswith("\\"): + pending += line[:-1] + " " + continue + lines.append(pending + line) + pending = "" + if pending: + raise HardeningError("sudo policy ends inside a line continuation") + return lines + + +def _sudo_user_aliases(lines: list[str]) -> dict[str, list[str]]: + """Collect User_Alias definitions so grant principals can be expanded.""" + aliases: dict[str, list[str]] = {} + for line in lines: + stripped = line.strip() + if not stripped.startswith("User_Alias"): + continue + for definition in stripped.removeprefix("User_Alias").split(":"): + name, separator, members = definition.partition("=") + name = name.strip() + values = [item.strip() for item in members.split(",")] + if ( + not separator + or not SUDO_ALIAS_NAME_RE.fullmatch(name) + or name in aliases + or not all(values) + ): + raise HardeningError("sudo alias definition is not auditable") + aliases[name] = values + return aliases + + +def _expand_sudo_principal( + principal: str, aliases: dict[str, list[str]], seen: frozenset[str] +) -> None: + """Reject grant principals that reach Hermes through aliases or wildcards.""" + value = principal.strip() + while value.startswith("!"): + value = value[1:].strip() + value = value.strip('"') + if not value: + raise HardeningError("sudo grant principal is malformed") + if value in aliases: + if value in seen: + raise HardeningError("sudo alias expansion is cyclic") + for member in aliases[value]: + _expand_sudo_principal(member, aliases, seen | {value}) + return + if value in {"ALL", "%ALL"}: + raise HardeningError("broad sudo authority includes Hermes") + if value.startswith("+") or value.startswith("%:"): + raise HardeningError("sudo netgroup authority cannot be audited") + if SUDO_ALIAS_NAME_RE.fullmatch(value): + raise HardeningError("sudo alias is undefined") + + +def _audit_sudo_grants(active: str) -> None: + """Expand every grant's user list; only literal non-Hermes principals pass.""" + lines = _joined_sudo_lines(active) + aliases = _sudo_user_aliases(lines) + skip = ("Defaults", "User_Alias", "Runas_Alias", "Host_Alias", "Cmnd_Alias") + for line in lines: + stripped = line.strip() + if not stripped or stripped.startswith(skip): + continue + head, separator, _rest = stripped.partition("=") + fields = head.split() + if not separator or len(fields) < 2: + raise HardeningError("sudo grant line is not auditable") + for principal in " ".join(fields[:-1]).split(","): + _expand_sudo_principal(principal, aliases, frozenset()) + + def audit_privilege_policies( account: str, account_uid: int, @@ -160,10 +242,7 @@ def audit_privilege_policies( active = _active_sudo_policy(text) if _mentions_dedicated_identity(active, account, account_uid): raise HardeningError("dedicated Hermes account has sudo authority") - for line in active.splitlines(): - fields = line.split() - if fields and fields[0] in {"ALL", "%ALL"} and "=" in line: - raise HardeningError("broad sudo authority includes Hermes") + _audit_sudo_grants(active) continue active = "\n".join( line for line in text.splitlines() if not line.lstrip().startswith("#") @@ -176,3 +255,4 @@ def audit_privilege_policies( broad = "unix-user:*" in active or "Identity=unix-user:*" in active if broad and grants and any(item in active for item in dangerous_polkit): raise HardeningError("root-equivalent broad polkit authority includes Hermes") + audit_polkit_policy(path, text) diff --git a/services/hermes/scripts/node_polkit_audit.py b/services/hermes/scripts/node_polkit_audit.py new file mode 100644 index 00000000..281f1484 --- /dev/null +++ b/services/hermes/scripts/node_polkit_audit.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +"""Literal-identity proof for polkit policies on hardened Atlas nodes. + +Polkit rules are JavaScript, so a grant can name its subject indirectly: +concatenated strings, escape sequences, bracket lookups, or arithmetic can +compute an identity that plain text matching never sees. Rather than model +the language, every grant must scope itself through exact literal identity +comparisons. Anything the audit cannot prove literal fails closed. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +from node_account_io import HardeningError + +_ALLOWED_ESCAPES = {'"', "'", "\\", "/", "n", "r", "t"} +_STRING_MARK = r"\x00[0-9]+\x00" +_GRANT_RE = re.compile(r"\bResult\s*\.\s*YES\b") +_PKLA_GRANT_RE = re.compile(r"(?im)^\s*Result(?:Any|Inactive|Active)\s*=\s*yes\s*$") +_PKLA_IDENTITY_RE = re.compile(r"unix-(?:user|group):[A-Za-z0-9][A-Za-z0-9._-]{0,99}\Z") +_IDENTITY_USE_RE = re.compile(r"subject\s*(\[|\.\s*(?:user|uid|isInGroup)\b)") +_COMPUTED_TOKEN_RE = re.compile( + r"\b(?:eval|Function|arguments|fromCharCode|fromCodePoint|atob|btoa|unescape" + r"|escape|decodeURI[A-Za-z]*|encodeURI[A-Za-z]*|charCodeAt|codePointAt" + r"|normalize|constructor|prototype|globalThis|Reflect|Proxy)\b" +) +# Logical && and || stay legal; every value-building operator next to a +# string or number literal is rejected so comparisons stay whole literals. +_OPERATOR_ADJACENT_RE = re.compile( + rf"(?:{_STRING_MARK}|[0-9])\s*[+\-*/%<>^~](?![&|=])" + rf"|(?])[+\-*/%<>^~]\s*(?:{_STRING_MARK}|[0-9])" +) +_USER_BEFORE_RE = re.compile(rf"{_STRING_MARK}\s*[=!]==?\s*\Z") +_USER_AFTER_RE = re.compile(rf"\s*[=!]==?\s*{_STRING_MARK}") +_UID_BEFORE_RE = re.compile(r"(? tuple[str, list[str]]: + """Replace string literals with markers and drop JavaScript comments.""" + code: list[str] = [] + strings: list[str] = [] + index = 0 + length = len(text) + while index < length: + character = text[index] + if character == "`": + raise HardeningError("polkit rule uses computed template strings") + if character in {'"', "'"}: + quote = character + index += 1 + value: list[str] = [] + while True: + if index >= length or text[index] in "\r\n": + raise HardeningError("polkit rule string is unterminated") + character = text[index] + if character == "\\": + if index + 1 >= length or text[index + 1] not in _ALLOWED_ESCAPES: + raise HardeningError("polkit rule string uses computed escapes") + escaped = text[index + 1] + value.append({"n": "\n", "r": "\r", "t": "\t"}.get(escaped, escaped)) + index += 2 + continue + index += 1 + if character == quote: + break + value.append(character) + strings.append("".join(value)) + code.append(f"\x00{len(strings) - 1}\x00") + continue + if text.startswith("//", index): + newline = text.find("\n", index) + index = length if newline < 0 else newline + continue + if text.startswith("/*", index): + closing = text.find("*/", index + 2) + if closing < 0: + raise HardeningError("polkit rule comment is unterminated") + code.append(" ") + index = closing + 2 + continue + code.append(character) + index += 1 + return "".join(code), strings + + +def _require_literal_identity_use(code: str, match: re.Match[str]) -> None: + """Reject one identity reference unless it is an exact literal comparison.""" + form = match.group(1) + if form.startswith("["): + raise HardeningError("polkit rule reads identity through a computed lookup") + member = form.removeprefix(".").strip() + before = code[max(match.start() - 48, 0) : match.start()] + after = code[match.end() : match.end() + 48] + if member == "user": + literal = _USER_BEFORE_RE.search(before) or _USER_AFTER_RE.match(after) + elif member == "uid": + literal = _UID_BEFORE_RE.search(before) or _UID_AFTER_RE.match(after) + else: + literal = _GROUP_AFTER_RE.match(after) + if not literal: + raise HardeningError("polkit rule identity test is not an exact literal") + + +def _audit_rules_javascript(text: str) -> None: + """Require granting rules to gate on exact literal subject identities. + + With computed tokens, bracket lookups, and subject aliasing rejected in + every rule file, a YES decision can only be written as the literal grant + token, so the grant test below cannot be evaded by construction. + """ + code, _strings = _split_strings(text) + if _COMPUTED_TOKEN_RE.search(code): + raise HardeningError("polkit rule computes values the audit cannot prove") + if _RESULT_USE_RE.search(code): + raise HardeningError("polkit rule reads results through a computed lookup") + for bare in _BARE_SUBJECT_RE.finditer(code): + if not _PARAMETER_BEFORE_RE.search(code[: bare.start()]): + raise HardeningError("polkit rule passes the subject to computed code") + if not _GRANT_RE.search(code): + return + if _OPERATOR_ADJACENT_RE.search(code): + raise HardeningError("polkit grant builds values with operators") + uses = 0 + for match in _IDENTITY_USE_RE.finditer(code): + uses += 1 + _require_literal_identity_use(code, match) + if not uses: + raise HardeningError("polkit grant is not scoped to an explicit identity") + + +def _audit_pkla(text: str) -> None: + """Require granting localauthority entries to name exact literal identities.""" + if not _PKLA_GRANT_RE.search(text): + return + identities: list[str] = [] + for line in text.splitlines(): + name, separator, value = line.strip().partition("=") + if name.strip() != "Identity" or not separator: + continue + entries = [entry.strip() for entry in value.split(";") if entry.strip()] + identities.extend(entries) + if not identities: + raise HardeningError("polkit authority grant lacks an identity scope") + for entry in identities: + if not _PKLA_IDENTITY_RE.fullmatch(entry): + raise HardeningError("polkit authority identity is not an exact literal") + + +def audit_polkit_policy(path: Path, text: str) -> None: + """Prove one polkit policy file cannot grant through a computed identity.""" + if "localauthority" in path.parts or path.suffix == ".pkla": + _audit_pkla(text) + else: + _audit_rules_javascript(text) diff --git a/testing/tests/test_hermes_node_polkit_identity.py b/testing/tests/test_hermes_node_polkit_identity.py new file mode 100644 index 00000000..059f05f2 --- /dev/null +++ b/testing/tests/test_hermes_node_polkit_identity.py @@ -0,0 +1,211 @@ +"""Literal-identity denial proofs for polkit grants near the Hermes account.""" + +from __future__ import annotations + +import importlib.util +import os +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).parents[2] +SCRIPTS = ROOT / "services/hermes/scripts" +sys.path.insert(0, str(SCRIPTS)) + + +def _load(): + spec = importlib.util.spec_from_file_location( + "node_polkit_audit_test", SCRIPTS / "node_polkit_audit.py" + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _rules(module, text: str): + module.audit_polkit_policy(Path("rules.d/50-test.rules"), text) + + +def test_literal_distribution_rule_passes(): + module = _load() + _rules( + module, + "// allow the network daemon\n" + "polkit.addRule(function(action, subject) {\n" + ' if (action.id == "org.freedesktop.network1.set-dns" &&\n' + ' subject.user == "systemd-network") {\n' + " return polkit.Result.YES;\n" + " }\n" + "});\n", + ) + _rules( + module, + "polkit.addRule(function(action, subject) {\n" + ' if (subject.isInGroup("wheel") && subject.uid === 4242) {\n' + " return polkit.Result.YES;\n" + " }\n" + "});\n", + ) + _rules( + module, + "/* deny-only rules may use constructs the audit cannot prove */\n" + "polkit.addRule(function(action, subject) {\n" + " if (subject.user.indexOf(\"x\") == 0) { return polkit.Result.NO; }\n" + "});\n", + ) + + +def test_reversed_literal_comparisons_pass(): + module = _load() + _rules( + module, + "polkit.addRule(function(action, subject) {\n" + ' if ("atlas" == subject.user || 4242 === subject.uid) {\n' + " return polkit.Result.YES;\n" + " }\n" + "});\n", + ) + + +def test_unconditional_grant_fails_closed(): + module = _load() + with pytest.raises(module.HardeningError, match="explicit identity"): + _rules(module, "polkit.addRule(function() { return polkit.Result.YES; });\n") + + +@pytest.mark.parametrize( + ("snippet", "match"), + [ + ('subject.user == "herme" + "s-agent"', "operators"), + ("subject.uid == 1199 + 1", "operators"), + ("subject.uid == 0x4B0", "exact literal"), + ("subject.uid >= 1000", "exact literal"), + ("subject.uid == -1", "operators"), + ("subject.user == name", "exact literal"), + ("subject.user.indexOf(\"h\") == 0", "exact literal"), + ("subject.isInGroup(group)", "exact literal"), + ('subject["user"] == "atlas"', "computed lookup"), + ], +) +def test_computed_identity_grants_fail_closed(snippet: str, match: str): + module = _load() + text = ( + "polkit.addRule(function(action, subject) {\n" + f" if ({snippet}) {{ return polkit.Result.YES; }}\n" + "});\n" + ) + with pytest.raises(module.HardeningError, match=match): + _rules(module, text) + + +@pytest.mark.parametrize( + ("text", "match"), + [ + ("var x = `hermes`;\n", "template strings"), + ('var x = "unterminated\n', "unterminated"), + ('var x = "bad\\x2descape";\n', "computed escapes"), + ('var x = "dangling\\', "computed escapes"), + ("/* never closed\n", "comment is unterminated"), + ("var f = eval;\n", "cannot prove"), + ('var s = String.fromCharCode(104);\n', "cannot prove"), + ("var a = arguments;\n", "cannot prove"), + ('var r = polkit.Result["YES"];\n', "computed lookup"), + ("var r = polkit.Result;\n", "computed lookup"), + ("var s = subject;\n", "computed code"), + ("check(subject);\n", "computed code"), + ], +) +def test_unprovable_rule_constructs_fail_closed(text: str, match: str): + module = _load() + with pytest.raises(module.HardeningError, match=match): + _rules(module, text) + + +def test_string_and_comment_handling_keeps_literal_rules_auditable(): + module = _load() + code, strings = module._split_strings( + 'var a = "with // not a comment"; // trailing\n' + "/* block */ var b = 'quo\\'te\\n';\n" + ) + assert strings == ["with // not a comment", "quo'te\n"] + assert "//" not in code and "block" not in code + assert "\x000\x00" in code and "\x001\x00" in code + + +def test_grant_detection_survives_spacing_and_strings(): + module = _load() + with pytest.raises(module.HardeningError, match="explicit identity"): + _rules(module, "polkit.addRule(function() { return polkit.Result . YES; });\n") + # A grant token inside a string is inert and needs no identity scope. + _rules(module, 'var label = "polkit.Result.YES";\n') + + +@pytest.mark.parametrize( + ("text", "match"), + [ + ( + "[All]\nIdentity=unix-user:*\nAction=org.example\nResultAny=yes\n", + "exact literal", + ), + ( + "[N]\nIdentity=unix-netgroup:ops\nAction=org.example\nResultActive=yes\n", + "exact literal", + ), + ("[N]\nAction=org.example\nResultInactive=yes\n", "identity scope"), + ("[N]\nIdentity=\nAction=org.example\nResultAny=yes\n", "identity scope"), + ], +) +def test_pkla_grants_require_exact_literal_identities(text: str, match: str): + module = _load() + with pytest.raises(module.HardeningError, match=match): + module.audit_polkit_policy(Path("localauthority/50-local.d/a.pkla"), text) + + +def test_pkla_literal_grants_and_deny_entries_pass(): + module = _load() + module.audit_polkit_policy( + Path("localauthority/50-local.d/a.pkla"), + "[Mount]\nIdentity=unix-user:atlas;unix-group:operators\n" + "Action=org.freedesktop.udisks2.filesystem-mount\nResultAny=yes\n", + ) + module.audit_polkit_policy( + Path("etc/b.pkla"), + "[Deny]\nIdentity=unix-user:*\nAction=org.example\nResultAny=no\n", + ) + + +def test_integration_rejects_previously_invisible_grants(tmp_path: Path): + spec = importlib.util.spec_from_file_location( + "node_account_audit_polkit_test", SCRIPTS / "node_account_audit.py" + ) + assert spec and spec.loader + audit = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = audit + spec.loader.exec_module(audit) + + host_etc = tmp_path / "etc" + rules = host_etc / "polkit-1/rules.d" + rules.mkdir(parents=True) + (rules / "90-open.rules").write_text( + "polkit.addRule(function() { return polkit.Result.YES; });\n", + encoding="utf-8", + ) + with pytest.raises(audit.HardeningError, match="explicit identity"): + audit.audit_privilege_policies( + "hermes-agent", 1200, os.getuid(), host_etc, tmp_path / "missing" + ) + + (rules / "90-open.rules").unlink() + local = host_etc / "polkit-1/localauthority/50-local.d" + local.mkdir(parents=True) + (local / "wide.pkla").write_text( + "[W]\nIdentity=unix-user:*\nAction=org.example.benign\nResultAny=yes\n", + encoding="utf-8", + ) + with pytest.raises(audit.HardeningError, match="exact literal"): + audit.audit_privilege_policies( + "hermes-agent", 1200, os.getuid(), host_etc, tmp_path / "missing" + ) \ No newline at end of file diff --git a/testing/tests/test_hermes_node_sudo_alias_audit.py b/testing/tests/test_hermes_node_sudo_alias_audit.py new file mode 100644 index 00000000..e62b82bf --- /dev/null +++ b/testing/tests/test_hermes_node_sudo_alias_audit.py @@ -0,0 +1,134 @@ +"""Sudo alias and computed-identity denial proofs for the Hermes account.""" + +from __future__ import annotations + +import importlib.util +import os +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).parents[2] +SCRIPTS = ROOT / "services/hermes/scripts" +sys.path.insert(0, str(SCRIPTS)) + + +def _load(): + spec = importlib.util.spec_from_file_location( + "node_account_audit_sudo_test", SCRIPTS / "node_account_audit.py" + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _audit(tmp_path: Path, sudoers: str): + module = _load() + host_etc = tmp_path / "etc" + (host_etc / "sudoers.d").mkdir(parents=True, exist_ok=True) + (host_etc / "sudoers.d/atlas").write_text(sudoers, encoding="utf-8") + module.audit_privilege_policies( + "hermes-agent", 1200, os.getuid(), host_etc, tmp_path / "missing-share" + ) + + +def test_benign_distribution_policy_passes(tmp_path: Path): + _audit( + tmp_path, + "Defaults secure_path=/usr/bin\n" + "root ALL=(ALL:ALL) ALL\n" + "%sudo ALL=(ALL:ALL) ALL\n" + "#999 ALL=(root) /usr/bin/uptime\n" + "alice, bob ALL=(root) NOPASSWD: /usr/bin/systemctl status\n", + ) + + +def test_alias_reaching_hermes_directly_is_denied(tmp_path: Path): + module = _load() + with pytest.raises(module.HardeningError, match="sudo authority"): + _audit( + tmp_path, + "User_Alias OPERATORS = atlas, hermes-agent\n" + "OPERATORS ALL=(ALL) ALL\n", + ) + + +def test_alias_reaching_hermes_by_numeric_uid_is_denied(tmp_path: Path): + module = _load() + with pytest.raises(module.HardeningError, match="sudo authority"): + _audit(tmp_path, "User_Alias OPERATORS = #1200\nOPERATORS ALL=(ALL) ALL\n") + + +def test_alias_expanding_to_all_is_denied(tmp_path: Path): + module = _load() + with pytest.raises(module.HardeningError, match="broad sudo authority"): + _audit(tmp_path, "User_Alias ADMINS = ALL\nADMINS ALL=(ALL) ALL\n") + + +def test_nested_alias_expanding_to_all_via_continuation_is_denied(tmp_path: Path): + module = _load() + with pytest.raises(module.HardeningError, match="broad sudo authority"): + _audit( + tmp_path, + "User_Alias INNER = atlas, \\\n ALL\n" + "User_Alias OUTER = INNER\n" + "OUTER ALL=(ALL) ALL\n", + ) + + +@pytest.mark.parametrize( + ("sudoers", "match"), + [ + ("+operators ALL=(ALL) ALL\n", "netgroup"), + ("%:S-1-5-32 ALL=(ALL) ALL\n", "netgroup"), + ("GHOSTS ALL=(ALL) ALL\n", "alias is undefined"), + ("User_Alias A = B\nUser_Alias B = A\nA ALL=(ALL) ALL\n", "cyclic"), + ("User_Alias broken\nroot ALL=(ALL) ALL\n", "not auditable"), + ("User_Alias lower = atlas\n", "not auditable"), + ("User_Alias A = atlas\nUser_Alias A = bob\nA ALL=(ALL) ALL\n", "not auditable"), + ("User_Alias A = atlas,,bob\nA ALL=(ALL) ALL\n", "not auditable"), + ("stray-line-without-equals\n", "not auditable"), + ("=orphan (ALL) ALL\n", "not auditable"), + ("! ALL=(ALL) ALL\n", "malformed"), + ("User_Alias A = atlas : B = ALL\nB ALL=(ALL) ALL\n", "broad sudo"), + ], +) +def test_unauditable_or_broad_sudo_policies_fail_closed( + tmp_path: Path, sudoers: str, match: str +): + module = _load() + with pytest.raises(module.HardeningError, match=match): + _audit(tmp_path, sudoers) + + +def test_negated_and_quoted_principals_are_expanded_before_judging(tmp_path: Path): + module = _load() + with pytest.raises(module.HardeningError, match="broad sudo authority"): + _audit(tmp_path, '!"ALL" ALL=(ALL) ALL\n') + _audit(tmp_path, "!alice ALL=(ALL) ALL\n") + + +def test_continuation_join_requires_a_complete_final_line(): + module = _load() + with pytest.raises(module.HardeningError, match="line continuation"): + module._joined_sudo_lines("root ALL=(ALL) ALL \\") + assert module._joined_sudo_lines("a \\\nb\nc") == ["a b", "c"] + + +def test_alias_definitions_parse_multiple_groups_per_line(): + module = _load() + aliases = module._sudo_user_aliases( + ["User_Alias A = atlas, bob : B = carol", "Runas_Alias R = root"] + ) + assert aliases == {"A": ["atlas", "bob"], "B": ["carol"]} + + +def test_expansion_accepts_nested_literal_users(): + module = _load() + aliases = {"A": ["atlas", "B"], "B": ["carol"]} + module._expand_sudo_principal("A", aliases, frozenset()) + module._expand_sudo_principal("%wheel", aliases, frozenset()) + module._expand_sudo_principal("#999", aliases, frozenset()) \ No newline at end of file -- 2.47.2 From 5020e49057c911469c04a002b97b1329d8b8266a Mon Sep 17 00:00:00 2001 From: jenkins Date: Mon, 17 Aug 2026 15:15:47 -0300 Subject: [PATCH 09/12] hermes: track new boundary modules in gate Co-Authored-By: Claude Fable 5 --- testing/quality_contract.json | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/testing/quality_contract.json b/testing/quality_contract.json index 805857d9..78fa2f6a 100644 --- a/testing/quality_contract.json +++ b/testing/quality_contract.json @@ -40,11 +40,15 @@ "testing/quality_gate.py", "ci/tests/glue/test_ariadne_schedules.py", "ci/tests/glue/test_glue_metrics.py", - "testing/tests/test_publish_test_metrics.py", - "testing/tests/test_supply_chain_report.py", - "testing/tests/test_semgrep_report.py", - "testing/tests/test_quality_contract.py", - "testing/tests/test_quality_gate.py" + "testing/tests/test_publish_test_metrics.py", + "testing/tests/test_supply_chain_report.py", + "testing/tests/test_semgrep_report.py", + "testing/tests/test_quality_contract.py", + "testing/tests/test_quality_gate.py", + "services/hermes/scm-common/scripts/deadline_http.py", + "services/hermes/scm-common/scripts/git_pack_objects.py", + "services/hermes/scm-common/scripts/receive_pack_scan.py", + "services/hermes/scripts/node_polkit_audit.py" ], "lint_paths": [ "ci/scripts/publish_test_metrics.py", @@ -63,7 +67,8 @@ "services/hermes/scripts/node_account_io.py", "services/hermes/scripts/stage_runtime_access.py", "testing/tests", - "testing" + "testing", + "services/hermes/scripts/node_polkit_audit.py" ], "pytest_suites": { "unit": { @@ -154,7 +159,8 @@ "services/hermes/scripts/node_account_io.py", "services/hermes/scripts/stage_runtime_access.py", "services/mailu/scripts/mailu_sync.py", - "services/mailu/scripts/mailu_sync_listener.py" + "services/mailu/scripts/mailu_sync_listener.py", + "services/hermes/scripts/node_polkit_audit.py" ], "naming_rules": [ { @@ -210,7 +216,11 @@ "services/hermes/scripts/node_account_hardening.py", "services/hermes/scripts/node_account_io.py", "services/hermes/scripts/stage_runtime_access.py", - "testing/quality_coverage.py" + "testing/quality_coverage.py", + "services/hermes/scm-common/scripts/deadline_http.py", + "services/hermes/scm-common/scripts/git_pack_objects.py", + "services/hermes/scm-common/scripts/receive_pack_scan.py", + "services/hermes/scripts/node_polkit_audit.py" ], "tracked_files": [ "ci/scripts/publish_test_metrics.py", @@ -235,7 +245,11 @@ "testing/quality_docs.py", "testing/quality_hygiene.py", "testing/quality_coverage.py", - "testing/quality_gate.py" + "testing/quality_gate.py", + "services/hermes/scm-common/scripts/deadline_http.py", + "services/hermes/scm-common/scripts/git_pack_objects.py", + "services/hermes/scm-common/scripts/receive_pack_scan.py", + "services/hermes/scripts/node_polkit_audit.py" ] } } -- 2.47.2 From 6e9e8c6ab5e5cc560e45a606c47ab83b8c261d64 Mon Sep 17 00:00:00 2001 From: jenkins Date: Mon, 17 Aug 2026 20:37:30 -0300 Subject: [PATCH 10/12] hermes: reject regex literals in polkit grants A regex literal embedding a quote desynced the string tokenizer and swallowed an unconditional Result.YES, so the grant read as unscoped and slipped past the literal-identity check. Fail closed on any slash outside a string or comment; neither regex nor division belongs in an identity-scoped grant. Co-Authored-By: Claude Fable 5 --- services/hermes/scripts/node_polkit_audit.py | 7 +++ .../tests/test_hermes_node_polkit_identity.py | 56 +++++++++++++++++++ 2 files changed, 63 insertions(+) diff --git a/services/hermes/scripts/node_polkit_audit.py b/services/hermes/scripts/node_polkit_audit.py index 281f1484..d45a0134 100644 --- a/services/hermes/scripts/node_polkit_audit.py +++ b/services/hermes/scripts/node_polkit_audit.py @@ -85,6 +85,13 @@ def _split_strings(text: str) -> tuple[str, list[str]]: code.append(" ") index = closing + 2 continue + if character == "/": + # Any remaining slash is a regex literal or a division operator. + # A regex literal may embed a quote character, which would desync + # the string scanner and swallow an unconditional grant. Neither + # regex nor division has a place in an identity-scoped grant, so + # both fail closed before they can hide a Result.YES. + raise HardeningError("polkit rule uses a regex or division operator") code.append(character) index += 1 return "".join(code), strings diff --git a/testing/tests/test_hermes_node_polkit_identity.py b/testing/tests/test_hermes_node_polkit_identity.py index 059f05f2..3af8d095 100644 --- a/testing/tests/test_hermes_node_polkit_identity.py +++ b/testing/tests/test_hermes_node_polkit_identity.py @@ -124,6 +124,62 @@ def test_unprovable_rule_constructs_fail_closed(text: str, match: str): _rules(module, text) +def test_regex_literal_tokenizer_desync_grant_is_rejected(): + """A regex literal embedding a quote must not open a phantom string. + + Without regex awareness the two /'/ literals fool the string scanner into + swallowing the unconditional Result.YES between their quotes; the grant + then reads as unscoped and would slip past every later check. + """ + module = _load() + exploit = ( + "polkit.addRule(function(action, subject) { " + "var a = /'/; " + 'if (action.id == "org.freedesktop.policykit.exec") ' + "{ return polkit.Result.YES; } " + "var b = /'/; });\n" + ) + with pytest.raises(module.HardeningError, match="regex or division"): + _rules(module, exploit) + + +@pytest.mark.parametrize( + "snippet", + [ + "var a = /'/;", + 'var a = /"/;', + "var a = /abc/;", + "var a = /[/'\"]/;", + "var a = /x\\/y/;", + "var a = subject.user / 2;", + "// a comment / with a slash then\nvar a = /'/;", + '/* block */ var a = /"/;', + "return polkit.Result.YES / 1;", + ], +) +def test_every_slash_form_outside_a_comment_fails_closed(snippet: str): + module = _load() + text = f"polkit.addRule(function(action, subject) {{ {snippet} }});\n" + with pytest.raises(module.HardeningError, match="regex or division"): + _rules(module, text) + + +def test_slashes_confined_to_strings_and_comments_stay_legal(): + module = _load() + # Slashes inside string literals and // or /* */ comments are inert. + _rules( + module, + "// path-like /usr/bin comment\n" + "/* another /etc/ comment */\n" + "polkit.addRule(function(action, subject) {\n" + ' if (action.id == "org.freedesktop.systemd1.manage-units" &&\n' + ' subject.user == "systemd-network") {\n' + " return polkit.Result.YES;\n" + " }\n" + "});\n", + ) + + def test_string_and_comment_handling_keeps_literal_rules_auditable(): module = _load() code, strings = module._split_strings( -- 2.47.2 From 6c1123201e7ce642a5fc1c89d014a97a9bd3f0ef Mon Sep 17 00:00:00 2001 From: jenkins Date: Mon, 17 Aug 2026 20:37:30 -0300 Subject: [PATCH 11/12] hermes: audit every sudo include spelling A quoted, space-bearing #include path failed the include regex and fell through to the comment branch, leaving a second authority file unenumerated. Detect any include directive before the comment rule and fail closed on every form except the exact bare includedir into the audited /etc/sudoers.d. Co-Authored-By: Claude Fable 5 --- services/hermes/scripts/node_account_audit.py | 23 +++++++--- .../tests/test_hermes_node_audit_coverage.py | 44 +++++++++++++++++++ 2 files changed, 60 insertions(+), 7 deletions(-) diff --git a/services/hermes/scripts/node_account_audit.py b/services/hermes/scripts/node_account_audit.py index 31b54302..fa6245ba 100644 --- a/services/hermes/scripts/node_account_audit.py +++ b/services/hermes/scripts/node_account_audit.py @@ -89,13 +89,22 @@ def _active_sudo_policy(text: str) -> str: stripped = line.lstrip() if not stripped: continue - include = re.fullmatch( - r"(?i)([#@]include|[#@]includedir)\s+([^\s]+)", stripped - ) - if include: - directive, target = include.groups() - if directive.lower().endswith("includedir") and target == "/etc/sudoers.d": - continue + # Detect any include spelling before the comment rule below. sudo + # honours both #include and @include; a quoted or space-bearing path + # must never fall through the "#" comment branch and leave a second + # authority file unaudited. Only the exact bare includedir into the + # audited /etc/sudoers.d passes; everything else fails closed. + if re.match(r"(?i)[#@]include(?:dir)?(?=\s|$)", stripped): + include = re.fullmatch( + r"(?i)([#@]include(?:dir)?)\s+([^\s\"'\\]+)", stripped + ) + if include: + directive, target = include.groups() + if ( + directive.lower().endswith("includedir") + and target == "/etc/sudoers.d" + ): + continue raise HardeningError("sudo policy includes an unaudited authority source") if stripped.startswith("#") and not re.match(r"#\d+(?:\s|$)", stripped): continue diff --git a/testing/tests/test_hermes_node_audit_coverage.py b/testing/tests/test_hermes_node_audit_coverage.py index 8c011b5b..20114ada 100644 --- a/testing/tests/test_hermes_node_audit_coverage.py +++ b/testing/tests/test_hermes_node_audit_coverage.py @@ -105,6 +105,50 @@ def test_active_sudo_policy_handles_blank_comments_numeric_and_includes(): module._active_sudo_policy("@include /external/policy\n") +@pytest.mark.parametrize( + "directive", + [ + '#include "/etc/evil path"', + "#include '/etc/evil path'", + "#include /etc/evil\\ path", + "#include\t/etc/evil", + "#include /etc/evil", + "#includedir /opt/other", + '#includedir "/etc/sudoers.d"', + "@include /external/policy", + '@include "/etc/evil path"', + "@includedir /opt/other", + "#include", + "@includedir", + ], +) +def test_every_include_spelling_outside_the_standard_dir_fails_closed(directive: str): + """No include form may reach the comment branch and stay unaudited.""" + module = _audit_module() + with pytest.raises(module.HardeningError, match="unaudited"): + module._active_sudo_policy(directive + "\n") + + +@pytest.mark.parametrize( + "directive", + ["#includedir /etc/sudoers.d", "@includedir /etc/sudoers.d"], +) +def test_standard_includedir_is_the_only_allowed_include(directive: str): + module = _audit_module() + active = module._active_sudo_policy(directive + "\nroot ALL=(ALL) ALL\n") + assert active == "root ALL=(ALL) ALL" + + +@pytest.mark.parametrize( + "line", + ["#included by an operator note", "#include-guard notes", "#includes history"], +) +def test_comment_lines_that_only_resemble_includes_stay_comments(line: str): + module = _audit_module() + active = module._active_sudo_policy(line + "\nroot ALL=(ALL) ALL\n") + assert active == "root ALL=(ALL) ALL" + + def test_identity_matcher_covers_account_numeric_and_unrelated_values(): module = _audit_module() assert module._mentions_dedicated_identity("hermes-agent ALL", "hermes-agent", 1200) -- 2.47.2 From 31439f096afb0acba710d4d0ebb4b52a84a3b370 Mon Sep 17 00:00:00 2001 From: jenkins Date: Mon, 17 Aug 2026 20:37:30 -0300 Subject: [PATCH 12/12] hermes: bound streaming git DNS resolution The watchdog can only close a live socket, but getaddrinfo runs before any socket exists and ignores socket timeouts, so a slow resolver outlived the stream deadline. Resolve under the same absolute deadline in a joinable worker before the real connect. Co-Authored-By: Claude Fable 5 --- .../scm-common/scripts/deadline_http.py | 46 +++++++++- testing/tests/test_hermes_deadline_stream.py | 87 +++++++++++++++++++ 2 files changed, 131 insertions(+), 2 deletions(-) diff --git a/services/hermes/scm-common/scripts/deadline_http.py b/services/hermes/scm-common/scripts/deadline_http.py index a19abccc..e513a446 100644 --- a/services/hermes/scm-common/scripts/deadline_http.py +++ b/services/hermes/scm-common/scripts/deadline_http.py @@ -227,6 +227,31 @@ def open_bounded( ) +def _resolve_within(host: str, port: int, timeout: float) -> None: + """Resolve one host under a hard timeout the socket layer never covers. + + ``getaddrinfo`` ignores socket timeouts and runs before any socket + exists, so a slow or hostile resolver would otherwise outlive the whole + stream deadline. Running it in a joinable worker bounds resolution by the + same wall clock; a stuck lookup leaks only one daemon thread. + """ + outcome: dict[str, object] = {} + + def run() -> None: + try: + outcome["value"] = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM) + except OSError as exc: + outcome["error"] = exc + + worker = threading.Thread(target=run, daemon=True) + worker.start() + worker.join(timeout) + if worker.is_alive(): + raise PolicyError("HTTP stream DNS resolution deadline exceeded") + if "error" in outcome: + raise outcome["error"] # type: ignore[misc] + + class StreamDeadline: """Force-close tracked connections once an absolute deadline passes.""" @@ -270,6 +295,23 @@ class StreamDeadline: raise PolicyError("HTTP stream deadline exceeded") return connection + def _bounded_connection(self, connection: http.client.HTTPConnection): + """Register one connection and bound its DNS phase by the deadline. + + The watchdog can only close a live socket, but ``connect`` resolves + the host before any socket exists. Wrapping ``connect`` runs that + resolution under the same absolute deadline first, then hands off to + the real connect for the socket phases the watchdog already covers. + """ + original_connect = connection.connect + + def connect() -> None: + _resolve_within(connection.host, connection.port, self.remaining()) + original_connect() + + connection.connect = connect # type: ignore[method-assign] + return self._track(connection) + def cancel(self) -> None: self._timer.cancel() @@ -280,7 +322,7 @@ class StreamDeadline: class GuardedHTTPHandler(urllib.request.HTTPHandler): def http_open(self, req): return self.do_open( - lambda host, **kwargs: deadline._track( + lambda host, **kwargs: deadline._bounded_connection( http.client.HTTPConnection(host, **kwargs) ), req, @@ -289,7 +331,7 @@ class StreamDeadline: class GuardedHTTPSHandler(urllib.request.HTTPSHandler): def https_open(self, req): return self.do_open( - lambda host, **kwargs: deadline._track( + lambda host, **kwargs: deadline._bounded_connection( http.client.HTTPSConnection(host, **kwargs) ), req, diff --git a/testing/tests/test_hermes_deadline_stream.py b/testing/tests/test_hermes_deadline_stream.py index d4f97efe..88022591 100644 --- a/testing/tests/test_hermes_deadline_stream.py +++ b/testing/tests/test_hermes_deadline_stream.py @@ -107,6 +107,93 @@ def test_handlers_build_guarded_connections_without_network(monkeypatch): guard.cancel() +def test_dns_resolution_is_bounded_by_the_stream_deadline(monkeypatch): + module = _load("deadline_http") + + def hang(*_args, **_kwargs): + time.sleep(30) + + monkeypatch.setattr(module.socket, "getaddrinfo", hang) + started = time.monotonic() + with pytest.raises(module.PolicyError, match="DNS resolution deadline"): + module._resolve_within("example.invalid", 443, 0.4) + assert time.monotonic() - started < 3 + + +def test_resolve_within_returns_on_success_and_propagates_errors(monkeypatch): + module = _load("deadline_http") + calls = [] + monkeypatch.setattr( + module.socket, "getaddrinfo", lambda *a, **k: calls.append((a, k)) + ) + module._resolve_within("127.0.0.1", 80, 5) + assert calls + + def boom(*_a, **_k): + raise OSError("name resolution failed") + + monkeypatch.setattr(module.socket, "getaddrinfo", boom) + with pytest.raises(OSError, match="name resolution failed"): + module._resolve_within("127.0.0.1", 80, 5) + + +def test_bounded_connection_wraps_connect_to_resolve_first(monkeypatch): + module = _load("deadline_http") + guard = module.StreamDeadline(30, timer=_FakeTimer) + order = [] + monkeypatch.setattr( + module, + "_resolve_within", + lambda host, port, timeout: order.append(("resolve", host, port)), + ) + + class FakeConnection: + host = "scm.bstein.dev" + port = 443 + + def __init__(self): + self.sock = None + + def connect(self): + order.append(("connect", self.host)) + + def close(self): + order.append(("close",)) + + connection = guard._bounded_connection(FakeConnection()) + assert connection in guard._connections + connection.connect() + assert order == [("resolve", "scm.bstein.dev", 443), ("connect", "scm.bstein.dev")] + guard.cancel() + + +def test_bounded_connection_connect_fails_closed_once_expired(monkeypatch): + module = _load("deadline_http") + guard = module.StreamDeadline(30, timer=_FakeTimer) + resolved = [] + monkeypatch.setattr( + module, "_resolve_within", lambda *a, **k: resolved.append(a) + ) + + class FakeConnection: + host = "scm.bstein.dev" + port = 443 + sock = None + + def connect(self): + resolved.append("connected") + + def close(self): + pass + + connection = guard._bounded_connection(FakeConnection()) + guard._expire() + with pytest.raises(module.PolicyError, match="deadline exceeded"): + connection.connect() + assert resolved == [] + guard.cancel() + + class _TrickleHandler(BaseHTTPRequestHandler): def do_GET(self): time.sleep(10) -- 2.47.2