From 762784da6b558eedce5ec3e3993fdf42e79a97ec Mon Sep 17 00:00:00 2001 From: jenkins Date: Sun, 16 Aug 2026 19:54:41 -0300 Subject: [PATCH 1/4] hermes: bound Atlas pull request client --- services/hermes/agent-configmap.yaml | 30 +- services/hermes/agent-deployment.yaml | 9 + services/hermes/kustomization.yaml | 7 + services/hermes/scripts/gitea_api.py | 422 ++++++++++++++---- .../manage-atlas-pull-requests/SKILL.md | 58 +++ .../agents/openai.yaml | 4 + testing/tests/test_hermes_chat_quality.py | 8 +- testing/tests/test_hermes_gitea_pr_client.py | 322 +++++++++++++ testing/tests/test_hermes_runtime_access.py | 40 +- 9 files changed, 784 insertions(+), 116 deletions(-) mode change 100644 => 100755 services/hermes/scripts/gitea_api.py create mode 100644 services/hermes/skills/manage-atlas-pull-requests/SKILL.md create mode 100644 services/hermes/skills/manage-atlas-pull-requests/agents/openai.yaml create mode 100644 testing/tests/test_hermes_gitea_pr_client.py diff --git a/services/hermes/agent-configmap.yaml b/services/hermes/agent-configmap.yaml index 0b81379d..b35da59a 100644 --- a/services/hermes/agent-configmap.yaml +++ b/services/hermes/agent-configmap.yaml @@ -315,15 +315,21 @@ data: real branch only when the requested implementation is review-ready. Never force-push. - `scm.bstein.dev` is Gitea, not GitHub. Never load or follow a GitHub/`gh` - skill for an Atlas remote, and do not interpret an unauthenticated Gitea - HTTP 404 as a missing private repository. Use authenticated Git for clone, - fetch, and push. For pull-request metadata, comments, and merges, call - `/opt/coordinator/gitea_api.py METHOD /api/v1/...`; it injects the runtime - Vault token without exposing it to the command line, environment, output, - or transcript. Before a consequential merge, independently verify the - exact base/head ancestry and diff, run the relevant tests, and confirm the - candidate Jenkins result. Prefer the internal endpoint in + `scm.bstein.dev` is Forgejo/Gitea, not GitHub. Never load or follow a + GitHub/`gh` skill for an Atlas remote, and do not interpret an + unauthenticated Gitea HTTP 404 as a missing private repository. Use + authenticated Git for clone, fetch, and non-force push. For repository and + pull-request reads, or to create/update a review-ready draft PR, load + `$manage-atlas-pull-requests` and use `/opt/coordinator/gitea_api.py`. The + client injects the runtime Vault token without exposing it to the command + line, environment, output, or transcript. It permits only Atlas-scoped + reads, same-repository draft creation, and title/body updates while the PR + remains draft. Merge, approve, close, delete, comments, repository + administration, and force-push are unavailable; never bypass the client + with raw HTTP. Leave every PR unmerged for Brad's review. Independently + verify the exact base/head ancestry and diff, run the relevant tests, and + confirm the candidate Jenkins result before presenting a handoff. Prefer + the internal endpoint in `JENKINS_BASE_URL`; when Jenkins API authorization prevents a read, use the existing cluster access to inspect the controller's job/build files and logs rather than guessing a public hostname. The preferred read-only path @@ -331,10 +337,8 @@ data: [--commit SHA] --wait`; it distinguishes a genuinely terminal build from nested Jenkins execution metadata and returns bounded JSON/log evidence. With a branch, the helper also tries the conventional `JOB-branches` - multibranch name. For JSON Gitea mutations, prefer repeatable shell-safe - `--field KEY=VALUE` arguments over hand-quoted JSON. - After merging, observe the default-branch build to a terminal result and - report exact commit, build, and test evidence. + multibranch name. After Brad merges, observe the default-branch build to a + terminal result and report exact commit, build, and test evidence. The workspace already has a non-secret Hermes Git author identity. Do not use `git reset --hard`, even in a fresh clone; use a detached worktree or a diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml index 5ed758cf..4b93f92d 100644 --- a/services/hermes/agent-deployment.yaml +++ b/services/hermes/agent-deployment.yaml @@ -663,6 +663,7 @@ spec: - {name: subprocess-secret-patch, mountPath: /opt/hermes/tools/process_registry.py, subPath: process_registry.py} - {name: kubeconfig, mountPath: /opt/data/home/.kube/config, subPath: config, readOnly: true} - {name: auto-router-plugin, mountPath: /opt/data/plugins/auto-router, readOnly: true} + - {name: atlas-pr-skill, mountPath: /opt/data/workspace/skills/manage-atlas-pull-requests, readOnly: true} - {name: routing-catalog, mountPath: /routing-catalog, readOnly: true} - {name: tmp, mountPath: /tmp} startupProbe: @@ -826,6 +827,7 @@ spec: - {name: subprocess-secret-patch, mountPath: /opt/hermes/tools/process_registry.py, subPath: process_registry.py} - {name: kubeconfig, mountPath: /opt/data/home/.kube/config, subPath: config, readOnly: true} - {name: auto-router-plugin, mountPath: /opt/data/plugins/auto-router, readOnly: true} + - {name: atlas-pr-skill, mountPath: /opt/data/workspace/skills/manage-atlas-pull-requests, readOnly: true} - {name: routing-catalog, mountPath: /routing-catalog, readOnly: true} - {name: tmp, mountPath: /tmp} - {name: ttyd-index, mountPath: /ttyd-index, readOnly: true} @@ -894,6 +896,7 @@ spec: - {name: auth-patch, mountPath: /opt/hermes/hermes_cli/auth.py, subPath: auth.py} - {name: kubeconfig, mountPath: /opt/data/home/.kube/config, subPath: config, readOnly: true} - {name: auto-router-plugin, mountPath: /opt/data/plugins/auto-router, readOnly: true} + - {name: atlas-pr-skill, mountPath: /opt/data/workspace/skills/manage-atlas-pull-requests, readOnly: true} - {name: tmp, mountPath: /tmp} resources: requests: {cpu: 100m, memory: 256Mi} @@ -1188,6 +1191,12 @@ spec: - {key: dashboard-api.py, path: dashboard/plugin_api.py} - {key: dashboard-index.js, path: dashboard/dist/index.js} - {key: dashboard-style.css, path: dashboard/dist/style.css} + - name: atlas-pr-skill + configMap: + name: hermes-atlas-pr-skill + items: + - {key: SKILL.md, path: SKILL.md} + - {key: openai.yaml, path: agents/openai.yaml} - name: image-policy configMap: name: hermes-image-policy diff --git a/services/hermes/kustomization.yaml b/services/hermes/kustomization.yaml index 3b9c2345..87984530 100644 --- a/services/hermes/kustomization.yaml +++ b/services/hermes/kustomization.yaml @@ -158,3 +158,10 @@ configMapGenerator: - alert-review.md=skills/tune-atlas-alerts/references/alert-review.md options: disableNameSuffixHash: true + - name: hermes-atlas-pr-skill + namespace: hermes + files: + - SKILL.md=skills/manage-atlas-pull-requests/SKILL.md + - openai.yaml=skills/manage-atlas-pull-requests/agents/openai.yaml + options: + disableNameSuffixHash: true diff --git a/services/hermes/scripts/gitea_api.py b/services/hermes/scripts/gitea_api.py old mode 100644 new mode 100755 index 5544c398..73a2bffd --- a/services/hermes/scripts/gitea_api.py +++ b/services/hermes/scripts/gitea_api.py @@ -1,43 +1,158 @@ #!/usr/bin/env python3 -"""Call the private Atlas Gitea API through a runtime-only token boundary.""" +"""Use the private Atlas Forgejo API through a least-authority PR boundary.""" from __future__ import annotations import argparse import json import os +import re import sys import urllib.error import urllib.parse import urllib.request +from collections.abc import Callable from pathlib import Path - -DEFAULT_BASE_URL = "https://scm.bstein.dev" +CANONICAL_BASE_URL = "https://scm.bstein.dev" +ALLOWED_OWNER = "atlas" DEFAULT_TOKEN_FILE = Path("/runtime-access/gitea-token") -ALLOWED_METHODS = ("GET", "POST", "PUT", "PATCH", "DELETE") +REPO_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,99}\Z") +PR_PATH_RE = re.compile(r"/api/v1/repos/atlas/([^/]+)/pulls/([1-9][0-9]*)\Z") +CREATE_PATH_RE = re.compile(r"/api/v1/repos/atlas/([^/]+)/pulls\Z") +READ_PATH_RE = re.compile(r"/api/v1/repos/atlas/([^/]+)(?:/.*)?\Z") +SAFE_REF_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._/-]{0,199}\Z") +MAX_ERROR_BYTES = 8192 +DRAFT_TITLE_PREFIX = "WIP: " + + +class PolicyError(ValueError): + """Raised when a requested Forgejo operation exceeds the safe boundary.""" def read_token(path: Path = DEFAULT_TOKEN_FILE) -> str: - """Read and validate the token from its in-memory Vault projection.""" + """Read the token from the pod-lifetime Vault projection.""" token = path.read_text(encoding="utf-8").strip() if not token: raise ValueError(f"runtime credential is empty: {path}") return token -def api_url(base_url: str, path: str) -> str: - """Return a same-origin Gitea API URL for a validated API path.""" - base = urllib.parse.urlsplit(base_url.rstrip("/")) +def configured_base_url() -> str: + """Reject attempts to redirect credentials away from the canonical origin.""" + configured = os.environ.get("GITEA_BASE_URL", CANONICAL_BASE_URL).rstrip("/") + if configured != CANONICAL_BASE_URL: + raise PolicyError("Forgejo origin is fixed to the private Atlas SCM service") + return configured + + +def _validate_repo(repo: str) -> str: + if not REPO_RE.fullmatch(repo) or repo in {".", ".."}: + raise PolicyError("repository name is outside the Atlas allowlist") + return repo + + +def _validate_ref(value: object, name: str) -> str: + if not isinstance(value, str) or not SAFE_REF_RE.fullmatch(value): + raise PolicyError(f"{name} must be a same-repository branch name") + if ( + value.startswith(".") + or value.endswith(("/", ".", ".lock")) + or "//" in value + or ".." in value + or "@{" in value + ): + raise PolicyError(f"{name} must be a safe same-repository branch name") + return value + + +def _validate_text(value: object, name: str, maximum: int, *, required: bool) -> str: + if not isinstance(value, str): + raise PolicyError(f"{name} must be text") + if required and not value.strip(): + raise PolicyError(f"{name} must not be empty") + if len(value) > maximum or "\x00" in value: + raise PolicyError(f"{name} exceeds the safe request limit") + return value + + +def _draft_title(value: object) -> str: + """Return a bounded title using Gitea's configured default draft prefix.""" + title = _validate_text(value, "title", 251, required=True).strip() + for prefix in ("WIP:", "[WIP]"): + if title.upper().startswith(prefix): + title = title[len(prefix) :].lstrip() + break + if not title: + raise PolicyError("title must contain text after the draft prefix") + return DRAFT_TITLE_PREFIX + title + + +def _split_api_path(path: str) -> urllib.parse.SplitResult: target = urllib.parse.urlsplit(path) - if base.scheme not in {"http", "https"} or not base.netloc: - raise ValueError("GITEA_BASE_URL must be an absolute HTTP(S) URL") if target.scheme or target.netloc or target.fragment: - raise ValueError("API path must be relative to the configured Gitea origin") + raise PolicyError("API path must be relative to the Atlas SCM origin") if not target.path.startswith("/api/v1/"): - raise ValueError("API path must start with /api/v1/") + raise PolicyError("API path must start with /api/v1/") + if "%" in target.path or "//" in target.path or "/../" in f"{target.path}/": + raise PolicyError("encoded or non-canonical API paths are not allowed") + return target + + +def api_url(base_url: str, path: str) -> str: + """Return a canonical same-origin URL without forwarding credentials.""" + if base_url.rstrip("/") != CANONICAL_BASE_URL: + raise PolicyError("Forgejo origin is fixed to the private Atlas SCM service") + target = _split_api_path(path) return urllib.parse.urlunsplit( - (base.scheme, base.netloc, target.path, target.query, "") + ("https", "scm.bstein.dev", target.path, target.query, "") + ) + + +def authorize_request(method: str, path: str, data: object | None) -> str: + """Validate one request and return its bounded operation name.""" + normalized_method = method.upper() + target = _split_api_path(path) + if normalized_method == "GET": + if data is not None: + raise PolicyError("read operations cannot include a request body") + match = READ_PATH_RE.fullmatch(target.path) + if not match: + raise PolicyError("reads are limited to repositories owned by atlas") + _validate_repo(match.group(1)) + return "read" + + if target.query: + raise PolicyError("mutating operations cannot include query parameters") + if normalized_method == "POST": + match = CREATE_PATH_RE.fullmatch(target.path) + if not match: + raise PolicyError("POST is limited to creating an Atlas draft pull request") + _validate_repo(match.group(1)) + if not isinstance(data, dict) or set(data) != {"base", "body", "head", "title"}: + raise PolicyError("draft creation accepts only base, body, head, and title") + _validate_ref(data["base"], "base") + _validate_ref(data["head"], "head") + if data["title"] != _draft_title(data["title"]): + raise PolicyError("new pull requests must use the Gitea draft-title prefix") + _validate_text(data["body"], "body", 65536, required=False) + return "create-draft" + + if normalized_method == "PATCH": + match = PR_PATH_RE.fullmatch(target.path) + if not match: + raise PolicyError("PATCH is limited to Atlas draft pull-request metadata") + _validate_repo(match.group(1)) + if not isinstance(data, dict) or not data or not set(data) <= {"body", "title"}: + raise PolicyError("draft updates accept only title and body") + if "title" in data and data["title"] != _draft_title(data["title"]): + raise PolicyError("updated pull-request titles must preserve draft state") + if "body" in data: + _validate_text(data["body"], "body", 65536, required=False) + return "update-draft" + + raise PolicyError( + "method is not available: merge, approve, close, delete, and force-push are outside this client" ) @@ -49,100 +164,255 @@ def build_request( token: str, data: object | None = None, ) -> urllib.request.Request: - """Build one authenticated request without placing the token in its URL.""" + """Build an authorized request without putting the token in its URL or body.""" + authorize_request(method, path, data) payload = None if data is not None: payload = json.dumps(data, separators=(",", ":")).encode("utf-8") return urllib.request.Request( api_url(base_url, path), data=payload, - method=method, + method=method.upper(), headers={ "Accept": "application/json", "Authorization": f"token {token}", "Content-Type": "application/json", - "User-Agent": "hermes-atlas-operator/1", + "User-Agent": "hermes-atlas-pr-client/1", }, ) -def parse_args(argv: list[str] | None = None) -> argparse.Namespace: - """Parse a bounded method, API path, and optional JSON request body.""" - parser = argparse.ArgumentParser( - description="Call the Atlas Gitea API using the runtime Vault token." +def redact_bytes(value: bytes, token: str) -> bytes: + """Remove exact runtime-token and authorization-header values from output.""" + redacted = value.replace(token.encode("utf-8"), b"[REDACTED]") + return re.sub( + rb"(?i)(authorization\s*[:=]\s*)(?:token|bearer)?\s*[^\s,;\"}]+", + rb"\1[REDACTED]", + redacted, ) - parser.add_argument("method", choices=ALLOWED_METHODS) - parser.add_argument("path", help="Gitea path beginning with /api/v1/") - data_group = parser.add_mutually_exclusive_group() - data_group.add_argument("--data-json", help="JSON object/array request body") - data_group.add_argument( - "--data-file", type=Path, help="path to a JSON request body" - ) - data_group.add_argument( - "--field", - action="append", - metavar="KEY=VALUE", - help=( - "repeatable JSON field; bare text remains a string while true, false, " - "null, numbers, objects, and arrays are decoded as JSON" - ), - ) - return parser.parse_args(argv) -def parse_fields(values: list[str]) -> dict[str, object]: - """Build a JSON object from shell-safe, repeatable key/value arguments.""" - result: dict[str, object] = {} - for value in values: - key, separator, raw = value.partition("=") - if not separator or not key or any(char.isspace() for char in key): - raise ValueError("each --field must be KEY=VALUE with a non-space key") - try: - result[key] = json.loads(raw) - except json.JSONDecodeError: - result[key] = raw +def _request( + method: str, + path: str, + data: object | None, + *, + token: str, + opener: Callable[..., object] = urllib.request.urlopen, +) -> bytes: + request = build_request( + method, + path, + base_url=configured_base_url(), + token=token, + data=data, + ) + with opener(request, timeout=30) as response: # type: ignore[attr-defined] + return redact_bytes(response.read(), token) # type: ignore[attr-defined] + + +def read( + path: str, *, token: str, opener: Callable[..., object] = urllib.request.urlopen +) -> bytes: + """Read one Atlas repository API resource.""" + return _request("GET", path, None, token=token, opener=opener) + + +def create_draft( + repo: str, + *, + base: str, + head: str, + title: str, + body: str, + token: str, + opener: Callable[..., object] = urllib.request.urlopen, +) -> bytes: + """Create a same-repository draft pull request for human review.""" + data = { + "base": base, + "body": body, + "head": head, + "title": _draft_title(title), + } + result = _request( + "POST", + f"/api/v1/repos/{ALLOWED_OWNER}/{_validate_repo(repo)}/pulls", + data, + token=token, + opener=opener, + ) + _require_draft_response(result) return result -def load_data(args: argparse.Namespace) -> object | None: - """Decode the optional JSON body without involving a shell expansion.""" - if args.data_json is not None: - return json.loads(args.data_json) - if args.data_file is not None: - return json.loads(args.data_file.read_text(encoding="utf-8")) - if args.field is not None: - return parse_fields(args.field) - return None +def _require_draft_response(value: bytes) -> dict[str, object]: + """Fail closed unless Forgejo confirms the resulting PR remains a draft.""" + try: + document = json.loads(value) + except json.JSONDecodeError as exc: + raise PolicyError("Forgejo returned invalid pull-request metadata") from exc + if not isinstance(document, dict) or document.get("draft") is not True: + raise PolicyError( + "Forgejo did not confirm draft state; human review is required" + ) + return document + + +def update_draft( + repo: str, + number: int, + updates: dict[str, str], + *, + token: str, + opener: Callable[..., object] = urllib.request.urlopen, +) -> bytes: + """Update title/body only after Forgejo confirms the PR is still a draft.""" + if not isinstance(number, int) or isinstance(number, bool) or number < 1: + raise PolicyError("pull-request number must be positive") + path = f"/api/v1/repos/{ALLOWED_OWNER}/{_validate_repo(repo)}/pulls/{number}" + current_raw = _request("GET", path, None, token=token, opener=opener) + try: + current = json.loads(current_raw) + except json.JSONDecodeError as exc: + raise PolicyError("Forgejo returned invalid pull-request metadata") from exc + if not isinstance(current, dict) or current.get("draft") is not True: + raise PolicyError("pull request is not a draft; human review now owns it") + safe_updates = dict(updates) + if "title" in safe_updates: + safe_updates["title"] = _draft_title(safe_updates["title"]) + result = _request("PATCH", path, safe_updates, token=token, opener=opener) + _require_draft_response(result) + return result + + +def _body(args: argparse.Namespace) -> str: + if args.body_file is None: + return "" + return args.body_file.read_text(encoding="utf-8") + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + """Parse the three intentionally narrow client operations.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--dry-run", + action="store_true", + help="validate and describe the request without reading a token or using the network", + ) + subparsers = parser.add_subparsers(dest="operation", required=True) + + read_parser = subparsers.add_parser( + "read", help="read an Atlas repository API path" + ) + read_parser.add_argument("path") + + create_parser = subparsers.add_parser( + "create-draft", help="create a same-repository draft pull request" + ) + create_parser.add_argument("repo") + create_parser.add_argument("--base", required=True) + create_parser.add_argument("--head", required=True) + create_parser.add_argument("--title", required=True) + create_parser.add_argument("--body-file", type=Path) + + update_parser = subparsers.add_parser( + "update-draft", help="update title/body of an existing draft pull request" + ) + update_parser.add_argument("repo") + update_parser.add_argument("number", type=int) + update_parser.add_argument("--title") + update_parser.add_argument("--body-file", type=Path) + return parser.parse_args(argv) + + +def _operation(args: argparse.Namespace) -> tuple[str, str, object | None]: + if args.operation == "read": + return "GET", args.path, None + repo = _validate_repo(args.repo) + if args.operation == "create-draft": + data = { + "base": args.base, + "body": _body(args), + "head": args.head, + "title": _draft_title(args.title), + } + return "POST", f"/api/v1/repos/{ALLOWED_OWNER}/{repo}/pulls", data + if args.number < 1: + raise PolicyError("pull-request number must be positive") + data = {} + if args.title is not None: + data["title"] = _draft_title(args.title) + if args.body_file is not None: + data["body"] = _body(args) + return "PATCH", f"/api/v1/repos/{ALLOWED_OWNER}/{repo}/pulls/{args.number}", data + + +def _write_body(body: bytes) -> None: + if body: + sys.stdout.buffer.write(body) + if not body.endswith(b"\n"): + sys.stdout.buffer.write(b"\n") def main(argv: list[str] | None = None) -> int: - """Execute the request, print only its response body, and return HTTP status.""" - args = parse_args(argv) + """Validate, optionally describe, and execute one safe Forgejo operation.""" + token = "" try: - request = build_request( - args.method, - args.path, - base_url=os.environ.get("GITEA_BASE_URL", DEFAULT_BASE_URL), - token=read_token(), - data=load_data(args), - ) - with urllib.request.urlopen(request, timeout=30) as response: - body = response.read() - if body: - sys.stdout.buffer.write(body) - if not body.endswith(b"\n"): - sys.stdout.buffer.write(b"\n") + args = parse_args(argv) + method, path, data = _operation(args) + operation = authorize_request(method, path, data) + if args.dry_run: + print( + json.dumps( + { + "dry_run": True, + "host": "scm.bstein.dev", + "operation": operation, + "owner": ALLOWED_OWNER, + "path": path, + "request_fields": sorted(data) + if isinstance(data, dict) + else [], + "requires_draft_preflight": operation == "update-draft", + }, + sort_keys=True, + ) + ) + return 0 + token = read_token() + if args.operation == "read": + body = read(path, token=token) + elif args.operation == "create-draft": + assert isinstance(data, dict) + body = create_draft( + args.repo, + base=str(data["base"]), + head=str(data["head"]), + title=str(data["title"]), + body=str(data["body"]), + token=token, + ) + else: + assert isinstance(data, dict) + body = update_draft(args.repo, args.number, data, token=token) + _write_body(body) return 0 except urllib.error.HTTPError as exc: - body = exc.read(65536) - print(f"Gitea API returned HTTP {exc.code}", file=sys.stderr) + body = redact_bytes(exc.read(MAX_ERROR_BYTES), token) if token else b"" + print(f"Forgejo request failed with HTTP {exc.code}", file=sys.stderr) if body: sys.stderr.buffer.write(body) if not body.endswith(b"\n"): sys.stderr.buffer.write(b"\n") return 1 - except (OSError, ValueError, json.JSONDecodeError) as exc: - print(f"Gitea API request failed: {exc}", file=sys.stderr) + except (OSError, PolicyError, ValueError, json.JSONDecodeError): + # Deliberately omit exception details. File contents, HTTP request + # objects, and provider errors can carry credential material. + print( + "Forgejo request rejected or unavailable; no credential was disclosed", + file=sys.stderr, + ) return 1 diff --git a/services/hermes/skills/manage-atlas-pull-requests/SKILL.md b/services/hermes/skills/manage-atlas-pull-requests/SKILL.md new file mode 100644 index 00000000..4e73a3cf --- /dev/null +++ b/services/hermes/skills/manage-atlas-pull-requests/SKILL.md @@ -0,0 +1,58 @@ +--- +name: manage-atlas-pull-requests +description: Read private Atlas repository and pull-request metadata, create a draft pull request after a tested branch is pushed, or update the title/body of an existing draft through Hermes' least-authority Forgejo client. Use for PR handoff in scm.bstein.dev/atlas repositories. Never use it to merge, approve, close, delete, or force-push. +--- + +# Manage Atlas pull requests + +Use `/opt/coordinator/gitea_api.py` for Forgejo API access. It reads its token +from the pod-lifetime Vault projection; never read that file, copy the token, +put it in an argument or environment variable, or replace this client with +`curl`. + +## Read repository or PR state + +Pass an Atlas repository API path to the read operation: + +```sh +/opt/coordinator/gitea_api.py read /api/v1/repos/atlas/REPO/pulls/NUMBER +``` + +Reads outside `scm.bstein.dev/atlas` are rejected. Treat returned state as +evidence, not authorization to change it. + +## Create a review handoff + +Before opening a PR, verify the remote, branch, clean worktree, diff, tests, and +exact pushed commit. Never force-push. Write the PR description to a workspace +file so multiline Markdown is not shell-quoted, then validate without reading a +credential or using the network: + +```sh +/opt/coordinator/gitea_api.py --dry-run create-draft REPO \ + --base main --head hermes/TASK --title "Focused change" \ + --body-file /tmp/pr-body.md +``` + +Run the same command without `--dry-run` only when the branch is review-ready. +The client always creates a draft in the same repository. Report its URL and +exact head commit to Brad. Leave it unmerged for human review. + +## Update an existing draft + +Use only a title, a body file, or both: + +```sh +/opt/coordinator/gitea_api.py update-draft REPO NUMBER \ + --body-file /tmp/pr-body.md +``` + +The client first reads the PR from Forgejo and refuses the update unless it is +still a draft. It never changes head/base, review state, or merge state. + +## Stop at the authority boundary + +Merge, approve, close, delete, branch mutation, comments, releases, repository +administration, and force-push are intentionally unavailable. Do not bypass the +client with a raw HTTP request. If one of those actions is needed, present the +tested commit and evidence to Brad and stop for human review. diff --git a/services/hermes/skills/manage-atlas-pull-requests/agents/openai.yaml b/services/hermes/skills/manage-atlas-pull-requests/agents/openai.yaml new file mode 100644 index 00000000..57cb8496 --- /dev/null +++ b/services/hermes/skills/manage-atlas-pull-requests/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Manage Atlas Pull Requests" + short_description: "Create safe draft PR handoffs for human review" + default_prompt: "Use $manage-atlas-pull-requests to inspect the requested Atlas repository or prepare a tested branch as an unmerged draft pull request. Preserve the human review boundary." diff --git a/testing/tests/test_hermes_chat_quality.py b/testing/tests/test_hermes_chat_quality.py index 818f7679..09f6b00c 100644 --- a/testing/tests/test_hermes_chat_quality.py +++ b/testing/tests/test_hermes_chat_quality.py @@ -99,9 +99,11 @@ def test_agent_config_keeps_delegated_reviewers_from_owning_task_lifecycle(): assert "Never call `kanban_show` without\na known, non-empty task ID" in soul assert "must load a skill only when its workflow\nmaterially applies" in soul assert "runtime-only `GIT_ASKPASS`" in soul - assert "`scm.bstein.dev` is Gitea, not GitHub" in instructions - assert "Never load or follow a GitHub/`gh`" in instructions - assert "/opt/coordinator/gitea_api.py METHOD /api/v1/..." in instructions + assert "`scm.bstein.dev` is Forgejo/Gitea, not GitHub" in instructions + assert "GitHub/`gh` skill for an Atlas remote" in instructions + assert "load\n`$manage-atlas-pull-requests`" in instructions + assert "Merge, approve, close, delete, comments" in instructions + assert "Leave every PR unmerged for Brad's review" in instructions assert "JENKINS_BASE_URL" in instructions assert "Do not\nuse `git reset --hard`" in instructions rendered = (HERMES / "agent-deployment.yaml").read_text() diff --git a/testing/tests/test_hermes_gitea_pr_client.py b/testing/tests/test_hermes_gitea_pr_client.py new file mode 100644 index 00000000..658af189 --- /dev/null +++ b/testing/tests/test_hermes_gitea_pr_client.py @@ -0,0 +1,322 @@ +"""Adversarial contracts for the Atlas-only draft pull-request client.""" + +from __future__ import annotations + +import importlib.util +import io +import json +import sys +import urllib.error +from pathlib import Path + +import pytest +import yaml + +ROOT = Path(__file__).parents[2] +CLIENT_PATH = ROOT / "services/hermes/scripts/gitea_api.py" + + +def _load(): + spec = importlib.util.spec_from_file_location("safe_gitea_api", CLIENT_PATH) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _draft_payload(**updates): + payload = { + "base": "main", + "body": "Review evidence", + "head": "hermes/review-fix", + "title": "WIP: Repair review findings", + } + payload.update(updates) + return payload + + +class Response: + def __init__(self, body: object): + self.body = body if isinstance(body, bytes) else json.dumps(body).encode() + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def read(self): + return self.body + + +@pytest.mark.parametrize( + ("base_url", "path"), + [ + ("https://evil.example", "/api/v1/repos/atlas/cassandra"), + ("http://scm.bstein.dev", "/api/v1/repos/atlas/cassandra"), + ("https://scm.bstein.dev:443", "/api/v1/repos/atlas/cassandra"), + ("https://scm.bstein.dev", "https://evil.example/api/v1/repos/atlas/cassandra"), + ("https://scm.bstein.dev", "/api/v1/repos/evil/cassandra"), + ("https://scm.bstein.dev", "/api/v1/repos/%61tlas/cassandra"), + ("https://scm.bstein.dev", "/api/v1/repos/atlas/../admin"), + ], +) +def test_host_owner_and_path_escape_attempts_are_rejected(base_url: str, path: str): + client = _load() + + with pytest.raises(client.PolicyError): + client.build_request("GET", path, base_url=base_url, token="secret") + + +@pytest.mark.parametrize( + ("method", "path", "data"), + [ + ("DELETE", "/api/v1/repos/atlas/cassandra/pulls/4", None), + ("POST", "/api/v1/repos/atlas/cassandra/pulls/4/merge", {}), + ( + "POST", + "/api/v1/repos/atlas/cassandra/pulls/4/reviews", + {"event": "APPROVED"}, + ), + ("PATCH", "/api/v1/repos/atlas/cassandra/pulls/4", {"state": "closed"}), + ("PATCH", "/api/v1/repos/atlas/cassandra/pulls/4", {"draft": False}), + ("PUT", "/api/v1/repos/atlas/cassandra/branches/main", {}), + ], +) +def test_merge_approve_close_delete_and_other_mutations_are_rejected( + method: str, path: str, data: object +): + client = _load() + + with pytest.raises(client.PolicyError): + client.build_request( + method, path, base_url=client.CANONICAL_BASE_URL, token="secret", data=data + ) + + +def test_create_forces_draft_title_and_same_repository_branch_names(): + client = _load() + assert ( + client.authorize_request( + "POST", "/api/v1/repos/atlas/cassandra/pulls", _draft_payload() + ) + == "create-draft" + ) + + with pytest.raises(client.PolicyError, match="draft-title prefix"): + client.authorize_request( + "POST", + "/api/v1/repos/atlas/cassandra/pulls", + _draft_payload(title="Not a draft"), + ) + with pytest.raises(client.PolicyError): + client.authorize_request( + "POST", + "/api/v1/repos/atlas/cassandra/pulls", + _draft_payload(head="someone:branch"), + ) + + +def test_update_checks_server_draft_state_before_patching(): + client = _load() + calls = [] + responses = iter( + [ + Response({"number": 7, "draft": True}), + Response({"number": 7, "draft": True}), + ] + ) + + def opener(request, timeout): + calls.append((request.method, request.full_url, request.data, timeout)) + return next(responses) + + result = client.update_draft( + "cassandra", 7, {"title": "Narrowed repair"}, token="runtime", opener=opener + ) + + assert json.loads(result) == {"number": 7, "draft": True} + assert [call[0] for call in calls] == ["GET", "PATCH"] + assert json.loads(calls[1][2]) == {"title": "WIP: Narrowed repair"} + + +def test_update_rejects_non_draft_without_sending_patch(): + client = _load() + calls = [] + + def opener(request, timeout): + calls.append((request.method, timeout)) + return Response({"number": 7, "draft": False}) + + with pytest.raises(client.PolicyError, match="human review now owns"): + client.update_draft( + "cassandra", 7, {"body": "new"}, token="runtime", opener=opener + ) + assert calls == [("GET", 30)] + + +def test_runtime_token_is_only_an_authorization_header(): + client = _load() + request = client.build_request( + "POST", + "/api/v1/repos/atlas/cassandra/pulls", + base_url=client.CANONICAL_BASE_URL, + token="do-not-leak", + data=_draft_payload(), + ) + + assert "do-not-leak" not in request.full_url + assert b"do-not-leak" not in request.data + assert request.get_header("Authorization") == "token do-not-leak" + + +def test_create_uses_live_gitea_schema_and_verifies_draft_response(): + client = _load() + calls = [] + + def opener(request, timeout): + calls.append((json.loads(request.data), timeout)) + return Response({"number": 3, "draft": True}) + + result = client.create_draft( + "cassandra", + base="main", + head="hermes/fix", + title="Focused fix", + body="Evidence", + token="runtime", + opener=opener, + ) + + assert json.loads(result)["draft"] is True + assert calls[0][0]["title"] == "WIP: Focused fix" + assert "draft" not in calls[0][0] + + +def test_create_fails_closed_when_server_does_not_confirm_draft(): + client = _load() + + with pytest.raises(client.PolicyError, match="did not confirm draft"): + client.create_draft( + "cassandra", + base="main", + head="hermes/fix", + title="Focused fix", + body="Evidence", + token="runtime", + opener=lambda *_a, **_k: Response({"number": 3, "draft": False}), + ) + + +def test_output_redaction_covers_exact_token_and_authorization_header(): + client = _load() + raw = b'{"message":"do-not-leak","debug":"Authorization: token do-not-leak"}' + redacted = client.redact_bytes(raw, "do-not-leak") + + assert b"do-not-leak" not in redacted + assert redacted.count(b"[REDACTED]") >= 1 + + +def test_dry_run_does_not_read_token_or_use_network( + tmp_path: Path, monkeypatch, capsys +): + client = _load() + body = tmp_path / "body.md" + body.write_text("Evidence only\n", encoding="utf-8") + monkeypatch.setattr(client, "read_token", lambda: pytest.fail("read token")) + monkeypatch.setattr( + client.urllib.request, "urlopen", lambda *_a, **_k: pytest.fail("network") + ) + + assert ( + client.main( + [ + "--dry-run", + "create-draft", + "cassandra", + "--base", + "main", + "--head", + "hermes/fix", + "--title", + "Repair", + "--body-file", + str(body), + ] + ) + == 0 + ) + output = json.loads(capsys.readouterr().out) + assert output["operation"] == "create-draft" + assert output["owner"] == "atlas" + assert "Evidence only" not in json.dumps(output) + + +def test_http_error_path_redacts_token(monkeypatch, capsys): + client = _load() + monkeypatch.setattr(client, "read_token", lambda: "do-not-leak") + + def fail(*_args, **_kwargs): + raise urllib.error.HTTPError( + "https://scm.bstein.dev/api/v1/repos/atlas/cassandra", + 403, + "forbidden", + {}, + io.BytesIO(b"Authorization: token do-not-leak"), + ) + + monkeypatch.setattr(client, "read", lambda *_a, **_k: fail()) + + assert client.main(["read", "/api/v1/repos/atlas/cassandra"]) == 1 + captured = capsys.readouterr() + assert "do-not-leak" not in captured.err + assert "HTTP 403" in captured.err + + +def test_flux_manifest_projects_runtime_vault_token_and_skill_only(): + client_source = CLIENT_PATH.read_text(encoding="utf-8") + assert "/runtime-access/gitea-token" in client_source + assert "GITEA_TOKEN" not in client_source + + deployment = yaml.safe_load( + (ROOT / "services/hermes/agent-deployment.yaml").read_text(encoding="utf-8") + ) + template = deployment["spec"]["template"] + annotations = template["metadata"]["annotations"] + assert annotations["vault.hashicorp.com/agent-inject-secret-gitea-token"] == ( + "kv/data/atlas/hermes/developer-gitea" + ) + runtime = next( + volume + for volume in template["spec"]["volumes"] + if volume["name"] == "runtime-access" + ) + assert runtime["emptyDir"]["medium"] == "Memory" + + expected = {"hermes", "terminal", "cli-lane-runner"} + mounted = { + container["name"] + for container in template["spec"]["containers"] + if any( + mount["name"] == "atlas-pr-skill" + and mount["mountPath"] + == "/opt/data/workspace/skills/manage-atlas-pull-requests" + and mount.get("readOnly") is True + for mount in container.get("volumeMounts", []) + ) + } + assert mounted == expected + + kustomization = yaml.safe_load( + (ROOT / "services/hermes/kustomization.yaml").read_text(encoding="utf-8") + ) + generator = next( + item + for item in kustomization["configMapGenerator"] + if item["name"] == "hermes-atlas-pr-skill" + ) + assert generator["files"] == [ + "SKILL.md=skills/manage-atlas-pull-requests/SKILL.md", + "openai.yaml=skills/manage-atlas-pull-requests/agents/openai.yaml", + ] diff --git a/testing/tests/test_hermes_runtime_access.py b/testing/tests/test_hermes_runtime_access.py index 7e455eb6..7ad5087a 100644 --- a/testing/tests/test_hermes_runtime_access.py +++ b/testing/tests/test_hermes_runtime_access.py @@ -11,7 +11,6 @@ from pathlib import Path import pytest import yaml - ROOT = Path(__file__).parents[2] HERMES = ROOT / "services" / "hermes" SCRIPTS = HERMES / "scripts" @@ -31,19 +30,23 @@ def test_gitea_api_builds_runtime_authenticated_same_origin_requests(): request = gitea_api.build_request( "POST", - "/api/v1/repos/atlas/cassandra/issues/1/comments", + "/api/v1/repos/atlas/cassandra/pulls", base_url="https://scm.bstein.dev", token="runtime-only-value", - data={"body": "reviewed"}, + data={ + "base": "main", + "body": "Ready for review.", + "head": "hermes/repair", + "title": "WIP: Repair semantic review findings", + }, ) assert isinstance(request, urllib.request.Request) - assert request.full_url == ( - "https://scm.bstein.dev/api/v1/repos/atlas/cassandra/issues/1/comments" - ) + assert request.full_url == "https://scm.bstein.dev/api/v1/repos/atlas/cassandra/pulls" assert request.method == "POST" assert request.get_header("Authorization") == "token runtime-only-value" - assert json.loads(request.data) == {"body": "reviewed"} + assert json.loads(request.data)["title"].startswith("WIP: ") + assert "draft" not in json.loads(request.data) @pytest.mark.parametrize( @@ -61,25 +64,14 @@ def test_gitea_api_rejects_foreign_or_non_api_targets(path: str): gitea_api.api_url("https://scm.bstein.dev", path) -def test_gitea_api_parses_shell_safe_fields_as_json_values(): +@pytest.mark.parametrize("method", ["DELETE", "PUT", "OPTIONS"]) +def test_gitea_api_rejects_unavailable_methods(method: str): gitea_api = _load("gitea_api") - assert gitea_api.parse_fields( - ["Do=manually-merged", "MergeCommitID=abc123", "enabled=true", "count=2"] - ) == { - "Do": "manually-merged", - "MergeCommitID": "abc123", - "enabled": True, - "count": 2, - } - - -@pytest.mark.parametrize("value", ["missing-separator", "=missing-key", "bad key=x"]) -def test_gitea_api_rejects_invalid_shell_safe_fields(value: str): - gitea_api = _load("gitea_api") - - with pytest.raises(ValueError): - gitea_api.parse_fields([value]) + with pytest.raises(gitea_api.PolicyError): + gitea_api.authorize_request( + method, "/api/v1/repos/atlas/cassandra/pulls/1", None + ) def test_jenkins_evidence_ignores_nested_execution_result(): From cf3e46c2b684270ff931d82fcadcee6777fb9543 Mon Sep 17 00:00:00 2001 From: jenkins Date: Sun, 16 Aug 2026 20:18:49 -0300 Subject: [PATCH 2/4] hermes: harden Forgejo client boundary --- services/hermes/agent-configmap.yaml | 33 +- services/hermes/scripts/gitea_api.py | 381 +++++++++------- .../manage-atlas-pull-requests/SKILL.md | 48 +- testing/tests/test_hermes_chat_quality.py | 8 +- testing/tests/test_hermes_gitea_pr_client.py | 417 +++++++++++------- .../tests/test_hermes_gitea_pr_integration.py | 147 ++++++ 6 files changed, 671 insertions(+), 363 deletions(-) create mode 100644 testing/tests/test_hermes_gitea_pr_integration.py diff --git a/services/hermes/agent-configmap.yaml b/services/hermes/agent-configmap.yaml index b35da59a..782876cf 100644 --- a/services/hermes/agent-configmap.yaml +++ b/services/hermes/agent-configmap.yaml @@ -318,15 +318,15 @@ data: `scm.bstein.dev` is Forgejo/Gitea, not GitHub. Never load or follow a GitHub/`gh` skill for an Atlas remote, and do not interpret an unauthenticated Gitea HTTP 404 as a missing private repository. Use - authenticated Git for clone, fetch, and non-force push. For repository and - pull-request reads, or to create/update a review-ready draft PR, load + authenticated Git for clone, fetch, and non-force push. For bounded + repository/pull-request evidence or to create a review-ready draft PR, load `$manage-atlas-pull-requests` and use `/opt/coordinator/gitea_api.py`. The client injects the runtime Vault token without exposing it to the command line, environment, output, or transcript. It permits only Atlas-scoped - reads, same-repository draft creation, and title/body updates while the PR - remains draft. Merge, approve, close, delete, comments, repository - administration, and force-push are unavailable; never bypass the client - with raw HTTP. Leave every PR unmerged for Brad's review. Independently + metadata reads and verified same-repository draft creation. Updates, merge, + approve, close, delete, comments, repository administration, and force-push + are unavailable; never bypass the client with raw HTTP. Leave every PR + unmerged for Brad's review. Independently verify the exact base/head ancestry and diff, run the relevant tests, and confirm the candidate Jenkins result before presenting a handoff. Prefer the internal endpoint in @@ -337,25 +337,24 @@ data: [--commit SHA] --wait`; it distinguishes a genuinely terminal build from nested Jenkins execution metadata and returns bounded JSON/log evidence. With a branch, the helper also tries the conventional `JOB-branches` - multibranch name. After Brad merges, observe the default-branch build to a - terminal result and report exact commit, build, and test evidence. + multibranch name. If Brad separately reports that he merged the PR, observe + the default-branch build to a terminal result and report exact commit, + build, and test evidence. The workspace already has a non-secret Hermes Git author identity. Do not use `git reset --hard`, even in a fresh clone; use a detached worktree or a clean branch switch when comparing revisions, and preserve any unexpected - file as user state. Use the Gitea pull-request merge endpoint for an open - PR so both Git history and PR state remain auditable. If an authorized - manual merge already reached the base branch, reconcile the open PR with - `Do=manually-merged` and its exact merge commit instead of leaving stale - review state. + file as user state. PR publication ends at the verified open draft; Brad + owns review and merge authority. The terminal PATH contains the pinned operator tools. Start cluster work with `kubectl config current-context`, read-only status/events/logs, and the relevant `titan-iac` manifests. Put durable desired-state changes on a - reviewable `titan-iac` branch, validate Kustomize and client dry-run, then - use Flux reconciliation after the tracked change is published. Direct - `kubectl` mutations are for explicit incident recovery or ephemeral - verification, not ordinary delivery. + reviewable `titan-iac` branch and validate Kustomize and client dry-run. + Brad owns merge and routine Flux reconciliation; perform either only under + a separate explicit operator request. Direct `kubectl` mutations are for + explicit incident recovery or ephemeral verification, not ordinary + delivery. Node SSH uses a dedicated audited Hermes identity: `ssh titan-04` (or any current Kubernetes node name). Host keys are pinned and password fallback diff --git a/services/hermes/scripts/gitea_api.py b/services/hermes/scripts/gitea_api.py index 73a2bffd..057a79a1 100755 --- a/services/hermes/scripts/gitea_api.py +++ b/services/hermes/scripts/gitea_api.py @@ -7,6 +7,7 @@ import argparse import json import os import re +import subprocess import sys import urllib.error import urllib.parse @@ -18,18 +19,41 @@ CANONICAL_BASE_URL = "https://scm.bstein.dev" ALLOWED_OWNER = "atlas" DEFAULT_TOKEN_FILE = Path("/runtime-access/gitea-token") REPO_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,99}\Z") -PR_PATH_RE = re.compile(r"/api/v1/repos/atlas/([^/]+)/pulls/([1-9][0-9]*)\Z") +SHA_RE = re.compile(r"[0-9a-fA-F]{40}\Z") CREATE_PATH_RE = re.compile(r"/api/v1/repos/atlas/([^/]+)/pulls\Z") -READ_PATH_RE = re.compile(r"/api/v1/repos/atlas/([^/]+)(?:/.*)?\Z") -SAFE_REF_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._/-]{0,199}\Z") MAX_ERROR_BYTES = 8192 +MAX_RESPONSE_BYTES = 2 * 1024 * 1024 DRAFT_TITLE_PREFIX = "WIP: " +GIT_BIN = "/usr/bin/git" +SENSITIVE_BODY_PATTERNS = ( + re.compile(r"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----"), + re.compile(r"(?i)\b(?:password|passwd|secret|token|api[_-]?key)\s*[:=]"), + re.compile(r"(?i)\bauthorization\s*:\s*(?:bearer|token|basic)\s+\S+"), + re.compile(r"\b(?:gh[pousr]_[A-Za-z0-9]{20,}|sk-[A-Za-z0-9]{20,})\b"), + re.compile(r"\b(?:AKIA[0-9A-Z]{16}|AIza[0-9A-Za-z_-]{35})\b"), + re.compile(r"\beyJ[A-Za-z0-9_-]{12,}\.[A-Za-z0-9_-]{12,}\.[A-Za-z0-9_-]{8,}\b"), +) class PolicyError(ValueError): """Raised when a requested Forgejo operation exceeds the safe boundary.""" +class RejectRedirectHandler(urllib.request.HTTPRedirectHandler): + """Reject every redirect before urllib can copy authentication headers.""" + + def redirect_request(self, req, fp, code, msg, headers, newurl): + raise PolicyError("Forgejo redirects are not allowed") + + +_SAFE_OPENER = urllib.request.build_opener(RejectRedirectHandler()) + + +def _safe_urlopen(request: urllib.request.Request, timeout: int): + """Open one request using a handler that never follows redirects.""" + return _SAFE_OPENER.open(request, timeout=timeout) + + def read_token(path: Path = DEFAULT_TOKEN_FILE) -> str: """Read the token from the pod-lifetime Vault projection.""" token = path.read_text(encoding="utf-8").strip() @@ -53,19 +77,29 @@ def _validate_repo(repo: str) -> str: def _validate_ref(value: object, name: str) -> str: - if not isinstance(value, str) or not SAFE_REF_RE.fullmatch(value): + """Validate the complete Git ref grammar using Git itself.""" + if not isinstance(value, str) or not value or value.startswith("-"): raise PolicyError(f"{name} must be a same-repository branch name") - if ( - value.startswith(".") - or value.endswith(("/", ".", ".lock")) - or "//" in value - or ".." in value - or "@{" in value - ): + if any(ord(character) < 32 or ord(character) == 127 for character in value): + raise PolicyError(f"{name} must be a safe same-repository branch name") + result = subprocess.run( + [GIT_BIN, "check-ref-format", f"refs/heads/{value}"], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + if result.returncode != 0: raise PolicyError(f"{name} must be a safe same-repository branch name") return value +def _validate_sha(value: object, name: str = "head SHA") -> str: + if not isinstance(value, str) or not SHA_RE.fullmatch(value): + raise PolicyError(f"{name} must be a full Git SHA-1") + return value.lower() + + def _validate_text(value: object, name: str, maximum: int, *, required: bool) -> str: if not isinstance(value, str): raise PolicyError(f"{name} must be text") @@ -76,6 +110,15 @@ def _validate_text(value: object, name: str, maximum: int, *, required: bool) -> return value +def _validate_body(value: object, *, forbidden: tuple[str, ...] = ()) -> str: + body = _validate_text(value, "body", 16384, required=False) + if any(secret and secret in body for secret in forbidden): + raise PolicyError("pull-request body contains runtime credential material") + if any(pattern.search(body) for pattern in SENSITIVE_BODY_PATTERNS): + raise PolicyError("pull-request body resembles credential material") + return body + + def _draft_title(value: object) -> str: """Return a bounded title using Gitea's configured default draft prefix.""" title = _validate_text(value, "title", 251, required=True).strip() @@ -99,6 +142,73 @@ def _split_api_path(path: str) -> urllib.parse.SplitResult: return target +def _validate_query(target: urllib.parse.SplitResult, allowed: set[str]) -> None: + try: + pairs = urllib.parse.parse_qsl( + target.query, keep_blank_values=True, strict_parsing=True + ) + except ValueError as exc: + raise PolicyError("invalid API query") from exc + if len({key for key, _ in pairs}) != len(pairs): + raise PolicyError("duplicate API query parameters are not allowed") + if any(key not in allowed for key, _ in pairs): + raise PolicyError("API query parameter is outside the read allowlist") + values = dict(pairs) + for name in ("page", "limit"): + if name not in values: + continue + if not values[name].isdigit() or int(values[name]) < 1: + raise PolicyError(f"{name} must be a positive integer") + if "limit" in values and int(values["limit"]) > 50: + raise PolicyError("read limit cannot exceed 50") + if "state" in values and values["state"] not in {"open", "closed", "all"}: + raise PolicyError("pull-request state is invalid") + + +def _authorize_read(target: urllib.parse.SplitResult) -> str: + """Allow only repository, PR, branch, commit, and status metadata reads.""" + prefix = "/api/v1/repos/atlas/" + remainder = target.path.removeprefix(prefix) + if remainder == target.path: + raise PolicyError("reads are limited to explicit Atlas repository metadata") + repo, separator, suffix = remainder.partition("/") + _validate_repo(repo) + + if not separator: + _validate_query(target, set()) + return "repository" + if suffix == "pulls": + _validate_query(target, {"page", "limit", "state"}) + return "pull-list" + if re.fullmatch(r"pulls/[1-9][0-9]*", suffix): + _validate_query(target, set()) + return "pull" + if re.fullmatch(r"pulls/[1-9][0-9]*/(?:commits|files)", suffix): + _validate_query(target, {"page", "limit"}) + return "pull-evidence" + if suffix == "branches": + _validate_query(target, {"page", "limit"}) + return "branch-list" + branch_match = re.fullmatch(r"branches/([^/]+)", suffix) + if branch_match: + _validate_ref(branch_match.group(1), "branch") + _validate_query(target, set()) + return "branch" + if suffix == "commits": + _validate_query(target, {"page", "limit"}) + return "commit-list" + if re.fullmatch(r"git/commits/[0-9a-fA-F]{40}", suffix): + _validate_query(target, set()) + return "commit" + if re.fullmatch( + r"(?:commits/[0-9a-fA-F]{40}/status|commits/[0-9a-fA-F]{40}/statuses|statuses/[0-9a-fA-F]{40})", + suffix, + ): + _validate_query(target, {"page", "limit"}) + return "status" + raise PolicyError("repository API route is outside the metadata read allowlist") + + def api_url(base_url: str, path: str) -> str: """Return a canonical same-origin URL without forwarding credentials.""" if base_url.rstrip("/") != CANONICAL_BASE_URL: @@ -116,44 +226,23 @@ def authorize_request(method: str, path: str, data: object | None) -> str: if normalized_method == "GET": if data is not None: raise PolicyError("read operations cannot include a request body") - match = READ_PATH_RE.fullmatch(target.path) - if not match: - raise PolicyError("reads are limited to repositories owned by atlas") - _validate_repo(match.group(1)) - return "read" - + return _authorize_read(target) if target.query: raise PolicyError("mutating operations cannot include query parameters") - if normalized_method == "POST": - match = CREATE_PATH_RE.fullmatch(target.path) - if not match: - raise PolicyError("POST is limited to creating an Atlas draft pull request") - _validate_repo(match.group(1)) - if not isinstance(data, dict) or set(data) != {"base", "body", "head", "title"}: - raise PolicyError("draft creation accepts only base, body, head, and title") - _validate_ref(data["base"], "base") - _validate_ref(data["head"], "head") - if data["title"] != _draft_title(data["title"]): - raise PolicyError("new pull requests must use the Gitea draft-title prefix") - _validate_text(data["body"], "body", 65536, required=False) - return "create-draft" - - if normalized_method == "PATCH": - match = PR_PATH_RE.fullmatch(target.path) - if not match: - raise PolicyError("PATCH is limited to Atlas draft pull-request metadata") - _validate_repo(match.group(1)) - if not isinstance(data, dict) or not data or not set(data) <= {"body", "title"}: - raise PolicyError("draft updates accept only title and body") - if "title" in data and data["title"] != _draft_title(data["title"]): - raise PolicyError("updated pull-request titles must preserve draft state") - if "body" in data: - _validate_text(data["body"], "body", 65536, required=False) - return "update-draft" - - raise PolicyError( - "method is not available: merge, approve, close, delete, and force-push are outside this client" - ) + if normalized_method != "POST": + raise PolicyError("only read and create-draft operations are available") + match = CREATE_PATH_RE.fullmatch(target.path) + if not match: + raise PolicyError("POST is limited to creating an Atlas draft pull request") + _validate_repo(match.group(1)) + if not isinstance(data, dict) or set(data) != {"base", "body", "head", "title"}: + raise PolicyError("draft creation accepts only base, body, head, and title") + _validate_ref(data["base"], "base") + _validate_ref(data["head"], "head") + if data["title"] != _draft_title(data["title"]): + raise PolicyError("new pull requests must use the Gitea draft-title prefix") + _validate_body(data["body"]) + return "create-draft" def build_request( @@ -177,7 +266,7 @@ def build_request( "Accept": "application/json", "Authorization": f"token {token}", "Content-Type": "application/json", - "User-Agent": "hermes-atlas-pr-client/1", + "User-Agent": "hermes-atlas-pr-client/2", }, ) @@ -198,102 +287,116 @@ def _request( data: object | None, *, token: str, - opener: Callable[..., object] = urllib.request.urlopen, + opener: Callable[..., object] = _safe_urlopen, ) -> bytes: request = build_request( - method, - path, - base_url=configured_base_url(), - token=token, - data=data, + method, path, base_url=configured_base_url(), token=token, data=data ) with opener(request, timeout=30) as response: # type: ignore[attr-defined] - return redact_bytes(response.read(), token) # type: ignore[attr-defined] + body = response.read(MAX_RESPONSE_BYTES + 1) # type: ignore[attr-defined] + if len(body) > MAX_RESPONSE_BYTES: + raise PolicyError("Forgejo response exceeds the safe size limit") + return redact_bytes(body, token) def read( - path: str, *, token: str, opener: Callable[..., object] = urllib.request.urlopen + path: str, *, token: str, opener: Callable[..., object] = _safe_urlopen ) -> bytes: - """Read one Atlas repository API resource.""" + """Read one explicitly allowed Atlas repository metadata resource.""" return _request("GET", path, None, token=token, opener=opener) +def _nested(document: dict[str, object], *keys: str) -> object: + value: object = document + for key in keys: + if not isinstance(value, dict) or key not in value: + raise PolicyError("Forgejo omitted required pull-request evidence") + value = value[key] + return value + + +def _require_create_response( + value: bytes, + *, + repo: str, + base: str, + head: str, + head_sha: str, + title: str, + body: str, +) -> dict[str, object]: + """Require exact server evidence for the intended open draft handoff.""" + try: + document = json.loads(value) + except json.JSONDecodeError as exc: + raise PolicyError("Forgejo returned invalid pull-request metadata") from exc + if not isinstance(document, dict): + raise PolicyError("Forgejo returned invalid pull-request metadata") + number = document.get("number") + if not isinstance(number, int) or isinstance(number, bool) or number < 1: + raise PolicyError("Forgejo omitted a valid pull-request number") + full_name = f"{ALLOWED_OWNER}/{repo}" + expected_html = f"{CANONICAL_BASE_URL}/{full_name}/pulls/{number}" + expected_api = f"{CANONICAL_BASE_URL}/api/v1/repos/{full_name}/pulls/{number}" + checks = ( + document.get("state") == "open", + document.get("draft") is True, + document.get("merged") is False, + document.get("html_url") == expected_html, + document.get("url") == expected_api, + document.get("title") == title, + document.get("body") == body, + _nested(document, "base", "ref") == base, + _nested(document, "base", "repo", "full_name") == full_name, + _nested(document, "head", "ref") == head, + _nested(document, "head", "sha") == head_sha, + _nested(document, "head", "repo", "full_name") == full_name, + ) + if not all(checks): + raise PolicyError("Forgejo did not confirm the exact draft pull-request state") + return document + + def create_draft( repo: str, *, base: str, head: str, + head_sha: str, title: str, body: str, token: str, - opener: Callable[..., object] = urllib.request.urlopen, + opener: Callable[..., object] = _safe_urlopen, ) -> bytes: - """Create a same-repository draft pull request for human review.""" - data = { - "base": base, - "body": body, - "head": head, - "title": _draft_title(title), - } + """Create and verify a same-repository draft pull request for human review.""" + repo = _validate_repo(repo) + base = _validate_ref(base, "base") + head = _validate_ref(head, "head") + head_sha = _validate_sha(head_sha) + title = _draft_title(title) + body = _validate_body(body, forbidden=(token,)) + data = {"base": base, "body": body, "head": head, "title": title} result = _request( "POST", - f"/api/v1/repos/{ALLOWED_OWNER}/{_validate_repo(repo)}/pulls", + f"/api/v1/repos/{ALLOWED_OWNER}/{repo}/pulls", data, token=token, opener=opener, ) - _require_draft_response(result) + _require_create_response( + result, + repo=repo, + base=base, + head=head, + head_sha=head_sha, + title=title, + body=body, + ) return result -def _require_draft_response(value: bytes) -> dict[str, object]: - """Fail closed unless Forgejo confirms the resulting PR remains a draft.""" - try: - document = json.loads(value) - except json.JSONDecodeError as exc: - raise PolicyError("Forgejo returned invalid pull-request metadata") from exc - if not isinstance(document, dict) or document.get("draft") is not True: - raise PolicyError( - "Forgejo did not confirm draft state; human review is required" - ) - return document - - -def update_draft( - repo: str, - number: int, - updates: dict[str, str], - *, - token: str, - opener: Callable[..., object] = urllib.request.urlopen, -) -> bytes: - """Update title/body only after Forgejo confirms the PR is still a draft.""" - if not isinstance(number, int) or isinstance(number, bool) or number < 1: - raise PolicyError("pull-request number must be positive") - path = f"/api/v1/repos/{ALLOWED_OWNER}/{_validate_repo(repo)}/pulls/{number}" - current_raw = _request("GET", path, None, token=token, opener=opener) - try: - current = json.loads(current_raw) - except json.JSONDecodeError as exc: - raise PolicyError("Forgejo returned invalid pull-request metadata") from exc - if not isinstance(current, dict) or current.get("draft") is not True: - raise PolicyError("pull request is not a draft; human review now owns it") - safe_updates = dict(updates) - if "title" in safe_updates: - safe_updates["title"] = _draft_title(safe_updates["title"]) - result = _request("PATCH", path, safe_updates, token=token, opener=opener) - _require_draft_response(result) - return result - - -def _body(args: argparse.Namespace) -> str: - if args.body_file is None: - return "" - return args.body_file.read_text(encoding="utf-8") - - def parse_args(argv: list[str] | None = None) -> argparse.Namespace: - """Parse the three intentionally narrow client operations.""" + """Parse the intentionally narrow read and create-draft operations.""" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--dry-run", @@ -301,28 +404,17 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: help="validate and describe the request without reading a token or using the network", ) subparsers = parser.add_subparsers(dest="operation", required=True) - - read_parser = subparsers.add_parser( - "read", help="read an Atlas repository API path" - ) + read_parser = subparsers.add_parser("read", help="read allowed Atlas metadata") read_parser.add_argument("path") - create_parser = subparsers.add_parser( "create-draft", help="create a same-repository draft pull request" ) create_parser.add_argument("repo") create_parser.add_argument("--base", required=True) create_parser.add_argument("--head", required=True) + create_parser.add_argument("--head-sha", required=True) create_parser.add_argument("--title", required=True) - create_parser.add_argument("--body-file", type=Path) - - update_parser = subparsers.add_parser( - "update-draft", help="update title/body of an existing draft pull request" - ) - update_parser.add_argument("repo") - update_parser.add_argument("number", type=int) - update_parser.add_argument("--title") - update_parser.add_argument("--body-file", type=Path) + create_parser.add_argument("--body", default="") return parser.parse_args(argv) @@ -330,22 +422,14 @@ def _operation(args: argparse.Namespace) -> tuple[str, str, object | None]: if args.operation == "read": return "GET", args.path, None repo = _validate_repo(args.repo) - if args.operation == "create-draft": - data = { - "base": args.base, - "body": _body(args), - "head": args.head, - "title": _draft_title(args.title), - } - return "POST", f"/api/v1/repos/{ALLOWED_OWNER}/{repo}/pulls", data - if args.number < 1: - raise PolicyError("pull-request number must be positive") - data = {} - if args.title is not None: - data["title"] = _draft_title(args.title) - if args.body_file is not None: - data["body"] = _body(args) - return "PATCH", f"/api/v1/repos/{ALLOWED_OWNER}/{repo}/pulls/{args.number}", data + _validate_sha(args.head_sha) + data = { + "base": _validate_ref(args.base, "base"), + "body": _validate_body(args.body), + "head": _validate_ref(args.head, "head"), + "title": _draft_title(args.title), + } + return "POST", f"/api/v1/repos/{ALLOWED_OWNER}/{repo}/pulls", data def _write_body(body: bytes) -> None: @@ -374,7 +458,6 @@ def main(argv: list[str] | None = None) -> int: "request_fields": sorted(data) if isinstance(data, dict) else [], - "requires_draft_preflight": operation == "update-draft", }, sort_keys=True, ) @@ -383,19 +466,17 @@ def main(argv: list[str] | None = None) -> int: token = read_token() if args.operation == "read": body = read(path, token=token) - elif args.operation == "create-draft": + else: assert isinstance(data, dict) body = create_draft( args.repo, base=str(data["base"]), head=str(data["head"]), + head_sha=args.head_sha, title=str(data["title"]), body=str(data["body"]), token=token, ) - else: - assert isinstance(data, dict) - body = update_draft(args.repo, args.number, data, token=token) _write_body(body) return 0 except urllib.error.HTTPError as exc: @@ -407,8 +488,6 @@ def main(argv: list[str] | None = None) -> int: sys.stderr.buffer.write(b"\n") return 1 except (OSError, PolicyError, ValueError, json.JSONDecodeError): - # Deliberately omit exception details. File contents, HTTP request - # objects, and provider errors can carry credential material. print( "Forgejo request rejected or unavailable; no credential was disclosed", file=sys.stderr, diff --git a/services/hermes/skills/manage-atlas-pull-requests/SKILL.md b/services/hermes/skills/manage-atlas-pull-requests/SKILL.md index 4e73a3cf..32fd1ccf 100644 --- a/services/hermes/skills/manage-atlas-pull-requests/SKILL.md +++ b/services/hermes/skills/manage-atlas-pull-requests/SKILL.md @@ -1,6 +1,6 @@ --- name: manage-atlas-pull-requests -description: Read private Atlas repository and pull-request metadata, create a draft pull request after a tested branch is pushed, or update the title/body of an existing draft through Hermes' least-authority Forgejo client. Use for PR handoff in scm.bstein.dev/atlas repositories. Never use it to merge, approve, close, delete, or force-push. +description: Read bounded private Atlas repository and pull-request metadata or create a verified draft pull request after a tested branch is pushed through Hermes' least-authority Forgejo client. Use for PR handoff in scm.bstein.dev/atlas repositories. Never use it to update, merge, approve, close, delete, or force-push. --- # Manage Atlas pull requests @@ -12,47 +12,43 @@ put it in an argument or environment variable, or replace this client with ## Read repository or PR state -Pass an Atlas repository API path to the read operation: +Pass one allowed Atlas metadata API path to the read operation: ```sh /opt/coordinator/gitea_api.py read /api/v1/repos/atlas/REPO/pulls/NUMBER ``` -Reads outside `scm.bstein.dev/atlas` are rejected. Treat returned state as +The client exposes only repository metadata, PRs and PR evidence, branches, +commits, and commit status. Hooks, Actions secrets/variables, collaborators, +protections, reviews, merge endpoints, releases, and administration are +rejected even as reads. Responses are size-bounded. Treat returned state as evidence, not authorization to change it. ## Create a review handoff Before opening a PR, verify the remote, branch, clean worktree, diff, tests, and -exact pushed commit. Never force-push. Write the PR description to a workspace -file so multiline Markdown is not shell-quoted, then validate without reading a -credential or using the network: +exact pushed commit. Never force-push. Pass that full 40-character pushed head +SHA explicitly. Keep the short body free of credentials and credential-like +text. Validate without reading a credential or using the network: ```sh /opt/coordinator/gitea_api.py --dry-run create-draft REPO \ - --base main --head hermes/TASK --title "Focused change" \ - --body-file /tmp/pr-body.md + --base main --head hermes/TASK --head-sha FULL_PUSHED_SHA \ + --title "Focused change" --body "Tests and review evidence" ``` Run the same command without `--dry-run` only when the branch is review-ready. -The client always creates a draft in the same repository. Report its URL and -exact head commit to Brad. Leave it unmerged for human review. - -## Update an existing draft - -Use only a title, a body file, or both: - -```sh -/opt/coordinator/gitea_api.py update-draft REPO NUMBER \ - --body-file /tmp/pr-body.md -``` - -The client first reads the PR from Forgejo and refuses the update unless it is -still a draft. It never changes head/base, review state, or merge state. +The client accepts success only when Forgejo returns the exact requested +repository, base/head refs, head SHA, title/body, canonical URLs, positive PR +number, and open/unmerged/draft state. Report its URL and exact head commit to +Brad. Leave it unmerged for human review. PR updates are intentionally absent; +publish a corrected tested commit and ask Brad how to proceed if metadata must +change. ## Stop at the authority boundary -Merge, approve, close, delete, branch mutation, comments, releases, repository -administration, and force-push are intentionally unavailable. Do not bypass the -client with a raw HTTP request. If one of those actions is needed, present the -tested commit and evidence to Brad and stop for human review. +Update, merge, approve, close, delete, branch mutation, comments, releases, +repository administration, and force-push are intentionally unavailable. Do +not bypass the client with a raw HTTP request. If one of those actions is +needed, present the tested commit and evidence to Brad and stop for human +review. diff --git a/testing/tests/test_hermes_chat_quality.py b/testing/tests/test_hermes_chat_quality.py index 09f6b00c..025f3bf7 100644 --- a/testing/tests/test_hermes_chat_quality.py +++ b/testing/tests/test_hermes_chat_quality.py @@ -102,8 +102,12 @@ def test_agent_config_keeps_delegated_reviewers_from_owning_task_lifecycle(): assert "`scm.bstein.dev` is Forgejo/Gitea, not GitHub" in instructions assert "GitHub/`gh` skill for an Atlas remote" in instructions assert "load\n`$manage-atlas-pull-requests`" in instructions - assert "Merge, approve, close, delete, comments" in instructions - assert "Leave every PR unmerged for Brad's review" in instructions + assert "Updates, merge,\napprove, close, delete, comments" in instructions + assert "Leave every PR\nunmerged for Brad's review" in instructions + assert "Use the Gitea pull-request merge endpoint" not in instructions + assert "reconcile the open PR" not in instructions + assert "use Flux reconciliation after" not in instructions + assert "PR publication ends at the verified open draft" in instructions assert "JENKINS_BASE_URL" in instructions assert "Do not\nuse `git reset --hard`" in instructions rendered = (HERMES / "agent-deployment.yaml").read_text() diff --git a/testing/tests/test_hermes_gitea_pr_client.py b/testing/tests/test_hermes_gitea_pr_client.py index 658af189..91475bcf 100644 --- a/testing/tests/test_hermes_gitea_pr_client.py +++ b/testing/tests/test_hermes_gitea_pr_client.py @@ -2,18 +2,18 @@ from __future__ import annotations +import copy import importlib.util -import io import json import sys -import urllib.error +import urllib.request from pathlib import Path import pytest -import yaml ROOT = Path(__file__).parents[2] CLIENT_PATH = ROOT / "services/hermes/scripts/gitea_api.py" +HEAD_SHA = "465cf9146b05c174a2a8d310aff6c64be58277b6" def _load(): @@ -36,6 +36,27 @@ def _draft_payload(**updates): return payload +def _draft_response(**updates): + response = { + "number": 3, + "state": "open", + "draft": True, + "merged": False, + "html_url": "https://scm.bstein.dev/atlas/cassandra/pulls/3", + "url": "https://scm.bstein.dev/api/v1/repos/atlas/cassandra/pulls/3", + "title": "WIP: Focused fix", + "body": "Review evidence", + "base": {"ref": "main", "repo": {"full_name": "atlas/cassandra"}}, + "head": { + "ref": "hermes/fix", + "sha": HEAD_SHA, + "repo": {"full_name": "atlas/cassandra"}, + }, + } + response.update(updates) + return response + + class Response: def __init__(self, body: object): self.body = body if isinstance(body, bytes) else json.dumps(body).encode() @@ -46,8 +67,32 @@ class Response: def __exit__(self, *_args): return False - def read(self): - return self.body + def read(self, limit=-1): + return self.body if limit < 0 else self.body[:limit] + + +def test_redirect_handler_rejects_cross_origin_with_sentinel_authorization(): + client = _load() + source = urllib.request.Request( + "https://scm.bstein.dev/api/v1/repos/atlas/cassandra", + headers={"Authorization": "token redirect-sentinel"}, + ) + + with pytest.raises(client.PolicyError, match="redirects are not allowed") as exc: + client.RejectRedirectHandler().redirect_request( + source, + None, + 302, + "Found", + {}, + "https://evil.example/collect", + ) + + assert "redirect-sentinel" not in str(exc.value) + assert any( + isinstance(handler, client.RejectRedirectHandler) + for handler in client._SAFE_OPENER.handlers + ) @pytest.mark.parametrize( @@ -69,6 +114,71 @@ def test_host_owner_and_path_escape_attempts_are_rejected(base_url: str, path: s client.build_request("GET", path, base_url=base_url, token="secret") +@pytest.mark.parametrize( + "path", + [ + "/api/v1/repos/atlas/cassandra", + "/api/v1/repos/atlas/cassandra/pulls?state=open&limit=20&page=1", + "/api/v1/repos/atlas/cassandra/pulls/7", + "/api/v1/repos/atlas/cassandra/pulls/7/commits?limit=20", + "/api/v1/repos/atlas/cassandra/pulls/7/files?page=1", + "/api/v1/repos/atlas/cassandra/branches", + "/api/v1/repos/atlas/cassandra/branches/main", + "/api/v1/repos/atlas/cassandra/commits?limit=10", + f"/api/v1/repos/atlas/cassandra/git/commits/{HEAD_SHA}", + f"/api/v1/repos/atlas/cassandra/commits/{HEAD_SHA}/status", + f"/api/v1/repos/atlas/cassandra/commits/{HEAD_SHA}/statuses?limit=10", + f"/api/v1/repos/atlas/cassandra/statuses/{HEAD_SHA}?page=1", + ], +) +def test_explicit_read_allowlist_accepts_only_engineering_metadata(path: str): + client = _load() + + request = client.build_request( + "GET", path, base_url=client.CANONICAL_BASE_URL, token="runtime" + ) + assert request.method == "GET" + + +@pytest.mark.parametrize( + "path", + [ + "/api/v1/repos/atlas/cassandra/hooks", + "/api/v1/repos/atlas/cassandra/actions/secrets", + "/api/v1/repos/atlas/cassandra/actions/variables", + "/api/v1/repos/atlas/cassandra/collaborators", + "/api/v1/repos/atlas/cassandra/branch_protections", + "/api/v1/repos/atlas/cassandra/keys", + "/api/v1/repos/atlas/cassandra/pulls/7/reviews", + "/api/v1/repos/atlas/cassandra/pulls/7/merge", + "/api/v1/repos/atlas/cassandra/pulls/7.diff", + "/api/v1/repos/atlas/cassandra/releases", + ], +) +def test_privileged_or_content_routes_are_denied_even_for_get(path: str): + client = _load() + + with pytest.raises(client.PolicyError, match="outside the metadata read allowlist"): + client.authorize_request("GET", path, None) + + +@pytest.mark.parametrize( + "path", + [ + "/api/v1/repos/atlas/cassandra/pulls?limit=51", + "/api/v1/repos/atlas/cassandra/pulls?state=merged", + "/api/v1/repos/atlas/cassandra/pulls?private=true", + "/api/v1/repos/atlas/cassandra/pulls?limit=1&limit=2", + "/api/v1/repos/atlas/cassandra?p=1", + ], +) +def test_read_query_is_bounded(path: str): + client = _load() + + with pytest.raises(client.PolicyError): + client.authorize_request("GET", path, None) + + @pytest.mark.parametrize( ("method", "path", "data"), [ @@ -79,12 +189,11 @@ def test_host_owner_and_path_escape_attempts_are_rejected(base_url: str, path: s "/api/v1/repos/atlas/cassandra/pulls/4/reviews", {"event": "APPROVED"}, ), - ("PATCH", "/api/v1/repos/atlas/cassandra/pulls/4", {"state": "closed"}), - ("PATCH", "/api/v1/repos/atlas/cassandra/pulls/4", {"draft": False}), + ("PATCH", "/api/v1/repos/atlas/cassandra/pulls/4", {"title": "WIP: x"}), ("PUT", "/api/v1/repos/atlas/cassandra/branches/main", {}), ], ) -def test_merge_approve_close_delete_and_other_mutations_are_rejected( +def test_merge_approve_close_delete_update_and_other_mutations_are_rejected( method: str, path: str, data: object ): client = _load() @@ -95,6 +204,36 @@ def test_merge_approve_close_delete_and_other_mutations_are_rejected( ) +@pytest.mark.parametrize( + "ref", + [ + "foo/.bar", + "foo/bar.lock/baz", + "foo..bar", + "foo@{bar", + "foo//bar", + "-danger", + "danger.", + "danger~one", + "danger^one", + "danger:one", + "danger one", + ], +) +def test_complete_git_ref_validation_rejects_invalid_names(ref: str): + client = _load() + + with pytest.raises(client.PolicyError): + client._validate_ref(ref, "head") + + +def test_git_ref_validation_uses_fixed_trusted_binary(): + client = _load() + + assert client.GIT_BIN == "/usr/bin/git" + assert client._validate_ref("hermes/valid-fix", "head") == "hermes/valid-fix" + + def test_create_forces_draft_title_and_same_repository_branch_names(): client = _load() assert ( @@ -103,7 +242,6 @@ def test_create_forces_draft_title_and_same_repository_branch_names(): ) == "create-draft" ) - with pytest.raises(client.PolicyError, match="draft-title prefix"): client.authorize_request( "POST", @@ -118,44 +256,6 @@ def test_create_forces_draft_title_and_same_repository_branch_names(): ) -def test_update_checks_server_draft_state_before_patching(): - client = _load() - calls = [] - responses = iter( - [ - Response({"number": 7, "draft": True}), - Response({"number": 7, "draft": True}), - ] - ) - - def opener(request, timeout): - calls.append((request.method, request.full_url, request.data, timeout)) - return next(responses) - - result = client.update_draft( - "cassandra", 7, {"title": "Narrowed repair"}, token="runtime", opener=opener - ) - - assert json.loads(result) == {"number": 7, "draft": True} - assert [call[0] for call in calls] == ["GET", "PATCH"] - assert json.loads(calls[1][2]) == {"title": "WIP: Narrowed repair"} - - -def test_update_rejects_non_draft_without_sending_patch(): - client = _load() - calls = [] - - def opener(request, timeout): - calls.append((request.method, timeout)) - return Response({"number": 7, "draft": False}) - - with pytest.raises(client.PolicyError, match="human review now owns"): - client.update_draft( - "cassandra", 7, {"body": "new"}, token="runtime", opener=opener - ) - assert calls == [("GET", 30)] - - def test_runtime_token_is_only_an_authorization_header(): client = _load() request = client.build_request( @@ -171,41 +271,128 @@ def test_runtime_token_is_only_an_authorization_header(): assert request.get_header("Authorization") == "token do-not-leak" -def test_create_uses_live_gitea_schema_and_verifies_draft_response(): +@pytest.mark.parametrize( + "body", + [ + "pass" + "word=not-a-real-credential", + "Authorization: " + "Bearer not-a-real-credential-value", + "token: " + "ghp_" + "notarealcredentialvalue123456", + "-----BEGIN OPENSSH " + "PRIVATE KEY-----", + "eyJnotarealheader." + "notarealpayloadvalue." + "notarealsignature", + ], +) +def test_body_rejects_common_credential_shapes(body: str): + client = _load() + + with pytest.raises(client.PolicyError, match="credential material"): + client._validate_body(body) + + +def test_create_rejects_exact_runtime_token_before_network(): + client = _load() + called = False + + def opener(*_args, **_kwargs): + nonlocal called + called = True + return Response(_draft_response()) + + with pytest.raises(client.PolicyError, match="runtime credential"): + client.create_draft( + "cassandra", + base="main", + head="hermes/fix", + head_sha=HEAD_SHA, + title="Focused fix", + body="accidental runtime-sentinel value", + token="runtime-sentinel", + opener=opener, + ) + assert called is False + + +def test_create_verifies_every_server_postcondition(): client = _load() calls = [] def opener(request, timeout): - calls.append((json.loads(request.data), timeout)) - return Response({"number": 3, "draft": True}) + calls.append((request, timeout)) + return Response(_draft_response()) result = client.create_draft( "cassandra", base="main", head="hermes/fix", + head_sha=HEAD_SHA, title="Focused fix", - body="Evidence", + body="Review evidence", token="runtime", opener=opener, ) - assert json.loads(result)["draft"] is True - assert calls[0][0]["title"] == "WIP: Focused fix" - assert "draft" not in calls[0][0] + assert json.loads(result) == _draft_response() + payload = json.loads(calls[0][0].data) + assert payload == { + "base": "main", + "body": "Review evidence", + "head": "hermes/fix", + "title": "WIP: Focused fix", + } + assert calls[0][1] == 30 -def test_create_fails_closed_when_server_does_not_confirm_draft(): +def test_create_postcondition_rejects_every_material_mismatch(): + client = _load() + mutations = [ + ("number", 0), + ("state", "closed"), + ("draft", False), + ("merged", True), + ("html_url", "https://evil.example/pulls/3"), + ("url", "https://evil.example/api/pulls/3"), + ("title", "Focused fix"), + ("body", "different"), + ] + documents = [] + for key, value in mutations: + document = _draft_response() + document[key] = value + documents.append(document) + for path, value in [ + (("base", "ref"), "master"), + (("base", "repo", "full_name"), "evil/cassandra"), + (("head", "ref"), "other"), + (("head", "sha"), "0" * 40), + (("head", "repo", "full_name"), "evil/cassandra"), + ]: + document = copy.deepcopy(_draft_response()) + target = document + for key in path[:-1]: + target = target[key] + target[path[-1]] = value + documents.append(document) + + for document in documents: + with pytest.raises(client.PolicyError): + client._require_create_response( + json.dumps(document).encode(), + repo="cassandra", + base="main", + head="hermes/fix", + head_sha=HEAD_SHA, + title="WIP: Focused fix", + body="Review evidence", + ) + + +def test_read_response_is_bounded(): client = _load() - with pytest.raises(client.PolicyError, match="did not confirm draft"): - client.create_draft( - "cassandra", - base="main", - head="hermes/fix", - title="Focused fix", - body="Evidence", + with pytest.raises(client.PolicyError, match="safe size limit"): + client.read( + "/api/v1/repos/atlas/cassandra", token="runtime", - opener=lambda *_a, **_k: Response({"number": 3, "draft": False}), + opener=lambda *_a, **_k: Response(b"x" * (client.MAX_RESPONSE_BYTES + 1)), ) @@ -216,107 +403,3 @@ def test_output_redaction_covers_exact_token_and_authorization_header(): assert b"do-not-leak" not in redacted assert redacted.count(b"[REDACTED]") >= 1 - - -def test_dry_run_does_not_read_token_or_use_network( - tmp_path: Path, monkeypatch, capsys -): - client = _load() - body = tmp_path / "body.md" - body.write_text("Evidence only\n", encoding="utf-8") - monkeypatch.setattr(client, "read_token", lambda: pytest.fail("read token")) - monkeypatch.setattr( - client.urllib.request, "urlopen", lambda *_a, **_k: pytest.fail("network") - ) - - assert ( - client.main( - [ - "--dry-run", - "create-draft", - "cassandra", - "--base", - "main", - "--head", - "hermes/fix", - "--title", - "Repair", - "--body-file", - str(body), - ] - ) - == 0 - ) - output = json.loads(capsys.readouterr().out) - assert output["operation"] == "create-draft" - assert output["owner"] == "atlas" - assert "Evidence only" not in json.dumps(output) - - -def test_http_error_path_redacts_token(monkeypatch, capsys): - client = _load() - monkeypatch.setattr(client, "read_token", lambda: "do-not-leak") - - def fail(*_args, **_kwargs): - raise urllib.error.HTTPError( - "https://scm.bstein.dev/api/v1/repos/atlas/cassandra", - 403, - "forbidden", - {}, - io.BytesIO(b"Authorization: token do-not-leak"), - ) - - monkeypatch.setattr(client, "read", lambda *_a, **_k: fail()) - - assert client.main(["read", "/api/v1/repos/atlas/cassandra"]) == 1 - captured = capsys.readouterr() - assert "do-not-leak" not in captured.err - assert "HTTP 403" in captured.err - - -def test_flux_manifest_projects_runtime_vault_token_and_skill_only(): - client_source = CLIENT_PATH.read_text(encoding="utf-8") - assert "/runtime-access/gitea-token" in client_source - assert "GITEA_TOKEN" not in client_source - - deployment = yaml.safe_load( - (ROOT / "services/hermes/agent-deployment.yaml").read_text(encoding="utf-8") - ) - template = deployment["spec"]["template"] - annotations = template["metadata"]["annotations"] - assert annotations["vault.hashicorp.com/agent-inject-secret-gitea-token"] == ( - "kv/data/atlas/hermes/developer-gitea" - ) - runtime = next( - volume - for volume in template["spec"]["volumes"] - if volume["name"] == "runtime-access" - ) - assert runtime["emptyDir"]["medium"] == "Memory" - - expected = {"hermes", "terminal", "cli-lane-runner"} - mounted = { - container["name"] - for container in template["spec"]["containers"] - if any( - mount["name"] == "atlas-pr-skill" - and mount["mountPath"] - == "/opt/data/workspace/skills/manage-atlas-pull-requests" - and mount.get("readOnly") is True - for mount in container.get("volumeMounts", []) - ) - } - assert mounted == expected - - kustomization = yaml.safe_load( - (ROOT / "services/hermes/kustomization.yaml").read_text(encoding="utf-8") - ) - generator = next( - item - for item in kustomization["configMapGenerator"] - if item["name"] == "hermes-atlas-pr-skill" - ) - assert generator["files"] == [ - "SKILL.md=skills/manage-atlas-pull-requests/SKILL.md", - "openai.yaml=skills/manage-atlas-pull-requests/agents/openai.yaml", - ] diff --git a/testing/tests/test_hermes_gitea_pr_integration.py b/testing/tests/test_hermes_gitea_pr_integration.py new file mode 100644 index 00000000..d3b47eef --- /dev/null +++ b/testing/tests/test_hermes_gitea_pr_integration.py @@ -0,0 +1,147 @@ +"""Runtime and Flux integration contracts for the safe Atlas PR client.""" + +from __future__ import annotations + +import importlib.util +import io +import json +import sys +import urllib.error +from pathlib import Path + +import pytest +import yaml + +ROOT = Path(__file__).parents[2] +CLIENT_PATH = ROOT / "services/hermes/scripts/gitea_api.py" +HEAD_SHA = "465cf9146b05c174a2a8d310aff6c64be58277b6" + + +def _load(): + spec = importlib.util.spec_from_file_location( + "safe_gitea_api_integration", CLIENT_PATH + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def test_dry_run_has_no_file_input_and_uses_no_token_or_network(monkeypatch, capsys): + client = _load() + monkeypatch.setattr(client, "read_token", lambda: pytest.fail("read token")) + monkeypatch.setattr( + client, "_safe_urlopen", lambda *_a, **_k: pytest.fail("network") + ) + + assert ( + client.main( + [ + "--dry-run", + "create-draft", + "cassandra", + "--base", + "main", + "--head", + "hermes/fix", + "--head-sha", + HEAD_SHA, + "--title", + "Repair", + "--body", + "Evidence only", + ] + ) + == 0 + ) + output = json.loads(capsys.readouterr().out) + assert output["operation"] == "create-draft" + assert output["owner"] == "atlas" + assert "Evidence only" not in json.dumps(output) + with pytest.raises(SystemExit): + client.parse_args( + [ + "create-draft", + "cassandra", + "--base", + "main", + "--head", + "hermes/fix", + "--head-sha", + HEAD_SHA, + "--title", + "Repair", + "--body-file", + "/runtime-access/gitea-token", + ] + ) + + +def test_http_error_path_redacts_token(monkeypatch, capsys): + client = _load() + monkeypatch.setattr(client, "read_token", lambda: "do-not-leak") + + def fail(*_args, **_kwargs): + raise urllib.error.HTTPError( + "https://scm.bstein.dev/api/v1/repos/atlas/cassandra", + 403, + "forbidden", + {}, + io.BytesIO(b"Authorization: token do-not-leak"), + ) + + monkeypatch.setattr(client, "read", lambda *_a, **_k: fail()) + + assert client.main(["read", "/api/v1/repos/atlas/cassandra"]) == 1 + captured = capsys.readouterr() + assert "do-not-leak" not in captured.err + assert "HTTP 403" in captured.err + + +def test_flux_manifest_projects_runtime_vault_token_and_skill_only(): + client_source = CLIENT_PATH.read_text(encoding="utf-8") + assert "/runtime-access/gitea-token" in client_source + assert "GITEA_TOKEN" not in client_source + + deployment = yaml.safe_load( + (ROOT / "services/hermes/agent-deployment.yaml").read_text(encoding="utf-8") + ) + template = deployment["spec"]["template"] + annotations = template["metadata"]["annotations"] + assert annotations["vault.hashicorp.com/agent-inject-secret-gitea-token"] == ( + "kv/data/atlas/hermes/developer-gitea" + ) + runtime = next( + volume + for volume in template["spec"]["volumes"] + if volume["name"] == "runtime-access" + ) + assert runtime["emptyDir"]["medium"] == "Memory" + + expected = {"hermes", "terminal", "cli-lane-runner"} + mounted = { + container["name"] + for container in template["spec"]["containers"] + if any( + mount["name"] == "atlas-pr-skill" + and mount["mountPath"] + == "/opt/data/workspace/skills/manage-atlas-pull-requests" + and mount.get("readOnly") is True + for mount in container.get("volumeMounts", []) + ) + } + assert mounted == expected + + kustomization = yaml.safe_load( + (ROOT / "services/hermes/kustomization.yaml").read_text(encoding="utf-8") + ) + generator = next( + item + for item in kustomization["configMapGenerator"] + if item["name"] == "hermes-atlas-pr-skill" + ) + assert generator["files"] == [ + "SKILL.md=skills/manage-atlas-pull-requests/SKILL.md", + "openai.yaml=skills/manage-atlas-pull-requests/agents/openai.yaml", + ] From 2c1d97e551e5b61a3aa1e821e412ef4948bbcc8d Mon Sep 17 00:00:00 2001 From: jenkins Date: Sun, 16 Aug 2026 20:21:04 -0300 Subject: [PATCH 3/4] hermes: match canonical Forgejo PR URL --- services/hermes/scripts/gitea_api.py | 3 +-- testing/tests/test_hermes_gitea_pr_client.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/services/hermes/scripts/gitea_api.py b/services/hermes/scripts/gitea_api.py index 057a79a1..b4a98eca 100755 --- a/services/hermes/scripts/gitea_api.py +++ b/services/hermes/scripts/gitea_api.py @@ -337,13 +337,12 @@ def _require_create_response( raise PolicyError("Forgejo omitted a valid pull-request number") full_name = f"{ALLOWED_OWNER}/{repo}" expected_html = f"{CANONICAL_BASE_URL}/{full_name}/pulls/{number}" - expected_api = f"{CANONICAL_BASE_URL}/api/v1/repos/{full_name}/pulls/{number}" checks = ( document.get("state") == "open", document.get("draft") is True, document.get("merged") is False, document.get("html_url") == expected_html, - document.get("url") == expected_api, + document.get("url") == expected_html, document.get("title") == title, document.get("body") == body, _nested(document, "base", "ref") == base, diff --git a/testing/tests/test_hermes_gitea_pr_client.py b/testing/tests/test_hermes_gitea_pr_client.py index 91475bcf..447f3067 100644 --- a/testing/tests/test_hermes_gitea_pr_client.py +++ b/testing/tests/test_hermes_gitea_pr_client.py @@ -43,7 +43,7 @@ def _draft_response(**updates): "draft": True, "merged": False, "html_url": "https://scm.bstein.dev/atlas/cassandra/pulls/3", - "url": "https://scm.bstein.dev/api/v1/repos/atlas/cassandra/pulls/3", + "url": "https://scm.bstein.dev/atlas/cassandra/pulls/3", "title": "WIP: Focused fix", "body": "Review evidence", "base": {"ref": "main", "repo": {"full_name": "atlas/cassandra"}}, From 8c6e3acdacb223f68b1716f0a74bd543e18890e6 Mon Sep 17 00:00:00 2001 From: jenkins Date: Sun, 16 Aug 2026 20:41:14 -0300 Subject: [PATCH 4/4] hermes: isolate Atlas SCM write authority --- .../kustomization.yaml | 19 + .../hermes-observer-rbac/kustomization.yaml | 16 + .../hermes-scm-broker-code/kustomization.yaml | 19 + .../hermes-scm-broker/kustomization.yaml | 27 + .../hermes-scm-namespace/kustomization.yaml | 16 + .../applications/hermes/kustomization.yaml | 2 + .../applications/kustomization.yaml | 5 + scripts/tests/test_dashboards_render_atlas.py | 98 - ...test_dashboards_render_atlas_drilldowns.py | 127 ++ .../gitea/atlas-identity-bootstrap-job.yaml | 20 +- services/gitea/kustomization.yaml | 1 + .../scripts/gitea_atlas_identity_ensure.sh | 45 + .../scripts/gitea_branch_protection_check.py | 207 ++ .../kustomization.yaml | 5 + .../rolebindings.yaml | 204 ++ .../hermes-observer-rbac/kustomization.yaml | 5 + services/hermes-observer-rbac/rbac.yaml | 112 + services/hermes-scm-broker/deployment.yaml | 98 + services/hermes-scm-broker/kustomization.yaml | 11 + services/hermes-scm-broker/networkpolicy.yaml | 46 + services/hermes-scm-broker/service.yaml | 13 + .../hermes-scm-namespace/kustomization.yaml | 5 + services/hermes-scm-namespace/namespace.yaml | 14 + services/hermes/agent-configmap.yaml | 74 +- services/hermes/agent-deployment.yaml | 20 +- services/hermes/agent-kubeconfig.yaml | 4 +- services/hermes/agent-rbac.yaml | 16 - services/hermes/kustomization.yaml | 12 +- services/hermes/networkpolicy.yaml | 62 +- services/hermes/node-ssh-access.yaml | 77 +- services/hermes/scm-common/kustomization.yaml | 14 + .../{ => scm-common}/scripts/gitea_api.py | 284 ++- .../scm-common/scripts/gitea_api_policy.py | 490 +++++ .../hermes/scm-common/scripts/scm_broker.py | 496 +++++ .../scm-common/scripts/scm_broker_client.py | 95 + .../scm-common/scripts/scm_broker_io.py | 72 + .../scm-common/scripts/scm_broker_server.py | 118 ++ services/hermes/scripts/gitea_askpass.sh | 19 - services/hermes/scripts/hermes_coordinator.py | 32 +- services/hermes/scripts/node_account_audit.py | 178 ++ .../hermes/scripts/node_account_hardening.py | 496 +++++ services/hermes/scripts/node_account_io.py | 205 ++ .../hermes/scripts/stage_runtime_access.py | 19 +- .../manage-atlas-pull-requests/SKILL.md | 24 +- .../vault/hermes-auth-role-bootstrap-job.yaml | 2 +- .../vault/scripts/vault_k8s_auth_configure.sh | 4 +- testing/quality_contract.json | 57 + testing/quality_coverage.py | 61 +- testing/tests/test_hermes_chat_images.py | 282 +++ .../tests/test_hermes_chat_local_runtime.py | 172 ++ .../tests/test_hermes_chat_provider_auth.py | 476 +++++ testing/tests/test_hermes_chat_quality.py | 1812 +---------------- .../tests/test_hermes_chat_routing_runtime.py | 486 +++++ .../test_hermes_chat_session_continuity.py | 478 +++++ testing/tests/test_hermes_chat_support.py | 38 + testing/tests/test_hermes_cli_lanes.py | 1591 +-------------- testing/tests/test_hermes_cli_lanes_access.py | 378 ++++ .../test_hermes_cli_lanes_configuration.py | 444 ++++ testing/tests/test_hermes_cli_lanes_kanban.py | 392 ++++ .../tests/test_hermes_cli_lanes_support.py | 79 + .../tests/test_hermes_cli_lanes_toolchain.py | 386 ++++ testing/tests/test_hermes_coordinator.py | 212 +- .../test_hermes_coordinator_cassandra.py | 194 ++ .../tests/test_hermes_coordinator_coverage.py | 267 +++ .../tests/test_hermes_coordinator_support.py | 35 + .../test_hermes_gitea_branch_coverage.py | 183 ++ .../test_hermes_gitea_internal_coverage.py | 375 ++++ testing/tests/test_hermes_gitea_pr_client.py | 398 ++-- .../tests/test_hermes_gitea_pr_creation.py | 203 ++ testing/tests/test_hermes_gitea_pr_dlp.py | 317 +++ .../tests/test_hermes_gitea_pr_integration.py | 41 +- .../test_hermes_gitea_pr_postconditions.py | 199 ++ testing/tests/test_hermes_gitea_support.py | 75 + .../test_hermes_node_account_hardening.py | 252 +++ testing/tests/test_hermes_node_account_io.py | 222 ++ ...est_hermes_node_account_privilege_audit.py | 196 ++ .../tests/test_hermes_node_account_support.py | 90 + .../tests/test_hermes_node_acl_coverage.py | 206 ++ .../tests/test_hermes_node_audit_coverage.py | 221 ++ .../test_hermes_node_hardening_coverage.py | 211 ++ testing/tests/test_hermes_node_io_coverage.py | 189 ++ testing/tests/test_hermes_runtime_access.py | 24 +- .../test_hermes_runtime_stage_coverage.py | 137 ++ testing/tests/test_hermes_scm_broker.py | 146 ++ .../test_hermes_scm_broker_client_coverage.py | 157 ++ ...test_hermes_scm_broker_handler_coverage.py | 265 +++ ...est_hermes_scm_broker_internal_coverage.py | 263 +++ .../tests/test_hermes_scm_broker_policy.py | 321 +++ .../tests/test_hermes_scm_broker_streaming.py | 237 +++ .../tests/test_hermes_scm_broker_support.py | 55 + .../tests/test_hermes_scm_server_coverage.py | 180 ++ .../tests/test_quality_coverage_helpers.py | 87 +- 92 files changed, 12731 insertions(+), 4287 deletions(-) create mode 100644 clusters/atlas/flux-system/applications/hermes-observer-bindings/kustomization.yaml create mode 100644 clusters/atlas/flux-system/applications/hermes-observer-rbac/kustomization.yaml create mode 100644 clusters/atlas/flux-system/applications/hermes-scm-broker-code/kustomization.yaml create mode 100644 clusters/atlas/flux-system/applications/hermes-scm-broker/kustomization.yaml create mode 100644 clusters/atlas/flux-system/applications/hermes-scm-namespace/kustomization.yaml create mode 100644 scripts/tests/test_dashboards_render_atlas_drilldowns.py create mode 100644 services/gitea/scripts/gitea_branch_protection_check.py create mode 100644 services/hermes-observer-bindings/kustomization.yaml create mode 100644 services/hermes-observer-bindings/rolebindings.yaml create mode 100644 services/hermes-observer-rbac/kustomization.yaml create mode 100644 services/hermes-observer-rbac/rbac.yaml create mode 100644 services/hermes-scm-broker/deployment.yaml create mode 100644 services/hermes-scm-broker/kustomization.yaml create mode 100644 services/hermes-scm-broker/networkpolicy.yaml create mode 100644 services/hermes-scm-broker/service.yaml create mode 100644 services/hermes-scm-namespace/kustomization.yaml create mode 100644 services/hermes-scm-namespace/namespace.yaml delete mode 100644 services/hermes/agent-rbac.yaml create mode 100644 services/hermes/scm-common/kustomization.yaml rename services/hermes/{ => scm-common}/scripts/gitea_api.py (65%) mode change 100755 => 100644 create mode 100644 services/hermes/scm-common/scripts/gitea_api_policy.py create mode 100644 services/hermes/scm-common/scripts/scm_broker.py create mode 100644 services/hermes/scm-common/scripts/scm_broker_client.py create mode 100644 services/hermes/scm-common/scripts/scm_broker_io.py create mode 100644 services/hermes/scm-common/scripts/scm_broker_server.py delete mode 100755 services/hermes/scripts/gitea_askpass.sh create mode 100644 services/hermes/scripts/node_account_audit.py create mode 100644 services/hermes/scripts/node_account_hardening.py create mode 100644 services/hermes/scripts/node_account_io.py create mode 100644 testing/tests/test_hermes_chat_images.py create mode 100644 testing/tests/test_hermes_chat_local_runtime.py create mode 100644 testing/tests/test_hermes_chat_provider_auth.py create mode 100644 testing/tests/test_hermes_chat_routing_runtime.py create mode 100644 testing/tests/test_hermes_chat_session_continuity.py create mode 100644 testing/tests/test_hermes_chat_support.py create mode 100644 testing/tests/test_hermes_cli_lanes_access.py create mode 100644 testing/tests/test_hermes_cli_lanes_configuration.py create mode 100644 testing/tests/test_hermes_cli_lanes_kanban.py create mode 100644 testing/tests/test_hermes_cli_lanes_support.py create mode 100644 testing/tests/test_hermes_cli_lanes_toolchain.py create mode 100644 testing/tests/test_hermes_coordinator_cassandra.py create mode 100644 testing/tests/test_hermes_coordinator_coverage.py create mode 100644 testing/tests/test_hermes_coordinator_support.py create mode 100644 testing/tests/test_hermes_gitea_branch_coverage.py create mode 100644 testing/tests/test_hermes_gitea_internal_coverage.py create mode 100644 testing/tests/test_hermes_gitea_pr_creation.py create mode 100644 testing/tests/test_hermes_gitea_pr_dlp.py create mode 100644 testing/tests/test_hermes_gitea_pr_postconditions.py create mode 100644 testing/tests/test_hermes_gitea_support.py create mode 100644 testing/tests/test_hermes_node_account_hardening.py create mode 100644 testing/tests/test_hermes_node_account_io.py create mode 100644 testing/tests/test_hermes_node_account_privilege_audit.py create mode 100644 testing/tests/test_hermes_node_account_support.py create mode 100644 testing/tests/test_hermes_node_acl_coverage.py create mode 100644 testing/tests/test_hermes_node_audit_coverage.py create mode 100644 testing/tests/test_hermes_node_hardening_coverage.py create mode 100644 testing/tests/test_hermes_node_io_coverage.py create mode 100644 testing/tests/test_hermes_runtime_stage_coverage.py create mode 100644 testing/tests/test_hermes_scm_broker.py create mode 100644 testing/tests/test_hermes_scm_broker_client_coverage.py create mode 100644 testing/tests/test_hermes_scm_broker_handler_coverage.py create mode 100644 testing/tests/test_hermes_scm_broker_internal_coverage.py create mode 100644 testing/tests/test_hermes_scm_broker_policy.py create mode 100644 testing/tests/test_hermes_scm_broker_streaming.py create mode 100644 testing/tests/test_hermes_scm_broker_support.py create mode 100644 testing/tests/test_hermes_scm_server_coverage.py diff --git a/clusters/atlas/flux-system/applications/hermes-observer-bindings/kustomization.yaml b/clusters/atlas/flux-system/applications/hermes-observer-bindings/kustomization.yaml new file mode 100644 index 00000000..b64913d5 --- /dev/null +++ b/clusters/atlas/flux-system/applications/hermes-observer-bindings/kustomization.yaml @@ -0,0 +1,19 @@ +# clusters/atlas/flux-system/applications/hermes-observer-bindings/kustomization.yaml +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: hermes-observer-bindings + namespace: flux-system +spec: + interval: 10m + path: ./services/hermes-observer-bindings + prune: true + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + wait: true + timeout: 5m + dependsOn: + - name: hermes + - name: hermes-observer-rbac diff --git a/clusters/atlas/flux-system/applications/hermes-observer-rbac/kustomization.yaml b/clusters/atlas/flux-system/applications/hermes-observer-rbac/kustomization.yaml new file mode 100644 index 00000000..c2749739 --- /dev/null +++ b/clusters/atlas/flux-system/applications/hermes-observer-rbac/kustomization.yaml @@ -0,0 +1,16 @@ +# clusters/atlas/flux-system/applications/hermes-observer-rbac/kustomization.yaml +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: hermes-observer-rbac + namespace: flux-system +spec: + interval: 10m + path: ./services/hermes-observer-rbac + prune: true + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + wait: true + timeout: 5m diff --git a/clusters/atlas/flux-system/applications/hermes-scm-broker-code/kustomization.yaml b/clusters/atlas/flux-system/applications/hermes-scm-broker-code/kustomization.yaml new file mode 100644 index 00000000..20ae82e2 --- /dev/null +++ b/clusters/atlas/flux-system/applications/hermes-scm-broker-code/kustomization.yaml @@ -0,0 +1,19 @@ +# clusters/atlas/flux-system/applications/hermes-scm-broker-code/kustomization.yaml +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: hermes-scm-broker-code + namespace: flux-system +spec: + interval: 10m + path: ./services/hermes/scm-common + targetNamespace: hermes-scm + prune: true + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + wait: true + timeout: 5m + dependsOn: + - name: hermes-scm-namespace diff --git a/clusters/atlas/flux-system/applications/hermes-scm-broker/kustomization.yaml b/clusters/atlas/flux-system/applications/hermes-scm-broker/kustomization.yaml new file mode 100644 index 00000000..70b3989e --- /dev/null +++ b/clusters/atlas/flux-system/applications/hermes-scm-broker/kustomization.yaml @@ -0,0 +1,27 @@ +# clusters/atlas/flux-system/applications/hermes-scm-broker/kustomization.yaml +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: hermes-scm-broker + namespace: flux-system +spec: + interval: 10m + path: ./services/hermes-scm-broker + targetNamespace: hermes-scm + prune: true + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + wait: true + timeout: 10m + healthChecks: + - apiVersion: apps/v1 + kind: Deployment + name: hermes-scm-broker + namespace: hermes-scm + dependsOn: + - name: vault + - name: gitea + - name: hermes-scm-namespace + - name: hermes-scm-broker-code diff --git a/clusters/atlas/flux-system/applications/hermes-scm-namespace/kustomization.yaml b/clusters/atlas/flux-system/applications/hermes-scm-namespace/kustomization.yaml new file mode 100644 index 00000000..9f066709 --- /dev/null +++ b/clusters/atlas/flux-system/applications/hermes-scm-namespace/kustomization.yaml @@ -0,0 +1,16 @@ +# clusters/atlas/flux-system/applications/hermes-scm-namespace/kustomization.yaml +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: hermes-scm-namespace + namespace: flux-system +spec: + interval: 10m + path: ./services/hermes-scm-namespace + prune: true + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + wait: true + timeout: 5m diff --git a/clusters/atlas/flux-system/applications/hermes/kustomization.yaml b/clusters/atlas/flux-system/applications/hermes/kustomization.yaml index 3949282b..df7a5560 100644 --- a/clusters/atlas/flux-system/applications/hermes/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/hermes/kustomization.yaml @@ -60,3 +60,5 @@ spec: - name: keycloak - name: longhorn - name: vault + - name: hermes-scm-broker + - name: hermes-observer-rbac diff --git a/clusters/atlas/flux-system/applications/kustomization.yaml b/clusters/atlas/flux-system/applications/kustomization.yaml index eb8bc86a..39a859d0 100644 --- a/clusters/atlas/flux-system/applications/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/kustomization.yaml @@ -27,7 +27,12 @@ resources: - jenkins/kustomization.yaml - ai-llm/kustomization.yaml - openclaw/kustomization.yaml + - hermes-scm-namespace/kustomization.yaml + - hermes-scm-broker-code/kustomization.yaml - hermes/kustomization.yaml + - hermes-observer-rbac/kustomization.yaml + - hermes-observer-bindings/kustomization.yaml + - hermes-scm-broker/kustomization.yaml - hermes-chat/kustomization.yaml - hermes-triage-demo/kustomization.yaml - game-stream/kustomization.yaml diff --git a/scripts/tests/test_dashboards_render_atlas.py b/scripts/tests/test_dashboards_render_atlas.py index bba16cdf..a522f849 100644 --- a/scripts/tests/test_dashboards_render_atlas.py +++ b/scripts/tests/test_dashboards_render_atlas.py @@ -416,101 +416,3 @@ def test_jobs_dashboard_bar_gauges_use_solid_threshold_colors(): assert {"color": "dark-yellow", "value": 93} in threshold_steps assert {"color": "dark-blue", "value": 100} in threshold_steps - - -def test_jobs_dashboard_collapses_heavy_drilldowns_for_light_first_paint(): - mod = load_module() - dashboard = mod.build_jobs_dashboard() - panels = dashboard["panels"] - rows = [panel for panel in panels if panel["type"] == "row"] - visible_query_panels = [panel for panel in panels if panel["type"] != "row"] - nested_panels_by_title = { - child["title"]: child - for row in rows - for child in row.get("panels", []) - } - - assert len(panels) == 18 - assert len(visible_query_panels) == 12 - assert sum(len(panel.get("targets", [])) for panel in visible_query_panels) == 12 - assert all( - panel["title"] != "Coverage Gap to 95% by Suite" - for panel in visible_query_panels - ) - assert [row["title"] for row in rows] == [ - "CI Runs And Test Result History", - "Check Failure Rates By Suite", - "Check Healthy Rates By Suite", - "Test Drilldowns And Problem Tests", - "Telemetry Completeness And Branches", - "SonarQube Project Health", - ] - assert all(row["collapsed"] for row in rows) - - assert "Coverage Failure Rate" in nested_panels_by_title - assert "Supply Chain Healthy Rate" in nested_panels_by_title - assert "Test Category Health History" in nested_panels_by_title - assert "Selected Test Pass Rate History" in nested_panels_by_title - assert "Coverage Metrics Present by Suite" in nested_panels_by_title - assert "SonarQube API Up" in nested_panels_by_title - - failure_rate_panel = nested_panels_by_title["Coverage Failure Rate"] - assert failure_rate_panel["type"] == "state-timeline" - assert failure_rate_panel["fieldConfig"]["defaults"]["unit"] == "percent" - assert failure_rate_panel["fieldConfig"]["defaults"]["max"] == 100 - assert failure_rate_panel["fieldConfig"]["defaults"]["thresholds"]["steps"][0]["color"] == "dark-blue" - assert "increase(" not in failure_rate_panel["targets"][0]["expr"] - assert "platform_quality:check_failed_flag:present_1h" in failure_rate_panel["targets"][0]["expr"] - assert "platform_quality:check_seen_flag:present_1h" in failure_rate_panel["targets"][0]["expr"] - assert "platform_quality:check_status:present_1h" not in failure_rate_panel["targets"][0]["expr"] - assert '.*_quality_gate_checks_total' not in failure_rate_panel["targets"][0]["expr"] - assert "last_over_time" not in failure_rate_panel["targets"][0]["expr"] - assert 'label_replace' not in failure_rate_panel["targets"][0]["expr"] - assert "0 *" in failure_rate_panel["targets"][0]["expr"] - assert "and on(suite)" not in failure_rate_panel["targets"][0]["expr"] - - pass_rate_panel = nested_panels_by_title["Selected Test Pass Rate History"] - assert pass_rate_panel["type"] == "state-timeline" - assert "platform_quality:test_case_pass_rate:percent_1h" in pass_rate_panel["targets"][0]["expr"] - assert "platform_quality_gate_test_case_result" not in pass_rate_panel["targets"][0]["expr"] - - pass_fail_panel = nested_panels_by_title["Selected Test Pass/Fail History"] - assert pass_fail_panel["fieldConfig"]["defaults"]["custom"]["drawStyle"] == "bars" - assert all( - "platform_quality:test_case_status:count_1h" in target["expr"] - for target in pass_fail_panel["targets"] - ) - - problematic_panel = nested_panels_by_title["Problematic Tests Over Time (Top failures)"] - assert problematic_panel["type"] == "state-timeline" - assert problematic_panel["gridPos"]["w"] == 24 - assert 'test!=""' in problematic_panel["targets"][0]["expr"] - assert "vector(0)" not in problematic_panel["targets"][0]["expr"] - assert problematic_panel["fieldConfig"]["defaults"]["thresholds"]["steps"] == [ - {"color": "dark-blue", "value": None}, - {"color": "dark-green", "value": 2}, - {"color": "dark-yellow", "value": 3}, - {"color": "dark-orange", "value": 5}, - {"color": "dark-red", "value": 8}, - ] - assert "rolling 24h failure count" in problematic_panel["description"] - assert "at least two recent failures" in problematic_panel["description"] - - sonar_mix_panel = nested_panels_by_title["Sonar Gate Status Mix (Selected)"] - sonar_health_panel = nested_panels_by_title["Sonar Gate Health by Project"] - assert sonar_mix_panel["gridPos"]["w"] == 4 - assert sonar_health_panel["gridPos"]["w"] == 8 - assert sonar_health_panel["type"] == "state-timeline" - assert "platform_quality:sonar_gate_health_percent:latest_1h" in sonar_health_panel["targets"][0]["expr"] - assert "sonarqube_project_quality_gate_pass" not in sonar_health_panel["targets"][0]["expr"] - - branch_panel = nested_panels_by_title["Primary Branch Clean by Suite (7d)"] - recent_branch_panel = nested_panels_by_title["Recent Branch Evidence by Suite (7d)"] - assert branch_panel["gridPos"]["x"] == 12 - assert recent_branch_panel["gridPos"]["x"] == 18 - assert "[7d:1h]" in recent_branch_panel["targets"][0]["expr"] - assert "[7d:1h]" in branch_panel["targets"][0]["expr"] - assert branch_panel["fieldConfig"]["defaults"]["unit"] == "percent" - assert "unless on(suite)" in branch_panel["targets"][0]["expr"] - assert "> bool 0" in branch_panel["targets"][0]["expr"] - assert branch_panel["targets"][0]["expr"].startswith("sort(") diff --git a/scripts/tests/test_dashboards_render_atlas_drilldowns.py b/scripts/tests/test_dashboards_render_atlas_drilldowns.py new file mode 100644 index 00000000..08871b71 --- /dev/null +++ b/scripts/tests/test_dashboards_render_atlas_drilldowns.py @@ -0,0 +1,127 @@ +"""Detailed Atlas Jobs dashboard collapse and drilldown contracts.""" + +from scripts.tests.test_dashboards_render_atlas import load_module + + +def test_jobs_dashboard_collapses_heavy_drilldowns_for_light_first_paint(): + mod = load_module() + dashboard = mod.build_jobs_dashboard() + panels = dashboard["panels"] + rows = [panel for panel in panels if panel["type"] == "row"] + visible_query_panels = [panel for panel in panels if panel["type"] != "row"] + nested_panels_by_title = { + child["title"]: child for row in rows for child in row.get("panels", []) + } + + assert len(panels) == 18 + assert len(visible_query_panels) == 12 + assert sum(len(panel.get("targets", [])) for panel in visible_query_panels) == 12 + assert all( + panel["title"] != "Coverage Gap to 95% by Suite" + for panel in visible_query_panels + ) + assert [row["title"] for row in rows] == [ + "CI Runs And Test Result History", + "Check Failure Rates By Suite", + "Check Healthy Rates By Suite", + "Test Drilldowns And Problem Tests", + "Telemetry Completeness And Branches", + "SonarQube Project Health", + ] + assert all(row["collapsed"] for row in rows) + + assert "Coverage Failure Rate" in nested_panels_by_title + assert "Supply Chain Healthy Rate" in nested_panels_by_title + assert "Test Category Health History" in nested_panels_by_title + assert "Selected Test Pass Rate History" in nested_panels_by_title + assert "Coverage Metrics Present by Suite" in nested_panels_by_title + assert "SonarQube API Up" in nested_panels_by_title + + failure_rate_panel = nested_panels_by_title["Coverage Failure Rate"] + assert failure_rate_panel["type"] == "state-timeline" + assert failure_rate_panel["fieldConfig"]["defaults"]["unit"] == "percent" + assert failure_rate_panel["fieldConfig"]["defaults"]["max"] == 100 + assert ( + failure_rate_panel["fieldConfig"]["defaults"]["thresholds"]["steps"][0]["color"] + == "dark-blue" + ) + assert "increase(" not in failure_rate_panel["targets"][0]["expr"] + assert ( + "platform_quality:check_failed_flag:present_1h" + in failure_rate_panel["targets"][0]["expr"] + ) + assert ( + "platform_quality:check_seen_flag:present_1h" + in failure_rate_panel["targets"][0]["expr"] + ) + assert ( + "platform_quality:check_status:present_1h" + not in failure_rate_panel["targets"][0]["expr"] + ) + assert ( + ".*_quality_gate_checks_total" not in failure_rate_panel["targets"][0]["expr"] + ) + assert "last_over_time" not in failure_rate_panel["targets"][0]["expr"] + assert "label_replace" not in failure_rate_panel["targets"][0]["expr"] + assert "0 *" in failure_rate_panel["targets"][0]["expr"] + assert "and on(suite)" not in failure_rate_panel["targets"][0]["expr"] + + pass_rate_panel = nested_panels_by_title["Selected Test Pass Rate History"] + assert pass_rate_panel["type"] == "state-timeline" + assert ( + "platform_quality:test_case_pass_rate:percent_1h" + in pass_rate_panel["targets"][0]["expr"] + ) + assert ( + "platform_quality_gate_test_case_result" + not in pass_rate_panel["targets"][0]["expr"] + ) + + pass_fail_panel = nested_panels_by_title["Selected Test Pass/Fail History"] + assert pass_fail_panel["fieldConfig"]["defaults"]["custom"]["drawStyle"] == "bars" + assert all( + "platform_quality:test_case_status:count_1h" in target["expr"] + for target in pass_fail_panel["targets"] + ) + + problematic_panel = nested_panels_by_title[ + "Problematic Tests Over Time (Top failures)" + ] + assert problematic_panel["type"] == "state-timeline" + assert problematic_panel["gridPos"]["w"] == 24 + assert 'test!=""' in problematic_panel["targets"][0]["expr"] + assert "vector(0)" not in problematic_panel["targets"][0]["expr"] + assert problematic_panel["fieldConfig"]["defaults"]["thresholds"]["steps"] == [ + {"color": "dark-blue", "value": None}, + {"color": "dark-green", "value": 2}, + {"color": "dark-yellow", "value": 3}, + {"color": "dark-orange", "value": 5}, + {"color": "dark-red", "value": 8}, + ] + assert "rolling 24h failure count" in problematic_panel["description"] + assert "at least two recent failures" in problematic_panel["description"] + + sonar_mix_panel = nested_panels_by_title["Sonar Gate Status Mix (Selected)"] + sonar_health_panel = nested_panels_by_title["Sonar Gate Health by Project"] + assert sonar_mix_panel["gridPos"]["w"] == 4 + assert sonar_health_panel["gridPos"]["w"] == 8 + assert sonar_health_panel["type"] == "state-timeline" + assert ( + "platform_quality:sonar_gate_health_percent:latest_1h" + in sonar_health_panel["targets"][0]["expr"] + ) + assert ( + "sonarqube_project_quality_gate_pass" + not in sonar_health_panel["targets"][0]["expr"] + ) + + branch_panel = nested_panels_by_title["Primary Branch Clean by Suite (7d)"] + recent_branch_panel = nested_panels_by_title["Recent Branch Evidence by Suite (7d)"] + assert branch_panel["gridPos"]["x"] == 12 + assert recent_branch_panel["gridPos"]["x"] == 18 + assert "[7d:1h]" in recent_branch_panel["targets"][0]["expr"] + assert "[7d:1h]" in branch_panel["targets"][0]["expr"] + assert branch_panel["fieldConfig"]["defaults"]["unit"] == "percent" + assert "unless on(suite)" in branch_panel["targets"][0]["expr"] + assert "> bool 0" in branch_panel["targets"][0]["expr"] + assert branch_panel["targets"][0]["expr"].startswith("sort(") diff --git a/services/gitea/atlas-identity-bootstrap-job.yaml b/services/gitea/atlas-identity-bootstrap-job.yaml index c7e2071f..bacecd01 100644 --- a/services/gitea/atlas-identity-bootstrap-job.yaml +++ b/services/gitea/atlas-identity-bootstrap-job.yaml @@ -2,7 +2,7 @@ apiVersion: batch/v1 kind: Job metadata: - name: gitea-atlas-identity-bootstrap-3 + name: gitea-atlas-identity-bootstrap-5 namespace: gitea labels: app.kubernetes.io/name: gitea @@ -34,6 +34,20 @@ spec: nodeSelector: node-role.kubernetes.io/worker: "true" hardware: rpi5 + initContainers: + - name: python-runtime + image: python@sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df + command: [/bin/sh, -ec] + args: ["cp -R /usr/local/. /python/"] + securityContext: + allowPrivilegeEscalation: false + capabilities: {drop: [ALL]} + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + volumeMounts: + - {name: python-runtime, mountPath: /python} containers: - name: bootstrap image: gitea/gitea:1.23 @@ -80,6 +94,7 @@ spec: - name: bootstrap mountPath: /opt/bootstrap readOnly: true + - {name: python-runtime, mountPath: /opt/python, readOnly: true} volumes: - name: data persistentVolumeClaim: @@ -88,3 +103,6 @@ spec: configMap: name: gitea-atlas-identity-bootstrap defaultMode: 0555 + - name: python-runtime + emptyDir: + sizeLimit: 128Mi diff --git a/services/gitea/kustomization.yaml b/services/gitea/kustomization.yaml index 8c473a38..6511418a 100644 --- a/services/gitea/kustomization.yaml +++ b/services/gitea/kustomization.yaml @@ -15,6 +15,7 @@ configMapGenerator: - name: gitea-atlas-identity-bootstrap files: - gitea_atlas_identity_ensure.sh=scripts/gitea_atlas_identity_ensure.sh + - gitea_branch_protection_check.py=scripts/gitea_branch_protection_check.py generatorOptions: disableNameSuffixHash: true diff --git a/services/gitea/scripts/gitea_atlas_identity_ensure.sh b/services/gitea/scripts/gitea_atlas_identity_ensure.sh index f98f08f5..6d5dbd90 100644 --- a/services/gitea/scripts/gitea_atlas_identity_ensure.sh +++ b/services/gitea/scripts/gitea_atlas_identity_ensure.sh @@ -19,6 +19,7 @@ source_owner="${GITEA_SOURCE_OWNER:-bstein}" managed_repositories="${GITEA_MANAGED_REPOSITORIES:-hermes-code-demo cassandra soteria pegasus metis ananke ariadne typhon titan-iac}" contributor_team=Contributors contributor_units='["repo.actions","repo.packages","repo.code","repo.issues","repo.wiki","repo.pulls","repo.releases","repo.projects"]' +protected_reviewer=bstein die() { echo "Gitea Atlas identity bootstrap failed: $*" >&2 @@ -272,6 +273,46 @@ transfer_managed_repositories() { trap - EXIT HUP INT TERM } +ensure_default_branch_protection() { + repository=$1 + repo_file=$(mktemp) + protections_file=$(mktemp) + response_file=$(mktemp) + + status=$(api_request GET "/repos/${organization}/${repository}" '' "${repo_file}") + expect_status "${status}" 200 "read ${repository} metadata for branch protection" + default_branch=$(sed -n 's/.*"default_branch":"\([^"]*\)".*/\1/p' "${repo_file}") + case "${default_branch}" in + main|master) ;; + *) die "${repository} has unsupported default branch ${default_branch:-missing}" ;; + esac + + status=$(api_request GET "/repos/${organization}/${repository}/branch_protections" '' "${protections_file}") + expect_status "${status}" 200 "list ${repository} branch protections" + if ! protection_state=$( + /opt/python/bin/python3 /opt/bootstrap/gitea_branch_protection_check.py \ + "${protections_file}" "${default_branch}" "${protected_reviewer}" + ); then + die "${repository} ${default_branch} protection differs from the human-review policy" + fi + + if [ "${protection_state}" = ABSENT ]; then + payload="{\"rule_name\":\"${default_branch}\",\"enable_push\":true,\"enable_push_whitelist\":true,\"push_whitelist_usernames\":[\"${protected_reviewer}\"],\"push_whitelist_teams\":[],\"push_whitelist_deploy_keys\":false,\"enable_force_push\":false,\"enable_force_push_allowlist\":false,\"force_push_allowlist_usernames\":[],\"force_push_allowlist_teams\":[],\"force_push_allowlist_deploy_keys\":false,\"enable_merge_whitelist\":true,\"merge_whitelist_usernames\":[\"${protected_reviewer}\"],\"merge_whitelist_teams\":[],\"enable_approvals_whitelist\":true,\"approvals_whitelist_username\":[\"${protected_reviewer}\"],\"approvals_whitelist_teams\":[],\"required_approvals\":1,\"block_on_rejected_reviews\":true,\"block_on_outdated_branch\":true,\"dismiss_stale_approvals\":true,\"block_admin_merge_override\":true}" + status=$(api_request POST "/repos/${organization}/${repository}/branch_protections" "${payload}" "${response_file}") + expect_status "${status}" 201 "protect ${repository} ${default_branch}" + status=$(api_request GET "/repos/${organization}/${repository}/branch_protections" '' "${protections_file}") + expect_status "${status}" 200 "read back ${repository} branch protections" + protection_state=$( + /opt/python/bin/python3 /opt/bootstrap/gitea_branch_protection_check.py \ + "${protections_file}" "${default_branch}" "${protected_reviewer}" + ) || die "${repository} ${default_branch} protection differs from the human-review policy" + fi + [ "${protection_state}" = PRESENT ] || \ + die "${repository} ${default_branch} protection did not become effective" + + rm -f "${repo_file}" "${protections_file}" "${response_file}" +} + wait_for_gitea ensure_user "${reconciler_user}" "${reconciler_email}" true ensure_user "${hermes_user}" "${hermes_email}" false @@ -318,6 +359,10 @@ expect_status "${status}" 204 "add Hermes to Atlas contributors" transfer_managed_repositories +for repository in ${managed_repositories}; do + ensure_default_branch_protection "${repository}" +done + vault_write gitea/atlas-reconciler "{\"data\":{\"username\":\"${reconciler_user}\",\"token\":\"${reconciler_token}\",\"base_url\":\"${public_url}\"}}" vault_write hermes/developer-gitea "{\"data\":{\"username\":\"${hermes_user}\",\"email\":\"${hermes_email}\",\"token\":\"${hermes_token}\",\"base_url\":\"${public_url}\"}}" diff --git a/services/gitea/scripts/gitea_branch_protection_check.py b/services/gitea/scripts/gitea_branch_protection_check.py new file mode 100644 index 00000000..e3f112c9 --- /dev/null +++ b/services/gitea/scripts/gitea_branch_protection_check.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python3 +"""Validate the first effective Gitea protection for an Atlas primary branch.""" + +from __future__ import annotations + +import argparse +import datetime +import json +import os +import re +import stat +from pathlib import Path + +MAX_INPUT = 1024 * 1024 +MAX_RULES = 100 +MAX_RULE_NAME = 255 +SPECIAL = frozenset("*?\\[]{}") + + +class PolicyError(RuntimeError): + """The protection response is ambiguous or violates human review policy.""" + + +def _read_bounded(path: Path) -> bytes: + descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + try: + metadata = os.fstat(descriptor) + if not stat.S_ISREG(metadata.st_mode) or metadata.st_size > MAX_INPUT: + raise PolicyError("branch protection response exceeds the safe limit") + value = os.read(descriptor, MAX_INPUT + 1) + finally: + os.close(descriptor) + if len(value) != metadata.st_size: + raise PolicyError("branch protection response changed while reading") + return value + + +def _glob_regex(pattern: str, position: int = 0, terminators: str = "") -> tuple[str, int]: + """Compile the gobwas/glob syntax Gitea v1.23.8 uses.""" + pieces: list[str] = [] + while position < len(pattern): + character = pattern[position] + if character in terminators: + break + if character == "\\": + position += 1 + if position >= len(pattern): + raise ValueError("trailing escape") + pieces.append(re.escape(pattern[position])) + elif character == "*": + if position + 1 < len(pattern) and pattern[position + 1] == "*": + pieces.append(".*") + position += 1 + else: + pieces.append("[^/]*") + elif character == "?": + pieces.append("[^/]") + elif character == "[": + end = pattern.find("]", position + 1) + if end < 0: + raise ValueError("unterminated range") + value = pattern[position + 1 : end] + if not value: + raise ValueError("empty range") + negate = value.startswith("!") + value = value[1:] if negate else value + if not value or "[" in value or "\\" in value: + raise ValueError("invalid range") + if "-" in value: + if value.count("-") != 1 or value.startswith("-") or value.endswith("-"): + raise ValueError("invalid range") + low, high = value.split("-") + if len(low) != 1 or len(high) != 1 or ord(high) < ord(low): + raise ValueError("invalid range") + content = re.escape(low) + "-" + re.escape(high) + else: + content = re.escape(value) + pieces.append("[" + ("^" if negate else "") + content + "]") + position = end + elif character == "{": + alternatives: list[str] = [] + position += 1 + while True: + alternative, position = _glob_regex(pattern, position, ",}") + alternatives.append(alternative) + if position >= len(pattern): + raise ValueError("unterminated alternatives") + if pattern[position] == "}": + break + position += 1 + if len(alternatives) < 2 or any(not item for item in alternatives): + raise ValueError("invalid alternatives") + pieces.append("(?:" + "|".join(alternatives) + ")") + elif character in "]}": + # A closing delimiter outside its grammar context is literal in + # gobwas/glob's lexer. + pieces.append(re.escape(character)) + else: + pieces.append(re.escape(character)) + position += 1 + return "".join(pieces), position + + +def _is_plain(pattern: str) -> bool: + return not any(character in SPECIAL for character in pattern) + + +def _matches(pattern: str, branch: str) -> bool: + if not pattern.isascii() or not 1 <= len(pattern) <= MAX_RULE_NAME: + raise PolicyError("branch protection rule name is invalid") + if _is_plain(pattern): + return pattern.casefold() == branch.casefold() + try: + expression, position = _glob_regex(pattern) + if position != len(pattern): + raise ValueError("incomplete glob") + except (re.error, ValueError): + # Gitea quotes an invalid special pattern and matches it literally. + # A literal containing a special byte cannot equal main or master. + return False + return re.fullmatch(expression, branch) is not None + + +def _created(value: object) -> datetime.datetime: + if not isinstance(value, str) or len(value) > 64: + raise PolicyError("branch protection creation time is invalid") + try: + result = datetime.datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as exc: + raise PolicyError("branch protection creation time is invalid") from exc + if result.tzinfo is None: + raise PolicyError("branch protection creation time is invalid") + return result + + +def _required(reviewer: str) -> dict[str, object]: + return { + "enable_push": True, + "enable_push_whitelist": True, + "push_whitelist_usernames": [reviewer], + "push_whitelist_deploy_keys": False, + "enable_force_push": False, + "enable_merge_whitelist": True, + "merge_whitelist_usernames": [reviewer], + "enable_approvals_whitelist": True, + "approvals_whitelist_username": [reviewer], + "required_approvals": 1, + "block_on_rejected_reviews": True, + "block_on_outdated_branch": True, + "dismiss_stale_approvals": True, + "block_admin_merge_override": True, + } + + +def evaluate(value: bytes, branch: str, reviewer: str) -> str: + """Return PRESENT/ABSENT or fail on ambiguous/effective policy drift.""" + if len(value) > MAX_INPUT or branch not in {"main", "master"}: + raise PolicyError("branch protection input is invalid") + data = json.loads(value) + if not isinstance(data, list) or len(data) > MAX_RULES: + raise PolicyError("branch protection response is invalid") + ordered: list[tuple[int, bool, datetime.datetime, int, dict[str, object]]] = [] + for index, raw in enumerate(data): + if not isinstance(raw, dict): + raise PolicyError("branch protection entry is invalid") + priority = raw.get("priority") + rule_name = raw.get("rule_name") + if ( + not isinstance(priority, int) + or isinstance(priority, bool) + or not 1 <= priority <= 1_000_000 + or not isinstance(rule_name, str) + ): + raise PolicyError("branch protection priority is ambiguous") + created = _created(raw.get("created_at")) + if _matches(rule_name, branch): + # Gitea v1.23.8 sorts by priority, then puts plain names before + # globs, then uses creation time. The API exposes every component. + ordered.append((priority, not _is_plain(rule_name), created, index, raw)) + if not ordered: + return "ABSENT" + effective = min(ordered)[4] + for key, expected in _required(reviewer).items(): + if effective.get(key) != expected: + raise PolicyError( + f"effective {branch} protection differs from human-review policy" + ) + return "PRESENT" + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("response", type=Path) + parser.add_argument("branch", choices=("main", "master")) + parser.add_argument("reviewer") + args = parser.parse_args() + try: + value = _read_bounded(args.response) + print(evaluate(value, args.branch, args.reviewer)) + except (OSError, json.JSONDecodeError, PolicyError) as exc: + print(f"branch protection check failed: {exc}", file=__import__("sys").stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/hermes-observer-bindings/kustomization.yaml b/services/hermes-observer-bindings/kustomization.yaml new file mode 100644 index 00000000..4dbf36b5 --- /dev/null +++ b/services/hermes-observer-bindings/kustomization.yaml @@ -0,0 +1,5 @@ +# services/hermes-observer-bindings/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - rolebindings.yaml diff --git a/services/hermes-observer-bindings/rolebindings.yaml b/services/hermes-observer-bindings/rolebindings.yaml new file mode 100644 index 00000000..403cd39a --- /dev/null +++ b/services/hermes-observer-bindings/rolebindings.yaml @@ -0,0 +1,204 @@ +# services/hermes-observer-bindings/rolebindings.yaml +apiVersion: v1 +kind: List +items: + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: ai} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: bstein-dev-home} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: cassandra} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: cert-manager} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: climate} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: comms} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: crypto} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: default} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: finance} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: flux-system} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: game-stream} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: gitea} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: harbor} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: health} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: hermes} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: hermes-chat} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: hermes-triage-demo} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: jellyfin} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: jenkins} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: kube-node-lease} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: kube-public} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: kube-system} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: logging} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: longhorn-system} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: mailu-mailserver} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: maintenance} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: metallb-system} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: monitoring} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: nextcloud} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: openclaw} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: outline} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: planka} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: postgres} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: quality} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: sso} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: sui-metrics} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: traefik} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: vault} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: vaultwarden} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: veles} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} diff --git a/services/hermes-observer-rbac/kustomization.yaml b/services/hermes-observer-rbac/kustomization.yaml new file mode 100644 index 00000000..cff6f4b9 --- /dev/null +++ b/services/hermes-observer-rbac/kustomization.yaml @@ -0,0 +1,5 @@ +# services/hermes-observer-rbac/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - rbac.yaml diff --git a/services/hermes-observer-rbac/rbac.yaml b/services/hermes-observer-rbac/rbac.yaml new file mode 100644 index 00000000..737ac4dd --- /dev/null +++ b/services/hermes-observer-rbac/rbac.yaml @@ -0,0 +1,112 @@ +# services/hermes-observer-rbac/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: hermes-agent-cluster-observer-v2 + labels: + app.kubernetes.io/name: hermes-agent + app.kubernetes.io/part-of: hermes +rules: + - apiGroups: [""] + resources: + - namespaces + - nodes + - persistentvolumes + verbs: ["get", "list", "watch"] + - apiGroups: ["networking.k8s.io"] + resources: ["ingressclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: + - csidrivers + - csinodes + - storageclasses + - volumeattachments + verbs: ["get", "list", "watch"] + - apiGroups: ["metrics.k8s.io"] + resources: ["nodes"] + verbs: ["get", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: hermes-agent-namespaced-observer-v2 + labels: + app.kubernetes.io/name: hermes-agent + app.kubernetes.io/part-of: hermes +rules: + - apiGroups: [""] + resources: + - configmaps + - endpoints + - events + - persistentvolumeclaims + - pods + - pods/log + - replicationcontrollers + - resourcequotas + - services + verbs: ["get", "list", "watch"] + - apiGroups: ["apps"] + resources: + - controllerrevisions + - daemonsets + - deployments + - replicasets + - statefulsets + verbs: ["get", "list", "watch"] + - apiGroups: ["batch"] + resources: ["cronjobs", "jobs"] + verbs: ["get", "list", "watch"] + - apiGroups: ["autoscaling"] + resources: ["horizontalpodautoscalers"] + verbs: ["get", "list", "watch"] + - apiGroups: ["networking.k8s.io"] + resources: ["ingresses", "networkpolicies"] + verbs: ["get", "list", "watch"] + - apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["get", "list", "watch"] + - apiGroups: ["policy"] + resources: ["poddisruptionbudgets"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csistoragecapacities"] + verbs: ["get", "list", "watch"] + - apiGroups: ["metrics.k8s.io"] + resources: ["pods"] + verbs: ["get", "list"] + - apiGroups: + - helm.toolkit.fluxcd.io + - image.toolkit.fluxcd.io + - kustomize.toolkit.fluxcd.io + - notification.toolkit.fluxcd.io + - source.toolkit.fluxcd.io + resources: ["*"] + verbs: ["get", "list", "watch"] + - apiGroups: ["longhorn.io"] + resources: + - backingimages + - engines + - instancemanagers + - nodes + - replicas + - settings + - volumes + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: hermes-agent-cluster-observer-v2 + labels: + app.kubernetes.io/name: hermes-agent + app.kubernetes.io/part-of: hermes +subjects: + - kind: ServiceAccount + name: hermes-agent + namespace: hermes +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: hermes-agent-cluster-observer-v2 diff --git a/services/hermes-scm-broker/deployment.yaml b/services/hermes-scm-broker/deployment.yaml new file mode 100644 index 00000000..649ada1d --- /dev/null +++ b/services/hermes-scm-broker/deployment.yaml @@ -0,0 +1,98 @@ +# services/hermes-scm-broker/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hermes-scm-broker + namespace: hermes-scm + labels: + app: hermes-scm-broker +spec: + replicas: 1 + revisionHistoryLimit: 2 + selector: + matchLabels: + app: hermes-scm-broker + template: + metadata: + labels: + app: hermes-scm-broker + annotations: + vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/role: hermes-scm-broker + vault.hashicorp.com/agent-inject-secret-gitea-token: kv/data/atlas/hermes/developer-gitea + vault.hashicorp.com/agent-inject-template-gitea-token: | + {{- with secret "kv/data/atlas/hermes/developer-gitea" -}} + {{ .Data.data.token }} + {{- end }} + vault.hashicorp.com/agent-pre-populate-only: "true" + vault.hashicorp.com/agent-init-first: "true" + vault.hashicorp.com/agent-requests-cpu: 10m + vault.hashicorp.com/agent-requests-mem: 32Mi + vault.hashicorp.com/agent-limits-cpu: 50m + vault.hashicorp.com/agent-limits-mem: 64Mi + spec: + serviceAccountName: hermes-scm-broker + automountServiceAccountToken: true + securityContext: + fsGroup: 10000 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: + type: RuntimeDefault + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: [arm64] + - key: node-role.kubernetes.io/worker + operator: In + values: ["true"] + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: hardware + operator: In + values: [rpi5] + containers: + - name: broker + image: registry.bstein.dev/bstein/hermes-agent + imagePullPolicy: IfNotPresent + command: + - /opt/hermes/.venv/bin/python + - /opt/broker/scm_broker.py + ports: + - {name: http, containerPort: 9081, protocol: TCP} + readinessProbe: + httpGet: {path: /healthz, port: http} + initialDelaySeconds: 3 + periodSeconds: 10 + livenessProbe: + httpGet: {path: /healthz, port: http} + initialDelaySeconds: 15 + periodSeconds: 30 + failureThreshold: 10 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 10000 + runAsGroup: 10000 + volumeMounts: + - {name: broker-code, mountPath: /opt/broker, readOnly: true} + - {name: tmp, mountPath: /tmp} + resources: + requests: {cpu: 50m, memory: 128Mi} + limits: {cpu: "1", memory: 768Mi} + volumes: + - name: broker-code + configMap: + name: hermes-scm-boundary + defaultMode: 0555 + - name: tmp + emptyDir: + sizeLimit: 1Gi diff --git a/services/hermes-scm-broker/kustomization.yaml b/services/hermes-scm-broker/kustomization.yaml new file mode 100644 index 00000000..5dac3620 --- /dev/null +++ b/services/hermes-scm-broker/kustomization.yaml @@ -0,0 +1,11 @@ +# services/hermes-scm-broker/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: hermes-scm +resources: + - service.yaml + - deployment.yaml + - networkpolicy.yaml +images: + - name: registry.bstein.dev/bstein/hermes-agent + digest: sha256:37ebf720c783ae908a602916ffccf88d43d205a157957f5dc4b487867aee45e7 diff --git a/services/hermes-scm-broker/networkpolicy.yaml b/services/hermes-scm-broker/networkpolicy.yaml new file mode 100644 index 00000000..8187e333 --- /dev/null +++ b/services/hermes-scm-broker/networkpolicy.yaml @@ -0,0 +1,46 @@ +# services/hermes-scm-broker/networkpolicy.yaml +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: hermes-scm-broker-isolation + namespace: hermes-scm +spec: + podSelector: + matchLabels: + app: hermes-scm-broker + policyTypes: [Ingress, Egress] + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: hermes + podSelector: + matchLabels: + app: hermes-agent + ports: + - {protocol: TCP, port: 9081} + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - {protocol: UDP, port: 53} + - {protocol: TCP, port: 53} + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: vault + podSelector: + matchLabels: + app: vault + ports: + - {protocol: TCP, port: 8200} + - to: + - ipBlock: + cidr: 192.168.22.9/32 + ports: + - {protocol: TCP, port: 443} diff --git a/services/hermes-scm-broker/service.yaml b/services/hermes-scm-broker/service.yaml new file mode 100644 index 00000000..0b08c7f6 --- /dev/null +++ b/services/hermes-scm-broker/service.yaml @@ -0,0 +1,13 @@ +# services/hermes-scm-broker/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: hermes-scm-broker + namespace: hermes-scm +spec: + selector: + app: hermes-scm-broker + ports: + - name: http + port: 9081 + targetPort: http diff --git a/services/hermes-scm-namespace/kustomization.yaml b/services/hermes-scm-namespace/kustomization.yaml new file mode 100644 index 00000000..a9e89c7b --- /dev/null +++ b/services/hermes-scm-namespace/kustomization.yaml @@ -0,0 +1,5 @@ +# services/hermes-scm-namespace/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - namespace.yaml diff --git a/services/hermes-scm-namespace/namespace.yaml b/services/hermes-scm-namespace/namespace.yaml new file mode 100644 index 00000000..542f9602 --- /dev/null +++ b/services/hermes-scm-namespace/namespace.yaml @@ -0,0 +1,14 @@ +# services/hermes-scm-namespace/namespace.yaml +apiVersion: v1 +kind: Namespace +metadata: + name: hermes-scm + labels: + app.kubernetes.io/name: hermes-scm-broker +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: hermes-scm-broker + namespace: hermes-scm +automountServiceAccountToken: true diff --git a/services/hermes/agent-configmap.yaml b/services/hermes/agent-configmap.yaml index 782876cf..b53b2c92 100644 --- a/services/hermes/agent-configmap.yaml +++ b/services/hermes/agent-configmap.yaml @@ -189,9 +189,9 @@ data: not replace these coordinator-wide rules. Never call `kanban_show` without a known, non-empty task ID. Ad-hoc inspection and acceptance checks do not need a synthetic Kanban lookup, and must load a skill only when its workflow - materially applies. Atlas HTTPS Git authentication is already supplied by - the runtime-only `GIT_ASKPASS`; use it without reading or exposing the - credential. Coordinator guidance lives at + materially applies. Atlas Git access is supplied by the isolated SCM + broker; no repository token is present in this pod. Use only configured + broker remotes and clients. Coordinator guidance lives at `/opt/data/workspace/AGENTS.md` when more detail is needed. The Jetson classifier is mandatory for AUTO selection. Switchyard may use @@ -295,21 +295,21 @@ data: objective's difficulty warrants it; otherwise synthesize at the original effort. - This owner-only pod has cluster-admin access across Atlas, including logs, - Secrets, exec, port-forwarding, rollout operations, and Flux reconciliation. - Prefer the titan-iac Git/Flux workflow for every durable cluster change; - direct operations are available for explicit operator requests, incident - recovery, and verification, and must be followed by a matching source-of- - truth change when they alter desired state. Never expose credentials in - chat or logs. Triage belongs at triage.hermes.bstein.dev. + This owner-only pod has scoped read-only Kubernetes diagnostics across + Atlas, including resource status, events, logs, and Flux/Helm evidence. It + cannot read Secrets, exec or attach to pods, create service-account tokens, + mutate workloads or RBAC, or reconcile Flux. Put every durable cluster + change on a reviewed titan-iac branch. Never expose credentials in chat or + logs. Triage belongs at triage.hermes.bstein.dev. ## Atlas engineering access The Atlas organization has private visibility. Repository visibility is preserved per project and may be public or private; do not infer a repository's visibility from the organization setting. Repositories are - canonical at `https://scm.bstein.dev/atlas/.git`. HTTPS Git - authentication is already supplied through `GIT_ASKPASS`. Verify the remote + canonical at `https://scm.bstein.dev/atlas/.git`; worker Git traffic + uses `http://hermes-scm-broker.hermes-scm.svc.cluster.local:9081/git/atlas/.git`. + Verify the broker remote and cleanly separate pre-existing changes, create a task branch, run the repository's tests, use `git push --dry-run` when proving access, and push a real branch only when the requested implementation is review-ready. Never @@ -318,11 +318,13 @@ data: `scm.bstein.dev` is Forgejo/Gitea, not GitHub. Never load or follow a GitHub/`gh` skill for an Atlas remote, and do not interpret an unauthenticated Gitea HTTP 404 as a missing private repository. Use - authenticated Git for clone, fetch, and non-force push. For bounded + brokered Git for clone, fetch, and creation of a new namespaced feature + branch. Existing-ref updates, protected refs, deletion, and force-push are + rejected. For bounded repository/pull-request evidence or to create a review-ready draft PR, load - `$manage-atlas-pull-requests` and use `/opt/coordinator/gitea_api.py`. The - client injects the runtime Vault token without exposing it to the command - line, environment, output, or transcript. It permits only Atlas-scoped + `$manage-atlas-pull-requests` and use `/opt/scm/gitea_api.py`. The + client carries no repository credential; a separate least-authority + workload performs the allowed operation. It permits only Atlas-scoped metadata reads and verified same-repository draft creation. Updates, merge, approve, close, delete, comments, repository administration, and force-push are unavailable; never bypass the client with raw HTTP. Leave every PR @@ -330,14 +332,10 @@ data: verify the exact base/head ancestry and diff, run the relevant tests, and confirm the candidate Jenkins result before presenting a handoff. Prefer the internal endpoint in - `JENKINS_BASE_URL`; when Jenkins API authorization prevents a read, use the - existing cluster access to inspect the controller's job/build files and - logs rather than guessing a public hostname. The preferred read-only path - is `/opt/coordinator/jenkins_build_evidence.py JOB [--branch BRANCH] - [--commit SHA] --wait`; it distinguishes a genuinely terminal build from - nested Jenkins execution metadata and returns bounded JSON/log evidence. - With a branch, the helper also tries the conventional `JOB-branches` - multibranch name. If Brad separately reports that he merged the PR, observe + `JENKINS_BASE_URL`; when Jenkins API authorization prevents a read, use + Kubernetes pod logs and status evidence without exec or secret access. + Do not bypass the boundary to inspect controller files. If Brad separately + reports that he merged the PR, observe the default-branch build to a terminal result and report exact commit, build, and test evidence. @@ -349,19 +347,21 @@ data: The terminal PATH contains the pinned operator tools. Start cluster work with `kubectl config current-context`, read-only status/events/logs, and the - relevant `titan-iac` manifests. Put durable desired-state changes on a + relevant `titan-iac` manifests. The broker namespace is deliberately + excluded from agent RBAC. Kubernetes cannot express a deny on one namespace + inside an all-namespace list, so enumerate namespaces and run namespaced + diagnostics instead of relying on `kubectl ... --all-namespaces`. Put + durable desired-state changes on a reviewable `titan-iac` branch and validate Kustomize and client dry-run. - Brad owns merge and routine Flux reconciliation; perform either only under - a separate explicit operator request. Direct `kubectl` mutations are for - explicit incident recovery or ephemeral verification, not ordinary - delivery. + Brad owns merge and Flux reconciliation. Direct `kubectl` mutation is not + available to the agent; hand an explicit incident mutation back to Brad. - Node SSH uses a dedicated audited Hermes identity: `ssh titan-04` (or any - current Kubernetes node name). Host keys are pinned and password fallback - is disabled. Use SSH only for host-level evidence or repairs that cannot be - performed through Kubernetes. Read first, identify the exact node and - impact, avoid fleet-wide destructive commands, and reflect persistent host - configuration in the appropriate tracked provisioning source. + Node SSH uses the dedicated locked-password `hermes-agent` OS account: + `ssh titan-04` (or any current Kubernetes node name). Host keys are pinned, + password fallback is disabled, and the account has no sudo, disk, runtime, + or Kubernetes-storage authority. Use it for unprivileged host evidence; + hand privileged host repair back to Brad and reflect persistent changes in + the tracked provisioning source. Treat every credential, SSH identity, host-trust record, and credential-bearing access-client state as runtime-only Vault data. Keep @@ -398,7 +398,7 @@ data: [model]` for a persistent override. The first native Codex worker requires one device-code login; refreshed provider credentials persist through Vault while client caches remain disposable. The owner - workspace includes cluster-admin Kubernetes - access plus `kubectl`, `flux`, `helm`, `kustomize`, `vault`, `sops`, `age`, + workspace includes scoped read-only Kubernetes diagnostics plus `kubectl`, + `flux`, `helm`, `kustomize`, `vault`, `sops`, `age`, `terraform`, `k9s`, `jq`, `yq`, `gh`, Git, SSH, Python, Node, the browser/computer tools, and the native provider CLIs. diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml index 4b93f92d..ccbc36ea 100644 --- a/services/hermes/agent-deployment.yaml +++ b/services/hermes/agent-deployment.yaml @@ -46,16 +46,6 @@ spec: {{- with secret "kv/data/atlas/hermes/agent-tokens" -}} {{ .Data.data.codex_auth_json }} {{- end }} - vault.hashicorp.com/agent-inject-secret-gitea-token: kv/data/atlas/hermes/developer-gitea - vault.hashicorp.com/agent-inject-template-gitea-token: | - {{- with secret "kv/data/atlas/hermes/developer-gitea" -}} - {{ .Data.data.token }} - {{- end }} - vault.hashicorp.com/agent-inject-secret-gitea-username: kv/data/atlas/hermes/developer-gitea - vault.hashicorp.com/agent-inject-template-gitea-username: | - {{- with secret "kv/data/atlas/hermes/developer-gitea" -}} - {{ .Data.data.username }} - {{- end }} vault.hashicorp.com/agent-inject-secret-node-ssh-private-key: kv/data/atlas/hermes/developer-ssh vault.hashicorp.com/agent-inject-template-node-ssh-private-key: | {{- with secret "kv/data/atlas/hermes/developer-ssh" -}} @@ -186,9 +176,8 @@ spec: chmod 0600 "${profile_env}" chown 10000:10000 "${profile_env}" done - upsert_env GIT_ASKPASS /opt/coordinator/gitea_askpass.sh upsert_env GIT_TERMINAL_PROMPT 0 - upsert_env GITEA_BASE_URL https://scm.bstein.dev + upsert_env HERMES_SCM_BROKER_URL http://hermes-scm-broker.hermes-scm.svc.cluster.local:9081 upsert_env JENKINS_BASE_URL http://jenkins.jenkins.svc.cluster.local:8080 upsert_env ARIADNE_BASE_URL http://ariadne.maintenance.svc.cluster.local upsert_env VICTORIA_METRICS_URL http://victoria-metrics-single-server.monitoring.svc.cluster.local:8428 @@ -664,6 +653,7 @@ spec: - {name: kubeconfig, mountPath: /opt/data/home/.kube/config, subPath: config, readOnly: true} - {name: auto-router-plugin, mountPath: /opt/data/plugins/auto-router, readOnly: true} - {name: atlas-pr-skill, mountPath: /opt/data/workspace/skills/manage-atlas-pull-requests, readOnly: true} + - {name: scm-boundary, mountPath: /opt/scm, readOnly: true} - {name: routing-catalog, mountPath: /routing-catalog, readOnly: true} - {name: tmp, mountPath: /tmp} startupProbe: @@ -828,6 +818,7 @@ spec: - {name: kubeconfig, mountPath: /opt/data/home/.kube/config, subPath: config, readOnly: true} - {name: auto-router-plugin, mountPath: /opt/data/plugins/auto-router, readOnly: true} - {name: atlas-pr-skill, mountPath: /opt/data/workspace/skills/manage-atlas-pull-requests, readOnly: true} + - {name: scm-boundary, mountPath: /opt/scm, readOnly: true} - {name: routing-catalog, mountPath: /routing-catalog, readOnly: true} - {name: tmp, mountPath: /tmp} - {name: ttyd-index, mountPath: /ttyd-index, readOnly: true} @@ -897,6 +888,7 @@ spec: - {name: kubeconfig, mountPath: /opt/data/home/.kube/config, subPath: config, readOnly: true} - {name: auto-router-plugin, mountPath: /opt/data/plugins/auto-router, readOnly: true} - {name: atlas-pr-skill, mountPath: /opt/data/workspace/skills/manage-atlas-pull-requests, readOnly: true} + - {name: scm-boundary, mountPath: /opt/scm, readOnly: true} - {name: tmp, mountPath: /tmp} resources: requests: {cpu: 100m, memory: 256Mi} @@ -1197,6 +1189,10 @@ spec: items: - {key: SKILL.md, path: SKILL.md} - {key: openai.yaml, path: agents/openai.yaml} + - name: scm-boundary + configMap: + name: hermes-scm-boundary + defaultMode: 0555 - name: image-policy configMap: name: hermes-image-policy diff --git a/services/hermes/agent-kubeconfig.yaml b/services/hermes/agent-kubeconfig.yaml index 15b38da9..9ad9242f 100644 --- a/services/hermes/agent-kubeconfig.yaml +++ b/services/hermes/agent-kubeconfig.yaml @@ -11,9 +11,9 @@ users: user: tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token contexts: - - name: atlas-owner + - name: atlas-observer context: cluster: atlas user: hermes-agent namespace: default -current-context: atlas-owner +current-context: atlas-observer diff --git a/services/hermes/agent-rbac.yaml b/services/hermes/agent-rbac.yaml deleted file mode 100644 index 9dbff9e3..00000000 --- a/services/hermes/agent-rbac.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# services/hermes/agent-rbac.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: hermes-agent-cluster-admin - labels: - app.kubernetes.io/name: hermes-agent - app.kubernetes.io/part-of: hermes -subjects: - - kind: ServiceAccount - name: hermes-agent - namespace: hermes -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cluster-admin diff --git a/services/hermes/kustomization.yaml b/services/hermes/kustomization.yaml index 87984530..fe60bef9 100644 --- a/services/hermes/kustomization.yaml +++ b/services/hermes/kustomization.yaml @@ -7,13 +7,13 @@ images: digest: sha256:37ebf720c783ae908a602916ffccf88d43d205a157957f5dc4b487867aee45e7 resources: - namespace.yaml + - scm-common - vault-serviceaccount.yaml - configmap.yaml - agent-configmap.yaml - chat-configmap.yaml - switchyard-configmap.yaml - rbac.yaml - - agent-rbac.yaml - node-ssh-access.yaml - pvc.yaml - switchyard-pvc.yaml @@ -69,8 +69,6 @@ configMapGenerator: - classifier_broker.py=scripts/classifier_broker.py - claude_oauth_broker.py=scripts/claude_oauth_broker.py - worker_route_broker.py=scripts/worker_route_broker.py - - gitea_api.py=scripts/gitea_api.py - - gitea_askpass.sh=scripts/gitea_askpass.sh - hermes_coordinator.py=scripts/hermes_coordinator.py - hermes_model_routing.py=scripts/hermes_model_routing.py - hermes_stt_client.py=scripts/hermes_stt_client.py @@ -104,6 +102,14 @@ configMapGenerator: - config=agent-kubeconfig.yaml options: disableNameSuffixHash: true + - name: hermes-node-account-hardener + namespace: hermes + files: + - node_account_hardening.py=scripts/node_account_hardening.py + - node_account_audit.py=scripts/node_account_audit.py + - node_account_io.py=scripts/node_account_io.py + options: + disableNameSuffixHash: true - name: hermes-auto-router-plugin namespace: hermes files: diff --git a/services/hermes/networkpolicy.yaml b/services/hermes/networkpolicy.yaml index d972889e..f9c3b09c 100644 --- a/services/hermes/networkpolicy.yaml +++ b/services/hermes/networkpolicy.yaml @@ -119,11 +119,65 @@ spec: app: server ports: - {protocol: TCP, port: 9010} - # agent.hermes.bstein.dev is an owner-only engineering workstation. The - # browser boundary remains OAuth-protected, while its workers need to reach - # every cluster namespace, Atlas LAN service, and hosted provider endpoint. egress: - - {} + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - {protocol: UDP, port: 53} + - {protocol: TCP, port: 53} + - to: + - namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: NotIn + values: [gitea, hermes-scm] + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: hermes-scm + podSelector: + matchLabels: + app: hermes-scm-broker + ports: + - {protocol: TCP, port: 9081} + - to: + - ipBlock: + cidr: 10.43.0.1/32 + ports: + - {protocol: TCP, port: 443} + - to: + - ipBlock: + cidr: 192.168.0.0/16 + except: + - 192.168.22.9/32 + - to: + - ipBlock: + cidr: 0.0.0.0/0 + except: + - 10.0.0.0/8 + - 100.64.0.0/10 + - 127.0.0.0/8 + - 169.254.0.0/16 + - 172.16.0.0/12 + - 192.168.0.0/16 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: hermes-node-ssh-access-isolation + namespace: hermes +spec: + podSelector: + matchLabels: + app: hermes-node-ssh-access + policyTypes: [Ingress, Egress] + ingress: [] + egress: [] --- apiVersion: networking.k8s.io/v1 kind: NetworkPolicy diff --git a/services/hermes/node-ssh-access.yaml b/services/hermes/node-ssh-access.yaml index d9c6b223..04b0c78b 100644 --- a/services/hermes/node-ssh-access.yaml +++ b/services/hermes/node-ssh-access.yaml @@ -45,32 +45,16 @@ spec: - operator: Exists containers: - name: key-reconciler - image: busybox:1.37 + # The pinned index supports the cluster's amd64 and arm64 nodes. + image: python@sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df imagePullPolicy: IfNotPresent command: [/bin/sh, -ec] args: - | - reconcile() { - key="$(cat /vault/secrets/node-ssh-public-key)" - found=0 - for user in atlas oceanus; do - home="/host-home/${user}" - [ -d "${home}" ] || continue - found=1 - identity="$(awk -F: -v name="${user}" '$1 == name {print $3 ":" $4; exit}' /host-etc/passwd)" - [ -n "${identity}" ] || identity="$(stat -c %u:%g "${home}")" - uid="${identity%%:*}" - gid="${identity##*:}" - install -d -m 0700 -o "${uid}" -g "${gid}" "${home}/.ssh" - touch "${home}/.ssh/authorized_keys" - grep -qxF "${key}" "${home}/.ssh/authorized_keys" || printf '%s\n' "${key}" >> "${home}/.ssh/authorized_keys" - chown "${uid}:${gid}" "${home}/.ssh/authorized_keys" - chmod 0600 "${home}/.ssh/authorized_keys" - done - [ "${found}" = 1 ] || { echo "no supported node SSH account found" >&2; return 1; } - } while true; do - reconcile + /usr/local/bin/python \ + /opt/node-hardener/node_account_hardening.py \ + --public-key-file /vault/secrets/node-ssh-public-key sleep 300 done securityContext: @@ -83,8 +67,21 @@ spec: volumeMounts: - name: host-home mountPath: /host-home - - name: host-passwd - mountPath: /host-etc/passwd + - name: host-etc + mountPath: /host-etc + - name: host-polkit-share + mountPath: /host-polkit-share + readOnly: true + - name: host-k3s + mountPath: /host-k3s + - name: host-kubelet + mountPath: /host-kubelet + - name: host-run-k3s + mountPath: /host-run-k3s + - name: host-run-containerd + mountPath: /host-run-containerd + - name: coordinator + mountPath: /opt/node-hardener readOnly: true - name: vault-secrets mountPath: /vault/secrets @@ -94,19 +91,43 @@ spec: resources: requests: cpu: 5m - memory: 8Mi + memory: 32Mi limits: cpu: 50m - memory: 32Mi + memory: 128Mi volumes: - name: host-home hostPath: path: /home type: Directory - - name: host-passwd + - name: host-etc hostPath: - path: /etc/passwd - type: File + path: /etc + type: Directory + - name: host-polkit-share + hostPath: + path: /usr/share/polkit-1 + type: Directory + - name: host-k3s + hostPath: + path: /var/lib/rancher/k3s + type: Directory + - name: host-kubelet + hostPath: + path: /var/lib/kubelet + type: Directory + - name: host-run-k3s + hostPath: + path: /run/k3s + type: Directory + - name: host-run-containerd + hostPath: + path: /run/containerd + type: Directory + - name: coordinator + configMap: + name: hermes-node-account-hardener + defaultMode: 0555 - name: vault-secrets csi: driver: secrets-store.csi.k8s.io diff --git a/services/hermes/scm-common/kustomization.yaml b/services/hermes/scm-common/kustomization.yaml new file mode 100644 index 00000000..17dc1da6 --- /dev/null +++ b/services/hermes/scm-common/kustomization.yaml @@ -0,0 +1,14 @@ +# services/hermes/scm-common/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +configMapGenerator: + - name: hermes-scm-boundary + files: + - gitea_api.py=scripts/gitea_api.py + - gitea_api_policy.py=scripts/gitea_api_policy.py + - scm_broker.py=scripts/scm_broker.py + - scm_broker_io.py=scripts/scm_broker_io.py + - scm_broker_server.py=scripts/scm_broker_server.py + - scm_broker_client.py=scripts/scm_broker_client.py + options: + disableNameSuffixHash: true diff --git a/services/hermes/scripts/gitea_api.py b/services/hermes/scm-common/scripts/gitea_api.py old mode 100755 new mode 100644 similarity index 65% rename from services/hermes/scripts/gitea_api.py rename to services/hermes/scm-common/scripts/gitea_api.py index b4a98eca..d207cc63 --- a/services/hermes/scripts/gitea_api.py +++ b/services/hermes/scm-common/scripts/gitea_api.py @@ -4,10 +4,10 @@ from __future__ import annotations import argparse +import base64 import json import os import re -import subprocess import sys import urllib.error import urllib.parse @@ -15,28 +15,29 @@ import urllib.request from collections.abc import Callable from pathlib import Path +from gitea_api_policy import ( + PolicyError, + _draft_title, + _reject_forbidden, + _validate_body, + _validate_pr_number, + _validate_pr_number_segment, + _validate_query, + _validate_ref, + _validate_ref_bounds, + _validate_repo, + _validate_sha, +) + CANONICAL_BASE_URL = "https://scm.bstein.dev" ALLOWED_OWNER = "atlas" -DEFAULT_TOKEN_FILE = Path("/runtime-access/gitea-token") -REPO_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,99}\Z") -SHA_RE = re.compile(r"[0-9a-fA-F]{40}\Z") +DEFAULT_TOKEN_FILE = Path("/vault/secrets/gitea-token") CREATE_PATH_RE = re.compile(r"/api/v1/repos/atlas/([^/]+)/pulls\Z") MAX_ERROR_BYTES = 8192 MAX_RESPONSE_BYTES = 2 * 1024 * 1024 -DRAFT_TITLE_PREFIX = "WIP: " -GIT_BIN = "/usr/bin/git" -SENSITIVE_BODY_PATTERNS = ( - re.compile(r"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----"), - re.compile(r"(?i)\b(?:password|passwd|secret|token|api[_-]?key)\s*[:=]"), - re.compile(r"(?i)\bauthorization\s*:\s*(?:bearer|token|basic)\s+\S+"), - re.compile(r"\b(?:gh[pousr]_[A-Za-z0-9]{20,}|sk-[A-Za-z0-9]{20,})\b"), - re.compile(r"\b(?:AKIA[0-9A-Z]{16}|AIza[0-9A-Za-z_-]{35})\b"), - re.compile(r"\beyJ[A-Za-z0-9_-]{12,}\.[A-Za-z0-9_-]{12,}\.[A-Za-z0-9_-]{8,}\b"), -) - - -class PolicyError(ValueError): - """Raised when a requested Forgejo operation exceeds the safe boundary.""" +MAX_API_TARGET_LENGTH = 768 +MAX_API_PATH_LENGTH = 512 +RAW_API_TARGET_RE = re.compile(r"[A-Za-z0-9/_.?&=-]+\Z") class RejectRedirectHandler(urllib.request.HTTPRedirectHandler): @@ -64,114 +65,48 @@ def read_token(path: Path = DEFAULT_TOKEN_FILE) -> str: def configured_base_url() -> str: """Reject attempts to redirect credentials away from the canonical origin.""" - configured = os.environ.get("GITEA_BASE_URL", CANONICAL_BASE_URL).rstrip("/") + configured = os.environ.get("GITEA_BASE_URL", CANONICAL_BASE_URL) if configured != CANONICAL_BASE_URL: raise PolicyError("Forgejo origin is fixed to the private Atlas SCM service") return configured -def _validate_repo(repo: str) -> str: - if not REPO_RE.fullmatch(repo) or repo in {".", ".."}: - raise PolicyError("repository name is outside the Atlas allowlist") - return repo - - -def _validate_ref(value: object, name: str) -> str: - """Validate the complete Git ref grammar using Git itself.""" - if not isinstance(value, str) or not value or value.startswith("-"): - raise PolicyError(f"{name} must be a same-repository branch name") - if any(ord(character) < 32 or ord(character) == 127 for character in value): - raise PolicyError(f"{name} must be a safe same-repository branch name") - result = subprocess.run( - [GIT_BIN, "check-ref-format", f"refs/heads/{value}"], - stdin=subprocess.DEVNULL, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - check=False, - ) - if result.returncode != 0: - raise PolicyError(f"{name} must be a safe same-repository branch name") - return value - - -def _validate_sha(value: object, name: str = "head SHA") -> str: - if not isinstance(value, str) or not SHA_RE.fullmatch(value): - raise PolicyError(f"{name} must be a full Git SHA-1") - return value.lower() - - -def _validate_text(value: object, name: str, maximum: int, *, required: bool) -> str: - if not isinstance(value, str): - raise PolicyError(f"{name} must be text") - if required and not value.strip(): - raise PolicyError(f"{name} must not be empty") - if len(value) > maximum or "\x00" in value: - raise PolicyError(f"{name} exceeds the safe request limit") - return value - - -def _validate_body(value: object, *, forbidden: tuple[str, ...] = ()) -> str: - body = _validate_text(value, "body", 16384, required=False) - if any(secret and secret in body for secret in forbidden): - raise PolicyError("pull-request body contains runtime credential material") - if any(pattern.search(body) for pattern in SENSITIVE_BODY_PATTERNS): - raise PolicyError("pull-request body resembles credential material") - return body - - -def _draft_title(value: object) -> str: - """Return a bounded title using Gitea's configured default draft prefix.""" - title = _validate_text(value, "title", 251, required=True).strip() - for prefix in ("WIP:", "[WIP]"): - if title.upper().startswith(prefix): - title = title[len(prefix) :].lstrip() - break - if not title: - raise PolicyError("title must contain text after the draft prefix") - return DRAFT_TITLE_PREFIX + title - - def _split_api_path(path: str) -> urllib.parse.SplitResult: + if not isinstance(path, str) or len(path) > MAX_API_TARGET_LENGTH: + raise PolicyError("API target exceeds the safe size limit") + if ( + not path.isascii() + or any(ord(character) < 32 or ord(character) == 127 for character in path) + or "\\" in path + or not RAW_API_TARGET_RE.fullmatch(path) + ): + raise PolicyError("API target contains non-canonical raw characters") target = urllib.parse.urlsplit(path) + canonical = urllib.parse.urlunsplit(("", "", target.path, target.query, "")) + if canonical != path: + raise PolicyError("API target is not in exact canonical form") if target.scheme or target.netloc or target.fragment: raise PolicyError("API path must be relative to the Atlas SCM origin") if not target.path.startswith("/api/v1/"): raise PolicyError("API path must start with /api/v1/") - if "%" in target.path or "//" in target.path or "/../" in f"{target.path}/": + if len(target.path) > MAX_API_PATH_LENGTH: + raise PolicyError("API path exceeds the safe size limit") + segments = target.path.split("/") + if "//" in target.path or any(segment in {".", ".."} for segment in segments): raise PolicyError("encoded or non-canonical API paths are not allowed") return target -def _validate_query(target: urllib.parse.SplitResult, allowed: set[str]) -> None: - try: - pairs = urllib.parse.parse_qsl( - target.query, keep_blank_values=True, strict_parsing=True - ) - except ValueError as exc: - raise PolicyError("invalid API query") from exc - if len({key for key, _ in pairs}) != len(pairs): - raise PolicyError("duplicate API query parameters are not allowed") - if any(key not in allowed for key, _ in pairs): - raise PolicyError("API query parameter is outside the read allowlist") - values = dict(pairs) - for name in ("page", "limit"): - if name not in values: - continue - if not values[name].isdigit() or int(values[name]) < 1: - raise PolicyError(f"{name} must be a positive integer") - if "limit" in values and int(values["limit"]) > 50: - raise PolicyError("read limit cannot exceed 50") - if "state" in values and values["state"] not in {"open", "closed", "all"}: - raise PolicyError("pull-request state is invalid") - - -def _authorize_read(target: urllib.parse.SplitResult) -> str: +def _authorize_read( + target: urllib.parse.SplitResult, *, forbidden: tuple[str, ...] = () +) -> str: """Allow only repository, PR, branch, commit, and status metadata reads.""" prefix = "/api/v1/repos/atlas/" remainder = target.path.removeprefix(prefix) if remainder == target.path: raise PolicyError("reads are limited to explicit Atlas repository metadata") repo, separator, suffix = remainder.partition("/") + _reject_forbidden(repo, "repository", forbidden) _validate_repo(repo) if not separator: @@ -180,10 +115,18 @@ def _authorize_read(target: urllib.parse.SplitResult) -> str: if suffix == "pulls": _validate_query(target, {"page", "limit", "state"}) return "pull-list" - if re.fullmatch(r"pulls/[1-9][0-9]*", suffix): + content_match = re.fullmatch(r"pulls/([^/]+)\.(?:patch|diff)", suffix) + if content_match: + _validate_pr_number_segment(content_match.group(1)) + raise PolicyError("repository API route is outside the metadata read allowlist") + pull_match = re.fullmatch(r"pulls/([^/]+)", suffix) + if pull_match: + _validate_pr_number_segment(pull_match.group(1)) _validate_query(target, set()) return "pull" - if re.fullmatch(r"pulls/[1-9][0-9]*/(?:commits|files)", suffix): + evidence_match = re.fullmatch(r"pulls/([^/]+)/(?:commits|files)", suffix) + if evidence_match: + _validate_pr_number_segment(evidence_match.group(1)) _validate_query(target, {"page", "limit"}) return "pull-evidence" if suffix == "branches": @@ -211,7 +154,7 @@ def _authorize_read(target: urllib.parse.SplitResult) -> str: def api_url(base_url: str, path: str) -> str: """Return a canonical same-origin URL without forwarding credentials.""" - if base_url.rstrip("/") != CANONICAL_BASE_URL: + if base_url != CANONICAL_BASE_URL: raise PolicyError("Forgejo origin is fixed to the private Atlas SCM service") target = _split_api_path(path) return urllib.parse.urlunsplit( @@ -219,14 +162,20 @@ def api_url(base_url: str, path: str) -> str: ) -def authorize_request(method: str, path: str, data: object | None) -> str: +def authorize_request( + method: str, + path: str, + data: object | None, + *, + forbidden: tuple[str, ...] = (), +) -> str: """Validate one request and return its bounded operation name.""" normalized_method = method.upper() target = _split_api_path(path) if normalized_method == "GET": if data is not None: raise PolicyError("read operations cannot include a request body") - return _authorize_read(target) + return _authorize_read(target, forbidden=forbidden) if target.query: raise PolicyError("mutating operations cannot include query parameters") if normalized_method != "POST": @@ -234,14 +183,17 @@ def authorize_request(method: str, path: str, data: object | None) -> str: match = CREATE_PATH_RE.fullmatch(target.path) if not match: raise PolicyError("POST is limited to creating an Atlas draft pull request") + _reject_forbidden(match.group(1), "repository", forbidden) _validate_repo(match.group(1)) if not isinstance(data, dict) or set(data) != {"base", "body", "head", "title"}: raise PolicyError("draft creation accepts only base, body, head, and title") - _validate_ref(data["base"], "base") - _validate_ref(data["head"], "head") - if data["title"] != _draft_title(data["title"]): + if data["title"] != _draft_title(data["title"], forbidden=forbidden): raise PolicyError("new pull requests must use the Gitea draft-title prefix") - _validate_body(data["body"]) + _validate_body(data["body"], forbidden=forbidden) + _validate_ref_bounds(data["base"], "base", forbidden=forbidden) + _validate_ref_bounds(data["head"], "head", forbidden=forbidden) + _validate_ref(data["base"], "base", forbidden=forbidden) + _validate_ref(data["head"], "head", forbidden=forbidden) return "create-draft" @@ -254,7 +206,19 @@ def build_request( data: object | None = None, ) -> urllib.request.Request: """Build an authorized request without putting the token in its URL or body.""" - authorize_request(method, path, data) + authorize_request(method, path, data, forbidden=(token,)) + return _build_request(method, path, base_url=base_url, token=token, data=data) + + +def _build_request( + method: str, + path: str, + *, + base_url: str, + token: str, + data: object | None = None, +) -> urllib.request.Request: + """Build one already-authorized upstream request.""" payload = None if data is not None: payload = json.dumps(data, separators=(",", ":")).encode("utf-8") @@ -281,6 +245,19 @@ def redact_bytes(value: bytes, token: str) -> bytes: ) +def _reject_response_credential(value: bytes, token: str) -> None: + """Fail closed if an upstream body reflects any ordinary token encoding.""" + raw = token.encode("utf-8") + forms = { + raw, + base64.b64encode(raw), + base64.b64encode(b"hermes-automation:" + raw), + urllib.parse.quote(token, safe="").encode("ascii"), + } + if any(form and form in value for form in forms): + raise PolicyError("Forgejo response contains runtime credential material") + + def _request( method: str, path: str, @@ -288,14 +265,23 @@ def _request( *, token: str, opener: Callable[..., object] = _safe_urlopen, + authorized: bool = False, + expected_status: int, ) -> bytes: - request = build_request( - method, path, base_url=configured_base_url(), token=token, data=data - ) + builder = _build_request if authorized else build_request + request = builder(method, path, base_url=configured_base_url(), token=token, data=data) with opener(request, timeout=30) as response: # type: ignore[attr-defined] + status = getattr(response, "status", None) + if status is None and hasattr(response, "getcode"): + status = response.getcode() # type: ignore[attr-defined] + if status != expected_status: + raise PolicyError("Forgejo returned an unexpected HTTP status") + if response.headers.get_content_type() != "application/json": # type: ignore[attr-defined] + raise PolicyError("Forgejo returned an unexpected response type") body = response.read(MAX_RESPONSE_BYTES + 1) # type: ignore[attr-defined] if len(body) > MAX_RESPONSE_BYTES: raise PolicyError("Forgejo response exceeds the safe size limit") + _reject_response_credential(body, token) return redact_bytes(body, token) @@ -303,7 +289,9 @@ def read( path: str, *, token: str, opener: Callable[..., object] = _safe_urlopen ) -> bytes: """Read one explicitly allowed Atlas repository metadata resource.""" - return _request("GET", path, None, token=token, opener=opener) + return _request( + "GET", path, None, token=token, opener=opener, expected_status=200 + ) def _nested(document: dict[str, object], *keys: str) -> object: @@ -332,9 +320,7 @@ def _require_create_response( raise PolicyError("Forgejo returned invalid pull-request metadata") from exc if not isinstance(document, dict): raise PolicyError("Forgejo returned invalid pull-request metadata") - number = document.get("number") - if not isinstance(number, int) or isinstance(number, bool) or number < 1: - raise PolicyError("Forgejo omitted a valid pull-request number") + number = _validate_pr_number(document.get("number")) full_name = f"{ALLOWED_OWNER}/{repo}" expected_html = f"{CANONICAL_BASE_URL}/{full_name}/pulls/{number}" checks = ( @@ -368,12 +354,15 @@ def create_draft( opener: Callable[..., object] = _safe_urlopen, ) -> bytes: """Create and verify a same-repository draft pull request for human review.""" + _reject_forbidden(repo, "repository", (token,)) repo = _validate_repo(repo) - base = _validate_ref(base, "base") - head = _validate_ref(head, "head") - head_sha = _validate_sha(head_sha) - title = _draft_title(title) + title = _draft_title(title, forbidden=(token,)) body = _validate_body(body, forbidden=(token,)) + head_sha = _validate_sha(head_sha) + _validate_ref_bounds(base, "base", forbidden=(token,)) + _validate_ref_bounds(head, "head", forbidden=(token,)) + base = _validate_ref(base, "base", forbidden=(token,)) + head = _validate_ref(head, "head", forbidden=(token,)) data = {"base": base, "body": body, "head": head, "title": title} result = _request( "POST", @@ -381,6 +370,8 @@ def create_draft( data, token=token, opener=opener, + authorized=True, + expected_status=201, ) _require_create_response( result, @@ -417,16 +408,24 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: return parser.parse_args(argv) -def _operation(args: argparse.Namespace) -> tuple[str, str, object | None]: +def _operation( + args: argparse.Namespace, *, validate_refs: bool = True +) -> tuple[str, str, object | None]: if args.operation == "read": return "GET", args.path, None repo = _validate_repo(args.repo) _validate_sha(args.head_sha) + body = _validate_body(args.body) + title = _draft_title(args.title) + _validate_ref_bounds(args.base, "base") + _validate_ref_bounds(args.head, "head") + base = _validate_ref(args.base, "base") if validate_refs else args.base + head = _validate_ref(args.head, "head") if validate_refs else args.head data = { - "base": _validate_ref(args.base, "base"), - "body": _validate_body(args.body), - "head": _validate_ref(args.head, "head"), - "title": _draft_title(args.title), + "base": base, + "body": body, + "head": head, + "title": title, } return "POST", f"/api/v1/repos/{ALLOWED_OWNER}/{repo}/pulls", data @@ -440,12 +439,11 @@ def _write_body(body: bytes) -> None: def main(argv: list[str] | None = None) -> int: """Validate, optionally describe, and execute one safe Forgejo operation.""" - token = "" try: args = parse_args(argv) - method, path, data = _operation(args) - operation = authorize_request(method, path, data) if args.dry_run: + method, path, data = _operation(args, validate_refs=False) + operation = authorize_request(method, path, data) print( json.dumps( { @@ -462,29 +460,27 @@ def main(argv: list[str] | None = None) -> int: ) ) return 0 - token = read_token() + method, path, data = _operation(args, validate_refs=False) + from scm_broker_client import create_draft as broker_create_draft + from scm_broker_client import read as broker_read + if args.operation == "read": - body = read(path, token=token) + body = broker_read(path) else: assert isinstance(data, dict) - body = create_draft( + body = broker_create_draft( args.repo, base=str(data["base"]), head=str(data["head"]), head_sha=args.head_sha, title=str(data["title"]), body=str(data["body"]), - token=token, ) _write_body(body) return 0 except urllib.error.HTTPError as exc: - body = redact_bytes(exc.read(MAX_ERROR_BYTES), token) if token else b"" - print(f"Forgejo request failed with HTTP {exc.code}", file=sys.stderr) - if body: - sys.stderr.buffer.write(body) - if not body.endswith(b"\n"): - sys.stderr.buffer.write(b"\n") + exc.read(MAX_ERROR_BYTES) + print(f"SCM broker request failed with HTTP {exc.code}", file=sys.stderr) return 1 except (OSError, PolicyError, ValueError, json.JSONDecodeError): print( diff --git a/services/hermes/scm-common/scripts/gitea_api_policy.py b/services/hermes/scm-common/scripts/gitea_api_policy.py new file mode 100644 index 00000000..db8833cf --- /dev/null +++ b/services/hermes/scm-common/scripts/gitea_api_policy.py @@ -0,0 +1,490 @@ +"""Validation policy for Hermes' least-authority Atlas Forgejo client. + +The screening catches structured credential assignments, known token formats, +and long high-entropy values. It is a fail-closed accident barrier, not proof +that text is secret-free: a short or multiword secret under an innocuous key is +not reliably distinguishable from prose and must never be supplied by callers. +JSON depth/nodes/documents and multiline gaps are deliberately bounded; large +or unusual configuration blobs are not a supported pull-request field format. +""" + +# ruff: noqa: SIM905 + +from __future__ import annotations + +import json +import re +import subprocess +import urllib.parse +from collections import Counter +from math import log2 + +REPO_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,99}\Z") +SHA_RE = re.compile(r"[0-9a-fA-F]{40}\Z") +DRAFT_TITLE_PREFIX = "WIP: " +GIT_BIN = "/usr/bin/git" +MAX_QUERY_LENGTH = 128 +MAX_QUERY_ITEMS = 3 +MAX_QUERY_KEY_LENGTH = 16 +MAX_QUERY_VALUE_LENGTH = 16 +MAX_PAGE = 10_000 +MAX_LIMIT = 50 +MAX_PR_NUMBER = 2_147_483_647 +MAX_PR_NUMBER_DIGITS = 10 +MAX_REF_CHARACTERS = 200 +MAX_REF_UTF8_BYTES = 255 +MAX_TITLE_UTF8_BYTES = 512 +MAX_BODY_UTF8_BYTES = 32_768 +MAX_JSON_DOCUMENTS = 32 +MAX_JSON_NODES = 2_048 +MAX_JSON_DEPTH = 32 +CANONICAL_QUERY_RE = re.compile(r"[A-Za-z0-9_=&-]*\Z") +ASSIGNMENT_RE = re.compile( + r"""(?mx) + (?[\"']?) + (?P\.?[A-Za-z][A-Za-z0-9_.-]{0,127}) + (?P=key_quote) + [ \t]*(?P[:=])[ \t]* + (?P + \"(?:\\.|[^\"\\\r\n])*\" | + '(?:\\.|[^'\\\r\n])*' | + [^\r\n,;}&]{0,2048} + ) + """ +) +JSON_KEY_RE = re.compile(r'"(?P[^"]{1,512})"(?P[\x00-\x20\x7f]{0,256}):') +MULTILINE_ASSIGNMENT_RE = re.compile( + r"""(?mx) + (?["']?) + (?P\.?[A-Za-z][A-Za-z0-9_.-]{0,127}) + (?P=key_quote) + [ \t]{0,64}(?:\r?\n[ \t]{0,64}){0,4} + (?P[:=]) + [ \t]{0,64}(?:\r?\n[ \t]{0,64}){1,4} + (?P[^\r\n,;}&]{1,2048}) + """ +) +CAMEL_BOUNDARY_RE = re.compile(r"(?<=[a-z0-9])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])") +CREDENTIAL_SCHEME_RE = re.compile( + r"(?i)\b(?:bearer|basic|token)\s+" + r"(?=[A-Za-z0-9+/_=.:-]{8,}\b)(?=[A-Za-z0-9+/_=.:-]*[0-9+/_=.-])" + r"[A-Za-z0-9+/_=.:-]{8,}" +) +STANDALONE_SECRET_PATTERNS = tuple( + re.compile(pattern, re.IGNORECASE) + for pattern in ( + r"(? str: + if not REPO_RE.fullmatch(repo) or repo in {".", ".."}: + raise PolicyError("repository name is outside the Atlas allowlist") + return repo + + +def _reject_forbidden(value: object, name: str, forbidden: tuple[str, ...]) -> None: + """Reject exact runtime credentials before invoking helpers or the network.""" + if not isinstance(value, str) or any(secret and secret in value for secret in forbidden): + raise PolicyError(f"{name} contains runtime credential material") + + +def _validate_ref_bounds( + value: object, name: str, *, forbidden: tuple[str, ...] = () +) -> str: + """Reject oversized or non-encodable refs before invoking Git.""" + _reject_forbidden(value, name, forbidden) + if not value or len(value) > MAX_REF_CHARACTERS: + raise PolicyError(f"{name} exceeds the safe branch-name limit") + try: + encoded = value.encode("utf-8", errors="strict") + except UnicodeEncodeError as exc: + raise PolicyError(f"{name} is not valid UTF-8 text") from exc + if len(encoded) > MAX_REF_UTF8_BYTES: + raise PolicyError(f"{name} exceeds the safe UTF-8 branch-name limit") + return value + + +def _validate_ref( + value: object, name: str, *, forbidden: tuple[str, ...] = () +) -> str: + """Validate the complete Git ref grammar using Git itself.""" + value = _validate_ref_bounds(value, name, forbidden=forbidden) + if value.startswith("-"): + raise PolicyError(f"{name} must be a same-repository branch name") + if any(ord(character) < 32 or ord(character) == 127 for character in value): + raise PolicyError(f"{name} must be a safe same-repository branch name") + result = subprocess.run( + [GIT_BIN, "check-ref-format", f"refs/heads/{value}"], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + if result.returncode != 0: + raise PolicyError(f"{name} must be a safe same-repository branch name") + return value + + +def _validate_sha(value: object, name: str = "head SHA") -> str: + if not isinstance(value, str) or not SHA_RE.fullmatch(value): + raise PolicyError(f"{name} must be a full Git SHA-1") + return value.lower() + + +def _validate_pr_number_segment(value: object) -> int: + """Validate a canonical bounded pull-request number from an API path.""" + if ( + not isinstance(value, str) + or len(value) > MAX_PR_NUMBER_DIGITS + or not re.fullmatch(r"[1-9][0-9]*", value) + ): + raise PolicyError("pull-request number must use bounded canonical ASCII digits") + number = int(value) + if number > MAX_PR_NUMBER: + raise PolicyError("pull-request number is outside its allowed range") + return number + + +def _validate_pr_number(value: object) -> int: + """Validate a bounded pull-request number returned by Forgejo.""" + if ( + not isinstance(value, int) + or isinstance(value, bool) + or not 1 <= value <= MAX_PR_NUMBER + ): + raise PolicyError("Forgejo omitted a valid pull-request number") + return value + + +def _validate_text( + value: object, + name: str, + maximum: int, + maximum_bytes: int, + *, + required: bool, +) -> str: + if not isinstance(value, str): + raise PolicyError(f"{name} must be text") + if required and not value.strip(): + raise PolicyError(f"{name} must not be empty") + try: + encoded = value.encode("utf-8", errors="strict") + except UnicodeEncodeError as exc: + raise PolicyError(f"{name} is not valid UTF-8 text") from exc + if len(value) > maximum or len(encoded) > maximum_bytes or "\x00" in value: + raise PolicyError(f"{name} exceeds the safe request limit") + return value + + +def _normalized_key(value: str) -> tuple[tuple[str, ...], str]: + """Split camelCase and separator-based assignment keys into semantics.""" + separated = CAMEL_BOUNDARY_RE.sub(" ", value.strip(".\"'")) + words = tuple(re.findall(r"[A-Za-z0-9]+", separated.lower())) + return words, "".join(words) + + +def _is_sensitive_key(value: str) -> bool: + words, compact = _normalized_key(value) + word_set = set(words) + if word_set & SENSITIVE_KEY_WORDS: + return True + if compact in SENSITIVE_COMPACT_KEYS: + return True + if compact.endswith(SENSITIVE_COMPACT_SUFFIXES): + return True + if "key" in word_set and word_set & KEY_MODIFIER_WORDS: + return True + if {"client", "email"} <= word_set or {"client", "id"} <= word_set: + return True + if {"access", "id"} <= word_set or {"connection", "string"} <= word_set: + return True + return compact.endswith("configjson") + + +def _strip_assignment_value(value: str) -> tuple[str, bool]: + stripped = value.strip() + quoted = ( + len(stripped) >= 2 and stripped[0] in {'"', "'"} and stripped[-1] == stripped[0] + ) + if quoted: + stripped = stripped[1:-1].strip() + return stripped, quoted + + +def _looks_like_prose(value: str) -> bool: + words = re.findall(r"[A-Za-z]+(?:[-'][A-Za-z]+)?", value) + if len(words) < 2: + return False + if re.search(r"[$`{}\[\]\\@/:=+_]", value): + return False + return words[0].lower() in PROSE_LEAD_WORDS + + +def _looks_sensitive_assignment_value(value: str) -> bool: + stripped, quoted = _strip_assignment_value(value) + if not stripped: + return False + if quoted or stripped.startswith(("{", "[", "|", ">", "!!", "&", "*")): + return True + if CREDENTIAL_SCHEME_RE.search(stripped): + return True + if any(pattern.search(stripped) for pattern in STANDALONE_SECRET_PATTERNS): + return True + if re.search(r"(?:https?://|[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+)", stripped): + return True + if re.search(r"(?:\$\{|\$\(|\\[nrt]|[A-Za-z0-9+/]{16,}={0,3}\Z)", stripped): + return True + return not _looks_like_prose(stripped) + + +def _is_structural_credential_assignment(key: str, value: str) -> bool: + words, compact_key = _normalized_key(key) + stripped, _quoted = _strip_assignment_value(value) + _value_words, compact_value = _normalized_key(stripped) + return ( + compact_key == "type" and compact_value in {"serviceaccount", "credential"} + ) or ("service" in words and "account" in words and bool(stripped)) + + +def _decode_json_key(raw: str) -> tuple[str, bool]: + """Decode one bounded JSON key without silently accepting bad escapes.""" + try: + decoded = json.loads(f'"{raw}"') + except (json.JSONDecodeError, UnicodeError): + approximate = re.sub(r"[^A-Za-z0-9]+", "_", raw.replace("\\", "")) + return approximate, False + return decoded if isinstance(decoded, str) else "", True + + +def _json_value_has_sensitive_assignment( + value: object, *, depth: int = 0, nodes: list[int] | None = None +) -> bool: + """Walk one decoded JSON value with explicit depth and node ceilings.""" + if nodes is None: + nodes = [0] + nodes[0] += 1 + if nodes[0] > MAX_JSON_NODES or depth > MAX_JSON_DEPTH: + raise PolicyError("structured pull-request text exceeds the scan limit") + if isinstance(value, dict): + for key, assigned in value.items(): + if not isinstance(key, str): + continue + if _is_sensitive_key(key): + return True + assigned_text = assigned if isinstance(assigned, str) else "" + if _is_structural_credential_assignment(key, assigned_text): + return True + if _json_value_has_sensitive_assignment( + assigned, depth=depth + 1, nodes=nodes + ): + return True + elif isinstance(value, list): + return any( + _json_value_has_sensitive_assignment(item, depth=depth + 1, nodes=nodes) + for item in value + ) + return False + + +def _decoded_json_documents(value: str): + """Yield bounded embedded JSON object or array fragments.""" + decoder = json.JSONDecoder() + index = 0 + attempts = 0 + while attempts < MAX_JSON_DOCUMENTS: + positions = [ + position for token in "{[" if (position := value.find(token, index)) >= 0 + ] + if not positions: + return + start = min(positions) + attempts += 1 + try: + document, end = decoder.raw_decode(value, start) + except (json.JSONDecodeError, RecursionError, ValueError): + index = start + 1 + continue + yield document + index = max(end, start + 1) + if "{" in value[index:] or "[" in value[index:]: + raise PolicyError("structured pull-request text exceeds the scan limit") + + +def _has_structured_sensitive_assignment(value: str) -> bool: + """Detect bounded JSON and multiline YAML/env credential assignments.""" + for candidate in JSON_KEY_RE.finditer(value): + decoded, valid = _decode_json_key(candidate.group("key")) + if _is_sensitive_key(decoded): + return True + if not valid and _is_sensitive_key(candidate.group("key").replace("\\", "")): + return True + for candidate in MULTILINE_ASSIGNMENT_RE.finditer(value): + key = candidate.group("key") + assigned = candidate.group("value") + if _is_structural_credential_assignment(key, assigned) or ( + _is_sensitive_key(key) and _looks_sensitive_assignment_value(assigned) + ): + return True + return any( + _json_value_has_sensitive_assignment(item) + for item in _decoded_json_documents(value) + ) + + +def _has_high_entropy_token(value: str) -> bool: + for match in HIGH_ENTROPY_TOKEN_RE.finditer(value): + token = match.group(0) + if len(token) > 256: + return True + groups = sum( + bool(re.search(pattern, token)) + for pattern in (r"[a-z]", r"[A-Z]", r"[0-9]", r"[+/_=-]") + ) + counts = Counter(token) + entropy = -sum( + (count / len(token)) * log2(count / len(token)) for count in counts.values() + ) + if groups >= 3 and entropy >= 4.0: + return True + return False + + +def _reject_sensitive(value: str, name: str, forbidden: tuple[str, ...]) -> None: + if any(secret and secret in value for secret in forbidden): + raise PolicyError(f"pull-request {name} contains runtime credential material") + if CREDENTIAL_SCHEME_RE.search(value): + raise PolicyError(f"pull-request {name} resembles credential material") + if any(pattern.search(value) for pattern in STANDALONE_SECRET_PATTERNS): + raise PolicyError(f"pull-request {name} resembles credential material") + if _has_structured_sensitive_assignment(value): + raise PolicyError(f"pull-request {name} resembles credential material") + for candidate in ASSIGNMENT_RE.finditer(value): + key = candidate.group("key") + assigned = candidate.group("value") + if _is_structural_credential_assignment(key, assigned) or ( + _is_sensitive_key(key) and _looks_sensitive_assignment_value(assigned) + ): + raise PolicyError(f"pull-request {name} resembles credential material") + if _has_high_entropy_token(value): + raise PolicyError(f"pull-request {name} resembles credential material") + + +def _validate_body(value: object, *, forbidden: tuple[str, ...] = ()) -> str: + body = _validate_text(value, "body", 16384, MAX_BODY_UTF8_BYTES, required=False) + _reject_sensitive(body, "body", forbidden) + return body + + +def _draft_title(value: object, *, forbidden: tuple[str, ...] = ()) -> str: + """Return a bounded, secret-screened Gitea draft title.""" + title = _validate_text( + value, "title", 251, MAX_TITLE_UTF8_BYTES, required=True + ).strip() + _reject_sensitive(title, "title", forbidden) + for prefix in ("WIP:", "[WIP]"): + if title.upper().startswith(prefix): + title = title[len(prefix) :].lstrip() + break + if not title: + raise PolicyError("title must contain text after the draft prefix") + return DRAFT_TITLE_PREFIX + title + + +def _validate_query(target: urllib.parse.SplitResult, allowed: set[str]) -> None: + """Accept only short canonical ASCII query strings with bounded pagination.""" + raw = target.query + if ( + len(raw) > MAX_QUERY_LENGTH + or not raw.isascii() + or not CANONICAL_QUERY_RE.fullmatch(raw) + or "%" in raw + ): + raise PolicyError("API query must use short canonical ASCII form") + try: + pairs = urllib.parse.parse_qsl(raw, keep_blank_values=True, strict_parsing=True) + except ValueError as exc: + raise PolicyError("invalid API query") from exc + if len(pairs) > MAX_QUERY_ITEMS: + raise PolicyError("too many API query parameters") + if len({key for key, _ in pairs}) != len(pairs): + raise PolicyError("duplicate API query parameters are not allowed") + if any( + not key + or len(key) > MAX_QUERY_KEY_LENGTH + or len(value) > MAX_QUERY_VALUE_LENGTH + for key, value in pairs + ): + raise PolicyError("API query key or value exceeds its safe limit") + if any(key not in allowed for key, _ in pairs): + raise PolicyError("API query parameter is outside the read allowlist") + values = dict(pairs) + for name in ("page", "limit"): + if name not in values: + continue + if not re.fullmatch(r"[0-9]+", values[name]): + raise PolicyError(f"{name} must use ASCII decimal digits") + number = int(values[name]) + if values[name] != str(number): + raise PolicyError(f"{name} must use canonical ASCII decimal form") + ceiling = MAX_PAGE if name == "page" else MAX_LIMIT + if not 1 <= number <= ceiling: + raise PolicyError(f"{name} is outside its allowed range") + if "state" in values and values["state"] not in {"open", "closed", "all"}: + raise PolicyError("pull-request state is invalid") diff --git a/services/hermes/scm-common/scripts/scm_broker.py b/services/hermes/scm-common/scripts/scm_broker.py new file mode 100644 index 00000000..a0c4043c --- /dev/null +++ b/services/hermes/scm-common/scripts/scm_broker.py @@ -0,0 +1,496 @@ +#!/usr/bin/env python3 +"""Serve the credential-isolated Atlas API and Git smart-HTTP boundary.""" + +from __future__ import annotations + +import argparse +import base64 +import json +import re +import socket +import tempfile +import time +import urllib.error +import urllib.parse +import urllib.request +from http.server import BaseHTTPRequestHandler +from typing import BinaryIO + +from gitea_api import ( + CANONICAL_BASE_URL, + PolicyError, + create_draft, + read, + read_token, +) +from gitea_api_policy import _reject_forbidden, _validate_ref, _validate_repo +from scm_broker_io import RejectRedirect, response_status as _status, spool_response +from scm_broker_server import AbsoluteHeaderDeadlineMixin, BoundedThreadingHTTPServer + +BROKER_PORT = 9081 +GIT_USER = "hermes-automation" +MAX_CONTROL_BODY = 64 * 1024 +MAX_GIT_REQUEST = 128 * 1024 * 1024 +MAX_GIT_RESPONSE = 128 * 1024 * 1024 +MAX_PUSH_COMMANDS = 16 +MAX_HEADERS = 32 +MAX_HEADER_BYTES = 16 * 1024 +SPOOL_MEMORY_LIMIT = 1024 * 1024 +STREAM_CHUNK = 64 * 1024 +INBOUND_HEADER_TIMEOUT = 10.0 +INBOUND_BODY_TIMEOUT = 120.0 +ZERO_SHA = b"0" * 40 +GIT_PATH_RE = re.compile( + r"/git/atlas/(?P[A-Za-z0-9][A-Za-z0-9._-]{0,99})\.git/" + r"(?Pinfo/refs|git-upload-pack|git-receive-pack)\Z" +) +FEATURE_REF_RE = re.compile( + r"refs/heads/(?:(?:feature|fix|hermes|handoff)/[A-Za-z0-9][A-Za-z0-9._/-]{0,190})\Z" +) + + +UPSTREAM_OPENER = urllib.request.build_opener(RejectRedirect()) + + +def _read_bounded( + stream, + maximum: int, + length: int | None = None, + *, + deadline: float | None = None, + set_timeout=None, +) -> bytes: + if length is not None and not 0 <= length <= maximum: + raise PolicyError("SCM request exceeds the safe size limit") + wanted = maximum + 1 if length is None else length + chunks: list[bytes] = [] + total = 0 + while total < wanted: + if deadline is not None: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise PolicyError("SCM request body deadline exceeded") + if set_timeout is not None: + set_timeout(remaining) + chunk = stream.read(min(STREAM_CHUNK, wanted - total)) + if not chunk: + break + chunks.append(chunk) + total += len(chunk) + value = b"".join(chunks) + if len(value) > maximum or (length is not None and len(value) != length): + raise PolicyError("SCM request exceeds the safe size limit") + return value + + +def _spool_bounded( + stream, + maximum: int, + length: int, + *, + token: str, + context: str, + deadline: float, + set_timeout=None, +) -> tuple[BinaryIO, int]: + """Copy a fixed-length exchange through a bounded-memory disk spool.""" + if not 0 <= length <= maximum: + raise PolicyError("SCM request exceeds the safe size limit") + spool = tempfile.SpooledTemporaryFile( # noqa: SIM115 - caller owns returned spool + max_size=SPOOL_MEMORY_LIMIT, dir="/tmp" + ) + remaining = length + forms = _credential_forms(token) + carry = b"" + try: + while remaining: + remaining_time = deadline - time.monotonic() + if remaining_time <= 0: + raise PolicyError("SCM request body deadline exceeded") + if set_timeout is not None: + set_timeout(remaining_time) + chunk = stream.read(min(STREAM_CHUNK, remaining)) + if not chunk: + raise PolicyError("SCM request body ended early") + candidate = carry + chunk + if any(form in candidate for form in forms): + raise PolicyError(f"{context} contains runtime credential material") + width = max(len(form) for form in forms) - 1 + carry = candidate[-width:] if width else b"" + spool.write(chunk) + remaining -= len(chunk) + spool.seek(0) + return spool, length + except Exception: + spool.close() + raise + + +def _spool_response(response, maximum: int, token: str) -> tuple[BinaryIO, int]: + return spool_response( + response, + maximum, + _credential_forms(token), + memory_limit=SPOOL_MEMORY_LIMIT, + chunk_size=STREAM_CHUNK, + deadline_seconds=INBOUND_BODY_TIMEOUT, + ) + + +def _load_json(handler: BaseHTTPRequestHandler) -> dict[str, object]: + if handler.headers.get("Transfer-Encoding"): + raise PolicyError("chunked broker control requests are not allowed") + if handler.headers.get_content_type() != "application/json": + raise PolicyError("broker control request must be JSON") + length = _content_length(handler.headers, MAX_CONTROL_BODY) + deadline = time.monotonic() + INBOUND_BODY_TIMEOUT + value = json.loads( + _read_bounded( + handler.rfile, + MAX_CONTROL_BODY, + length, + deadline=deadline, + set_timeout=handler.connection.settimeout, + ) + ) + if not isinstance(value, dict): + raise PolicyError("broker control request must be an object") + return value + + +def _git_target(raw: str) -> tuple[str, str, str]: + if ( + not raw.isascii() + or any(ord(character) < 32 or ord(character) == 127 for character in raw) + or "%" in raw + or "\\" in raw + or len(raw) > 512 + ): + raise PolicyError("Git target is not canonical ASCII") + target = urllib.parse.urlsplit(raw) + canonical = urllib.parse.urlunsplit(("", "", target.path, target.query, "")) + if canonical != raw: + raise PolicyError("Git target is not in exact canonical form") + if target.scheme or target.netloc or target.fragment: + raise PolicyError("Git target must be relative") + match = GIT_PATH_RE.fullmatch(target.path) + if not match: + raise PolicyError("Git target is outside the Atlas allowlist") + repo = _validate_repo(match.group("repo")) + operation = match.group("operation") + query = target.query + if operation == "info/refs": + if query not in {"service=git-upload-pack", "service=git-receive-pack"}: + raise PolicyError("Git discovery service is outside the allowlist") + service = query.removeprefix("service=") + elif query: + raise PolicyError("Git RPC query parameters are not allowed") + else: + service = operation + return repo, operation, service + + +def _content_length(headers: object, maximum: int) -> int: + """Parse one short canonical bounded Content-Length header.""" + raw = headers.get("Content-Length", "") # type: ignore[attr-defined] + if ( + not isinstance(raw, str) + or not raw.isascii() + or len(raw) > 10 + or not raw.isdigit() + ): + raise PolicyError("SCM request length is invalid") + length = int(raw) + if raw != str(length) or not 0 <= length <= maximum: + raise PolicyError("SCM request length is invalid") + return length + + +def _credential_forms(token: str) -> tuple[bytes, ...]: + basic = base64.b64encode(f"{GIT_USER}:{token}".encode()) + return token.encode("utf-8"), basic, b"Basic " + basic + + +def _reject_credential_bytes(value: bytes, token: str, context: str) -> None: + if any(form in value for form in _credential_forms(token)): + raise PolicyError(f"{context} contains runtime credential material") + + +def _receive_prefix(body: bytes | BinaryIO) -> bytes: + if isinstance(body, bytes): + return body + position = body.tell() + try: + body.seek(0) + return body.read(64 * 1024) + finally: + body.seek(position) + + +def _validate_receive_pack(body: bytes | BinaryIO, token: str) -> None: + """Permit only creation of new, namespaced feature branches.""" + body = _receive_prefix(body) + _reject_credential_bytes(body, token, "Git request") + position = 0 + commands = 0 + while position + 4 <= len(body): + header = body[position : position + 4] + if not re.fullmatch(rb"[0-9a-f]{4}", header): + raise PolicyError("Git receive-pack command framing is invalid") + size = int(header, 16) + if size == 0: + if commands == 0: + raise PolicyError("Git receive-pack request has no ref command") + return + if size < 4 or position + size > len(body): + raise PolicyError("Git receive-pack command framing is invalid") + command = body[position + 4 : position + size].rstrip(b"\n") + command = command.split(b"\x00", 1)[0] + fields = command.split(b" ") + if len(fields) != 3 or not all( + re.fullmatch(rb"[0-9a-f]{40}", item) for item in fields[:2] + ): + raise PolicyError("Git receive-pack ref command is invalid") + old_sha, new_sha, raw_ref = fields + if old_sha != ZERO_SHA or new_sha == ZERO_SHA: + raise PolicyError("Git broker permits only new feature-branch creation") + try: + ref = raw_ref.decode("ascii") + except UnicodeDecodeError as exc: + raise PolicyError("Git ref must be canonical ASCII") from exc + if not FEATURE_REF_RE.fullmatch(ref): + raise PolicyError("Git push is limited to namespaced feature branches") + _validate_ref(ref.removeprefix("refs/heads/"), "head", forbidden=(token,)) + commands += 1 + if commands > MAX_PUSH_COMMANDS: + raise PolicyError("Git push contains too many ref commands") + position += size + raise PolicyError("Git receive-pack request omitted its command terminator") + + +def _upstream_git_request( + target: str, + *, + method: str, + body: bytes | BinaryIO | None, + content_type: str | None, + expected_type: str, + token: str, + body_length: int | None = None, + stream_result: bool = False, + opener=UPSTREAM_OPENER.open, +) -> bytes | tuple[BinaryIO, int]: + credentials = base64.b64encode(f"{GIT_USER}:{token}".encode()).decode("ascii") + headers = {"Accept": expected_type, "Authorization": f"Basic {credentials}"} + if content_type: + headers["Content-Type"] = content_type + if body is not None: + if body_length is None: + if not isinstance(body, bytes): + raise PolicyError("Git upstream request length is missing") + body_length = len(body) + headers["Content-Length"] = str(body_length) + request = urllib.request.Request( + CANONICAL_BASE_URL + target, + data=body, + method=method, + headers=headers, + ) + with opener(request, timeout=120) as response: + if _status(response) != 200: + raise PolicyError("Git upstream returned an unexpected HTTP status") + if response.headers.get_content_type() != expected_type: + raise PolicyError("Git upstream returned an unexpected response type") + result, result_length = _spool_response(response, MAX_GIT_RESPONSE, token) + if stream_result: + return result, result_length + try: + return result.read() + finally: + result.close() + + +class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler): + """Expose only bounded metadata, draft creation, and smart-HTTP Git.""" + + server_version = "HermesSCMBroker/1" + sys_version = "" + header_deadline_seconds = INBOUND_HEADER_TIMEOUT + + def _validate_headers(self) -> None: + items = list(self.headers.items()) + if len(items) > MAX_HEADERS: + raise PolicyError("SCM request has too many headers") + total = 0 + for name, value in items: + if not name.isascii() or not value.isascii(): + raise PolicyError("SCM request headers must be ASCII") + total += len(name) + len(value) + 4 + if any(ord(character) < 32 and character != "\t" for character in value): + raise PolicyError("SCM request header contains controls") + if total > MAX_HEADER_BYTES: + raise PolicyError("SCM request headers exceed the safe limit") + if len(self.headers.get_all("Content-Length", [])) > 1: + raise PolicyError("SCM request has duplicate Content-Length") + + def _stream( + self, status: int, content_type: str, body: BinaryIO, length: int + ) -> None: + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(length)) + self.end_headers() + deadline = time.monotonic() + INBOUND_BODY_TIMEOUT + while True: + chunk = body.read(STREAM_CHUNK) + if not chunk: + break + remaining = deadline - time.monotonic() + if remaining <= 0: + raise PolicyError("SCM response write deadline exceeded") + self.connection.settimeout(remaining) + self.wfile.write(chunk) + + def log_message(self, _format: str, *_args: object) -> None: + return + + def _json(self, status: int, body: bytes) -> None: + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.send_header("Cache-Control", "no-store") + self.end_headers() + self.wfile.write(body) + + def _reject(self, status: int = 400) -> None: + self._json(status, b'{"error":"request rejected"}') + + def do_GET(self) -> None: + try: + self._validate_headers() + if self.path == "/healthz": + self._json(200, b'{"status":"ok"}') + return + repo, operation, service = _git_target(self.path) + if operation != "info/refs": + raise PolicyError("Git RPC requires POST") + token = read_token() + _reject_forbidden(repo, "repository", (token,)) + expected = f"application/x-{service}-advertisement" + streamed = _upstream_git_request( + f"/atlas/{repo}.git/info/refs?service={service}", + method="GET", + body=None, + content_type=None, + expected_type=expected, + token=token, + stream_result=True, + ) + body, length = streamed + try: + self._stream(200, expected, body, length) + finally: + body.close() + except ( + OSError, + PolicyError, + socket.timeout, + urllib.error.URLError, + ValueError, + ): + self._reject() + + def do_POST(self) -> None: + try: + self._validate_headers() + self.connection.settimeout(INBOUND_BODY_TIMEOUT) + if self.path in {"/v1/metadata", "/v1/drafts"}: + self._control() + else: + self._git_rpc() + except ( + OSError, + PolicyError, + socket.timeout, + urllib.error.URLError, + ValueError, + json.JSONDecodeError, + ): + self._reject() + + def _control(self) -> None: + data = _load_json(self) + token = read_token() + if self.path == "/v1/metadata": + if set(data) != {"path"} or not isinstance(data["path"], str): + raise PolicyError("metadata request fields are invalid") + result = read(data["path"], token=token) + else: + expected = {"base", "body", "head", "head_sha", "repo", "title"} + if set(data) != expected or not all( + isinstance(data[key], str) for key in expected + ): + raise PolicyError("draft request fields are invalid") + result = create_draft(token=token, **data) # type: ignore[arg-type] + if token.encode("utf-8") in result: + raise PolicyError("SCM upstream reflected credential material") + self._json(200, result) + + def _git_rpc(self) -> None: + repo, operation, service = _git_target(self.path) + if ( + operation not in {"git-upload-pack", "git-receive-pack"} + or service != operation + ): + raise PolicyError("Git RPC operation is outside the allowlist") + expected_request = f"application/x-{service}-request" + if self.headers.get_content_type() != expected_request or self.headers.get( + "Transfer-Encoding" + ): + raise PolicyError("Git RPC request type is invalid") + length = _content_length(self.headers, MAX_GIT_REQUEST) + token = read_token() + _reject_forbidden(repo, "repository", (token,)) + body, body_length = _spool_bounded( + self.rfile, + MAX_GIT_REQUEST, + length, + token=token, + context="Git request", + deadline=time.monotonic() + INBOUND_BODY_TIMEOUT, + set_timeout=self.connection.settimeout, + ) + try: + if service == "git-receive-pack": + _validate_receive_pack(body, token) + expected = f"application/x-{service}-result" + streamed = _upstream_git_request( + f"/atlas/{repo}.git/{service}", + method="POST", + body=body, + body_length=body_length, + content_type=expected_request, + expected_type=expected, + token=token, + stream_result=True, + ) + result, result_length = streamed + try: + self._stream(200, expected, result, result_length) + finally: + result.close() + finally: + body.close() + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--listen", default="0.0.0.0") + parser.add_argument("--port", type=int, default=BROKER_PORT) + args = parser.parse_args() + BoundedThreadingHTTPServer((args.listen, args.port), BrokerHandler).serve_forever() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/hermes/scm-common/scripts/scm_broker_client.py b/services/hermes/scm-common/scripts/scm_broker_client.py new file mode 100644 index 00000000..6c57ddfd --- /dev/null +++ b/services/hermes/scm-common/scripts/scm_broker_client.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Call the credential-isolated Hermes SCM broker over its fixed cluster origin.""" + +from __future__ import annotations + +import json +import urllib.request +from collections.abc import Callable + +from gitea_api_policy import PolicyError + +BROKER_ORIGIN = "http://hermes-scm-broker.hermes-scm.svc.cluster.local:9081" +MAX_RESPONSE_BYTES = 2 * 1024 * 1024 + + +class RejectRedirectHandler(urllib.request.HTTPRedirectHandler): + """Keep every broker request on its fixed in-cluster origin.""" + + def redirect_request(self, req, fp, code, msg, headers, newurl): + raise PolicyError("SCM broker redirects are not allowed") + + +_OPENER = urllib.request.build_opener(RejectRedirectHandler()) + + +def _open(request: urllib.request.Request, timeout: int): + return _OPENER.open(request, timeout=timeout) + + +def request( + endpoint: str, + payload: dict[str, object], + *, + opener: Callable[..., object] = _open, +) -> bytes: + """Send one bounded broker operation without any repository credential.""" + if endpoint not in {"/v1/metadata", "/v1/drafts"}: + raise PolicyError("SCM broker operation is outside the client allowlist") + body = json.dumps(payload, separators=(",", ":")).encode("utf-8") + if len(body) > 64 * 1024: + raise PolicyError("SCM broker request exceeds the safe size limit") + outgoing = urllib.request.Request( + BROKER_ORIGIN + endpoint, + data=body, + method="POST", + headers={ + "Accept": "application/json", + "Content-Type": "application/json", + "User-Agent": "hermes-scm-broker-client/1", + }, + ) + with opener(outgoing, timeout=30) as response: # type: ignore[attr-defined] + status = getattr(response, "status", None) + if status is None and hasattr(response, "getcode"): + status = response.getcode() # type: ignore[attr-defined] + if status != 200: + raise PolicyError("SCM broker returned an unexpected HTTP status") + content_type = response.headers.get_content_type() # type: ignore[attr-defined] + if content_type != "application/json": + raise PolicyError("SCM broker returned an unexpected response type") + result = response.read(MAX_RESPONSE_BYTES + 1) # type: ignore[attr-defined] + if len(result) > MAX_RESPONSE_BYTES: + raise PolicyError("SCM broker response exceeds the safe size limit") + json.loads(result) + return result + + +def read(path: str, *, opener: Callable[..., object] = _open) -> bytes: + """Read explicitly allowed Atlas metadata through the broker.""" + return request("/v1/metadata", {"path": path}, opener=opener) + + +def create_draft( + repo: str, + *, + base: str, + head: str, + head_sha: str, + title: str, + body: str, + opener: Callable[..., object] = _open, +) -> bytes: + """Create one verified draft through the broker for human review.""" + return request( + "/v1/drafts", + { + "base": base, + "body": body, + "head": head, + "head_sha": head_sha, + "repo": repo, + "title": title, + }, + opener=opener, + ) diff --git a/services/hermes/scm-common/scripts/scm_broker_io.py b/services/hermes/scm-common/scripts/scm_broker_io.py new file mode 100644 index 00000000..96df1fe0 --- /dev/null +++ b/services/hermes/scm-common/scripts/scm_broker_io.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""Bounded-memory stream helpers for the Hermes SCM broker.""" + +from __future__ import annotations + +import tempfile +import time +import urllib.request +from typing import BinaryIO + +from gitea_api_policy import PolicyError + + +class RejectRedirect(urllib.request.HTTPRedirectHandler): + """Reject every upstream redirect before credentials can be forwarded.""" + + def redirect_request(self, req, fp, code, msg, headers, newurl): + raise PolicyError("SCM upstream redirects are not allowed") + + +def response_status(response: object) -> int | None: + """Read an HTTP status from either urllib response interface.""" + value = getattr(response, "status", None) + if value is None and hasattr(response, "getcode"): + value = response.getcode() # type: ignore[attr-defined] + return value + + +def spool_response( + response, + maximum: int, + forbidden: tuple[bytes, ...], + *, + memory_limit: int, + chunk_size: int, + deadline_seconds: float, +) -> tuple[BinaryIO, int]: + """Spool an upstream response to disk while scanning chunk boundaries.""" + spool = tempfile.SpooledTemporaryFile( # noqa: SIM115 - caller owns returned spool + max_size=memory_limit, dir="/tmp" + ) + carry = b"" + total = 0 + deadline = time.monotonic() + deadline_seconds + try: + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise PolicyError("SCM upstream response deadline exceeded") + raw = getattr(getattr(response, "fp", None), "raw", None) + sock = getattr(raw, "_sock", raw) + if hasattr(sock, "settimeout"): + sock.settimeout(remaining) + chunk = response.read(chunk_size) + if not chunk: + break + total += len(chunk) + if total > maximum: + raise PolicyError("SCM upstream response exceeds the safe size limit") + candidate = carry + chunk + if any(value in candidate for value in forbidden): + raise PolicyError( + "Git upstream response contains runtime credential material" + ) + width = max(len(value) for value in forbidden) - 1 + carry = candidate[-width:] if width else b"" + spool.write(chunk) + spool.seek(0) + return spool, total + except Exception: + spool.close() + raise diff --git a/services/hermes/scm-common/scripts/scm_broker_server.py b/services/hermes/scm-common/scripts/scm_broker_server.py new file mode 100644 index 00000000..f29cd314 --- /dev/null +++ b/services/hermes/scm-common/scripts/scm_broker_server.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +"""Concurrency-bounded threaded HTTP server for the Hermes SCM broker.""" + +from __future__ import annotations + +import threading +import time +from http.server import ThreadingHTTPServer + +MAX_CONCURRENT_REQUESTS = 3 + + +class _AbsoluteDeadlineReader: + """Read header lines against one wall-clock deadline, not idle timeouts.""" + + def __init__(self, stream, connection): + self._stream = stream + self._connection = connection + self._deadline: float | None = None + + def begin(self, timeout: float) -> None: + self._deadline = time.monotonic() + timeout + + def end(self) -> None: + self._deadline = None + + def readline(self, limit: int = -1) -> bytes: + if self._deadline is None: + return self._stream.readline(limit) + value = bytearray() + while limit < 0 or len(value) < limit: + remaining = self._deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError("absolute SCM header deadline exceeded") + self._connection.settimeout(remaining) + character = self._stream.read(1) + if not character: + break + value.extend(character) + if character == b"\n": + break + return bytes(value) + + def __getattr__(self, name: str): + return getattr(self._stream, name) + + +class AbsoluteHeaderDeadlineMixin: + """Apply an absolute request-line plus header deadline to HTTP handlers.""" + + header_deadline_seconds = 10.0 + + def setup(self) -> None: + super().setup() + self._header_reader = _AbsoluteDeadlineReader(self.rfile, self.connection) + self.rfile = self._header_reader + + def handle_one_request(self) -> None: + try: + self._header_reader.begin(self.header_deadline_seconds) + try: + self.raw_requestline = self.rfile.readline(65537) + if len(self.raw_requestline) > 65536: + self.requestline = "" + self.request_version = "" + self.command = "" + self.send_error(414) + return + if not self.raw_requestline: + self.close_connection = True + return + if not self.parse_request(): + return + finally: + self._header_reader.end() + method_name = "do_" + self.command + if not hasattr(self, method_name): + self.send_error(501, "Unsupported method") + return + getattr(self, method_name)() + self.wfile.flush() + except TimeoutError: + self.close_connection = True + + +class BoundedThreadingHTTPServer(ThreadingHTTPServer): + """Cap active handlers so large Git exchanges cannot exhaust memory.""" + + daemon_threads = True + block_on_close = True + request_queue_size = 16 + + def __init__(self, *args, **kwargs): + self._slots = threading.BoundedSemaphore(MAX_CONCURRENT_REQUESTS) + super().__init__(*args, **kwargs) + + def process_request(self, request, client_address) -> None: + if not self._slots.acquire(blocking=False): + try: + request.sendall( + b"HTTP/1.1 503 Service Unavailable\r\n" + b"Connection: close\r\nContent-Length: 0\r\n\r\n" + ) + finally: + self.shutdown_request(request) + return + try: + super().process_request(request, client_address) + except Exception: + self._slots.release() + self.shutdown_request(request) + raise + + def process_request_thread(self, request, client_address) -> None: + try: + super().process_request_thread(request, client_address) + finally: + self._slots.release() diff --git a/services/hermes/scripts/gitea_askpass.sh b/services/hermes/scripts/gitea_askpass.sh deleted file mode 100755 index d6146d80..00000000 --- a/services/hermes/scripts/gitea_askpass.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env sh -set -eu - -case "${1:-}" in - *Username*) - if [ -s /runtime-access/gitea-username ]; then - tr -d '\r\n' Path: @@ -139,24 +141,8 @@ def bootstrap_cassandra_state(root: Path) -> dict[str, str]: return {"state": "ready"} -def _gitea_token_path() -> Path: - """Return the runtime-only Gitea credential path without reading it.""" - configured = os.environ.get("HERMES_GITEA_TOKEN_FILE", "").strip() - return Path(configured) if configured else DEFAULT_GITEA_TOKEN_PATH - - -def _gitea_token_available() -> bool: - """Check credential readiness without loading it into process memory.""" - try: - path = _gitea_token_path() - return path.is_file() and path.stat().st_size > 0 - except OSError: - return False - - def sync_cassandra_repo(env_values: dict[str, str]) -> str: - """Clone or fetch Cassandra through the runtime-only askpass credential.""" - token_available = _gitea_token_available() + """Clone or fetch Cassandra through the credential-isolated SCM broker.""" if shutil.which("git") is None: return "git-unavailable" child_env = os.environ.copy() @@ -173,9 +159,7 @@ def sync_cassandra_repo(env_values: dict[str, str]) -> str: } } ) - child_env["GIT_ASKPASS"] = env_values.get( - "GIT_ASKPASS", "/opt/coordinator/gitea_askpass.sh" - ) + child_env.pop("GIT_ASKPASS", None) child_env["GIT_TERMINAL_PROMPT"] = "0" if (CASSANDRA_BASE_PATH / ".git").exists(): try: @@ -232,8 +216,6 @@ def sync_cassandra_repo(env_values: dict[str, str]) -> str: return f"remote-repair-failed-{repaired.returncode}" except (OSError, subprocess.TimeoutExpired): return "remote-repair-failed" - if not token_available: - return "ready; fetch skipped until Gitea token is configured" command = [ "git", "-C", @@ -247,8 +229,6 @@ def sync_cassandra_repo(env_values: dict[str, str]) -> str: CASSANDRA_BASE_PATH.mkdir(parents=True, exist_ok=True) if any(CASSANDRA_BASE_PATH.iterdir()): return "unmanaged-nonempty-directory" - if not token_available: - return "awaiting-gitea-token" command = [ "git", "clone", diff --git a/services/hermes/scripts/node_account_audit.py b/services/hermes/scripts/node_account_audit.py new file mode 100644 index 00000000..b1795e99 --- /dev/null +++ b/services/hermes/scripts/node_account_audit.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +"""Fail-closed supplementary-group and host privilege-policy audits.""" + +from __future__ import annotations + +import re +import stat +import struct +from pathlib import Path + +from node_account_io import HardeningError, read_regular + + +def _members(field: str, context: str) -> set[str]: + if not field: + return set() + values = field.split(",") + if any(not value or value.strip() != value for value in values): + raise HardeningError(f"{context} has a malformed member list") + if len(values) != len(set(values)): + raise HardeningError(f"{context} has duplicate members") + return set(values) + + +def audit_membership( + account: str, groups: list[list[str]], gshadow: list[list[str]] +) -> None: + """Reject the dedicated account in any supplementary group/admin list.""" + for record in groups: + if account in _members(record[3], f"group {record[0]}"): + raise HardeningError("dedicated Hermes account has supplementary group access") + for record in gshadow: + principals = _members(record[2], f"gshadow {record[0]} admins") + principals |= _members(record[3], f"gshadow {record[0]} members") + if account in principals: + raise HardeningError("dedicated Hermes account has gshadow group access") + + +def _policy_files(host_etc: Path, polkit_share: Path, root_uid: int) -> list[Path]: + files: list[Path] = [] + sudoers = host_etc / "sudoers" + if sudoers.exists(): + files.append(sudoers) + roots = ( + host_etc / "sudoers.d", + host_etc / "polkit-1/rules.d", + host_etc / "polkit-1/localauthority", + polkit_share / "rules.d", + polkit_share / "localauthority", + ) + for root in roots: + if not root.exists(): + continue + metadata = root.lstat() + if ( + not stat.S_ISDIR(metadata.st_mode) + or stat.S_ISLNK(metadata.st_mode) + or metadata.st_uid != root_uid + or stat.S_IMODE(metadata.st_mode) & 0o022 + ): + raise HardeningError("unsafe sudo/polkit policy directory") + files.extend(path for path in root.rglob("*") if path.is_file()) + if len(files) > 256: + raise HardeningError("too many sudo/polkit policy files") + return sorted(files) + + +def _audit_policy_metadata(snapshot, root_uid: int, account_uid: int) -> None: + if snapshot.uid != root_uid or snapshot.mode & 0o022: + raise HardeningError("sudo/polkit policy metadata permits unsafe mutation") + acl = dict(snapshot.xattrs).get("system.posix_acl_access", b"") + if not acl: + return + header = struct.Struct(" str: + """Remove comments without discarding sudoers' numeric-UID principals.""" + active: list[str] = [] + for line in text.splitlines(): + stripped = line.lstrip() + if not stripped: + continue + include = re.fullmatch( + r"(?i)([#@]include|[#@]includedir)\s+([^\s]+)", stripped + ) + if include: + directive, target = include.groups() + if directive.lower().endswith("includedir") and target == "/etc/sudoers.d": + continue + raise HardeningError("sudo policy includes an unaudited authority source") + if stripped.startswith("#") and not re.match(r"#\d+(?:\s|$)", stripped): + continue + active.append(line) + return "\n".join(active) + + +def _mentions_dedicated_identity(text: str, account: str, account_uid: int) -> bool: + account_pattern = rf"(? None: + """Reject direct, group, wildcard sudo, or root-equivalent polkit grants.""" + dangerous_polkit = ( + "org.freedesktop.policykit.exec", + "org.freedesktop.systemd1.manage-unit", + "org.freedesktop.udisks2.modify-device", + "org.freedesktop.packagekit", + ) + nsswitch = host_etc / "nsswitch.conf" + if nsswitch.exists(): + try: + nss_snapshot = read_regular(nsswitch, 64 * 1024) + _audit_policy_metadata(nss_snapshot, root_uid, account_uid) + text = nss_snapshot.value.decode("utf-8") + except UnicodeDecodeError as exc: + raise HardeningError("nsswitch policy is not UTF-8") from exc + for line in text.splitlines(): + name, separator, sources = line.partition(":") + if name.strip() in { + "passwd", + "group", + "initgroups", + "shadow", + "sudoers", + } and separator: + active_sources = [item for item in sources.split() if not item.startswith("[")] + if not active_sources or any( + item not in {"files", "systemd"} for item in active_sources + ): + raise HardeningError("external group/account/sudo authority source is unsafe") + total = 0 + for path in _policy_files(host_etc, polkit_share, root_uid): + snapshot = read_regular(path, 256 * 1024) + _audit_policy_metadata(snapshot, root_uid, account_uid) + value = snapshot.value + total += len(value) + if total > 2 * 1024 * 1024: + raise HardeningError("sudo/polkit policy input exceeds safe limit") + try: + text = value.decode("utf-8") + except UnicodeDecodeError as exc: + raise HardeningError("sudo/polkit policy is not UTF-8") from exc + if path.name == "sudoers" or "sudoers.d" in path.parts: + active = _active_sudo_policy(text) + if _mentions_dedicated_identity(active, account, account_uid): + raise HardeningError("dedicated Hermes account has sudo authority") + for line in active.splitlines(): + fields = line.split() + if fields and fields[0] in {"ALL", "%ALL"} and "=" in line: + raise HardeningError("broad sudo authority includes Hermes") + continue + active = "\n".join( + line for line in text.splitlines() if not line.lstrip().startswith("#") + ) + grants = "polkit.Result.YES" in active or re.search( + r"(?im)^\s*Result(?:Any|Inactive|Active)\s*=\s*yes\s*$", active + ) + if grants and _mentions_dedicated_identity(active, account, account_uid): + raise HardeningError("dedicated Hermes account has polkit authority") + broad = "unix-user:*" in active or "Identity=unix-user:*" in active + if broad and grants and any(item in active for item in dangerous_polkit): + raise HardeningError("root-equivalent broad polkit authority includes Hermes") diff --git a/services/hermes/scripts/node_account_hardening.py b/services/hermes/scripts/node_account_hardening.py new file mode 100644 index 00000000..a545af47 --- /dev/null +++ b/services/hermes/scripts/node_account_hardening.py @@ -0,0 +1,496 @@ +#!/usr/bin/env python3 +"""Reconcile a dedicated unprivileged Hermes SSH account on one Atlas node.""" + +from __future__ import annotations + +import argparse +import base64 +import errno +import os +import re +import shlex +import stat +import struct +from contextlib import suppress +from pathlib import Path + +from node_account_audit import audit_membership, audit_privilege_policies +from node_account_io import ( + FileSnapshot, + HardeningError, + account_lock, + assert_unchanged, + atomic_write, + backup_once, + read_regular, +) + +HOST_ETC = Path("/host-etc") +HOST_HOME = Path("/host-home") +HOST_K3S = Path("/host-k3s") +HOST_KUBELET = Path("/host-kubelet") +HOST_RUN_K3S = Path("/host-run-k3s") +HOST_RUN_CONTAINERD = Path("/host-run-containerd") +HOST_POLKIT_SHARE = Path("/host-polkit-share") +ACCOUNT = "hermes-agent" +ACCOUNT_UID = 1200 +ACCOUNT_GID = 1200 +ACCOUNT_HOME = "/home/hermes-agent" +ACCOUNT_SHELL = "/bin/bash" +HOST_ROOT_UID = 0 +HOST_ROOT_GID = 0 +LEGACY_ACCOUNTS = ("atlas", "oceanus") +MAX_ACCOUNT_FILE = 2 * 1024 * 1024 +MAX_AUTHORIZED_KEYS = 1024 * 1024 +ACL_VERSION = 2 +ACL_UNDEFINED_ID = 0xFFFFFFFF +ACL_USER_OBJ = 0x01 +ACL_USER = 0x02 +ACL_GROUP_OBJ = 0x04 +ACL_GROUP = 0x08 +ACL_MASK = 0x10 +ACL_OTHER = 0x20 +ACL_HEADER = struct.Struct(" tuple[bytes, os.stat_result]: + snapshot = read_regular(path, maximum) + return snapshot.value, path.stat(follow_symlinks=False) + + +def _records(value: bytes, fields: int, name: str) -> list[list[str]]: + try: + text = value.decode("utf-8") + except UnicodeDecodeError as exc: + raise HardeningError(f"{name} is not UTF-8") from exc + if not text.endswith("\n"): + raise HardeningError(f"{name} is missing its final newline") + records = [] + names = set() + for line in text.splitlines(): + parts = line.split(":") + if len(parts) != fields or not parts[0] or parts[0] in names: + raise HardeningError(f"{name} has an invalid record") + names.add(parts[0]) + records.append(parts) + return records + + +def _encode(records: list[list[str]]) -> bytes: + return ("\n".join(":".join(record) for record in records) + "\n").encode() + + +def _expected_records() -> dict[str, list[str]]: + return { + "passwd": [ + ACCOUNT, + "x", + str(ACCOUNT_UID), + str(ACCOUNT_GID), + "Hermes Agent", + ACCOUNT_HOME, + ACCOUNT_SHELL, + ], + "group": [ACCOUNT, "x", str(ACCOUNT_GID), ""], + "shadow": [ACCOUNT, "!", "1", "0", "99999", "7", "", "", ""], + "gshadow": [ACCOUNT, "!", "", ""], + } + + +def _reconcile_record( + records: list[list[str]], expected: list[str], *, identity_index: int +) -> list[list[str]]: + for record in records: + same_name = record[0] == expected[0] + same_identity = record[identity_index] == expected[identity_index] + if same_name or same_identity: + if record != expected: + raise HardeningError("dedicated Hermes account identity conflicts") + return records + return [*records, expected] + + +def _reconcile_databases() -> None: + expected = _expected_records() + definitions = ( + ("passwd", 7, 2), + ("group", 4, 2), + ("shadow", 9, 0), + ("gshadow", 4, 0), + ) + planned: list[tuple[Path, FileSnapshot, bytes]] = [] + account_present = False + parsed: dict[str, list[list[str]]] = {} + for name, fields, identity_index in definitions: + path = HOST_ETC / name + snapshot = read_regular(path, MAX_ACCOUNT_FILE) + records = _records(snapshot.value, fields, name) + parsed[name] = records + updated = _reconcile_record( + records, expected[name], identity_index=identity_index + ) + if name == "passwd": + account_present = expected[name] in records + planned.append((path, snapshot, _encode(updated))) + audit_membership(ACCOUNT, parsed["group"], parsed["gshadow"]) + audit_privilege_policies( + ACCOUNT, ACCOUNT_UID, HOST_ROOT_UID, HOST_ETC, HOST_POLKIT_SHARE + ) + try: + (HOST_HOME / ACCOUNT).lstat() + except FileNotFoundError: + pass + else: + if not account_present: + raise HardeningError("dedicated Hermes account home already exists") + for path, snapshot, _updated in planned: + assert_unchanged(path, snapshot, MAX_ACCOUNT_FILE) + for path, snapshot, _updated in planned: + backup_once(path, snapshot, MAX_ACCOUNT_FILE) + for path, snapshot, _updated in planned: + assert_unchanged(path, snapshot, MAX_ACCOUNT_FILE) + written: list[tuple[Path, FileSnapshot, bytes]] = [] + try: + for path, snapshot, updated in planned: + assert_unchanged(path, snapshot, MAX_ACCOUNT_FILE) + atomic_write(path, updated, snapshot) + written.append((path, snapshot, updated)) + for path, _snapshot, _updated in planned: + name = path.name + fields = next(item[1] for item in definitions if item[0] == name) + records = _records(_read_regular(path)[0], fields, name) + if expected[name] not in records: + raise HardeningError("dedicated Hermes account validation failed") + except Exception as error: + for path, snapshot, updated in reversed(written): + current = read_regular(path, MAX_ACCOUNT_FILE) + if ( + current.value != updated + or current.mode != snapshot.mode + or current.uid != snapshot.uid + or current.gid != snapshot.gid + or current.xattrs != snapshot.xattrs + ): + raise HardeningError( + f"concurrent host account change prevents rollback: {path.name}" + ) from error + atomic_write(path, snapshot.value, snapshot) + raise + + +SUPPORTED_KEY_TYPES = {"ssh-ed25519", "ecdsa-sha2-nistp256", "ssh-rsa"} +KEY_TYPE_RE = re.compile(r"(?:sk-)?(?:ssh|ecdsa)-[A-Za-z0-9@._+-]+\Z") + + +def _key_identity(line: bytes) -> tuple[str, bytes] | None: + """Return OpenSSH key type/blob, ignoring options and comments.""" + stripped = line.strip(b"\r\n") + if not stripped or stripped.lstrip().startswith(b"#"): + return None + try: + fields = shlex.split(stripped.decode("ascii"), posix=True) + except (UnicodeDecodeError, ValueError) as exc: + raise HardeningError("authorized key line is malformed") from exc + indexes = [ + index for index, field in enumerate(fields) if KEY_TYPE_RE.fullmatch(field) + ] + if len(indexes) != 1 or indexes[0] + 1 >= len(fields): + raise HardeningError("authorized key line has ambiguous key material") + key_type = fields[indexes[0]] + try: + blob = base64.b64decode(fields[indexes[0] + 1], validate=True) + except ValueError as exc: + raise HardeningError("authorized key payload is invalid") from exc + encoded_type = key_type.encode("ascii") + if len(blob) < 4 or int.from_bytes(blob[:4], "big") != len(encoded_type): + raise HardeningError("authorized key blob is malformed") + if blob[4 : 4 + len(encoded_type)] != encoded_type: + raise HardeningError("authorized key type does not match its blob") + return key_type, blob + + +def _validated_public_key(path: Path) -> tuple[bytes, tuple[str, bytes]]: + value, _ = _read_regular(path, 16 * 1024) + line = value.strip() + if b"\n" in line or b"\r" in line: + raise HardeningError("Hermes public key must contain one line") + identity = _key_identity(line) + fields = line.split() + if ( + identity is None + or identity[0] not in SUPPORTED_KEY_TYPES + or len(fields) not in {2, 3} + or fields[0].decode("ascii") != identity[0] + ): + raise HardeningError("Hermes public key format is unsupported") + return line, identity + + +def _directory(path: Path, *, mode: int, uid: int, gid: int) -> None: + with suppress(FileExistsError): + path.mkdir(mode=mode) + metadata = path.lstat() + if not stat.S_ISDIR(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode): + raise HardeningError(f"unsafe account directory: {path.name}") + if metadata.st_uid not in {0, uid} or metadata.st_gid not in {0, gid}: + raise HardeningError(f"account directory ownership conflicts: {path.name}") + os.chown(path, uid, gid) + path.chmod(mode) + + +def _without_key(value: bytes, identity: tuple[str, bytes]) -> bytes: + kept = [] + for line in value.splitlines(keepends=True): + if _key_identity(line) != identity: + kept.append(line) + return b"".join(kept) + + +def _move_key(public_key: Path) -> None: + key, identity = _validated_public_key(public_key) + target = HOST_HOME / ACCOUNT / ".ssh" / "authorized_keys" + legacy_plan: list[tuple[Path, FileSnapshot, bytes]] = [] + for legacy in LEGACY_ACCOUNTS: + authorized = HOST_HOME / legacy / ".ssh" / "authorized_keys" + try: + snapshot = read_regular(authorized, MAX_AUTHORIZED_KEYS) + except FileNotFoundError: + continue + updated = _without_key(snapshot.value, identity) + if updated != snapshot.value: + legacy_plan.append((authorized, snapshot, updated)) + if legacy_plan and target.exists(): + target_snapshot = read_regular(target, MAX_AUTHORIZED_KEYS) + target_without_key = _without_key(target_snapshot.value, identity) + if target_without_key != target_snapshot.value: + backup_once(target, target_snapshot, MAX_AUTHORIZED_KEYS) + assert_unchanged(target, target_snapshot, MAX_AUTHORIZED_KEYS) + atomic_write(target, target_without_key, target_snapshot) + verified_target = read_regular(target, MAX_AUTHORIZED_KEYS).value + if _without_key(verified_target, identity) != verified_target: + raise HardeningError("preexisting Hermes authorization removal failed") + for authorized, snapshot, _updated in legacy_plan: + assert_unchanged(authorized, snapshot, MAX_AUTHORIZED_KEYS) + backup_once(authorized, snapshot, MAX_AUTHORIZED_KEYS) + for authorized, snapshot, updated in legacy_plan: + assert_unchanged(authorized, snapshot, MAX_AUTHORIZED_KEYS) + atomic_write(authorized, updated, snapshot) + for legacy in LEGACY_ACCOUNTS: + authorized = HOST_HOME / legacy / ".ssh" / "authorized_keys" + try: + value = read_regular(authorized, MAX_AUTHORIZED_KEYS).value + except FileNotFoundError: + continue + if _without_key(value, identity) != value: + raise HardeningError("legacy Hermes authorization key removal failed") + + # Installation happens only after every legacy authorization is absent. A + # crash can therefore remove access temporarily, but can never duplicate + # the machine credential across privileged and unprivileged accounts. + home = HOST_HOME / ACCOUNT + ssh = home / ".ssh" + _directory(home, mode=0o700, uid=ACCOUNT_UID, gid=ACCOUNT_GID) + _directory(ssh, mode=0o700, uid=ACCOUNT_UID, gid=ACCOUNT_GID) + target = ssh / "authorized_keys" + if target.exists(): + current = read_regular(target, MAX_AUTHORIZED_KEYS) + if current.value != key + b"\n": + backup_once(target, current, MAX_AUTHORIZED_KEYS) + else: + current = FileSnapshot( + value=b"", + device=0, + inode=0, + mode=0o600, + uid=ACCOUNT_UID, + gid=ACCOUNT_GID, + size=0, + mtime_ns=0, + ctime_ns=0, + xattrs=(), + ) + atomic_write(target, key + b"\n", current) + if _read_regular(target, MAX_AUTHORIZED_KEYS)[0] != key + b"\n": + raise HardeningError("dedicated Hermes authorized key validation failed") + + +def _decode_acl(value: bytes, mode: int) -> list[tuple[int, int, int]]: + """Decode a bounded POSIX ACL or derive one from ordinary mode bits.""" + if not value: + group = (mode >> 3) & 0o7 + return [ + (ACL_USER_OBJ, (mode >> 6) & 0o7, ACL_UNDEFINED_ID), + (ACL_GROUP_OBJ, group, ACL_UNDEFINED_ID), + (ACL_MASK, group, ACL_UNDEFINED_ID), + (ACL_OTHER, mode & 0o7, ACL_UNDEFINED_ID), + ] + if len(value) < ACL_HEADER.size or (len(value) - ACL_HEADER.size) % ACL_ENTRY.size: + raise HardeningError("sensitive directory ACL is malformed") + if ACL_HEADER.unpack_from(value)[0] != ACL_VERSION: + raise HardeningError("sensitive directory ACL version is unsupported") + entries = [ + ACL_ENTRY.unpack_from(value, offset) + for offset in range(ACL_HEADER.size, len(value), ACL_ENTRY.size) + ] + if any(permission > 0o7 for _tag, permission, _identifier in entries): + raise HardeningError("sensitive directory ACL permission is malformed") + required = {ACL_USER_OBJ, ACL_GROUP_OBJ, ACL_OTHER} + if not required <= {tag for tag, _permission, _identifier in entries}: + raise HardeningError("sensitive directory ACL is incomplete") + return entries + + +def _encode_acl(entries: list[tuple[int, int, int]]) -> bytes: + return ACL_HEADER.pack(ACL_VERSION) + b"".join( + ACL_ENTRY.pack(*entry) for entry in entries + ) + + +def _acl_with_deny(value: bytes, mode: int) -> bytes: + entries = _decode_acl(value, mode) + entries = [ + entry + for entry in entries + if not (entry[0] == ACL_USER and entry[2] == ACCOUNT_UID) + ] + entries.append((ACL_USER, 0, ACCOUNT_UID)) + if not any(tag == ACL_MASK for tag, _permission, _identifier in entries): + entries.append((ACL_MASK, (mode >> 3) & 0o7, ACL_UNDEFINED_ID)) + order = { + ACL_USER_OBJ: 0, + ACL_USER: 1, + ACL_GROUP_OBJ: 2, + ACL_GROUP: 3, + ACL_MASK: 4, + ACL_OTHER: 5, + } + entries.sort(key=lambda entry: (order.get(entry[0], 99), entry[2])) + return _encode_acl(entries) + + +def _read_acl(path: Path) -> bytes: + try: + return os.getxattr(path, ACL_XATTR, follow_symlinks=False) + except OSError as exc: + if exc.errno in {errno.ENODATA, getattr(errno, "ENOATTR", errno.ENODATA)}: + return b"" + raise + + +def _validate_acl_backup(value: bytes) -> None: + if value[:1] not in {b"A", b"N"}: + raise HardeningError("sensitive directory ACL backup is malformed") + if value[:1] == b"A": + if len(value) == 1: + raise HardeningError("sensitive directory ACL backup is malformed") + _decode_acl(value[1:], 0) + elif value != b"N": + raise HardeningError("sensitive directory ACL backup is malformed") + + +def _acl_backup_once(path: Path, value: bytes, backup_name: str | None = None) -> None: + """Persist the original ACL as a private, durable host recovery file.""" + root = HOST_ETC / "hermes-node-boundary" + _directory(root, mode=0o700, uid=HOST_ROOT_UID, gid=HOST_ROOT_GID) + backup = root / f"{backup_name or path.name}.acl" + encoded = b"A" + value if value else b"N" + try: + existing, _metadata = _read_regular(backup, 64 * 1024) + except FileNotFoundError: + pass + else: + _validate_acl_backup(existing) + return + + temporary = root / f".{path.name}.acl.hermes-{os.getpid()}" + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(temporary, flags, 0o600) + try: + if os.write(descriptor, encoded) != len(encoded): + raise HardeningError("short sensitive directory ACL backup write") + os.fchmod(descriptor, 0o600) + os.fchown(descriptor, HOST_ROOT_UID, HOST_ROOT_GID) + os.fsync(descriptor) + except Exception: + temporary.unlink(missing_ok=True) + raise + finally: + os.close(descriptor) + try: + os.link(temporary, backup, follow_symlinks=False) + except FileExistsError: + # Another reconciler won the race; never overwrite the first backup. + pass + finally: + temporary.unlink(missing_ok=True) + stored, _metadata = _read_regular(backup, 64 * 1024) + _validate_acl_backup(stored) + directory = os.open(root, os.O_RDONLY) + try: + os.fsync(directory) + finally: + os.close(directory) + + +def _deny_sensitive_root(path: Path, backup_name: str | None = None) -> None: + metadata = path.lstat() + if not stat.S_ISDIR(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode): + raise HardeningError(f"unsafe sensitive directory: {path.name}") + if metadata.st_uid != HOST_ROOT_UID: + raise HardeningError(f"sensitive directory is not root-owned: {path.name}") + current = _read_acl(path) + _acl_backup_once(path, current, backup_name) + updated = _acl_with_deny(current, stat.S_IMODE(metadata.st_mode)) + try: + os.setxattr(path, ACL_XATTR, updated, follow_symlinks=False) + verified = _decode_acl(_read_acl(path), stat.S_IMODE(metadata.st_mode)) + if (ACL_USER, 0, ACCOUNT_UID) not in verified: + raise HardeningError("sensitive directory ACL validation failed") + except Exception: + if current: + os.setxattr(path, ACL_XATTR, current, follow_symlinks=False) + else: + try: + os.removexattr(path, ACL_XATTR, follow_symlinks=False) + except OSError as exc: + if exc.errno not in { + errno.ENODATA, + getattr(errno, "ENOATTR", errno.ENODATA), + }: + raise + raise + + +def _deny_sensitive_roots() -> None: + for path, backup_name in ( + (HOST_K3S, "var-lib-rancher-k3s"), + (HOST_KUBELET, "var-lib-kubelet"), + (HOST_RUN_K3S, "run-k3s"), + (HOST_RUN_CONTAINERD, "run-containerd"), + ): + _deny_sensitive_root(path, backup_name) + + +def reconcile(public_key: Path) -> None: + """Create the locked account and move only the Hermes authorization key.""" + with account_lock(HOST_ETC, expected_uid=HOST_ROOT_UID): + _reconcile_databases() + _deny_sensitive_roots() + _move_key(public_key) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--public-key-file", type=Path, required=True) + args = parser.parse_args() + reconcile(args.public_key_file) + print("Dedicated Hermes node account reconciled.", flush=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/hermes/scripts/node_account_io.py b/services/hermes/scripts/node_account_io.py new file mode 100644 index 00000000..4fb2ae9e --- /dev/null +++ b/services/hermes/scripts/node_account_io.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +"""Crash-safe, metadata-preserving host account database I/O.""" + +from __future__ import annotations + +import fcntl +import os +import stat +import time +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Iterator + +MAX_XATTRS = 64 +MAX_XATTR_NAME = 255 +MAX_XATTR_VALUE = 256 * 1024 +MAX_XATTR_TOTAL = 1024 * 1024 +LOCK_TIMEOUT_SECONDS = 15.0 + + +class HardeningError(RuntimeError): + """Raised before an unsafe or ambiguous host-account change.""" + + +@dataclass(frozen=True) +class FileSnapshot: + """Bounded file contents and all security-relevant inode metadata.""" + + value: bytes + device: int + inode: int + mode: int + uid: int + gid: int + size: int + mtime_ns: int + ctime_ns: int + xattrs: tuple[tuple[str, bytes], ...] + + +def _stat_identity(item: os.stat_result) -> tuple[int, ...]: + return ( + item.st_dev, + item.st_ino, + item.st_mode, + item.st_uid, + item.st_gid, + item.st_size, + item.st_mtime_ns, + item.st_ctime_ns, + ) + + +def _xattrs(path_or_fd: Path | int) -> tuple[tuple[str, bytes], ...]: + try: + names = sorted(os.listxattr(path_or_fd, follow_symlinks=False)) + except (TypeError, ValueError): + names = sorted(os.listxattr(path_or_fd)) + if len(names) > MAX_XATTRS: + raise HardeningError("host account file has too many extended attributes") + values: list[tuple[str, bytes]] = [] + total = 0 + for name in names: + if len(name.encode("utf-8")) > MAX_XATTR_NAME: + raise HardeningError("host account file xattr name is too long") + try: + value = os.getxattr(path_or_fd, name, follow_symlinks=False) + except (TypeError, ValueError): + value = os.getxattr(path_or_fd, name) + total += len(value) + if len(value) > MAX_XATTR_VALUE or total > MAX_XATTR_TOTAL: + raise HardeningError("host account file xattrs exceed the safe limit") + values.append((name, value)) + return tuple(values) + + +def _restore_xattrs(descriptor: int, expected: tuple[tuple[str, bytes], ...]) -> None: + expected_names = {name for name, _value in expected} + for name, _value in _xattrs(descriptor): + if name not in expected_names: + os.removexattr(descriptor, name) + for name, value in expected: + os.setxattr(descriptor, name, value) + + +def read_regular(path: Path, maximum: int) -> FileSnapshot: + """Read one bounded, non-symlink regular file and capture its metadata.""" + flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(path, flags) + try: + metadata = os.fstat(descriptor) + if not stat.S_ISREG(metadata.st_mode) or metadata.st_size > maximum: + raise HardeningError(f"unsafe regular file: {path.name}") + value = os.read(descriptor, maximum + 1) + attrs = _xattrs(descriptor) + after = os.fstat(descriptor) + finally: + os.close(descriptor) + if len(value) != metadata.st_size or _stat_identity(metadata) != _stat_identity(after): + raise HardeningError(f"short file read: {path.name}") + return FileSnapshot( + value=value, + device=metadata.st_dev, + inode=metadata.st_ino, + mode=stat.S_IMODE(metadata.st_mode), + uid=metadata.st_uid, + gid=metadata.st_gid, + size=metadata.st_size, + mtime_ns=metadata.st_mtime_ns, + ctime_ns=metadata.st_ctime_ns, + xattrs=attrs, + ) + + +def assert_unchanged(path: Path, expected: FileSnapshot, maximum: int) -> None: + """Fail if an account file changed since planning began.""" + current = read_regular(path, maximum) + if current != expected: + raise HardeningError(f"concurrent host account change detected: {path.name}") + + +def atomic_write(path: Path, value: bytes, metadata: FileSnapshot) -> None: + """Atomically replace a file while retaining ACLs, labels, and xattrs.""" + temporary = path.with_name(f".{path.name}.hermes-{os.getpid()}") + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(temporary, flags, metadata.mode) + try: + if os.write(descriptor, value) != len(value): + raise HardeningError(f"short atomic write: {path.name}") + os.fchmod(descriptor, metadata.mode) + os.fchown(descriptor, metadata.uid, metadata.gid) + _restore_xattrs(descriptor, metadata.xattrs) + os.fsync(descriptor) + except Exception: + temporary.unlink(missing_ok=True) + raise + finally: + os.close(descriptor) + os.replace(temporary, path) + directory = os.open(path.parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) + try: + os.fsync(directory) + finally: + os.close(directory) + + +def backup_once(path: Path, snapshot: FileSnapshot, maximum: int) -> Path: + """Create the first durable recovery copy without replacing an older one.""" + backup = path.with_name(path.name + ".hermes-boundary-backup") + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(backup, flags, snapshot.mode) + except FileExistsError: + read_regular(backup, maximum) + return backup + try: + if os.write(descriptor, snapshot.value) != len(snapshot.value): + raise HardeningError(f"short backup write: {path.name}") + os.fchmod(descriptor, snapshot.mode) + os.fchown(descriptor, snapshot.uid, snapshot.gid) + _restore_xattrs(descriptor, snapshot.xattrs) + os.fsync(descriptor) + except Exception: + backup.unlink(missing_ok=True) + raise + finally: + os.close(descriptor) + directory = os.open(path.parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) + try: + os.fsync(directory) + finally: + os.close(directory) + return backup + + +@contextmanager +def account_lock(host_etc: Path, *, expected_uid: int = 0) -> Iterator[None]: + """Hold the standard shadow-utils account lock for the full transaction.""" + path = host_etc / ".pwd.lock" + flags = os.O_WRONLY | os.O_CREAT | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(path, flags, 0o600) + try: + metadata = os.fstat(descriptor) + if ( + not stat.S_ISREG(metadata.st_mode) + or metadata.st_uid != expected_uid + or stat.S_IMODE(metadata.st_mode) != 0o600 + ): + raise HardeningError("unsafe standard account lock") + deadline = time.monotonic() + LOCK_TIMEOUT_SECONDS + while True: + try: + fcntl.lockf(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except BlockingIOError as exc: + if time.monotonic() >= deadline: + raise HardeningError("standard account lock is busy") from exc + time.sleep(0.05) + yield + finally: + try: + fcntl.lockf(descriptor, fcntl.LOCK_UN) + finally: + os.close(descriptor) diff --git a/services/hermes/scripts/stage_runtime_access.py b/services/hermes/scripts/stage_runtime_access.py index f4d73160..07423d70 100644 --- a/services/hermes/scripts/stage_runtime_access.py +++ b/services/hermes/scripts/stage_runtime_access.py @@ -86,6 +86,21 @@ def _write_empty_auth_store() -> None: os.chown(path, OWNER_UID, OWNER_GID) +def _stage_node_ssh_config() -> None: + """Force every Atlas node alias onto the dedicated Hermes OS account.""" + destination = RUNTIME_ROOT / "node-ssh-config" + source = _copy_secret("node-ssh-config", destination) + if "\x00" in source: + destination.unlink(missing_ok=True) + raise RuntimeError("node SSH config contains invalid control data") + destination.write_text( + "Host titan-*\n User hermes-agent\n" + source.rstrip() + "\n", + encoding="utf-8", + ) + destination.chmod(0o600) + os.chown(destination, OWNER_UID, OWNER_GID) + + def stage_agent() -> None: """Stage the owner agent's complete runtime access set.""" for path in (RUNTIME_ROOT, RUNTIME_ROOT / "claude", RUNTIME_ROOT / "codex"): @@ -93,13 +108,11 @@ def stage_agent() -> None: for name in ( "agent-api-key", "chat-relay-key", - "gitea-token", - "gitea-username", "node-ssh-private-key", - "node-ssh-config", "node-ssh-known-hosts", ): _copy_secret(name, RUNTIME_ROOT / name) + _stage_node_ssh_config() _validated_json( "claude-credentials", RUNTIME_ROOT / "claude" / ".credentials.json", diff --git a/services/hermes/skills/manage-atlas-pull-requests/SKILL.md b/services/hermes/skills/manage-atlas-pull-requests/SKILL.md index 32fd1ccf..6194f7f9 100644 --- a/services/hermes/skills/manage-atlas-pull-requests/SKILL.md +++ b/services/hermes/skills/manage-atlas-pull-requests/SKILL.md @@ -5,17 +5,21 @@ description: Read bounded private Atlas repository and pull-request metadata or # Manage Atlas pull requests -Use `/opt/coordinator/gitea_api.py` for Forgejo API access. It reads its token -from the pod-lifetime Vault projection; never read that file, copy the token, -put it in an argument or environment variable, or replace this client with -`curl`. +Use `/opt/scm/gitea_api.py` for Forgejo API access. The agent pod has no +repository credential. The client calls a separate least-authority broker; +never bypass it with `curl` or direct Gitea HTTP. + +Use the configured broker Git remote for clone, fetch, and creation of a new +namespaced feature branch. Existing-ref updates, protected refs, deletion, +and force-push are rejected by the broker. Never replace that remote with a +credential-bearing URL. ## Read repository or PR state Pass one allowed Atlas metadata API path to the read operation: ```sh -/opt/coordinator/gitea_api.py read /api/v1/repos/atlas/REPO/pulls/NUMBER +/opt/scm/gitea_api.py read /api/v1/repos/atlas/REPO/pulls/NUMBER ``` The client exposes only repository metadata, PRs and PR evidence, branches, @@ -29,10 +33,16 @@ evidence, not authorization to change it. Before opening a PR, verify the remote, branch, clean worktree, diff, tests, and exact pushed commit. Never force-push. Pass that full 40-character pushed head SHA explicitly. Keep the short body free of credentials and credential-like -text. Validate without reading a credential or using the network: +text. The client screens structured assignments, known token formats, and long +high-entropy values as an accidental-disclosure barrier. It cannot prove text +is secret-free: short or multiword secrets under innocuous names may resemble +ordinary prose, so never place credential material in either field. Structured +inspection is deliberately bounded; large, deeply nested, or unusual config +blobs do not belong in these fields and may be rejected. Validate without +reading a credential or using the network: ```sh -/opt/coordinator/gitea_api.py --dry-run create-draft REPO \ +/opt/scm/gitea_api.py --dry-run create-draft REPO \ --base main --head hermes/TASK --head-sha FULL_PUSHED_SHA \ --title "Focused change" --body "Tests and review evidence" ``` diff --git a/services/vault/hermes-auth-role-bootstrap-job.yaml b/services/vault/hermes-auth-role-bootstrap-job.yaml index 07c223f0..7804a9f3 100644 --- a/services/vault/hermes-auth-role-bootstrap-job.yaml +++ b/services/vault/hermes-auth-role-bootstrap-job.yaml @@ -3,7 +3,7 @@ apiVersion: batch/v1 kind: Job metadata: - name: vault-k8s-auth-hermes-8 + name: vault-k8s-auth-hermes-9 namespace: vault spec: backoffLimit: 2 diff --git a/services/vault/scripts/vault_k8s_auth_configure.sh b/services/vault/scripts/vault_k8s_auth_configure.sh index 73572e62..612391c9 100644 --- a/services/vault/scripts/vault_k8s_auth_configure.sh +++ b/services/vault/scripts/vault_k8s_auth_configure.sh @@ -256,7 +256,9 @@ write_policy_and_role "game-stream" "game-stream" "game-stream-vault" \ write_policy_and_role "hermes" "hermes" "hermes-vault,hermes-triage" \ "hermes/triage-oidc hermes/agent-tokens hermes/triage-api" "" write_policy_and_role "hermes-agent" "hermes" "hermes-agent,hermes-switchyard" \ - "hermes/agent-oidc hermes/agent-tokens hermes/chat-telegram hermes/developer-keycloak hermes/developer-gitea hermes/developer-harbor hermes/developer-jenkins hermes/developer-ssh" "" + "hermes/agent-oidc hermes/agent-tokens hermes/chat-telegram hermes/developer-keycloak hermes/developer-harbor hermes/developer-jenkins hermes/developer-ssh" "" +write_policy_and_role "hermes-scm-broker" "hermes-scm" "hermes-scm-broker" \ + "hermes/developer-gitea" "" write_policy_and_role "hermes-credential-sync" "hermes" "hermes-agent" \ "" "hermes/agent-tokens" write_policy_and_role "hermes-node-ssh" "hermes" "hermes-node-ssh-access" \ diff --git a/testing/quality_contract.json b/testing/quality_contract.json index e8793d84..805857d9 100644 --- a/testing/quality_contract.json +++ b/testing/quality_contract.json @@ -20,6 +20,18 @@ "ci/scripts/supply_chain_report.py", "services/mailu/scripts/mailu_sync.py", "services/mailu/scripts/mailu_sync_listener.py", + "services/gitea/scripts/gitea_branch_protection_check.py", + "services/hermes/scm-common/scripts/gitea_api.py", + "services/hermes/scm-common/scripts/gitea_api_policy.py", + "services/hermes/scm-common/scripts/scm_broker.py", + "services/hermes/scm-common/scripts/scm_broker_client.py", + "services/hermes/scm-common/scripts/scm_broker_io.py", + "services/hermes/scm-common/scripts/scm_broker_server.py", + "services/hermes/scripts/hermes_coordinator.py", + "services/hermes/scripts/node_account_audit.py", + "services/hermes/scripts/node_account_hardening.py", + "services/hermes/scripts/node_account_io.py", + "services/hermes/scripts/stage_runtime_access.py", "testing/__init__.py", "testing/quality_contract.py", "testing/quality_docs.py", @@ -43,6 +55,13 @@ "services/comms/scripts/tests", "services/mailu/scripts/mailu_sync.py", "services/mailu/scripts/mailu_sync_listener.py", + "services/gitea/scripts/gitea_branch_protection_check.py", + "services/hermes/scm-common/scripts", + "services/hermes/scripts/hermes_coordinator.py", + "services/hermes/scripts/node_account_audit.py", + "services/hermes/scripts/node_account_hardening.py", + "services/hermes/scripts/node_account_io.py", + "services/hermes/scripts/stage_runtime_access.py", "testing/tests", "testing" ], @@ -58,6 +77,9 @@ "coverage_sources": [ "ci/scripts", "scripts.render.dashboards_render_atlas", + "services/gitea/scripts", + "services/hermes/scm-common/scripts", + "services/hermes/scripts", "services/mailu/scripts", "testing" ], @@ -124,6 +146,13 @@ "ci/tests/**/*.py", "scripts/tests/**/*.py", "services/*/scripts/tests/**/*.py", + "services/gitea/scripts/gitea_branch_protection_check.py", + "services/hermes/scm-common/scripts/*.py", + "services/hermes/scripts/hermes_coordinator.py", + "services/hermes/scripts/node_account_audit.py", + "services/hermes/scripts/node_account_hardening.py", + "services/hermes/scripts/node_account_io.py", + "services/hermes/scripts/stage_runtime_access.py", "services/mailu/scripts/mailu_sync.py", "services/mailu/scripts/mailu_sync_listener.py" ], @@ -167,6 +196,22 @@ }, "coverage": { "minimum_percent": 95.0, + "minimum_branch_percent": 95.0, + "branch_tracked_files": [ + "services/gitea/scripts/gitea_branch_protection_check.py", + "services/hermes/scm-common/scripts/gitea_api.py", + "services/hermes/scm-common/scripts/gitea_api_policy.py", + "services/hermes/scm-common/scripts/scm_broker.py", + "services/hermes/scm-common/scripts/scm_broker_client.py", + "services/hermes/scm-common/scripts/scm_broker_io.py", + "services/hermes/scm-common/scripts/scm_broker_server.py", + "services/hermes/scripts/hermes_coordinator.py", + "services/hermes/scripts/node_account_audit.py", + "services/hermes/scripts/node_account_hardening.py", + "services/hermes/scripts/node_account_io.py", + "services/hermes/scripts/stage_runtime_access.py", + "testing/quality_coverage.py" + ], "tracked_files": [ "ci/scripts/publish_test_metrics.py", "ci/scripts/publish_test_metrics_quality.py", @@ -174,6 +219,18 @@ "ci/scripts/supply_chain_report.py", "services/mailu/scripts/mailu_sync.py", "services/mailu/scripts/mailu_sync_listener.py", + "services/gitea/scripts/gitea_branch_protection_check.py", + "services/hermes/scm-common/scripts/gitea_api.py", + "services/hermes/scm-common/scripts/gitea_api_policy.py", + "services/hermes/scm-common/scripts/scm_broker.py", + "services/hermes/scm-common/scripts/scm_broker_client.py", + "services/hermes/scm-common/scripts/scm_broker_io.py", + "services/hermes/scm-common/scripts/scm_broker_server.py", + "services/hermes/scripts/hermes_coordinator.py", + "services/hermes/scripts/node_account_audit.py", + "services/hermes/scripts/node_account_hardening.py", + "services/hermes/scripts/node_account_io.py", + "services/hermes/scripts/stage_runtime_access.py", "testing/quality_contract.py", "testing/quality_docs.py", "testing/quality_hygiene.py", diff --git a/testing/quality_coverage.py b/testing/quality_coverage.py index 78d6649f..98cc65d5 100644 --- a/testing/quality_coverage.py +++ b/testing/quality_coverage.py @@ -3,23 +3,31 @@ from __future__ import annotations import xml.etree.ElementTree as ET +from dataclasses import dataclass from pathlib import Path from typing import Any -def _load_percentages(xml_path: Path, root: Path) -> dict[str, float]: - """Load per-file line-rate percentages from a Cobertura XML report.""" +@dataclass(frozen=True) +class CoverageRates: + """Line and branch percentages reported for one source file.""" + + line: float + branch: float | None + + +def _load_percentages(xml_path: Path, root: Path) -> dict[str, CoverageRates]: + """Load per-file line and branch percentages from a Cobertura report.""" tree = ET.parse(xml_path) xml_root = tree.getroot() source_roots = [ - Path(node.text) - for node in xml_root.findall("./sources/source") - if node.text + Path(node.text) for node in xml_root.findall("./sources/source") if node.text ] - percentages: dict[str, float] = {} + percentages: dict[str, CoverageRates] = {} for class_node in xml_root.findall(".//class"): filename = class_node.attrib.get("filename") line_rate = class_node.attrib.get("line-rate") + branch_rate = class_node.attrib.get("branch-rate") if not filename or line_rate is None: continue normalized = filename.replace("\\", "/") @@ -32,7 +40,10 @@ def _load_percentages(xml_path: Path, root: Path) -> dict[str, float]: if candidate.exists(): key = candidate.relative_to(root).as_posix() break - percentages[key] = float(line_rate) * 100.0 + percentages[key] = CoverageRates( + line=float(line_rate) * 100.0, + branch=float(branch_rate) * 100.0 if branch_rate is not None else None, + ) return percentages @@ -47,17 +58,39 @@ def run_check(contract: dict[str, Any], root: Path, xml_path: Path) -> list[str] percentages = _load_percentages(xml_path, root) minimum = float(contract.get("coverage", {}).get("minimum_percent", 95.0)) + coverage_contract = contract.get("coverage", {}) + branch_minimum = coverage_contract.get("minimum_branch_percent") + if branch_minimum is not None: + branch_minimum = float(branch_minimum) + branch_tracked = { + path.replace("\\", "/") + for path in coverage_contract.get( + "branch_tracked_files", + coverage_contract.get("tracked_files", []) + if branch_minimum is not None + else [], + ) + } issues: list[str] = [] for relative_path in contract.get("coverage", {}).get("tracked_files", []): normalized = relative_path.replace("\\", "/") - percent = percentages.get(normalized) - if percent is None: + rates = percentages.get(normalized) + if rates is None: issues.append(f"coverage missing for tracked file: {relative_path}") continue - if percent + 1e-9 < minimum: + if rates.line + 1e-9 < minimum: issues.append( - f"coverage below {minimum:.1f}%: {relative_path} ({percent:.1f}%)" + f"coverage below {minimum:.1f}%: {relative_path} ({rates.line:.1f}%)" + ) + if branch_minimum is None or normalized not in branch_tracked: + continue + if rates.branch is None: + issues.append(f"branch coverage missing for tracked file: {relative_path}") + elif rates.branch + 1e-9 < branch_minimum: + issues.append( + "branch coverage below " + f"{branch_minimum:.1f}%: {relative_path} ({rates.branch:.1f}%)" ) return issues @@ -77,9 +110,9 @@ def compute_workspace_line_coverage( samples: list[float] = [] for relative_path in contract.get("coverage", {}).get("tracked_files", []): normalized = relative_path.replace("\\", "/") - percent = percentages.get(normalized) - if percent is not None: - samples.append(percent) + rates = percentages.get(normalized) + if rates is not None: + samples.append(rates.line) if not samples: return 0.0 return round(sum(samples) / len(samples), 3) diff --git a/testing/tests/test_hermes_chat_images.py b/testing/tests/test_hermes_chat_images.py new file mode 100644 index 00000000..48c6c2e5 --- /dev/null +++ b/testing/tests/test_hermes_chat_images.py @@ -0,0 +1,282 @@ +"""Image generation and sandbox-boundary contracts for Hermes chat.""" + +from __future__ import annotations + +import importlib.util +import sys +import time +from types import SimpleNamespace + +import yaml + +from testing.tests.test_hermes_chat_support import ( + HERMES, + VAULT, + _documents, +) + + +def test_chat_image_generation_uses_private_owner_broker(): + """Family pods get image bytes without receiving the owner's OAuth file.""" + configmap = _documents(HERMES / "chat-configmap.yaml")[0] + assert "shared desktop/Wolf lane" in configmap["data"]["SOUL.md"] + assert "local FLUX waits" in configmap["data"]["SOUL.md"] + assert "Use `image_generate_local`" in configmap["data"]["SOUL.md"] + assert "Use `image_generate_hosted`" in configmap["data"]["SOUL.md"] + assert "ComfyUI endpoint" in configmap["data"]["SOUL.md"] + assert "`MEDIA:` path" in configmap["data"]["SOUL.md"] + assert "`image_edit_latest`" in configmap["data"]["SOUL.md"] + assert ( + "Preserve the most recently selected image lane" in configmap["data"]["SOUL.md"] + ) + config = yaml.safe_load(configmap["data"]["config.yaml"]) + assert config["image_gen"] == { + "provider": "atlas-broker", + "model": "atlas-image-auto-high", + } + assert config["plugins"]["enabled"] == ["atlas-broker", "auto-router"] + + statefulset = _documents(HERMES / "chat-statefulset.yaml")[0] + pod = statefulset["spec"]["template"]["spec"] + hermes = next(item for item in pod["containers"] if item["name"] == "hermes") + mounts = {item["name"]: item for item in hermes["volumeMounts"]} + assert mounts["image-plugin"]["mountPath"] == ( + "/opt/hermes/plugins/image_gen/atlas-broker" + ) + assert mounts["runtime-access"]["mountPath"] == "/runtime-access" + assert "provider-auth" not in mounts + assert not any( + mount["name"] == "home" and "agent" in str(mount) + for mount in hermes["volumeMounts"] + ) + env = {item["name"]: item["value"] for item in hermes["env"]} + assert env["HERMES_IMAGE_BROKER_URL"].startswith("http://hermes-image-broker.") + + plugin = (HERMES / "plugins" / "image-gen-broker" / "__init__.py").read_text() + assert '"local": "flux-2-klein-4b-local"' in plugin + assert '"hosted": "gpt-image-2-high"' in plugin + assert 'name="image_generate_local"' in plugin + assert 'name="image_generate_hosted"' in plugin + assert '"name": "image_edit_latest"' in plugin + assert '"name": "image_edit_latest_local"' in plugin + assert '"name": "image_edit_latest_hosted"' in plugin + assert "def _latest_generated_image" in plugin + assert "newest MEDIA: path from the conversation" in plugin + assert 'candidate.upper().startswith("MEDIA:")' in plugin + assert "override=True" not in plugin + + agent = _documents(HERMES / "agent-deployment.yaml")[0] + containers = agent["spec"]["template"]["spec"]["containers"] + broker = next(item for item in containers if item["name"] == "image-broker") + assert broker["ports"] == [ + {"name": "image-broker", "containerPort": 9002, "protocol": "TCP"} + ] + assert broker["securityContext"]["readOnlyRootFilesystem"] is True + assert broker["securityContext"]["runAsNonRoot"] is True + + services = _documents(HERMES / "service.yaml") + service = next( + item for item in services if item["metadata"]["name"] == "hermes-image-broker" + ) + assert service["spec"]["selector"] == {"app": "hermes-agent"} + + oauth_store = _documents(HERMES / "oauth-session-store.yaml") + redis = next(item for item in oauth_store if item["kind"] == "Deployment") + assert redis["spec"]["strategy"]["type"] == "Recreate" + assert "--appendonly" in redis["spec"]["template"]["spec"]["containers"][0]["args"] + + policies = _documents(HERMES / "networkpolicy.yaml") + agent_policy = next( + item + for item in policies + if item["metadata"]["name"] == "hermes-agent-isolation" + ) + broker_ingress = next( + rule + for rule in agent_policy["spec"]["ingress"] + if {port["port"] for port in rule["ports"]} == {9002, 9003} + ) + assert broker_ingress["from"][0]["podSelector"]["matchLabels"] == { + "app": "hermes-chat-tenant" + } + vault_policy = (VAULT / "scripts" / "vault_k8s_auth_configure.sh").read_text() + agent_role = vault_policy[ + vault_policy.index('write_policy_and_role "hermes-agent"') : vault_policy.index( + "write_policy_and_role", + vault_policy.index('write_policy_and_role "hermes-agent"') + 1, + ) + ] + assert "hermes/developer-gitea" not in agent_role + assert 'write_policy_and_role "hermes-scm-broker" "hermes-scm"' in vault_policy + assert ( + 'write_policy_and_role "hermes-node-ssh" "hermes" ' + '"hermes-node-ssh-access"' in vault_policy + ) + + +def test_compact_image_edit_resolves_latest_tenant_artifact(tmp_path, monkeypatch): + """Follow-up edits resolve the source server-side and keep tool JSON small.""" + provider_module = SimpleNamespace( + DEFAULT_ASPECT_RATIO="square", + ImageGenProvider=object, + error_response=lambda **value: value, + normalize_reference_images=lambda value: value, + resolve_aspect_ratio=lambda value: value, + save_b64_image=lambda *_args, **_kwargs: tmp_path / "saved.png", + success_response=lambda **value: value, + ) + monkeypatch.setitem(sys.modules, "agent", SimpleNamespace()) + monkeypatch.setitem(sys.modules, "agent.image_gen_provider", provider_module) + spec = importlib.util.spec_from_file_location( + "hermes_image_plugin", + HERMES / "plugins" / "image-gen-broker" / "__init__.py", + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + image_dir = tmp_path / "cache" / "images" + image_dir.mkdir(parents=True) + older = image_dir / "atlas_flux-old.png" + newest = image_dir / "atlas_gpt-image-new.png" + older.write_bytes(b"older") + newest.write_bytes(b"newest") + older.touch() + time.sleep(0.001) + newest.touch() + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + + calls = [] + monkeypatch.setattr( + module, + "_handle_image_generate", + lambda args, route: calls.append((args, route)) or "ok", + ) + assert module._handle_hosted_edit({"prompt": "make it a clown"}) == "ok" + assert calls == [ + ( + { + "prompt": "make it a clown", + "image_url": str(newest.resolve()), + }, + "hosted", + ) + ] + + +def test_chat_reasoning_uses_switchyard_without_owner_credentials(): + """Family pods use AUTO/manual routes without mounting owner credentials.""" + configmap = _documents(HERMES / "chat-configmap.yaml")[0] + config = yaml.safe_load(configmap["data"]["config.yaml"]) + assert config["model"] == { + "provider": "atlas-switchyard", + "default": "atlas/auto/fast", + "model": "atlas/auto/fast", + } + assert config["providers"]["atlas-switchyard"] == { + "name": "Automatic Router", + "api": "http://hermes-switchyard.hermes.svc.cluster.local:9005/v1", + "api_key": "atlas-switchyard", + "default_model": "atlas/auto/fast", + "transport": "chat_completions", + } + assert config["platforms"]["api_server"]["extra"]["model_routes"] == { + route: {"provider": "atlas-switchyard", "model": route} + for route in [ + "atlas/auto/fast", + "atlas/auto/balanced", + "atlas/auto/deep", + "atlas/auto/maximum", + "atlas/manual/codex/luna", + "atlas/manual/codex/terra", + "atlas/manual/codex/sol", + "atlas/manual/claude/haiku", + "atlas/manual/claude/fable", + "atlas/manual/claude/sonnet", + "atlas/manual/claude/opus", + "atlas/manual/local/qwen-14b", + ] + } + + agent = _documents(HERMES / "agent-deployment.yaml")[0] + containers = agent["spec"]["template"]["spec"]["containers"] + broker = next(item for item in containers if item["name"] == "codex-broker") + assert broker["ports"] == [ + {"name": "codex-broker", "containerPort": 9003, "protocol": "TCP"} + ] + assert broker["securityContext"]["readOnlyRootFilesystem"] is True + assert broker["securityContext"]["runAsNonRoot"] is True + assert {item["name"]: item["value"] for item in broker["env"]}.items() >= { + "PYTHONPATH": "/opt/hermes", + "HERMES_CODEX_BROKER_LISTEN_PORT": "9003", + "HERMES_ROUTING_CATALOG_PATH": "/routing-catalog/catalog.json", + }.items() + + services = _documents(HERMES / "service.yaml") + service = next( + item for item in services if item["metadata"]["name"] == "hermes-codex-broker" + ) + assert service["spec"]["selector"] == {"app": "hermes-agent"} + assert service["spec"]["ports"] == [ + { + "name": "http", + "port": 9003, + "targetPort": "codex-broker", + "protocol": "TCP", + } + ] + + statefulset = _documents(HERMES / "chat-statefulset.yaml")[0] + assert ( + statefulset["spec"]["template"]["metadata"]["annotations"][ + "ai.bstein.dev/config-rev" + ] + == "20260816-telegram-topics" + ) + pod_spec = statefulset["spec"]["template"]["spec"] + patch_init = next( + item + for item in pod_spec["initContainers"] + if item["name"] == "patch-stream-recovery" + ) + assert patch_init["command"][-2:] == [ + "/opt/hermes/agent/conversation_loop.py", + "/patched/conversation_loop.py", + ] + hermes = next(item for item in pod_spec["containers"] if item["name"] == "hermes") + assert { + "name": "stream-recovery-patch", + "mountPath": "/opt/hermes/agent/conversation_loop.py", + "subPath": "conversation_loop.py", + } in hermes["volumeMounts"] + api_session_init = next( + item + for item in pod_spec["initContainers"] + if item["name"] == "patch-api-server-sessions" + ) + assert "patch_api_server_sessions.py" in api_session_init["args"][0] + assert "migrate_telegram_api_sessions.py" in api_session_init["args"][0] + assert { + "name": "api-server-patch", + "mountPath": "/opt/hermes/gateway/platforms/api_server.py", + "subPath": "api_server.py", + } in hermes["volumeMounts"] + assert any(volume["name"] == "api-server-patch" for volume in pod_spec["volumes"]) + assert not any( + mount["mountPath"].endswith("/.codex") for mount in hermes["volumeMounts"] + ) + + policies = _documents(HERMES / "networkpolicy.yaml") + agent_policy = next( + item + for item in policies + if item["metadata"]["name"] == "hermes-agent-isolation" + ) + broker_ingress = next( + rule + for rule in agent_policy["spec"]["ingress"] + if {port["port"] for port in rule["ports"]} == {9002, 9003} + ) + assert broker_ingress["from"][0]["podSelector"]["matchLabels"] == { + "app": "hermes-chat-tenant" + } diff --git a/testing/tests/test_hermes_chat_local_runtime.py b/testing/tests/test_hermes_chat_local_runtime.py new file mode 100644 index 00000000..ddb0ef40 --- /dev/null +++ b/testing/tests/test_hermes_chat_local_runtime.py @@ -0,0 +1,172 @@ +"""Local inference and sandbox execution contracts for Hermes chat.""" + +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path +from types import SimpleNamespace + + +from testing.tests.test_hermes_chat_support import ( + HERMES, + ROOT, + _documents, +) + + +def test_titan20_serializes_classifier_and_local_chat_model_residency(): + """Classifier and local chat share one serialized resident Qwen weight.""" + deployment = _documents( + Path(__file__).parents[2] / "services/ai-llm/deployment.yaml" + )[0] + pod = deployment["spec"]["template"]["spec"] + required = pod["affinity"]["nodeAffinity"][ + "requiredDuringSchedulingIgnoredDuringExecution" + ]["nodeSelectorTerms"][0]["matchExpressions"][0] + assert required["values"] == ["titan-20"] + container = pod["containers"][0] + env = {item["name"]: item["value"] for item in container["env"]} + assert env["OLLAMA_MAX_LOADED_MODELS"] == "1" + assert env["OLLAMA_NUM_PARALLEL"] == "1" + assert env["OLLAMA_KEEP_ALIVE"] == "-1" + assert env["OLLAMA_CONTEXT_LENGTH"] == "8192" + warm_command = " ".join(container["command"]) + assert "--keepalive=-1" not in warm_command + models = next(item for item in pod["volumes"] if item["name"] == "models") + assert models["persistentVolumeClaim"]["claimName"] == ("ollama-models-titan20") + + +def test_local_image_gpu_guard_distinguishes_background_and_saturated_gpu(monkeypatch): + """Lease-idle desktop spikes may coexist, but saturation still blocks FLUX.""" + source = ROOT / "dockerfiles" / "hermes-local-image-server.py" + spec = importlib.util.spec_from_file_location("hermes_local_image_server", source) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + monkeypatch.setattr(module, "GPU_ACTIVITY_NODE", "titan-24") + monkeypatch.setattr(module, "GPU_ACTIVE_SM_PERCENT", 80.0) + monkeypatch.setattr(module, "GPU_MAX_EXTERNAL_MEMORY_BYTES", 3 << 30) + + idle = module._parse_gpu_activity( + "\n".join( + [ + 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="Xorg"} 0', + 'nvidia_process_gpu_memory_used_bytes{node="titan-24",namespace="host",process="Xorg"} 1900000000', + 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="game-stream",process="wolf"} 3', + 'nvidia_process_gpu_memory_used_bytes{node="titan-24",namespace="hermes",process="python"} 9000000000', + ] + ) + ) + assert idle["interactive_active"] is False + assert idle["external_gpu_memory_bytes"] == 1900000000 + assert idle["external_gpu_sm_percent"] == 3 + + background_spike = module._parse_gpu_activity( + 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="sway"} 41\n' + ) + assert background_spike["interactive_active"] is False + + active = module._parse_gpu_activity( + 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="steam"} 91\n' + ) + assert active["interactive_active"] is True + assert "91%" in active["gpu_guard_reason"] + + +def test_local_flux_renderer_uses_a_disposable_cuda_worker(monkeypatch): + """A completed render must not leave its CUDA context in the API process.""" + source = ROOT / "dockerfiles" / "hermes-local-image-server.py" + spec = importlib.util.spec_from_file_location("hermes_local_image_worker", source) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + calls = [] + + def run(command, **kwargs): + calls.append((command, kwargs)) + return SimpleNamespace( + returncode=0, + stdout=b'{"success":true,"route":"local","image_b64":"cG5n"}', + stderr=b"", + ) + + monkeypatch.setattr(module.subprocess, "run", run) + result = module._render({"prompt": "black cat", "aspect_ratio": "square"}) + + assert result["route"] == "local" + command, options = calls[0] + assert command[-1] == "--render-worker" + assert json.loads(options["input"]) == { + "prompt": "black cat", + "aspect_ratio": "square", + } + assert options["timeout"] == module.RENDER_TIMEOUT_SECONDS + assert options["check"] is False + + +def test_local_flux_uses_low_vram_offload_without_reducing_resolution(): + """The shared 3080 lane must trade time, not image size, for headroom.""" + source = (ROOT / "dockerfiles" / "hermes-local-image-server.py").read_text() + assert "OFFLOAD_MODE = os.environ.get(" in source + assert '"HERMES_LOCAL_IMAGE_OFFLOAD_MODE", "sequential"' in source + assert "pipe.enable_sequential_cpu_offload()" in source + assert '"square": (1024, 1024)' in source + + +def test_chat_auth_and_relay_are_pod_lifetime_only(): + statefulset = _documents(HERMES / "chat-statefulset.yaml")[0] + pod = statefulset["spec"]["template"]["spec"] + containers = statefulset["spec"]["template"]["spec"]["containers"] + + for name in ("hermes", "webui"): + container = next(item for item in containers if item["name"] == name) + env = {item["name"]: item["value"] for item in container["env"]} + assert env["HERMES_AUTH_FILE"] == "/runtime-access/hermes-auth.json" + mount = next( + item + for item in container["volumeMounts"] + if item["name"] == "runtime-access" + ) + assert mount["mountPath"] == "/runtime-access" + assert "subPath" not in mount + + hermes_env = { + item["name"]: item["value"] + for item in next(item for item in containers if item["name"] == "hermes")["env"] + } + runtime = next(item for item in pod["volumes"] if item["name"] == "runtime-access") + assert runtime["emptyDir"] == {"medium": "Memory", "sizeLimit": "2Mi"} + assert not any(item["name"] == "provider-auth" for item in pod["volumes"]) + init_command = next( + item for item in pod["initContainers"] if item["name"] == "init-config" + )["command"][2] + for key in ( + "ANTHROPIC_API_KEY", + "API_SERVER_KEY", + "CLAUDE_CODE_OAUTH_TOKEN", + "GITEA_TOKEN", + "HERMES_IMAGE_BROKER_KEY", + "OPENAI_API_KEY", + ): + assert key in init_command + assert "printf 'API_SERVER_KEY=%s" not in init_command + assert hermes_env["AGENT_BROWSER_EXECUTABLE_PATH"].endswith( + "/chrome-linux/headless_shell" + ) + assert "--no-sandbox" in hermes_env["AGENT_BROWSER_ARGS"] + + +def test_sandbox_executes_python_with_bounded_output(tmp_path: Path, monkeypatch): + source = ROOT / "dockerfiles" / "hermes-chat-sandbox-server.py" + spec = importlib.util.spec_from_file_location("hermes_chat_sandbox_server", source) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + monkeypatch.setattr(module, "WORKSPACE", tmp_path) + + result = module._execute("import math\nprint(math.comb(10, 3))") + + assert result["success"] is True + assert result["stdout"] == "120\n" + assert result["stderr"] == "" diff --git a/testing/tests/test_hermes_chat_provider_auth.py b/testing/tests/test_hermes_chat_provider_auth.py new file mode 100644 index 00000000..8e432131 --- /dev/null +++ b/testing/tests/test_hermes_chat_provider_auth.py @@ -0,0 +1,476 @@ +"""Provider subscription and OAuth contracts for Hermes chat.""" + +from __future__ import annotations + +import base64 +import json +import sys +import time +from pathlib import Path +from types import ModuleType + +import pytest + +from testing.tests.test_hermes_chat_support import ( + _load_broker_module, +) + + +def test_codex_broker_auth_and_request_contract(tmp_path: Path, monkeypatch): + """The relay is bounded, stateless, and rejects unapproved models.""" + module = _load_broker_module("hermes_codex_broker", "codex_broker.py", monkeypatch) + monkeypatch.setattr(module, "TOKEN", "relay-secret") + + assert module._authorized("Bearer relay-secret") is True + assert module._authorized("Bearer wrong") is False + assert module._real_model("route/codex/gpt-5.6-sol/xhigh") == "gpt-5.6-sol" + payload = module._validate_payload( + { + "model": "gpt-5.6-terra", + "input": "route this chat turn", + "store": True, + "stream": False, + "max_output_tokens": 96, + "max_completion_tokens": 96, + "max_tokens": 96, + "temperature": 0.7, + "top_p": 0.9, + } + ) + assert payload["store"] is False + assert payload["stream"] is True + assert "max_output_tokens" not in payload + assert "max_completion_tokens" not in payload + assert "max_tokens" not in payload + assert "temperature" not in payload + assert "top_p" not in payload + assert payload["input"] == [ + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "route this chat turn"}], + } + ] + response_item = { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "keep this item"}], + } + assert module._validate_payload({"model": "gpt-5.6-terra", "input": response_item})[ + "input" + ] == [response_item] + response_items = [response_item] + assert ( + module._validate_payload({"model": "gpt-5.6-terra", "input": response_items})[ + "input" + ] + is response_items + ) + image_items = [ + { + "type": "message", + "role": "user", + "content": [ + {"type": "input_text", "text": "What color is this?"}, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,cHJpdmF0ZQ==", + "detail": "high", + }, + }, + ], + } + ] + assert module._validate_payload({"model": "gpt-5.6-terra", "input": image_items})[ + "input" + ][0]["content"][1] == { + "type": "input_image", + "image_url": "data:image/png;base64,cHJpdmF0ZQ==", + "detail": "high", + } + switchyard_image_items = [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "What color is this?"}, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,cHJpdmF0ZQ==", + "detail": "auto", + }, + }, + ], + } + ] + assert module._validate_payload( + {"model": "gpt-5.6-terra", "input": switchyard_image_items} + )["input"][0]["content"][1] == { + "type": "input_image", + "image_url": "data:image/png;base64,cHJpdmF0ZQ==", + "detail": "auto", + } + switchyard_base64_items = [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "What color is this?"}, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "cHJpdmF0ZQ==", + }, + }, + ], + } + ] + normalized_base64 = module._validate_payload( + {"model": "gpt-5.6-terra", "input": switchyard_base64_items} + )["input"][0]["content"][1] + assert normalized_base64 == { + "type": "input_image", + "image_url": "data:image/png;base64,cHJpdmF0ZQ==", + } + switchyard_enum_items = [ + { + "role": "user", + "content": [ + { + "type": "input_image", + "image_url": { + "type": "url", + "data": { + "url": "data:image/png;base64,cHJpdmF0ZQ==", + "detail": "high", + }, + }, + } + ], + } + ] + nested_image = module._validate_payload( + {"model": "gpt-5.6-terra", "input": switchyard_enum_items} + )["input"][0]["content"][0] + assert nested_image["image_url"] == "data:image/png;base64,cHJpdmF0ZQ==" + assert nested_image["detail"] == "high" + with pytest.raises(ValueError, match=r"non-empty Responses image URL.*str\[4\]"): + module._validate_payload( + { + "model": "gpt-5.6-terra", + "input": [ + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_image", + "image_url": {"detail": "high"}, + } + ], + } + ], + } + ) + routed = module._validate_payload( + { + "model": "route/codex/gpt-5.6-luna/low", + "input": "use the low route", + "stream": False, + } + ) + assert routed["model"] == "gpt-5.6-luna" + with pytest.raises(ValueError, match="unsupported Codex model"): + module._validate_payload({"model": "unapproved-model", "input": "hello"}) + with pytest.raises(ValueError, match="non-empty Responses input"): + module._validate_payload({"model": "gpt-5.6-terra", "input": ""}) + with pytest.raises(ValueError, match="non-empty Responses input list"): + module._validate_payload({"model": "gpt-5.6-terra", "input": []}) + + completed = { + "id": "resp_test", + "object": "response", + "status": "completed", + "output": [], + } + completed_item = { + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "done"}], + } + assert module._completed_response( + [ + "event: response.created", + 'data: {"type":"response.created","response":{}}', + "event: response.output_item.done", + "data: " + + json.dumps( + { + "type": "response.output_item.done", + "output_index": 0, + "item": completed_item, + } + ), + "event: response.completed", + "data: " + + json.dumps({"type": "response.completed", "response": completed}), + "data: [DONE]", + ] + )["output"] == [completed_item] + raw_stream = ( + "event: response.output_item.done\n" + "data: " + + json.dumps( + { + "type": "response.output_item.done", + "output_index": 0, + "item": completed_item, + } + ) + + "\n\nevent: response.completed\ndata: " + + json.dumps({"type": "response.completed", "response": completed}) + + "\n\n" + ).encode() + normalized = module._normalized_stream( + raw_stream, {**completed, "output": [completed_item]} + ).decode() + terminal_data = next( + line for line in normalized.splitlines() if '"response.completed"' in line + ) + assert json.loads(terminal_data.removeprefix("data: "))["response"]["output"] == [ + completed_item + ] + assert normalized.endswith("\n\n") + streamed_function_item = { + "type": "function_call", + "name": "read_file", + "status": "completed", + "arguments": '{"path":"/tmp"}', + } + streamed_function_body = ( + "event: response.function_call_arguments.delta\n" + 'data: {"type":"response.function_call_arguments.delta",' + '"item_id":"call_1","delta":"{\\"path\\":\\"/tmp\\"}"}\n\n' + "event: response.function_call_arguments.done\n" + 'data: {"type":"response.function_call_arguments.done",' + '"item_id":"call_1","arguments":"{\\"path\\":\\"/tmp\\"}"}\n\n' + "event: response.output_item.done\n" + 'data: {"type":"response.output_item.done","output_index":0,' + '"item":{"type":"function_call","name":"read_file",' + '"arguments":"{\\"path\\":\\"/tmp\\"}"}}\n\n' + "event: response.completed\n" + "data: " + + json.dumps( + { + "type": "response.completed", + "response": {**completed, "output": [streamed_function_item]}, + } + ) + + "\n\n" + ).encode() + normalized_function_stream = module._normalized_stream( + streamed_function_body, {**completed, "output": [streamed_function_item]} + ).decode() + assert "response.function_call_arguments.delta" in normalized_function_stream + assert "response.function_call_arguments.done" not in normalized_function_stream + assert "response.output_item.done" not in normalized_function_stream + normalized_terminal = next( + line + for line in normalized_function_stream.splitlines() + if '"response.completed"' in line + ) + assert ( + json.loads(normalized_terminal.removeprefix("data: "))["response"]["output"] + == [] + ) + with pytest.raises(RuntimeError, match="retryable incomplete response"): + module._completed_response( + [ + "event: response.incomplete", + 'data: {"type":"response.incomplete","response":' + '{"status":"incomplete","incomplete_details":' + '{"reason":"max_output_tokens"}}}', + ] + ) + with pytest.raises(RuntimeError, match="provider unavailable"): + module._completed_response( + [ + "event: error", + 'data: {"type":"error","error":{"message":"provider unavailable"}}', + ] + ) + malformed_tool_item = { + "type": "function_call", + "name": "search_files", + "status": "completed", + "arguments": '{"path":"","offset":', + } + with pytest.raises(RuntimeError, match="malformed function arguments"): + module._completed_response( + [ + "event: response.output_item.done", + "data: " + + json.dumps( + { + "type": "response.output_item.done", + "output_index": 0, + "item": malformed_tool_item, + } + ), + "event: response.completed", + "data: " + + json.dumps({"type": "response.completed", "response": completed}), + ] + ) + valid_tool_item = { + **malformed_tool_item, + "arguments": '{"path":"","offset":0}', + } + assert module._completed_response( + [ + "event: response.output_item.done", + "data: " + + json.dumps( + { + "type": "response.output_item.done", + "output_index": 0, + "item": valid_tool_item, + } + ), + "event: response.completed", + "data: " + + json.dumps({"type": "response.completed", "response": completed}), + ] + )["output"] == [valid_tool_item] + with pytest.raises(RuntimeError, match="malformed function arguments"): + module._completed_response( + [ + "event: response.function_call_arguments.delta", + 'data: {"type":"response.function_call_arguments.delta",' + '"item_id":"call_1","output_index":0,' + '"delta":"{\\"path\\":\\"/tmp\\",\\"offset\\":"}', + "event: response.completed", + "data: " + + json.dumps({"type": "response.completed", "response": completed}), + ] + ) + streamed_tool = module._completed_response( + [ + "event: response.function_call_arguments.delta", + 'data: {"type":"response.function_call_arguments.delta",' + '"item_id":"call_2","output_index":0,' + '"delta":"{\\"path\\":\\"/tmp\\",\\"offset\\":"}', + "event: response.function_call_arguments.done", + 'data: {"type":"response.function_call_arguments.done",' + '"item_id":"call_2","output_index":0,' + '"arguments":"{\\"path\\":\\"/tmp\\",\\"offset\\":0}"}', + "event: response.completed", + "data: " + + json.dumps({"type": "response.completed", "response": completed}), + ] + ) + assert streamed_tool["status"] == "completed" + + auth_dir = tmp_path / ".codex" + auth_dir.mkdir() + # The token payload need only prove the broker reads CODEX_HOME directly. + encoded = ( + base64.urlsafe_b64encode(json.dumps({"exp": time.time() + 3600}).encode()) + .decode() + .rstrip("=") + ) + (auth_dir / "auth.json").write_text( + json.dumps({"tokens": {"access_token": f"header.{encoded}.signature"}}) + ) + monkeypatch.setenv("CODEX_HOME", str(auth_dir)) + assert module._access_token().startswith("header.") + + +def test_codex_broker_refreshes_and_persists_first_party_oauth( + tmp_path: Path, monkeypatch +): + """Expired ChatGPT OAuth refreshes in the canonical Codex CLI store.""" + module = _load_broker_module( + "hermes_codex_refresh_broker", "codex_broker.py", monkeypatch + ) + auth_dir = tmp_path / ".codex" + auth_dir.mkdir() + + def jwt(expires_at: float) -> str: + payload = ( + base64.urlsafe_b64encode(json.dumps({"exp": expires_at}).encode()) + .decode() + .rstrip("=") + ) + return f"header.{payload}.signature" + + expired = jwt(time.time() - 60) + live = jwt(time.time() + 3600) + auth_path = auth_dir / "auth.json" + auth_path.write_text( + json.dumps( + { + "auth_mode": "chatgpt", + "tokens": { + "access_token": expired, + "refresh_token": "refresh-old", + }, + } + ) + ) + calls = [] + auth_module = ModuleType("hermes_cli.auth") + + def refresh(access_token, refresh_token, *, timeout_seconds): + calls.append((access_token, refresh_token, timeout_seconds)) + return { + "access_token": live, + "refresh_token": "refresh-new", + "last_refresh": "2026-08-12T20:00:00Z", + } + + auth_module.refresh_codex_oauth_pure = refresh + package = ModuleType("hermes_cli") + package.auth = auth_module + monkeypatch.setitem(sys.modules, "hermes_cli", package) + monkeypatch.setitem(sys.modules, "hermes_cli.auth", auth_module) + monkeypatch.setenv("CODEX_HOME", str(auth_dir)) + + assert module._access_token() == live + persisted = json.loads(auth_path.read_text()) + assert persisted["tokens"]["access_token"] == live + assert persisted["tokens"]["refresh_token"] == "refresh-new" + assert persisted["last_refresh"] == "2026-08-12T20:00:00Z" + assert calls == [(expired, "refresh-old", 30.0)] + assert auth_path.stat().st_mode & 0o777 == 0o600 + + # A healthy token is reused, so repeated routed turns do not spend a + # refresh token or create a second billing/authentication path. + assert module._access_token() == live + assert len(calls) == 1 + + +def test_claude_broker_uses_native_subscription_without_api_billing(monkeypatch): + """Claude traffic must use the native first-party CLI subscription lane.""" + module = _load_broker_module( + "hermes_claude_broker", "claude_oauth_broker.py", monkeypatch + ) + monkeypatch.setenv("ANTHROPIC_API_KEY", "must-not-leak") + monkeypatch.setenv("CLAUDE_API_KEY", "must-not-leak") + monkeypatch.setattr( + module, + "resolve_route", + lambda route: "claude-fable-5" if "/fable/" in route else route, + ) + + model, effort = module._route( + "route/claude/fable/xhigh", {"output_config": {"effort": "xhigh"}} + ) + + assert (model, effort) == ("claude-fable-5", "xhigh") + assert "ANTHROPIC_API_KEY" not in module._claude_environment() + assert "CLAUDE_API_KEY" not in module._claude_environment() + assert module.CAPACITY_PATTERN.search("weekly usage limit exhausted") diff --git a/testing/tests/test_hermes_chat_quality.py b/testing/tests/test_hermes_chat_quality.py index 025f3bf7..b513b452 100644 --- a/testing/tests/test_hermes_chat_quality.py +++ b/testing/tests/test_hermes_chat_quality.py @@ -1,49 +1,18 @@ -"""Contracts for isolated high-quality Hermes chat capabilities.""" +"""Core configuration and workload contracts for Hermes chat.""" from __future__ import annotations -import base64 import importlib.util -import json -import sqlite3 import sys -import time -import tomllib -from pathlib import Path -from types import ModuleType, SimpleNamespace +from types import SimpleNamespace -import pytest import yaml - -ROOT = Path(__file__).parents[2] -HERMES = ROOT / "services" / "hermes" -VAULT = ROOT / "services" / "vault" - - -def _documents(path: Path) -> list[dict]: - return [doc for doc in yaml.safe_load_all(path.read_text()) if doc] - - -def _load_broker_module(name: str, filename: str, monkeypatch): - """Load one broker with its mounted routing-catalog dependency.""" - catalog_path = HERMES / "scripts" / "routing_catalog.py" - catalog_spec = importlib.util.spec_from_file_location( - "routing_catalog", catalog_path - ) - assert catalog_spec and catalog_spec.loader - catalog = importlib.util.module_from_spec(catalog_spec) - catalog_spec.loader.exec_module(catalog) - monkeypatch.setitem(sys.modules, "routing_catalog", catalog) - monkeypatch.setitem(sys.modules, "httpx", SimpleNamespace()) - - broker_path = HERMES / "scripts" / filename - spec = importlib.util.spec_from_file_location(name, broker_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - +from testing.tests.test_hermes_chat_support import ( + HERMES, + ROOT, + _documents, +) def test_chat_config_enables_real_research_compute_and_delegation(): configmap = _documents(HERMES / "chat-configmap.yaml")[0] @@ -92,13 +61,14 @@ def test_agent_config_keeps_delegated_reviewers_from_owning_task_lifecycle(): assert "Atlas organization has private visibility" in instructions assert "may be public or private" in instructions assert "do not infer a\nrepository's visibility" in instructions - assert "already supplied through `GIT_ASKPASS`" in instructions + assert "brokered Git for clone, fetch" in instructions + assert "client carries no repository credential" in instructions assert "Never call `kanban_show` without a known, non-empty task ID" in instructions assert "bounded ad-hoc inspection and acceptance checks may" in instructions assert "load implementation or TDD skills" in instructions assert "Never call `kanban_show` without\na known, non-empty task ID" in soul assert "must load a skill only when its workflow\nmaterially applies" in soul - assert "runtime-only `GIT_ASKPASS`" in soul + assert "no repository token is present in this pod" in soul assert "`scm.bstein.dev` is Forgejo/Gitea, not GitHub" in instructions assert "GitHub/`gh` skill for an Atlas remote" in instructions assert "load\n`$manage-atlas-pull-requests`" in instructions @@ -484,1765 +454,3 @@ def test_voice_workloads_have_deliberate_xavier_placement(): assert tts["containers"][0]["resources"]["limits"]["cpu"] == "4" assert all("hostPath" not in volume for volume in stt["volumes"]) assert all("hostPath" not in volume for volume in tts["volumes"]) - - -def test_chat_image_generation_uses_private_owner_broker(): - """Family pods get image bytes without receiving the owner's OAuth file.""" - configmap = _documents(HERMES / "chat-configmap.yaml")[0] - assert "shared desktop/Wolf lane" in configmap["data"]["SOUL.md"] - assert "local FLUX waits" in configmap["data"]["SOUL.md"] - assert "Use `image_generate_local`" in configmap["data"]["SOUL.md"] - assert "Use `image_generate_hosted`" in configmap["data"]["SOUL.md"] - assert "ComfyUI endpoint" in configmap["data"]["SOUL.md"] - assert "`MEDIA:` path" in configmap["data"]["SOUL.md"] - assert "`image_edit_latest`" in configmap["data"]["SOUL.md"] - assert "Preserve the most recently selected image lane" in configmap["data"]["SOUL.md"] - config = yaml.safe_load(configmap["data"]["config.yaml"]) - assert config["image_gen"] == { - "provider": "atlas-broker", - "model": "atlas-image-auto-high", - } - assert config["plugins"]["enabled"] == ["atlas-broker", "auto-router"] - - statefulset = _documents(HERMES / "chat-statefulset.yaml")[0] - pod = statefulset["spec"]["template"]["spec"] - hermes = next(item for item in pod["containers"] if item["name"] == "hermes") - mounts = {item["name"]: item for item in hermes["volumeMounts"]} - assert mounts["image-plugin"]["mountPath"] == ( - "/opt/hermes/plugins/image_gen/atlas-broker" - ) - assert mounts["runtime-access"]["mountPath"] == "/runtime-access" - assert "provider-auth" not in mounts - assert not any(mount["name"] == "home" and "agent" in str(mount) for mount in hermes["volumeMounts"]) - env = {item["name"]: item["value"] for item in hermes["env"]} - assert env["HERMES_IMAGE_BROKER_URL"].startswith("http://hermes-image-broker.") - - plugin = (HERMES / "plugins" / "image-gen-broker" / "__init__.py").read_text() - assert '"local": "flux-2-klein-4b-local"' in plugin - assert '"hosted": "gpt-image-2-high"' in plugin - assert 'name="image_generate_local"' in plugin - assert 'name="image_generate_hosted"' in plugin - assert '"name": "image_edit_latest"' in plugin - assert '"name": "image_edit_latest_local"' in plugin - assert '"name": "image_edit_latest_hosted"' in plugin - assert "def _latest_generated_image" in plugin - assert "newest MEDIA: path from the conversation" in plugin - assert 'candidate.upper().startswith("MEDIA:")' in plugin - assert "override=True" not in plugin - - agent = _documents(HERMES / "agent-deployment.yaml")[0] - containers = agent["spec"]["template"]["spec"]["containers"] - broker = next(item for item in containers if item["name"] == "image-broker") - assert broker["ports"] == [ - {"name": "image-broker", "containerPort": 9002, "protocol": "TCP"} - ] - assert broker["securityContext"]["readOnlyRootFilesystem"] is True - assert broker["securityContext"]["runAsNonRoot"] is True - - services = _documents(HERMES / "service.yaml") - service = next( - item for item in services if item["metadata"]["name"] == "hermes-image-broker" - ) - assert service["spec"]["selector"] == {"app": "hermes-agent"} - - oauth_store = _documents(HERMES / "oauth-session-store.yaml") - redis = next(item for item in oauth_store if item["kind"] == "Deployment") - assert redis["spec"]["strategy"]["type"] == "Recreate" - assert "--appendonly" in redis["spec"]["template"]["spec"]["containers"][0]["args"] - - policies = _documents(HERMES / "networkpolicy.yaml") - agent_policy = next( - item for item in policies if item["metadata"]["name"] == "hermes-agent-isolation" - ) - broker_ingress = next( - rule - for rule in agent_policy["spec"]["ingress"] - if {port["port"] for port in rule["ports"]} == {9002, 9003} - ) - assert broker_ingress["from"][0]["podSelector"]["matchLabels"] == { - "app": "hermes-chat-tenant" - } - vault_policy = (VAULT / "scripts" / "vault_k8s_auth_configure.sh").read_text() - assert ( - '"hermes/agent-oidc hermes/agent-tokens hermes/chat-telegram ' - 'hermes/developer-keycloak hermes/developer-gitea ' - 'hermes/developer-harbor hermes/developer-jenkins ' - 'hermes/developer-ssh"' - in vault_policy - ) - assert ( - 'write_policy_and_role "hermes-node-ssh" "hermes" ' - '"hermes-node-ssh-access"' in vault_policy - ) - - -def test_compact_image_edit_resolves_latest_tenant_artifact(tmp_path, monkeypatch): - """Follow-up edits resolve the source server-side and keep tool JSON small.""" - provider_module = SimpleNamespace( - DEFAULT_ASPECT_RATIO="square", - ImageGenProvider=object, - error_response=lambda **value: value, - normalize_reference_images=lambda value: value, - resolve_aspect_ratio=lambda value: value, - save_b64_image=lambda *_args, **_kwargs: tmp_path / "saved.png", - success_response=lambda **value: value, - ) - monkeypatch.setitem(sys.modules, "agent", SimpleNamespace()) - monkeypatch.setitem(sys.modules, "agent.image_gen_provider", provider_module) - spec = importlib.util.spec_from_file_location( - "hermes_image_plugin", - HERMES / "plugins" / "image-gen-broker" / "__init__.py", - ) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - - image_dir = tmp_path / "cache" / "images" - image_dir.mkdir(parents=True) - older = image_dir / "atlas_flux-old.png" - newest = image_dir / "atlas_gpt-image-new.png" - older.write_bytes(b"older") - newest.write_bytes(b"newest") - older.touch() - time.sleep(0.001) - newest.touch() - monkeypatch.setenv("HERMES_HOME", str(tmp_path)) - - calls = [] - monkeypatch.setattr( - module, - "_handle_image_generate", - lambda args, route: calls.append((args, route)) or "ok", - ) - assert module._handle_hosted_edit({"prompt": "make it a clown"}) == "ok" - assert calls == [ - ( - { - "prompt": "make it a clown", - "image_url": str(newest.resolve()), - }, - "hosted", - ) - ] - - -def test_chat_reasoning_uses_switchyard_without_owner_credentials(): - """Family pods use AUTO/manual routes without mounting owner credentials.""" - configmap = _documents(HERMES / "chat-configmap.yaml")[0] - config = yaml.safe_load(configmap["data"]["config.yaml"]) - assert config["model"] == { - "provider": "atlas-switchyard", - "default": "atlas/auto/fast", - "model": "atlas/auto/fast", - } - assert config["providers"]["atlas-switchyard"] == { - "name": "Automatic Router", - "api": "http://hermes-switchyard.hermes.svc.cluster.local:9005/v1", - "api_key": "atlas-switchyard", - "default_model": "atlas/auto/fast", - "transport": "chat_completions", - } - assert config["platforms"]["api_server"]["extra"]["model_routes"] == { - route: {"provider": "atlas-switchyard", "model": route} - for route in [ - "atlas/auto/fast", - "atlas/auto/balanced", - "atlas/auto/deep", - "atlas/auto/maximum", - "atlas/manual/codex/luna", - "atlas/manual/codex/terra", - "atlas/manual/codex/sol", - "atlas/manual/claude/haiku", - "atlas/manual/claude/fable", - "atlas/manual/claude/sonnet", - "atlas/manual/claude/opus", - "atlas/manual/local/qwen-14b", - ] - } - - agent = _documents(HERMES / "agent-deployment.yaml")[0] - containers = agent["spec"]["template"]["spec"]["containers"] - broker = next(item for item in containers if item["name"] == "codex-broker") - assert broker["ports"] == [ - {"name": "codex-broker", "containerPort": 9003, "protocol": "TCP"} - ] - assert broker["securityContext"]["readOnlyRootFilesystem"] is True - assert broker["securityContext"]["runAsNonRoot"] is True - assert {item["name"]: item["value"] for item in broker["env"]}.items() >= { - "PYTHONPATH": "/opt/hermes", - "HERMES_CODEX_BROKER_LISTEN_PORT": "9003", - "HERMES_ROUTING_CATALOG_PATH": "/routing-catalog/catalog.json", - }.items() - - services = _documents(HERMES / "service.yaml") - service = next( - item for item in services if item["metadata"]["name"] == "hermes-codex-broker" - ) - assert service["spec"]["selector"] == {"app": "hermes-agent"} - assert service["spec"]["ports"] == [ - { - "name": "http", - "port": 9003, - "targetPort": "codex-broker", - "protocol": "TCP", - } - ] - - statefulset = _documents(HERMES / "chat-statefulset.yaml")[0] - assert statefulset["spec"]["template"]["metadata"]["annotations"][ - "ai.bstein.dev/config-rev" - ] == "20260816-telegram-topics" - pod_spec = statefulset["spec"]["template"]["spec"] - patch_init = next( - item for item in pod_spec["initContainers"] - if item["name"] == "patch-stream-recovery" - ) - assert patch_init["command"][-2:] == [ - "/opt/hermes/agent/conversation_loop.py", - "/patched/conversation_loop.py", - ] - hermes = next( - item - for item in pod_spec["containers"] - if item["name"] == "hermes" - ) - assert { - "name": "stream-recovery-patch", - "mountPath": "/opt/hermes/agent/conversation_loop.py", - "subPath": "conversation_loop.py", - } in hermes["volumeMounts"] - api_session_init = next( - item for item in pod_spec["initContainers"] - if item["name"] == "patch-api-server-sessions" - ) - assert "patch_api_server_sessions.py" in api_session_init["args"][0] - assert "migrate_telegram_api_sessions.py" in api_session_init["args"][0] - assert { - "name": "api-server-patch", - "mountPath": "/opt/hermes/gateway/platforms/api_server.py", - "subPath": "api_server.py", - } in hermes["volumeMounts"] - assert any( - volume["name"] == "api-server-patch" for volume in pod_spec["volumes"] - ) - assert not any( - mount["mountPath"].endswith("/.codex") - for mount in hermes["volumeMounts"] - ) - - policies = _documents(HERMES / "networkpolicy.yaml") - agent_policy = next( - item for item in policies if item["metadata"]["name"] == "hermes-agent-isolation" - ) - broker_ingress = next( - rule - for rule in agent_policy["spec"]["ingress"] - if {port["port"] for port in rule["ports"]} == {9002, 9003} - ) - assert broker_ingress["from"][0]["podSelector"]["matchLabels"] == { - "app": "hermes-chat-tenant" - } - - -def test_codex_broker_auth_and_request_contract(tmp_path: Path, monkeypatch): - """The relay is bounded, stateless, and rejects unapproved models.""" - module = _load_broker_module( - "hermes_codex_broker", "codex_broker.py", monkeypatch - ) - monkeypatch.setattr(module, "TOKEN", "relay-secret") - - assert module._authorized("Bearer relay-secret") is True - assert module._authorized("Bearer wrong") is False - assert module._real_model("route/codex/gpt-5.6-sol/xhigh") == "gpt-5.6-sol" - payload = module._validate_payload( - { - "model": "gpt-5.6-terra", - "input": "route this chat turn", - "store": True, - "stream": False, - "max_output_tokens": 96, - "max_completion_tokens": 96, - "max_tokens": 96, - "temperature": 0.7, - "top_p": 0.9, - } - ) - assert payload["store"] is False - assert payload["stream"] is True - assert "max_output_tokens" not in payload - assert "max_completion_tokens" not in payload - assert "max_tokens" not in payload - assert "temperature" not in payload - assert "top_p" not in payload - assert payload["input"] == [ - { - "type": "message", - "role": "user", - "content": [{"type": "input_text", "text": "route this chat turn"}], - } - ] - response_item = { - "type": "message", - "role": "user", - "content": [{"type": "input_text", "text": "keep this item"}], - } - assert module._validate_payload( - {"model": "gpt-5.6-terra", "input": response_item} - )["input"] == [response_item] - response_items = [response_item] - assert module._validate_payload( - {"model": "gpt-5.6-terra", "input": response_items} - )["input"] is response_items - image_items = [ - { - "type": "message", - "role": "user", - "content": [ - {"type": "input_text", "text": "What color is this?"}, - { - "type": "image_url", - "image_url": { - "url": "data:image/png;base64,cHJpdmF0ZQ==", - "detail": "high", - }, - }, - ], - } - ] - assert module._validate_payload( - {"model": "gpt-5.6-terra", "input": image_items} - )["input"][0]["content"][1] == { - "type": "input_image", - "image_url": "data:image/png;base64,cHJpdmF0ZQ==", - "detail": "high", - } - switchyard_image_items = [ - { - "role": "user", - "content": [ - {"type": "input_text", "text": "What color is this?"}, - { - "type": "image_url", - "image_url": { - "url": "data:image/png;base64,cHJpdmF0ZQ==", - "detail": "auto", - }, - }, - ], - } - ] - assert module._validate_payload( - {"model": "gpt-5.6-terra", "input": switchyard_image_items} - )["input"][0]["content"][1] == { - "type": "input_image", - "image_url": "data:image/png;base64,cHJpdmF0ZQ==", - "detail": "auto", - } - switchyard_base64_items = [ - { - "role": "user", - "content": [ - {"type": "input_text", "text": "What color is this?"}, - { - "type": "image", - "source": { - "type": "base64", - "media_type": "image/png", - "data": "cHJpdmF0ZQ==", - }, - }, - ], - } - ] - normalized_base64 = module._validate_payload( - {"model": "gpt-5.6-terra", "input": switchyard_base64_items} - )["input"][0]["content"][1] - assert normalized_base64 == { - "type": "input_image", - "image_url": "data:image/png;base64,cHJpdmF0ZQ==", - } - switchyard_enum_items = [ - { - "role": "user", - "content": [ - { - "type": "input_image", - "image_url": { - "type": "url", - "data": { - "url": "data:image/png;base64,cHJpdmF0ZQ==", - "detail": "high", - }, - }, - } - ], - } - ] - nested_image = module._validate_payload( - {"model": "gpt-5.6-terra", "input": switchyard_enum_items} - )["input"][0]["content"][0] - assert nested_image["image_url"] == "data:image/png;base64,cHJpdmF0ZQ==" - assert nested_image["detail"] == "high" - with pytest.raises(ValueError, match=r"non-empty Responses image URL.*str\[4\]"): - module._validate_payload( - { - "model": "gpt-5.6-terra", - "input": [ - { - "type": "message", - "role": "user", - "content": [ - { - "type": "input_image", - "image_url": {"detail": "high"}, - } - ], - } - ], - } - ) - routed = module._validate_payload( - { - "model": "route/codex/gpt-5.6-luna/low", - "input": "use the low route", - "stream": False, - } - ) - assert routed["model"] == "gpt-5.6-luna" - with pytest.raises(ValueError, match="unsupported Codex model"): - module._validate_payload({"model": "unapproved-model", "input": "hello"}) - with pytest.raises(ValueError, match="non-empty Responses input"): - module._validate_payload({"model": "gpt-5.6-terra", "input": ""}) - with pytest.raises(ValueError, match="non-empty Responses input list"): - module._validate_payload({"model": "gpt-5.6-terra", "input": []}) - - completed = { - "id": "resp_test", - "object": "response", - "status": "completed", - "output": [], - } - completed_item = { - "type": "message", - "role": "assistant", - "status": "completed", - "content": [{"type": "output_text", "text": "done"}], - } - assert module._completed_response( - [ - "event: response.created", - 'data: {"type":"response.created","response":{}}', - "event: response.output_item.done", - "data: " - + json.dumps( - { - "type": "response.output_item.done", - "output_index": 0, - "item": completed_item, - } - ), - "event: response.completed", - "data: " - + json.dumps({"type": "response.completed", "response": completed}), - "data: [DONE]", - ] - )["output"] == [completed_item] - raw_stream = ( - "event: response.output_item.done\n" - "data: " - + json.dumps( - { - "type": "response.output_item.done", - "output_index": 0, - "item": completed_item, - } - ) - + "\n\nevent: response.completed\ndata: " - + json.dumps({"type": "response.completed", "response": completed}) - + "\n\n" - ).encode() - normalized = module._normalized_stream( - raw_stream, {**completed, "output": [completed_item]} - ).decode() - terminal_data = next( - line for line in normalized.splitlines() if '"response.completed"' in line - ) - assert json.loads(terminal_data.removeprefix("data: "))["response"][ - "output" - ] == [completed_item] - assert normalized.endswith("\n\n") - streamed_function_item = { - "type": "function_call", - "name": "read_file", - "status": "completed", - "arguments": '{"path":"/tmp"}', - } - streamed_function_body = ( - "event: response.function_call_arguments.delta\n" - 'data: {"type":"response.function_call_arguments.delta",' - '"item_id":"call_1","delta":"{\\"path\\":\\"/tmp\\"}"}\n\n' - "event: response.function_call_arguments.done\n" - 'data: {"type":"response.function_call_arguments.done",' - '"item_id":"call_1","arguments":"{\\"path\\":\\"/tmp\\"}"}\n\n' - "event: response.output_item.done\n" - 'data: {"type":"response.output_item.done","output_index":0,' - '"item":{"type":"function_call","name":"read_file",' - '"arguments":"{\\"path\\":\\"/tmp\\"}"}}\n\n' - "event: response.completed\n" - "data: " - + json.dumps( - { - "type": "response.completed", - "response": {**completed, "output": [streamed_function_item]}, - } - ) - + "\n\n" - ).encode() - normalized_function_stream = module._normalized_stream( - streamed_function_body, {**completed, "output": [streamed_function_item]} - ).decode() - assert "response.function_call_arguments.delta" in normalized_function_stream - assert "response.function_call_arguments.done" not in normalized_function_stream - assert "response.output_item.done" not in normalized_function_stream - normalized_terminal = next( - line - for line in normalized_function_stream.splitlines() - if '"response.completed"' in line - ) - assert json.loads(normalized_terminal.removeprefix("data: "))["response"][ - "output" - ] == [] - with pytest.raises(RuntimeError, match="retryable incomplete response"): - module._completed_response( - [ - "event: response.incomplete", - 'data: {"type":"response.incomplete","response":' - '{"status":"incomplete","incomplete_details":' - '{"reason":"max_output_tokens"}}}', - ] - ) - with pytest.raises(RuntimeError, match="provider unavailable"): - module._completed_response( - [ - "event: error", - 'data: {"type":"error","error":{"message":"provider unavailable"}}', - ] - ) - malformed_tool_item = { - "type": "function_call", - "name": "search_files", - "status": "completed", - "arguments": '{"path":"","offset":', - } - with pytest.raises(RuntimeError, match="malformed function arguments"): - module._completed_response( - [ - "event: response.output_item.done", - "data: " - + json.dumps( - { - "type": "response.output_item.done", - "output_index": 0, - "item": malformed_tool_item, - } - ), - "event: response.completed", - "data: " - + json.dumps({"type": "response.completed", "response": completed}), - ] - ) - valid_tool_item = { - **malformed_tool_item, - "arguments": '{"path":"","offset":0}', - } - assert module._completed_response( - [ - "event: response.output_item.done", - "data: " - + json.dumps( - { - "type": "response.output_item.done", - "output_index": 0, - "item": valid_tool_item, - } - ), - "event: response.completed", - "data: " - + json.dumps({"type": "response.completed", "response": completed}), - ] - )["output"] == [valid_tool_item] - with pytest.raises(RuntimeError, match="malformed function arguments"): - module._completed_response( - [ - "event: response.function_call_arguments.delta", - 'data: {"type":"response.function_call_arguments.delta",' - '"item_id":"call_1","output_index":0,' - '"delta":"{\\"path\\":\\"/tmp\\",\\"offset\\":"}', - "event: response.completed", - "data: " - + json.dumps({"type": "response.completed", "response": completed}), - ] - ) - streamed_tool = module._completed_response( - [ - "event: response.function_call_arguments.delta", - 'data: {"type":"response.function_call_arguments.delta",' - '"item_id":"call_2","output_index":0,' - '"delta":"{\\"path\\":\\"/tmp\\",\\"offset\\":"}', - "event: response.function_call_arguments.done", - 'data: {"type":"response.function_call_arguments.done",' - '"item_id":"call_2","output_index":0,' - '"arguments":"{\\"path\\":\\"/tmp\\",\\"offset\\":0}"}', - "event: response.completed", - "data: " - + json.dumps({"type": "response.completed", "response": completed}), - ] - ) - assert streamed_tool["status"] == "completed" - - auth_dir = tmp_path / ".codex" - auth_dir.mkdir() - # The token payload need only prove the broker reads CODEX_HOME directly. - encoded = base64.urlsafe_b64encode( - json.dumps({"exp": time.time() + 3600}).encode() - ).decode().rstrip("=") - (auth_dir / "auth.json").write_text( - json.dumps({"tokens": {"access_token": f"header.{encoded}.signature"}}) - ) - monkeypatch.setenv("CODEX_HOME", str(auth_dir)) - assert module._access_token().startswith("header.") - - -def test_codex_broker_refreshes_and_persists_first_party_oauth( - tmp_path: Path, monkeypatch -): - """Expired ChatGPT OAuth refreshes in the canonical Codex CLI store.""" - module = _load_broker_module( - "hermes_codex_refresh_broker", "codex_broker.py", monkeypatch - ) - auth_dir = tmp_path / ".codex" - auth_dir.mkdir() - - def jwt(expires_at: float) -> str: - payload = base64.urlsafe_b64encode( - json.dumps({"exp": expires_at}).encode() - ).decode().rstrip("=") - return f"header.{payload}.signature" - - expired = jwt(time.time() - 60) - live = jwt(time.time() + 3600) - auth_path = auth_dir / "auth.json" - auth_path.write_text( - json.dumps( - { - "auth_mode": "chatgpt", - "tokens": { - "access_token": expired, - "refresh_token": "refresh-old", - }, - } - ) - ) - calls = [] - auth_module = ModuleType("hermes_cli.auth") - - def refresh(access_token, refresh_token, *, timeout_seconds): - calls.append((access_token, refresh_token, timeout_seconds)) - return { - "access_token": live, - "refresh_token": "refresh-new", - "last_refresh": "2026-08-12T20:00:00Z", - } - - auth_module.refresh_codex_oauth_pure = refresh - package = ModuleType("hermes_cli") - package.auth = auth_module - monkeypatch.setitem(sys.modules, "hermes_cli", package) - monkeypatch.setitem(sys.modules, "hermes_cli.auth", auth_module) - monkeypatch.setenv("CODEX_HOME", str(auth_dir)) - - assert module._access_token() == live - persisted = json.loads(auth_path.read_text()) - assert persisted["tokens"]["access_token"] == live - assert persisted["tokens"]["refresh_token"] == "refresh-new" - assert persisted["last_refresh"] == "2026-08-12T20:00:00Z" - assert calls == [(expired, "refresh-old", 30.0)] - assert auth_path.stat().st_mode & 0o777 == 0o600 - - # A healthy token is reused, so repeated routed turns do not spend a - # refresh token or create a second billing/authentication path. - assert module._access_token() == live - assert len(calls) == 1 - - -def test_claude_broker_uses_native_subscription_without_api_billing(monkeypatch): - """Claude traffic must use the native first-party CLI subscription lane.""" - module = _load_broker_module( - "hermes_claude_broker", "claude_oauth_broker.py", monkeypatch - ) - monkeypatch.setenv("ANTHROPIC_API_KEY", "must-not-leak") - monkeypatch.setenv("CLAUDE_API_KEY", "must-not-leak") - monkeypatch.setattr( - module, - "resolve_route", - lambda route: "claude-fable-5" if "/fable/" in route else route, - ) - - model, effort = module._route( - "route/claude/fable/xhigh", {"output_config": {"effort": "xhigh"}} - ) - - assert (model, effort) == ("claude-fable-5", "xhigh") - assert "ANTHROPIC_API_KEY" not in module._claude_environment() - assert "CLAUDE_API_KEY" not in module._claude_environment() - assert module.CAPACITY_PATTERN.search("weekly usage limit exhausted") - - -def test_codex_native_health_overrides_historical_router_errors( - tmp_path: Path, monkeypatch -): - """Fresh first-party health is authoritative over old Switchyard probes.""" - plugin_path = HERMES / "plugins" / "auto-router" / "provider_status.py" - spec = importlib.util.spec_from_file_location("hermes_provider_status", plugin_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - health_path = tmp_path / "codex.json" - health_path.write_text( - json.dumps( - { - "state": "available", - "authenticated": True, - "transport": "codex-chatgpt-subscription", - } - ) - ) - monkeypatch.setattr(module, "CODEX_HEALTH_PATH", health_path) - monkeypatch.setattr(module, "CLAUDE_HEALTH_PATH", tmp_path / "missing.json") - monkeypatch.setattr( - module, - "_get_json", - lambda url: {"status": "ok"} - if url.endswith("/health") - else { - "models": { - "route/codex/terra/medium": { - "calls": 1, - "errors": 99, - "total_tokens": 12, - } - } - }, - ) - monkeypatch.setattr(module, "_codex_account", lambda: {}) - monkeypatch.setattr(module, "_claude_account", lambda: {}) - - codex = module.provider_status_payload()["providers"]["codex"] - - assert codex["errors"] == 99 - assert codex["state"] == "available" - assert codex["native_health"]["transport"] == "codex-chatgpt-subscription" - - -def test_api_session_patch_accepts_parent_lineage(tmp_path: Path): - """API-created workers must persist the originating Hermes session.""" - module_path = HERMES / "scripts" / "patch_api_server_sessions.py" - spec = importlib.util.spec_from_file_location("patch_api_sessions", module_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - source = tmp_path / "api_server.py" - destination = tmp_path / "patched.py" - source.write_text( - "prefix\n" - + module.BEFORE - + "middle\n" - + module.RUNS_BEFORE - + "run body\n" - + module.RUN_CLOSE_BEFORE - + module.RESPONSES_SESSION_BEFORE - + module.EVENT_CALLBACK_SIGNATURE_BEFORE - + "callback docstring and push helper\n" - + module.EVENT_CALLBACK_BODY_BEFORE - + "tool start body\n" - + module.EVENT_CALLBACK_END_BEFORE - + module.EVENT_CALLBACK_CALL_BEFORE - + module.RUN_SWEEP_BEFORE - + "suffix\n", - encoding="utf-8", - ) - - module.patch(source, destination) - patched = destination.read_text(encoding="utf-8") - - assert "X-Hermes-Parent-Session-Id" in patched - assert "parent_session_id=parent_session_id" in patched - assert "Parent session not found" in patched - assert "HERMES_API_DEFAULT_PARENT_MATCH_PREFIXES" in patched - assert "user_message.startswith(default_prefixes)" in patched - assert "session_parent_conflict" in patched - assert "X-Hermes-Conversation-Platform" in patched - assert "X-Hermes-Conversation-Title" in patched - assert 'conversation_platform != "telegram"' in patched - assert "db.record_gateway_session_peer(" in patched - assert 'display_name="Telegram"' in patched - assert "db.reopen_session(session_id)" in patched - assert 'db.end_session(session_id, f"api_run_{terminal_status}")' in patched - assert "def _record_run_activity(" in patched - assert '"_thinking": "Hermes is reasoning"' in patched - assert '"run.started": "Worker started"' in patched - assert '"run.completed": "Worker completed"' in patched - assert '"reasoning.available": "Hermes finished a reasoning step"' in patched - assert '"subagent.progress": "Nested worker progress"' in patched - assert "redact_sensitive_text" in patched - assert 'getattr(os, "O_NOFOLLOW", 0)' in patched - assert "os.fchmod(fd, 0o600)" in patched - assert "session_id=session_id" in patched - assert 'self._record_run_activity(session_id, "run.started")' in patched - assert 'detail = tool_name if event_type in {' in patched - assert 'if event_type == "subagent.tool"' in patched - assert "_RUN_ACTIVITY_HEARTBEAT_SECONDS = 15.0" in patched - assert 'heartbeats.get(session_id, 0.0)' in patched - assert '"subagent.thinking",' in patched - assert "Stream retention and run lifetime are separate" in patched - assert 'terminal_status in {"completed", "failed", "cancelled"}' in patched - assert patched.index("terminal_status = self._run_statuses") < patched.index( - "self._active_run_tasks.pop(run_id, None)" - ) - - -def test_telegram_api_session_migration_is_bounded_and_idempotent(tmp_path: Path): - """Only named Telegram conversations receive presentation metadata.""" - module_path = HERMES / "scripts" / "migrate_telegram_api_sessions.py" - spec = importlib.util.spec_from_file_location("migrate_telegram_sessions", module_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - state = tmp_path / "state.db" - responses = tmp_path / "response_store.db" - with sqlite3.connect(state) as connection: - connection.execute( - """CREATE TABLE sessions ( - id TEXT PRIMARY KEY, source TEXT, session_key TEXT, chat_type TEXT, - display_name TEXT, origin_json TEXT, title TEXT - )""" - ) - connection.executemany( - "INSERT INTO sessions (id, source) VALUES (?, ?)", - (("telegram-session", "api_server"), ("other-session", "api_server")), - ) - with sqlite3.connect(responses) as connection: - connection.execute( - "CREATE TABLE conversations (name TEXT PRIMARY KEY, response_id TEXT NOT NULL)" - ) - connection.execute( - "CREATE TABLE responses (response_id TEXT PRIMARY KEY, data TEXT NOT NULL, accessed_at REAL NOT NULL)" - ) - connection.executemany( - "INSERT INTO responses VALUES (?, ?, 0)", - ( - ("telegram-response", json.dumps({"session_id": "telegram-session"})), - ("other-response", json.dumps({"session_id": "other-session"})), - ), - ) - connection.executemany( - "INSERT INTO conversations VALUES (?, ?)", - (("telegram", "telegram-response"), ("unrelated", "other-response")), - ) - - assert module.migrate(state, responses) == 1 - assert module.migrate(state, responses) == 0 - with sqlite3.connect(state) as connection: - telegram = connection.execute( - "SELECT session_key, chat_type, display_name, origin_json, title " - "FROM sessions WHERE id = 'telegram-session'" - ).fetchone() - other = connection.execute( - "SELECT session_key, title FROM sessions WHERE id = 'other-session'" - ).fetchone() - assert telegram[:3] == ("telegram", "private", "Telegram") - assert json.loads(telegram[3]) == { - "platform": "telegram", - "session_key": "telegram", - } - assert telegram[4] == "Telegram · General" - assert other == (None, None) - - -def test_web_session_activity_patch_projects_bounded_events(tmp_path: Path): - """The DOM transcript includes events without polluting agent history.""" - module_path = HERMES / "scripts" / "patch_web_session_activity.py" - spec = importlib.util.spec_from_file_location("patch_web_activity", module_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - source = tmp_path / "web_server.py" - destination = tmp_path / "patched.py" - source.write_text( - "prefix\n" - + module.LATEST_ROWS_BEFORE - + module.LATEST_SELECTION_BEFORE - + module.HELPER_MARKER - + " db = object()\n" - + module.MESSAGES_BEFORE - + "suffix\n", - encoding="utf-8", - ) - - module.patch(source, destination) - patched = destination.read_text(encoding="utf-8") - - assert "def _run_activity_messages(" in patched - assert 'root = get_hermes_home() / "run-activity"' in patched - assert "path.stat().st_size > 600_000" in patched - assert "entries[-1_000:]" in patched - assert "*_run_activity_messages(sid)" in patched - assert "messages.sort(" in patched - assert "limit: Optional[int] = None" in patched - assert "total_messages = len(messages)" in patched - assert "min(int(limit), 10_000)" in patched - assert '"total_messages": total_messages' in patched - assert "SELECT id, parent_session_id, started_at, ended_at" in patched - assert "newest still-open member of the lineage" in patched - assert "immediate objective parent" in patched - assert "mixes unrelated workstreams" in patched - assert "orphaned children" in patched - assert 'item[0].get("ended_at") is None' in patched - - -def test_web_session_lineage_returns_to_resumed_parent(): - """An ended reviewer must not strand the live view away from its parent.""" - module_path = HERMES / "scripts" / "patch_web_session_activity.py" - spec = importlib.util.spec_from_file_location("patch_web_lineage", module_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - - namespace: dict[str, object] = {} - exec( - "def select_active(sid, db, rows):\n" + module.LATEST_SELECTION_AFTER, - namespace, - ) - - class FakeDB: - def get_session(self, session_id): - return next((row for row in rows if row["id"] == session_id), None) - - rows = [ - { - "id": "umbrella", - "parent_session_id": None, - "started_at": 0.0, - "ended_at": None, - }, - { - "id": "root", - "parent_session_id": "umbrella", - "started_at": 1.0, - "ended_at": None, - }, - { - "id": "review-1", - "parent_session_id": "root", - "started_at": 2.0, - "ended_at": 3.0, - }, - { - "id": "review-2", - "parent_session_id": "root", - "started_at": 4.0, - "ended_at": None, - }, - ] - select_active = namespace["select_active"] - - assert select_active("root", FakeDB(), rows) == ( - "review-2", - ["root", "review-2"], - ) - assert select_active("review-1", FakeDB(), rows) == ( - "review-2", - ["root", "review-2"], - ) - rows[2]["ended_at"] = 5.0 - rows[3]["ended_at"] = 5.0 - assert select_active("root", FakeDB(), rows) == ("root", ["root"]) - - rows[1]["ended_at"] = 9.0 - rows.append( - { - "id": "orphaned-review", - "parent_session_id": "root", - "started_at": 6.0, - "ended_at": None, - } - ) - rows.append( - { - "id": "accepted-review", - "parent_session_id": "root", - "started_at": 7.0, - "ended_at": 8.0, - } - ) - assert select_active("review-1", FakeDB(), rows) == ( - "accepted-review", - ["root", "accepted-review"], - ) - - -def test_api_activity_patch_coalesces_streaming_heartbeats( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -): - """Token callbacks stay bounded while every tool transition is retained.""" - module_path = HERMES / "scripts" / "patch_api_server_sessions.py" - spec = importlib.util.spec_from_file_location("patch_api_activity", module_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - activity_body = module.EVENT_CALLBACK_SIGNATURE_AFTER.split( - " def _make_run_event_callback(", 1 - )[0] - namespace: dict[str, object] = {} - exec( - "import hashlib, json, logging, os, time\n" - "from pathlib import Path\n" - "logger = logging.getLogger(__name__)\n" - "def redact_sensitive_text(value): return value\n" - "class ActivityRecorder:\n" - + activity_body, - namespace, - ) - recorder = namespace["ActivityRecorder"]() - monkeypatch.setenv("HERMES_HOME", str(tmp_path)) - - recorder._record_run_activity("session", "_thinking") - recorder._record_run_activity("session", "reasoning.available") - recorder._record_run_activity("session", "subagent.thinking") - recorder._record_run_activity("session", "tool.started", tool_name="terminal") - recorder._record_run_activity("session", "tool.completed", tool_name="terminal") - - journals = list((tmp_path / "run-activity").glob("*.jsonl")) - assert len(journals) == 1 - entries = [json.loads(line) for line in journals[0].read_text().splitlines()] - assert [entry["activity_event"] for entry in entries] == [ - "_thinking", - "tool.started", - "tool.completed", - ] - - -def test_legacy_api_sessions_are_nested_idempotently(tmp_path: Path): - """Known standalone API workers move under Cassandra without data loss.""" - module_path = HERMES / "scripts" / "migrate_api_session_lineage.py" - spec = importlib.util.spec_from_file_location("migrate_api_sessions", module_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - database = tmp_path / "state.db" - with sqlite3.connect(database) as connection: - connection.execute( - "CREATE TABLE sessions (id TEXT PRIMARY KEY, source TEXT, " - "parent_session_id TEXT, title TEXT, transcript TEXT, archived INTEGER DEFAULT 0)" - ) - connection.execute( - "INSERT INTO sessions (id, source, parent_session_id, title, transcript) " - "VALUES (?, 'tui', NULL, 'Cassandra', 'parent-data')", - (module.LEGACY_CASSANDRA_PARENT,), - ) - worker_id = next(iter(module.LEGACY_CASSANDRA_WORKERS)) - connection.execute( - "INSERT INTO sessions (id, source, parent_session_id, title, transcript) " - "VALUES (?, 'api_server', NULL, 'old', 'keep-me')", - (worker_id,), - ) - connection.executemany( - "INSERT INTO sessions (id, source, parent_session_id, title, transcript) " - "VALUES (?, 'api_server', NULL, 'old smoke', 'keep-smoke')", - ((orphan_id,) for orphan_id in module.LEGACY_ORPHANED_SMOKE_SESSIONS), - ) - - assert module.migrate(database) == 1 + len(module.LEGACY_ORPHANED_SMOKE_SESSIONS) - assert module.migrate(database) == 0 - with sqlite3.connect(database) as connection: - row = connection.execute( - "SELECT parent_session_id, title, transcript FROM sessions WHERE id = ?", - (worker_id,), - ).fetchone() - assert row == ( - module.LEGACY_CASSANDRA_PARENT, - module.LEGACY_CASSANDRA_WORKERS[worker_id], - "keep-me", - ) - with sqlite3.connect(database) as connection: - orphans = connection.execute( - "SELECT id, archived, title, transcript FROM sessions " - "WHERE id IN ({}) ORDER BY id".format( - ",".join("?" for _ in module.LEGACY_ORPHANED_SMOKE_SESSIONS) - ), - tuple(module.LEGACY_ORPHANED_SMOKE_SESSIONS), - ).fetchall() - assert orphans == sorted( - ( - orphan_id, - 1, - module.LEGACY_ORPHANED_SMOKE_SESSIONS[orphan_id], - "keep-smoke", - ) - for orphan_id in module.LEGACY_ORPHANED_SMOKE_SESSIONS - ) - - -def test_automated_triage_sessions_are_grouped_without_touching_interactive_runs(tmp_path: Path): - """Only the stable Ariadne contract moves below the triage parent.""" - module_path = HERMES / "scripts" / "migrate_api_session_lineage.py" - spec = importlib.util.spec_from_file_location("migrate_triage_sessions", module_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - database = tmp_path / "state.db" - with sqlite3.connect(database) as connection: - connection.execute( - "CREATE TABLE sessions (id TEXT PRIMARY KEY, source TEXT, " - "parent_session_id TEXT, title TEXT, started_at REAL, archived INTEGER DEFAULT 0)" - ) - connection.execute( - "CREATE TABLE messages (session_id TEXT, role TEXT, content TEXT)" - ) - connection.executemany( - "INSERT INTO sessions (id, source, started_at) VALUES (?, 'api_server', ?)", - (("triage-run", 1.0), ("jenkins-run", 1.5), ("interactive-run", 2.0)), - ) - connection.executemany( - "INSERT INTO messages (session_id, role, content) VALUES (?, 'user', ?)", - ( - ( - "triage-run", - module.TRIAGE_MESSAGE_PREFIXES[0] - + " Fix for incident sonar/bstein_home/python:S2208/finding-key.", - ), - ( - "jenkins-run", - module.TRIAGE_MESSAGE_PREFIXES[1] - + "\nAnalyze incident soteria/291 for the Jenkins job soteria.", - ), - ("interactive-run", "Please explain this alert to me."), - ), - ) - - assert module.migrate(database, group_triage=True) == 2 - assert module.migrate(database, group_triage=True) == 0 - with sqlite3.connect(database) as connection: - parent = connection.execute( - "SELECT title FROM sessions WHERE id = ?", (module.TRIAGE_PARENT,) - ).fetchone() - triage = connection.execute( - "SELECT parent_session_id, title FROM sessions WHERE id = 'triage-run'" - ).fetchone() - interactive = connection.execute( - "SELECT parent_session_id FROM sessions WHERE id = 'interactive-run'" - ).fetchone() - jenkins = connection.execute( - "SELECT parent_session_id, title FROM sessions WHERE id = 'jenkins-run'" - ).fetchone() - assert parent == (module.TRIAGE_PARENT_TITLE,) - assert triage == ( - module.TRIAGE_PARENT, - "Sonar · bstein_home · python:S2208 · iage-run", - ) - assert interactive == (None,) - assert jenkins == ( - module.TRIAGE_PARENT, - "Sonar · soteria/291 · kins-run", - ) - - -def test_stale_parent_linked_api_workers_close_on_startup(tmp_path: Path): - """A previous gateway lifetime cannot leave phantom active workers.""" - module_path = HERMES / "scripts" / "migrate_api_session_lineage.py" - spec = importlib.util.spec_from_file_location("close_stale_api_workers", module_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - database = tmp_path / "state.db" - with sqlite3.connect(database) as connection: - connection.execute( - "CREATE TABLE sessions (id TEXT PRIMARY KEY, source TEXT, " - "parent_session_id TEXT, title TEXT, started_at REAL, ended_at REAL, " - "end_reason TEXT, archived INTEGER DEFAULT 0)" - ) - connection.executemany( - "INSERT INTO sessions " - "(id, source, parent_session_id, started_at, ended_at, end_reason) " - "VALUES (?, ?, ?, 1, ?, ?)", - ( - ("stale", "api_server", "parent", None, None), - ("root", "api_server", None, None, None), - ("finished", "api_server", "parent", 2.0, "api_run_completed"), - ("interactive", "tui", "parent", None, None), - ), - ) - - assert module.migrate(database) == 1 - assert module.migrate(database) == 0 - with sqlite3.connect(database) as connection: - rows = connection.execute( - "SELECT id, ended_at, end_reason FROM sessions ORDER BY id" - ).fetchall() - by_id = {row[0]: row[1:] for row in rows} - assert by_id["stale"][0] is not None - assert by_id["stale"][1] == "api_run_recovered_stale" - assert by_id["root"] == (None, None) - assert by_id["finished"] == (2.0, "api_run_completed") - assert by_id["interactive"] == (None, None) - - -def test_switchyard_brokers_and_native_claude_lane_use_the_right_images(): - """Thin brokers stay small while native Claude runs beside owner auth.""" - dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-switchyard-brokers").read_text() - assert "httpx==0.28.1" in dockerfile - assert "worker_route_broker.py" in dockerfile - assert "routing_catalog.py" in dockerfile - - deployment = _documents(HERMES / "switchyard-deployment.yaml")[0] - containers = { - container["name"]: container - for container in deployment["spec"]["template"]["spec"]["containers"] - } - expected = ( - "registry.bstein.dev/bstein/hermes-switchyard-brokers@" - "sha256:ee7e95e060ef8083da505162d7e9030daba15fdd828cc047bbcbe6aa409d2083" - ) - assert containers["worker-route-broker"]["image"] == expected - assert containers["classifier-broker"]["image"] == expected - assert "claude-oauth-broker" not in containers - - agent = _documents(HERMES / "agent-deployment.yaml")[0] - agent_containers = { - container["name"]: container - for container in agent["spec"]["template"]["spec"]["containers"] - } - for container_name in ("hermes", "terminal"): - container = agent_containers[container_name] - environment = {item["name"]: item["value"] for item in container["env"]} - mounts = {item["name"]: item for item in container["volumeMounts"]} - assert environment["HERMES_ROUTING_CATALOG_PATH"] == "/routing-catalog/catalog.json" - assert environment["HERMES_CODEX_HEALTH_PATH"] == "/opt/data/provider-health/codex.json" - assert environment["HERMES_CLAUDE_HEALTH_PATH"] == "/opt/data/provider-health/claude.json" - assert mounts["routing-catalog"]["mountPath"] == "/routing-catalog" - assert mounts["routing-catalog"]["readOnly"] is True - codex = agent_containers["codex-broker"] - codex_environment = {item["name"]: item["value"] for item in codex["env"]} - assert codex_environment["HERMES_CODEX_HEALTH_PATH"] == "/opt/data/provider-health/codex.json" - claude = agent_containers["claude-broker"] - assert claude["image"].startswith("registry.bstein.dev/bstein/hermes-agent@") - assert "unset ANTHROPIC_API_KEY CLAUDE_API_KEY" in claude["args"][0] - assert any( - mount["name"] == "home" and mount["mountPath"] == "/opt/data" - for mount in claude["volumeMounts"] - ) - - -def test_classifier_broker_bounds_history_without_losing_routing_intent(monkeypatch): - """AUTO classification must fit the Jetson context without losing intent.""" - broker_path = HERMES / "scripts" / "classifier_broker.py" - monkeypatch.setitem(sys.modules, "httpx", SimpleNamespace()) - spec = importlib.util.spec_from_file_location("hermes_classifier_broker", broker_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - - payload = { - "model": "qwen2.5:14b-instruct-q4_0", - "messages": [ - { - "role": "system", - "content": "routing contract\n" + ("candidate policy " * 1000), - }, - { - "role": "user", - "content": "Build and verify the Cassandra release safely.", - }, - { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call_large", - "type": "function", - "function": { - "name": "large_tool", - "arguments": '{"command":"' + ("x" * 5000) + '"}', - }, - } - ], - }, - { - "role": "tool", - "content": "unbounded test output " * 10000, - "tool_call_id": "call_large", - }, - { - "role": "user", - "content": [ - {"type": "image_url", "image_url": {"url": "data:image/png;base64,private"}}, - {"type": "text", "text": "Turn this cat into a cute clown."}, - ], - }, - ], - "tools": [{"type": "function", "function": {"name": "large_tool"}}], - "tool_choice": "auto", - "parallel_tool_calls": True, - "response_format": {"type": "json_object"}, - } - - compacted = module.compact_payload(payload) - encoded = json.dumps(compacted) - - assert compacted["model"] == payload["model"] - assert compacted["response_format"] == payload["response_format"] - assert "tools" not in compacted - assert "tool_choice" not in compacted - assert "parallel_tool_calls" not in compacted - assert all(message.get("role") != "tool" for message in compacted["messages"]) - assert all("tool_call_id" not in message for message in compacted["messages"]) - assert all("tool_calls" not in message for message in compacted["messages"]) - assert "[tool evidence]" in encoded - assert "[assistant requested an external tool]" in encoded - assert "Build and verify the Cassandra release safely." in encoded - assert "Turn this cat into a cute clown." in encoded - assert "image attachment available to the selected worker" in encoded - assert "data:image/png;base64" not in encoded - assert "unbounded test output " * 100 not in encoded - assert len(encoded) < 18_000 - - -def test_switchyard_classifier_is_bounded_and_fails_open_once(): - """A sick local judge must not hold chat through repeated long retries.""" - config = tomllib.loads( - _documents(HERMES / "switchyard-configmap.yaml")[0]["data"]["routes.toml"] - ) - classifier = config["llm_clients"]["classifier"] - assert classifier["base_url"] == "http://127.0.0.1:9008/v1" - assert classifier["max_retries"] == 0 - for route in ("auto_fast", "auto_balanced"): - assert config["routes"][route]["recent_turn_window"] == 4 - for route in ("auto_deep", "auto_maximum", "worker_auto_maximum"): - assert config["routes"][route]["recent_turn_window"] == 6 - - deployment = _documents(HERMES / "switchyard-deployment.yaml")[0] - containers = { - item["name"]: item - for item in deployment["spec"]["template"]["spec"]["containers"] - } - classifier_container = containers["classifier-broker"] - env = {item["name"]: item["value"] for item in classifier_container["env"]} - assert env["HERMES_CLASSIFIER_BROKER_READ_TIMEOUT"] == "60" - assert classifier_container["readinessProbe"]["httpGet"]["port"] == "classifier" - - -def test_image_broker_returns_bytes_and_removes_owner_cache(tmp_path: Path, monkeypatch): - """The broker must not retain a family user's generated image.""" - broker_path = HERMES / "scripts" / "image_broker.py" - spec = importlib.util.spec_from_file_location("hermes_image_broker", broker_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - generated = tmp_path / "generated.png" - generated.write_bytes(b"\x89PNG\r\n\x1a\nprivate-image") - - class Provider: - def generate(self, prompt, aspect, **kwargs): - assert prompt == "paint a blue sphere" - assert aspect == "square" - return { - "success": True, - "image": str(generated), - "model": "gpt-image-2-high", - "quality": "high", - } - - monkeypatch.setattr(module, "_PROVIDER", Provider()) - result = module._generate( - { - "prompt": "paint a blue sphere", - "aspect_ratio": "square", - "model": "gpt-image-2-high", - } - ) - - assert result["success"] is True - assert result["image_b64"] - assert "image" not in result - assert not generated.exists() - - -def test_image_broker_auto_falls_back_to_local_and_honors_explicit_routes( - monkeypatch, -): - """AUTO is hosted-first while explicit local never calls the hosted lane.""" - broker_path = HERMES / "scripts" / "image_broker.py" - spec = importlib.util.spec_from_file_location("hermes_image_router", broker_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - calls = [] - - def hosted(payload, model, prompt, aspect): - calls.append(("hosted", model, prompt, aspect)) - return {"success": False, "error": "hosted refusal"} - - def local(payload, timeout=1800.0): - calls.append(("local", payload["model"], timeout)) - return { - "success": True, - "image_b64": "aW1hZ2U=", - "model": "flux-2-klein-4b-local", - "route": "local", - } - - monkeypatch.setattr(module, "_generate_hosted", hosted) - monkeypatch.setattr(module, "_local_request", local) - auto = module._generate( - { - "prompt": "colorize this family photograph", - "aspect_ratio": "portrait", - "model": "atlas-image-auto-high", - } - ) - assert auto["success"] is True - assert auto["route"] == "local" - assert auto["hosted_fallback_reason"] == "hosted refusal" - assert [call[0] for call in calls] == ["hosted", "local"] - - calls.clear() - explicit = module._generate( - { - "prompt": "make a local landscape", - "aspect_ratio": "landscape", - "model": "flux-2-klein-4b-local", - } - ) - assert explicit["route"] == "local" - assert [call[0] for call in calls] == ["local"] - - -def test_image_broker_policy_is_narrow_and_operator_extensible(tmp_path: Path, monkeypatch): - """Family-photo restoration stays allowed while the hard boundary remains.""" - broker_path = HERMES / "scripts" / "image_broker.py" - spec = importlib.util.spec_from_file_location("hermes_image_policy", broker_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - policy = tmp_path / "policy.json" - policy.write_text('{"additional_blocked_phrases":["site-specific block"]}') - monkeypatch.setattr(module, "POLICY_PATH", policy) - - assert module._policy_error( - "Colorize my baby photograph with a lighter natural skin tone" - ) is None - assert "minors" in module._policy_error("Create a sexual image of a child") - assert module._policy_error("A site-specific block request") == ( - "request is blocked by the operator image policy" - ) - - -def test_local_flux_runtime_and_gpu_handoff_are_flux_managed(): - """FLUX and Wolf share titan-24 while text stays on titan-20.""" - deployment = _documents(HERMES / "local-image-deployment.yaml")[0] - assert deployment["metadata"]["name"] == "hermes-local-image" - pod = deployment["spec"]["template"]["spec"] - assert pod["serviceAccountName"] == "hermes-gpu-runtime" - local = next(item for item in pod["containers"] if item["name"] == "local-image") - assert len(pod["containers"]) == 1 - assert local["resources"]["requests"]["nvidia.com/gpu.shared"] == 1 - assert local["ports"] == [{"name": "local-image", "containerPort": 9004}] - assert any(mount["mountPath"] == "/models" for mount in local["volumeMounts"]) - model_env = {item["name"]: item["value"] for item in local["env"]} - assert model_env["HERMES_LOCAL_IMAGE_LISTEN_PORT"] == "9004" - assert model_env["HERMES_LOCAL_IMAGE_REVISION"] == ( - "e7b7dc27f91deacad38e78976d1f2b499d76a294" - ) - assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVITY_NODE"] == "titan-24" - assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVE_SM_PERCENT"] == "80" - assert model_env["HERMES_LOCAL_IMAGE_GPU_MAX_EXTERNAL_MEMORY_BYTES"] == ( - "3221225472" - ) - assert model_env["HERMES_LOCAL_IMAGE_OFFLOAD_MODE"] == "sequential" - assert "nvidia-process-exporter-local.monitoring.svc.cluster.local" in model_env[ - "HERMES_LOCAL_IMAGE_GPU_ACTIVITY_URL" - ] - models_volume = next(item for item in pod["volumes"] if item["name"] == "models") - assert models_volume["persistentVolumeClaim"]["claimName"] == ( - "hermes-image-models" - ) - - services = _documents(HERMES / "service.yaml") - image_service = next( - item for item in services if item["metadata"]["name"] == "hermes-local-image" - ) - assert image_service["spec"]["selector"] == {"app": "hermes-local-image"} - - handoff_services = _documents(HERMES / "model-gate-deployment.yaml") - handoff = next( - item - for item in handoff_services - if item["kind"] == "Service" - and item["metadata"]["name"] == "hermes-gpu-handoff" - ) - assert handoff["spec"]["ports"][0]["targetPort"] == "handoff" - - ariadne = _documents( - Path(__file__).parents[2] - / "services/maintenance/apps/ariadne-deployment.yaml" - )[0] - env = { - item["name"]: item["value"] - for item in ariadne["spec"]["template"]["spec"]["containers"][0]["env"] - if "value" in item - } - assert env["GAME_MODE_OLLAMA_URL"] == ( - "http://hermes-gpu-handoff.hermes.svc.cluster.local:11434" - ) - assert env["GAME_MODE_OLLAMA_MODEL"] == "flux-2-klein-4b-local" - - for config_name in ("configmap.yaml", "agent-configmap.yaml", "chat-configmap.yaml"): - config = _documents(HERMES / config_name)[0]["data"]["config.yaml"] - assert "gpt-oss:20b" not in config - assert "atlas-switchyard" in config - switchyard = _documents(HERMES / "switchyard-configmap.yaml")[0]["data"][ - "routes.toml" - ] - assert 'id = "qwen2.5:14b-instruct-q4_0"' in switchyard - assert "qwen2.5:3b-instruct-q4_0" not in switchyard - assert "route/local/qwen2.5-14b/medium" in switchyard - assert "Anthropic and Claude name the same provider" in switchyard - assert "OpenAI and Codex name the same provider" in switchyard - assert "Choose across every configured Codex and Claude family" in switchyard - assert "Claude Fable" in switchyard - assert switchyard.count('Treat "think hard"') == 4 - assert switchyard.count("Never choose below the") >= 5 - switchyard_config = tomllib.loads(switchyard) - routes = switchyard_config["routes"] - configured_targets = switchyard_config["targets"] - for route_name in ("auto_fast", "auto_balanced", "auto_deep", "auto_maximum"): - leading_targets = set(routes[route_name]["targets"][:2]) - assert leading_targets == {"codex_sol_xhigh", "claude_opus_xhigh"} - assert "max_output_tokens" not in routes[route_name] - for route_name in ("auto_fast", "auto_balanced", "auto_deep", "auto_maximum"): - targets = routes[route_name]["targets"] - selector_targets = routes[route_name]["response_schema"] - assert not any(target.startswith("local_") for target in targets) - assert any("fable" in target for target in targets) - assert "local_qwen" not in selector_targets - assert "not eligible for foreground" in routes[route_name]["prompt"] - for route_name in ("auto_deep", "auto_maximum"): - targets = routes[route_name]["targets"] - selector_targets = routes[route_name]["response_schema"] - assert not any(target.endswith("_low") for target in targets) - assert "_low" not in selector_targets - maximum_targets = routes["auto_maximum"]["targets"] - maximum_selector_targets = routes["auto_maximum"]["response_schema"] - assert not any(target.endswith("_medium") for target in maximum_targets) - assert "_medium" not in maximum_selector_targets - assert "absolute high effort floor" in routes["auto_maximum"]["prompt"] - assert "quality mark was missed" in routes["auto_balanced"]["prompt"] - assert "raises the next boundary to xhigh" in routes["auto_maximum"]["prompt"] - assert "Repeated quality misses require xhigh" in routes[ - "worker_auto_maximum" - ]["prompt"] - assert any( - target.startswith("local_") - for target in routes["manual_local_qwen"]["targets"] - ) - for route_name in ( - "manual_codex_luna", - "manual_codex_terra", - "manual_codex_sol", - "manual_claude_haiku", - "manual_claude_fable", - "manual_claude_sonnet", - "manual_claude_opus", - ): - assert not any( - target.startswith("local_") for target in routes[route_name]["targets"] - ) - for provider, families in { - "codex": ("luna", "terra", "sol"), - "claude": ("haiku", "fable", "sonnet", "opus"), - }.items(): - for family in families: - for effort in ("low", "medium", "high", "xhigh"): - route = routes[f"manual_{provider}_{family}_{effort}"] - assert route["id"] == f"atlas/manual/{provider}/{family}/{effort}" - assert route["targets"][0] == f"{provider}_{family}_{effort}" - worker_target = f"worker_{provider}_{family}_{effort}" - assert worker_target in routes["worker_auto_maximum"]["targets"] - assert configured_targets[worker_target]["id"] == ( - f"worker/{provider}/{family}/{effort}" - ) - for route_name in ("auto_fast", "auto_balanced"): - prompt = routes[route_name]["prompt"] - assert "image tool—not the conversational model" in prompt - assert "Do not select a" in prompt - assert "local Qwen or Claude target" in prompt - assert "max_output_tokens" not in routes["worker_auto_maximum"] - model_gate = _documents(HERMES / "model-gate-configmap.yaml")[0]["data"][ - "model_gate.py" - ] - assert "qwen2.5:14b-instruct-q4_0" in model_gate - - -def test_titan20_serializes_classifier_and_local_chat_model_residency(): - """Classifier and local chat share one serialized resident Qwen weight.""" - deployment = _documents( - Path(__file__).parents[2] / "services/ai-llm/deployment.yaml" - )[0] - pod = deployment["spec"]["template"]["spec"] - required = pod["affinity"]["nodeAffinity"][ - "requiredDuringSchedulingIgnoredDuringExecution" - ]["nodeSelectorTerms"][0]["matchExpressions"][0] - assert required["values"] == ["titan-20"] - container = pod["containers"][0] - env = {item["name"]: item["value"] for item in container["env"]} - assert env["OLLAMA_MAX_LOADED_MODELS"] == "1" - assert env["OLLAMA_NUM_PARALLEL"] == "1" - assert env["OLLAMA_KEEP_ALIVE"] == "-1" - assert env["OLLAMA_CONTEXT_LENGTH"] == "8192" - warm_command = " ".join(container["command"]) - assert "--keepalive=-1" not in warm_command - models = next(item for item in pod["volumes"] if item["name"] == "models") - assert models["persistentVolumeClaim"]["claimName"] == ( - "ollama-models-titan20" - ) - - -def test_local_image_gpu_guard_distinguishes_background_and_saturated_gpu(monkeypatch): - """Lease-idle desktop spikes may coexist, but saturation still blocks FLUX.""" - source = ROOT / "dockerfiles" / "hermes-local-image-server.py" - spec = importlib.util.spec_from_file_location("hermes_local_image_server", source) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - monkeypatch.setattr(module, "GPU_ACTIVITY_NODE", "titan-24") - monkeypatch.setattr(module, "GPU_ACTIVE_SM_PERCENT", 80.0) - monkeypatch.setattr(module, "GPU_MAX_EXTERNAL_MEMORY_BYTES", 3 << 30) - - idle = module._parse_gpu_activity( - '\n'.join( - [ - 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="Xorg"} 0', - 'nvidia_process_gpu_memory_used_bytes{node="titan-24",namespace="host",process="Xorg"} 1900000000', - 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="game-stream",process="wolf"} 3', - 'nvidia_process_gpu_memory_used_bytes{node="titan-24",namespace="hermes",process="python"} 9000000000', - ] - ) - ) - assert idle["interactive_active"] is False - assert idle["external_gpu_memory_bytes"] == 1900000000 - assert idle["external_gpu_sm_percent"] == 3 - - background_spike = module._parse_gpu_activity( - 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="sway"} 41\n' - ) - assert background_spike["interactive_active"] is False - - active = module._parse_gpu_activity( - 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="steam"} 91\n' - ) - assert active["interactive_active"] is True - assert "91%" in active["gpu_guard_reason"] - - -def test_local_flux_renderer_uses_a_disposable_cuda_worker(monkeypatch): - """A completed render must not leave its CUDA context in the API process.""" - source = ROOT / "dockerfiles" / "hermes-local-image-server.py" - spec = importlib.util.spec_from_file_location("hermes_local_image_worker", source) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - calls = [] - - def run(command, **kwargs): - calls.append((command, kwargs)) - return SimpleNamespace( - returncode=0, - stdout=b'{"success":true,"route":"local","image_b64":"cG5n"}', - stderr=b"", - ) - - monkeypatch.setattr(module.subprocess, "run", run) - result = module._render({"prompt": "black cat", "aspect_ratio": "square"}) - - assert result["route"] == "local" - command, options = calls[0] - assert command[-1] == "--render-worker" - assert json.loads(options["input"]) == { - "prompt": "black cat", - "aspect_ratio": "square", - } - assert options["timeout"] == module.RENDER_TIMEOUT_SECONDS - assert options["check"] is False - - -def test_local_flux_uses_low_vram_offload_without_reducing_resolution(): - """The shared 3080 lane must trade time, not image size, for headroom.""" - source = (ROOT / "dockerfiles" / "hermes-local-image-server.py").read_text() - assert 'OFFLOAD_MODE = os.environ.get(' in source - assert '"HERMES_LOCAL_IMAGE_OFFLOAD_MODE", "sequential"' in source - assert "pipe.enable_sequential_cpu_offload()" in source - assert '"square": (1024, 1024)' in source - - -def test_chat_auth_and_relay_are_pod_lifetime_only(): - statefulset = _documents(HERMES / "chat-statefulset.yaml")[0] - pod = statefulset["spec"]["template"]["spec"] - containers = statefulset["spec"]["template"]["spec"]["containers"] - - for name in ("hermes", "webui"): - container = next(item for item in containers if item["name"] == name) - env = {item["name"]: item["value"] for item in container["env"]} - assert env["HERMES_AUTH_FILE"] == "/runtime-access/hermes-auth.json" - mount = next( - item for item in container["volumeMounts"] if item["name"] == "runtime-access" - ) - assert mount["mountPath"] == "/runtime-access" - assert "subPath" not in mount - - hermes_env = { - item["name"]: item["value"] - for item in next(item for item in containers if item["name"] == "hermes")["env"] - } - runtime = next(item for item in pod["volumes"] if item["name"] == "runtime-access") - assert runtime["emptyDir"] == {"medium": "Memory", "sizeLimit": "2Mi"} - assert not any(item["name"] == "provider-auth" for item in pod["volumes"]) - init_command = next( - item for item in pod["initContainers"] if item["name"] == "init-config" - )["command"][2] - for key in ( - "ANTHROPIC_API_KEY", - "API_SERVER_KEY", - "CLAUDE_CODE_OAUTH_TOKEN", - "GITEA_TOKEN", - "HERMES_IMAGE_BROKER_KEY", - "OPENAI_API_KEY", - ): - assert key in init_command - assert "printf 'API_SERVER_KEY=%s" not in init_command - assert hermes_env["AGENT_BROWSER_EXECUTABLE_PATH"].endswith("/chrome-linux/headless_shell") - assert "--no-sandbox" in hermes_env["AGENT_BROWSER_ARGS"] - - -def test_sandbox_executes_python_with_bounded_output(tmp_path: Path, monkeypatch): - source = ROOT / "dockerfiles" / "hermes-chat-sandbox-server.py" - spec = importlib.util.spec_from_file_location("hermes_chat_sandbox_server", source) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - monkeypatch.setattr(module, "WORKSPACE", tmp_path) - - result = module._execute("import math\nprint(math.comb(10, 3))") - - assert result["success"] is True - assert result["stdout"] == "120\n" - assert result["stderr"] == "" diff --git a/testing/tests/test_hermes_chat_routing_runtime.py b/testing/tests/test_hermes_chat_routing_runtime.py new file mode 100644 index 00000000..6a81d625 --- /dev/null +++ b/testing/tests/test_hermes_chat_routing_runtime.py @@ -0,0 +1,486 @@ +"""Routing and runtime workload contracts for Hermes chat.""" + +from __future__ import annotations + +import importlib.util +import json +import sqlite3 +import sys +import tomllib +from pathlib import Path +from types import SimpleNamespace + + +from testing.tests.test_hermes_chat_support import ( + HERMES, + ROOT, + _documents, +) + + +def test_stale_parent_linked_api_workers_close_on_startup(tmp_path: Path): + """A previous gateway lifetime cannot leave phantom active workers.""" + module_path = HERMES / "scripts" / "migrate_api_session_lineage.py" + spec = importlib.util.spec_from_file_location( + "close_stale_api_workers", module_path + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + database = tmp_path / "state.db" + with sqlite3.connect(database) as connection: + connection.execute( + "CREATE TABLE sessions (id TEXT PRIMARY KEY, source TEXT, " + "parent_session_id TEXT, title TEXT, started_at REAL, ended_at REAL, " + "end_reason TEXT, archived INTEGER DEFAULT 0)" + ) + connection.executemany( + "INSERT INTO sessions " + "(id, source, parent_session_id, started_at, ended_at, end_reason) " + "VALUES (?, ?, ?, 1, ?, ?)", + ( + ("stale", "api_server", "parent", None, None), + ("root", "api_server", None, None, None), + ("finished", "api_server", "parent", 2.0, "api_run_completed"), + ("interactive", "tui", "parent", None, None), + ), + ) + + assert module.migrate(database) == 1 + assert module.migrate(database) == 0 + with sqlite3.connect(database) as connection: + rows = connection.execute( + "SELECT id, ended_at, end_reason FROM sessions ORDER BY id" + ).fetchall() + by_id = {row[0]: row[1:] for row in rows} + assert by_id["stale"][0] is not None + assert by_id["stale"][1] == "api_run_recovered_stale" + assert by_id["root"] == (None, None) + assert by_id["finished"] == (2.0, "api_run_completed") + assert by_id["interactive"] == (None, None) + + +def test_switchyard_brokers_and_native_claude_lane_use_the_right_images(): + """Thin brokers stay small while native Claude runs beside owner auth.""" + dockerfile = ( + ROOT / "dockerfiles" / "Dockerfile.hermes-switchyard-brokers" + ).read_text() + assert "httpx==0.28.1" in dockerfile + assert "worker_route_broker.py" in dockerfile + assert "routing_catalog.py" in dockerfile + + deployment = _documents(HERMES / "switchyard-deployment.yaml")[0] + containers = { + container["name"]: container + for container in deployment["spec"]["template"]["spec"]["containers"] + } + expected = ( + "registry.bstein.dev/bstein/hermes-switchyard-brokers@" + "sha256:ee7e95e060ef8083da505162d7e9030daba15fdd828cc047bbcbe6aa409d2083" + ) + assert containers["worker-route-broker"]["image"] == expected + assert containers["classifier-broker"]["image"] == expected + assert "claude-oauth-broker" not in containers + + agent = _documents(HERMES / "agent-deployment.yaml")[0] + agent_containers = { + container["name"]: container + for container in agent["spec"]["template"]["spec"]["containers"] + } + for container_name in ("hermes", "terminal"): + container = agent_containers[container_name] + environment = {item["name"]: item["value"] for item in container["env"]} + mounts = {item["name"]: item for item in container["volumeMounts"]} + assert ( + environment["HERMES_ROUTING_CATALOG_PATH"] + == "/routing-catalog/catalog.json" + ) + assert ( + environment["HERMES_CODEX_HEALTH_PATH"] + == "/opt/data/provider-health/codex.json" + ) + assert ( + environment["HERMES_CLAUDE_HEALTH_PATH"] + == "/opt/data/provider-health/claude.json" + ) + assert mounts["routing-catalog"]["mountPath"] == "/routing-catalog" + assert mounts["routing-catalog"]["readOnly"] is True + codex = agent_containers["codex-broker"] + codex_environment = {item["name"]: item["value"] for item in codex["env"]} + assert ( + codex_environment["HERMES_CODEX_HEALTH_PATH"] + == "/opt/data/provider-health/codex.json" + ) + claude = agent_containers["claude-broker"] + assert claude["image"].startswith("registry.bstein.dev/bstein/hermes-agent@") + assert "unset ANTHROPIC_API_KEY CLAUDE_API_KEY" in claude["args"][0] + assert any( + mount["name"] == "home" and mount["mountPath"] == "/opt/data" + for mount in claude["volumeMounts"] + ) + + +def test_classifier_broker_bounds_history_without_losing_routing_intent(monkeypatch): + """AUTO classification must fit the Jetson context without losing intent.""" + broker_path = HERMES / "scripts" / "classifier_broker.py" + monkeypatch.setitem(sys.modules, "httpx", SimpleNamespace()) + spec = importlib.util.spec_from_file_location( + "hermes_classifier_broker", broker_path + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + payload = { + "model": "qwen2.5:14b-instruct-q4_0", + "messages": [ + { + "role": "system", + "content": "routing contract\n" + ("candidate policy " * 1000), + }, + { + "role": "user", + "content": "Build and verify the Cassandra release safely.", + }, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_large", + "type": "function", + "function": { + "name": "large_tool", + "arguments": '{"command":"' + ("x" * 5000) + '"}', + }, + } + ], + }, + { + "role": "tool", + "content": "unbounded test output " * 10000, + "tool_call_id": "call_large", + }, + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": "data:image/png;base64,private"}, + }, + {"type": "text", "text": "Turn this cat into a cute clown."}, + ], + }, + ], + "tools": [{"type": "function", "function": {"name": "large_tool"}}], + "tool_choice": "auto", + "parallel_tool_calls": True, + "response_format": {"type": "json_object"}, + } + + compacted = module.compact_payload(payload) + encoded = json.dumps(compacted) + + assert compacted["model"] == payload["model"] + assert compacted["response_format"] == payload["response_format"] + assert "tools" not in compacted + assert "tool_choice" not in compacted + assert "parallel_tool_calls" not in compacted + assert all(message.get("role") != "tool" for message in compacted["messages"]) + assert all("tool_call_id" not in message for message in compacted["messages"]) + assert all("tool_calls" not in message for message in compacted["messages"]) + assert "[tool evidence]" in encoded + assert "[assistant requested an external tool]" in encoded + assert "Build and verify the Cassandra release safely." in encoded + assert "Turn this cat into a cute clown." in encoded + assert "image attachment available to the selected worker" in encoded + assert "data:image/png;base64" not in encoded + assert "unbounded test output " * 100 not in encoded + assert len(encoded) < 18_000 + + +def test_switchyard_classifier_is_bounded_and_fails_open_once(): + """A sick local judge must not hold chat through repeated long retries.""" + config = tomllib.loads( + _documents(HERMES / "switchyard-configmap.yaml")[0]["data"]["routes.toml"] + ) + classifier = config["llm_clients"]["classifier"] + assert classifier["base_url"] == "http://127.0.0.1:9008/v1" + assert classifier["max_retries"] == 0 + for route in ("auto_fast", "auto_balanced"): + assert config["routes"][route]["recent_turn_window"] == 4 + for route in ("auto_deep", "auto_maximum", "worker_auto_maximum"): + assert config["routes"][route]["recent_turn_window"] == 6 + + deployment = _documents(HERMES / "switchyard-deployment.yaml")[0] + containers = { + item["name"]: item + for item in deployment["spec"]["template"]["spec"]["containers"] + } + classifier_container = containers["classifier-broker"] + env = {item["name"]: item["value"] for item in classifier_container["env"]} + assert env["HERMES_CLASSIFIER_BROKER_READ_TIMEOUT"] == "60" + assert classifier_container["readinessProbe"]["httpGet"]["port"] == "classifier" + + +def test_image_broker_returns_bytes_and_removes_owner_cache( + tmp_path: Path, monkeypatch +): + """The broker must not retain a family user's generated image.""" + broker_path = HERMES / "scripts" / "image_broker.py" + spec = importlib.util.spec_from_file_location("hermes_image_broker", broker_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + generated = tmp_path / "generated.png" + generated.write_bytes(b"\x89PNG\r\n\x1a\nprivate-image") + + class Provider: + def generate(self, prompt, aspect, **kwargs): + assert prompt == "paint a blue sphere" + assert aspect == "square" + return { + "success": True, + "image": str(generated), + "model": "gpt-image-2-high", + "quality": "high", + } + + monkeypatch.setattr(module, "_PROVIDER", Provider()) + result = module._generate( + { + "prompt": "paint a blue sphere", + "aspect_ratio": "square", + "model": "gpt-image-2-high", + } + ) + + assert result["success"] is True + assert result["image_b64"] + assert "image" not in result + assert not generated.exists() + + +def test_image_broker_auto_falls_back_to_local_and_honors_explicit_routes( + monkeypatch, +): + """AUTO is hosted-first while explicit local never calls the hosted lane.""" + broker_path = HERMES / "scripts" / "image_broker.py" + spec = importlib.util.spec_from_file_location("hermes_image_router", broker_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + calls = [] + + def hosted(payload, model, prompt, aspect): + calls.append(("hosted", model, prompt, aspect)) + return {"success": False, "error": "hosted refusal"} + + def local(payload, timeout=1800.0): + calls.append(("local", payload["model"], timeout)) + return { + "success": True, + "image_b64": "aW1hZ2U=", + "model": "flux-2-klein-4b-local", + "route": "local", + } + + monkeypatch.setattr(module, "_generate_hosted", hosted) + monkeypatch.setattr(module, "_local_request", local) + auto = module._generate( + { + "prompt": "colorize this family photograph", + "aspect_ratio": "portrait", + "model": "atlas-image-auto-high", + } + ) + assert auto["success"] is True + assert auto["route"] == "local" + assert auto["hosted_fallback_reason"] == "hosted refusal" + assert [call[0] for call in calls] == ["hosted", "local"] + + calls.clear() + explicit = module._generate( + { + "prompt": "make a local landscape", + "aspect_ratio": "landscape", + "model": "flux-2-klein-4b-local", + } + ) + assert explicit["route"] == "local" + assert [call[0] for call in calls] == ["local"] + + +def test_image_broker_policy_is_narrow_and_operator_extensible( + tmp_path: Path, monkeypatch +): + """Family-photo restoration stays allowed while the hard boundary remains.""" + broker_path = HERMES / "scripts" / "image_broker.py" + spec = importlib.util.spec_from_file_location("hermes_image_policy", broker_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + policy = tmp_path / "policy.json" + policy.write_text('{"additional_blocked_phrases":["site-specific block"]}') + monkeypatch.setattr(module, "POLICY_PATH", policy) + + assert ( + module._policy_error( + "Colorize my baby photograph with a lighter natural skin tone" + ) + is None + ) + assert "minors" in module._policy_error("Create a sexual image of a child") + assert module._policy_error("A site-specific block request") == ( + "request is blocked by the operator image policy" + ) + + +def test_local_flux_runtime_and_gpu_handoff_are_flux_managed(): + """FLUX and Wolf share titan-24 while text stays on titan-20.""" + deployment = _documents(HERMES / "local-image-deployment.yaml")[0] + assert deployment["metadata"]["name"] == "hermes-local-image" + pod = deployment["spec"]["template"]["spec"] + assert pod["serviceAccountName"] == "hermes-gpu-runtime" + local = next(item for item in pod["containers"] if item["name"] == "local-image") + assert len(pod["containers"]) == 1 + assert local["resources"]["requests"]["nvidia.com/gpu.shared"] == 1 + assert local["ports"] == [{"name": "local-image", "containerPort": 9004}] + assert any(mount["mountPath"] == "/models" for mount in local["volumeMounts"]) + model_env = {item["name"]: item["value"] for item in local["env"]} + assert model_env["HERMES_LOCAL_IMAGE_LISTEN_PORT"] == "9004" + assert model_env["HERMES_LOCAL_IMAGE_REVISION"] == ( + "e7b7dc27f91deacad38e78976d1f2b499d76a294" + ) + assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVITY_NODE"] == "titan-24" + assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVE_SM_PERCENT"] == "80" + assert model_env["HERMES_LOCAL_IMAGE_GPU_MAX_EXTERNAL_MEMORY_BYTES"] == ( + "3221225472" + ) + assert model_env["HERMES_LOCAL_IMAGE_OFFLOAD_MODE"] == "sequential" + assert ( + "nvidia-process-exporter-local.monitoring.svc.cluster.local" + in model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVITY_URL"] + ) + models_volume = next(item for item in pod["volumes"] if item["name"] == "models") + assert models_volume["persistentVolumeClaim"]["claimName"] == ( + "hermes-image-models" + ) + + services = _documents(HERMES / "service.yaml") + image_service = next( + item for item in services if item["metadata"]["name"] == "hermes-local-image" + ) + assert image_service["spec"]["selector"] == {"app": "hermes-local-image"} + + handoff_services = _documents(HERMES / "model-gate-deployment.yaml") + handoff = next( + item + for item in handoff_services + if item["kind"] == "Service" + and item["metadata"]["name"] == "hermes-gpu-handoff" + ) + assert handoff["spec"]["ports"][0]["targetPort"] == "handoff" + + ariadne = _documents( + Path(__file__).parents[2] / "services/maintenance/apps/ariadne-deployment.yaml" + )[0] + env = { + item["name"]: item["value"] + for item in ariadne["spec"]["template"]["spec"]["containers"][0]["env"] + if "value" in item + } + assert env["GAME_MODE_OLLAMA_URL"] == ( + "http://hermes-gpu-handoff.hermes.svc.cluster.local:11434" + ) + assert env["GAME_MODE_OLLAMA_MODEL"] == "flux-2-klein-4b-local" + + for config_name in ( + "configmap.yaml", + "agent-configmap.yaml", + "chat-configmap.yaml", + ): + config = _documents(HERMES / config_name)[0]["data"]["config.yaml"] + assert "gpt-oss:20b" not in config + assert "atlas-switchyard" in config + switchyard = _documents(HERMES / "switchyard-configmap.yaml")[0]["data"][ + "routes.toml" + ] + assert 'id = "qwen2.5:14b-instruct-q4_0"' in switchyard + assert "qwen2.5:3b-instruct-q4_0" not in switchyard + assert "route/local/qwen2.5-14b/medium" in switchyard + assert "Anthropic and Claude name the same provider" in switchyard + assert "OpenAI and Codex name the same provider" in switchyard + assert "Choose across every configured Codex and Claude family" in switchyard + assert "Claude Fable" in switchyard + assert switchyard.count('Treat "think hard"') == 4 + assert switchyard.count("Never choose below the") >= 5 + switchyard_config = tomllib.loads(switchyard) + routes = switchyard_config["routes"] + configured_targets = switchyard_config["targets"] + for route_name in ("auto_fast", "auto_balanced", "auto_deep", "auto_maximum"): + leading_targets = set(routes[route_name]["targets"][:2]) + assert leading_targets == {"codex_sol_xhigh", "claude_opus_xhigh"} + assert "max_output_tokens" not in routes[route_name] + for route_name in ("auto_fast", "auto_balanced", "auto_deep", "auto_maximum"): + targets = routes[route_name]["targets"] + selector_targets = routes[route_name]["response_schema"] + assert not any(target.startswith("local_") for target in targets) + assert any("fable" in target for target in targets) + assert "local_qwen" not in selector_targets + assert "not eligible for foreground" in routes[route_name]["prompt"] + for route_name in ("auto_deep", "auto_maximum"): + targets = routes[route_name]["targets"] + selector_targets = routes[route_name]["response_schema"] + assert not any(target.endswith("_low") for target in targets) + assert "_low" not in selector_targets + maximum_targets = routes["auto_maximum"]["targets"] + maximum_selector_targets = routes["auto_maximum"]["response_schema"] + assert not any(target.endswith("_medium") for target in maximum_targets) + assert "_medium" not in maximum_selector_targets + assert "absolute high effort floor" in routes["auto_maximum"]["prompt"] + assert "quality mark was missed" in routes["auto_balanced"]["prompt"] + assert "raises the next boundary to xhigh" in routes["auto_maximum"]["prompt"] + assert ( + "Repeated quality misses require xhigh" + in routes["worker_auto_maximum"]["prompt"] + ) + assert any( + target.startswith("local_") for target in routes["manual_local_qwen"]["targets"] + ) + for route_name in ( + "manual_codex_luna", + "manual_codex_terra", + "manual_codex_sol", + "manual_claude_haiku", + "manual_claude_fable", + "manual_claude_sonnet", + "manual_claude_opus", + ): + assert not any( + target.startswith("local_") for target in routes[route_name]["targets"] + ) + for provider, families in { + "codex": ("luna", "terra", "sol"), + "claude": ("haiku", "fable", "sonnet", "opus"), + }.items(): + for family in families: + for effort in ("low", "medium", "high", "xhigh"): + route = routes[f"manual_{provider}_{family}_{effort}"] + assert route["id"] == f"atlas/manual/{provider}/{family}/{effort}" + assert route["targets"][0] == f"{provider}_{family}_{effort}" + worker_target = f"worker_{provider}_{family}_{effort}" + assert worker_target in routes["worker_auto_maximum"]["targets"] + assert configured_targets[worker_target]["id"] == ( + f"worker/{provider}/{family}/{effort}" + ) + for route_name in ("auto_fast", "auto_balanced"): + prompt = routes[route_name]["prompt"] + assert "image tool—not the conversational model" in prompt + assert "Do not select a" in prompt + assert "local Qwen or Claude target" in prompt + assert "max_output_tokens" not in routes["worker_auto_maximum"] + model_gate = _documents(HERMES / "model-gate-configmap.yaml")[0]["data"][ + "model_gate.py" + ] + assert "qwen2.5:14b-instruct-q4_0" in model_gate diff --git a/testing/tests/test_hermes_chat_session_continuity.py b/testing/tests/test_hermes_chat_session_continuity.py new file mode 100644 index 00000000..0da542cb --- /dev/null +++ b/testing/tests/test_hermes_chat_session_continuity.py @@ -0,0 +1,478 @@ +"""Session lineage and continuity contracts for Hermes chat.""" + +from __future__ import annotations + +import importlib.util +import json +import sqlite3 +from pathlib import Path + +import pytest + +from testing.tests.test_hermes_chat_support import ( + HERMES, +) + + +def test_codex_native_health_overrides_historical_router_errors( + tmp_path: Path, monkeypatch +): + """Fresh first-party health is authoritative over old Switchyard probes.""" + plugin_path = HERMES / "plugins" / "auto-router" / "provider_status.py" + spec = importlib.util.spec_from_file_location("hermes_provider_status", plugin_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + health_path = tmp_path / "codex.json" + health_path.write_text( + json.dumps( + { + "state": "available", + "authenticated": True, + "transport": "codex-chatgpt-subscription", + } + ) + ) + monkeypatch.setattr(module, "CODEX_HEALTH_PATH", health_path) + monkeypatch.setattr(module, "CLAUDE_HEALTH_PATH", tmp_path / "missing.json") + monkeypatch.setattr( + module, + "_get_json", + lambda url: {"status": "ok"} + if url.endswith("/health") + else { + "models": { + "route/codex/terra/medium": { + "calls": 1, + "errors": 99, + "total_tokens": 12, + } + } + }, + ) + monkeypatch.setattr(module, "_codex_account", lambda: {}) + monkeypatch.setattr(module, "_claude_account", lambda: {}) + + codex = module.provider_status_payload()["providers"]["codex"] + + assert codex["errors"] == 99 + assert codex["state"] == "available" + assert codex["native_health"]["transport"] == "codex-chatgpt-subscription" + + +def test_api_session_patch_accepts_parent_lineage(tmp_path: Path): + """API-created workers must persist the originating Hermes session.""" + module_path = HERMES / "scripts" / "patch_api_server_sessions.py" + spec = importlib.util.spec_from_file_location("patch_api_sessions", module_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + source = tmp_path / "api_server.py" + destination = tmp_path / "patched.py" + source.write_text( + "prefix\n" + + module.BEFORE + + "middle\n" + + module.RUNS_BEFORE + + "run body\n" + + module.RUN_CLOSE_BEFORE + + module.RESPONSES_SESSION_BEFORE + + module.EVENT_CALLBACK_SIGNATURE_BEFORE + + "callback docstring and push helper\n" + + module.EVENT_CALLBACK_BODY_BEFORE + + "tool start body\n" + + module.EVENT_CALLBACK_END_BEFORE + + module.EVENT_CALLBACK_CALL_BEFORE + + module.RUN_SWEEP_BEFORE + + "suffix\n", + encoding="utf-8", + ) + + module.patch(source, destination) + patched = destination.read_text(encoding="utf-8") + + assert "X-Hermes-Parent-Session-Id" in patched + assert "parent_session_id=parent_session_id" in patched + assert "Parent session not found" in patched + assert "HERMES_API_DEFAULT_PARENT_MATCH_PREFIXES" in patched + assert "user_message.startswith(default_prefixes)" in patched + assert "session_parent_conflict" in patched + assert "X-Hermes-Conversation-Platform" in patched + assert "X-Hermes-Conversation-Title" in patched + assert 'conversation_platform != "telegram"' in patched + assert "db.record_gateway_session_peer(" in patched + assert 'display_name="Telegram"' in patched + assert "db.reopen_session(session_id)" in patched + assert 'db.end_session(session_id, f"api_run_{terminal_status}")' in patched + assert "def _record_run_activity(" in patched + assert '"_thinking": "Hermes is reasoning"' in patched + assert '"run.started": "Worker started"' in patched + assert '"run.completed": "Worker completed"' in patched + assert '"reasoning.available": "Hermes finished a reasoning step"' in patched + assert '"subagent.progress": "Nested worker progress"' in patched + assert "redact_sensitive_text" in patched + assert 'getattr(os, "O_NOFOLLOW", 0)' in patched + assert "os.fchmod(fd, 0o600)" in patched + assert "session_id=session_id" in patched + assert 'self._record_run_activity(session_id, "run.started")' in patched + assert "detail = tool_name if event_type in {" in patched + assert 'if event_type == "subagent.tool"' in patched + assert "_RUN_ACTIVITY_HEARTBEAT_SECONDS = 15.0" in patched + assert "heartbeats.get(session_id, 0.0)" in patched + assert '"subagent.thinking",' in patched + assert "Stream retention and run lifetime are separate" in patched + assert 'terminal_status in {"completed", "failed", "cancelled"}' in patched + assert patched.index("terminal_status = self._run_statuses") < patched.index( + "self._active_run_tasks.pop(run_id, None)" + ) + + +def test_telegram_api_session_migration_is_bounded_and_idempotent(tmp_path: Path): + """Only named Telegram conversations receive presentation metadata.""" + module_path = HERMES / "scripts" / "migrate_telegram_api_sessions.py" + spec = importlib.util.spec_from_file_location( + "migrate_telegram_sessions", module_path + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + state = tmp_path / "state.db" + responses = tmp_path / "response_store.db" + with sqlite3.connect(state) as connection: + connection.execute( + """CREATE TABLE sessions ( + id TEXT PRIMARY KEY, source TEXT, session_key TEXT, chat_type TEXT, + display_name TEXT, origin_json TEXT, title TEXT + )""" + ) + connection.executemany( + "INSERT INTO sessions (id, source) VALUES (?, ?)", + (("telegram-session", "api_server"), ("other-session", "api_server")), + ) + with sqlite3.connect(responses) as connection: + connection.execute( + "CREATE TABLE conversations (name TEXT PRIMARY KEY, response_id TEXT NOT NULL)" + ) + connection.execute( + "CREATE TABLE responses (response_id TEXT PRIMARY KEY, data TEXT NOT NULL, accessed_at REAL NOT NULL)" + ) + connection.executemany( + "INSERT INTO responses VALUES (?, ?, 0)", + ( + ("telegram-response", json.dumps({"session_id": "telegram-session"})), + ("other-response", json.dumps({"session_id": "other-session"})), + ), + ) + connection.executemany( + "INSERT INTO conversations VALUES (?, ?)", + (("telegram", "telegram-response"), ("unrelated", "other-response")), + ) + + assert module.migrate(state, responses) == 1 + assert module.migrate(state, responses) == 0 + with sqlite3.connect(state) as connection: + telegram = connection.execute( + "SELECT session_key, chat_type, display_name, origin_json, title " + "FROM sessions WHERE id = 'telegram-session'" + ).fetchone() + other = connection.execute( + "SELECT session_key, title FROM sessions WHERE id = 'other-session'" + ).fetchone() + assert telegram[:3] == ("telegram", "private", "Telegram") + assert json.loads(telegram[3]) == { + "platform": "telegram", + "session_key": "telegram", + } + assert telegram[4] == "Telegram · General" + assert other == (None, None) + + +def test_web_session_activity_patch_projects_bounded_events(tmp_path: Path): + """The DOM transcript includes events without polluting agent history.""" + module_path = HERMES / "scripts" / "patch_web_session_activity.py" + spec = importlib.util.spec_from_file_location("patch_web_activity", module_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + source = tmp_path / "web_server.py" + destination = tmp_path / "patched.py" + source.write_text( + "prefix\n" + + module.LATEST_ROWS_BEFORE + + module.LATEST_SELECTION_BEFORE + + module.HELPER_MARKER + + " db = object()\n" + + module.MESSAGES_BEFORE + + "suffix\n", + encoding="utf-8", + ) + + module.patch(source, destination) + patched = destination.read_text(encoding="utf-8") + + assert "def _run_activity_messages(" in patched + assert 'root = get_hermes_home() / "run-activity"' in patched + assert "path.stat().st_size > 600_000" in patched + assert "entries[-1_000:]" in patched + assert "*_run_activity_messages(sid)" in patched + assert "messages.sort(" in patched + assert "limit: Optional[int] = None" in patched + assert "total_messages = len(messages)" in patched + assert "min(int(limit), 10_000)" in patched + assert '"total_messages": total_messages' in patched + assert "SELECT id, parent_session_id, started_at, ended_at" in patched + assert "newest still-open member of the lineage" in patched + assert "immediate objective parent" in patched + assert "mixes unrelated workstreams" in patched + assert "orphaned children" in patched + assert 'item[0].get("ended_at") is None' in patched + + +def test_web_session_lineage_returns_to_resumed_parent(): + """An ended reviewer must not strand the live view away from its parent.""" + module_path = HERMES / "scripts" / "patch_web_session_activity.py" + spec = importlib.util.spec_from_file_location("patch_web_lineage", module_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + namespace: dict[str, object] = {} + exec( + "def select_active(sid, db, rows):\n" + module.LATEST_SELECTION_AFTER, + namespace, + ) + + class FakeDB: + def get_session(self, session_id): + return next((row for row in rows if row["id"] == session_id), None) + + rows = [ + { + "id": "umbrella", + "parent_session_id": None, + "started_at": 0.0, + "ended_at": None, + }, + { + "id": "root", + "parent_session_id": "umbrella", + "started_at": 1.0, + "ended_at": None, + }, + { + "id": "review-1", + "parent_session_id": "root", + "started_at": 2.0, + "ended_at": 3.0, + }, + { + "id": "review-2", + "parent_session_id": "root", + "started_at": 4.0, + "ended_at": None, + }, + ] + select_active = namespace["select_active"] + + assert select_active("root", FakeDB(), rows) == ( + "review-2", + ["root", "review-2"], + ) + assert select_active("review-1", FakeDB(), rows) == ( + "review-2", + ["root", "review-2"], + ) + rows[2]["ended_at"] = 5.0 + rows[3]["ended_at"] = 5.0 + assert select_active("root", FakeDB(), rows) == ("root", ["root"]) + + rows[1]["ended_at"] = 9.0 + rows.append( + { + "id": "orphaned-review", + "parent_session_id": "root", + "started_at": 6.0, + "ended_at": None, + } + ) + rows.append( + { + "id": "accepted-review", + "parent_session_id": "root", + "started_at": 7.0, + "ended_at": 8.0, + } + ) + assert select_active("review-1", FakeDB(), rows) == ( + "accepted-review", + ["root", "accepted-review"], + ) + + +def test_api_activity_patch_coalesces_streaming_heartbeats( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +): + """Token callbacks stay bounded while every tool transition is retained.""" + module_path = HERMES / "scripts" / "patch_api_server_sessions.py" + spec = importlib.util.spec_from_file_location("patch_api_activity", module_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + activity_body = module.EVENT_CALLBACK_SIGNATURE_AFTER.split( + " def _make_run_event_callback(", 1 + )[0] + namespace: dict[str, object] = {} + exec( + "import hashlib, json, logging, os, time\n" + "from pathlib import Path\n" + "logger = logging.getLogger(__name__)\n" + "def redact_sensitive_text(value): return value\n" + "class ActivityRecorder:\n" + activity_body, + namespace, + ) + recorder = namespace["ActivityRecorder"]() + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + + recorder._record_run_activity("session", "_thinking") + recorder._record_run_activity("session", "reasoning.available") + recorder._record_run_activity("session", "subagent.thinking") + recorder._record_run_activity("session", "tool.started", tool_name="terminal") + recorder._record_run_activity("session", "tool.completed", tool_name="terminal") + + journals = list((tmp_path / "run-activity").glob("*.jsonl")) + assert len(journals) == 1 + entries = [json.loads(line) for line in journals[0].read_text().splitlines()] + assert [entry["activity_event"] for entry in entries] == [ + "_thinking", + "tool.started", + "tool.completed", + ] + + +def test_legacy_api_sessions_are_nested_idempotently(tmp_path: Path): + """Known standalone API workers move under Cassandra without data loss.""" + module_path = HERMES / "scripts" / "migrate_api_session_lineage.py" + spec = importlib.util.spec_from_file_location("migrate_api_sessions", module_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + database = tmp_path / "state.db" + with sqlite3.connect(database) as connection: + connection.execute( + "CREATE TABLE sessions (id TEXT PRIMARY KEY, source TEXT, " + "parent_session_id TEXT, title TEXT, transcript TEXT, archived INTEGER DEFAULT 0)" + ) + connection.execute( + "INSERT INTO sessions (id, source, parent_session_id, title, transcript) " + "VALUES (?, 'tui', NULL, 'Cassandra', 'parent-data')", + (module.LEGACY_CASSANDRA_PARENT,), + ) + worker_id = next(iter(module.LEGACY_CASSANDRA_WORKERS)) + connection.execute( + "INSERT INTO sessions (id, source, parent_session_id, title, transcript) " + "VALUES (?, 'api_server', NULL, 'old', 'keep-me')", + (worker_id,), + ) + connection.executemany( + "INSERT INTO sessions (id, source, parent_session_id, title, transcript) " + "VALUES (?, 'api_server', NULL, 'old smoke', 'keep-smoke')", + ((orphan_id,) for orphan_id in module.LEGACY_ORPHANED_SMOKE_SESSIONS), + ) + + assert module.migrate(database) == 1 + len(module.LEGACY_ORPHANED_SMOKE_SESSIONS) + assert module.migrate(database) == 0 + with sqlite3.connect(database) as connection: + row = connection.execute( + "SELECT parent_session_id, title, transcript FROM sessions WHERE id = ?", + (worker_id,), + ).fetchone() + assert row == ( + module.LEGACY_CASSANDRA_PARENT, + module.LEGACY_CASSANDRA_WORKERS[worker_id], + "keep-me", + ) + with sqlite3.connect(database) as connection: + orphans = connection.execute( + "SELECT id, archived, title, transcript FROM sessions " + "WHERE id IN ({}) ORDER BY id".format( + ",".join("?" for _ in module.LEGACY_ORPHANED_SMOKE_SESSIONS) + ), + tuple(module.LEGACY_ORPHANED_SMOKE_SESSIONS), + ).fetchall() + assert orphans == sorted( + ( + orphan_id, + 1, + module.LEGACY_ORPHANED_SMOKE_SESSIONS[orphan_id], + "keep-smoke", + ) + for orphan_id in module.LEGACY_ORPHANED_SMOKE_SESSIONS + ) + + +def test_automated_triage_sessions_are_grouped_without_touching_interactive_runs( + tmp_path: Path, +): + """Only the stable Ariadne contract moves below the triage parent.""" + module_path = HERMES / "scripts" / "migrate_api_session_lineage.py" + spec = importlib.util.spec_from_file_location( + "migrate_triage_sessions", module_path + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + database = tmp_path / "state.db" + with sqlite3.connect(database) as connection: + connection.execute( + "CREATE TABLE sessions (id TEXT PRIMARY KEY, source TEXT, " + "parent_session_id TEXT, title TEXT, started_at REAL, archived INTEGER DEFAULT 0)" + ) + connection.execute( + "CREATE TABLE messages (session_id TEXT, role TEXT, content TEXT)" + ) + connection.executemany( + "INSERT INTO sessions (id, source, started_at) VALUES (?, 'api_server', ?)", + (("triage-run", 1.0), ("jenkins-run", 1.5), ("interactive-run", 2.0)), + ) + connection.executemany( + "INSERT INTO messages (session_id, role, content) VALUES (?, 'user', ?)", + ( + ( + "triage-run", + module.TRIAGE_MESSAGE_PREFIXES[0] + + " Fix for incident sonar/bstein_home/python:S2208/finding-key.", + ), + ( + "jenkins-run", + module.TRIAGE_MESSAGE_PREFIXES[1] + + "\nAnalyze incident soteria/291 for the Jenkins job soteria.", + ), + ("interactive-run", "Please explain this alert to me."), + ), + ) + + assert module.migrate(database, group_triage=True) == 2 + assert module.migrate(database, group_triage=True) == 0 + with sqlite3.connect(database) as connection: + parent = connection.execute( + "SELECT title FROM sessions WHERE id = ?", (module.TRIAGE_PARENT,) + ).fetchone() + triage = connection.execute( + "SELECT parent_session_id, title FROM sessions WHERE id = 'triage-run'" + ).fetchone() + interactive = connection.execute( + "SELECT parent_session_id FROM sessions WHERE id = 'interactive-run'" + ).fetchone() + jenkins = connection.execute( + "SELECT parent_session_id, title FROM sessions WHERE id = 'jenkins-run'" + ).fetchone() + assert parent == (module.TRIAGE_PARENT_TITLE,) + assert triage == ( + module.TRIAGE_PARENT, + "Sonar · bstein_home · python:S2208 · iage-run", + ) + assert interactive == (None,) + assert jenkins == ( + module.TRIAGE_PARENT, + "Sonar · soteria/291 · kins-run", + ) diff --git a/testing/tests/test_hermes_chat_support.py b/testing/tests/test_hermes_chat_support.py new file mode 100644 index 00000000..bf7d998c --- /dev/null +++ b/testing/tests/test_hermes_chat_support.py @@ -0,0 +1,38 @@ +"""Shared paths and loaders for Hermes chat capability contracts.""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path +from types import SimpleNamespace + +import yaml + +ROOT = Path(__file__).parents[2] +HERMES = ROOT / "services" / "hermes" +VAULT = ROOT / "services" / "vault" + + +def _documents(path: Path) -> list[dict]: + return [doc for doc in yaml.safe_load_all(path.read_text()) if doc] + + +def _load_broker_module(name: str, filename: str, monkeypatch): + """Load one broker with its mounted routing-catalog dependency.""" + catalog_path = HERMES / "scripts" / "routing_catalog.py" + catalog_spec = importlib.util.spec_from_file_location( + "routing_catalog", catalog_path + ) + assert catalog_spec and catalog_spec.loader + catalog = importlib.util.module_from_spec(catalog_spec) + catalog_spec.loader.exec_module(catalog) + monkeypatch.setitem(sys.modules, "routing_catalog", catalog) + monkeypatch.setitem(sys.modules, "httpx", SimpleNamespace()) + + broker_path = HERMES / "scripts" / filename + spec = importlib.util.spec_from_file_location(name, broker_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module diff --git a/testing/tests/test_hermes_cli_lanes.py b/testing/tests/test_hermes_cli_lanes.py index fe5272da..c40740aa 100644 --- a/testing/tests/test_hermes_cli_lanes.py +++ b/testing/tests/test_hermes_cli_lanes.py @@ -1,93 +1,19 @@ -"""Focused tests for Agent Hermes' direct Codex and Claude Kanban lanes.""" +"""Provider routing and process-isolation contracts for Hermes CLI lanes.""" from __future__ import annotations -import importlib.util import json import os import signal import sys -from contextlib import nullcontext from pathlib import Path -from types import SimpleNamespace - -import pytest -import yaml -SCRIPTS = Path(__file__).parents[2] / "services/hermes/scripts" -sys.path.insert(0, str(SCRIPTS)) -HERMES = Path(__file__).parents[2] / "services/hermes" -KEYCLOAK = Path(__file__).parents[2] / "services/keycloak" -FLUX_HERMES = ( - Path(__file__).parents[2] - / "clusters/atlas/flux-system/applications/hermes/kustomization.yaml" +from testing.tests.test_hermes_cli_lanes_support import ( + _SwitchyardResponse, + lanes, ) - -def _load(name: str): - spec = importlib.util.spec_from_file_location(name, SCRIPTS / f"{name}.py") - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - sys.modules[spec.name] = module - spec.loader.exec_module(module) - return module - - -lanes = _load("cli_lane_runner") -policy = _load("claude_command_policy") -migration = _load("migrate_herdr_state") -auth_patch = _load("patch_hermes_auth") -tui_gateway_patch = _load("patch_tui_gateway") -codex_runtime_patch = _load("patch_codex_runtime") -ttyd_patch = _load("patch_ttyd_index") -client_config = _load("configure_agent_clients") - - -def _agent_deployment() -> dict: - return yaml.safe_load((HERMES / "agent-deployment.yaml").read_text()) - - -def _services() -> dict[str, dict]: - return { - item["metadata"]["name"]: item - for item in yaml.safe_load_all((HERMES / "service.yaml").read_text()) - if item - } - - -def _oauth_deployment(name: str) -> dict: - documents = [ - item - for item in yaml.safe_load_all((HERMES / "oauth2-proxy.yaml").read_text()) - if item - ] - return next( - item - for item in documents - if item["kind"] == "Deployment" and item["metadata"]["name"] == name - ) - - -class _SwitchyardResponse: - """Minimal context-managed response used by routing contract tests.""" - - def __init__(self, selected: str, rationale: str = "local classifier vote"): - self.headers = { - "x-model-router-selected-model": selected, - "x-model-router-rationale": rationale, - } - - def __enter__(self): - return self - - def __exit__(self, *_args): - return False - - def read(self): - return b"{}" - - def test_auto_lane_uses_switchyard_worker_decision(): observed = {} @@ -460,1512 +386,3 @@ def test_worker_terminal_descendants_in_separate_groups_are_killed(monkeypatch): (5000, signal.SIGKILL), } assert process_signals == [(5000, signal.SIGTERM), (5000, signal.SIGKILL)] - - -def test_unassigned_ready_task_is_persistently_routed_to_auto_lane(monkeypatch): - task = SimpleNamespace(id="t_auto", assignee=None, status="ready") - assigned = [] - - class Connection: - def close(self): - return None - - def assign_task(_conn, task_id, profile): - assigned.append((task_id, profile)) - task.assignee = profile - return True - - fake_db = SimpleNamespace( - list_boards=lambda include_archived=False: [{"slug": "cassandra"}], - scoped_current_board=lambda _board: nullcontext(), - connect=lambda board: Connection(), - recompute_ready=lambda _conn: None, - list_tasks=lambda _conn: [task], - assign_task=assign_task, - get_task=lambda _conn, _task_id: task, - claim_task=lambda _conn, _task_id, **_kwargs: task, - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - - assert lanes.claim_ready(set(), 1) == [("cassandra", "t_auto")] - assert assigned == [("t_auto", "cli-auto")] - - -def test_corrupt_board_is_quarantined_without_stopping_healthy_lanes(monkeypatch, capsys): - class CorruptBoardError(Exception): - pass - - task = SimpleNamespace(id="t_healthy", assignee="cli-auto", status="ready") - - class Connection: - def close(self): - return None - - def connect(*, board): - if board == "cassandra": - raise CorruptBoardError("integrity_check failed") - return Connection() - - fake_db = SimpleNamespace( - KanbanDbCorruptError=CorruptBoardError, - list_boards=lambda include_archived=False: [ - {"slug": "cassandra"}, - {"slug": "healthy"}, - ], - scoped_current_board=lambda _board: nullcontext(), - connect=connect, - recompute_ready=lambda _conn: None, - list_tasks=lambda _conn: [task], - claim_task=lambda _conn, _task_id, **_kwargs: task, - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - lanes.BOARD_CORRUPTION_ERRORS.clear() - - assert lanes.claim_ready(set(), 1) == [("healthy", "t_healthy")] - assert "temporarily skipping Kanban board 'cassandra'" in capsys.readouterr().err - - -def test_transient_board_scan_failure_does_not_stop_healthy_lanes(monkeypatch, capsys): - task = SimpleNamespace(id="t_healthy", assignee="cli-auto", status="ready") - - class Connection: - def __init__(self, board): - self.board = board - - def close(self): - return None - - def recompute_ready(connection): - if connection.board == "cassandra": - raise lanes.sqlite3.OperationalError("disk I/O error") - - fake_db = SimpleNamespace( - list_boards=lambda include_archived=False: [ - {"slug": "cassandra"}, - {"slug": "healthy"}, - ], - scoped_current_board=lambda _board: nullcontext(), - connect=lambda board: Connection(board), - recompute_ready=recompute_ready, - list_tasks=lambda _conn: [task], - claim_task=lambda _conn, _task_id, **_kwargs: task, - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - lanes.BOARD_CORRUPTION_ERRORS.clear() - - assert lanes.claim_ready(set(), 1) == [("healthy", "t_healthy")] - error = capsys.readouterr().err - assert "temporarily skipping Kanban board 'cassandra'" in error - assert "storage OperationalError: disk I/O error" in error - - -def test_board_call_retries_storage_faults_on_fresh_connections(): - connections = [] - - class Connection: - def __init__(self): - self.closed = False - - def close(self): - self.closed = True - - def connect(*, board): - assert board == "cassandra" - connection = Connection() - connections.append(connection) - return connection - - attempts = [] - - def operation(_connection): - attempts.append(1) - if len(attempts) < 3: - raise lanes.sqlite3.OperationalError("disk I/O error") - return "healthy" - - fake_db = SimpleNamespace( - scoped_current_board=lambda _board: nullcontext(), - connect=connect, - ) - lanes.BOARD_CORRUPTION_ERRORS.clear() - - assert lanes._board_call(fake_db, "cassandra", operation) == "healthy" - assert len(connections) == 3 - assert all(connection.closed for connection in connections) - - -@pytest.mark.parametrize( - ("result", "expected_action"), - [ - (lanes.ProcessResult(0, "plain text only", None, False), "block"), - ( - lanes.ProcessResult( - 0, - "", - { - "status": "completed", - "summary": "done", - "changed_files": ["src/a.py"], - "tests_run": ["pytest -q"], - "artifacts": ["reports/result.json"], - "blockers": [], - }, - False, - ), - "complete", - ), - ( - lanes.ProcessResult( - 0, - "", - { - "status": "completed", - "summary": "The full test suite is still running.", - "changed_files": ["src/a.py"], - "tests_run": ["pytest -q — in progress"], - "artifacts": [], - "blockers": [], - }, - False, - ), - "block", - ), - ], -) -def test_claim_requires_structured_evidence_and_surfaces_artifacts( - tmp_path: Path, - monkeypatch, - result, - expected_action, -): - task = SimpleNamespace( - id="t_worker", - current_run_id=4, - assignee="cli-auto", - max_runtime_seconds=60, - ) - calls = [] - heartbeats = [] - connections = [] - artifact = tmp_path / "reports/result.json" - artifact.parent.mkdir() - artifact.write_text("{}\n", encoding="utf-8") - - class Connection: - def __init__(self): - self.closed = False - - def close(self): - self.closed = True - - def connect(*, board): - assert board == "cassandra" - connection = Connection() - connections.append(connection) - return connection - - fake_db = SimpleNamespace( - scoped_current_board=lambda _board: nullcontext(), - connect=connect, - get_task=lambda _conn, _task_id: task, - worker_log_path=lambda _task_id, board: tmp_path / "worker.log", - _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_worker"), - set_branch_name=lambda *_args: None, - set_workspace_path=lambda *_args: None, - build_worker_context=lambda *_args: "bounded objective", - heartbeat_worker=lambda _conn, _task_id, *, note, expected_run_id: ( - heartbeats.append((note, expected_run_id)) or True - ), - add_comment=lambda *_args: None, - complete_task=lambda *_args, **kwargs: calls.append(("complete", kwargs)), - block_task=lambda *_args, **kwargs: calls.append(("block", kwargs)), - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - monkeypatch.setattr(lanes, "state_path", lambda _board, _task_id: tmp_path / "state.json") - monkeypatch.setattr( - lanes, - "select_route", - lambda *_args, **_kwargs: lanes.Route( - "codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, () - ), - ) - def run_provider(*args, **_kwargs): - assert connections[0].closed - before_heartbeat = len(connections) - assert args[6]("working") is True - assert len(connections) == before_heartbeat + 1 - assert connections[-1].closed - return result - - monkeypatch.setattr(lanes, "run_provider", run_provider) - - lanes.execute_claim("cassandra", "t_worker") - - assert calls[0][0] == expected_action - assert heartbeats == [("working", 4)] - if expected_action == "complete": - assert calls[0][1]["metadata"]["artifacts"] == [str(artifact)] - assert calls[0][1]["metadata"]["tests_run"] == ["pytest -q"] - else: - assert calls[0][1]["kind"] == "capability" - assert all(connection.closed for connection in connections) - - -def test_goal_card_continues_after_local_judge_rejects_progress( - tmp_path: Path, - monkeypatch, -): - task = SimpleNamespace( - id="t_goal", - current_run_id=12, - assignee="cli-auto", - max_runtime_seconds=300, - goal_mode=True, - goal_max_turns=3, - ) - calls = [] - comments = [] - - class Connection: - def close(self): - return None - - fake_db = SimpleNamespace( - scoped_current_board=lambda _board: nullcontext(), - connect=lambda board: Connection(), - get_task=lambda _conn, _task_id: task, - worker_log_path=lambda _task_id, board: tmp_path / "worker.log", - _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_goal"), - set_branch_name=lambda *_args: None, - set_workspace_path=lambda *_args: None, - build_worker_context=lambda *_args: "Run tests, commit, push, and verify remote HEAD.", - heartbeat_worker=lambda *_args, **_kwargs: True, - add_comment=lambda _conn, _task_id, _author, body: comments.append(body), - complete_task=lambda *_args, **kwargs: calls.append(("complete", kwargs)), - block_task=lambda *_args, **kwargs: calls.append(("block", kwargs)), - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - monkeypatch.setattr( - lanes, - "state_path", - lambda _board, _task_id: tmp_path / "state.json", - ) - claude_low = lanes.Route( - "claude", "claude-fable-5", "low", "claude-low", "jetson", "vote", 1, () - ) - codex_low = lanes.Route( - "codex", "gpt-5.6-luna", "low", "codex-low", "manual", "fallback", 1, () - ) - codex_xhigh = lanes.Route( - "codex", "gpt-5.6-sol", "xhigh", "codex-xhigh", "jetson", "escalated", 1, () - ) - route_calls = [] - - def select_route(_prompt, assignee, **kwargs): - route_calls.append((assignee, kwargs)) - if assignee == "cli-codex-low": - return codex_low - if len(route_calls) == 1: - return claude_low - return codex_xhigh - - monkeypatch.setattr(lanes, "select_route", select_route) - monkeypatch.setattr(lanes, "fresh_unavailable_provider", lambda: None) - reports = [ - lanes.ProcessResult(1, "authentication expired", None, True), - lanes.ProcessResult( - 0, - "first turn", - { - "status": "completed", - "summary": "Focused tests passed.", - "changed_files": ["src/a.py"], - "tests_run": ["pytest focused: passed"], - "artifacts": [], - "blockers": [], - }, - False, - ), - lanes.ProcessResult( - 0, - "second turn", - { - "status": "completed", - "summary": "Full tests passed; commit pushed and remote HEAD verified.", - "changed_files": ["src/a.py"], - "tests_run": ["pytest full: passed"], - "artifacts": [], - "blockers": [], - }, - False, - ), - ] - monkeypatch.setattr(lanes, "run_provider", lambda *_args, **_kwargs: reports.pop(0)) - verdicts = iter( - [ - (False, "commit, push, and remote verification are missing"), - (True, "all explicit acceptance criteria have evidence"), - ] - ) - judge_contexts = [] - - def judge_goal_completion(objective, *_args, **_kwargs): - judge_contexts.append(objective) - return next(verdicts) - - monkeypatch.setattr( - lanes.cli_lane_goal, - "judge_goal_completion", - judge_goal_completion, - ) - - lanes.execute_claim("cassandra", "t_goal") - - assert calls[0][0] == "complete" - assert calls[0][1]["metadata"]["goal_turn"] == 2 - assert any("Goal completion rejected; continuing turn 2/3" in item for item in comments) - assert any("Goal route 2/3: codex/gpt-5.6-sol at xhigh" in item for item in comments) - assert route_calls[2][1]["exclude_provider"] == "claude" - assert "prior rejected reports" in judge_contexts[1] - assert "commit, push, and remote verification are missing" in judge_contexts[1] - assert reports == [] - - -def test_workspace_preparation_failure_durably_blocks_the_claim(tmp_path: Path, monkeypatch): - task = SimpleNamespace(id="t_bad_worktree", current_run_id=7, assignee="cli-auto") - calls = [] - - class Connection: - def close(self): - return None - - fake_db = SimpleNamespace( - scoped_current_board=lambda _board: nullcontext(), - connect=lambda board: Connection(), - get_task=lambda _conn, _task_id: task, - worker_log_path=lambda _task_id, board: tmp_path / "worker.log", - _resolve_worktree_workspace=lambda _task, board: (_ for _ in ()).throw( - ValueError("not a Git repository") - ), - block_task=lambda *_args, **kwargs: calls.append(kwargs), - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - monkeypatch.setattr(lanes, "state_path", lambda _board, _task_id: tmp_path / "state.json") - - lanes.execute_claim("cassandra", "t_bad_worktree") - - assert calls[0]["kind"] == "capability" - assert calls[0]["expected_run_id"] == 7 - assert "not a Git repository" in calls[0]["reason"] - - -def test_artifacts_cannot_escape_the_task_worktree(tmp_path: Path): - workspace = tmp_path / "workspace" - workspace.mkdir() - inside = workspace / "report.json" - outside = tmp_path / "auth.json" - inside.write_text("{}\n", encoding="utf-8") - outside.write_text("secret\n", encoding="utf-8") - - assert lanes.workspace_artifacts( - workspace, - ["report.json", str(outside), "missing.json"], - ) == [str(inside)] - - -def test_restart_provider_change_includes_explicit_workspace_handoff(tmp_path: Path, monkeypatch): - task = SimpleNamespace( - id="t_resume", - current_run_id=9, - assignee="cli-auto", - max_runtime_seconds=60, - ) - state_file = tmp_path / "state.json" - state_file.write_text( - json.dumps({"current_route": {"provider": "claude"}}), - encoding="utf-8", - ) - (tmp_path / "worker.log").write_text("prior provider evidence", encoding="utf-8") - prompts = [] - - class Connection: - def close(self): - return None - - fake_db = SimpleNamespace( - scoped_current_board=lambda _board: nullcontext(), - connect=lambda board: Connection(), - get_task=lambda _conn, _task_id: task, - worker_log_path=lambda _task_id, board: tmp_path / "worker.log", - _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_resume"), - set_branch_name=lambda *_args: None, - set_workspace_path=lambda *_args: None, - build_worker_context=lambda *_args: "resume objective", - add_comment=lambda *_args: None, - complete_task=lambda *_args, **_kwargs: None, - block_task=lambda *_args, **_kwargs: None, - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - monkeypatch.setattr(lanes, "state_path", lambda _board, _task_id: state_file) - monkeypatch.setattr( - lanes, - "select_route", - lambda *_args, **_kwargs: lanes.Route( - "codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, () - ), - ) - monkeypatch.setattr( - lanes, - "git_handoff", - lambda _workspace, output: f"HANDOFF:{output}", - ) - monkeypatch.setattr( - lanes, - "run_provider", - lambda _route, prompt, *_args, **_kwargs: ( - prompts.append(prompt) - or lanes.ProcessResult( - 0, - "", - { - "status": "completed", - "summary": "done", - "changed_files": [], - "tests_run": [], - "artifacts": [], - "blockers": [], - }, - False, - ) - ), - ) - - lanes.execute_claim("cassandra", "t_resume") - - assert "HANDOFF:prior provider evidence" in prompts[0] - - -def test_provider_commands_are_structured_unattended_and_capped(tmp_path: Path): - route = lanes.Route("codex", "gpt-5.6-sol", "xhigh", "codex-xhigh", "jetson", "vote", 1, ()) - command = lanes._codex_command(route, "Work.", tmp_path, {}, tmp_path / "result.json") - assert "--dangerously-bypass-approvals-and-sandbox" in command - assert "--json" in command - assert "--output-schema" in command - assert 'model_reasoning_effort="xhigh"' in command - - claude_state = {"claude_session_id": "13864642-2985-4f91-bef5-53f145f878e8"} - claude = lanes._claude_command( - lanes.Route("claude", "claude-opus-5", "xhigh", "claude-xhigh", "jetson", "vote", 1, ()), - "Review.", - claude_state, - False, - ) - assert "--dangerously-skip-permissions" in claude - assert "--output-format" in claude and "stream-json" in claude - assert "--json-schema" in claude - assert "--disallowedTools" in claude - assert "Bash(kubectl apply *)" not in claude - assert "Bash(flux reconcile *)" not in claude - assert "max" not in claude - - -def test_worker_contract_separates_review_findings_from_task_blockers(tmp_path: Path): - prompt = lanes.build_prompt("Review the change.", tmp_path) - - assert "put defects and risks in findings" in prompt - assert "blockers array must be empty whenever status is completed" in prompt - assert "findings" in lanes.RESULT_SCHEMA["properties"] - assert set(lanes.RESULT_SCHEMA["required"]) == set( - lanes.RESULT_SCHEMA["properties"] - ) - assert "assigned task itself" in lanes.RESULT_SCHEMA["properties"]["blockers"]["description"] - - -@pytest.mark.parametrize( - "command", - [ - "git push --force origin main", - "git reset --hard HEAD~1", - "git clean -fd", - ], -) -def test_claude_pretool_hook_blocks_hard_denies(command: str): - assert policy.denial_reason(command) - - -def test_claude_pretool_hook_allows_normal_engineering(): - assert policy.denial_reason("pytest -q testing/tests") is None - assert policy.denial_reason("git push origin feature/hermes") is None - assert policy.denial_reason("kubectl delete pod -n cassandra stuck-worker") is None - assert policy.denial_reason("flux reconcile kustomization hermes") is None - assert policy.denial_reason("vault kv get kv/atlas/hermes") is None - - -def test_claude_settings_preserve_state_and_install_three_guardrail_layers(tmp_path: Path): - state = tmp_path / ".claude.json" - settings = tmp_path / "settings.json" - state.write_text('{"promptQueueUseCount": 4}\n', encoding="utf-8") - settings.write_text( - json.dumps( - { - "theme": "dark", - "permissions": { - "deny": [ - "Bash(kubectl apply *)", - "Bash(flux reconcile *)", - "Bash(vault kv *)", - "Bash(custom-owner-rule *)", - ] - }, - } - ) - + "\n", - encoding="utf-8", - ) - - client_config.configure_claude_state(state) - client_config.configure_claude_settings(settings) - - state_value = json.loads(state.read_text()) - settings_value = json.loads(settings.read_text()) - assert state_value["promptQueueUseCount"] == 4 - assert state_value["bypassPermissionsModeAccepted"] is True - assert settings_value["theme"] == "dark" - assert "Bash(git reset --hard *)" in settings_value["permissions"]["deny"] - assert "Bash(custom-owner-rule *)" in settings_value["permissions"]["deny"] - assert "Bash(kubectl apply *)" not in settings_value["permissions"]["deny"] - assert "Bash(flux reconcile *)" not in settings_value["permissions"]["deny"] - assert "Bash(vault kv *)" not in settings_value["permissions"]["deny"] - hook = settings_value["hooks"]["PreToolUse"][0]["hooks"][0] - assert "claude_command_policy.py" in hook["command"] - - -def test_legacy_state_is_archived_without_removing_provider_transcripts(tmp_path: Path): - session = tmp_path / "home/.config/herdr/session.json" - session.parent.mkdir(parents=True) - session.write_text('{"agents":[{"agent":"claude","session_id":"abc"}]}', encoding="utf-8") - binary = tmp_path / "tools/bin/herdr" - binary.parent.mkdir(parents=True) - binary.write_text("legacy", encoding="utf-8") - (tmp_path / "home/.claude").mkdir() - (tmp_path / "home/.codex").mkdir() - - archive = migration.archive_legacy_state(tmp_path) - - value = json.loads(archive.read_text()) - assert value["legacy_session"]["agents"][0]["session_id"] == "abc" - assert (tmp_path / "home/.claude").is_dir() - assert (tmp_path / "home/.codex").is_dir() - assert not binary.exists() - assert not (tmp_path / "home/.config/herdr").exists() - - -def test_agent_uses_one_native_kanban_control_plane(): - configmap = yaml.safe_load((HERMES / "agent-configmap.yaml").read_text()) - config = yaml.safe_load(configmap["data"]["config.yaml"]) - assert config["model"] == { - "provider": "atlas-switchyard", - "default": "atlas/auto/balanced", - "model": "atlas/auto/balanced", - } - assert config["kanban"]["dispatch_in_gateway"] is True - assert config["kanban"]["default_assignee"] == "cli-auto" - assert config["plugins"]["enabled"] == ["auto-router"] - - deployment = _agent_deployment() - pod = deployment["spec"]["template"]["spec"] - assert pod["enableServiceLinks"] is False - names = {item["name"] for item in pod["containers"]} - assert "cli-lane-runner" in names - assert "terminal" in names - assert not any("herdr" in name for name in names) - rendered = (HERMES / "agent-deployment.yaml").read_text() - assert "herdr server" not in rendered - assert "herdr-dispatch" not in rendered - - -def test_cli_lane_reserves_cpu_headroom_for_ui_and_auth(): - deployment = _agent_deployment() - containers = { - item["name"]: item - for item in deployment["spec"]["template"]["spec"]["containers"] - } - lane = containers["cli-lane-runner"] - environment = {item["name"]: item["value"] for item in lane["env"]} - - assert environment["HERMES_CLI_LANE_CONCURRENCY"] == "2" - assert lane["resources"] == { - "requests": {"cpu": "100m", "memory": "256Mi"}, - "limits": {"cpu": "2", "memory": "6Gi"}, - } - - -def test_agent_avoids_unhealthy_nodes_and_fits_its_remaining_capacity(): - """Placement correction: keep the agent off nodes that cannot hold it. - - titan-04 is cordoned after repeated kernel undervoltage and kubelet - failure, and titan-19 was probe/Longhorn unstable under worker load, so - both must join the existing hard exclusions. That leaves titan-05 as the - healthy candidate, which is tight enough on requested CPU that the main - container has to give back 50m to schedule there. - """ - pod = _agent_deployment()["spec"]["template"]["spec"] - hostnames = next( - item - for item in pod["affinity"]["nodeAffinity"][ - "requiredDuringSchedulingIgnoredDuringExecution" - ]["nodeSelectorTerms"][0]["matchExpressions"] - if item["key"] == "kubernetes.io/hostname" - ) - - assert hostnames["operator"] == "NotIn" - assert set(hostnames["values"]) >= {"titan-04", "titan-19"} - - hermes = next( - item for item in pod["containers"] if item["name"] == "hermes" - ) - assert hermes["resources"]["requests"]["cpu"] == "300m" - - -def test_agent_root_is_stock_dashboard_and_terminal_is_a_separate_path(): - deployment = _agent_deployment() - pod = deployment["spec"]["template"]["spec"] - containers = {item["name"]: item for item in pod["containers"]} - assert "webui" not in containers - - assert "dashboard" not in containers - hermes = containers["hermes"] - hermes_env = {item["name"]: item["value"] for item in hermes["env"]} - assert hermes_env["HERMES_STREAM_STALE_TIMEOUT"] == "600" - assert hermes_env["HERMES_API_CALL_STALE_TIMEOUT"] == "600" - assert hermes["command"] == ["/bin/sh", "-ec"] - startup = hermes["args"][0] - assert ". /opt/data/.env" in startup - assert "exec /init /opt/hermes/docker/main-wrapper.sh gateway run" in startup - hermes_env = {item["name"]: item["value"] for item in hermes["env"]} - assert hermes_env["HERMES_DASHBOARD"] == "1" - assert hermes_env["HERMES_DASHBOARD_HOST"] == "127.0.0.1" - assert hermes_env["HERMES_DASHBOARD_PORT"] == "9119" - assert hermes_env["HERMES_TUI_AGENT_INIT_TIMEOUT_S"] == "180" - assert hermes["securityContext"]["runAsUser"] == 0 - assert hermes["securityContext"]["runAsGroup"] == 0 - for probe_name in ("startupProbe", "readinessProbe", "livenessProbe"): - probe = hermes[probe_name] - assert probe["exec"]["command"] == [ - "curl", - "-fsS", - "http://127.0.0.1:9119/api/status", - ] - - terminal = containers["terminal"] - command = terminal["args"][0] - assert "--base-path /terminal" in command - assert "--check-origin" not in command - assert "/usr/bin/tmux new-session -A" in command - assert "--continue" in command - assert "--yolo" in command - terminal_env = {item["name"]: item["value"] for item in terminal["env"]} - assert terminal_env["HERMES_TUI_AGENT_INIT_TIMEOUT_S"] == "180" - - claude_broker = containers["claude-broker"] - claude_env = {item["name"]: item["value"] for item in claude_broker["env"]} - assert claude_env["HERMES_CLAUDE_BROKER_CONCURRENCY"] == "2" - assert claude_broker["readinessProbe"]["tcpSocket"] == {"port": "claude-broker"} - assert claude_broker["livenessProbe"]["tcpSocket"] == {"port": "claude-broker"} - - args = containers["oauth2-proxy"]["args"] - terminal_upstream = "--upstream=http://127.0.0.1:7681/terminal/" - dashboard_upstream = "--upstream=http://127.0.0.1:9119/" - assert terminal_upstream in args - assert dashboard_upstream in args - assert args.index(terminal_upstream) < args.index(dashboard_upstream) - assert "--pass-host-header=false" in args - assert "--cookie-refresh=19m" in args - assert "--session-store-type=redis" in args - assert any( - arg.startswith("--redis-connection-url=redis://hermes-oauth-sessions.") - for arg in args - ) - - patch_init = next( - item for item in pod["initContainers"] - if item["name"] == "patch-tui-gateway" - ) - assert patch_init["command"][-1] == "/patched/server.py" - for name in ("hermes", "terminal"): - mounts = containers[name]["volumeMounts"] - assert { - "name": "tui-gateway-patch", - "mountPath": "/opt/hermes/tui_gateway/server.py", - "subPath": "server.py", - } in mounts - - ingress_documents = [ - item - for item in yaml.safe_load_all((HERMES / "agent-ingress.yaml").read_text()) - if item - ] - middlewares = { - item["metadata"]["name"]: item - for item in ingress_documents - if item["kind"] == "Middleware" - } - assert middlewares["hermes-agent-terminal-slash"]["spec"]["redirectRegex"][ - "replacement" - ].endswith("/terminal/") - assert middlewares["hermes-agent-stock-dashboard-headers"]["spec"]["headers"][ - "customRequestHeaders" - ]["Origin"] == "http://127.0.0.1:9119" - ingresses = { - item["metadata"]["name"]: item - for item in ingress_documents - if item["kind"] == "Ingress" - } - assert ingresses["hermes-agent-dashboard"]["metadata"]["annotations"][ - "traefik.ingress.kubernetes.io/router.middlewares" - ] == "hermes-hermes-agent-stock-dashboard-headers@kubernetescrd" - assert ingresses["hermes-agent-terminal"]["metadata"]["annotations"][ - "traefik.ingress.kubernetes.io/router.middlewares" - ] == "hermes-hermes-agent-terminal-slash@kubernetescrd" - - -def test_broker_services_survive_sibling_container_readiness_loss(): - services = _services() - - for name in ( - "hermes-image-broker", - "hermes-codex-broker", - "hermes-local-image", - "hermes-claude-broker", - ): - assert services[name]["spec"]["publishNotReadyAddresses"] is True - - -def test_agent_dashboard_reconnects_all_transient_websockets(): - dockerfile = ( - HERMES.parents[1] / "dockerfiles/Dockerfile.hermes-agent" - ).read_text(encoding="utf-8") - assert "eventsRetryAttempt.current" in dockerfile - assert "if (!unmounting) setVersion((v) => v + 1);" in dockerfile - assert "events feed rejected (${ev.code}) — reload the page" in dockerfile - assert 'url = await api.buildWsUrl("/api/pty", params);' in dockerfile - assert 'url = await buildWsUrl("/api/events", { channel });' in dockerfile - assert dockerfile.count("' await api.getSessions(1, 0,") == 2 - assert ".then(() => gw.connect())" in dockerfile - assert 'api.getSessions(1, 0, profile ?? "")' in dockerfile - assert "dashboard token rotated by a server restart" in dockerfile - - -def test_agent_refreshes_routes_after_restoring_cli_logins(): - deployment = _agent_deployment() - init_containers = { - item["name"]: item - for item in deployment["spec"]["template"]["spec"]["initContainers"] - } - configure = init_containers["configure-agent-clients"] - command = configure["command"][-1] - assert "configure_agent_clients.py" in command - assert command.index("configure_agent_clients.py") < command.index( - "hermes_coordinator.py --once" - ) - env = {item["name"]: item["value"] for item in configure["env"]} - assert env["HERMES_AUTH_FILE"] == "/runtime-access/hermes-auth.json" - assert env["PYTHONPATH"] == "/opt/hermes" - for name in ("bootstrap-coordinator", "configure-agent-clients"): - route_env = { - item["name"]: item["value"] - for item in init_containers[name]["env"] - } - assert route_env["CODEX_HOME"] == "/runtime-access/codex" - assert route_env["CLAUDE_CONFIG_DIR"] == "/runtime-access/claude" - assert "/opt/data/tools/bin" in route_env["PATH"] - - assert "patch-web-session-activity" in init_containers - web_patch = init_containers["patch-web-session-activity"] - assert "/opt/coordinator/patch_web_session_activity.py" in web_patch["command"] - - containers = { - item["name"]: item - for item in deployment["spec"]["template"]["spec"]["containers"] - } - steward_env = { - item["name"]: item["value"] - for item in containers["model-steward"]["env"] - } - assert steward_env["CODEX_HOME"] == "/runtime-access/codex" - assert steward_env["CLAUDE_CONFIG_DIR"] == "/runtime-access/claude" - assert "/opt/data/tools/bin" in steward_env["PATH"] - hermes_mounts = { - (item["name"], item["mountPath"], item.get("subPath")) - for item in containers["hermes"]["volumeMounts"] - } - assert ( - "web-server-patch", - "/opt/hermes/hermes_cli/web_server.py", - "web_server.py", - ) in hermes_mounts - - -def test_flux_health_checks_follow_the_owner_oauth_sidecar(): - flux = yaml.safe_load(FLUX_HERMES.read_text()) - checks = { - (item["kind"], item["name"]) - for item in flux["spec"]["healthChecks"] - } - assert ("Deployment", "hermes-agent") in checks - assert ("DaemonSet", "hermes-node-ssh-access") in checks - assert ("Deployment", "oauth2-proxy-hermes-agent") not in checks - - -def test_agent_auth_is_bstein_group_and_email_bounded(): - deployment = _agent_deployment() - oauth = next( - item for item in deployment["spec"]["template"]["spec"]["containers"] - if item["name"] == "oauth2-proxy" - ) - args = oauth["args"] - assert "--user-id-claim=sub" in args - assert "--oidc-groups-claim=groups" in args - assert "--allowed-group=/hermes-owner" in args - assert "--authenticated-emails-file=/etc/oauth2-proxy/allowed-emails" in args - script = (KEYCLOAK / "scripts/hermes_access_oidc_ensure.sh").read_text() - assert 'group_name="hermes-owner"' in script - assert "username=bstein&exact=true" in script - assert '"full.path":"true"' in script - - -def test_agent_network_boundary_allows_only_authenticated_and_metrics_surfaces(): - documents = [ - item - for item in yaml.safe_load_all((HERMES / "networkpolicy.yaml").read_text()) - if item - ] - isolation = next(item for item in documents if item.get("metadata", {}).get("name") == "hermes-agent-isolation") - assert isolation["spec"]["ingress"] == [ - { - "from": [ - { - "namespaceSelector": { - "matchLabels": { - "kubernetes.io/metadata.name": "traefik" - } - }, - "podSelector": { - "matchLabels": {"app.kubernetes.io/name": "traefik"} - }, - } - ], - "ports": [{"protocol": "TCP", "port": 4180}], - }, - { - "from": [ - { - "podSelector": { - "matchLabels": {"app": "hermes-chat-tenant"} - } - } - ], - "ports": [ - {"protocol": "TCP", "port": 9002}, - {"protocol": "TCP", "port": 9003}, - ], - }, - { - "from": [ - { - "podSelector": { - "matchLabels": {"app": "hermes-switchyard"} - } - } - ], - "ports": [ - {"protocol": "TCP", "port": 9003}, - {"protocol": "TCP", "port": 9006}, - ], - }, - { - "from": [ - { - "namespaceSelector": { - "matchLabels": { - "kubernetes.io/metadata.name": "monitoring" - } - }, - "podSelector": {"matchLabels": {"app": "server"}}, - } - ], - "ports": [{"protocol": "TCP", "port": 9010}], - }, - ] - assert isolation["spec"]["egress"] == [{}] - - -def test_owner_agent_has_cluster_admin_kubernetes_context(): - config = yaml.safe_load((HERMES / "agent-kubeconfig.yaml").read_text()) - assert config["current-context"] == "atlas-owner" - assert config["contexts"][0]["context"]["namespace"] == "default" - rbac_path = HERMES / "agent-rbac.yaml" - documents = [item for item in yaml.safe_load_all(rbac_path.read_text()) if item] - binding = next(item for item in documents if item["kind"] == "ClusterRoleBinding") - assert binding["roleRef"] == { - "apiGroup": "rbac.authorization.k8s.io", - "kind": "ClusterRole", - "name": "cluster-admin", - } - assert binding["subjects"] == [ - {"kind": "ServiceAccount", "name": "hermes-agent", "namespace": "hermes"} - ] - - -def test_owner_agent_has_pinned_dedicated_node_ssh_access(): - deployment = _agent_deployment() - annotations = deployment["spec"]["template"]["metadata"]["annotations"] - assert annotations[ - "vault.hashicorp.com/agent-inject-secret-node-ssh-private-key" - ] == "kv/data/atlas/hermes/developer-ssh" - assert annotations[ - "vault.hashicorp.com/agent-inject-secret-node-ssh-config" - ] == "kv/data/atlas/hermes/developer-ssh" - assert annotations[ - "vault.hashicorp.com/agent-inject-secret-node-ssh-known-hosts" - ] == "kv/data/atlas/hermes/developer-ssh" - - init = next( - item - for item in deployment["spec"]["template"]["spec"]["initContainers"] - if item["name"] == "init-config" - ) - command = init["command"][2] - assert "ln -s /runtime-access/node-ssh-config /opt/data/home/.ssh/config" in command - assert ( - "ln -s /runtime-access/node-ssh-known-hosts /opt/data/home/.ssh/known_hosts" - in command - ) - assert "ln -s home/.ssh /opt/data/.ssh" in command - assert "chmod 0700 /opt/data/home/.ssh" in command - assert ( - "ln -s /runtime-access/node-ssh-private-key " - "/opt/data/home/.ssh/id_ed25519_atlas_nodes" - ) in command - - config = yaml.safe_load((HERMES / "agent-configmap.yaml").read_text())["data"] - assert "ssh_config" not in config - assert "ssh_known_hosts" not in config - assert "ssh-ed25519" not in (HERMES / "agent-configmap.yaml").read_text() - - resources = yaml.safe_load((HERMES / "kustomization.yaml").read_text())[ - "resources" - ] - assert "node-ssh-access.yaml" in resources - access = [ - item - for item in yaml.safe_load_all((HERMES / "node-ssh-access.yaml").read_text()) - if item - ] - service_account = next(item for item in access if item["kind"] == "ServiceAccount") - assert service_account["metadata"]["name"] == "hermes-node-ssh-access" - provider = next(item for item in access if item["kind"] == "SecretProviderClass") - assert provider["metadata"]["name"] == "hermes-node-ssh-access" - assert provider["spec"]["provider"] == "vault" - parameters = provider["spec"]["parameters"] - assert parameters["roleName"] == "hermes-node-ssh" - assert 'secretPath: "kv/data/atlas/hermes/developer-ssh"' in parameters["objects"] - assert 'secretKey: "public_key"' in parameters["objects"] - daemonset = next(item for item in access if item["kind"] == "DaemonSet") - pod = daemonset["spec"]["template"]["spec"] - assert pod["serviceAccountName"] == "hermes-node-ssh-access" - assert pod["automountServiceAccountToken"] is True - host_home = next(item for item in pod["volumes"] if item["name"] == "host-home") - assert host_home["hostPath"] == {"path": "/home", "type": "Directory"} - vault_secrets = next( - item for item in pod["volumes"] if item["name"] == "vault-secrets" - ) - assert vault_secrets["csi"]["driver"] == "secrets-store.csi.k8s.io" - assert vault_secrets["csi"]["volumeAttributes"] == { - "secretProviderClass": "hermes-node-ssh-access" - } - reconciler = pod["containers"][0]["args"][0] - assert "cat /vault/secrets/node-ssh-public-key" in reconciler - assert "grep -qxF" in reconciler - assert "for user in atlas oceanus" in reconciler - assert "/host-etc/passwd" in reconciler - assert "chown \"${uid}:${gid}\"" in reconciler - host_passwd = next( - item for item in pod["volumes"] if item["name"] == "host-passwd" - ) - assert host_passwd["hostPath"] == {"path": "/etc/passwd", "type": "File"} - - -def test_owner_agent_tracks_no_ssh_identity_or_host_key_material(): - """Vault references may be tracked; SSH identities and trust data may not.""" - forbidden = ( - "BEGIN OPENSSH PRIVATE KEY", - "ssh-ed25519 AAAA", - "ssh-rsa AAAA", - "IdentityFile ", - "UserKnownHostsFile ", - "StrictHostKeyChecking ", - "ssh_config:", - "ssh_known_hosts:", - ) - text_suffixes = { - ".conf", - ".json", - ".md", - ".py", - ".sh", - ".toml", - ".yaml", - ".yml", - } - tracked = "\n".join( - path.read_text(encoding="utf-8") - for path in HERMES.rglob("*") - if path.is_file() and path.suffix in text_suffixes - ) - for marker in forbidden: - assert marker not in tracked - - -def test_switchyard_has_a_dedicated_non_owner_identity_and_read_only_catalog(): - """Routing must not inherit the owner agent's cluster-admin capability.""" - service_accounts = [ - item - for item in yaml.safe_load_all( - (HERMES / "vault-serviceaccount.yaml").read_text() - ) - if item - ] - assert any( - item["kind"] == "ServiceAccount" - and item["metadata"]["name"] == "hermes-switchyard" - for item in service_accounts - ) - - switchyard = yaml.safe_load( - (HERMES / "switchyard-deployment.yaml").read_text() - ) - switchyard_pod = switchyard["spec"]["template"]["spec"] - assert switchyard_pod["serviceAccountName"] == "hermes-switchyard" - agent_pod = _agent_deployment()["spec"]["template"]["spec"] - for pod, container_name in ( - (switchyard_pod, "worker-route-broker"), - (agent_pod, "claude-broker"), - ): - container = next(item for item in pod["containers"] if item["name"] == container_name) - catalog = next( - item - for item in container["volumeMounts"] - if item["mountPath"] == "/routing-catalog" - ) - assert catalog["readOnly"] is True - - rbac = [ - item - for item in yaml.safe_load_all((HERMES / "agent-rbac.yaml").read_text()) - if item - ] - binding = next(item for item in rbac if item["kind"] == "ClusterRoleBinding") - assert binding["subjects"] == [ - {"kind": "ServiceAccount", "name": "hermes-agent", "namespace": "hermes"} - ] - - -def test_switchyard_active_state_uses_a_relocatable_rwx_claim(): - """A stale node attachment must not strand the routing authority.""" - claims = [ - item - for item in yaml.safe_load_all((HERMES / "switchyard-pvc.yaml").read_text()) - if item - ] - active_claim = next( - item - for item in claims - if item["metadata"]["name"] == "hermes-switchyard-state-rwx" - ) - assert active_claim["spec"]["accessModes"] == ["ReadWriteMany"] - - deployment = yaml.safe_load((HERMES / "switchyard-deployment.yaml").read_text()) - strategy = deployment["spec"]["strategy"] - assert strategy == { - "type": "RollingUpdate", - "rollingUpdate": {"maxSurge": 1, "maxUnavailable": 0}, - } - pod = deployment["spec"]["template"]["spec"] - state = next(item for item in pod["volumes"] if item["name"] == "state") - assert state["persistentVolumeClaim"]["claimName"] == active_claim["metadata"][ - "name" - ] - - -def test_worker_route_broker_accepts_pod_network_health_checks(): - """Kubelet probes the pod IP, so the broker cannot bind to loopback only.""" - script = (SCRIPTS / "worker_route_broker.py").read_text() - assert 'ThreadingHTTPServer(("0.0.0.0", PORT), Handler)' in script - - -def test_switchyard_network_boundary_allows_vault_bootstrap(): - """The pre-populate init container must reach Vault before routing starts.""" - documents = [ - item - for item in yaml.safe_load_all((HERMES / "networkpolicy.yaml").read_text()) - if item - ] - isolation = next( - item - for item in documents - if item.get("metadata", {}).get("name") == "hermes-switchyard-isolation" - ) - assert any( - rule.get("to") - == [ - { - "namespaceSelector": { - "matchLabels": {"kubernetes.io/metadata.name": "vault"} - }, - "podSelector": {"matchLabels": {"app": "vault"}}, - } - ] - and rule.get("ports") == [{"protocol": "TCP", "port": 8200}] - for rule in isolation["spec"]["egress"] - ) - - -def test_switchyard_network_boundary_allows_metrics_scraping(): - """VictoriaMetrics may scrape Switchyard without widening its API boundary.""" - documents = [ - item - for item in yaml.safe_load_all((HERMES / "networkpolicy.yaml").read_text()) - if item - ] - isolation = next( - item - for item in documents - if item.get("metadata", {}).get("name") == "hermes-switchyard-isolation" - ) - assert any( - rule.get("from") - == [ - { - "namespaceSelector": { - "matchLabels": { - "kubernetes.io/metadata.name": "monitoring" - } - }, - "podSelector": {"matchLabels": {"app": "server"}}, - } - ] - and rule.get("ports") == [{"protocol": "TCP", "port": 9005}] - for rule in isolation["spec"]["ingress"] - ) - - -def test_owner_agent_installs_the_pinned_operator_toolchain(): - script = (SCRIPTS / "install_agent_tools.sh").read_text() - for value in [ - "flux", - "helm", - "kustomize", - "jq", - "yq", - "gh", - "vault", - "sops", - "age", - "age-keygen", - "k9s", - "terraform", - "go", - "gofmt", - ]: - assert value in script - assert "go1.26.5.linux-arm64.tar.gz" in script - assert ( - "fe4789e92b1f33358680864bbe8704289e7bb5fc207d80623c308935bd696d49" - in script - ) - assert script.count("sha256sum -c -") == 1 - - deployment = _agent_deployment() - installer = next( - item - for item in deployment["spec"]["template"]["spec"]["initContainers"] - if item["name"] == "install-agent-tools" - ) - assert "/bin/sh /opt/coordinator/install_agent_tools.sh" in installer["command"][2] - assert any(mount["name"] == "coordinator" for mount in installer["volumeMounts"]) - init_config = next( - item - for item in deployment["spec"]["template"]["spec"]["initContainers"] - if item["name"] == "init-config" - ) - init_command = init_config["command"][2] - assert "# Hermes managed operator PATH." in init_command - assert "/opt/coordinator:/opt/data/tools/bin:/opt/hermes/.venv/bin" in init_command - assert 'chmod 0644 "${profile_file}"' in init_command - - -def test_owner_agent_uses_only_the_canonical_hostname(): - paths = [ - HERMES / "agent-configmap.yaml", - HERMES / "agent-deployment.yaml", - HERMES / "agent-ingress.yaml", - Path(__file__).parents[2] / "scripts/ops/hermes_triage_monitor.py", - ] - for path in paths: - content = path.read_text() - assert "agent.bstein.dev" not in content - assert "agent.hermes.bstein.dev" in content - - -def test_agent_reconnect_retains_complete_history_and_long_tool_budget(): - configmap = yaml.safe_load((HERMES / "agent-configmap.yaml").read_text()) - config = yaml.safe_load(configmap["data"]["config.yaml"]) - display = config["display"] - assert display["resume_exchanges"] >= 10000 - assert display["resume_max_user_chars"] >= 10000000 - assert display["resume_max_assistant_chars"] >= 10000000 - assert config["agent"]["max_turns"] == 180 - assert config["delegation"]["max_iterations"] == 120 - - -def test_auth_patch_honors_explicit_shared_store(tmp_path: Path): - source = tmp_path / "auth.py" - destination = tmp_path / "patched/auth.py" - source.write_text( - 'from pathlib import Path\nimport os\n\ndef _auth_file_path() -> Path:\n path = get_hermes_home() / "auth.json"\n return path\n', - encoding="utf-8", - ) - auth_patch.patch(source, destination) - content = destination.read_text() - assert 'os.environ.get("HERMES_AUTH_FILE"' in content - - -def test_auth_patch_fails_closed_on_upstream_drift(tmp_path: Path): - source = tmp_path / "auth.py" - source.write_text("def changed():\n pass\n", encoding="utf-8") - with pytest.raises(RuntimeError, match="context changed"): - auth_patch.patch(source, tmp_path / "patched.py") - - -def test_codex_runtime_patch_uses_cli_and_forwards_route(tmp_path: Path): - provider = tmp_path / "runtime_provider.py" - provider.write_text(codex_runtime_patch.PROVIDER_BEFORE, encoding="utf-8") - provider_out = tmp_path / "patched/runtime_provider.py" - codex_runtime_patch.patch_provider(provider, provider_out) - assert '"api_mode": "codex_app_server"' in provider_out.read_text() - - session = tmp_path / "codex_app_server_session.py" - session.write_text( - codex_runtime_patch.SESSION_SIGNATURE_BEFORE - + codex_runtime_patch.SESSION_REQUEST_BEFORE, - encoding="utf-8", - ) - session_out = tmp_path / "patched/codex_app_server_session.py" - codex_runtime_patch.patch_session(session, session_out) - session_content = session_out.read_text() - assert 'turn_params["model"] = model' in session_content - assert 'turn_params["effort"] = effort' in session_content - assert '"approvalPolicy": "never"' in session_content - assert '"sandboxPolicy": {"type": "dangerFullAccess"}' in session_content - - turn = tmp_path / "codex_runtime.py" - turn.write_text( - codex_runtime_patch.FALLBACK_CONTEXT_BEFORE - + codex_runtime_patch.TURN_BEFORE, - encoding="utf-8", - ) - turn_out = tmp_path / "patched/codex_runtime.py" - codex_runtime_patch.patch_turn(turn, turn_out) - turn_content = turn_out.read_text() - assert "model=str(getattr(agent" in turn_content - assert "build_cross_provider_codex_prompt" in turn_content - - fallback = tmp_path / "chat_completion_helpers.py" - fallback.write_text( - codex_runtime_patch.FALLBACK_RESOLUTION_BEFORE, - encoding="utf-8", - ) - fallback_out = tmp_path / "patched/chat_completion_helpers.py" - codex_runtime_patch.patch_fallback(fallback, fallback_out) - fallback_content = fallback_out.read_text() - assert 'agent.api_mode = "codex_app_server"' in fallback_content - assert "agent._codex_cross_provider_fallback = True" in fallback_content - - loop = tmp_path / "conversation_loop.py" - loop.write_text( - codex_runtime_patch.FALLBACK_DISPATCH_BEFORE - + codex_runtime_patch.RETRY_FALLBACK_DISPATCH_BEFORE - + codex_runtime_patch.STREAM_RECOVERY_BEFORE, - encoding="utf-8", - ) - loop_out = tmp_path / "patched/conversation_loop.py" - codex_runtime_patch.patch_loop(loop, loop_out) - loop_content = loop_out.read_text() - assert loop_content.count('if agent.api_mode == "codex_app_server"') == 2 - assert "build_cross_provider_codex_prompt" in loop_content - retry_dispatch = loop_content.index( - "Fallback activation happens inside this retry loop" - ) - api_kwargs = loop_content.find("agent._build_api_kwargs", retry_dispatch) - assert api_kwargs == -1 or retry_dispatch < api_kwargs - assert "Provider stream ended before a complete response" in loop_content - assert "_is_transport_stub" in loop_content - assert "rerouting ({truncated_tool_call_retries}/4)" in loop_content - - auxiliary = tmp_path / "auxiliary_client.py" - auxiliary.write_text( - codex_runtime_patch.AUXILIARY_TOKEN_BEFORE, - encoding="utf-8", - ) - auxiliary_out = tmp_path / "patched/auxiliary_client.py" - codex_runtime_patch.patch_auxiliary(auxiliary, auxiliary_out) - auxiliary_content = auxiliary_out.read_text() - assert 'os.environ.get("CODEX_HOME"' in auxiliary_content - assert 'Path(codex_home).expanduser() / "auth.json"' in auxiliary_content - assert "never creates a metered API-key lane" in auxiliary_content - - -def test_agent_mounts_codex_auxiliary_runtime_patch(): - deployment = _agent_deployment() - pod = deployment["spec"]["template"]["spec"] - patch_init = next( - item for item in pod["initContainers"] - if item["name"] == "patch-codex-runtime" - ) - assert patch_init["command"][-2:] == [ - "/opt/hermes/agent/auxiliary_client.py", - "/patched/auxiliary_client.py", - ] - expected_mount = { - "name": "codex-runtime-patch", - "mountPath": "/opt/hermes/agent/auxiliary_client.py", - "subPath": "auxiliary_client.py", - } - containers = {item["name"]: item for item in pod["containers"]} - for name in ("hermes", "terminal"): - assert expected_mount in containers[name]["volumeMounts"] - - -def test_codex_auxiliary_patch_reads_cli_token_without_copying_it( - tmp_path: Path, - monkeypatch, -): - codex_home = tmp_path / ".codex" - codex_home.mkdir() - (codex_home / "auth.json").write_text( - json.dumps({"tokens": {"access_token": "cli-access-token"}}), - encoding="utf-8", - ) - monkeypatch.setenv("CODEX_HOME", str(codex_home)) - - source = tmp_path / "auxiliary_client.py" - source.write_text( - "import json, logging, os, time\n" - "from pathlib import Path\n" - "logger = logging.getLogger(__name__)\n" - "def read_token():\n" - " try:\n" - " raise RuntimeError('Hermes provider store intentionally empty')\n" - + codex_runtime_patch.AUXILIARY_TOKEN_BEFORE, - encoding="utf-8", - ) - destination = tmp_path / "patched/auxiliary_client.py" - codex_runtime_patch.patch_auxiliary(source, destination) - namespace: dict = {} - exec(compile(destination.read_text(), str(destination), "exec"), namespace) - - assert namespace["read_token"]() == "cli-access-token" - - -def test_codex_runtime_patch_fails_closed_on_upstream_drift(tmp_path: Path): - source = tmp_path / "runtime_provider.py" - source.write_text("def changed():\n pass\n", encoding="utf-8") - with pytest.raises(RuntimeError, match="context changed"): - codex_runtime_patch.patch_provider(source, tmp_path / "patched.py") - - -def test_codex_runtime_migration_uses_owner_unsafe_mode(tmp_path: Path): - config = tmp_path / "config.yaml" - config.write_text("model: {}\n", encoding="utf-8") - codex_home = tmp_path / ".codex" - calls = [] - - class Report: - errors = [] - - @staticmethod - def summary(): - return "configured" - - def migrate(value, **kwargs): - calls.append((value, kwargs)) - return Report() - - client_config.configure_codex_runtime(config, migrate, codex_home) - - assert calls[0][1]["default_permission_profile"] is None - assert calls[0][1]["codex_home"] == codex_home - content = (codex_home / "config.toml").read_text(encoding="utf-8") - assert 'approval_policy = "never"' in content - assert 'sandbox_mode = "danger-full-access"' in content - assert "default_permissions" not in content - - -def test_codex_owner_permissions_replace_stale_profile(tmp_path: Path): - config = tmp_path / "config.toml" - config.write_text( - 'default_permissions = ":danger-no-sandbox"\n\n[features]\nhooks = true\n', - encoding="utf-8", - ) - - client_config.configure_codex_owner_permissions(config) - client_config.configure_codex_owner_permissions(config) - - content = config.read_text(encoding="utf-8") - assert content.count(client_config.OWNER_PERMISSIONS_BEGIN) == 1 - assert content.count('approval_policy = "never"') == 1 - assert content.count('sandbox_mode = "danger-full-access"') == 1 - assert "default_permissions" not in content - assert "[features]\nhooks = true" in content - - -def test_tui_gateway_patch_extends_and_bounds_agent_startup(tmp_path: Path): - source = tmp_path / "server.py" - destination = tmp_path / "patched/server.py" - source.write_text( - "import os\n\n" + tui_gateway_patch.BEFORE + "\ndef unchanged():\n pass\n", - encoding="utf-8", - ) - - tui_gateway_patch.patch(source, destination) - - content = destination.read_text(encoding="utf-8") - assert "HERMES_TUI_AGENT_INIT_TIMEOUT_S" in content - assert 'configured = 180.0' in content - assert "return max(30.0, min(configured, 900.0))" in content - assert "timeout: float | None = None" in content - assert "ready.wait(timeout=wait_timeout)" in content - assert "def unchanged():" in content - - -def test_tui_gateway_patch_fails_closed_on_upstream_drift(tmp_path: Path): - source = tmp_path / "server.py" - source.write_text("def changed():\n pass\n", encoding="utf-8") - with pytest.raises(RuntimeError, match="context changed"): - tui_gateway_patch.patch(source, tmp_path / "patched.py") - - -def test_ttyd_clipboard_and_reconnect_patch_remain_enabled(): - source = '' - content = ttyd_patch.patch_html(source) - assert 'id="atlas-ttyd-clipboard"' in content - assert "navigator.clipboard.writeText(text)" in content - assert "class AtlasRecoveringWebSocket" in content - assert "window.location.reload()" in content - assert "event.stopImmediatePropagation()" in content - - -def test_ttyd_patch_fails_closed_on_upstream_drift(): - with pytest.raises(RuntimeError, match="context changed"): - ttyd_patch.patch_html("changed") diff --git a/testing/tests/test_hermes_cli_lanes_access.py b/testing/tests/test_hermes_cli_lanes_access.py new file mode 100644 index 00000000..01d9f4bf --- /dev/null +++ b/testing/tests/test_hermes_cli_lanes_access.py @@ -0,0 +1,378 @@ +"""Network, OAuth, Kubernetes, and SSH access contracts for Hermes CLI lanes.""" + +from __future__ import annotations + + +import yaml + +from testing.tests.test_hermes_cli_lanes_support import ( + FLUX_HERMES, + HERMES, + KEYCLOAK, + ROOT, + _agent_deployment, + _services, +) + + +def test_broker_services_survive_sibling_container_readiness_loss(): + services = _services() + + for name in ( + "hermes-image-broker", + "hermes-codex-broker", + "hermes-local-image", + "hermes-claude-broker", + ): + assert services[name]["spec"]["publishNotReadyAddresses"] is True + + +def test_agent_dashboard_reconnects_all_transient_websockets(): + dockerfile = (HERMES.parents[1] / "dockerfiles/Dockerfile.hermes-agent").read_text( + encoding="utf-8" + ) + assert "eventsRetryAttempt.current" in dockerfile + assert "if (!unmounting) setVersion((v) => v + 1);" in dockerfile + assert "events feed rejected (${ev.code}) — reload the page" in dockerfile + assert 'url = await api.buildWsUrl("/api/pty", params);' in dockerfile + assert 'url = await buildWsUrl("/api/events", { channel });' in dockerfile + assert dockerfile.count("' await api.getSessions(1, 0,") == 2 + assert ".then(() => gw.connect())" in dockerfile + assert 'api.getSessions(1, 0, profile ?? "")' in dockerfile + assert "dashboard token rotated by a server restart" in dockerfile + + +def test_agent_refreshes_routes_after_restoring_cli_logins(): + deployment = _agent_deployment() + init_containers = { + item["name"]: item + for item in deployment["spec"]["template"]["spec"]["initContainers"] + } + configure = init_containers["configure-agent-clients"] + command = configure["command"][-1] + assert "configure_agent_clients.py" in command + assert command.index("configure_agent_clients.py") < command.index( + "hermes_coordinator.py --once" + ) + env = {item["name"]: item["value"] for item in configure["env"]} + assert env["HERMES_AUTH_FILE"] == "/runtime-access/hermes-auth.json" + assert env["PYTHONPATH"] == "/opt/hermes" + for name in ("bootstrap-coordinator", "configure-agent-clients"): + route_env = { + item["name"]: item["value"] for item in init_containers[name]["env"] + } + assert route_env["CODEX_HOME"] == "/runtime-access/codex" + assert route_env["CLAUDE_CONFIG_DIR"] == "/runtime-access/claude" + assert "/opt/data/tools/bin" in route_env["PATH"] + + assert "patch-web-session-activity" in init_containers + web_patch = init_containers["patch-web-session-activity"] + assert "/opt/coordinator/patch_web_session_activity.py" in web_patch["command"] + + containers = { + item["name"]: item + for item in deployment["spec"]["template"]["spec"]["containers"] + } + steward_env = { + item["name"]: item["value"] for item in containers["model-steward"]["env"] + } + assert steward_env["CODEX_HOME"] == "/runtime-access/codex" + assert steward_env["CLAUDE_CONFIG_DIR"] == "/runtime-access/claude" + assert "/opt/data/tools/bin" in steward_env["PATH"] + hermes_mounts = { + (item["name"], item["mountPath"], item.get("subPath")) + for item in containers["hermes"]["volumeMounts"] + } + assert ( + "web-server-patch", + "/opt/hermes/hermes_cli/web_server.py", + "web_server.py", + ) in hermes_mounts + + +def test_flux_health_checks_follow_the_owner_oauth_sidecar(): + flux = yaml.safe_load(FLUX_HERMES.read_text()) + checks = {(item["kind"], item["name"]) for item in flux["spec"]["healthChecks"]} + assert ("Deployment", "hermes-agent") in checks + assert ("DaemonSet", "hermes-node-ssh-access") in checks + assert ("Deployment", "oauth2-proxy-hermes-agent") not in checks + + +def test_agent_auth_is_bstein_group_and_email_bounded(): + deployment = _agent_deployment() + oauth = next( + item + for item in deployment["spec"]["template"]["spec"]["containers"] + if item["name"] == "oauth2-proxy" + ) + args = oauth["args"] + assert "--user-id-claim=sub" in args + assert "--oidc-groups-claim=groups" in args + assert "--allowed-group=/hermes-owner" in args + assert "--authenticated-emails-file=/etc/oauth2-proxy/allowed-emails" in args + script = (KEYCLOAK / "scripts/hermes_access_oidc_ensure.sh").read_text() + assert 'group_name="hermes-owner"' in script + assert "username=bstein&exact=true" in script + assert '"full.path":"true"' in script + + +def test_agent_network_boundary_allows_only_authenticated_and_metrics_surfaces(): + documents = [ + item + for item in yaml.safe_load_all((HERMES / "networkpolicy.yaml").read_text()) + if item + ] + isolation = next( + item + for item in documents + if item.get("metadata", {}).get("name") == "hermes-agent-isolation" + ) + assert isolation["spec"]["ingress"] == [ + { + "from": [ + { + "namespaceSelector": { + "matchLabels": {"kubernetes.io/metadata.name": "traefik"} + }, + "podSelector": { + "matchLabels": {"app.kubernetes.io/name": "traefik"} + }, + } + ], + "ports": [{"protocol": "TCP", "port": 4180}], + }, + { + "from": [{"podSelector": {"matchLabels": {"app": "hermes-chat-tenant"}}}], + "ports": [ + {"protocol": "TCP", "port": 9002}, + {"protocol": "TCP", "port": 9003}, + ], + }, + { + "from": [{"podSelector": {"matchLabels": {"app": "hermes-switchyard"}}}], + "ports": [ + {"protocol": "TCP", "port": 9003}, + {"protocol": "TCP", "port": 9006}, + ], + }, + { + "from": [ + { + "namespaceSelector": { + "matchLabels": {"kubernetes.io/metadata.name": "monitoring"} + }, + "podSelector": {"matchLabels": {"app": "server"}}, + } + ], + "ports": [{"protocol": "TCP", "port": 9010}], + }, + ] + egress = isolation["spec"]["egress"] + assert any( + rule.get("ports") == [{"protocol": "TCP", "port": 9081}] for rule in egress + ) + assert "hermes-scm" in yaml.safe_dump(egress) + assert "gitea" in yaml.safe_dump(egress) + assert egress != [{}] + + +def test_owner_agent_has_scoped_read_only_kubernetes_context(): + config = yaml.safe_load((HERMES / "agent-kubeconfig.yaml").read_text()) + assert config["current-context"] == "atlas-observer" + assert config["contexts"][0]["context"]["namespace"] == "default" + rbac_path = ROOT / "services/hermes-observer-rbac/rbac.yaml" + documents = [item for item in yaml.safe_load_all(rbac_path.read_text()) if item] + binding = next(item for item in documents if item["kind"] == "ClusterRoleBinding") + assert binding["roleRef"] == { + "apiGroup": "rbac.authorization.k8s.io", + "kind": "ClusterRole", + "name": "hermes-agent-cluster-observer-v2", + } + assert binding["subjects"] == [ + {"kind": "ServiceAccount", "name": "hermes-agent", "namespace": "hermes"} + ] + + +def test_owner_agent_has_pinned_dedicated_node_ssh_access(): + deployment = _agent_deployment() + annotations = deployment["spec"]["template"]["metadata"]["annotations"] + assert ( + annotations["vault.hashicorp.com/agent-inject-secret-node-ssh-private-key"] + == "kv/data/atlas/hermes/developer-ssh" + ) + assert ( + annotations["vault.hashicorp.com/agent-inject-secret-node-ssh-config"] + == "kv/data/atlas/hermes/developer-ssh" + ) + assert ( + annotations["vault.hashicorp.com/agent-inject-secret-node-ssh-known-hosts"] + == "kv/data/atlas/hermes/developer-ssh" + ) + + init = next( + item + for item in deployment["spec"]["template"]["spec"]["initContainers"] + if item["name"] == "init-config" + ) + command = init["command"][2] + assert "ln -s /runtime-access/node-ssh-config /opt/data/home/.ssh/config" in command + assert ( + "ln -s /runtime-access/node-ssh-known-hosts /opt/data/home/.ssh/known_hosts" + in command + ) + assert "ln -s home/.ssh /opt/data/.ssh" in command + assert "chmod 0700 /opt/data/home/.ssh" in command + assert ( + "ln -s /runtime-access/node-ssh-private-key " + "/opt/data/home/.ssh/id_ed25519_atlas_nodes" + ) in command + + config = yaml.safe_load((HERMES / "agent-configmap.yaml").read_text())["data"] + assert "ssh_config" not in config + assert "ssh_known_hosts" not in config + assert "ssh-ed25519" not in (HERMES / "agent-configmap.yaml").read_text() + + resources = yaml.safe_load((HERMES / "kustomization.yaml").read_text())["resources"] + assert "node-ssh-access.yaml" in resources + access = [ + item + for item in yaml.safe_load_all((HERMES / "node-ssh-access.yaml").read_text()) + if item + ] + service_account = next(item for item in access if item["kind"] == "ServiceAccount") + assert service_account["metadata"]["name"] == "hermes-node-ssh-access" + provider = next(item for item in access if item["kind"] == "SecretProviderClass") + assert provider["metadata"]["name"] == "hermes-node-ssh-access" + assert provider["spec"]["provider"] == "vault" + parameters = provider["spec"]["parameters"] + assert parameters["roleName"] == "hermes-node-ssh" + assert 'secretPath: "kv/data/atlas/hermes/developer-ssh"' in parameters["objects"] + assert 'secretKey: "public_key"' in parameters["objects"] + daemonset = next(item for item in access if item["kind"] == "DaemonSet") + pod = daemonset["spec"]["template"]["spec"] + assert pod["serviceAccountName"] == "hermes-node-ssh-access" + assert pod["automountServiceAccountToken"] is True + host_home = next(item for item in pod["volumes"] if item["name"] == "host-home") + assert host_home["hostPath"] == {"path": "/home", "type": "Directory"} + vault_secrets = next( + item for item in pod["volumes"] if item["name"] == "vault-secrets" + ) + assert vault_secrets["csi"]["driver"] == "secrets-store.csi.k8s.io" + assert vault_secrets["csi"]["volumeAttributes"] == { + "secretProviderClass": "hermes-node-ssh-access" + } + reconciler = pod["containers"][0]["args"][0] + assert "/opt/node-hardener/node_account_hardening.py" in reconciler + assert "--public-key-file /vault/secrets/node-ssh-public-key" in reconciler + assert "sleep 300" in reconciler + host_etc = next(item for item in pod["volumes"] if item["name"] == "host-etc") + assert host_etc["hostPath"] == {"path": "/etc", "type": "Directory"} + hardener = (HERMES / "scripts/node_account_hardening.py").read_text() + assert 'ACCOUNT = "hermes-agent"' in hardener + assert 'LEGACY_ACCOUNTS = ("atlas", "oceanus")' in hardener + assert "ACCOUNT_UID = 1200" in hardener + assert "ACCOUNT_GID = 1200" in hardener + + +def test_owner_agent_tracks_no_ssh_identity_or_host_key_material(): + """Vault references may be tracked; SSH identities and trust data may not.""" + forbidden = ( + "BEGIN OPENSSH PRIVATE KEY", + "ssh-ed25519 AAAA", + "ssh-rsa AAAA", + "IdentityFile ", + "UserKnownHostsFile ", + "StrictHostKeyChecking ", + "ssh_config:", + "ssh_known_hosts:", + ) + text_suffixes = { + ".conf", + ".json", + ".md", + ".py", + ".sh", + ".toml", + ".yaml", + ".yml", + } + tracked = "\n".join( + path.read_text(encoding="utf-8") + for path in HERMES.rglob("*") + if path.is_file() and path.suffix in text_suffixes + ) + for marker in forbidden: + assert marker not in tracked + + +def test_switchyard_has_a_dedicated_non_owner_identity_and_read_only_catalog(): + """Routing must not inherit the owner agent's cluster-admin capability.""" + service_accounts = [ + item + for item in yaml.safe_load_all( + (HERMES / "vault-serviceaccount.yaml").read_text() + ) + if item + ] + assert any( + item["kind"] == "ServiceAccount" + and item["metadata"]["name"] == "hermes-switchyard" + for item in service_accounts + ) + + switchyard = yaml.safe_load((HERMES / "switchyard-deployment.yaml").read_text()) + switchyard_pod = switchyard["spec"]["template"]["spec"] + assert switchyard_pod["serviceAccountName"] == "hermes-switchyard" + agent_pod = _agent_deployment()["spec"]["template"]["spec"] + for pod, container_name in ( + (switchyard_pod, "worker-route-broker"), + (agent_pod, "claude-broker"), + ): + container = next( + item for item in pod["containers"] if item["name"] == container_name + ) + catalog = next( + item + for item in container["volumeMounts"] + if item["mountPath"] == "/routing-catalog" + ) + assert catalog["readOnly"] is True + + rbac = [ + item + for item in yaml.safe_load_all( + (ROOT / "services/hermes-observer-rbac/rbac.yaml").read_text() + ) + if item + ] + binding = next(item for item in rbac if item["kind"] == "ClusterRoleBinding") + assert binding["subjects"] == [ + {"kind": "ServiceAccount", "name": "hermes-agent", "namespace": "hermes"} + ] + + +def test_switchyard_active_state_uses_a_relocatable_rwx_claim(): + """A stale node attachment must not strand the routing authority.""" + claims = [ + item + for item in yaml.safe_load_all((HERMES / "switchyard-pvc.yaml").read_text()) + if item + ] + active_claim = next( + item + for item in claims + if item["metadata"]["name"] == "hermes-switchyard-state-rwx" + ) + assert active_claim["spec"]["accessModes"] == ["ReadWriteMany"] + + deployment = yaml.safe_load((HERMES / "switchyard-deployment.yaml").read_text()) + strategy = deployment["spec"]["strategy"] + assert strategy == { + "type": "RollingUpdate", + "rollingUpdate": {"maxSurge": 1, "maxUnavailable": 0}, + } + pod = deployment["spec"]["template"]["spec"] + state = next(item for item in pod["volumes"] if item["name"] == "state") + assert ( + state["persistentVolumeClaim"]["claimName"] == active_claim["metadata"]["name"] + ) diff --git a/testing/tests/test_hermes_cli_lanes_configuration.py b/testing/tests/test_hermes_cli_lanes_configuration.py new file mode 100644 index 00000000..bf1807f4 --- /dev/null +++ b/testing/tests/test_hermes_cli_lanes_configuration.py @@ -0,0 +1,444 @@ +"""Agent configuration and dashboard contracts for Hermes CLI lanes.""" + +from __future__ import annotations + +import json +import sys +from contextlib import nullcontext +from pathlib import Path +from types import SimpleNamespace + +import pytest +import yaml + +from testing.tests.test_hermes_cli_lanes_support import ( + HERMES, + _agent_deployment, + client_config, + lanes, + migration, + policy, +) + + +def test_workspace_preparation_failure_durably_blocks_the_claim( + tmp_path: Path, monkeypatch +): + task = SimpleNamespace(id="t_bad_worktree", current_run_id=7, assignee="cli-auto") + calls = [] + + class Connection: + def close(self): + return None + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + get_task=lambda _conn, _task_id: task, + worker_log_path=lambda _task_id, board: tmp_path / "worker.log", + _resolve_worktree_workspace=lambda _task, board: (_ for _ in ()).throw( + ValueError("not a Git repository") + ), + block_task=lambda *_args, **kwargs: calls.append(kwargs), + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + monkeypatch.setattr( + lanes, "state_path", lambda _board, _task_id: tmp_path / "state.json" + ) + + lanes.execute_claim("cassandra", "t_bad_worktree") + + assert calls[0]["kind"] == "capability" + assert calls[0]["expected_run_id"] == 7 + assert "not a Git repository" in calls[0]["reason"] + + +def test_artifacts_cannot_escape_the_task_worktree(tmp_path: Path): + workspace = tmp_path / "workspace" + workspace.mkdir() + inside = workspace / "report.json" + outside = tmp_path / "auth.json" + inside.write_text("{}\n", encoding="utf-8") + outside.write_text("secret\n", encoding="utf-8") + + assert lanes.workspace_artifacts( + workspace, + ["report.json", str(outside), "missing.json"], + ) == [str(inside)] + + +def test_restart_provider_change_includes_explicit_workspace_handoff( + tmp_path: Path, monkeypatch +): + task = SimpleNamespace( + id="t_resume", + current_run_id=9, + assignee="cli-auto", + max_runtime_seconds=60, + ) + state_file = tmp_path / "state.json" + state_file.write_text( + json.dumps({"current_route": {"provider": "claude"}}), + encoding="utf-8", + ) + (tmp_path / "worker.log").write_text("prior provider evidence", encoding="utf-8") + prompts = [] + + class Connection: + def close(self): + return None + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + get_task=lambda _conn, _task_id: task, + worker_log_path=lambda _task_id, board: tmp_path / "worker.log", + _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_resume"), + set_branch_name=lambda *_args: None, + set_workspace_path=lambda *_args: None, + build_worker_context=lambda *_args: "resume objective", + add_comment=lambda *_args: None, + complete_task=lambda *_args, **_kwargs: None, + block_task=lambda *_args, **_kwargs: None, + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + monkeypatch.setattr(lanes, "state_path", lambda _board, _task_id: state_file) + monkeypatch.setattr( + lanes, + "select_route", + lambda *_args, **_kwargs: lanes.Route( + "codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, () + ), + ) + monkeypatch.setattr( + lanes, + "git_handoff", + lambda _workspace, output: f"HANDOFF:{output}", + ) + monkeypatch.setattr( + lanes, + "run_provider", + lambda _route, prompt, *_args, **_kwargs: ( + prompts.append(prompt) + or lanes.ProcessResult( + 0, + "", + { + "status": "completed", + "summary": "done", + "changed_files": [], + "tests_run": [], + "artifacts": [], + "blockers": [], + }, + False, + ) + ), + ) + + lanes.execute_claim("cassandra", "t_resume") + + assert "HANDOFF:prior provider evidence" in prompts[0] + + +def test_provider_commands_are_structured_unattended_and_capped(tmp_path: Path): + route = lanes.Route( + "codex", "gpt-5.6-sol", "xhigh", "codex-xhigh", "jetson", "vote", 1, () + ) + command = lanes._codex_command( + route, "Work.", tmp_path, {}, tmp_path / "result.json" + ) + assert "--dangerously-bypass-approvals-and-sandbox" in command + assert "--json" in command + assert "--output-schema" in command + assert 'model_reasoning_effort="xhigh"' in command + + claude_state = {"claude_session_id": "13864642-2985-4f91-bef5-53f145f878e8"} + claude = lanes._claude_command( + lanes.Route( + "claude", "claude-opus-5", "xhigh", "claude-xhigh", "jetson", "vote", 1, () + ), + "Review.", + claude_state, + False, + ) + assert "--dangerously-skip-permissions" in claude + assert "--output-format" in claude and "stream-json" in claude + assert "--json-schema" in claude + assert "--disallowedTools" in claude + assert "Bash(kubectl apply *)" not in claude + assert "Bash(flux reconcile *)" not in claude + assert "max" not in claude + + +def test_worker_contract_separates_review_findings_from_task_blockers(tmp_path: Path): + prompt = lanes.build_prompt("Review the change.", tmp_path) + + assert "put defects and risks in findings" in prompt + assert "blockers array must be empty whenever status is completed" in prompt + assert "findings" in lanes.RESULT_SCHEMA["properties"] + assert set(lanes.RESULT_SCHEMA["required"]) == set( + lanes.RESULT_SCHEMA["properties"] + ) + assert ( + "assigned task itself" + in lanes.RESULT_SCHEMA["properties"]["blockers"]["description"] + ) + + +@pytest.mark.parametrize( + "command", + [ + "git push --force origin main", + "git reset --hard HEAD~1", + "git clean -fd", + ], +) +def test_claude_pretool_hook_blocks_hard_denies(command: str): + assert policy.denial_reason(command) + + +def test_claude_pretool_hook_allows_normal_engineering(): + assert policy.denial_reason("pytest -q testing/tests") is None + assert policy.denial_reason("git push origin feature/hermes") is None + assert policy.denial_reason("kubectl delete pod -n cassandra stuck-worker") is None + assert policy.denial_reason("flux reconcile kustomization hermes") is None + assert policy.denial_reason("vault kv get kv/atlas/hermes") is None + + +def test_claude_settings_preserve_state_and_install_three_guardrail_layers( + tmp_path: Path, +): + state = tmp_path / ".claude.json" + settings = tmp_path / "settings.json" + state.write_text('{"promptQueueUseCount": 4}\n', encoding="utf-8") + settings.write_text( + json.dumps( + { + "theme": "dark", + "permissions": { + "deny": [ + "Bash(kubectl apply *)", + "Bash(flux reconcile *)", + "Bash(vault kv *)", + "Bash(custom-owner-rule *)", + ] + }, + } + ) + + "\n", + encoding="utf-8", + ) + + client_config.configure_claude_state(state) + client_config.configure_claude_settings(settings) + + state_value = json.loads(state.read_text()) + settings_value = json.loads(settings.read_text()) + assert state_value["promptQueueUseCount"] == 4 + assert state_value["bypassPermissionsModeAccepted"] is True + assert settings_value["theme"] == "dark" + assert "Bash(git reset --hard *)" in settings_value["permissions"]["deny"] + assert "Bash(custom-owner-rule *)" in settings_value["permissions"]["deny"] + assert "Bash(kubectl apply *)" not in settings_value["permissions"]["deny"] + assert "Bash(flux reconcile *)" not in settings_value["permissions"]["deny"] + assert "Bash(vault kv *)" not in settings_value["permissions"]["deny"] + hook = settings_value["hooks"]["PreToolUse"][0]["hooks"][0] + assert "claude_command_policy.py" in hook["command"] + + +def test_legacy_state_is_archived_without_removing_provider_transcripts(tmp_path: Path): + session = tmp_path / "home/.config/herdr/session.json" + session.parent.mkdir(parents=True) + session.write_text( + '{"agents":[{"agent":"claude","session_id":"abc"}]}', encoding="utf-8" + ) + binary = tmp_path / "tools/bin/herdr" + binary.parent.mkdir(parents=True) + binary.write_text("legacy", encoding="utf-8") + (tmp_path / "home/.claude").mkdir() + (tmp_path / "home/.codex").mkdir() + + archive = migration.archive_legacy_state(tmp_path) + + value = json.loads(archive.read_text()) + assert value["legacy_session"]["agents"][0]["session_id"] == "abc" + assert (tmp_path / "home/.claude").is_dir() + assert (tmp_path / "home/.codex").is_dir() + assert not binary.exists() + assert not (tmp_path / "home/.config/herdr").exists() + + +def test_agent_uses_one_native_kanban_control_plane(): + configmap = yaml.safe_load((HERMES / "agent-configmap.yaml").read_text()) + config = yaml.safe_load(configmap["data"]["config.yaml"]) + assert config["model"] == { + "provider": "atlas-switchyard", + "default": "atlas/auto/balanced", + "model": "atlas/auto/balanced", + } + assert config["kanban"]["dispatch_in_gateway"] is True + assert config["kanban"]["default_assignee"] == "cli-auto" + assert config["plugins"]["enabled"] == ["auto-router"] + + deployment = _agent_deployment() + pod = deployment["spec"]["template"]["spec"] + assert pod["enableServiceLinks"] is False + names = {item["name"] for item in pod["containers"]} + assert "cli-lane-runner" in names + assert "terminal" in names + assert not any("herdr" in name for name in names) + rendered = (HERMES / "agent-deployment.yaml").read_text() + assert "herdr server" not in rendered + assert "herdr-dispatch" not in rendered + + +def test_cli_lane_reserves_cpu_headroom_for_ui_and_auth(): + deployment = _agent_deployment() + containers = { + item["name"]: item + for item in deployment["spec"]["template"]["spec"]["containers"] + } + lane = containers["cli-lane-runner"] + environment = {item["name"]: item["value"] for item in lane["env"]} + + assert environment["HERMES_CLI_LANE_CONCURRENCY"] == "2" + assert lane["resources"] == { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "2", "memory": "6Gi"}, + } + + +def test_agent_avoids_unhealthy_nodes_and_fits_its_remaining_capacity(): + """Placement correction: keep the agent off nodes that cannot hold it. + + titan-04 is cordoned after repeated kernel undervoltage and kubelet + failure, and titan-19 was probe/Longhorn unstable under worker load, so + both must join the existing hard exclusions. That leaves titan-05 as the + healthy candidate, which is tight enough on requested CPU that the main + container has to give back 50m to schedule there. + """ + pod = _agent_deployment()["spec"]["template"]["spec"] + hostnames = next( + item + for item in pod["affinity"]["nodeAffinity"][ + "requiredDuringSchedulingIgnoredDuringExecution" + ]["nodeSelectorTerms"][0]["matchExpressions"] + if item["key"] == "kubernetes.io/hostname" + ) + + assert hostnames["operator"] == "NotIn" + assert set(hostnames["values"]) >= {"titan-04", "titan-19"} + + hermes = next(item for item in pod["containers"] if item["name"] == "hermes") + assert hermes["resources"]["requests"]["cpu"] == "300m" + + +def test_agent_root_is_stock_dashboard_and_terminal_is_a_separate_path(): + deployment = _agent_deployment() + pod = deployment["spec"]["template"]["spec"] + containers = {item["name"]: item for item in pod["containers"]} + assert "webui" not in containers + + assert "dashboard" not in containers + hermes = containers["hermes"] + hermes_env = {item["name"]: item["value"] for item in hermes["env"]} + assert hermes_env["HERMES_STREAM_STALE_TIMEOUT"] == "600" + assert hermes_env["HERMES_API_CALL_STALE_TIMEOUT"] == "600" + assert hermes["command"] == ["/bin/sh", "-ec"] + startup = hermes["args"][0] + assert ". /opt/data/.env" in startup + assert "exec /init /opt/hermes/docker/main-wrapper.sh gateway run" in startup + hermes_env = {item["name"]: item["value"] for item in hermes["env"]} + assert hermes_env["HERMES_DASHBOARD"] == "1" + assert hermes_env["HERMES_DASHBOARD_HOST"] == "127.0.0.1" + assert hermes_env["HERMES_DASHBOARD_PORT"] == "9119" + assert hermes_env["HERMES_TUI_AGENT_INIT_TIMEOUT_S"] == "180" + assert hermes["securityContext"]["runAsUser"] == 0 + assert hermes["securityContext"]["runAsGroup"] == 0 + for probe_name in ("startupProbe", "readinessProbe", "livenessProbe"): + probe = hermes[probe_name] + assert probe["exec"]["command"] == [ + "curl", + "-fsS", + "http://127.0.0.1:9119/api/status", + ] + + terminal = containers["terminal"] + command = terminal["args"][0] + assert "--base-path /terminal" in command + assert "--check-origin" not in command + assert "/usr/bin/tmux new-session -A" in command + assert "--continue" in command + assert "--yolo" in command + terminal_env = {item["name"]: item["value"] for item in terminal["env"]} + assert terminal_env["HERMES_TUI_AGENT_INIT_TIMEOUT_S"] == "180" + + claude_broker = containers["claude-broker"] + claude_env = {item["name"]: item["value"] for item in claude_broker["env"]} + assert claude_env["HERMES_CLAUDE_BROKER_CONCURRENCY"] == "2" + assert claude_broker["readinessProbe"]["tcpSocket"] == {"port": "claude-broker"} + assert claude_broker["livenessProbe"]["tcpSocket"] == {"port": "claude-broker"} + + args = containers["oauth2-proxy"]["args"] + terminal_upstream = "--upstream=http://127.0.0.1:7681/terminal/" + dashboard_upstream = "--upstream=http://127.0.0.1:9119/" + assert terminal_upstream in args + assert dashboard_upstream in args + assert args.index(terminal_upstream) < args.index(dashboard_upstream) + assert "--pass-host-header=false" in args + assert "--cookie-refresh=19m" in args + assert "--session-store-type=redis" in args + assert any( + arg.startswith("--redis-connection-url=redis://hermes-oauth-sessions.") + for arg in args + ) + + patch_init = next( + item for item in pod["initContainers"] if item["name"] == "patch-tui-gateway" + ) + assert patch_init["command"][-1] == "/patched/server.py" + for name in ("hermes", "terminal"): + mounts = containers[name]["volumeMounts"] + assert { + "name": "tui-gateway-patch", + "mountPath": "/opt/hermes/tui_gateway/server.py", + "subPath": "server.py", + } in mounts + + ingress_documents = [ + item + for item in yaml.safe_load_all((HERMES / "agent-ingress.yaml").read_text()) + if item + ] + middlewares = { + item["metadata"]["name"]: item + for item in ingress_documents + if item["kind"] == "Middleware" + } + assert middlewares["hermes-agent-terminal-slash"]["spec"]["redirectRegex"][ + "replacement" + ].endswith("/terminal/") + assert ( + middlewares["hermes-agent-stock-dashboard-headers"]["spec"]["headers"][ + "customRequestHeaders" + ]["Origin"] + == "http://127.0.0.1:9119" + ) + ingresses = { + item["metadata"]["name"]: item + for item in ingress_documents + if item["kind"] == "Ingress" + } + assert ( + ingresses["hermes-agent-dashboard"]["metadata"]["annotations"][ + "traefik.ingress.kubernetes.io/router.middlewares" + ] + == "hermes-hermes-agent-stock-dashboard-headers@kubernetescrd" + ) + assert ( + ingresses["hermes-agent-terminal"]["metadata"]["annotations"][ + "traefik.ingress.kubernetes.io/router.middlewares" + ] + == "hermes-hermes-agent-terminal-slash@kubernetescrd" + ) diff --git a/testing/tests/test_hermes_cli_lanes_kanban.py b/testing/tests/test_hermes_cli_lanes_kanban.py new file mode 100644 index 00000000..279b74c5 --- /dev/null +++ b/testing/tests/test_hermes_cli_lanes_kanban.py @@ -0,0 +1,392 @@ +"""Kanban claim and lifecycle contracts for Hermes CLI lanes.""" + +from __future__ import annotations + +import sys +from contextlib import nullcontext +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from testing.tests.test_hermes_cli_lanes_support import ( + lanes, +) + + +def test_unassigned_ready_task_is_persistently_routed_to_auto_lane(monkeypatch): + task = SimpleNamespace(id="t_auto", assignee=None, status="ready") + assigned = [] + + class Connection: + def close(self): + return None + + def assign_task(_conn, task_id, profile): + assigned.append((task_id, profile)) + task.assignee = profile + return True + + fake_db = SimpleNamespace( + list_boards=lambda include_archived=False: [{"slug": "cassandra"}], + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + recompute_ready=lambda _conn: None, + list_tasks=lambda _conn: [task], + assign_task=assign_task, + get_task=lambda _conn, _task_id: task, + claim_task=lambda _conn, _task_id, **_kwargs: task, + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + + assert lanes.claim_ready(set(), 1) == [("cassandra", "t_auto")] + assert assigned == [("t_auto", "cli-auto")] + + +def test_corrupt_board_is_quarantined_without_stopping_healthy_lanes( + monkeypatch, capsys +): + class CorruptBoardError(Exception): + pass + + task = SimpleNamespace(id="t_healthy", assignee="cli-auto", status="ready") + + class Connection: + def close(self): + return None + + def connect(*, board): + if board == "cassandra": + raise CorruptBoardError("integrity_check failed") + return Connection() + + fake_db = SimpleNamespace( + KanbanDbCorruptError=CorruptBoardError, + list_boards=lambda include_archived=False: [ + {"slug": "cassandra"}, + {"slug": "healthy"}, + ], + scoped_current_board=lambda _board: nullcontext(), + connect=connect, + recompute_ready=lambda _conn: None, + list_tasks=lambda _conn: [task], + claim_task=lambda _conn, _task_id, **_kwargs: task, + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + lanes.BOARD_CORRUPTION_ERRORS.clear() + + assert lanes.claim_ready(set(), 1) == [("healthy", "t_healthy")] + assert "temporarily skipping Kanban board 'cassandra'" in capsys.readouterr().err + + +def test_transient_board_scan_failure_does_not_stop_healthy_lanes(monkeypatch, capsys): + task = SimpleNamespace(id="t_healthy", assignee="cli-auto", status="ready") + + class Connection: + def __init__(self, board): + self.board = board + + def close(self): + return None + + def recompute_ready(connection): + if connection.board == "cassandra": + raise lanes.sqlite3.OperationalError("disk I/O error") + + fake_db = SimpleNamespace( + list_boards=lambda include_archived=False: [ + {"slug": "cassandra"}, + {"slug": "healthy"}, + ], + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(board), + recompute_ready=recompute_ready, + list_tasks=lambda _conn: [task], + claim_task=lambda _conn, _task_id, **_kwargs: task, + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + lanes.BOARD_CORRUPTION_ERRORS.clear() + + assert lanes.claim_ready(set(), 1) == [("healthy", "t_healthy")] + error = capsys.readouterr().err + assert "temporarily skipping Kanban board 'cassandra'" in error + assert "storage OperationalError: disk I/O error" in error + + +def test_board_call_retries_storage_faults_on_fresh_connections(): + connections = [] + + class Connection: + def __init__(self): + self.closed = False + + def close(self): + self.closed = True + + def connect(*, board): + assert board == "cassandra" + connection = Connection() + connections.append(connection) + return connection + + attempts = [] + + def operation(_connection): + attempts.append(1) + if len(attempts) < 3: + raise lanes.sqlite3.OperationalError("disk I/O error") + return "healthy" + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=connect, + ) + lanes.BOARD_CORRUPTION_ERRORS.clear() + + assert lanes._board_call(fake_db, "cassandra", operation) == "healthy" + assert len(connections) == 3 + assert all(connection.closed for connection in connections) + + +@pytest.mark.parametrize( + ("result", "expected_action"), + [ + (lanes.ProcessResult(0, "plain text only", None, False), "block"), + ( + lanes.ProcessResult( + 0, + "", + { + "status": "completed", + "summary": "done", + "changed_files": ["src/a.py"], + "tests_run": ["pytest -q"], + "artifacts": ["reports/result.json"], + "blockers": [], + }, + False, + ), + "complete", + ), + ( + lanes.ProcessResult( + 0, + "", + { + "status": "completed", + "summary": "The full test suite is still running.", + "changed_files": ["src/a.py"], + "tests_run": ["pytest -q — in progress"], + "artifacts": [], + "blockers": [], + }, + False, + ), + "block", + ), + ], +) +def test_claim_requires_structured_evidence_and_surfaces_artifacts( + tmp_path: Path, + monkeypatch, + result, + expected_action, +): + task = SimpleNamespace( + id="t_worker", + current_run_id=4, + assignee="cli-auto", + max_runtime_seconds=60, + ) + calls = [] + heartbeats = [] + connections = [] + artifact = tmp_path / "reports/result.json" + artifact.parent.mkdir() + artifact.write_text("{}\n", encoding="utf-8") + + class Connection: + def __init__(self): + self.closed = False + + def close(self): + self.closed = True + + def connect(*, board): + assert board == "cassandra" + connection = Connection() + connections.append(connection) + return connection + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=connect, + get_task=lambda _conn, _task_id: task, + worker_log_path=lambda _task_id, board: tmp_path / "worker.log", + _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_worker"), + set_branch_name=lambda *_args: None, + set_workspace_path=lambda *_args: None, + build_worker_context=lambda *_args: "bounded objective", + heartbeat_worker=lambda _conn, _task_id, *, note, expected_run_id: ( + heartbeats.append((note, expected_run_id)) or True + ), + add_comment=lambda *_args: None, + complete_task=lambda *_args, **kwargs: calls.append(("complete", kwargs)), + block_task=lambda *_args, **kwargs: calls.append(("block", kwargs)), + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + monkeypatch.setattr( + lanes, "state_path", lambda _board, _task_id: tmp_path / "state.json" + ) + monkeypatch.setattr( + lanes, + "select_route", + lambda *_args, **_kwargs: lanes.Route( + "codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, () + ), + ) + + def run_provider(*args, **_kwargs): + assert connections[0].closed + before_heartbeat = len(connections) + assert args[6]("working") is True + assert len(connections) == before_heartbeat + 1 + assert connections[-1].closed + return result + + monkeypatch.setattr(lanes, "run_provider", run_provider) + + lanes.execute_claim("cassandra", "t_worker") + + assert calls[0][0] == expected_action + assert heartbeats == [("working", 4)] + if expected_action == "complete": + assert calls[0][1]["metadata"]["artifacts"] == [str(artifact)] + assert calls[0][1]["metadata"]["tests_run"] == ["pytest -q"] + else: + assert calls[0][1]["kind"] == "capability" + assert all(connection.closed for connection in connections) + + +def test_goal_card_continues_after_local_judge_rejects_progress( + tmp_path: Path, + monkeypatch, +): + task = SimpleNamespace( + id="t_goal", + current_run_id=12, + assignee="cli-auto", + max_runtime_seconds=300, + goal_mode=True, + goal_max_turns=3, + ) + calls = [] + comments = [] + + class Connection: + def close(self): + return None + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + get_task=lambda _conn, _task_id: task, + worker_log_path=lambda _task_id, board: tmp_path / "worker.log", + _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_goal"), + set_branch_name=lambda *_args: None, + set_workspace_path=lambda *_args: None, + build_worker_context=lambda *_args: "Run tests, commit, push, and verify remote HEAD.", + heartbeat_worker=lambda *_args, **_kwargs: True, + add_comment=lambda _conn, _task_id, _author, body: comments.append(body), + complete_task=lambda *_args, **kwargs: calls.append(("complete", kwargs)), + block_task=lambda *_args, **kwargs: calls.append(("block", kwargs)), + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + monkeypatch.setattr( + lanes, + "state_path", + lambda _board, _task_id: tmp_path / "state.json", + ) + claude_low = lanes.Route( + "claude", "claude-fable-5", "low", "claude-low", "jetson", "vote", 1, () + ) + codex_low = lanes.Route( + "codex", "gpt-5.6-luna", "low", "codex-low", "manual", "fallback", 1, () + ) + codex_xhigh = lanes.Route( + "codex", "gpt-5.6-sol", "xhigh", "codex-xhigh", "jetson", "escalated", 1, () + ) + route_calls = [] + + def select_route(_prompt, assignee, **kwargs): + route_calls.append((assignee, kwargs)) + if assignee == "cli-codex-low": + return codex_low + if len(route_calls) == 1: + return claude_low + return codex_xhigh + + monkeypatch.setattr(lanes, "select_route", select_route) + monkeypatch.setattr(lanes, "fresh_unavailable_provider", lambda: None) + reports = [ + lanes.ProcessResult(1, "authentication expired", None, True), + lanes.ProcessResult( + 0, + "first turn", + { + "status": "completed", + "summary": "Focused tests passed.", + "changed_files": ["src/a.py"], + "tests_run": ["pytest focused: passed"], + "artifacts": [], + "blockers": [], + }, + False, + ), + lanes.ProcessResult( + 0, + "second turn", + { + "status": "completed", + "summary": "Full tests passed; commit pushed and remote HEAD verified.", + "changed_files": ["src/a.py"], + "tests_run": ["pytest full: passed"], + "artifacts": [], + "blockers": [], + }, + False, + ), + ] + monkeypatch.setattr(lanes, "run_provider", lambda *_args, **_kwargs: reports.pop(0)) + verdicts = iter( + [ + (False, "commit, push, and remote verification are missing"), + (True, "all explicit acceptance criteria have evidence"), + ] + ) + judge_contexts = [] + + def judge_goal_completion(objective, *_args, **_kwargs): + judge_contexts.append(objective) + return next(verdicts) + + monkeypatch.setattr( + lanes.cli_lane_goal, + "judge_goal_completion", + judge_goal_completion, + ) + + lanes.execute_claim("cassandra", "t_goal") + + assert calls[0][0] == "complete" + assert calls[0][1]["metadata"]["goal_turn"] == 2 + assert any( + "Goal completion rejected; continuing turn 2/3" in item for item in comments + ) + assert any( + "Goal route 2/3: codex/gpt-5.6-sol at xhigh" in item for item in comments + ) + assert route_calls[2][1]["exclude_provider"] == "claude" + assert "prior rejected reports" in judge_contexts[1] + assert "commit, push, and remote verification are missing" in judge_contexts[1] + assert reports == [] diff --git a/testing/tests/test_hermes_cli_lanes_support.py b/testing/tests/test_hermes_cli_lanes_support.py new file mode 100644 index 00000000..242351b9 --- /dev/null +++ b/testing/tests/test_hermes_cli_lanes_support.py @@ -0,0 +1,79 @@ +"""Shared loaders and manifest helpers for Hermes CLI-lane contracts.""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +import yaml + +ROOT = Path(__file__).parents[2] +SCRIPTS = ROOT / "services/hermes/scripts" +sys.path.insert(0, str(SCRIPTS)) +HERMES = ROOT / "services/hermes" +KEYCLOAK = ROOT / "services/keycloak" +FLUX_HERMES = ROOT / "clusters/atlas/flux-system/applications/hermes/kustomization.yaml" + + +def _load(name: str): + spec = importlib.util.spec_from_file_location(name, SCRIPTS / f"{name}.py") + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +lanes = _load("cli_lane_runner") +policy = _load("claude_command_policy") +migration = _load("migrate_herdr_state") +auth_patch = _load("patch_hermes_auth") +tui_gateway_patch = _load("patch_tui_gateway") +codex_runtime_patch = _load("patch_codex_runtime") +ttyd_patch = _load("patch_ttyd_index") +client_config = _load("configure_agent_clients") + + +def _agent_deployment() -> dict: + return yaml.safe_load((HERMES / "agent-deployment.yaml").read_text()) + + +def _services() -> dict[str, dict]: + return { + item["metadata"]["name"]: item + for item in yaml.safe_load_all((HERMES / "service.yaml").read_text()) + if item + } + + +def _oauth_deployment(name: str) -> dict: + documents = [ + item + for item in yaml.safe_load_all((HERMES / "oauth2-proxy.yaml").read_text()) + if item + ] + return next( + item + for item in documents + if item["kind"] == "Deployment" and item["metadata"]["name"] == name + ) + + +class _SwitchyardResponse: + """Minimal context-managed response used by routing contract tests.""" + + def __init__(self, selected: str, rationale: str = "local classifier vote"): + self.headers = { + "x-model-router-selected-model": selected, + "x-model-router-rationale": rationale, + } + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def read(self): + return b"{}" diff --git a/testing/tests/test_hermes_cli_lanes_toolchain.py b/testing/tests/test_hermes_cli_lanes_toolchain.py new file mode 100644 index 00000000..2ed232cb --- /dev/null +++ b/testing/tests/test_hermes_cli_lanes_toolchain.py @@ -0,0 +1,386 @@ +"""Operator toolchain and runtime-patch contracts for Hermes CLI lanes.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +import yaml + +from testing.tests.test_hermes_cli_lanes_support import ( + HERMES, + SCRIPTS, + _agent_deployment, + auth_patch, + client_config, + codex_runtime_patch, + tui_gateway_patch, + ttyd_patch, +) + + +def test_worker_route_broker_accepts_pod_network_health_checks(): + """Kubelet probes the pod IP, so the broker cannot bind to loopback only.""" + script = (SCRIPTS / "worker_route_broker.py").read_text() + assert 'ThreadingHTTPServer(("0.0.0.0", PORT), Handler)' in script + + +def test_switchyard_network_boundary_allows_vault_bootstrap(): + """The pre-populate init container must reach Vault before routing starts.""" + documents = [ + item + for item in yaml.safe_load_all((HERMES / "networkpolicy.yaml").read_text()) + if item + ] + isolation = next( + item + for item in documents + if item.get("metadata", {}).get("name") == "hermes-switchyard-isolation" + ) + assert any( + rule.get("to") + == [ + { + "namespaceSelector": { + "matchLabels": {"kubernetes.io/metadata.name": "vault"} + }, + "podSelector": {"matchLabels": {"app": "vault"}}, + } + ] + and rule.get("ports") == [{"protocol": "TCP", "port": 8200}] + for rule in isolation["spec"]["egress"] + ) + + +def test_switchyard_network_boundary_allows_metrics_scraping(): + """VictoriaMetrics may scrape Switchyard without widening its API boundary.""" + documents = [ + item + for item in yaml.safe_load_all((HERMES / "networkpolicy.yaml").read_text()) + if item + ] + isolation = next( + item + for item in documents + if item.get("metadata", {}).get("name") == "hermes-switchyard-isolation" + ) + assert any( + rule.get("from") + == [ + { + "namespaceSelector": { + "matchLabels": {"kubernetes.io/metadata.name": "monitoring"} + }, + "podSelector": {"matchLabels": {"app": "server"}}, + } + ] + and rule.get("ports") == [{"protocol": "TCP", "port": 9005}] + for rule in isolation["spec"]["ingress"] + ) + + +def test_owner_agent_installs_the_pinned_operator_toolchain(): + script = (SCRIPTS / "install_agent_tools.sh").read_text() + for value in [ + "flux", + "helm", + "kustomize", + "jq", + "yq", + "gh", + "vault", + "sops", + "age", + "age-keygen", + "k9s", + "terraform", + "go", + "gofmt", + ]: + assert value in script + assert "go1.26.5.linux-arm64.tar.gz" in script + assert "fe4789e92b1f33358680864bbe8704289e7bb5fc207d80623c308935bd696d49" in script + assert script.count("sha256sum -c -") == 1 + + deployment = _agent_deployment() + installer = next( + item + for item in deployment["spec"]["template"]["spec"]["initContainers"] + if item["name"] == "install-agent-tools" + ) + assert "/bin/sh /opt/coordinator/install_agent_tools.sh" in installer["command"][2] + assert any(mount["name"] == "coordinator" for mount in installer["volumeMounts"]) + init_config = next( + item + for item in deployment["spec"]["template"]["spec"]["initContainers"] + if item["name"] == "init-config" + ) + init_command = init_config["command"][2] + assert "# Hermes managed operator PATH." in init_command + assert "/opt/coordinator:/opt/data/tools/bin:/opt/hermes/.venv/bin" in init_command + assert 'chmod 0644 "${profile_file}"' in init_command + + +def test_owner_agent_uses_only_the_canonical_hostname(): + paths = [ + HERMES / "agent-configmap.yaml", + HERMES / "agent-deployment.yaml", + HERMES / "agent-ingress.yaml", + Path(__file__).parents[2] / "scripts/ops/hermes_triage_monitor.py", + ] + for path in paths: + content = path.read_text() + assert "agent.bstein.dev" not in content + assert "agent.hermes.bstein.dev" in content + + +def test_agent_reconnect_retains_complete_history_and_long_tool_budget(): + configmap = yaml.safe_load((HERMES / "agent-configmap.yaml").read_text()) + config = yaml.safe_load(configmap["data"]["config.yaml"]) + display = config["display"] + assert display["resume_exchanges"] >= 10000 + assert display["resume_max_user_chars"] >= 10000000 + assert display["resume_max_assistant_chars"] >= 10000000 + assert config["agent"]["max_turns"] == 180 + assert config["delegation"]["max_iterations"] == 120 + + +def test_auth_patch_honors_explicit_shared_store(tmp_path: Path): + source = tmp_path / "auth.py" + destination = tmp_path / "patched/auth.py" + source.write_text( + 'from pathlib import Path\nimport os\n\ndef _auth_file_path() -> Path:\n path = get_hermes_home() / "auth.json"\n return path\n', + encoding="utf-8", + ) + auth_patch.patch(source, destination) + content = destination.read_text() + assert 'os.environ.get("HERMES_AUTH_FILE"' in content + + +def test_auth_patch_fails_closed_on_upstream_drift(tmp_path: Path): + source = tmp_path / "auth.py" + source.write_text("def changed():\n pass\n", encoding="utf-8") + with pytest.raises(RuntimeError, match="context changed"): + auth_patch.patch(source, tmp_path / "patched.py") + + +def test_codex_runtime_patch_uses_cli_and_forwards_route(tmp_path: Path): + provider = tmp_path / "runtime_provider.py" + provider.write_text(codex_runtime_patch.PROVIDER_BEFORE, encoding="utf-8") + provider_out = tmp_path / "patched/runtime_provider.py" + codex_runtime_patch.patch_provider(provider, provider_out) + assert '"api_mode": "codex_app_server"' in provider_out.read_text() + + session = tmp_path / "codex_app_server_session.py" + session.write_text( + codex_runtime_patch.SESSION_SIGNATURE_BEFORE + + codex_runtime_patch.SESSION_REQUEST_BEFORE, + encoding="utf-8", + ) + session_out = tmp_path / "patched/codex_app_server_session.py" + codex_runtime_patch.patch_session(session, session_out) + session_content = session_out.read_text() + assert 'turn_params["model"] = model' in session_content + assert 'turn_params["effort"] = effort' in session_content + assert '"approvalPolicy": "never"' in session_content + assert '"sandboxPolicy": {"type": "dangerFullAccess"}' in session_content + + turn = tmp_path / "codex_runtime.py" + turn.write_text( + codex_runtime_patch.FALLBACK_CONTEXT_BEFORE + codex_runtime_patch.TURN_BEFORE, + encoding="utf-8", + ) + turn_out = tmp_path / "patched/codex_runtime.py" + codex_runtime_patch.patch_turn(turn, turn_out) + turn_content = turn_out.read_text() + assert "model=str(getattr(agent" in turn_content + assert "build_cross_provider_codex_prompt" in turn_content + + fallback = tmp_path / "chat_completion_helpers.py" + fallback.write_text( + codex_runtime_patch.FALLBACK_RESOLUTION_BEFORE, + encoding="utf-8", + ) + fallback_out = tmp_path / "patched/chat_completion_helpers.py" + codex_runtime_patch.patch_fallback(fallback, fallback_out) + fallback_content = fallback_out.read_text() + assert 'agent.api_mode = "codex_app_server"' in fallback_content + assert "agent._codex_cross_provider_fallback = True" in fallback_content + + loop = tmp_path / "conversation_loop.py" + loop.write_text( + codex_runtime_patch.FALLBACK_DISPATCH_BEFORE + + codex_runtime_patch.RETRY_FALLBACK_DISPATCH_BEFORE + + codex_runtime_patch.STREAM_RECOVERY_BEFORE, + encoding="utf-8", + ) + loop_out = tmp_path / "patched/conversation_loop.py" + codex_runtime_patch.patch_loop(loop, loop_out) + loop_content = loop_out.read_text() + assert loop_content.count('if agent.api_mode == "codex_app_server"') == 2 + assert "build_cross_provider_codex_prompt" in loop_content + retry_dispatch = loop_content.index( + "Fallback activation happens inside this retry loop" + ) + api_kwargs = loop_content.find("agent._build_api_kwargs", retry_dispatch) + assert api_kwargs == -1 or retry_dispatch < api_kwargs + assert "Provider stream ended before a complete response" in loop_content + assert "_is_transport_stub" in loop_content + assert "rerouting ({truncated_tool_call_retries}/4)" in loop_content + + auxiliary = tmp_path / "auxiliary_client.py" + auxiliary.write_text( + codex_runtime_patch.AUXILIARY_TOKEN_BEFORE, + encoding="utf-8", + ) + auxiliary_out = tmp_path / "patched/auxiliary_client.py" + codex_runtime_patch.patch_auxiliary(auxiliary, auxiliary_out) + auxiliary_content = auxiliary_out.read_text() + assert 'os.environ.get("CODEX_HOME"' in auxiliary_content + assert 'Path(codex_home).expanduser() / "auth.json"' in auxiliary_content + assert "never creates a metered API-key lane" in auxiliary_content + + +def test_agent_mounts_codex_auxiliary_runtime_patch(): + deployment = _agent_deployment() + pod = deployment["spec"]["template"]["spec"] + patch_init = next( + item for item in pod["initContainers"] if item["name"] == "patch-codex-runtime" + ) + assert patch_init["command"][-2:] == [ + "/opt/hermes/agent/auxiliary_client.py", + "/patched/auxiliary_client.py", + ] + expected_mount = { + "name": "codex-runtime-patch", + "mountPath": "/opt/hermes/agent/auxiliary_client.py", + "subPath": "auxiliary_client.py", + } + containers = {item["name"]: item for item in pod["containers"]} + for name in ("hermes", "terminal"): + assert expected_mount in containers[name]["volumeMounts"] + + +def test_codex_auxiliary_patch_reads_cli_token_without_copying_it( + tmp_path: Path, + monkeypatch, +): + codex_home = tmp_path / ".codex" + codex_home.mkdir() + (codex_home / "auth.json").write_text( + json.dumps({"tokens": {"access_token": "cli-access-token"}}), + encoding="utf-8", + ) + monkeypatch.setenv("CODEX_HOME", str(codex_home)) + + source = tmp_path / "auxiliary_client.py" + source.write_text( + "import json, logging, os, time\n" + "from pathlib import Path\n" + "logger = logging.getLogger(__name__)\n" + "def read_token():\n" + " try:\n" + " raise RuntimeError('Hermes provider store intentionally empty')\n" + + codex_runtime_patch.AUXILIARY_TOKEN_BEFORE, + encoding="utf-8", + ) + destination = tmp_path / "patched/auxiliary_client.py" + codex_runtime_patch.patch_auxiliary(source, destination) + namespace: dict = {} + exec(compile(destination.read_text(), str(destination), "exec"), namespace) + + assert namespace["read_token"]() == "cli-access-token" + + +def test_codex_runtime_patch_fails_closed_on_upstream_drift(tmp_path: Path): + source = tmp_path / "runtime_provider.py" + source.write_text("def changed():\n pass\n", encoding="utf-8") + with pytest.raises(RuntimeError, match="context changed"): + codex_runtime_patch.patch_provider(source, tmp_path / "patched.py") + + +def test_codex_runtime_migration_uses_owner_unsafe_mode(tmp_path: Path): + config = tmp_path / "config.yaml" + config.write_text("model: {}\n", encoding="utf-8") + codex_home = tmp_path / ".codex" + calls = [] + + class Report: + errors = [] + + @staticmethod + def summary(): + return "configured" + + def migrate(value, **kwargs): + calls.append((value, kwargs)) + return Report() + + client_config.configure_codex_runtime(config, migrate, codex_home) + + assert calls[0][1]["default_permission_profile"] is None + assert calls[0][1]["codex_home"] == codex_home + content = (codex_home / "config.toml").read_text(encoding="utf-8") + assert 'approval_policy = "never"' in content + assert 'sandbox_mode = "danger-full-access"' in content + assert "default_permissions" not in content + + +def test_codex_owner_permissions_replace_stale_profile(tmp_path: Path): + config = tmp_path / "config.toml" + config.write_text( + 'default_permissions = ":danger-no-sandbox"\n\n[features]\nhooks = true\n', + encoding="utf-8", + ) + + client_config.configure_codex_owner_permissions(config) + client_config.configure_codex_owner_permissions(config) + + content = config.read_text(encoding="utf-8") + assert content.count(client_config.OWNER_PERMISSIONS_BEGIN) == 1 + assert content.count('approval_policy = "never"') == 1 + assert content.count('sandbox_mode = "danger-full-access"') == 1 + assert "default_permissions" not in content + assert "[features]\nhooks = true" in content + + +def test_tui_gateway_patch_extends_and_bounds_agent_startup(tmp_path: Path): + source = tmp_path / "server.py" + destination = tmp_path / "patched/server.py" + source.write_text( + "import os\n\n" + tui_gateway_patch.BEFORE + "\ndef unchanged():\n pass\n", + encoding="utf-8", + ) + + tui_gateway_patch.patch(source, destination) + + content = destination.read_text(encoding="utf-8") + assert "HERMES_TUI_AGENT_INIT_TIMEOUT_S" in content + assert "configured = 180.0" in content + assert "return max(30.0, min(configured, 900.0))" in content + assert "timeout: float | None = None" in content + assert "ready.wait(timeout=wait_timeout)" in content + assert "def unchanged():" in content + + +def test_tui_gateway_patch_fails_closed_on_upstream_drift(tmp_path: Path): + source = tmp_path / "server.py" + source.write_text("def changed():\n pass\n", encoding="utf-8") + with pytest.raises(RuntimeError, match="context changed"): + tui_gateway_patch.patch(source, tmp_path / "patched.py") + + +def test_ttyd_clipboard_and_reconnect_patch_remain_enabled(): + source = '' + content = ttyd_patch.patch_html(source) + assert 'id="atlas-ttyd-clipboard"' in content + assert "navigator.clipboard.writeText(text)" in content + assert "class AtlasRecoveringWebSocket" in content + assert "window.location.reload()" in content + assert "event.stopImmediatePropagation()" in content + + +def test_ttyd_patch_fails_closed_on_upstream_drift(): + with pytest.raises(RuntimeError, match="context changed"): + ttyd_patch.patch_html("changed") diff --git a/testing/tests/test_hermes_coordinator.py b/testing/tests/test_hermes_coordinator.py index bc2d5b5f..683c0201 100644 --- a/testing/tests/test_hermes_coordinator.py +++ b/testing/tests/test_hermes_coordinator.py @@ -2,44 +2,20 @@ from __future__ import annotations -import importlib.util import json import subprocess -import sys import tomllib from pathlib import Path -import pytest import yaml - -SCRIPT = Path(__file__).parents[2] / "services/hermes/scripts/hermes_coordinator.py" -sys.path.insert(0, str(SCRIPT.parent)) -routing = importlib.import_module("hermes_model_routing") -catalog_resolver = importlib.import_module("routing_catalog") -SPEC = importlib.util.spec_from_file_location("hermes_coordinator", SCRIPT) -assert SPEC and SPEC.loader -coordinator = importlib.util.module_from_spec(SPEC) -sys.modules[SPEC.name] = coordinator -SPEC.loader.exec_module(coordinator) - - -def _base_config() -> dict: - """Return the minimal coordinator configuration used by routing tests.""" - return { - "model": { - "provider": "openai-codex", - "default": "gpt-5.6-terra", - "model": "gpt-5.6-terra", - }, - "fallback_providers": [ - {"provider": "anthropic", "model": "claude-opus-5"}, - dict(routing.LOCAL_FALLBACK), - ], - "toolsets": ["kanban"], - "terminal": {"cwd": "/opt/data/workspace"}, - } - +from testing.tests.test_hermes_coordinator_support import ( + SCRIPT, + _base_config, + catalog_resolver, + coordinator, + routing, +) def test_model_version_and_quality_selection_handle_new_and_small_models(): assert routing.model_version("claude-3-5-sonnet-20241022") == (3, 5) @@ -334,177 +310,3 @@ def test_refresh_writes_non_secret_routing_status(tmp_path: Path, monkeypatch): assert status["projects"]["cassandra"]["state"] == "ready" assert status["projects"]["cassandra"]["board_state"] == {"state": "ready"} assert "do-not-report" not in status_path.read_text(encoding="utf-8") - - -def test_corrupt_cassandra_board_is_preserved_without_blocking_refresh( - tmp_path: Path, monkeypatch -): - """A damaged project board degrades Kanban instead of stopping all services.""" - - class FakeKanbanDbCorruptError(Exception): - def __init__(self): - super().__init__("integrity check failed") - self.db_path = tmp_path / "kanban.db" - self.backup_path = tmp_path / "kanban.db.corrupt.backup" - self.reason = "row out of order" - - def raise_corruption(_root: Path) -> None: - raise FakeKanbanDbCorruptError() - - monkeypatch.setattr(coordinator, "bootstrap_cassandra", raise_corruption) - monkeypatch.setattr( - coordinator, "_kanban_corruption_type", lambda: FakeKanbanDbCorruptError - ) - - status = coordinator.bootstrap_cassandra_state(tmp_path) - - assert status == { - "state": "corrupt-preserved", - "database": str(tmp_path / "kanban.db"), - "backup": str(tmp_path / "kanban.db.corrupt.backup"), - "reason": "row out of order", - } - - -def test_unrelated_cassandra_bootstrap_failure_remains_fatal( - tmp_path: Path, monkeypatch -): - """Only the known fail-closed corruption state may be degraded.""" - - class FakeKanbanDbCorruptError(Exception): - pass - - def raise_permission_error(_root: Path) -> None: - raise PermissionError("cannot access board") - - monkeypatch.setattr(coordinator, "bootstrap_cassandra", raise_permission_error) - monkeypatch.setattr( - coordinator, "_kanban_corruption_type", lambda: FakeKanbanDbCorruptError - ) - - try: - coordinator.bootstrap_cassandra_state(tmp_path) - except PermissionError as error: - assert str(error) == "cannot access board" - else: - raise AssertionError("unrelated board errors must fail the coordinator refresh") - - -def test_cassandra_workspace_uses_valid_configured_worktree( - tmp_path: Path, monkeypatch -): - """The coordinator selects an explicit Git worktree but rejects stale paths.""" - active = tmp_path / "cassandra-v69" - active.mkdir() - (active / ".git").write_text("gitdir: elsewhere\n", encoding="utf-8") - monkeypatch.setenv("HERMES_CASSANDRA_ACTIVE_WORKTREE", str(active)) - - assert coordinator.cassandra_workspace() == active - - (active / ".git").unlink() - assert coordinator.cassandra_workspace() == coordinator.CASSANDRA_BASE_PATH - - -@pytest.mark.parametrize( - ("current_remote", "repair_action"), - [ - ("https://scm.bstein.dev/bstein/cassandra.git", "set-url"), - (None, "add"), - (coordinator.CASSANDRA_REMOTE, None), - ], - ids=["old-origin", "missing-origin", "canonical-origin"], -) -def test_cassandra_sync_repairs_existing_worktree( - tmp_path: Path, monkeypatch, current_remote: str | None, repair_action: str | None -): - """Existing worktrees use the canonical Atlas origin before fetching.""" - workspace = tmp_path / "cassandra" - (workspace / ".git").mkdir(parents=True) - monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", workspace) - monkeypatch.setattr(coordinator.shutil, "which", lambda _name: "/usr/bin/git") - token_path = tmp_path / "gitea-token" - token_path.write_text("configured\n", encoding="utf-8") - monkeypatch.setenv("HERMES_GITEA_TOKEN_FILE", str(token_path)) - commands: list[list[str]] = [] - environments: list[dict[str, str]] = [] - - def run(command, **kwargs): - commands.append(command) - environments.append(kwargs.get("env", {})) - if command[-2:] == ["get-url", "origin"]: - return subprocess.CompletedProcess( - command, 0 if current_remote else 2, stdout=f"{current_remote or ''}\n" - ) - return subprocess.CompletedProcess(command, 0) - - monkeypatch.setattr(coordinator.subprocess, "run", run) - assert coordinator.sync_cassandra_repo({"GITEA_TOKEN": "must-not-pass"}) == "ready" - repair_commands = [command for command in commands if repair_action in command] - assert bool(repair_commands) is bool(repair_action) - assert commands[-1][-4:] == ["fetch", "--quiet", "--prune", "origin"] - assert all("GITEA_TOKEN" not in environment for environment in environments) - - -def test_cassandra_sync_repairs_origin_without_token(tmp_path: Path, monkeypatch): - """Missing credentials skip only the fetch, not the local origin repair.""" - workspace = tmp_path / "cassandra" - (workspace / ".git").mkdir(parents=True) - monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", workspace) - monkeypatch.setattr(coordinator.shutil, "which", lambda _name: "/usr/bin/git") - commands: list[list[str]] = [] - - def run(command, **_kwargs): - commands.append(command) - stdout = "old-origin\n" if command[-2:] == ["get-url", "origin"] else None - return subprocess.CompletedProcess(command, 0, stdout=stdout) - - monkeypatch.setattr(coordinator.subprocess, "run", run) - state = coordinator.sync_cassandra_repo({}) - - assert state == "ready; fetch skipped until Gitea token is configured" - assert [command[-3] for command in commands] == ["remote", "set-url"] - - -def test_migrate_open_cassandra_tasks_preserves_running_and_done_tasks(): - """A project switch moves queued work without relocating active evidence.""" - - class Task: - def __init__(self, task_id: str, status: str): - self.id = task_id - self.status = status - self.workspace_kind = "dir" - self.workspace_path = str(coordinator.CASSANDRA_BASE_PATH) - - class Closing: - def __enter__(self): - return object() - - def __exit__(self, *_args): - return None - - class FakeKanban: - tasks = [ - Task("queued", "todo"), - Task("active", "running"), - Task("done", "done"), - ] - moved: list[tuple[str, str]] = [] - - @staticmethod - def connect_closing(*, board: str): - assert board == "cassandra" - return Closing() - - @classmethod - def list_tasks(cls, _connection, *, include_archived: bool): - assert include_archived is False - return cls.tasks - - @classmethod - def set_workspace_path(cls, _connection, task_id: str, path: Path): - cls.moved.append((task_id, str(path))) - - active = Path("/opt/data/workspace/projects/cassandra-hermes-v69") - coordinator._migrate_open_cassandra_tasks(FakeKanban, active) - - assert FakeKanban.moved == [("queued", str(active))] diff --git a/testing/tests/test_hermes_coordinator_cassandra.py b/testing/tests/test_hermes_coordinator_cassandra.py new file mode 100644 index 00000000..646db291 --- /dev/null +++ b/testing/tests/test_hermes_coordinator_cassandra.py @@ -0,0 +1,194 @@ +"""Cassandra workspace and board contracts for the Hermes coordinator.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from testing.tests.test_hermes_coordinator_support import coordinator + + +def test_corrupt_cassandra_board_is_preserved_without_blocking_refresh( + tmp_path: Path, monkeypatch +): + """A damaged project board degrades Kanban instead of stopping all services.""" + + class FakeKanbanDbCorruptError(Exception): + def __init__(self): + super().__init__("integrity check failed") + self.db_path = tmp_path / "kanban.db" + self.backup_path = tmp_path / "kanban.db.corrupt.backup" + self.reason = "row out of order" + + def raise_corruption(_root: Path) -> None: + raise FakeKanbanDbCorruptError() + + monkeypatch.setattr(coordinator, "bootstrap_cassandra", raise_corruption) + monkeypatch.setattr( + coordinator, "_kanban_corruption_type", lambda: FakeKanbanDbCorruptError + ) + + status = coordinator.bootstrap_cassandra_state(tmp_path) + + assert status == { + "state": "corrupt-preserved", + "database": str(tmp_path / "kanban.db"), + "backup": str(tmp_path / "kanban.db.corrupt.backup"), + "reason": "row out of order", + } + + +def test_unrelated_cassandra_bootstrap_failure_remains_fatal( + tmp_path: Path, monkeypatch +): + """Only the known fail-closed corruption state may be degraded.""" + + class FakeKanbanDbCorruptError(Exception): + pass + + def raise_permission_error(_root: Path) -> None: + raise PermissionError("cannot access board") + + monkeypatch.setattr(coordinator, "bootstrap_cassandra", raise_permission_error) + monkeypatch.setattr( + coordinator, "_kanban_corruption_type", lambda: FakeKanbanDbCorruptError + ) + + try: + coordinator.bootstrap_cassandra_state(tmp_path) + except PermissionError as error: + assert str(error) == "cannot access board" + else: + raise AssertionError("unrelated board errors must fail the coordinator refresh") + + +def test_cassandra_workspace_uses_valid_configured_worktree( + tmp_path: Path, monkeypatch +): + """The coordinator selects an explicit Git worktree but rejects stale paths.""" + active = tmp_path / "cassandra-v69" + active.mkdir() + (active / ".git").write_text("gitdir: elsewhere\n", encoding="utf-8") + monkeypatch.setenv("HERMES_CASSANDRA_ACTIVE_WORKTREE", str(active)) + + assert coordinator.cassandra_workspace() == active + + (active / ".git").unlink() + assert coordinator.cassandra_workspace() == coordinator.CASSANDRA_BASE_PATH + + +@pytest.mark.parametrize( + ("current_remote", "repair_action"), + [ + ("https://scm.bstein.dev/bstein/cassandra.git", "set-url"), + (None, "add"), + (coordinator.CASSANDRA_REMOTE, None), + ], + ids=["old-origin", "missing-origin", "canonical-origin"], +) +def test_cassandra_sync_repairs_existing_worktree( + tmp_path: Path, monkeypatch, current_remote: str | None, repair_action: str | None +): + """Existing worktrees use the canonical Atlas origin before fetching.""" + workspace = tmp_path / "cassandra" + (workspace / ".git").mkdir(parents=True) + monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", workspace) + monkeypatch.setattr(coordinator.shutil, "which", lambda _name: "/usr/bin/git") + token_path = tmp_path / "gitea-token" + token_path.write_text("configured\n", encoding="utf-8") + monkeypatch.setenv("HERMES_GITEA_TOKEN_FILE", str(token_path)) + commands: list[list[str]] = [] + environments: list[dict[str, str]] = [] + + def run(command, **kwargs): + commands.append(command) + environments.append(kwargs.get("env", {})) + if command[-2:] == ["get-url", "origin"]: + return subprocess.CompletedProcess( + command, 0 if current_remote else 2, stdout=f"{current_remote or ''}\n" + ) + return subprocess.CompletedProcess(command, 0) + + monkeypatch.setattr(coordinator.subprocess, "run", run) + assert coordinator.sync_cassandra_repo({"GITEA_TOKEN": "must-not-pass"}) == "ready" + repair_commands = [command for command in commands if repair_action in command] + assert bool(repair_commands) is bool(repair_action) + assert commands[-1][-4:] == ["fetch", "--quiet", "--prune", "origin"] + assert all("GITEA_TOKEN" not in environment for environment in environments) + + +def test_cassandra_sync_repairs_origin_through_broker_without_token( + tmp_path: Path, monkeypatch +): + """Repository sync uses the credential-isolated broker remote.""" + monkeypatch.setenv( + "HERMES_GITEA_TOKEN_FILE", str(tmp_path / "explicitly-unavailable-token") + ) + workspace = tmp_path / "cassandra" + (workspace / ".git").mkdir(parents=True) + monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", workspace) + monkeypatch.setattr(coordinator.shutil, "which", lambda _name: "/usr/bin/git") + commands: list[list[str]] = [] + + def run(command, **_kwargs): + commands.append(command) + stdout = "old-origin\n" if command[-2:] == ["get-url", "origin"] else None + return subprocess.CompletedProcess(command, 0, stdout=stdout) + + monkeypatch.setattr(coordinator.subprocess, "run", run) + state = coordinator.sync_cassandra_repo({}) + + assert state == "ready" + assert any( + "hermes-scm-broker.hermes-scm.svc.cluster.local" in " ".join(cmd) + for cmd in commands + ) + assert all("token" not in " ".join(cmd).lower() for cmd in commands) + assert commands[-1][-4:] == ["fetch", "--quiet", "--prune", "origin"] + + +def test_migrate_open_cassandra_tasks_preserves_running_and_done_tasks(): + """A project switch moves queued work without relocating active evidence.""" + + class Task: + def __init__(self, task_id: str, status: str): + self.id = task_id + self.status = status + self.workspace_kind = "dir" + self.workspace_path = str(coordinator.CASSANDRA_BASE_PATH) + + class Closing: + def __enter__(self): + return object() + + def __exit__(self, *_args): + return None + + class FakeKanban: + tasks = [ + Task("queued", "todo"), + Task("active", "running"), + Task("done", "done"), + ] + moved: list[tuple[str, str]] = [] + + @staticmethod + def connect_closing(*, board: str): + assert board == "cassandra" + return Closing() + + @classmethod + def list_tasks(cls, _connection, *, include_archived: bool): + assert include_archived is False + return cls.tasks + + @classmethod + def set_workspace_path(cls, _connection, task_id: str, path: Path): + cls.moved.append((task_id, str(path))) + + active = Path("/opt/data/workspace/projects/cassandra-hermes-v69") + coordinator._migrate_open_cassandra_tasks(FakeKanban, active) + + assert FakeKanban.moved == [("queued", str(active))] diff --git a/testing/tests/test_hermes_coordinator_coverage.py b/testing/tests/test_hermes_coordinator_coverage.py new file mode 100644 index 00000000..fe4f3168 --- /dev/null +++ b/testing/tests/test_hermes_coordinator_coverage.py @@ -0,0 +1,267 @@ +"""Behavioral branch coverage for Hermes project coordination.""" + +from __future__ import annotations + +import json +import subprocess +import sys +import types +from contextlib import nullcontext +from pathlib import Path + +import pytest + +from testing.tests.test_hermes_coordinator_support import coordinator, routing + + +def test_workspace_falls_back_when_unset_or_invalid(tmp_path: Path, monkeypatch): + monkeypatch.delenv("HERMES_CASSANDRA_ACTIVE_WORKTREE", raising=False) + assert coordinator.cassandra_workspace() == coordinator.CASSANDRA_BASE_PATH + monkeypatch.setenv("HERMES_CASSANDRA_ACTIVE_WORKTREE", str(tmp_path / "missing")) + assert coordinator.cassandra_workspace() == coordinator.CASSANDRA_BASE_PATH + + +def test_task_migration_is_noop_for_base_workspace(): + class Board: + @staticmethod + def connect_closing(**_kwargs): + raise AssertionError("base workspace must not access Kanban") + + coordinator._migrate_open_cassandra_tasks(Board, coordinator.CASSANDRA_BASE_PATH) + + +class _Project: + id = "project-id" + + +def _fake_hermes_modules(monkeypatch, *, existing: bool, active: bool): + events = [] + kb = types.ModuleType("hermes_cli.kanban_db") + kb.board_exists = lambda slug: existing + kb.create_board = lambda *args, **kwargs: events.append( + ("create-board", args, kwargs) + ) + kb.set_current_board = lambda slug: events.append(("current", slug)) + kb.connect_closing = lambda **kwargs: nullcontext(object()) + kb.list_tasks = lambda *_a, **_k: [] + + pdb = types.ModuleType("hermes_cli.projects_db") + pdb.connect_closing = lambda: nullcontext(object()) + pdb.get_project = lambda *_a: _Project() if existing else None + pdb.get_active_id = lambda *_a: "active" if active else None + pdb.create_project = ( + lambda *args, **kwargs: events.append(("create-project", kwargs)) or "new-id" + ) + pdb.set_active = lambda *args: events.append(("active", args[-1])) + pdb.update_project = lambda *args, **kwargs: events.append(("update", kwargs)) + pdb.add_folder = lambda *args, **kwargs: events.append(("folder", args[-1], kwargs)) + + package = types.ModuleType("hermes_cli") + package.kanban_db = kb + package.projects_db = pdb + monkeypatch.setitem(sys.modules, "hermes_cli", package) + monkeypatch.setitem(sys.modules, "hermes_cli.kanban_db", kb) + monkeypatch.setitem(sys.modules, "hermes_cli.projects_db", pdb) + return events + + +@pytest.mark.parametrize( + ("existing", "active", "expected"), + [ + (False, False, {"current", "create-project", "active"}), + (False, True, {"current", "create-project"}), + (True, True, {"update", "folder"}), + ], +) +def test_bootstrap_cassandra_creates_or_updates_state( + tmp_path: Path, monkeypatch, existing, active, expected +): + events = _fake_hermes_modules(monkeypatch, existing=existing, active=active) + base = tmp_path / "projects/cassandra" + worktree = tmp_path / "projects/cassandra-worktree" + worktree.mkdir(parents=True) + (worktree / ".git").write_text("gitdir: elsewhere\n", encoding="utf-8") + monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", base) + monkeypatch.setenv("HERMES_CASSANDRA_ACTIVE_WORKTREE", str(worktree)) + coordinator.bootstrap_cassandra(tmp_path / "hermes") + names = {event[0] for event in events} + assert expected <= names + assert "create-board" in names + + +def test_bootstrap_state_ready_path(tmp_path: Path, monkeypatch): + monkeypatch.setattr(coordinator, "bootstrap_cassandra", lambda _root: None) + assert coordinator.bootstrap_cassandra_state(tmp_path) == {"state": "ready"} + + +def test_corruption_type_imports_pinned_exception(monkeypatch): + class Expected(Exception): + pass + + package = types.ModuleType("hermes_cli") + kanban = types.ModuleType("hermes_cli.kanban_db") + kanban.KanbanDbCorruptError = Expected + package.kanban_db = kanban + monkeypatch.setitem(sys.modules, "hermes_cli", package) + monkeypatch.setitem(sys.modules, "hermes_cli.kanban_db", kanban) + assert coordinator._kanban_corruption_type() is Expected + + +def test_repo_sync_reports_missing_git_and_unmanaged_directory( + tmp_path: Path, monkeypatch +): + monkeypatch.setattr(coordinator.shutil, "which", lambda _name: None) + assert coordinator.sync_cassandra_repo({}) == "git-unavailable" + + workspace = tmp_path / "cassandra" + workspace.mkdir() + (workspace / "user-file").write_text("preserve", encoding="utf-8") + monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", workspace) + monkeypatch.setattr(coordinator.shutil, "which", lambda _name: "/usr/bin/git") + assert coordinator.sync_cassandra_repo({}) == "unmanaged-nonempty-directory" + + +@pytest.mark.parametrize( + ("failure", "expected"), + [ + ("get-url-error", "remote-repair-failed"), + ("repair-exit", "remote-repair-failed-9"), + ("repair-error", "remote-repair-failed"), + ("fetch-exit", "sync-failed-8"), + ("fetch-error", "sync-failed"), + ], +) +def test_existing_repo_sync_reports_each_failure( + tmp_path: Path, monkeypatch, failure, expected +): + workspace = tmp_path / "cassandra" + (workspace / ".git").mkdir(parents=True) + monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", workspace) + monkeypatch.setattr(coordinator.shutil, "which", lambda _name: "/usr/bin/git") + + def run(command, **_kwargs): + action = command[-2:] if len(command) >= 2 else [] + if action == ["get-url", "origin"]: + if failure == "get-url-error": + raise OSError("git unavailable") + return subprocess.CompletedProcess(command, 0, stdout="old\n") + if "set-url" in command: + if failure == "repair-error": + raise subprocess.TimeoutExpired(command, 30) + return subprocess.CompletedProcess( + command, 9 if failure == "repair-exit" else 0 + ) + if failure == "fetch-error": + raise OSError("fetch unavailable") + return subprocess.CompletedProcess(command, 8 if failure == "fetch-exit" else 0) + + monkeypatch.setattr(coordinator.subprocess, "run", run) + assert coordinator.sync_cassandra_repo({}) == expected + + +@pytest.mark.parametrize( + ("exit_code", "expected"), [(0, "ready"), (7, "sync-failed-7")] +) +def test_empty_repo_clones_through_broker( + tmp_path: Path, monkeypatch, exit_code, expected +): + workspace = tmp_path / "cassandra" + monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", workspace) + monkeypatch.setattr(coordinator.shutil, "which", lambda _name: "/usr/bin/git") + seen = [] + + def run(command, **kwargs): + seen.append((command, kwargs["env"])) + return subprocess.CompletedProcess(command, exit_code) + + monkeypatch.setattr(coordinator.subprocess, "run", run) + assert ( + coordinator.sync_cassandra_repo( + {"API_SERVER_KEY": "not-forwarded", "SAFE": "yes"} + ) + == expected + ) + assert coordinator.CASSANDRA_REMOTE in seen[0][0] + assert seen[0][1]["SAFE"] == "yes" + assert "API_SERVER_KEY" not in seen[0][1] + + +def _catalog(provider): + return routing.Catalog(provider, ["model"], True, True, "connected") + + +def test_refresh_once_writes_complete_status(tmp_path: Path, monkeypatch): + monkeypatch.setattr(coordinator, "_read_env", lambda _path: {"SAFE": "yes"}) + monkeypatch.setattr(coordinator, "discover_codex_models", lambda: _catalog("codex")) + monkeypatch.setattr( + coordinator, "discover_claude_models", lambda: _catalog("claude") + ) + monkeypatch.setattr( + coordinator, "configure_routes", lambda *_a: {"coordinator": ["auto"]} + ) + monkeypatch.setattr( + coordinator, "bootstrap_cassandra_state", lambda _root: {"state": "ready"} + ) + monkeypatch.setattr(coordinator, "sync_cassandra_repo", lambda _env: "ready") + monkeypatch.setattr( + coordinator, "cassandra_workspace", lambda: tmp_path / "worktree" + ) + written = [] + monkeypatch.setattr( + coordinator, "_atomic_write", lambda path, value: written.append((path, value)) + ) + status = coordinator.refresh_once(tmp_path) + assert status["projects"]["cassandra"]["state"] == "ready" + assert json.loads(written[0][1])["providers"]["codex"]["connected"] is True + + +def test_main_once_success_and_failure(monkeypatch, capsys): + status = {"providers": {"codex": {"state": "ready"}}} + monkeypatch.setattr(sys, "argv", ["coordinator", "--once"]) + monkeypatch.setattr(coordinator, "refresh_once", lambda _root: status) + assert coordinator.main() == 0 + assert "codex=ready" in capsys.readouterr().out + monkeypatch.setattr( + coordinator, + "refresh_once", + lambda _root: (_ for _ in ()).throw(RuntimeError("failed")), + ) + assert coordinator.main() == 1 + assert "RuntimeError" in capsys.readouterr().out + + +@pytest.mark.parametrize(("interval", "expected"), [(1, 300), (400, 400)]) +def test_main_loop_sleeps_with_floor(monkeypatch, interval, expected): + calls = iter(({"providers": {}}, KeyboardInterrupt())) + + def refresh(_root): + value = next(calls) + if isinstance(value, BaseException): + raise value + return value + + sleeps = [] + monkeypatch.setattr(coordinator, "refresh_once", refresh) + monkeypatch.setattr(coordinator.time, "sleep", sleeps.append) + monkeypatch.setattr( + sys, "argv", ["coordinator", "--loop", "--interval", str(interval)] + ) + with pytest.raises(KeyboardInterrupt): + coordinator.main() + assert sleeps == [expected] + + +def test_main_loop_recovers_from_refresh_failure(monkeypatch): + calls = iter((RuntimeError("temporary"), KeyboardInterrupt())) + + def refresh(_root): + error = next(calls) + raise error + + sleeps = [] + monkeypatch.setattr(coordinator, "refresh_once", refresh) + monkeypatch.setattr(coordinator.time, "sleep", sleeps.append) + monkeypatch.setattr(sys, "argv", ["coordinator", "--loop", "--interval", "300"]) + with pytest.raises(KeyboardInterrupt): + coordinator.main() + assert sleeps == [300] diff --git a/testing/tests/test_hermes_coordinator_support.py b/testing/tests/test_hermes_coordinator_support.py new file mode 100644 index 00000000..2a41f070 --- /dev/null +++ b/testing/tests/test_hermes_coordinator_support.py @@ -0,0 +1,35 @@ +"""Shared loader and configuration for Hermes coordinator contracts.""" + +from __future__ import annotations + +import importlib +import importlib.util +import sys +from pathlib import Path + +SCRIPT = Path(__file__).parents[2] / "services/hermes/scripts/hermes_coordinator.py" +sys.path.insert(0, str(SCRIPT.parent)) +routing = importlib.import_module("hermes_model_routing") +catalog_resolver = importlib.import_module("routing_catalog") +SPEC = importlib.util.spec_from_file_location("hermes_coordinator", SCRIPT) +assert SPEC and SPEC.loader +coordinator = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = coordinator +SPEC.loader.exec_module(coordinator) + + +def _base_config() -> dict: + """Return the minimal coordinator configuration used by routing tests.""" + return { + "model": { + "provider": "openai-codex", + "default": "gpt-5.6-terra", + "model": "gpt-5.6-terra", + }, + "fallback_providers": [ + {"provider": "anthropic", "model": "claude-opus-5"}, + dict(routing.LOCAL_FALLBACK), + ], + "toolsets": ["kanban"], + "terminal": {"cwd": "/opt/data/workspace"}, + } diff --git a/testing/tests/test_hermes_gitea_branch_coverage.py b/testing/tests/test_hermes_gitea_branch_coverage.py new file mode 100644 index 00000000..362e8726 --- /dev/null +++ b/testing/tests/test_hermes_gitea_branch_coverage.py @@ -0,0 +1,183 @@ +"""Behavioral branch coverage for the Gitea protection policy checker.""" + +from __future__ import annotations + +import json +import os +import stat +import sys +from pathlib import Path + +import pytest + +from testing.tests.test_hermes_scm_broker_support import ROOT, _load_path + + +def _module(name: str = "branch_protection_coverage"): + return _load_path( + name, + ROOT / "services/gitea/scripts/gitea_branch_protection_check.py", + ) + + +def _rule(module, name: str = "main", **updates): + value = { + "rule_name": name, + "priority": 1, + "created_at": "2026-01-01T00:00:00Z", + **module._required("bstein"), + } + value.update(updates) + return value + + +def test_read_bounded_accepts_regular_file_and_closes_descriptor( + tmp_path: Path, monkeypatch +): + module = _module("branch_read_regular") + source = tmp_path / "rules.json" + source.write_bytes(b"[]") + closed: list[int] = [] + real_close = os.close + + def close(descriptor: int) -> None: + closed.append(descriptor) + real_close(descriptor) + + monkeypatch.setattr(module.os, "close", close) + assert module._read_bounded(source) == b"[]" + assert len(closed) == 1 + + +@pytest.mark.parametrize("kind", ["directory", "oversized", "short"]) +def test_read_bounded_rejects_unsafe_or_changed_input( + tmp_path: Path, monkeypatch, kind +): + module = _module(f"branch_read_{kind}") + source = tmp_path / "rules" + source.write_bytes(b"[]") + real_fstat = module.os.fstat + real_read = module.os.read + + if kind == "directory": + monkeypatch.setattr( + module.os, + "fstat", + lambda descriptor: os.stat_result( + (stat.S_IFDIR | 0o700, 0, 0, 0, 0, 0, 2, 0, 0, 0) + ), + ) + elif kind == "oversized": + monkeypatch.setattr( + module.os, + "fstat", + lambda descriptor: os.stat_result( + (stat.S_IFREG | 0o600, 0, 0, 0, 0, 0, module.MAX_INPUT + 1, 0, 0, 0) + ), + ) + else: + monkeypatch.setattr(module.os, "fstat", real_fstat) + monkeypatch.setattr(module.os, "read", lambda descriptor, maximum: b"[") + with pytest.raises(module.PolicyError): + module._read_bounded(source) + monkeypatch.setattr(module.os, "read", real_read) + + +@pytest.mark.parametrize( + ("pattern", "branch", "expected"), + [ + ("m?in", "main", True), + ("m[ai]in", "main", True), + ("m[!z]in", "main", True), + ("m]ain", "m]ain", True), + ("m}ain", "m}ain", True), + ("m[a-z]in", "main", True), + ("m{ain,aster}", "main", True), + ("m\\ain", "main", True), + ], +) +def test_glob_grammar_accepts_supported_constructs(pattern, branch, expected): + module = _module("branch_glob_supported") + assert module._matches(pattern, branch) is expected + + +@pytest.mark.parametrize( + "pattern", + [ + "m\\", + "m[", + "m[]", + "m[!]", + "m[[a]", + "m[\\a]", + "m[-a]", + "m[a-]", + "m[a-b-c]", + "m[z-a]", + "m[aa-b]", + "m{ain}", + "m{ain,}", + "m{ain,aster", + ], +) +def test_invalid_globs_are_literal_nonmatches(pattern): + module = _module("branch_glob_invalid") + assert module._matches(pattern, "main") is False + + +@pytest.mark.parametrize("pattern", ["", "x" * 256, "máin"]) +def test_rule_names_are_ascii_and_bounded(pattern): + module = _module("branch_name_bounds") + with pytest.raises(module.PolicyError, match="rule name"): + module._matches(pattern, "main") + + +@pytest.mark.parametrize("value", [None, "x" * 65, "invalid", "2026-01-01T00:00:00"]) +def test_created_at_requires_short_timezone_aware_iso(value): + module = _module("branch_created_bounds") + with pytest.raises(module.PolicyError, match="creation time"): + module._created(value) + + +@pytest.mark.parametrize( + "value", + [b"x" * (1024 * 1024 + 1), b"{}", json.dumps([None]).encode()], +) +def test_evaluate_rejects_invalid_top_level_inputs(value): + module = _module("branch_evaluate_top") + with pytest.raises((module.PolicyError, json.JSONDecodeError)): + module.evaluate(value, "main", "bstein") + + +def test_evaluate_rejects_unknown_branch_and_excess_rules(): + module = _module("branch_evaluate_bounds") + with pytest.raises(module.PolicyError, match="input"): + module.evaluate(b"[]", "release", "bstein") + with pytest.raises(module.PolicyError, match="response"): + module.evaluate(json.dumps([{}] * 101).encode(), "main", "bstein") + + +@pytest.mark.parametrize("priority", [True, 0, 1_000_001, "1"]) +def test_evaluate_rejects_ambiguous_priority_types(priority): + module = _module("branch_priority_bounds") + rule = _rule(module, priority=priority) + with pytest.raises(module.PolicyError, match="priority"): + module.evaluate(json.dumps([rule]).encode(), "main", "bstein") + + +def test_main_reports_success_and_failures(tmp_path: Path, monkeypatch, capsys): + module = _module("branch_main_paths") + source = tmp_path / "rules.json" + source.write_text(json.dumps([_rule(module)]), encoding="utf-8") + monkeypatch.setattr(sys, "argv", ["checker", str(source), "main", "bstein"]) + assert module.main() == 0 + assert capsys.readouterr().out.strip() == "PRESENT" + + source.write_text("not-json", encoding="utf-8") + assert module.main() == 1 + assert "branch protection check failed" in capsys.readouterr().err + + monkeypatch.setattr( + module, "_read_bounded", lambda _path: (_ for _ in ()).throw(OSError()) + ) + assert module.main() == 1 diff --git a/testing/tests/test_hermes_gitea_internal_coverage.py b/testing/tests/test_hermes_gitea_internal_coverage.py new file mode 100644 index 00000000..f811827c --- /dev/null +++ b/testing/tests/test_hermes_gitea_internal_coverage.py @@ -0,0 +1,375 @@ +"""Behavioral coverage for internal safe-Gitea client and policy paths.""" + +from __future__ import annotations + +import json +import subprocess +import sys +import types +import urllib.error +import urllib.parse +from email.message import Message +from pathlib import Path + +import pytest + +from testing.tests.test_hermes_gitea_support import CLIENT_PATH, Response, _load +from testing.tests.test_hermes_scm_broker_support import _load_path + + +def _policy(name: str = "gitea_policy_coverage"): + return _load_path(name, CLIENT_PATH.parent / "gitea_api_policy.py") + + +def test_runtime_token_reader_and_fixed_origin(tmp_path: Path, monkeypatch): + module = _load() + token = tmp_path / "token" + token.write_text(" synthetic-runtime-value \n", encoding="utf-8") + assert module.read_token(token) == "synthetic-runtime-value" + token.write_text(" \n", encoding="utf-8") + with pytest.raises(ValueError, match="empty"): + module.read_token(token) + + monkeypatch.setenv("GITEA_BASE_URL", module.CANONICAL_BASE_URL) + assert module.configured_base_url() == module.CANONICAL_BASE_URL + monkeypatch.setenv("GITEA_BASE_URL", "https://example.invalid") + with pytest.raises(module.PolicyError, match="fixed"): + module.configured_base_url() + + +def test_safe_urlopen_delegates_only_to_redirect_rejecting_opener(monkeypatch): + module = _load() + sentinel = object() + monkeypatch.setattr( + module._SAFE_OPENER, + "open", + lambda request, timeout: (request, timeout, sentinel), + ) + request = object() + assert module._safe_urlopen(request, 7) == (request, 7, sentinel) + + +def test_api_target_and_request_body_edge_paths(monkeypatch): + module = _load() + with pytest.raises(module.PolicyError, match="path exceeds"): + module._split_api_path("/api/v1/" + "x" * 510) + with pytest.raises(module.PolicyError, match="request body"): + module.authorize_request( + "GET", "/api/v1/repos/atlas/cassandra", {"unexpected": True} + ) + with pytest.raises(module.PolicyError, match="query"): + module.authorize_request( + "POST", + "/api/v1/repos/atlas/cassandra/pulls?page=1", + {}, + ) + with pytest.raises(module.PolicyError, match="only read"): + module.authorize_request("TRACE", "/api/v1/repos/atlas/cassandra/pulls", None) + with pytest.raises(module.PolicyError, match="accepts only"): + module.authorize_request("POST", "/api/v1/repos/atlas/cassandra/pulls", []) + + +def test_response_status_fallback_and_nested_fail_closed(): + module = _load() + + class GetCodeResponse(Response): + def __init__(self): + super().__init__({"name": "cassandra"}, status=200) + del self.status + + def getcode(self): + return 200 + + assert ( + module.read( + "/api/v1/repos/atlas/cassandra", + token="synthetic", + opener=lambda *_a, **_k: GetCodeResponse(), + ) + == b'{"name": "cassandra"}' + ) + with pytest.raises(module.PolicyError, match="omitted required"): + module._nested({"base": None}, "base", "ref") + + +@pytest.mark.parametrize("body", [b"not-json", b"[]"]) +def test_create_response_requires_json_object(body: bytes): + module = _load() + with pytest.raises(module.PolicyError, match="invalid pull-request metadata"): + module._require_create_response( + body, + repo="cassandra", + base="main", + head="feature/test", + head_sha="a" * 40, + title="WIP: Test", + body="Evidence", + ) + + +def test_write_body_handles_empty_newline_and_missing_newline(monkeypatch): + module = _load() + + class Buffer: + value = bytearray() + + @classmethod + def write(cls, value): + cls.value.extend(value) + + monkeypatch.setattr(module.sys, "stdout", types.SimpleNamespace(buffer=Buffer)) + module._write_body(b"") + module._write_body(b"one\n") + module._write_body(b"two") + assert bytes(Buffer.value) == b"one\ntwo\n" + + +def test_main_executes_broker_read_and_create_paths(monkeypatch): + module = _load() + outputs: list[bytes] = [] + client = types.ModuleType("scm_broker_client") + client.read = lambda path: json.dumps({"path": path}).encode() + client.create_draft = lambda repo, **data: json.dumps( + {"repo": repo, **data}, sort_keys=True + ).encode() + monkeypatch.setitem(sys.modules, "scm_broker_client", client) + monkeypatch.setattr(module, "_write_body", outputs.append) + + assert module.main(["read", "/api/v1/repos/atlas/cassandra"]) == 0 + assert json.loads(outputs.pop()) == {"path": "/api/v1/repos/atlas/cassandra"} + assert ( + module.main( + [ + "create-draft", + "cassandra", + "--base", + "main", + "--head", + "feature/coverage", + "--head-sha", + "a" * 40, + "--title", + "Coverage repair", + "--body", + "Review the focused tests", + ] + ) + == 0 + ) + assert json.loads(outputs.pop())["repo"] == "cassandra" + + +def test_main_handles_http_and_policy_failures(monkeypatch, capsys): + module = _load() + client = types.ModuleType("scm_broker_client") + + def http_failure(_path): + headers = Message() + raise urllib.error.HTTPError("url", 503, "unavailable", headers, None) + + client.read = http_failure + client.create_draft = lambda *_a, **_k: b"{}" + monkeypatch.setitem(sys.modules, "scm_broker_client", client) + assert module.main(["read", "/api/v1/repos/atlas/cassandra"]) == 1 + assert "HTTP 503" in capsys.readouterr().err + client.read = lambda _path: (_ for _ in ()).throw(module.PolicyError("rejected")) + assert module.main(["read", "/not-allowed"]) == 1 + assert "no credential" in capsys.readouterr().err + + +@pytest.mark.parametrize( + ("function", "value"), + [ + ("_validate_repo", "."), + ("_validate_repo", ".."), + ("_validate_sha", "short"), + ("_validate_pr_number", True), + ("_validate_pr_number", 0), + ("_validate_pr_number", 2_147_483_648), + ], +) +def test_policy_scalar_validators_reject_ambiguous_values(function, value): + module = _policy("gitea_policy_scalar") + with pytest.raises(module.PolicyError): + getattr(module, function)(value) + + +def test_ref_bounds_cover_utf8_controls_git_failure_and_success(monkeypatch): + module = _policy("gitea_policy_refs") + with pytest.raises(module.PolicyError, match="valid UTF-8"): + module._validate_ref_bounds("\ud800", "head") + with pytest.raises(module.PolicyError, match="UTF-8"): + module._validate_ref_bounds("🧪" * 64, "head") + with pytest.raises(module.PolicyError, match="safe same"): + module._validate_ref("bad\nref", "head") + monkeypatch.setattr( + module.subprocess, + "run", + lambda *_a, **_k: subprocess.CompletedProcess([], 1), + ) + with pytest.raises(module.PolicyError, match="same-repository"): + module._validate_ref("invalid-ref", "head") + monkeypatch.setattr( + module.subprocess, + "run", + lambda *_a, **_k: subprocess.CompletedProcess([], 0), + ) + assert module._validate_ref("feature/valid", "head") == "feature/valid" + + +@pytest.mark.parametrize( + ("value", "required", "match"), + [ + (None, False, "must be text"), + (" ", True, "must not be empty"), + ("\ud800", False, "valid UTF-8"), + ("x\x00y", False, "safe request limit"), + ], +) +def test_text_validation_rejects_nontext_empty_invalid_and_controls( + value, required, match +): + module = _policy("gitea_policy_text") + with pytest.raises(module.PolicyError, match=match): + module._validate_text(value, "field", 20, 20, required=required) + + +@pytest.mark.parametrize( + ("key", "expected"), + [ + ("password", True), + ("accountKey", True), + ("registry-key", True), + ("clientEmail", True), + ("clientId", True), + ("accessId", True), + ("connectionString", True), + ("dockerConfigJson", True), + ("release_note", False), + ], +) +def test_sensitive_key_semantics_cover_generic_forms(key, expected): + module = _policy("gitea_policy_keys") + assert module._is_sensitive_key(key) is expected + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("", False), + ('"synthetic"', True), + ("{encoded}", True), + ("Bearer value-12345678", True), + ("ghp_" + "a" * 24, True), + ("https://example.invalid/value", True), + ("name@example.invalid", True), + ("${RUNTIME_VALUE}", True), + ("AbCdEf0123456789", True), + ("reject empty values", False), + ("ordinary", True), + ], +) +def test_assignment_value_shape_distinguishes_prose_from_credentials(value, expected): + module = _policy("gitea_policy_assignment_values") + assert module._looks_sensitive_assignment_value(value) is expected + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("single", False), + ("add regression coverage", True), + ("add regression_coverage", False), + ("ordinary engineering prose", False), + ], +) +def test_prose_classifier_covers_length_punctuation_and_lead_words(value, expected): + module = _policy("gitea_policy_prose") + assert module._looks_like_prose(value) is expected + + +def test_structured_json_walk_covers_lists_nonstring_keys_and_limits(): + module = _policy("gitea_policy_json_walk") + assert module._json_value_has_sensitive_assignment({1: "ignored"}) is False + assert module._json_value_has_sensitive_assignment([{"release": "safe"}]) is False + assert ( + module._json_value_has_sensitive_assignment({"client_secret": "value"}) is True + ) + assert ( + module._json_value_has_sensitive_assignment( + {"outer": {"client_secret": "value"}} + ) + is True + ) + assert ( + module._json_value_has_sensitive_assignment({"type": "service-account"}) is True + ) + with pytest.raises(module.PolicyError, match="scan limit"): + module._json_value_has_sensitive_assignment([], depth=33) + with pytest.raises(module.PolicyError, match="scan limit"): + module._json_value_has_sensitive_assignment([], nodes=[2048]) + + +def test_json_key_decoder_and_embedded_document_scanner_cover_failures(): + module = _policy("gitea_policy_json_decoder") + assert module._decode_json_key("client\\u005fsecret") == ("client_secret", True) + decoded, valid = module._decode_json_key("client\\qsecret") + assert valid is False and "client" in decoded + assert list(module._decoded_json_documents("prose only")) == [] + assert list(module._decoded_json_documents("x {bad y [1, 2]")) == [[1, 2]] + with pytest.raises(module.PolicyError, match="scan limit"): + list(module._decoded_json_documents("{" * 33)) + assert ( + module._has_structured_sensitive_assignment('"client\\qsecret": value') is True + ) + assert ( + module._has_structured_sensitive_assignment("type:\n service-account") is True + ) + + +def test_high_entropy_detector_covers_long_mixed_and_low_entropy_values(): + module = _policy("gitea_policy_entropy") + assert module._has_high_entropy_token("A1_" * 100) is True + assert ( + module._has_high_entropy_token( + "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789+/_=" + ) + is True + ) + assert module._has_high_entropy_token("a" * 64) is False + assert module._has_high_entropy_token("short prose") is False + + +@pytest.mark.parametrize( + "query", + [ + "bad", + "a=1&b=2&c=3&d=4", + "page=1&page=2", + "=value", + "unexpected=1", + "page=abc", + "page=01", + "page=10001", + "limit=51", + "state=merged", + ], +) +def test_query_validator_covers_each_rejection_class(query): + module = _policy("gitea_policy_query") + target = urllib.parse.urlsplit("/api/v1/repos/atlas/cassandra/pulls?" + query) + with pytest.raises(module.PolicyError): + module._validate_query(target, {"page", "limit", "state"}) + + +def test_draft_title_rejects_prefix_without_content(): + module = _policy("gitea_policy_empty_draft") + with pytest.raises(module.PolicyError, match="after the draft prefix"): + module._draft_title("WIP:") + + +def test_query_validator_rejects_noncanonical_raw_form(): + module = _policy("gitea_policy_raw_query") + target = urllib.parse.SplitResult("", "", "/api/v1/repos/atlas/cassandra", "%", "") + with pytest.raises(module.PolicyError, match="canonical ASCII"): + module._validate_query(target, set()) diff --git a/testing/tests/test_hermes_gitea_pr_client.py b/testing/tests/test_hermes_gitea_pr_client.py index 447f3067..ae419e10 100644 --- a/testing/tests/test_hermes_gitea_pr_client.py +++ b/testing/tests/test_hermes_gitea_pr_client.py @@ -1,75 +1,12 @@ -"""Adversarial contracts for the Atlas-only draft pull-request client.""" +"""Route and request-boundary contracts for the Atlas Gitea client.""" from __future__ import annotations -import copy -import importlib.util -import json -import sys import urllib.request -from pathlib import Path import pytest -ROOT = Path(__file__).parents[2] -CLIENT_PATH = ROOT / "services/hermes/scripts/gitea_api.py" -HEAD_SHA = "465cf9146b05c174a2a8d310aff6c64be58277b6" - - -def _load(): - spec = importlib.util.spec_from_file_location("safe_gitea_api", CLIENT_PATH) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - sys.modules[spec.name] = module - spec.loader.exec_module(module) - return module - - -def _draft_payload(**updates): - payload = { - "base": "main", - "body": "Review evidence", - "head": "hermes/review-fix", - "title": "WIP: Repair review findings", - } - payload.update(updates) - return payload - - -def _draft_response(**updates): - response = { - "number": 3, - "state": "open", - "draft": True, - "merged": False, - "html_url": "https://scm.bstein.dev/atlas/cassandra/pulls/3", - "url": "https://scm.bstein.dev/atlas/cassandra/pulls/3", - "title": "WIP: Focused fix", - "body": "Review evidence", - "base": {"ref": "main", "repo": {"full_name": "atlas/cassandra"}}, - "head": { - "ref": "hermes/fix", - "sha": HEAD_SHA, - "repo": {"full_name": "atlas/cassandra"}, - }, - } - response.update(updates) - return response - - -class Response: - def __init__(self, body: object): - self.body = body if isinstance(body, bytes) else json.dumps(body).encode() - - def __enter__(self): - return self - - def __exit__(self, *_args): - return False - - def read(self, limit=-1): - return self.body if limit < 0 else self.body[:limit] - +from testing.tests.test_hermes_gitea_support import HEAD_SHA, Response, _load def test_redirect_handler_rejects_cross_origin_with_sentinel_authorization(): client = _load() @@ -101,6 +38,10 @@ def test_redirect_handler_rejects_cross_origin_with_sentinel_authorization(): ("https://evil.example", "/api/v1/repos/atlas/cassandra"), ("http://scm.bstein.dev", "/api/v1/repos/atlas/cassandra"), ("https://scm.bstein.dev:443", "/api/v1/repos/atlas/cassandra"), + ("https://scm.bstein.dev/", "/api/v1/repos/atlas/cassandra"), + ("HTTPS://scm.bstein.dev", "/api/v1/repos/atlas/cassandra"), + ("https://SCM.bstein.dev", "/api/v1/repos/atlas/cassandra"), + ("https://user@scm.bstein.dev", "/api/v1/repos/atlas/cassandra"), ("https://scm.bstein.dev", "https://evil.example/api/v1/repos/atlas/cassandra"), ("https://scm.bstein.dev", "/api/v1/repos/evil/cassandra"), ("https://scm.bstein.dev", "/api/v1/repos/%61tlas/cassandra"), @@ -114,6 +55,72 @@ def test_host_owner_and_path_escape_attempts_are_rejected(base_url: str, path: s client.build_request("GET", path, base_url=base_url, token="secret") +@pytest.mark.parametrize( + "path", + [ + "/api/v1/repos/atlas/cassandra/pulls/1\nHost: evil.example", + "/api/v1/repos/atlas/cassandra/pulls/1\r\nX-Test: value", + "/api/v1/repos/atlas/cassandra/pulls/1\tignored", + "/api/v1/repos/atlas/cassandra/pulls/1\x00ignored", + "/api/v1/repos/atlas/cassandra/pulls/1\x1fignored", + "/api/v1/repos/atlas/cassandra/pulls/1\x7fignored", + "/api/v1/repos/atlas/cassandra\\pulls\\1", + "/api/v1/repos/atlas/cassandra/pulls/%31", + "/api/v1/repos/atlas/cassandra/pulls/１", + " https://scm.bstein.dev/api/v1/repos/atlas/cassandra", + "https://scm.bstein.dev/api/v1/repos/atlas/cassandra", + ], +) +def test_raw_noncanonical_target_is_rejected_before_urlsplit_and_opener( + path: str, monkeypatch +): + client = _load() + split_called = False + opener_called = False + original_urlsplit = client.urllib.parse.urlsplit + + def urlsplit(*args, **kwargs): + nonlocal split_called + split_called = True + return original_urlsplit(*args, **kwargs) + + def opener(*_args, **_kwargs): + nonlocal opener_called + opener_called = True + return Response(b"{}") + + monkeypatch.setattr(client.urllib.parse, "urlsplit", urlsplit) + with pytest.raises(client.PolicyError): + client.read(path, token="runtime", opener=opener) + assert split_called is False + assert opener_called is False + + +@pytest.mark.parametrize( + "path", + [ + "/api/v1/repos/atlas/cassandra/pulls/1?", + "//scm.bstein.dev/api/v1/repos/atlas/cassandra", + "/api/v1/repos/atlas/cassandra/./pulls/1", + "/api/v1/repos/atlas/cassandra/../admin", + "/api/v1/repos/atlas/cassandra//pulls/1", + "/api/v1/repos/atlas/cassandra/pulls/1?limit=01", + ], +) +def test_noncanonical_round_trip_or_segments_never_reach_opener(path: str): + client = _load() + opener_called = False + + def opener(*_args, **_kwargs): + nonlocal opener_called + opener_called = True + return Response(b"{}") + + with pytest.raises(client.PolicyError): + client.read(path, token="runtime", opener=opener) + assert opener_called is False + + @pytest.mark.parametrize( "path", [ @@ -180,226 +187,103 @@ def test_read_query_is_bounded(path: str): @pytest.mark.parametrize( - ("method", "path", "data"), + "query", [ - ("DELETE", "/api/v1/repos/atlas/cassandra/pulls/4", None), - ("POST", "/api/v1/repos/atlas/cassandra/pulls/4/merge", {}), - ( - "POST", - "/api/v1/repos/atlas/cassandra/pulls/4/reviews", - {"event": "APPROVED"}, - ), - ("PATCH", "/api/v1/repos/atlas/cassandra/pulls/4", {"title": "WIP: x"}), - ("PUT", "/api/v1/repos/atlas/cassandra/branches/main", {}), + "page=0", + "page=10001", + "page=01", + "limit=0", + "limit=51", + "limit=01", + "page=" + "9" * 4000, + "page=０", + "page=%EF%BC%90", + "state=%6fpen", + "p%61ge=1", + "page=1&page=2", + "page=1&" + "x" * 17 + "=1", + "state=" + "x" * 17, + "page=1&limit=2&state=open&extra=3", ], ) -def test_merge_approve_close_delete_update_and_other_mutations_are_rejected( - method: str, path: str, data: object -): +def test_read_query_requires_canonical_bounded_ascii(query: str): client = _load() with pytest.raises(client.PolicyError): - client.build_request( - method, path, base_url=client.CANONICAL_BASE_URL, token="secret", data=data + client.authorize_request( + "GET", f"/api/v1/repos/atlas/cassandra/pulls?{query}", None ) @pytest.mark.parametrize( - "ref", + "suffix", [ - "foo/.bar", - "foo/bar.lock/baz", - "foo..bar", - "foo@{bar", - "foo//bar", - "-danger", - "danger.", - "danger~one", - "danger^one", - "danger:one", - "danger one", + "pulls/{number}", + "pulls/{number}.patch", + "pulls/{number}.diff", + "pulls/{number}/commits", + "pulls/{number}/files", + "commits/{number}/status", + "commits/{number}/statuses", + "statuses/{number}", + "branches/{number}", ], ) -def test_complete_git_ref_validation_rejects_invalid_names(ref: str): - client = _load() - - with pytest.raises(client.PolicyError): - client._validate_ref(ref, "head") - - -def test_git_ref_validation_uses_fixed_trusted_binary(): - client = _load() - - assert client.GIT_BIN == "/usr/bin/git" - assert client._validate_ref("hermes/valid-fix", "head") == "hermes/valid-fix" - - -def test_create_forces_draft_title_and_same_repository_branch_names(): - client = _load() - assert ( - client.authorize_request( - "POST", "/api/v1/repos/atlas/cassandra/pulls", _draft_payload() - ) - == "create-draft" - ) - with pytest.raises(client.PolicyError, match="draft-title prefix"): - client.authorize_request( - "POST", - "/api/v1/repos/atlas/cassandra/pulls", - _draft_payload(title="Not a draft"), - ) - with pytest.raises(client.PolicyError): - client.authorize_request( - "POST", - "/api/v1/repos/atlas/cassandra/pulls", - _draft_payload(head="someone:branch"), - ) - - -def test_runtime_token_is_only_an_authorization_header(): - client = _load() - request = client.build_request( - "POST", - "/api/v1/repos/atlas/cassandra/pulls", - base_url=client.CANONICAL_BASE_URL, - token="do-not-leak", - data=_draft_payload(), - ) - - assert "do-not-leak" not in request.full_url - assert b"do-not-leak" not in request.data - assert request.get_header("Authorization") == "token do-not-leak" - - -@pytest.mark.parametrize( - "body", - [ - "pass" + "word=not-a-real-credential", - "Authorization: " + "Bearer not-a-real-credential-value", - "token: " + "ghp_" + "notarealcredentialvalue123456", - "-----BEGIN OPENSSH " + "PRIVATE KEY-----", - "eyJnotarealheader." + "notarealpayloadvalue." + "notarealsignature", - ], -) -def test_body_rejects_common_credential_shapes(body: str): - client = _load() - - with pytest.raises(client.PolicyError, match="credential material"): - client._validate_body(body) - - -def test_create_rejects_exact_runtime_token_before_network(): +def test_oversized_numeric_or_captured_path_never_reaches_opener(suffix: str): client = _load() called = False def opener(*_args, **_kwargs): nonlocal called called = True - return Response(_draft_response()) + return Response(b"{}") - with pytest.raises(client.PolicyError, match="runtime credential"): - client.create_draft( - "cassandra", - base="main", - head="hermes/fix", - head_sha=HEAD_SHA, - title="Focused fix", - body="accidental runtime-sentinel value", - token="runtime-sentinel", + path = "/api/v1/repos/atlas/cassandra/" + suffix.format(number="9" * 4000) + with pytest.raises(client.PolicyError): + client.read(path, token="runtime", opener=opener) + assert called is False + + +@pytest.mark.parametrize( + "number", + ["0", "01", "2147483648", "１２", "%31", "12345678901"], +) +@pytest.mark.parametrize("tail", ["", ".patch", ".diff", "/commits", "/files"]) +def test_noncanonical_or_out_of_range_pr_number_never_reaches_opener( + number: str, tail: str +): + client = _load() + called = False + + def opener(*_args, **_kwargs): + nonlocal called + called = True + return Response(b"{}") + + with pytest.raises(client.PolicyError): + client.read( + f"/api/v1/repos/atlas/cassandra/pulls/{number}{tail}", + token="runtime", opener=opener, ) assert called is False -def test_create_verifies_every_server_postcondition(): +def test_maximum_bounded_pr_number_is_readable(): client = _load() - calls = [] + called = False - def opener(request, timeout): - calls.append((request, timeout)) - return Response(_draft_response()) + def opener(*_args, **_kwargs): + nonlocal called + called = True + return Response(b"{}") - result = client.create_draft( - "cassandra", - base="main", - head="hermes/fix", - head_sha=HEAD_SHA, - title="Focused fix", - body="Review evidence", - token="runtime", - opener=opener, - ) - - assert json.loads(result) == _draft_response() - payload = json.loads(calls[0][0].data) - assert payload == { - "base": "main", - "body": "Review evidence", - "head": "hermes/fix", - "title": "WIP: Focused fix", - } - assert calls[0][1] == 30 - - -def test_create_postcondition_rejects_every_material_mismatch(): - client = _load() - mutations = [ - ("number", 0), - ("state", "closed"), - ("draft", False), - ("merged", True), - ("html_url", "https://evil.example/pulls/3"), - ("url", "https://evil.example/api/pulls/3"), - ("title", "Focused fix"), - ("body", "different"), - ] - documents = [] - for key, value in mutations: - document = _draft_response() - document[key] = value - documents.append(document) - for path, value in [ - (("base", "ref"), "master"), - (("base", "repo", "full_name"), "evil/cassandra"), - (("head", "ref"), "other"), - (("head", "sha"), "0" * 40), - (("head", "repo", "full_name"), "evil/cassandra"), - ]: - document = copy.deepcopy(_draft_response()) - target = document - for key in path[:-1]: - target = target[key] - target[path[-1]] = value - documents.append(document) - - for document in documents: - with pytest.raises(client.PolicyError): - client._require_create_response( - json.dumps(document).encode(), - repo="cassandra", - base="main", - head="hermes/fix", - head_sha=HEAD_SHA, - title="WIP: Focused fix", - body="Review evidence", - ) - - -def test_read_response_is_bounded(): - client = _load() - - with pytest.raises(client.PolicyError, match="safe size limit"): + assert ( client.read( - "/api/v1/repos/atlas/cassandra", + "/api/v1/repos/atlas/cassandra/pulls/2147483647", token="runtime", - opener=lambda *_a, **_k: Response(b"x" * (client.MAX_RESPONSE_BYTES + 1)), + opener=opener, ) - - -def test_output_redaction_covers_exact_token_and_authorization_header(): - client = _load() - raw = b'{"message":"do-not-leak","debug":"Authorization: token do-not-leak"}' - redacted = client.redact_bytes(raw, "do-not-leak") - - assert b"do-not-leak" not in redacted - assert redacted.count(b"[REDACTED]") >= 1 + == b"{}" + ) + assert called is True diff --git a/testing/tests/test_hermes_gitea_pr_creation.py b/testing/tests/test_hermes_gitea_pr_creation.py new file mode 100644 index 00000000..ad2967ed --- /dev/null +++ b/testing/tests/test_hermes_gitea_pr_creation.py @@ -0,0 +1,203 @@ +"""Draft-creation input contracts for the Atlas Gitea client.""" + +from __future__ import annotations + +import pytest + +from testing.tests.test_hermes_gitea_support import ( + HEAD_SHA, + Response, + _draft_payload, + _draft_response, + _load, +) + + +@pytest.mark.parametrize( + ("method", "path", "data"), + [ + ("DELETE", "/api/v1/repos/atlas/cassandra/pulls/4", None), + ("POST", "/api/v1/repos/atlas/cassandra/pulls/4/merge", {}), + ( + "POST", + "/api/v1/repos/atlas/cassandra/pulls/4/reviews", + {"event": "APPROVED"}, + ), + ("PATCH", "/api/v1/repos/atlas/cassandra/pulls/4", {"title": "WIP: x"}), + ("PUT", "/api/v1/repos/atlas/cassandra/branches/main", {}), + ], +) +def test_merge_approve_close_delete_update_and_other_mutations_are_rejected( + method: str, path: str, data: object +): + client = _load() + + with pytest.raises(client.PolicyError): + client.build_request( + method, path, base_url=client.CANONICAL_BASE_URL, token="secret", data=data + ) + + +@pytest.mark.parametrize( + "ref", + [ + "foo/.bar", + "foo/bar.lock/baz", + "foo..bar", + "foo@{bar", + "foo//bar", + "-danger", + "danger.", + "danger~one", + "danger^one", + "danger:one", + "danger one", + ], +) +def test_complete_git_ref_validation_rejects_invalid_names(ref: str): + client = _load() + + with pytest.raises(client.PolicyError): + client._validate_ref(ref, "head") + + +def test_git_ref_validation_uses_fixed_trusted_binary(): + client = _load() + + assert client._validate_ref.__globals__["GIT_BIN"] == "/usr/bin/git" + assert client._validate_ref("hermes/valid-fix", "head") == "hermes/valid-fix" + + +@pytest.mark.parametrize("field", ["base", "head"]) +@pytest.mark.parametrize("oversized", ["r" * 100_000, "🧪" * 128]) +def test_oversized_ref_never_invokes_git_request_or_opener( + field: str, oversized: str, monkeypatch +): + client = _load() + git_called = False + request_built = False + opener_called = False + original_build_request = client.build_request + + def git_run(*_args, **_kwargs): + nonlocal git_called + git_called = True + raise AssertionError("Git must not receive an oversized ref") + + def build_request(*args, **kwargs): + nonlocal request_built + request_built = True + return original_build_request(*args, **kwargs) + + def opener(*_args, **_kwargs): + nonlocal opener_called + opener_called = True + return Response(_draft_response()) + + monkeypatch.setattr(client._validate_ref.__globals__["subprocess"], "run", git_run) + client.build_request = build_request + refs = {"base": "main", "head": "hermes/fix"} + refs[field] = oversized + with pytest.raises(client.PolicyError, match="branch-name limit"): + client.create_draft( + "cassandra", + base=refs["base"], + head=refs["head"], + head_sha=HEAD_SHA, + title="Focused fix", + body="Review evidence", + token="runtime", + opener=opener, + ) + assert git_called is False + assert request_built is False + assert opener_called is False + + +@pytest.mark.parametrize( + ("field", "oversized"), + [("repo", "r" * 101), ("title", "🧪" * 200), ("body", "🧪" * 9_000)], +) +def test_other_text_bounds_fail_before_git_request_or_opener( + field: str, oversized: str, monkeypatch +): + client = _load() + git_called = False + request_built = False + opener_called = False + original_build_request = client.build_request + + def git_run(*_args, **_kwargs): + nonlocal git_called + git_called = True + raise AssertionError("Git must not run before cheap input bounds") + + def build_request(*args, **kwargs): + nonlocal request_built + request_built = True + return original_build_request(*args, **kwargs) + + def opener(*_args, **_kwargs): + nonlocal opener_called + opener_called = True + return Response(_draft_response()) + + monkeypatch.setattr(client._validate_ref.__globals__["subprocess"], "run", git_run) + client.build_request = build_request + values = { + "repo": "cassandra", + "title": "Focused fix", + "body": "Review evidence", + } + values[field] = oversized + with pytest.raises(client.PolicyError): + client.create_draft( + values["repo"], + base="main", + head="hermes/fix", + head_sha=HEAD_SHA, + title=values["title"], + body=values["body"], + token="runtime", + opener=opener, + ) + assert git_called is False + assert request_built is False + assert opener_called is False + + +def test_create_forces_draft_title_and_same_repository_branch_names(): + client = _load() + assert ( + client.authorize_request( + "POST", "/api/v1/repos/atlas/cassandra/pulls", _draft_payload() + ) + == "create-draft" + ) + with pytest.raises(client.PolicyError, match="draft-title prefix"): + client.authorize_request( + "POST", + "/api/v1/repos/atlas/cassandra/pulls", + _draft_payload(title="Not a draft"), + ) + with pytest.raises(client.PolicyError): + client.authorize_request( + "POST", + "/api/v1/repos/atlas/cassandra/pulls", + _draft_payload(head="someone:branch"), + ) + + +def test_runtime_token_is_only_an_authorization_header(): + client = _load() + request = client.build_request( + "POST", + "/api/v1/repos/atlas/cassandra/pulls", + base_url=client.CANONICAL_BASE_URL, + token="do-not-leak", + data=_draft_payload(), + ) + + assert "do-not-leak" not in request.full_url + assert b"do-not-leak" not in request.data + assert request.get_header("Authorization") == "token do-not-leak" diff --git a/testing/tests/test_hermes_gitea_pr_dlp.py b/testing/tests/test_hermes_gitea_pr_dlp.py new file mode 100644 index 00000000..4a2b1d27 --- /dev/null +++ b/testing/tests/test_hermes_gitea_pr_dlp.py @@ -0,0 +1,317 @@ +"""Credential-loss-prevention contracts for Atlas draft creation.""" + +from __future__ import annotations + +import pytest + +from testing.tests.test_hermes_gitea_support import ( + HEAD_SHA, + Response, + _draft_response, + _load, +) + + +@pytest.mark.parametrize( + "sensitive", + [ + "client_" + "secret=not-a-real-value", + '{"client_' + 'secret":\n"synthetic-value"}', + '{"client_' + 'secret"\n:\n"synthetic-value"}', + '{"client\\u005f' + 'secret":"synthetic-value"}', + "client_" + "secret:\n synthetic-value", + '{"client\\q' + 'secret":"synthetic-value"}', + '{"client\n' + 'secret":"synthetic-value"}', + '"client_' + 'secret"\x0b:\n"synthetic-value"', + "ACCESS_" + "TOKEN = 'not-a-real-value'", + '{"refresh_' + 'token": "not-a-real-value"}', + "private_" + "key: not-a-real-value", + "AWS_SECRET_ACCESS_" + "KEY=not-a-real-value", + "aws_access_key_" + "id: not-a-real-value", + "AWS_SESSION_" + "TOKEN = 'not-a-real-value'", + '{"AccessKey' + 'Id":"not-a-real-value"}', + '{"SecretAccess' + 'Key":"not-a-real-value"}', + '{"Session' + 'Token":"not-a-real-value"}', + "aws-security-" + "token: not-a-real-value", + "Account" + "Key=not-a-real-value", + "SharedAccess" + "Signature: not-a-real-value", + "AZURE_STORAGE_CONNECTION_" + "STRING='not-a-real-value'", + "DefaultEndpointsProtocol=https;AccountName=fake;Account" + + "Key=not-a-real-value;EndpointSuffix=example", + "?sv=2024-11-04&ss=b&srt=sco&sp=rwdlac&se=2099-01-01&sig=" + "not-a-real-value", + "DOCKER_AUTH_" + "CONFIG='not-a-real-value'", + '{"auths":{"registry.example":{"auth":"bm90LXJlYWw="}}}', + '{"identity' + 'token":"not-a-real-value"}', + '{"type":"service_' + 'account","client_email":"fake@example.test"}', + '{"private_key_' + 'id":"not-a-real-value"}', + '{"client_' + 'email":"fake@example.test"}', + "GOOGLE_CREDENTIALS" + "=not-a-real-value", + "personal_access_" + "token: not-a-real-value", + "GITEA_" + "TOKEN=not-a-real-value", + "gitlab-token" + ": not-a-real-value", + "pat" + "=not-a-real-value", + "Authorization: " + "Bearer not-a-real-credential-value", + "authorization = " + '"Basic not-a-real-credential-value"', + "Bearer" + "=not-a-real-credential-value", + "Basic" + ": not-a-real-credential-value", + "pass" + "word=not-a-real-credential", + "ghp_" + "notarealcredentialvalue123456", + "github_pat_" + "notarealcredentialvalue123456", + "glpat-" + "notarealcredentialvalue123456", + "xoxb-" + "not-a-real-credential-value-123456", + "sk-ant-" + "notarealcredentialvalue123456", + "sk-proj-" + "notarealcredentialvalue123456", + "sk_live_" + "notarealcredentialvalue123456", + "ya29." + "notarealcredentialvalue123456", + "gta_" + "notarealcredentialvalue123456", + "whsec_" + "notarealcredentialvalue123456", + "npm_" + "notarealcredentialvalue123456", + "pypi-" + "notarealcredentialvalue123456789012345", + "hf_" + "notarealcredentialvalue123456", + "SG." + "notarealvalue1234" + ".notarealcredentialvalue123456", + "SK" + "a" * 32, + "https://hooks.slack.com/services/" + "T000/B000/notarealvalue123456", + "https://discord.com/api/webhooks/123456789/" + "notarealcredentialvalue123456", + "https://fake.webhook.office.com/" + "notarealcredentialvalue123456", + "webhook_" + "url=https://example.test/not-real", + "FutureCloudSigning" + "Credential=not-a-real-value", + "future-client-signing-" + "key: not-a-real-value", + "future_client_signing_" + "key='not-a-real-value'", + '{"serviceAccountPrivate' + 'Key":"not-a-real-value"}', + "CONTAINER_REGISTRY_" + "CREDENTIAL=not-a-real-value", + "someWebhookSigning" + "Secret: not-a-real-value", + "client" + "Key=not-a-real-value", + "session" + "Key: not-a-real-value", + "access" + "Id=not-a-real-value", + "credentials" + ": {user: fake}", + "private" + "Key: |", + "nuget_api_" + "key=not-a-real-value", + "oy2" + "a" * 44, + "sk_test_" + "notarealcredentialvalue123456", + "A1b2C3d4E5f6G7h8I9j0K_l-M+n/O=pQ2rS3tU4vW5xY6zZ7aB8cC9d", + "AKIA" + "A" * 16, + "AIza" + "a" * 35, + "-----BEGIN OPENSSH " + "PRIVATE KEY-----", + "ssh-ed25519 " + "bm90YXJlYWxjcmVkZW50aWFsdmFsdWU=", + "eyJnotarealheader." + "notarealpayloadvalue." + "notarealsignature", + ], +) +@pytest.mark.parametrize("field", ["title", "body"]) +def test_create_rejects_expanded_credential_shapes_before_network( + sensitive: str, field: str +): + client = _load() + called = False + request_built = False + + original_build_request = client.build_request + + def build_request(*args, **kwargs): + nonlocal request_built + request_built = True + return original_build_request(*args, **kwargs) + + def opener(*_args, **_kwargs): + nonlocal called + called = True + return Response(_draft_response()) + + client.build_request = build_request + values = {"title": "Focused fix", "body": "Review evidence"} + values[field] = sensitive + + with pytest.raises(client.PolicyError, match="credential material"): + client.create_draft( + "cassandra", + base="main", + head="hermes/fix", + head_sha=HEAD_SHA, + title=values["title"], + body=values["body"], + token="runtime-sentinel", + opener=opener, + ) + assert request_built is False + assert called is False + + +def test_very_long_compact_body_is_rejected_before_request_or_network(): + client = _load() + request_built = False + opener_called = False + original_build_request = client.build_request + + def build_request(*args, **kwargs): + nonlocal request_built + request_built = True + return original_build_request(*args, **kwargs) + + def opener(*_args, **_kwargs): + nonlocal opener_called + opener_called = True + return Response(_draft_response()) + + client.build_request = build_request + with pytest.raises(client.PolicyError, match="credential material"): + client.create_draft( + "cassandra", + base="main", + head="hermes/fix", + head_sha=HEAD_SHA, + title="Focused fix", + body="a" * 300, + token="runtime-sentinel", + opener=opener, + ) + assert request_built is False + assert opener_called is False + + +@pytest.mark.parametrize( + "safe_text", + [ + "AWS_SECRET_ACCESS_KEY is injected at runtime", + "Token: reject empty values", + "Authorization = preserve header behavior", + "Password: add regression", + "Review the Authorization header behavior", + "Bearer authentication is required for this route", + "Document Docker auths payload rejection", + "The client_email field belongs to service accounts", + "AccountKey assignments must be blocked", + "This patch changes token validation without including a value", + "FutureCloudSigningCredential handling needs a regression test", + "The clientKey name is documented without an assigned value", + "A SHA-256 digest 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef is evidence", + ], +) +def test_credential_policy_keeps_normal_engineering_prose_usable(safe_text: str): + client = _load() + + assert client._validate_body(safe_text) == safe_text + assert client._draft_title(safe_text) == f"WIP: {safe_text}" + + +@pytest.mark.parametrize("field", ["title", "body"]) +def test_create_rejects_exact_runtime_token_before_network(field: str): + client = _load() + called = False + request_built = False + + original_build_request = client.build_request + + def build_request(*args, **kwargs): + nonlocal request_built + request_built = True + return original_build_request(*args, **kwargs) + + def opener(*_args, **_kwargs): + nonlocal called + called = True + return Response(_draft_response()) + + client.build_request = build_request + runtime_token = "exact-random-runtime-sentinel-7b73ac61" + values = {"title": "Focused fix", "body": "Review evidence"} + values[field] = f"Accidental {runtime_token} value" + with pytest.raises(client.PolicyError, match="runtime credential"): + client.create_draft( + "cassandra", + base="main", + head="hermes/fix", + head_sha=HEAD_SHA, + title=values["title"], + body=values["body"], + token=runtime_token, + opener=opener, + ) + assert request_built is False + assert called is False + + +@pytest.mark.parametrize("field", ["repo", "base", "head", "title", "body"]) +def test_every_public_field_rejects_exact_runtime_token_before_git_or_network( + field: str, monkeypatch +): + client = _load() + git_called = False + request_built = False + opener_called = False + runtime_token = "runtime-sentinel" + values = { + "repo": "cassandra", + "base": "main", + "head": "hermes/fix", + "title": "Focused fix", + "body": "Review evidence", + } + values[field] = runtime_token + + def git_run(*_args, **_kwargs): + nonlocal git_called + git_called = True + raise AssertionError("Git must not run for a credential-bearing field") + + def build_request(*_args, **_kwargs): + nonlocal request_built + request_built = True + raise AssertionError("a request must not be built") + + def opener(*_args, **_kwargs): + nonlocal opener_called + opener_called = True + raise AssertionError("the opener must not be called") + + monkeypatch.setattr(client._validate_ref.__globals__["subprocess"], "run", git_run) + monkeypatch.setattr(client, "build_request", build_request) + with pytest.raises(client.PolicyError, match="runtime credential"): + client.create_draft( + values["repo"], + base=values["base"], + head=values["head"], + head_sha=HEAD_SHA, + title=values["title"], + body=values["body"], + token=runtime_token, + opener=opener, + ) + assert git_called is False + assert request_built is False + assert opener_called is False + + +@pytest.mark.parametrize( + "sensitive", + [ + " ".join(["{}"] * 32) + ' {"client_secret":"synthetic-value"}', + "prefix_" + "ghp_" + "notarealcredentialvalue123456_suffix", + "client_secret: correct horse battery staple", + "'client_secret':\n synthetic-value", + ], +) +@pytest.mark.parametrize("field", ["title", "body"]) +def test_structured_scanner_closes_bounded_and_wrapped_token_bypasses( + sensitive: str, field: str +): + client = _load() + values = {"title": "Focused fix", "body": "Review evidence"} + values[field] = sensitive + called = False + + def opener(*_args, **_kwargs): + nonlocal called + called = True + return Response(_draft_response()) + + with pytest.raises(client.PolicyError, match="credential material"): + client.create_draft( + "cassandra", + base="main", + head="hermes/fix", + head_sha=HEAD_SHA, + title=values["title"], + body=values["body"], + token="runtime-sentinel", + opener=opener, + ) + assert called is False diff --git a/testing/tests/test_hermes_gitea_pr_integration.py b/testing/tests/test_hermes_gitea_pr_integration.py index d3b47eef..ee482cfe 100644 --- a/testing/tests/test_hermes_gitea_pr_integration.py +++ b/testing/tests/test_hermes_gitea_pr_integration.py @@ -13,8 +13,10 @@ import pytest import yaml ROOT = Path(__file__).parents[2] -CLIENT_PATH = ROOT / "services/hermes/scripts/gitea_api.py" +CLIENT_PATH = ROOT / "services/hermes/scm-common/scripts/gitea_api.py" HEAD_SHA = "465cf9146b05c174a2a8d310aff6c64be58277b6" +if str(CLIENT_PATH.parent) not in sys.path: + sys.path.insert(0, str(CLIENT_PATH.parent)) def _load(): @@ -96,12 +98,12 @@ def test_http_error_path_redacts_token(monkeypatch, capsys): assert client.main(["read", "/api/v1/repos/atlas/cassandra"]) == 1 captured = capsys.readouterr() assert "do-not-leak" not in captured.err - assert "HTTP 403" in captured.err + assert "credential was disclosed" in captured.err -def test_flux_manifest_projects_runtime_vault_token_and_skill_only(): +def test_flux_manifest_isolates_vault_token_in_separate_broker_only(): client_source = CLIENT_PATH.read_text(encoding="utf-8") - assert "/runtime-access/gitea-token" in client_source + assert "scm_broker_client" in client_source assert "GITEA_TOKEN" not in client_source deployment = yaml.safe_load( @@ -109,9 +111,7 @@ def test_flux_manifest_projects_runtime_vault_token_and_skill_only(): ) template = deployment["spec"]["template"] annotations = template["metadata"]["annotations"] - assert annotations["vault.hashicorp.com/agent-inject-secret-gitea-token"] == ( - "kv/data/atlas/hermes/developer-gitea" - ) + assert not any("gitea" in key.lower() for key in annotations) runtime = next( volume for volume in template["spec"]["volumes"] @@ -136,6 +136,17 @@ def test_flux_manifest_projects_runtime_vault_token_and_skill_only(): kustomization = yaml.safe_load( (ROOT / "services/hermes/kustomization.yaml").read_text(encoding="utf-8") ) + common = yaml.safe_load( + (ROOT / "services/hermes/scm-common/kustomization.yaml").read_text( + encoding="utf-8" + ) + ) + boundary = common["configMapGenerator"][0] + assert boundary["name"] == "hermes-scm-boundary" + assert "gitea_api.py=scripts/gitea_api.py" in boundary["files"] + assert "gitea_api_policy.py=scripts/gitea_api_policy.py" in boundary["files"] + assert "scm_broker_client.py=scripts/scm_broker_client.py" in boundary["files"] + assert not any("gitea_askpass" in item for item in boundary["files"]) generator = next( item for item in kustomization["configMapGenerator"] @@ -145,3 +156,19 @@ def test_flux_manifest_projects_runtime_vault_token_and_skill_only(): "SKILL.md=skills/manage-atlas-pull-requests/SKILL.md", "openai.yaml=skills/manage-atlas-pull-requests/agents/openai.yaml", ] + + broker = yaml.safe_load( + (ROOT / "services/hermes-scm-broker/deployment.yaml").read_text( + encoding="utf-8" + ) + ) + assert broker["metadata"]["namespace"] == "hermes-scm" + pod = broker["spec"]["template"] + assert pod["spec"]["serviceAccountName"] == "hermes-scm-broker" + assert pod["metadata"]["annotations"][ + "vault.hashicorp.com/agent-inject-secret-gitea-token" + ] == "kv/data/atlas/hermes/developer-gitea" + assert not any( + volume.get("hostPath") or volume.get("persistentVolumeClaim") + for volume in pod["spec"]["volumes"] + ) diff --git a/testing/tests/test_hermes_gitea_pr_postconditions.py b/testing/tests/test_hermes_gitea_pr_postconditions.py new file mode 100644 index 00000000..c06a7784 --- /dev/null +++ b/testing/tests/test_hermes_gitea_pr_postconditions.py @@ -0,0 +1,199 @@ +"""Response and postcondition contracts for Atlas draft creation.""" + +from __future__ import annotations + +import base64 +import copy +import json + +import pytest + +from testing.tests.test_hermes_gitea_support import ( + HEAD_SHA, + Response, + _draft_response, + _load, +) + +@pytest.mark.parametrize("status", [200, 202, 204, 206]) +def test_create_accepts_only_http_201(status: int): + client = _load() + + with pytest.raises(client.PolicyError, match="unexpected HTTP status"): + client.create_draft( + "cassandra", + base="main", + head="hermes/fix", + head_sha=HEAD_SHA, + title="Focused fix", + body="Review evidence", + token="runtime", + opener=lambda *_a, **_k: Response(_draft_response(), status=status), + ) + + +@pytest.mark.parametrize("status", [201, 202, 204, 206]) +def test_read_accepts_only_http_200(status: int): + client = _load() + + with pytest.raises(client.PolicyError, match="unexpected HTTP status"): + client.read( + "/api/v1/repos/atlas/cassandra", + token="runtime", + opener=lambda *_a, **_k: Response(b"{}", status=status), + ) + + +def test_successful_create_validates_each_ref_once(monkeypatch): + client = _load() + calls = [] + + def git_run(command, **_kwargs): + calls.append(command) + return type("Result", (), {"returncode": 0})() + + monkeypatch.setattr(client._validate_ref.__globals__["subprocess"], "run", git_run) + client.create_draft( + "cassandra", + base="main", + head="hermes/fix", + head_sha=HEAD_SHA, + title="Focused fix", + body="Review evidence", + token="runtime", + opener=lambda *_a, **_k: Response(_draft_response()), + ) + + assert [command[-1] for command in calls] == [ + "refs/heads/main", + "refs/heads/hermes/fix", + ] + + +def test_create_verifies_every_server_postcondition(): + client = _load() + calls = [] + + def opener(request, timeout): + calls.append((request, timeout)) + return Response(_draft_response()) + + result = client.create_draft( + "cassandra", + base="main", + head="hermes/fix", + head_sha=HEAD_SHA, + title="Focused fix", + body="Review evidence", + token="runtime", + opener=opener, + ) + + assert json.loads(result) == _draft_response() + payload = json.loads(calls[0][0].data) + assert payload == { + "base": "main", + "body": "Review evidence", + "head": "hermes/fix", + "title": "WIP: Focused fix", + } + assert calls[0][1] == 30 + + +def test_create_postcondition_rejects_every_material_mismatch(): + client = _load() + mutations = [ + ("number", 0), + ("number", 2_147_483_648), + ("state", "closed"), + ("draft", False), + ("merged", True), + ("html_url", "https://evil.example/pulls/3"), + ("url", "https://evil.example/api/pulls/3"), + ("title", "Focused fix"), + ("body", "different"), + ] + documents = [] + for key, value in mutations: + document = _draft_response() + document[key] = value + documents.append(document) + for path, value in [ + (("base", "ref"), "master"), + (("base", "repo", "full_name"), "evil/cassandra"), + (("head", "ref"), "other"), + (("head", "sha"), "0" * 40), + (("head", "repo", "full_name"), "evil/cassandra"), + ]: + document = copy.deepcopy(_draft_response()) + target = document + for key in path[:-1]: + target = target[key] + target[path[-1]] = value + documents.append(document) + + for document in documents: + with pytest.raises(client.PolicyError): + client._require_create_response( + json.dumps(document).encode(), + repo="cassandra", + base="main", + head="hermes/fix", + head_sha=HEAD_SHA, + title="WIP: Focused fix", + body="Review evidence", + ) + + +def test_read_response_is_bounded(): + client = _load() + + with pytest.raises(client.PolicyError, match="safe size limit"): + client.read( + "/api/v1/repos/atlas/cassandra", + token="runtime", + opener=lambda *_a, **_k: Response(b"x" * (client.MAX_RESPONSE_BYTES + 1)), + ) + + +def test_direct_api_rejects_unexpected_success_content_type(): + client = _load() + response = Response(b"{}") + response.headers.replace_header("Content-Type", "text/html") + + with pytest.raises(client.PolicyError, match="unexpected response type"): + client.read( + "/api/v1/repos/atlas/cassandra", + token="runtime", + opener=lambda *_a, **_k: response, + ) + + +def test_output_redaction_covers_exact_token_and_authorization_header(): + client = _load() + raw = b'{"message":"do-not-leak","debug":"Authorization: token do-not-leak"}' + redacted = client.redact_bytes(raw, "do-not-leak") + + assert b"do-not-leak" not in redacted + assert redacted.count(b"[REDACTED]") >= 1 + + +@pytest.mark.parametrize( + "reflected", + [ + b"runtime-sentinel", + base64.b64encode(b"runtime-sentinel"), + base64.b64encode(b"hermes-automation:runtime-sentinel"), + ], +) +def test_direct_api_rejects_credential_reflection(reflected: bytes): + client = _load() + + with pytest.raises(client.PolicyError, match="credential material"): + client.read( + "/api/v1/repos/atlas/cassandra", + token="runtime-sentinel", + opener=lambda *_a, **_k: Response( + b'{"unexpected":"' + reflected + b'"}' + ), + ) diff --git a/testing/tests/test_hermes_gitea_support.py b/testing/tests/test_hermes_gitea_support.py new file mode 100644 index 00000000..8b00e784 --- /dev/null +++ b/testing/tests/test_hermes_gitea_support.py @@ -0,0 +1,75 @@ +"""Shared fixtures for the Atlas-only Gitea client contracts.""" + +from __future__ import annotations + +import importlib.util +import json +import sys +from email.message import Message +from pathlib import Path + +ROOT = Path(__file__).parents[2] +CLIENT_PATH = ROOT / "services/hermes/scm-common/scripts/gitea_api.py" +HEAD_SHA = "465cf9146b05c174a2a8d310aff6c64be58277b6" +if str(CLIENT_PATH.parent) not in sys.path: + sys.path.insert(0, str(CLIENT_PATH.parent)) + + +def _load(): + spec = importlib.util.spec_from_file_location("safe_gitea_api", CLIENT_PATH) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _draft_payload(**updates): + payload = { + "base": "main", + "body": "Review evidence", + "head": "hermes/review-fix", + "title": "WIP: Repair review findings", + } + payload.update(updates) + return payload + + +def _draft_response(**updates): + response = { + "number": 3, + "state": "open", + "draft": True, + "merged": False, + "html_url": "https://scm.bstein.dev/atlas/cassandra/pulls/3", + "url": "https://scm.bstein.dev/atlas/cassandra/pulls/3", + "title": "WIP: Focused fix", + "body": "Review evidence", + "base": {"ref": "main", "repo": {"full_name": "atlas/cassandra"}}, + "head": { + "ref": "hermes/fix", + "sha": HEAD_SHA, + "repo": {"full_name": "atlas/cassandra"}, + }, + } + response.update(updates) + return response + + +class Response: + def __init__(self, body: object, status: int | None = None): + self.body = body if isinstance(body, bytes) else json.dumps(body).encode() + self.status = ( + status if status is not None else (201 if isinstance(body, dict) else 200) + ) + self.headers = Message() + self.headers["Content-Type"] = "application/json" + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def read(self, limit=-1): + return self.body if limit < 0 else self.body[:limit] diff --git a/testing/tests/test_hermes_node_account_hardening.py b/testing/tests/test_hermes_node_account_hardening.py new file mode 100644 index 00000000..9cf2bb3a --- /dev/null +++ b/testing/tests/test_hermes_node_account_hardening.py @@ -0,0 +1,252 @@ +"""Core contracts for the dedicated Hermes node SSH identity.""" + +from __future__ import annotations + +import os +from pathlib import Path + +import pytest +import yaml + +from testing.tests.test_hermes_node_account_support import ( + ROOT, + SCRIPT, + _fixture, + _key, + _load, +) + +def test_reconciler_creates_locked_groupless_account_and_moves_only_exact_key( + tmp_path: Path, monkeypatch +): + module, originals, key, other, public_key = _fixture(tmp_path, monkeypatch) + + module.reconcile(public_key) + first = { + name: (module.HOST_ETC / name).read_text(encoding="utf-8") + for name in originals + } + module.reconcile(public_key) + second = { + name: (module.HOST_ETC / name).read_text(encoding="utf-8") + for name in originals + } + + assert first == second + expected = module._expected_records() + for name, original in originals.items(): + assert first[name] == original + ":".join(expected[name]) + "\n" + backup = module.HOST_ETC / f"{name}.hermes-boundary-backup" + assert backup.read_text(encoding="utf-8") == original + assert expected["shadow"][1] == "!" + assert expected["group"][-1] == "" + assert "disk:x:6:atlas\n" in first["group"] + assert "sudo:x:27:oceanus\n" in first["group"] + + hermes_keys = ( + module.HOST_HOME / module.ACCOUNT / ".ssh/authorized_keys" + ).read_text(encoding="utf-8") + assert hermes_keys == key + "\n" + for user in module.LEGACY_ACCOUNTS: + legacy = ( + module.HOST_HOME / user / ".ssh/authorized_keys" + ).read_text(encoding="utf-8") + assert legacy == f"{other} {user}\n" + backup = module.HOST_HOME / user / ".ssh/authorized_keys.hermes-boundary-backup" + assert backup.read_text(encoding="utf-8") == f"{other} {user}\n{key}\n" + + +def test_identity_conflict_fails_closed_without_editing_account_databases( + tmp_path: Path, monkeypatch +): + module, originals, _key, _other, public_key = _fixture(tmp_path, monkeypatch) + conflict = originals["passwd"] + ( + f"unrelated:x:{module.ACCOUNT_UID}:{module.ACCOUNT_GID}:Other:/home/other:/bin/bash\n" + ) + (module.HOST_ETC / "passwd").write_text(conflict, encoding="utf-8") + + with pytest.raises(module.HardeningError, match="identity conflicts"): + module.reconcile(public_key) + + assert (module.HOST_ETC / "passwd").read_text(encoding="utf-8") == conflict + for name in ("group", "shadow", "gshadow"): + assert (module.HOST_ETC / name).read_text(encoding="utf-8") == originals[name] + assert not (module.HOST_HOME / module.ACCOUNT).exists() + + +def test_malformed_account_file_fails_closed(tmp_path: Path, monkeypatch): + module, originals, _key, _other, public_key = _fixture(tmp_path, monkeypatch) + (module.HOST_ETC / "group").write_text("malformed\n", encoding="utf-8") + + with pytest.raises(module.HardeningError, match="invalid record"): + module.reconcile(public_key) + + assert (module.HOST_ETC / "passwd").read_text(encoding="utf-8") == originals[ + "passwd" + ] + assert (module.HOST_ETC / "group").read_text(encoding="utf-8") == "malformed\n" + + +def test_preexisting_home_fails_before_account_database_or_key_changes( + tmp_path: Path, monkeypatch +): + module, originals, _key, _other, public_key = _fixture(tmp_path, monkeypatch) + home = module.HOST_HOME / module.ACCOUNT + home.mkdir() + (home / "unrelated").write_text("preserve\n", encoding="utf-8") + + with pytest.raises(module.HardeningError, match="home already exists"): + module.reconcile(public_key) + + for name, original in originals.items(): + assert (module.HOST_ETC / name).read_text(encoding="utf-8") == original + assert not (module.HOST_ETC / f"{name}.hermes-boundary-backup").exists() + assert (home / "unrelated").read_text(encoding="utf-8") == "preserve\n" + + +def test_flux_daemonset_reconciles_every_node_without_mutating_human_groups(): + documents = list( + yaml.safe_load_all( + (ROOT / "services/hermes/node-ssh-access.yaml").read_text(encoding="utf-8") + ) + ) + daemonset = next(item for item in documents if item["kind"] == "DaemonSet") + spec = daemonset["spec"]["template"]["spec"] + assert spec["tolerations"] == [{"operator": "Exists"}] + command = spec["containers"][0]["args"][0] + assert "/opt/node-hardener/node_account_hardening.py" in command + assert "sleep 300" in command + mounts = {item["name"]: item for item in spec["volumes"]} + assert mounts["host-home"]["hostPath"]["path"] == "/home" + assert mounts["host-etc"]["hostPath"]["path"] == "/etc" + assert mounts["host-k3s"]["hostPath"]["path"] == "/var/lib/rancher/k3s" + assert mounts["host-kubelet"]["hostPath"]["path"] == "/var/lib/kubelet" + assert mounts["host-run-k3s"]["hostPath"]["path"] == "/run/k3s" + assert mounts["host-run-containerd"]["hostPath"]["path"] == "/run/containerd" + assert "usermod" not in command + assert "groupmod" not in command + + policies = list( + yaml.safe_load_all( + (ROOT / "services/hermes/networkpolicy.yaml").read_text(encoding="utf-8") + ) + ) + isolation = next( + item + for item in policies + if item["metadata"]["name"] == "hermes-node-ssh-access-isolation" + ) + assert isolation["spec"]["ingress"] == [] + assert isolation["spec"]["egress"] == [] + + source = SCRIPT.read_text(encoding="utf-8") + assert "ACCOUNT = \"hermes-agent\"" in source + assert "ACCOUNT_UID = 1200" in source + assert "ACCOUNT_GID = 1200" in source + assert 'LEGACY_ACCOUNTS = ("atlas", "oceanus")' in source + assert "audit_membership" in source + image = spec["containers"][0]["image"] + assert image == ( + "python@sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df" + ) + assert mounts["host-polkit-share"]["hostPath"]["path"] == "/usr/share/polkit-1" + + +def test_sensitive_roots_get_explicit_zero_permission_acl_for_hermes( + tmp_path: Path, monkeypatch +): + module = _load() + sensitive = tmp_path / "k3s" + sensitive.mkdir(mode=0o755) + host_etc = tmp_path / "etc" + host_etc.mkdir() + monkeypatch.setattr(module, "ACCOUNT_UID", 1200) + monkeypatch.setattr(module, "HOST_ETC", host_etc) + monkeypatch.setattr(module, "HOST_ROOT_UID", os.getuid()) + monkeypatch.setattr(module, "HOST_ROOT_GID", os.getgid()) + # Production requires uid 0. The test process owns its fixture but runs the + # real xattr/ACL implementation using the test owner's uid as that boundary. + + module._deny_sensitive_root(sensitive) + first = module._read_acl(sensitive) + module._deny_sensitive_root(sensitive) + second = module._read_acl(sensitive) + + assert first == second + entries = module._decode_acl(first, 0o755) + assert (module.ACL_USER, 0, 1200) in entries + assert (module.ACL_USER_OBJ, 0o7, module.ACL_UNDEFINED_ID) in entries + assert (module.ACL_GROUP_OBJ, 0o5, module.ACL_UNDEFINED_ID) in entries + assert (module.ACL_OTHER, 0o5, module.ACL_UNDEFINED_ID) in entries + backup = host_etc / "hermes-node-boundary/k3s.acl" + assert backup.read_bytes() == b"N" + assert backup.stat().st_mode & 0o777 == 0o600 + + +def test_sensitive_root_rejects_non_directory_and_non_root_owner( + tmp_path: Path, monkeypatch +): + module = _load() + regular = tmp_path / "not-a-directory" + regular.write_text("preserve", encoding="utf-8") + with pytest.raises(module.HardeningError, match="unsafe sensitive directory"): + module._deny_sensitive_root(regular) + + directory = tmp_path / "not-root-owned" + directory.mkdir() + if directory.stat().st_uid == 0: + pytest.skip("cannot construct a non-root-owned fixture as root") + with pytest.raises(module.HardeningError, match="not root-owned"): + module._deny_sensitive_root(directory) + + +def test_acl_parser_rejects_malformed_or_incomplete_values(): + module = _load() + with pytest.raises(module.HardeningError, match="malformed"): + module._decode_acl(b"too short", 0o755) + incomplete = module._encode_acl( + [(module.ACL_USER_OBJ, 0o7, module.ACL_UNDEFINED_ID)] + ) + with pytest.raises(module.HardeningError, match="incomplete"): + module._decode_acl(incomplete, 0o755) + + +def test_acl_failure_stops_before_installing_the_ssh_key(tmp_path: Path, monkeypatch): + module = _load() + host_etc = tmp_path / "etc" + host_etc.mkdir() + monkeypatch.setattr(module, "HOST_ETC", host_etc) + monkeypatch.setattr(module, "HOST_ROOT_UID", os.getuid()) + public_key = tmp_path / "public-key" + public_key.write_text( + _key(b"synthetic-key") + "\n", + encoding="utf-8", + ) + moved = False + monkeypatch.setattr(module, "_reconcile_databases", lambda: None) + + def fail_acl(): + raise module.HardeningError("ACL unavailable") + + def move_key(_path): + nonlocal moved + moved = True + + monkeypatch.setattr(module, "_deny_sensitive_roots", fail_acl) + monkeypatch.setattr(module, "_move_key", move_key) + + with pytest.raises(module.HardeningError, match="ACL unavailable"): + module.reconcile(public_key) + assert moved is False + + +def test_runtime_ssh_config_forces_dedicated_account_for_every_titan(): + stage = ( + ROOT / "services/hermes/scripts/stage_runtime_access.py" + ).read_text(encoding="utf-8") + assert '"Host titan-*\\n User hermes-agent\\n"' in stage + agent = yaml.safe_load( + (ROOT / "services/hermes/agent-deployment.yaml").read_text(encoding="utf-8") + ) + assert "User atlas" not in yaml.safe_dump(agent) + assert "User oceanus" not in yaml.safe_dump(agent) diff --git a/testing/tests/test_hermes_node_account_io.py b/testing/tests/test_hermes_node_account_io.py new file mode 100644 index 00000000..6b05cc0a --- /dev/null +++ b/testing/tests/test_hermes_node_account_io.py @@ -0,0 +1,222 @@ +"""Crash-safety and metadata contracts for Hermes node-account writes.""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path + +import pytest +import yaml + +from testing.tests.test_hermes_node_account_support import ( + ROOT, + _fixture, + _load, + _typed_key, +) + +def test_key_identity_ignores_options_and_comments_and_removes_legacy_first( + tmp_path: Path, monkeypatch +): + module, _originals, key, other, public_key = _fixture(tmp_path, monkeypatch) + key_fields = key.split() + decorated = f'restrict,command="echo denied" {key_fields[0]} {key_fields[1]} old-comment' + for user in module.LEGACY_ACCOUNTS: + path = module.HOST_HOME / user / ".ssh/authorized_keys" + path.write_text(f"{other} {user}\n{decorated}\n", encoding="utf-8") + writes: list[Path] = [] + real_write = module.atomic_write + + def recording_write(path, value, metadata): + writes.append(path) + real_write(path, value, metadata) + + monkeypatch.setattr(module, "atomic_write", recording_write) + module.reconcile(public_key) + + key_writes = [path for path in writes if path.name == "authorized_keys"] + assert key_writes[-1].parts[-3:] == (module.ACCOUNT, ".ssh", "authorized_keys") + assert all(module.ACCOUNT not in path.parts for path in key_writes[:-1]) + + +def test_unrelated_modern_human_key_type_is_preserved(tmp_path: Path, monkeypatch): + module, _originals, key, _other, public_key = _fixture(tmp_path, monkeypatch) + modern = _typed_key("sk-ssh-ed25519@openssh.com", b"human-security-key", "human") + for user in module.LEGACY_ACCOUNTS: + path = module.HOST_HOME / user / ".ssh/authorized_keys" + path.write_text(f"{modern}\n{key}\n", encoding="utf-8") + + module.reconcile(public_key) + + for user in module.LEGACY_ACCOUNTS: + path = module.HOST_HOME / user / ".ssh/authorized_keys" + assert path.read_text() == modern + "\n" + + +def test_crash_during_legacy_key_removal_never_installs_duplicate_key( + tmp_path: Path, monkeypatch +): + module, _originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + real_write = module.atomic_write + key_writes = 0 + + def crash_on_second_key(path, value, metadata): + nonlocal key_writes + if path.name == "authorized_keys": + key_writes += 1 + if key_writes == 2: + raise module.HardeningError("synthetic crash") + real_write(path, value, metadata) + + monkeypatch.setattr(module, "atomic_write", crash_on_second_key) + with pytest.raises(module.HardeningError, match="synthetic crash"): + module.reconcile(public_key) + assert not (module.HOST_HOME / module.ACCOUNT / ".ssh/authorized_keys").exists() + + +def test_preexisting_target_key_is_removed_before_legacy_cleanup_and_reinstalled( + tmp_path: Path, monkeypatch +): + module, _originals, key, _other, public_key = _fixture(tmp_path, monkeypatch) + target = module.HOST_HOME / module.ACCOUNT / ".ssh/authorized_keys" + target.parent.mkdir(parents=True) + target.write_text(key + " existing\n") + writes: list[Path] = [] + real_write = module.atomic_write + + def recording_write(path, value, metadata): + if path.name == "authorized_keys": + writes.append(path) + real_write(path, value, metadata) + + monkeypatch.setattr(module, "atomic_write", recording_write) + module._move_key(public_key) + + assert writes[0] == target + assert writes[-1] == target + assert all(path != target for path in writes[1:-1]) + assert target.read_text() == key + "\n" + + +def test_account_database_write_preserves_mode_owner_and_xattrs( + tmp_path: Path, monkeypatch +): + module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + passwd = module.HOST_ETC / "passwd" + passwd.chmod(0o640) + os.setxattr(passwd, "user.hermes-test", b"preserve") + before = passwd.stat() + + module.reconcile(public_key) + + after = passwd.stat() + assert after.st_mode & 0o777 == 0o640 + assert (after.st_uid, after.st_gid) == (before.st_uid, before.st_gid) + assert os.getxattr(passwd, "user.hermes-test") == b"preserve" + backup = module.HOST_ETC / "passwd.hermes-boundary-backup" + assert os.getxattr(backup, "user.hermes-test") == b"preserve" + assert backup.read_text() == originals["passwd"] + + +def test_atomic_write_restores_acl_and_security_label_xattrs(tmp_path: Path, monkeypatch): + module = _load() + io_module = sys.modules[module.atomic_write.__module__] + target = tmp_path / "passwd" + target.write_bytes(b"old\n") + metadata = io_module.FileSnapshot( + value=b"old\n", + device=target.stat().st_dev, + inode=target.stat().st_ino, + mode=0o600, + uid=os.getuid(), + gid=os.getgid(), + size=4, + mtime_ns=target.stat().st_mtime_ns, + ctime_ns=target.stat().st_ctime_ns, + xattrs=( + ("system.posix_acl_access", b"synthetic-acl"), + ("security.selinux", b"synthetic-label"), + ("user.audit", b"synthetic-xattr"), + ), + ) + restored: list[tuple[str, bytes]] = [] + monkeypatch.setattr( + io_module.os, + "setxattr", + lambda _fd, name, value: restored.append((name, value)), + ) + + io_module.atomic_write(target, b"new\n", metadata) + + assert target.read_bytes() == b"new\n" + assert restored == list(metadata.xattrs) + + +def test_concurrent_database_change_is_detected_before_any_account_write( + tmp_path: Path, monkeypatch +): + module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + real_backup = module.backup_once + backup_count = 0 + writes: list[Path] = [] + + def racing_backup(path, snapshot, maximum): + nonlocal backup_count + result = real_backup(path, snapshot, maximum) + backup_count += 1 + if backup_count == 4: + group = module.HOST_ETC / "group" + group.write_text(originals["group"] + "race:x:4000:\n") + return result + + monkeypatch.setattr(module, "backup_once", racing_backup) + monkeypatch.setattr(module, "atomic_write", lambda path, *_args: writes.append(path)) + with pytest.raises(module.HardeningError, match="concurrent host account change"): + module.reconcile(public_key) + assert writes == [] + + +def test_flux_orders_observer_rbac_before_hermes_prunes_old_authority(): + observer = yaml.safe_load( + (ROOT / "clusters/atlas/flux-system/applications/hermes-observer-rbac/kustomization.yaml").read_text() + ) + hermes = yaml.safe_load( + (ROOT / "clusters/atlas/flux-system/applications/hermes/kustomization.yaml").read_text() + ) + bindings = yaml.safe_load( + ( + ROOT + / "clusters/atlas/flux-system/applications/hermes-observer-bindings/kustomization.yaml" + ).read_text() + ) + assert "dependsOn" not in observer["spec"] + assert {item["name"] for item in hermes["spec"]["dependsOn"]} >= { + "hermes-observer-rbac", + "hermes-scm-broker", + } + assert {item["name"] for item in bindings["spec"]["dependsOn"]} == { + "hermes-observer-rbac", + "hermes", + } + assert "hermes-observer-bindings" not in { + item["name"] for item in hermes["spec"]["dependsOn"] + } + + +def test_sensitive_root_acl_backups_have_unique_host_path_names(monkeypatch): + module = _load() + seen = [] + monkeypatch.setattr( + module, + "_deny_sensitive_root", + lambda path, backup_name=None: seen.append((path, backup_name)), + ) + module._deny_sensitive_roots() + assert [name for _path, name in seen] == [ + "var-lib-rancher-k3s", + "var-lib-kubelet", + "run-k3s", + "run-containerd", + ] + assert len({name for _path, name in seen}) == 4 diff --git a/testing/tests/test_hermes_node_account_privilege_audit.py b/testing/tests/test_hermes_node_account_privilege_audit.py new file mode 100644 index 00000000..3370fe4f --- /dev/null +++ b/testing/tests/test_hermes_node_account_privilege_audit.py @@ -0,0 +1,196 @@ +"""Host privilege-policy audits for the Hermes node SSH identity.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from testing.tests.test_hermes_node_account_support import _fixture + + +@pytest.mark.parametrize( + "database,record", + [ + ("group", "disk:x:6:atlas,hermes-agent\n"), + ("gshadow", "disk:!:hermes-agent:atlas\n"), + ("gshadow", "disk:!::atlas,hermes-agent\n"), + ], +) +def test_unexpected_group_or_gshadow_membership_fails_before_writes( + tmp_path: Path, monkeypatch, database: str, record: str +): + module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + path = module.HOST_ETC / database + path.write_text( + "\n".join( + line + for line in originals[database].splitlines() + if not line.startswith("disk:") + ) + + "\n" + + record, + encoding="utf-8", + ) + before = {name: (module.HOST_ETC / name).read_bytes() for name in originals} + + with pytest.raises(module.HardeningError, match="group access"): + module.reconcile(public_key) + + assert {name: (module.HOST_ETC / name).read_bytes() for name in originals} == before + + +@pytest.mark.parametrize( + "relative,value", + [ + ("sudoers.d/hermes", "hermes-agent ALL=(ALL:ALL) NOPASSWD: ALL\n"), + ("sudoers", "ALL ALL=(ALL:ALL) ALL\n"), + ( + "polkit-1/rules.d/90-root.rules", + 'if (subject.user == "hermes-agent") return polkit.Result.YES;\n', + ), + ( + "polkit-1/rules.d/90-root.rules", + "unix-user:* org.freedesktop.policykit.exec polkit.Result.YES\n", + ), + ], +) +def test_sudo_and_root_equivalent_polkit_authority_fail_closed( + tmp_path: Path, monkeypatch, relative: str, value: str +): + module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + policy = module.HOST_ETC / relative + policy.parent.mkdir(parents=True, exist_ok=True) + policy.write_text(value, encoding="utf-8") + + with pytest.raises(module.HardeningError, match="sudo|polkit"): + module.reconcile(public_key) + + assert all( + (module.HOST_ETC / name).read_text() == original + for name, original in originals.items() + ) + + +def test_external_group_authority_source_is_rejected(tmp_path: Path, monkeypatch): + module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + (module.HOST_ETC / "nsswitch.conf").write_text("passwd: files\ngroup: files ldap\n") + with pytest.raises(module.HardeningError, match="external group"): + module.reconcile(public_key) + assert all( + (module.HOST_ETC / name).read_text() == original + for name, original in originals.items() + ) + + +@pytest.mark.parametrize("database", ["passwd", "group", "initgroups", "shadow"]) +def test_external_identity_authority_sources_fail_closed( + tmp_path: Path, monkeypatch, database: str +): + module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + (module.HOST_ETC / "nsswitch.conf").write_text( + f"passwd: files\n{database}: files ldap\n", encoding="utf-8" + ) + with pytest.raises(module.HardeningError, match="external group/account"): + module.reconcile(public_key) + assert all( + (module.HOST_ETC / name).read_text() == original + for name, original in originals.items() + ) + + +@pytest.mark.parametrize( + "value", + [ + "#1200 ALL=(ALL:ALL) NOPASSWD: ALL\n", + "User_Alias HERMES_IDS = #1200\nHERMES_IDS ALL=(ALL:ALL) NOPASSWD: ALL\n", + "User_Alias HERMES_GROUP = %#1200\nHERMES_GROUP ALL=(ALL:ALL) NOPASSWD: ALL\n", + ], +) +def test_visudo_valid_numeric_and_alias_grants_fail_closed( + tmp_path: Path, monkeypatch, value: str +): + module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + monkeypatch.setattr(module, "ACCOUNT_UID", 1200) + policy = module.HOST_ETC / "sudoers" + policy.write_text(value, encoding="utf-8") + validation = subprocess.run( + ["/usr/bin/visudo", "-c", "-f", str(policy)], + check=False, + capture_output=True, + text=True, + ) + assert validation.returncode == 0, validation.stderr + + with pytest.raises(module.HardeningError, match="sudo authority"): + module.reconcile(public_key) + + assert all( + (module.HOST_ETC / name).read_text() == original + for name, original in originals.items() + ) + + +def test_sudo_include_is_bounded_to_the_audited_standard_directory( + tmp_path: Path, monkeypatch +): + module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + sudoers = module.HOST_ETC / "sudoers" + sudoers.write_text("#includedir /opt/external-sudoers\n", encoding="utf-8") + with pytest.raises(module.HardeningError, match="unaudited authority source"): + module.reconcile(public_key) + + sudoers.write_text("#includedir /etc/sudoers.d\n", encoding="utf-8") + module.reconcile(public_key) + assert all( + (module.HOST_ETC / name).read_text().startswith(original) + for name, original in originals.items() + ) + + +@pytest.mark.parametrize( + "relative,value", + [ + ( + "polkit-1/rules.d/90-hermes.rules", + "polkit.addRule(function(action, subject) {\n" + " if (subject.uid == 1200) return polkit.Result.YES;\n" + "});\n", + ), + ( + "polkit-1/localauthority/50-local.d/hermes.pkla", + "[Hermes]\nIdentity=unix-user:1200\n" + "Action=org.freedesktop.policykit.exec\nResultActive=yes\n", + ), + ], +) +def test_numeric_polkit_grants_fail_closed( + tmp_path: Path, monkeypatch, relative: str, value: str +): + module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + monkeypatch.setattr(module, "ACCOUNT_UID", 1200) + policy = module.HOST_ETC / relative + policy.parent.mkdir(parents=True, exist_ok=True) + policy.write_text(value, encoding="utf-8") + + with pytest.raises(module.HardeningError, match="polkit authority"): + module.reconcile(public_key) + + assert all( + (module.HOST_ETC / name).read_text() == original + for name, original in originals.items() + ) + + +def test_writable_privilege_policy_is_rejected(tmp_path: Path, monkeypatch): + module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + sudoers = module.HOST_ETC / "sudoers" + sudoers.write_text("root ALL=(ALL:ALL) ALL\n") + sudoers.chmod(0o666) + with pytest.raises(module.HardeningError, match="unsafe mutation"): + module.reconcile(public_key) + assert all( + (module.HOST_ETC / name).read_text() == original + for name, original in originals.items() + ) diff --git a/testing/tests/test_hermes_node_account_support.py b/testing/tests/test_hermes_node_account_support.py new file mode 100644 index 00000000..2d74d124 --- /dev/null +++ b/testing/tests/test_hermes_node_account_support.py @@ -0,0 +1,90 @@ +"""Shared fixtures for dedicated Hermes node-account contracts.""" + +from __future__ import annotations + +import base64 +import importlib.util +import os +import sys +from pathlib import Path + +ROOT = Path(__file__).parents[2] +SCRIPT = ROOT / "services/hermes/scripts/node_account_hardening.py" +sys.path.insert(0, str(SCRIPT.parent)) + + +def _key(material: bytes, comment: str = "") -> str: + key_type = b"ssh-ed25519" + blob = len(key_type).to_bytes(4, "big") + key_type + material + suffix = f" {comment}" if comment else "" + return "ssh-ed25519 " + base64.b64encode(blob).decode() + suffix + + +def _typed_key(key_type: str, material: bytes, comment: str = "") -> str: + encoded_type = key_type.encode() + blob = len(encoded_type).to_bytes(4, "big") + encoded_type + material + suffix = f" {comment}" if comment else "" + return f"{key_type} {base64.b64encode(blob).decode()}{suffix}" + + +def _load(): + spec = importlib.util.spec_from_file_location("node_account_hardening_test", SCRIPT) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _fixture(tmp_path: Path, monkeypatch): + module = _load() + host_etc = tmp_path / "etc" + host_home = tmp_path / "home" + host_etc.mkdir() + host_home.mkdir() + originals = { + "passwd": ( + "root:x:0:0:root:/root:/bin/bash\n" + "atlas:x:2000:2000:Atlas:/home/atlas:/bin/bash\n" + "oceanus:x:2001:2001:Oceanus:/home/oceanus:/bin/bash\n" + ), + "group": ( + "root:x:0:\n" + "atlas:x:2000:\n" + "oceanus:x:2001:\n" + "disk:x:6:atlas\n" + "sudo:x:27:oceanus\n" + ), + "shadow": ( + "root:!:1:0:99999:7:::\n" + "atlas:!:1:0:99999:7:::\n" + "oceanus:!:1:0:99999:7:::\n" + ), + "gshadow": ( + "root:!::\n" + "atlas:!::\n" + "oceanus:!::\n" + "disk:!::atlas\n" + "sudo:!::oceanus\n" + ), + } + for name, value in originals.items(): + (host_etc / name).write_text(value, encoding="utf-8") + key = _key(b"synthetic-hermes-key") + other = _key(b"human-operator-key") + for user in ("atlas", "oceanus"): + ssh = host_home / user / ".ssh" + ssh.mkdir(parents=True) + (ssh / "authorized_keys").write_text( + f"{other} {user}\n{key}\n", encoding="utf-8" + ) + public_key = tmp_path / "public-key" + public_key.write_text(key + "\n", encoding="utf-8") + monkeypatch.setattr(module, "HOST_ETC", host_etc) + monkeypatch.setattr(module, "HOST_HOME", host_home) + monkeypatch.setattr(module, "ACCOUNT_UID", os.getuid()) + monkeypatch.setattr(module, "ACCOUNT_GID", os.getgid()) + monkeypatch.setattr(module, "HOST_ROOT_UID", os.getuid()) + monkeypatch.setattr(module, "HOST_POLKIT_SHARE", tmp_path / "missing-polkit") + monkeypatch.setattr(module, "_deny_sensitive_roots", lambda: None) + return module, originals, key, other, public_key diff --git a/testing/tests/test_hermes_node_acl_coverage.py b/testing/tests/test_hermes_node_acl_coverage.py new file mode 100644 index 00000000..572fd0d2 --- /dev/null +++ b/testing/tests/test_hermes_node_acl_coverage.py @@ -0,0 +1,206 @@ +"""Behavioral branch coverage for node-storage ACL hardening.""" + +from __future__ import annotations + +import errno +import os +import pytest + +from testing.tests.test_hermes_node_account_support import _fixture, _load + + +def test_acl_decoder_rejects_version_permissions_and_adds_missing_mask(): + module = _load() + valid = [ + (module.ACL_USER_OBJ, 7, module.ACL_UNDEFINED_ID), + (module.ACL_GROUP_OBJ, 5, module.ACL_UNDEFINED_ID), + (module.ACL_OTHER, 5, module.ACL_UNDEFINED_ID), + ] + wrong_version = module.ACL_HEADER.pack(99) + b"".join( + module.ACL_ENTRY.pack(*entry) for entry in valid + ) + with pytest.raises(module.HardeningError, match="version"): + module._decode_acl(wrong_version, 0o755) + bad_permission = module._encode_acl([*valid, (module.ACL_USER, 8, 1200)]) + with pytest.raises(module.HardeningError, match="permission"): + module._decode_acl(bad_permission, 0o755) + updated = module._decode_acl( + module._acl_with_deny(module._encode_acl(valid), 0o755), 0o755 + ) + assert (module.ACL_MASK, 5, module.ACL_UNDEFINED_ID) in updated + assert (module.ACL_USER, 0, module.ACCOUNT_UID) in updated + + +def test_read_acl_handles_absence_and_propagates_other_errors(monkeypatch, tmp_path): + module = _load() + path = tmp_path / "root" + path.mkdir() + + def missing(*_args, **_kwargs): + raise OSError(errno.ENODATA, "missing") + + monkeypatch.setattr(module.os, "getxattr", missing) + assert module._read_acl(path) == b"" + + def denied(*_args, **_kwargs): + raise OSError(errno.EPERM, "denied") + + monkeypatch.setattr(module.os, "getxattr", denied) + with pytest.raises(OSError): + module._read_acl(path) + + +@pytest.mark.parametrize("value", [b"", b"X", b"A", b"Nextra"]) +def test_acl_backup_validator_rejects_malformed_encodings(value): + module = _load() + with pytest.raises(module.HardeningError, match="backup is malformed"): + module._validate_acl_backup(value) + module._validate_acl_backup(b"N") + valid = module._encode_acl( + [ + (module.ACL_USER_OBJ, 7, module.ACL_UNDEFINED_ID), + (module.ACL_GROUP_OBJ, 5, module.ACL_UNDEFINED_ID), + (module.ACL_OTHER, 5, module.ACL_UNDEFINED_ID), + ] + ) + module._validate_acl_backup(b"A" + valid) + + +def test_acl_backup_reuses_valid_existing_copy(tmp_path, monkeypatch): + module = _load() + host_etc = tmp_path / "etc" + root = host_etc / "hermes-node-boundary" + root.mkdir(parents=True) + backup = root / "k3s.acl" + backup.write_bytes(b"N") + monkeypatch.setattr(module, "HOST_ETC", host_etc) + monkeypatch.setattr(module, "HOST_ROOT_UID", os.getuid()) + monkeypatch.setattr(module, "HOST_ROOT_GID", os.getgid()) + module._acl_backup_once(tmp_path / "k3s", b"") + assert backup.read_bytes() == b"N" + + +def test_acl_backup_short_write_cleans_temporary(tmp_path, monkeypatch): + module = _load() + host_etc = tmp_path / "etc" + host_etc.mkdir() + monkeypatch.setattr(module, "HOST_ETC", host_etc) + monkeypatch.setattr(module, "HOST_ROOT_UID", os.getuid()) + monkeypatch.setattr(module, "HOST_ROOT_GID", os.getgid()) + monkeypatch.setattr(module.os, "write", lambda _fd, _value: 0) + with pytest.raises(module.HardeningError, match="short sensitive"): + module._acl_backup_once(tmp_path / "k3s", b"") + assert not list((host_etc / "hermes-node-boundary").glob(".*.hermes-*")) + + +def test_acl_backup_tolerates_concurrent_first_writer(tmp_path, monkeypatch): + module = _load() + host_etc = tmp_path / "etc" + host_etc.mkdir() + monkeypatch.setattr(module, "HOST_ETC", host_etc) + monkeypatch.setattr(module, "HOST_ROOT_UID", os.getuid()) + monkeypatch.setattr(module, "HOST_ROOT_GID", os.getgid()) + real_link = os.link + + def race(source, target, **kwargs): + real_link(source, target, **kwargs) + raise FileExistsError() + + monkeypatch.setattr(module.os, "link", race) + module._acl_backup_once(tmp_path / "k3s", b"") + assert (host_etc / "hermes-node-boundary/k3s.acl").read_bytes() == b"N" + + +def test_sensitive_root_failure_restores_existing_acl(tmp_path, monkeypatch): + module = _load() + root = tmp_path / "k3s" + root.mkdir() + monkeypatch.setattr(module, "HOST_ROOT_UID", os.getuid()) + current = module._encode_acl( + [ + (module.ACL_USER_OBJ, 7, module.ACL_UNDEFINED_ID), + (module.ACL_GROUP_OBJ, 5, module.ACL_UNDEFINED_ID), + (module.ACL_MASK, 5, module.ACL_UNDEFINED_ID), + (module.ACL_OTHER, 5, module.ACL_UNDEFINED_ID), + ] + ) + reads = iter((current, current)) + writes = [] + monkeypatch.setattr(module, "_read_acl", lambda _path: next(reads)) + monkeypatch.setattr(module, "_acl_backup_once", lambda *_a, **_k: None) + monkeypatch.setattr( + module.os, "setxattr", lambda *args, **kwargs: writes.append(args[2]) + ) + with pytest.raises(module.HardeningError, match="validation failed"): + module._deny_sensitive_root(root) + assert writes[-1] == current + + +@pytest.mark.parametrize("remove_errno", [errno.ENODATA, errno.EPERM]) +def test_sensitive_root_failure_removes_new_acl_or_propagates_cleanup_error( + tmp_path, monkeypatch, remove_errno +): + module = _load() + root = tmp_path / "k3s" + root.mkdir() + monkeypatch.setattr(module, "HOST_ROOT_UID", os.getuid()) + monkeypatch.setattr(module, "_read_acl", lambda _path: b"") + monkeypatch.setattr(module, "_acl_backup_once", lambda *_a, **_k: None) + monkeypatch.setattr( + module.os, + "setxattr", + lambda *_a, **_k: (_ for _ in ()).throw(RuntimeError("set failed")), + ) + monkeypatch.setattr( + module.os, + "removexattr", + lambda *_a, **_k: (_ for _ in ()).throw(OSError(remove_errno, "remove")), + ) + expected = OSError if remove_errno == errno.EPERM else RuntimeError + with pytest.raises(expected): + module._deny_sensitive_root(root) + + +def test_database_validation_failure_rolls_back_all_writes(tmp_path, monkeypatch): + module, originals, _key_value, _other, _public = _fixture(tmp_path, monkeypatch) + real_read = module._read_regular + calls = 0 + + def missing_expected(path, maximum=module.MAX_ACCOUNT_FILE): + nonlocal calls + calls += 1 + if calls == 1: + return originals["passwd"].encode(), path.stat(follow_symlinks=False) + return real_read(path, maximum) + + monkeypatch.setattr(module, "_read_regular", missing_expected) + with pytest.raises(module.HardeningError, match="validation failed"): + module._reconcile_databases() + for name, value in originals.items(): + assert (module.HOST_ETC / name).read_text() == value + + +def test_preexisting_target_key_removal_is_verified(tmp_path, monkeypatch): + module, _originals, key, _other, public = _fixture(tmp_path, monkeypatch) + target = module.HOST_HOME / module.ACCOUNT / ".ssh/authorized_keys" + target.parent.mkdir(parents=True) + target.write_text(key + "\n", encoding="utf-8") + real_write = module.atomic_write + + def ignore_preinstall_target(path, value, metadata): + if path == target and value == b"": + return + real_write(path, value, metadata) + + monkeypatch.setattr(module, "atomic_write", ignore_preinstall_target) + with pytest.raises(module.HardeningError, match="preexisting Hermes authorization"): + module._move_key(public) + + +def test_existing_target_without_key_does_not_need_precleanup(tmp_path, monkeypatch): + module, _originals, _key_value, other, public = _fixture(tmp_path, monkeypatch) + target = module.HOST_HOME / module.ACCOUNT / ".ssh/authorized_keys" + target.parent.mkdir(parents=True) + target.write_text(other + "\n", encoding="utf-8") + module._move_key(public) + assert target.read_text().endswith("\n") diff --git a/testing/tests/test_hermes_node_audit_coverage.py b/testing/tests/test_hermes_node_audit_coverage.py new file mode 100644 index 00000000..8c011b5b --- /dev/null +++ b/testing/tests/test_hermes_node_audit_coverage.py @@ -0,0 +1,221 @@ +"""Behavioral branch coverage for dedicated node-account privilege audits.""" + +from __future__ import annotations + +import os +import struct +import sys +from types import SimpleNamespace + +import pytest + +from testing.tests.test_hermes_node_account_support import _load + + +def _audit_module(): + hardening = _load() + return sys.modules[hardening.audit_membership.__module__] + + +def test_member_parser_handles_empty_valid_malformed_and_duplicates(): + module = _audit_module() + assert module._members("", "group") == set() + assert module._members("one,two", "group") == {"one", "two"} + for value in ("one,", " one", "one,one"): + with pytest.raises(module.HardeningError): + module._members(value, "group") + + +def test_membership_audit_accepts_unrelated_records_and_rejects_account(): + module = _audit_module() + module.audit_membership( + "hermes-agent", [["disk", "x", "6", "atlas"]], [["disk", "!", "atlas", ""]] + ) + with pytest.raises(module.HardeningError, match="supplementary"): + module.audit_membership( + "hermes-agent", [["disk", "x", "6", "hermes-agent"]], [] + ) + with pytest.raises(module.HardeningError, match="gshadow"): + module.audit_membership("hermes-agent", [], [["disk", "!", "hermes-agent", ""]]) + + +@pytest.mark.parametrize("kind", ["mode", "symlink"]) +def test_policy_files_reject_unsafe_directories(tmp_path, kind): + module = _audit_module() + host_etc = tmp_path / "etc" + host_etc.mkdir() + root = host_etc / "sudoers.d" + if kind == "mode": + root.mkdir(mode=0o777) + root.chmod(0o777) + else: + target = tmp_path / "target" + target.mkdir() + root.symlink_to(target) + with pytest.raises(module.HardeningError, match="unsafe sudo/polkit"): + module._policy_files(host_etc, tmp_path / "share", os.getuid()) + + +def test_policy_file_count_is_bounded(tmp_path): + module = _audit_module() + root = tmp_path / "etc/sudoers.d" + root.mkdir(parents=True) + for index in range(257): + (root / str(index)).write_text("# safe\n", encoding="utf-8") + with pytest.raises(module.HardeningError, match="too many"): + module._policy_files(tmp_path / "etc", tmp_path / "share", os.getuid()) + + +def _acl(*entries): + return struct.pack(" bytes: + command = old + b" " + new + b" " + ref + b"\n" + framed = f"{len(command) + 4:04x}".encode() + command + return framed + (b"0000" if terminator else b"") + + +def test_receive_pack_rejects_nonascii_many_commands_and_missing_terminator( + monkeypatch, +): + broker = _load("scm_broker") + zero = b"0" * 40 + commit = b"1" * 40 + with pytest.raises(broker.PolicyError, match="canonical ASCII"): + broker._validate_receive_pack( + _packet(zero, commit, b"refs/heads/hermes/\xff"), "sentinel" + ) + command = _packet(zero, commit, b"refs/heads/hermes/fix", terminator=False) + with pytest.raises(broker.PolicyError, match="terminator"): + broker._validate_receive_pack(command, "sentinel") + many = command * (broker.MAX_PUSH_COMMANDS + 1) + b"0000" + with pytest.raises(broker.PolicyError, match="too many"): + broker._validate_receive_pack(many, "sentinel") + + +def test_upstream_request_infers_byte_length_and_requires_stream_length(): + broker = _load("scm_broker") + seen = [] + + def opener(request, timeout): + seen.append((request, timeout)) + return Response(b"result", content_type="application/x-git-upload-pack-result") + + assert ( + broker._upstream_git_request( + "/atlas/cassandra.git/git-upload-pack", + method="POST", + body=b"request", + content_type=None, + expected_type="application/x-git-upload-pack-result", + token="sentinel", + opener=opener, + ) + == b"result" + ) + assert seen[0][0].get_header("Content-length") == "7" + with pytest.raises(broker.PolicyError, match="length is missing"): + broker._upstream_git_request( + "/atlas/cassandra.git/git-upload-pack", + method="POST", + body=io.BytesIO(b"request"), + content_type=None, + expected_type="application/x-git-upload-pack-result", + token="sentinel", + opener=opener, + ) + + +def test_upstream_request_rejects_wrong_type_and_supports_getcode_status(): + broker = _load("scm_broker") + + class CodeOnly(Response): + def __init__(self, body, content_type): + super().__init__(body, content_type=content_type) + del self.status + + def getcode(self): + return 200 + + assert ( + broker._upstream_git_request( + "/atlas/cassandra.git/info/refs?service=git-upload-pack", + method="GET", + body=None, + content_type=None, + expected_type="application/x-git-upload-pack-advertisement", + token="sentinel", + opener=lambda *_a, **_k: CodeOnly( + b"advertisement", "application/x-git-upload-pack-advertisement" + ), + ) + == b"advertisement" + ) + with pytest.raises(broker.PolicyError, match="response type"): + broker._upstream_git_request( + "/atlas/cassandra.git/info/refs?service=git-upload-pack", + method="GET", + body=None, + content_type=None, + expected_type="application/x-git-upload-pack-advertisement", + token="sentinel", + opener=lambda *_a, **_k: Response(b"bad", content_type="text/plain"), + ) diff --git a/testing/tests/test_hermes_scm_broker_policy.py b/testing/tests/test_hermes_scm_broker_policy.py new file mode 100644 index 00000000..25d94fa0 --- /dev/null +++ b/testing/tests/test_hermes_scm_broker_policy.py @@ -0,0 +1,321 @@ +"""RBAC, Flux, and human-review contracts for the Hermes SCM broker.""" + +from __future__ import annotations + +import json + +import pytest +import yaml + +from testing.tests.test_hermes_scm_broker_support import ROOT, _load_path + +def _can_i(role: dict, api_group: str, resource: str, verb: str) -> bool: + return any( + (api_group in rule["apiGroups"] or "*" in rule["apiGroups"]) + and (resource in rule["resources"] or "*" in rule["resources"]) + and (verb in rule["verbs"] or "*" in rule["verbs"]) + for rule in role["rules"] + ) + + +def test_agent_rbac_preserves_read_diagnostics_without_administrator_authority(): + documents = list( + yaml.safe_load_all( + (ROOT / "services/hermes-observer-rbac/rbac.yaml").read_text( + encoding="utf-8" + ) + ) + ) + cluster_role, namespaced_role, binding = documents + assert binding["roleRef"]["name"] == "hermes-agent-cluster-observer-v2" + assert _can_i(cluster_role, "", "nodes", "get") + assert _can_i(namespaced_role, "", "pods", "get") + assert _can_i(namespaced_role, "", "pods/log", "get") + assert _can_i( + namespaced_role, + "kustomize.toolkit.fluxcd.io", + "kustomizations", + "list", + ) + + denied = ( + ("", "secrets", "get"), + ("", "serviceaccounts/token", "create"), + ("", "pods", "create"), + ("", "pods/exec", "create"), + ("", "pods/attach", "create"), + ("", "pods/portforward", "create"), + ("rbac.authorization.k8s.io", "clusterrolebindings", "create"), + ("authorization.k8s.io", "selfsubjectaccessreviews", "create"), + ("apps", "deployments", "patch"), + ) + assert all( + not _can_i(role, *request) + for request in denied + for role in (cluster_role, namespaced_role) + ) + + bindings = yaml.safe_load( + (ROOT / "services/hermes-observer-bindings/rolebindings.yaml").read_text( + encoding="utf-8" + ) + )["items"] + namespaces = {item["metadata"]["namespace"] for item in bindings} + assert "hermes-scm" not in namespaces + assert {"cassandra", "flux-system", "hermes", "kube-system"} <= namespaces + assert all(item["roleRef"]["name"].endswith("namespaced-observer-v2") for item in bindings) + + +def test_flux_boundary_keeps_broker_secret_and_network_separate_from_agent(): + agent = yaml.safe_load( + (ROOT / "services/hermes/agent-deployment.yaml").read_text(encoding="utf-8") + ) + annotations = agent["spec"]["template"]["metadata"]["annotations"] + assert not any("gitea" in key.lower() for key in annotations) + assert "GIT_ASKPASS" not in json.dumps(agent) + + broker = yaml.safe_load( + (ROOT / "services/hermes-scm-broker/deployment.yaml").read_text( + encoding="utf-8" + ) + ) + assert broker["metadata"]["namespace"] == "hermes-scm" + assert broker["spec"]["template"]["spec"]["serviceAccountName"] == ( + "hermes-scm-broker" + ) + assert "gitea-token" in json.dumps(broker) + container = broker["spec"]["template"]["spec"]["containers"][0] + assert container["resources"]["limits"]["memory"] == "768Mi" + assert container["livenessProbe"]["failureThreshold"] == 10 + tmp = next( + item for item in broker["spec"]["template"]["spec"]["volumes"] if item["name"] == "tmp" + ) + assert tmp["emptyDir"]["sizeLimit"] == "1Gi" + + policy = yaml.safe_load( + (ROOT / "services/hermes-scm-broker/networkpolicy.yaml").read_text( + encoding="utf-8" + ) + ) + ingress = policy["spec"]["ingress"] + assert ingress[0]["from"][0]["namespaceSelector"]["matchLabels"] == { + "kubernetes.io/metadata.name": "hermes" + } + assert ingress[0]["from"][0]["podSelector"]["matchLabels"] == { + "app": "hermes-agent" + } + + vault = ( + ROOT / "services/vault/scripts/vault_k8s_auth_configure.sh" + ).read_text(encoding="utf-8") + assert 'write_policy_and_role "hermes-scm-broker" "hermes-scm"' in vault + assert '"hermes/developer-gitea" ""' in vault + agent_start = vault.index('write_policy_and_role "hermes-agent"') + agent_end = vault.index("write_policy_and_role", agent_start + 1) + assert "developer-gitea" not in vault[agent_start:agent_end] + + +def test_flux_bootstrap_has_no_agent_namespace_dependency_cycle(): + hermes_resources = yaml.safe_load( + (ROOT / "services/hermes/kustomization.yaml").read_text(encoding="utf-8") + )["resources"] + assert "namespace.yaml" in hermes_resources + assert "scm-common" in hermes_resources + + observer = yaml.safe_load( + ( + ROOT + / "clusters/atlas/flux-system/applications/hermes-observer-rbac/kustomization.yaml" + ).read_text(encoding="utf-8") + ) + assert "dependsOn" not in observer["spec"] + observer_objects = [ + item + for item in yaml.safe_load_all( + (ROOT / "services/hermes-observer-rbac/rbac.yaml").read_text( + encoding="utf-8" + ) + ) + if item + ] + assert {item["kind"] for item in observer_objects} <= { + "ClusterRole", + "ClusterRoleBinding", + } + + hermes_flux = yaml.safe_load( + ( + ROOT / "clusters/atlas/flux-system/applications/hermes/kustomization.yaml" + ).read_text(encoding="utf-8") + ) + assert "hermes-observer-rbac" in { + item["name"] for item in hermes_flux["spec"]["dependsOn"] + } + bindings_flux = yaml.safe_load( + ( + ROOT + / "clusters/atlas/flux-system/applications/hermes-observer-bindings/kustomization.yaml" + ).read_text(encoding="utf-8") + ) + assert {item["name"] for item in bindings_flux["spec"]["dependsOn"]} == { + "hermes", + "hermes-observer-rbac", + } + assert bindings_flux["metadata"]["name"] not in { + item["name"] for item in hermes_flux["spec"]["dependsOn"] + } + + applications = ( + ROOT / "clusters/atlas/flux-system/applications/kustomization.yaml" + ).read_text(encoding="utf-8") + assert "hermes-scm-agent-code" not in applications + + broker_code = yaml.safe_load( + ( + ROOT + / "clusters/atlas/flux-system/applications/hermes-scm-broker-code/kustomization.yaml" + ).read_text(encoding="utf-8") + ) + assert broker_code["spec"]["path"] == "./services/hermes/scm-common" + assert broker_code["spec"]["targetNamespace"] == "hermes-scm" + assert broker_code["spec"]["dependsOn"] == [{"name": "hermes-scm-namespace"}] + + +def test_gitea_bootstrap_enforces_human_review_without_overwriting_drift(): + script = ( + ROOT / "services/gitea/scripts/gitea_atlas_identity_ensure.sh" + ).read_text(encoding="utf-8") + + assert 'required_approvals\\\":1' in script + assert 'enable_push_whitelist\\\":true' in script + assert 'push_whitelist_usernames\\\":[\\\"${protected_reviewer}\\\"]' in script + assert 'enable_merge_whitelist\\\":true' in script + assert 'merge_whitelist_usernames\\\":[\\\"${protected_reviewer}\\\"]' in script + assert "protection differs from the human-review policy" in script + assert "api_request PATCH" not in script + job = yaml.safe_load( + (ROOT / "services/gitea/atlas-identity-bootstrap-job.yaml").read_text() + ) + init = job["spec"]["template"]["spec"]["initContainers"][0] + assert init["image"] == ( + "python@sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df" + ) + assert init["securityContext"]["runAsNonRoot"] is True + assert init["securityContext"]["capabilities"]["drop"] == ["ALL"] + assert "/opt/python/bin/python3" in script + + +def _protection(rule_name: str, priority: int, helper, **overrides): + value = { + "rule_name": rule_name, + "priority": priority, + "created_at": "2026-01-01T00:00:00Z", + **helper._required("bstein"), + } + value.update(overrides) + return value + + +def test_branch_protection_uses_first_effective_glob_by_priority(): + helper = _load_path( + "branch_protection_test", + ROOT / "services/gitea/scripts/gitea_branch_protection_check.py", + ) + earlier_drift = _protection("m*", 1, helper, required_approvals=0) + later_exact = _protection("main", 2, helper) + + with pytest.raises(helper.PolicyError, match="effective main protection differs"): + helper.evaluate(json.dumps([later_exact, earlier_drift]).encode(), "main", "bstein") + + earlier_drift["required_approvals"] = 1 + assert ( + helper.evaluate(json.dumps([later_exact, earlier_drift]).encode(), "main", "bstein") + == "PRESENT" + ) + brace_drift = _protection("m{ain,aster}", 1, helper, required_approvals=0) + with pytest.raises(helper.PolicyError, match="effective master protection differs"): + helper.evaluate(json.dumps([brace_drift]).encode(), "master", "bstein") + + upper_glob = _protection("M*", 1, helper, required_approvals=0) + assert ( + helper.evaluate(json.dumps([upper_glob, later_exact]).encode(), "main", "bstein") + == "PRESENT" + ) + + +def test_branch_protection_matches_gitea_v1238_tie_order(): + helper = _load_path( + "branch_protection_order_test", + ROOT / "services/gitea/scripts/gitea_branch_protection_check.py", + ) + older_glob = _protection( + "m*", 1, helper, created_at="2025-01-01T00:00:00Z", required_approvals=0 + ) + newer_plain = _protection( + "MAIN", 1, helper, created_at="2026-01-01T00:00:00Z" + ) + assert ( + helper.evaluate(json.dumps([older_glob, newer_plain]).encode(), "main", "bstein") + == "PRESENT" + ) + + older_glob["created_at"] = "2026-01-02T00:00:00Z" + earlier_glob = _protection( + "m{ain,aster}", + 1, + helper, + created_at="2026-01-01T00:00:00Z", + required_approvals=0, + ) + with pytest.raises(helper.PolicyError, match="effective main protection differs"): + helper.evaluate(json.dumps([older_glob, earlier_glob]).encode(), "main", "bstein") + + +@pytest.mark.parametrize( + ("rule_name", "branch", "expected"), + [ + ("release/*", "release/v1.17", True), + ("release/**/v1.17", "release/test/1/v1.17", True), + ("release/*/v1.17", "release/test/1/v1.17", False), + ("*", "release/v1.16", False), + ("**", "release/v1.16", True), + ("MAIN", "main", True), + ("M*", "main", False), + ("m{ain,aster}", "master", True), + (r"m\ain", "main", True), + ], +) +def test_branch_glob_matches_gitea_v1238_source_corpus( + rule_name: str, branch: str, expected: bool +): + helper = _load_path( + "branch_protection_glob_parity_test", + ROOT / "services/gitea/scripts/gitea_branch_protection_check.py", + ) + assert helper._matches(rule_name, branch) is expected + + +@pytest.mark.parametrize( + "rules", + [ + [{"rule_name": "main", "priority": 0, "created_at": "2026-01-01T00:00:00Z"}], + [{"rule_name": "main", "priority": 1, "created_at": "not-a-time"}], + [{"rule_name": "main"}], + ], +) +def test_branch_protection_rejects_ambiguous_rules_before_creation(rules): + helper = _load_path( + "branch_protection_adversarial_test", + ROOT / "services/gitea/scripts/gitea_branch_protection_check.py", + ) + with pytest.raises(helper.PolicyError): + helper.evaluate(json.dumps(rules).encode(), "main", "bstein") + + +def test_branch_protection_reports_absent_only_when_no_rule_matches(): + helper = _load_path( + "branch_protection_absent_test", + ROOT / "services/gitea/scripts/gitea_branch_protection_check.py", + ) + rules = [_protection("release/*", 1, helper)] + assert helper.evaluate(json.dumps(rules).encode(), "master", "bstein") == "ABSENT" diff --git a/testing/tests/test_hermes_scm_broker_streaming.py b/testing/tests/test_hermes_scm_broker_streaming.py new file mode 100644 index 00000000..2fa9683f --- /dev/null +++ b/testing/tests/test_hermes_scm_broker_streaming.py @@ -0,0 +1,237 @@ +"""Bounded I/O and concurrency contracts for the Hermes SCM broker.""" + +from __future__ import annotations + +import io +import json +from email.message import Message + +import pytest + +from testing.tests.test_hermes_scm_broker_support import Response, _load + + +def test_agent_broker_client_sends_no_credential_or_authorization_header(): + client = _load("scm_broker_client") + seen = [] + + def opener(request, timeout): + seen.append((request, timeout)) + return Response(b"{}", content_type="application/json") + + assert client.read("/api/v1/repos/atlas/cassandra", opener=opener) == b"{}" + request = seen[0][0] + assert request.full_url == client.BROKER_ORIGIN + "/v1/metadata" + assert request.get_header("Authorization") is None + assert json.loads(request.data) == {"path": "/api/v1/repos/atlas/cassandra"} + + +@pytest.mark.parametrize("raw", ["", "01", "9" * 4000, "134217729", "１２"]) +def test_broker_content_length_is_canonical_and_bounded(raw: str): + broker = _load("scm_broker") + headers = Message() + headers["Content-Length"] = raw + + with pytest.raises(broker.PolicyError, match="request length"): + broker._content_length(headers, broker.MAX_GIT_REQUEST) + + +def test_large_git_exchange_rolls_to_disk_and_scans_chunk_boundaries(): + broker = _load("scm_broker") + token = "runtime-sentinel" + safe = b"x" * (broker.SPOOL_MEMORY_LIMIT + 1) + inbound_timeouts = [] + spool, length = broker._spool_bounded( + io.BytesIO(safe), + broker.MAX_GIT_REQUEST, + len(safe), + token=token, + context="Git request", + deadline=broker.time.monotonic() + 30, + set_timeout=inbound_timeouts.append, + ) + try: + assert length == len(safe) + assert spool._rolled is True + assert spool.read(4) == b"xxxx" + assert inbound_timeouts + assert all(0 < value <= 30 for value in inbound_timeouts) + finally: + spool.close() + + crossing = b"x" * (broker.STREAM_CHUNK - 5) + token.encode() + b"tail" + with pytest.raises(broker.PolicyError, match="credential material"): + broker._spool_bounded( + io.BytesIO(crossing), + broker.MAX_GIT_REQUEST, + len(crossing), + token=token, + context="Git request", + deadline=broker.time.monotonic() + 30, + ) + + +def test_large_upstream_response_is_streamed_through_bounded_spool(): + broker = _load("scm_broker") + body = b"z" * (broker.SPOOL_MEMORY_LIMIT + 1) + seen = [] + + def opener(request, **_kwargs): + seen.append(request) + assert request.get_header("Content-length") == "7" + assert request.data.read() == b"request" + return Response(body, content_type="application/x-git-upload-pack-result") + + streamed = broker._upstream_git_request( + "/atlas/cassandra.git/git-upload-pack", + method="POST", + body=io.BytesIO(b"request"), + body_length=7, + content_type="application/x-git-upload-pack-request", + expected_type="application/x-git-upload-pack-result", + token="runtime-sentinel", + stream_result=True, + opener=opener, + ) + spool, length = streamed + try: + assert length == len(body) + assert spool._rolled is True + assert spool.read(3) == b"zzz" + assert len(seen) == 1 + finally: + spool.close() + + +def test_body_deadline_fails_closed_before_reading(): + broker = _load("scm_broker") + with pytest.raises(broker.PolicyError, match="deadline"): + broker._spool_bounded( + io.BytesIO(b"body"), + broker.MAX_GIT_REQUEST, + 4, + token="runtime-sentinel", + context="Git request", + deadline=broker.time.monotonic() - 1, + ) + + +def test_upstream_stream_applies_absolute_socket_deadline(): + broker = _load("scm_broker") + + class DeadlineSocket: + def __init__(self): + self.values = [] + + def settimeout(self, value): + self.values.append(value) + + response = Response(b"safe", content_type="application/x-git-upload-pack-result") + sock = DeadlineSocket() + response.fp = type("FP", (), {"raw": type("Raw", (), {"_sock": sock})()})() + spool, length = broker._spool_response( + response, broker.MAX_GIT_RESPONSE, "sentinel" + ) + try: + assert length == 4 + assert sock.values + assert all(0 < value <= broker.INBOUND_BODY_TIMEOUT for value in sock.values) + finally: + spool.close() + + +def test_broker_rejects_work_when_concurrency_slots_are_exhausted(): + broker = _load("scm_broker") + + class FakeSocket: + def __init__(self): + self.value = b"" + + def sendall(self, value): + self.value += value + + def shutdown(self, _how): + return None + + def close(self): + return None + + server = broker.BoundedThreadingHTTPServer(("127.0.0.1", 0), broker.BrokerHandler) + try: + assert broker.BoundedThreadingHTTPServer.__module__ == "scm_broker_server" + assert server._slots.acquire(blocking=False) + assert server._slots.acquire(blocking=False) + assert server._slots.acquire(blocking=False) + request = FakeSocket() + server.process_request(request, ("127.0.0.1", 1)) + assert b"503 Service Unavailable" in request.value + finally: + server._slots.release() + server._slots.release() + server._slots.release() + server.server_close() + + +def test_three_simultaneous_broker_calls_have_bounded_capacity(): + server_module = _load("scm_broker_server") + semaphore = server_module.threading.BoundedSemaphore( + server_module.MAX_CONCURRENT_REQUESTS + ) + assert server_module.MAX_CONCURRENT_REQUESTS >= 3 + assert [semaphore.acquire(blocking=False) for _ in range(3)] == [True, True, True] + assert semaphore.acquire(blocking=False) is False + + +def test_header_deadline_is_absolute_against_slow_trickle(monkeypatch): + server_module = _load("scm_broker_server") + + class Stream: + def __init__(self): + self.value = io.BytesIO(b"GET /healthz HTTP/1.1\r\n") + + def read(self, length): + return self.value.read(length) + + class Connection: + def __init__(self): + self.timeouts = [] + + def settimeout(self, value): + self.timeouts.append(value) + + ticks = iter((0.0, 0.02, 0.04, 0.06, 0.08, 0.11)) + monkeypatch.setattr(server_module.time, "monotonic", lambda: next(ticks)) + connection = Connection() + reader = server_module._AbsoluteDeadlineReader(Stream(), connection) + reader.begin(0.1) + + with pytest.raises(TimeoutError, match="absolute SCM header deadline"): + reader.readline(65537) + + assert connection.timeouts == pytest.approx([0.08, 0.06, 0.04, 0.02]) + + +def test_broker_bounds_header_count_size_and_duplicate_lengths(): + broker = _load("scm_broker") + + def validate(headers): + handler = object.__new__(broker.BrokerHandler) + handler.headers = headers + handler._validate_headers() + + too_many = Message() + for index in range(broker.MAX_HEADERS + 1): + too_many[f"X-Test-{index}"] = "safe" + with pytest.raises(broker.PolicyError, match="too many headers"): + validate(too_many) + + too_large = Message() + too_large["X-Test"] = "x" * broker.MAX_HEADER_BYTES + with pytest.raises(broker.PolicyError, match="safe limit"): + validate(too_large) + + duplicate = Message() + duplicate["Content-Length"] = "1" + duplicate["Content-Length"] = "1" + with pytest.raises(broker.PolicyError, match="duplicate Content-Length"): + validate(duplicate) diff --git a/testing/tests/test_hermes_scm_broker_support.py b/testing/tests/test_hermes_scm_broker_support.py new file mode 100644 index 00000000..6e2184d9 --- /dev/null +++ b/testing/tests/test_hermes_scm_broker_support.py @@ -0,0 +1,55 @@ +"""Shared fixtures for credential-isolated Hermes SCM broker tests.""" + +from __future__ import annotations + +import importlib.util +import io +import sys +from email.message import Message +from pathlib import Path + +ROOT = Path(__file__).parents[2] +SCRIPTS = ROOT / "services/hermes/scm-common/scripts" +sys.path.insert(0, str(SCRIPTS)) + + +def _load(name: str): + path = SCRIPTS / f"{name}.py" + spec = importlib.util.spec_from_file_location(f"test_{name}", path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _load_path(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +class Response: + def __init__(self, body: bytes, *, status: int = 200, content_type: str): + self.body = body + self.stream = io.BytesIO(body) + self.status = status + self.headers = Message() + self.headers["Content-Type"] = content_type + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def read(self, limit=-1): + return self.stream.read(limit) + + +def _receive_command(old: bytes, new: bytes, ref: bytes) -> bytes: + command = old + b" " + new + b" " + ref + b"\x00report-status\n" + return f"{len(command) + 4:04x}".encode() + command + b"0000PACK" diff --git a/testing/tests/test_hermes_scm_server_coverage.py b/testing/tests/test_hermes_scm_server_coverage.py new file mode 100644 index 00000000..7a358640 --- /dev/null +++ b/testing/tests/test_hermes_scm_server_coverage.py @@ -0,0 +1,180 @@ +"""Behavioral branch coverage for the SCM broker HTTP server primitives.""" + +from __future__ import annotations + +import io + +import pytest + +from testing.tests.test_hermes_scm_broker_support import _load + + +class _Connection: + def __init__(self): + self.timeouts = [] + + def settimeout(self, value): + self.timeouts.append(value) + + +class _FlushBuffer(io.BytesIO): + def __init__(self): + super().__init__() + self.flushes = 0 + + def flush(self): + self.flushes += 1 + + +def test_absolute_reader_supports_idle_limit_newline_eof_and_attribute_delegation(): + module = _load("scm_broker_server") + stream = io.BytesIO(b"first\nsecond") + reader = module._AbsoluteDeadlineReader(stream, _Connection()) + assert reader.readline(3) == b"fir" + reader.begin(5) + assert reader.readline() == b"st\n" + assert reader.readline() == b"second" + assert reader.readline() == b"" + assert reader.closed is False + reader.end() + assert reader.readline() == b"" + + +def _handler_type(module): + class StubHandler: + request_version = "HTTP/1.1" + command = "GET" + + def setup(self): + self.connection = _Connection() + self.rfile = io.BytesIO(b"") + self.wfile = _FlushBuffer() + + def parse_request(self): + return self.parse_result + + def send_error(self, code, *_args): + self.errors.append(code) + + def do_GET(self): + self.calls.append("GET") + + return type( + "DeadlineHandler", (module.AbsoluteHeaderDeadlineMixin, StubHandler), {} + ) + + +def _handler(module, raw: bytes, *, parse=True): + handler_type = _handler_type(module) + handler = object.__new__(handler_type) + handler.setup() + handler._header_reader._stream = io.BytesIO(raw) + handler.parse_result = parse + handler.errors = [] + handler.calls = [] + return handler + + +def test_header_mixin_setup_and_known_request_dispatch(): + module = _load("scm_broker_server") + handler = _handler(module, b"GET /healthz HTTP/1.1\r\n") + handler.handle_one_request() + assert handler.calls == ["GET"] + assert handler.wfile.flushes == 1 + assert handler._header_reader._deadline is None + + +@pytest.mark.parametrize( + ("raw", "parse", "command", "expected_error", "closed"), + [ + (b"x" * 65537, True, "GET", 414, False), + (b"", True, "GET", None, True), + (b"GET / HTTP/1.1\r\n", False, "GET", None, False), + (b"TRACE / HTTP/1.1\r\n", True, "TRACE", 501, False), + ], +) +def test_header_mixin_rejects_long_empty_unparsed_and_unknown_requests( + raw, parse, command, expected_error, closed +): + module = _load("scm_broker_server") + handler = _handler(module, raw, parse=parse) + handler.command = command + handler.close_connection = False + handler.handle_one_request() + assert handler.errors == ([] if expected_error is None else [expected_error]) + assert handler.close_connection is closed + + +def test_header_mixin_closes_on_absolute_timeout(monkeypatch): + module = _load("scm_broker_server") + handler = _handler(module, b"GET / HTTP/1.1\r\n") + handler.close_connection = False + monkeypatch.setattr( + handler._header_reader, + "readline", + lambda *_a, **_k: (_ for _ in ()).throw(TimeoutError()), + ) + handler.handle_one_request() + assert handler.close_connection is True + + +class _Slots: + def __init__(self, available=True): + self.available = available + self.releases = 0 + + def acquire(self, *, blocking): + assert blocking is False + return self.available + + def release(self): + self.releases += 1 + + +def _server_shell(module, available=True): + server = object.__new__(module.BoundedThreadingHTTPServer) + server._slots = _Slots(available) + server.shutdowns = [] + server.shutdown_request = server.shutdowns.append + return server + + +def test_bounded_server_delegates_accepted_request(monkeypatch): + module = _load("scm_broker_server") + server = _server_shell(module) + calls = [] + monkeypatch.setattr( + module.ThreadingHTTPServer, + "process_request", + lambda self, request, address: calls.append((request, address)), + ) + server.process_request("socket", ("127.0.0.1", 1)) + assert calls == [("socket", ("127.0.0.1", 1))] + assert server._slots.releases == 0 + + +def test_bounded_server_releases_on_dispatch_failure(monkeypatch): + module = _load("scm_broker_server") + server = _server_shell(module) + monkeypatch.setattr( + module.ThreadingHTTPServer, + "process_request", + lambda *_a, **_k: (_ for _ in ()).throw(RuntimeError("dispatch")), + ) + with pytest.raises(RuntimeError, match="dispatch"): + server.process_request("socket", ("127.0.0.1", 1)) + assert server._slots.releases == 1 + assert server.shutdowns == ["socket"] + + +def test_bounded_server_thread_always_releases(monkeypatch): + module = _load("scm_broker_server") + server = _server_shell(module) + monkeypatch.setattr( + module.ThreadingHTTPServer, + "process_request_thread", + lambda *_a, **_k: (_ for _ in ()).throw(RuntimeError("handler")), + ) + with pytest.raises(RuntimeError, match="handler"): + server.process_request_thread("socket", ("127.0.0.1", 1)) + assert server._slots.releases == 1 diff --git a/testing/tests/test_quality_coverage_helpers.py b/testing/tests/test_quality_coverage_helpers.py index 5eaa624e..fc22a8ab 100644 --- a/testing/tests/test_quality_coverage_helpers.py +++ b/testing/tests/test_quality_coverage_helpers.py @@ -12,10 +12,15 @@ def test_compute_workspace_line_coverage_handles_missing_xml(tmp_path: Path) -> """Missing coverage XML should produce a zero workspace coverage score.""" contract = {"coverage": {"tracked_files": ["managed.py"]}} - assert compute_workspace_line_coverage(contract, tmp_path, tmp_path / "missing.xml") == 0.0 + assert ( + compute_workspace_line_coverage(contract, tmp_path, tmp_path / "missing.xml") + == 0.0 + ) -def test_compute_workspace_line_coverage_averages_present_tracked_files(tmp_path: Path) -> None: +def test_compute_workspace_line_coverage_averages_present_tracked_files( + tmp_path: Path, +) -> None: """Workspace coverage should average only tracked files that appear in the report.""" coverage_xml = tmp_path / "coverage.xml" @@ -26,8 +31,8 @@ def test_compute_workspace_line_coverage_averages_present_tracked_files(tmp_path - - + + @@ -41,7 +46,9 @@ def test_compute_workspace_line_coverage_averages_present_tracked_files(tmp_path assert compute_workspace_line_coverage(contract, tmp_path, coverage_xml) == 75.0 -def test_run_check_keeps_relative_names_when_source_roots_do_not_match(tmp_path: Path) -> None: +def test_run_check_keeps_relative_names_when_source_roots_do_not_match( + tmp_path: Path, +) -> None: """Relative filenames should remain relative when no declared source root contains them.""" coverage_xml = tmp_path / "coverage.xml" @@ -74,3 +81,73 @@ def test_run_check_keeps_relative_names_when_source_roots_do_not_match(tmp_path: ) assert issues == ["coverage below 95.0%: relative.py (80.0%)"] + + +def test_run_check_enforces_branch_floor_and_requires_branch_data( + tmp_path: Path, +) -> None: + """Tracked modules must report and meet the configured branch floor.""" + + coverage_xml = tmp_path / "coverage.xml" + coverage_xml.write_text( + textwrap.dedent( + """\ + + + + + + + + + + + """ + ), + encoding="utf-8", + ) + + issues = run_check( + { + "coverage": { + "minimum_percent": 95.0, + "minimum_branch_percent": 95.0, + "tracked_files": ["low.py", "missing.py"], + } + }, + tmp_path, + coverage_xml, + ) + + assert issues == [ + "branch coverage below 95.0%: low.py (90.0%)", + "branch coverage missing for tracked file: missing.py", + ] + + +def test_run_check_limits_branch_floor_to_explicit_branch_tracked_files( + tmp_path: Path, +) -> None: + """A branch rollout can cover selected security modules without hiding lines.""" + + coverage_xml = tmp_path / "coverage.xml" + coverage_xml.write_text( + """ + + + """, + encoding="utf-8", + ) + issues = run_check( + { + "coverage": { + "minimum_percent": 95, + "minimum_branch_percent": 95, + "tracked_files": ["new.py", "legacy.py"], + "branch_tracked_files": ["new.py"], + } + }, + tmp_path, + coverage_xml, + ) + assert issues == []