diff --git a/clusters/atlas/flux-system/applications/hermes-observer-bindings/kustomization.yaml b/clusters/atlas/flux-system/applications/hermes-observer-bindings/kustomization.yaml new file mode 100644 index 00000000..b64913d5 --- /dev/null +++ b/clusters/atlas/flux-system/applications/hermes-observer-bindings/kustomization.yaml @@ -0,0 +1,19 @@ +# clusters/atlas/flux-system/applications/hermes-observer-bindings/kustomization.yaml +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: hermes-observer-bindings + namespace: flux-system +spec: + interval: 10m + path: ./services/hermes-observer-bindings + prune: true + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + wait: true + timeout: 5m + dependsOn: + - name: hermes + - name: hermes-observer-rbac diff --git a/clusters/atlas/flux-system/applications/hermes-observer-rbac/kustomization.yaml b/clusters/atlas/flux-system/applications/hermes-observer-rbac/kustomization.yaml new file mode 100644 index 00000000..c2749739 --- /dev/null +++ b/clusters/atlas/flux-system/applications/hermes-observer-rbac/kustomization.yaml @@ -0,0 +1,16 @@ +# clusters/atlas/flux-system/applications/hermes-observer-rbac/kustomization.yaml +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: hermes-observer-rbac + namespace: flux-system +spec: + interval: 10m + path: ./services/hermes-observer-rbac + prune: true + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + wait: true + timeout: 5m diff --git a/clusters/atlas/flux-system/applications/hermes-scm-broker-code/kustomization.yaml b/clusters/atlas/flux-system/applications/hermes-scm-broker-code/kustomization.yaml new file mode 100644 index 00000000..20ae82e2 --- /dev/null +++ b/clusters/atlas/flux-system/applications/hermes-scm-broker-code/kustomization.yaml @@ -0,0 +1,19 @@ +# clusters/atlas/flux-system/applications/hermes-scm-broker-code/kustomization.yaml +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: hermes-scm-broker-code + namespace: flux-system +spec: + interval: 10m + path: ./services/hermes/scm-common + targetNamespace: hermes-scm + prune: true + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + wait: true + timeout: 5m + dependsOn: + - name: hermes-scm-namespace diff --git a/clusters/atlas/flux-system/applications/hermes-scm-broker/kustomization.yaml b/clusters/atlas/flux-system/applications/hermes-scm-broker/kustomization.yaml new file mode 100644 index 00000000..70b3989e --- /dev/null +++ b/clusters/atlas/flux-system/applications/hermes-scm-broker/kustomization.yaml @@ -0,0 +1,27 @@ +# clusters/atlas/flux-system/applications/hermes-scm-broker/kustomization.yaml +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: hermes-scm-broker + namespace: flux-system +spec: + interval: 10m + path: ./services/hermes-scm-broker + targetNamespace: hermes-scm + prune: true + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + wait: true + timeout: 10m + healthChecks: + - apiVersion: apps/v1 + kind: Deployment + name: hermes-scm-broker + namespace: hermes-scm + dependsOn: + - name: vault + - name: gitea + - name: hermes-scm-namespace + - name: hermes-scm-broker-code diff --git a/clusters/atlas/flux-system/applications/hermes-scm-namespace/kustomization.yaml b/clusters/atlas/flux-system/applications/hermes-scm-namespace/kustomization.yaml new file mode 100644 index 00000000..9f066709 --- /dev/null +++ b/clusters/atlas/flux-system/applications/hermes-scm-namespace/kustomization.yaml @@ -0,0 +1,16 @@ +# clusters/atlas/flux-system/applications/hermes-scm-namespace/kustomization.yaml +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: hermes-scm-namespace + namespace: flux-system +spec: + interval: 10m + path: ./services/hermes-scm-namespace + prune: true + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + wait: true + timeout: 5m diff --git a/clusters/atlas/flux-system/applications/hermes/kustomization.yaml b/clusters/atlas/flux-system/applications/hermes/kustomization.yaml index 9a8365ea..b4b2759f 100644 --- a/clusters/atlas/flux-system/applications/hermes/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/hermes/kustomization.yaml @@ -61,3 +61,5 @@ spec: - name: longhorn - name: vault - name: jenkins + - name: hermes-scm-broker + - name: hermes-observer-rbac diff --git a/clusters/atlas/flux-system/applications/kustomization.yaml b/clusters/atlas/flux-system/applications/kustomization.yaml index 3303f4fa..794f0869 100644 --- a/clusters/atlas/flux-system/applications/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/kustomization.yaml @@ -28,7 +28,12 @@ resources: - jenkins/kustomization.yaml - ai-llm/kustomization.yaml - openclaw/kustomization.yaml + - hermes-scm-namespace/kustomization.yaml + - hermes-scm-broker-code/kustomization.yaml - hermes/kustomization.yaml + - hermes-observer-rbac/kustomization.yaml + - hermes-observer-bindings/kustomization.yaml + - hermes-scm-broker/kustomization.yaml - hermes-chat/kustomization.yaml - hermes-triage-demo/kustomization.yaml - game-stream/kustomization.yaml diff --git a/clusters/atlas/flux-system/applications/vault/kustomization.yaml b/clusters/atlas/flux-system/applications/vault/kustomization.yaml index 58f8ef7d..c3408a6e 100644 --- a/clusters/atlas/flux-system/applications/vault/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/vault/kustomization.yaml @@ -19,7 +19,7 @@ spec: healthChecks: - apiVersion: batch/v1 kind: Job - name: vault-k8s-auth-hermes-9 + name: vault-k8s-auth-hermes-10 namespace: vault dependsOn: - name: longhorn diff --git a/scripts/tests/test_dashboards_render_atlas.py b/scripts/tests/test_dashboards_render_atlas.py index bba16cdf..a522f849 100644 --- a/scripts/tests/test_dashboards_render_atlas.py +++ b/scripts/tests/test_dashboards_render_atlas.py @@ -416,101 +416,3 @@ def test_jobs_dashboard_bar_gauges_use_solid_threshold_colors(): assert {"color": "dark-yellow", "value": 93} in threshold_steps assert {"color": "dark-blue", "value": 100} in threshold_steps - - -def test_jobs_dashboard_collapses_heavy_drilldowns_for_light_first_paint(): - mod = load_module() - dashboard = mod.build_jobs_dashboard() - panels = dashboard["panels"] - rows = [panel for panel in panels if panel["type"] == "row"] - visible_query_panels = [panel for panel in panels if panel["type"] != "row"] - nested_panels_by_title = { - child["title"]: child - for row in rows - for child in row.get("panels", []) - } - - assert len(panels) == 18 - assert len(visible_query_panels) == 12 - assert sum(len(panel.get("targets", [])) for panel in visible_query_panels) == 12 - assert all( - panel["title"] != "Coverage Gap to 95% by Suite" - for panel in visible_query_panels - ) - assert [row["title"] for row in rows] == [ - "CI Runs And Test Result History", - "Check Failure Rates By Suite", - "Check Healthy Rates By Suite", - "Test Drilldowns And Problem Tests", - "Telemetry Completeness And Branches", - "SonarQube Project Health", - ] - assert all(row["collapsed"] for row in rows) - - assert "Coverage Failure Rate" in nested_panels_by_title - assert "Supply Chain Healthy Rate" in nested_panels_by_title - assert "Test Category Health History" in nested_panels_by_title - assert "Selected Test Pass Rate History" in nested_panels_by_title - assert "Coverage Metrics Present by Suite" in nested_panels_by_title - assert "SonarQube API Up" in nested_panels_by_title - - failure_rate_panel = nested_panels_by_title["Coverage Failure Rate"] - assert failure_rate_panel["type"] == "state-timeline" - assert failure_rate_panel["fieldConfig"]["defaults"]["unit"] == "percent" - assert failure_rate_panel["fieldConfig"]["defaults"]["max"] == 100 - assert failure_rate_panel["fieldConfig"]["defaults"]["thresholds"]["steps"][0]["color"] == "dark-blue" - assert "increase(" not in failure_rate_panel["targets"][0]["expr"] - assert "platform_quality:check_failed_flag:present_1h" in failure_rate_panel["targets"][0]["expr"] - assert "platform_quality:check_seen_flag:present_1h" in failure_rate_panel["targets"][0]["expr"] - assert "platform_quality:check_status:present_1h" not in failure_rate_panel["targets"][0]["expr"] - assert '.*_quality_gate_checks_total' not in failure_rate_panel["targets"][0]["expr"] - assert "last_over_time" not in failure_rate_panel["targets"][0]["expr"] - assert 'label_replace' not in failure_rate_panel["targets"][0]["expr"] - assert "0 *" in failure_rate_panel["targets"][0]["expr"] - assert "and on(suite)" not in failure_rate_panel["targets"][0]["expr"] - - pass_rate_panel = nested_panels_by_title["Selected Test Pass Rate History"] - assert pass_rate_panel["type"] == "state-timeline" - assert "platform_quality:test_case_pass_rate:percent_1h" in pass_rate_panel["targets"][0]["expr"] - assert "platform_quality_gate_test_case_result" not in pass_rate_panel["targets"][0]["expr"] - - pass_fail_panel = nested_panels_by_title["Selected Test Pass/Fail History"] - assert pass_fail_panel["fieldConfig"]["defaults"]["custom"]["drawStyle"] == "bars" - assert all( - "platform_quality:test_case_status:count_1h" in target["expr"] - for target in pass_fail_panel["targets"] - ) - - problematic_panel = nested_panels_by_title["Problematic Tests Over Time (Top failures)"] - assert problematic_panel["type"] == "state-timeline" - assert problematic_panel["gridPos"]["w"] == 24 - assert 'test!=""' in problematic_panel["targets"][0]["expr"] - assert "vector(0)" not in problematic_panel["targets"][0]["expr"] - assert problematic_panel["fieldConfig"]["defaults"]["thresholds"]["steps"] == [ - {"color": "dark-blue", "value": None}, - {"color": "dark-green", "value": 2}, - {"color": "dark-yellow", "value": 3}, - {"color": "dark-orange", "value": 5}, - {"color": "dark-red", "value": 8}, - ] - assert "rolling 24h failure count" in problematic_panel["description"] - assert "at least two recent failures" in problematic_panel["description"] - - sonar_mix_panel = nested_panels_by_title["Sonar Gate Status Mix (Selected)"] - sonar_health_panel = nested_panels_by_title["Sonar Gate Health by Project"] - assert sonar_mix_panel["gridPos"]["w"] == 4 - assert sonar_health_panel["gridPos"]["w"] == 8 - assert sonar_health_panel["type"] == "state-timeline" - assert "platform_quality:sonar_gate_health_percent:latest_1h" in sonar_health_panel["targets"][0]["expr"] - assert "sonarqube_project_quality_gate_pass" not in sonar_health_panel["targets"][0]["expr"] - - branch_panel = nested_panels_by_title["Primary Branch Clean by Suite (7d)"] - recent_branch_panel = nested_panels_by_title["Recent Branch Evidence by Suite (7d)"] - assert branch_panel["gridPos"]["x"] == 12 - assert recent_branch_panel["gridPos"]["x"] == 18 - assert "[7d:1h]" in recent_branch_panel["targets"][0]["expr"] - assert "[7d:1h]" in branch_panel["targets"][0]["expr"] - assert branch_panel["fieldConfig"]["defaults"]["unit"] == "percent" - assert "unless on(suite)" in branch_panel["targets"][0]["expr"] - assert "> bool 0" in branch_panel["targets"][0]["expr"] - assert branch_panel["targets"][0]["expr"].startswith("sort(") diff --git a/scripts/tests/test_dashboards_render_atlas_drilldowns.py b/scripts/tests/test_dashboards_render_atlas_drilldowns.py new file mode 100644 index 00000000..08871b71 --- /dev/null +++ b/scripts/tests/test_dashboards_render_atlas_drilldowns.py @@ -0,0 +1,127 @@ +"""Detailed Atlas Jobs dashboard collapse and drilldown contracts.""" + +from scripts.tests.test_dashboards_render_atlas import load_module + + +def test_jobs_dashboard_collapses_heavy_drilldowns_for_light_first_paint(): + mod = load_module() + dashboard = mod.build_jobs_dashboard() + panels = dashboard["panels"] + rows = [panel for panel in panels if panel["type"] == "row"] + visible_query_panels = [panel for panel in panels if panel["type"] != "row"] + nested_panels_by_title = { + child["title"]: child for row in rows for child in row.get("panels", []) + } + + assert len(panels) == 18 + assert len(visible_query_panels) == 12 + assert sum(len(panel.get("targets", [])) for panel in visible_query_panels) == 12 + assert all( + panel["title"] != "Coverage Gap to 95% by Suite" + for panel in visible_query_panels + ) + assert [row["title"] for row in rows] == [ + "CI Runs And Test Result History", + "Check Failure Rates By Suite", + "Check Healthy Rates By Suite", + "Test Drilldowns And Problem Tests", + "Telemetry Completeness And Branches", + "SonarQube Project Health", + ] + assert all(row["collapsed"] for row in rows) + + assert "Coverage Failure Rate" in nested_panels_by_title + assert "Supply Chain Healthy Rate" in nested_panels_by_title + assert "Test Category Health History" in nested_panels_by_title + assert "Selected Test Pass Rate History" in nested_panels_by_title + assert "Coverage Metrics Present by Suite" in nested_panels_by_title + assert "SonarQube API Up" in nested_panels_by_title + + failure_rate_panel = nested_panels_by_title["Coverage Failure Rate"] + assert failure_rate_panel["type"] == "state-timeline" + assert failure_rate_panel["fieldConfig"]["defaults"]["unit"] == "percent" + assert failure_rate_panel["fieldConfig"]["defaults"]["max"] == 100 + assert ( + failure_rate_panel["fieldConfig"]["defaults"]["thresholds"]["steps"][0]["color"] + == "dark-blue" + ) + assert "increase(" not in failure_rate_panel["targets"][0]["expr"] + assert ( + "platform_quality:check_failed_flag:present_1h" + in failure_rate_panel["targets"][0]["expr"] + ) + assert ( + "platform_quality:check_seen_flag:present_1h" + in failure_rate_panel["targets"][0]["expr"] + ) + assert ( + "platform_quality:check_status:present_1h" + not in failure_rate_panel["targets"][0]["expr"] + ) + assert ( + ".*_quality_gate_checks_total" not in failure_rate_panel["targets"][0]["expr"] + ) + assert "last_over_time" not in failure_rate_panel["targets"][0]["expr"] + assert "label_replace" not in failure_rate_panel["targets"][0]["expr"] + assert "0 *" in failure_rate_panel["targets"][0]["expr"] + assert "and on(suite)" not in failure_rate_panel["targets"][0]["expr"] + + pass_rate_panel = nested_panels_by_title["Selected Test Pass Rate History"] + assert pass_rate_panel["type"] == "state-timeline" + assert ( + "platform_quality:test_case_pass_rate:percent_1h" + in pass_rate_panel["targets"][0]["expr"] + ) + assert ( + "platform_quality_gate_test_case_result" + not in pass_rate_panel["targets"][0]["expr"] + ) + + pass_fail_panel = nested_panels_by_title["Selected Test Pass/Fail History"] + assert pass_fail_panel["fieldConfig"]["defaults"]["custom"]["drawStyle"] == "bars" + assert all( + "platform_quality:test_case_status:count_1h" in target["expr"] + for target in pass_fail_panel["targets"] + ) + + problematic_panel = nested_panels_by_title[ + "Problematic Tests Over Time (Top failures)" + ] + assert problematic_panel["type"] == "state-timeline" + assert problematic_panel["gridPos"]["w"] == 24 + assert 'test!=""' in problematic_panel["targets"][0]["expr"] + assert "vector(0)" not in problematic_panel["targets"][0]["expr"] + assert problematic_panel["fieldConfig"]["defaults"]["thresholds"]["steps"] == [ + {"color": "dark-blue", "value": None}, + {"color": "dark-green", "value": 2}, + {"color": "dark-yellow", "value": 3}, + {"color": "dark-orange", "value": 5}, + {"color": "dark-red", "value": 8}, + ] + assert "rolling 24h failure count" in problematic_panel["description"] + assert "at least two recent failures" in problematic_panel["description"] + + sonar_mix_panel = nested_panels_by_title["Sonar Gate Status Mix (Selected)"] + sonar_health_panel = nested_panels_by_title["Sonar Gate Health by Project"] + assert sonar_mix_panel["gridPos"]["w"] == 4 + assert sonar_health_panel["gridPos"]["w"] == 8 + assert sonar_health_panel["type"] == "state-timeline" + assert ( + "platform_quality:sonar_gate_health_percent:latest_1h" + in sonar_health_panel["targets"][0]["expr"] + ) + assert ( + "sonarqube_project_quality_gate_pass" + not in sonar_health_panel["targets"][0]["expr"] + ) + + branch_panel = nested_panels_by_title["Primary Branch Clean by Suite (7d)"] + recent_branch_panel = nested_panels_by_title["Recent Branch Evidence by Suite (7d)"] + assert branch_panel["gridPos"]["x"] == 12 + assert recent_branch_panel["gridPos"]["x"] == 18 + assert "[7d:1h]" in recent_branch_panel["targets"][0]["expr"] + assert "[7d:1h]" in branch_panel["targets"][0]["expr"] + assert branch_panel["fieldConfig"]["defaults"]["unit"] == "percent" + assert "unless on(suite)" in branch_panel["targets"][0]["expr"] + assert "> bool 0" in branch_panel["targets"][0]["expr"] + assert branch_panel["targets"][0]["expr"].startswith("sort(") diff --git a/services/gitea/atlas-identity-bootstrap-job.yaml b/services/gitea/atlas-identity-bootstrap-job.yaml index c7e2071f..bacecd01 100644 --- a/services/gitea/atlas-identity-bootstrap-job.yaml +++ b/services/gitea/atlas-identity-bootstrap-job.yaml @@ -2,7 +2,7 @@ apiVersion: batch/v1 kind: Job metadata: - name: gitea-atlas-identity-bootstrap-3 + name: gitea-atlas-identity-bootstrap-5 namespace: gitea labels: app.kubernetes.io/name: gitea @@ -34,6 +34,20 @@ spec: nodeSelector: node-role.kubernetes.io/worker: "true" hardware: rpi5 + initContainers: + - name: python-runtime + image: python@sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df + command: [/bin/sh, -ec] + args: ["cp -R /usr/local/. /python/"] + securityContext: + allowPrivilegeEscalation: false + capabilities: {drop: [ALL]} + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + volumeMounts: + - {name: python-runtime, mountPath: /python} containers: - name: bootstrap image: gitea/gitea:1.23 @@ -80,6 +94,7 @@ spec: - name: bootstrap mountPath: /opt/bootstrap readOnly: true + - {name: python-runtime, mountPath: /opt/python, readOnly: true} volumes: - name: data persistentVolumeClaim: @@ -88,3 +103,6 @@ spec: configMap: name: gitea-atlas-identity-bootstrap defaultMode: 0555 + - name: python-runtime + emptyDir: + sizeLimit: 128Mi diff --git a/services/gitea/kustomization.yaml b/services/gitea/kustomization.yaml index 8c473a38..6511418a 100644 --- a/services/gitea/kustomization.yaml +++ b/services/gitea/kustomization.yaml @@ -15,6 +15,7 @@ configMapGenerator: - name: gitea-atlas-identity-bootstrap files: - gitea_atlas_identity_ensure.sh=scripts/gitea_atlas_identity_ensure.sh + - gitea_branch_protection_check.py=scripts/gitea_branch_protection_check.py generatorOptions: disableNameSuffixHash: true diff --git a/services/gitea/scripts/gitea_atlas_identity_ensure.sh b/services/gitea/scripts/gitea_atlas_identity_ensure.sh index f98f08f5..6d5dbd90 100644 --- a/services/gitea/scripts/gitea_atlas_identity_ensure.sh +++ b/services/gitea/scripts/gitea_atlas_identity_ensure.sh @@ -19,6 +19,7 @@ source_owner="${GITEA_SOURCE_OWNER:-bstein}" managed_repositories="${GITEA_MANAGED_REPOSITORIES:-hermes-code-demo cassandra soteria pegasus metis ananke ariadne typhon titan-iac}" contributor_team=Contributors contributor_units='["repo.actions","repo.packages","repo.code","repo.issues","repo.wiki","repo.pulls","repo.releases","repo.projects"]' +protected_reviewer=bstein die() { echo "Gitea Atlas identity bootstrap failed: $*" >&2 @@ -272,6 +273,46 @@ transfer_managed_repositories() { trap - EXIT HUP INT TERM } +ensure_default_branch_protection() { + repository=$1 + repo_file=$(mktemp) + protections_file=$(mktemp) + response_file=$(mktemp) + + status=$(api_request GET "/repos/${organization}/${repository}" '' "${repo_file}") + expect_status "${status}" 200 "read ${repository} metadata for branch protection" + default_branch=$(sed -n 's/.*"default_branch":"\([^"]*\)".*/\1/p' "${repo_file}") + case "${default_branch}" in + main|master) ;; + *) die "${repository} has unsupported default branch ${default_branch:-missing}" ;; + esac + + status=$(api_request GET "/repos/${organization}/${repository}/branch_protections" '' "${protections_file}") + expect_status "${status}" 200 "list ${repository} branch protections" + if ! protection_state=$( + /opt/python/bin/python3 /opt/bootstrap/gitea_branch_protection_check.py \ + "${protections_file}" "${default_branch}" "${protected_reviewer}" + ); then + die "${repository} ${default_branch} protection differs from the human-review policy" + fi + + if [ "${protection_state}" = ABSENT ]; then + payload="{\"rule_name\":\"${default_branch}\",\"enable_push\":true,\"enable_push_whitelist\":true,\"push_whitelist_usernames\":[\"${protected_reviewer}\"],\"push_whitelist_teams\":[],\"push_whitelist_deploy_keys\":false,\"enable_force_push\":false,\"enable_force_push_allowlist\":false,\"force_push_allowlist_usernames\":[],\"force_push_allowlist_teams\":[],\"force_push_allowlist_deploy_keys\":false,\"enable_merge_whitelist\":true,\"merge_whitelist_usernames\":[\"${protected_reviewer}\"],\"merge_whitelist_teams\":[],\"enable_approvals_whitelist\":true,\"approvals_whitelist_username\":[\"${protected_reviewer}\"],\"approvals_whitelist_teams\":[],\"required_approvals\":1,\"block_on_rejected_reviews\":true,\"block_on_outdated_branch\":true,\"dismiss_stale_approvals\":true,\"block_admin_merge_override\":true}" + status=$(api_request POST "/repos/${organization}/${repository}/branch_protections" "${payload}" "${response_file}") + expect_status "${status}" 201 "protect ${repository} ${default_branch}" + status=$(api_request GET "/repos/${organization}/${repository}/branch_protections" '' "${protections_file}") + expect_status "${status}" 200 "read back ${repository} branch protections" + protection_state=$( + /opt/python/bin/python3 /opt/bootstrap/gitea_branch_protection_check.py \ + "${protections_file}" "${default_branch}" "${protected_reviewer}" + ) || die "${repository} ${default_branch} protection differs from the human-review policy" + fi + [ "${protection_state}" = PRESENT ] || \ + die "${repository} ${default_branch} protection did not become effective" + + rm -f "${repo_file}" "${protections_file}" "${response_file}" +} + wait_for_gitea ensure_user "${reconciler_user}" "${reconciler_email}" true ensure_user "${hermes_user}" "${hermes_email}" false @@ -318,6 +359,10 @@ expect_status "${status}" 204 "add Hermes to Atlas contributors" transfer_managed_repositories +for repository in ${managed_repositories}; do + ensure_default_branch_protection "${repository}" +done + vault_write gitea/atlas-reconciler "{\"data\":{\"username\":\"${reconciler_user}\",\"token\":\"${reconciler_token}\",\"base_url\":\"${public_url}\"}}" vault_write hermes/developer-gitea "{\"data\":{\"username\":\"${hermes_user}\",\"email\":\"${hermes_email}\",\"token\":\"${hermes_token}\",\"base_url\":\"${public_url}\"}}" diff --git a/services/gitea/scripts/gitea_branch_protection_check.py b/services/gitea/scripts/gitea_branch_protection_check.py new file mode 100644 index 00000000..5cb9717d --- /dev/null +++ b/services/gitea/scripts/gitea_branch_protection_check.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +"""Validate the first effective Gitea protection for an Atlas primary branch.""" + +from __future__ import annotations + +import argparse +import datetime +import json +import os +import stat +from pathlib import Path + +MAX_INPUT = 1024 * 1024 +MAX_RULES = 100 +MAX_RULE_NAME = 255 +SPECIAL = frozenset("*?\\[]{}") + + +class PolicyError(RuntimeError): + """The protection response is ambiguous or violates human review policy.""" + + +def _read_bounded(path: Path) -> bytes: + descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + try: + metadata = os.fstat(descriptor) + if not stat.S_ISREG(metadata.st_mode) or metadata.st_size > MAX_INPUT: + raise PolicyError("branch protection response exceeds the safe limit") + value = os.read(descriptor, MAX_INPUT + 1) + finally: + os.close(descriptor) + if len(value) != metadata.st_size: + raise PolicyError("branch protection response changed while reading") + return value + + +def _is_plain(pattern: str) -> bool: + return not any(character in SPECIAL for character in pattern) + + +def _literal_rule(pattern: str) -> str: + """Return one exact rule name or reject Gitea glob interpretation. + + Gitea 1.23.8 delegates matching to gobwas/glob. Reimplementing that + grammar and its error quoting rules here would create a second policy + engine. The reconciler instead accepts only plain ASCII literals. This + deliberately rejects braces, ranges, escapes, malformed globs, and even + special rules that appear unrelated: an operator must remove ambiguity + before this job may attest the primary-branch boundary. + """ + if not pattern.isascii() or not 1 <= len(pattern) <= MAX_RULE_NAME: + raise PolicyError("branch protection rule name is invalid") + if not _is_plain(pattern): + raise PolicyError("glob branch protection rules require human review") + return pattern + + +def _created(value: object) -> datetime.datetime: + if not isinstance(value, str) or len(value) > 64: + raise PolicyError("branch protection creation time is invalid") + try: + result = datetime.datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as exc: + raise PolicyError("branch protection creation time is invalid") from exc + if result.tzinfo is None: + raise PolicyError("branch protection creation time is invalid") + return result + + +def _required(reviewer: str) -> dict[str, object]: + return { + "enable_push": True, + "enable_push_whitelist": True, + "push_whitelist_usernames": [reviewer], + "push_whitelist_deploy_keys": False, + "enable_force_push": False, + "enable_merge_whitelist": True, + "merge_whitelist_usernames": [reviewer], + "enable_approvals_whitelist": True, + "approvals_whitelist_username": [reviewer], + "required_approvals": 1, + "block_on_rejected_reviews": True, + "block_on_outdated_branch": True, + "dismiss_stale_approvals": True, + "block_admin_merge_override": True, + } + + +def evaluate(value: bytes, branch: str, reviewer: str) -> str: + """Return PRESENT/ABSENT or fail on ambiguous/effective policy drift.""" + if len(value) > MAX_INPUT or branch not in {"main", "master"}: + raise PolicyError("branch protection input is invalid") + data = json.loads(value) + if not isinstance(data, list) or len(data) > MAX_RULES: + raise PolicyError("branch protection response is invalid") + ordered: list[tuple[int, datetime.datetime, int, dict[str, object]]] = [] + for index, raw in enumerate(data): + if not isinstance(raw, dict): + raise PolicyError("branch protection entry is invalid") + priority = raw.get("priority") + rule_name = raw.get("rule_name") + if ( + not isinstance(priority, int) + or isinstance(priority, bool) + or not 0 <= priority <= 1_000_000 + or not isinstance(rule_name, str) + ): + raise PolicyError("branch protection priority is ambiguous") + created = _created(raw.get("created_at")) + literal = _literal_rule(rule_name) + if literal == branch: + # With every rule constrained to a plain literal, Gitea's glob + # comparator cannot affect this result. Preserve API ordering: + # priority first (including legacy zero), creation time, then the + # stable response order for an otherwise exact tie. + ordered.append((priority, created, index, raw)) + if not ordered: + return "ABSENT" + if len(ordered) != 1: + raise PolicyError("duplicate primary-branch protections are ambiguous") + effective = ordered[0][3] + for key, expected in _required(reviewer).items(): + if effective.get(key) != expected: + raise PolicyError( + f"effective {branch} protection differs from human-review policy" + ) + return "PRESENT" + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("response", type=Path) + parser.add_argument("branch", choices=("main", "master")) + parser.add_argument("reviewer") + args = parser.parse_args() + try: + value = _read_bounded(args.response) + print(evaluate(value, args.branch, args.reviewer)) + except (OSError, json.JSONDecodeError, PolicyError) as exc: + print(f"branch protection check failed: {exc}", file=__import__("sys").stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/hermes-observer-bindings/kustomization.yaml b/services/hermes-observer-bindings/kustomization.yaml new file mode 100644 index 00000000..4dbf36b5 --- /dev/null +++ b/services/hermes-observer-bindings/kustomization.yaml @@ -0,0 +1,5 @@ +# services/hermes-observer-bindings/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - rolebindings.yaml diff --git a/services/hermes-observer-bindings/rolebindings.yaml b/services/hermes-observer-bindings/rolebindings.yaml new file mode 100644 index 00000000..403cd39a --- /dev/null +++ b/services/hermes-observer-bindings/rolebindings.yaml @@ -0,0 +1,204 @@ +# services/hermes-observer-bindings/rolebindings.yaml +apiVersion: v1 +kind: List +items: + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: ai} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: bstein-dev-home} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: cassandra} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: cert-manager} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: climate} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: comms} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: crypto} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: default} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: finance} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: flux-system} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: game-stream} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: gitea} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: harbor} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: health} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: hermes} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: hermes-chat} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: hermes-triage-demo} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: jellyfin} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: jenkins} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: kube-node-lease} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: kube-public} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: kube-system} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: logging} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: longhorn-system} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: mailu-mailserver} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: maintenance} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: metallb-system} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: monitoring} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: nextcloud} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: openclaw} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: outline} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: planka} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: postgres} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: quality} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: sso} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: sui-metrics} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: traefik} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: vault} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: vaultwarden} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: {name: hermes-agent-observer, namespace: veles} + subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}] + roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2} diff --git a/services/hermes-observer-rbac/kustomization.yaml b/services/hermes-observer-rbac/kustomization.yaml new file mode 100644 index 00000000..cff6f4b9 --- /dev/null +++ b/services/hermes-observer-rbac/kustomization.yaml @@ -0,0 +1,5 @@ +# services/hermes-observer-rbac/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - rbac.yaml diff --git a/services/hermes-observer-rbac/rbac.yaml b/services/hermes-observer-rbac/rbac.yaml new file mode 100644 index 00000000..737ac4dd --- /dev/null +++ b/services/hermes-observer-rbac/rbac.yaml @@ -0,0 +1,112 @@ +# services/hermes-observer-rbac/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: hermes-agent-cluster-observer-v2 + labels: + app.kubernetes.io/name: hermes-agent + app.kubernetes.io/part-of: hermes +rules: + - apiGroups: [""] + resources: + - namespaces + - nodes + - persistentvolumes + verbs: ["get", "list", "watch"] + - apiGroups: ["networking.k8s.io"] + resources: ["ingressclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: + - csidrivers + - csinodes + - storageclasses + - volumeattachments + verbs: ["get", "list", "watch"] + - apiGroups: ["metrics.k8s.io"] + resources: ["nodes"] + verbs: ["get", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: hermes-agent-namespaced-observer-v2 + labels: + app.kubernetes.io/name: hermes-agent + app.kubernetes.io/part-of: hermes +rules: + - apiGroups: [""] + resources: + - configmaps + - endpoints + - events + - persistentvolumeclaims + - pods + - pods/log + - replicationcontrollers + - resourcequotas + - services + verbs: ["get", "list", "watch"] + - apiGroups: ["apps"] + resources: + - controllerrevisions + - daemonsets + - deployments + - replicasets + - statefulsets + verbs: ["get", "list", "watch"] + - apiGroups: ["batch"] + resources: ["cronjobs", "jobs"] + verbs: ["get", "list", "watch"] + - apiGroups: ["autoscaling"] + resources: ["horizontalpodautoscalers"] + verbs: ["get", "list", "watch"] + - apiGroups: ["networking.k8s.io"] + resources: ["ingresses", "networkpolicies"] + verbs: ["get", "list", "watch"] + - apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["get", "list", "watch"] + - apiGroups: ["policy"] + resources: ["poddisruptionbudgets"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csistoragecapacities"] + verbs: ["get", "list", "watch"] + - apiGroups: ["metrics.k8s.io"] + resources: ["pods"] + verbs: ["get", "list"] + - apiGroups: + - helm.toolkit.fluxcd.io + - image.toolkit.fluxcd.io + - kustomize.toolkit.fluxcd.io + - notification.toolkit.fluxcd.io + - source.toolkit.fluxcd.io + resources: ["*"] + verbs: ["get", "list", "watch"] + - apiGroups: ["longhorn.io"] + resources: + - backingimages + - engines + - instancemanagers + - nodes + - replicas + - settings + - volumes + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: hermes-agent-cluster-observer-v2 + labels: + app.kubernetes.io/name: hermes-agent + app.kubernetes.io/part-of: hermes +subjects: + - kind: ServiceAccount + name: hermes-agent + namespace: hermes +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: hermes-agent-cluster-observer-v2 diff --git a/services/hermes-scm-broker/deployment.yaml b/services/hermes-scm-broker/deployment.yaml new file mode 100644 index 00000000..649ada1d --- /dev/null +++ b/services/hermes-scm-broker/deployment.yaml @@ -0,0 +1,98 @@ +# services/hermes-scm-broker/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hermes-scm-broker + namespace: hermes-scm + labels: + app: hermes-scm-broker +spec: + replicas: 1 + revisionHistoryLimit: 2 + selector: + matchLabels: + app: hermes-scm-broker + template: + metadata: + labels: + app: hermes-scm-broker + annotations: + vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/role: hermes-scm-broker + vault.hashicorp.com/agent-inject-secret-gitea-token: kv/data/atlas/hermes/developer-gitea + vault.hashicorp.com/agent-inject-template-gitea-token: | + {{- with secret "kv/data/atlas/hermes/developer-gitea" -}} + {{ .Data.data.token }} + {{- end }} + vault.hashicorp.com/agent-pre-populate-only: "true" + vault.hashicorp.com/agent-init-first: "true" + vault.hashicorp.com/agent-requests-cpu: 10m + vault.hashicorp.com/agent-requests-mem: 32Mi + vault.hashicorp.com/agent-limits-cpu: 50m + vault.hashicorp.com/agent-limits-mem: 64Mi + spec: + serviceAccountName: hermes-scm-broker + automountServiceAccountToken: true + securityContext: + fsGroup: 10000 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: + type: RuntimeDefault + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: [arm64] + - key: node-role.kubernetes.io/worker + operator: In + values: ["true"] + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: hardware + operator: In + values: [rpi5] + containers: + - name: broker + image: registry.bstein.dev/bstein/hermes-agent + imagePullPolicy: IfNotPresent + command: + - /opt/hermes/.venv/bin/python + - /opt/broker/scm_broker.py + ports: + - {name: http, containerPort: 9081, protocol: TCP} + readinessProbe: + httpGet: {path: /healthz, port: http} + initialDelaySeconds: 3 + periodSeconds: 10 + livenessProbe: + httpGet: {path: /healthz, port: http} + initialDelaySeconds: 15 + periodSeconds: 30 + failureThreshold: 10 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 10000 + runAsGroup: 10000 + volumeMounts: + - {name: broker-code, mountPath: /opt/broker, readOnly: true} + - {name: tmp, mountPath: /tmp} + resources: + requests: {cpu: 50m, memory: 128Mi} + limits: {cpu: "1", memory: 768Mi} + volumes: + - name: broker-code + configMap: + name: hermes-scm-boundary + defaultMode: 0555 + - name: tmp + emptyDir: + sizeLimit: 1Gi diff --git a/services/hermes-scm-broker/kustomization.yaml b/services/hermes-scm-broker/kustomization.yaml new file mode 100644 index 00000000..5dac3620 --- /dev/null +++ b/services/hermes-scm-broker/kustomization.yaml @@ -0,0 +1,11 @@ +# services/hermes-scm-broker/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: hermes-scm +resources: + - service.yaml + - deployment.yaml + - networkpolicy.yaml +images: + - name: registry.bstein.dev/bstein/hermes-agent + digest: sha256:37ebf720c783ae908a602916ffccf88d43d205a157957f5dc4b487867aee45e7 diff --git a/services/hermes-scm-broker/networkpolicy.yaml b/services/hermes-scm-broker/networkpolicy.yaml new file mode 100644 index 00000000..8187e333 --- /dev/null +++ b/services/hermes-scm-broker/networkpolicy.yaml @@ -0,0 +1,46 @@ +# services/hermes-scm-broker/networkpolicy.yaml +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: hermes-scm-broker-isolation + namespace: hermes-scm +spec: + podSelector: + matchLabels: + app: hermes-scm-broker + policyTypes: [Ingress, Egress] + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: hermes + podSelector: + matchLabels: + app: hermes-agent + ports: + - {protocol: TCP, port: 9081} + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - {protocol: UDP, port: 53} + - {protocol: TCP, port: 53} + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: vault + podSelector: + matchLabels: + app: vault + ports: + - {protocol: TCP, port: 8200} + - to: + - ipBlock: + cidr: 192.168.22.9/32 + ports: + - {protocol: TCP, port: 443} diff --git a/services/hermes-scm-broker/service.yaml b/services/hermes-scm-broker/service.yaml new file mode 100644 index 00000000..0b08c7f6 --- /dev/null +++ b/services/hermes-scm-broker/service.yaml @@ -0,0 +1,13 @@ +# services/hermes-scm-broker/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: hermes-scm-broker + namespace: hermes-scm +spec: + selector: + app: hermes-scm-broker + ports: + - name: http + port: 9081 + targetPort: http diff --git a/services/hermes-scm-namespace/kustomization.yaml b/services/hermes-scm-namespace/kustomization.yaml new file mode 100644 index 00000000..a9e89c7b --- /dev/null +++ b/services/hermes-scm-namespace/kustomization.yaml @@ -0,0 +1,5 @@ +# services/hermes-scm-namespace/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - namespace.yaml diff --git a/services/hermes-scm-namespace/namespace.yaml b/services/hermes-scm-namespace/namespace.yaml new file mode 100644 index 00000000..542f9602 --- /dev/null +++ b/services/hermes-scm-namespace/namespace.yaml @@ -0,0 +1,14 @@ +# services/hermes-scm-namespace/namespace.yaml +apiVersion: v1 +kind: Namespace +metadata: + name: hermes-scm + labels: + app.kubernetes.io/name: hermes-scm-broker +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: hermes-scm-broker + namespace: hermes-scm +automountServiceAccountToken: true diff --git a/services/hermes/agent-configmap.yaml b/services/hermes/agent-configmap.yaml index 0b81379d..28d520fb 100644 --- a/services/hermes/agent-configmap.yaml +++ b/services/hermes/agent-configmap.yaml @@ -189,9 +189,9 @@ data: not replace these coordinator-wide rules. Never call `kanban_show` without a known, non-empty task ID. Ad-hoc inspection and acceptance checks do not need a synthetic Kanban lookup, and must load a skill only when its workflow - materially applies. Atlas HTTPS Git authentication is already supplied by - the runtime-only `GIT_ASKPASS`; use it without reading or exposing the - credential. Coordinator guidance lives at + materially applies. Atlas Git access is supplied by the isolated SCM + broker; no repository token is present in this pod. Use only configured + broker remotes and clients. Coordinator guidance lives at `/opt/data/workspace/AGENTS.md` when more detail is needed. The Jetson classifier is mandatory for AUTO selection. Switchyard may use @@ -295,70 +295,75 @@ data: objective's difficulty warrants it; otherwise synthesize at the original effort. - This owner-only pod has cluster-admin access across Atlas, including logs, - Secrets, exec, port-forwarding, rollout operations, and Flux reconciliation. - Prefer the titan-iac Git/Flux workflow for every durable cluster change; - direct operations are available for explicit operator requests, incident - recovery, and verification, and must be followed by a matching source-of- - truth change when they alter desired state. Never expose credentials in - chat or logs. Triage belongs at triage.hermes.bstein.dev. + This owner-only pod has scoped read-only Kubernetes diagnostics across + Atlas, including resource status, events, logs, and Flux/Helm evidence. It + cannot read Secrets, exec or attach to pods, create service-account tokens, + mutate workloads or RBAC, or reconcile Flux. Put every durable cluster + change on a reviewed titan-iac branch. Never expose credentials in chat or + logs. Triage belongs at triage.hermes.bstein.dev. ## Atlas engineering access The Atlas organization has private visibility. Repository visibility is preserved per project and may be public or private; do not infer a repository's visibility from the organization setting. Repositories are - canonical at `https://scm.bstein.dev/atlas/.git`. HTTPS Git - authentication is already supplied through `GIT_ASKPASS`. Verify the remote + canonical at `https://scm.bstein.dev/atlas/.git`; worker Git traffic + uses `http://hermes-scm-broker.hermes-scm.svc.cluster.local:9081/git/atlas/.git`. + Verify the broker remote and cleanly separate pre-existing changes, create a task branch, run the repository's tests, use `git push --dry-run` when proving access, and push a real branch only when the requested implementation is review-ready. Never force-push. - `scm.bstein.dev` is Gitea, not GitHub. Never load or follow a GitHub/`gh` - skill for an Atlas remote, and do not interpret an unauthenticated Gitea - HTTP 404 as a missing private repository. Use authenticated Git for clone, - fetch, and push. For pull-request metadata, comments, and merges, call - `/opt/coordinator/gitea_api.py METHOD /api/v1/...`; it injects the runtime - Vault token without exposing it to the command line, environment, output, - or transcript. Before a consequential merge, independently verify the - exact base/head ancestry and diff, run the relevant tests, and confirm the - candidate Jenkins result. Prefer the internal endpoint in - `JENKINS_BASE_URL`; when Jenkins API authorization prevents a read, use the - existing cluster access to inspect the controller's job/build files and - logs rather than guessing a public hostname. The preferred read-only path - is `/opt/coordinator/jenkins_build_evidence.py JOB [--branch BRANCH] - [--commit SHA] --wait`; it distinguishes a genuinely terminal build from - nested Jenkins execution metadata and returns bounded JSON/log evidence. - With a branch, the helper also tries the conventional `JOB-branches` - multibranch name. For JSON Gitea mutations, prefer repeatable shell-safe - `--field KEY=VALUE` arguments over hand-quoted JSON. - After merging, observe the default-branch build to a terminal result and - report exact commit, build, and test evidence. + `scm.bstein.dev` is Forgejo/Gitea, not GitHub. Never load or follow a + GitHub/`gh` skill for an Atlas remote, and do not interpret an + unauthenticated Gitea HTTP 404 as a missing private repository. Use + brokered Git for clone, fetch, and creation of a new namespaced feature + branch. Existing-ref updates, protected refs, deletion, and force-push are + rejected. The broker inflates and scans every pushed object, so thin + packs are rejected; always push with `git push --no-thin` so the pack + is self-contained. For bounded + repository/pull-request evidence or to create a review-ready draft PR, load + `$manage-atlas-pull-requests` and use `/opt/scm/gitea_api.py`. The + client carries no repository credential; a separate least-authority + workload performs the allowed operation. It permits only Atlas-scoped + metadata reads and verified same-repository draft creation. Updates, merge, + approve, close, delete, comments, repository administration, and force-push + are unavailable; never bypass the client with raw HTTP. Leave every PR + unmerged for Brad's review. Independently + verify the exact base/head ancestry and diff, run the relevant tests, and + confirm the candidate Jenkins result before presenting a handoff. Prefer + the internal endpoint in + `JENKINS_BASE_URL`; when Jenkins API authorization prevents a read, use + Kubernetes pod logs and status evidence without exec or secret access. + Do not bypass the boundary to inspect controller files. If Brad separately + reports that he merged the PR, observe + the default-branch build to a terminal result and report exact commit, + build, and test evidence. The workspace already has a non-secret Hermes Git author identity. Do not use `git reset --hard`, even in a fresh clone; use a detached worktree or a clean branch switch when comparing revisions, and preserve any unexpected - file as user state. Use the Gitea pull-request merge endpoint for an open - PR so both Git history and PR state remain auditable. If an authorized - manual merge already reached the base branch, reconcile the open PR with - `Do=manually-merged` and its exact merge commit instead of leaving stale - review state. + file as user state. PR publication ends at the verified open draft; Brad + owns review and merge authority. The terminal PATH contains the pinned operator tools. Start cluster work with `kubectl config current-context`, read-only status/events/logs, and the - relevant `titan-iac` manifests. Put durable desired-state changes on a - reviewable `titan-iac` branch, validate Kustomize and client dry-run, then - use Flux reconciliation after the tracked change is published. Direct - `kubectl` mutations are for explicit incident recovery or ephemeral - verification, not ordinary delivery. + relevant `titan-iac` manifests. The broker namespace is deliberately + excluded from agent RBAC. Kubernetes cannot express a deny on one namespace + inside an all-namespace list, so enumerate namespaces and run namespaced + diagnostics instead of relying on `kubectl ... --all-namespaces`. Put + durable desired-state changes on a + reviewable `titan-iac` branch and validate Kustomize and client dry-run. + Brad owns merge and Flux reconciliation. Direct `kubectl` mutation is not + available to the agent; hand an explicit incident mutation back to Brad. - Node SSH uses a dedicated audited Hermes identity: `ssh titan-04` (or any - current Kubernetes node name). Host keys are pinned and password fallback - is disabled. Use SSH only for host-level evidence or repairs that cannot be - performed through Kubernetes. Read first, identify the exact node and - impact, avoid fleet-wide destructive commands, and reflect persistent host - configuration in the appropriate tracked provisioning source. + Node SSH uses the dedicated locked-password `hermes-agent` OS account: + `ssh titan-04` (or any current Kubernetes node name). Host keys are pinned, + password fallback is disabled, and the account has no sudo, disk, runtime, + or Kubernetes-storage authority. Use it for unprivileged host evidence; + hand privileged host repair back to Brad and reflect persistent changes in + the tracked provisioning source. Treat every credential, SSH identity, host-trust record, and credential-bearing access-client state as runtime-only Vault data. Keep @@ -395,7 +400,7 @@ data: [model]` for a persistent override. The first native Codex worker requires one device-code login; refreshed provider credentials persist through Vault while client caches remain disposable. The owner - workspace includes cluster-admin Kubernetes - access plus `kubectl`, `flux`, `helm`, `kustomize`, `vault`, `sops`, `age`, + workspace includes scoped read-only Kubernetes diagnostics plus `kubectl`, + `flux`, `helm`, `kustomize`, `vault`, `sops`, `age`, `terraform`, `k9s`, `jq`, `yq`, `gh`, Git, SSH, Python, Node, the browser/computer tools, and the native provider CLIs. diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml index 028c5d70..3fef73a4 100644 --- a/services/hermes/agent-deployment.yaml +++ b/services/hermes/agent-deployment.yaml @@ -46,16 +46,6 @@ spec: {{- with secret "kv/data/atlas/hermes/agent-tokens" -}} {{ .Data.data.codex_auth_json }} {{- end }} - vault.hashicorp.com/agent-inject-secret-gitea-token: kv/data/atlas/hermes/developer-gitea - vault.hashicorp.com/agent-inject-template-gitea-token: | - {{- with secret "kv/data/atlas/hermes/developer-gitea" -}} - {{ .Data.data.token }} - {{- end }} - vault.hashicorp.com/agent-inject-secret-gitea-username: kv/data/atlas/hermes/developer-gitea - vault.hashicorp.com/agent-inject-template-gitea-username: | - {{- with secret "kv/data/atlas/hermes/developer-gitea" -}} - {{ .Data.data.username }} - {{- end }} vault.hashicorp.com/agent-inject-secret-jenkins-image-build-token: kv/data/atlas/hermes/developer-jenkins vault.hashicorp.com/agent-inject-template-jenkins-image-build-token: | {{- with secret "kv/data/atlas/hermes/developer-jenkins" -}} @@ -191,9 +181,8 @@ spec: chmod 0600 "${profile_env}" chown 10000:10000 "${profile_env}" done - upsert_env GIT_ASKPASS /opt/coordinator/gitea_askpass.sh upsert_env GIT_TERMINAL_PROMPT 0 - upsert_env GITEA_BASE_URL https://scm.bstein.dev + upsert_env HERMES_SCM_BROKER_URL http://hermes-scm-broker.hermes-scm.svc.cluster.local:9081 upsert_env JENKINS_BASE_URL http://jenkins.jenkins.svc.cluster.local:8080 upsert_env ARIADNE_BASE_URL http://ariadne.maintenance.svc.cluster.local upsert_env VICTORIA_METRICS_URL http://victoria-metrics-single-server.monitoring.svc.cluster.local:8428 @@ -668,6 +657,8 @@ spec: - {name: subprocess-secret-patch, mountPath: /opt/hermes/tools/process_registry.py, subPath: process_registry.py} - {name: kubeconfig, mountPath: /opt/data/home/.kube/config, subPath: config, readOnly: true} - {name: auto-router-plugin, mountPath: /opt/data/plugins/auto-router, readOnly: true} + - {name: atlas-pr-skill, mountPath: /opt/data/workspace/skills/manage-atlas-pull-requests, readOnly: true} + - {name: scm-boundary, mountPath: /opt/scm, readOnly: true} - {name: routing-catalog, mountPath: /routing-catalog, readOnly: true} - {name: tmp, mountPath: /tmp} startupProbe: @@ -831,6 +822,8 @@ spec: - {name: subprocess-secret-patch, mountPath: /opt/hermes/tools/process_registry.py, subPath: process_registry.py} - {name: kubeconfig, mountPath: /opt/data/home/.kube/config, subPath: config, readOnly: true} - {name: auto-router-plugin, mountPath: /opt/data/plugins/auto-router, readOnly: true} + - {name: atlas-pr-skill, mountPath: /opt/data/workspace/skills/manage-atlas-pull-requests, readOnly: true} + - {name: scm-boundary, mountPath: /opt/scm, readOnly: true} - {name: routing-catalog, mountPath: /routing-catalog, readOnly: true} - {name: tmp, mountPath: /tmp} - {name: ttyd-index, mountPath: /ttyd-index, readOnly: true} @@ -899,6 +892,8 @@ spec: - {name: auth-patch, mountPath: /opt/hermes/hermes_cli/auth.py, subPath: auth.py} - {name: kubeconfig, mountPath: /opt/data/home/.kube/config, subPath: config, readOnly: true} - {name: auto-router-plugin, mountPath: /opt/data/plugins/auto-router, readOnly: true} + - {name: atlas-pr-skill, mountPath: /opt/data/workspace/skills/manage-atlas-pull-requests, readOnly: true} + - {name: scm-boundary, mountPath: /opt/scm, readOnly: true} - {name: tmp, mountPath: /tmp} resources: requests: {cpu: 100m, memory: 256Mi} @@ -1193,6 +1188,16 @@ spec: - {key: dashboard-api.py, path: dashboard/plugin_api.py} - {key: dashboard-index.js, path: dashboard/dist/index.js} - {key: dashboard-style.css, path: dashboard/dist/style.css} + - name: atlas-pr-skill + configMap: + name: hermes-atlas-pr-skill + items: + - {key: SKILL.md, path: SKILL.md} + - {key: openai.yaml, path: agents/openai.yaml} + - name: scm-boundary + configMap: + name: hermes-scm-boundary + defaultMode: 0555 - name: image-policy configMap: name: hermes-image-policy diff --git a/services/hermes/agent-kubeconfig.yaml b/services/hermes/agent-kubeconfig.yaml index 15b38da9..9ad9242f 100644 --- a/services/hermes/agent-kubeconfig.yaml +++ b/services/hermes/agent-kubeconfig.yaml @@ -11,9 +11,9 @@ users: user: tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token contexts: - - name: atlas-owner + - name: atlas-observer context: cluster: atlas user: hermes-agent namespace: default -current-context: atlas-owner +current-context: atlas-observer diff --git a/services/hermes/agent-rbac.yaml b/services/hermes/agent-rbac.yaml deleted file mode 100644 index 9dbff9e3..00000000 --- a/services/hermes/agent-rbac.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# services/hermes/agent-rbac.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: hermes-agent-cluster-admin - labels: - app.kubernetes.io/name: hermes-agent - app.kubernetes.io/part-of: hermes -subjects: - - kind: ServiceAccount - name: hermes-agent - namespace: hermes -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cluster-admin diff --git a/services/hermes/kustomization.yaml b/services/hermes/kustomization.yaml index 6009cd7b..8d6fc315 100644 --- a/services/hermes/kustomization.yaml +++ b/services/hermes/kustomization.yaml @@ -7,13 +7,13 @@ images: digest: sha256:37ebf720c783ae908a602916ffccf88d43d205a157957f5dc4b487867aee45e7 resources: - namespace.yaml + - scm-common - vault-serviceaccount.yaml - configmap.yaml - agent-configmap.yaml - chat-configmap.yaml - switchyard-configmap.yaml - rbac.yaml - - agent-rbac.yaml - node-ssh-access.yaml - pvc.yaml - switchyard-pvc.yaml @@ -69,8 +69,6 @@ configMapGenerator: - classifier_broker.py=scripts/classifier_broker.py - claude_oauth_broker.py=scripts/claude_oauth_broker.py - worker_route_broker.py=scripts/worker_route_broker.py - - gitea_api.py=scripts/gitea_api.py - - gitea_askpass.sh=scripts/gitea_askpass.sh - hermes_coordinator.py=scripts/hermes_coordinator.py - hermes_model_routing.py=scripts/hermes_model_routing.py - hermes_stt_client.py=scripts/hermes_stt_client.py @@ -105,6 +103,15 @@ configMapGenerator: - config=agent-kubeconfig.yaml options: disableNameSuffixHash: true + - name: hermes-node-account-hardener + namespace: hermes + files: + - node_account_hardening.py=scripts/node_account_hardening.py + - node_account_audit.py=scripts/node_account_audit.py + - node_account_io.py=scripts/node_account_io.py + - node_polkit_audit.py=scripts/node_polkit_audit.py + options: + disableNameSuffixHash: true - name: hermes-auto-router-plugin namespace: hermes files: @@ -159,3 +166,10 @@ configMapGenerator: - alert-review.md=skills/tune-atlas-alerts/references/alert-review.md options: disableNameSuffixHash: true + - name: hermes-atlas-pr-skill + namespace: hermes + files: + - SKILL.md=skills/manage-atlas-pull-requests/SKILL.md + - openai.yaml=skills/manage-atlas-pull-requests/agents/openai.yaml + options: + disableNameSuffixHash: true diff --git a/services/hermes/networkpolicy.yaml b/services/hermes/networkpolicy.yaml index d972889e..f9c3b09c 100644 --- a/services/hermes/networkpolicy.yaml +++ b/services/hermes/networkpolicy.yaml @@ -119,11 +119,65 @@ spec: app: server ports: - {protocol: TCP, port: 9010} - # agent.hermes.bstein.dev is an owner-only engineering workstation. The - # browser boundary remains OAuth-protected, while its workers need to reach - # every cluster namespace, Atlas LAN service, and hosted provider endpoint. egress: - - {} + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - {protocol: UDP, port: 53} + - {protocol: TCP, port: 53} + - to: + - namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: NotIn + values: [gitea, hermes-scm] + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: hermes-scm + podSelector: + matchLabels: + app: hermes-scm-broker + ports: + - {protocol: TCP, port: 9081} + - to: + - ipBlock: + cidr: 10.43.0.1/32 + ports: + - {protocol: TCP, port: 443} + - to: + - ipBlock: + cidr: 192.168.0.0/16 + except: + - 192.168.22.9/32 + - to: + - ipBlock: + cidr: 0.0.0.0/0 + except: + - 10.0.0.0/8 + - 100.64.0.0/10 + - 127.0.0.0/8 + - 169.254.0.0/16 + - 172.16.0.0/12 + - 192.168.0.0/16 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: hermes-node-ssh-access-isolation + namespace: hermes +spec: + podSelector: + matchLabels: + app: hermes-node-ssh-access + policyTypes: [Ingress, Egress] + ingress: [] + egress: [] --- apiVersion: networking.k8s.io/v1 kind: NetworkPolicy diff --git a/services/hermes/node-ssh-access.yaml b/services/hermes/node-ssh-access.yaml index d9c6b223..04b0c78b 100644 --- a/services/hermes/node-ssh-access.yaml +++ b/services/hermes/node-ssh-access.yaml @@ -45,32 +45,16 @@ spec: - operator: Exists containers: - name: key-reconciler - image: busybox:1.37 + # The pinned index supports the cluster's amd64 and arm64 nodes. + image: python@sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df imagePullPolicy: IfNotPresent command: [/bin/sh, -ec] args: - | - reconcile() { - key="$(cat /vault/secrets/node-ssh-public-key)" - found=0 - for user in atlas oceanus; do - home="/host-home/${user}" - [ -d "${home}" ] || continue - found=1 - identity="$(awk -F: -v name="${user}" '$1 == name {print $3 ":" $4; exit}' /host-etc/passwd)" - [ -n "${identity}" ] || identity="$(stat -c %u:%g "${home}")" - uid="${identity%%:*}" - gid="${identity##*:}" - install -d -m 0700 -o "${uid}" -g "${gid}" "${home}/.ssh" - touch "${home}/.ssh/authorized_keys" - grep -qxF "${key}" "${home}/.ssh/authorized_keys" || printf '%s\n' "${key}" >> "${home}/.ssh/authorized_keys" - chown "${uid}:${gid}" "${home}/.ssh/authorized_keys" - chmod 0600 "${home}/.ssh/authorized_keys" - done - [ "${found}" = 1 ] || { echo "no supported node SSH account found" >&2; return 1; } - } while true; do - reconcile + /usr/local/bin/python \ + /opt/node-hardener/node_account_hardening.py \ + --public-key-file /vault/secrets/node-ssh-public-key sleep 300 done securityContext: @@ -83,8 +67,21 @@ spec: volumeMounts: - name: host-home mountPath: /host-home - - name: host-passwd - mountPath: /host-etc/passwd + - name: host-etc + mountPath: /host-etc + - name: host-polkit-share + mountPath: /host-polkit-share + readOnly: true + - name: host-k3s + mountPath: /host-k3s + - name: host-kubelet + mountPath: /host-kubelet + - name: host-run-k3s + mountPath: /host-run-k3s + - name: host-run-containerd + mountPath: /host-run-containerd + - name: coordinator + mountPath: /opt/node-hardener readOnly: true - name: vault-secrets mountPath: /vault/secrets @@ -94,19 +91,43 @@ spec: resources: requests: cpu: 5m - memory: 8Mi + memory: 32Mi limits: cpu: 50m - memory: 32Mi + memory: 128Mi volumes: - name: host-home hostPath: path: /home type: Directory - - name: host-passwd + - name: host-etc hostPath: - path: /etc/passwd - type: File + path: /etc + type: Directory + - name: host-polkit-share + hostPath: + path: /usr/share/polkit-1 + type: Directory + - name: host-k3s + hostPath: + path: /var/lib/rancher/k3s + type: Directory + - name: host-kubelet + hostPath: + path: /var/lib/kubelet + type: Directory + - name: host-run-k3s + hostPath: + path: /run/k3s + type: Directory + - name: host-run-containerd + hostPath: + path: /run/containerd + type: Directory + - name: coordinator + configMap: + name: hermes-node-account-hardener + defaultMode: 0555 - name: vault-secrets csi: driver: secrets-store.csi.k8s.io diff --git a/services/hermes/scm-common/kustomization.yaml b/services/hermes/scm-common/kustomization.yaml new file mode 100644 index 00000000..a4756ccd --- /dev/null +++ b/services/hermes/scm-common/kustomization.yaml @@ -0,0 +1,17 @@ +# services/hermes/scm-common/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +configMapGenerator: + - name: hermes-scm-boundary + files: + - deadline_http.py=scripts/deadline_http.py + - git_pack_objects.py=scripts/git_pack_objects.py + - gitea_api.py=scripts/gitea_api.py + - gitea_api_policy.py=scripts/gitea_api_policy.py + - receive_pack_scan.py=scripts/receive_pack_scan.py + - scm_broker.py=scripts/scm_broker.py + - scm_broker_io.py=scripts/scm_broker_io.py + - scm_broker_server.py=scripts/scm_broker_server.py + - scm_broker_client.py=scripts/scm_broker_client.py + options: + disableNameSuffixHash: true diff --git a/services/hermes/scm-common/scripts/deadline_http.py b/services/hermes/scm-common/scripts/deadline_http.py new file mode 100644 index 00000000..e513a446 --- /dev/null +++ b/services/hermes/scm-common/scripts/deadline_http.py @@ -0,0 +1,344 @@ +#!/usr/bin/env python3 +"""Absolute wall-clock deadlines for every outbound Hermes HTTP exchange. + +Small control exchanges run inside a killable helper process, so connect, +send, read, and DNS all share one hard bound that no blocking socket state +can outlive. Large streaming Git exchanges cannot cross a process boundary; +they instead register their connection with a watchdog that force-closes the +socket when the same kind of absolute deadline expires. +""" + +from __future__ import annotations + +import base64 +import contextlib +import email.message +import http.client +import io +import json +import os +import signal +import socket +import subprocess +import sys +import threading +import time +import urllib.error +import urllib.request +from dataclasses import dataclass + +from gitea_api_policy import PolicyError + +MAX_CONTROL_BYTES = 128 * 1024 +MAX_PROTOCOL_BYTES = 4 * 1024 * 1024 +MAX_STREAM_SECONDS = 900.0 +READ_CHUNK = 64 * 1024 + + +class _RejectRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + raise urllib.error.URLError("redirect rejected") + + +@dataclass(frozen=True) +class Result: + """Minimal HTTP evidence returned across the process boundary.""" + + status: int + content_type: str + body: bytes + + +def _read_response(response, maximum: int, deadline: float) -> bytes: + chunks: list[bytes] = [] + total = 0 + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError + raw = getattr(getattr(response, "fp", None), "raw", None) + sock = getattr(raw, "_sock", raw) + if hasattr(sock, "settimeout"): + sock.settimeout(remaining) + chunk = response.read(min(READ_CHUNK, maximum + 1 - total)) + if not chunk: + break + chunks.append(chunk) + total += len(chunk) + if total > maximum: + raise ValueError("response too large") + return b"".join(chunks) + + +def _child_exchange(control: dict[str, object]) -> Result: + url = control.get("url") + method = control.get("method") + headers = control.get("headers") + encoded = control.get("body") + maximum = control.get("maximum") + timeout = control.get("timeout") + if ( + not isinstance(url, str) + or not isinstance(method, str) + or not isinstance(headers, dict) + or not all(isinstance(key, str) and isinstance(value, str) for key, value in headers.items()) + or not isinstance(encoded, str) + or not isinstance(maximum, int) + or not 0 <= maximum <= 2 * 1024 * 1024 + or not isinstance(timeout, (int, float)) + or not 0 < timeout <= 120 + ): + raise ValueError("invalid control") + body = base64.b64decode(encoded, validate=True) if encoded else None + if body is not None and len(body) > MAX_CONTROL_BYTES: + raise ValueError("request too large") + deadline = time.monotonic() + float(timeout) + request = urllib.request.Request( + url, data=body, method=method, headers=headers # type: ignore[arg-type] + ) + opener = urllib.request.build_opener(_RejectRedirect()) + try: + response = opener.open(request, timeout=max(deadline - time.monotonic(), 0.001)) + except urllib.error.HTTPError as error: + response = error + with response: + status = getattr(response, "status", response.getcode()) + content_type = response.headers.get_content_type() + value = _read_response(response, maximum, deadline) + return Result(int(status), str(content_type), value) + + +def _child_main() -> int: + try: + raw = sys.stdin.buffer.read(MAX_CONTROL_BYTES + 1) + if len(raw) > MAX_CONTROL_BYTES: + raise ValueError("control too large") + control = json.loads(raw) + if not isinstance(control, dict): + raise ValueError("invalid control") + result = _child_exchange(control) + output = { + "ok": True, + "status": result.status, + "content_type": result.content_type, + "body": base64.b64encode(result.body).decode("ascii"), + } + except Exception: + output = {"ok": False} + sys.stdout.write(json.dumps(output, separators=(",", ":"))) + return 0 if output["ok"] else 1 + + +def _terminate(process: subprocess.Popen[bytes]) -> None: + with contextlib.suppress(ProcessLookupError): + os.killpg(process.pid, signal.SIGKILL) + process.communicate() + + +def exchange( + request: urllib.request.Request, + *, + maximum: int, + timeout: float, + popen=subprocess.Popen, +) -> Result: + """Execute a whole request under one monotonic deadline and size limit.""" + if not 0 < timeout <= 120 or not 0 <= maximum <= 2 * 1024 * 1024: + raise PolicyError("HTTP control request bounds are invalid") + body = request.data or b"" + if not isinstance(body, bytes) or len(body) > MAX_CONTROL_BYTES: + raise PolicyError("HTTP control request exceeds the safe size limit") + control = json.dumps( + { + "url": request.full_url, + "method": request.get_method(), + "headers": dict(request.header_items()), + "body": base64.b64encode(body).decode("ascii"), + "maximum": maximum, + "timeout": timeout, + }, + separators=(",", ":"), + ).encode() + deadline = time.monotonic() + timeout + process = popen( + [sys.executable, os.path.realpath(__file__), "--child"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + start_new_session=True, + env={"LANG": "C.UTF-8", "PATH": "/usr/bin:/bin"}, + ) + try: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise subprocess.TimeoutExpired(process.args, timeout) + output, _error = process.communicate(control, timeout=remaining) + except subprocess.TimeoutExpired as exc: + _terminate(process) + raise PolicyError("HTTP control request deadline exceeded") from exc + if process.returncode != 0 or len(output) > MAX_PROTOCOL_BYTES: + raise PolicyError("HTTP control request failed") + try: + result = json.loads(output) + if not isinstance(result, dict) or result.get("ok") is not True: + raise ValueError + status = result["status"] + content_type = result["content_type"] + encoded = result["body"] + if not isinstance(status, int) or not isinstance(content_type, str) or not isinstance(encoded, str): + raise ValueError + value = base64.b64decode(encoded, validate=True) + except (KeyError, ValueError, TypeError, json.JSONDecodeError) as exc: + raise PolicyError("HTTP control helper returned invalid evidence") from exc + if len(value) > maximum: + raise PolicyError("HTTP control response exceeds the safe size limit") + return Result(status, content_type, value) + + +class _AdaptedResponse: + """Present one finished exchange through the urlopen reader interface.""" + + def __init__(self, result: Result): + self.status = result.status + self.headers = email.message.Message() + self.headers["Content-Type"] = result.content_type + self._stream = io.BytesIO(result.body) + + def read(self, limit: int = -1) -> bytes: + return self._stream.read(limit) + + def __enter__(self): + return self + + def __exit__(self, *_excinfo: object) -> bool: + return False + + +def open_bounded( + request: urllib.request.Request, + *, + maximum: int, + timeout: float, + popen=subprocess.Popen, +) -> _AdaptedResponse: + """Open one control exchange whose whole lifetime shares one deadline.""" + return _AdaptedResponse( + exchange(request, maximum=maximum, timeout=timeout, popen=popen) + ) + + +def _resolve_within(host: str, port: int, timeout: float) -> None: + """Resolve one host under a hard timeout the socket layer never covers. + + ``getaddrinfo`` ignores socket timeouts and runs before any socket + exists, so a slow or hostile resolver would otherwise outlive the whole + stream deadline. Running it in a joinable worker bounds resolution by the + same wall clock; a stuck lookup leaks only one daemon thread. + """ + outcome: dict[str, object] = {} + + def run() -> None: + try: + outcome["value"] = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM) + except OSError as exc: + outcome["error"] = exc + + worker = threading.Thread(target=run, daemon=True) + worker.start() + worker.join(timeout) + if worker.is_alive(): + raise PolicyError("HTTP stream DNS resolution deadline exceeded") + if "error" in outcome: + raise outcome["error"] # type: ignore[misc] + + +class StreamDeadline: + """Force-close tracked connections once an absolute deadline passes.""" + + def __init__(self, timeout: float, *, timer=threading.Timer): + if not 0 < timeout <= MAX_STREAM_SECONDS: + raise PolicyError("HTTP stream deadline bounds are invalid") + self._deadline = time.monotonic() + timeout + self._lock = threading.Lock() + self._connections: list[http.client.HTTPConnection] = [] + self.expired = False + self._timer = timer(timeout, self._expire) + self._timer.daemon = True + self._timer.start() + + def remaining(self) -> float: + value = self._deadline - time.monotonic() + if value <= 0 or self.expired: + raise PolicyError("HTTP stream deadline exceeded") + return value + + def _expire(self) -> None: + with self._lock: + self.expired = True + connections = list(self._connections) + for connection in connections: + # Shut the raw socket down first so a blocked send or recv wakes + # immediately; close alone leaves peers waiting on the old fd. + sock = getattr(connection, "sock", None) + if sock is not None: + with contextlib.suppress(OSError): + sock.shutdown(socket.SHUT_RDWR) + connection.close() + + def _track(self, connection: http.client.HTTPConnection): + with self._lock: + expired = self.expired + if not expired: + self._connections.append(connection) + if expired: + connection.close() + raise PolicyError("HTTP stream deadline exceeded") + return connection + + def _bounded_connection(self, connection: http.client.HTTPConnection): + """Register one connection and bound its DNS phase by the deadline. + + The watchdog can only close a live socket, but ``connect`` resolves + the host before any socket exists. Wrapping ``connect`` runs that + resolution under the same absolute deadline first, then hands off to + the real connect for the socket phases the watchdog already covers. + """ + original_connect = connection.connect + + def connect() -> None: + _resolve_within(connection.host, connection.port, self.remaining()) + original_connect() + + connection.connect = connect # type: ignore[method-assign] + return self._track(connection) + + def cancel(self) -> None: + self._timer.cancel() + + def handlers(self) -> tuple[urllib.request.BaseHandler, ...]: + """Build urllib handlers whose connections obey this deadline.""" + deadline = self + + class GuardedHTTPHandler(urllib.request.HTTPHandler): + def http_open(self, req): + return self.do_open( + lambda host, **kwargs: deadline._bounded_connection( + http.client.HTTPConnection(host, **kwargs) + ), + req, + ) + + class GuardedHTTPSHandler(urllib.request.HTTPSHandler): + def https_open(self, req): + return self.do_open( + lambda host, **kwargs: deadline._bounded_connection( + http.client.HTTPSConnection(host, **kwargs) + ), + req, + ) + + return (GuardedHTTPHandler(), GuardedHTTPSHandler()) + + +if __name__ == "__main__": + raise SystemExit(_child_main() if sys.argv[1:] == ["--child"] else 2) diff --git a/services/hermes/scm-common/scripts/git_pack_objects.py b/services/hermes/scm-common/scripts/git_pack_objects.py new file mode 100644 index 00000000..d9a3e715 --- /dev/null +++ b/services/hermes/scm-common/scripts/git_pack_objects.py @@ -0,0 +1,308 @@ +#!/usr/bin/env python3 +"""Bounded Git pack inflation and delta resolution for quarantine scanning. + +Pushed objects arrive zlib-compressed inside a packfile, so any content +screening must inflate the real payloads first. This module unpacks one +self-contained pack under explicit object, size, and checksum bounds and +resolves every delta against bases inside the same pack. Thin packs are +rejected outright: a delta against a base the broker cannot see would leave +part of the pushed content unscanned. +""" + +from __future__ import annotations + +import hashlib +import tempfile +import zlib +from typing import BinaryIO + +from gitea_api_policy import PolicyError + +OBJECT_TYPE_NAMES = {1: b"commit", 2: b"tree", 3: b"blob", 4: b"tag"} +OFS_DELTA = 6 +REF_DELTA = 7 +MAX_PACK_OBJECTS = 50_000 +MAX_OBJECT_BYTES = 32 * 1024 * 1024 +MAX_TOTAL_BYTES = 256 * 1024 * 1024 +MAX_RESOLVE_PASSES = 64 +SPOOL_MEMORY_LIMIT = 256 * 1024 +READ_CHUNK = 64 * 1024 + + +def _read_exact(stream: BinaryIO, size: int) -> bytes: + value = stream.read(size) + if len(value) != size: + raise PolicyError("Git pack data ended early") + return value + + +def _object_header(stream: BinaryIO) -> tuple[int, int]: + """Parse one object's type and declared inflated size.""" + byte = _read_exact(stream, 1)[0] + type_code = (byte >> 4) & 0x7 + size = byte & 0x0F + shift = 4 + while byte & 0x80: + if shift > 60: + raise PolicyError("Git object header is invalid") + byte = _read_exact(stream, 1)[0] + size |= (byte & 0x7F) << shift + shift += 7 + return type_code, size + + +def _delta_distance(stream: BinaryIO) -> int: + """Parse one ofs-delta base distance.""" + byte = _read_exact(stream, 1)[0] + value = byte & 0x7F + while byte & 0x80: + byte = _read_exact(stream, 1)[0] + value = ((value + 1) << 7) | (byte & 0x7F) + if value > 1 << 48: + raise PolicyError("Git delta base distance is invalid") + return value + + +def _inflate(stream: BinaryIO, declared: int) -> BinaryIO: + """Inflate one object into a quarantine spool, bounded by its header.""" + spool = tempfile.SpooledTemporaryFile( # noqa: SIM115 - caller owns returned spool + max_size=SPOOL_MEMORY_LIMIT, dir="/tmp" + ) + decompressor = zlib.decompressobj() + produced = 0 + try: + while not decompressor.eof: + data = stream.read(READ_CHUNK) + if not data: + raise PolicyError("Git pack object data ended early") + while True: + value = decompressor.decompress(data, READ_CHUNK) + produced += len(value) + if produced > declared: + raise PolicyError("Git object size does not match its header") + spool.write(value) + data = decompressor.unconsumed_tail + if decompressor.eof or not data: + break + if decompressor.unused_data: + stream.seek(-len(decompressor.unused_data), 1) + if produced != declared: + raise PolicyError("Git object size does not match its header") + spool.seek(0) + return spool + except zlib.error as exc: + spool.close() + raise PolicyError("Git pack object data is corrupt") from exc + except Exception: + spool.close() + raise + + +def _delta_size(delta: bytes, position: int) -> tuple[int, int]: + size = 0 + shift = 0 + while True: + if position >= len(delta) or shift > 60: + raise PolicyError("Git delta header is invalid") + byte = delta[position] + position += 1 + size |= (byte & 0x7F) << shift + shift += 7 + if not byte & 0x80: + return size, position + + +def _apply_delta(base: bytes, delta: bytes) -> bytes: + """Apply one Git delta and require exact declared sizes throughout.""" + source_size, position = _delta_size(delta, 0) + target_size, position = _delta_size(delta, position) + if source_size != len(base) or target_size > MAX_OBJECT_BYTES: + raise PolicyError("Git delta sizes are invalid") + parts: list[bytes] = [] + produced = 0 + while position < len(delta): + opcode = delta[position] + position += 1 + if opcode & 0x80: + operands = [bit for bit in (1, 2, 4, 8, 16, 32, 64) if opcode & bit] + if position + len(operands) > len(delta): + raise PolicyError("Git delta copy is truncated") + offset = 0 + size = 0 + for shift, bit in enumerate((1, 2, 4, 8)): + if opcode & bit: + offset |= delta[position] << (8 * shift) + position += 1 + for shift, bit in enumerate((16, 32, 64)): + if opcode & bit: + size |= delta[position] << (8 * shift) + position += 1 + size = size or 0x10000 + if offset + size > len(base): + raise PolicyError("Git delta copy is out of bounds") + parts.append(base[offset : offset + size]) + produced += size + elif opcode: + if position + opcode > len(delta): + raise PolicyError("Git delta insert is truncated") + parts.append(delta[position : position + opcode]) + position += opcode + produced += opcode + else: + raise PolicyError("Git delta instruction is invalid") + if produced > target_size: + raise PolicyError("Git delta result exceeds its declared size") + if produced != target_size: + raise PolicyError("Git delta result size is invalid") + return b"".join(parts) + + +def _payload_sha(type_code: int, payload: BinaryIO, size: int) -> bytes: + digest = hashlib.sha1(OBJECT_TYPE_NAMES[type_code] + b" %d\x00" % size) + payload.seek(0) + while True: + chunk = payload.read(READ_CHUNK) + if not chunk: + break + digest.update(chunk) + payload.seek(0) + return digest.digest() + + +def _verify_checksum(stream: BinaryIO, start: int, end: int) -> None: + trailer = _read_exact(stream, 20) + stream.seek(start) + digest = hashlib.sha1() + remaining = end - start + while remaining: + chunk = stream.read(min(READ_CHUNK, remaining)) + digest.update(chunk) + remaining -= len(chunk) + if digest.digest() != trailer: + raise PolicyError("Git pack checksum is invalid") + stream.seek(end + 20) + + +class _Entry: + """One pack entry: either resolved content or an unresolved delta.""" + + def __init__(self, type_code: int, payload: BinaryIO, size: int, base): + self.type_code = type_code + self.payload = payload + self.size = size + self.base = base + + +def _resolve(entries: dict[int, _Entry], total: list[int]) -> None: + """Resolve every delta against in-pack bases only, to a fixed point.""" + by_sha: dict[bytes, int] = {} + for offset, entry in entries.items(): + if entry.base is None: + by_sha.setdefault( + _payload_sha(entry.type_code, entry.payload, entry.size), offset + ) + for _ in range(MAX_RESOLVE_PASSES): + pending = [item for item in entries.items() if item[1].base is not None] + if not pending: + return + progressed = False + for offset, entry in pending: + base_key = entry.base + base_offset = base_key if isinstance(base_key, int) else by_sha.get(base_key) + base = entries.get(base_offset) if base_offset is not None else None + if base is None or base.base is not None: + if isinstance(base_key, int) and base is None: + raise PolicyError("Git delta base offset is invalid") + continue + base.payload.seek(0) + entry.payload.seek(0) + value = _apply_delta(base.payload.read(), entry.payload.read()) + total[0] += len(value) + if total[0] > MAX_TOTAL_BYTES: + raise PolicyError("Git pack contents exceed the safe size limit") + entry.payload.close() + spool = tempfile.SpooledTemporaryFile( # noqa: SIM115 - entry owns spool + max_size=SPOOL_MEMORY_LIMIT, dir="/tmp" + ) + spool.write(value) + spool.seek(0) + entry.type_code = base.type_code + entry.payload = spool + entry.size = len(value) + entry.base = None + by_sha.setdefault(_payload_sha(entry.type_code, spool, entry.size), offset) + progressed = True + if not progressed: + raise PolicyError("Git delta base is outside the pack; push full packs") + raise PolicyError("Git delta chains exceed the safe resolution limit") + + +def unpack_objects(stream: BinaryIO) -> list[tuple[int, BinaryIO]]: + """Inflate one self-contained pack and return every resolved payload.""" + start = stream.tell() + header = _read_exact(stream, 12) + version = int.from_bytes(header[4:8], "big") + count = int.from_bytes(header[8:12], "big") + if header[:4] != b"PACK" or version not in {2, 3}: + raise PolicyError("Git pack header is invalid") + if count > MAX_PACK_OBJECTS: + raise PolicyError("Git pack contains too many objects") + entries: dict[int, _Entry] = {} + total = [0] + try: + for _ in range(count): + offset = stream.tell() - start + type_code, declared = _object_header(stream) + if declared > MAX_OBJECT_BYTES: + raise PolicyError("Git object exceeds the safe size limit") + total[0] += declared + if total[0] > MAX_TOTAL_BYTES: + raise PolicyError("Git pack contents exceed the safe size limit") + base: int | bytes | None = None + if type_code == OFS_DELTA: + distance = _delta_distance(stream) + base = offset - distance + if base < 0: + raise PolicyError("Git delta base offset is invalid") + elif type_code == REF_DELTA: + base = _read_exact(stream, 20) + elif type_code not in OBJECT_TYPE_NAMES: + raise PolicyError("Git pack object type is invalid") + entries[offset] = _Entry(type_code, _inflate(stream, declared), declared, base) + _verify_checksum(stream, start, stream.tell()) + _resolve(entries, total) + except Exception: + for entry in entries.values(): + entry.payload.close() + raise + for entry in entries.values(): + entry.payload.seek(0) + return [ + (entries[offset].type_code, entries[offset].payload) + for offset in sorted(entries) + ] + + +def parse_loose_object(value: bytes) -> tuple[int, bytes]: + """Inflate one quarantined loose object and return its typed payload.""" + if len(value) > MAX_OBJECT_BYTES: + raise PolicyError("Git object exceeds the safe size limit") + decompressor = zlib.decompressobj() + try: + inflated = decompressor.decompress(value, MAX_OBJECT_BYTES + 1) + except zlib.error as exc: + raise PolicyError("Git loose object data is corrupt") from exc + if not decompressor.eof or decompressor.unused_data or len(inflated) > MAX_OBJECT_BYTES: + raise PolicyError("Git loose object framing is invalid") + kind, separator, remainder = inflated.partition(b"\x00") + name, space, size = kind.partition(b" ") + types = {label: code for code, label in OBJECT_TYPE_NAMES.items()} + if ( + not separator + or not space + or name not in types + or not size.isdigit() + or int(size) != len(remainder) + ): + raise PolicyError("Git loose object header is invalid") + return types[name], remainder diff --git a/services/hermes/scm-common/scripts/gitea_api.py b/services/hermes/scm-common/scripts/gitea_api.py new file mode 100644 index 00000000..3a7de241 --- /dev/null +++ b/services/hermes/scm-common/scripts/gitea_api.py @@ -0,0 +1,492 @@ +#!/usr/bin/env python3 +"""Use the private Atlas Forgejo API through a least-authority PR boundary.""" + +from __future__ import annotations + +import argparse +import base64 +import json +import os +import re +import sys +import urllib.error +import urllib.parse +import urllib.request +from collections.abc import Callable +from pathlib import Path + +import deadline_http +from gitea_api_policy import ( + PolicyError, + _draft_title, + _reject_forbidden, + _validate_body, + _validate_pr_number, + _validate_pr_number_segment, + _validate_query, + _validate_ref, + _validate_ref_bounds, + _validate_repo, + _validate_sha, +) + +CANONICAL_BASE_URL = "https://scm.bstein.dev" +ALLOWED_OWNER = "atlas" +DEFAULT_TOKEN_FILE = Path("/vault/secrets/gitea-token") +CREATE_PATH_RE = re.compile(r"/api/v1/repos/atlas/([^/]+)/pulls\Z") +MAX_ERROR_BYTES = 8192 +MAX_RESPONSE_BYTES = 2 * 1024 * 1024 +MAX_API_TARGET_LENGTH = 768 +MAX_API_PATH_LENGTH = 512 +RAW_API_TARGET_RE = re.compile(r"[A-Za-z0-9/_.?&=-]+\Z") + + +def _safe_urlopen(request: urllib.request.Request, timeout: int): + """Open one exchange under a killable absolute wall-clock deadline. + + The helper process rejects redirects before urllib can copy + authentication headers, and its whole connect/send/read lifetime + shares one hard deadline. + """ + return deadline_http.open_bounded( + request, maximum=MAX_RESPONSE_BYTES, timeout=timeout + ) + + +def read_token(path: Path = DEFAULT_TOKEN_FILE) -> str: + """Read the token from the pod-lifetime Vault projection.""" + token = path.read_text(encoding="utf-8").strip() + if not token: + raise ValueError(f"runtime credential is empty: {path}") + return token + + +def configured_base_url() -> str: + """Reject attempts to redirect credentials away from the canonical origin.""" + configured = os.environ.get("GITEA_BASE_URL", CANONICAL_BASE_URL) + if configured != CANONICAL_BASE_URL: + raise PolicyError("Forgejo origin is fixed to the private Atlas SCM service") + return configured + + +def _split_api_path(path: str) -> urllib.parse.SplitResult: + if not isinstance(path, str) or len(path) > MAX_API_TARGET_LENGTH: + raise PolicyError("API target exceeds the safe size limit") + if ( + not path.isascii() + or any(ord(character) < 32 or ord(character) == 127 for character in path) + or "\\" in path + or not RAW_API_TARGET_RE.fullmatch(path) + ): + raise PolicyError("API target contains non-canonical raw characters") + target = urllib.parse.urlsplit(path) + canonical = urllib.parse.urlunsplit(("", "", target.path, target.query, "")) + if canonical != path: + raise PolicyError("API target is not in exact canonical form") + if target.scheme or target.netloc or target.fragment: + raise PolicyError("API path must be relative to the Atlas SCM origin") + if not target.path.startswith("/api/v1/"): + raise PolicyError("API path must start with /api/v1/") + if len(target.path) > MAX_API_PATH_LENGTH: + raise PolicyError("API path exceeds the safe size limit") + segments = target.path.split("/") + if "//" in target.path or any(segment in {".", ".."} for segment in segments): + raise PolicyError("encoded or non-canonical API paths are not allowed") + return target + + +def _authorize_read( + target: urllib.parse.SplitResult, *, forbidden: tuple[str, ...] = () +) -> str: + """Allow only repository, PR, branch, commit, and status metadata reads.""" + prefix = "/api/v1/repos/atlas/" + remainder = target.path.removeprefix(prefix) + if remainder == target.path: + raise PolicyError("reads are limited to explicit Atlas repository metadata") + repo, separator, suffix = remainder.partition("/") + _reject_forbidden(repo, "repository", forbidden) + _validate_repo(repo) + + if not separator: + _validate_query(target, set()) + return "repository" + if suffix == "pulls": + _validate_query(target, {"page", "limit", "state"}) + return "pull-list" + content_match = re.fullmatch(r"pulls/([^/]+)\.(?:patch|diff)", suffix) + if content_match: + _validate_pr_number_segment(content_match.group(1)) + raise PolicyError("repository API route is outside the metadata read allowlist") + pull_match = re.fullmatch(r"pulls/([^/]+)", suffix) + if pull_match: + _validate_pr_number_segment(pull_match.group(1)) + _validate_query(target, set()) + return "pull" + evidence_match = re.fullmatch(r"pulls/([^/]+)/(?:commits|files)", suffix) + if evidence_match: + _validate_pr_number_segment(evidence_match.group(1)) + _validate_query(target, {"page", "limit"}) + return "pull-evidence" + if suffix == "branches": + _validate_query(target, {"page", "limit"}) + return "branch-list" + branch_match = re.fullmatch(r"branches/([^/]+)", suffix) + if branch_match: + _validate_ref(branch_match.group(1), "branch") + _validate_query(target, set()) + return "branch" + if suffix == "commits": + _validate_query(target, {"page", "limit"}) + return "commit-list" + if re.fullmatch(r"git/commits/[0-9a-fA-F]{40}", suffix): + _validate_query(target, set()) + return "commit" + if re.fullmatch( + r"(?:commits/[0-9a-fA-F]{40}/status|commits/[0-9a-fA-F]{40}/statuses|statuses/[0-9a-fA-F]{40})", + suffix, + ): + _validate_query(target, {"page", "limit"}) + return "status" + raise PolicyError("repository API route is outside the metadata read allowlist") + + +def api_url(base_url: str, path: str) -> str: + """Return a canonical same-origin URL without forwarding credentials.""" + if base_url != CANONICAL_BASE_URL: + raise PolicyError("Forgejo origin is fixed to the private Atlas SCM service") + target = _split_api_path(path) + return urllib.parse.urlunsplit( + ("https", "scm.bstein.dev", target.path, target.query, "") + ) + + +def authorize_request( + method: str, + path: str, + data: object | None, + *, + forbidden: tuple[str, ...] = (), +) -> str: + """Validate one request and return its bounded operation name.""" + normalized_method = method.upper() + target = _split_api_path(path) + if normalized_method == "GET": + if data is not None: + raise PolicyError("read operations cannot include a request body") + return _authorize_read(target, forbidden=forbidden) + if target.query: + raise PolicyError("mutating operations cannot include query parameters") + if normalized_method != "POST": + raise PolicyError("only read and create-draft operations are available") + match = CREATE_PATH_RE.fullmatch(target.path) + if not match: + raise PolicyError("POST is limited to creating an Atlas draft pull request") + _reject_forbidden(match.group(1), "repository", forbidden) + _validate_repo(match.group(1)) + if not isinstance(data, dict) or set(data) != {"base", "body", "head", "title"}: + raise PolicyError("draft creation accepts only base, body, head, and title") + if data["title"] != _draft_title(data["title"], forbidden=forbidden): + raise PolicyError("new pull requests must use the Gitea draft-title prefix") + _validate_body(data["body"], forbidden=forbidden) + _validate_ref_bounds(data["base"], "base", forbidden=forbidden) + _validate_ref_bounds(data["head"], "head", forbidden=forbidden) + _validate_ref(data["base"], "base", forbidden=forbidden) + _validate_ref(data["head"], "head", forbidden=forbidden) + return "create-draft" + + +def build_request( + method: str, + path: str, + *, + base_url: str, + token: str, + data: object | None = None, +) -> urllib.request.Request: + """Build an authorized request without putting the token in its URL or body.""" + authorize_request(method, path, data, forbidden=(token,)) + return _build_request(method, path, base_url=base_url, token=token, data=data) + + +def _build_request( + method: str, + path: str, + *, + base_url: str, + token: str, + data: object | None = None, +) -> urllib.request.Request: + """Build one already-authorized upstream request.""" + payload = None + if data is not None: + payload = json.dumps(data, separators=(",", ":")).encode("utf-8") + return urllib.request.Request( + api_url(base_url, path), + data=payload, + method=method.upper(), + headers={ + "Accept": "application/json", + "Authorization": f"token {token}", + "Content-Type": "application/json", + "User-Agent": "hermes-atlas-pr-client/2", + }, + ) + + +def redact_bytes(value: bytes, token: str) -> bytes: + """Remove exact runtime-token and authorization-header values from output.""" + redacted = value.replace(token.encode("utf-8"), b"[REDACTED]") + return re.sub( + rb"(?i)(authorization\s*[:=]\s*)(?:token|bearer)?\s*[^\s,;\"}]+", + rb"\1[REDACTED]", + redacted, + ) + + +def _reject_response_credential(value: bytes, token: str) -> None: + """Fail closed if an upstream body reflects any ordinary token encoding.""" + raw = token.encode("utf-8") + forms = { + raw, + base64.b64encode(raw), + base64.b64encode(b"hermes-automation:" + raw), + urllib.parse.quote(token, safe="").encode("ascii"), + } + if any(form and form in value for form in forms): + raise PolicyError("Forgejo response contains runtime credential material") + + +def _request( + method: str, + path: str, + data: object | None, + *, + token: str, + opener: Callable[..., object] = _safe_urlopen, + authorized: bool = False, + expected_status: int, +) -> bytes: + builder = _build_request if authorized else build_request + request = builder(method, path, base_url=configured_base_url(), token=token, data=data) + with opener(request, timeout=30) as response: # type: ignore[attr-defined] + status = getattr(response, "status", None) + if status is None and hasattr(response, "getcode"): + status = response.getcode() # type: ignore[attr-defined] + if status != expected_status: + raise PolicyError("Forgejo returned an unexpected HTTP status") + if response.headers.get_content_type() != "application/json": # type: ignore[attr-defined] + raise PolicyError("Forgejo returned an unexpected response type") + body = response.read(MAX_RESPONSE_BYTES + 1) # type: ignore[attr-defined] + if len(body) > MAX_RESPONSE_BYTES: + raise PolicyError("Forgejo response exceeds the safe size limit") + _reject_response_credential(body, token) + return redact_bytes(body, token) + + +def read( + path: str, *, token: str, opener: Callable[..., object] = _safe_urlopen +) -> bytes: + """Read one explicitly allowed Atlas repository metadata resource.""" + return _request( + "GET", path, None, token=token, opener=opener, expected_status=200 + ) + + +def _nested(document: dict[str, object], *keys: str) -> object: + value: object = document + for key in keys: + if not isinstance(value, dict) or key not in value: + raise PolicyError("Forgejo omitted required pull-request evidence") + value = value[key] + return value + + +def _require_create_response( + value: bytes, + *, + repo: str, + base: str, + head: str, + head_sha: str, + title: str, + body: str, +) -> dict[str, object]: + """Require exact server evidence for the intended open draft handoff.""" + try: + document = json.loads(value) + except json.JSONDecodeError as exc: + raise PolicyError("Forgejo returned invalid pull-request metadata") from exc + if not isinstance(document, dict): + raise PolicyError("Forgejo returned invalid pull-request metadata") + number = _validate_pr_number(document.get("number")) + full_name = f"{ALLOWED_OWNER}/{repo}" + expected_html = f"{CANONICAL_BASE_URL}/{full_name}/pulls/{number}" + checks = ( + document.get("state") == "open", + document.get("draft") is True, + document.get("merged") is False, + document.get("html_url") == expected_html, + document.get("url") == expected_html, + document.get("title") == title, + document.get("body") == body, + _nested(document, "base", "ref") == base, + _nested(document, "base", "repo", "full_name") == full_name, + _nested(document, "head", "ref") == head, + _nested(document, "head", "sha") == head_sha, + _nested(document, "head", "repo", "full_name") == full_name, + ) + if not all(checks): + raise PolicyError("Forgejo did not confirm the exact draft pull-request state") + return document + + +def create_draft( + repo: str, + *, + base: str, + head: str, + head_sha: str, + title: str, + body: str, + token: str, + opener: Callable[..., object] = _safe_urlopen, +) -> bytes: + """Create and verify a same-repository draft pull request for human review.""" + _reject_forbidden(repo, "repository", (token,)) + repo = _validate_repo(repo) + title = _draft_title(title, forbidden=(token,)) + body = _validate_body(body, forbidden=(token,)) + head_sha = _validate_sha(head_sha) + _validate_ref_bounds(base, "base", forbidden=(token,)) + _validate_ref_bounds(head, "head", forbidden=(token,)) + base = _validate_ref(base, "base", forbidden=(token,)) + head = _validate_ref(head, "head", forbidden=(token,)) + data = {"base": base, "body": body, "head": head, "title": title} + result = _request( + "POST", + f"/api/v1/repos/{ALLOWED_OWNER}/{repo}/pulls", + data, + token=token, + opener=opener, + authorized=True, + expected_status=201, + ) + _require_create_response( + result, + repo=repo, + base=base, + head=head, + head_sha=head_sha, + title=title, + body=body, + ) + return result + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + """Parse the intentionally narrow read and create-draft operations.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--dry-run", + action="store_true", + help="validate and describe the request without reading a token or using the network", + ) + subparsers = parser.add_subparsers(dest="operation", required=True) + read_parser = subparsers.add_parser("read", help="read allowed Atlas metadata") + read_parser.add_argument("path") + create_parser = subparsers.add_parser( + "create-draft", help="create a same-repository draft pull request" + ) + create_parser.add_argument("repo") + create_parser.add_argument("--base", required=True) + create_parser.add_argument("--head", required=True) + create_parser.add_argument("--head-sha", required=True) + create_parser.add_argument("--title", required=True) + create_parser.add_argument("--body", default="") + return parser.parse_args(argv) + + +def _operation( + args: argparse.Namespace, *, validate_refs: bool = True +) -> tuple[str, str, object | None]: + if args.operation == "read": + return "GET", args.path, None + repo = _validate_repo(args.repo) + _validate_sha(args.head_sha) + body = _validate_body(args.body) + title = _draft_title(args.title) + _validate_ref_bounds(args.base, "base") + _validate_ref_bounds(args.head, "head") + base = _validate_ref(args.base, "base") if validate_refs else args.base + head = _validate_ref(args.head, "head") if validate_refs else args.head + data = { + "base": base, + "body": body, + "head": head, + "title": title, + } + return "POST", f"/api/v1/repos/{ALLOWED_OWNER}/{repo}/pulls", data + + +def _write_body(body: bytes) -> None: + if body: + sys.stdout.buffer.write(body) + if not body.endswith(b"\n"): + sys.stdout.buffer.write(b"\n") + + +def main(argv: list[str] | None = None) -> int: + """Validate, optionally describe, and execute one safe Forgejo operation.""" + try: + args = parse_args(argv) + if args.dry_run: + method, path, data = _operation(args, validate_refs=False) + operation = authorize_request(method, path, data) + print( + json.dumps( + { + "dry_run": True, + "host": "scm.bstein.dev", + "operation": operation, + "owner": ALLOWED_OWNER, + "path": path, + "request_fields": sorted(data) + if isinstance(data, dict) + else [], + }, + sort_keys=True, + ) + ) + return 0 + method, path, data = _operation(args, validate_refs=False) + from scm_broker_client import create_draft as broker_create_draft + from scm_broker_client import read as broker_read + + if args.operation == "read": + body = broker_read(path) + else: + assert isinstance(data, dict) + body = broker_create_draft( + args.repo, + base=str(data["base"]), + head=str(data["head"]), + head_sha=args.head_sha, + title=str(data["title"]), + body=str(data["body"]), + ) + _write_body(body) + return 0 + except urllib.error.HTTPError as exc: + exc.read(MAX_ERROR_BYTES) + print(f"SCM broker request failed with HTTP {exc.code}", file=sys.stderr) + return 1 + except (OSError, PolicyError, ValueError, json.JSONDecodeError): + print( + "Forgejo request rejected or unavailable; no credential was disclosed", + file=sys.stderr, + ) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/hermes/scm-common/scripts/gitea_api_policy.py b/services/hermes/scm-common/scripts/gitea_api_policy.py new file mode 100644 index 00000000..db8833cf --- /dev/null +++ b/services/hermes/scm-common/scripts/gitea_api_policy.py @@ -0,0 +1,490 @@ +"""Validation policy for Hermes' least-authority Atlas Forgejo client. + +The screening catches structured credential assignments, known token formats, +and long high-entropy values. It is a fail-closed accident barrier, not proof +that text is secret-free: a short or multiword secret under an innocuous key is +not reliably distinguishable from prose and must never be supplied by callers. +JSON depth/nodes/documents and multiline gaps are deliberately bounded; large +or unusual configuration blobs are not a supported pull-request field format. +""" + +# ruff: noqa: SIM905 + +from __future__ import annotations + +import json +import re +import subprocess +import urllib.parse +from collections import Counter +from math import log2 + +REPO_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,99}\Z") +SHA_RE = re.compile(r"[0-9a-fA-F]{40}\Z") +DRAFT_TITLE_PREFIX = "WIP: " +GIT_BIN = "/usr/bin/git" +MAX_QUERY_LENGTH = 128 +MAX_QUERY_ITEMS = 3 +MAX_QUERY_KEY_LENGTH = 16 +MAX_QUERY_VALUE_LENGTH = 16 +MAX_PAGE = 10_000 +MAX_LIMIT = 50 +MAX_PR_NUMBER = 2_147_483_647 +MAX_PR_NUMBER_DIGITS = 10 +MAX_REF_CHARACTERS = 200 +MAX_REF_UTF8_BYTES = 255 +MAX_TITLE_UTF8_BYTES = 512 +MAX_BODY_UTF8_BYTES = 32_768 +MAX_JSON_DOCUMENTS = 32 +MAX_JSON_NODES = 2_048 +MAX_JSON_DEPTH = 32 +CANONICAL_QUERY_RE = re.compile(r"[A-Za-z0-9_=&-]*\Z") +ASSIGNMENT_RE = re.compile( + r"""(?mx) + (?[\"']?) + (?P\.?[A-Za-z][A-Za-z0-9_.-]{0,127}) + (?P=key_quote) + [ \t]*(?P[:=])[ \t]* + (?P + \"(?:\\.|[^\"\\\r\n])*\" | + '(?:\\.|[^'\\\r\n])*' | + [^\r\n,;}&]{0,2048} + ) + """ +) +JSON_KEY_RE = re.compile(r'"(?P[^"]{1,512})"(?P[\x00-\x20\x7f]{0,256}):') +MULTILINE_ASSIGNMENT_RE = re.compile( + r"""(?mx) + (?["']?) + (?P\.?[A-Za-z][A-Za-z0-9_.-]{0,127}) + (?P=key_quote) + [ \t]{0,64}(?:\r?\n[ \t]{0,64}){0,4} + (?P[:=]) + [ \t]{0,64}(?:\r?\n[ \t]{0,64}){1,4} + (?P[^\r\n,;}&]{1,2048}) + """ +) +CAMEL_BOUNDARY_RE = re.compile(r"(?<=[a-z0-9])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])") +CREDENTIAL_SCHEME_RE = re.compile( + r"(?i)\b(?:bearer|basic|token)\s+" + r"(?=[A-Za-z0-9+/_=.:-]{8,}\b)(?=[A-Za-z0-9+/_=.:-]*[0-9+/_=.-])" + r"[A-Za-z0-9+/_=.:-]{8,}" +) +STANDALONE_SECRET_PATTERNS = tuple( + re.compile(pattern, re.IGNORECASE) + for pattern in ( + r"(? str: + if not REPO_RE.fullmatch(repo) or repo in {".", ".."}: + raise PolicyError("repository name is outside the Atlas allowlist") + return repo + + +def _reject_forbidden(value: object, name: str, forbidden: tuple[str, ...]) -> None: + """Reject exact runtime credentials before invoking helpers or the network.""" + if not isinstance(value, str) or any(secret and secret in value for secret in forbidden): + raise PolicyError(f"{name} contains runtime credential material") + + +def _validate_ref_bounds( + value: object, name: str, *, forbidden: tuple[str, ...] = () +) -> str: + """Reject oversized or non-encodable refs before invoking Git.""" + _reject_forbidden(value, name, forbidden) + if not value or len(value) > MAX_REF_CHARACTERS: + raise PolicyError(f"{name} exceeds the safe branch-name limit") + try: + encoded = value.encode("utf-8", errors="strict") + except UnicodeEncodeError as exc: + raise PolicyError(f"{name} is not valid UTF-8 text") from exc + if len(encoded) > MAX_REF_UTF8_BYTES: + raise PolicyError(f"{name} exceeds the safe UTF-8 branch-name limit") + return value + + +def _validate_ref( + value: object, name: str, *, forbidden: tuple[str, ...] = () +) -> str: + """Validate the complete Git ref grammar using Git itself.""" + value = _validate_ref_bounds(value, name, forbidden=forbidden) + if value.startswith("-"): + raise PolicyError(f"{name} must be a same-repository branch name") + if any(ord(character) < 32 or ord(character) == 127 for character in value): + raise PolicyError(f"{name} must be a safe same-repository branch name") + result = subprocess.run( + [GIT_BIN, "check-ref-format", f"refs/heads/{value}"], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + if result.returncode != 0: + raise PolicyError(f"{name} must be a safe same-repository branch name") + return value + + +def _validate_sha(value: object, name: str = "head SHA") -> str: + if not isinstance(value, str) or not SHA_RE.fullmatch(value): + raise PolicyError(f"{name} must be a full Git SHA-1") + return value.lower() + + +def _validate_pr_number_segment(value: object) -> int: + """Validate a canonical bounded pull-request number from an API path.""" + if ( + not isinstance(value, str) + or len(value) > MAX_PR_NUMBER_DIGITS + or not re.fullmatch(r"[1-9][0-9]*", value) + ): + raise PolicyError("pull-request number must use bounded canonical ASCII digits") + number = int(value) + if number > MAX_PR_NUMBER: + raise PolicyError("pull-request number is outside its allowed range") + return number + + +def _validate_pr_number(value: object) -> int: + """Validate a bounded pull-request number returned by Forgejo.""" + if ( + not isinstance(value, int) + or isinstance(value, bool) + or not 1 <= value <= MAX_PR_NUMBER + ): + raise PolicyError("Forgejo omitted a valid pull-request number") + return value + + +def _validate_text( + value: object, + name: str, + maximum: int, + maximum_bytes: int, + *, + required: bool, +) -> str: + if not isinstance(value, str): + raise PolicyError(f"{name} must be text") + if required and not value.strip(): + raise PolicyError(f"{name} must not be empty") + try: + encoded = value.encode("utf-8", errors="strict") + except UnicodeEncodeError as exc: + raise PolicyError(f"{name} is not valid UTF-8 text") from exc + if len(value) > maximum or len(encoded) > maximum_bytes or "\x00" in value: + raise PolicyError(f"{name} exceeds the safe request limit") + return value + + +def _normalized_key(value: str) -> tuple[tuple[str, ...], str]: + """Split camelCase and separator-based assignment keys into semantics.""" + separated = CAMEL_BOUNDARY_RE.sub(" ", value.strip(".\"'")) + words = tuple(re.findall(r"[A-Za-z0-9]+", separated.lower())) + return words, "".join(words) + + +def _is_sensitive_key(value: str) -> bool: + words, compact = _normalized_key(value) + word_set = set(words) + if word_set & SENSITIVE_KEY_WORDS: + return True + if compact in SENSITIVE_COMPACT_KEYS: + return True + if compact.endswith(SENSITIVE_COMPACT_SUFFIXES): + return True + if "key" in word_set and word_set & KEY_MODIFIER_WORDS: + return True + if {"client", "email"} <= word_set or {"client", "id"} <= word_set: + return True + if {"access", "id"} <= word_set or {"connection", "string"} <= word_set: + return True + return compact.endswith("configjson") + + +def _strip_assignment_value(value: str) -> tuple[str, bool]: + stripped = value.strip() + quoted = ( + len(stripped) >= 2 and stripped[0] in {'"', "'"} and stripped[-1] == stripped[0] + ) + if quoted: + stripped = stripped[1:-1].strip() + return stripped, quoted + + +def _looks_like_prose(value: str) -> bool: + words = re.findall(r"[A-Za-z]+(?:[-'][A-Za-z]+)?", value) + if len(words) < 2: + return False + if re.search(r"[$`{}\[\]\\@/:=+_]", value): + return False + return words[0].lower() in PROSE_LEAD_WORDS + + +def _looks_sensitive_assignment_value(value: str) -> bool: + stripped, quoted = _strip_assignment_value(value) + if not stripped: + return False + if quoted or stripped.startswith(("{", "[", "|", ">", "!!", "&", "*")): + return True + if CREDENTIAL_SCHEME_RE.search(stripped): + return True + if any(pattern.search(stripped) for pattern in STANDALONE_SECRET_PATTERNS): + return True + if re.search(r"(?:https?://|[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+)", stripped): + return True + if re.search(r"(?:\$\{|\$\(|\\[nrt]|[A-Za-z0-9+/]{16,}={0,3}\Z)", stripped): + return True + return not _looks_like_prose(stripped) + + +def _is_structural_credential_assignment(key: str, value: str) -> bool: + words, compact_key = _normalized_key(key) + stripped, _quoted = _strip_assignment_value(value) + _value_words, compact_value = _normalized_key(stripped) + return ( + compact_key == "type" and compact_value in {"serviceaccount", "credential"} + ) or ("service" in words and "account" in words and bool(stripped)) + + +def _decode_json_key(raw: str) -> tuple[str, bool]: + """Decode one bounded JSON key without silently accepting bad escapes.""" + try: + decoded = json.loads(f'"{raw}"') + except (json.JSONDecodeError, UnicodeError): + approximate = re.sub(r"[^A-Za-z0-9]+", "_", raw.replace("\\", "")) + return approximate, False + return decoded if isinstance(decoded, str) else "", True + + +def _json_value_has_sensitive_assignment( + value: object, *, depth: int = 0, nodes: list[int] | None = None +) -> bool: + """Walk one decoded JSON value with explicit depth and node ceilings.""" + if nodes is None: + nodes = [0] + nodes[0] += 1 + if nodes[0] > MAX_JSON_NODES or depth > MAX_JSON_DEPTH: + raise PolicyError("structured pull-request text exceeds the scan limit") + if isinstance(value, dict): + for key, assigned in value.items(): + if not isinstance(key, str): + continue + if _is_sensitive_key(key): + return True + assigned_text = assigned if isinstance(assigned, str) else "" + if _is_structural_credential_assignment(key, assigned_text): + return True + if _json_value_has_sensitive_assignment( + assigned, depth=depth + 1, nodes=nodes + ): + return True + elif isinstance(value, list): + return any( + _json_value_has_sensitive_assignment(item, depth=depth + 1, nodes=nodes) + for item in value + ) + return False + + +def _decoded_json_documents(value: str): + """Yield bounded embedded JSON object or array fragments.""" + decoder = json.JSONDecoder() + index = 0 + attempts = 0 + while attempts < MAX_JSON_DOCUMENTS: + positions = [ + position for token in "{[" if (position := value.find(token, index)) >= 0 + ] + if not positions: + return + start = min(positions) + attempts += 1 + try: + document, end = decoder.raw_decode(value, start) + except (json.JSONDecodeError, RecursionError, ValueError): + index = start + 1 + continue + yield document + index = max(end, start + 1) + if "{" in value[index:] or "[" in value[index:]: + raise PolicyError("structured pull-request text exceeds the scan limit") + + +def _has_structured_sensitive_assignment(value: str) -> bool: + """Detect bounded JSON and multiline YAML/env credential assignments.""" + for candidate in JSON_KEY_RE.finditer(value): + decoded, valid = _decode_json_key(candidate.group("key")) + if _is_sensitive_key(decoded): + return True + if not valid and _is_sensitive_key(candidate.group("key").replace("\\", "")): + return True + for candidate in MULTILINE_ASSIGNMENT_RE.finditer(value): + key = candidate.group("key") + assigned = candidate.group("value") + if _is_structural_credential_assignment(key, assigned) or ( + _is_sensitive_key(key) and _looks_sensitive_assignment_value(assigned) + ): + return True + return any( + _json_value_has_sensitive_assignment(item) + for item in _decoded_json_documents(value) + ) + + +def _has_high_entropy_token(value: str) -> bool: + for match in HIGH_ENTROPY_TOKEN_RE.finditer(value): + token = match.group(0) + if len(token) > 256: + return True + groups = sum( + bool(re.search(pattern, token)) + for pattern in (r"[a-z]", r"[A-Z]", r"[0-9]", r"[+/_=-]") + ) + counts = Counter(token) + entropy = -sum( + (count / len(token)) * log2(count / len(token)) for count in counts.values() + ) + if groups >= 3 and entropy >= 4.0: + return True + return False + + +def _reject_sensitive(value: str, name: str, forbidden: tuple[str, ...]) -> None: + if any(secret and secret in value for secret in forbidden): + raise PolicyError(f"pull-request {name} contains runtime credential material") + if CREDENTIAL_SCHEME_RE.search(value): + raise PolicyError(f"pull-request {name} resembles credential material") + if any(pattern.search(value) for pattern in STANDALONE_SECRET_PATTERNS): + raise PolicyError(f"pull-request {name} resembles credential material") + if _has_structured_sensitive_assignment(value): + raise PolicyError(f"pull-request {name} resembles credential material") + for candidate in ASSIGNMENT_RE.finditer(value): + key = candidate.group("key") + assigned = candidate.group("value") + if _is_structural_credential_assignment(key, assigned) or ( + _is_sensitive_key(key) and _looks_sensitive_assignment_value(assigned) + ): + raise PolicyError(f"pull-request {name} resembles credential material") + if _has_high_entropy_token(value): + raise PolicyError(f"pull-request {name} resembles credential material") + + +def _validate_body(value: object, *, forbidden: tuple[str, ...] = ()) -> str: + body = _validate_text(value, "body", 16384, MAX_BODY_UTF8_BYTES, required=False) + _reject_sensitive(body, "body", forbidden) + return body + + +def _draft_title(value: object, *, forbidden: tuple[str, ...] = ()) -> str: + """Return a bounded, secret-screened Gitea draft title.""" + title = _validate_text( + value, "title", 251, MAX_TITLE_UTF8_BYTES, required=True + ).strip() + _reject_sensitive(title, "title", forbidden) + for prefix in ("WIP:", "[WIP]"): + if title.upper().startswith(prefix): + title = title[len(prefix) :].lstrip() + break + if not title: + raise PolicyError("title must contain text after the draft prefix") + return DRAFT_TITLE_PREFIX + title + + +def _validate_query(target: urllib.parse.SplitResult, allowed: set[str]) -> None: + """Accept only short canonical ASCII query strings with bounded pagination.""" + raw = target.query + if ( + len(raw) > MAX_QUERY_LENGTH + or not raw.isascii() + or not CANONICAL_QUERY_RE.fullmatch(raw) + or "%" in raw + ): + raise PolicyError("API query must use short canonical ASCII form") + try: + pairs = urllib.parse.parse_qsl(raw, keep_blank_values=True, strict_parsing=True) + except ValueError as exc: + raise PolicyError("invalid API query") from exc + if len(pairs) > MAX_QUERY_ITEMS: + raise PolicyError("too many API query parameters") + if len({key for key, _ in pairs}) != len(pairs): + raise PolicyError("duplicate API query parameters are not allowed") + if any( + not key + or len(key) > MAX_QUERY_KEY_LENGTH + or len(value) > MAX_QUERY_VALUE_LENGTH + for key, value in pairs + ): + raise PolicyError("API query key or value exceeds its safe limit") + if any(key not in allowed for key, _ in pairs): + raise PolicyError("API query parameter is outside the read allowlist") + values = dict(pairs) + for name in ("page", "limit"): + if name not in values: + continue + if not re.fullmatch(r"[0-9]+", values[name]): + raise PolicyError(f"{name} must use ASCII decimal digits") + number = int(values[name]) + if values[name] != str(number): + raise PolicyError(f"{name} must use canonical ASCII decimal form") + ceiling = MAX_PAGE if name == "page" else MAX_LIMIT + if not 1 <= number <= ceiling: + raise PolicyError(f"{name} is outside its allowed range") + if "state" in values and values["state"] not in {"open", "closed", "all"}: + raise PolicyError("pull-request state is invalid") diff --git a/services/hermes/scm-common/scripts/receive_pack_scan.py b/services/hermes/scm-common/scripts/receive_pack_scan.py new file mode 100644 index 00000000..76980880 --- /dev/null +++ b/services/hermes/scm-common/scripts/receive_pack_scan.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +"""Validate Git receive-pack requests and scan pushed object content. + +Ref commands may only create new namespaced feature branches. Every pushed +object is then inflated in quarantine and its decompressed payload scanned, +so credentials cannot ride through the broker inside compressed pack data. +The screening is a fail-closed accident barrier for credential-shaped +content: exact runtime-token forms, private keys, SSH key material, and +known provider token formats. Entropy heuristics are deliberately excluded +so SOPS ciphertext, pinned digests, and lock files remain pushable; a short +secret disguised as prose is not reliably detectable and must never be +supplied by callers. +""" + +from __future__ import annotations + +import io +import re +from typing import BinaryIO + +import git_pack_objects +from gitea_api_policy import ( + STANDALONE_SECRET_PATTERNS, + PolicyError, + _validate_ref, +) + +ZERO_SHA = b"0" * 40 +MAX_PUSH_COMMANDS = 16 +SCAN_CHUNK = 256 * 1024 +SCAN_CARRY = 8 * 1024 +FEATURE_REF_RE = re.compile( + r"refs/heads/(?:(?:feature|fix|hermes|handoff)/[A-Za-z0-9][A-Za-z0-9._/-]{0,190})\Z" +) +PKT_HEADER_RE = re.compile(rb"[0-9a-f]{4}\Z") +CONTENT_SECRET_PATTERNS = tuple( + re.compile(pattern.pattern.encode("ascii"), re.IGNORECASE) + for pattern in STANDALONE_SECRET_PATTERNS +) + ( + re.compile(rb"(?i)(? bytes | None: + """Read one pkt-line; None marks the flush packet before the pack.""" + header = stream.read(4) + if not header: + raise PolicyError("Git receive-pack request omitted its command terminator") + if not PKT_HEADER_RE.fullmatch(header): + raise PolicyError("Git receive-pack command framing is invalid") + size = int(header, 16) + if size == 0: + return None + if size < 4: + raise PolicyError("Git receive-pack command framing is invalid") + value = stream.read(size - 4) + if len(value) != size - 4: + raise PolicyError("Git receive-pack command framing is invalid") + return value + + +def _validate_command(line: bytes, token: str, forbidden: tuple[bytes, ...]) -> None: + """Permit only creation of one new, namespaced feature branch.""" + if any(form in line for form in forbidden): + raise PolicyError("Git request contains runtime credential material") + command = line.rstrip(b"\n").split(b"\x00", 1)[0] + fields = command.split(b" ") + if len(fields) != 3 or not all( + re.fullmatch(rb"[0-9a-f]{40}", item) for item in fields[:2] + ): + raise PolicyError("Git receive-pack ref command is invalid") + old_sha, new_sha, raw_ref = fields + if old_sha != ZERO_SHA or new_sha == ZERO_SHA: + raise PolicyError("Git broker permits only new feature-branch creation") + try: + ref = raw_ref.decode("ascii") + except UnicodeDecodeError as exc: + raise PolicyError("Git ref must be canonical ASCII") from exc + if not FEATURE_REF_RE.fullmatch(ref): + raise PolicyError("Git push is limited to namespaced feature branches") + _validate_ref(ref.removeprefix("refs/heads/"), "head", forbidden=(token,)) + + +def _scan_payload(payload: BinaryIO, forbidden: tuple[bytes, ...]) -> None: + """Scan one decompressed object payload across chunk boundaries.""" + carry = b"" + while True: + chunk = payload.read(SCAN_CHUNK) + if not chunk: + return + window = carry + chunk + if any(form in window for form in forbidden): + raise PolicyError("Git push contains runtime credential material") + if any(pattern.search(window) for pattern in CONTENT_SECRET_PATTERNS): + raise PolicyError("Git push contains credential-shaped content") + carry = window[-SCAN_CARRY:] + + +def _scan_pack(stream: BinaryIO, forbidden: tuple[bytes, ...]) -> None: + objects = git_pack_objects.unpack_objects(stream) + try: + if stream.read(1): + raise PolicyError("Git request has bytes after its pack") + for _type_code, payload in objects: + _scan_payload(payload, forbidden) + finally: + for _type_code, payload in objects: + payload.close() + + +def validate_receive_pack( + body: bytes | BinaryIO, token: str, forbidden: tuple[bytes, ...] +) -> None: + """Validate one whole receive-pack request: commands, pack, and content.""" + stream = io.BytesIO(body) if isinstance(body, bytes) else body + stream.seek(0) + try: + commands = 0 + while True: + line = _read_pkt_line(stream) + if line is None: + break + _validate_command(line, token, forbidden) + commands += 1 + if commands > MAX_PUSH_COMMANDS: + raise PolicyError("Git push contains too many ref commands") + if commands == 0: + raise PolicyError("Git receive-pack request has no ref command") + _scan_pack(stream, forbidden) + finally: + stream.seek(0) + + +def scan_loose_object(value: bytes, forbidden: tuple[bytes, ...]) -> int: + """Scan one quarantined loose object and return its parsed type code.""" + type_code, payload = git_pack_objects.parse_loose_object(value) + _scan_payload(io.BytesIO(payload), forbidden) + return type_code diff --git a/services/hermes/scm-common/scripts/scm_broker.py b/services/hermes/scm-common/scripts/scm_broker.py new file mode 100644 index 00000000..a9cb3685 --- /dev/null +++ b/services/hermes/scm-common/scripts/scm_broker.py @@ -0,0 +1,455 @@ +#!/usr/bin/env python3 +"""Serve the credential-isolated Atlas API and Git smart-HTTP boundary.""" + +from __future__ import annotations + +import argparse +import base64 +import json +import re +import socket +import tempfile +import time +import urllib.error +import urllib.parse +import urllib.request +from http.server import BaseHTTPRequestHandler +from typing import BinaryIO + +import deadline_http +from gitea_api import ( + CANONICAL_BASE_URL, + PolicyError, + create_draft, + read, + read_token, +) +from gitea_api_policy import _reject_forbidden, _validate_repo +from receive_pack_scan import validate_receive_pack +from scm_broker_io import RejectRedirect, response_status as _status, spool_response +from scm_broker_server import AbsoluteHeaderDeadlineMixin, BoundedThreadingHTTPServer + +BROKER_PORT = 9081 +GIT_USER = "hermes-automation" +MAX_CONTROL_BODY = 64 * 1024 +MAX_GIT_REQUEST = 128 * 1024 * 1024 +MAX_GIT_RESPONSE = 128 * 1024 * 1024 +MAX_HEADERS = 32 +MAX_HEADER_BYTES = 16 * 1024 +SPOOL_MEMORY_LIMIT = 1024 * 1024 +STREAM_CHUNK = 64 * 1024 +INBOUND_HEADER_TIMEOUT = 10.0 +INBOUND_BODY_TIMEOUT = 120.0 +UPSTREAM_DEADLINE_SECONDS = 300.0 +GIT_PATH_RE = re.compile( + r"/git/atlas/(?P[A-Za-z0-9][A-Za-z0-9._-]{0,99})\.git/" + r"(?Pinfo/refs|git-upload-pack|git-receive-pack)\Z" +) + + +def _read_bounded( + stream, + maximum: int, + length: int | None = None, + *, + deadline: float | None = None, + set_timeout=None, +) -> bytes: + if length is not None and not 0 <= length <= maximum: + raise PolicyError("SCM request exceeds the safe size limit") + wanted = maximum + 1 if length is None else length + chunks: list[bytes] = [] + total = 0 + while total < wanted: + if deadline is not None: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise PolicyError("SCM request body deadline exceeded") + if set_timeout is not None: + set_timeout(remaining) + chunk = stream.read(min(STREAM_CHUNK, wanted - total)) + if not chunk: + break + chunks.append(chunk) + total += len(chunk) + value = b"".join(chunks) + if len(value) > maximum or (length is not None and len(value) != length): + raise PolicyError("SCM request exceeds the safe size limit") + return value + + +def _spool_bounded( + stream, + maximum: int, + length: int, + *, + token: str, + context: str, + deadline: float, + set_timeout=None, +) -> tuple[BinaryIO, int]: + """Copy a fixed-length exchange through a bounded-memory disk spool.""" + if not 0 <= length <= maximum: + raise PolicyError("SCM request exceeds the safe size limit") + spool = tempfile.SpooledTemporaryFile( # noqa: SIM115 - caller owns returned spool + max_size=SPOOL_MEMORY_LIMIT, dir="/tmp" + ) + remaining = length + forms = _credential_forms(token) + carry = b"" + try: + while remaining: + remaining_time = deadline - time.monotonic() + if remaining_time <= 0: + raise PolicyError("SCM request body deadline exceeded") + if set_timeout is not None: + set_timeout(remaining_time) + chunk = stream.read(min(STREAM_CHUNK, remaining)) + if not chunk: + raise PolicyError("SCM request body ended early") + candidate = carry + chunk + if any(form in candidate for form in forms): + raise PolicyError(f"{context} contains runtime credential material") + width = max(len(form) for form in forms) - 1 + carry = candidate[-width:] if width else b"" + spool.write(chunk) + remaining -= len(chunk) + spool.seek(0) + return spool, length + except Exception: + spool.close() + raise + + +def _spool_response( + response, maximum: int, token: str, deadline_seconds: float +) -> tuple[BinaryIO, int]: + return spool_response( + response, + maximum, + _credential_forms(token), + memory_limit=SPOOL_MEMORY_LIMIT, + chunk_size=STREAM_CHUNK, + deadline_seconds=deadline_seconds, + ) + + +def _load_json(handler: BaseHTTPRequestHandler) -> dict[str, object]: + if handler.headers.get("Transfer-Encoding"): + raise PolicyError("chunked broker control requests are not allowed") + if handler.headers.get_content_type() != "application/json": + raise PolicyError("broker control request must be JSON") + length = _content_length(handler.headers, MAX_CONTROL_BODY) + deadline = time.monotonic() + INBOUND_BODY_TIMEOUT + value = json.loads( + _read_bounded( + handler.rfile, + MAX_CONTROL_BODY, + length, + deadline=deadline, + set_timeout=handler.connection.settimeout, + ) + ) + if not isinstance(value, dict): + raise PolicyError("broker control request must be an object") + return value + + +def _git_target(raw: str) -> tuple[str, str, str]: + if ( + not raw.isascii() + or any(ord(character) < 32 or ord(character) == 127 for character in raw) + or "%" in raw + or "\\" in raw + or len(raw) > 512 + ): + raise PolicyError("Git target is not canonical ASCII") + target = urllib.parse.urlsplit(raw) + canonical = urllib.parse.urlunsplit(("", "", target.path, target.query, "")) + if canonical != raw: + raise PolicyError("Git target is not in exact canonical form") + if target.scheme or target.netloc or target.fragment: + raise PolicyError("Git target must be relative") + match = GIT_PATH_RE.fullmatch(target.path) + if not match: + raise PolicyError("Git target is outside the Atlas allowlist") + repo = _validate_repo(match.group("repo")) + operation = match.group("operation") + query = target.query + if operation == "info/refs": + if query not in {"service=git-upload-pack", "service=git-receive-pack"}: + raise PolicyError("Git discovery service is outside the allowlist") + service = query.removeprefix("service=") + elif query: + raise PolicyError("Git RPC query parameters are not allowed") + else: + service = operation + return repo, operation, service + + +def _content_length(headers: object, maximum: int) -> int: + """Parse one short canonical bounded Content-Length header.""" + raw = headers.get("Content-Length", "") # type: ignore[attr-defined] + if ( + not isinstance(raw, str) + or not raw.isascii() + or len(raw) > 10 + or not raw.isdigit() + ): + raise PolicyError("SCM request length is invalid") + length = int(raw) + if raw != str(length) or not 0 <= length <= maximum: + raise PolicyError("SCM request length is invalid") + return length + + +def _credential_forms(token: str) -> tuple[bytes, ...]: + basic = base64.b64encode(f"{GIT_USER}:{token}".encode()) + return token.encode("utf-8"), basic, b"Basic " + basic + + +def _validate_receive_pack(body: bytes | BinaryIO, token: str) -> None: + """Permit only vetted new-feature-branch pushes with scanned contents.""" + validate_receive_pack(body, token, _credential_forms(token)) + + +def _guarded_opener(guard: deadline_http.StreamDeadline): + """Build an opener whose connection obeys one absolute stream deadline.""" + return urllib.request.build_opener(RejectRedirect(), *guard.handlers()).open + + +def _upstream_git_request( + target: str, + *, + method: str, + body: bytes | BinaryIO | None, + content_type: str | None, + expected_type: str, + token: str, + body_length: int | None = None, + stream_result: bool = False, + opener=None, +) -> bytes | tuple[BinaryIO, int]: + credentials = base64.b64encode(f"{GIT_USER}:{token}".encode()).decode("ascii") + headers = {"Accept": expected_type, "Authorization": f"Basic {credentials}"} + if content_type: + headers["Content-Type"] = content_type + if body is not None: + if body_length is None: + if not isinstance(body, bytes): + raise PolicyError("Git upstream request length is missing") + body_length = len(body) + headers["Content-Length"] = str(body_length) + request = urllib.request.Request( + CANONICAL_BASE_URL + target, + data=body, + method=method, + headers=headers, + ) + # One wall-clock budget covers connect, send, headers, and body read; the + # watchdog closes the upstream socket if any phase overstays it. + guard = deadline_http.StreamDeadline(UPSTREAM_DEADLINE_SECONDS) + try: + open_upstream = opener if opener is not None else _guarded_opener(guard) + with open_upstream(request, timeout=guard.remaining()) as response: + if _status(response) != 200: + raise PolicyError("Git upstream returned an unexpected HTTP status") + if response.headers.get_content_type() != expected_type: + raise PolicyError("Git upstream returned an unexpected response type") + result, result_length = _spool_response( + response, MAX_GIT_RESPONSE, token, guard.remaining() + ) + finally: + guard.cancel() + if stream_result: + return result, result_length + try: + return result.read() + finally: + result.close() + + +class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler): + """Expose only bounded metadata, draft creation, and smart-HTTP Git.""" + + server_version = "HermesSCMBroker/1" + sys_version = "" + header_deadline_seconds = INBOUND_HEADER_TIMEOUT + + def _validate_headers(self) -> None: + items = list(self.headers.items()) + if len(items) > MAX_HEADERS: + raise PolicyError("SCM request has too many headers") + total = 0 + for name, value in items: + if not name.isascii() or not value.isascii(): + raise PolicyError("SCM request headers must be ASCII") + total += len(name) + len(value) + 4 + if any(ord(character) < 32 and character != "\t" for character in value): + raise PolicyError("SCM request header contains controls") + if total > MAX_HEADER_BYTES: + raise PolicyError("SCM request headers exceed the safe limit") + if len(self.headers.get_all("Content-Length", [])) > 1: + raise PolicyError("SCM request has duplicate Content-Length") + + def _stream( + self, status: int, content_type: str, body: BinaryIO, length: int + ) -> None: + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(length)) + self.end_headers() + deadline = time.monotonic() + INBOUND_BODY_TIMEOUT + while True: + chunk = body.read(STREAM_CHUNK) + if not chunk: + break + remaining = deadline - time.monotonic() + if remaining <= 0: + raise PolicyError("SCM response write deadline exceeded") + self.connection.settimeout(remaining) + self.wfile.write(chunk) + + def log_message(self, _format: str, *_args: object) -> None: + return + + def _json(self, status: int, body: bytes) -> None: + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.send_header("Cache-Control", "no-store") + self.end_headers() + self.wfile.write(body) + + def _reject(self, status: int = 400) -> None: + self._json(status, b'{"error":"request rejected"}') + + def do_GET(self) -> None: + try: + self._validate_headers() + if self.path == "/healthz": + self._json(200, b'{"status":"ok"}') + return + repo, operation, service = _git_target(self.path) + if operation != "info/refs": + raise PolicyError("Git RPC requires POST") + token = read_token() + _reject_forbidden(repo, "repository", (token,)) + expected = f"application/x-{service}-advertisement" + streamed = _upstream_git_request( + f"/atlas/{repo}.git/info/refs?service={service}", + method="GET", + body=None, + content_type=None, + expected_type=expected, + token=token, + stream_result=True, + ) + body, length = streamed + try: + self._stream(200, expected, body, length) + finally: + body.close() + except ( + OSError, + PolicyError, + socket.timeout, + urllib.error.URLError, + ValueError, + ): + self._reject() + + def do_POST(self) -> None: + try: + self._validate_headers() + self.connection.settimeout(INBOUND_BODY_TIMEOUT) + if self.path in {"/v1/metadata", "/v1/drafts"}: + self._control() + else: + self._git_rpc() + except ( + OSError, + PolicyError, + socket.timeout, + urllib.error.URLError, + ValueError, + json.JSONDecodeError, + ): + self._reject() + + def _control(self) -> None: + data = _load_json(self) + token = read_token() + if self.path == "/v1/metadata": + if set(data) != {"path"} or not isinstance(data["path"], str): + raise PolicyError("metadata request fields are invalid") + result = read(data["path"], token=token) + else: + expected = {"base", "body", "head", "head_sha", "repo", "title"} + if set(data) != expected or not all( + isinstance(data[key], str) for key in expected + ): + raise PolicyError("draft request fields are invalid") + result = create_draft(token=token, **data) # type: ignore[arg-type] + if token.encode("utf-8") in result: + raise PolicyError("SCM upstream reflected credential material") + self._json(200, result) + + def _git_rpc(self) -> None: + repo, operation, service = _git_target(self.path) + if ( + operation not in {"git-upload-pack", "git-receive-pack"} + or service != operation + ): + raise PolicyError("Git RPC operation is outside the allowlist") + expected_request = f"application/x-{service}-request" + if self.headers.get_content_type() != expected_request or self.headers.get( + "Transfer-Encoding" + ): + raise PolicyError("Git RPC request type is invalid") + length = _content_length(self.headers, MAX_GIT_REQUEST) + token = read_token() + _reject_forbidden(repo, "repository", (token,)) + body, body_length = _spool_bounded( + self.rfile, + MAX_GIT_REQUEST, + length, + token=token, + context="Git request", + deadline=time.monotonic() + INBOUND_BODY_TIMEOUT, + set_timeout=self.connection.settimeout, + ) + try: + if service == "git-receive-pack": + _validate_receive_pack(body, token) + expected = f"application/x-{service}-result" + streamed = _upstream_git_request( + f"/atlas/{repo}.git/{service}", + method="POST", + body=body, + body_length=body_length, + content_type=expected_request, + expected_type=expected, + token=token, + stream_result=True, + ) + result, result_length = streamed + try: + self._stream(200, expected, result, result_length) + finally: + result.close() + finally: + body.close() + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--listen", default="0.0.0.0") + parser.add_argument("--port", type=int, default=BROKER_PORT) + args = parser.parse_args() + BoundedThreadingHTTPServer((args.listen, args.port), BrokerHandler).serve_forever() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/hermes/scm-common/scripts/scm_broker_client.py b/services/hermes/scm-common/scripts/scm_broker_client.py new file mode 100644 index 00000000..9a5265b6 --- /dev/null +++ b/services/hermes/scm-common/scripts/scm_broker_client.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""Call the credential-isolated Hermes SCM broker over its fixed cluster origin.""" + +from __future__ import annotations + +import json +import urllib.request +from collections.abc import Callable + +import deadline_http +from gitea_api_policy import PolicyError + +BROKER_ORIGIN = "http://hermes-scm-broker.hermes-scm.svc.cluster.local:9081" +MAX_RESPONSE_BYTES = 2 * 1024 * 1024 + + +def _open(request: urllib.request.Request, timeout: int): + """Open one broker exchange under a killable absolute deadline. + + The helper process keeps every request on its fixed in-cluster origin by + rejecting redirects before they are followed. + """ + return deadline_http.open_bounded( + request, maximum=MAX_RESPONSE_BYTES, timeout=timeout + ) + + +def request( + endpoint: str, + payload: dict[str, object], + *, + opener: Callable[..., object] = _open, +) -> bytes: + """Send one bounded broker operation without any repository credential.""" + if endpoint not in {"/v1/metadata", "/v1/drafts"}: + raise PolicyError("SCM broker operation is outside the client allowlist") + body = json.dumps(payload, separators=(",", ":")).encode("utf-8") + if len(body) > 64 * 1024: + raise PolicyError("SCM broker request exceeds the safe size limit") + outgoing = urllib.request.Request( + BROKER_ORIGIN + endpoint, + data=body, + method="POST", + headers={ + "Accept": "application/json", + "Content-Type": "application/json", + "User-Agent": "hermes-scm-broker-client/1", + }, + ) + with opener(outgoing, timeout=30) as response: # type: ignore[attr-defined] + status = getattr(response, "status", None) + if status is None and hasattr(response, "getcode"): + status = response.getcode() # type: ignore[attr-defined] + if status != 200: + raise PolicyError("SCM broker returned an unexpected HTTP status") + content_type = response.headers.get_content_type() # type: ignore[attr-defined] + if content_type != "application/json": + raise PolicyError("SCM broker returned an unexpected response type") + result = response.read(MAX_RESPONSE_BYTES + 1) # type: ignore[attr-defined] + if len(result) > MAX_RESPONSE_BYTES: + raise PolicyError("SCM broker response exceeds the safe size limit") + json.loads(result) + return result + + +def read(path: str, *, opener: Callable[..., object] = _open) -> bytes: + """Read explicitly allowed Atlas metadata through the broker.""" + return request("/v1/metadata", {"path": path}, opener=opener) + + +def create_draft( + repo: str, + *, + base: str, + head: str, + head_sha: str, + title: str, + body: str, + opener: Callable[..., object] = _open, +) -> bytes: + """Create one verified draft through the broker for human review.""" + return request( + "/v1/drafts", + { + "base": base, + "body": body, + "head": head, + "head_sha": head_sha, + "repo": repo, + "title": title, + }, + opener=opener, + ) diff --git a/services/hermes/scm-common/scripts/scm_broker_io.py b/services/hermes/scm-common/scripts/scm_broker_io.py new file mode 100644 index 00000000..96df1fe0 --- /dev/null +++ b/services/hermes/scm-common/scripts/scm_broker_io.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""Bounded-memory stream helpers for the Hermes SCM broker.""" + +from __future__ import annotations + +import tempfile +import time +import urllib.request +from typing import BinaryIO + +from gitea_api_policy import PolicyError + + +class RejectRedirect(urllib.request.HTTPRedirectHandler): + """Reject every upstream redirect before credentials can be forwarded.""" + + def redirect_request(self, req, fp, code, msg, headers, newurl): + raise PolicyError("SCM upstream redirects are not allowed") + + +def response_status(response: object) -> int | None: + """Read an HTTP status from either urllib response interface.""" + value = getattr(response, "status", None) + if value is None and hasattr(response, "getcode"): + value = response.getcode() # type: ignore[attr-defined] + return value + + +def spool_response( + response, + maximum: int, + forbidden: tuple[bytes, ...], + *, + memory_limit: int, + chunk_size: int, + deadline_seconds: float, +) -> tuple[BinaryIO, int]: + """Spool an upstream response to disk while scanning chunk boundaries.""" + spool = tempfile.SpooledTemporaryFile( # noqa: SIM115 - caller owns returned spool + max_size=memory_limit, dir="/tmp" + ) + carry = b"" + total = 0 + deadline = time.monotonic() + deadline_seconds + try: + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise PolicyError("SCM upstream response deadline exceeded") + raw = getattr(getattr(response, "fp", None), "raw", None) + sock = getattr(raw, "_sock", raw) + if hasattr(sock, "settimeout"): + sock.settimeout(remaining) + chunk = response.read(chunk_size) + if not chunk: + break + total += len(chunk) + if total > maximum: + raise PolicyError("SCM upstream response exceeds the safe size limit") + candidate = carry + chunk + if any(value in candidate for value in forbidden): + raise PolicyError( + "Git upstream response contains runtime credential material" + ) + width = max(len(value) for value in forbidden) - 1 + carry = candidate[-width:] if width else b"" + spool.write(chunk) + spool.seek(0) + return spool, total + except Exception: + spool.close() + raise diff --git a/services/hermes/scm-common/scripts/scm_broker_server.py b/services/hermes/scm-common/scripts/scm_broker_server.py new file mode 100644 index 00000000..f29cd314 --- /dev/null +++ b/services/hermes/scm-common/scripts/scm_broker_server.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +"""Concurrency-bounded threaded HTTP server for the Hermes SCM broker.""" + +from __future__ import annotations + +import threading +import time +from http.server import ThreadingHTTPServer + +MAX_CONCURRENT_REQUESTS = 3 + + +class _AbsoluteDeadlineReader: + """Read header lines against one wall-clock deadline, not idle timeouts.""" + + def __init__(self, stream, connection): + self._stream = stream + self._connection = connection + self._deadline: float | None = None + + def begin(self, timeout: float) -> None: + self._deadline = time.monotonic() + timeout + + def end(self) -> None: + self._deadline = None + + def readline(self, limit: int = -1) -> bytes: + if self._deadline is None: + return self._stream.readline(limit) + value = bytearray() + while limit < 0 or len(value) < limit: + remaining = self._deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError("absolute SCM header deadline exceeded") + self._connection.settimeout(remaining) + character = self._stream.read(1) + if not character: + break + value.extend(character) + if character == b"\n": + break + return bytes(value) + + def __getattr__(self, name: str): + return getattr(self._stream, name) + + +class AbsoluteHeaderDeadlineMixin: + """Apply an absolute request-line plus header deadline to HTTP handlers.""" + + header_deadline_seconds = 10.0 + + def setup(self) -> None: + super().setup() + self._header_reader = _AbsoluteDeadlineReader(self.rfile, self.connection) + self.rfile = self._header_reader + + def handle_one_request(self) -> None: + try: + self._header_reader.begin(self.header_deadline_seconds) + try: + self.raw_requestline = self.rfile.readline(65537) + if len(self.raw_requestline) > 65536: + self.requestline = "" + self.request_version = "" + self.command = "" + self.send_error(414) + return + if not self.raw_requestline: + self.close_connection = True + return + if not self.parse_request(): + return + finally: + self._header_reader.end() + method_name = "do_" + self.command + if not hasattr(self, method_name): + self.send_error(501, "Unsupported method") + return + getattr(self, method_name)() + self.wfile.flush() + except TimeoutError: + self.close_connection = True + + +class BoundedThreadingHTTPServer(ThreadingHTTPServer): + """Cap active handlers so large Git exchanges cannot exhaust memory.""" + + daemon_threads = True + block_on_close = True + request_queue_size = 16 + + def __init__(self, *args, **kwargs): + self._slots = threading.BoundedSemaphore(MAX_CONCURRENT_REQUESTS) + super().__init__(*args, **kwargs) + + def process_request(self, request, client_address) -> None: + if not self._slots.acquire(blocking=False): + try: + request.sendall( + b"HTTP/1.1 503 Service Unavailable\r\n" + b"Connection: close\r\nContent-Length: 0\r\n\r\n" + ) + finally: + self.shutdown_request(request) + return + try: + super().process_request(request, client_address) + except Exception: + self._slots.release() + self.shutdown_request(request) + raise + + def process_request_thread(self, request, client_address) -> None: + try: + super().process_request_thread(request, client_address) + finally: + self._slots.release() diff --git a/services/hermes/scripts/gitea_api.py b/services/hermes/scripts/gitea_api.py deleted file mode 100644 index 5544c398..00000000 --- a/services/hermes/scripts/gitea_api.py +++ /dev/null @@ -1,150 +0,0 @@ -#!/usr/bin/env python3 -"""Call the private Atlas Gitea API through a runtime-only token boundary.""" - -from __future__ import annotations - -import argparse -import json -import os -import sys -import urllib.error -import urllib.parse -import urllib.request -from pathlib import Path - - -DEFAULT_BASE_URL = "https://scm.bstein.dev" -DEFAULT_TOKEN_FILE = Path("/runtime-access/gitea-token") -ALLOWED_METHODS = ("GET", "POST", "PUT", "PATCH", "DELETE") - - -def read_token(path: Path = DEFAULT_TOKEN_FILE) -> str: - """Read and validate the token from its in-memory Vault projection.""" - token = path.read_text(encoding="utf-8").strip() - if not token: - raise ValueError(f"runtime credential is empty: {path}") - return token - - -def api_url(base_url: str, path: str) -> str: - """Return a same-origin Gitea API URL for a validated API path.""" - base = urllib.parse.urlsplit(base_url.rstrip("/")) - target = urllib.parse.urlsplit(path) - if base.scheme not in {"http", "https"} or not base.netloc: - raise ValueError("GITEA_BASE_URL must be an absolute HTTP(S) URL") - if target.scheme or target.netloc or target.fragment: - raise ValueError("API path must be relative to the configured Gitea origin") - if not target.path.startswith("/api/v1/"): - raise ValueError("API path must start with /api/v1/") - return urllib.parse.urlunsplit( - (base.scheme, base.netloc, target.path, target.query, "") - ) - - -def build_request( - method: str, - path: str, - *, - base_url: str, - token: str, - data: object | None = None, -) -> urllib.request.Request: - """Build one authenticated request without placing the token in its URL.""" - payload = None - if data is not None: - payload = json.dumps(data, separators=(",", ":")).encode("utf-8") - return urllib.request.Request( - api_url(base_url, path), - data=payload, - method=method, - headers={ - "Accept": "application/json", - "Authorization": f"token {token}", - "Content-Type": "application/json", - "User-Agent": "hermes-atlas-operator/1", - }, - ) - - -def parse_args(argv: list[str] | None = None) -> argparse.Namespace: - """Parse a bounded method, API path, and optional JSON request body.""" - parser = argparse.ArgumentParser( - description="Call the Atlas Gitea API using the runtime Vault token." - ) - parser.add_argument("method", choices=ALLOWED_METHODS) - parser.add_argument("path", help="Gitea path beginning with /api/v1/") - data_group = parser.add_mutually_exclusive_group() - data_group.add_argument("--data-json", help="JSON object/array request body") - data_group.add_argument( - "--data-file", type=Path, help="path to a JSON request body" - ) - data_group.add_argument( - "--field", - action="append", - metavar="KEY=VALUE", - help=( - "repeatable JSON field; bare text remains a string while true, false, " - "null, numbers, objects, and arrays are decoded as JSON" - ), - ) - return parser.parse_args(argv) - - -def parse_fields(values: list[str]) -> dict[str, object]: - """Build a JSON object from shell-safe, repeatable key/value arguments.""" - result: dict[str, object] = {} - for value in values: - key, separator, raw = value.partition("=") - if not separator or not key or any(char.isspace() for char in key): - raise ValueError("each --field must be KEY=VALUE with a non-space key") - try: - result[key] = json.loads(raw) - except json.JSONDecodeError: - result[key] = raw - return result - - -def load_data(args: argparse.Namespace) -> object | None: - """Decode the optional JSON body without involving a shell expansion.""" - if args.data_json is not None: - return json.loads(args.data_json) - if args.data_file is not None: - return json.loads(args.data_file.read_text(encoding="utf-8")) - if args.field is not None: - return parse_fields(args.field) - return None - - -def main(argv: list[str] | None = None) -> int: - """Execute the request, print only its response body, and return HTTP status.""" - args = parse_args(argv) - try: - request = build_request( - args.method, - args.path, - base_url=os.environ.get("GITEA_BASE_URL", DEFAULT_BASE_URL), - token=read_token(), - data=load_data(args), - ) - with urllib.request.urlopen(request, timeout=30) as response: - body = response.read() - if body: - sys.stdout.buffer.write(body) - if not body.endswith(b"\n"): - sys.stdout.buffer.write(b"\n") - return 0 - except urllib.error.HTTPError as exc: - body = exc.read(65536) - print(f"Gitea API returned HTTP {exc.code}", file=sys.stderr) - if body: - sys.stderr.buffer.write(body) - if not body.endswith(b"\n"): - sys.stderr.buffer.write(b"\n") - return 1 - except (OSError, ValueError, json.JSONDecodeError) as exc: - print(f"Gitea API request failed: {exc}", file=sys.stderr) - return 1 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/services/hermes/scripts/gitea_askpass.sh b/services/hermes/scripts/gitea_askpass.sh deleted file mode 100755 index d6146d80..00000000 --- a/services/hermes/scripts/gitea_askpass.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env sh -set -eu - -case "${1:-}" in - *Username*) - if [ -s /runtime-access/gitea-username ]; then - tr -d '\r\n' Path: @@ -139,24 +141,8 @@ def bootstrap_cassandra_state(root: Path) -> dict[str, str]: return {"state": "ready"} -def _gitea_token_path() -> Path: - """Return the runtime-only Gitea credential path without reading it.""" - configured = os.environ.get("HERMES_GITEA_TOKEN_FILE", "").strip() - return Path(configured) if configured else DEFAULT_GITEA_TOKEN_PATH - - -def _gitea_token_available() -> bool: - """Check credential readiness without loading it into process memory.""" - try: - path = _gitea_token_path() - return path.is_file() and path.stat().st_size > 0 - except OSError: - return False - - def sync_cassandra_repo(env_values: dict[str, str]) -> str: - """Clone or fetch Cassandra through the runtime-only askpass credential.""" - token_available = _gitea_token_available() + """Clone or fetch Cassandra through the credential-isolated SCM broker.""" if shutil.which("git") is None: return "git-unavailable" child_env = os.environ.copy() @@ -173,9 +159,7 @@ def sync_cassandra_repo(env_values: dict[str, str]) -> str: } } ) - child_env["GIT_ASKPASS"] = env_values.get( - "GIT_ASKPASS", "/opt/coordinator/gitea_askpass.sh" - ) + child_env.pop("GIT_ASKPASS", None) child_env["GIT_TERMINAL_PROMPT"] = "0" if (CASSANDRA_BASE_PATH / ".git").exists(): try: @@ -232,8 +216,6 @@ def sync_cassandra_repo(env_values: dict[str, str]) -> str: return f"remote-repair-failed-{repaired.returncode}" except (OSError, subprocess.TimeoutExpired): return "remote-repair-failed" - if not token_available: - return "ready; fetch skipped until Gitea token is configured" command = [ "git", "-C", @@ -247,8 +229,6 @@ def sync_cassandra_repo(env_values: dict[str, str]) -> str: CASSANDRA_BASE_PATH.mkdir(parents=True, exist_ok=True) if any(CASSANDRA_BASE_PATH.iterdir()): return "unmanaged-nonempty-directory" - if not token_available: - return "awaiting-gitea-token" command = [ "git", "clone", diff --git a/services/hermes/scripts/node_account_audit.py b/services/hermes/scripts/node_account_audit.py new file mode 100644 index 00000000..fa6245ba --- /dev/null +++ b/services/hermes/scripts/node_account_audit.py @@ -0,0 +1,267 @@ +#!/usr/bin/env python3 +"""Fail-closed supplementary-group and host privilege-policy audits.""" + +from __future__ import annotations + +import re +import stat +import struct +from pathlib import Path + +from node_account_io import HardeningError, read_regular +from node_polkit_audit import audit_polkit_policy + + +def _members(field: str, context: str) -> set[str]: + if not field: + return set() + values = field.split(",") + if any(not value or value.strip() != value for value in values): + raise HardeningError(f"{context} has a malformed member list") + if len(values) != len(set(values)): + raise HardeningError(f"{context} has duplicate members") + return set(values) + + +def audit_membership( + account: str, groups: list[list[str]], gshadow: list[list[str]] +) -> None: + """Reject the dedicated account in any supplementary group/admin list.""" + for record in groups: + if account in _members(record[3], f"group {record[0]}"): + raise HardeningError("dedicated Hermes account has supplementary group access") + for record in gshadow: + principals = _members(record[2], f"gshadow {record[0]} admins") + principals |= _members(record[3], f"gshadow {record[0]} members") + if account in principals: + raise HardeningError("dedicated Hermes account has gshadow group access") + + +def _policy_files(host_etc: Path, polkit_share: Path, root_uid: int) -> list[Path]: + files: list[Path] = [] + sudoers = host_etc / "sudoers" + if sudoers.exists(): + files.append(sudoers) + roots = ( + host_etc / "sudoers.d", + host_etc / "polkit-1/rules.d", + host_etc / "polkit-1/localauthority", + polkit_share / "rules.d", + polkit_share / "localauthority", + ) + for root in roots: + if not root.exists(): + continue + metadata = root.lstat() + if ( + not stat.S_ISDIR(metadata.st_mode) + or stat.S_ISLNK(metadata.st_mode) + or metadata.st_uid != root_uid + or stat.S_IMODE(metadata.st_mode) & 0o022 + ): + raise HardeningError("unsafe sudo/polkit policy directory") + files.extend(path for path in root.rglob("*") if path.is_file()) + if len(files) > 256: + raise HardeningError("too many sudo/polkit policy files") + return sorted(files) + + +def _audit_policy_metadata(snapshot, root_uid: int, account_uid: int) -> None: + if snapshot.uid != root_uid or snapshot.mode & 0o022: + raise HardeningError("sudo/polkit policy metadata permits unsafe mutation") + acl = dict(snapshot.xattrs).get("system.posix_acl_access", b"") + if not acl: + return + header = struct.Struct(" str: + """Remove comments without discarding sudoers' numeric-UID principals.""" + active: list[str] = [] + for line in text.splitlines(): + stripped = line.lstrip() + if not stripped: + continue + # Detect any include spelling before the comment rule below. sudo + # honours both #include and @include; a quoted or space-bearing path + # must never fall through the "#" comment branch and leave a second + # authority file unaudited. Only the exact bare includedir into the + # audited /etc/sudoers.d passes; everything else fails closed. + if re.match(r"(?i)[#@]include(?:dir)?(?=\s|$)", stripped): + include = re.fullmatch( + r"(?i)([#@]include(?:dir)?)\s+([^\s\"'\\]+)", stripped + ) + if include: + directive, target = include.groups() + if ( + directive.lower().endswith("includedir") + and target == "/etc/sudoers.d" + ): + continue + raise HardeningError("sudo policy includes an unaudited authority source") + if stripped.startswith("#") and not re.match(r"#\d+(?:\s|$)", stripped): + continue + active.append(line) + return "\n".join(active) + + +def _mentions_dedicated_identity(text: str, account: str, account_uid: int) -> bool: + account_pattern = rf"(? list[str]: + """Merge sudoers backslash continuations into whole logical lines.""" + lines: list[str] = [] + pending = "" + for line in active.splitlines(): + if line.endswith("\\"): + pending += line[:-1] + " " + continue + lines.append(pending + line) + pending = "" + if pending: + raise HardeningError("sudo policy ends inside a line continuation") + return lines + + +def _sudo_user_aliases(lines: list[str]) -> dict[str, list[str]]: + """Collect User_Alias definitions so grant principals can be expanded.""" + aliases: dict[str, list[str]] = {} + for line in lines: + stripped = line.strip() + if not stripped.startswith("User_Alias"): + continue + for definition in stripped.removeprefix("User_Alias").split(":"): + name, separator, members = definition.partition("=") + name = name.strip() + values = [item.strip() for item in members.split(",")] + if ( + not separator + or not SUDO_ALIAS_NAME_RE.fullmatch(name) + or name in aliases + or not all(values) + ): + raise HardeningError("sudo alias definition is not auditable") + aliases[name] = values + return aliases + + +def _expand_sudo_principal( + principal: str, aliases: dict[str, list[str]], seen: frozenset[str] +) -> None: + """Reject grant principals that reach Hermes through aliases or wildcards.""" + value = principal.strip() + while value.startswith("!"): + value = value[1:].strip() + value = value.strip('"') + if not value: + raise HardeningError("sudo grant principal is malformed") + if value in aliases: + if value in seen: + raise HardeningError("sudo alias expansion is cyclic") + for member in aliases[value]: + _expand_sudo_principal(member, aliases, seen | {value}) + return + if value in {"ALL", "%ALL"}: + raise HardeningError("broad sudo authority includes Hermes") + if value.startswith("+") or value.startswith("%:"): + raise HardeningError("sudo netgroup authority cannot be audited") + if SUDO_ALIAS_NAME_RE.fullmatch(value): + raise HardeningError("sudo alias is undefined") + + +def _audit_sudo_grants(active: str) -> None: + """Expand every grant's user list; only literal non-Hermes principals pass.""" + lines = _joined_sudo_lines(active) + aliases = _sudo_user_aliases(lines) + skip = ("Defaults", "User_Alias", "Runas_Alias", "Host_Alias", "Cmnd_Alias") + for line in lines: + stripped = line.strip() + if not stripped or stripped.startswith(skip): + continue + head, separator, _rest = stripped.partition("=") + fields = head.split() + if not separator or len(fields) < 2: + raise HardeningError("sudo grant line is not auditable") + for principal in " ".join(fields[:-1]).split(","): + _expand_sudo_principal(principal, aliases, frozenset()) + + +def audit_privilege_policies( + account: str, + account_uid: int, + root_uid: int, + host_etc: Path, + polkit_share: Path, +) -> None: + """Reject direct, group, wildcard sudo, or root-equivalent polkit grants.""" + dangerous_polkit = ( + "org.freedesktop.policykit.exec", + "org.freedesktop.systemd1.manage-unit", + "org.freedesktop.udisks2.modify-device", + "org.freedesktop.packagekit", + ) + nsswitch = host_etc / "nsswitch.conf" + if nsswitch.exists(): + try: + nss_snapshot = read_regular(nsswitch, 64 * 1024) + _audit_policy_metadata(nss_snapshot, root_uid, account_uid) + text = nss_snapshot.value.decode("utf-8") + except UnicodeDecodeError as exc: + raise HardeningError("nsswitch policy is not UTF-8") from exc + for line in text.splitlines(): + name, separator, sources = line.partition(":") + if name.strip() in { + "passwd", + "group", + "initgroups", + "shadow", + "sudoers", + } and separator: + active_sources = [item for item in sources.split() if not item.startswith("[")] + if not active_sources or any( + item not in {"files", "systemd"} for item in active_sources + ): + raise HardeningError("external group/account/sudo authority source is unsafe") + total = 0 + for path in _policy_files(host_etc, polkit_share, root_uid): + snapshot = read_regular(path, 256 * 1024) + _audit_policy_metadata(snapshot, root_uid, account_uid) + value = snapshot.value + total += len(value) + if total > 2 * 1024 * 1024: + raise HardeningError("sudo/polkit policy input exceeds safe limit") + try: + text = value.decode("utf-8") + except UnicodeDecodeError as exc: + raise HardeningError("sudo/polkit policy is not UTF-8") from exc + if path.name == "sudoers" or "sudoers.d" in path.parts: + active = _active_sudo_policy(text) + if _mentions_dedicated_identity(active, account, account_uid): + raise HardeningError("dedicated Hermes account has sudo authority") + _audit_sudo_grants(active) + continue + active = "\n".join( + line for line in text.splitlines() if not line.lstrip().startswith("#") + ) + grants = "polkit.Result.YES" in active or re.search( + r"(?im)^\s*Result(?:Any|Inactive|Active)\s*=\s*yes\s*$", active + ) + if grants and _mentions_dedicated_identity(active, account, account_uid): + raise HardeningError("dedicated Hermes account has polkit authority") + broad = "unix-user:*" in active or "Identity=unix-user:*" in active + if broad and grants and any(item in active for item in dangerous_polkit): + raise HardeningError("root-equivalent broad polkit authority includes Hermes") + audit_polkit_policy(path, text) diff --git a/services/hermes/scripts/node_account_hardening.py b/services/hermes/scripts/node_account_hardening.py new file mode 100644 index 00000000..a545af47 --- /dev/null +++ b/services/hermes/scripts/node_account_hardening.py @@ -0,0 +1,496 @@ +#!/usr/bin/env python3 +"""Reconcile a dedicated unprivileged Hermes SSH account on one Atlas node.""" + +from __future__ import annotations + +import argparse +import base64 +import errno +import os +import re +import shlex +import stat +import struct +from contextlib import suppress +from pathlib import Path + +from node_account_audit import audit_membership, audit_privilege_policies +from node_account_io import ( + FileSnapshot, + HardeningError, + account_lock, + assert_unchanged, + atomic_write, + backup_once, + read_regular, +) + +HOST_ETC = Path("/host-etc") +HOST_HOME = Path("/host-home") +HOST_K3S = Path("/host-k3s") +HOST_KUBELET = Path("/host-kubelet") +HOST_RUN_K3S = Path("/host-run-k3s") +HOST_RUN_CONTAINERD = Path("/host-run-containerd") +HOST_POLKIT_SHARE = Path("/host-polkit-share") +ACCOUNT = "hermes-agent" +ACCOUNT_UID = 1200 +ACCOUNT_GID = 1200 +ACCOUNT_HOME = "/home/hermes-agent" +ACCOUNT_SHELL = "/bin/bash" +HOST_ROOT_UID = 0 +HOST_ROOT_GID = 0 +LEGACY_ACCOUNTS = ("atlas", "oceanus") +MAX_ACCOUNT_FILE = 2 * 1024 * 1024 +MAX_AUTHORIZED_KEYS = 1024 * 1024 +ACL_VERSION = 2 +ACL_UNDEFINED_ID = 0xFFFFFFFF +ACL_USER_OBJ = 0x01 +ACL_USER = 0x02 +ACL_GROUP_OBJ = 0x04 +ACL_GROUP = 0x08 +ACL_MASK = 0x10 +ACL_OTHER = 0x20 +ACL_HEADER = struct.Struct(" tuple[bytes, os.stat_result]: + snapshot = read_regular(path, maximum) + return snapshot.value, path.stat(follow_symlinks=False) + + +def _records(value: bytes, fields: int, name: str) -> list[list[str]]: + try: + text = value.decode("utf-8") + except UnicodeDecodeError as exc: + raise HardeningError(f"{name} is not UTF-8") from exc + if not text.endswith("\n"): + raise HardeningError(f"{name} is missing its final newline") + records = [] + names = set() + for line in text.splitlines(): + parts = line.split(":") + if len(parts) != fields or not parts[0] or parts[0] in names: + raise HardeningError(f"{name} has an invalid record") + names.add(parts[0]) + records.append(parts) + return records + + +def _encode(records: list[list[str]]) -> bytes: + return ("\n".join(":".join(record) for record in records) + "\n").encode() + + +def _expected_records() -> dict[str, list[str]]: + return { + "passwd": [ + ACCOUNT, + "x", + str(ACCOUNT_UID), + str(ACCOUNT_GID), + "Hermes Agent", + ACCOUNT_HOME, + ACCOUNT_SHELL, + ], + "group": [ACCOUNT, "x", str(ACCOUNT_GID), ""], + "shadow": [ACCOUNT, "!", "1", "0", "99999", "7", "", "", ""], + "gshadow": [ACCOUNT, "!", "", ""], + } + + +def _reconcile_record( + records: list[list[str]], expected: list[str], *, identity_index: int +) -> list[list[str]]: + for record in records: + same_name = record[0] == expected[0] + same_identity = record[identity_index] == expected[identity_index] + if same_name or same_identity: + if record != expected: + raise HardeningError("dedicated Hermes account identity conflicts") + return records + return [*records, expected] + + +def _reconcile_databases() -> None: + expected = _expected_records() + definitions = ( + ("passwd", 7, 2), + ("group", 4, 2), + ("shadow", 9, 0), + ("gshadow", 4, 0), + ) + planned: list[tuple[Path, FileSnapshot, bytes]] = [] + account_present = False + parsed: dict[str, list[list[str]]] = {} + for name, fields, identity_index in definitions: + path = HOST_ETC / name + snapshot = read_regular(path, MAX_ACCOUNT_FILE) + records = _records(snapshot.value, fields, name) + parsed[name] = records + updated = _reconcile_record( + records, expected[name], identity_index=identity_index + ) + if name == "passwd": + account_present = expected[name] in records + planned.append((path, snapshot, _encode(updated))) + audit_membership(ACCOUNT, parsed["group"], parsed["gshadow"]) + audit_privilege_policies( + ACCOUNT, ACCOUNT_UID, HOST_ROOT_UID, HOST_ETC, HOST_POLKIT_SHARE + ) + try: + (HOST_HOME / ACCOUNT).lstat() + except FileNotFoundError: + pass + else: + if not account_present: + raise HardeningError("dedicated Hermes account home already exists") + for path, snapshot, _updated in planned: + assert_unchanged(path, snapshot, MAX_ACCOUNT_FILE) + for path, snapshot, _updated in planned: + backup_once(path, snapshot, MAX_ACCOUNT_FILE) + for path, snapshot, _updated in planned: + assert_unchanged(path, snapshot, MAX_ACCOUNT_FILE) + written: list[tuple[Path, FileSnapshot, bytes]] = [] + try: + for path, snapshot, updated in planned: + assert_unchanged(path, snapshot, MAX_ACCOUNT_FILE) + atomic_write(path, updated, snapshot) + written.append((path, snapshot, updated)) + for path, _snapshot, _updated in planned: + name = path.name + fields = next(item[1] for item in definitions if item[0] == name) + records = _records(_read_regular(path)[0], fields, name) + if expected[name] not in records: + raise HardeningError("dedicated Hermes account validation failed") + except Exception as error: + for path, snapshot, updated in reversed(written): + current = read_regular(path, MAX_ACCOUNT_FILE) + if ( + current.value != updated + or current.mode != snapshot.mode + or current.uid != snapshot.uid + or current.gid != snapshot.gid + or current.xattrs != snapshot.xattrs + ): + raise HardeningError( + f"concurrent host account change prevents rollback: {path.name}" + ) from error + atomic_write(path, snapshot.value, snapshot) + raise + + +SUPPORTED_KEY_TYPES = {"ssh-ed25519", "ecdsa-sha2-nistp256", "ssh-rsa"} +KEY_TYPE_RE = re.compile(r"(?:sk-)?(?:ssh|ecdsa)-[A-Za-z0-9@._+-]+\Z") + + +def _key_identity(line: bytes) -> tuple[str, bytes] | None: + """Return OpenSSH key type/blob, ignoring options and comments.""" + stripped = line.strip(b"\r\n") + if not stripped or stripped.lstrip().startswith(b"#"): + return None + try: + fields = shlex.split(stripped.decode("ascii"), posix=True) + except (UnicodeDecodeError, ValueError) as exc: + raise HardeningError("authorized key line is malformed") from exc + indexes = [ + index for index, field in enumerate(fields) if KEY_TYPE_RE.fullmatch(field) + ] + if len(indexes) != 1 or indexes[0] + 1 >= len(fields): + raise HardeningError("authorized key line has ambiguous key material") + key_type = fields[indexes[0]] + try: + blob = base64.b64decode(fields[indexes[0] + 1], validate=True) + except ValueError as exc: + raise HardeningError("authorized key payload is invalid") from exc + encoded_type = key_type.encode("ascii") + if len(blob) < 4 or int.from_bytes(blob[:4], "big") != len(encoded_type): + raise HardeningError("authorized key blob is malformed") + if blob[4 : 4 + len(encoded_type)] != encoded_type: + raise HardeningError("authorized key type does not match its blob") + return key_type, blob + + +def _validated_public_key(path: Path) -> tuple[bytes, tuple[str, bytes]]: + value, _ = _read_regular(path, 16 * 1024) + line = value.strip() + if b"\n" in line or b"\r" in line: + raise HardeningError("Hermes public key must contain one line") + identity = _key_identity(line) + fields = line.split() + if ( + identity is None + or identity[0] not in SUPPORTED_KEY_TYPES + or len(fields) not in {2, 3} + or fields[0].decode("ascii") != identity[0] + ): + raise HardeningError("Hermes public key format is unsupported") + return line, identity + + +def _directory(path: Path, *, mode: int, uid: int, gid: int) -> None: + with suppress(FileExistsError): + path.mkdir(mode=mode) + metadata = path.lstat() + if not stat.S_ISDIR(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode): + raise HardeningError(f"unsafe account directory: {path.name}") + if metadata.st_uid not in {0, uid} or metadata.st_gid not in {0, gid}: + raise HardeningError(f"account directory ownership conflicts: {path.name}") + os.chown(path, uid, gid) + path.chmod(mode) + + +def _without_key(value: bytes, identity: tuple[str, bytes]) -> bytes: + kept = [] + for line in value.splitlines(keepends=True): + if _key_identity(line) != identity: + kept.append(line) + return b"".join(kept) + + +def _move_key(public_key: Path) -> None: + key, identity = _validated_public_key(public_key) + target = HOST_HOME / ACCOUNT / ".ssh" / "authorized_keys" + legacy_plan: list[tuple[Path, FileSnapshot, bytes]] = [] + for legacy in LEGACY_ACCOUNTS: + authorized = HOST_HOME / legacy / ".ssh" / "authorized_keys" + try: + snapshot = read_regular(authorized, MAX_AUTHORIZED_KEYS) + except FileNotFoundError: + continue + updated = _without_key(snapshot.value, identity) + if updated != snapshot.value: + legacy_plan.append((authorized, snapshot, updated)) + if legacy_plan and target.exists(): + target_snapshot = read_regular(target, MAX_AUTHORIZED_KEYS) + target_without_key = _without_key(target_snapshot.value, identity) + if target_without_key != target_snapshot.value: + backup_once(target, target_snapshot, MAX_AUTHORIZED_KEYS) + assert_unchanged(target, target_snapshot, MAX_AUTHORIZED_KEYS) + atomic_write(target, target_without_key, target_snapshot) + verified_target = read_regular(target, MAX_AUTHORIZED_KEYS).value + if _without_key(verified_target, identity) != verified_target: + raise HardeningError("preexisting Hermes authorization removal failed") + for authorized, snapshot, _updated in legacy_plan: + assert_unchanged(authorized, snapshot, MAX_AUTHORIZED_KEYS) + backup_once(authorized, snapshot, MAX_AUTHORIZED_KEYS) + for authorized, snapshot, updated in legacy_plan: + assert_unchanged(authorized, snapshot, MAX_AUTHORIZED_KEYS) + atomic_write(authorized, updated, snapshot) + for legacy in LEGACY_ACCOUNTS: + authorized = HOST_HOME / legacy / ".ssh" / "authorized_keys" + try: + value = read_regular(authorized, MAX_AUTHORIZED_KEYS).value + except FileNotFoundError: + continue + if _without_key(value, identity) != value: + raise HardeningError("legacy Hermes authorization key removal failed") + + # Installation happens only after every legacy authorization is absent. A + # crash can therefore remove access temporarily, but can never duplicate + # the machine credential across privileged and unprivileged accounts. + home = HOST_HOME / ACCOUNT + ssh = home / ".ssh" + _directory(home, mode=0o700, uid=ACCOUNT_UID, gid=ACCOUNT_GID) + _directory(ssh, mode=0o700, uid=ACCOUNT_UID, gid=ACCOUNT_GID) + target = ssh / "authorized_keys" + if target.exists(): + current = read_regular(target, MAX_AUTHORIZED_KEYS) + if current.value != key + b"\n": + backup_once(target, current, MAX_AUTHORIZED_KEYS) + else: + current = FileSnapshot( + value=b"", + device=0, + inode=0, + mode=0o600, + uid=ACCOUNT_UID, + gid=ACCOUNT_GID, + size=0, + mtime_ns=0, + ctime_ns=0, + xattrs=(), + ) + atomic_write(target, key + b"\n", current) + if _read_regular(target, MAX_AUTHORIZED_KEYS)[0] != key + b"\n": + raise HardeningError("dedicated Hermes authorized key validation failed") + + +def _decode_acl(value: bytes, mode: int) -> list[tuple[int, int, int]]: + """Decode a bounded POSIX ACL or derive one from ordinary mode bits.""" + if not value: + group = (mode >> 3) & 0o7 + return [ + (ACL_USER_OBJ, (mode >> 6) & 0o7, ACL_UNDEFINED_ID), + (ACL_GROUP_OBJ, group, ACL_UNDEFINED_ID), + (ACL_MASK, group, ACL_UNDEFINED_ID), + (ACL_OTHER, mode & 0o7, ACL_UNDEFINED_ID), + ] + if len(value) < ACL_HEADER.size or (len(value) - ACL_HEADER.size) % ACL_ENTRY.size: + raise HardeningError("sensitive directory ACL is malformed") + if ACL_HEADER.unpack_from(value)[0] != ACL_VERSION: + raise HardeningError("sensitive directory ACL version is unsupported") + entries = [ + ACL_ENTRY.unpack_from(value, offset) + for offset in range(ACL_HEADER.size, len(value), ACL_ENTRY.size) + ] + if any(permission > 0o7 for _tag, permission, _identifier in entries): + raise HardeningError("sensitive directory ACL permission is malformed") + required = {ACL_USER_OBJ, ACL_GROUP_OBJ, ACL_OTHER} + if not required <= {tag for tag, _permission, _identifier in entries}: + raise HardeningError("sensitive directory ACL is incomplete") + return entries + + +def _encode_acl(entries: list[tuple[int, int, int]]) -> bytes: + return ACL_HEADER.pack(ACL_VERSION) + b"".join( + ACL_ENTRY.pack(*entry) for entry in entries + ) + + +def _acl_with_deny(value: bytes, mode: int) -> bytes: + entries = _decode_acl(value, mode) + entries = [ + entry + for entry in entries + if not (entry[0] == ACL_USER and entry[2] == ACCOUNT_UID) + ] + entries.append((ACL_USER, 0, ACCOUNT_UID)) + if not any(tag == ACL_MASK for tag, _permission, _identifier in entries): + entries.append((ACL_MASK, (mode >> 3) & 0o7, ACL_UNDEFINED_ID)) + order = { + ACL_USER_OBJ: 0, + ACL_USER: 1, + ACL_GROUP_OBJ: 2, + ACL_GROUP: 3, + ACL_MASK: 4, + ACL_OTHER: 5, + } + entries.sort(key=lambda entry: (order.get(entry[0], 99), entry[2])) + return _encode_acl(entries) + + +def _read_acl(path: Path) -> bytes: + try: + return os.getxattr(path, ACL_XATTR, follow_symlinks=False) + except OSError as exc: + if exc.errno in {errno.ENODATA, getattr(errno, "ENOATTR", errno.ENODATA)}: + return b"" + raise + + +def _validate_acl_backup(value: bytes) -> None: + if value[:1] not in {b"A", b"N"}: + raise HardeningError("sensitive directory ACL backup is malformed") + if value[:1] == b"A": + if len(value) == 1: + raise HardeningError("sensitive directory ACL backup is malformed") + _decode_acl(value[1:], 0) + elif value != b"N": + raise HardeningError("sensitive directory ACL backup is malformed") + + +def _acl_backup_once(path: Path, value: bytes, backup_name: str | None = None) -> None: + """Persist the original ACL as a private, durable host recovery file.""" + root = HOST_ETC / "hermes-node-boundary" + _directory(root, mode=0o700, uid=HOST_ROOT_UID, gid=HOST_ROOT_GID) + backup = root / f"{backup_name or path.name}.acl" + encoded = b"A" + value if value else b"N" + try: + existing, _metadata = _read_regular(backup, 64 * 1024) + except FileNotFoundError: + pass + else: + _validate_acl_backup(existing) + return + + temporary = root / f".{path.name}.acl.hermes-{os.getpid()}" + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(temporary, flags, 0o600) + try: + if os.write(descriptor, encoded) != len(encoded): + raise HardeningError("short sensitive directory ACL backup write") + os.fchmod(descriptor, 0o600) + os.fchown(descriptor, HOST_ROOT_UID, HOST_ROOT_GID) + os.fsync(descriptor) + except Exception: + temporary.unlink(missing_ok=True) + raise + finally: + os.close(descriptor) + try: + os.link(temporary, backup, follow_symlinks=False) + except FileExistsError: + # Another reconciler won the race; never overwrite the first backup. + pass + finally: + temporary.unlink(missing_ok=True) + stored, _metadata = _read_regular(backup, 64 * 1024) + _validate_acl_backup(stored) + directory = os.open(root, os.O_RDONLY) + try: + os.fsync(directory) + finally: + os.close(directory) + + +def _deny_sensitive_root(path: Path, backup_name: str | None = None) -> None: + metadata = path.lstat() + if not stat.S_ISDIR(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode): + raise HardeningError(f"unsafe sensitive directory: {path.name}") + if metadata.st_uid != HOST_ROOT_UID: + raise HardeningError(f"sensitive directory is not root-owned: {path.name}") + current = _read_acl(path) + _acl_backup_once(path, current, backup_name) + updated = _acl_with_deny(current, stat.S_IMODE(metadata.st_mode)) + try: + os.setxattr(path, ACL_XATTR, updated, follow_symlinks=False) + verified = _decode_acl(_read_acl(path), stat.S_IMODE(metadata.st_mode)) + if (ACL_USER, 0, ACCOUNT_UID) not in verified: + raise HardeningError("sensitive directory ACL validation failed") + except Exception: + if current: + os.setxattr(path, ACL_XATTR, current, follow_symlinks=False) + else: + try: + os.removexattr(path, ACL_XATTR, follow_symlinks=False) + except OSError as exc: + if exc.errno not in { + errno.ENODATA, + getattr(errno, "ENOATTR", errno.ENODATA), + }: + raise + raise + + +def _deny_sensitive_roots() -> None: + for path, backup_name in ( + (HOST_K3S, "var-lib-rancher-k3s"), + (HOST_KUBELET, "var-lib-kubelet"), + (HOST_RUN_K3S, "run-k3s"), + (HOST_RUN_CONTAINERD, "run-containerd"), + ): + _deny_sensitive_root(path, backup_name) + + +def reconcile(public_key: Path) -> None: + """Create the locked account and move only the Hermes authorization key.""" + with account_lock(HOST_ETC, expected_uid=HOST_ROOT_UID): + _reconcile_databases() + _deny_sensitive_roots() + _move_key(public_key) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--public-key-file", type=Path, required=True) + args = parser.parse_args() + reconcile(args.public_key_file) + print("Dedicated Hermes node account reconciled.", flush=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/hermes/scripts/node_account_io.py b/services/hermes/scripts/node_account_io.py new file mode 100644 index 00000000..4fb2ae9e --- /dev/null +++ b/services/hermes/scripts/node_account_io.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +"""Crash-safe, metadata-preserving host account database I/O.""" + +from __future__ import annotations + +import fcntl +import os +import stat +import time +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Iterator + +MAX_XATTRS = 64 +MAX_XATTR_NAME = 255 +MAX_XATTR_VALUE = 256 * 1024 +MAX_XATTR_TOTAL = 1024 * 1024 +LOCK_TIMEOUT_SECONDS = 15.0 + + +class HardeningError(RuntimeError): + """Raised before an unsafe or ambiguous host-account change.""" + + +@dataclass(frozen=True) +class FileSnapshot: + """Bounded file contents and all security-relevant inode metadata.""" + + value: bytes + device: int + inode: int + mode: int + uid: int + gid: int + size: int + mtime_ns: int + ctime_ns: int + xattrs: tuple[tuple[str, bytes], ...] + + +def _stat_identity(item: os.stat_result) -> tuple[int, ...]: + return ( + item.st_dev, + item.st_ino, + item.st_mode, + item.st_uid, + item.st_gid, + item.st_size, + item.st_mtime_ns, + item.st_ctime_ns, + ) + + +def _xattrs(path_or_fd: Path | int) -> tuple[tuple[str, bytes], ...]: + try: + names = sorted(os.listxattr(path_or_fd, follow_symlinks=False)) + except (TypeError, ValueError): + names = sorted(os.listxattr(path_or_fd)) + if len(names) > MAX_XATTRS: + raise HardeningError("host account file has too many extended attributes") + values: list[tuple[str, bytes]] = [] + total = 0 + for name in names: + if len(name.encode("utf-8")) > MAX_XATTR_NAME: + raise HardeningError("host account file xattr name is too long") + try: + value = os.getxattr(path_or_fd, name, follow_symlinks=False) + except (TypeError, ValueError): + value = os.getxattr(path_or_fd, name) + total += len(value) + if len(value) > MAX_XATTR_VALUE or total > MAX_XATTR_TOTAL: + raise HardeningError("host account file xattrs exceed the safe limit") + values.append((name, value)) + return tuple(values) + + +def _restore_xattrs(descriptor: int, expected: tuple[tuple[str, bytes], ...]) -> None: + expected_names = {name for name, _value in expected} + for name, _value in _xattrs(descriptor): + if name not in expected_names: + os.removexattr(descriptor, name) + for name, value in expected: + os.setxattr(descriptor, name, value) + + +def read_regular(path: Path, maximum: int) -> FileSnapshot: + """Read one bounded, non-symlink regular file and capture its metadata.""" + flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(path, flags) + try: + metadata = os.fstat(descriptor) + if not stat.S_ISREG(metadata.st_mode) or metadata.st_size > maximum: + raise HardeningError(f"unsafe regular file: {path.name}") + value = os.read(descriptor, maximum + 1) + attrs = _xattrs(descriptor) + after = os.fstat(descriptor) + finally: + os.close(descriptor) + if len(value) != metadata.st_size or _stat_identity(metadata) != _stat_identity(after): + raise HardeningError(f"short file read: {path.name}") + return FileSnapshot( + value=value, + device=metadata.st_dev, + inode=metadata.st_ino, + mode=stat.S_IMODE(metadata.st_mode), + uid=metadata.st_uid, + gid=metadata.st_gid, + size=metadata.st_size, + mtime_ns=metadata.st_mtime_ns, + ctime_ns=metadata.st_ctime_ns, + xattrs=attrs, + ) + + +def assert_unchanged(path: Path, expected: FileSnapshot, maximum: int) -> None: + """Fail if an account file changed since planning began.""" + current = read_regular(path, maximum) + if current != expected: + raise HardeningError(f"concurrent host account change detected: {path.name}") + + +def atomic_write(path: Path, value: bytes, metadata: FileSnapshot) -> None: + """Atomically replace a file while retaining ACLs, labels, and xattrs.""" + temporary = path.with_name(f".{path.name}.hermes-{os.getpid()}") + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(temporary, flags, metadata.mode) + try: + if os.write(descriptor, value) != len(value): + raise HardeningError(f"short atomic write: {path.name}") + os.fchmod(descriptor, metadata.mode) + os.fchown(descriptor, metadata.uid, metadata.gid) + _restore_xattrs(descriptor, metadata.xattrs) + os.fsync(descriptor) + except Exception: + temporary.unlink(missing_ok=True) + raise + finally: + os.close(descriptor) + os.replace(temporary, path) + directory = os.open(path.parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) + try: + os.fsync(directory) + finally: + os.close(directory) + + +def backup_once(path: Path, snapshot: FileSnapshot, maximum: int) -> Path: + """Create the first durable recovery copy without replacing an older one.""" + backup = path.with_name(path.name + ".hermes-boundary-backup") + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(backup, flags, snapshot.mode) + except FileExistsError: + read_regular(backup, maximum) + return backup + try: + if os.write(descriptor, snapshot.value) != len(snapshot.value): + raise HardeningError(f"short backup write: {path.name}") + os.fchmod(descriptor, snapshot.mode) + os.fchown(descriptor, snapshot.uid, snapshot.gid) + _restore_xattrs(descriptor, snapshot.xattrs) + os.fsync(descriptor) + except Exception: + backup.unlink(missing_ok=True) + raise + finally: + os.close(descriptor) + directory = os.open(path.parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) + try: + os.fsync(directory) + finally: + os.close(directory) + return backup + + +@contextmanager +def account_lock(host_etc: Path, *, expected_uid: int = 0) -> Iterator[None]: + """Hold the standard shadow-utils account lock for the full transaction.""" + path = host_etc / ".pwd.lock" + flags = os.O_WRONLY | os.O_CREAT | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(path, flags, 0o600) + try: + metadata = os.fstat(descriptor) + if ( + not stat.S_ISREG(metadata.st_mode) + or metadata.st_uid != expected_uid + or stat.S_IMODE(metadata.st_mode) != 0o600 + ): + raise HardeningError("unsafe standard account lock") + deadline = time.monotonic() + LOCK_TIMEOUT_SECONDS + while True: + try: + fcntl.lockf(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except BlockingIOError as exc: + if time.monotonic() >= deadline: + raise HardeningError("standard account lock is busy") from exc + time.sleep(0.05) + yield + finally: + try: + fcntl.lockf(descriptor, fcntl.LOCK_UN) + finally: + os.close(descriptor) diff --git a/services/hermes/scripts/node_polkit_audit.py b/services/hermes/scripts/node_polkit_audit.py new file mode 100644 index 00000000..d45a0134 --- /dev/null +++ b/services/hermes/scripts/node_polkit_audit.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +"""Literal-identity proof for polkit policies on hardened Atlas nodes. + +Polkit rules are JavaScript, so a grant can name its subject indirectly: +concatenated strings, escape sequences, bracket lookups, or arithmetic can +compute an identity that plain text matching never sees. Rather than model +the language, every grant must scope itself through exact literal identity +comparisons. Anything the audit cannot prove literal fails closed. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +from node_account_io import HardeningError + +_ALLOWED_ESCAPES = {'"', "'", "\\", "/", "n", "r", "t"} +_STRING_MARK = r"\x00[0-9]+\x00" +_GRANT_RE = re.compile(r"\bResult\s*\.\s*YES\b") +_PKLA_GRANT_RE = re.compile(r"(?im)^\s*Result(?:Any|Inactive|Active)\s*=\s*yes\s*$") +_PKLA_IDENTITY_RE = re.compile(r"unix-(?:user|group):[A-Za-z0-9][A-Za-z0-9._-]{0,99}\Z") +_IDENTITY_USE_RE = re.compile(r"subject\s*(\[|\.\s*(?:user|uid|isInGroup)\b)") +_COMPUTED_TOKEN_RE = re.compile( + r"\b(?:eval|Function|arguments|fromCharCode|fromCodePoint|atob|btoa|unescape" + r"|escape|decodeURI[A-Za-z]*|encodeURI[A-Za-z]*|charCodeAt|codePointAt" + r"|normalize|constructor|prototype|globalThis|Reflect|Proxy)\b" +) +# Logical && and || stay legal; every value-building operator next to a +# string or number literal is rejected so comparisons stay whole literals. +_OPERATOR_ADJACENT_RE = re.compile( + rf"(?:{_STRING_MARK}|[0-9])\s*[+\-*/%<>^~](?![&|=])" + rf"|(?])[+\-*/%<>^~]\s*(?:{_STRING_MARK}|[0-9])" +) +_USER_BEFORE_RE = re.compile(rf"{_STRING_MARK}\s*[=!]==?\s*\Z") +_USER_AFTER_RE = re.compile(rf"\s*[=!]==?\s*{_STRING_MARK}") +_UID_BEFORE_RE = re.compile(r"(? tuple[str, list[str]]: + """Replace string literals with markers and drop JavaScript comments.""" + code: list[str] = [] + strings: list[str] = [] + index = 0 + length = len(text) + while index < length: + character = text[index] + if character == "`": + raise HardeningError("polkit rule uses computed template strings") + if character in {'"', "'"}: + quote = character + index += 1 + value: list[str] = [] + while True: + if index >= length or text[index] in "\r\n": + raise HardeningError("polkit rule string is unterminated") + character = text[index] + if character == "\\": + if index + 1 >= length or text[index + 1] not in _ALLOWED_ESCAPES: + raise HardeningError("polkit rule string uses computed escapes") + escaped = text[index + 1] + value.append({"n": "\n", "r": "\r", "t": "\t"}.get(escaped, escaped)) + index += 2 + continue + index += 1 + if character == quote: + break + value.append(character) + strings.append("".join(value)) + code.append(f"\x00{len(strings) - 1}\x00") + continue + if text.startswith("//", index): + newline = text.find("\n", index) + index = length if newline < 0 else newline + continue + if text.startswith("/*", index): + closing = text.find("*/", index + 2) + if closing < 0: + raise HardeningError("polkit rule comment is unterminated") + code.append(" ") + index = closing + 2 + continue + if character == "/": + # Any remaining slash is a regex literal or a division operator. + # A regex literal may embed a quote character, which would desync + # the string scanner and swallow an unconditional grant. Neither + # regex nor division has a place in an identity-scoped grant, so + # both fail closed before they can hide a Result.YES. + raise HardeningError("polkit rule uses a regex or division operator") + code.append(character) + index += 1 + return "".join(code), strings + + +def _require_literal_identity_use(code: str, match: re.Match[str]) -> None: + """Reject one identity reference unless it is an exact literal comparison.""" + form = match.group(1) + if form.startswith("["): + raise HardeningError("polkit rule reads identity through a computed lookup") + member = form.removeprefix(".").strip() + before = code[max(match.start() - 48, 0) : match.start()] + after = code[match.end() : match.end() + 48] + if member == "user": + literal = _USER_BEFORE_RE.search(before) or _USER_AFTER_RE.match(after) + elif member == "uid": + literal = _UID_BEFORE_RE.search(before) or _UID_AFTER_RE.match(after) + else: + literal = _GROUP_AFTER_RE.match(after) + if not literal: + raise HardeningError("polkit rule identity test is not an exact literal") + + +def _audit_rules_javascript(text: str) -> None: + """Require granting rules to gate on exact literal subject identities. + + With computed tokens, bracket lookups, and subject aliasing rejected in + every rule file, a YES decision can only be written as the literal grant + token, so the grant test below cannot be evaded by construction. + """ + code, _strings = _split_strings(text) + if _COMPUTED_TOKEN_RE.search(code): + raise HardeningError("polkit rule computes values the audit cannot prove") + if _RESULT_USE_RE.search(code): + raise HardeningError("polkit rule reads results through a computed lookup") + for bare in _BARE_SUBJECT_RE.finditer(code): + if not _PARAMETER_BEFORE_RE.search(code[: bare.start()]): + raise HardeningError("polkit rule passes the subject to computed code") + if not _GRANT_RE.search(code): + return + if _OPERATOR_ADJACENT_RE.search(code): + raise HardeningError("polkit grant builds values with operators") + uses = 0 + for match in _IDENTITY_USE_RE.finditer(code): + uses += 1 + _require_literal_identity_use(code, match) + if not uses: + raise HardeningError("polkit grant is not scoped to an explicit identity") + + +def _audit_pkla(text: str) -> None: + """Require granting localauthority entries to name exact literal identities.""" + if not _PKLA_GRANT_RE.search(text): + return + identities: list[str] = [] + for line in text.splitlines(): + name, separator, value = line.strip().partition("=") + if name.strip() != "Identity" or not separator: + continue + entries = [entry.strip() for entry in value.split(";") if entry.strip()] + identities.extend(entries) + if not identities: + raise HardeningError("polkit authority grant lacks an identity scope") + for entry in identities: + if not _PKLA_IDENTITY_RE.fullmatch(entry): + raise HardeningError("polkit authority identity is not an exact literal") + + +def audit_polkit_policy(path: Path, text: str) -> None: + """Prove one polkit policy file cannot grant through a computed identity.""" + if "localauthority" in path.parts or path.suffix == ".pkla": + _audit_pkla(text) + else: + _audit_rules_javascript(text) diff --git a/services/hermes/scripts/stage_runtime_access.py b/services/hermes/scripts/stage_runtime_access.py index e8276a5b..9c71bb7e 100644 --- a/services/hermes/scripts/stage_runtime_access.py +++ b/services/hermes/scripts/stage_runtime_access.py @@ -86,6 +86,21 @@ def _write_empty_auth_store() -> None: os.chown(path, OWNER_UID, OWNER_GID) +def _stage_node_ssh_config() -> None: + """Force every Atlas node alias onto the dedicated Hermes OS account.""" + destination = RUNTIME_ROOT / "node-ssh-config" + source = _copy_secret("node-ssh-config", destination) + if "\x00" in source: + destination.unlink(missing_ok=True) + raise RuntimeError("node SSH config contains invalid control data") + destination.write_text( + "Host titan-*\n User hermes-agent\n" + source.rstrip() + "\n", + encoding="utf-8", + ) + destination.chmod(0o600) + os.chown(destination, OWNER_UID, OWNER_GID) + + def stage_agent() -> None: """Stage the owner agent's complete runtime access set.""" for path in (RUNTIME_ROOT, RUNTIME_ROOT / "claude", RUNTIME_ROOT / "codex"): @@ -93,14 +108,12 @@ def stage_agent() -> None: for name in ( "agent-api-key", "chat-relay-key", - "gitea-token", - "gitea-username", "jenkins-image-build-token", "node-ssh-private-key", - "node-ssh-config", "node-ssh-known-hosts", ): _copy_secret(name, RUNTIME_ROOT / name) + _stage_node_ssh_config() _validated_json( "claude-credentials", RUNTIME_ROOT / "claude" / ".credentials.json", diff --git a/services/hermes/skills/manage-atlas-pull-requests/SKILL.md b/services/hermes/skills/manage-atlas-pull-requests/SKILL.md new file mode 100644 index 00000000..6194f7f9 --- /dev/null +++ b/services/hermes/skills/manage-atlas-pull-requests/SKILL.md @@ -0,0 +1,64 @@ +--- +name: manage-atlas-pull-requests +description: Read bounded private Atlas repository and pull-request metadata or create a verified draft pull request after a tested branch is pushed through Hermes' least-authority Forgejo client. Use for PR handoff in scm.bstein.dev/atlas repositories. Never use it to update, merge, approve, close, delete, or force-push. +--- + +# Manage Atlas pull requests + +Use `/opt/scm/gitea_api.py` for Forgejo API access. The agent pod has no +repository credential. The client calls a separate least-authority broker; +never bypass it with `curl` or direct Gitea HTTP. + +Use the configured broker Git remote for clone, fetch, and creation of a new +namespaced feature branch. Existing-ref updates, protected refs, deletion, +and force-push are rejected by the broker. Never replace that remote with a +credential-bearing URL. + +## Read repository or PR state + +Pass one allowed Atlas metadata API path to the read operation: + +```sh +/opt/scm/gitea_api.py read /api/v1/repos/atlas/REPO/pulls/NUMBER +``` + +The client exposes only repository metadata, PRs and PR evidence, branches, +commits, and commit status. Hooks, Actions secrets/variables, collaborators, +protections, reviews, merge endpoints, releases, and administration are +rejected even as reads. Responses are size-bounded. Treat returned state as +evidence, not authorization to change it. + +## Create a review handoff + +Before opening a PR, verify the remote, branch, clean worktree, diff, tests, and +exact pushed commit. Never force-push. Pass that full 40-character pushed head +SHA explicitly. Keep the short body free of credentials and credential-like +text. The client screens structured assignments, known token formats, and long +high-entropy values as an accidental-disclosure barrier. It cannot prove text +is secret-free: short or multiword secrets under innocuous names may resemble +ordinary prose, so never place credential material in either field. Structured +inspection is deliberately bounded; large, deeply nested, or unusual config +blobs do not belong in these fields and may be rejected. Validate without +reading a credential or using the network: + +```sh +/opt/scm/gitea_api.py --dry-run create-draft REPO \ + --base main --head hermes/TASK --head-sha FULL_PUSHED_SHA \ + --title "Focused change" --body "Tests and review evidence" +``` + +Run the same command without `--dry-run` only when the branch is review-ready. +The client accepts success only when Forgejo returns the exact requested +repository, base/head refs, head SHA, title/body, canonical URLs, positive PR +number, and open/unmerged/draft state. Report its URL and exact head commit to +Brad. Leave it unmerged for human review. PR updates are intentionally absent; +publish a corrected tested commit and ask Brad how to proceed if metadata must +change. + +## Stop at the authority boundary + +Update, merge, approve, close, delete, branch mutation, comments, releases, +repository administration, and force-push are intentionally unavailable. Do +not bypass the client with a raw HTTP request. If one of those actions is +needed, present the tested commit and evidence to Brad and stop for human +review. diff --git a/services/hermes/skills/manage-atlas-pull-requests/agents/openai.yaml b/services/hermes/skills/manage-atlas-pull-requests/agents/openai.yaml new file mode 100644 index 00000000..57cb8496 --- /dev/null +++ b/services/hermes/skills/manage-atlas-pull-requests/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Manage Atlas Pull Requests" + short_description: "Create safe draft PR handoffs for human review" + default_prompt: "Use $manage-atlas-pull-requests to inspect the requested Atlas repository or prepare a tested branch as an unmerged draft pull request. Preserve the human review boundary." diff --git a/services/vault/hermes-auth-role-bootstrap-job.yaml b/services/vault/hermes-auth-role-bootstrap-job.yaml index 4bcee103..c383f0e7 100644 --- a/services/vault/hermes-auth-role-bootstrap-job.yaml +++ b/services/vault/hermes-auth-role-bootstrap-job.yaml @@ -3,7 +3,7 @@ apiVersion: batch/v1 kind: Job metadata: - name: vault-k8s-auth-hermes-9 + name: vault-k8s-auth-hermes-10 namespace: vault spec: backoffLimit: 2 diff --git a/services/vault/scripts/vault_k8s_auth_configure.sh b/services/vault/scripts/vault_k8s_auth_configure.sh index 48e839e6..89c27121 100644 --- a/services/vault/scripts/vault_k8s_auth_configure.sh +++ b/services/vault/scripts/vault_k8s_auth_configure.sh @@ -269,8 +269,11 @@ write_policy_and_role "game-stream" "game-stream" "game-stream-vault" \ "game-stream/*" "" write_policy_and_role "hermes" "hermes" "hermes-vault,hermes-triage" \ "hermes/triage-oidc hermes/agent-tokens hermes/triage-api" "" +# The agent keeps its own credentials but no longer holds developer-gitea: +# the least-authority hermes-scm-broker role (below) carries the Gitea token +# so a compromised agent cannot reach the SCM write path directly. write_policy_and_role "hermes-agent" "hermes" "hermes-agent" \ - "hermes/agent-oidc hermes/agent-tokens hermes/chat-telegram hermes/developer-keycloak hermes/developer-gitea hermes/developer-harbor hermes/developer-jenkins hermes/developer-ssh" "" + "hermes/agent-oidc hermes/agent-tokens hermes/chat-telegram hermes/developer-keycloak hermes/developer-harbor hermes/developer-jenkins hermes/developer-ssh" "" write_policy_and_role "hermes-switchyard" "hermes" "hermes-switchyard" \ "hermes/chat-telegram" "" hermes_jenkins_token_seed_policy=' @@ -288,6 +291,8 @@ vault_cmd write "auth/kubernetes/role/hermes-jenkins-token-seed" \ bound_service_account_namespaces="vault" \ policies="hermes-jenkins-token-seed" \ ttl="${role_ttl}" +write_policy_and_role "hermes-scm-broker" "hermes-scm" "hermes-scm-broker" \ + "hermes/developer-gitea" "" write_policy_and_role "hermes-credential-sync" "hermes" "hermes-agent" \ "" "hermes/agent-tokens" write_policy_and_role "hermes-node-ssh" "hermes" "hermes-node-ssh-access" \ diff --git a/testing/quality_contract.json b/testing/quality_contract.json index 73d68fa2..bf6ace5c 100644 --- a/testing/quality_contract.json +++ b/testing/quality_contract.json @@ -32,11 +32,27 @@ "testing/quality_gate.py", "ci/tests/glue/test_ariadne_schedules.py", "ci/tests/glue/test_glue_metrics.py", - "testing/tests/test_publish_test_metrics.py", - "testing/tests/test_supply_chain_report.py", - "testing/tests/test_semgrep_report.py", - "testing/tests/test_quality_contract.py", - "testing/tests/test_quality_gate.py" + "testing/tests/test_publish_test_metrics.py", + "testing/tests/test_supply_chain_report.py", + "testing/tests/test_semgrep_report.py", + "testing/tests/test_quality_contract.py", + "testing/tests/test_quality_gate.py", + "services/gitea/scripts/gitea_branch_protection_check.py", + "services/hermes/scm-common/scripts/gitea_api.py", + "services/hermes/scm-common/scripts/gitea_api_policy.py", + "services/hermes/scm-common/scripts/scm_broker.py", + "services/hermes/scm-common/scripts/scm_broker_client.py", + "services/hermes/scm-common/scripts/scm_broker_io.py", + "services/hermes/scm-common/scripts/scm_broker_server.py", + "services/hermes/scripts/hermes_coordinator.py", + "services/hermes/scripts/node_account_audit.py", + "services/hermes/scripts/node_account_hardening.py", + "services/hermes/scripts/node_account_io.py", + "services/hermes/scripts/stage_runtime_access.py", + "services/hermes/scm-common/scripts/deadline_http.py", + "services/hermes/scm-common/scripts/git_pack_objects.py", + "services/hermes/scm-common/scripts/receive_pack_scan.py", + "services/hermes/scripts/node_polkit_audit.py" ], "lint_paths": [ "ci/scripts/hermes_image_release.py", @@ -52,7 +68,15 @@ "services/mailu/scripts/mailu_sync.py", "services/mailu/scripts/mailu_sync_listener.py", "testing/tests", - "testing" + "testing", + "services/gitea/scripts/gitea_branch_protection_check.py", + "services/hermes/scm-common/scripts", + "services/hermes/scripts/hermes_coordinator.py", + "services/hermes/scripts/node_account_audit.py", + "services/hermes/scripts/node_account_hardening.py", + "services/hermes/scripts/node_account_io.py", + "services/hermes/scripts/stage_runtime_access.py", + "services/hermes/scripts/node_polkit_audit.py" ], "pytest_suites": { "unit": { @@ -70,7 +94,9 @@ "services/harbor/scripts", "services/hermes/scripts", "services/mailu/scripts", - "testing" + "testing", + "services/gitea/scripts", + "services/hermes/scm-common/scripts" ], "coverage_xml": "build/coverage-unit.xml" }, @@ -136,7 +162,15 @@ "scripts/tests/**/*.py", "services/*/scripts/tests/**/*.py", "services/mailu/scripts/mailu_sync.py", - "services/mailu/scripts/mailu_sync_listener.py" + "services/mailu/scripts/mailu_sync_listener.py", + "services/gitea/scripts/gitea_branch_protection_check.py", + "services/hermes/scm-common/scripts/*.py", + "services/hermes/scripts/hermes_coordinator.py", + "services/hermes/scripts/node_account_audit.py", + "services/hermes/scripts/node_account_hardening.py", + "services/hermes/scripts/node_account_io.py", + "services/hermes/scripts/stage_runtime_access.py", + "services/hermes/scripts/node_polkit_audit.py" ], "naming_rules": [ { @@ -193,7 +227,23 @@ "testing/quality_docs.py", "testing/quality_hygiene.py", "testing/quality_coverage.py", - "testing/quality_gate.py" + "testing/quality_gate.py", + "services/gitea/scripts/gitea_branch_protection_check.py", + "services/hermes/scm-common/scripts/gitea_api.py", + "services/hermes/scm-common/scripts/gitea_api_policy.py", + "services/hermes/scm-common/scripts/scm_broker.py", + "services/hermes/scm-common/scripts/scm_broker_client.py", + "services/hermes/scm-common/scripts/scm_broker_io.py", + "services/hermes/scm-common/scripts/scm_broker_server.py", + "services/hermes/scripts/hermes_coordinator.py", + "services/hermes/scripts/node_account_audit.py", + "services/hermes/scripts/node_account_hardening.py", + "services/hermes/scripts/node_account_io.py", + "services/hermes/scripts/stage_runtime_access.py", + "services/hermes/scm-common/scripts/deadline_http.py", + "services/hermes/scm-common/scripts/git_pack_objects.py", + "services/hermes/scm-common/scripts/receive_pack_scan.py", + "services/hermes/scripts/node_polkit_audit.py" ] } } diff --git a/testing/tests/test_hermes_chat_images.py b/testing/tests/test_hermes_chat_images.py new file mode 100644 index 00000000..48c6c2e5 --- /dev/null +++ b/testing/tests/test_hermes_chat_images.py @@ -0,0 +1,282 @@ +"""Image generation and sandbox-boundary contracts for Hermes chat.""" + +from __future__ import annotations + +import importlib.util +import sys +import time +from types import SimpleNamespace + +import yaml + +from testing.tests.test_hermes_chat_support import ( + HERMES, + VAULT, + _documents, +) + + +def test_chat_image_generation_uses_private_owner_broker(): + """Family pods get image bytes without receiving the owner's OAuth file.""" + configmap = _documents(HERMES / "chat-configmap.yaml")[0] + assert "shared desktop/Wolf lane" in configmap["data"]["SOUL.md"] + assert "local FLUX waits" in configmap["data"]["SOUL.md"] + assert "Use `image_generate_local`" in configmap["data"]["SOUL.md"] + assert "Use `image_generate_hosted`" in configmap["data"]["SOUL.md"] + assert "ComfyUI endpoint" in configmap["data"]["SOUL.md"] + assert "`MEDIA:` path" in configmap["data"]["SOUL.md"] + assert "`image_edit_latest`" in configmap["data"]["SOUL.md"] + assert ( + "Preserve the most recently selected image lane" in configmap["data"]["SOUL.md"] + ) + config = yaml.safe_load(configmap["data"]["config.yaml"]) + assert config["image_gen"] == { + "provider": "atlas-broker", + "model": "atlas-image-auto-high", + } + assert config["plugins"]["enabled"] == ["atlas-broker", "auto-router"] + + statefulset = _documents(HERMES / "chat-statefulset.yaml")[0] + pod = statefulset["spec"]["template"]["spec"] + hermes = next(item for item in pod["containers"] if item["name"] == "hermes") + mounts = {item["name"]: item for item in hermes["volumeMounts"]} + assert mounts["image-plugin"]["mountPath"] == ( + "/opt/hermes/plugins/image_gen/atlas-broker" + ) + assert mounts["runtime-access"]["mountPath"] == "/runtime-access" + assert "provider-auth" not in mounts + assert not any( + mount["name"] == "home" and "agent" in str(mount) + for mount in hermes["volumeMounts"] + ) + env = {item["name"]: item["value"] for item in hermes["env"]} + assert env["HERMES_IMAGE_BROKER_URL"].startswith("http://hermes-image-broker.") + + plugin = (HERMES / "plugins" / "image-gen-broker" / "__init__.py").read_text() + assert '"local": "flux-2-klein-4b-local"' in plugin + assert '"hosted": "gpt-image-2-high"' in plugin + assert 'name="image_generate_local"' in plugin + assert 'name="image_generate_hosted"' in plugin + assert '"name": "image_edit_latest"' in plugin + assert '"name": "image_edit_latest_local"' in plugin + assert '"name": "image_edit_latest_hosted"' in plugin + assert "def _latest_generated_image" in plugin + assert "newest MEDIA: path from the conversation" in plugin + assert 'candidate.upper().startswith("MEDIA:")' in plugin + assert "override=True" not in plugin + + agent = _documents(HERMES / "agent-deployment.yaml")[0] + containers = agent["spec"]["template"]["spec"]["containers"] + broker = next(item for item in containers if item["name"] == "image-broker") + assert broker["ports"] == [ + {"name": "image-broker", "containerPort": 9002, "protocol": "TCP"} + ] + assert broker["securityContext"]["readOnlyRootFilesystem"] is True + assert broker["securityContext"]["runAsNonRoot"] is True + + services = _documents(HERMES / "service.yaml") + service = next( + item for item in services if item["metadata"]["name"] == "hermes-image-broker" + ) + assert service["spec"]["selector"] == {"app": "hermes-agent"} + + oauth_store = _documents(HERMES / "oauth-session-store.yaml") + redis = next(item for item in oauth_store if item["kind"] == "Deployment") + assert redis["spec"]["strategy"]["type"] == "Recreate" + assert "--appendonly" in redis["spec"]["template"]["spec"]["containers"][0]["args"] + + policies = _documents(HERMES / "networkpolicy.yaml") + agent_policy = next( + item + for item in policies + if item["metadata"]["name"] == "hermes-agent-isolation" + ) + broker_ingress = next( + rule + for rule in agent_policy["spec"]["ingress"] + if {port["port"] for port in rule["ports"]} == {9002, 9003} + ) + assert broker_ingress["from"][0]["podSelector"]["matchLabels"] == { + "app": "hermes-chat-tenant" + } + vault_policy = (VAULT / "scripts" / "vault_k8s_auth_configure.sh").read_text() + agent_role = vault_policy[ + vault_policy.index('write_policy_and_role "hermes-agent"') : vault_policy.index( + "write_policy_and_role", + vault_policy.index('write_policy_and_role "hermes-agent"') + 1, + ) + ] + assert "hermes/developer-gitea" not in agent_role + assert 'write_policy_and_role "hermes-scm-broker" "hermes-scm"' in vault_policy + assert ( + 'write_policy_and_role "hermes-node-ssh" "hermes" ' + '"hermes-node-ssh-access"' in vault_policy + ) + + +def test_compact_image_edit_resolves_latest_tenant_artifact(tmp_path, monkeypatch): + """Follow-up edits resolve the source server-side and keep tool JSON small.""" + provider_module = SimpleNamespace( + DEFAULT_ASPECT_RATIO="square", + ImageGenProvider=object, + error_response=lambda **value: value, + normalize_reference_images=lambda value: value, + resolve_aspect_ratio=lambda value: value, + save_b64_image=lambda *_args, **_kwargs: tmp_path / "saved.png", + success_response=lambda **value: value, + ) + monkeypatch.setitem(sys.modules, "agent", SimpleNamespace()) + monkeypatch.setitem(sys.modules, "agent.image_gen_provider", provider_module) + spec = importlib.util.spec_from_file_location( + "hermes_image_plugin", + HERMES / "plugins" / "image-gen-broker" / "__init__.py", + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + image_dir = tmp_path / "cache" / "images" + image_dir.mkdir(parents=True) + older = image_dir / "atlas_flux-old.png" + newest = image_dir / "atlas_gpt-image-new.png" + older.write_bytes(b"older") + newest.write_bytes(b"newest") + older.touch() + time.sleep(0.001) + newest.touch() + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + + calls = [] + monkeypatch.setattr( + module, + "_handle_image_generate", + lambda args, route: calls.append((args, route)) or "ok", + ) + assert module._handle_hosted_edit({"prompt": "make it a clown"}) == "ok" + assert calls == [ + ( + { + "prompt": "make it a clown", + "image_url": str(newest.resolve()), + }, + "hosted", + ) + ] + + +def test_chat_reasoning_uses_switchyard_without_owner_credentials(): + """Family pods use AUTO/manual routes without mounting owner credentials.""" + configmap = _documents(HERMES / "chat-configmap.yaml")[0] + config = yaml.safe_load(configmap["data"]["config.yaml"]) + assert config["model"] == { + "provider": "atlas-switchyard", + "default": "atlas/auto/fast", + "model": "atlas/auto/fast", + } + assert config["providers"]["atlas-switchyard"] == { + "name": "Automatic Router", + "api": "http://hermes-switchyard.hermes.svc.cluster.local:9005/v1", + "api_key": "atlas-switchyard", + "default_model": "atlas/auto/fast", + "transport": "chat_completions", + } + assert config["platforms"]["api_server"]["extra"]["model_routes"] == { + route: {"provider": "atlas-switchyard", "model": route} + for route in [ + "atlas/auto/fast", + "atlas/auto/balanced", + "atlas/auto/deep", + "atlas/auto/maximum", + "atlas/manual/codex/luna", + "atlas/manual/codex/terra", + "atlas/manual/codex/sol", + "atlas/manual/claude/haiku", + "atlas/manual/claude/fable", + "atlas/manual/claude/sonnet", + "atlas/manual/claude/opus", + "atlas/manual/local/qwen-14b", + ] + } + + agent = _documents(HERMES / "agent-deployment.yaml")[0] + containers = agent["spec"]["template"]["spec"]["containers"] + broker = next(item for item in containers if item["name"] == "codex-broker") + assert broker["ports"] == [ + {"name": "codex-broker", "containerPort": 9003, "protocol": "TCP"} + ] + assert broker["securityContext"]["readOnlyRootFilesystem"] is True + assert broker["securityContext"]["runAsNonRoot"] is True + assert {item["name"]: item["value"] for item in broker["env"]}.items() >= { + "PYTHONPATH": "/opt/hermes", + "HERMES_CODEX_BROKER_LISTEN_PORT": "9003", + "HERMES_ROUTING_CATALOG_PATH": "/routing-catalog/catalog.json", + }.items() + + services = _documents(HERMES / "service.yaml") + service = next( + item for item in services if item["metadata"]["name"] == "hermes-codex-broker" + ) + assert service["spec"]["selector"] == {"app": "hermes-agent"} + assert service["spec"]["ports"] == [ + { + "name": "http", + "port": 9003, + "targetPort": "codex-broker", + "protocol": "TCP", + } + ] + + statefulset = _documents(HERMES / "chat-statefulset.yaml")[0] + assert ( + statefulset["spec"]["template"]["metadata"]["annotations"][ + "ai.bstein.dev/config-rev" + ] + == "20260816-telegram-topics" + ) + pod_spec = statefulset["spec"]["template"]["spec"] + patch_init = next( + item + for item in pod_spec["initContainers"] + if item["name"] == "patch-stream-recovery" + ) + assert patch_init["command"][-2:] == [ + "/opt/hermes/agent/conversation_loop.py", + "/patched/conversation_loop.py", + ] + hermes = next(item for item in pod_spec["containers"] if item["name"] == "hermes") + assert { + "name": "stream-recovery-patch", + "mountPath": "/opt/hermes/agent/conversation_loop.py", + "subPath": "conversation_loop.py", + } in hermes["volumeMounts"] + api_session_init = next( + item + for item in pod_spec["initContainers"] + if item["name"] == "patch-api-server-sessions" + ) + assert "patch_api_server_sessions.py" in api_session_init["args"][0] + assert "migrate_telegram_api_sessions.py" in api_session_init["args"][0] + assert { + "name": "api-server-patch", + "mountPath": "/opt/hermes/gateway/platforms/api_server.py", + "subPath": "api_server.py", + } in hermes["volumeMounts"] + assert any(volume["name"] == "api-server-patch" for volume in pod_spec["volumes"]) + assert not any( + mount["mountPath"].endswith("/.codex") for mount in hermes["volumeMounts"] + ) + + policies = _documents(HERMES / "networkpolicy.yaml") + agent_policy = next( + item + for item in policies + if item["metadata"]["name"] == "hermes-agent-isolation" + ) + broker_ingress = next( + rule + for rule in agent_policy["spec"]["ingress"] + if {port["port"] for port in rule["ports"]} == {9002, 9003} + ) + assert broker_ingress["from"][0]["podSelector"]["matchLabels"] == { + "app": "hermes-chat-tenant" + } diff --git a/testing/tests/test_hermes_chat_local_runtime.py b/testing/tests/test_hermes_chat_local_runtime.py new file mode 100644 index 00000000..ddb0ef40 --- /dev/null +++ b/testing/tests/test_hermes_chat_local_runtime.py @@ -0,0 +1,172 @@ +"""Local inference and sandbox execution contracts for Hermes chat.""" + +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path +from types import SimpleNamespace + + +from testing.tests.test_hermes_chat_support import ( + HERMES, + ROOT, + _documents, +) + + +def test_titan20_serializes_classifier_and_local_chat_model_residency(): + """Classifier and local chat share one serialized resident Qwen weight.""" + deployment = _documents( + Path(__file__).parents[2] / "services/ai-llm/deployment.yaml" + )[0] + pod = deployment["spec"]["template"]["spec"] + required = pod["affinity"]["nodeAffinity"][ + "requiredDuringSchedulingIgnoredDuringExecution" + ]["nodeSelectorTerms"][0]["matchExpressions"][0] + assert required["values"] == ["titan-20"] + container = pod["containers"][0] + env = {item["name"]: item["value"] for item in container["env"]} + assert env["OLLAMA_MAX_LOADED_MODELS"] == "1" + assert env["OLLAMA_NUM_PARALLEL"] == "1" + assert env["OLLAMA_KEEP_ALIVE"] == "-1" + assert env["OLLAMA_CONTEXT_LENGTH"] == "8192" + warm_command = " ".join(container["command"]) + assert "--keepalive=-1" not in warm_command + models = next(item for item in pod["volumes"] if item["name"] == "models") + assert models["persistentVolumeClaim"]["claimName"] == ("ollama-models-titan20") + + +def test_local_image_gpu_guard_distinguishes_background_and_saturated_gpu(monkeypatch): + """Lease-idle desktop spikes may coexist, but saturation still blocks FLUX.""" + source = ROOT / "dockerfiles" / "hermes-local-image-server.py" + spec = importlib.util.spec_from_file_location("hermes_local_image_server", source) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + monkeypatch.setattr(module, "GPU_ACTIVITY_NODE", "titan-24") + monkeypatch.setattr(module, "GPU_ACTIVE_SM_PERCENT", 80.0) + monkeypatch.setattr(module, "GPU_MAX_EXTERNAL_MEMORY_BYTES", 3 << 30) + + idle = module._parse_gpu_activity( + "\n".join( + [ + 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="Xorg"} 0', + 'nvidia_process_gpu_memory_used_bytes{node="titan-24",namespace="host",process="Xorg"} 1900000000', + 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="game-stream",process="wolf"} 3', + 'nvidia_process_gpu_memory_used_bytes{node="titan-24",namespace="hermes",process="python"} 9000000000', + ] + ) + ) + assert idle["interactive_active"] is False + assert idle["external_gpu_memory_bytes"] == 1900000000 + assert idle["external_gpu_sm_percent"] == 3 + + background_spike = module._parse_gpu_activity( + 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="sway"} 41\n' + ) + assert background_spike["interactive_active"] is False + + active = module._parse_gpu_activity( + 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="steam"} 91\n' + ) + assert active["interactive_active"] is True + assert "91%" in active["gpu_guard_reason"] + + +def test_local_flux_renderer_uses_a_disposable_cuda_worker(monkeypatch): + """A completed render must not leave its CUDA context in the API process.""" + source = ROOT / "dockerfiles" / "hermes-local-image-server.py" + spec = importlib.util.spec_from_file_location("hermes_local_image_worker", source) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + calls = [] + + def run(command, **kwargs): + calls.append((command, kwargs)) + return SimpleNamespace( + returncode=0, + stdout=b'{"success":true,"route":"local","image_b64":"cG5n"}', + stderr=b"", + ) + + monkeypatch.setattr(module.subprocess, "run", run) + result = module._render({"prompt": "black cat", "aspect_ratio": "square"}) + + assert result["route"] == "local" + command, options = calls[0] + assert command[-1] == "--render-worker" + assert json.loads(options["input"]) == { + "prompt": "black cat", + "aspect_ratio": "square", + } + assert options["timeout"] == module.RENDER_TIMEOUT_SECONDS + assert options["check"] is False + + +def test_local_flux_uses_low_vram_offload_without_reducing_resolution(): + """The shared 3080 lane must trade time, not image size, for headroom.""" + source = (ROOT / "dockerfiles" / "hermes-local-image-server.py").read_text() + assert "OFFLOAD_MODE = os.environ.get(" in source + assert '"HERMES_LOCAL_IMAGE_OFFLOAD_MODE", "sequential"' in source + assert "pipe.enable_sequential_cpu_offload()" in source + assert '"square": (1024, 1024)' in source + + +def test_chat_auth_and_relay_are_pod_lifetime_only(): + statefulset = _documents(HERMES / "chat-statefulset.yaml")[0] + pod = statefulset["spec"]["template"]["spec"] + containers = statefulset["spec"]["template"]["spec"]["containers"] + + for name in ("hermes", "webui"): + container = next(item for item in containers if item["name"] == name) + env = {item["name"]: item["value"] for item in container["env"]} + assert env["HERMES_AUTH_FILE"] == "/runtime-access/hermes-auth.json" + mount = next( + item + for item in container["volumeMounts"] + if item["name"] == "runtime-access" + ) + assert mount["mountPath"] == "/runtime-access" + assert "subPath" not in mount + + hermes_env = { + item["name"]: item["value"] + for item in next(item for item in containers if item["name"] == "hermes")["env"] + } + runtime = next(item for item in pod["volumes"] if item["name"] == "runtime-access") + assert runtime["emptyDir"] == {"medium": "Memory", "sizeLimit": "2Mi"} + assert not any(item["name"] == "provider-auth" for item in pod["volumes"]) + init_command = next( + item for item in pod["initContainers"] if item["name"] == "init-config" + )["command"][2] + for key in ( + "ANTHROPIC_API_KEY", + "API_SERVER_KEY", + "CLAUDE_CODE_OAUTH_TOKEN", + "GITEA_TOKEN", + "HERMES_IMAGE_BROKER_KEY", + "OPENAI_API_KEY", + ): + assert key in init_command + assert "printf 'API_SERVER_KEY=%s" not in init_command + assert hermes_env["AGENT_BROWSER_EXECUTABLE_PATH"].endswith( + "/chrome-linux/headless_shell" + ) + assert "--no-sandbox" in hermes_env["AGENT_BROWSER_ARGS"] + + +def test_sandbox_executes_python_with_bounded_output(tmp_path: Path, monkeypatch): + source = ROOT / "dockerfiles" / "hermes-chat-sandbox-server.py" + spec = importlib.util.spec_from_file_location("hermes_chat_sandbox_server", source) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + monkeypatch.setattr(module, "WORKSPACE", tmp_path) + + result = module._execute("import math\nprint(math.comb(10, 3))") + + assert result["success"] is True + assert result["stdout"] == "120\n" + assert result["stderr"] == "" diff --git a/testing/tests/test_hermes_chat_provider_auth.py b/testing/tests/test_hermes_chat_provider_auth.py new file mode 100644 index 00000000..8e432131 --- /dev/null +++ b/testing/tests/test_hermes_chat_provider_auth.py @@ -0,0 +1,476 @@ +"""Provider subscription and OAuth contracts for Hermes chat.""" + +from __future__ import annotations + +import base64 +import json +import sys +import time +from pathlib import Path +from types import ModuleType + +import pytest + +from testing.tests.test_hermes_chat_support import ( + _load_broker_module, +) + + +def test_codex_broker_auth_and_request_contract(tmp_path: Path, monkeypatch): + """The relay is bounded, stateless, and rejects unapproved models.""" + module = _load_broker_module("hermes_codex_broker", "codex_broker.py", monkeypatch) + monkeypatch.setattr(module, "TOKEN", "relay-secret") + + assert module._authorized("Bearer relay-secret") is True + assert module._authorized("Bearer wrong") is False + assert module._real_model("route/codex/gpt-5.6-sol/xhigh") == "gpt-5.6-sol" + payload = module._validate_payload( + { + "model": "gpt-5.6-terra", + "input": "route this chat turn", + "store": True, + "stream": False, + "max_output_tokens": 96, + "max_completion_tokens": 96, + "max_tokens": 96, + "temperature": 0.7, + "top_p": 0.9, + } + ) + assert payload["store"] is False + assert payload["stream"] is True + assert "max_output_tokens" not in payload + assert "max_completion_tokens" not in payload + assert "max_tokens" not in payload + assert "temperature" not in payload + assert "top_p" not in payload + assert payload["input"] == [ + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "route this chat turn"}], + } + ] + response_item = { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "keep this item"}], + } + assert module._validate_payload({"model": "gpt-5.6-terra", "input": response_item})[ + "input" + ] == [response_item] + response_items = [response_item] + assert ( + module._validate_payload({"model": "gpt-5.6-terra", "input": response_items})[ + "input" + ] + is response_items + ) + image_items = [ + { + "type": "message", + "role": "user", + "content": [ + {"type": "input_text", "text": "What color is this?"}, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,cHJpdmF0ZQ==", + "detail": "high", + }, + }, + ], + } + ] + assert module._validate_payload({"model": "gpt-5.6-terra", "input": image_items})[ + "input" + ][0]["content"][1] == { + "type": "input_image", + "image_url": "data:image/png;base64,cHJpdmF0ZQ==", + "detail": "high", + } + switchyard_image_items = [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "What color is this?"}, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,cHJpdmF0ZQ==", + "detail": "auto", + }, + }, + ], + } + ] + assert module._validate_payload( + {"model": "gpt-5.6-terra", "input": switchyard_image_items} + )["input"][0]["content"][1] == { + "type": "input_image", + "image_url": "data:image/png;base64,cHJpdmF0ZQ==", + "detail": "auto", + } + switchyard_base64_items = [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "What color is this?"}, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "cHJpdmF0ZQ==", + }, + }, + ], + } + ] + normalized_base64 = module._validate_payload( + {"model": "gpt-5.6-terra", "input": switchyard_base64_items} + )["input"][0]["content"][1] + assert normalized_base64 == { + "type": "input_image", + "image_url": "data:image/png;base64,cHJpdmF0ZQ==", + } + switchyard_enum_items = [ + { + "role": "user", + "content": [ + { + "type": "input_image", + "image_url": { + "type": "url", + "data": { + "url": "data:image/png;base64,cHJpdmF0ZQ==", + "detail": "high", + }, + }, + } + ], + } + ] + nested_image = module._validate_payload( + {"model": "gpt-5.6-terra", "input": switchyard_enum_items} + )["input"][0]["content"][0] + assert nested_image["image_url"] == "data:image/png;base64,cHJpdmF0ZQ==" + assert nested_image["detail"] == "high" + with pytest.raises(ValueError, match=r"non-empty Responses image URL.*str\[4\]"): + module._validate_payload( + { + "model": "gpt-5.6-terra", + "input": [ + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_image", + "image_url": {"detail": "high"}, + } + ], + } + ], + } + ) + routed = module._validate_payload( + { + "model": "route/codex/gpt-5.6-luna/low", + "input": "use the low route", + "stream": False, + } + ) + assert routed["model"] == "gpt-5.6-luna" + with pytest.raises(ValueError, match="unsupported Codex model"): + module._validate_payload({"model": "unapproved-model", "input": "hello"}) + with pytest.raises(ValueError, match="non-empty Responses input"): + module._validate_payload({"model": "gpt-5.6-terra", "input": ""}) + with pytest.raises(ValueError, match="non-empty Responses input list"): + module._validate_payload({"model": "gpt-5.6-terra", "input": []}) + + completed = { + "id": "resp_test", + "object": "response", + "status": "completed", + "output": [], + } + completed_item = { + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "done"}], + } + assert module._completed_response( + [ + "event: response.created", + 'data: {"type":"response.created","response":{}}', + "event: response.output_item.done", + "data: " + + json.dumps( + { + "type": "response.output_item.done", + "output_index": 0, + "item": completed_item, + } + ), + "event: response.completed", + "data: " + + json.dumps({"type": "response.completed", "response": completed}), + "data: [DONE]", + ] + )["output"] == [completed_item] + raw_stream = ( + "event: response.output_item.done\n" + "data: " + + json.dumps( + { + "type": "response.output_item.done", + "output_index": 0, + "item": completed_item, + } + ) + + "\n\nevent: response.completed\ndata: " + + json.dumps({"type": "response.completed", "response": completed}) + + "\n\n" + ).encode() + normalized = module._normalized_stream( + raw_stream, {**completed, "output": [completed_item]} + ).decode() + terminal_data = next( + line for line in normalized.splitlines() if '"response.completed"' in line + ) + assert json.loads(terminal_data.removeprefix("data: "))["response"]["output"] == [ + completed_item + ] + assert normalized.endswith("\n\n") + streamed_function_item = { + "type": "function_call", + "name": "read_file", + "status": "completed", + "arguments": '{"path":"/tmp"}', + } + streamed_function_body = ( + "event: response.function_call_arguments.delta\n" + 'data: {"type":"response.function_call_arguments.delta",' + '"item_id":"call_1","delta":"{\\"path\\":\\"/tmp\\"}"}\n\n' + "event: response.function_call_arguments.done\n" + 'data: {"type":"response.function_call_arguments.done",' + '"item_id":"call_1","arguments":"{\\"path\\":\\"/tmp\\"}"}\n\n' + "event: response.output_item.done\n" + 'data: {"type":"response.output_item.done","output_index":0,' + '"item":{"type":"function_call","name":"read_file",' + '"arguments":"{\\"path\\":\\"/tmp\\"}"}}\n\n' + "event: response.completed\n" + "data: " + + json.dumps( + { + "type": "response.completed", + "response": {**completed, "output": [streamed_function_item]}, + } + ) + + "\n\n" + ).encode() + normalized_function_stream = module._normalized_stream( + streamed_function_body, {**completed, "output": [streamed_function_item]} + ).decode() + assert "response.function_call_arguments.delta" in normalized_function_stream + assert "response.function_call_arguments.done" not in normalized_function_stream + assert "response.output_item.done" not in normalized_function_stream + normalized_terminal = next( + line + for line in normalized_function_stream.splitlines() + if '"response.completed"' in line + ) + assert ( + json.loads(normalized_terminal.removeprefix("data: "))["response"]["output"] + == [] + ) + with pytest.raises(RuntimeError, match="retryable incomplete response"): + module._completed_response( + [ + "event: response.incomplete", + 'data: {"type":"response.incomplete","response":' + '{"status":"incomplete","incomplete_details":' + '{"reason":"max_output_tokens"}}}', + ] + ) + with pytest.raises(RuntimeError, match="provider unavailable"): + module._completed_response( + [ + "event: error", + 'data: {"type":"error","error":{"message":"provider unavailable"}}', + ] + ) + malformed_tool_item = { + "type": "function_call", + "name": "search_files", + "status": "completed", + "arguments": '{"path":"","offset":', + } + with pytest.raises(RuntimeError, match="malformed function arguments"): + module._completed_response( + [ + "event: response.output_item.done", + "data: " + + json.dumps( + { + "type": "response.output_item.done", + "output_index": 0, + "item": malformed_tool_item, + } + ), + "event: response.completed", + "data: " + + json.dumps({"type": "response.completed", "response": completed}), + ] + ) + valid_tool_item = { + **malformed_tool_item, + "arguments": '{"path":"","offset":0}', + } + assert module._completed_response( + [ + "event: response.output_item.done", + "data: " + + json.dumps( + { + "type": "response.output_item.done", + "output_index": 0, + "item": valid_tool_item, + } + ), + "event: response.completed", + "data: " + + json.dumps({"type": "response.completed", "response": completed}), + ] + )["output"] == [valid_tool_item] + with pytest.raises(RuntimeError, match="malformed function arguments"): + module._completed_response( + [ + "event: response.function_call_arguments.delta", + 'data: {"type":"response.function_call_arguments.delta",' + '"item_id":"call_1","output_index":0,' + '"delta":"{\\"path\\":\\"/tmp\\",\\"offset\\":"}', + "event: response.completed", + "data: " + + json.dumps({"type": "response.completed", "response": completed}), + ] + ) + streamed_tool = module._completed_response( + [ + "event: response.function_call_arguments.delta", + 'data: {"type":"response.function_call_arguments.delta",' + '"item_id":"call_2","output_index":0,' + '"delta":"{\\"path\\":\\"/tmp\\",\\"offset\\":"}', + "event: response.function_call_arguments.done", + 'data: {"type":"response.function_call_arguments.done",' + '"item_id":"call_2","output_index":0,' + '"arguments":"{\\"path\\":\\"/tmp\\",\\"offset\\":0}"}', + "event: response.completed", + "data: " + + json.dumps({"type": "response.completed", "response": completed}), + ] + ) + assert streamed_tool["status"] == "completed" + + auth_dir = tmp_path / ".codex" + auth_dir.mkdir() + # The token payload need only prove the broker reads CODEX_HOME directly. + encoded = ( + base64.urlsafe_b64encode(json.dumps({"exp": time.time() + 3600}).encode()) + .decode() + .rstrip("=") + ) + (auth_dir / "auth.json").write_text( + json.dumps({"tokens": {"access_token": f"header.{encoded}.signature"}}) + ) + monkeypatch.setenv("CODEX_HOME", str(auth_dir)) + assert module._access_token().startswith("header.") + + +def test_codex_broker_refreshes_and_persists_first_party_oauth( + tmp_path: Path, monkeypatch +): + """Expired ChatGPT OAuth refreshes in the canonical Codex CLI store.""" + module = _load_broker_module( + "hermes_codex_refresh_broker", "codex_broker.py", monkeypatch + ) + auth_dir = tmp_path / ".codex" + auth_dir.mkdir() + + def jwt(expires_at: float) -> str: + payload = ( + base64.urlsafe_b64encode(json.dumps({"exp": expires_at}).encode()) + .decode() + .rstrip("=") + ) + return f"header.{payload}.signature" + + expired = jwt(time.time() - 60) + live = jwt(time.time() + 3600) + auth_path = auth_dir / "auth.json" + auth_path.write_text( + json.dumps( + { + "auth_mode": "chatgpt", + "tokens": { + "access_token": expired, + "refresh_token": "refresh-old", + }, + } + ) + ) + calls = [] + auth_module = ModuleType("hermes_cli.auth") + + def refresh(access_token, refresh_token, *, timeout_seconds): + calls.append((access_token, refresh_token, timeout_seconds)) + return { + "access_token": live, + "refresh_token": "refresh-new", + "last_refresh": "2026-08-12T20:00:00Z", + } + + auth_module.refresh_codex_oauth_pure = refresh + package = ModuleType("hermes_cli") + package.auth = auth_module + monkeypatch.setitem(sys.modules, "hermes_cli", package) + monkeypatch.setitem(sys.modules, "hermes_cli.auth", auth_module) + monkeypatch.setenv("CODEX_HOME", str(auth_dir)) + + assert module._access_token() == live + persisted = json.loads(auth_path.read_text()) + assert persisted["tokens"]["access_token"] == live + assert persisted["tokens"]["refresh_token"] == "refresh-new" + assert persisted["last_refresh"] == "2026-08-12T20:00:00Z" + assert calls == [(expired, "refresh-old", 30.0)] + assert auth_path.stat().st_mode & 0o777 == 0o600 + + # A healthy token is reused, so repeated routed turns do not spend a + # refresh token or create a second billing/authentication path. + assert module._access_token() == live + assert len(calls) == 1 + + +def test_claude_broker_uses_native_subscription_without_api_billing(monkeypatch): + """Claude traffic must use the native first-party CLI subscription lane.""" + module = _load_broker_module( + "hermes_claude_broker", "claude_oauth_broker.py", monkeypatch + ) + monkeypatch.setenv("ANTHROPIC_API_KEY", "must-not-leak") + monkeypatch.setenv("CLAUDE_API_KEY", "must-not-leak") + monkeypatch.setattr( + module, + "resolve_route", + lambda route: "claude-fable-5" if "/fable/" in route else route, + ) + + model, effort = module._route( + "route/claude/fable/xhigh", {"output_config": {"effort": "xhigh"}} + ) + + assert (model, effort) == ("claude-fable-5", "xhigh") + assert "ANTHROPIC_API_KEY" not in module._claude_environment() + assert "CLAUDE_API_KEY" not in module._claude_environment() + assert module.CAPACITY_PATTERN.search("weekly usage limit exhausted") diff --git a/testing/tests/test_hermes_chat_quality.py b/testing/tests/test_hermes_chat_quality.py index 818f7679..b513b452 100644 --- a/testing/tests/test_hermes_chat_quality.py +++ b/testing/tests/test_hermes_chat_quality.py @@ -1,49 +1,18 @@ -"""Contracts for isolated high-quality Hermes chat capabilities.""" +"""Core configuration and workload contracts for Hermes chat.""" from __future__ import annotations -import base64 import importlib.util -import json -import sqlite3 import sys -import time -import tomllib -from pathlib import Path -from types import ModuleType, SimpleNamespace +from types import SimpleNamespace -import pytest import yaml - -ROOT = Path(__file__).parents[2] -HERMES = ROOT / "services" / "hermes" -VAULT = ROOT / "services" / "vault" - - -def _documents(path: Path) -> list[dict]: - return [doc for doc in yaml.safe_load_all(path.read_text()) if doc] - - -def _load_broker_module(name: str, filename: str, monkeypatch): - """Load one broker with its mounted routing-catalog dependency.""" - catalog_path = HERMES / "scripts" / "routing_catalog.py" - catalog_spec = importlib.util.spec_from_file_location( - "routing_catalog", catalog_path - ) - assert catalog_spec and catalog_spec.loader - catalog = importlib.util.module_from_spec(catalog_spec) - catalog_spec.loader.exec_module(catalog) - monkeypatch.setitem(sys.modules, "routing_catalog", catalog) - monkeypatch.setitem(sys.modules, "httpx", SimpleNamespace()) - - broker_path = HERMES / "scripts" / filename - spec = importlib.util.spec_from_file_location(name, broker_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - +from testing.tests.test_hermes_chat_support import ( + HERMES, + ROOT, + _documents, +) def test_chat_config_enables_real_research_compute_and_delegation(): configmap = _documents(HERMES / "chat-configmap.yaml")[0] @@ -92,16 +61,23 @@ def test_agent_config_keeps_delegated_reviewers_from_owning_task_lifecycle(): assert "Atlas organization has private visibility" in instructions assert "may be public or private" in instructions assert "do not infer a\nrepository's visibility" in instructions - assert "already supplied through `GIT_ASKPASS`" in instructions + assert "brokered Git for clone, fetch" in instructions + assert "client carries no repository credential" in instructions assert "Never call `kanban_show` without a known, non-empty task ID" in instructions assert "bounded ad-hoc inspection and acceptance checks may" in instructions assert "load implementation or TDD skills" in instructions assert "Never call `kanban_show` without\na known, non-empty task ID" in soul assert "must load a skill only when its workflow\nmaterially applies" in soul - assert "runtime-only `GIT_ASKPASS`" in soul - assert "`scm.bstein.dev` is Gitea, not GitHub" in instructions - assert "Never load or follow a GitHub/`gh`" in instructions - assert "/opt/coordinator/gitea_api.py METHOD /api/v1/..." in instructions + assert "no repository token is present in this pod" in soul + assert "`scm.bstein.dev` is Forgejo/Gitea, not GitHub" in instructions + assert "GitHub/`gh` skill for an Atlas remote" in instructions + assert "load\n`$manage-atlas-pull-requests`" in instructions + assert "Updates, merge,\napprove, close, delete, comments" in instructions + assert "Leave every PR\nunmerged for Brad's review" in instructions + assert "Use the Gitea pull-request merge endpoint" not in instructions + assert "reconcile the open PR" not in instructions + assert "use Flux reconciliation after" not in instructions + assert "PR publication ends at the verified open draft" in instructions assert "JENKINS_BASE_URL" in instructions assert "Do not\nuse `git reset --hard`" in instructions rendered = (HERMES / "agent-deployment.yaml").read_text() @@ -478,1765 +454,3 @@ def test_voice_workloads_have_deliberate_xavier_placement(): assert tts["containers"][0]["resources"]["limits"]["cpu"] == "4" assert all("hostPath" not in volume for volume in stt["volumes"]) assert all("hostPath" not in volume for volume in tts["volumes"]) - - -def test_chat_image_generation_uses_private_owner_broker(): - """Family pods get image bytes without receiving the owner's OAuth file.""" - configmap = _documents(HERMES / "chat-configmap.yaml")[0] - assert "shared desktop/Wolf lane" in configmap["data"]["SOUL.md"] - assert "local FLUX waits" in configmap["data"]["SOUL.md"] - assert "Use `image_generate_local`" in configmap["data"]["SOUL.md"] - assert "Use `image_generate_hosted`" in configmap["data"]["SOUL.md"] - assert "ComfyUI endpoint" in configmap["data"]["SOUL.md"] - assert "`MEDIA:` path" in configmap["data"]["SOUL.md"] - assert "`image_edit_latest`" in configmap["data"]["SOUL.md"] - assert "Preserve the most recently selected image lane" in configmap["data"]["SOUL.md"] - config = yaml.safe_load(configmap["data"]["config.yaml"]) - assert config["image_gen"] == { - "provider": "atlas-broker", - "model": "atlas-image-auto-high", - } - assert config["plugins"]["enabled"] == ["atlas-broker", "auto-router"] - - statefulset = _documents(HERMES / "chat-statefulset.yaml")[0] - pod = statefulset["spec"]["template"]["spec"] - hermes = next(item for item in pod["containers"] if item["name"] == "hermes") - mounts = {item["name"]: item for item in hermes["volumeMounts"]} - assert mounts["image-plugin"]["mountPath"] == ( - "/opt/hermes/plugins/image_gen/atlas-broker" - ) - assert mounts["runtime-access"]["mountPath"] == "/runtime-access" - assert "provider-auth" not in mounts - assert not any(mount["name"] == "home" and "agent" in str(mount) for mount in hermes["volumeMounts"]) - env = {item["name"]: item["value"] for item in hermes["env"]} - assert env["HERMES_IMAGE_BROKER_URL"].startswith("http://hermes-image-broker.") - - plugin = (HERMES / "plugins" / "image-gen-broker" / "__init__.py").read_text() - assert '"local": "flux-2-klein-4b-local"' in plugin - assert '"hosted": "gpt-image-2-high"' in plugin - assert 'name="image_generate_local"' in plugin - assert 'name="image_generate_hosted"' in plugin - assert '"name": "image_edit_latest"' in plugin - assert '"name": "image_edit_latest_local"' in plugin - assert '"name": "image_edit_latest_hosted"' in plugin - assert "def _latest_generated_image" in plugin - assert "newest MEDIA: path from the conversation" in plugin - assert 'candidate.upper().startswith("MEDIA:")' in plugin - assert "override=True" not in plugin - - agent = _documents(HERMES / "agent-deployment.yaml")[0] - containers = agent["spec"]["template"]["spec"]["containers"] - broker = next(item for item in containers if item["name"] == "image-broker") - assert broker["ports"] == [ - {"name": "image-broker", "containerPort": 9002, "protocol": "TCP"} - ] - assert broker["securityContext"]["readOnlyRootFilesystem"] is True - assert broker["securityContext"]["runAsNonRoot"] is True - - services = _documents(HERMES / "service.yaml") - service = next( - item for item in services if item["metadata"]["name"] == "hermes-image-broker" - ) - assert service["spec"]["selector"] == {"app": "hermes-agent"} - - oauth_store = _documents(HERMES / "oauth-session-store.yaml") - redis = next(item for item in oauth_store if item["kind"] == "Deployment") - assert redis["spec"]["strategy"]["type"] == "Recreate" - assert "--appendonly" in redis["spec"]["template"]["spec"]["containers"][0]["args"] - - policies = _documents(HERMES / "networkpolicy.yaml") - agent_policy = next( - item for item in policies if item["metadata"]["name"] == "hermes-agent-isolation" - ) - broker_ingress = next( - rule - for rule in agent_policy["spec"]["ingress"] - if {port["port"] for port in rule["ports"]} == {9002, 9003} - ) - assert broker_ingress["from"][0]["podSelector"]["matchLabels"] == { - "app": "hermes-chat-tenant" - } - vault_policy = (VAULT / "scripts" / "vault_k8s_auth_configure.sh").read_text() - assert ( - '"hermes/agent-oidc hermes/agent-tokens hermes/chat-telegram ' - 'hermes/developer-keycloak hermes/developer-gitea ' - 'hermes/developer-harbor hermes/developer-jenkins ' - 'hermes/developer-ssh"' - in vault_policy - ) - assert ( - 'write_policy_and_role "hermes-node-ssh" "hermes" ' - '"hermes-node-ssh-access"' in vault_policy - ) - - -def test_compact_image_edit_resolves_latest_tenant_artifact(tmp_path, monkeypatch): - """Follow-up edits resolve the source server-side and keep tool JSON small.""" - provider_module = SimpleNamespace( - DEFAULT_ASPECT_RATIO="square", - ImageGenProvider=object, - error_response=lambda **value: value, - normalize_reference_images=lambda value: value, - resolve_aspect_ratio=lambda value: value, - save_b64_image=lambda *_args, **_kwargs: tmp_path / "saved.png", - success_response=lambda **value: value, - ) - monkeypatch.setitem(sys.modules, "agent", SimpleNamespace()) - monkeypatch.setitem(sys.modules, "agent.image_gen_provider", provider_module) - spec = importlib.util.spec_from_file_location( - "hermes_image_plugin", - HERMES / "plugins" / "image-gen-broker" / "__init__.py", - ) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - - image_dir = tmp_path / "cache" / "images" - image_dir.mkdir(parents=True) - older = image_dir / "atlas_flux-old.png" - newest = image_dir / "atlas_gpt-image-new.png" - older.write_bytes(b"older") - newest.write_bytes(b"newest") - older.touch() - time.sleep(0.001) - newest.touch() - monkeypatch.setenv("HERMES_HOME", str(tmp_path)) - - calls = [] - monkeypatch.setattr( - module, - "_handle_image_generate", - lambda args, route: calls.append((args, route)) or "ok", - ) - assert module._handle_hosted_edit({"prompt": "make it a clown"}) == "ok" - assert calls == [ - ( - { - "prompt": "make it a clown", - "image_url": str(newest.resolve()), - }, - "hosted", - ) - ] - - -def test_chat_reasoning_uses_switchyard_without_owner_credentials(): - """Family pods use AUTO/manual routes without mounting owner credentials.""" - configmap = _documents(HERMES / "chat-configmap.yaml")[0] - config = yaml.safe_load(configmap["data"]["config.yaml"]) - assert config["model"] == { - "provider": "atlas-switchyard", - "default": "atlas/auto/fast", - "model": "atlas/auto/fast", - } - assert config["providers"]["atlas-switchyard"] == { - "name": "Automatic Router", - "api": "http://hermes-switchyard.hermes.svc.cluster.local:9005/v1", - "api_key": "atlas-switchyard", - "default_model": "atlas/auto/fast", - "transport": "chat_completions", - } - assert config["platforms"]["api_server"]["extra"]["model_routes"] == { - route: {"provider": "atlas-switchyard", "model": route} - for route in [ - "atlas/auto/fast", - "atlas/auto/balanced", - "atlas/auto/deep", - "atlas/auto/maximum", - "atlas/manual/codex/luna", - "atlas/manual/codex/terra", - "atlas/manual/codex/sol", - "atlas/manual/claude/haiku", - "atlas/manual/claude/fable", - "atlas/manual/claude/sonnet", - "atlas/manual/claude/opus", - "atlas/manual/local/qwen-14b", - ] - } - - agent = _documents(HERMES / "agent-deployment.yaml")[0] - containers = agent["spec"]["template"]["spec"]["containers"] - broker = next(item for item in containers if item["name"] == "codex-broker") - assert broker["ports"] == [ - {"name": "codex-broker", "containerPort": 9003, "protocol": "TCP"} - ] - assert broker["securityContext"]["readOnlyRootFilesystem"] is True - assert broker["securityContext"]["runAsNonRoot"] is True - assert {item["name"]: item["value"] for item in broker["env"]}.items() >= { - "PYTHONPATH": "/opt/hermes", - "HERMES_CODEX_BROKER_LISTEN_PORT": "9003", - "HERMES_ROUTING_CATALOG_PATH": "/routing-catalog/catalog.json", - }.items() - - services = _documents(HERMES / "service.yaml") - service = next( - item for item in services if item["metadata"]["name"] == "hermes-codex-broker" - ) - assert service["spec"]["selector"] == {"app": "hermes-agent"} - assert service["spec"]["ports"] == [ - { - "name": "http", - "port": 9003, - "targetPort": "codex-broker", - "protocol": "TCP", - } - ] - - statefulset = _documents(HERMES / "chat-statefulset.yaml")[0] - assert statefulset["spec"]["template"]["metadata"]["annotations"][ - "ai.bstein.dev/config-rev" - ] == "20260816-telegram-topics" - pod_spec = statefulset["spec"]["template"]["spec"] - patch_init = next( - item for item in pod_spec["initContainers"] - if item["name"] == "patch-stream-recovery" - ) - assert patch_init["command"][-2:] == [ - "/opt/hermes/agent/conversation_loop.py", - "/patched/conversation_loop.py", - ] - hermes = next( - item - for item in pod_spec["containers"] - if item["name"] == "hermes" - ) - assert { - "name": "stream-recovery-patch", - "mountPath": "/opt/hermes/agent/conversation_loop.py", - "subPath": "conversation_loop.py", - } in hermes["volumeMounts"] - api_session_init = next( - item for item in pod_spec["initContainers"] - if item["name"] == "patch-api-server-sessions" - ) - assert "patch_api_server_sessions.py" in api_session_init["args"][0] - assert "migrate_telegram_api_sessions.py" in api_session_init["args"][0] - assert { - "name": "api-server-patch", - "mountPath": "/opt/hermes/gateway/platforms/api_server.py", - "subPath": "api_server.py", - } in hermes["volumeMounts"] - assert any( - volume["name"] == "api-server-patch" for volume in pod_spec["volumes"] - ) - assert not any( - mount["mountPath"].endswith("/.codex") - for mount in hermes["volumeMounts"] - ) - - policies = _documents(HERMES / "networkpolicy.yaml") - agent_policy = next( - item for item in policies if item["metadata"]["name"] == "hermes-agent-isolation" - ) - broker_ingress = next( - rule - for rule in agent_policy["spec"]["ingress"] - if {port["port"] for port in rule["ports"]} == {9002, 9003} - ) - assert broker_ingress["from"][0]["podSelector"]["matchLabels"] == { - "app": "hermes-chat-tenant" - } - - -def test_codex_broker_auth_and_request_contract(tmp_path: Path, monkeypatch): - """The relay is bounded, stateless, and rejects unapproved models.""" - module = _load_broker_module( - "hermes_codex_broker", "codex_broker.py", monkeypatch - ) - monkeypatch.setattr(module, "TOKEN", "relay-secret") - - assert module._authorized("Bearer relay-secret") is True - assert module._authorized("Bearer wrong") is False - assert module._real_model("route/codex/gpt-5.6-sol/xhigh") == "gpt-5.6-sol" - payload = module._validate_payload( - { - "model": "gpt-5.6-terra", - "input": "route this chat turn", - "store": True, - "stream": False, - "max_output_tokens": 96, - "max_completion_tokens": 96, - "max_tokens": 96, - "temperature": 0.7, - "top_p": 0.9, - } - ) - assert payload["store"] is False - assert payload["stream"] is True - assert "max_output_tokens" not in payload - assert "max_completion_tokens" not in payload - assert "max_tokens" not in payload - assert "temperature" not in payload - assert "top_p" not in payload - assert payload["input"] == [ - { - "type": "message", - "role": "user", - "content": [{"type": "input_text", "text": "route this chat turn"}], - } - ] - response_item = { - "type": "message", - "role": "user", - "content": [{"type": "input_text", "text": "keep this item"}], - } - assert module._validate_payload( - {"model": "gpt-5.6-terra", "input": response_item} - )["input"] == [response_item] - response_items = [response_item] - assert module._validate_payload( - {"model": "gpt-5.6-terra", "input": response_items} - )["input"] is response_items - image_items = [ - { - "type": "message", - "role": "user", - "content": [ - {"type": "input_text", "text": "What color is this?"}, - { - "type": "image_url", - "image_url": { - "url": "data:image/png;base64,cHJpdmF0ZQ==", - "detail": "high", - }, - }, - ], - } - ] - assert module._validate_payload( - {"model": "gpt-5.6-terra", "input": image_items} - )["input"][0]["content"][1] == { - "type": "input_image", - "image_url": "data:image/png;base64,cHJpdmF0ZQ==", - "detail": "high", - } - switchyard_image_items = [ - { - "role": "user", - "content": [ - {"type": "input_text", "text": "What color is this?"}, - { - "type": "image_url", - "image_url": { - "url": "data:image/png;base64,cHJpdmF0ZQ==", - "detail": "auto", - }, - }, - ], - } - ] - assert module._validate_payload( - {"model": "gpt-5.6-terra", "input": switchyard_image_items} - )["input"][0]["content"][1] == { - "type": "input_image", - "image_url": "data:image/png;base64,cHJpdmF0ZQ==", - "detail": "auto", - } - switchyard_base64_items = [ - { - "role": "user", - "content": [ - {"type": "input_text", "text": "What color is this?"}, - { - "type": "image", - "source": { - "type": "base64", - "media_type": "image/png", - "data": "cHJpdmF0ZQ==", - }, - }, - ], - } - ] - normalized_base64 = module._validate_payload( - {"model": "gpt-5.6-terra", "input": switchyard_base64_items} - )["input"][0]["content"][1] - assert normalized_base64 == { - "type": "input_image", - "image_url": "data:image/png;base64,cHJpdmF0ZQ==", - } - switchyard_enum_items = [ - { - "role": "user", - "content": [ - { - "type": "input_image", - "image_url": { - "type": "url", - "data": { - "url": "data:image/png;base64,cHJpdmF0ZQ==", - "detail": "high", - }, - }, - } - ], - } - ] - nested_image = module._validate_payload( - {"model": "gpt-5.6-terra", "input": switchyard_enum_items} - )["input"][0]["content"][0] - assert nested_image["image_url"] == "data:image/png;base64,cHJpdmF0ZQ==" - assert nested_image["detail"] == "high" - with pytest.raises(ValueError, match=r"non-empty Responses image URL.*str\[4\]"): - module._validate_payload( - { - "model": "gpt-5.6-terra", - "input": [ - { - "type": "message", - "role": "user", - "content": [ - { - "type": "input_image", - "image_url": {"detail": "high"}, - } - ], - } - ], - } - ) - routed = module._validate_payload( - { - "model": "route/codex/gpt-5.6-luna/low", - "input": "use the low route", - "stream": False, - } - ) - assert routed["model"] == "gpt-5.6-luna" - with pytest.raises(ValueError, match="unsupported Codex model"): - module._validate_payload({"model": "unapproved-model", "input": "hello"}) - with pytest.raises(ValueError, match="non-empty Responses input"): - module._validate_payload({"model": "gpt-5.6-terra", "input": ""}) - with pytest.raises(ValueError, match="non-empty Responses input list"): - module._validate_payload({"model": "gpt-5.6-terra", "input": []}) - - completed = { - "id": "resp_test", - "object": "response", - "status": "completed", - "output": [], - } - completed_item = { - "type": "message", - "role": "assistant", - "status": "completed", - "content": [{"type": "output_text", "text": "done"}], - } - assert module._completed_response( - [ - "event: response.created", - 'data: {"type":"response.created","response":{}}', - "event: response.output_item.done", - "data: " - + json.dumps( - { - "type": "response.output_item.done", - "output_index": 0, - "item": completed_item, - } - ), - "event: response.completed", - "data: " - + json.dumps({"type": "response.completed", "response": completed}), - "data: [DONE]", - ] - )["output"] == [completed_item] - raw_stream = ( - "event: response.output_item.done\n" - "data: " - + json.dumps( - { - "type": "response.output_item.done", - "output_index": 0, - "item": completed_item, - } - ) - + "\n\nevent: response.completed\ndata: " - + json.dumps({"type": "response.completed", "response": completed}) - + "\n\n" - ).encode() - normalized = module._normalized_stream( - raw_stream, {**completed, "output": [completed_item]} - ).decode() - terminal_data = next( - line for line in normalized.splitlines() if '"response.completed"' in line - ) - assert json.loads(terminal_data.removeprefix("data: "))["response"][ - "output" - ] == [completed_item] - assert normalized.endswith("\n\n") - streamed_function_item = { - "type": "function_call", - "name": "read_file", - "status": "completed", - "arguments": '{"path":"/tmp"}', - } - streamed_function_body = ( - "event: response.function_call_arguments.delta\n" - 'data: {"type":"response.function_call_arguments.delta",' - '"item_id":"call_1","delta":"{\\"path\\":\\"/tmp\\"}"}\n\n' - "event: response.function_call_arguments.done\n" - 'data: {"type":"response.function_call_arguments.done",' - '"item_id":"call_1","arguments":"{\\"path\\":\\"/tmp\\"}"}\n\n' - "event: response.output_item.done\n" - 'data: {"type":"response.output_item.done","output_index":0,' - '"item":{"type":"function_call","name":"read_file",' - '"arguments":"{\\"path\\":\\"/tmp\\"}"}}\n\n' - "event: response.completed\n" - "data: " - + json.dumps( - { - "type": "response.completed", - "response": {**completed, "output": [streamed_function_item]}, - } - ) - + "\n\n" - ).encode() - normalized_function_stream = module._normalized_stream( - streamed_function_body, {**completed, "output": [streamed_function_item]} - ).decode() - assert "response.function_call_arguments.delta" in normalized_function_stream - assert "response.function_call_arguments.done" not in normalized_function_stream - assert "response.output_item.done" not in normalized_function_stream - normalized_terminal = next( - line - for line in normalized_function_stream.splitlines() - if '"response.completed"' in line - ) - assert json.loads(normalized_terminal.removeprefix("data: "))["response"][ - "output" - ] == [] - with pytest.raises(RuntimeError, match="retryable incomplete response"): - module._completed_response( - [ - "event: response.incomplete", - 'data: {"type":"response.incomplete","response":' - '{"status":"incomplete","incomplete_details":' - '{"reason":"max_output_tokens"}}}', - ] - ) - with pytest.raises(RuntimeError, match="provider unavailable"): - module._completed_response( - [ - "event: error", - 'data: {"type":"error","error":{"message":"provider unavailable"}}', - ] - ) - malformed_tool_item = { - "type": "function_call", - "name": "search_files", - "status": "completed", - "arguments": '{"path":"","offset":', - } - with pytest.raises(RuntimeError, match="malformed function arguments"): - module._completed_response( - [ - "event: response.output_item.done", - "data: " - + json.dumps( - { - "type": "response.output_item.done", - "output_index": 0, - "item": malformed_tool_item, - } - ), - "event: response.completed", - "data: " - + json.dumps({"type": "response.completed", "response": completed}), - ] - ) - valid_tool_item = { - **malformed_tool_item, - "arguments": '{"path":"","offset":0}', - } - assert module._completed_response( - [ - "event: response.output_item.done", - "data: " - + json.dumps( - { - "type": "response.output_item.done", - "output_index": 0, - "item": valid_tool_item, - } - ), - "event: response.completed", - "data: " - + json.dumps({"type": "response.completed", "response": completed}), - ] - )["output"] == [valid_tool_item] - with pytest.raises(RuntimeError, match="malformed function arguments"): - module._completed_response( - [ - "event: response.function_call_arguments.delta", - 'data: {"type":"response.function_call_arguments.delta",' - '"item_id":"call_1","output_index":0,' - '"delta":"{\\"path\\":\\"/tmp\\",\\"offset\\":"}', - "event: response.completed", - "data: " - + json.dumps({"type": "response.completed", "response": completed}), - ] - ) - streamed_tool = module._completed_response( - [ - "event: response.function_call_arguments.delta", - 'data: {"type":"response.function_call_arguments.delta",' - '"item_id":"call_2","output_index":0,' - '"delta":"{\\"path\\":\\"/tmp\\",\\"offset\\":"}', - "event: response.function_call_arguments.done", - 'data: {"type":"response.function_call_arguments.done",' - '"item_id":"call_2","output_index":0,' - '"arguments":"{\\"path\\":\\"/tmp\\",\\"offset\\":0}"}', - "event: response.completed", - "data: " - + json.dumps({"type": "response.completed", "response": completed}), - ] - ) - assert streamed_tool["status"] == "completed" - - auth_dir = tmp_path / ".codex" - auth_dir.mkdir() - # The token payload need only prove the broker reads CODEX_HOME directly. - encoded = base64.urlsafe_b64encode( - json.dumps({"exp": time.time() + 3600}).encode() - ).decode().rstrip("=") - (auth_dir / "auth.json").write_text( - json.dumps({"tokens": {"access_token": f"header.{encoded}.signature"}}) - ) - monkeypatch.setenv("CODEX_HOME", str(auth_dir)) - assert module._access_token().startswith("header.") - - -def test_codex_broker_refreshes_and_persists_first_party_oauth( - tmp_path: Path, monkeypatch -): - """Expired ChatGPT OAuth refreshes in the canonical Codex CLI store.""" - module = _load_broker_module( - "hermes_codex_refresh_broker", "codex_broker.py", monkeypatch - ) - auth_dir = tmp_path / ".codex" - auth_dir.mkdir() - - def jwt(expires_at: float) -> str: - payload = base64.urlsafe_b64encode( - json.dumps({"exp": expires_at}).encode() - ).decode().rstrip("=") - return f"header.{payload}.signature" - - expired = jwt(time.time() - 60) - live = jwt(time.time() + 3600) - auth_path = auth_dir / "auth.json" - auth_path.write_text( - json.dumps( - { - "auth_mode": "chatgpt", - "tokens": { - "access_token": expired, - "refresh_token": "refresh-old", - }, - } - ) - ) - calls = [] - auth_module = ModuleType("hermes_cli.auth") - - def refresh(access_token, refresh_token, *, timeout_seconds): - calls.append((access_token, refresh_token, timeout_seconds)) - return { - "access_token": live, - "refresh_token": "refresh-new", - "last_refresh": "2026-08-12T20:00:00Z", - } - - auth_module.refresh_codex_oauth_pure = refresh - package = ModuleType("hermes_cli") - package.auth = auth_module - monkeypatch.setitem(sys.modules, "hermes_cli", package) - monkeypatch.setitem(sys.modules, "hermes_cli.auth", auth_module) - monkeypatch.setenv("CODEX_HOME", str(auth_dir)) - - assert module._access_token() == live - persisted = json.loads(auth_path.read_text()) - assert persisted["tokens"]["access_token"] == live - assert persisted["tokens"]["refresh_token"] == "refresh-new" - assert persisted["last_refresh"] == "2026-08-12T20:00:00Z" - assert calls == [(expired, "refresh-old", 30.0)] - assert auth_path.stat().st_mode & 0o777 == 0o600 - - # A healthy token is reused, so repeated routed turns do not spend a - # refresh token or create a second billing/authentication path. - assert module._access_token() == live - assert len(calls) == 1 - - -def test_claude_broker_uses_native_subscription_without_api_billing(monkeypatch): - """Claude traffic must use the native first-party CLI subscription lane.""" - module = _load_broker_module( - "hermes_claude_broker", "claude_oauth_broker.py", monkeypatch - ) - monkeypatch.setenv("ANTHROPIC_API_KEY", "must-not-leak") - monkeypatch.setenv("CLAUDE_API_KEY", "must-not-leak") - monkeypatch.setattr( - module, - "resolve_route", - lambda route: "claude-fable-5" if "/fable/" in route else route, - ) - - model, effort = module._route( - "route/claude/fable/xhigh", {"output_config": {"effort": "xhigh"}} - ) - - assert (model, effort) == ("claude-fable-5", "xhigh") - assert "ANTHROPIC_API_KEY" not in module._claude_environment() - assert "CLAUDE_API_KEY" not in module._claude_environment() - assert module.CAPACITY_PATTERN.search("weekly usage limit exhausted") - - -def test_codex_native_health_overrides_historical_router_errors( - tmp_path: Path, monkeypatch -): - """Fresh first-party health is authoritative over old Switchyard probes.""" - plugin_path = HERMES / "plugins" / "auto-router" / "provider_status.py" - spec = importlib.util.spec_from_file_location("hermes_provider_status", plugin_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - health_path = tmp_path / "codex.json" - health_path.write_text( - json.dumps( - { - "state": "available", - "authenticated": True, - "transport": "codex-chatgpt-subscription", - } - ) - ) - monkeypatch.setattr(module, "CODEX_HEALTH_PATH", health_path) - monkeypatch.setattr(module, "CLAUDE_HEALTH_PATH", tmp_path / "missing.json") - monkeypatch.setattr( - module, - "_get_json", - lambda url: {"status": "ok"} - if url.endswith("/health") - else { - "models": { - "route/codex/terra/medium": { - "calls": 1, - "errors": 99, - "total_tokens": 12, - } - } - }, - ) - monkeypatch.setattr(module, "_codex_account", lambda: {}) - monkeypatch.setattr(module, "_claude_account", lambda: {}) - - codex = module.provider_status_payload()["providers"]["codex"] - - assert codex["errors"] == 99 - assert codex["state"] == "available" - assert codex["native_health"]["transport"] == "codex-chatgpt-subscription" - - -def test_api_session_patch_accepts_parent_lineage(tmp_path: Path): - """API-created workers must persist the originating Hermes session.""" - module_path = HERMES / "scripts" / "patch_api_server_sessions.py" - spec = importlib.util.spec_from_file_location("patch_api_sessions", module_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - source = tmp_path / "api_server.py" - destination = tmp_path / "patched.py" - source.write_text( - "prefix\n" - + module.BEFORE - + "middle\n" - + module.RUNS_BEFORE - + "run body\n" - + module.RUN_CLOSE_BEFORE - + module.RESPONSES_SESSION_BEFORE - + module.EVENT_CALLBACK_SIGNATURE_BEFORE - + "callback docstring and push helper\n" - + module.EVENT_CALLBACK_BODY_BEFORE - + "tool start body\n" - + module.EVENT_CALLBACK_END_BEFORE - + module.EVENT_CALLBACK_CALL_BEFORE - + module.RUN_SWEEP_BEFORE - + "suffix\n", - encoding="utf-8", - ) - - module.patch(source, destination) - patched = destination.read_text(encoding="utf-8") - - assert "X-Hermes-Parent-Session-Id" in patched - assert "parent_session_id=parent_session_id" in patched - assert "Parent session not found" in patched - assert "HERMES_API_DEFAULT_PARENT_MATCH_PREFIXES" in patched - assert "user_message.startswith(default_prefixes)" in patched - assert "session_parent_conflict" in patched - assert "X-Hermes-Conversation-Platform" in patched - assert "X-Hermes-Conversation-Title" in patched - assert 'conversation_platform != "telegram"' in patched - assert "db.record_gateway_session_peer(" in patched - assert 'display_name="Telegram"' in patched - assert "db.reopen_session(session_id)" in patched - assert 'db.end_session(session_id, f"api_run_{terminal_status}")' in patched - assert "def _record_run_activity(" in patched - assert '"_thinking": "Hermes is reasoning"' in patched - assert '"run.started": "Worker started"' in patched - assert '"run.completed": "Worker completed"' in patched - assert '"reasoning.available": "Hermes finished a reasoning step"' in patched - assert '"subagent.progress": "Nested worker progress"' in patched - assert "redact_sensitive_text" in patched - assert 'getattr(os, "O_NOFOLLOW", 0)' in patched - assert "os.fchmod(fd, 0o600)" in patched - assert "session_id=session_id" in patched - assert 'self._record_run_activity(session_id, "run.started")' in patched - assert 'detail = tool_name if event_type in {' in patched - assert 'if event_type == "subagent.tool"' in patched - assert "_RUN_ACTIVITY_HEARTBEAT_SECONDS = 15.0" in patched - assert 'heartbeats.get(session_id, 0.0)' in patched - assert '"subagent.thinking",' in patched - assert "Stream retention and run lifetime are separate" in patched - assert 'terminal_status in {"completed", "failed", "cancelled"}' in patched - assert patched.index("terminal_status = self._run_statuses") < patched.index( - "self._active_run_tasks.pop(run_id, None)" - ) - - -def test_telegram_api_session_migration_is_bounded_and_idempotent(tmp_path: Path): - """Only named Telegram conversations receive presentation metadata.""" - module_path = HERMES / "scripts" / "migrate_telegram_api_sessions.py" - spec = importlib.util.spec_from_file_location("migrate_telegram_sessions", module_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - state = tmp_path / "state.db" - responses = tmp_path / "response_store.db" - with sqlite3.connect(state) as connection: - connection.execute( - """CREATE TABLE sessions ( - id TEXT PRIMARY KEY, source TEXT, session_key TEXT, chat_type TEXT, - display_name TEXT, origin_json TEXT, title TEXT - )""" - ) - connection.executemany( - "INSERT INTO sessions (id, source) VALUES (?, ?)", - (("telegram-session", "api_server"), ("other-session", "api_server")), - ) - with sqlite3.connect(responses) as connection: - connection.execute( - "CREATE TABLE conversations (name TEXT PRIMARY KEY, response_id TEXT NOT NULL)" - ) - connection.execute( - "CREATE TABLE responses (response_id TEXT PRIMARY KEY, data TEXT NOT NULL, accessed_at REAL NOT NULL)" - ) - connection.executemany( - "INSERT INTO responses VALUES (?, ?, 0)", - ( - ("telegram-response", json.dumps({"session_id": "telegram-session"})), - ("other-response", json.dumps({"session_id": "other-session"})), - ), - ) - connection.executemany( - "INSERT INTO conversations VALUES (?, ?)", - (("telegram", "telegram-response"), ("unrelated", "other-response")), - ) - - assert module.migrate(state, responses) == 1 - assert module.migrate(state, responses) == 0 - with sqlite3.connect(state) as connection: - telegram = connection.execute( - "SELECT session_key, chat_type, display_name, origin_json, title " - "FROM sessions WHERE id = 'telegram-session'" - ).fetchone() - other = connection.execute( - "SELECT session_key, title FROM sessions WHERE id = 'other-session'" - ).fetchone() - assert telegram[:3] == ("telegram", "private", "Telegram") - assert json.loads(telegram[3]) == { - "platform": "telegram", - "session_key": "telegram", - } - assert telegram[4] == "Telegram · General" - assert other == (None, None) - - -def test_web_session_activity_patch_projects_bounded_events(tmp_path: Path): - """The DOM transcript includes events without polluting agent history.""" - module_path = HERMES / "scripts" / "patch_web_session_activity.py" - spec = importlib.util.spec_from_file_location("patch_web_activity", module_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - source = tmp_path / "web_server.py" - destination = tmp_path / "patched.py" - source.write_text( - "prefix\n" - + module.LATEST_ROWS_BEFORE - + module.LATEST_SELECTION_BEFORE - + module.HELPER_MARKER - + " db = object()\n" - + module.MESSAGES_BEFORE - + "suffix\n", - encoding="utf-8", - ) - - module.patch(source, destination) - patched = destination.read_text(encoding="utf-8") - - assert "def _run_activity_messages(" in patched - assert 'root = get_hermes_home() / "run-activity"' in patched - assert "path.stat().st_size > 600_000" in patched - assert "entries[-1_000:]" in patched - assert "*_run_activity_messages(sid)" in patched - assert "messages.sort(" in patched - assert "limit: Optional[int] = None" in patched - assert "total_messages = len(messages)" in patched - assert "min(int(limit), 10_000)" in patched - assert '"total_messages": total_messages' in patched - assert "SELECT id, parent_session_id, started_at, ended_at" in patched - assert "newest still-open member of the lineage" in patched - assert "immediate objective parent" in patched - assert "mixes unrelated workstreams" in patched - assert "orphaned children" in patched - assert 'item[0].get("ended_at") is None' in patched - - -def test_web_session_lineage_returns_to_resumed_parent(): - """An ended reviewer must not strand the live view away from its parent.""" - module_path = HERMES / "scripts" / "patch_web_session_activity.py" - spec = importlib.util.spec_from_file_location("patch_web_lineage", module_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - - namespace: dict[str, object] = {} - exec( - "def select_active(sid, db, rows):\n" + module.LATEST_SELECTION_AFTER, - namespace, - ) - - class FakeDB: - def get_session(self, session_id): - return next((row for row in rows if row["id"] == session_id), None) - - rows = [ - { - "id": "umbrella", - "parent_session_id": None, - "started_at": 0.0, - "ended_at": None, - }, - { - "id": "root", - "parent_session_id": "umbrella", - "started_at": 1.0, - "ended_at": None, - }, - { - "id": "review-1", - "parent_session_id": "root", - "started_at": 2.0, - "ended_at": 3.0, - }, - { - "id": "review-2", - "parent_session_id": "root", - "started_at": 4.0, - "ended_at": None, - }, - ] - select_active = namespace["select_active"] - - assert select_active("root", FakeDB(), rows) == ( - "review-2", - ["root", "review-2"], - ) - assert select_active("review-1", FakeDB(), rows) == ( - "review-2", - ["root", "review-2"], - ) - rows[2]["ended_at"] = 5.0 - rows[3]["ended_at"] = 5.0 - assert select_active("root", FakeDB(), rows) == ("root", ["root"]) - - rows[1]["ended_at"] = 9.0 - rows.append( - { - "id": "orphaned-review", - "parent_session_id": "root", - "started_at": 6.0, - "ended_at": None, - } - ) - rows.append( - { - "id": "accepted-review", - "parent_session_id": "root", - "started_at": 7.0, - "ended_at": 8.0, - } - ) - assert select_active("review-1", FakeDB(), rows) == ( - "accepted-review", - ["root", "accepted-review"], - ) - - -def test_api_activity_patch_coalesces_streaming_heartbeats( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -): - """Token callbacks stay bounded while every tool transition is retained.""" - module_path = HERMES / "scripts" / "patch_api_server_sessions.py" - spec = importlib.util.spec_from_file_location("patch_api_activity", module_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - activity_body = module.EVENT_CALLBACK_SIGNATURE_AFTER.split( - " def _make_run_event_callback(", 1 - )[0] - namespace: dict[str, object] = {} - exec( - "import hashlib, json, logging, os, time\n" - "from pathlib import Path\n" - "logger = logging.getLogger(__name__)\n" - "def redact_sensitive_text(value): return value\n" - "class ActivityRecorder:\n" - + activity_body, - namespace, - ) - recorder = namespace["ActivityRecorder"]() - monkeypatch.setenv("HERMES_HOME", str(tmp_path)) - - recorder._record_run_activity("session", "_thinking") - recorder._record_run_activity("session", "reasoning.available") - recorder._record_run_activity("session", "subagent.thinking") - recorder._record_run_activity("session", "tool.started", tool_name="terminal") - recorder._record_run_activity("session", "tool.completed", tool_name="terminal") - - journals = list((tmp_path / "run-activity").glob("*.jsonl")) - assert len(journals) == 1 - entries = [json.loads(line) for line in journals[0].read_text().splitlines()] - assert [entry["activity_event"] for entry in entries] == [ - "_thinking", - "tool.started", - "tool.completed", - ] - - -def test_legacy_api_sessions_are_nested_idempotently(tmp_path: Path): - """Known standalone API workers move under Cassandra without data loss.""" - module_path = HERMES / "scripts" / "migrate_api_session_lineage.py" - spec = importlib.util.spec_from_file_location("migrate_api_sessions", module_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - database = tmp_path / "state.db" - with sqlite3.connect(database) as connection: - connection.execute( - "CREATE TABLE sessions (id TEXT PRIMARY KEY, source TEXT, " - "parent_session_id TEXT, title TEXT, transcript TEXT, archived INTEGER DEFAULT 0)" - ) - connection.execute( - "INSERT INTO sessions (id, source, parent_session_id, title, transcript) " - "VALUES (?, 'tui', NULL, 'Cassandra', 'parent-data')", - (module.LEGACY_CASSANDRA_PARENT,), - ) - worker_id = next(iter(module.LEGACY_CASSANDRA_WORKERS)) - connection.execute( - "INSERT INTO sessions (id, source, parent_session_id, title, transcript) " - "VALUES (?, 'api_server', NULL, 'old', 'keep-me')", - (worker_id,), - ) - connection.executemany( - "INSERT INTO sessions (id, source, parent_session_id, title, transcript) " - "VALUES (?, 'api_server', NULL, 'old smoke', 'keep-smoke')", - ((orphan_id,) for orphan_id in module.LEGACY_ORPHANED_SMOKE_SESSIONS), - ) - - assert module.migrate(database) == 1 + len(module.LEGACY_ORPHANED_SMOKE_SESSIONS) - assert module.migrate(database) == 0 - with sqlite3.connect(database) as connection: - row = connection.execute( - "SELECT parent_session_id, title, transcript FROM sessions WHERE id = ?", - (worker_id,), - ).fetchone() - assert row == ( - module.LEGACY_CASSANDRA_PARENT, - module.LEGACY_CASSANDRA_WORKERS[worker_id], - "keep-me", - ) - with sqlite3.connect(database) as connection: - orphans = connection.execute( - "SELECT id, archived, title, transcript FROM sessions " - "WHERE id IN ({}) ORDER BY id".format( - ",".join("?" for _ in module.LEGACY_ORPHANED_SMOKE_SESSIONS) - ), - tuple(module.LEGACY_ORPHANED_SMOKE_SESSIONS), - ).fetchall() - assert orphans == sorted( - ( - orphan_id, - 1, - module.LEGACY_ORPHANED_SMOKE_SESSIONS[orphan_id], - "keep-smoke", - ) - for orphan_id in module.LEGACY_ORPHANED_SMOKE_SESSIONS - ) - - -def test_automated_triage_sessions_are_grouped_without_touching_interactive_runs(tmp_path: Path): - """Only the stable Ariadne contract moves below the triage parent.""" - module_path = HERMES / "scripts" / "migrate_api_session_lineage.py" - spec = importlib.util.spec_from_file_location("migrate_triage_sessions", module_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - database = tmp_path / "state.db" - with sqlite3.connect(database) as connection: - connection.execute( - "CREATE TABLE sessions (id TEXT PRIMARY KEY, source TEXT, " - "parent_session_id TEXT, title TEXT, started_at REAL, archived INTEGER DEFAULT 0)" - ) - connection.execute( - "CREATE TABLE messages (session_id TEXT, role TEXT, content TEXT)" - ) - connection.executemany( - "INSERT INTO sessions (id, source, started_at) VALUES (?, 'api_server', ?)", - (("triage-run", 1.0), ("jenkins-run", 1.5), ("interactive-run", 2.0)), - ) - connection.executemany( - "INSERT INTO messages (session_id, role, content) VALUES (?, 'user', ?)", - ( - ( - "triage-run", - module.TRIAGE_MESSAGE_PREFIXES[0] - + " Fix for incident sonar/bstein_home/python:S2208/finding-key.", - ), - ( - "jenkins-run", - module.TRIAGE_MESSAGE_PREFIXES[1] - + "\nAnalyze incident soteria/291 for the Jenkins job soteria.", - ), - ("interactive-run", "Please explain this alert to me."), - ), - ) - - assert module.migrate(database, group_triage=True) == 2 - assert module.migrate(database, group_triage=True) == 0 - with sqlite3.connect(database) as connection: - parent = connection.execute( - "SELECT title FROM sessions WHERE id = ?", (module.TRIAGE_PARENT,) - ).fetchone() - triage = connection.execute( - "SELECT parent_session_id, title FROM sessions WHERE id = 'triage-run'" - ).fetchone() - interactive = connection.execute( - "SELECT parent_session_id FROM sessions WHERE id = 'interactive-run'" - ).fetchone() - jenkins = connection.execute( - "SELECT parent_session_id, title FROM sessions WHERE id = 'jenkins-run'" - ).fetchone() - assert parent == (module.TRIAGE_PARENT_TITLE,) - assert triage == ( - module.TRIAGE_PARENT, - "Sonar · bstein_home · python:S2208 · iage-run", - ) - assert interactive == (None,) - assert jenkins == ( - module.TRIAGE_PARENT, - "Sonar · soteria/291 · kins-run", - ) - - -def test_stale_parent_linked_api_workers_close_on_startup(tmp_path: Path): - """A previous gateway lifetime cannot leave phantom active workers.""" - module_path = HERMES / "scripts" / "migrate_api_session_lineage.py" - spec = importlib.util.spec_from_file_location("close_stale_api_workers", module_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - database = tmp_path / "state.db" - with sqlite3.connect(database) as connection: - connection.execute( - "CREATE TABLE sessions (id TEXT PRIMARY KEY, source TEXT, " - "parent_session_id TEXT, title TEXT, started_at REAL, ended_at REAL, " - "end_reason TEXT, archived INTEGER DEFAULT 0)" - ) - connection.executemany( - "INSERT INTO sessions " - "(id, source, parent_session_id, started_at, ended_at, end_reason) " - "VALUES (?, ?, ?, 1, ?, ?)", - ( - ("stale", "api_server", "parent", None, None), - ("root", "api_server", None, None, None), - ("finished", "api_server", "parent", 2.0, "api_run_completed"), - ("interactive", "tui", "parent", None, None), - ), - ) - - assert module.migrate(database) == 1 - assert module.migrate(database) == 0 - with sqlite3.connect(database) as connection: - rows = connection.execute( - "SELECT id, ended_at, end_reason FROM sessions ORDER BY id" - ).fetchall() - by_id = {row[0]: row[1:] for row in rows} - assert by_id["stale"][0] is not None - assert by_id["stale"][1] == "api_run_recovered_stale" - assert by_id["root"] == (None, None) - assert by_id["finished"] == (2.0, "api_run_completed") - assert by_id["interactive"] == (None, None) - - -def test_switchyard_brokers_and_native_claude_lane_use_the_right_images(): - """Thin brokers stay small while native Claude runs beside owner auth.""" - dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-switchyard-brokers").read_text() - assert "httpx==0.28.1" in dockerfile - assert "worker_route_broker.py" in dockerfile - assert "routing_catalog.py" in dockerfile - - deployment = _documents(HERMES / "switchyard-deployment.yaml")[0] - containers = { - container["name"]: container - for container in deployment["spec"]["template"]["spec"]["containers"] - } - expected = ( - "registry.bstein.dev/bstein/hermes-switchyard-brokers@" - "sha256:ee7e95e060ef8083da505162d7e9030daba15fdd828cc047bbcbe6aa409d2083" - ) - assert containers["worker-route-broker"]["image"] == expected - assert containers["classifier-broker"]["image"] == expected - assert "claude-oauth-broker" not in containers - - agent = _documents(HERMES / "agent-deployment.yaml")[0] - agent_containers = { - container["name"]: container - for container in agent["spec"]["template"]["spec"]["containers"] - } - for container_name in ("hermes", "terminal"): - container = agent_containers[container_name] - environment = {item["name"]: item["value"] for item in container["env"]} - mounts = {item["name"]: item for item in container["volumeMounts"]} - assert environment["HERMES_ROUTING_CATALOG_PATH"] == "/routing-catalog/catalog.json" - assert environment["HERMES_CODEX_HEALTH_PATH"] == "/opt/data/provider-health/codex.json" - assert environment["HERMES_CLAUDE_HEALTH_PATH"] == "/opt/data/provider-health/claude.json" - assert mounts["routing-catalog"]["mountPath"] == "/routing-catalog" - assert mounts["routing-catalog"]["readOnly"] is True - codex = agent_containers["codex-broker"] - codex_environment = {item["name"]: item["value"] for item in codex["env"]} - assert codex_environment["HERMES_CODEX_HEALTH_PATH"] == "/opt/data/provider-health/codex.json" - claude = agent_containers["claude-broker"] - assert claude["image"].startswith("registry.bstein.dev/bstein/hermes-agent@") - assert "unset ANTHROPIC_API_KEY CLAUDE_API_KEY" in claude["args"][0] - assert any( - mount["name"] == "home" and mount["mountPath"] == "/opt/data" - for mount in claude["volumeMounts"] - ) - - -def test_classifier_broker_bounds_history_without_losing_routing_intent(monkeypatch): - """AUTO classification must fit the Jetson context without losing intent.""" - broker_path = HERMES / "scripts" / "classifier_broker.py" - monkeypatch.setitem(sys.modules, "httpx", SimpleNamespace()) - spec = importlib.util.spec_from_file_location("hermes_classifier_broker", broker_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - - payload = { - "model": "qwen2.5:14b-instruct-q4_0", - "messages": [ - { - "role": "system", - "content": "routing contract\n" + ("candidate policy " * 1000), - }, - { - "role": "user", - "content": "Build and verify the Cassandra release safely.", - }, - { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call_large", - "type": "function", - "function": { - "name": "large_tool", - "arguments": '{"command":"' + ("x" * 5000) + '"}', - }, - } - ], - }, - { - "role": "tool", - "content": "unbounded test output " * 10000, - "tool_call_id": "call_large", - }, - { - "role": "user", - "content": [ - {"type": "image_url", "image_url": {"url": "data:image/png;base64,private"}}, - {"type": "text", "text": "Turn this cat into a cute clown."}, - ], - }, - ], - "tools": [{"type": "function", "function": {"name": "large_tool"}}], - "tool_choice": "auto", - "parallel_tool_calls": True, - "response_format": {"type": "json_object"}, - } - - compacted = module.compact_payload(payload) - encoded = json.dumps(compacted) - - assert compacted["model"] == payload["model"] - assert compacted["response_format"] == payload["response_format"] - assert "tools" not in compacted - assert "tool_choice" not in compacted - assert "parallel_tool_calls" not in compacted - assert all(message.get("role") != "tool" for message in compacted["messages"]) - assert all("tool_call_id" not in message for message in compacted["messages"]) - assert all("tool_calls" not in message for message in compacted["messages"]) - assert "[tool evidence]" in encoded - assert "[assistant requested an external tool]" in encoded - assert "Build and verify the Cassandra release safely." in encoded - assert "Turn this cat into a cute clown." in encoded - assert "image attachment available to the selected worker" in encoded - assert "data:image/png;base64" not in encoded - assert "unbounded test output " * 100 not in encoded - assert len(encoded) < 18_000 - - -def test_switchyard_classifier_is_bounded_and_fails_open_once(): - """A sick local judge must not hold chat through repeated long retries.""" - config = tomllib.loads( - _documents(HERMES / "switchyard-configmap.yaml")[0]["data"]["routes.toml"] - ) - classifier = config["llm_clients"]["classifier"] - assert classifier["base_url"] == "http://127.0.0.1:9008/v1" - assert classifier["max_retries"] == 0 - for route in ("auto_fast", "auto_balanced"): - assert config["routes"][route]["recent_turn_window"] == 4 - for route in ("auto_deep", "auto_maximum", "worker_auto_maximum"): - assert config["routes"][route]["recent_turn_window"] == 6 - - deployment = _documents(HERMES / "switchyard-deployment.yaml")[0] - containers = { - item["name"]: item - for item in deployment["spec"]["template"]["spec"]["containers"] - } - classifier_container = containers["classifier-broker"] - env = {item["name"]: item["value"] for item in classifier_container["env"]} - assert env["HERMES_CLASSIFIER_BROKER_READ_TIMEOUT"] == "60" - assert classifier_container["readinessProbe"]["httpGet"]["port"] == "classifier" - - -def test_image_broker_returns_bytes_and_removes_owner_cache(tmp_path: Path, monkeypatch): - """The broker must not retain a family user's generated image.""" - broker_path = HERMES / "scripts" / "image_broker.py" - spec = importlib.util.spec_from_file_location("hermes_image_broker", broker_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - generated = tmp_path / "generated.png" - generated.write_bytes(b"\x89PNG\r\n\x1a\nprivate-image") - - class Provider: - def generate(self, prompt, aspect, **kwargs): - assert prompt == "paint a blue sphere" - assert aspect == "square" - return { - "success": True, - "image": str(generated), - "model": "gpt-image-2-high", - "quality": "high", - } - - monkeypatch.setattr(module, "_PROVIDER", Provider()) - result = module._generate( - { - "prompt": "paint a blue sphere", - "aspect_ratio": "square", - "model": "gpt-image-2-high", - } - ) - - assert result["success"] is True - assert result["image_b64"] - assert "image" not in result - assert not generated.exists() - - -def test_image_broker_auto_falls_back_to_local_and_honors_explicit_routes( - monkeypatch, -): - """AUTO is hosted-first while explicit local never calls the hosted lane.""" - broker_path = HERMES / "scripts" / "image_broker.py" - spec = importlib.util.spec_from_file_location("hermes_image_router", broker_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - calls = [] - - def hosted(payload, model, prompt, aspect): - calls.append(("hosted", model, prompt, aspect)) - return {"success": False, "error": "hosted refusal"} - - def local(payload, timeout=1800.0): - calls.append(("local", payload["model"], timeout)) - return { - "success": True, - "image_b64": "aW1hZ2U=", - "model": "flux-2-klein-4b-local", - "route": "local", - } - - monkeypatch.setattr(module, "_generate_hosted", hosted) - monkeypatch.setattr(module, "_local_request", local) - auto = module._generate( - { - "prompt": "colorize this family photograph", - "aspect_ratio": "portrait", - "model": "atlas-image-auto-high", - } - ) - assert auto["success"] is True - assert auto["route"] == "local" - assert auto["hosted_fallback_reason"] == "hosted refusal" - assert [call[0] for call in calls] == ["hosted", "local"] - - calls.clear() - explicit = module._generate( - { - "prompt": "make a local landscape", - "aspect_ratio": "landscape", - "model": "flux-2-klein-4b-local", - } - ) - assert explicit["route"] == "local" - assert [call[0] for call in calls] == ["local"] - - -def test_image_broker_policy_is_narrow_and_operator_extensible(tmp_path: Path, monkeypatch): - """Family-photo restoration stays allowed while the hard boundary remains.""" - broker_path = HERMES / "scripts" / "image_broker.py" - spec = importlib.util.spec_from_file_location("hermes_image_policy", broker_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - policy = tmp_path / "policy.json" - policy.write_text('{"additional_blocked_phrases":["site-specific block"]}') - monkeypatch.setattr(module, "POLICY_PATH", policy) - - assert module._policy_error( - "Colorize my baby photograph with a lighter natural skin tone" - ) is None - assert "minors" in module._policy_error("Create a sexual image of a child") - assert module._policy_error("A site-specific block request") == ( - "request is blocked by the operator image policy" - ) - - -def test_local_flux_runtime_and_gpu_handoff_are_flux_managed(): - """FLUX and Wolf share titan-24 while text stays on titan-20.""" - deployment = _documents(HERMES / "local-image-deployment.yaml")[0] - assert deployment["metadata"]["name"] == "hermes-local-image" - pod = deployment["spec"]["template"]["spec"] - assert pod["serviceAccountName"] == "hermes-gpu-runtime" - local = next(item for item in pod["containers"] if item["name"] == "local-image") - assert len(pod["containers"]) == 1 - assert local["resources"]["requests"]["nvidia.com/gpu.shared"] == 1 - assert local["ports"] == [{"name": "local-image", "containerPort": 9004}] - assert any(mount["mountPath"] == "/models" for mount in local["volumeMounts"]) - model_env = {item["name"]: item["value"] for item in local["env"]} - assert model_env["HERMES_LOCAL_IMAGE_LISTEN_PORT"] == "9004" - assert model_env["HERMES_LOCAL_IMAGE_REVISION"] == ( - "e7b7dc27f91deacad38e78976d1f2b499d76a294" - ) - assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVITY_NODE"] == "titan-24" - assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVE_SM_PERCENT"] == "80" - assert model_env["HERMES_LOCAL_IMAGE_GPU_MAX_EXTERNAL_MEMORY_BYTES"] == ( - "3221225472" - ) - assert model_env["HERMES_LOCAL_IMAGE_OFFLOAD_MODE"] == "sequential" - assert "nvidia-process-exporter-local.monitoring.svc.cluster.local" in model_env[ - "HERMES_LOCAL_IMAGE_GPU_ACTIVITY_URL" - ] - models_volume = next(item for item in pod["volumes"] if item["name"] == "models") - assert models_volume["persistentVolumeClaim"]["claimName"] == ( - "hermes-image-models" - ) - - services = _documents(HERMES / "service.yaml") - image_service = next( - item for item in services if item["metadata"]["name"] == "hermes-local-image" - ) - assert image_service["spec"]["selector"] == {"app": "hermes-local-image"} - - handoff_services = _documents(HERMES / "model-gate-deployment.yaml") - handoff = next( - item - for item in handoff_services - if item["kind"] == "Service" - and item["metadata"]["name"] == "hermes-gpu-handoff" - ) - assert handoff["spec"]["ports"][0]["targetPort"] == "handoff" - - ariadne = _documents( - Path(__file__).parents[2] - / "services/maintenance/apps/ariadne-deployment.yaml" - )[0] - env = { - item["name"]: item["value"] - for item in ariadne["spec"]["template"]["spec"]["containers"][0]["env"] - if "value" in item - } - assert env["GAME_MODE_OLLAMA_URL"] == ( - "http://hermes-gpu-handoff.hermes.svc.cluster.local:11434" - ) - assert env["GAME_MODE_OLLAMA_MODEL"] == "flux-2-klein-4b-local" - - for config_name in ("configmap.yaml", "agent-configmap.yaml", "chat-configmap.yaml"): - config = _documents(HERMES / config_name)[0]["data"]["config.yaml"] - assert "gpt-oss:20b" not in config - assert "atlas-switchyard" in config - switchyard = _documents(HERMES / "switchyard-configmap.yaml")[0]["data"][ - "routes.toml" - ] - assert 'id = "qwen2.5:14b-instruct-q4_0"' in switchyard - assert "qwen2.5:3b-instruct-q4_0" not in switchyard - assert "route/local/qwen2.5-14b/medium" in switchyard - assert "Anthropic and Claude name the same provider" in switchyard - assert "OpenAI and Codex name the same provider" in switchyard - assert "Choose across every configured Codex and Claude family" in switchyard - assert "Claude Fable" in switchyard - assert switchyard.count('Treat "think hard"') == 4 - assert switchyard.count("Never choose below the") >= 5 - switchyard_config = tomllib.loads(switchyard) - routes = switchyard_config["routes"] - configured_targets = switchyard_config["targets"] - for route_name in ("auto_fast", "auto_balanced", "auto_deep", "auto_maximum"): - leading_targets = set(routes[route_name]["targets"][:2]) - assert leading_targets == {"codex_sol_xhigh", "claude_opus_xhigh"} - assert "max_output_tokens" not in routes[route_name] - for route_name in ("auto_fast", "auto_balanced", "auto_deep", "auto_maximum"): - targets = routes[route_name]["targets"] - selector_targets = routes[route_name]["response_schema"] - assert not any(target.startswith("local_") for target in targets) - assert any("fable" in target for target in targets) - assert "local_qwen" not in selector_targets - assert "not eligible for foreground" in routes[route_name]["prompt"] - for route_name in ("auto_deep", "auto_maximum"): - targets = routes[route_name]["targets"] - selector_targets = routes[route_name]["response_schema"] - assert not any(target.endswith("_low") for target in targets) - assert "_low" not in selector_targets - maximum_targets = routes["auto_maximum"]["targets"] - maximum_selector_targets = routes["auto_maximum"]["response_schema"] - assert not any(target.endswith("_medium") for target in maximum_targets) - assert "_medium" not in maximum_selector_targets - assert "absolute high effort floor" in routes["auto_maximum"]["prompt"] - assert "quality mark was missed" in routes["auto_balanced"]["prompt"] - assert "raises the next boundary to xhigh" in routes["auto_maximum"]["prompt"] - assert "Repeated quality misses require xhigh" in routes[ - "worker_auto_maximum" - ]["prompt"] - assert any( - target.startswith("local_") - for target in routes["manual_local_qwen"]["targets"] - ) - for route_name in ( - "manual_codex_luna", - "manual_codex_terra", - "manual_codex_sol", - "manual_claude_haiku", - "manual_claude_fable", - "manual_claude_sonnet", - "manual_claude_opus", - ): - assert not any( - target.startswith("local_") for target in routes[route_name]["targets"] - ) - for provider, families in { - "codex": ("luna", "terra", "sol"), - "claude": ("haiku", "fable", "sonnet", "opus"), - }.items(): - for family in families: - for effort in ("low", "medium", "high", "xhigh"): - route = routes[f"manual_{provider}_{family}_{effort}"] - assert route["id"] == f"atlas/manual/{provider}/{family}/{effort}" - assert route["targets"][0] == f"{provider}_{family}_{effort}" - worker_target = f"worker_{provider}_{family}_{effort}" - assert worker_target in routes["worker_auto_maximum"]["targets"] - assert configured_targets[worker_target]["id"] == ( - f"worker/{provider}/{family}/{effort}" - ) - for route_name in ("auto_fast", "auto_balanced"): - prompt = routes[route_name]["prompt"] - assert "image tool—not the conversational model" in prompt - assert "Do not select a" in prompt - assert "local Qwen or Claude target" in prompt - assert "max_output_tokens" not in routes["worker_auto_maximum"] - model_gate = _documents(HERMES / "model-gate-configmap.yaml")[0]["data"][ - "model_gate.py" - ] - assert "qwen2.5:14b-instruct-q4_0" in model_gate - - -def test_titan20_serializes_classifier_and_local_chat_model_residency(): - """Classifier and local chat share one serialized resident Qwen weight.""" - deployment = _documents( - Path(__file__).parents[2] / "services/ai-llm/deployment.yaml" - )[0] - pod = deployment["spec"]["template"]["spec"] - required = pod["affinity"]["nodeAffinity"][ - "requiredDuringSchedulingIgnoredDuringExecution" - ]["nodeSelectorTerms"][0]["matchExpressions"][0] - assert required["values"] == ["titan-20"] - container = pod["containers"][0] - env = {item["name"]: item["value"] for item in container["env"]} - assert env["OLLAMA_MAX_LOADED_MODELS"] == "1" - assert env["OLLAMA_NUM_PARALLEL"] == "1" - assert env["OLLAMA_KEEP_ALIVE"] == "-1" - assert env["OLLAMA_CONTEXT_LENGTH"] == "8192" - warm_command = " ".join(container["command"]) - assert "--keepalive=-1" not in warm_command - models = next(item for item in pod["volumes"] if item["name"] == "models") - assert models["persistentVolumeClaim"]["claimName"] == ( - "ollama-models-titan20" - ) - - -def test_local_image_gpu_guard_distinguishes_background_and_saturated_gpu(monkeypatch): - """Lease-idle desktop spikes may coexist, but saturation still blocks FLUX.""" - source = ROOT / "dockerfiles" / "hermes-local-image-server.py" - spec = importlib.util.spec_from_file_location("hermes_local_image_server", source) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - monkeypatch.setattr(module, "GPU_ACTIVITY_NODE", "titan-24") - monkeypatch.setattr(module, "GPU_ACTIVE_SM_PERCENT", 80.0) - monkeypatch.setattr(module, "GPU_MAX_EXTERNAL_MEMORY_BYTES", 3 << 30) - - idle = module._parse_gpu_activity( - '\n'.join( - [ - 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="Xorg"} 0', - 'nvidia_process_gpu_memory_used_bytes{node="titan-24",namespace="host",process="Xorg"} 1900000000', - 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="game-stream",process="wolf"} 3', - 'nvidia_process_gpu_memory_used_bytes{node="titan-24",namespace="hermes",process="python"} 9000000000', - ] - ) - ) - assert idle["interactive_active"] is False - assert idle["external_gpu_memory_bytes"] == 1900000000 - assert idle["external_gpu_sm_percent"] == 3 - - background_spike = module._parse_gpu_activity( - 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="sway"} 41\n' - ) - assert background_spike["interactive_active"] is False - - active = module._parse_gpu_activity( - 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="steam"} 91\n' - ) - assert active["interactive_active"] is True - assert "91%" in active["gpu_guard_reason"] - - -def test_local_flux_renderer_uses_a_disposable_cuda_worker(monkeypatch): - """A completed render must not leave its CUDA context in the API process.""" - source = ROOT / "dockerfiles" / "hermes-local-image-server.py" - spec = importlib.util.spec_from_file_location("hermes_local_image_worker", source) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - calls = [] - - def run(command, **kwargs): - calls.append((command, kwargs)) - return SimpleNamespace( - returncode=0, - stdout=b'{"success":true,"route":"local","image_b64":"cG5n"}', - stderr=b"", - ) - - monkeypatch.setattr(module.subprocess, "run", run) - result = module._render({"prompt": "black cat", "aspect_ratio": "square"}) - - assert result["route"] == "local" - command, options = calls[0] - assert command[-1] == "--render-worker" - assert json.loads(options["input"]) == { - "prompt": "black cat", - "aspect_ratio": "square", - } - assert options["timeout"] == module.RENDER_TIMEOUT_SECONDS - assert options["check"] is False - - -def test_local_flux_uses_low_vram_offload_without_reducing_resolution(): - """The shared 3080 lane must trade time, not image size, for headroom.""" - source = (ROOT / "dockerfiles" / "hermes-local-image-server.py").read_text() - assert 'OFFLOAD_MODE = os.environ.get(' in source - assert '"HERMES_LOCAL_IMAGE_OFFLOAD_MODE", "sequential"' in source - assert "pipe.enable_sequential_cpu_offload()" in source - assert '"square": (1024, 1024)' in source - - -def test_chat_auth_and_relay_are_pod_lifetime_only(): - statefulset = _documents(HERMES / "chat-statefulset.yaml")[0] - pod = statefulset["spec"]["template"]["spec"] - containers = statefulset["spec"]["template"]["spec"]["containers"] - - for name in ("hermes", "webui"): - container = next(item for item in containers if item["name"] == name) - env = {item["name"]: item["value"] for item in container["env"]} - assert env["HERMES_AUTH_FILE"] == "/runtime-access/hermes-auth.json" - mount = next( - item for item in container["volumeMounts"] if item["name"] == "runtime-access" - ) - assert mount["mountPath"] == "/runtime-access" - assert "subPath" not in mount - - hermes_env = { - item["name"]: item["value"] - for item in next(item for item in containers if item["name"] == "hermes")["env"] - } - runtime = next(item for item in pod["volumes"] if item["name"] == "runtime-access") - assert runtime["emptyDir"] == {"medium": "Memory", "sizeLimit": "2Mi"} - assert not any(item["name"] == "provider-auth" for item in pod["volumes"]) - init_command = next( - item for item in pod["initContainers"] if item["name"] == "init-config" - )["command"][2] - for key in ( - "ANTHROPIC_API_KEY", - "API_SERVER_KEY", - "CLAUDE_CODE_OAUTH_TOKEN", - "GITEA_TOKEN", - "HERMES_IMAGE_BROKER_KEY", - "OPENAI_API_KEY", - ): - assert key in init_command - assert "printf 'API_SERVER_KEY=%s" not in init_command - assert hermes_env["AGENT_BROWSER_EXECUTABLE_PATH"].endswith("/chrome-linux/headless_shell") - assert "--no-sandbox" in hermes_env["AGENT_BROWSER_ARGS"] - - -def test_sandbox_executes_python_with_bounded_output(tmp_path: Path, monkeypatch): - source = ROOT / "dockerfiles" / "hermes-chat-sandbox-server.py" - spec = importlib.util.spec_from_file_location("hermes_chat_sandbox_server", source) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - monkeypatch.setattr(module, "WORKSPACE", tmp_path) - - result = module._execute("import math\nprint(math.comb(10, 3))") - - assert result["success"] is True - assert result["stdout"] == "120\n" - assert result["stderr"] == "" diff --git a/testing/tests/test_hermes_chat_routing_runtime.py b/testing/tests/test_hermes_chat_routing_runtime.py new file mode 100644 index 00000000..6a81d625 --- /dev/null +++ b/testing/tests/test_hermes_chat_routing_runtime.py @@ -0,0 +1,486 @@ +"""Routing and runtime workload contracts for Hermes chat.""" + +from __future__ import annotations + +import importlib.util +import json +import sqlite3 +import sys +import tomllib +from pathlib import Path +from types import SimpleNamespace + + +from testing.tests.test_hermes_chat_support import ( + HERMES, + ROOT, + _documents, +) + + +def test_stale_parent_linked_api_workers_close_on_startup(tmp_path: Path): + """A previous gateway lifetime cannot leave phantom active workers.""" + module_path = HERMES / "scripts" / "migrate_api_session_lineage.py" + spec = importlib.util.spec_from_file_location( + "close_stale_api_workers", module_path + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + database = tmp_path / "state.db" + with sqlite3.connect(database) as connection: + connection.execute( + "CREATE TABLE sessions (id TEXT PRIMARY KEY, source TEXT, " + "parent_session_id TEXT, title TEXT, started_at REAL, ended_at REAL, " + "end_reason TEXT, archived INTEGER DEFAULT 0)" + ) + connection.executemany( + "INSERT INTO sessions " + "(id, source, parent_session_id, started_at, ended_at, end_reason) " + "VALUES (?, ?, ?, 1, ?, ?)", + ( + ("stale", "api_server", "parent", None, None), + ("root", "api_server", None, None, None), + ("finished", "api_server", "parent", 2.0, "api_run_completed"), + ("interactive", "tui", "parent", None, None), + ), + ) + + assert module.migrate(database) == 1 + assert module.migrate(database) == 0 + with sqlite3.connect(database) as connection: + rows = connection.execute( + "SELECT id, ended_at, end_reason FROM sessions ORDER BY id" + ).fetchall() + by_id = {row[0]: row[1:] for row in rows} + assert by_id["stale"][0] is not None + assert by_id["stale"][1] == "api_run_recovered_stale" + assert by_id["root"] == (None, None) + assert by_id["finished"] == (2.0, "api_run_completed") + assert by_id["interactive"] == (None, None) + + +def test_switchyard_brokers_and_native_claude_lane_use_the_right_images(): + """Thin brokers stay small while native Claude runs beside owner auth.""" + dockerfile = ( + ROOT / "dockerfiles" / "Dockerfile.hermes-switchyard-brokers" + ).read_text() + assert "httpx==0.28.1" in dockerfile + assert "worker_route_broker.py" in dockerfile + assert "routing_catalog.py" in dockerfile + + deployment = _documents(HERMES / "switchyard-deployment.yaml")[0] + containers = { + container["name"]: container + for container in deployment["spec"]["template"]["spec"]["containers"] + } + expected = ( + "registry.bstein.dev/bstein/hermes-switchyard-brokers@" + "sha256:ee7e95e060ef8083da505162d7e9030daba15fdd828cc047bbcbe6aa409d2083" + ) + assert containers["worker-route-broker"]["image"] == expected + assert containers["classifier-broker"]["image"] == expected + assert "claude-oauth-broker" not in containers + + agent = _documents(HERMES / "agent-deployment.yaml")[0] + agent_containers = { + container["name"]: container + for container in agent["spec"]["template"]["spec"]["containers"] + } + for container_name in ("hermes", "terminal"): + container = agent_containers[container_name] + environment = {item["name"]: item["value"] for item in container["env"]} + mounts = {item["name"]: item for item in container["volumeMounts"]} + assert ( + environment["HERMES_ROUTING_CATALOG_PATH"] + == "/routing-catalog/catalog.json" + ) + assert ( + environment["HERMES_CODEX_HEALTH_PATH"] + == "/opt/data/provider-health/codex.json" + ) + assert ( + environment["HERMES_CLAUDE_HEALTH_PATH"] + == "/opt/data/provider-health/claude.json" + ) + assert mounts["routing-catalog"]["mountPath"] == "/routing-catalog" + assert mounts["routing-catalog"]["readOnly"] is True + codex = agent_containers["codex-broker"] + codex_environment = {item["name"]: item["value"] for item in codex["env"]} + assert ( + codex_environment["HERMES_CODEX_HEALTH_PATH"] + == "/opt/data/provider-health/codex.json" + ) + claude = agent_containers["claude-broker"] + assert claude["image"].startswith("registry.bstein.dev/bstein/hermes-agent@") + assert "unset ANTHROPIC_API_KEY CLAUDE_API_KEY" in claude["args"][0] + assert any( + mount["name"] == "home" and mount["mountPath"] == "/opt/data" + for mount in claude["volumeMounts"] + ) + + +def test_classifier_broker_bounds_history_without_losing_routing_intent(monkeypatch): + """AUTO classification must fit the Jetson context without losing intent.""" + broker_path = HERMES / "scripts" / "classifier_broker.py" + monkeypatch.setitem(sys.modules, "httpx", SimpleNamespace()) + spec = importlib.util.spec_from_file_location( + "hermes_classifier_broker", broker_path + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + payload = { + "model": "qwen2.5:14b-instruct-q4_0", + "messages": [ + { + "role": "system", + "content": "routing contract\n" + ("candidate policy " * 1000), + }, + { + "role": "user", + "content": "Build and verify the Cassandra release safely.", + }, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_large", + "type": "function", + "function": { + "name": "large_tool", + "arguments": '{"command":"' + ("x" * 5000) + '"}', + }, + } + ], + }, + { + "role": "tool", + "content": "unbounded test output " * 10000, + "tool_call_id": "call_large", + }, + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": "data:image/png;base64,private"}, + }, + {"type": "text", "text": "Turn this cat into a cute clown."}, + ], + }, + ], + "tools": [{"type": "function", "function": {"name": "large_tool"}}], + "tool_choice": "auto", + "parallel_tool_calls": True, + "response_format": {"type": "json_object"}, + } + + compacted = module.compact_payload(payload) + encoded = json.dumps(compacted) + + assert compacted["model"] == payload["model"] + assert compacted["response_format"] == payload["response_format"] + assert "tools" not in compacted + assert "tool_choice" not in compacted + assert "parallel_tool_calls" not in compacted + assert all(message.get("role") != "tool" for message in compacted["messages"]) + assert all("tool_call_id" not in message for message in compacted["messages"]) + assert all("tool_calls" not in message for message in compacted["messages"]) + assert "[tool evidence]" in encoded + assert "[assistant requested an external tool]" in encoded + assert "Build and verify the Cassandra release safely." in encoded + assert "Turn this cat into a cute clown." in encoded + assert "image attachment available to the selected worker" in encoded + assert "data:image/png;base64" not in encoded + assert "unbounded test output " * 100 not in encoded + assert len(encoded) < 18_000 + + +def test_switchyard_classifier_is_bounded_and_fails_open_once(): + """A sick local judge must not hold chat through repeated long retries.""" + config = tomllib.loads( + _documents(HERMES / "switchyard-configmap.yaml")[0]["data"]["routes.toml"] + ) + classifier = config["llm_clients"]["classifier"] + assert classifier["base_url"] == "http://127.0.0.1:9008/v1" + assert classifier["max_retries"] == 0 + for route in ("auto_fast", "auto_balanced"): + assert config["routes"][route]["recent_turn_window"] == 4 + for route in ("auto_deep", "auto_maximum", "worker_auto_maximum"): + assert config["routes"][route]["recent_turn_window"] == 6 + + deployment = _documents(HERMES / "switchyard-deployment.yaml")[0] + containers = { + item["name"]: item + for item in deployment["spec"]["template"]["spec"]["containers"] + } + classifier_container = containers["classifier-broker"] + env = {item["name"]: item["value"] for item in classifier_container["env"]} + assert env["HERMES_CLASSIFIER_BROKER_READ_TIMEOUT"] == "60" + assert classifier_container["readinessProbe"]["httpGet"]["port"] == "classifier" + + +def test_image_broker_returns_bytes_and_removes_owner_cache( + tmp_path: Path, monkeypatch +): + """The broker must not retain a family user's generated image.""" + broker_path = HERMES / "scripts" / "image_broker.py" + spec = importlib.util.spec_from_file_location("hermes_image_broker", broker_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + generated = tmp_path / "generated.png" + generated.write_bytes(b"\x89PNG\r\n\x1a\nprivate-image") + + class Provider: + def generate(self, prompt, aspect, **kwargs): + assert prompt == "paint a blue sphere" + assert aspect == "square" + return { + "success": True, + "image": str(generated), + "model": "gpt-image-2-high", + "quality": "high", + } + + monkeypatch.setattr(module, "_PROVIDER", Provider()) + result = module._generate( + { + "prompt": "paint a blue sphere", + "aspect_ratio": "square", + "model": "gpt-image-2-high", + } + ) + + assert result["success"] is True + assert result["image_b64"] + assert "image" not in result + assert not generated.exists() + + +def test_image_broker_auto_falls_back_to_local_and_honors_explicit_routes( + monkeypatch, +): + """AUTO is hosted-first while explicit local never calls the hosted lane.""" + broker_path = HERMES / "scripts" / "image_broker.py" + spec = importlib.util.spec_from_file_location("hermes_image_router", broker_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + calls = [] + + def hosted(payload, model, prompt, aspect): + calls.append(("hosted", model, prompt, aspect)) + return {"success": False, "error": "hosted refusal"} + + def local(payload, timeout=1800.0): + calls.append(("local", payload["model"], timeout)) + return { + "success": True, + "image_b64": "aW1hZ2U=", + "model": "flux-2-klein-4b-local", + "route": "local", + } + + monkeypatch.setattr(module, "_generate_hosted", hosted) + monkeypatch.setattr(module, "_local_request", local) + auto = module._generate( + { + "prompt": "colorize this family photograph", + "aspect_ratio": "portrait", + "model": "atlas-image-auto-high", + } + ) + assert auto["success"] is True + assert auto["route"] == "local" + assert auto["hosted_fallback_reason"] == "hosted refusal" + assert [call[0] for call in calls] == ["hosted", "local"] + + calls.clear() + explicit = module._generate( + { + "prompt": "make a local landscape", + "aspect_ratio": "landscape", + "model": "flux-2-klein-4b-local", + } + ) + assert explicit["route"] == "local" + assert [call[0] for call in calls] == ["local"] + + +def test_image_broker_policy_is_narrow_and_operator_extensible( + tmp_path: Path, monkeypatch +): + """Family-photo restoration stays allowed while the hard boundary remains.""" + broker_path = HERMES / "scripts" / "image_broker.py" + spec = importlib.util.spec_from_file_location("hermes_image_policy", broker_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + policy = tmp_path / "policy.json" + policy.write_text('{"additional_blocked_phrases":["site-specific block"]}') + monkeypatch.setattr(module, "POLICY_PATH", policy) + + assert ( + module._policy_error( + "Colorize my baby photograph with a lighter natural skin tone" + ) + is None + ) + assert "minors" in module._policy_error("Create a sexual image of a child") + assert module._policy_error("A site-specific block request") == ( + "request is blocked by the operator image policy" + ) + + +def test_local_flux_runtime_and_gpu_handoff_are_flux_managed(): + """FLUX and Wolf share titan-24 while text stays on titan-20.""" + deployment = _documents(HERMES / "local-image-deployment.yaml")[0] + assert deployment["metadata"]["name"] == "hermes-local-image" + pod = deployment["spec"]["template"]["spec"] + assert pod["serviceAccountName"] == "hermes-gpu-runtime" + local = next(item for item in pod["containers"] if item["name"] == "local-image") + assert len(pod["containers"]) == 1 + assert local["resources"]["requests"]["nvidia.com/gpu.shared"] == 1 + assert local["ports"] == [{"name": "local-image", "containerPort": 9004}] + assert any(mount["mountPath"] == "/models" for mount in local["volumeMounts"]) + model_env = {item["name"]: item["value"] for item in local["env"]} + assert model_env["HERMES_LOCAL_IMAGE_LISTEN_PORT"] == "9004" + assert model_env["HERMES_LOCAL_IMAGE_REVISION"] == ( + "e7b7dc27f91deacad38e78976d1f2b499d76a294" + ) + assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVITY_NODE"] == "titan-24" + assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVE_SM_PERCENT"] == "80" + assert model_env["HERMES_LOCAL_IMAGE_GPU_MAX_EXTERNAL_MEMORY_BYTES"] == ( + "3221225472" + ) + assert model_env["HERMES_LOCAL_IMAGE_OFFLOAD_MODE"] == "sequential" + assert ( + "nvidia-process-exporter-local.monitoring.svc.cluster.local" + in model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVITY_URL"] + ) + models_volume = next(item for item in pod["volumes"] if item["name"] == "models") + assert models_volume["persistentVolumeClaim"]["claimName"] == ( + "hermes-image-models" + ) + + services = _documents(HERMES / "service.yaml") + image_service = next( + item for item in services if item["metadata"]["name"] == "hermes-local-image" + ) + assert image_service["spec"]["selector"] == {"app": "hermes-local-image"} + + handoff_services = _documents(HERMES / "model-gate-deployment.yaml") + handoff = next( + item + for item in handoff_services + if item["kind"] == "Service" + and item["metadata"]["name"] == "hermes-gpu-handoff" + ) + assert handoff["spec"]["ports"][0]["targetPort"] == "handoff" + + ariadne = _documents( + Path(__file__).parents[2] / "services/maintenance/apps/ariadne-deployment.yaml" + )[0] + env = { + item["name"]: item["value"] + for item in ariadne["spec"]["template"]["spec"]["containers"][0]["env"] + if "value" in item + } + assert env["GAME_MODE_OLLAMA_URL"] == ( + "http://hermes-gpu-handoff.hermes.svc.cluster.local:11434" + ) + assert env["GAME_MODE_OLLAMA_MODEL"] == "flux-2-klein-4b-local" + + for config_name in ( + "configmap.yaml", + "agent-configmap.yaml", + "chat-configmap.yaml", + ): + config = _documents(HERMES / config_name)[0]["data"]["config.yaml"] + assert "gpt-oss:20b" not in config + assert "atlas-switchyard" in config + switchyard = _documents(HERMES / "switchyard-configmap.yaml")[0]["data"][ + "routes.toml" + ] + assert 'id = "qwen2.5:14b-instruct-q4_0"' in switchyard + assert "qwen2.5:3b-instruct-q4_0" not in switchyard + assert "route/local/qwen2.5-14b/medium" in switchyard + assert "Anthropic and Claude name the same provider" in switchyard + assert "OpenAI and Codex name the same provider" in switchyard + assert "Choose across every configured Codex and Claude family" in switchyard + assert "Claude Fable" in switchyard + assert switchyard.count('Treat "think hard"') == 4 + assert switchyard.count("Never choose below the") >= 5 + switchyard_config = tomllib.loads(switchyard) + routes = switchyard_config["routes"] + configured_targets = switchyard_config["targets"] + for route_name in ("auto_fast", "auto_balanced", "auto_deep", "auto_maximum"): + leading_targets = set(routes[route_name]["targets"][:2]) + assert leading_targets == {"codex_sol_xhigh", "claude_opus_xhigh"} + assert "max_output_tokens" not in routes[route_name] + for route_name in ("auto_fast", "auto_balanced", "auto_deep", "auto_maximum"): + targets = routes[route_name]["targets"] + selector_targets = routes[route_name]["response_schema"] + assert not any(target.startswith("local_") for target in targets) + assert any("fable" in target for target in targets) + assert "local_qwen" not in selector_targets + assert "not eligible for foreground" in routes[route_name]["prompt"] + for route_name in ("auto_deep", "auto_maximum"): + targets = routes[route_name]["targets"] + selector_targets = routes[route_name]["response_schema"] + assert not any(target.endswith("_low") for target in targets) + assert "_low" not in selector_targets + maximum_targets = routes["auto_maximum"]["targets"] + maximum_selector_targets = routes["auto_maximum"]["response_schema"] + assert not any(target.endswith("_medium") for target in maximum_targets) + assert "_medium" not in maximum_selector_targets + assert "absolute high effort floor" in routes["auto_maximum"]["prompt"] + assert "quality mark was missed" in routes["auto_balanced"]["prompt"] + assert "raises the next boundary to xhigh" in routes["auto_maximum"]["prompt"] + assert ( + "Repeated quality misses require xhigh" + in routes["worker_auto_maximum"]["prompt"] + ) + assert any( + target.startswith("local_") for target in routes["manual_local_qwen"]["targets"] + ) + for route_name in ( + "manual_codex_luna", + "manual_codex_terra", + "manual_codex_sol", + "manual_claude_haiku", + "manual_claude_fable", + "manual_claude_sonnet", + "manual_claude_opus", + ): + assert not any( + target.startswith("local_") for target in routes[route_name]["targets"] + ) + for provider, families in { + "codex": ("luna", "terra", "sol"), + "claude": ("haiku", "fable", "sonnet", "opus"), + }.items(): + for family in families: + for effort in ("low", "medium", "high", "xhigh"): + route = routes[f"manual_{provider}_{family}_{effort}"] + assert route["id"] == f"atlas/manual/{provider}/{family}/{effort}" + assert route["targets"][0] == f"{provider}_{family}_{effort}" + worker_target = f"worker_{provider}_{family}_{effort}" + assert worker_target in routes["worker_auto_maximum"]["targets"] + assert configured_targets[worker_target]["id"] == ( + f"worker/{provider}/{family}/{effort}" + ) + for route_name in ("auto_fast", "auto_balanced"): + prompt = routes[route_name]["prompt"] + assert "image tool—not the conversational model" in prompt + assert "Do not select a" in prompt + assert "local Qwen or Claude target" in prompt + assert "max_output_tokens" not in routes["worker_auto_maximum"] + model_gate = _documents(HERMES / "model-gate-configmap.yaml")[0]["data"][ + "model_gate.py" + ] + assert "qwen2.5:14b-instruct-q4_0" in model_gate diff --git a/testing/tests/test_hermes_chat_session_continuity.py b/testing/tests/test_hermes_chat_session_continuity.py new file mode 100644 index 00000000..0da542cb --- /dev/null +++ b/testing/tests/test_hermes_chat_session_continuity.py @@ -0,0 +1,478 @@ +"""Session lineage and continuity contracts for Hermes chat.""" + +from __future__ import annotations + +import importlib.util +import json +import sqlite3 +from pathlib import Path + +import pytest + +from testing.tests.test_hermes_chat_support import ( + HERMES, +) + + +def test_codex_native_health_overrides_historical_router_errors( + tmp_path: Path, monkeypatch +): + """Fresh first-party health is authoritative over old Switchyard probes.""" + plugin_path = HERMES / "plugins" / "auto-router" / "provider_status.py" + spec = importlib.util.spec_from_file_location("hermes_provider_status", plugin_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + health_path = tmp_path / "codex.json" + health_path.write_text( + json.dumps( + { + "state": "available", + "authenticated": True, + "transport": "codex-chatgpt-subscription", + } + ) + ) + monkeypatch.setattr(module, "CODEX_HEALTH_PATH", health_path) + monkeypatch.setattr(module, "CLAUDE_HEALTH_PATH", tmp_path / "missing.json") + monkeypatch.setattr( + module, + "_get_json", + lambda url: {"status": "ok"} + if url.endswith("/health") + else { + "models": { + "route/codex/terra/medium": { + "calls": 1, + "errors": 99, + "total_tokens": 12, + } + } + }, + ) + monkeypatch.setattr(module, "_codex_account", lambda: {}) + monkeypatch.setattr(module, "_claude_account", lambda: {}) + + codex = module.provider_status_payload()["providers"]["codex"] + + assert codex["errors"] == 99 + assert codex["state"] == "available" + assert codex["native_health"]["transport"] == "codex-chatgpt-subscription" + + +def test_api_session_patch_accepts_parent_lineage(tmp_path: Path): + """API-created workers must persist the originating Hermes session.""" + module_path = HERMES / "scripts" / "patch_api_server_sessions.py" + spec = importlib.util.spec_from_file_location("patch_api_sessions", module_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + source = tmp_path / "api_server.py" + destination = tmp_path / "patched.py" + source.write_text( + "prefix\n" + + module.BEFORE + + "middle\n" + + module.RUNS_BEFORE + + "run body\n" + + module.RUN_CLOSE_BEFORE + + module.RESPONSES_SESSION_BEFORE + + module.EVENT_CALLBACK_SIGNATURE_BEFORE + + "callback docstring and push helper\n" + + module.EVENT_CALLBACK_BODY_BEFORE + + "tool start body\n" + + module.EVENT_CALLBACK_END_BEFORE + + module.EVENT_CALLBACK_CALL_BEFORE + + module.RUN_SWEEP_BEFORE + + "suffix\n", + encoding="utf-8", + ) + + module.patch(source, destination) + patched = destination.read_text(encoding="utf-8") + + assert "X-Hermes-Parent-Session-Id" in patched + assert "parent_session_id=parent_session_id" in patched + assert "Parent session not found" in patched + assert "HERMES_API_DEFAULT_PARENT_MATCH_PREFIXES" in patched + assert "user_message.startswith(default_prefixes)" in patched + assert "session_parent_conflict" in patched + assert "X-Hermes-Conversation-Platform" in patched + assert "X-Hermes-Conversation-Title" in patched + assert 'conversation_platform != "telegram"' in patched + assert "db.record_gateway_session_peer(" in patched + assert 'display_name="Telegram"' in patched + assert "db.reopen_session(session_id)" in patched + assert 'db.end_session(session_id, f"api_run_{terminal_status}")' in patched + assert "def _record_run_activity(" in patched + assert '"_thinking": "Hermes is reasoning"' in patched + assert '"run.started": "Worker started"' in patched + assert '"run.completed": "Worker completed"' in patched + assert '"reasoning.available": "Hermes finished a reasoning step"' in patched + assert '"subagent.progress": "Nested worker progress"' in patched + assert "redact_sensitive_text" in patched + assert 'getattr(os, "O_NOFOLLOW", 0)' in patched + assert "os.fchmod(fd, 0o600)" in patched + assert "session_id=session_id" in patched + assert 'self._record_run_activity(session_id, "run.started")' in patched + assert "detail = tool_name if event_type in {" in patched + assert 'if event_type == "subagent.tool"' in patched + assert "_RUN_ACTIVITY_HEARTBEAT_SECONDS = 15.0" in patched + assert "heartbeats.get(session_id, 0.0)" in patched + assert '"subagent.thinking",' in patched + assert "Stream retention and run lifetime are separate" in patched + assert 'terminal_status in {"completed", "failed", "cancelled"}' in patched + assert patched.index("terminal_status = self._run_statuses") < patched.index( + "self._active_run_tasks.pop(run_id, None)" + ) + + +def test_telegram_api_session_migration_is_bounded_and_idempotent(tmp_path: Path): + """Only named Telegram conversations receive presentation metadata.""" + module_path = HERMES / "scripts" / "migrate_telegram_api_sessions.py" + spec = importlib.util.spec_from_file_location( + "migrate_telegram_sessions", module_path + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + state = tmp_path / "state.db" + responses = tmp_path / "response_store.db" + with sqlite3.connect(state) as connection: + connection.execute( + """CREATE TABLE sessions ( + id TEXT PRIMARY KEY, source TEXT, session_key TEXT, chat_type TEXT, + display_name TEXT, origin_json TEXT, title TEXT + )""" + ) + connection.executemany( + "INSERT INTO sessions (id, source) VALUES (?, ?)", + (("telegram-session", "api_server"), ("other-session", "api_server")), + ) + with sqlite3.connect(responses) as connection: + connection.execute( + "CREATE TABLE conversations (name TEXT PRIMARY KEY, response_id TEXT NOT NULL)" + ) + connection.execute( + "CREATE TABLE responses (response_id TEXT PRIMARY KEY, data TEXT NOT NULL, accessed_at REAL NOT NULL)" + ) + connection.executemany( + "INSERT INTO responses VALUES (?, ?, 0)", + ( + ("telegram-response", json.dumps({"session_id": "telegram-session"})), + ("other-response", json.dumps({"session_id": "other-session"})), + ), + ) + connection.executemany( + "INSERT INTO conversations VALUES (?, ?)", + (("telegram", "telegram-response"), ("unrelated", "other-response")), + ) + + assert module.migrate(state, responses) == 1 + assert module.migrate(state, responses) == 0 + with sqlite3.connect(state) as connection: + telegram = connection.execute( + "SELECT session_key, chat_type, display_name, origin_json, title " + "FROM sessions WHERE id = 'telegram-session'" + ).fetchone() + other = connection.execute( + "SELECT session_key, title FROM sessions WHERE id = 'other-session'" + ).fetchone() + assert telegram[:3] == ("telegram", "private", "Telegram") + assert json.loads(telegram[3]) == { + "platform": "telegram", + "session_key": "telegram", + } + assert telegram[4] == "Telegram · General" + assert other == (None, None) + + +def test_web_session_activity_patch_projects_bounded_events(tmp_path: Path): + """The DOM transcript includes events without polluting agent history.""" + module_path = HERMES / "scripts" / "patch_web_session_activity.py" + spec = importlib.util.spec_from_file_location("patch_web_activity", module_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + source = tmp_path / "web_server.py" + destination = tmp_path / "patched.py" + source.write_text( + "prefix\n" + + module.LATEST_ROWS_BEFORE + + module.LATEST_SELECTION_BEFORE + + module.HELPER_MARKER + + " db = object()\n" + + module.MESSAGES_BEFORE + + "suffix\n", + encoding="utf-8", + ) + + module.patch(source, destination) + patched = destination.read_text(encoding="utf-8") + + assert "def _run_activity_messages(" in patched + assert 'root = get_hermes_home() / "run-activity"' in patched + assert "path.stat().st_size > 600_000" in patched + assert "entries[-1_000:]" in patched + assert "*_run_activity_messages(sid)" in patched + assert "messages.sort(" in patched + assert "limit: Optional[int] = None" in patched + assert "total_messages = len(messages)" in patched + assert "min(int(limit), 10_000)" in patched + assert '"total_messages": total_messages' in patched + assert "SELECT id, parent_session_id, started_at, ended_at" in patched + assert "newest still-open member of the lineage" in patched + assert "immediate objective parent" in patched + assert "mixes unrelated workstreams" in patched + assert "orphaned children" in patched + assert 'item[0].get("ended_at") is None' in patched + + +def test_web_session_lineage_returns_to_resumed_parent(): + """An ended reviewer must not strand the live view away from its parent.""" + module_path = HERMES / "scripts" / "patch_web_session_activity.py" + spec = importlib.util.spec_from_file_location("patch_web_lineage", module_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + namespace: dict[str, object] = {} + exec( + "def select_active(sid, db, rows):\n" + module.LATEST_SELECTION_AFTER, + namespace, + ) + + class FakeDB: + def get_session(self, session_id): + return next((row for row in rows if row["id"] == session_id), None) + + rows = [ + { + "id": "umbrella", + "parent_session_id": None, + "started_at": 0.0, + "ended_at": None, + }, + { + "id": "root", + "parent_session_id": "umbrella", + "started_at": 1.0, + "ended_at": None, + }, + { + "id": "review-1", + "parent_session_id": "root", + "started_at": 2.0, + "ended_at": 3.0, + }, + { + "id": "review-2", + "parent_session_id": "root", + "started_at": 4.0, + "ended_at": None, + }, + ] + select_active = namespace["select_active"] + + assert select_active("root", FakeDB(), rows) == ( + "review-2", + ["root", "review-2"], + ) + assert select_active("review-1", FakeDB(), rows) == ( + "review-2", + ["root", "review-2"], + ) + rows[2]["ended_at"] = 5.0 + rows[3]["ended_at"] = 5.0 + assert select_active("root", FakeDB(), rows) == ("root", ["root"]) + + rows[1]["ended_at"] = 9.0 + rows.append( + { + "id": "orphaned-review", + "parent_session_id": "root", + "started_at": 6.0, + "ended_at": None, + } + ) + rows.append( + { + "id": "accepted-review", + "parent_session_id": "root", + "started_at": 7.0, + "ended_at": 8.0, + } + ) + assert select_active("review-1", FakeDB(), rows) == ( + "accepted-review", + ["root", "accepted-review"], + ) + + +def test_api_activity_patch_coalesces_streaming_heartbeats( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +): + """Token callbacks stay bounded while every tool transition is retained.""" + module_path = HERMES / "scripts" / "patch_api_server_sessions.py" + spec = importlib.util.spec_from_file_location("patch_api_activity", module_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + activity_body = module.EVENT_CALLBACK_SIGNATURE_AFTER.split( + " def _make_run_event_callback(", 1 + )[0] + namespace: dict[str, object] = {} + exec( + "import hashlib, json, logging, os, time\n" + "from pathlib import Path\n" + "logger = logging.getLogger(__name__)\n" + "def redact_sensitive_text(value): return value\n" + "class ActivityRecorder:\n" + activity_body, + namespace, + ) + recorder = namespace["ActivityRecorder"]() + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + + recorder._record_run_activity("session", "_thinking") + recorder._record_run_activity("session", "reasoning.available") + recorder._record_run_activity("session", "subagent.thinking") + recorder._record_run_activity("session", "tool.started", tool_name="terminal") + recorder._record_run_activity("session", "tool.completed", tool_name="terminal") + + journals = list((tmp_path / "run-activity").glob("*.jsonl")) + assert len(journals) == 1 + entries = [json.loads(line) for line in journals[0].read_text().splitlines()] + assert [entry["activity_event"] for entry in entries] == [ + "_thinking", + "tool.started", + "tool.completed", + ] + + +def test_legacy_api_sessions_are_nested_idempotently(tmp_path: Path): + """Known standalone API workers move under Cassandra without data loss.""" + module_path = HERMES / "scripts" / "migrate_api_session_lineage.py" + spec = importlib.util.spec_from_file_location("migrate_api_sessions", module_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + database = tmp_path / "state.db" + with sqlite3.connect(database) as connection: + connection.execute( + "CREATE TABLE sessions (id TEXT PRIMARY KEY, source TEXT, " + "parent_session_id TEXT, title TEXT, transcript TEXT, archived INTEGER DEFAULT 0)" + ) + connection.execute( + "INSERT INTO sessions (id, source, parent_session_id, title, transcript) " + "VALUES (?, 'tui', NULL, 'Cassandra', 'parent-data')", + (module.LEGACY_CASSANDRA_PARENT,), + ) + worker_id = next(iter(module.LEGACY_CASSANDRA_WORKERS)) + connection.execute( + "INSERT INTO sessions (id, source, parent_session_id, title, transcript) " + "VALUES (?, 'api_server', NULL, 'old', 'keep-me')", + (worker_id,), + ) + connection.executemany( + "INSERT INTO sessions (id, source, parent_session_id, title, transcript) " + "VALUES (?, 'api_server', NULL, 'old smoke', 'keep-smoke')", + ((orphan_id,) for orphan_id in module.LEGACY_ORPHANED_SMOKE_SESSIONS), + ) + + assert module.migrate(database) == 1 + len(module.LEGACY_ORPHANED_SMOKE_SESSIONS) + assert module.migrate(database) == 0 + with sqlite3.connect(database) as connection: + row = connection.execute( + "SELECT parent_session_id, title, transcript FROM sessions WHERE id = ?", + (worker_id,), + ).fetchone() + assert row == ( + module.LEGACY_CASSANDRA_PARENT, + module.LEGACY_CASSANDRA_WORKERS[worker_id], + "keep-me", + ) + with sqlite3.connect(database) as connection: + orphans = connection.execute( + "SELECT id, archived, title, transcript FROM sessions " + "WHERE id IN ({}) ORDER BY id".format( + ",".join("?" for _ in module.LEGACY_ORPHANED_SMOKE_SESSIONS) + ), + tuple(module.LEGACY_ORPHANED_SMOKE_SESSIONS), + ).fetchall() + assert orphans == sorted( + ( + orphan_id, + 1, + module.LEGACY_ORPHANED_SMOKE_SESSIONS[orphan_id], + "keep-smoke", + ) + for orphan_id in module.LEGACY_ORPHANED_SMOKE_SESSIONS + ) + + +def test_automated_triage_sessions_are_grouped_without_touching_interactive_runs( + tmp_path: Path, +): + """Only the stable Ariadne contract moves below the triage parent.""" + module_path = HERMES / "scripts" / "migrate_api_session_lineage.py" + spec = importlib.util.spec_from_file_location( + "migrate_triage_sessions", module_path + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + database = tmp_path / "state.db" + with sqlite3.connect(database) as connection: + connection.execute( + "CREATE TABLE sessions (id TEXT PRIMARY KEY, source TEXT, " + "parent_session_id TEXT, title TEXT, started_at REAL, archived INTEGER DEFAULT 0)" + ) + connection.execute( + "CREATE TABLE messages (session_id TEXT, role TEXT, content TEXT)" + ) + connection.executemany( + "INSERT INTO sessions (id, source, started_at) VALUES (?, 'api_server', ?)", + (("triage-run", 1.0), ("jenkins-run", 1.5), ("interactive-run", 2.0)), + ) + connection.executemany( + "INSERT INTO messages (session_id, role, content) VALUES (?, 'user', ?)", + ( + ( + "triage-run", + module.TRIAGE_MESSAGE_PREFIXES[0] + + " Fix for incident sonar/bstein_home/python:S2208/finding-key.", + ), + ( + "jenkins-run", + module.TRIAGE_MESSAGE_PREFIXES[1] + + "\nAnalyze incident soteria/291 for the Jenkins job soteria.", + ), + ("interactive-run", "Please explain this alert to me."), + ), + ) + + assert module.migrate(database, group_triage=True) == 2 + assert module.migrate(database, group_triage=True) == 0 + with sqlite3.connect(database) as connection: + parent = connection.execute( + "SELECT title FROM sessions WHERE id = ?", (module.TRIAGE_PARENT,) + ).fetchone() + triage = connection.execute( + "SELECT parent_session_id, title FROM sessions WHERE id = 'triage-run'" + ).fetchone() + interactive = connection.execute( + "SELECT parent_session_id FROM sessions WHERE id = 'interactive-run'" + ).fetchone() + jenkins = connection.execute( + "SELECT parent_session_id, title FROM sessions WHERE id = 'jenkins-run'" + ).fetchone() + assert parent == (module.TRIAGE_PARENT_TITLE,) + assert triage == ( + module.TRIAGE_PARENT, + "Sonar · bstein_home · python:S2208 · iage-run", + ) + assert interactive == (None,) + assert jenkins == ( + module.TRIAGE_PARENT, + "Sonar · soteria/291 · kins-run", + ) diff --git a/testing/tests/test_hermes_chat_support.py b/testing/tests/test_hermes_chat_support.py new file mode 100644 index 00000000..bf7d998c --- /dev/null +++ b/testing/tests/test_hermes_chat_support.py @@ -0,0 +1,38 @@ +"""Shared paths and loaders for Hermes chat capability contracts.""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path +from types import SimpleNamespace + +import yaml + +ROOT = Path(__file__).parents[2] +HERMES = ROOT / "services" / "hermes" +VAULT = ROOT / "services" / "vault" + + +def _documents(path: Path) -> list[dict]: + return [doc for doc in yaml.safe_load_all(path.read_text()) if doc] + + +def _load_broker_module(name: str, filename: str, monkeypatch): + """Load one broker with its mounted routing-catalog dependency.""" + catalog_path = HERMES / "scripts" / "routing_catalog.py" + catalog_spec = importlib.util.spec_from_file_location( + "routing_catalog", catalog_path + ) + assert catalog_spec and catalog_spec.loader + catalog = importlib.util.module_from_spec(catalog_spec) + catalog_spec.loader.exec_module(catalog) + monkeypatch.setitem(sys.modules, "routing_catalog", catalog) + monkeypatch.setitem(sys.modules, "httpx", SimpleNamespace()) + + broker_path = HERMES / "scripts" / filename + spec = importlib.util.spec_from_file_location(name, broker_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module diff --git a/testing/tests/test_hermes_cli_lanes.py b/testing/tests/test_hermes_cli_lanes.py index fe5272da..c40740aa 100644 --- a/testing/tests/test_hermes_cli_lanes.py +++ b/testing/tests/test_hermes_cli_lanes.py @@ -1,93 +1,19 @@ -"""Focused tests for Agent Hermes' direct Codex and Claude Kanban lanes.""" +"""Provider routing and process-isolation contracts for Hermes CLI lanes.""" from __future__ import annotations -import importlib.util import json import os import signal import sys -from contextlib import nullcontext from pathlib import Path -from types import SimpleNamespace - -import pytest -import yaml -SCRIPTS = Path(__file__).parents[2] / "services/hermes/scripts" -sys.path.insert(0, str(SCRIPTS)) -HERMES = Path(__file__).parents[2] / "services/hermes" -KEYCLOAK = Path(__file__).parents[2] / "services/keycloak" -FLUX_HERMES = ( - Path(__file__).parents[2] - / "clusters/atlas/flux-system/applications/hermes/kustomization.yaml" +from testing.tests.test_hermes_cli_lanes_support import ( + _SwitchyardResponse, + lanes, ) - -def _load(name: str): - spec = importlib.util.spec_from_file_location(name, SCRIPTS / f"{name}.py") - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - sys.modules[spec.name] = module - spec.loader.exec_module(module) - return module - - -lanes = _load("cli_lane_runner") -policy = _load("claude_command_policy") -migration = _load("migrate_herdr_state") -auth_patch = _load("patch_hermes_auth") -tui_gateway_patch = _load("patch_tui_gateway") -codex_runtime_patch = _load("patch_codex_runtime") -ttyd_patch = _load("patch_ttyd_index") -client_config = _load("configure_agent_clients") - - -def _agent_deployment() -> dict: - return yaml.safe_load((HERMES / "agent-deployment.yaml").read_text()) - - -def _services() -> dict[str, dict]: - return { - item["metadata"]["name"]: item - for item in yaml.safe_load_all((HERMES / "service.yaml").read_text()) - if item - } - - -def _oauth_deployment(name: str) -> dict: - documents = [ - item - for item in yaml.safe_load_all((HERMES / "oauth2-proxy.yaml").read_text()) - if item - ] - return next( - item - for item in documents - if item["kind"] == "Deployment" and item["metadata"]["name"] == name - ) - - -class _SwitchyardResponse: - """Minimal context-managed response used by routing contract tests.""" - - def __init__(self, selected: str, rationale: str = "local classifier vote"): - self.headers = { - "x-model-router-selected-model": selected, - "x-model-router-rationale": rationale, - } - - def __enter__(self): - return self - - def __exit__(self, *_args): - return False - - def read(self): - return b"{}" - - def test_auto_lane_uses_switchyard_worker_decision(): observed = {} @@ -460,1512 +386,3 @@ def test_worker_terminal_descendants_in_separate_groups_are_killed(monkeypatch): (5000, signal.SIGKILL), } assert process_signals == [(5000, signal.SIGTERM), (5000, signal.SIGKILL)] - - -def test_unassigned_ready_task_is_persistently_routed_to_auto_lane(monkeypatch): - task = SimpleNamespace(id="t_auto", assignee=None, status="ready") - assigned = [] - - class Connection: - def close(self): - return None - - def assign_task(_conn, task_id, profile): - assigned.append((task_id, profile)) - task.assignee = profile - return True - - fake_db = SimpleNamespace( - list_boards=lambda include_archived=False: [{"slug": "cassandra"}], - scoped_current_board=lambda _board: nullcontext(), - connect=lambda board: Connection(), - recompute_ready=lambda _conn: None, - list_tasks=lambda _conn: [task], - assign_task=assign_task, - get_task=lambda _conn, _task_id: task, - claim_task=lambda _conn, _task_id, **_kwargs: task, - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - - assert lanes.claim_ready(set(), 1) == [("cassandra", "t_auto")] - assert assigned == [("t_auto", "cli-auto")] - - -def test_corrupt_board_is_quarantined_without_stopping_healthy_lanes(monkeypatch, capsys): - class CorruptBoardError(Exception): - pass - - task = SimpleNamespace(id="t_healthy", assignee="cli-auto", status="ready") - - class Connection: - def close(self): - return None - - def connect(*, board): - if board == "cassandra": - raise CorruptBoardError("integrity_check failed") - return Connection() - - fake_db = SimpleNamespace( - KanbanDbCorruptError=CorruptBoardError, - list_boards=lambda include_archived=False: [ - {"slug": "cassandra"}, - {"slug": "healthy"}, - ], - scoped_current_board=lambda _board: nullcontext(), - connect=connect, - recompute_ready=lambda _conn: None, - list_tasks=lambda _conn: [task], - claim_task=lambda _conn, _task_id, **_kwargs: task, - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - lanes.BOARD_CORRUPTION_ERRORS.clear() - - assert lanes.claim_ready(set(), 1) == [("healthy", "t_healthy")] - assert "temporarily skipping Kanban board 'cassandra'" in capsys.readouterr().err - - -def test_transient_board_scan_failure_does_not_stop_healthy_lanes(monkeypatch, capsys): - task = SimpleNamespace(id="t_healthy", assignee="cli-auto", status="ready") - - class Connection: - def __init__(self, board): - self.board = board - - def close(self): - return None - - def recompute_ready(connection): - if connection.board == "cassandra": - raise lanes.sqlite3.OperationalError("disk I/O error") - - fake_db = SimpleNamespace( - list_boards=lambda include_archived=False: [ - {"slug": "cassandra"}, - {"slug": "healthy"}, - ], - scoped_current_board=lambda _board: nullcontext(), - connect=lambda board: Connection(board), - recompute_ready=recompute_ready, - list_tasks=lambda _conn: [task], - claim_task=lambda _conn, _task_id, **_kwargs: task, - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - lanes.BOARD_CORRUPTION_ERRORS.clear() - - assert lanes.claim_ready(set(), 1) == [("healthy", "t_healthy")] - error = capsys.readouterr().err - assert "temporarily skipping Kanban board 'cassandra'" in error - assert "storage OperationalError: disk I/O error" in error - - -def test_board_call_retries_storage_faults_on_fresh_connections(): - connections = [] - - class Connection: - def __init__(self): - self.closed = False - - def close(self): - self.closed = True - - def connect(*, board): - assert board == "cassandra" - connection = Connection() - connections.append(connection) - return connection - - attempts = [] - - def operation(_connection): - attempts.append(1) - if len(attempts) < 3: - raise lanes.sqlite3.OperationalError("disk I/O error") - return "healthy" - - fake_db = SimpleNamespace( - scoped_current_board=lambda _board: nullcontext(), - connect=connect, - ) - lanes.BOARD_CORRUPTION_ERRORS.clear() - - assert lanes._board_call(fake_db, "cassandra", operation) == "healthy" - assert len(connections) == 3 - assert all(connection.closed for connection in connections) - - -@pytest.mark.parametrize( - ("result", "expected_action"), - [ - (lanes.ProcessResult(0, "plain text only", None, False), "block"), - ( - lanes.ProcessResult( - 0, - "", - { - "status": "completed", - "summary": "done", - "changed_files": ["src/a.py"], - "tests_run": ["pytest -q"], - "artifacts": ["reports/result.json"], - "blockers": [], - }, - False, - ), - "complete", - ), - ( - lanes.ProcessResult( - 0, - "", - { - "status": "completed", - "summary": "The full test suite is still running.", - "changed_files": ["src/a.py"], - "tests_run": ["pytest -q — in progress"], - "artifacts": [], - "blockers": [], - }, - False, - ), - "block", - ), - ], -) -def test_claim_requires_structured_evidence_and_surfaces_artifacts( - tmp_path: Path, - monkeypatch, - result, - expected_action, -): - task = SimpleNamespace( - id="t_worker", - current_run_id=4, - assignee="cli-auto", - max_runtime_seconds=60, - ) - calls = [] - heartbeats = [] - connections = [] - artifact = tmp_path / "reports/result.json" - artifact.parent.mkdir() - artifact.write_text("{}\n", encoding="utf-8") - - class Connection: - def __init__(self): - self.closed = False - - def close(self): - self.closed = True - - def connect(*, board): - assert board == "cassandra" - connection = Connection() - connections.append(connection) - return connection - - fake_db = SimpleNamespace( - scoped_current_board=lambda _board: nullcontext(), - connect=connect, - get_task=lambda _conn, _task_id: task, - worker_log_path=lambda _task_id, board: tmp_path / "worker.log", - _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_worker"), - set_branch_name=lambda *_args: None, - set_workspace_path=lambda *_args: None, - build_worker_context=lambda *_args: "bounded objective", - heartbeat_worker=lambda _conn, _task_id, *, note, expected_run_id: ( - heartbeats.append((note, expected_run_id)) or True - ), - add_comment=lambda *_args: None, - complete_task=lambda *_args, **kwargs: calls.append(("complete", kwargs)), - block_task=lambda *_args, **kwargs: calls.append(("block", kwargs)), - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - monkeypatch.setattr(lanes, "state_path", lambda _board, _task_id: tmp_path / "state.json") - monkeypatch.setattr( - lanes, - "select_route", - lambda *_args, **_kwargs: lanes.Route( - "codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, () - ), - ) - def run_provider(*args, **_kwargs): - assert connections[0].closed - before_heartbeat = len(connections) - assert args[6]("working") is True - assert len(connections) == before_heartbeat + 1 - assert connections[-1].closed - return result - - monkeypatch.setattr(lanes, "run_provider", run_provider) - - lanes.execute_claim("cassandra", "t_worker") - - assert calls[0][0] == expected_action - assert heartbeats == [("working", 4)] - if expected_action == "complete": - assert calls[0][1]["metadata"]["artifacts"] == [str(artifact)] - assert calls[0][1]["metadata"]["tests_run"] == ["pytest -q"] - else: - assert calls[0][1]["kind"] == "capability" - assert all(connection.closed for connection in connections) - - -def test_goal_card_continues_after_local_judge_rejects_progress( - tmp_path: Path, - monkeypatch, -): - task = SimpleNamespace( - id="t_goal", - current_run_id=12, - assignee="cli-auto", - max_runtime_seconds=300, - goal_mode=True, - goal_max_turns=3, - ) - calls = [] - comments = [] - - class Connection: - def close(self): - return None - - fake_db = SimpleNamespace( - scoped_current_board=lambda _board: nullcontext(), - connect=lambda board: Connection(), - get_task=lambda _conn, _task_id: task, - worker_log_path=lambda _task_id, board: tmp_path / "worker.log", - _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_goal"), - set_branch_name=lambda *_args: None, - set_workspace_path=lambda *_args: None, - build_worker_context=lambda *_args: "Run tests, commit, push, and verify remote HEAD.", - heartbeat_worker=lambda *_args, **_kwargs: True, - add_comment=lambda _conn, _task_id, _author, body: comments.append(body), - complete_task=lambda *_args, **kwargs: calls.append(("complete", kwargs)), - block_task=lambda *_args, **kwargs: calls.append(("block", kwargs)), - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - monkeypatch.setattr( - lanes, - "state_path", - lambda _board, _task_id: tmp_path / "state.json", - ) - claude_low = lanes.Route( - "claude", "claude-fable-5", "low", "claude-low", "jetson", "vote", 1, () - ) - codex_low = lanes.Route( - "codex", "gpt-5.6-luna", "low", "codex-low", "manual", "fallback", 1, () - ) - codex_xhigh = lanes.Route( - "codex", "gpt-5.6-sol", "xhigh", "codex-xhigh", "jetson", "escalated", 1, () - ) - route_calls = [] - - def select_route(_prompt, assignee, **kwargs): - route_calls.append((assignee, kwargs)) - if assignee == "cli-codex-low": - return codex_low - if len(route_calls) == 1: - return claude_low - return codex_xhigh - - monkeypatch.setattr(lanes, "select_route", select_route) - monkeypatch.setattr(lanes, "fresh_unavailable_provider", lambda: None) - reports = [ - lanes.ProcessResult(1, "authentication expired", None, True), - lanes.ProcessResult( - 0, - "first turn", - { - "status": "completed", - "summary": "Focused tests passed.", - "changed_files": ["src/a.py"], - "tests_run": ["pytest focused: passed"], - "artifacts": [], - "blockers": [], - }, - False, - ), - lanes.ProcessResult( - 0, - "second turn", - { - "status": "completed", - "summary": "Full tests passed; commit pushed and remote HEAD verified.", - "changed_files": ["src/a.py"], - "tests_run": ["pytest full: passed"], - "artifacts": [], - "blockers": [], - }, - False, - ), - ] - monkeypatch.setattr(lanes, "run_provider", lambda *_args, **_kwargs: reports.pop(0)) - verdicts = iter( - [ - (False, "commit, push, and remote verification are missing"), - (True, "all explicit acceptance criteria have evidence"), - ] - ) - judge_contexts = [] - - def judge_goal_completion(objective, *_args, **_kwargs): - judge_contexts.append(objective) - return next(verdicts) - - monkeypatch.setattr( - lanes.cli_lane_goal, - "judge_goal_completion", - judge_goal_completion, - ) - - lanes.execute_claim("cassandra", "t_goal") - - assert calls[0][0] == "complete" - assert calls[0][1]["metadata"]["goal_turn"] == 2 - assert any("Goal completion rejected; continuing turn 2/3" in item for item in comments) - assert any("Goal route 2/3: codex/gpt-5.6-sol at xhigh" in item for item in comments) - assert route_calls[2][1]["exclude_provider"] == "claude" - assert "prior rejected reports" in judge_contexts[1] - assert "commit, push, and remote verification are missing" in judge_contexts[1] - assert reports == [] - - -def test_workspace_preparation_failure_durably_blocks_the_claim(tmp_path: Path, monkeypatch): - task = SimpleNamespace(id="t_bad_worktree", current_run_id=7, assignee="cli-auto") - calls = [] - - class Connection: - def close(self): - return None - - fake_db = SimpleNamespace( - scoped_current_board=lambda _board: nullcontext(), - connect=lambda board: Connection(), - get_task=lambda _conn, _task_id: task, - worker_log_path=lambda _task_id, board: tmp_path / "worker.log", - _resolve_worktree_workspace=lambda _task, board: (_ for _ in ()).throw( - ValueError("not a Git repository") - ), - block_task=lambda *_args, **kwargs: calls.append(kwargs), - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - monkeypatch.setattr(lanes, "state_path", lambda _board, _task_id: tmp_path / "state.json") - - lanes.execute_claim("cassandra", "t_bad_worktree") - - assert calls[0]["kind"] == "capability" - assert calls[0]["expected_run_id"] == 7 - assert "not a Git repository" in calls[0]["reason"] - - -def test_artifacts_cannot_escape_the_task_worktree(tmp_path: Path): - workspace = tmp_path / "workspace" - workspace.mkdir() - inside = workspace / "report.json" - outside = tmp_path / "auth.json" - inside.write_text("{}\n", encoding="utf-8") - outside.write_text("secret\n", encoding="utf-8") - - assert lanes.workspace_artifacts( - workspace, - ["report.json", str(outside), "missing.json"], - ) == [str(inside)] - - -def test_restart_provider_change_includes_explicit_workspace_handoff(tmp_path: Path, monkeypatch): - task = SimpleNamespace( - id="t_resume", - current_run_id=9, - assignee="cli-auto", - max_runtime_seconds=60, - ) - state_file = tmp_path / "state.json" - state_file.write_text( - json.dumps({"current_route": {"provider": "claude"}}), - encoding="utf-8", - ) - (tmp_path / "worker.log").write_text("prior provider evidence", encoding="utf-8") - prompts = [] - - class Connection: - def close(self): - return None - - fake_db = SimpleNamespace( - scoped_current_board=lambda _board: nullcontext(), - connect=lambda board: Connection(), - get_task=lambda _conn, _task_id: task, - worker_log_path=lambda _task_id, board: tmp_path / "worker.log", - _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_resume"), - set_branch_name=lambda *_args: None, - set_workspace_path=lambda *_args: None, - build_worker_context=lambda *_args: "resume objective", - add_comment=lambda *_args: None, - complete_task=lambda *_args, **_kwargs: None, - block_task=lambda *_args, **_kwargs: None, - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - monkeypatch.setattr(lanes, "state_path", lambda _board, _task_id: state_file) - monkeypatch.setattr( - lanes, - "select_route", - lambda *_args, **_kwargs: lanes.Route( - "codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, () - ), - ) - monkeypatch.setattr( - lanes, - "git_handoff", - lambda _workspace, output: f"HANDOFF:{output}", - ) - monkeypatch.setattr( - lanes, - "run_provider", - lambda _route, prompt, *_args, **_kwargs: ( - prompts.append(prompt) - or lanes.ProcessResult( - 0, - "", - { - "status": "completed", - "summary": "done", - "changed_files": [], - "tests_run": [], - "artifacts": [], - "blockers": [], - }, - False, - ) - ), - ) - - lanes.execute_claim("cassandra", "t_resume") - - assert "HANDOFF:prior provider evidence" in prompts[0] - - -def test_provider_commands_are_structured_unattended_and_capped(tmp_path: Path): - route = lanes.Route("codex", "gpt-5.6-sol", "xhigh", "codex-xhigh", "jetson", "vote", 1, ()) - command = lanes._codex_command(route, "Work.", tmp_path, {}, tmp_path / "result.json") - assert "--dangerously-bypass-approvals-and-sandbox" in command - assert "--json" in command - assert "--output-schema" in command - assert 'model_reasoning_effort="xhigh"' in command - - claude_state = {"claude_session_id": "13864642-2985-4f91-bef5-53f145f878e8"} - claude = lanes._claude_command( - lanes.Route("claude", "claude-opus-5", "xhigh", "claude-xhigh", "jetson", "vote", 1, ()), - "Review.", - claude_state, - False, - ) - assert "--dangerously-skip-permissions" in claude - assert "--output-format" in claude and "stream-json" in claude - assert "--json-schema" in claude - assert "--disallowedTools" in claude - assert "Bash(kubectl apply *)" not in claude - assert "Bash(flux reconcile *)" not in claude - assert "max" not in claude - - -def test_worker_contract_separates_review_findings_from_task_blockers(tmp_path: Path): - prompt = lanes.build_prompt("Review the change.", tmp_path) - - assert "put defects and risks in findings" in prompt - assert "blockers array must be empty whenever status is completed" in prompt - assert "findings" in lanes.RESULT_SCHEMA["properties"] - assert set(lanes.RESULT_SCHEMA["required"]) == set( - lanes.RESULT_SCHEMA["properties"] - ) - assert "assigned task itself" in lanes.RESULT_SCHEMA["properties"]["blockers"]["description"] - - -@pytest.mark.parametrize( - "command", - [ - "git push --force origin main", - "git reset --hard HEAD~1", - "git clean -fd", - ], -) -def test_claude_pretool_hook_blocks_hard_denies(command: str): - assert policy.denial_reason(command) - - -def test_claude_pretool_hook_allows_normal_engineering(): - assert policy.denial_reason("pytest -q testing/tests") is None - assert policy.denial_reason("git push origin feature/hermes") is None - assert policy.denial_reason("kubectl delete pod -n cassandra stuck-worker") is None - assert policy.denial_reason("flux reconcile kustomization hermes") is None - assert policy.denial_reason("vault kv get kv/atlas/hermes") is None - - -def test_claude_settings_preserve_state_and_install_three_guardrail_layers(tmp_path: Path): - state = tmp_path / ".claude.json" - settings = tmp_path / "settings.json" - state.write_text('{"promptQueueUseCount": 4}\n', encoding="utf-8") - settings.write_text( - json.dumps( - { - "theme": "dark", - "permissions": { - "deny": [ - "Bash(kubectl apply *)", - "Bash(flux reconcile *)", - "Bash(vault kv *)", - "Bash(custom-owner-rule *)", - ] - }, - } - ) - + "\n", - encoding="utf-8", - ) - - client_config.configure_claude_state(state) - client_config.configure_claude_settings(settings) - - state_value = json.loads(state.read_text()) - settings_value = json.loads(settings.read_text()) - assert state_value["promptQueueUseCount"] == 4 - assert state_value["bypassPermissionsModeAccepted"] is True - assert settings_value["theme"] == "dark" - assert "Bash(git reset --hard *)" in settings_value["permissions"]["deny"] - assert "Bash(custom-owner-rule *)" in settings_value["permissions"]["deny"] - assert "Bash(kubectl apply *)" not in settings_value["permissions"]["deny"] - assert "Bash(flux reconcile *)" not in settings_value["permissions"]["deny"] - assert "Bash(vault kv *)" not in settings_value["permissions"]["deny"] - hook = settings_value["hooks"]["PreToolUse"][0]["hooks"][0] - assert "claude_command_policy.py" in hook["command"] - - -def test_legacy_state_is_archived_without_removing_provider_transcripts(tmp_path: Path): - session = tmp_path / "home/.config/herdr/session.json" - session.parent.mkdir(parents=True) - session.write_text('{"agents":[{"agent":"claude","session_id":"abc"}]}', encoding="utf-8") - binary = tmp_path / "tools/bin/herdr" - binary.parent.mkdir(parents=True) - binary.write_text("legacy", encoding="utf-8") - (tmp_path / "home/.claude").mkdir() - (tmp_path / "home/.codex").mkdir() - - archive = migration.archive_legacy_state(tmp_path) - - value = json.loads(archive.read_text()) - assert value["legacy_session"]["agents"][0]["session_id"] == "abc" - assert (tmp_path / "home/.claude").is_dir() - assert (tmp_path / "home/.codex").is_dir() - assert not binary.exists() - assert not (tmp_path / "home/.config/herdr").exists() - - -def test_agent_uses_one_native_kanban_control_plane(): - configmap = yaml.safe_load((HERMES / "agent-configmap.yaml").read_text()) - config = yaml.safe_load(configmap["data"]["config.yaml"]) - assert config["model"] == { - "provider": "atlas-switchyard", - "default": "atlas/auto/balanced", - "model": "atlas/auto/balanced", - } - assert config["kanban"]["dispatch_in_gateway"] is True - assert config["kanban"]["default_assignee"] == "cli-auto" - assert config["plugins"]["enabled"] == ["auto-router"] - - deployment = _agent_deployment() - pod = deployment["spec"]["template"]["spec"] - assert pod["enableServiceLinks"] is False - names = {item["name"] for item in pod["containers"]} - assert "cli-lane-runner" in names - assert "terminal" in names - assert not any("herdr" in name for name in names) - rendered = (HERMES / "agent-deployment.yaml").read_text() - assert "herdr server" not in rendered - assert "herdr-dispatch" not in rendered - - -def test_cli_lane_reserves_cpu_headroom_for_ui_and_auth(): - deployment = _agent_deployment() - containers = { - item["name"]: item - for item in deployment["spec"]["template"]["spec"]["containers"] - } - lane = containers["cli-lane-runner"] - environment = {item["name"]: item["value"] for item in lane["env"]} - - assert environment["HERMES_CLI_LANE_CONCURRENCY"] == "2" - assert lane["resources"] == { - "requests": {"cpu": "100m", "memory": "256Mi"}, - "limits": {"cpu": "2", "memory": "6Gi"}, - } - - -def test_agent_avoids_unhealthy_nodes_and_fits_its_remaining_capacity(): - """Placement correction: keep the agent off nodes that cannot hold it. - - titan-04 is cordoned after repeated kernel undervoltage and kubelet - failure, and titan-19 was probe/Longhorn unstable under worker load, so - both must join the existing hard exclusions. That leaves titan-05 as the - healthy candidate, which is tight enough on requested CPU that the main - container has to give back 50m to schedule there. - """ - pod = _agent_deployment()["spec"]["template"]["spec"] - hostnames = next( - item - for item in pod["affinity"]["nodeAffinity"][ - "requiredDuringSchedulingIgnoredDuringExecution" - ]["nodeSelectorTerms"][0]["matchExpressions"] - if item["key"] == "kubernetes.io/hostname" - ) - - assert hostnames["operator"] == "NotIn" - assert set(hostnames["values"]) >= {"titan-04", "titan-19"} - - hermes = next( - item for item in pod["containers"] if item["name"] == "hermes" - ) - assert hermes["resources"]["requests"]["cpu"] == "300m" - - -def test_agent_root_is_stock_dashboard_and_terminal_is_a_separate_path(): - deployment = _agent_deployment() - pod = deployment["spec"]["template"]["spec"] - containers = {item["name"]: item for item in pod["containers"]} - assert "webui" not in containers - - assert "dashboard" not in containers - hermes = containers["hermes"] - hermes_env = {item["name"]: item["value"] for item in hermes["env"]} - assert hermes_env["HERMES_STREAM_STALE_TIMEOUT"] == "600" - assert hermes_env["HERMES_API_CALL_STALE_TIMEOUT"] == "600" - assert hermes["command"] == ["/bin/sh", "-ec"] - startup = hermes["args"][0] - assert ". /opt/data/.env" in startup - assert "exec /init /opt/hermes/docker/main-wrapper.sh gateway run" in startup - hermes_env = {item["name"]: item["value"] for item in hermes["env"]} - assert hermes_env["HERMES_DASHBOARD"] == "1" - assert hermes_env["HERMES_DASHBOARD_HOST"] == "127.0.0.1" - assert hermes_env["HERMES_DASHBOARD_PORT"] == "9119" - assert hermes_env["HERMES_TUI_AGENT_INIT_TIMEOUT_S"] == "180" - assert hermes["securityContext"]["runAsUser"] == 0 - assert hermes["securityContext"]["runAsGroup"] == 0 - for probe_name in ("startupProbe", "readinessProbe", "livenessProbe"): - probe = hermes[probe_name] - assert probe["exec"]["command"] == [ - "curl", - "-fsS", - "http://127.0.0.1:9119/api/status", - ] - - terminal = containers["terminal"] - command = terminal["args"][0] - assert "--base-path /terminal" in command - assert "--check-origin" not in command - assert "/usr/bin/tmux new-session -A" in command - assert "--continue" in command - assert "--yolo" in command - terminal_env = {item["name"]: item["value"] for item in terminal["env"]} - assert terminal_env["HERMES_TUI_AGENT_INIT_TIMEOUT_S"] == "180" - - claude_broker = containers["claude-broker"] - claude_env = {item["name"]: item["value"] for item in claude_broker["env"]} - assert claude_env["HERMES_CLAUDE_BROKER_CONCURRENCY"] == "2" - assert claude_broker["readinessProbe"]["tcpSocket"] == {"port": "claude-broker"} - assert claude_broker["livenessProbe"]["tcpSocket"] == {"port": "claude-broker"} - - args = containers["oauth2-proxy"]["args"] - terminal_upstream = "--upstream=http://127.0.0.1:7681/terminal/" - dashboard_upstream = "--upstream=http://127.0.0.1:9119/" - assert terminal_upstream in args - assert dashboard_upstream in args - assert args.index(terminal_upstream) < args.index(dashboard_upstream) - assert "--pass-host-header=false" in args - assert "--cookie-refresh=19m" in args - assert "--session-store-type=redis" in args - assert any( - arg.startswith("--redis-connection-url=redis://hermes-oauth-sessions.") - for arg in args - ) - - patch_init = next( - item for item in pod["initContainers"] - if item["name"] == "patch-tui-gateway" - ) - assert patch_init["command"][-1] == "/patched/server.py" - for name in ("hermes", "terminal"): - mounts = containers[name]["volumeMounts"] - assert { - "name": "tui-gateway-patch", - "mountPath": "/opt/hermes/tui_gateway/server.py", - "subPath": "server.py", - } in mounts - - ingress_documents = [ - item - for item in yaml.safe_load_all((HERMES / "agent-ingress.yaml").read_text()) - if item - ] - middlewares = { - item["metadata"]["name"]: item - for item in ingress_documents - if item["kind"] == "Middleware" - } - assert middlewares["hermes-agent-terminal-slash"]["spec"]["redirectRegex"][ - "replacement" - ].endswith("/terminal/") - assert middlewares["hermes-agent-stock-dashboard-headers"]["spec"]["headers"][ - "customRequestHeaders" - ]["Origin"] == "http://127.0.0.1:9119" - ingresses = { - item["metadata"]["name"]: item - for item in ingress_documents - if item["kind"] == "Ingress" - } - assert ingresses["hermes-agent-dashboard"]["metadata"]["annotations"][ - "traefik.ingress.kubernetes.io/router.middlewares" - ] == "hermes-hermes-agent-stock-dashboard-headers@kubernetescrd" - assert ingresses["hermes-agent-terminal"]["metadata"]["annotations"][ - "traefik.ingress.kubernetes.io/router.middlewares" - ] == "hermes-hermes-agent-terminal-slash@kubernetescrd" - - -def test_broker_services_survive_sibling_container_readiness_loss(): - services = _services() - - for name in ( - "hermes-image-broker", - "hermes-codex-broker", - "hermes-local-image", - "hermes-claude-broker", - ): - assert services[name]["spec"]["publishNotReadyAddresses"] is True - - -def test_agent_dashboard_reconnects_all_transient_websockets(): - dockerfile = ( - HERMES.parents[1] / "dockerfiles/Dockerfile.hermes-agent" - ).read_text(encoding="utf-8") - assert "eventsRetryAttempt.current" in dockerfile - assert "if (!unmounting) setVersion((v) => v + 1);" in dockerfile - assert "events feed rejected (${ev.code}) — reload the page" in dockerfile - assert 'url = await api.buildWsUrl("/api/pty", params);' in dockerfile - assert 'url = await buildWsUrl("/api/events", { channel });' in dockerfile - assert dockerfile.count("' await api.getSessions(1, 0,") == 2 - assert ".then(() => gw.connect())" in dockerfile - assert 'api.getSessions(1, 0, profile ?? "")' in dockerfile - assert "dashboard token rotated by a server restart" in dockerfile - - -def test_agent_refreshes_routes_after_restoring_cli_logins(): - deployment = _agent_deployment() - init_containers = { - item["name"]: item - for item in deployment["spec"]["template"]["spec"]["initContainers"] - } - configure = init_containers["configure-agent-clients"] - command = configure["command"][-1] - assert "configure_agent_clients.py" in command - assert command.index("configure_agent_clients.py") < command.index( - "hermes_coordinator.py --once" - ) - env = {item["name"]: item["value"] for item in configure["env"]} - assert env["HERMES_AUTH_FILE"] == "/runtime-access/hermes-auth.json" - assert env["PYTHONPATH"] == "/opt/hermes" - for name in ("bootstrap-coordinator", "configure-agent-clients"): - route_env = { - item["name"]: item["value"] - for item in init_containers[name]["env"] - } - assert route_env["CODEX_HOME"] == "/runtime-access/codex" - assert route_env["CLAUDE_CONFIG_DIR"] == "/runtime-access/claude" - assert "/opt/data/tools/bin" in route_env["PATH"] - - assert "patch-web-session-activity" in init_containers - web_patch = init_containers["patch-web-session-activity"] - assert "/opt/coordinator/patch_web_session_activity.py" in web_patch["command"] - - containers = { - item["name"]: item - for item in deployment["spec"]["template"]["spec"]["containers"] - } - steward_env = { - item["name"]: item["value"] - for item in containers["model-steward"]["env"] - } - assert steward_env["CODEX_HOME"] == "/runtime-access/codex" - assert steward_env["CLAUDE_CONFIG_DIR"] == "/runtime-access/claude" - assert "/opt/data/tools/bin" in steward_env["PATH"] - hermes_mounts = { - (item["name"], item["mountPath"], item.get("subPath")) - for item in containers["hermes"]["volumeMounts"] - } - assert ( - "web-server-patch", - "/opt/hermes/hermes_cli/web_server.py", - "web_server.py", - ) in hermes_mounts - - -def test_flux_health_checks_follow_the_owner_oauth_sidecar(): - flux = yaml.safe_load(FLUX_HERMES.read_text()) - checks = { - (item["kind"], item["name"]) - for item in flux["spec"]["healthChecks"] - } - assert ("Deployment", "hermes-agent") in checks - assert ("DaemonSet", "hermes-node-ssh-access") in checks - assert ("Deployment", "oauth2-proxy-hermes-agent") not in checks - - -def test_agent_auth_is_bstein_group_and_email_bounded(): - deployment = _agent_deployment() - oauth = next( - item for item in deployment["spec"]["template"]["spec"]["containers"] - if item["name"] == "oauth2-proxy" - ) - args = oauth["args"] - assert "--user-id-claim=sub" in args - assert "--oidc-groups-claim=groups" in args - assert "--allowed-group=/hermes-owner" in args - assert "--authenticated-emails-file=/etc/oauth2-proxy/allowed-emails" in args - script = (KEYCLOAK / "scripts/hermes_access_oidc_ensure.sh").read_text() - assert 'group_name="hermes-owner"' in script - assert "username=bstein&exact=true" in script - assert '"full.path":"true"' in script - - -def test_agent_network_boundary_allows_only_authenticated_and_metrics_surfaces(): - documents = [ - item - for item in yaml.safe_load_all((HERMES / "networkpolicy.yaml").read_text()) - if item - ] - isolation = next(item for item in documents if item.get("metadata", {}).get("name") == "hermes-agent-isolation") - assert isolation["spec"]["ingress"] == [ - { - "from": [ - { - "namespaceSelector": { - "matchLabels": { - "kubernetes.io/metadata.name": "traefik" - } - }, - "podSelector": { - "matchLabels": {"app.kubernetes.io/name": "traefik"} - }, - } - ], - "ports": [{"protocol": "TCP", "port": 4180}], - }, - { - "from": [ - { - "podSelector": { - "matchLabels": {"app": "hermes-chat-tenant"} - } - } - ], - "ports": [ - {"protocol": "TCP", "port": 9002}, - {"protocol": "TCP", "port": 9003}, - ], - }, - { - "from": [ - { - "podSelector": { - "matchLabels": {"app": "hermes-switchyard"} - } - } - ], - "ports": [ - {"protocol": "TCP", "port": 9003}, - {"protocol": "TCP", "port": 9006}, - ], - }, - { - "from": [ - { - "namespaceSelector": { - "matchLabels": { - "kubernetes.io/metadata.name": "monitoring" - } - }, - "podSelector": {"matchLabels": {"app": "server"}}, - } - ], - "ports": [{"protocol": "TCP", "port": 9010}], - }, - ] - assert isolation["spec"]["egress"] == [{}] - - -def test_owner_agent_has_cluster_admin_kubernetes_context(): - config = yaml.safe_load((HERMES / "agent-kubeconfig.yaml").read_text()) - assert config["current-context"] == "atlas-owner" - assert config["contexts"][0]["context"]["namespace"] == "default" - rbac_path = HERMES / "agent-rbac.yaml" - documents = [item for item in yaml.safe_load_all(rbac_path.read_text()) if item] - binding = next(item for item in documents if item["kind"] == "ClusterRoleBinding") - assert binding["roleRef"] == { - "apiGroup": "rbac.authorization.k8s.io", - "kind": "ClusterRole", - "name": "cluster-admin", - } - assert binding["subjects"] == [ - {"kind": "ServiceAccount", "name": "hermes-agent", "namespace": "hermes"} - ] - - -def test_owner_agent_has_pinned_dedicated_node_ssh_access(): - deployment = _agent_deployment() - annotations = deployment["spec"]["template"]["metadata"]["annotations"] - assert annotations[ - "vault.hashicorp.com/agent-inject-secret-node-ssh-private-key" - ] == "kv/data/atlas/hermes/developer-ssh" - assert annotations[ - "vault.hashicorp.com/agent-inject-secret-node-ssh-config" - ] == "kv/data/atlas/hermes/developer-ssh" - assert annotations[ - "vault.hashicorp.com/agent-inject-secret-node-ssh-known-hosts" - ] == "kv/data/atlas/hermes/developer-ssh" - - init = next( - item - for item in deployment["spec"]["template"]["spec"]["initContainers"] - if item["name"] == "init-config" - ) - command = init["command"][2] - assert "ln -s /runtime-access/node-ssh-config /opt/data/home/.ssh/config" in command - assert ( - "ln -s /runtime-access/node-ssh-known-hosts /opt/data/home/.ssh/known_hosts" - in command - ) - assert "ln -s home/.ssh /opt/data/.ssh" in command - assert "chmod 0700 /opt/data/home/.ssh" in command - assert ( - "ln -s /runtime-access/node-ssh-private-key " - "/opt/data/home/.ssh/id_ed25519_atlas_nodes" - ) in command - - config = yaml.safe_load((HERMES / "agent-configmap.yaml").read_text())["data"] - assert "ssh_config" not in config - assert "ssh_known_hosts" not in config - assert "ssh-ed25519" not in (HERMES / "agent-configmap.yaml").read_text() - - resources = yaml.safe_load((HERMES / "kustomization.yaml").read_text())[ - "resources" - ] - assert "node-ssh-access.yaml" in resources - access = [ - item - for item in yaml.safe_load_all((HERMES / "node-ssh-access.yaml").read_text()) - if item - ] - service_account = next(item for item in access if item["kind"] == "ServiceAccount") - assert service_account["metadata"]["name"] == "hermes-node-ssh-access" - provider = next(item for item in access if item["kind"] == "SecretProviderClass") - assert provider["metadata"]["name"] == "hermes-node-ssh-access" - assert provider["spec"]["provider"] == "vault" - parameters = provider["spec"]["parameters"] - assert parameters["roleName"] == "hermes-node-ssh" - assert 'secretPath: "kv/data/atlas/hermes/developer-ssh"' in parameters["objects"] - assert 'secretKey: "public_key"' in parameters["objects"] - daemonset = next(item for item in access if item["kind"] == "DaemonSet") - pod = daemonset["spec"]["template"]["spec"] - assert pod["serviceAccountName"] == "hermes-node-ssh-access" - assert pod["automountServiceAccountToken"] is True - host_home = next(item for item in pod["volumes"] if item["name"] == "host-home") - assert host_home["hostPath"] == {"path": "/home", "type": "Directory"} - vault_secrets = next( - item for item in pod["volumes"] if item["name"] == "vault-secrets" - ) - assert vault_secrets["csi"]["driver"] == "secrets-store.csi.k8s.io" - assert vault_secrets["csi"]["volumeAttributes"] == { - "secretProviderClass": "hermes-node-ssh-access" - } - reconciler = pod["containers"][0]["args"][0] - assert "cat /vault/secrets/node-ssh-public-key" in reconciler - assert "grep -qxF" in reconciler - assert "for user in atlas oceanus" in reconciler - assert "/host-etc/passwd" in reconciler - assert "chown \"${uid}:${gid}\"" in reconciler - host_passwd = next( - item for item in pod["volumes"] if item["name"] == "host-passwd" - ) - assert host_passwd["hostPath"] == {"path": "/etc/passwd", "type": "File"} - - -def test_owner_agent_tracks_no_ssh_identity_or_host_key_material(): - """Vault references may be tracked; SSH identities and trust data may not.""" - forbidden = ( - "BEGIN OPENSSH PRIVATE KEY", - "ssh-ed25519 AAAA", - "ssh-rsa AAAA", - "IdentityFile ", - "UserKnownHostsFile ", - "StrictHostKeyChecking ", - "ssh_config:", - "ssh_known_hosts:", - ) - text_suffixes = { - ".conf", - ".json", - ".md", - ".py", - ".sh", - ".toml", - ".yaml", - ".yml", - } - tracked = "\n".join( - path.read_text(encoding="utf-8") - for path in HERMES.rglob("*") - if path.is_file() and path.suffix in text_suffixes - ) - for marker in forbidden: - assert marker not in tracked - - -def test_switchyard_has_a_dedicated_non_owner_identity_and_read_only_catalog(): - """Routing must not inherit the owner agent's cluster-admin capability.""" - service_accounts = [ - item - for item in yaml.safe_load_all( - (HERMES / "vault-serviceaccount.yaml").read_text() - ) - if item - ] - assert any( - item["kind"] == "ServiceAccount" - and item["metadata"]["name"] == "hermes-switchyard" - for item in service_accounts - ) - - switchyard = yaml.safe_load( - (HERMES / "switchyard-deployment.yaml").read_text() - ) - switchyard_pod = switchyard["spec"]["template"]["spec"] - assert switchyard_pod["serviceAccountName"] == "hermes-switchyard" - agent_pod = _agent_deployment()["spec"]["template"]["spec"] - for pod, container_name in ( - (switchyard_pod, "worker-route-broker"), - (agent_pod, "claude-broker"), - ): - container = next(item for item in pod["containers"] if item["name"] == container_name) - catalog = next( - item - for item in container["volumeMounts"] - if item["mountPath"] == "/routing-catalog" - ) - assert catalog["readOnly"] is True - - rbac = [ - item - for item in yaml.safe_load_all((HERMES / "agent-rbac.yaml").read_text()) - if item - ] - binding = next(item for item in rbac if item["kind"] == "ClusterRoleBinding") - assert binding["subjects"] == [ - {"kind": "ServiceAccount", "name": "hermes-agent", "namespace": "hermes"} - ] - - -def test_switchyard_active_state_uses_a_relocatable_rwx_claim(): - """A stale node attachment must not strand the routing authority.""" - claims = [ - item - for item in yaml.safe_load_all((HERMES / "switchyard-pvc.yaml").read_text()) - if item - ] - active_claim = next( - item - for item in claims - if item["metadata"]["name"] == "hermes-switchyard-state-rwx" - ) - assert active_claim["spec"]["accessModes"] == ["ReadWriteMany"] - - deployment = yaml.safe_load((HERMES / "switchyard-deployment.yaml").read_text()) - strategy = deployment["spec"]["strategy"] - assert strategy == { - "type": "RollingUpdate", - "rollingUpdate": {"maxSurge": 1, "maxUnavailable": 0}, - } - pod = deployment["spec"]["template"]["spec"] - state = next(item for item in pod["volumes"] if item["name"] == "state") - assert state["persistentVolumeClaim"]["claimName"] == active_claim["metadata"][ - "name" - ] - - -def test_worker_route_broker_accepts_pod_network_health_checks(): - """Kubelet probes the pod IP, so the broker cannot bind to loopback only.""" - script = (SCRIPTS / "worker_route_broker.py").read_text() - assert 'ThreadingHTTPServer(("0.0.0.0", PORT), Handler)' in script - - -def test_switchyard_network_boundary_allows_vault_bootstrap(): - """The pre-populate init container must reach Vault before routing starts.""" - documents = [ - item - for item in yaml.safe_load_all((HERMES / "networkpolicy.yaml").read_text()) - if item - ] - isolation = next( - item - for item in documents - if item.get("metadata", {}).get("name") == "hermes-switchyard-isolation" - ) - assert any( - rule.get("to") - == [ - { - "namespaceSelector": { - "matchLabels": {"kubernetes.io/metadata.name": "vault"} - }, - "podSelector": {"matchLabels": {"app": "vault"}}, - } - ] - and rule.get("ports") == [{"protocol": "TCP", "port": 8200}] - for rule in isolation["spec"]["egress"] - ) - - -def test_switchyard_network_boundary_allows_metrics_scraping(): - """VictoriaMetrics may scrape Switchyard without widening its API boundary.""" - documents = [ - item - for item in yaml.safe_load_all((HERMES / "networkpolicy.yaml").read_text()) - if item - ] - isolation = next( - item - for item in documents - if item.get("metadata", {}).get("name") == "hermes-switchyard-isolation" - ) - assert any( - rule.get("from") - == [ - { - "namespaceSelector": { - "matchLabels": { - "kubernetes.io/metadata.name": "monitoring" - } - }, - "podSelector": {"matchLabels": {"app": "server"}}, - } - ] - and rule.get("ports") == [{"protocol": "TCP", "port": 9005}] - for rule in isolation["spec"]["ingress"] - ) - - -def test_owner_agent_installs_the_pinned_operator_toolchain(): - script = (SCRIPTS / "install_agent_tools.sh").read_text() - for value in [ - "flux", - "helm", - "kustomize", - "jq", - "yq", - "gh", - "vault", - "sops", - "age", - "age-keygen", - "k9s", - "terraform", - "go", - "gofmt", - ]: - assert value in script - assert "go1.26.5.linux-arm64.tar.gz" in script - assert ( - "fe4789e92b1f33358680864bbe8704289e7bb5fc207d80623c308935bd696d49" - in script - ) - assert script.count("sha256sum -c -") == 1 - - deployment = _agent_deployment() - installer = next( - item - for item in deployment["spec"]["template"]["spec"]["initContainers"] - if item["name"] == "install-agent-tools" - ) - assert "/bin/sh /opt/coordinator/install_agent_tools.sh" in installer["command"][2] - assert any(mount["name"] == "coordinator" for mount in installer["volumeMounts"]) - init_config = next( - item - for item in deployment["spec"]["template"]["spec"]["initContainers"] - if item["name"] == "init-config" - ) - init_command = init_config["command"][2] - assert "# Hermes managed operator PATH." in init_command - assert "/opt/coordinator:/opt/data/tools/bin:/opt/hermes/.venv/bin" in init_command - assert 'chmod 0644 "${profile_file}"' in init_command - - -def test_owner_agent_uses_only_the_canonical_hostname(): - paths = [ - HERMES / "agent-configmap.yaml", - HERMES / "agent-deployment.yaml", - HERMES / "agent-ingress.yaml", - Path(__file__).parents[2] / "scripts/ops/hermes_triage_monitor.py", - ] - for path in paths: - content = path.read_text() - assert "agent.bstein.dev" not in content - assert "agent.hermes.bstein.dev" in content - - -def test_agent_reconnect_retains_complete_history_and_long_tool_budget(): - configmap = yaml.safe_load((HERMES / "agent-configmap.yaml").read_text()) - config = yaml.safe_load(configmap["data"]["config.yaml"]) - display = config["display"] - assert display["resume_exchanges"] >= 10000 - assert display["resume_max_user_chars"] >= 10000000 - assert display["resume_max_assistant_chars"] >= 10000000 - assert config["agent"]["max_turns"] == 180 - assert config["delegation"]["max_iterations"] == 120 - - -def test_auth_patch_honors_explicit_shared_store(tmp_path: Path): - source = tmp_path / "auth.py" - destination = tmp_path / "patched/auth.py" - source.write_text( - 'from pathlib import Path\nimport os\n\ndef _auth_file_path() -> Path:\n path = get_hermes_home() / "auth.json"\n return path\n', - encoding="utf-8", - ) - auth_patch.patch(source, destination) - content = destination.read_text() - assert 'os.environ.get("HERMES_AUTH_FILE"' in content - - -def test_auth_patch_fails_closed_on_upstream_drift(tmp_path: Path): - source = tmp_path / "auth.py" - source.write_text("def changed():\n pass\n", encoding="utf-8") - with pytest.raises(RuntimeError, match="context changed"): - auth_patch.patch(source, tmp_path / "patched.py") - - -def test_codex_runtime_patch_uses_cli_and_forwards_route(tmp_path: Path): - provider = tmp_path / "runtime_provider.py" - provider.write_text(codex_runtime_patch.PROVIDER_BEFORE, encoding="utf-8") - provider_out = tmp_path / "patched/runtime_provider.py" - codex_runtime_patch.patch_provider(provider, provider_out) - assert '"api_mode": "codex_app_server"' in provider_out.read_text() - - session = tmp_path / "codex_app_server_session.py" - session.write_text( - codex_runtime_patch.SESSION_SIGNATURE_BEFORE - + codex_runtime_patch.SESSION_REQUEST_BEFORE, - encoding="utf-8", - ) - session_out = tmp_path / "patched/codex_app_server_session.py" - codex_runtime_patch.patch_session(session, session_out) - session_content = session_out.read_text() - assert 'turn_params["model"] = model' in session_content - assert 'turn_params["effort"] = effort' in session_content - assert '"approvalPolicy": "never"' in session_content - assert '"sandboxPolicy": {"type": "dangerFullAccess"}' in session_content - - turn = tmp_path / "codex_runtime.py" - turn.write_text( - codex_runtime_patch.FALLBACK_CONTEXT_BEFORE - + codex_runtime_patch.TURN_BEFORE, - encoding="utf-8", - ) - turn_out = tmp_path / "patched/codex_runtime.py" - codex_runtime_patch.patch_turn(turn, turn_out) - turn_content = turn_out.read_text() - assert "model=str(getattr(agent" in turn_content - assert "build_cross_provider_codex_prompt" in turn_content - - fallback = tmp_path / "chat_completion_helpers.py" - fallback.write_text( - codex_runtime_patch.FALLBACK_RESOLUTION_BEFORE, - encoding="utf-8", - ) - fallback_out = tmp_path / "patched/chat_completion_helpers.py" - codex_runtime_patch.patch_fallback(fallback, fallback_out) - fallback_content = fallback_out.read_text() - assert 'agent.api_mode = "codex_app_server"' in fallback_content - assert "agent._codex_cross_provider_fallback = True" in fallback_content - - loop = tmp_path / "conversation_loop.py" - loop.write_text( - codex_runtime_patch.FALLBACK_DISPATCH_BEFORE - + codex_runtime_patch.RETRY_FALLBACK_DISPATCH_BEFORE - + codex_runtime_patch.STREAM_RECOVERY_BEFORE, - encoding="utf-8", - ) - loop_out = tmp_path / "patched/conversation_loop.py" - codex_runtime_patch.patch_loop(loop, loop_out) - loop_content = loop_out.read_text() - assert loop_content.count('if agent.api_mode == "codex_app_server"') == 2 - assert "build_cross_provider_codex_prompt" in loop_content - retry_dispatch = loop_content.index( - "Fallback activation happens inside this retry loop" - ) - api_kwargs = loop_content.find("agent._build_api_kwargs", retry_dispatch) - assert api_kwargs == -1 or retry_dispatch < api_kwargs - assert "Provider stream ended before a complete response" in loop_content - assert "_is_transport_stub" in loop_content - assert "rerouting ({truncated_tool_call_retries}/4)" in loop_content - - auxiliary = tmp_path / "auxiliary_client.py" - auxiliary.write_text( - codex_runtime_patch.AUXILIARY_TOKEN_BEFORE, - encoding="utf-8", - ) - auxiliary_out = tmp_path / "patched/auxiliary_client.py" - codex_runtime_patch.patch_auxiliary(auxiliary, auxiliary_out) - auxiliary_content = auxiliary_out.read_text() - assert 'os.environ.get("CODEX_HOME"' in auxiliary_content - assert 'Path(codex_home).expanduser() / "auth.json"' in auxiliary_content - assert "never creates a metered API-key lane" in auxiliary_content - - -def test_agent_mounts_codex_auxiliary_runtime_patch(): - deployment = _agent_deployment() - pod = deployment["spec"]["template"]["spec"] - patch_init = next( - item for item in pod["initContainers"] - if item["name"] == "patch-codex-runtime" - ) - assert patch_init["command"][-2:] == [ - "/opt/hermes/agent/auxiliary_client.py", - "/patched/auxiliary_client.py", - ] - expected_mount = { - "name": "codex-runtime-patch", - "mountPath": "/opt/hermes/agent/auxiliary_client.py", - "subPath": "auxiliary_client.py", - } - containers = {item["name"]: item for item in pod["containers"]} - for name in ("hermes", "terminal"): - assert expected_mount in containers[name]["volumeMounts"] - - -def test_codex_auxiliary_patch_reads_cli_token_without_copying_it( - tmp_path: Path, - monkeypatch, -): - codex_home = tmp_path / ".codex" - codex_home.mkdir() - (codex_home / "auth.json").write_text( - json.dumps({"tokens": {"access_token": "cli-access-token"}}), - encoding="utf-8", - ) - monkeypatch.setenv("CODEX_HOME", str(codex_home)) - - source = tmp_path / "auxiliary_client.py" - source.write_text( - "import json, logging, os, time\n" - "from pathlib import Path\n" - "logger = logging.getLogger(__name__)\n" - "def read_token():\n" - " try:\n" - " raise RuntimeError('Hermes provider store intentionally empty')\n" - + codex_runtime_patch.AUXILIARY_TOKEN_BEFORE, - encoding="utf-8", - ) - destination = tmp_path / "patched/auxiliary_client.py" - codex_runtime_patch.patch_auxiliary(source, destination) - namespace: dict = {} - exec(compile(destination.read_text(), str(destination), "exec"), namespace) - - assert namespace["read_token"]() == "cli-access-token" - - -def test_codex_runtime_patch_fails_closed_on_upstream_drift(tmp_path: Path): - source = tmp_path / "runtime_provider.py" - source.write_text("def changed():\n pass\n", encoding="utf-8") - with pytest.raises(RuntimeError, match="context changed"): - codex_runtime_patch.patch_provider(source, tmp_path / "patched.py") - - -def test_codex_runtime_migration_uses_owner_unsafe_mode(tmp_path: Path): - config = tmp_path / "config.yaml" - config.write_text("model: {}\n", encoding="utf-8") - codex_home = tmp_path / ".codex" - calls = [] - - class Report: - errors = [] - - @staticmethod - def summary(): - return "configured" - - def migrate(value, **kwargs): - calls.append((value, kwargs)) - return Report() - - client_config.configure_codex_runtime(config, migrate, codex_home) - - assert calls[0][1]["default_permission_profile"] is None - assert calls[0][1]["codex_home"] == codex_home - content = (codex_home / "config.toml").read_text(encoding="utf-8") - assert 'approval_policy = "never"' in content - assert 'sandbox_mode = "danger-full-access"' in content - assert "default_permissions" not in content - - -def test_codex_owner_permissions_replace_stale_profile(tmp_path: Path): - config = tmp_path / "config.toml" - config.write_text( - 'default_permissions = ":danger-no-sandbox"\n\n[features]\nhooks = true\n', - encoding="utf-8", - ) - - client_config.configure_codex_owner_permissions(config) - client_config.configure_codex_owner_permissions(config) - - content = config.read_text(encoding="utf-8") - assert content.count(client_config.OWNER_PERMISSIONS_BEGIN) == 1 - assert content.count('approval_policy = "never"') == 1 - assert content.count('sandbox_mode = "danger-full-access"') == 1 - assert "default_permissions" not in content - assert "[features]\nhooks = true" in content - - -def test_tui_gateway_patch_extends_and_bounds_agent_startup(tmp_path: Path): - source = tmp_path / "server.py" - destination = tmp_path / "patched/server.py" - source.write_text( - "import os\n\n" + tui_gateway_patch.BEFORE + "\ndef unchanged():\n pass\n", - encoding="utf-8", - ) - - tui_gateway_patch.patch(source, destination) - - content = destination.read_text(encoding="utf-8") - assert "HERMES_TUI_AGENT_INIT_TIMEOUT_S" in content - assert 'configured = 180.0' in content - assert "return max(30.0, min(configured, 900.0))" in content - assert "timeout: float | None = None" in content - assert "ready.wait(timeout=wait_timeout)" in content - assert "def unchanged():" in content - - -def test_tui_gateway_patch_fails_closed_on_upstream_drift(tmp_path: Path): - source = tmp_path / "server.py" - source.write_text("def changed():\n pass\n", encoding="utf-8") - with pytest.raises(RuntimeError, match="context changed"): - tui_gateway_patch.patch(source, tmp_path / "patched.py") - - -def test_ttyd_clipboard_and_reconnect_patch_remain_enabled(): - source = '' - content = ttyd_patch.patch_html(source) - assert 'id="atlas-ttyd-clipboard"' in content - assert "navigator.clipboard.writeText(text)" in content - assert "class AtlasRecoveringWebSocket" in content - assert "window.location.reload()" in content - assert "event.stopImmediatePropagation()" in content - - -def test_ttyd_patch_fails_closed_on_upstream_drift(): - with pytest.raises(RuntimeError, match="context changed"): - ttyd_patch.patch_html("changed") diff --git a/testing/tests/test_hermes_cli_lanes_access.py b/testing/tests/test_hermes_cli_lanes_access.py new file mode 100644 index 00000000..01d9f4bf --- /dev/null +++ b/testing/tests/test_hermes_cli_lanes_access.py @@ -0,0 +1,378 @@ +"""Network, OAuth, Kubernetes, and SSH access contracts for Hermes CLI lanes.""" + +from __future__ import annotations + + +import yaml + +from testing.tests.test_hermes_cli_lanes_support import ( + FLUX_HERMES, + HERMES, + KEYCLOAK, + ROOT, + _agent_deployment, + _services, +) + + +def test_broker_services_survive_sibling_container_readiness_loss(): + services = _services() + + for name in ( + "hermes-image-broker", + "hermes-codex-broker", + "hermes-local-image", + "hermes-claude-broker", + ): + assert services[name]["spec"]["publishNotReadyAddresses"] is True + + +def test_agent_dashboard_reconnects_all_transient_websockets(): + dockerfile = (HERMES.parents[1] / "dockerfiles/Dockerfile.hermes-agent").read_text( + encoding="utf-8" + ) + assert "eventsRetryAttempt.current" in dockerfile + assert "if (!unmounting) setVersion((v) => v + 1);" in dockerfile + assert "events feed rejected (${ev.code}) — reload the page" in dockerfile + assert 'url = await api.buildWsUrl("/api/pty", params);' in dockerfile + assert 'url = await buildWsUrl("/api/events", { channel });' in dockerfile + assert dockerfile.count("' await api.getSessions(1, 0,") == 2 + assert ".then(() => gw.connect())" in dockerfile + assert 'api.getSessions(1, 0, profile ?? "")' in dockerfile + assert "dashboard token rotated by a server restart" in dockerfile + + +def test_agent_refreshes_routes_after_restoring_cli_logins(): + deployment = _agent_deployment() + init_containers = { + item["name"]: item + for item in deployment["spec"]["template"]["spec"]["initContainers"] + } + configure = init_containers["configure-agent-clients"] + command = configure["command"][-1] + assert "configure_agent_clients.py" in command + assert command.index("configure_agent_clients.py") < command.index( + "hermes_coordinator.py --once" + ) + env = {item["name"]: item["value"] for item in configure["env"]} + assert env["HERMES_AUTH_FILE"] == "/runtime-access/hermes-auth.json" + assert env["PYTHONPATH"] == "/opt/hermes" + for name in ("bootstrap-coordinator", "configure-agent-clients"): + route_env = { + item["name"]: item["value"] for item in init_containers[name]["env"] + } + assert route_env["CODEX_HOME"] == "/runtime-access/codex" + assert route_env["CLAUDE_CONFIG_DIR"] == "/runtime-access/claude" + assert "/opt/data/tools/bin" in route_env["PATH"] + + assert "patch-web-session-activity" in init_containers + web_patch = init_containers["patch-web-session-activity"] + assert "/opt/coordinator/patch_web_session_activity.py" in web_patch["command"] + + containers = { + item["name"]: item + for item in deployment["spec"]["template"]["spec"]["containers"] + } + steward_env = { + item["name"]: item["value"] for item in containers["model-steward"]["env"] + } + assert steward_env["CODEX_HOME"] == "/runtime-access/codex" + assert steward_env["CLAUDE_CONFIG_DIR"] == "/runtime-access/claude" + assert "/opt/data/tools/bin" in steward_env["PATH"] + hermes_mounts = { + (item["name"], item["mountPath"], item.get("subPath")) + for item in containers["hermes"]["volumeMounts"] + } + assert ( + "web-server-patch", + "/opt/hermes/hermes_cli/web_server.py", + "web_server.py", + ) in hermes_mounts + + +def test_flux_health_checks_follow_the_owner_oauth_sidecar(): + flux = yaml.safe_load(FLUX_HERMES.read_text()) + checks = {(item["kind"], item["name"]) for item in flux["spec"]["healthChecks"]} + assert ("Deployment", "hermes-agent") in checks + assert ("DaemonSet", "hermes-node-ssh-access") in checks + assert ("Deployment", "oauth2-proxy-hermes-agent") not in checks + + +def test_agent_auth_is_bstein_group_and_email_bounded(): + deployment = _agent_deployment() + oauth = next( + item + for item in deployment["spec"]["template"]["spec"]["containers"] + if item["name"] == "oauth2-proxy" + ) + args = oauth["args"] + assert "--user-id-claim=sub" in args + assert "--oidc-groups-claim=groups" in args + assert "--allowed-group=/hermes-owner" in args + assert "--authenticated-emails-file=/etc/oauth2-proxy/allowed-emails" in args + script = (KEYCLOAK / "scripts/hermes_access_oidc_ensure.sh").read_text() + assert 'group_name="hermes-owner"' in script + assert "username=bstein&exact=true" in script + assert '"full.path":"true"' in script + + +def test_agent_network_boundary_allows_only_authenticated_and_metrics_surfaces(): + documents = [ + item + for item in yaml.safe_load_all((HERMES / "networkpolicy.yaml").read_text()) + if item + ] + isolation = next( + item + for item in documents + if item.get("metadata", {}).get("name") == "hermes-agent-isolation" + ) + assert isolation["spec"]["ingress"] == [ + { + "from": [ + { + "namespaceSelector": { + "matchLabels": {"kubernetes.io/metadata.name": "traefik"} + }, + "podSelector": { + "matchLabels": {"app.kubernetes.io/name": "traefik"} + }, + } + ], + "ports": [{"protocol": "TCP", "port": 4180}], + }, + { + "from": [{"podSelector": {"matchLabels": {"app": "hermes-chat-tenant"}}}], + "ports": [ + {"protocol": "TCP", "port": 9002}, + {"protocol": "TCP", "port": 9003}, + ], + }, + { + "from": [{"podSelector": {"matchLabels": {"app": "hermes-switchyard"}}}], + "ports": [ + {"protocol": "TCP", "port": 9003}, + {"protocol": "TCP", "port": 9006}, + ], + }, + { + "from": [ + { + "namespaceSelector": { + "matchLabels": {"kubernetes.io/metadata.name": "monitoring"} + }, + "podSelector": {"matchLabels": {"app": "server"}}, + } + ], + "ports": [{"protocol": "TCP", "port": 9010}], + }, + ] + egress = isolation["spec"]["egress"] + assert any( + rule.get("ports") == [{"protocol": "TCP", "port": 9081}] for rule in egress + ) + assert "hermes-scm" in yaml.safe_dump(egress) + assert "gitea" in yaml.safe_dump(egress) + assert egress != [{}] + + +def test_owner_agent_has_scoped_read_only_kubernetes_context(): + config = yaml.safe_load((HERMES / "agent-kubeconfig.yaml").read_text()) + assert config["current-context"] == "atlas-observer" + assert config["contexts"][0]["context"]["namespace"] == "default" + rbac_path = ROOT / "services/hermes-observer-rbac/rbac.yaml" + documents = [item for item in yaml.safe_load_all(rbac_path.read_text()) if item] + binding = next(item for item in documents if item["kind"] == "ClusterRoleBinding") + assert binding["roleRef"] == { + "apiGroup": "rbac.authorization.k8s.io", + "kind": "ClusterRole", + "name": "hermes-agent-cluster-observer-v2", + } + assert binding["subjects"] == [ + {"kind": "ServiceAccount", "name": "hermes-agent", "namespace": "hermes"} + ] + + +def test_owner_agent_has_pinned_dedicated_node_ssh_access(): + deployment = _agent_deployment() + annotations = deployment["spec"]["template"]["metadata"]["annotations"] + assert ( + annotations["vault.hashicorp.com/agent-inject-secret-node-ssh-private-key"] + == "kv/data/atlas/hermes/developer-ssh" + ) + assert ( + annotations["vault.hashicorp.com/agent-inject-secret-node-ssh-config"] + == "kv/data/atlas/hermes/developer-ssh" + ) + assert ( + annotations["vault.hashicorp.com/agent-inject-secret-node-ssh-known-hosts"] + == "kv/data/atlas/hermes/developer-ssh" + ) + + init = next( + item + for item in deployment["spec"]["template"]["spec"]["initContainers"] + if item["name"] == "init-config" + ) + command = init["command"][2] + assert "ln -s /runtime-access/node-ssh-config /opt/data/home/.ssh/config" in command + assert ( + "ln -s /runtime-access/node-ssh-known-hosts /opt/data/home/.ssh/known_hosts" + in command + ) + assert "ln -s home/.ssh /opt/data/.ssh" in command + assert "chmod 0700 /opt/data/home/.ssh" in command + assert ( + "ln -s /runtime-access/node-ssh-private-key " + "/opt/data/home/.ssh/id_ed25519_atlas_nodes" + ) in command + + config = yaml.safe_load((HERMES / "agent-configmap.yaml").read_text())["data"] + assert "ssh_config" not in config + assert "ssh_known_hosts" not in config + assert "ssh-ed25519" not in (HERMES / "agent-configmap.yaml").read_text() + + resources = yaml.safe_load((HERMES / "kustomization.yaml").read_text())["resources"] + assert "node-ssh-access.yaml" in resources + access = [ + item + for item in yaml.safe_load_all((HERMES / "node-ssh-access.yaml").read_text()) + if item + ] + service_account = next(item for item in access if item["kind"] == "ServiceAccount") + assert service_account["metadata"]["name"] == "hermes-node-ssh-access" + provider = next(item for item in access if item["kind"] == "SecretProviderClass") + assert provider["metadata"]["name"] == "hermes-node-ssh-access" + assert provider["spec"]["provider"] == "vault" + parameters = provider["spec"]["parameters"] + assert parameters["roleName"] == "hermes-node-ssh" + assert 'secretPath: "kv/data/atlas/hermes/developer-ssh"' in parameters["objects"] + assert 'secretKey: "public_key"' in parameters["objects"] + daemonset = next(item for item in access if item["kind"] == "DaemonSet") + pod = daemonset["spec"]["template"]["spec"] + assert pod["serviceAccountName"] == "hermes-node-ssh-access" + assert pod["automountServiceAccountToken"] is True + host_home = next(item for item in pod["volumes"] if item["name"] == "host-home") + assert host_home["hostPath"] == {"path": "/home", "type": "Directory"} + vault_secrets = next( + item for item in pod["volumes"] if item["name"] == "vault-secrets" + ) + assert vault_secrets["csi"]["driver"] == "secrets-store.csi.k8s.io" + assert vault_secrets["csi"]["volumeAttributes"] == { + "secretProviderClass": "hermes-node-ssh-access" + } + reconciler = pod["containers"][0]["args"][0] + assert "/opt/node-hardener/node_account_hardening.py" in reconciler + assert "--public-key-file /vault/secrets/node-ssh-public-key" in reconciler + assert "sleep 300" in reconciler + host_etc = next(item for item in pod["volumes"] if item["name"] == "host-etc") + assert host_etc["hostPath"] == {"path": "/etc", "type": "Directory"} + hardener = (HERMES / "scripts/node_account_hardening.py").read_text() + assert 'ACCOUNT = "hermes-agent"' in hardener + assert 'LEGACY_ACCOUNTS = ("atlas", "oceanus")' in hardener + assert "ACCOUNT_UID = 1200" in hardener + assert "ACCOUNT_GID = 1200" in hardener + + +def test_owner_agent_tracks_no_ssh_identity_or_host_key_material(): + """Vault references may be tracked; SSH identities and trust data may not.""" + forbidden = ( + "BEGIN OPENSSH PRIVATE KEY", + "ssh-ed25519 AAAA", + "ssh-rsa AAAA", + "IdentityFile ", + "UserKnownHostsFile ", + "StrictHostKeyChecking ", + "ssh_config:", + "ssh_known_hosts:", + ) + text_suffixes = { + ".conf", + ".json", + ".md", + ".py", + ".sh", + ".toml", + ".yaml", + ".yml", + } + tracked = "\n".join( + path.read_text(encoding="utf-8") + for path in HERMES.rglob("*") + if path.is_file() and path.suffix in text_suffixes + ) + for marker in forbidden: + assert marker not in tracked + + +def test_switchyard_has_a_dedicated_non_owner_identity_and_read_only_catalog(): + """Routing must not inherit the owner agent's cluster-admin capability.""" + service_accounts = [ + item + for item in yaml.safe_load_all( + (HERMES / "vault-serviceaccount.yaml").read_text() + ) + if item + ] + assert any( + item["kind"] == "ServiceAccount" + and item["metadata"]["name"] == "hermes-switchyard" + for item in service_accounts + ) + + switchyard = yaml.safe_load((HERMES / "switchyard-deployment.yaml").read_text()) + switchyard_pod = switchyard["spec"]["template"]["spec"] + assert switchyard_pod["serviceAccountName"] == "hermes-switchyard" + agent_pod = _agent_deployment()["spec"]["template"]["spec"] + for pod, container_name in ( + (switchyard_pod, "worker-route-broker"), + (agent_pod, "claude-broker"), + ): + container = next( + item for item in pod["containers"] if item["name"] == container_name + ) + catalog = next( + item + for item in container["volumeMounts"] + if item["mountPath"] == "/routing-catalog" + ) + assert catalog["readOnly"] is True + + rbac = [ + item + for item in yaml.safe_load_all( + (ROOT / "services/hermes-observer-rbac/rbac.yaml").read_text() + ) + if item + ] + binding = next(item for item in rbac if item["kind"] == "ClusterRoleBinding") + assert binding["subjects"] == [ + {"kind": "ServiceAccount", "name": "hermes-agent", "namespace": "hermes"} + ] + + +def test_switchyard_active_state_uses_a_relocatable_rwx_claim(): + """A stale node attachment must not strand the routing authority.""" + claims = [ + item + for item in yaml.safe_load_all((HERMES / "switchyard-pvc.yaml").read_text()) + if item + ] + active_claim = next( + item + for item in claims + if item["metadata"]["name"] == "hermes-switchyard-state-rwx" + ) + assert active_claim["spec"]["accessModes"] == ["ReadWriteMany"] + + deployment = yaml.safe_load((HERMES / "switchyard-deployment.yaml").read_text()) + strategy = deployment["spec"]["strategy"] + assert strategy == { + "type": "RollingUpdate", + "rollingUpdate": {"maxSurge": 1, "maxUnavailable": 0}, + } + pod = deployment["spec"]["template"]["spec"] + state = next(item for item in pod["volumes"] if item["name"] == "state") + assert ( + state["persistentVolumeClaim"]["claimName"] == active_claim["metadata"]["name"] + ) diff --git a/testing/tests/test_hermes_cli_lanes_configuration.py b/testing/tests/test_hermes_cli_lanes_configuration.py new file mode 100644 index 00000000..bf1807f4 --- /dev/null +++ b/testing/tests/test_hermes_cli_lanes_configuration.py @@ -0,0 +1,444 @@ +"""Agent configuration and dashboard contracts for Hermes CLI lanes.""" + +from __future__ import annotations + +import json +import sys +from contextlib import nullcontext +from pathlib import Path +from types import SimpleNamespace + +import pytest +import yaml + +from testing.tests.test_hermes_cli_lanes_support import ( + HERMES, + _agent_deployment, + client_config, + lanes, + migration, + policy, +) + + +def test_workspace_preparation_failure_durably_blocks_the_claim( + tmp_path: Path, monkeypatch +): + task = SimpleNamespace(id="t_bad_worktree", current_run_id=7, assignee="cli-auto") + calls = [] + + class Connection: + def close(self): + return None + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + get_task=lambda _conn, _task_id: task, + worker_log_path=lambda _task_id, board: tmp_path / "worker.log", + _resolve_worktree_workspace=lambda _task, board: (_ for _ in ()).throw( + ValueError("not a Git repository") + ), + block_task=lambda *_args, **kwargs: calls.append(kwargs), + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + monkeypatch.setattr( + lanes, "state_path", lambda _board, _task_id: tmp_path / "state.json" + ) + + lanes.execute_claim("cassandra", "t_bad_worktree") + + assert calls[0]["kind"] == "capability" + assert calls[0]["expected_run_id"] == 7 + assert "not a Git repository" in calls[0]["reason"] + + +def test_artifacts_cannot_escape_the_task_worktree(tmp_path: Path): + workspace = tmp_path / "workspace" + workspace.mkdir() + inside = workspace / "report.json" + outside = tmp_path / "auth.json" + inside.write_text("{}\n", encoding="utf-8") + outside.write_text("secret\n", encoding="utf-8") + + assert lanes.workspace_artifacts( + workspace, + ["report.json", str(outside), "missing.json"], + ) == [str(inside)] + + +def test_restart_provider_change_includes_explicit_workspace_handoff( + tmp_path: Path, monkeypatch +): + task = SimpleNamespace( + id="t_resume", + current_run_id=9, + assignee="cli-auto", + max_runtime_seconds=60, + ) + state_file = tmp_path / "state.json" + state_file.write_text( + json.dumps({"current_route": {"provider": "claude"}}), + encoding="utf-8", + ) + (tmp_path / "worker.log").write_text("prior provider evidence", encoding="utf-8") + prompts = [] + + class Connection: + def close(self): + return None + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + get_task=lambda _conn, _task_id: task, + worker_log_path=lambda _task_id, board: tmp_path / "worker.log", + _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_resume"), + set_branch_name=lambda *_args: None, + set_workspace_path=lambda *_args: None, + build_worker_context=lambda *_args: "resume objective", + add_comment=lambda *_args: None, + complete_task=lambda *_args, **_kwargs: None, + block_task=lambda *_args, **_kwargs: None, + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + monkeypatch.setattr(lanes, "state_path", lambda _board, _task_id: state_file) + monkeypatch.setattr( + lanes, + "select_route", + lambda *_args, **_kwargs: lanes.Route( + "codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, () + ), + ) + monkeypatch.setattr( + lanes, + "git_handoff", + lambda _workspace, output: f"HANDOFF:{output}", + ) + monkeypatch.setattr( + lanes, + "run_provider", + lambda _route, prompt, *_args, **_kwargs: ( + prompts.append(prompt) + or lanes.ProcessResult( + 0, + "", + { + "status": "completed", + "summary": "done", + "changed_files": [], + "tests_run": [], + "artifacts": [], + "blockers": [], + }, + False, + ) + ), + ) + + lanes.execute_claim("cassandra", "t_resume") + + assert "HANDOFF:prior provider evidence" in prompts[0] + + +def test_provider_commands_are_structured_unattended_and_capped(tmp_path: Path): + route = lanes.Route( + "codex", "gpt-5.6-sol", "xhigh", "codex-xhigh", "jetson", "vote", 1, () + ) + command = lanes._codex_command( + route, "Work.", tmp_path, {}, tmp_path / "result.json" + ) + assert "--dangerously-bypass-approvals-and-sandbox" in command + assert "--json" in command + assert "--output-schema" in command + assert 'model_reasoning_effort="xhigh"' in command + + claude_state = {"claude_session_id": "13864642-2985-4f91-bef5-53f145f878e8"} + claude = lanes._claude_command( + lanes.Route( + "claude", "claude-opus-5", "xhigh", "claude-xhigh", "jetson", "vote", 1, () + ), + "Review.", + claude_state, + False, + ) + assert "--dangerously-skip-permissions" in claude + assert "--output-format" in claude and "stream-json" in claude + assert "--json-schema" in claude + assert "--disallowedTools" in claude + assert "Bash(kubectl apply *)" not in claude + assert "Bash(flux reconcile *)" not in claude + assert "max" not in claude + + +def test_worker_contract_separates_review_findings_from_task_blockers(tmp_path: Path): + prompt = lanes.build_prompt("Review the change.", tmp_path) + + assert "put defects and risks in findings" in prompt + assert "blockers array must be empty whenever status is completed" in prompt + assert "findings" in lanes.RESULT_SCHEMA["properties"] + assert set(lanes.RESULT_SCHEMA["required"]) == set( + lanes.RESULT_SCHEMA["properties"] + ) + assert ( + "assigned task itself" + in lanes.RESULT_SCHEMA["properties"]["blockers"]["description"] + ) + + +@pytest.mark.parametrize( + "command", + [ + "git push --force origin main", + "git reset --hard HEAD~1", + "git clean -fd", + ], +) +def test_claude_pretool_hook_blocks_hard_denies(command: str): + assert policy.denial_reason(command) + + +def test_claude_pretool_hook_allows_normal_engineering(): + assert policy.denial_reason("pytest -q testing/tests") is None + assert policy.denial_reason("git push origin feature/hermes") is None + assert policy.denial_reason("kubectl delete pod -n cassandra stuck-worker") is None + assert policy.denial_reason("flux reconcile kustomization hermes") is None + assert policy.denial_reason("vault kv get kv/atlas/hermes") is None + + +def test_claude_settings_preserve_state_and_install_three_guardrail_layers( + tmp_path: Path, +): + state = tmp_path / ".claude.json" + settings = tmp_path / "settings.json" + state.write_text('{"promptQueueUseCount": 4}\n', encoding="utf-8") + settings.write_text( + json.dumps( + { + "theme": "dark", + "permissions": { + "deny": [ + "Bash(kubectl apply *)", + "Bash(flux reconcile *)", + "Bash(vault kv *)", + "Bash(custom-owner-rule *)", + ] + }, + } + ) + + "\n", + encoding="utf-8", + ) + + client_config.configure_claude_state(state) + client_config.configure_claude_settings(settings) + + state_value = json.loads(state.read_text()) + settings_value = json.loads(settings.read_text()) + assert state_value["promptQueueUseCount"] == 4 + assert state_value["bypassPermissionsModeAccepted"] is True + assert settings_value["theme"] == "dark" + assert "Bash(git reset --hard *)" in settings_value["permissions"]["deny"] + assert "Bash(custom-owner-rule *)" in settings_value["permissions"]["deny"] + assert "Bash(kubectl apply *)" not in settings_value["permissions"]["deny"] + assert "Bash(flux reconcile *)" not in settings_value["permissions"]["deny"] + assert "Bash(vault kv *)" not in settings_value["permissions"]["deny"] + hook = settings_value["hooks"]["PreToolUse"][0]["hooks"][0] + assert "claude_command_policy.py" in hook["command"] + + +def test_legacy_state_is_archived_without_removing_provider_transcripts(tmp_path: Path): + session = tmp_path / "home/.config/herdr/session.json" + session.parent.mkdir(parents=True) + session.write_text( + '{"agents":[{"agent":"claude","session_id":"abc"}]}', encoding="utf-8" + ) + binary = tmp_path / "tools/bin/herdr" + binary.parent.mkdir(parents=True) + binary.write_text("legacy", encoding="utf-8") + (tmp_path / "home/.claude").mkdir() + (tmp_path / "home/.codex").mkdir() + + archive = migration.archive_legacy_state(tmp_path) + + value = json.loads(archive.read_text()) + assert value["legacy_session"]["agents"][0]["session_id"] == "abc" + assert (tmp_path / "home/.claude").is_dir() + assert (tmp_path / "home/.codex").is_dir() + assert not binary.exists() + assert not (tmp_path / "home/.config/herdr").exists() + + +def test_agent_uses_one_native_kanban_control_plane(): + configmap = yaml.safe_load((HERMES / "agent-configmap.yaml").read_text()) + config = yaml.safe_load(configmap["data"]["config.yaml"]) + assert config["model"] == { + "provider": "atlas-switchyard", + "default": "atlas/auto/balanced", + "model": "atlas/auto/balanced", + } + assert config["kanban"]["dispatch_in_gateway"] is True + assert config["kanban"]["default_assignee"] == "cli-auto" + assert config["plugins"]["enabled"] == ["auto-router"] + + deployment = _agent_deployment() + pod = deployment["spec"]["template"]["spec"] + assert pod["enableServiceLinks"] is False + names = {item["name"] for item in pod["containers"]} + assert "cli-lane-runner" in names + assert "terminal" in names + assert not any("herdr" in name for name in names) + rendered = (HERMES / "agent-deployment.yaml").read_text() + assert "herdr server" not in rendered + assert "herdr-dispatch" not in rendered + + +def test_cli_lane_reserves_cpu_headroom_for_ui_and_auth(): + deployment = _agent_deployment() + containers = { + item["name"]: item + for item in deployment["spec"]["template"]["spec"]["containers"] + } + lane = containers["cli-lane-runner"] + environment = {item["name"]: item["value"] for item in lane["env"]} + + assert environment["HERMES_CLI_LANE_CONCURRENCY"] == "2" + assert lane["resources"] == { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "2", "memory": "6Gi"}, + } + + +def test_agent_avoids_unhealthy_nodes_and_fits_its_remaining_capacity(): + """Placement correction: keep the agent off nodes that cannot hold it. + + titan-04 is cordoned after repeated kernel undervoltage and kubelet + failure, and titan-19 was probe/Longhorn unstable under worker load, so + both must join the existing hard exclusions. That leaves titan-05 as the + healthy candidate, which is tight enough on requested CPU that the main + container has to give back 50m to schedule there. + """ + pod = _agent_deployment()["spec"]["template"]["spec"] + hostnames = next( + item + for item in pod["affinity"]["nodeAffinity"][ + "requiredDuringSchedulingIgnoredDuringExecution" + ]["nodeSelectorTerms"][0]["matchExpressions"] + if item["key"] == "kubernetes.io/hostname" + ) + + assert hostnames["operator"] == "NotIn" + assert set(hostnames["values"]) >= {"titan-04", "titan-19"} + + hermes = next(item for item in pod["containers"] if item["name"] == "hermes") + assert hermes["resources"]["requests"]["cpu"] == "300m" + + +def test_agent_root_is_stock_dashboard_and_terminal_is_a_separate_path(): + deployment = _agent_deployment() + pod = deployment["spec"]["template"]["spec"] + containers = {item["name"]: item for item in pod["containers"]} + assert "webui" not in containers + + assert "dashboard" not in containers + hermes = containers["hermes"] + hermes_env = {item["name"]: item["value"] for item in hermes["env"]} + assert hermes_env["HERMES_STREAM_STALE_TIMEOUT"] == "600" + assert hermes_env["HERMES_API_CALL_STALE_TIMEOUT"] == "600" + assert hermes["command"] == ["/bin/sh", "-ec"] + startup = hermes["args"][0] + assert ". /opt/data/.env" in startup + assert "exec /init /opt/hermes/docker/main-wrapper.sh gateway run" in startup + hermes_env = {item["name"]: item["value"] for item in hermes["env"]} + assert hermes_env["HERMES_DASHBOARD"] == "1" + assert hermes_env["HERMES_DASHBOARD_HOST"] == "127.0.0.1" + assert hermes_env["HERMES_DASHBOARD_PORT"] == "9119" + assert hermes_env["HERMES_TUI_AGENT_INIT_TIMEOUT_S"] == "180" + assert hermes["securityContext"]["runAsUser"] == 0 + assert hermes["securityContext"]["runAsGroup"] == 0 + for probe_name in ("startupProbe", "readinessProbe", "livenessProbe"): + probe = hermes[probe_name] + assert probe["exec"]["command"] == [ + "curl", + "-fsS", + "http://127.0.0.1:9119/api/status", + ] + + terminal = containers["terminal"] + command = terminal["args"][0] + assert "--base-path /terminal" in command + assert "--check-origin" not in command + assert "/usr/bin/tmux new-session -A" in command + assert "--continue" in command + assert "--yolo" in command + terminal_env = {item["name"]: item["value"] for item in terminal["env"]} + assert terminal_env["HERMES_TUI_AGENT_INIT_TIMEOUT_S"] == "180" + + claude_broker = containers["claude-broker"] + claude_env = {item["name"]: item["value"] for item in claude_broker["env"]} + assert claude_env["HERMES_CLAUDE_BROKER_CONCURRENCY"] == "2" + assert claude_broker["readinessProbe"]["tcpSocket"] == {"port": "claude-broker"} + assert claude_broker["livenessProbe"]["tcpSocket"] == {"port": "claude-broker"} + + args = containers["oauth2-proxy"]["args"] + terminal_upstream = "--upstream=http://127.0.0.1:7681/terminal/" + dashboard_upstream = "--upstream=http://127.0.0.1:9119/" + assert terminal_upstream in args + assert dashboard_upstream in args + assert args.index(terminal_upstream) < args.index(dashboard_upstream) + assert "--pass-host-header=false" in args + assert "--cookie-refresh=19m" in args + assert "--session-store-type=redis" in args + assert any( + arg.startswith("--redis-connection-url=redis://hermes-oauth-sessions.") + for arg in args + ) + + patch_init = next( + item for item in pod["initContainers"] if item["name"] == "patch-tui-gateway" + ) + assert patch_init["command"][-1] == "/patched/server.py" + for name in ("hermes", "terminal"): + mounts = containers[name]["volumeMounts"] + assert { + "name": "tui-gateway-patch", + "mountPath": "/opt/hermes/tui_gateway/server.py", + "subPath": "server.py", + } in mounts + + ingress_documents = [ + item + for item in yaml.safe_load_all((HERMES / "agent-ingress.yaml").read_text()) + if item + ] + middlewares = { + item["metadata"]["name"]: item + for item in ingress_documents + if item["kind"] == "Middleware" + } + assert middlewares["hermes-agent-terminal-slash"]["spec"]["redirectRegex"][ + "replacement" + ].endswith("/terminal/") + assert ( + middlewares["hermes-agent-stock-dashboard-headers"]["spec"]["headers"][ + "customRequestHeaders" + ]["Origin"] + == "http://127.0.0.1:9119" + ) + ingresses = { + item["metadata"]["name"]: item + for item in ingress_documents + if item["kind"] == "Ingress" + } + assert ( + ingresses["hermes-agent-dashboard"]["metadata"]["annotations"][ + "traefik.ingress.kubernetes.io/router.middlewares" + ] + == "hermes-hermes-agent-stock-dashboard-headers@kubernetescrd" + ) + assert ( + ingresses["hermes-agent-terminal"]["metadata"]["annotations"][ + "traefik.ingress.kubernetes.io/router.middlewares" + ] + == "hermes-hermes-agent-terminal-slash@kubernetescrd" + ) diff --git a/testing/tests/test_hermes_cli_lanes_kanban.py b/testing/tests/test_hermes_cli_lanes_kanban.py new file mode 100644 index 00000000..279b74c5 --- /dev/null +++ b/testing/tests/test_hermes_cli_lanes_kanban.py @@ -0,0 +1,392 @@ +"""Kanban claim and lifecycle contracts for Hermes CLI lanes.""" + +from __future__ import annotations + +import sys +from contextlib import nullcontext +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from testing.tests.test_hermes_cli_lanes_support import ( + lanes, +) + + +def test_unassigned_ready_task_is_persistently_routed_to_auto_lane(monkeypatch): + task = SimpleNamespace(id="t_auto", assignee=None, status="ready") + assigned = [] + + class Connection: + def close(self): + return None + + def assign_task(_conn, task_id, profile): + assigned.append((task_id, profile)) + task.assignee = profile + return True + + fake_db = SimpleNamespace( + list_boards=lambda include_archived=False: [{"slug": "cassandra"}], + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + recompute_ready=lambda _conn: None, + list_tasks=lambda _conn: [task], + assign_task=assign_task, + get_task=lambda _conn, _task_id: task, + claim_task=lambda _conn, _task_id, **_kwargs: task, + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + + assert lanes.claim_ready(set(), 1) == [("cassandra", "t_auto")] + assert assigned == [("t_auto", "cli-auto")] + + +def test_corrupt_board_is_quarantined_without_stopping_healthy_lanes( + monkeypatch, capsys +): + class CorruptBoardError(Exception): + pass + + task = SimpleNamespace(id="t_healthy", assignee="cli-auto", status="ready") + + class Connection: + def close(self): + return None + + def connect(*, board): + if board == "cassandra": + raise CorruptBoardError("integrity_check failed") + return Connection() + + fake_db = SimpleNamespace( + KanbanDbCorruptError=CorruptBoardError, + list_boards=lambda include_archived=False: [ + {"slug": "cassandra"}, + {"slug": "healthy"}, + ], + scoped_current_board=lambda _board: nullcontext(), + connect=connect, + recompute_ready=lambda _conn: None, + list_tasks=lambda _conn: [task], + claim_task=lambda _conn, _task_id, **_kwargs: task, + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + lanes.BOARD_CORRUPTION_ERRORS.clear() + + assert lanes.claim_ready(set(), 1) == [("healthy", "t_healthy")] + assert "temporarily skipping Kanban board 'cassandra'" in capsys.readouterr().err + + +def test_transient_board_scan_failure_does_not_stop_healthy_lanes(monkeypatch, capsys): + task = SimpleNamespace(id="t_healthy", assignee="cli-auto", status="ready") + + class Connection: + def __init__(self, board): + self.board = board + + def close(self): + return None + + def recompute_ready(connection): + if connection.board == "cassandra": + raise lanes.sqlite3.OperationalError("disk I/O error") + + fake_db = SimpleNamespace( + list_boards=lambda include_archived=False: [ + {"slug": "cassandra"}, + {"slug": "healthy"}, + ], + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(board), + recompute_ready=recompute_ready, + list_tasks=lambda _conn: [task], + claim_task=lambda _conn, _task_id, **_kwargs: task, + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + lanes.BOARD_CORRUPTION_ERRORS.clear() + + assert lanes.claim_ready(set(), 1) == [("healthy", "t_healthy")] + error = capsys.readouterr().err + assert "temporarily skipping Kanban board 'cassandra'" in error + assert "storage OperationalError: disk I/O error" in error + + +def test_board_call_retries_storage_faults_on_fresh_connections(): + connections = [] + + class Connection: + def __init__(self): + self.closed = False + + def close(self): + self.closed = True + + def connect(*, board): + assert board == "cassandra" + connection = Connection() + connections.append(connection) + return connection + + attempts = [] + + def operation(_connection): + attempts.append(1) + if len(attempts) < 3: + raise lanes.sqlite3.OperationalError("disk I/O error") + return "healthy" + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=connect, + ) + lanes.BOARD_CORRUPTION_ERRORS.clear() + + assert lanes._board_call(fake_db, "cassandra", operation) == "healthy" + assert len(connections) == 3 + assert all(connection.closed for connection in connections) + + +@pytest.mark.parametrize( + ("result", "expected_action"), + [ + (lanes.ProcessResult(0, "plain text only", None, False), "block"), + ( + lanes.ProcessResult( + 0, + "", + { + "status": "completed", + "summary": "done", + "changed_files": ["src/a.py"], + "tests_run": ["pytest -q"], + "artifacts": ["reports/result.json"], + "blockers": [], + }, + False, + ), + "complete", + ), + ( + lanes.ProcessResult( + 0, + "", + { + "status": "completed", + "summary": "The full test suite is still running.", + "changed_files": ["src/a.py"], + "tests_run": ["pytest -q — in progress"], + "artifacts": [], + "blockers": [], + }, + False, + ), + "block", + ), + ], +) +def test_claim_requires_structured_evidence_and_surfaces_artifacts( + tmp_path: Path, + monkeypatch, + result, + expected_action, +): + task = SimpleNamespace( + id="t_worker", + current_run_id=4, + assignee="cli-auto", + max_runtime_seconds=60, + ) + calls = [] + heartbeats = [] + connections = [] + artifact = tmp_path / "reports/result.json" + artifact.parent.mkdir() + artifact.write_text("{}\n", encoding="utf-8") + + class Connection: + def __init__(self): + self.closed = False + + def close(self): + self.closed = True + + def connect(*, board): + assert board == "cassandra" + connection = Connection() + connections.append(connection) + return connection + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=connect, + get_task=lambda _conn, _task_id: task, + worker_log_path=lambda _task_id, board: tmp_path / "worker.log", + _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_worker"), + set_branch_name=lambda *_args: None, + set_workspace_path=lambda *_args: None, + build_worker_context=lambda *_args: "bounded objective", + heartbeat_worker=lambda _conn, _task_id, *, note, expected_run_id: ( + heartbeats.append((note, expected_run_id)) or True + ), + add_comment=lambda *_args: None, + complete_task=lambda *_args, **kwargs: calls.append(("complete", kwargs)), + block_task=lambda *_args, **kwargs: calls.append(("block", kwargs)), + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + monkeypatch.setattr( + lanes, "state_path", lambda _board, _task_id: tmp_path / "state.json" + ) + monkeypatch.setattr( + lanes, + "select_route", + lambda *_args, **_kwargs: lanes.Route( + "codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, () + ), + ) + + def run_provider(*args, **_kwargs): + assert connections[0].closed + before_heartbeat = len(connections) + assert args[6]("working") is True + assert len(connections) == before_heartbeat + 1 + assert connections[-1].closed + return result + + monkeypatch.setattr(lanes, "run_provider", run_provider) + + lanes.execute_claim("cassandra", "t_worker") + + assert calls[0][0] == expected_action + assert heartbeats == [("working", 4)] + if expected_action == "complete": + assert calls[0][1]["metadata"]["artifacts"] == [str(artifact)] + assert calls[0][1]["metadata"]["tests_run"] == ["pytest -q"] + else: + assert calls[0][1]["kind"] == "capability" + assert all(connection.closed for connection in connections) + + +def test_goal_card_continues_after_local_judge_rejects_progress( + tmp_path: Path, + monkeypatch, +): + task = SimpleNamespace( + id="t_goal", + current_run_id=12, + assignee="cli-auto", + max_runtime_seconds=300, + goal_mode=True, + goal_max_turns=3, + ) + calls = [] + comments = [] + + class Connection: + def close(self): + return None + + fake_db = SimpleNamespace( + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: Connection(), + get_task=lambda _conn, _task_id: task, + worker_log_path=lambda _task_id, board: tmp_path / "worker.log", + _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_goal"), + set_branch_name=lambda *_args: None, + set_workspace_path=lambda *_args: None, + build_worker_context=lambda *_args: "Run tests, commit, push, and verify remote HEAD.", + heartbeat_worker=lambda *_args, **_kwargs: True, + add_comment=lambda _conn, _task_id, _author, body: comments.append(body), + complete_task=lambda *_args, **kwargs: calls.append(("complete", kwargs)), + block_task=lambda *_args, **kwargs: calls.append(("block", kwargs)), + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) + monkeypatch.setattr( + lanes, + "state_path", + lambda _board, _task_id: tmp_path / "state.json", + ) + claude_low = lanes.Route( + "claude", "claude-fable-5", "low", "claude-low", "jetson", "vote", 1, () + ) + codex_low = lanes.Route( + "codex", "gpt-5.6-luna", "low", "codex-low", "manual", "fallback", 1, () + ) + codex_xhigh = lanes.Route( + "codex", "gpt-5.6-sol", "xhigh", "codex-xhigh", "jetson", "escalated", 1, () + ) + route_calls = [] + + def select_route(_prompt, assignee, **kwargs): + route_calls.append((assignee, kwargs)) + if assignee == "cli-codex-low": + return codex_low + if len(route_calls) == 1: + return claude_low + return codex_xhigh + + monkeypatch.setattr(lanes, "select_route", select_route) + monkeypatch.setattr(lanes, "fresh_unavailable_provider", lambda: None) + reports = [ + lanes.ProcessResult(1, "authentication expired", None, True), + lanes.ProcessResult( + 0, + "first turn", + { + "status": "completed", + "summary": "Focused tests passed.", + "changed_files": ["src/a.py"], + "tests_run": ["pytest focused: passed"], + "artifacts": [], + "blockers": [], + }, + False, + ), + lanes.ProcessResult( + 0, + "second turn", + { + "status": "completed", + "summary": "Full tests passed; commit pushed and remote HEAD verified.", + "changed_files": ["src/a.py"], + "tests_run": ["pytest full: passed"], + "artifacts": [], + "blockers": [], + }, + False, + ), + ] + monkeypatch.setattr(lanes, "run_provider", lambda *_args, **_kwargs: reports.pop(0)) + verdicts = iter( + [ + (False, "commit, push, and remote verification are missing"), + (True, "all explicit acceptance criteria have evidence"), + ] + ) + judge_contexts = [] + + def judge_goal_completion(objective, *_args, **_kwargs): + judge_contexts.append(objective) + return next(verdicts) + + monkeypatch.setattr( + lanes.cli_lane_goal, + "judge_goal_completion", + judge_goal_completion, + ) + + lanes.execute_claim("cassandra", "t_goal") + + assert calls[0][0] == "complete" + assert calls[0][1]["metadata"]["goal_turn"] == 2 + assert any( + "Goal completion rejected; continuing turn 2/3" in item for item in comments + ) + assert any( + "Goal route 2/3: codex/gpt-5.6-sol at xhigh" in item for item in comments + ) + assert route_calls[2][1]["exclude_provider"] == "claude" + assert "prior rejected reports" in judge_contexts[1] + assert "commit, push, and remote verification are missing" in judge_contexts[1] + assert reports == [] diff --git a/testing/tests/test_hermes_cli_lanes_support.py b/testing/tests/test_hermes_cli_lanes_support.py new file mode 100644 index 00000000..242351b9 --- /dev/null +++ b/testing/tests/test_hermes_cli_lanes_support.py @@ -0,0 +1,79 @@ +"""Shared loaders and manifest helpers for Hermes CLI-lane contracts.""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +import yaml + +ROOT = Path(__file__).parents[2] +SCRIPTS = ROOT / "services/hermes/scripts" +sys.path.insert(0, str(SCRIPTS)) +HERMES = ROOT / "services/hermes" +KEYCLOAK = ROOT / "services/keycloak" +FLUX_HERMES = ROOT / "clusters/atlas/flux-system/applications/hermes/kustomization.yaml" + + +def _load(name: str): + spec = importlib.util.spec_from_file_location(name, SCRIPTS / f"{name}.py") + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +lanes = _load("cli_lane_runner") +policy = _load("claude_command_policy") +migration = _load("migrate_herdr_state") +auth_patch = _load("patch_hermes_auth") +tui_gateway_patch = _load("patch_tui_gateway") +codex_runtime_patch = _load("patch_codex_runtime") +ttyd_patch = _load("patch_ttyd_index") +client_config = _load("configure_agent_clients") + + +def _agent_deployment() -> dict: + return yaml.safe_load((HERMES / "agent-deployment.yaml").read_text()) + + +def _services() -> dict[str, dict]: + return { + item["metadata"]["name"]: item + for item in yaml.safe_load_all((HERMES / "service.yaml").read_text()) + if item + } + + +def _oauth_deployment(name: str) -> dict: + documents = [ + item + for item in yaml.safe_load_all((HERMES / "oauth2-proxy.yaml").read_text()) + if item + ] + return next( + item + for item in documents + if item["kind"] == "Deployment" and item["metadata"]["name"] == name + ) + + +class _SwitchyardResponse: + """Minimal context-managed response used by routing contract tests.""" + + def __init__(self, selected: str, rationale: str = "local classifier vote"): + self.headers = { + "x-model-router-selected-model": selected, + "x-model-router-rationale": rationale, + } + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def read(self): + return b"{}" diff --git a/testing/tests/test_hermes_cli_lanes_toolchain.py b/testing/tests/test_hermes_cli_lanes_toolchain.py new file mode 100644 index 00000000..2ed232cb --- /dev/null +++ b/testing/tests/test_hermes_cli_lanes_toolchain.py @@ -0,0 +1,386 @@ +"""Operator toolchain and runtime-patch contracts for Hermes CLI lanes.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +import yaml + +from testing.tests.test_hermes_cli_lanes_support import ( + HERMES, + SCRIPTS, + _agent_deployment, + auth_patch, + client_config, + codex_runtime_patch, + tui_gateway_patch, + ttyd_patch, +) + + +def test_worker_route_broker_accepts_pod_network_health_checks(): + """Kubelet probes the pod IP, so the broker cannot bind to loopback only.""" + script = (SCRIPTS / "worker_route_broker.py").read_text() + assert 'ThreadingHTTPServer(("0.0.0.0", PORT), Handler)' in script + + +def test_switchyard_network_boundary_allows_vault_bootstrap(): + """The pre-populate init container must reach Vault before routing starts.""" + documents = [ + item + for item in yaml.safe_load_all((HERMES / "networkpolicy.yaml").read_text()) + if item + ] + isolation = next( + item + for item in documents + if item.get("metadata", {}).get("name") == "hermes-switchyard-isolation" + ) + assert any( + rule.get("to") + == [ + { + "namespaceSelector": { + "matchLabels": {"kubernetes.io/metadata.name": "vault"} + }, + "podSelector": {"matchLabels": {"app": "vault"}}, + } + ] + and rule.get("ports") == [{"protocol": "TCP", "port": 8200}] + for rule in isolation["spec"]["egress"] + ) + + +def test_switchyard_network_boundary_allows_metrics_scraping(): + """VictoriaMetrics may scrape Switchyard without widening its API boundary.""" + documents = [ + item + for item in yaml.safe_load_all((HERMES / "networkpolicy.yaml").read_text()) + if item + ] + isolation = next( + item + for item in documents + if item.get("metadata", {}).get("name") == "hermes-switchyard-isolation" + ) + assert any( + rule.get("from") + == [ + { + "namespaceSelector": { + "matchLabels": {"kubernetes.io/metadata.name": "monitoring"} + }, + "podSelector": {"matchLabels": {"app": "server"}}, + } + ] + and rule.get("ports") == [{"protocol": "TCP", "port": 9005}] + for rule in isolation["spec"]["ingress"] + ) + + +def test_owner_agent_installs_the_pinned_operator_toolchain(): + script = (SCRIPTS / "install_agent_tools.sh").read_text() + for value in [ + "flux", + "helm", + "kustomize", + "jq", + "yq", + "gh", + "vault", + "sops", + "age", + "age-keygen", + "k9s", + "terraform", + "go", + "gofmt", + ]: + assert value in script + assert "go1.26.5.linux-arm64.tar.gz" in script + assert "fe4789e92b1f33358680864bbe8704289e7bb5fc207d80623c308935bd696d49" in script + assert script.count("sha256sum -c -") == 1 + + deployment = _agent_deployment() + installer = next( + item + for item in deployment["spec"]["template"]["spec"]["initContainers"] + if item["name"] == "install-agent-tools" + ) + assert "/bin/sh /opt/coordinator/install_agent_tools.sh" in installer["command"][2] + assert any(mount["name"] == "coordinator" for mount in installer["volumeMounts"]) + init_config = next( + item + for item in deployment["spec"]["template"]["spec"]["initContainers"] + if item["name"] == "init-config" + ) + init_command = init_config["command"][2] + assert "# Hermes managed operator PATH." in init_command + assert "/opt/coordinator:/opt/data/tools/bin:/opt/hermes/.venv/bin" in init_command + assert 'chmod 0644 "${profile_file}"' in init_command + + +def test_owner_agent_uses_only_the_canonical_hostname(): + paths = [ + HERMES / "agent-configmap.yaml", + HERMES / "agent-deployment.yaml", + HERMES / "agent-ingress.yaml", + Path(__file__).parents[2] / "scripts/ops/hermes_triage_monitor.py", + ] + for path in paths: + content = path.read_text() + assert "agent.bstein.dev" not in content + assert "agent.hermes.bstein.dev" in content + + +def test_agent_reconnect_retains_complete_history_and_long_tool_budget(): + configmap = yaml.safe_load((HERMES / "agent-configmap.yaml").read_text()) + config = yaml.safe_load(configmap["data"]["config.yaml"]) + display = config["display"] + assert display["resume_exchanges"] >= 10000 + assert display["resume_max_user_chars"] >= 10000000 + assert display["resume_max_assistant_chars"] >= 10000000 + assert config["agent"]["max_turns"] == 180 + assert config["delegation"]["max_iterations"] == 120 + + +def test_auth_patch_honors_explicit_shared_store(tmp_path: Path): + source = tmp_path / "auth.py" + destination = tmp_path / "patched/auth.py" + source.write_text( + 'from pathlib import Path\nimport os\n\ndef _auth_file_path() -> Path:\n path = get_hermes_home() / "auth.json"\n return path\n', + encoding="utf-8", + ) + auth_patch.patch(source, destination) + content = destination.read_text() + assert 'os.environ.get("HERMES_AUTH_FILE"' in content + + +def test_auth_patch_fails_closed_on_upstream_drift(tmp_path: Path): + source = tmp_path / "auth.py" + source.write_text("def changed():\n pass\n", encoding="utf-8") + with pytest.raises(RuntimeError, match="context changed"): + auth_patch.patch(source, tmp_path / "patched.py") + + +def test_codex_runtime_patch_uses_cli_and_forwards_route(tmp_path: Path): + provider = tmp_path / "runtime_provider.py" + provider.write_text(codex_runtime_patch.PROVIDER_BEFORE, encoding="utf-8") + provider_out = tmp_path / "patched/runtime_provider.py" + codex_runtime_patch.patch_provider(provider, provider_out) + assert '"api_mode": "codex_app_server"' in provider_out.read_text() + + session = tmp_path / "codex_app_server_session.py" + session.write_text( + codex_runtime_patch.SESSION_SIGNATURE_BEFORE + + codex_runtime_patch.SESSION_REQUEST_BEFORE, + encoding="utf-8", + ) + session_out = tmp_path / "patched/codex_app_server_session.py" + codex_runtime_patch.patch_session(session, session_out) + session_content = session_out.read_text() + assert 'turn_params["model"] = model' in session_content + assert 'turn_params["effort"] = effort' in session_content + assert '"approvalPolicy": "never"' in session_content + assert '"sandboxPolicy": {"type": "dangerFullAccess"}' in session_content + + turn = tmp_path / "codex_runtime.py" + turn.write_text( + codex_runtime_patch.FALLBACK_CONTEXT_BEFORE + codex_runtime_patch.TURN_BEFORE, + encoding="utf-8", + ) + turn_out = tmp_path / "patched/codex_runtime.py" + codex_runtime_patch.patch_turn(turn, turn_out) + turn_content = turn_out.read_text() + assert "model=str(getattr(agent" in turn_content + assert "build_cross_provider_codex_prompt" in turn_content + + fallback = tmp_path / "chat_completion_helpers.py" + fallback.write_text( + codex_runtime_patch.FALLBACK_RESOLUTION_BEFORE, + encoding="utf-8", + ) + fallback_out = tmp_path / "patched/chat_completion_helpers.py" + codex_runtime_patch.patch_fallback(fallback, fallback_out) + fallback_content = fallback_out.read_text() + assert 'agent.api_mode = "codex_app_server"' in fallback_content + assert "agent._codex_cross_provider_fallback = True" in fallback_content + + loop = tmp_path / "conversation_loop.py" + loop.write_text( + codex_runtime_patch.FALLBACK_DISPATCH_BEFORE + + codex_runtime_patch.RETRY_FALLBACK_DISPATCH_BEFORE + + codex_runtime_patch.STREAM_RECOVERY_BEFORE, + encoding="utf-8", + ) + loop_out = tmp_path / "patched/conversation_loop.py" + codex_runtime_patch.patch_loop(loop, loop_out) + loop_content = loop_out.read_text() + assert loop_content.count('if agent.api_mode == "codex_app_server"') == 2 + assert "build_cross_provider_codex_prompt" in loop_content + retry_dispatch = loop_content.index( + "Fallback activation happens inside this retry loop" + ) + api_kwargs = loop_content.find("agent._build_api_kwargs", retry_dispatch) + assert api_kwargs == -1 or retry_dispatch < api_kwargs + assert "Provider stream ended before a complete response" in loop_content + assert "_is_transport_stub" in loop_content + assert "rerouting ({truncated_tool_call_retries}/4)" in loop_content + + auxiliary = tmp_path / "auxiliary_client.py" + auxiliary.write_text( + codex_runtime_patch.AUXILIARY_TOKEN_BEFORE, + encoding="utf-8", + ) + auxiliary_out = tmp_path / "patched/auxiliary_client.py" + codex_runtime_patch.patch_auxiliary(auxiliary, auxiliary_out) + auxiliary_content = auxiliary_out.read_text() + assert 'os.environ.get("CODEX_HOME"' in auxiliary_content + assert 'Path(codex_home).expanduser() / "auth.json"' in auxiliary_content + assert "never creates a metered API-key lane" in auxiliary_content + + +def test_agent_mounts_codex_auxiliary_runtime_patch(): + deployment = _agent_deployment() + pod = deployment["spec"]["template"]["spec"] + patch_init = next( + item for item in pod["initContainers"] if item["name"] == "patch-codex-runtime" + ) + assert patch_init["command"][-2:] == [ + "/opt/hermes/agent/auxiliary_client.py", + "/patched/auxiliary_client.py", + ] + expected_mount = { + "name": "codex-runtime-patch", + "mountPath": "/opt/hermes/agent/auxiliary_client.py", + "subPath": "auxiliary_client.py", + } + containers = {item["name"]: item for item in pod["containers"]} + for name in ("hermes", "terminal"): + assert expected_mount in containers[name]["volumeMounts"] + + +def test_codex_auxiliary_patch_reads_cli_token_without_copying_it( + tmp_path: Path, + monkeypatch, +): + codex_home = tmp_path / ".codex" + codex_home.mkdir() + (codex_home / "auth.json").write_text( + json.dumps({"tokens": {"access_token": "cli-access-token"}}), + encoding="utf-8", + ) + monkeypatch.setenv("CODEX_HOME", str(codex_home)) + + source = tmp_path / "auxiliary_client.py" + source.write_text( + "import json, logging, os, time\n" + "from pathlib import Path\n" + "logger = logging.getLogger(__name__)\n" + "def read_token():\n" + " try:\n" + " raise RuntimeError('Hermes provider store intentionally empty')\n" + + codex_runtime_patch.AUXILIARY_TOKEN_BEFORE, + encoding="utf-8", + ) + destination = tmp_path / "patched/auxiliary_client.py" + codex_runtime_patch.patch_auxiliary(source, destination) + namespace: dict = {} + exec(compile(destination.read_text(), str(destination), "exec"), namespace) + + assert namespace["read_token"]() == "cli-access-token" + + +def test_codex_runtime_patch_fails_closed_on_upstream_drift(tmp_path: Path): + source = tmp_path / "runtime_provider.py" + source.write_text("def changed():\n pass\n", encoding="utf-8") + with pytest.raises(RuntimeError, match="context changed"): + codex_runtime_patch.patch_provider(source, tmp_path / "patched.py") + + +def test_codex_runtime_migration_uses_owner_unsafe_mode(tmp_path: Path): + config = tmp_path / "config.yaml" + config.write_text("model: {}\n", encoding="utf-8") + codex_home = tmp_path / ".codex" + calls = [] + + class Report: + errors = [] + + @staticmethod + def summary(): + return "configured" + + def migrate(value, **kwargs): + calls.append((value, kwargs)) + return Report() + + client_config.configure_codex_runtime(config, migrate, codex_home) + + assert calls[0][1]["default_permission_profile"] is None + assert calls[0][1]["codex_home"] == codex_home + content = (codex_home / "config.toml").read_text(encoding="utf-8") + assert 'approval_policy = "never"' in content + assert 'sandbox_mode = "danger-full-access"' in content + assert "default_permissions" not in content + + +def test_codex_owner_permissions_replace_stale_profile(tmp_path: Path): + config = tmp_path / "config.toml" + config.write_text( + 'default_permissions = ":danger-no-sandbox"\n\n[features]\nhooks = true\n', + encoding="utf-8", + ) + + client_config.configure_codex_owner_permissions(config) + client_config.configure_codex_owner_permissions(config) + + content = config.read_text(encoding="utf-8") + assert content.count(client_config.OWNER_PERMISSIONS_BEGIN) == 1 + assert content.count('approval_policy = "never"') == 1 + assert content.count('sandbox_mode = "danger-full-access"') == 1 + assert "default_permissions" not in content + assert "[features]\nhooks = true" in content + + +def test_tui_gateway_patch_extends_and_bounds_agent_startup(tmp_path: Path): + source = tmp_path / "server.py" + destination = tmp_path / "patched/server.py" + source.write_text( + "import os\n\n" + tui_gateway_patch.BEFORE + "\ndef unchanged():\n pass\n", + encoding="utf-8", + ) + + tui_gateway_patch.patch(source, destination) + + content = destination.read_text(encoding="utf-8") + assert "HERMES_TUI_AGENT_INIT_TIMEOUT_S" in content + assert "configured = 180.0" in content + assert "return max(30.0, min(configured, 900.0))" in content + assert "timeout: float | None = None" in content + assert "ready.wait(timeout=wait_timeout)" in content + assert "def unchanged():" in content + + +def test_tui_gateway_patch_fails_closed_on_upstream_drift(tmp_path: Path): + source = tmp_path / "server.py" + source.write_text("def changed():\n pass\n", encoding="utf-8") + with pytest.raises(RuntimeError, match="context changed"): + tui_gateway_patch.patch(source, tmp_path / "patched.py") + + +def test_ttyd_clipboard_and_reconnect_patch_remain_enabled(): + source = '' + content = ttyd_patch.patch_html(source) + assert 'id="atlas-ttyd-clipboard"' in content + assert "navigator.clipboard.writeText(text)" in content + assert "class AtlasRecoveringWebSocket" in content + assert "window.location.reload()" in content + assert "event.stopImmediatePropagation()" in content + + +def test_ttyd_patch_fails_closed_on_upstream_drift(): + with pytest.raises(RuntimeError, match="context changed"): + ttyd_patch.patch_html("changed") diff --git a/testing/tests/test_hermes_coordinator.py b/testing/tests/test_hermes_coordinator.py index bc2d5b5f..683c0201 100644 --- a/testing/tests/test_hermes_coordinator.py +++ b/testing/tests/test_hermes_coordinator.py @@ -2,44 +2,20 @@ from __future__ import annotations -import importlib.util import json import subprocess -import sys import tomllib from pathlib import Path -import pytest import yaml - -SCRIPT = Path(__file__).parents[2] / "services/hermes/scripts/hermes_coordinator.py" -sys.path.insert(0, str(SCRIPT.parent)) -routing = importlib.import_module("hermes_model_routing") -catalog_resolver = importlib.import_module("routing_catalog") -SPEC = importlib.util.spec_from_file_location("hermes_coordinator", SCRIPT) -assert SPEC and SPEC.loader -coordinator = importlib.util.module_from_spec(SPEC) -sys.modules[SPEC.name] = coordinator -SPEC.loader.exec_module(coordinator) - - -def _base_config() -> dict: - """Return the minimal coordinator configuration used by routing tests.""" - return { - "model": { - "provider": "openai-codex", - "default": "gpt-5.6-terra", - "model": "gpt-5.6-terra", - }, - "fallback_providers": [ - {"provider": "anthropic", "model": "claude-opus-5"}, - dict(routing.LOCAL_FALLBACK), - ], - "toolsets": ["kanban"], - "terminal": {"cwd": "/opt/data/workspace"}, - } - +from testing.tests.test_hermes_coordinator_support import ( + SCRIPT, + _base_config, + catalog_resolver, + coordinator, + routing, +) def test_model_version_and_quality_selection_handle_new_and_small_models(): assert routing.model_version("claude-3-5-sonnet-20241022") == (3, 5) @@ -334,177 +310,3 @@ def test_refresh_writes_non_secret_routing_status(tmp_path: Path, monkeypatch): assert status["projects"]["cassandra"]["state"] == "ready" assert status["projects"]["cassandra"]["board_state"] == {"state": "ready"} assert "do-not-report" not in status_path.read_text(encoding="utf-8") - - -def test_corrupt_cassandra_board_is_preserved_without_blocking_refresh( - tmp_path: Path, monkeypatch -): - """A damaged project board degrades Kanban instead of stopping all services.""" - - class FakeKanbanDbCorruptError(Exception): - def __init__(self): - super().__init__("integrity check failed") - self.db_path = tmp_path / "kanban.db" - self.backup_path = tmp_path / "kanban.db.corrupt.backup" - self.reason = "row out of order" - - def raise_corruption(_root: Path) -> None: - raise FakeKanbanDbCorruptError() - - monkeypatch.setattr(coordinator, "bootstrap_cassandra", raise_corruption) - monkeypatch.setattr( - coordinator, "_kanban_corruption_type", lambda: FakeKanbanDbCorruptError - ) - - status = coordinator.bootstrap_cassandra_state(tmp_path) - - assert status == { - "state": "corrupt-preserved", - "database": str(tmp_path / "kanban.db"), - "backup": str(tmp_path / "kanban.db.corrupt.backup"), - "reason": "row out of order", - } - - -def test_unrelated_cassandra_bootstrap_failure_remains_fatal( - tmp_path: Path, monkeypatch -): - """Only the known fail-closed corruption state may be degraded.""" - - class FakeKanbanDbCorruptError(Exception): - pass - - def raise_permission_error(_root: Path) -> None: - raise PermissionError("cannot access board") - - monkeypatch.setattr(coordinator, "bootstrap_cassandra", raise_permission_error) - monkeypatch.setattr( - coordinator, "_kanban_corruption_type", lambda: FakeKanbanDbCorruptError - ) - - try: - coordinator.bootstrap_cassandra_state(tmp_path) - except PermissionError as error: - assert str(error) == "cannot access board" - else: - raise AssertionError("unrelated board errors must fail the coordinator refresh") - - -def test_cassandra_workspace_uses_valid_configured_worktree( - tmp_path: Path, monkeypatch -): - """The coordinator selects an explicit Git worktree but rejects stale paths.""" - active = tmp_path / "cassandra-v69" - active.mkdir() - (active / ".git").write_text("gitdir: elsewhere\n", encoding="utf-8") - monkeypatch.setenv("HERMES_CASSANDRA_ACTIVE_WORKTREE", str(active)) - - assert coordinator.cassandra_workspace() == active - - (active / ".git").unlink() - assert coordinator.cassandra_workspace() == coordinator.CASSANDRA_BASE_PATH - - -@pytest.mark.parametrize( - ("current_remote", "repair_action"), - [ - ("https://scm.bstein.dev/bstein/cassandra.git", "set-url"), - (None, "add"), - (coordinator.CASSANDRA_REMOTE, None), - ], - ids=["old-origin", "missing-origin", "canonical-origin"], -) -def test_cassandra_sync_repairs_existing_worktree( - tmp_path: Path, monkeypatch, current_remote: str | None, repair_action: str | None -): - """Existing worktrees use the canonical Atlas origin before fetching.""" - workspace = tmp_path / "cassandra" - (workspace / ".git").mkdir(parents=True) - monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", workspace) - monkeypatch.setattr(coordinator.shutil, "which", lambda _name: "/usr/bin/git") - token_path = tmp_path / "gitea-token" - token_path.write_text("configured\n", encoding="utf-8") - monkeypatch.setenv("HERMES_GITEA_TOKEN_FILE", str(token_path)) - commands: list[list[str]] = [] - environments: list[dict[str, str]] = [] - - def run(command, **kwargs): - commands.append(command) - environments.append(kwargs.get("env", {})) - if command[-2:] == ["get-url", "origin"]: - return subprocess.CompletedProcess( - command, 0 if current_remote else 2, stdout=f"{current_remote or ''}\n" - ) - return subprocess.CompletedProcess(command, 0) - - monkeypatch.setattr(coordinator.subprocess, "run", run) - assert coordinator.sync_cassandra_repo({"GITEA_TOKEN": "must-not-pass"}) == "ready" - repair_commands = [command for command in commands if repair_action in command] - assert bool(repair_commands) is bool(repair_action) - assert commands[-1][-4:] == ["fetch", "--quiet", "--prune", "origin"] - assert all("GITEA_TOKEN" not in environment for environment in environments) - - -def test_cassandra_sync_repairs_origin_without_token(tmp_path: Path, monkeypatch): - """Missing credentials skip only the fetch, not the local origin repair.""" - workspace = tmp_path / "cassandra" - (workspace / ".git").mkdir(parents=True) - monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", workspace) - monkeypatch.setattr(coordinator.shutil, "which", lambda _name: "/usr/bin/git") - commands: list[list[str]] = [] - - def run(command, **_kwargs): - commands.append(command) - stdout = "old-origin\n" if command[-2:] == ["get-url", "origin"] else None - return subprocess.CompletedProcess(command, 0, stdout=stdout) - - monkeypatch.setattr(coordinator.subprocess, "run", run) - state = coordinator.sync_cassandra_repo({}) - - assert state == "ready; fetch skipped until Gitea token is configured" - assert [command[-3] for command in commands] == ["remote", "set-url"] - - -def test_migrate_open_cassandra_tasks_preserves_running_and_done_tasks(): - """A project switch moves queued work without relocating active evidence.""" - - class Task: - def __init__(self, task_id: str, status: str): - self.id = task_id - self.status = status - self.workspace_kind = "dir" - self.workspace_path = str(coordinator.CASSANDRA_BASE_PATH) - - class Closing: - def __enter__(self): - return object() - - def __exit__(self, *_args): - return None - - class FakeKanban: - tasks = [ - Task("queued", "todo"), - Task("active", "running"), - Task("done", "done"), - ] - moved: list[tuple[str, str]] = [] - - @staticmethod - def connect_closing(*, board: str): - assert board == "cassandra" - return Closing() - - @classmethod - def list_tasks(cls, _connection, *, include_archived: bool): - assert include_archived is False - return cls.tasks - - @classmethod - def set_workspace_path(cls, _connection, task_id: str, path: Path): - cls.moved.append((task_id, str(path))) - - active = Path("/opt/data/workspace/projects/cassandra-hermes-v69") - coordinator._migrate_open_cassandra_tasks(FakeKanban, active) - - assert FakeKanban.moved == [("queued", str(active))] diff --git a/testing/tests/test_hermes_coordinator_cassandra.py b/testing/tests/test_hermes_coordinator_cassandra.py new file mode 100644 index 00000000..646db291 --- /dev/null +++ b/testing/tests/test_hermes_coordinator_cassandra.py @@ -0,0 +1,194 @@ +"""Cassandra workspace and board contracts for the Hermes coordinator.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from testing.tests.test_hermes_coordinator_support import coordinator + + +def test_corrupt_cassandra_board_is_preserved_without_blocking_refresh( + tmp_path: Path, monkeypatch +): + """A damaged project board degrades Kanban instead of stopping all services.""" + + class FakeKanbanDbCorruptError(Exception): + def __init__(self): + super().__init__("integrity check failed") + self.db_path = tmp_path / "kanban.db" + self.backup_path = tmp_path / "kanban.db.corrupt.backup" + self.reason = "row out of order" + + def raise_corruption(_root: Path) -> None: + raise FakeKanbanDbCorruptError() + + monkeypatch.setattr(coordinator, "bootstrap_cassandra", raise_corruption) + monkeypatch.setattr( + coordinator, "_kanban_corruption_type", lambda: FakeKanbanDbCorruptError + ) + + status = coordinator.bootstrap_cassandra_state(tmp_path) + + assert status == { + "state": "corrupt-preserved", + "database": str(tmp_path / "kanban.db"), + "backup": str(tmp_path / "kanban.db.corrupt.backup"), + "reason": "row out of order", + } + + +def test_unrelated_cassandra_bootstrap_failure_remains_fatal( + tmp_path: Path, monkeypatch +): + """Only the known fail-closed corruption state may be degraded.""" + + class FakeKanbanDbCorruptError(Exception): + pass + + def raise_permission_error(_root: Path) -> None: + raise PermissionError("cannot access board") + + monkeypatch.setattr(coordinator, "bootstrap_cassandra", raise_permission_error) + monkeypatch.setattr( + coordinator, "_kanban_corruption_type", lambda: FakeKanbanDbCorruptError + ) + + try: + coordinator.bootstrap_cassandra_state(tmp_path) + except PermissionError as error: + assert str(error) == "cannot access board" + else: + raise AssertionError("unrelated board errors must fail the coordinator refresh") + + +def test_cassandra_workspace_uses_valid_configured_worktree( + tmp_path: Path, monkeypatch +): + """The coordinator selects an explicit Git worktree but rejects stale paths.""" + active = tmp_path / "cassandra-v69" + active.mkdir() + (active / ".git").write_text("gitdir: elsewhere\n", encoding="utf-8") + monkeypatch.setenv("HERMES_CASSANDRA_ACTIVE_WORKTREE", str(active)) + + assert coordinator.cassandra_workspace() == active + + (active / ".git").unlink() + assert coordinator.cassandra_workspace() == coordinator.CASSANDRA_BASE_PATH + + +@pytest.mark.parametrize( + ("current_remote", "repair_action"), + [ + ("https://scm.bstein.dev/bstein/cassandra.git", "set-url"), + (None, "add"), + (coordinator.CASSANDRA_REMOTE, None), + ], + ids=["old-origin", "missing-origin", "canonical-origin"], +) +def test_cassandra_sync_repairs_existing_worktree( + tmp_path: Path, monkeypatch, current_remote: str | None, repair_action: str | None +): + """Existing worktrees use the canonical Atlas origin before fetching.""" + workspace = tmp_path / "cassandra" + (workspace / ".git").mkdir(parents=True) + monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", workspace) + monkeypatch.setattr(coordinator.shutil, "which", lambda _name: "/usr/bin/git") + token_path = tmp_path / "gitea-token" + token_path.write_text("configured\n", encoding="utf-8") + monkeypatch.setenv("HERMES_GITEA_TOKEN_FILE", str(token_path)) + commands: list[list[str]] = [] + environments: list[dict[str, str]] = [] + + def run(command, **kwargs): + commands.append(command) + environments.append(kwargs.get("env", {})) + if command[-2:] == ["get-url", "origin"]: + return subprocess.CompletedProcess( + command, 0 if current_remote else 2, stdout=f"{current_remote or ''}\n" + ) + return subprocess.CompletedProcess(command, 0) + + monkeypatch.setattr(coordinator.subprocess, "run", run) + assert coordinator.sync_cassandra_repo({"GITEA_TOKEN": "must-not-pass"}) == "ready" + repair_commands = [command for command in commands if repair_action in command] + assert bool(repair_commands) is bool(repair_action) + assert commands[-1][-4:] == ["fetch", "--quiet", "--prune", "origin"] + assert all("GITEA_TOKEN" not in environment for environment in environments) + + +def test_cassandra_sync_repairs_origin_through_broker_without_token( + tmp_path: Path, monkeypatch +): + """Repository sync uses the credential-isolated broker remote.""" + monkeypatch.setenv( + "HERMES_GITEA_TOKEN_FILE", str(tmp_path / "explicitly-unavailable-token") + ) + workspace = tmp_path / "cassandra" + (workspace / ".git").mkdir(parents=True) + monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", workspace) + monkeypatch.setattr(coordinator.shutil, "which", lambda _name: "/usr/bin/git") + commands: list[list[str]] = [] + + def run(command, **_kwargs): + commands.append(command) + stdout = "old-origin\n" if command[-2:] == ["get-url", "origin"] else None + return subprocess.CompletedProcess(command, 0, stdout=stdout) + + monkeypatch.setattr(coordinator.subprocess, "run", run) + state = coordinator.sync_cassandra_repo({}) + + assert state == "ready" + assert any( + "hermes-scm-broker.hermes-scm.svc.cluster.local" in " ".join(cmd) + for cmd in commands + ) + assert all("token" not in " ".join(cmd).lower() for cmd in commands) + assert commands[-1][-4:] == ["fetch", "--quiet", "--prune", "origin"] + + +def test_migrate_open_cassandra_tasks_preserves_running_and_done_tasks(): + """A project switch moves queued work without relocating active evidence.""" + + class Task: + def __init__(self, task_id: str, status: str): + self.id = task_id + self.status = status + self.workspace_kind = "dir" + self.workspace_path = str(coordinator.CASSANDRA_BASE_PATH) + + class Closing: + def __enter__(self): + return object() + + def __exit__(self, *_args): + return None + + class FakeKanban: + tasks = [ + Task("queued", "todo"), + Task("active", "running"), + Task("done", "done"), + ] + moved: list[tuple[str, str]] = [] + + @staticmethod + def connect_closing(*, board: str): + assert board == "cassandra" + return Closing() + + @classmethod + def list_tasks(cls, _connection, *, include_archived: bool): + assert include_archived is False + return cls.tasks + + @classmethod + def set_workspace_path(cls, _connection, task_id: str, path: Path): + cls.moved.append((task_id, str(path))) + + active = Path("/opt/data/workspace/projects/cassandra-hermes-v69") + coordinator._migrate_open_cassandra_tasks(FakeKanban, active) + + assert FakeKanban.moved == [("queued", str(active))] diff --git a/testing/tests/test_hermes_coordinator_coverage.py b/testing/tests/test_hermes_coordinator_coverage.py new file mode 100644 index 00000000..fe4f3168 --- /dev/null +++ b/testing/tests/test_hermes_coordinator_coverage.py @@ -0,0 +1,267 @@ +"""Behavioral branch coverage for Hermes project coordination.""" + +from __future__ import annotations + +import json +import subprocess +import sys +import types +from contextlib import nullcontext +from pathlib import Path + +import pytest + +from testing.tests.test_hermes_coordinator_support import coordinator, routing + + +def test_workspace_falls_back_when_unset_or_invalid(tmp_path: Path, monkeypatch): + monkeypatch.delenv("HERMES_CASSANDRA_ACTIVE_WORKTREE", raising=False) + assert coordinator.cassandra_workspace() == coordinator.CASSANDRA_BASE_PATH + monkeypatch.setenv("HERMES_CASSANDRA_ACTIVE_WORKTREE", str(tmp_path / "missing")) + assert coordinator.cassandra_workspace() == coordinator.CASSANDRA_BASE_PATH + + +def test_task_migration_is_noop_for_base_workspace(): + class Board: + @staticmethod + def connect_closing(**_kwargs): + raise AssertionError("base workspace must not access Kanban") + + coordinator._migrate_open_cassandra_tasks(Board, coordinator.CASSANDRA_BASE_PATH) + + +class _Project: + id = "project-id" + + +def _fake_hermes_modules(monkeypatch, *, existing: bool, active: bool): + events = [] + kb = types.ModuleType("hermes_cli.kanban_db") + kb.board_exists = lambda slug: existing + kb.create_board = lambda *args, **kwargs: events.append( + ("create-board", args, kwargs) + ) + kb.set_current_board = lambda slug: events.append(("current", slug)) + kb.connect_closing = lambda **kwargs: nullcontext(object()) + kb.list_tasks = lambda *_a, **_k: [] + + pdb = types.ModuleType("hermes_cli.projects_db") + pdb.connect_closing = lambda: nullcontext(object()) + pdb.get_project = lambda *_a: _Project() if existing else None + pdb.get_active_id = lambda *_a: "active" if active else None + pdb.create_project = ( + lambda *args, **kwargs: events.append(("create-project", kwargs)) or "new-id" + ) + pdb.set_active = lambda *args: events.append(("active", args[-1])) + pdb.update_project = lambda *args, **kwargs: events.append(("update", kwargs)) + pdb.add_folder = lambda *args, **kwargs: events.append(("folder", args[-1], kwargs)) + + package = types.ModuleType("hermes_cli") + package.kanban_db = kb + package.projects_db = pdb + monkeypatch.setitem(sys.modules, "hermes_cli", package) + monkeypatch.setitem(sys.modules, "hermes_cli.kanban_db", kb) + monkeypatch.setitem(sys.modules, "hermes_cli.projects_db", pdb) + return events + + +@pytest.mark.parametrize( + ("existing", "active", "expected"), + [ + (False, False, {"current", "create-project", "active"}), + (False, True, {"current", "create-project"}), + (True, True, {"update", "folder"}), + ], +) +def test_bootstrap_cassandra_creates_or_updates_state( + tmp_path: Path, monkeypatch, existing, active, expected +): + events = _fake_hermes_modules(monkeypatch, existing=existing, active=active) + base = tmp_path / "projects/cassandra" + worktree = tmp_path / "projects/cassandra-worktree" + worktree.mkdir(parents=True) + (worktree / ".git").write_text("gitdir: elsewhere\n", encoding="utf-8") + monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", base) + monkeypatch.setenv("HERMES_CASSANDRA_ACTIVE_WORKTREE", str(worktree)) + coordinator.bootstrap_cassandra(tmp_path / "hermes") + names = {event[0] for event in events} + assert expected <= names + assert "create-board" in names + + +def test_bootstrap_state_ready_path(tmp_path: Path, monkeypatch): + monkeypatch.setattr(coordinator, "bootstrap_cassandra", lambda _root: None) + assert coordinator.bootstrap_cassandra_state(tmp_path) == {"state": "ready"} + + +def test_corruption_type_imports_pinned_exception(monkeypatch): + class Expected(Exception): + pass + + package = types.ModuleType("hermes_cli") + kanban = types.ModuleType("hermes_cli.kanban_db") + kanban.KanbanDbCorruptError = Expected + package.kanban_db = kanban + monkeypatch.setitem(sys.modules, "hermes_cli", package) + monkeypatch.setitem(sys.modules, "hermes_cli.kanban_db", kanban) + assert coordinator._kanban_corruption_type() is Expected + + +def test_repo_sync_reports_missing_git_and_unmanaged_directory( + tmp_path: Path, monkeypatch +): + monkeypatch.setattr(coordinator.shutil, "which", lambda _name: None) + assert coordinator.sync_cassandra_repo({}) == "git-unavailable" + + workspace = tmp_path / "cassandra" + workspace.mkdir() + (workspace / "user-file").write_text("preserve", encoding="utf-8") + monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", workspace) + monkeypatch.setattr(coordinator.shutil, "which", lambda _name: "/usr/bin/git") + assert coordinator.sync_cassandra_repo({}) == "unmanaged-nonempty-directory" + + +@pytest.mark.parametrize( + ("failure", "expected"), + [ + ("get-url-error", "remote-repair-failed"), + ("repair-exit", "remote-repair-failed-9"), + ("repair-error", "remote-repair-failed"), + ("fetch-exit", "sync-failed-8"), + ("fetch-error", "sync-failed"), + ], +) +def test_existing_repo_sync_reports_each_failure( + tmp_path: Path, monkeypatch, failure, expected +): + workspace = tmp_path / "cassandra" + (workspace / ".git").mkdir(parents=True) + monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", workspace) + monkeypatch.setattr(coordinator.shutil, "which", lambda _name: "/usr/bin/git") + + def run(command, **_kwargs): + action = command[-2:] if len(command) >= 2 else [] + if action == ["get-url", "origin"]: + if failure == "get-url-error": + raise OSError("git unavailable") + return subprocess.CompletedProcess(command, 0, stdout="old\n") + if "set-url" in command: + if failure == "repair-error": + raise subprocess.TimeoutExpired(command, 30) + return subprocess.CompletedProcess( + command, 9 if failure == "repair-exit" else 0 + ) + if failure == "fetch-error": + raise OSError("fetch unavailable") + return subprocess.CompletedProcess(command, 8 if failure == "fetch-exit" else 0) + + monkeypatch.setattr(coordinator.subprocess, "run", run) + assert coordinator.sync_cassandra_repo({}) == expected + + +@pytest.mark.parametrize( + ("exit_code", "expected"), [(0, "ready"), (7, "sync-failed-7")] +) +def test_empty_repo_clones_through_broker( + tmp_path: Path, monkeypatch, exit_code, expected +): + workspace = tmp_path / "cassandra" + monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", workspace) + monkeypatch.setattr(coordinator.shutil, "which", lambda _name: "/usr/bin/git") + seen = [] + + def run(command, **kwargs): + seen.append((command, kwargs["env"])) + return subprocess.CompletedProcess(command, exit_code) + + monkeypatch.setattr(coordinator.subprocess, "run", run) + assert ( + coordinator.sync_cassandra_repo( + {"API_SERVER_KEY": "not-forwarded", "SAFE": "yes"} + ) + == expected + ) + assert coordinator.CASSANDRA_REMOTE in seen[0][0] + assert seen[0][1]["SAFE"] == "yes" + assert "API_SERVER_KEY" not in seen[0][1] + + +def _catalog(provider): + return routing.Catalog(provider, ["model"], True, True, "connected") + + +def test_refresh_once_writes_complete_status(tmp_path: Path, monkeypatch): + monkeypatch.setattr(coordinator, "_read_env", lambda _path: {"SAFE": "yes"}) + monkeypatch.setattr(coordinator, "discover_codex_models", lambda: _catalog("codex")) + monkeypatch.setattr( + coordinator, "discover_claude_models", lambda: _catalog("claude") + ) + monkeypatch.setattr( + coordinator, "configure_routes", lambda *_a: {"coordinator": ["auto"]} + ) + monkeypatch.setattr( + coordinator, "bootstrap_cassandra_state", lambda _root: {"state": "ready"} + ) + monkeypatch.setattr(coordinator, "sync_cassandra_repo", lambda _env: "ready") + monkeypatch.setattr( + coordinator, "cassandra_workspace", lambda: tmp_path / "worktree" + ) + written = [] + monkeypatch.setattr( + coordinator, "_atomic_write", lambda path, value: written.append((path, value)) + ) + status = coordinator.refresh_once(tmp_path) + assert status["projects"]["cassandra"]["state"] == "ready" + assert json.loads(written[0][1])["providers"]["codex"]["connected"] is True + + +def test_main_once_success_and_failure(monkeypatch, capsys): + status = {"providers": {"codex": {"state": "ready"}}} + monkeypatch.setattr(sys, "argv", ["coordinator", "--once"]) + monkeypatch.setattr(coordinator, "refresh_once", lambda _root: status) + assert coordinator.main() == 0 + assert "codex=ready" in capsys.readouterr().out + monkeypatch.setattr( + coordinator, + "refresh_once", + lambda _root: (_ for _ in ()).throw(RuntimeError("failed")), + ) + assert coordinator.main() == 1 + assert "RuntimeError" in capsys.readouterr().out + + +@pytest.mark.parametrize(("interval", "expected"), [(1, 300), (400, 400)]) +def test_main_loop_sleeps_with_floor(monkeypatch, interval, expected): + calls = iter(({"providers": {}}, KeyboardInterrupt())) + + def refresh(_root): + value = next(calls) + if isinstance(value, BaseException): + raise value + return value + + sleeps = [] + monkeypatch.setattr(coordinator, "refresh_once", refresh) + monkeypatch.setattr(coordinator.time, "sleep", sleeps.append) + monkeypatch.setattr( + sys, "argv", ["coordinator", "--loop", "--interval", str(interval)] + ) + with pytest.raises(KeyboardInterrupt): + coordinator.main() + assert sleeps == [expected] + + +def test_main_loop_recovers_from_refresh_failure(monkeypatch): + calls = iter((RuntimeError("temporary"), KeyboardInterrupt())) + + def refresh(_root): + error = next(calls) + raise error + + sleeps = [] + monkeypatch.setattr(coordinator, "refresh_once", refresh) + monkeypatch.setattr(coordinator.time, "sleep", sleeps.append) + monkeypatch.setattr(sys, "argv", ["coordinator", "--loop", "--interval", "300"]) + with pytest.raises(KeyboardInterrupt): + coordinator.main() + assert sleeps == [300] diff --git a/testing/tests/test_hermes_coordinator_support.py b/testing/tests/test_hermes_coordinator_support.py new file mode 100644 index 00000000..2a41f070 --- /dev/null +++ b/testing/tests/test_hermes_coordinator_support.py @@ -0,0 +1,35 @@ +"""Shared loader and configuration for Hermes coordinator contracts.""" + +from __future__ import annotations + +import importlib +import importlib.util +import sys +from pathlib import Path + +SCRIPT = Path(__file__).parents[2] / "services/hermes/scripts/hermes_coordinator.py" +sys.path.insert(0, str(SCRIPT.parent)) +routing = importlib.import_module("hermes_model_routing") +catalog_resolver = importlib.import_module("routing_catalog") +SPEC = importlib.util.spec_from_file_location("hermes_coordinator", SCRIPT) +assert SPEC and SPEC.loader +coordinator = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = coordinator +SPEC.loader.exec_module(coordinator) + + +def _base_config() -> dict: + """Return the minimal coordinator configuration used by routing tests.""" + return { + "model": { + "provider": "openai-codex", + "default": "gpt-5.6-terra", + "model": "gpt-5.6-terra", + }, + "fallback_providers": [ + {"provider": "anthropic", "model": "claude-opus-5"}, + dict(routing.LOCAL_FALLBACK), + ], + "toolsets": ["kanban"], + "terminal": {"cwd": "/opt/data/workspace"}, + } diff --git a/testing/tests/test_hermes_deadline_http.py b/testing/tests/test_hermes_deadline_http.py new file mode 100644 index 00000000..45fd07e4 --- /dev/null +++ b/testing/tests/test_hermes_deadline_http.py @@ -0,0 +1,309 @@ +"""Absolute-deadline contracts for the killable HTTP control helper.""" + +from __future__ import annotations + +import base64 +import io +import json +import subprocess +import threading +import time +import urllib.request +from http.server import BaseHTTPRequestHandler, HTTPServer + +import pytest + +from testing.tests.test_hermes_scm_broker_support import _load + + +class _Handler(BaseHTTPRequestHandler): + def _respond(self): + if self.path == "/slow": + time.sleep(10) + if self.path == "/redirect": + self.send_response(302) + self.send_header("Location", "http://127.0.0.1:1/evil") + self.send_header("Content-Length", "0") + self.end_headers() + return + status = 404 if self.path == "/missing" else 200 + body = b"x" * 64 if self.path == "/large" else b'{"ok":true}' + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + do_GET = _respond + do_POST = _respond + + def log_message(self, *_args): + return + + +@pytest.fixture(scope="module") +def server(): + value = HTTPServer(("127.0.0.1", 0), _Handler) + thread = threading.Thread(target=value.serve_forever, daemon=True) + thread.start() + yield f"http://127.0.0.1:{value.server_port}" + value.shutdown() + + +def _control(url: str, **overrides): + value = { + "url": url, + "method": "GET", + "headers": {"Accept": "application/json"}, + "body": "", + "maximum": 1024, + "timeout": 10, + } + value.update(overrides) + return value + + +def test_child_exchange_round_trips_and_reports_http_errors(server): + module = _load("deadline_http") + result = module._child_exchange(_control(server + "/ok")) + assert (result.status, result.content_type) == (200, "application/json") + assert result.body == b'{"ok":true}' + + posted = module._child_exchange( + _control( + server + "/ok", + method="POST", + body=base64.b64encode(b'{"a":1}').decode(), + ) + ) + assert posted.status == 200 + + missing = module._child_exchange(_control(server + "/missing")) + assert missing.status == 404 + + +def test_child_exchange_rejects_redirects_and_oversized_responses(server): + module = _load("deadline_http") + with pytest.raises(Exception): + module._child_exchange(_control(server + "/redirect")) + with pytest.raises(ValueError, match="too large"): + module._child_exchange(_control(server + "/large", maximum=8)) + + +@pytest.mark.parametrize( + "overrides", + [ + {"url": 1}, + {"method": None}, + {"headers": ["not", "dict"]}, + {"headers": {"Accept": 5}}, + {"body": b"raw"}, + {"maximum": "big"}, + {"maximum": -1}, + {"maximum": 4 * 1024 * 1024}, + {"timeout": "soon"}, + {"timeout": 0}, + {"timeout": 121}, + ], +) +def test_child_exchange_rejects_invalid_control_documents(overrides): + module = _load("deadline_http") + control = _control("http://127.0.0.1:1/") + control.update(overrides) + with pytest.raises(ValueError, match="invalid control"): + module._child_exchange(control) + + +def test_child_exchange_rejects_bad_or_oversized_encoded_bodies(): + module = _load("deadline_http") + with pytest.raises(ValueError): + module._child_exchange(_control("http://127.0.0.1:1/", body="!!not-b64!!")) + huge = base64.b64encode(b"x" * (module.MAX_CONTROL_BYTES + 1)).decode() + with pytest.raises(ValueError, match="request too large"): + module._child_exchange(_control("http://127.0.0.1:1/", body=huge)) + + +def test_read_response_enforces_deadline_and_size(): + module = _load("deadline_http") + + class Reader: + def __init__(self, body: bytes): + self.stream = io.BytesIO(body) + + def read(self, limit: int): + return self.stream.read(limit) + + with pytest.raises(TimeoutError): + module._read_response(Reader(b"abc"), 10, time.monotonic() - 1) + with pytest.raises(ValueError, match="too large"): + module._read_response(Reader(b"abcdef"), 4, time.monotonic() + 5) + value = module._read_response(Reader(b"abc"), 10, time.monotonic() + 5) + assert value == b"abc" + + +def _run_child_main(module, monkeypatch, raw: bytes) -> dict: + stdin = type("Stdin", (), {"buffer": io.BytesIO(raw)})() + stdout = io.StringIO() + monkeypatch.setattr(module.sys, "stdin", stdin) + monkeypatch.setattr(module.sys, "stdout", stdout) + code = module._child_main() + return code, json.loads(stdout.getvalue()) + + +def test_child_main_reports_success_and_failure_documents(monkeypatch, server): + module = _load("deadline_http") + code, output = _run_child_main( + module, monkeypatch, json.dumps(_control(server + "/ok")).encode() + ) + assert code == 0 and output["ok"] is True + assert base64.b64decode(output["body"]) == b'{"ok":true}' + + for raw in ( + b"x" * (module.MAX_CONTROL_BYTES + 1), + b"not json", + b"[1,2]", + ): + code, output = _run_child_main(module, monkeypatch, raw) + assert code == 1 and output == {"ok": False} + + +class _FakeProcess: + def __init__(self, output: bytes, returncode: int = 0): + self.args = ["fake-child"] + self.pid = 0 + self.returncode = returncode + self.output = output + self.stdin_payload = None + + def communicate(self, payload=None, timeout=None): + self.stdin_payload = payload + return self.output, b"" + + +def _fake_popen(output: bytes, returncode: int = 0): + def popen(*_args, **_kwargs): + return _FakeProcess(output, returncode) + + return popen + + +def _success_output(body: bytes, **overrides) -> bytes: + value = { + "ok": True, + "status": 200, + "content_type": "application/json", + "body": base64.b64encode(body).decode("ascii"), + } + value.update(overrides) + return json.dumps(value).encode() + + +def test_exchange_validates_bounds_and_request_body(): + module = _load("deadline_http") + request = urllib.request.Request("https://scm.bstein.dev/api/v1/x") + with pytest.raises(module.PolicyError, match="bounds are invalid"): + module.exchange(request, maximum=1, timeout=0) + with pytest.raises(module.PolicyError, match="bounds are invalid"): + module.exchange(request, maximum=-1, timeout=10) + request.data = "not-bytes" + with pytest.raises(module.PolicyError, match="safe size"): + module.exchange(request, maximum=1, timeout=10) + request.data = b"x" * (module.MAX_CONTROL_BYTES + 1) + with pytest.raises(module.PolicyError, match="safe size"): + module.exchange(request, maximum=1, timeout=10) + + +def test_exchange_returns_result_and_enforces_response_limit(): + module = _load("deadline_http") + request = urllib.request.Request( + "https://scm.bstein.dev/api/v1/x", data=b"{}", method="POST" + ) + result = module.exchange( + request, maximum=64, timeout=10, popen=_fake_popen(_success_output(b"body")) + ) + assert (result.status, result.content_type, result.body) == ( + 200, + "application/json", + b"body", + ) + with pytest.raises(module.PolicyError, match="safe size"): + module.exchange( + request, + maximum=2, + timeout=10, + popen=_fake_popen(_success_output(b"body")), + ) + + +@pytest.mark.parametrize( + ("output", "returncode", "match"), + [ + (b"", 1, "request failed"), + (b"x" * (4 * 1024 * 1024 + 1), 0, "request failed"), + (b"not json", 0, "invalid evidence"), + (b'{"ok":false}', 0, "invalid evidence"), + (b"[1]", 0, "invalid evidence"), + (b'{"ok":true}', 0, "invalid evidence"), + (b'{"ok":true,"status":"200","content_type":"a","body":""}', 0, "invalid evidence"), + (b'{"ok":true,"status":200,"content_type":"a","body":"!!"}', 0, "invalid evidence"), + ], +) +def test_exchange_rejects_invalid_helper_evidence(output, returncode, match): + module = _load("deadline_http") + request = urllib.request.Request("https://scm.bstein.dev/api/v1/x") + with pytest.raises(module.PolicyError, match=match): + module.exchange( + request, maximum=64, timeout=10, popen=_fake_popen(output, returncode) + ) + + +def test_exchange_kills_the_helper_when_the_deadline_expires(server): + module = _load("deadline_http") + request = urllib.request.Request(server + "/slow") + started = time.monotonic() + with pytest.raises(module.PolicyError, match="deadline exceeded"): + module.exchange(request, maximum=64, timeout=1.0) + assert time.monotonic() - started < 5 + + +def test_exchange_fails_before_spawn_wait_when_time_is_exhausted(monkeypatch): + module = _load("deadline_http") + request = urllib.request.Request("https://scm.bstein.dev/api/v1/x") + killed = [] + monkeypatch.setattr(module.os, "killpg", lambda pid, sig: killed.append((pid, sig))) + clock = iter([0.0, 1_000_000.0, 1_000_000.0, 1_000_000.0]) + monkeypatch.setattr(module.time, "monotonic", lambda: next(clock)) + with pytest.raises(module.PolicyError, match="deadline exceeded"): + module.exchange( + request, maximum=64, timeout=10, popen=_fake_popen(_success_output(b"")) + ) + assert killed + + +def test_terminate_tolerates_already_finished_helpers(): + module = _load("deadline_http") + process = subprocess.Popen( + ["/bin/true"], start_new_session=True, stdout=subprocess.DEVNULL + ) + process.wait() + module._terminate(process) + assert process.returncode == 0 + + +def test_child_entrypoint_round_trip_through_real_subprocess(server): + module = _load("deadline_http") + request = urllib.request.Request(server + "/ok") + result = module.exchange(request, maximum=1024, timeout=20) + assert result.status == 200 and result.body == b'{"ok":true}' + + +def test_open_bounded_adapts_result_to_the_urlopen_interface(): + module = _load("deadline_http") + request = urllib.request.Request("https://scm.bstein.dev/api/v1/x") + with module.open_bounded( + request, maximum=64, timeout=10, popen=_fake_popen(_success_output(b"body")) + ) as response: + assert response.status == 200 + assert response.headers.get_content_type() == "application/json" + assert response.read(2) == b"bo" + assert response.read() == b"dy" diff --git a/testing/tests/test_hermes_deadline_stream.py b/testing/tests/test_hermes_deadline_stream.py new file mode 100644 index 00000000..88022591 --- /dev/null +++ b/testing/tests/test_hermes_deadline_stream.py @@ -0,0 +1,304 @@ +"""Watchdog contracts for absolute deadlines on streaming Git exchanges.""" + +from __future__ import annotations + +import threading +import time +import urllib.request +from http.server import BaseHTTPRequestHandler, HTTPServer + +import pytest + +from testing.tests.test_hermes_scm_broker_support import Response, _load + + +def test_stream_deadline_validates_bounds_and_counts_down(): + module = _load("deadline_http") + for timeout in (0, -1, module.MAX_STREAM_SECONDS + 1): + with pytest.raises(module.PolicyError, match="bounds are invalid"): + module.StreamDeadline(timeout) + guard = module.StreamDeadline(30) + try: + first = guard.remaining() + assert 0 < first <= 30 + finally: + guard.cancel() + + +class _FakeTimer: + def __init__(self, timeout, callback): + self.timeout = timeout + self.callback = callback + self.daemon = False + self.started = False + self.cancelled = False + + def start(self): + self.started = True + + def cancel(self): + self.cancelled = True + + +class _FakeSocket: + def __init__(self, fail: bool = False): + self.fail = fail + self.shut = False + + def shutdown(self, _how): + if self.fail: + raise OSError("already gone") + self.shut = True + + +class _FakeConnection: + def __init__(self, sock=None): + self.sock = sock + self.closed = False + + def close(self): + self.closed = True + + +def test_expiry_shuts_down_and_closes_every_tracked_connection(): + module = _load("deadline_http") + guard = module.StreamDeadline(30, timer=_FakeTimer) + assert guard._timer.started and guard._timer.daemon + healthy = _FakeConnection(_FakeSocket()) + broken = _FakeConnection(_FakeSocket(fail=True)) + bare = _FakeConnection() + for connection in (healthy, broken, bare): + assert guard._track(connection) is connection + guard._expire() + assert guard.expired + assert healthy.sock.shut and healthy.closed + assert broken.closed and not broken.sock.shut + assert bare.closed + with pytest.raises(module.PolicyError, match="deadline exceeded"): + guard.remaining() + late = _FakeConnection() + with pytest.raises(module.PolicyError, match="deadline exceeded"): + guard._track(late) + assert late.closed + guard.cancel() + assert guard._timer.cancelled + + +def test_handlers_build_guarded_connections_without_network(monkeypatch): + module = _load("deadline_http") + guard = module.StreamDeadline(30, timer=_FakeTimer) + http_handler, https_handler = guard.handlers() + assert isinstance(http_handler, urllib.request.HTTPHandler) + assert isinstance(https_handler, urllib.request.HTTPSHandler) + for handler, opener_name in ( + (http_handler, "http_open"), + (https_handler, "https_open"), + ): + seen = {} + monkeypatch.setattr( + handler, + "do_open", + lambda factory, req, seen=seen: seen.update(factory=factory, req=req), + ) + getattr(handler, opener_name)("request") + connection = seen["factory"]("127.0.0.1", timeout=1) + assert connection in guard._connections + connection.close() + guard.cancel() + + +def test_dns_resolution_is_bounded_by_the_stream_deadline(monkeypatch): + module = _load("deadline_http") + + def hang(*_args, **_kwargs): + time.sleep(30) + + monkeypatch.setattr(module.socket, "getaddrinfo", hang) + started = time.monotonic() + with pytest.raises(module.PolicyError, match="DNS resolution deadline"): + module._resolve_within("example.invalid", 443, 0.4) + assert time.monotonic() - started < 3 + + +def test_resolve_within_returns_on_success_and_propagates_errors(monkeypatch): + module = _load("deadline_http") + calls = [] + monkeypatch.setattr( + module.socket, "getaddrinfo", lambda *a, **k: calls.append((a, k)) + ) + module._resolve_within("127.0.0.1", 80, 5) + assert calls + + def boom(*_a, **_k): + raise OSError("name resolution failed") + + monkeypatch.setattr(module.socket, "getaddrinfo", boom) + with pytest.raises(OSError, match="name resolution failed"): + module._resolve_within("127.0.0.1", 80, 5) + + +def test_bounded_connection_wraps_connect_to_resolve_first(monkeypatch): + module = _load("deadline_http") + guard = module.StreamDeadline(30, timer=_FakeTimer) + order = [] + monkeypatch.setattr( + module, + "_resolve_within", + lambda host, port, timeout: order.append(("resolve", host, port)), + ) + + class FakeConnection: + host = "scm.bstein.dev" + port = 443 + + def __init__(self): + self.sock = None + + def connect(self): + order.append(("connect", self.host)) + + def close(self): + order.append(("close",)) + + connection = guard._bounded_connection(FakeConnection()) + assert connection in guard._connections + connection.connect() + assert order == [("resolve", "scm.bstein.dev", 443), ("connect", "scm.bstein.dev")] + guard.cancel() + + +def test_bounded_connection_connect_fails_closed_once_expired(monkeypatch): + module = _load("deadline_http") + guard = module.StreamDeadline(30, timer=_FakeTimer) + resolved = [] + monkeypatch.setattr( + module, "_resolve_within", lambda *a, **k: resolved.append(a) + ) + + class FakeConnection: + host = "scm.bstein.dev" + port = 443 + sock = None + + def connect(self): + resolved.append("connected") + + def close(self): + pass + + connection = guard._bounded_connection(FakeConnection()) + guard._expire() + with pytest.raises(module.PolicyError, match="deadline exceeded"): + connection.connect() + assert resolved == [] + guard.cancel() + + +class _TrickleHandler(BaseHTTPRequestHandler): + def do_GET(self): + time.sleep(10) + + def log_message(self, *_args): + return + + +def test_watchdog_interrupts_a_hung_streaming_exchange(): + module = _load("deadline_http") + server = HTTPServer(("127.0.0.1", 0), _TrickleHandler) + threading.Thread(target=server.serve_forever, daemon=True).start() + guard = module.StreamDeadline(1.0) + opener = urllib.request.build_opener(*guard.handlers()) + started = time.monotonic() + try: + with pytest.raises(OSError): + opener.open( + f"http://127.0.0.1:{server.server_port}/slow", timeout=30 + ).read() + assert time.monotonic() - started < 5 + assert guard.expired + finally: + guard.cancel() + server.shutdown() + + +def test_upstream_git_request_runs_under_one_wall_clock_budget(monkeypatch): + broker = _load("scm_broker") + seen = {} + + class RecordingGuard: + def __init__(self, timeout, **_kwargs): + seen["timeout"] = timeout + self.cancelled = False + seen["guard"] = self + + def remaining(self): + return 12.5 + + def cancel(self): + self.cancelled = True + + def handlers(self): + return () + + monkeypatch.setattr(broker.deadline_http, "StreamDeadline", RecordingGuard) + + def opener(request, timeout): + seen["opener_timeout"] = timeout + return Response(b"result", content_type="application/x-git-upload-pack-result") + + result = broker._upstream_git_request( + "/atlas/cassandra.git/git-upload-pack", + method="POST", + body=b"request", + content_type="application/x-git-upload-pack-request", + expected_type="application/x-git-upload-pack-result", + token="sentinel", + opener=opener, + ) + assert result == b"result" + assert seen["timeout"] == broker.UPSTREAM_DEADLINE_SECONDS + assert seen["opener_timeout"] == 12.5 + assert seen["guard"].cancelled + + +def test_upstream_git_request_cancels_guard_when_deadline_already_passed(monkeypatch): + broker = _load("scm_broker") + cancelled = [] + + class ExpiredGuard: + def __init__(self, _timeout, **_kwargs): + pass + + def remaining(self): + raise broker.PolicyError("HTTP stream deadline exceeded") + + def cancel(self): + cancelled.append(True) + + def handlers(self): + return () + + monkeypatch.setattr(broker.deadline_http, "StreamDeadline", ExpiredGuard) + with pytest.raises(broker.PolicyError, match="deadline exceeded"): + broker._upstream_git_request( + "/atlas/cassandra.git/git-upload-pack", + method="POST", + body=b"request", + content_type="application/x-git-upload-pack-request", + expected_type="application/x-git-upload-pack-result", + token="sentinel", + opener=lambda *_a, **_k: None, + ) + assert cancelled == [True] + + +def test_guarded_opener_wires_redirect_rejection_and_deadline_handlers(): + broker = _load("scm_broker") + module = _load("deadline_http") + guard = module.StreamDeadline(30, timer=_FakeTimer) + opener = broker._guarded_opener(guard) + assert callable(opener) + handlers = opener.__self__.handlers + assert any(isinstance(item, broker.RejectRedirect) for item in handlers) + assert any(isinstance(item, urllib.request.HTTPSHandler) for item in handlers) + guard.cancel() diff --git a/testing/tests/test_hermes_git_pack_objects.py b/testing/tests/test_hermes_git_pack_objects.py new file mode 100644 index 00000000..f3a249ec --- /dev/null +++ b/testing/tests/test_hermes_git_pack_objects.py @@ -0,0 +1,288 @@ +"""Bounded pack inflation and delta-resolution contracts for pushed objects.""" + +from __future__ import annotations + +import hashlib +import io +import zlib + +import pytest + +from testing.tests.test_hermes_scm_broker_support import ( + _load, + _object_entry, + _pack_of, +) + + +def _delta_header(type_code: int, size: int, suffix: bytes = b"") -> bytes: + byte = (type_code << 4) | (size & 0x0F) + size >>= 4 + header = bytearray() + while size: + header.append(byte | 0x80) + byte = size & 0x7F + size >>= 7 + header.append(byte) + return bytes(header) + suffix + + +def _size_varint(value: int) -> bytes: + out = bytearray() + while True: + byte = value & 0x7F + value >>= 7 + if value: + out.append(byte | 0x80) + else: + out.append(byte) + return bytes(out) + + +def _ofs_distance(value: int) -> bytes: + out = [value & 0x7F] + value >>= 7 + while value: + value -= 1 + out.insert(0, 0x80 | (value & 0x7F)) + value >>= 7 + return bytes(out) + + +def _delta(source: bytes, target_size: int, instructions: bytes) -> bytes: + return _size_varint(len(source)) + _size_varint(target_size) + instructions + + +def _blob_sha(payload: bytes) -> bytes: + return hashlib.sha1(b"blob %d\x00" % len(payload) + payload).digest() + + +def _read_all(objects): + try: + return [(code, payload.read()) for code, payload in objects] + finally: + for _code, payload in objects: + payload.close() + + +def test_unpack_returns_typed_payloads_for_every_base_object_kind(): + module = _load("git_pack_objects") + entries = [ + _object_entry(1, b"tree 1\n"), + _object_entry(2, b"\x00tree-bytes"), + _object_entry(3, b"blob contents"), + _object_entry(4, b"tag contents"), + ] + result = _read_all(module.unpack_objects(io.BytesIO(_pack_of(entries)))) + assert result == [ + (1, b"tree 1\n"), + (2, b"\x00tree-bytes"), + (3, b"blob contents"), + (4, b"tag contents"), + ] + + +def test_unpack_accepts_the_empty_pack(): + module = _load("git_pack_objects") + assert module.unpack_objects(io.BytesIO(_pack_of([]))) == [] + + +def test_ofs_delta_resolves_against_its_in_pack_base(): + module = _load("git_pack_objects") + base = b"base payload for copying" + base_entry = _object_entry(3, base) + # Copy the first 4 bytes, then insert new bytes. + instructions = bytes([0x80 | 0x10, 4]) + bytes([5]) + b"+tail" + delta = _delta(base, 9, instructions) + delta_entry = _delta_header( + 6, len(delta), _ofs_distance(len(base_entry)) + ) + zlib.compress(delta) + result = _read_all(module.unpack_objects(io.BytesIO(_pack_of([base_entry, delta_entry])))) + assert result == [(3, base), (3, b"base+tail")] + + +def test_ref_delta_resolves_by_sha_and_supports_chains(): + module = _load("git_pack_objects") + base = b"chain base" + first = _delta(base, 4, bytes([4]) + b"one!") + second = _delta(b"one!", 4, bytes([4]) + b"two!") + entries = [ + _object_entry(3, base), + _delta_header(7, len(first), _blob_sha(base)) + zlib.compress(first), + _delta_header(7, len(second), _blob_sha(b"one!")) + zlib.compress(second), + ] + result = _read_all(module.unpack_objects(io.BytesIO(_pack_of(entries)))) + assert [payload for _code, payload in result] == [base, b"one!", b"two!"] + + +def test_thin_pack_ref_delta_with_external_base_is_rejected(): + module = _load("git_pack_objects") + delta = _delta(b"absent", 1, bytes([1]) + b"x") + entry = _delta_header(7, len(delta), b"\xaa" * 20) + zlib.compress(delta) + with pytest.raises(module.PolicyError, match="push full packs"): + module.unpack_objects(io.BytesIO(_pack_of([entry]))) + + +def test_ofs_delta_bad_offsets_fail_closed(): + module = _load("git_pack_objects") + base_entry = _object_entry(3, b"base") + delta = _delta(b"base", 1, bytes([1]) + b"x") + # Distance beyond the pack start. + early = _delta_header(6, len(delta), _ofs_distance(999)) + zlib.compress(delta) + with pytest.raises(module.PolicyError, match="base offset is invalid"): + module.unpack_objects(io.BytesIO(_pack_of([base_entry, early]))) + # Distance landing between object boundaries. + misaligned = _delta_header(6, len(delta), _ofs_distance(3)) + zlib.compress(delta) + with pytest.raises(module.PolicyError, match="base offset is invalid"): + module.unpack_objects(io.BytesIO(_pack_of([base_entry, misaligned]))) + + +@pytest.mark.parametrize( + ("mutate", "match"), + [ + (lambda pack: b"JUNK" + pack[4:], "header is invalid"), + (lambda pack: pack[:4] + (9).to_bytes(4, "big") + pack[8:], "header is invalid"), + (lambda pack: pack[:8] + (2).to_bytes(4, "big") + pack[12:], "Git"), + (lambda pack: pack[:-1], "ended early"), + (lambda pack: pack[:-20] + b"\x00" * 20, "checksum is invalid"), + (lambda pack: pack[:11], "ended early"), + ], +) +def test_malformed_pack_containers_fail_closed(mutate, match): + module = _load("git_pack_objects") + pack = _pack_of([_object_entry(3, b"payload")]) + with pytest.raises(module.PolicyError, match=match): + module.unpack_objects(io.BytesIO(mutate(pack))) + + +def test_object_count_and_type_bounds_fail_closed(monkeypatch): + module = _load("git_pack_objects") + body = b"PACK" + (2).to_bytes(4, "big") + (module.MAX_PACK_OBJECTS + 1).to_bytes( + 4, "big" + ) + with pytest.raises(module.PolicyError, match="too many objects"): + module.unpack_objects(io.BytesIO(body + hashlib.sha1(body).digest())) + reserved = _delta_header(5, 1) + zlib.compress(b"x") + with pytest.raises(module.PolicyError, match="object type is invalid"): + module.unpack_objects(io.BytesIO(_pack_of([reserved]))) + + +def test_declared_size_mismatches_and_zlib_bombs_fail_closed(monkeypatch): + module = _load("git_pack_objects") + longer = _delta_header(3, 2) + zlib.compress(b"longer-than-two") + with pytest.raises(module.PolicyError, match="does not match its header"): + module.unpack_objects(io.BytesIO(_pack_of([longer]))) + shorter = _delta_header(3, 9) + zlib.compress(b"x") + with pytest.raises(module.PolicyError, match="does not match its header"): + module.unpack_objects(io.BytesIO(_pack_of([shorter]))) + corrupt = _delta_header(3, 4) + b"\x00not-zlib\x00" + with pytest.raises(module.PolicyError, match="corrupt"): + module.unpack_objects(io.BytesIO(_pack_of([corrupt]))) + + monkeypatch.setattr(module, "MAX_OBJECT_BYTES", 4) + with pytest.raises(module.PolicyError, match="exceeds the safe size limit"): + module.unpack_objects(io.BytesIO(_pack_of([_object_entry(3, b"12345")]))) + monkeypatch.setattr(module, "MAX_OBJECT_BYTES", 1024) + monkeypatch.setattr(module, "MAX_TOTAL_BYTES", 8) + with pytest.raises(module.PolicyError, match="contents exceed"): + module.unpack_objects( + io.BytesIO(_pack_of([_object_entry(3, b"12345"), _object_entry(3, b"12345")])) + ) + + +def test_object_and_distance_varints_are_bounded(): + module = _load("git_pack_objects") + with pytest.raises(module.PolicyError, match="object header is invalid"): + module._object_header(io.BytesIO(b"\xff" * 12)) + with pytest.raises(module.PolicyError, match="ended early"): + module._object_header(io.BytesIO(b"\x80")) + with pytest.raises(module.PolicyError, match="distance is invalid"): + module._delta_distance(io.BytesIO(b"\xff" * 12)) + + +@pytest.mark.parametrize( + ("base", "delta", "match"), + [ + (b"base", b"", "delta header is invalid"), + (b"base", _size_varint(4), "delta header is invalid"), + (b"base", _size_varint(9) + _size_varint(1) + bytes([1]) + b"x", "sizes are invalid"), + (b"base", _size_varint(4) + _size_varint(2) + bytes([0]), "instruction is invalid"), + (b"base", _size_varint(4) + _size_varint(2) + bytes([5]) + b"ab", "insert is truncated"), + (b"base", _size_varint(4) + _size_varint(2) + bytes([0x80 | 0x10]), "copy is truncated"), + ( + b"base", + _size_varint(4) + _size_varint(2) + bytes([0x80 | 0x01 | 0x10, 3, 9]), + "copy is out of bounds", + ), + (b"base", _size_varint(4) + _size_varint(1) + bytes([2]) + b"ab", "exceeds its declared size"), + (b"base", _size_varint(4) + _size_varint(3) + bytes([2]) + b"ab", "result size is invalid"), + ], +) +def test_delta_application_rejects_malformed_instructions(base, delta, match): + module = _load("git_pack_objects") + with pytest.raises(module.PolicyError, match=match): + module._apply_delta(base, delta) + + +def test_delta_copy_with_zero_size_field_means_the_git_default(): + module = _load("git_pack_objects") + base = b"z" * 0x10000 + delta = _delta(base, 0x10000, bytes([0x80 | 0x10 | 0x20, 0, 0])) + assert module._apply_delta(base, delta) == base + + +def test_delta_target_size_cap_applies(monkeypatch): + module = _load("git_pack_objects") + monkeypatch.setattr(module, "MAX_OBJECT_BYTES", 4) + delta = _delta(b"base", 5, bytes([5]) + b"abcde") + with pytest.raises(module.PolicyError, match="sizes are invalid"): + module._apply_delta(b"base", delta) + + +def test_resolution_pass_and_total_caps_apply(monkeypatch): + module = _load("git_pack_objects") + base = b"pass base" + first = _delta(base, 4, bytes([4]) + b"one!") + second = _delta(b"one!", 4, bytes([4]) + b"two!") + entries = [ + _delta_header(7, len(second), _blob_sha(b"one!")) + zlib.compress(second), + _object_entry(3, base), + _delta_header(7, len(first), _blob_sha(base)) + zlib.compress(first), + ] + pack = _pack_of(entries) + monkeypatch.setattr(module, "MAX_RESOLVE_PASSES", 1) + with pytest.raises(module.PolicyError, match="resolution limit"): + module.unpack_objects(io.BytesIO(pack)) + monkeypatch.setattr(module, "MAX_RESOLVE_PASSES", 64) + result = _read_all(module.unpack_objects(io.BytesIO(pack))) + assert [payload for _code, payload in result] == [b"two!", base, b"one!"] + + monkeypatch.setattr(module, "MAX_TOTAL_BYTES", len(base) + len(first) + len(second) + 1) + with pytest.raises(module.PolicyError, match="contents exceed"): + module.unpack_objects(io.BytesIO(pack)) + + +def test_parse_loose_object_round_trip_and_bounds(monkeypatch): + module = _load("git_pack_objects") + payload = b"loose blob" + value = zlib.compress(b"blob %d\x00" % len(payload) + payload) + assert module.parse_loose_object(value) == (3, payload) + + with pytest.raises(module.PolicyError, match="corrupt"): + module.parse_loose_object(b"\x00junk") + with pytest.raises(module.PolicyError, match="framing is invalid"): + module.parse_loose_object(value + b"trailing") + with pytest.raises(module.PolicyError, match="framing is invalid"): + module.parse_loose_object(value[:-4]) + for header in ( + b"noodle 3\x00abc", + b"blob3\x00abc", + b"blob x\x00abc", + b"blob 4\x00abc", + b"blob 3", + ): + with pytest.raises(module.PolicyError, match="header is invalid"): + module.parse_loose_object(zlib.compress(header)) + monkeypatch.setattr(module, "MAX_OBJECT_BYTES", 4) + with pytest.raises(module.PolicyError, match="exceeds the safe size limit"): + module.parse_loose_object(b"12345") \ No newline at end of file diff --git a/testing/tests/test_hermes_gitea_branch_coverage.py b/testing/tests/test_hermes_gitea_branch_coverage.py new file mode 100644 index 00000000..48421f01 --- /dev/null +++ b/testing/tests/test_hermes_gitea_branch_coverage.py @@ -0,0 +1,209 @@ +"""Behavioral branch coverage for the Gitea protection policy checker.""" + +from __future__ import annotations + +import json +import os +import stat +import sys +from pathlib import Path + +import pytest + +from testing.tests.test_hermes_scm_broker_support import ROOT, _load_path + + +def _module(name: str = "branch_protection_coverage"): + return _load_path( + name, + ROOT / "services/gitea/scripts/gitea_branch_protection_check.py", + ) + + +def _rule(module, name: str = "main", **updates): + value = { + "rule_name": name, + "priority": 1, + "created_at": "2026-01-01T00:00:00Z", + **module._required("bstein"), + } + value.update(updates) + return value + + +def test_read_bounded_accepts_regular_file_and_closes_descriptor( + tmp_path: Path, monkeypatch +): + module = _module("branch_read_regular") + source = tmp_path / "rules.json" + source.write_bytes(b"[]") + closed: list[int] = [] + real_close = os.close + + def close(descriptor: int) -> None: + closed.append(descriptor) + real_close(descriptor) + + monkeypatch.setattr(module.os, "close", close) + assert module._read_bounded(source) == b"[]" + assert len(closed) == 1 + + +@pytest.mark.parametrize("kind", ["directory", "oversized", "short"]) +def test_read_bounded_rejects_unsafe_or_changed_input( + tmp_path: Path, monkeypatch, kind +): + module = _module(f"branch_read_{kind}") + source = tmp_path / "rules" + source.write_bytes(b"[]") + real_fstat = module.os.fstat + real_read = module.os.read + + if kind == "directory": + monkeypatch.setattr( + module.os, + "fstat", + lambda descriptor: os.stat_result( + (stat.S_IFDIR | 0o700, 0, 0, 0, 0, 0, 2, 0, 0, 0) + ), + ) + elif kind == "oversized": + monkeypatch.setattr( + module.os, + "fstat", + lambda descriptor: os.stat_result( + (stat.S_IFREG | 0o600, 0, 0, 0, 0, 0, module.MAX_INPUT + 1, 0, 0, 0) + ), + ) + else: + monkeypatch.setattr(module.os, "fstat", real_fstat) + monkeypatch.setattr(module.os, "read", lambda descriptor, maximum: b"[") + with pytest.raises(module.PolicyError): + module._read_bounded(source) + monkeypatch.setattr(module.os, "read", real_read) + + +@pytest.mark.parametrize( + "pattern", + [ + "{main}", + "{main,master}", + "m?in", + "m[ai]in", + "m[!z]in", + "m[a-z]in", + "m\\ain", + "m\\", + "m[", + "m[]", + "m[!]", + "m[[a]", + "m[\\a]", + "m[-a]", + "m[a-]", + "m[a-b-c]", + "m[z-a]", + "m[aa-b]", + "m{ain}", + "m{ain,}", + "m{ain,aster", + ], +) +def test_every_special_or_malformed_glob_fails_closed(pattern): + module = _module("branch_glob_invalid") + with pytest.raises(module.PolicyError, match="glob branch protection"): + module._literal_rule(pattern) + + +@pytest.mark.parametrize("pattern", ["", "x" * 256, "máin"]) +def test_rule_names_are_ascii_and_bounded(pattern): + module = _module("branch_name_bounds") + with pytest.raises(module.PolicyError, match="rule name"): + module._literal_rule(pattern) + + +def test_plain_rule_matching_is_exact_and_case_sensitive(): + module = _module("branch_literal_case") + assert module._literal_rule("main") == "main" + assert module._literal_rule("MAIN") == "MAIN" + assert module.evaluate( + json.dumps([_rule(module, name="MAIN")]).encode(), "main", "bstein" + ) == "ABSENT" + + +@pytest.mark.parametrize("value", [None, "x" * 65, "invalid", "2026-01-01T00:00:00"]) +def test_created_at_requires_short_timezone_aware_iso(value): + module = _module("branch_created_bounds") + with pytest.raises(module.PolicyError, match="creation time"): + module._created(value) + + +@pytest.mark.parametrize( + "value", + [b"x" * (1024 * 1024 + 1), b"{}", json.dumps([None]).encode()], +) +def test_evaluate_rejects_invalid_top_level_inputs(value): + module = _module("branch_evaluate_top") + with pytest.raises((module.PolicyError, json.JSONDecodeError)): + module.evaluate(value, "main", "bstein") + + +def test_evaluate_rejects_unknown_branch_and_excess_rules(): + module = _module("branch_evaluate_bounds") + with pytest.raises(module.PolicyError, match="input"): + module.evaluate(b"[]", "release", "bstein") + with pytest.raises(module.PolicyError, match="response"): + module.evaluate(json.dumps([{}] * 101).encode(), "main", "bstein") + + +@pytest.mark.parametrize("priority", [True, -1, 1_000_001, "1"]) +def test_evaluate_rejects_ambiguous_priority_types(priority): + module = _module("branch_priority_bounds") + rule = _rule(module, priority=priority) + with pytest.raises(module.PolicyError, match="priority"): + module.evaluate(json.dumps([rule]).encode(), "main", "bstein") + + +def test_legacy_zero_priority_is_accepted_but_duplicate_ties_fail_closed(): + module = _module("branch_priority_zero") + assert module.evaluate( + json.dumps([_rule(module, priority=0)]).encode(), "main", "bstein" + ) == "PRESENT" + + first = _rule(module, priority=0, created_at="2026-01-01T00:00:00Z") + second = _rule( + module, + priority=0, + created_at="2026-01-01T00:00:00Z", + enable_force_push=True, + ) + with pytest.raises(module.PolicyError, match="duplicate primary-branch"): + module.evaluate(json.dumps([first, second]).encode(), "main", "bstein") + + +def test_any_special_rule_blocks_absent_create_and_present_readback(): + module = _module("branch_special_readback") + for rules in ( + [_rule(module, name="feature/*")], + [_rule(module), _rule(module, name="{main}", priority=0)], + ): + with pytest.raises(module.PolicyError, match="human review"): + module.evaluate(json.dumps(rules).encode(), "main", "bstein") + + +def test_main_reports_success_and_failures(tmp_path: Path, monkeypatch, capsys): + module = _module("branch_main_paths") + source = tmp_path / "rules.json" + source.write_text(json.dumps([_rule(module)]), encoding="utf-8") + monkeypatch.setattr(sys, "argv", ["checker", str(source), "main", "bstein"]) + assert module.main() == 0 + assert capsys.readouterr().out.strip() == "PRESENT" + + source.write_text("not-json", encoding="utf-8") + assert module.main() == 1 + assert "branch protection check failed" in capsys.readouterr().err + + monkeypatch.setattr( + module, "_read_bounded", lambda _path: (_ for _ in ()).throw(OSError()) + ) + assert module.main() == 1 diff --git a/testing/tests/test_hermes_gitea_internal_coverage.py b/testing/tests/test_hermes_gitea_internal_coverage.py new file mode 100644 index 00000000..da720878 --- /dev/null +++ b/testing/tests/test_hermes_gitea_internal_coverage.py @@ -0,0 +1,380 @@ +"""Behavioral coverage for internal safe-Gitea client and policy paths.""" + +from __future__ import annotations + +import json +import subprocess +import sys +import types +import urllib.error +import urllib.parse +from email.message import Message +from pathlib import Path + +import pytest + +from testing.tests.test_hermes_gitea_support import CLIENT_PATH, Response, _load +from testing.tests.test_hermes_scm_broker_support import _load_path + + +def _policy(name: str = "gitea_policy_coverage"): + return _load_path(name, CLIENT_PATH.parent / "gitea_api_policy.py") + + +def test_runtime_token_reader_and_fixed_origin(tmp_path: Path, monkeypatch): + module = _load() + token = tmp_path / "token" + token.write_text(" synthetic-runtime-value \n", encoding="utf-8") + assert module.read_token(token) == "synthetic-runtime-value" + token.write_text(" \n", encoding="utf-8") + with pytest.raises(ValueError, match="empty"): + module.read_token(token) + + monkeypatch.setenv("GITEA_BASE_URL", module.CANONICAL_BASE_URL) + assert module.configured_base_url() == module.CANONICAL_BASE_URL + monkeypatch.setenv("GITEA_BASE_URL", "https://example.invalid") + with pytest.raises(module.PolicyError, match="fixed"): + module.configured_base_url() + + +def test_safe_urlopen_delegates_only_to_deadline_bounded_exchange(monkeypatch): + module = _load() + sentinel = object() + monkeypatch.setattr( + module.deadline_http, + "open_bounded", + lambda request, *, maximum, timeout: (request, maximum, timeout, sentinel), + ) + request = object() + assert module._safe_urlopen(request, 7) == ( + request, + module.MAX_RESPONSE_BYTES, + 7, + sentinel, + ) + + +def test_api_target_and_request_body_edge_paths(monkeypatch): + module = _load() + with pytest.raises(module.PolicyError, match="path exceeds"): + module._split_api_path("/api/v1/" + "x" * 510) + with pytest.raises(module.PolicyError, match="request body"): + module.authorize_request( + "GET", "/api/v1/repos/atlas/cassandra", {"unexpected": True} + ) + with pytest.raises(module.PolicyError, match="query"): + module.authorize_request( + "POST", + "/api/v1/repos/atlas/cassandra/pulls?page=1", + {}, + ) + with pytest.raises(module.PolicyError, match="only read"): + module.authorize_request("TRACE", "/api/v1/repos/atlas/cassandra/pulls", None) + with pytest.raises(module.PolicyError, match="accepts only"): + module.authorize_request("POST", "/api/v1/repos/atlas/cassandra/pulls", []) + + +def test_response_status_fallback_and_nested_fail_closed(): + module = _load() + + class GetCodeResponse(Response): + def __init__(self): + super().__init__({"name": "cassandra"}, status=200) + del self.status + + def getcode(self): + return 200 + + assert ( + module.read( + "/api/v1/repos/atlas/cassandra", + token="synthetic", + opener=lambda *_a, **_k: GetCodeResponse(), + ) + == b'{"name": "cassandra"}' + ) + with pytest.raises(module.PolicyError, match="omitted required"): + module._nested({"base": None}, "base", "ref") + + +@pytest.mark.parametrize("body", [b"not-json", b"[]"]) +def test_create_response_requires_json_object(body: bytes): + module = _load() + with pytest.raises(module.PolicyError, match="invalid pull-request metadata"): + module._require_create_response( + body, + repo="cassandra", + base="main", + head="feature/test", + head_sha="a" * 40, + title="WIP: Test", + body="Evidence", + ) + + +def test_write_body_handles_empty_newline_and_missing_newline(monkeypatch): + module = _load() + + class Buffer: + value = bytearray() + + @classmethod + def write(cls, value): + cls.value.extend(value) + + monkeypatch.setattr(module.sys, "stdout", types.SimpleNamespace(buffer=Buffer)) + module._write_body(b"") + module._write_body(b"one\n") + module._write_body(b"two") + assert bytes(Buffer.value) == b"one\ntwo\n" + + +def test_main_executes_broker_read_and_create_paths(monkeypatch): + module = _load() + outputs: list[bytes] = [] + client = types.ModuleType("scm_broker_client") + client.read = lambda path: json.dumps({"path": path}).encode() + client.create_draft = lambda repo, **data: json.dumps( + {"repo": repo, **data}, sort_keys=True + ).encode() + monkeypatch.setitem(sys.modules, "scm_broker_client", client) + monkeypatch.setattr(module, "_write_body", outputs.append) + + assert module.main(["read", "/api/v1/repos/atlas/cassandra"]) == 0 + assert json.loads(outputs.pop()) == {"path": "/api/v1/repos/atlas/cassandra"} + assert ( + module.main( + [ + "create-draft", + "cassandra", + "--base", + "main", + "--head", + "feature/coverage", + "--head-sha", + "a" * 40, + "--title", + "Coverage repair", + "--body", + "Review the focused tests", + ] + ) + == 0 + ) + assert json.loads(outputs.pop())["repo"] == "cassandra" + + +def test_main_handles_http_and_policy_failures(monkeypatch, capsys): + module = _load() + client = types.ModuleType("scm_broker_client") + + def http_failure(_path): + headers = Message() + raise urllib.error.HTTPError("url", 503, "unavailable", headers, None) + + client.read = http_failure + client.create_draft = lambda *_a, **_k: b"{}" + monkeypatch.setitem(sys.modules, "scm_broker_client", client) + assert module.main(["read", "/api/v1/repos/atlas/cassandra"]) == 1 + assert "HTTP 503" in capsys.readouterr().err + client.read = lambda _path: (_ for _ in ()).throw(module.PolicyError("rejected")) + assert module.main(["read", "/not-allowed"]) == 1 + assert "no credential" in capsys.readouterr().err + + +@pytest.mark.parametrize( + ("function", "value"), + [ + ("_validate_repo", "."), + ("_validate_repo", ".."), + ("_validate_sha", "short"), + ("_validate_pr_number", True), + ("_validate_pr_number", 0), + ("_validate_pr_number", 2_147_483_648), + ], +) +def test_policy_scalar_validators_reject_ambiguous_values(function, value): + module = _policy("gitea_policy_scalar") + with pytest.raises(module.PolicyError): + getattr(module, function)(value) + + +def test_ref_bounds_cover_utf8_controls_git_failure_and_success(monkeypatch): + module = _policy("gitea_policy_refs") + with pytest.raises(module.PolicyError, match="valid UTF-8"): + module._validate_ref_bounds("\ud800", "head") + with pytest.raises(module.PolicyError, match="UTF-8"): + module._validate_ref_bounds("🧪" * 64, "head") + with pytest.raises(module.PolicyError, match="safe same"): + module._validate_ref("bad\nref", "head") + monkeypatch.setattr( + module.subprocess, + "run", + lambda *_a, **_k: subprocess.CompletedProcess([], 1), + ) + with pytest.raises(module.PolicyError, match="same-repository"): + module._validate_ref("invalid-ref", "head") + monkeypatch.setattr( + module.subprocess, + "run", + lambda *_a, **_k: subprocess.CompletedProcess([], 0), + ) + assert module._validate_ref("feature/valid", "head") == "feature/valid" + + +@pytest.mark.parametrize( + ("value", "required", "match"), + [ + (None, False, "must be text"), + (" ", True, "must not be empty"), + ("\ud800", False, "valid UTF-8"), + ("x\x00y", False, "safe request limit"), + ], +) +def test_text_validation_rejects_nontext_empty_invalid_and_controls( + value, required, match +): + module = _policy("gitea_policy_text") + with pytest.raises(module.PolicyError, match=match): + module._validate_text(value, "field", 20, 20, required=required) + + +@pytest.mark.parametrize( + ("key", "expected"), + [ + ("password", True), + ("accountKey", True), + ("registry-key", True), + ("clientEmail", True), + ("clientId", True), + ("accessId", True), + ("connectionString", True), + ("dockerConfigJson", True), + ("release_note", False), + ], +) +def test_sensitive_key_semantics_cover_generic_forms(key, expected): + module = _policy("gitea_policy_keys") + assert module._is_sensitive_key(key) is expected + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("", False), + ('"synthetic"', True), + ("{encoded}", True), + ("Bearer value-12345678", True), + ("ghp_" + "a" * 24, True), + ("https://example.invalid/value", True), + ("name@example.invalid", True), + ("${RUNTIME_VALUE}", True), + ("AbCdEf0123456789", True), + ("reject empty values", False), + ("ordinary", True), + ], +) +def test_assignment_value_shape_distinguishes_prose_from_credentials(value, expected): + module = _policy("gitea_policy_assignment_values") + assert module._looks_sensitive_assignment_value(value) is expected + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("single", False), + ("add regression coverage", True), + ("add regression_coverage", False), + ("ordinary engineering prose", False), + ], +) +def test_prose_classifier_covers_length_punctuation_and_lead_words(value, expected): + module = _policy("gitea_policy_prose") + assert module._looks_like_prose(value) is expected + + +def test_structured_json_walk_covers_lists_nonstring_keys_and_limits(): + module = _policy("gitea_policy_json_walk") + assert module._json_value_has_sensitive_assignment({1: "ignored"}) is False + assert module._json_value_has_sensitive_assignment([{"release": "safe"}]) is False + assert ( + module._json_value_has_sensitive_assignment({"client_secret": "value"}) is True + ) + assert ( + module._json_value_has_sensitive_assignment( + {"outer": {"client_secret": "value"}} + ) + is True + ) + assert ( + module._json_value_has_sensitive_assignment({"type": "service-account"}) is True + ) + with pytest.raises(module.PolicyError, match="scan limit"): + module._json_value_has_sensitive_assignment([], depth=33) + with pytest.raises(module.PolicyError, match="scan limit"): + module._json_value_has_sensitive_assignment([], nodes=[2048]) + + +def test_json_key_decoder_and_embedded_document_scanner_cover_failures(): + module = _policy("gitea_policy_json_decoder") + assert module._decode_json_key("client\\u005fsecret") == ("client_secret", True) + decoded, valid = module._decode_json_key("client\\qsecret") + assert valid is False and "client" in decoded + assert list(module._decoded_json_documents("prose only")) == [] + assert list(module._decoded_json_documents("x {bad y [1, 2]")) == [[1, 2]] + with pytest.raises(module.PolicyError, match="scan limit"): + list(module._decoded_json_documents("{" * 33)) + assert ( + module._has_structured_sensitive_assignment('"client\\qsecret": value') is True + ) + assert ( + module._has_structured_sensitive_assignment("type:\n service-account") is True + ) + + +def test_high_entropy_detector_covers_long_mixed_and_low_entropy_values(): + module = _policy("gitea_policy_entropy") + assert module._has_high_entropy_token("A1_" * 100) is True + assert ( + module._has_high_entropy_token( + "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789+/_=" + ) + is True + ) + assert module._has_high_entropy_token("a" * 64) is False + assert module._has_high_entropy_token("short prose") is False + + +@pytest.mark.parametrize( + "query", + [ + "bad", + "a=1&b=2&c=3&d=4", + "page=1&page=2", + "=value", + "unexpected=1", + "page=abc", + "page=01", + "page=10001", + "limit=51", + "state=merged", + ], +) +def test_query_validator_covers_each_rejection_class(query): + module = _policy("gitea_policy_query") + target = urllib.parse.urlsplit("/api/v1/repos/atlas/cassandra/pulls?" + query) + with pytest.raises(module.PolicyError): + module._validate_query(target, {"page", "limit", "state"}) + + +def test_draft_title_rejects_prefix_without_content(): + module = _policy("gitea_policy_empty_draft") + with pytest.raises(module.PolicyError, match="after the draft prefix"): + module._draft_title("WIP:") + + +def test_query_validator_rejects_noncanonical_raw_form(): + module = _policy("gitea_policy_raw_query") + target = urllib.parse.SplitResult("", "", "/api/v1/repos/atlas/cassandra", "%", "") + with pytest.raises(module.PolicyError, match="canonical ASCII"): + module._validate_query(target, set()) diff --git a/testing/tests/test_hermes_gitea_pr_client.py b/testing/tests/test_hermes_gitea_pr_client.py new file mode 100644 index 00000000..4ddb9830 --- /dev/null +++ b/testing/tests/test_hermes_gitea_pr_client.py @@ -0,0 +1,296 @@ +"""Route and request-boundary contracts for the Atlas Gitea client.""" + +from __future__ import annotations + +import urllib.error +import urllib.request + +import pytest + +from testing.tests.test_hermes_gitea_support import HEAD_SHA, Response, _load + +def test_default_opener_rejects_redirects_under_an_absolute_deadline(monkeypatch): + client = _load() + source = urllib.request.Request( + "https://scm.bstein.dev/api/v1/repos/atlas/cassandra", + headers={"Authorization": "token redirect-sentinel"}, + ) + + with pytest.raises(urllib.error.URLError) as exc: + client.deadline_http._RejectRedirect().redirect_request( + source, + None, + 302, + "Found", + {}, + "https://evil.example/collect", + ) + + assert "redirect-sentinel" not in str(exc.value) + seen = [] + monkeypatch.setattr( + client.deadline_http, + "open_bounded", + lambda request, *, maximum, timeout: seen.append( + (request, maximum, timeout) + ), + ) + client._safe_urlopen(source, 30) + assert seen == [(source, client.MAX_RESPONSE_BYTES, 30)] + + +@pytest.mark.parametrize( + ("base_url", "path"), + [ + ("https://evil.example", "/api/v1/repos/atlas/cassandra"), + ("http://scm.bstein.dev", "/api/v1/repos/atlas/cassandra"), + ("https://scm.bstein.dev:443", "/api/v1/repos/atlas/cassandra"), + ("https://scm.bstein.dev/", "/api/v1/repos/atlas/cassandra"), + ("HTTPS://scm.bstein.dev", "/api/v1/repos/atlas/cassandra"), + ("https://SCM.bstein.dev", "/api/v1/repos/atlas/cassandra"), + ("https://user@scm.bstein.dev", "/api/v1/repos/atlas/cassandra"), + ("https://scm.bstein.dev", "https://evil.example/api/v1/repos/atlas/cassandra"), + ("https://scm.bstein.dev", "/api/v1/repos/evil/cassandra"), + ("https://scm.bstein.dev", "/api/v1/repos/%61tlas/cassandra"), + ("https://scm.bstein.dev", "/api/v1/repos/atlas/../admin"), + ], +) +def test_host_owner_and_path_escape_attempts_are_rejected(base_url: str, path: str): + client = _load() + + with pytest.raises(client.PolicyError): + client.build_request("GET", path, base_url=base_url, token="secret") + + +@pytest.mark.parametrize( + "path", + [ + "/api/v1/repos/atlas/cassandra/pulls/1\nHost: evil.example", + "/api/v1/repos/atlas/cassandra/pulls/1\r\nX-Test: value", + "/api/v1/repos/atlas/cassandra/pulls/1\tignored", + "/api/v1/repos/atlas/cassandra/pulls/1\x00ignored", + "/api/v1/repos/atlas/cassandra/pulls/1\x1fignored", + "/api/v1/repos/atlas/cassandra/pulls/1\x7fignored", + "/api/v1/repos/atlas/cassandra\\pulls\\1", + "/api/v1/repos/atlas/cassandra/pulls/%31", + "/api/v1/repos/atlas/cassandra/pulls/１", + " https://scm.bstein.dev/api/v1/repos/atlas/cassandra", + "https://scm.bstein.dev/api/v1/repos/atlas/cassandra", + ], +) +def test_raw_noncanonical_target_is_rejected_before_urlsplit_and_opener( + path: str, monkeypatch +): + client = _load() + split_called = False + opener_called = False + original_urlsplit = client.urllib.parse.urlsplit + + def urlsplit(*args, **kwargs): + nonlocal split_called + split_called = True + return original_urlsplit(*args, **kwargs) + + def opener(*_args, **_kwargs): + nonlocal opener_called + opener_called = True + return Response(b"{}") + + monkeypatch.setattr(client.urllib.parse, "urlsplit", urlsplit) + with pytest.raises(client.PolicyError): + client.read(path, token="runtime", opener=opener) + assert split_called is False + assert opener_called is False + + +@pytest.mark.parametrize( + "path", + [ + "/api/v1/repos/atlas/cassandra/pulls/1?", + "//scm.bstein.dev/api/v1/repos/atlas/cassandra", + "/api/v1/repos/atlas/cassandra/./pulls/1", + "/api/v1/repos/atlas/cassandra/../admin", + "/api/v1/repos/atlas/cassandra//pulls/1", + "/api/v1/repos/atlas/cassandra/pulls/1?limit=01", + ], +) +def test_noncanonical_round_trip_or_segments_never_reach_opener(path: str): + client = _load() + opener_called = False + + def opener(*_args, **_kwargs): + nonlocal opener_called + opener_called = True + return Response(b"{}") + + with pytest.raises(client.PolicyError): + client.read(path, token="runtime", opener=opener) + assert opener_called is False + + +@pytest.mark.parametrize( + "path", + [ + "/api/v1/repos/atlas/cassandra", + "/api/v1/repos/atlas/cassandra/pulls?state=open&limit=20&page=1", + "/api/v1/repos/atlas/cassandra/pulls/7", + "/api/v1/repos/atlas/cassandra/pulls/7/commits?limit=20", + "/api/v1/repos/atlas/cassandra/pulls/7/files?page=1", + "/api/v1/repos/atlas/cassandra/branches", + "/api/v1/repos/atlas/cassandra/branches/main", + "/api/v1/repos/atlas/cassandra/commits?limit=10", + f"/api/v1/repos/atlas/cassandra/git/commits/{HEAD_SHA}", + f"/api/v1/repos/atlas/cassandra/commits/{HEAD_SHA}/status", + f"/api/v1/repos/atlas/cassandra/commits/{HEAD_SHA}/statuses?limit=10", + f"/api/v1/repos/atlas/cassandra/statuses/{HEAD_SHA}?page=1", + ], +) +def test_explicit_read_allowlist_accepts_only_engineering_metadata(path: str): + client = _load() + + request = client.build_request( + "GET", path, base_url=client.CANONICAL_BASE_URL, token="runtime" + ) + assert request.method == "GET" + + +@pytest.mark.parametrize( + "path", + [ + "/api/v1/repos/atlas/cassandra/hooks", + "/api/v1/repos/atlas/cassandra/actions/secrets", + "/api/v1/repos/atlas/cassandra/actions/variables", + "/api/v1/repos/atlas/cassandra/collaborators", + "/api/v1/repos/atlas/cassandra/branch_protections", + "/api/v1/repos/atlas/cassandra/keys", + "/api/v1/repos/atlas/cassandra/pulls/7/reviews", + "/api/v1/repos/atlas/cassandra/pulls/7/merge", + "/api/v1/repos/atlas/cassandra/pulls/7.diff", + "/api/v1/repos/atlas/cassandra/releases", + ], +) +def test_privileged_or_content_routes_are_denied_even_for_get(path: str): + client = _load() + + with pytest.raises(client.PolicyError, match="outside the metadata read allowlist"): + client.authorize_request("GET", path, None) + + +@pytest.mark.parametrize( + "path", + [ + "/api/v1/repos/atlas/cassandra/pulls?limit=51", + "/api/v1/repos/atlas/cassandra/pulls?state=merged", + "/api/v1/repos/atlas/cassandra/pulls?private=true", + "/api/v1/repos/atlas/cassandra/pulls?limit=1&limit=2", + "/api/v1/repos/atlas/cassandra?p=1", + ], +) +def test_read_query_is_bounded(path: str): + client = _load() + + with pytest.raises(client.PolicyError): + client.authorize_request("GET", path, None) + + +@pytest.mark.parametrize( + "query", + [ + "page=0", + "page=10001", + "page=01", + "limit=0", + "limit=51", + "limit=01", + "page=" + "9" * 4000, + "page=０", + "page=%EF%BC%90", + "state=%6fpen", + "p%61ge=1", + "page=1&page=2", + "page=1&" + "x" * 17 + "=1", + "state=" + "x" * 17, + "page=1&limit=2&state=open&extra=3", + ], +) +def test_read_query_requires_canonical_bounded_ascii(query: str): + client = _load() + + with pytest.raises(client.PolicyError): + client.authorize_request( + "GET", f"/api/v1/repos/atlas/cassandra/pulls?{query}", None + ) + + +@pytest.mark.parametrize( + "suffix", + [ + "pulls/{number}", + "pulls/{number}.patch", + "pulls/{number}.diff", + "pulls/{number}/commits", + "pulls/{number}/files", + "commits/{number}/status", + "commits/{number}/statuses", + "statuses/{number}", + "branches/{number}", + ], +) +def test_oversized_numeric_or_captured_path_never_reaches_opener(suffix: str): + client = _load() + called = False + + def opener(*_args, **_kwargs): + nonlocal called + called = True + return Response(b"{}") + + path = "/api/v1/repos/atlas/cassandra/" + suffix.format(number="9" * 4000) + with pytest.raises(client.PolicyError): + client.read(path, token="runtime", opener=opener) + assert called is False + + +@pytest.mark.parametrize( + "number", + ["0", "01", "2147483648", "１２", "%31", "12345678901"], +) +@pytest.mark.parametrize("tail", ["", ".patch", ".diff", "/commits", "/files"]) +def test_noncanonical_or_out_of_range_pr_number_never_reaches_opener( + number: str, tail: str +): + client = _load() + called = False + + def opener(*_args, **_kwargs): + nonlocal called + called = True + return Response(b"{}") + + with pytest.raises(client.PolicyError): + client.read( + f"/api/v1/repos/atlas/cassandra/pulls/{number}{tail}", + token="runtime", + opener=opener, + ) + assert called is False + + +def test_maximum_bounded_pr_number_is_readable(): + client = _load() + called = False + + def opener(*_args, **_kwargs): + nonlocal called + called = True + return Response(b"{}") + + assert ( + client.read( + "/api/v1/repos/atlas/cassandra/pulls/2147483647", + token="runtime", + opener=opener, + ) + == b"{}" + ) + assert called is True diff --git a/testing/tests/test_hermes_gitea_pr_creation.py b/testing/tests/test_hermes_gitea_pr_creation.py new file mode 100644 index 00000000..ad2967ed --- /dev/null +++ b/testing/tests/test_hermes_gitea_pr_creation.py @@ -0,0 +1,203 @@ +"""Draft-creation input contracts for the Atlas Gitea client.""" + +from __future__ import annotations + +import pytest + +from testing.tests.test_hermes_gitea_support import ( + HEAD_SHA, + Response, + _draft_payload, + _draft_response, + _load, +) + + +@pytest.mark.parametrize( + ("method", "path", "data"), + [ + ("DELETE", "/api/v1/repos/atlas/cassandra/pulls/4", None), + ("POST", "/api/v1/repos/atlas/cassandra/pulls/4/merge", {}), + ( + "POST", + "/api/v1/repos/atlas/cassandra/pulls/4/reviews", + {"event": "APPROVED"}, + ), + ("PATCH", "/api/v1/repos/atlas/cassandra/pulls/4", {"title": "WIP: x"}), + ("PUT", "/api/v1/repos/atlas/cassandra/branches/main", {}), + ], +) +def test_merge_approve_close_delete_update_and_other_mutations_are_rejected( + method: str, path: str, data: object +): + client = _load() + + with pytest.raises(client.PolicyError): + client.build_request( + method, path, base_url=client.CANONICAL_BASE_URL, token="secret", data=data + ) + + +@pytest.mark.parametrize( + "ref", + [ + "foo/.bar", + "foo/bar.lock/baz", + "foo..bar", + "foo@{bar", + "foo//bar", + "-danger", + "danger.", + "danger~one", + "danger^one", + "danger:one", + "danger one", + ], +) +def test_complete_git_ref_validation_rejects_invalid_names(ref: str): + client = _load() + + with pytest.raises(client.PolicyError): + client._validate_ref(ref, "head") + + +def test_git_ref_validation_uses_fixed_trusted_binary(): + client = _load() + + assert client._validate_ref.__globals__["GIT_BIN"] == "/usr/bin/git" + assert client._validate_ref("hermes/valid-fix", "head") == "hermes/valid-fix" + + +@pytest.mark.parametrize("field", ["base", "head"]) +@pytest.mark.parametrize("oversized", ["r" * 100_000, "🧪" * 128]) +def test_oversized_ref_never_invokes_git_request_or_opener( + field: str, oversized: str, monkeypatch +): + client = _load() + git_called = False + request_built = False + opener_called = False + original_build_request = client.build_request + + def git_run(*_args, **_kwargs): + nonlocal git_called + git_called = True + raise AssertionError("Git must not receive an oversized ref") + + def build_request(*args, **kwargs): + nonlocal request_built + request_built = True + return original_build_request(*args, **kwargs) + + def opener(*_args, **_kwargs): + nonlocal opener_called + opener_called = True + return Response(_draft_response()) + + monkeypatch.setattr(client._validate_ref.__globals__["subprocess"], "run", git_run) + client.build_request = build_request + refs = {"base": "main", "head": "hermes/fix"} + refs[field] = oversized + with pytest.raises(client.PolicyError, match="branch-name limit"): + client.create_draft( + "cassandra", + base=refs["base"], + head=refs["head"], + head_sha=HEAD_SHA, + title="Focused fix", + body="Review evidence", + token="runtime", + opener=opener, + ) + assert git_called is False + assert request_built is False + assert opener_called is False + + +@pytest.mark.parametrize( + ("field", "oversized"), + [("repo", "r" * 101), ("title", "🧪" * 200), ("body", "🧪" * 9_000)], +) +def test_other_text_bounds_fail_before_git_request_or_opener( + field: str, oversized: str, monkeypatch +): + client = _load() + git_called = False + request_built = False + opener_called = False + original_build_request = client.build_request + + def git_run(*_args, **_kwargs): + nonlocal git_called + git_called = True + raise AssertionError("Git must not run before cheap input bounds") + + def build_request(*args, **kwargs): + nonlocal request_built + request_built = True + return original_build_request(*args, **kwargs) + + def opener(*_args, **_kwargs): + nonlocal opener_called + opener_called = True + return Response(_draft_response()) + + monkeypatch.setattr(client._validate_ref.__globals__["subprocess"], "run", git_run) + client.build_request = build_request + values = { + "repo": "cassandra", + "title": "Focused fix", + "body": "Review evidence", + } + values[field] = oversized + with pytest.raises(client.PolicyError): + client.create_draft( + values["repo"], + base="main", + head="hermes/fix", + head_sha=HEAD_SHA, + title=values["title"], + body=values["body"], + token="runtime", + opener=opener, + ) + assert git_called is False + assert request_built is False + assert opener_called is False + + +def test_create_forces_draft_title_and_same_repository_branch_names(): + client = _load() + assert ( + client.authorize_request( + "POST", "/api/v1/repos/atlas/cassandra/pulls", _draft_payload() + ) + == "create-draft" + ) + with pytest.raises(client.PolicyError, match="draft-title prefix"): + client.authorize_request( + "POST", + "/api/v1/repos/atlas/cassandra/pulls", + _draft_payload(title="Not a draft"), + ) + with pytest.raises(client.PolicyError): + client.authorize_request( + "POST", + "/api/v1/repos/atlas/cassandra/pulls", + _draft_payload(head="someone:branch"), + ) + + +def test_runtime_token_is_only_an_authorization_header(): + client = _load() + request = client.build_request( + "POST", + "/api/v1/repos/atlas/cassandra/pulls", + base_url=client.CANONICAL_BASE_URL, + token="do-not-leak", + data=_draft_payload(), + ) + + assert "do-not-leak" not in request.full_url + assert b"do-not-leak" not in request.data + assert request.get_header("Authorization") == "token do-not-leak" diff --git a/testing/tests/test_hermes_gitea_pr_dlp.py b/testing/tests/test_hermes_gitea_pr_dlp.py new file mode 100644 index 00000000..4a2b1d27 --- /dev/null +++ b/testing/tests/test_hermes_gitea_pr_dlp.py @@ -0,0 +1,317 @@ +"""Credential-loss-prevention contracts for Atlas draft creation.""" + +from __future__ import annotations + +import pytest + +from testing.tests.test_hermes_gitea_support import ( + HEAD_SHA, + Response, + _draft_response, + _load, +) + + +@pytest.mark.parametrize( + "sensitive", + [ + "client_" + "secret=not-a-real-value", + '{"client_' + 'secret":\n"synthetic-value"}', + '{"client_' + 'secret"\n:\n"synthetic-value"}', + '{"client\\u005f' + 'secret":"synthetic-value"}', + "client_" + "secret:\n synthetic-value", + '{"client\\q' + 'secret":"synthetic-value"}', + '{"client\n' + 'secret":"synthetic-value"}', + '"client_' + 'secret"\x0b:\n"synthetic-value"', + "ACCESS_" + "TOKEN = 'not-a-real-value'", + '{"refresh_' + 'token": "not-a-real-value"}', + "private_" + "key: not-a-real-value", + "AWS_SECRET_ACCESS_" + "KEY=not-a-real-value", + "aws_access_key_" + "id: not-a-real-value", + "AWS_SESSION_" + "TOKEN = 'not-a-real-value'", + '{"AccessKey' + 'Id":"not-a-real-value"}', + '{"SecretAccess' + 'Key":"not-a-real-value"}', + '{"Session' + 'Token":"not-a-real-value"}', + "aws-security-" + "token: not-a-real-value", + "Account" + "Key=not-a-real-value", + "SharedAccess" + "Signature: not-a-real-value", + "AZURE_STORAGE_CONNECTION_" + "STRING='not-a-real-value'", + "DefaultEndpointsProtocol=https;AccountName=fake;Account" + + "Key=not-a-real-value;EndpointSuffix=example", + "?sv=2024-11-04&ss=b&srt=sco&sp=rwdlac&se=2099-01-01&sig=" + "not-a-real-value", + "DOCKER_AUTH_" + "CONFIG='not-a-real-value'", + '{"auths":{"registry.example":{"auth":"bm90LXJlYWw="}}}', + '{"identity' + 'token":"not-a-real-value"}', + '{"type":"service_' + 'account","client_email":"fake@example.test"}', + '{"private_key_' + 'id":"not-a-real-value"}', + '{"client_' + 'email":"fake@example.test"}', + "GOOGLE_CREDENTIALS" + "=not-a-real-value", + "personal_access_" + "token: not-a-real-value", + "GITEA_" + "TOKEN=not-a-real-value", + "gitlab-token" + ": not-a-real-value", + "pat" + "=not-a-real-value", + "Authorization: " + "Bearer not-a-real-credential-value", + "authorization = " + '"Basic not-a-real-credential-value"', + "Bearer" + "=not-a-real-credential-value", + "Basic" + ": not-a-real-credential-value", + "pass" + "word=not-a-real-credential", + "ghp_" + "notarealcredentialvalue123456", + "github_pat_" + "notarealcredentialvalue123456", + "glpat-" + "notarealcredentialvalue123456", + "xoxb-" + "not-a-real-credential-value-123456", + "sk-ant-" + "notarealcredentialvalue123456", + "sk-proj-" + "notarealcredentialvalue123456", + "sk_live_" + "notarealcredentialvalue123456", + "ya29." + "notarealcredentialvalue123456", + "gta_" + "notarealcredentialvalue123456", + "whsec_" + "notarealcredentialvalue123456", + "npm_" + "notarealcredentialvalue123456", + "pypi-" + "notarealcredentialvalue123456789012345", + "hf_" + "notarealcredentialvalue123456", + "SG." + "notarealvalue1234" + ".notarealcredentialvalue123456", + "SK" + "a" * 32, + "https://hooks.slack.com/services/" + "T000/B000/notarealvalue123456", + "https://discord.com/api/webhooks/123456789/" + "notarealcredentialvalue123456", + "https://fake.webhook.office.com/" + "notarealcredentialvalue123456", + "webhook_" + "url=https://example.test/not-real", + "FutureCloudSigning" + "Credential=not-a-real-value", + "future-client-signing-" + "key: not-a-real-value", + "future_client_signing_" + "key='not-a-real-value'", + '{"serviceAccountPrivate' + 'Key":"not-a-real-value"}', + "CONTAINER_REGISTRY_" + "CREDENTIAL=not-a-real-value", + "someWebhookSigning" + "Secret: not-a-real-value", + "client" + "Key=not-a-real-value", + "session" + "Key: not-a-real-value", + "access" + "Id=not-a-real-value", + "credentials" + ": {user: fake}", + "private" + "Key: |", + "nuget_api_" + "key=not-a-real-value", + "oy2" + "a" * 44, + "sk_test_" + "notarealcredentialvalue123456", + "A1b2C3d4E5f6G7h8I9j0K_l-M+n/O=pQ2rS3tU4vW5xY6zZ7aB8cC9d", + "AKIA" + "A" * 16, + "AIza" + "a" * 35, + "-----BEGIN OPENSSH " + "PRIVATE KEY-----", + "ssh-ed25519 " + "bm90YXJlYWxjcmVkZW50aWFsdmFsdWU=", + "eyJnotarealheader." + "notarealpayloadvalue." + "notarealsignature", + ], +) +@pytest.mark.parametrize("field", ["title", "body"]) +def test_create_rejects_expanded_credential_shapes_before_network( + sensitive: str, field: str +): + client = _load() + called = False + request_built = False + + original_build_request = client.build_request + + def build_request(*args, **kwargs): + nonlocal request_built + request_built = True + return original_build_request(*args, **kwargs) + + def opener(*_args, **_kwargs): + nonlocal called + called = True + return Response(_draft_response()) + + client.build_request = build_request + values = {"title": "Focused fix", "body": "Review evidence"} + values[field] = sensitive + + with pytest.raises(client.PolicyError, match="credential material"): + client.create_draft( + "cassandra", + base="main", + head="hermes/fix", + head_sha=HEAD_SHA, + title=values["title"], + body=values["body"], + token="runtime-sentinel", + opener=opener, + ) + assert request_built is False + assert called is False + + +def test_very_long_compact_body_is_rejected_before_request_or_network(): + client = _load() + request_built = False + opener_called = False + original_build_request = client.build_request + + def build_request(*args, **kwargs): + nonlocal request_built + request_built = True + return original_build_request(*args, **kwargs) + + def opener(*_args, **_kwargs): + nonlocal opener_called + opener_called = True + return Response(_draft_response()) + + client.build_request = build_request + with pytest.raises(client.PolicyError, match="credential material"): + client.create_draft( + "cassandra", + base="main", + head="hermes/fix", + head_sha=HEAD_SHA, + title="Focused fix", + body="a" * 300, + token="runtime-sentinel", + opener=opener, + ) + assert request_built is False + assert opener_called is False + + +@pytest.mark.parametrize( + "safe_text", + [ + "AWS_SECRET_ACCESS_KEY is injected at runtime", + "Token: reject empty values", + "Authorization = preserve header behavior", + "Password: add regression", + "Review the Authorization header behavior", + "Bearer authentication is required for this route", + "Document Docker auths payload rejection", + "The client_email field belongs to service accounts", + "AccountKey assignments must be blocked", + "This patch changes token validation without including a value", + "FutureCloudSigningCredential handling needs a regression test", + "The clientKey name is documented without an assigned value", + "A SHA-256 digest 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef is evidence", + ], +) +def test_credential_policy_keeps_normal_engineering_prose_usable(safe_text: str): + client = _load() + + assert client._validate_body(safe_text) == safe_text + assert client._draft_title(safe_text) == f"WIP: {safe_text}" + + +@pytest.mark.parametrize("field", ["title", "body"]) +def test_create_rejects_exact_runtime_token_before_network(field: str): + client = _load() + called = False + request_built = False + + original_build_request = client.build_request + + def build_request(*args, **kwargs): + nonlocal request_built + request_built = True + return original_build_request(*args, **kwargs) + + def opener(*_args, **_kwargs): + nonlocal called + called = True + return Response(_draft_response()) + + client.build_request = build_request + runtime_token = "exact-random-runtime-sentinel-7b73ac61" + values = {"title": "Focused fix", "body": "Review evidence"} + values[field] = f"Accidental {runtime_token} value" + with pytest.raises(client.PolicyError, match="runtime credential"): + client.create_draft( + "cassandra", + base="main", + head="hermes/fix", + head_sha=HEAD_SHA, + title=values["title"], + body=values["body"], + token=runtime_token, + opener=opener, + ) + assert request_built is False + assert called is False + + +@pytest.mark.parametrize("field", ["repo", "base", "head", "title", "body"]) +def test_every_public_field_rejects_exact_runtime_token_before_git_or_network( + field: str, monkeypatch +): + client = _load() + git_called = False + request_built = False + opener_called = False + runtime_token = "runtime-sentinel" + values = { + "repo": "cassandra", + "base": "main", + "head": "hermes/fix", + "title": "Focused fix", + "body": "Review evidence", + } + values[field] = runtime_token + + def git_run(*_args, **_kwargs): + nonlocal git_called + git_called = True + raise AssertionError("Git must not run for a credential-bearing field") + + def build_request(*_args, **_kwargs): + nonlocal request_built + request_built = True + raise AssertionError("a request must not be built") + + def opener(*_args, **_kwargs): + nonlocal opener_called + opener_called = True + raise AssertionError("the opener must not be called") + + monkeypatch.setattr(client._validate_ref.__globals__["subprocess"], "run", git_run) + monkeypatch.setattr(client, "build_request", build_request) + with pytest.raises(client.PolicyError, match="runtime credential"): + client.create_draft( + values["repo"], + base=values["base"], + head=values["head"], + head_sha=HEAD_SHA, + title=values["title"], + body=values["body"], + token=runtime_token, + opener=opener, + ) + assert git_called is False + assert request_built is False + assert opener_called is False + + +@pytest.mark.parametrize( + "sensitive", + [ + " ".join(["{}"] * 32) + ' {"client_secret":"synthetic-value"}', + "prefix_" + "ghp_" + "notarealcredentialvalue123456_suffix", + "client_secret: correct horse battery staple", + "'client_secret':\n synthetic-value", + ], +) +@pytest.mark.parametrize("field", ["title", "body"]) +def test_structured_scanner_closes_bounded_and_wrapped_token_bypasses( + sensitive: str, field: str +): + client = _load() + values = {"title": "Focused fix", "body": "Review evidence"} + values[field] = sensitive + called = False + + def opener(*_args, **_kwargs): + nonlocal called + called = True + return Response(_draft_response()) + + with pytest.raises(client.PolicyError, match="credential material"): + client.create_draft( + "cassandra", + base="main", + head="hermes/fix", + head_sha=HEAD_SHA, + title=values["title"], + body=values["body"], + token="runtime-sentinel", + opener=opener, + ) + assert called is False diff --git a/testing/tests/test_hermes_gitea_pr_integration.py b/testing/tests/test_hermes_gitea_pr_integration.py new file mode 100644 index 00000000..ee482cfe --- /dev/null +++ b/testing/tests/test_hermes_gitea_pr_integration.py @@ -0,0 +1,174 @@ +"""Runtime and Flux integration contracts for the safe Atlas PR client.""" + +from __future__ import annotations + +import importlib.util +import io +import json +import sys +import urllib.error +from pathlib import Path + +import pytest +import yaml + +ROOT = Path(__file__).parents[2] +CLIENT_PATH = ROOT / "services/hermes/scm-common/scripts/gitea_api.py" +HEAD_SHA = "465cf9146b05c174a2a8d310aff6c64be58277b6" +if str(CLIENT_PATH.parent) not in sys.path: + sys.path.insert(0, str(CLIENT_PATH.parent)) + + +def _load(): + spec = importlib.util.spec_from_file_location( + "safe_gitea_api_integration", CLIENT_PATH + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def test_dry_run_has_no_file_input_and_uses_no_token_or_network(monkeypatch, capsys): + client = _load() + monkeypatch.setattr(client, "read_token", lambda: pytest.fail("read token")) + monkeypatch.setattr( + client, "_safe_urlopen", lambda *_a, **_k: pytest.fail("network") + ) + + assert ( + client.main( + [ + "--dry-run", + "create-draft", + "cassandra", + "--base", + "main", + "--head", + "hermes/fix", + "--head-sha", + HEAD_SHA, + "--title", + "Repair", + "--body", + "Evidence only", + ] + ) + == 0 + ) + output = json.loads(capsys.readouterr().out) + assert output["operation"] == "create-draft" + assert output["owner"] == "atlas" + assert "Evidence only" not in json.dumps(output) + with pytest.raises(SystemExit): + client.parse_args( + [ + "create-draft", + "cassandra", + "--base", + "main", + "--head", + "hermes/fix", + "--head-sha", + HEAD_SHA, + "--title", + "Repair", + "--body-file", + "/runtime-access/gitea-token", + ] + ) + + +def test_http_error_path_redacts_token(monkeypatch, capsys): + client = _load() + monkeypatch.setattr(client, "read_token", lambda: "do-not-leak") + + def fail(*_args, **_kwargs): + raise urllib.error.HTTPError( + "https://scm.bstein.dev/api/v1/repos/atlas/cassandra", + 403, + "forbidden", + {}, + io.BytesIO(b"Authorization: token do-not-leak"), + ) + + monkeypatch.setattr(client, "read", lambda *_a, **_k: fail()) + + assert client.main(["read", "/api/v1/repos/atlas/cassandra"]) == 1 + captured = capsys.readouterr() + assert "do-not-leak" not in captured.err + assert "credential was disclosed" in captured.err + + +def test_flux_manifest_isolates_vault_token_in_separate_broker_only(): + client_source = CLIENT_PATH.read_text(encoding="utf-8") + assert "scm_broker_client" in client_source + assert "GITEA_TOKEN" not in client_source + + deployment = yaml.safe_load( + (ROOT / "services/hermes/agent-deployment.yaml").read_text(encoding="utf-8") + ) + template = deployment["spec"]["template"] + annotations = template["metadata"]["annotations"] + assert not any("gitea" in key.lower() for key in annotations) + runtime = next( + volume + for volume in template["spec"]["volumes"] + if volume["name"] == "runtime-access" + ) + assert runtime["emptyDir"]["medium"] == "Memory" + + expected = {"hermes", "terminal", "cli-lane-runner"} + mounted = { + container["name"] + for container in template["spec"]["containers"] + if any( + mount["name"] == "atlas-pr-skill" + and mount["mountPath"] + == "/opt/data/workspace/skills/manage-atlas-pull-requests" + and mount.get("readOnly") is True + for mount in container.get("volumeMounts", []) + ) + } + assert mounted == expected + + kustomization = yaml.safe_load( + (ROOT / "services/hermes/kustomization.yaml").read_text(encoding="utf-8") + ) + common = yaml.safe_load( + (ROOT / "services/hermes/scm-common/kustomization.yaml").read_text( + encoding="utf-8" + ) + ) + boundary = common["configMapGenerator"][0] + assert boundary["name"] == "hermes-scm-boundary" + assert "gitea_api.py=scripts/gitea_api.py" in boundary["files"] + assert "gitea_api_policy.py=scripts/gitea_api_policy.py" in boundary["files"] + assert "scm_broker_client.py=scripts/scm_broker_client.py" in boundary["files"] + assert not any("gitea_askpass" in item for item in boundary["files"]) + generator = next( + item + for item in kustomization["configMapGenerator"] + if item["name"] == "hermes-atlas-pr-skill" + ) + assert generator["files"] == [ + "SKILL.md=skills/manage-atlas-pull-requests/SKILL.md", + "openai.yaml=skills/manage-atlas-pull-requests/agents/openai.yaml", + ] + + broker = yaml.safe_load( + (ROOT / "services/hermes-scm-broker/deployment.yaml").read_text( + encoding="utf-8" + ) + ) + assert broker["metadata"]["namespace"] == "hermes-scm" + pod = broker["spec"]["template"] + assert pod["spec"]["serviceAccountName"] == "hermes-scm-broker" + assert pod["metadata"]["annotations"][ + "vault.hashicorp.com/agent-inject-secret-gitea-token" + ] == "kv/data/atlas/hermes/developer-gitea" + assert not any( + volume.get("hostPath") or volume.get("persistentVolumeClaim") + for volume in pod["spec"]["volumes"] + ) diff --git a/testing/tests/test_hermes_gitea_pr_postconditions.py b/testing/tests/test_hermes_gitea_pr_postconditions.py new file mode 100644 index 00000000..c06a7784 --- /dev/null +++ b/testing/tests/test_hermes_gitea_pr_postconditions.py @@ -0,0 +1,199 @@ +"""Response and postcondition contracts for Atlas draft creation.""" + +from __future__ import annotations + +import base64 +import copy +import json + +import pytest + +from testing.tests.test_hermes_gitea_support import ( + HEAD_SHA, + Response, + _draft_response, + _load, +) + +@pytest.mark.parametrize("status", [200, 202, 204, 206]) +def test_create_accepts_only_http_201(status: int): + client = _load() + + with pytest.raises(client.PolicyError, match="unexpected HTTP status"): + client.create_draft( + "cassandra", + base="main", + head="hermes/fix", + head_sha=HEAD_SHA, + title="Focused fix", + body="Review evidence", + token="runtime", + opener=lambda *_a, **_k: Response(_draft_response(), status=status), + ) + + +@pytest.mark.parametrize("status", [201, 202, 204, 206]) +def test_read_accepts_only_http_200(status: int): + client = _load() + + with pytest.raises(client.PolicyError, match="unexpected HTTP status"): + client.read( + "/api/v1/repos/atlas/cassandra", + token="runtime", + opener=lambda *_a, **_k: Response(b"{}", status=status), + ) + + +def test_successful_create_validates_each_ref_once(monkeypatch): + client = _load() + calls = [] + + def git_run(command, **_kwargs): + calls.append(command) + return type("Result", (), {"returncode": 0})() + + monkeypatch.setattr(client._validate_ref.__globals__["subprocess"], "run", git_run) + client.create_draft( + "cassandra", + base="main", + head="hermes/fix", + head_sha=HEAD_SHA, + title="Focused fix", + body="Review evidence", + token="runtime", + opener=lambda *_a, **_k: Response(_draft_response()), + ) + + assert [command[-1] for command in calls] == [ + "refs/heads/main", + "refs/heads/hermes/fix", + ] + + +def test_create_verifies_every_server_postcondition(): + client = _load() + calls = [] + + def opener(request, timeout): + calls.append((request, timeout)) + return Response(_draft_response()) + + result = client.create_draft( + "cassandra", + base="main", + head="hermes/fix", + head_sha=HEAD_SHA, + title="Focused fix", + body="Review evidence", + token="runtime", + opener=opener, + ) + + assert json.loads(result) == _draft_response() + payload = json.loads(calls[0][0].data) + assert payload == { + "base": "main", + "body": "Review evidence", + "head": "hermes/fix", + "title": "WIP: Focused fix", + } + assert calls[0][1] == 30 + + +def test_create_postcondition_rejects_every_material_mismatch(): + client = _load() + mutations = [ + ("number", 0), + ("number", 2_147_483_648), + ("state", "closed"), + ("draft", False), + ("merged", True), + ("html_url", "https://evil.example/pulls/3"), + ("url", "https://evil.example/api/pulls/3"), + ("title", "Focused fix"), + ("body", "different"), + ] + documents = [] + for key, value in mutations: + document = _draft_response() + document[key] = value + documents.append(document) + for path, value in [ + (("base", "ref"), "master"), + (("base", "repo", "full_name"), "evil/cassandra"), + (("head", "ref"), "other"), + (("head", "sha"), "0" * 40), + (("head", "repo", "full_name"), "evil/cassandra"), + ]: + document = copy.deepcopy(_draft_response()) + target = document + for key in path[:-1]: + target = target[key] + target[path[-1]] = value + documents.append(document) + + for document in documents: + with pytest.raises(client.PolicyError): + client._require_create_response( + json.dumps(document).encode(), + repo="cassandra", + base="main", + head="hermes/fix", + head_sha=HEAD_SHA, + title="WIP: Focused fix", + body="Review evidence", + ) + + +def test_read_response_is_bounded(): + client = _load() + + with pytest.raises(client.PolicyError, match="safe size limit"): + client.read( + "/api/v1/repos/atlas/cassandra", + token="runtime", + opener=lambda *_a, **_k: Response(b"x" * (client.MAX_RESPONSE_BYTES + 1)), + ) + + +def test_direct_api_rejects_unexpected_success_content_type(): + client = _load() + response = Response(b"{}") + response.headers.replace_header("Content-Type", "text/html") + + with pytest.raises(client.PolicyError, match="unexpected response type"): + client.read( + "/api/v1/repos/atlas/cassandra", + token="runtime", + opener=lambda *_a, **_k: response, + ) + + +def test_output_redaction_covers_exact_token_and_authorization_header(): + client = _load() + raw = b'{"message":"do-not-leak","debug":"Authorization: token do-not-leak"}' + redacted = client.redact_bytes(raw, "do-not-leak") + + assert b"do-not-leak" not in redacted + assert redacted.count(b"[REDACTED]") >= 1 + + +@pytest.mark.parametrize( + "reflected", + [ + b"runtime-sentinel", + base64.b64encode(b"runtime-sentinel"), + base64.b64encode(b"hermes-automation:runtime-sentinel"), + ], +) +def test_direct_api_rejects_credential_reflection(reflected: bytes): + client = _load() + + with pytest.raises(client.PolicyError, match="credential material"): + client.read( + "/api/v1/repos/atlas/cassandra", + token="runtime-sentinel", + opener=lambda *_a, **_k: Response( + b'{"unexpected":"' + reflected + b'"}' + ), + ) diff --git a/testing/tests/test_hermes_gitea_support.py b/testing/tests/test_hermes_gitea_support.py new file mode 100644 index 00000000..8b00e784 --- /dev/null +++ b/testing/tests/test_hermes_gitea_support.py @@ -0,0 +1,75 @@ +"""Shared fixtures for the Atlas-only Gitea client contracts.""" + +from __future__ import annotations + +import importlib.util +import json +import sys +from email.message import Message +from pathlib import Path + +ROOT = Path(__file__).parents[2] +CLIENT_PATH = ROOT / "services/hermes/scm-common/scripts/gitea_api.py" +HEAD_SHA = "465cf9146b05c174a2a8d310aff6c64be58277b6" +if str(CLIENT_PATH.parent) not in sys.path: + sys.path.insert(0, str(CLIENT_PATH.parent)) + + +def _load(): + spec = importlib.util.spec_from_file_location("safe_gitea_api", CLIENT_PATH) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _draft_payload(**updates): + payload = { + "base": "main", + "body": "Review evidence", + "head": "hermes/review-fix", + "title": "WIP: Repair review findings", + } + payload.update(updates) + return payload + + +def _draft_response(**updates): + response = { + "number": 3, + "state": "open", + "draft": True, + "merged": False, + "html_url": "https://scm.bstein.dev/atlas/cassandra/pulls/3", + "url": "https://scm.bstein.dev/atlas/cassandra/pulls/3", + "title": "WIP: Focused fix", + "body": "Review evidence", + "base": {"ref": "main", "repo": {"full_name": "atlas/cassandra"}}, + "head": { + "ref": "hermes/fix", + "sha": HEAD_SHA, + "repo": {"full_name": "atlas/cassandra"}, + }, + } + response.update(updates) + return response + + +class Response: + def __init__(self, body: object, status: int | None = None): + self.body = body if isinstance(body, bytes) else json.dumps(body).encode() + self.status = ( + status if status is not None else (201 if isinstance(body, dict) else 200) + ) + self.headers = Message() + self.headers["Content-Type"] = "application/json" + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def read(self, limit=-1): + return self.body if limit < 0 else self.body[:limit] diff --git a/testing/tests/test_hermes_image_builder_vault.py b/testing/tests/test_hermes_image_builder_vault.py index f46823f5..6dadf10c 100644 --- a/testing/tests/test_hermes_image_builder_vault.py +++ b/testing/tests/test_hermes_image_builder_vault.py @@ -56,7 +56,7 @@ def test_flux_orders_seed_then_jenkins_then_hermes() -> None: role_check = { "apiVersion": "batch/v1", "kind": "Job", - "name": "vault-k8s-auth-hermes-9", + "name": "vault-k8s-auth-hermes-10", "namespace": "vault", } seed_check = { diff --git a/testing/tests/test_hermes_node_account_hardening.py b/testing/tests/test_hermes_node_account_hardening.py new file mode 100644 index 00000000..9cf2bb3a --- /dev/null +++ b/testing/tests/test_hermes_node_account_hardening.py @@ -0,0 +1,252 @@ +"""Core contracts for the dedicated Hermes node SSH identity.""" + +from __future__ import annotations + +import os +from pathlib import Path + +import pytest +import yaml + +from testing.tests.test_hermes_node_account_support import ( + ROOT, + SCRIPT, + _fixture, + _key, + _load, +) + +def test_reconciler_creates_locked_groupless_account_and_moves_only_exact_key( + tmp_path: Path, monkeypatch +): + module, originals, key, other, public_key = _fixture(tmp_path, monkeypatch) + + module.reconcile(public_key) + first = { + name: (module.HOST_ETC / name).read_text(encoding="utf-8") + for name in originals + } + module.reconcile(public_key) + second = { + name: (module.HOST_ETC / name).read_text(encoding="utf-8") + for name in originals + } + + assert first == second + expected = module._expected_records() + for name, original in originals.items(): + assert first[name] == original + ":".join(expected[name]) + "\n" + backup = module.HOST_ETC / f"{name}.hermes-boundary-backup" + assert backup.read_text(encoding="utf-8") == original + assert expected["shadow"][1] == "!" + assert expected["group"][-1] == "" + assert "disk:x:6:atlas\n" in first["group"] + assert "sudo:x:27:oceanus\n" in first["group"] + + hermes_keys = ( + module.HOST_HOME / module.ACCOUNT / ".ssh/authorized_keys" + ).read_text(encoding="utf-8") + assert hermes_keys == key + "\n" + for user in module.LEGACY_ACCOUNTS: + legacy = ( + module.HOST_HOME / user / ".ssh/authorized_keys" + ).read_text(encoding="utf-8") + assert legacy == f"{other} {user}\n" + backup = module.HOST_HOME / user / ".ssh/authorized_keys.hermes-boundary-backup" + assert backup.read_text(encoding="utf-8") == f"{other} {user}\n{key}\n" + + +def test_identity_conflict_fails_closed_without_editing_account_databases( + tmp_path: Path, monkeypatch +): + module, originals, _key, _other, public_key = _fixture(tmp_path, monkeypatch) + conflict = originals["passwd"] + ( + f"unrelated:x:{module.ACCOUNT_UID}:{module.ACCOUNT_GID}:Other:/home/other:/bin/bash\n" + ) + (module.HOST_ETC / "passwd").write_text(conflict, encoding="utf-8") + + with pytest.raises(module.HardeningError, match="identity conflicts"): + module.reconcile(public_key) + + assert (module.HOST_ETC / "passwd").read_text(encoding="utf-8") == conflict + for name in ("group", "shadow", "gshadow"): + assert (module.HOST_ETC / name).read_text(encoding="utf-8") == originals[name] + assert not (module.HOST_HOME / module.ACCOUNT).exists() + + +def test_malformed_account_file_fails_closed(tmp_path: Path, monkeypatch): + module, originals, _key, _other, public_key = _fixture(tmp_path, monkeypatch) + (module.HOST_ETC / "group").write_text("malformed\n", encoding="utf-8") + + with pytest.raises(module.HardeningError, match="invalid record"): + module.reconcile(public_key) + + assert (module.HOST_ETC / "passwd").read_text(encoding="utf-8") == originals[ + "passwd" + ] + assert (module.HOST_ETC / "group").read_text(encoding="utf-8") == "malformed\n" + + +def test_preexisting_home_fails_before_account_database_or_key_changes( + tmp_path: Path, monkeypatch +): + module, originals, _key, _other, public_key = _fixture(tmp_path, monkeypatch) + home = module.HOST_HOME / module.ACCOUNT + home.mkdir() + (home / "unrelated").write_text("preserve\n", encoding="utf-8") + + with pytest.raises(module.HardeningError, match="home already exists"): + module.reconcile(public_key) + + for name, original in originals.items(): + assert (module.HOST_ETC / name).read_text(encoding="utf-8") == original + assert not (module.HOST_ETC / f"{name}.hermes-boundary-backup").exists() + assert (home / "unrelated").read_text(encoding="utf-8") == "preserve\n" + + +def test_flux_daemonset_reconciles_every_node_without_mutating_human_groups(): + documents = list( + yaml.safe_load_all( + (ROOT / "services/hermes/node-ssh-access.yaml").read_text(encoding="utf-8") + ) + ) + daemonset = next(item for item in documents if item["kind"] == "DaemonSet") + spec = daemonset["spec"]["template"]["spec"] + assert spec["tolerations"] == [{"operator": "Exists"}] + command = spec["containers"][0]["args"][0] + assert "/opt/node-hardener/node_account_hardening.py" in command + assert "sleep 300" in command + mounts = {item["name"]: item for item in spec["volumes"]} + assert mounts["host-home"]["hostPath"]["path"] == "/home" + assert mounts["host-etc"]["hostPath"]["path"] == "/etc" + assert mounts["host-k3s"]["hostPath"]["path"] == "/var/lib/rancher/k3s" + assert mounts["host-kubelet"]["hostPath"]["path"] == "/var/lib/kubelet" + assert mounts["host-run-k3s"]["hostPath"]["path"] == "/run/k3s" + assert mounts["host-run-containerd"]["hostPath"]["path"] == "/run/containerd" + assert "usermod" not in command + assert "groupmod" not in command + + policies = list( + yaml.safe_load_all( + (ROOT / "services/hermes/networkpolicy.yaml").read_text(encoding="utf-8") + ) + ) + isolation = next( + item + for item in policies + if item["metadata"]["name"] == "hermes-node-ssh-access-isolation" + ) + assert isolation["spec"]["ingress"] == [] + assert isolation["spec"]["egress"] == [] + + source = SCRIPT.read_text(encoding="utf-8") + assert "ACCOUNT = \"hermes-agent\"" in source + assert "ACCOUNT_UID = 1200" in source + assert "ACCOUNT_GID = 1200" in source + assert 'LEGACY_ACCOUNTS = ("atlas", "oceanus")' in source + assert "audit_membership" in source + image = spec["containers"][0]["image"] + assert image == ( + "python@sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df" + ) + assert mounts["host-polkit-share"]["hostPath"]["path"] == "/usr/share/polkit-1" + + +def test_sensitive_roots_get_explicit_zero_permission_acl_for_hermes( + tmp_path: Path, monkeypatch +): + module = _load() + sensitive = tmp_path / "k3s" + sensitive.mkdir(mode=0o755) + host_etc = tmp_path / "etc" + host_etc.mkdir() + monkeypatch.setattr(module, "ACCOUNT_UID", 1200) + monkeypatch.setattr(module, "HOST_ETC", host_etc) + monkeypatch.setattr(module, "HOST_ROOT_UID", os.getuid()) + monkeypatch.setattr(module, "HOST_ROOT_GID", os.getgid()) + # Production requires uid 0. The test process owns its fixture but runs the + # real xattr/ACL implementation using the test owner's uid as that boundary. + + module._deny_sensitive_root(sensitive) + first = module._read_acl(sensitive) + module._deny_sensitive_root(sensitive) + second = module._read_acl(sensitive) + + assert first == second + entries = module._decode_acl(first, 0o755) + assert (module.ACL_USER, 0, 1200) in entries + assert (module.ACL_USER_OBJ, 0o7, module.ACL_UNDEFINED_ID) in entries + assert (module.ACL_GROUP_OBJ, 0o5, module.ACL_UNDEFINED_ID) in entries + assert (module.ACL_OTHER, 0o5, module.ACL_UNDEFINED_ID) in entries + backup = host_etc / "hermes-node-boundary/k3s.acl" + assert backup.read_bytes() == b"N" + assert backup.stat().st_mode & 0o777 == 0o600 + + +def test_sensitive_root_rejects_non_directory_and_non_root_owner( + tmp_path: Path, monkeypatch +): + module = _load() + regular = tmp_path / "not-a-directory" + regular.write_text("preserve", encoding="utf-8") + with pytest.raises(module.HardeningError, match="unsafe sensitive directory"): + module._deny_sensitive_root(regular) + + directory = tmp_path / "not-root-owned" + directory.mkdir() + if directory.stat().st_uid == 0: + pytest.skip("cannot construct a non-root-owned fixture as root") + with pytest.raises(module.HardeningError, match="not root-owned"): + module._deny_sensitive_root(directory) + + +def test_acl_parser_rejects_malformed_or_incomplete_values(): + module = _load() + with pytest.raises(module.HardeningError, match="malformed"): + module._decode_acl(b"too short", 0o755) + incomplete = module._encode_acl( + [(module.ACL_USER_OBJ, 0o7, module.ACL_UNDEFINED_ID)] + ) + with pytest.raises(module.HardeningError, match="incomplete"): + module._decode_acl(incomplete, 0o755) + + +def test_acl_failure_stops_before_installing_the_ssh_key(tmp_path: Path, monkeypatch): + module = _load() + host_etc = tmp_path / "etc" + host_etc.mkdir() + monkeypatch.setattr(module, "HOST_ETC", host_etc) + monkeypatch.setattr(module, "HOST_ROOT_UID", os.getuid()) + public_key = tmp_path / "public-key" + public_key.write_text( + _key(b"synthetic-key") + "\n", + encoding="utf-8", + ) + moved = False + monkeypatch.setattr(module, "_reconcile_databases", lambda: None) + + def fail_acl(): + raise module.HardeningError("ACL unavailable") + + def move_key(_path): + nonlocal moved + moved = True + + monkeypatch.setattr(module, "_deny_sensitive_roots", fail_acl) + monkeypatch.setattr(module, "_move_key", move_key) + + with pytest.raises(module.HardeningError, match="ACL unavailable"): + module.reconcile(public_key) + assert moved is False + + +def test_runtime_ssh_config_forces_dedicated_account_for_every_titan(): + stage = ( + ROOT / "services/hermes/scripts/stage_runtime_access.py" + ).read_text(encoding="utf-8") + assert '"Host titan-*\\n User hermes-agent\\n"' in stage + agent = yaml.safe_load( + (ROOT / "services/hermes/agent-deployment.yaml").read_text(encoding="utf-8") + ) + assert "User atlas" not in yaml.safe_dump(agent) + assert "User oceanus" not in yaml.safe_dump(agent) diff --git a/testing/tests/test_hermes_node_account_io.py b/testing/tests/test_hermes_node_account_io.py new file mode 100644 index 00000000..6b05cc0a --- /dev/null +++ b/testing/tests/test_hermes_node_account_io.py @@ -0,0 +1,222 @@ +"""Crash-safety and metadata contracts for Hermes node-account writes.""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path + +import pytest +import yaml + +from testing.tests.test_hermes_node_account_support import ( + ROOT, + _fixture, + _load, + _typed_key, +) + +def test_key_identity_ignores_options_and_comments_and_removes_legacy_first( + tmp_path: Path, monkeypatch +): + module, _originals, key, other, public_key = _fixture(tmp_path, monkeypatch) + key_fields = key.split() + decorated = f'restrict,command="echo denied" {key_fields[0]} {key_fields[1]} old-comment' + for user in module.LEGACY_ACCOUNTS: + path = module.HOST_HOME / user / ".ssh/authorized_keys" + path.write_text(f"{other} {user}\n{decorated}\n", encoding="utf-8") + writes: list[Path] = [] + real_write = module.atomic_write + + def recording_write(path, value, metadata): + writes.append(path) + real_write(path, value, metadata) + + monkeypatch.setattr(module, "atomic_write", recording_write) + module.reconcile(public_key) + + key_writes = [path for path in writes if path.name == "authorized_keys"] + assert key_writes[-1].parts[-3:] == (module.ACCOUNT, ".ssh", "authorized_keys") + assert all(module.ACCOUNT not in path.parts for path in key_writes[:-1]) + + +def test_unrelated_modern_human_key_type_is_preserved(tmp_path: Path, monkeypatch): + module, _originals, key, _other, public_key = _fixture(tmp_path, monkeypatch) + modern = _typed_key("sk-ssh-ed25519@openssh.com", b"human-security-key", "human") + for user in module.LEGACY_ACCOUNTS: + path = module.HOST_HOME / user / ".ssh/authorized_keys" + path.write_text(f"{modern}\n{key}\n", encoding="utf-8") + + module.reconcile(public_key) + + for user in module.LEGACY_ACCOUNTS: + path = module.HOST_HOME / user / ".ssh/authorized_keys" + assert path.read_text() == modern + "\n" + + +def test_crash_during_legacy_key_removal_never_installs_duplicate_key( + tmp_path: Path, monkeypatch +): + module, _originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + real_write = module.atomic_write + key_writes = 0 + + def crash_on_second_key(path, value, metadata): + nonlocal key_writes + if path.name == "authorized_keys": + key_writes += 1 + if key_writes == 2: + raise module.HardeningError("synthetic crash") + real_write(path, value, metadata) + + monkeypatch.setattr(module, "atomic_write", crash_on_second_key) + with pytest.raises(module.HardeningError, match="synthetic crash"): + module.reconcile(public_key) + assert not (module.HOST_HOME / module.ACCOUNT / ".ssh/authorized_keys").exists() + + +def test_preexisting_target_key_is_removed_before_legacy_cleanup_and_reinstalled( + tmp_path: Path, monkeypatch +): + module, _originals, key, _other, public_key = _fixture(tmp_path, monkeypatch) + target = module.HOST_HOME / module.ACCOUNT / ".ssh/authorized_keys" + target.parent.mkdir(parents=True) + target.write_text(key + " existing\n") + writes: list[Path] = [] + real_write = module.atomic_write + + def recording_write(path, value, metadata): + if path.name == "authorized_keys": + writes.append(path) + real_write(path, value, metadata) + + monkeypatch.setattr(module, "atomic_write", recording_write) + module._move_key(public_key) + + assert writes[0] == target + assert writes[-1] == target + assert all(path != target for path in writes[1:-1]) + assert target.read_text() == key + "\n" + + +def test_account_database_write_preserves_mode_owner_and_xattrs( + tmp_path: Path, monkeypatch +): + module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + passwd = module.HOST_ETC / "passwd" + passwd.chmod(0o640) + os.setxattr(passwd, "user.hermes-test", b"preserve") + before = passwd.stat() + + module.reconcile(public_key) + + after = passwd.stat() + assert after.st_mode & 0o777 == 0o640 + assert (after.st_uid, after.st_gid) == (before.st_uid, before.st_gid) + assert os.getxattr(passwd, "user.hermes-test") == b"preserve" + backup = module.HOST_ETC / "passwd.hermes-boundary-backup" + assert os.getxattr(backup, "user.hermes-test") == b"preserve" + assert backup.read_text() == originals["passwd"] + + +def test_atomic_write_restores_acl_and_security_label_xattrs(tmp_path: Path, monkeypatch): + module = _load() + io_module = sys.modules[module.atomic_write.__module__] + target = tmp_path / "passwd" + target.write_bytes(b"old\n") + metadata = io_module.FileSnapshot( + value=b"old\n", + device=target.stat().st_dev, + inode=target.stat().st_ino, + mode=0o600, + uid=os.getuid(), + gid=os.getgid(), + size=4, + mtime_ns=target.stat().st_mtime_ns, + ctime_ns=target.stat().st_ctime_ns, + xattrs=( + ("system.posix_acl_access", b"synthetic-acl"), + ("security.selinux", b"synthetic-label"), + ("user.audit", b"synthetic-xattr"), + ), + ) + restored: list[tuple[str, bytes]] = [] + monkeypatch.setattr( + io_module.os, + "setxattr", + lambda _fd, name, value: restored.append((name, value)), + ) + + io_module.atomic_write(target, b"new\n", metadata) + + assert target.read_bytes() == b"new\n" + assert restored == list(metadata.xattrs) + + +def test_concurrent_database_change_is_detected_before_any_account_write( + tmp_path: Path, monkeypatch +): + module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + real_backup = module.backup_once + backup_count = 0 + writes: list[Path] = [] + + def racing_backup(path, snapshot, maximum): + nonlocal backup_count + result = real_backup(path, snapshot, maximum) + backup_count += 1 + if backup_count == 4: + group = module.HOST_ETC / "group" + group.write_text(originals["group"] + "race:x:4000:\n") + return result + + monkeypatch.setattr(module, "backup_once", racing_backup) + monkeypatch.setattr(module, "atomic_write", lambda path, *_args: writes.append(path)) + with pytest.raises(module.HardeningError, match="concurrent host account change"): + module.reconcile(public_key) + assert writes == [] + + +def test_flux_orders_observer_rbac_before_hermes_prunes_old_authority(): + observer = yaml.safe_load( + (ROOT / "clusters/atlas/flux-system/applications/hermes-observer-rbac/kustomization.yaml").read_text() + ) + hermes = yaml.safe_load( + (ROOT / "clusters/atlas/flux-system/applications/hermes/kustomization.yaml").read_text() + ) + bindings = yaml.safe_load( + ( + ROOT + / "clusters/atlas/flux-system/applications/hermes-observer-bindings/kustomization.yaml" + ).read_text() + ) + assert "dependsOn" not in observer["spec"] + assert {item["name"] for item in hermes["spec"]["dependsOn"]} >= { + "hermes-observer-rbac", + "hermes-scm-broker", + } + assert {item["name"] for item in bindings["spec"]["dependsOn"]} == { + "hermes-observer-rbac", + "hermes", + } + assert "hermes-observer-bindings" not in { + item["name"] for item in hermes["spec"]["dependsOn"] + } + + +def test_sensitive_root_acl_backups_have_unique_host_path_names(monkeypatch): + module = _load() + seen = [] + monkeypatch.setattr( + module, + "_deny_sensitive_root", + lambda path, backup_name=None: seen.append((path, backup_name)), + ) + module._deny_sensitive_roots() + assert [name for _path, name in seen] == [ + "var-lib-rancher-k3s", + "var-lib-kubelet", + "run-k3s", + "run-containerd", + ] + assert len({name for _path, name in seen}) == 4 diff --git a/testing/tests/test_hermes_node_account_privilege_audit.py b/testing/tests/test_hermes_node_account_privilege_audit.py new file mode 100644 index 00000000..3370fe4f --- /dev/null +++ b/testing/tests/test_hermes_node_account_privilege_audit.py @@ -0,0 +1,196 @@ +"""Host privilege-policy audits for the Hermes node SSH identity.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from testing.tests.test_hermes_node_account_support import _fixture + + +@pytest.mark.parametrize( + "database,record", + [ + ("group", "disk:x:6:atlas,hermes-agent\n"), + ("gshadow", "disk:!:hermes-agent:atlas\n"), + ("gshadow", "disk:!::atlas,hermes-agent\n"), + ], +) +def test_unexpected_group_or_gshadow_membership_fails_before_writes( + tmp_path: Path, monkeypatch, database: str, record: str +): + module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + path = module.HOST_ETC / database + path.write_text( + "\n".join( + line + for line in originals[database].splitlines() + if not line.startswith("disk:") + ) + + "\n" + + record, + encoding="utf-8", + ) + before = {name: (module.HOST_ETC / name).read_bytes() for name in originals} + + with pytest.raises(module.HardeningError, match="group access"): + module.reconcile(public_key) + + assert {name: (module.HOST_ETC / name).read_bytes() for name in originals} == before + + +@pytest.mark.parametrize( + "relative,value", + [ + ("sudoers.d/hermes", "hermes-agent ALL=(ALL:ALL) NOPASSWD: ALL\n"), + ("sudoers", "ALL ALL=(ALL:ALL) ALL\n"), + ( + "polkit-1/rules.d/90-root.rules", + 'if (subject.user == "hermes-agent") return polkit.Result.YES;\n', + ), + ( + "polkit-1/rules.d/90-root.rules", + "unix-user:* org.freedesktop.policykit.exec polkit.Result.YES\n", + ), + ], +) +def test_sudo_and_root_equivalent_polkit_authority_fail_closed( + tmp_path: Path, monkeypatch, relative: str, value: str +): + module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + policy = module.HOST_ETC / relative + policy.parent.mkdir(parents=True, exist_ok=True) + policy.write_text(value, encoding="utf-8") + + with pytest.raises(module.HardeningError, match="sudo|polkit"): + module.reconcile(public_key) + + assert all( + (module.HOST_ETC / name).read_text() == original + for name, original in originals.items() + ) + + +def test_external_group_authority_source_is_rejected(tmp_path: Path, monkeypatch): + module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + (module.HOST_ETC / "nsswitch.conf").write_text("passwd: files\ngroup: files ldap\n") + with pytest.raises(module.HardeningError, match="external group"): + module.reconcile(public_key) + assert all( + (module.HOST_ETC / name).read_text() == original + for name, original in originals.items() + ) + + +@pytest.mark.parametrize("database", ["passwd", "group", "initgroups", "shadow"]) +def test_external_identity_authority_sources_fail_closed( + tmp_path: Path, monkeypatch, database: str +): + module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + (module.HOST_ETC / "nsswitch.conf").write_text( + f"passwd: files\n{database}: files ldap\n", encoding="utf-8" + ) + with pytest.raises(module.HardeningError, match="external group/account"): + module.reconcile(public_key) + assert all( + (module.HOST_ETC / name).read_text() == original + for name, original in originals.items() + ) + + +@pytest.mark.parametrize( + "value", + [ + "#1200 ALL=(ALL:ALL) NOPASSWD: ALL\n", + "User_Alias HERMES_IDS = #1200\nHERMES_IDS ALL=(ALL:ALL) NOPASSWD: ALL\n", + "User_Alias HERMES_GROUP = %#1200\nHERMES_GROUP ALL=(ALL:ALL) NOPASSWD: ALL\n", + ], +) +def test_visudo_valid_numeric_and_alias_grants_fail_closed( + tmp_path: Path, monkeypatch, value: str +): + module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + monkeypatch.setattr(module, "ACCOUNT_UID", 1200) + policy = module.HOST_ETC / "sudoers" + policy.write_text(value, encoding="utf-8") + validation = subprocess.run( + ["/usr/bin/visudo", "-c", "-f", str(policy)], + check=False, + capture_output=True, + text=True, + ) + assert validation.returncode == 0, validation.stderr + + with pytest.raises(module.HardeningError, match="sudo authority"): + module.reconcile(public_key) + + assert all( + (module.HOST_ETC / name).read_text() == original + for name, original in originals.items() + ) + + +def test_sudo_include_is_bounded_to_the_audited_standard_directory( + tmp_path: Path, monkeypatch +): + module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + sudoers = module.HOST_ETC / "sudoers" + sudoers.write_text("#includedir /opt/external-sudoers\n", encoding="utf-8") + with pytest.raises(module.HardeningError, match="unaudited authority source"): + module.reconcile(public_key) + + sudoers.write_text("#includedir /etc/sudoers.d\n", encoding="utf-8") + module.reconcile(public_key) + assert all( + (module.HOST_ETC / name).read_text().startswith(original) + for name, original in originals.items() + ) + + +@pytest.mark.parametrize( + "relative,value", + [ + ( + "polkit-1/rules.d/90-hermes.rules", + "polkit.addRule(function(action, subject) {\n" + " if (subject.uid == 1200) return polkit.Result.YES;\n" + "});\n", + ), + ( + "polkit-1/localauthority/50-local.d/hermes.pkla", + "[Hermes]\nIdentity=unix-user:1200\n" + "Action=org.freedesktop.policykit.exec\nResultActive=yes\n", + ), + ], +) +def test_numeric_polkit_grants_fail_closed( + tmp_path: Path, monkeypatch, relative: str, value: str +): + module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + monkeypatch.setattr(module, "ACCOUNT_UID", 1200) + policy = module.HOST_ETC / relative + policy.parent.mkdir(parents=True, exist_ok=True) + policy.write_text(value, encoding="utf-8") + + with pytest.raises(module.HardeningError, match="polkit authority"): + module.reconcile(public_key) + + assert all( + (module.HOST_ETC / name).read_text() == original + for name, original in originals.items() + ) + + +def test_writable_privilege_policy_is_rejected(tmp_path: Path, monkeypatch): + module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch) + sudoers = module.HOST_ETC / "sudoers" + sudoers.write_text("root ALL=(ALL:ALL) ALL\n") + sudoers.chmod(0o666) + with pytest.raises(module.HardeningError, match="unsafe mutation"): + module.reconcile(public_key) + assert all( + (module.HOST_ETC / name).read_text() == original + for name, original in originals.items() + ) diff --git a/testing/tests/test_hermes_node_account_support.py b/testing/tests/test_hermes_node_account_support.py new file mode 100644 index 00000000..2d74d124 --- /dev/null +++ b/testing/tests/test_hermes_node_account_support.py @@ -0,0 +1,90 @@ +"""Shared fixtures for dedicated Hermes node-account contracts.""" + +from __future__ import annotations + +import base64 +import importlib.util +import os +import sys +from pathlib import Path + +ROOT = Path(__file__).parents[2] +SCRIPT = ROOT / "services/hermes/scripts/node_account_hardening.py" +sys.path.insert(0, str(SCRIPT.parent)) + + +def _key(material: bytes, comment: str = "") -> str: + key_type = b"ssh-ed25519" + blob = len(key_type).to_bytes(4, "big") + key_type + material + suffix = f" {comment}" if comment else "" + return "ssh-ed25519 " + base64.b64encode(blob).decode() + suffix + + +def _typed_key(key_type: str, material: bytes, comment: str = "") -> str: + encoded_type = key_type.encode() + blob = len(encoded_type).to_bytes(4, "big") + encoded_type + material + suffix = f" {comment}" if comment else "" + return f"{key_type} {base64.b64encode(blob).decode()}{suffix}" + + +def _load(): + spec = importlib.util.spec_from_file_location("node_account_hardening_test", SCRIPT) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _fixture(tmp_path: Path, monkeypatch): + module = _load() + host_etc = tmp_path / "etc" + host_home = tmp_path / "home" + host_etc.mkdir() + host_home.mkdir() + originals = { + "passwd": ( + "root:x:0:0:root:/root:/bin/bash\n" + "atlas:x:2000:2000:Atlas:/home/atlas:/bin/bash\n" + "oceanus:x:2001:2001:Oceanus:/home/oceanus:/bin/bash\n" + ), + "group": ( + "root:x:0:\n" + "atlas:x:2000:\n" + "oceanus:x:2001:\n" + "disk:x:6:atlas\n" + "sudo:x:27:oceanus\n" + ), + "shadow": ( + "root:!:1:0:99999:7:::\n" + "atlas:!:1:0:99999:7:::\n" + "oceanus:!:1:0:99999:7:::\n" + ), + "gshadow": ( + "root:!::\n" + "atlas:!::\n" + "oceanus:!::\n" + "disk:!::atlas\n" + "sudo:!::oceanus\n" + ), + } + for name, value in originals.items(): + (host_etc / name).write_text(value, encoding="utf-8") + key = _key(b"synthetic-hermes-key") + other = _key(b"human-operator-key") + for user in ("atlas", "oceanus"): + ssh = host_home / user / ".ssh" + ssh.mkdir(parents=True) + (ssh / "authorized_keys").write_text( + f"{other} {user}\n{key}\n", encoding="utf-8" + ) + public_key = tmp_path / "public-key" + public_key.write_text(key + "\n", encoding="utf-8") + monkeypatch.setattr(module, "HOST_ETC", host_etc) + monkeypatch.setattr(module, "HOST_HOME", host_home) + monkeypatch.setattr(module, "ACCOUNT_UID", os.getuid()) + monkeypatch.setattr(module, "ACCOUNT_GID", os.getgid()) + monkeypatch.setattr(module, "HOST_ROOT_UID", os.getuid()) + monkeypatch.setattr(module, "HOST_POLKIT_SHARE", tmp_path / "missing-polkit") + monkeypatch.setattr(module, "_deny_sensitive_roots", lambda: None) + return module, originals, key, other, public_key diff --git a/testing/tests/test_hermes_node_acl_coverage.py b/testing/tests/test_hermes_node_acl_coverage.py new file mode 100644 index 00000000..572fd0d2 --- /dev/null +++ b/testing/tests/test_hermes_node_acl_coverage.py @@ -0,0 +1,206 @@ +"""Behavioral branch coverage for node-storage ACL hardening.""" + +from __future__ import annotations + +import errno +import os +import pytest + +from testing.tests.test_hermes_node_account_support import _fixture, _load + + +def test_acl_decoder_rejects_version_permissions_and_adds_missing_mask(): + module = _load() + valid = [ + (module.ACL_USER_OBJ, 7, module.ACL_UNDEFINED_ID), + (module.ACL_GROUP_OBJ, 5, module.ACL_UNDEFINED_ID), + (module.ACL_OTHER, 5, module.ACL_UNDEFINED_ID), + ] + wrong_version = module.ACL_HEADER.pack(99) + b"".join( + module.ACL_ENTRY.pack(*entry) for entry in valid + ) + with pytest.raises(module.HardeningError, match="version"): + module._decode_acl(wrong_version, 0o755) + bad_permission = module._encode_acl([*valid, (module.ACL_USER, 8, 1200)]) + with pytest.raises(module.HardeningError, match="permission"): + module._decode_acl(bad_permission, 0o755) + updated = module._decode_acl( + module._acl_with_deny(module._encode_acl(valid), 0o755), 0o755 + ) + assert (module.ACL_MASK, 5, module.ACL_UNDEFINED_ID) in updated + assert (module.ACL_USER, 0, module.ACCOUNT_UID) in updated + + +def test_read_acl_handles_absence_and_propagates_other_errors(monkeypatch, tmp_path): + module = _load() + path = tmp_path / "root" + path.mkdir() + + def missing(*_args, **_kwargs): + raise OSError(errno.ENODATA, "missing") + + monkeypatch.setattr(module.os, "getxattr", missing) + assert module._read_acl(path) == b"" + + def denied(*_args, **_kwargs): + raise OSError(errno.EPERM, "denied") + + monkeypatch.setattr(module.os, "getxattr", denied) + with pytest.raises(OSError): + module._read_acl(path) + + +@pytest.mark.parametrize("value", [b"", b"X", b"A", b"Nextra"]) +def test_acl_backup_validator_rejects_malformed_encodings(value): + module = _load() + with pytest.raises(module.HardeningError, match="backup is malformed"): + module._validate_acl_backup(value) + module._validate_acl_backup(b"N") + valid = module._encode_acl( + [ + (module.ACL_USER_OBJ, 7, module.ACL_UNDEFINED_ID), + (module.ACL_GROUP_OBJ, 5, module.ACL_UNDEFINED_ID), + (module.ACL_OTHER, 5, module.ACL_UNDEFINED_ID), + ] + ) + module._validate_acl_backup(b"A" + valid) + + +def test_acl_backup_reuses_valid_existing_copy(tmp_path, monkeypatch): + module = _load() + host_etc = tmp_path / "etc" + root = host_etc / "hermes-node-boundary" + root.mkdir(parents=True) + backup = root / "k3s.acl" + backup.write_bytes(b"N") + monkeypatch.setattr(module, "HOST_ETC", host_etc) + monkeypatch.setattr(module, "HOST_ROOT_UID", os.getuid()) + monkeypatch.setattr(module, "HOST_ROOT_GID", os.getgid()) + module._acl_backup_once(tmp_path / "k3s", b"") + assert backup.read_bytes() == b"N" + + +def test_acl_backup_short_write_cleans_temporary(tmp_path, monkeypatch): + module = _load() + host_etc = tmp_path / "etc" + host_etc.mkdir() + monkeypatch.setattr(module, "HOST_ETC", host_etc) + monkeypatch.setattr(module, "HOST_ROOT_UID", os.getuid()) + monkeypatch.setattr(module, "HOST_ROOT_GID", os.getgid()) + monkeypatch.setattr(module.os, "write", lambda _fd, _value: 0) + with pytest.raises(module.HardeningError, match="short sensitive"): + module._acl_backup_once(tmp_path / "k3s", b"") + assert not list((host_etc / "hermes-node-boundary").glob(".*.hermes-*")) + + +def test_acl_backup_tolerates_concurrent_first_writer(tmp_path, monkeypatch): + module = _load() + host_etc = tmp_path / "etc" + host_etc.mkdir() + monkeypatch.setattr(module, "HOST_ETC", host_etc) + monkeypatch.setattr(module, "HOST_ROOT_UID", os.getuid()) + monkeypatch.setattr(module, "HOST_ROOT_GID", os.getgid()) + real_link = os.link + + def race(source, target, **kwargs): + real_link(source, target, **kwargs) + raise FileExistsError() + + monkeypatch.setattr(module.os, "link", race) + module._acl_backup_once(tmp_path / "k3s", b"") + assert (host_etc / "hermes-node-boundary/k3s.acl").read_bytes() == b"N" + + +def test_sensitive_root_failure_restores_existing_acl(tmp_path, monkeypatch): + module = _load() + root = tmp_path / "k3s" + root.mkdir() + monkeypatch.setattr(module, "HOST_ROOT_UID", os.getuid()) + current = module._encode_acl( + [ + (module.ACL_USER_OBJ, 7, module.ACL_UNDEFINED_ID), + (module.ACL_GROUP_OBJ, 5, module.ACL_UNDEFINED_ID), + (module.ACL_MASK, 5, module.ACL_UNDEFINED_ID), + (module.ACL_OTHER, 5, module.ACL_UNDEFINED_ID), + ] + ) + reads = iter((current, current)) + writes = [] + monkeypatch.setattr(module, "_read_acl", lambda _path: next(reads)) + monkeypatch.setattr(module, "_acl_backup_once", lambda *_a, **_k: None) + monkeypatch.setattr( + module.os, "setxattr", lambda *args, **kwargs: writes.append(args[2]) + ) + with pytest.raises(module.HardeningError, match="validation failed"): + module._deny_sensitive_root(root) + assert writes[-1] == current + + +@pytest.mark.parametrize("remove_errno", [errno.ENODATA, errno.EPERM]) +def test_sensitive_root_failure_removes_new_acl_or_propagates_cleanup_error( + tmp_path, monkeypatch, remove_errno +): + module = _load() + root = tmp_path / "k3s" + root.mkdir() + monkeypatch.setattr(module, "HOST_ROOT_UID", os.getuid()) + monkeypatch.setattr(module, "_read_acl", lambda _path: b"") + monkeypatch.setattr(module, "_acl_backup_once", lambda *_a, **_k: None) + monkeypatch.setattr( + module.os, + "setxattr", + lambda *_a, **_k: (_ for _ in ()).throw(RuntimeError("set failed")), + ) + monkeypatch.setattr( + module.os, + "removexattr", + lambda *_a, **_k: (_ for _ in ()).throw(OSError(remove_errno, "remove")), + ) + expected = OSError if remove_errno == errno.EPERM else RuntimeError + with pytest.raises(expected): + module._deny_sensitive_root(root) + + +def test_database_validation_failure_rolls_back_all_writes(tmp_path, monkeypatch): + module, originals, _key_value, _other, _public = _fixture(tmp_path, monkeypatch) + real_read = module._read_regular + calls = 0 + + def missing_expected(path, maximum=module.MAX_ACCOUNT_FILE): + nonlocal calls + calls += 1 + if calls == 1: + return originals["passwd"].encode(), path.stat(follow_symlinks=False) + return real_read(path, maximum) + + monkeypatch.setattr(module, "_read_regular", missing_expected) + with pytest.raises(module.HardeningError, match="validation failed"): + module._reconcile_databases() + for name, value in originals.items(): + assert (module.HOST_ETC / name).read_text() == value + + +def test_preexisting_target_key_removal_is_verified(tmp_path, monkeypatch): + module, _originals, key, _other, public = _fixture(tmp_path, monkeypatch) + target = module.HOST_HOME / module.ACCOUNT / ".ssh/authorized_keys" + target.parent.mkdir(parents=True) + target.write_text(key + "\n", encoding="utf-8") + real_write = module.atomic_write + + def ignore_preinstall_target(path, value, metadata): + if path == target and value == b"": + return + real_write(path, value, metadata) + + monkeypatch.setattr(module, "atomic_write", ignore_preinstall_target) + with pytest.raises(module.HardeningError, match="preexisting Hermes authorization"): + module._move_key(public) + + +def test_existing_target_without_key_does_not_need_precleanup(tmp_path, monkeypatch): + module, _originals, _key_value, other, public = _fixture(tmp_path, monkeypatch) + target = module.HOST_HOME / module.ACCOUNT / ".ssh/authorized_keys" + target.parent.mkdir(parents=True) + target.write_text(other + "\n", encoding="utf-8") + module._move_key(public) + assert target.read_text().endswith("\n") diff --git a/testing/tests/test_hermes_node_audit_coverage.py b/testing/tests/test_hermes_node_audit_coverage.py new file mode 100644 index 00000000..20114ada --- /dev/null +++ b/testing/tests/test_hermes_node_audit_coverage.py @@ -0,0 +1,265 @@ +"""Behavioral branch coverage for dedicated node-account privilege audits.""" + +from __future__ import annotations + +import os +import struct +import sys +from types import SimpleNamespace + +import pytest + +from testing.tests.test_hermes_node_account_support import _load + + +def _audit_module(): + hardening = _load() + return sys.modules[hardening.audit_membership.__module__] + + +def test_member_parser_handles_empty_valid_malformed_and_duplicates(): + module = _audit_module() + assert module._members("", "group") == set() + assert module._members("one,two", "group") == {"one", "two"} + for value in ("one,", " one", "one,one"): + with pytest.raises(module.HardeningError): + module._members(value, "group") + + +def test_membership_audit_accepts_unrelated_records_and_rejects_account(): + module = _audit_module() + module.audit_membership( + "hermes-agent", [["disk", "x", "6", "atlas"]], [["disk", "!", "atlas", ""]] + ) + with pytest.raises(module.HardeningError, match="supplementary"): + module.audit_membership( + "hermes-agent", [["disk", "x", "6", "hermes-agent"]], [] + ) + with pytest.raises(module.HardeningError, match="gshadow"): + module.audit_membership("hermes-agent", [], [["disk", "!", "hermes-agent", ""]]) + + +@pytest.mark.parametrize("kind", ["mode", "symlink"]) +def test_policy_files_reject_unsafe_directories(tmp_path, kind): + module = _audit_module() + host_etc = tmp_path / "etc" + host_etc.mkdir() + root = host_etc / "sudoers.d" + if kind == "mode": + root.mkdir(mode=0o777) + root.chmod(0o777) + else: + target = tmp_path / "target" + target.mkdir() + root.symlink_to(target) + with pytest.raises(module.HardeningError, match="unsafe sudo/polkit"): + module._policy_files(host_etc, tmp_path / "share", os.getuid()) + + +def test_policy_file_count_is_bounded(tmp_path): + module = _audit_module() + root = tmp_path / "etc/sudoers.d" + root.mkdir(parents=True) + for index in range(257): + (root / str(index)).write_text("# safe\n", encoding="utf-8") + with pytest.raises(module.HardeningError, match="too many"): + module._policy_files(tmp_path / "etc", tmp_path / "share", os.getuid()) + + +def _acl(*entries): + return struct.pack("= 1000", "exact literal"), + ("subject.uid == -1", "operators"), + ("subject.user == name", "exact literal"), + ("subject.user.indexOf(\"h\") == 0", "exact literal"), + ("subject.isInGroup(group)", "exact literal"), + ('subject["user"] == "atlas"', "computed lookup"), + ], +) +def test_computed_identity_grants_fail_closed(snippet: str, match: str): + module = _load() + text = ( + "polkit.addRule(function(action, subject) {\n" + f" if ({snippet}) {{ return polkit.Result.YES; }}\n" + "});\n" + ) + with pytest.raises(module.HardeningError, match=match): + _rules(module, text) + + +@pytest.mark.parametrize( + ("text", "match"), + [ + ("var x = `hermes`;\n", "template strings"), + ('var x = "unterminated\n', "unterminated"), + ('var x = "bad\\x2descape";\n', "computed escapes"), + ('var x = "dangling\\', "computed escapes"), + ("/* never closed\n", "comment is unterminated"), + ("var f = eval;\n", "cannot prove"), + ('var s = String.fromCharCode(104);\n', "cannot prove"), + ("var a = arguments;\n", "cannot prove"), + ('var r = polkit.Result["YES"];\n', "computed lookup"), + ("var r = polkit.Result;\n", "computed lookup"), + ("var s = subject;\n", "computed code"), + ("check(subject);\n", "computed code"), + ], +) +def test_unprovable_rule_constructs_fail_closed(text: str, match: str): + module = _load() + with pytest.raises(module.HardeningError, match=match): + _rules(module, text) + + +def test_regex_literal_tokenizer_desync_grant_is_rejected(): + """A regex literal embedding a quote must not open a phantom string. + + Without regex awareness the two /'/ literals fool the string scanner into + swallowing the unconditional Result.YES between their quotes; the grant + then reads as unscoped and would slip past every later check. + """ + module = _load() + exploit = ( + "polkit.addRule(function(action, subject) { " + "var a = /'/; " + 'if (action.id == "org.freedesktop.policykit.exec") ' + "{ return polkit.Result.YES; } " + "var b = /'/; });\n" + ) + with pytest.raises(module.HardeningError, match="regex or division"): + _rules(module, exploit) + + +@pytest.mark.parametrize( + "snippet", + [ + "var a = /'/;", + 'var a = /"/;', + "var a = /abc/;", + "var a = /[/'\"]/;", + "var a = /x\\/y/;", + "var a = subject.user / 2;", + "// a comment / with a slash then\nvar a = /'/;", + '/* block */ var a = /"/;', + "return polkit.Result.YES / 1;", + ], +) +def test_every_slash_form_outside_a_comment_fails_closed(snippet: str): + module = _load() + text = f"polkit.addRule(function(action, subject) {{ {snippet} }});\n" + with pytest.raises(module.HardeningError, match="regex or division"): + _rules(module, text) + + +def test_slashes_confined_to_strings_and_comments_stay_legal(): + module = _load() + # Slashes inside string literals and // or /* */ comments are inert. + _rules( + module, + "// path-like /usr/bin comment\n" + "/* another /etc/ comment */\n" + "polkit.addRule(function(action, subject) {\n" + ' if (action.id == "org.freedesktop.systemd1.manage-units" &&\n' + ' subject.user == "systemd-network") {\n' + " return polkit.Result.YES;\n" + " }\n" + "});\n", + ) + + +def test_string_and_comment_handling_keeps_literal_rules_auditable(): + module = _load() + code, strings = module._split_strings( + 'var a = "with // not a comment"; // trailing\n' + "/* block */ var b = 'quo\\'te\\n';\n" + ) + assert strings == ["with // not a comment", "quo'te\n"] + assert "//" not in code and "block" not in code + assert "\x000\x00" in code and "\x001\x00" in code + + +def test_grant_detection_survives_spacing_and_strings(): + module = _load() + with pytest.raises(module.HardeningError, match="explicit identity"): + _rules(module, "polkit.addRule(function() { return polkit.Result . YES; });\n") + # A grant token inside a string is inert and needs no identity scope. + _rules(module, 'var label = "polkit.Result.YES";\n') + + +@pytest.mark.parametrize( + ("text", "match"), + [ + ( + "[All]\nIdentity=unix-user:*\nAction=org.example\nResultAny=yes\n", + "exact literal", + ), + ( + "[N]\nIdentity=unix-netgroup:ops\nAction=org.example\nResultActive=yes\n", + "exact literal", + ), + ("[N]\nAction=org.example\nResultInactive=yes\n", "identity scope"), + ("[N]\nIdentity=\nAction=org.example\nResultAny=yes\n", "identity scope"), + ], +) +def test_pkla_grants_require_exact_literal_identities(text: str, match: str): + module = _load() + with pytest.raises(module.HardeningError, match=match): + module.audit_polkit_policy(Path("localauthority/50-local.d/a.pkla"), text) + + +def test_pkla_literal_grants_and_deny_entries_pass(): + module = _load() + module.audit_polkit_policy( + Path("localauthority/50-local.d/a.pkla"), + "[Mount]\nIdentity=unix-user:atlas;unix-group:operators\n" + "Action=org.freedesktop.udisks2.filesystem-mount\nResultAny=yes\n", + ) + module.audit_polkit_policy( + Path("etc/b.pkla"), + "[Deny]\nIdentity=unix-user:*\nAction=org.example\nResultAny=no\n", + ) + + +def test_integration_rejects_previously_invisible_grants(tmp_path: Path): + spec = importlib.util.spec_from_file_location( + "node_account_audit_polkit_test", SCRIPTS / "node_account_audit.py" + ) + assert spec and spec.loader + audit = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = audit + spec.loader.exec_module(audit) + + host_etc = tmp_path / "etc" + rules = host_etc / "polkit-1/rules.d" + rules.mkdir(parents=True) + (rules / "90-open.rules").write_text( + "polkit.addRule(function() { return polkit.Result.YES; });\n", + encoding="utf-8", + ) + with pytest.raises(audit.HardeningError, match="explicit identity"): + audit.audit_privilege_policies( + "hermes-agent", 1200, os.getuid(), host_etc, tmp_path / "missing" + ) + + (rules / "90-open.rules").unlink() + local = host_etc / "polkit-1/localauthority/50-local.d" + local.mkdir(parents=True) + (local / "wide.pkla").write_text( + "[W]\nIdentity=unix-user:*\nAction=org.example.benign\nResultAny=yes\n", + encoding="utf-8", + ) + with pytest.raises(audit.HardeningError, match="exact literal"): + audit.audit_privilege_policies( + "hermes-agent", 1200, os.getuid(), host_etc, tmp_path / "missing" + ) \ No newline at end of file diff --git a/testing/tests/test_hermes_node_sudo_alias_audit.py b/testing/tests/test_hermes_node_sudo_alias_audit.py new file mode 100644 index 00000000..e62b82bf --- /dev/null +++ b/testing/tests/test_hermes_node_sudo_alias_audit.py @@ -0,0 +1,134 @@ +"""Sudo alias and computed-identity denial proofs for the Hermes account.""" + +from __future__ import annotations + +import importlib.util +import os +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).parents[2] +SCRIPTS = ROOT / "services/hermes/scripts" +sys.path.insert(0, str(SCRIPTS)) + + +def _load(): + spec = importlib.util.spec_from_file_location( + "node_account_audit_sudo_test", SCRIPTS / "node_account_audit.py" + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _audit(tmp_path: Path, sudoers: str): + module = _load() + host_etc = tmp_path / "etc" + (host_etc / "sudoers.d").mkdir(parents=True, exist_ok=True) + (host_etc / "sudoers.d/atlas").write_text(sudoers, encoding="utf-8") + module.audit_privilege_policies( + "hermes-agent", 1200, os.getuid(), host_etc, tmp_path / "missing-share" + ) + + +def test_benign_distribution_policy_passes(tmp_path: Path): + _audit( + tmp_path, + "Defaults secure_path=/usr/bin\n" + "root ALL=(ALL:ALL) ALL\n" + "%sudo ALL=(ALL:ALL) ALL\n" + "#999 ALL=(root) /usr/bin/uptime\n" + "alice, bob ALL=(root) NOPASSWD: /usr/bin/systemctl status\n", + ) + + +def test_alias_reaching_hermes_directly_is_denied(tmp_path: Path): + module = _load() + with pytest.raises(module.HardeningError, match="sudo authority"): + _audit( + tmp_path, + "User_Alias OPERATORS = atlas, hermes-agent\n" + "OPERATORS ALL=(ALL) ALL\n", + ) + + +def test_alias_reaching_hermes_by_numeric_uid_is_denied(tmp_path: Path): + module = _load() + with pytest.raises(module.HardeningError, match="sudo authority"): + _audit(tmp_path, "User_Alias OPERATORS = #1200\nOPERATORS ALL=(ALL) ALL\n") + + +def test_alias_expanding_to_all_is_denied(tmp_path: Path): + module = _load() + with pytest.raises(module.HardeningError, match="broad sudo authority"): + _audit(tmp_path, "User_Alias ADMINS = ALL\nADMINS ALL=(ALL) ALL\n") + + +def test_nested_alias_expanding_to_all_via_continuation_is_denied(tmp_path: Path): + module = _load() + with pytest.raises(module.HardeningError, match="broad sudo authority"): + _audit( + tmp_path, + "User_Alias INNER = atlas, \\\n ALL\n" + "User_Alias OUTER = INNER\n" + "OUTER ALL=(ALL) ALL\n", + ) + + +@pytest.mark.parametrize( + ("sudoers", "match"), + [ + ("+operators ALL=(ALL) ALL\n", "netgroup"), + ("%:S-1-5-32 ALL=(ALL) ALL\n", "netgroup"), + ("GHOSTS ALL=(ALL) ALL\n", "alias is undefined"), + ("User_Alias A = B\nUser_Alias B = A\nA ALL=(ALL) ALL\n", "cyclic"), + ("User_Alias broken\nroot ALL=(ALL) ALL\n", "not auditable"), + ("User_Alias lower = atlas\n", "not auditable"), + ("User_Alias A = atlas\nUser_Alias A = bob\nA ALL=(ALL) ALL\n", "not auditable"), + ("User_Alias A = atlas,,bob\nA ALL=(ALL) ALL\n", "not auditable"), + ("stray-line-without-equals\n", "not auditable"), + ("=orphan (ALL) ALL\n", "not auditable"), + ("! ALL=(ALL) ALL\n", "malformed"), + ("User_Alias A = atlas : B = ALL\nB ALL=(ALL) ALL\n", "broad sudo"), + ], +) +def test_unauditable_or_broad_sudo_policies_fail_closed( + tmp_path: Path, sudoers: str, match: str +): + module = _load() + with pytest.raises(module.HardeningError, match=match): + _audit(tmp_path, sudoers) + + +def test_negated_and_quoted_principals_are_expanded_before_judging(tmp_path: Path): + module = _load() + with pytest.raises(module.HardeningError, match="broad sudo authority"): + _audit(tmp_path, '!"ALL" ALL=(ALL) ALL\n') + _audit(tmp_path, "!alice ALL=(ALL) ALL\n") + + +def test_continuation_join_requires_a_complete_final_line(): + module = _load() + with pytest.raises(module.HardeningError, match="line continuation"): + module._joined_sudo_lines("root ALL=(ALL) ALL \\") + assert module._joined_sudo_lines("a \\\nb\nc") == ["a b", "c"] + + +def test_alias_definitions_parse_multiple_groups_per_line(): + module = _load() + aliases = module._sudo_user_aliases( + ["User_Alias A = atlas, bob : B = carol", "Runas_Alias R = root"] + ) + assert aliases == {"A": ["atlas", "bob"], "B": ["carol"]} + + +def test_expansion_accepts_nested_literal_users(): + module = _load() + aliases = {"A": ["atlas", "B"], "B": ["carol"]} + module._expand_sudo_principal("A", aliases, frozenset()) + module._expand_sudo_principal("%wheel", aliases, frozenset()) + module._expand_sudo_principal("#999", aliases, frozenset()) \ No newline at end of file diff --git a/testing/tests/test_hermes_receive_pack_scan.py b/testing/tests/test_hermes_receive_pack_scan.py new file mode 100644 index 00000000..7d17fa6f --- /dev/null +++ b/testing/tests/test_hermes_receive_pack_scan.py @@ -0,0 +1,217 @@ +"""Quarantine content-scanning contracts for brokered Git pushes.""" + +from __future__ import annotations + +import hashlib +import io +import zlib + +import pytest + +from testing.tests.test_hermes_scm_broker_support import ( + _blob_pack, + _load, + _object_entry, + _pack_of, + _receive_command, +) + +ZERO = b"0" * 40 +COMMIT = b"1" * 40 +TOKEN = "runtime-sentinel" +FORMS = (b"runtime-sentinel",) + + +def _request(pack: bytes, ref: bytes = b"refs/heads/hermes/scan") -> bytes: + return _receive_command(ZERO, COMMIT, ref, pack) + + +def test_clean_push_with_real_content_passes(): + scan = _load("receive_pack_scan") + pack = _blob_pack( + b"# services/example/deployment.yaml\napiVersion: apps/v1\n", + b"tree 5b3a\nparent none\n", + ) + scan.validate_receive_pack(_request(pack), TOKEN, FORMS) + + +@pytest.mark.parametrize( + "payload", + [ + b"-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaA==\n", + b"-----BEGIN RSA PRIVATE KEY-----\nMIIE\n", + b"key = ghp_" + b"a1" * 12, + b"aws_access_key_id = AKIAABCDEFGHIJKLMNOP", + b"jwt eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxIn0.c2lnbmF0dXJlLXNlZ21lbnQ", + b"hook https://hooks.slack.com/services/T000/B000/tokentokentokentoken", + b"AGE-SECRET-KEY-1QQPCXR7QQPCXR7QQPCXR7QQPCXR7", + b"PuTTY-User-Key-File-3: ssh-ed25519", + b"Authorization: Bearer abc123-tokenvalue", + b"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF2c hermes@atlas", + b"gitea gta_" + b"z9" * 12, + ], +) +def test_credential_shaped_object_content_is_rejected(payload): + scan = _load("receive_pack_scan") + with pytest.raises(scan.PolicyError, match="credential-shaped"): + scan.validate_receive_pack(_request(_blob_pack(payload)), TOKEN, FORMS) + + +def test_runtime_token_hidden_by_compression_is_still_caught(): + scan = _load("receive_pack_scan") + pack = _blob_pack(b"config value: runtime-sentinel\n") + # The compressed wire bytes do not contain the token; only the + # decompressed payload does. + assert b"runtime-sentinel" not in pack + with pytest.raises(scan.PolicyError, match="runtime credential material"): + scan.validate_receive_pack(_request(pack), TOKEN, FORMS) + + +def _size_varint(value: int) -> bytes: + out = bytearray() + while True: + byte = value & 0x7F + value >>= 7 + if value: + out.append(byte | 0x80) + else: + out.append(byte) + return bytes(out) + + +def _typed_header(type_code: int, size: int) -> bytes: + byte = (type_code << 4) | (size & 0x0F) + size >>= 4 + header = bytearray() + while size: + header.append(byte | 0x80) + byte = size & 0x7F + size >>= 7 + header.append(byte) + return bytes(header) + + +def test_secret_assembled_only_by_delta_resolution_is_caught(): + scan = _load("receive_pack_scan") + base = b"docs: AKIAABCD" + completion = b"EFGHIJKLMNOP" + delta = ( + _size_varint(len(base)) + + _size_varint(len(base) + len(completion)) + + bytes([0x80 | 0x10, len(base)]) + + bytes([len(completion)]) + + completion + ) + base_sha = hashlib.sha1(b"blob %d\x00" % len(base) + base).digest() + delta_entry = _typed_header(7, len(delta)) + base_sha + zlib.compress(delta) + pack = _pack_of([_object_entry(3, base), delta_entry]) + # Neither the base nor the delta insert alone matches; the resolved + # payload is the only place the credential exists. + with pytest.raises(scan.PolicyError, match="runtime credential|credential-shaped"): + scan.validate_receive_pack(_request(pack), TOKEN, FORMS) + + +def test_thin_pack_deltas_are_rejected_outright(): + scan = _load("receive_pack_scan") + delta = _size_varint(4) + _size_varint(1) + bytes([1]) + b"x" + entry = _typed_header(7, len(delta)) + b"\xbb" * 20 + zlib.compress(delta) + with pytest.raises(scan.PolicyError, match="push full packs"): + scan.validate_receive_pack(_request(_pack_of([entry])), TOKEN, FORMS) + + +def test_trailing_bytes_after_the_pack_are_rejected(): + scan = _load("receive_pack_scan") + body = _request(_pack_of([])) + b"extra" + with pytest.raises(scan.PolicyError, match="bytes after its pack"): + scan.validate_receive_pack(body, TOKEN, FORMS) + + +@pytest.mark.parametrize( + ("line", "match"), + [ + (b"shallow " + COMMIT, "ref command is invalid"), + (COMMIT + b" " + COMMIT + b" refs/heads/hermes/x", "feature-branch creation"), + (ZERO + b" " + ZERO + b" refs/heads/hermes/x", "feature-branch creation"), + (ZERO + b" " + COMMIT + b" refs/heads/main", "namespaced feature branches"), + (ZERO + b" " + COMMIT + b" refs/tags/v1", "namespaced feature branches"), + (ZERO + b" " + COMMIT + b" refs/heads/hermes/\xff", "canonical ASCII"), + (b"zz" * 20 + b" " + COMMIT + b" refs/heads/hermes/x", "ref command is invalid"), + ], +) +def test_push_commands_outside_the_narrow_grant_are_rejected(line, match): + scan = _load("receive_pack_scan") + framed = f"{len(line) + 5:04x}".encode() + line + b"\n" + b"0000" + with pytest.raises(scan.PolicyError, match=match): + scan.validate_receive_pack(framed + _pack_of([]), TOKEN, FORMS) + + +def test_command_stream_bounds_are_enforced(): + scan = _load("receive_pack_scan") + with pytest.raises(scan.PolicyError, match="no ref command"): + scan.validate_receive_pack(b"0000" + _pack_of([]), TOKEN, FORMS) + with pytest.raises(scan.PolicyError, match="omitted its command terminator"): + scan.validate_receive_pack(b"", TOKEN, FORMS) + line = ZERO + b" " + COMMIT + b" refs/heads/hermes/x\n" + framed = f"{len(line) + 4:04x}".encode() + line + many = framed * (scan.MAX_PUSH_COMMANDS + 1) + b"0000" + _pack_of([]) + with pytest.raises(scan.PolicyError, match="too many ref commands"): + scan.validate_receive_pack(many, TOKEN, FORMS) + with pytest.raises(scan.PolicyError, match="framing is invalid"): + scan.validate_receive_pack(b"zzzz", TOKEN, FORMS) + with pytest.raises(scan.PolicyError, match="framing is invalid"): + scan.validate_receive_pack(b"0003", TOKEN, FORMS) + with pytest.raises(scan.PolicyError, match="framing is invalid"): + scan.validate_receive_pack(b"0009abc", TOKEN, FORMS) + + +def test_credential_forms_in_command_lines_are_rejected(): + scan = _load("receive_pack_scan") + line = ZERO + b" " + COMMIT + b" refs/heads/hermes/x runtime-sentinel\n" + framed = f"{len(line) + 4:04x}".encode() + line + b"0000" + with pytest.raises(scan.PolicyError, match="runtime credential material"): + scan.validate_receive_pack(framed + _pack_of([]), TOKEN, FORMS) + + +def test_scan_crosses_chunk_boundaries_with_carry(monkeypatch): + scan = _load("receive_pack_scan") + monkeypatch.setattr(scan, "SCAN_CHUNK", 8) + monkeypatch.setattr(scan, "SCAN_CARRY", 48) + payload = b"x" * 20 + b" AKIAABCDEFGHIJKLMNOP " + b"y" * 20 + with pytest.raises(scan.PolicyError, match="credential-shaped"): + scan._scan_payload(io.BytesIO(payload), FORMS) + scan._scan_payload(io.BytesIO(b"clean " * 40), FORMS) + scan._scan_payload(io.BytesIO(b""), FORMS) + + +def test_scan_loose_object_reports_type_and_scans_content(): + scan = _load("receive_pack_scan") + clean = zlib.compress(b"blob 5\x00clean") + assert scan.scan_loose_object(clean, FORMS) == 3 + secret = b"-----BEGIN EC PRIVATE KEY-----" + dirty = zlib.compress(b"blob %d\x00" % len(secret) + secret) + with pytest.raises(scan.PolicyError, match="credential-shaped"): + scan.scan_loose_object(dirty, FORMS) + leaked = b"the runtime-sentinel value" + with pytest.raises(scan.PolicyError, match="runtime credential material"): + scan.scan_loose_object( + zlib.compress(b"blob %d\x00" % len(leaked) + leaked), FORMS + ) + + +def test_content_patterns_extend_the_shared_standalone_corpus(): + scan = _load("receive_pack_scan") + policy = _load("gitea_api_policy") + assert len(scan.CONTENT_SECRET_PATTERNS) == len(policy.STANDALONE_SECRET_PATTERNS) + 3 + assert all( + isinstance(pattern.pattern, bytes) for pattern in scan.CONTENT_SECRET_PATTERNS + ) + + +def test_validation_rewinds_the_stream_for_upstream_forwarding(): + scan = _load("receive_pack_scan") + body = _request(_blob_pack(b"safe content")) + stream = io.BytesIO(body) + scan.validate_receive_pack(stream, TOKEN, FORMS) + assert stream.tell() == 0 + with pytest.raises(scan.PolicyError): + scan.validate_receive_pack(io.BytesIO(b"zzzz"), TOKEN, FORMS) \ No newline at end of file diff --git a/testing/tests/test_hermes_runtime_access.py b/testing/tests/test_hermes_runtime_access.py index fa4c8021..83bf11f2 100644 --- a/testing/tests/test_hermes_runtime_access.py +++ b/testing/tests/test_hermes_runtime_access.py @@ -11,14 +11,19 @@ from pathlib import Path import pytest import yaml - ROOT = Path(__file__).parents[2] HERMES = ROOT / "services" / "hermes" SCRIPTS = HERMES / "scripts" +SCM_SCRIPTS = HERMES / "scm-common" / "scripts" +if str(SCRIPTS) not in sys.path: + sys.path.insert(0, str(SCRIPTS)) +if str(SCM_SCRIPTS) not in sys.path: + sys.path.insert(0, str(SCM_SCRIPTS)) def _load(name: str): - spec = importlib.util.spec_from_file_location(name, SCRIPTS / f"{name}.py") + root = SCM_SCRIPTS if name == "gitea_api" else SCRIPTS + spec = importlib.util.spec_from_file_location(name, root / f"{name}.py") assert spec and spec.loader module = importlib.util.module_from_spec(spec) sys.modules[spec.name] = module @@ -31,19 +36,23 @@ def test_gitea_api_builds_runtime_authenticated_same_origin_requests(): request = gitea_api.build_request( "POST", - "/api/v1/repos/atlas/cassandra/issues/1/comments", + "/api/v1/repos/atlas/cassandra/pulls", base_url="https://scm.bstein.dev", token="runtime-only-value", - data={"body": "reviewed"}, + data={ + "base": "main", + "body": "Ready for review.", + "head": "hermes/repair", + "title": "WIP: Repair semantic review findings", + }, ) assert isinstance(request, urllib.request.Request) - assert request.full_url == ( - "https://scm.bstein.dev/api/v1/repos/atlas/cassandra/issues/1/comments" - ) + assert request.full_url == "https://scm.bstein.dev/api/v1/repos/atlas/cassandra/pulls" assert request.method == "POST" assert request.get_header("Authorization") == "token runtime-only-value" - assert json.loads(request.data) == {"body": "reviewed"} + assert json.loads(request.data)["title"].startswith("WIP: ") + assert "draft" not in json.loads(request.data) @pytest.mark.parametrize( @@ -61,25 +70,14 @@ def test_gitea_api_rejects_foreign_or_non_api_targets(path: str): gitea_api.api_url("https://scm.bstein.dev", path) -def test_gitea_api_parses_shell_safe_fields_as_json_values(): +@pytest.mark.parametrize("method", ["DELETE", "PUT", "OPTIONS"]) +def test_gitea_api_rejects_unavailable_methods(method: str): gitea_api = _load("gitea_api") - assert gitea_api.parse_fields( - ["Do=manually-merged", "MergeCommitID=abc123", "enabled=true", "count=2"] - ) == { - "Do": "manually-merged", - "MergeCommitID": "abc123", - "enabled": True, - "count": 2, - } - - -@pytest.mark.parametrize("value", ["missing-separator", "=missing-key", "bad key=x"]) -def test_gitea_api_rejects_invalid_shell_safe_fields(value: str): - gitea_api = _load("gitea_api") - - with pytest.raises(ValueError): - gitea_api.parse_fields([value]) + with pytest.raises(gitea_api.PolicyError): + gitea_api.authorize_request( + method, "/api/v1/repos/atlas/cassandra/pulls/1", None + ) def test_jenkins_evidence_ignores_nested_execution_result(): @@ -175,6 +173,9 @@ def test_agent_runtime_stage_keeps_credentials_in_memory(tmp_path: Path, monkeyp values = { "agent-api-key": "agent-key", "chat-relay-key": "relay-key", + # gitea-token/username are present in Vault but the agent staging must + # deliberately not copy them (the least-authority broker holds Gitea + # access); the assertions below prove they never reach the runtime. "gitea-token": "gitea-key", "gitea-username": "hermes-automation", "jenkins-image-build-token": "job-scoped-token", @@ -202,6 +203,10 @@ def test_agent_runtime_stage_keeps_credentials_in_memory(tmp_path: Path, monkeyp assert (runtime / "codex/skills").is_symlink() assert not (runtime / "claude/backups").exists() assert not (runtime / "codex/sessions").exists() + assert not (runtime / "gitea-token").exists() + assert not (runtime / "gitea-username").exists() + ssh_config = (runtime / "node-ssh-config").read_text(encoding="utf-8") + assert ssh_config.startswith("Host titan-*\n User hermes-agent\n") auth = json.loads((runtime / "hermes-auth.json").read_text(encoding="utf-8")) assert auth == {"version": 1, "providers": {}, "credential_pool": {}} @@ -365,12 +370,12 @@ def test_manifests_never_seed_access_material_into_persistent_env(): assert triage_annotations[ "vault.hashicorp.com/agent-inject-secret-triage-api-key" ] == "kv/data/atlas/hermes/triage-api" - askpass = (SCRIPTS / "gitea_askpass.sh").read_text(encoding="utf-8") - assert "/runtime-access/gitea-token" in askpass - assert "GITEA_TOKEN" not in askpass - gitea_api = (SCRIPTS / "gitea_api.py").read_text(encoding="utf-8") - assert "/runtime-access/gitea-token" in gitea_api + gitea_api = ( + ROOT / "services/hermes/scm-common/scripts/gitea_api.py" + ).read_text(encoding="utf-8") + assert "scm_broker_client" in gitea_api assert "GITEA_TOKEN" not in gitea_api + assert "gitea-token" not in annotations def test_chat_media_reads_one_raw_runtime_secret(tmp_path: Path): diff --git a/testing/tests/test_hermes_runtime_stage_coverage.py b/testing/tests/test_hermes_runtime_stage_coverage.py new file mode 100644 index 00000000..004853c2 --- /dev/null +++ b/testing/tests/test_hermes_runtime_stage_coverage.py @@ -0,0 +1,137 @@ +"""Behavioral branch coverage for Vault-to-memory runtime access staging.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import pytest + +from testing.tests.test_hermes_runtime_access import _load + + +def _stage(tmp_path: Path, monkeypatch): + module = _load("stage_runtime_access") + vault = tmp_path / "vault" + runtime = tmp_path / "runtime" + home = tmp_path / "home" + vault.mkdir() + home.mkdir() + monkeypatch.setattr(module, "VAULT_ROOT", vault) + monkeypatch.setattr(module, "RUNTIME_ROOT", runtime) + monkeypatch.setattr(module, "PERSISTENT_HOME", home) + monkeypatch.setattr(module.os, "chown", lambda *_args: None) + return module, vault, runtime, home + + +def test_private_directory_and_secret_copy_validate_mode_and_content( + tmp_path, monkeypatch +): + module, vault, runtime, _home = _stage(tmp_path, monkeypatch) + module._owned_directory(runtime) + assert runtime.stat().st_mode & 0o777 == 0o700 + (vault / "empty").write_text(" \n", encoding="utf-8") + with pytest.raises(RuntimeError, match="is empty"): + module._copy_secret("empty", runtime / "empty") + (vault / "value").write_text(" staged \n", encoding="utf-8") + assert module._copy_secret("value", runtime / "value") == "staged" + assert (runtime / "value").read_text(encoding="utf-8") == "staged\n" + + +@pytest.mark.parametrize( + ("document", "required", "match"), + [ + ({"outer": {}}, ("outer", "token"), "invalid shape"), + ({"outer": {"token": ""}}, ("outer", "token"), "empty credential"), + ({"outer": {"token": 7}}, ("outer", "token"), "empty credential"), + ], +) +def test_json_staging_rejects_missing_nontext_and_empty_credentials( + tmp_path, monkeypatch, document, required, match +): + module, vault, runtime, _home = _stage(tmp_path, monkeypatch) + runtime.mkdir() + (vault / "credential").write_text(json.dumps(document), encoding="utf-8") + destination = runtime / "credential.json" + with pytest.raises(RuntimeError, match=match): + module._validated_json("credential", destination, required) + assert not destination.exists() + + +def test_noncredential_linker_skips_absent_symlink_and_existing_entries( + tmp_path, monkeypatch +): + module, _vault, runtime, home = _stage(tmp_path, monkeypatch) + runtime.mkdir() + source = home / "state" + destination = runtime / "state" + destination.mkdir() + module._link_noncredential_state(source, destination, ("allowed",)) + + source.mkdir() + (source / "allowed").write_text("safe", encoding="utf-8") + (source / "existing").write_text("new", encoding="utf-8") + (source / "target-link").write_text("new", encoding="utf-8") + (source / "linked").symlink_to(source / "allowed") + (destination / "existing").write_text("preserve", encoding="utf-8") + (destination / "target-link").symlink_to(destination / "existing") + module._link_noncredential_state( + source, + destination, + ("missing", "linked", "existing", "target-link", "allowed"), + ) + assert (destination / "allowed").is_symlink() + assert (destination / "existing").read_text(encoding="utf-8") == "preserve" + + +def test_empty_auth_store_is_private_and_noncredential(tmp_path, monkeypatch): + module, _vault, runtime, _home = _stage(tmp_path, monkeypatch) + runtime.mkdir() + module._write_empty_auth_store() + path = runtime / "hermes-auth.json" + assert json.loads(path.read_text(encoding="utf-8")) == { + "version": 1, + "providers": {}, + "credential_pool": {}, + } + assert path.stat().st_mode & 0o777 == 0o600 + + +def test_node_ssh_config_rejects_nul_and_removes_partial_file(tmp_path, monkeypatch): + module, vault, runtime, _home = _stage(tmp_path, monkeypatch) + runtime.mkdir() + (vault / "node-ssh-config").write_bytes(b"Host titan-01\x00\n") + with pytest.raises(RuntimeError, match="control data"): + module._stage_node_ssh_config() + assert not (runtime / "node-ssh-config").exists() + + +@pytest.mark.parametrize( + ("mode", "secret_name"), + [("chat", "chat-relay-key"), ("triage", "triage-api-key")], +) +def test_isolated_tenant_staging_exposes_only_required_secret( + tmp_path, monkeypatch, mode, secret_name +): + module, vault, runtime, _home = _stage(tmp_path, monkeypatch) + (vault / secret_name).write_text("synthetic-value\n", encoding="utf-8") + getattr(module, f"stage_{mode}")() + assert (runtime / secret_name).read_text(encoding="utf-8") == "synthetic-value\n" + assert (runtime / "hermes-auth.json").exists() + assert sorted(path.name for path in runtime.iterdir()) == sorted( + ["hermes-auth.json", secret_name] + ) + + +@pytest.mark.parametrize("mode", ["agent", "chat", "triage"]) +def test_main_dispatches_each_mode(mode, monkeypatch, capsys): + module = _load("stage_runtime_access") + calls = [] + monkeypatch.setattr(module, "stage_agent", lambda: calls.append("agent")) + monkeypatch.setattr(module, "stage_chat", lambda: calls.append("chat")) + monkeypatch.setattr(module, "stage_triage", lambda: calls.append("triage")) + monkeypatch.setattr(sys, "argv", ["stage-runtime-access", mode]) + assert module.main() == 0 + assert calls == [mode] + assert "staged from Vault" in capsys.readouterr().out diff --git a/testing/tests/test_hermes_scm_broker.py b/testing/tests/test_hermes_scm_broker.py new file mode 100644 index 00000000..2a0fad44 --- /dev/null +++ b/testing/tests/test_hermes_scm_broker.py @@ -0,0 +1,146 @@ +"""Atlas route and Git protocol contracts for the Hermes SCM broker.""" + +from __future__ import annotations + +import pytest + +from testing.tests.test_hermes_scm_broker_support import ( + Response, + _load, + _receive_command, +) + +@pytest.mark.parametrize( + "target", + [ + "/api/v1/repos/atlas/cassandra/pulls/1/merge", + "/api/v1/repos/atlas/cassandra/pulls/1/reviews", + "/api/v1/admin/users", + "/git/evil/cassandra.git/info/refs?service=git-upload-pack", + "/git/atlas/cassandra.git/hooks", + "/git/atlas/cassandra.git/git-receive-pack?force=true", + "/git/atlas/cassandra.git/info/refs?service=git-upload-pack\nignored", + "/git/atlas/cassandra.git/info/refs?service=git-upload-pack\tignored", + "/git/atlas/cassandra.git/info/refs?service=git-upload-pack#fragment", + ], +) +def test_broker_rejects_privileged_or_non_atlas_routes(target: str): + broker = _load("scm_broker") + + with pytest.raises(broker.PolicyError): + broker._git_target(target) + + +def test_broker_metadata_boundary_reuses_explicit_read_allowlist(): + api = _load("gitea_api") + + assert ( + api.authorize_request("GET", "/api/v1/repos/atlas/cassandra/pulls/1", None) + == "pull" + ) + for path in ( + "/api/v1/repos/atlas/cassandra/pulls/1/merge", + "/api/v1/repos/atlas/cassandra/pulls/1/reviews", + "/api/v1/repos/atlas/cassandra/hooks", + "/api/v1/admin/users", + ): + with pytest.raises(api.PolicyError): + api.authorize_request("GET", path, None) + + +def test_receive_pack_allows_only_new_namespaced_feature_branch(): + broker = _load("scm_broker") + zero = b"0" * 40 + commit = b"1" * 40 + + broker._validate_receive_pack( + _receive_command(zero, commit, b"refs/heads/hermes/focused-fix"), + "runtime-sentinel", + ) + for old, new, ref in ( + (commit, b"2" * 40, b"refs/heads/hermes/focused-fix"), + (commit, zero, b"refs/heads/hermes/focused-fix"), + (zero, commit, b"refs/heads/main"), + (zero, commit, b"refs/heads/master"), + (zero, commit, b"refs/tags/release"), + ): + with pytest.raises(broker.PolicyError): + broker._validate_receive_pack( + _receive_command(old, new, ref), "runtime-sentinel" + ) + + +def test_git_proxy_uses_fixed_origin_and_never_reflects_credential(): + broker = _load("scm_broker") + seen = [] + + def opener(request, timeout): + seen.append((request, timeout)) + return Response( + b"git-result", + content_type="application/x-git-upload-pack-result", + ) + + result = broker._upstream_git_request( + "/atlas/cassandra.git/git-upload-pack", + method="POST", + body=b"request", + content_type="application/x-git-upload-pack-request", + expected_type="application/x-git-upload-pack-result", + token="runtime-sentinel", + opener=opener, + ) + + assert result == b"git-result" + request = seen[0][0] + assert request.full_url == ( + "https://scm.bstein.dev/atlas/cassandra.git/git-upload-pack" + ) + assert b"runtime-sentinel" not in request.data + assert "runtime-sentinel" not in request.full_url + + with pytest.raises(broker.PolicyError, match="credential material"): + broker._upstream_git_request( + "/atlas/cassandra.git/git-upload-pack", + method="POST", + body=b"request", + content_type="application/x-git-upload-pack-request", + expected_type="application/x-git-upload-pack-result", + token="runtime-sentinel", + opener=lambda *_a, **_k: Response( + b"runtime-sentinel", + content_type="application/x-git-upload-pack-result", + ), + ) + encoded = broker._credential_forms("runtime-sentinel")[1] + with pytest.raises(broker.PolicyError, match="credential material"): + broker._upstream_git_request( + "/atlas/cassandra.git/git-upload-pack", + method="POST", + body=b"request", + content_type="application/x-git-upload-pack-request", + expected_type="application/x-git-upload-pack-result", + token="runtime-sentinel", + opener=lambda *_a, **_k: Response( + b"Authorization: Basic " + encoded, + content_type="application/x-git-upload-pack-result", + ), + ) +@pytest.mark.parametrize("status", [201, 202, 204, 206, 301, 302, 307, 308]) +def test_git_proxy_requires_exact_http_200(status: int): + broker = _load("scm_broker") + + with pytest.raises(broker.PolicyError, match="unexpected HTTP status"): + broker._upstream_git_request( + "/atlas/cassandra.git/git-upload-pack", + method="POST", + body=b"request", + content_type="application/x-git-upload-pack-request", + expected_type="application/x-git-upload-pack-result", + token="runtime-sentinel", + opener=lambda *_a, **_k: Response( + b"result", + status=status, + content_type="application/x-git-upload-pack-result", + ), + ) diff --git a/testing/tests/test_hermes_scm_broker_client_coverage.py b/testing/tests/test_hermes_scm_broker_client_coverage.py new file mode 100644 index 00000000..9fa65fa6 --- /dev/null +++ b/testing/tests/test_hermes_scm_broker_client_coverage.py @@ -0,0 +1,160 @@ +"""Behavioral coverage for the credential-free SCM broker client and I/O.""" + +from __future__ import annotations + +import io +import json +import pytest + +from testing.tests.test_hermes_scm_broker_support import Response, _load + + +def test_client_default_opener_uses_deadline_bounded_exchange(monkeypatch): + client = _load("scm_broker_client") + sentinel = object() + monkeypatch.setattr( + client.deadline_http, + "open_bounded", + lambda request, *, maximum, timeout: (request, maximum, timeout, sentinel), + ) + assert client._open("request", 4) == ( + "request", + client.MAX_RESPONSE_BYTES, + 4, + sentinel, + ) + + +@pytest.mark.parametrize( + ("endpoint", "payload", "match"), + [ + ("/v1/admin", {}, "outside"), + ("/v1/metadata", {"path": "x" * (64 * 1024)}, "safe size"), + ], +) +def test_client_rejects_unknown_or_oversized_operations(endpoint, payload, match): + client = _load("scm_broker_client") + with pytest.raises(client.PolicyError, match=match): + client.request(endpoint, payload, opener=lambda *_a, **_k: None) + + +class _GetCodeResponse(Response): + def __init__(self, body: bytes, *, content_type="application/json", code=200): + super().__init__(body, content_type=content_type) + del self.status + self._code = code + + def getcode(self): + return self._code + + +def test_client_accepts_getcode_fallback_and_create_payload(): + client = _load("scm_broker_client") + seen = [] + + def opener(request, timeout): + seen.append((request, timeout)) + return _GetCodeResponse(b"{}") + + result = client.create_draft( + "cassandra", + base="main", + head="feature/coverage", + head_sha="a" * 40, + title="WIP: Coverage", + body="Review evidence", + opener=opener, + ) + assert result == b"{}" + assert json.loads(seen[0][0].data)["repo"] == "cassandra" + + +@pytest.mark.parametrize( + ("response", "match"), + [ + (_GetCodeResponse(b"{}", code=201), "HTTP status"), + (_GetCodeResponse(b"{}", content_type="text/plain"), "response type"), + ( + Response(b"x" * (2 * 1024 * 1024 + 1), content_type="application/json"), + "safe size", + ), + (Response(b"not-json", content_type="application/json"), "Expecting value"), + ], +) +def test_client_rejects_bad_status_type_size_and_json(response, match): + client = _load("scm_broker_client") + expected = ( + json.JSONDecodeError if match == "Expecting value" else client.PolicyError + ) + with pytest.raises(expected, match=match): + client.request( + "/v1/metadata", + {"path": "/api/v1/repos/atlas/cassandra"}, + opener=lambda *_a, **_k: response, + ) + + +class _Socket: + def __init__(self): + self.timeouts = [] + + def settimeout(self, value): + self.timeouts.append(value) + + +class _Stream: + def __init__(self, body: bytes, socket_shape: str = "direct"): + self.body = io.BytesIO(body) + self.socket = _Socket() + if socket_shape == "nested": + self.fp = type( + "FP", (), {"raw": type("Raw", (), {"_sock": self.socket})()} + )() + elif socket_shape == "direct": + self.fp = type("FP", (), {"raw": self.socket})() + else: + self.fp = None + + def read(self, size): + return self.body.read(size) + + +def test_response_spool_handles_direct_and_absent_socket_shapes(): + module = _load("scm_broker_io") + for shape in ("direct", "absent"): + stream = _Stream(b"safe", shape) + spool, length = module.spool_response( + stream, + 16, + (b"forbidden",), + memory_limit=8, + chunk_size=2, + deadline_seconds=5, + ) + try: + assert length == 4 and spool.read() == b"safe" + finally: + spool.close() + + +@pytest.mark.parametrize("failure", ["deadline", "size", "credential", "read"]) +def test_response_spool_closes_and_rejects_each_failure(monkeypatch, failure): + module = _load("scm_broker_io") + stream = _Stream(b"safe-forbidden-value") + kwargs = { + "maximum": 128, + "forbidden": (b"forbidden",), + "memory_limit": 8, + "chunk_size": 5, + "deadline_seconds": 5, + } + if failure == "deadline": + ticks = iter((10.0, 16.0)) + monkeypatch.setattr(module.time, "monotonic", lambda: next(ticks)) + elif failure == "size": + kwargs["maximum"] = 2 + elif failure == "read": + stream.read = lambda _size: (_ for _ in ()).throw(OSError("read failed")) + expected = module.PolicyError if failure != "read" else OSError + with pytest.raises(expected): + module.spool_response(stream, **kwargs) diff --git a/testing/tests/test_hermes_scm_broker_handler_coverage.py b/testing/tests/test_hermes_scm_broker_handler_coverage.py new file mode 100644 index 00000000..57245a6a --- /dev/null +++ b/testing/tests/test_hermes_scm_broker_handler_coverage.py @@ -0,0 +1,265 @@ +"""Behavioral branch coverage for SCM broker request handling.""" + +from __future__ import annotations + +import io +import json +import sys +from email.message import Message + +import pytest + +from testing.tests.test_hermes_scm_broker_support import _load, _receive_command + + +class _Connection: + def __init__(self): + self.timeouts = [] + + def settimeout(self, value): + self.timeouts.append(value) + + +def _headers(*, content_type="application/json", body=b""): + value = Message() + value["Content-Type"] = content_type + value["Content-Length"] = str(len(body)) + return value + + +def _handler(broker, *, path="/healthz", body=b"", content_type="application/json"): + handler = object.__new__(broker.BrokerHandler) + handler.path = path + handler.headers = _headers(content_type=content_type, body=body) + handler.rfile = io.BytesIO(body) + handler.wfile = io.BytesIO() + handler.connection = _Connection() + handler.events = [] + handler.send_response = lambda status: handler.events.append(("status", status)) + handler.send_header = lambda name, value: handler.events.append((name, value)) + handler.end_headers = lambda: handler.events.append(("headers", "done")) + return handler + + +def test_header_validation_accepts_safe_ascii_and_rejects_controls(): + broker = _load("scm_broker") + handler = _handler(broker) + handler._validate_headers() + + class HeaderBag: + def items(self): + return [("X-Test", "bad\x01value")] + + def get_all(self, _name, default): + return default + + handler.headers = HeaderBag() + with pytest.raises(broker.PolicyError, match="controls"): + handler._validate_headers() + + class UnicodeHeader(HeaderBag): + def items(self): + return [("X-Test", "máin")] + + handler.headers = UnicodeHeader() + with pytest.raises(broker.PolicyError, match="ASCII"): + handler._validate_headers() + + +def test_json_reject_and_stream_response_methods(monkeypatch): + broker = _load("scm_broker") + handler = _handler(broker) + handler._json(202, b'{"ok":true}') + assert ("status", 202) in handler.events + assert handler.wfile.getvalue() == b'{"ok":true}' + + handler = _handler(broker) + handler._reject(403) + assert ("status", 403) in handler.events + assert json.loads(handler.wfile.getvalue()) == {"error": "request rejected"} + + handler = _handler(broker) + handler._stream(200, "application/test", io.BytesIO(b"streamed"), 8) + assert handler.wfile.getvalue() == b"streamed" + assert handler.connection.timeouts + handler.log_message("ignored %s", "value") + + handler = _handler(broker) + ticks = iter((0.0, 121.0)) + monkeypatch.setattr(broker.time, "monotonic", lambda: next(ticks)) + with pytest.raises(broker.PolicyError, match="write deadline"): + handler._stream(200, "application/test", io.BytesIO(b"x"), 1) + + +def test_get_health_and_git_discovery_paths(monkeypatch): + broker = _load("scm_broker") + handler = _handler(broker) + handler.do_GET() + assert json.loads(handler.wfile.getvalue()) == {"status": "ok"} + + closed = [] + + class Body(io.BytesIO): + def close(self): + closed.append(True) + super().close() + + handler = _handler( + broker, + path="/git/atlas/cassandra.git/info/refs?service=git-upload-pack", + ) + monkeypatch.setattr(broker, "read_token", lambda: "sentinel") + monkeypatch.setattr( + broker, + "_upstream_git_request", + lambda *args, **kwargs: (Body(b"advertisement"), 13), + ) + handler.do_GET() + assert handler.wfile.getvalue() == b"advertisement" + assert closed == [True] + + handler = _handler(broker, path="/git/atlas/cassandra.git/git-upload-pack") + handler.do_GET() + assert json.loads(handler.wfile.getvalue()) == {"error": "request rejected"} + + +def test_post_dispatches_control_and_git_or_rejects(monkeypatch): + broker = _load("scm_broker") + control = _handler(broker, path="/v1/metadata", body=b"{}") + git = _handler(broker, path="/git/atlas/cassandra.git/git-upload-pack", body=b"") + calls = [] + control._control = lambda: calls.append("control") + control._git_rpc = lambda: calls.append("git") + git._control = lambda: calls.append("control") + git._git_rpc = lambda: calls.append("git") + control.do_POST() + git.do_POST() + assert calls == ["control", "git"] + + rejected = _handler(broker, path="/v1/metadata", body=b"{}") + rejected._control = lambda: (_ for _ in ()).throw(broker.PolicyError("no")) + rejected.do_POST() + assert json.loads(rejected.wfile.getvalue()) == {"error": "request rejected"} + + +def test_control_metadata_and_draft_fields(monkeypatch): + broker = _load("scm_broker") + monkeypatch.setattr(broker, "read_token", lambda: "sentinel") + monkeypatch.setattr( + broker, "read", lambda path, token: json.dumps({"path": path}).encode() + ) + monkeypatch.setattr( + broker, + "create_draft", + lambda token, **data: json.dumps({"token_used": bool(token), **data}).encode(), + ) + + metadata_body = b'{"path":"/api/v1/repos/atlas/cassandra"}' + metadata = _handler(broker, path="/v1/metadata", body=metadata_body) + metadata._control() + assert json.loads(metadata.wfile.getvalue())["path"].endswith("cassandra") + + draft_data = { + "base": "main", + "body": "Review evidence", + "head": "feature/coverage", + "head_sha": "a" * 40, + "repo": "cassandra", + "title": "WIP: Coverage", + } + draft_body = json.dumps(draft_data).encode() + draft = _handler(broker, path="/v1/drafts", body=draft_body) + draft._control() + assert json.loads(draft.wfile.getvalue())["repo"] == "cassandra" + + for path, body, match in ( + ("/v1/metadata", b'{"wrong":"field"}', "fields"), + ("/v1/drafts", b'{"repo":1}', "fields"), + ): + with pytest.raises(broker.PolicyError, match=match): + _handler(broker, path=path, body=body)._control() + + monkeypatch.setattr(broker, "read", lambda *_a, **_k: b"sentinel") + with pytest.raises(broker.PolicyError, match="reflected"): + _handler(broker, path="/v1/metadata", body=metadata_body)._control() + + +@pytest.mark.parametrize("service", ["git-upload-pack", "git-receive-pack"]) +def test_git_rpc_streams_allowed_upload_and_receive(monkeypatch, service): + broker = _load("scm_broker") + if service == "git-receive-pack": + body = _receive_command(b"0" * 40, b"1" * 40, b"refs/heads/hermes/coverage") + else: + body = b"upload-request" + handler = _handler( + broker, + path=f"/git/atlas/cassandra.git/{service}", + body=body, + content_type=f"application/x-{service}-request", + ) + monkeypatch.setattr(broker, "read_token", lambda: "sentinel") + seen = [] + + def upstream(target, **kwargs): + seen.append((target, kwargs)) + return io.BytesIO(b"result"), 6 + + monkeypatch.setattr(broker, "_upstream_git_request", upstream) + handler._git_rpc() + assert handler.wfile.getvalue() == b"result" + assert seen[0][0].endswith(service) + assert seen[0][1]["body_length"] == len(body) + + +@pytest.mark.parametrize( + ("path", "content_type", "transfer", "match"), + [ + ( + "/git/atlas/cassandra.git/info/refs?service=git-upload-pack", + "application/x-git-upload-pack-request", + None, + "operation", + ), + ( + "/git/atlas/cassandra.git/git-upload-pack", + "text/plain", + None, + "request type", + ), + ( + "/git/atlas/cassandra.git/git-upload-pack", + "application/x-git-upload-pack-request", + "chunked", + "request type", + ), + ], +) +def test_git_rpc_rejects_discovery_wrong_type_and_chunking( + path, content_type, transfer, match +): + broker = _load("scm_broker") + handler = _handler(broker, path=path, content_type=content_type) + if transfer: + handler.headers["Transfer-Encoding"] = transfer + with pytest.raises(broker.PolicyError, match=match): + handler._git_rpc() + + +def test_broker_main_constructs_bounded_server(monkeypatch): + broker = _load("scm_broker") + seen = [] + + class Server: + def __init__(self, address, handler): + seen.append((address, handler)) + + def serve_forever(self): + seen.append("served") + + monkeypatch.setattr(broker, "BoundedThreadingHTTPServer", Server) + monkeypatch.setattr( + sys, "argv", ["broker", "--listen", "127.0.0.1", "--port", "9191"] + ) + assert broker.main() == 0 + assert seen[0] == (("127.0.0.1", 9191), broker.BrokerHandler) + assert seen[1] == "served" diff --git a/testing/tests/test_hermes_scm_broker_internal_coverage.py b/testing/tests/test_hermes_scm_broker_internal_coverage.py new file mode 100644 index 00000000..b38702ac --- /dev/null +++ b/testing/tests/test_hermes_scm_broker_internal_coverage.py @@ -0,0 +1,272 @@ +"""Behavioral branch coverage for SCM broker protocol helpers.""" + +from __future__ import annotations + +import io +from email.message import Message + +import pytest + +from testing.tests.test_hermes_scm_broker_support import ( + Response, + _load, + _receive_command, +) + + +def test_broker_redirect_status_and_bounded_read_helpers(monkeypatch): + broker = _load("scm_broker") + with pytest.raises(broker.PolicyError, match="redirects"): + broker.RejectRedirect().redirect_request( + object(), None, 302, "redirect", {}, "https://elsewhere.invalid" + ) + + class CodeOnly: + def getcode(self): + return 204 + + assert broker._status(type("Status", (), {"status": 200})()) == 200 + assert broker._status(CodeOnly()) == 204 + assert broker._status(object()) is None + + with pytest.raises(broker.PolicyError, match="safe size"): + broker._read_bounded(io.BytesIO(b"x"), 4, -1) + with pytest.raises(broker.PolicyError, match="safe size"): + broker._read_bounded(io.BytesIO(b"xxxxx"), 4) + with pytest.raises(broker.PolicyError, match="safe size"): + broker._read_bounded(io.BytesIO(b"x"), 4, 2) + with pytest.raises(broker.PolicyError, match="deadline"): + broker._read_bounded(io.BytesIO(b"x"), 4, 1, deadline=0) + + timeouts = [] + assert ( + broker._read_bounded( + io.BytesIO(b"xy"), + 4, + 2, + deadline=broker.time.monotonic() + 5, + set_timeout=timeouts.append, + ) + == b"xy" + ) + assert timeouts + + +def test_spool_bounded_rejects_bounds_early_eof_and_closes_on_read_error(): + broker = _load("scm_broker") + for length in (-1, 5): + with pytest.raises(broker.PolicyError, match="safe size"): + broker._spool_bounded( + io.BytesIO(b"x"), + 4, + length, + token="sentinel", + context="request", + deadline=999999999, + ) + with pytest.raises(broker.PolicyError, match="ended early"): + broker._spool_bounded( + io.BytesIO(b"x"), + 4, + 2, + token="sentinel", + context="request", + deadline=999999999, + ) + + class Broken: + def read(self, _size): + raise OSError("broken stream") + + with pytest.raises(OSError, match="broken"): + broker._spool_bounded( + Broken(), 4, 1, token="sentinel", context="request", deadline=999999999 + ) + + +class _HandlerInput: + def __init__(self, body: bytes, *, content_type="application/json", length=None): + self.headers = Message() + self.headers["Content-Type"] = content_type + if length is None: + length = len(body) + self.headers["Content-Length"] = str(length) + self.rfile = io.BytesIO(body) + self.connection = type( + "Connection", (), {"settimeout": lambda self, value: None} + )() + + +def test_control_json_loader_requires_fixed_json_object(): + broker = _load("scm_broker") + handler = _HandlerInput(b"{}") + handler.headers["Transfer-Encoding"] = "chunked" + with pytest.raises(broker.PolicyError, match="chunked"): + broker._load_json(handler) + with pytest.raises(broker.PolicyError, match="must be JSON"): + broker._load_json(_HandlerInput(b"{}", content_type="text/plain")) + with pytest.raises(broker.PolicyError, match="must be an object"): + broker._load_json(_HandlerInput(b"[]")) + assert broker._load_json(_HandlerInput(b'{"path":"safe"}')) == {"path": "safe"} + + +@pytest.mark.parametrize( + ("target", "expected"), + [ + ( + "/git/atlas/cassandra.git/info/refs?service=git-upload-pack", + ("cassandra", "info/refs", "git-upload-pack"), + ), + ( + "/git/atlas/cassandra.git/info/refs?service=git-receive-pack", + ("cassandra", "info/refs", "git-receive-pack"), + ), + ( + "/git/atlas/cassandra.git/git-upload-pack", + ("cassandra", "git-upload-pack", "git-upload-pack"), + ), + ( + "/git/atlas/cassandra.git/git-receive-pack", + ("cassandra", "git-receive-pack", "git-receive-pack"), + ), + ], +) +def test_git_target_accepts_each_exact_smart_http_route(target, expected): + broker = _load("scm_broker") + assert broker._git_target(target) == expected + + +def test_content_length_and_credential_helpers_cover_safe_and_rejected_values(): + broker = _load("scm_broker") + headers = Message() + headers["Content-Length"] = "0" + assert broker._content_length(headers, 1) == 0 + forms = broker._credential_forms("sentinel") + assert len(forms) == 3 and forms[0] == b"sentinel" + zero = b"0" * 40 + commit = b"1" * 40 + line = zero + b" " + commit + b" refs/heads/hermes/x " + forms[2] + b"\n" + framed = f"{len(line) + 4:04x}".encode() + line + b"0000" + with pytest.raises(broker.PolicyError, match="credential material"): + broker._validate_receive_pack(framed, "sentinel") + + +def test_receive_pack_validation_rewinds_stream_and_accepts_bytes(): + broker = _load("scm_broker") + body = _receive_command(b"0" * 40, b"1" * 40, b"refs/heads/hermes/rewind") + stream = io.BytesIO(body) + stream.seek(9) + broker._validate_receive_pack(stream, "sentinel") + assert stream.tell() == 0 + broker._validate_receive_pack(body, "sentinel") + + +@pytest.mark.parametrize( + ("body", "match"), + [ + (b"bad!", "framing"), + (b"0000", "no ref command"), + (b"0003", "framing"), + (b"0008abc", "framing"), + (b"0008a b\n0000", "ref command"), + ], +) +def test_receive_pack_rejects_bad_framing_and_commands(body, match): + broker = _load("scm_broker") + with pytest.raises(broker.PolicyError, match=match): + broker._validate_receive_pack(body, "sentinel") + + +def _packet(old: bytes, new: bytes, ref: bytes, *, terminator=True) -> bytes: + command = old + b" " + new + b" " + ref + b"\n" + framed = f"{len(command) + 4:04x}".encode() + command + return framed + (b"0000" if terminator else b"") + + +def test_receive_pack_rejects_nonascii_many_commands_and_missing_terminator( + monkeypatch, +): + broker = _load("scm_broker") + zero = b"0" * 40 + commit = b"1" * 40 + with pytest.raises(broker.PolicyError, match="canonical ASCII"): + broker._validate_receive_pack( + _packet(zero, commit, b"refs/heads/hermes/\xff"), "sentinel" + ) + command = _packet(zero, commit, b"refs/heads/hermes/fix", terminator=False) + with pytest.raises(broker.PolicyError, match="terminator"): + broker._validate_receive_pack(command, "sentinel") + limit = _load("receive_pack_scan").MAX_PUSH_COMMANDS + many = command * (limit + 1) + b"0000" + with pytest.raises(broker.PolicyError, match="too many"): + broker._validate_receive_pack(many, "sentinel") + + +def test_upstream_request_infers_byte_length_and_requires_stream_length(): + broker = _load("scm_broker") + seen = [] + + def opener(request, timeout): + seen.append((request, timeout)) + return Response(b"result", content_type="application/x-git-upload-pack-result") + + assert ( + broker._upstream_git_request( + "/atlas/cassandra.git/git-upload-pack", + method="POST", + body=b"request", + content_type=None, + expected_type="application/x-git-upload-pack-result", + token="sentinel", + opener=opener, + ) + == b"result" + ) + assert seen[0][0].get_header("Content-length") == "7" + with pytest.raises(broker.PolicyError, match="length is missing"): + broker._upstream_git_request( + "/atlas/cassandra.git/git-upload-pack", + method="POST", + body=io.BytesIO(b"request"), + content_type=None, + expected_type="application/x-git-upload-pack-result", + token="sentinel", + opener=opener, + ) + + +def test_upstream_request_rejects_wrong_type_and_supports_getcode_status(): + broker = _load("scm_broker") + + class CodeOnly(Response): + def __init__(self, body, content_type): + super().__init__(body, content_type=content_type) + del self.status + + def getcode(self): + return 200 + + assert ( + broker._upstream_git_request( + "/atlas/cassandra.git/info/refs?service=git-upload-pack", + method="GET", + body=None, + content_type=None, + expected_type="application/x-git-upload-pack-advertisement", + token="sentinel", + opener=lambda *_a, **_k: CodeOnly( + b"advertisement", "application/x-git-upload-pack-advertisement" + ), + ) + == b"advertisement" + ) + with pytest.raises(broker.PolicyError, match="response type"): + broker._upstream_git_request( + "/atlas/cassandra.git/info/refs?service=git-upload-pack", + method="GET", + body=None, + content_type=None, + expected_type="application/x-git-upload-pack-advertisement", + token="sentinel", + opener=lambda *_a, **_k: Response(b"bad", content_type="text/plain"), + ) diff --git a/testing/tests/test_hermes_scm_broker_policy.py b/testing/tests/test_hermes_scm_broker_policy.py new file mode 100644 index 00000000..163cbc91 --- /dev/null +++ b/testing/tests/test_hermes_scm_broker_policy.py @@ -0,0 +1,285 @@ +"""RBAC, Flux, and human-review contracts for the Hermes SCM broker.""" + +from __future__ import annotations + +import json + +import pytest +import yaml + +from testing.tests.test_hermes_scm_broker_support import ROOT, _load_path + +def _can_i(role: dict, api_group: str, resource: str, verb: str) -> bool: + return any( + (api_group in rule["apiGroups"] or "*" in rule["apiGroups"]) + and (resource in rule["resources"] or "*" in rule["resources"]) + and (verb in rule["verbs"] or "*" in rule["verbs"]) + for rule in role["rules"] + ) + + +def test_agent_rbac_preserves_read_diagnostics_without_administrator_authority(): + documents = list( + yaml.safe_load_all( + (ROOT / "services/hermes-observer-rbac/rbac.yaml").read_text( + encoding="utf-8" + ) + ) + ) + cluster_role, namespaced_role, binding = documents + assert binding["roleRef"]["name"] == "hermes-agent-cluster-observer-v2" + assert _can_i(cluster_role, "", "nodes", "get") + assert _can_i(namespaced_role, "", "pods", "get") + assert _can_i(namespaced_role, "", "pods/log", "get") + assert _can_i( + namespaced_role, + "kustomize.toolkit.fluxcd.io", + "kustomizations", + "list", + ) + + denied = ( + ("", "secrets", "get"), + ("", "serviceaccounts/token", "create"), + ("", "pods", "create"), + ("", "pods/exec", "create"), + ("", "pods/attach", "create"), + ("", "pods/portforward", "create"), + ("rbac.authorization.k8s.io", "clusterrolebindings", "create"), + ("authorization.k8s.io", "selfsubjectaccessreviews", "create"), + ("apps", "deployments", "patch"), + ) + assert all( + not _can_i(role, *request) + for request in denied + for role in (cluster_role, namespaced_role) + ) + + bindings = yaml.safe_load( + (ROOT / "services/hermes-observer-bindings/rolebindings.yaml").read_text( + encoding="utf-8" + ) + )["items"] + namespaces = {item["metadata"]["namespace"] for item in bindings} + assert "hermes-scm" not in namespaces + assert {"cassandra", "flux-system", "hermes", "kube-system"} <= namespaces + assert all(item["roleRef"]["name"].endswith("namespaced-observer-v2") for item in bindings) + + +def test_flux_boundary_keeps_broker_secret_and_network_separate_from_agent(): + agent = yaml.safe_load( + (ROOT / "services/hermes/agent-deployment.yaml").read_text(encoding="utf-8") + ) + annotations = agent["spec"]["template"]["metadata"]["annotations"] + assert not any("gitea" in key.lower() for key in annotations) + assert "GIT_ASKPASS" not in json.dumps(agent) + + broker = yaml.safe_load( + (ROOT / "services/hermes-scm-broker/deployment.yaml").read_text( + encoding="utf-8" + ) + ) + assert broker["metadata"]["namespace"] == "hermes-scm" + assert broker["spec"]["template"]["spec"]["serviceAccountName"] == ( + "hermes-scm-broker" + ) + assert "gitea-token" in json.dumps(broker) + container = broker["spec"]["template"]["spec"]["containers"][0] + assert container["resources"]["limits"]["memory"] == "768Mi" + assert container["livenessProbe"]["failureThreshold"] == 10 + tmp = next( + item for item in broker["spec"]["template"]["spec"]["volumes"] if item["name"] == "tmp" + ) + assert tmp["emptyDir"]["sizeLimit"] == "1Gi" + + policy = yaml.safe_load( + (ROOT / "services/hermes-scm-broker/networkpolicy.yaml").read_text( + encoding="utf-8" + ) + ) + ingress = policy["spec"]["ingress"] + assert ingress[0]["from"][0]["namespaceSelector"]["matchLabels"] == { + "kubernetes.io/metadata.name": "hermes" + } + assert ingress[0]["from"][0]["podSelector"]["matchLabels"] == { + "app": "hermes-agent" + } + + vault = ( + ROOT / "services/vault/scripts/vault_k8s_auth_configure.sh" + ).read_text(encoding="utf-8") + assert 'write_policy_and_role "hermes-scm-broker" "hermes-scm"' in vault + assert '"hermes/developer-gitea" ""' in vault + agent_start = vault.index('write_policy_and_role "hermes-agent"') + agent_end = vault.index("write_policy_and_role", agent_start + 1) + assert "developer-gitea" not in vault[agent_start:agent_end] + + +def test_flux_bootstrap_has_no_agent_namespace_dependency_cycle(): + hermes_resources = yaml.safe_load( + (ROOT / "services/hermes/kustomization.yaml").read_text(encoding="utf-8") + )["resources"] + assert "namespace.yaml" in hermes_resources + assert "scm-common" in hermes_resources + + observer = yaml.safe_load( + ( + ROOT + / "clusters/atlas/flux-system/applications/hermes-observer-rbac/kustomization.yaml" + ).read_text(encoding="utf-8") + ) + assert "dependsOn" not in observer["spec"] + observer_objects = [ + item + for item in yaml.safe_load_all( + (ROOT / "services/hermes-observer-rbac/rbac.yaml").read_text( + encoding="utf-8" + ) + ) + if item + ] + assert {item["kind"] for item in observer_objects} <= { + "ClusterRole", + "ClusterRoleBinding", + } + + hermes_flux = yaml.safe_load( + ( + ROOT / "clusters/atlas/flux-system/applications/hermes/kustomization.yaml" + ).read_text(encoding="utf-8") + ) + assert "hermes-observer-rbac" in { + item["name"] for item in hermes_flux["spec"]["dependsOn"] + } + bindings_flux = yaml.safe_load( + ( + ROOT + / "clusters/atlas/flux-system/applications/hermes-observer-bindings/kustomization.yaml" + ).read_text(encoding="utf-8") + ) + assert {item["name"] for item in bindings_flux["spec"]["dependsOn"]} == { + "hermes", + "hermes-observer-rbac", + } + assert bindings_flux["metadata"]["name"] not in { + item["name"] for item in hermes_flux["spec"]["dependsOn"] + } + + applications = ( + ROOT / "clusters/atlas/flux-system/applications/kustomization.yaml" + ).read_text(encoding="utf-8") + assert "hermes-scm-agent-code" not in applications + + broker_code = yaml.safe_load( + ( + ROOT + / "clusters/atlas/flux-system/applications/hermes-scm-broker-code/kustomization.yaml" + ).read_text(encoding="utf-8") + ) + assert broker_code["spec"]["path"] == "./services/hermes/scm-common" + assert broker_code["spec"]["targetNamespace"] == "hermes-scm" + assert broker_code["spec"]["dependsOn"] == [{"name": "hermes-scm-namespace"}] + + +def test_gitea_bootstrap_enforces_human_review_without_overwriting_drift(): + script = ( + ROOT / "services/gitea/scripts/gitea_atlas_identity_ensure.sh" + ).read_text(encoding="utf-8") + + assert 'required_approvals\\\":1' in script + assert 'enable_push_whitelist\\\":true' in script + assert 'push_whitelist_usernames\\\":[\\\"${protected_reviewer}\\\"]' in script + assert 'enable_merge_whitelist\\\":true' in script + assert 'merge_whitelist_usernames\\\":[\\\"${protected_reviewer}\\\"]' in script + assert "protection differs from the human-review policy" in script + assert "api_request PATCH" not in script + job = yaml.safe_load( + (ROOT / "services/gitea/atlas-identity-bootstrap-job.yaml").read_text() + ) + init = job["spec"]["template"]["spec"]["initContainers"][0] + assert init["image"] == ( + "python@sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df" + ) + assert init["securityContext"]["runAsNonRoot"] is True + assert init["securityContext"]["capabilities"]["drop"] == ["ALL"] + assert "/opt/python/bin/python3" in script + + +def _protection(rule_name: str, priority: int, helper, **overrides): + value = { + "rule_name": rule_name, + "priority": priority, + "created_at": "2026-01-01T00:00:00Z", + **helper._required("bstein"), + } + value.update(overrides) + return value + + +def test_branch_protection_requires_exact_literal_rules_only(): + helper = _load_path( + "branch_protection_test", + ROOT / "services/gitea/scripts/gitea_branch_protection_check.py", + ) + exact = _protection("main", 2, helper) + assert ( + helper.evaluate(json.dumps([exact]).encode(), "main", "bstein") == "PRESENT" + ) + + drifted = _protection("main", 2, helper, required_approvals=0) + with pytest.raises(helper.PolicyError, match="effective main protection differs"): + helper.evaluate(json.dumps([drifted]).encode(), "main", "bstein") + + duplicate = _protection("main", 1, helper) + with pytest.raises(helper.PolicyError, match="duplicate primary-branch"): + helper.evaluate(json.dumps([exact, duplicate]).encode(), "main", "bstein") + + +@pytest.mark.parametrize( + "rule_name", + [ + "release/*", + "release/**/v1.17", + "*", + "**", + "M*", + "m{ain,aster}", + r"m\ain", + "m?in", + "m[ai]in", + ], +) +def test_branch_glob_rules_fail_closed_pending_human_review(rule_name: str): + helper = _load_path( + "branch_protection_glob_parity_test", + ROOT / "services/gitea/scripts/gitea_branch_protection_check.py", + ) + rules = [_protection(rule_name, 1, helper)] + with pytest.raises(helper.PolicyError, match="human review"): + helper.evaluate(json.dumps(rules).encode(), "main", "bstein") + + +@pytest.mark.parametrize( + "rules", + [ + [{"rule_name": "main", "priority": 0, "created_at": "2026-01-01T00:00:00Z"}], + [{"rule_name": "main", "priority": 1, "created_at": "not-a-time"}], + [{"rule_name": "main"}], + ], +) +def test_branch_protection_rejects_ambiguous_rules_before_creation(rules): + helper = _load_path( + "branch_protection_adversarial_test", + ROOT / "services/gitea/scripts/gitea_branch_protection_check.py", + ) + with pytest.raises(helper.PolicyError): + helper.evaluate(json.dumps(rules).encode(), "main", "bstein") + + +def test_branch_protection_reports_absent_only_when_no_rule_matches(): + helper = _load_path( + "branch_protection_absent_test", + ROOT / "services/gitea/scripts/gitea_branch_protection_check.py", + ) + rules = [_protection("develop", 1, helper)] + assert helper.evaluate(json.dumps(rules).encode(), "master", "bstein") == "ABSENT" diff --git a/testing/tests/test_hermes_scm_broker_streaming.py b/testing/tests/test_hermes_scm_broker_streaming.py new file mode 100644 index 00000000..78a63f27 --- /dev/null +++ b/testing/tests/test_hermes_scm_broker_streaming.py @@ -0,0 +1,237 @@ +"""Bounded I/O and concurrency contracts for the Hermes SCM broker.""" + +from __future__ import annotations + +import io +import json +from email.message import Message + +import pytest + +from testing.tests.test_hermes_scm_broker_support import Response, _load + + +def test_agent_broker_client_sends_no_credential_or_authorization_header(): + client = _load("scm_broker_client") + seen = [] + + def opener(request, timeout): + seen.append((request, timeout)) + return Response(b"{}", content_type="application/json") + + assert client.read("/api/v1/repos/atlas/cassandra", opener=opener) == b"{}" + request = seen[0][0] + assert request.full_url == client.BROKER_ORIGIN + "/v1/metadata" + assert request.get_header("Authorization") is None + assert json.loads(request.data) == {"path": "/api/v1/repos/atlas/cassandra"} + + +@pytest.mark.parametrize("raw", ["", "01", "9" * 4000, "134217729", "１２"]) +def test_broker_content_length_is_canonical_and_bounded(raw: str): + broker = _load("scm_broker") + headers = Message() + headers["Content-Length"] = raw + + with pytest.raises(broker.PolicyError, match="request length"): + broker._content_length(headers, broker.MAX_GIT_REQUEST) + + +def test_large_git_exchange_rolls_to_disk_and_scans_chunk_boundaries(): + broker = _load("scm_broker") + token = "runtime-sentinel" + safe = b"x" * (broker.SPOOL_MEMORY_LIMIT + 1) + inbound_timeouts = [] + spool, length = broker._spool_bounded( + io.BytesIO(safe), + broker.MAX_GIT_REQUEST, + len(safe), + token=token, + context="Git request", + deadline=broker.time.monotonic() + 30, + set_timeout=inbound_timeouts.append, + ) + try: + assert length == len(safe) + assert spool._rolled is True + assert spool.read(4) == b"xxxx" + assert inbound_timeouts + assert all(0 < value <= 30 for value in inbound_timeouts) + finally: + spool.close() + + crossing = b"x" * (broker.STREAM_CHUNK - 5) + token.encode() + b"tail" + with pytest.raises(broker.PolicyError, match="credential material"): + broker._spool_bounded( + io.BytesIO(crossing), + broker.MAX_GIT_REQUEST, + len(crossing), + token=token, + context="Git request", + deadline=broker.time.monotonic() + 30, + ) + + +def test_large_upstream_response_is_streamed_through_bounded_spool(): + broker = _load("scm_broker") + body = b"z" * (broker.SPOOL_MEMORY_LIMIT + 1) + seen = [] + + def opener(request, **_kwargs): + seen.append(request) + assert request.get_header("Content-length") == "7" + assert request.data.read() == b"request" + return Response(body, content_type="application/x-git-upload-pack-result") + + streamed = broker._upstream_git_request( + "/atlas/cassandra.git/git-upload-pack", + method="POST", + body=io.BytesIO(b"request"), + body_length=7, + content_type="application/x-git-upload-pack-request", + expected_type="application/x-git-upload-pack-result", + token="runtime-sentinel", + stream_result=True, + opener=opener, + ) + spool, length = streamed + try: + assert length == len(body) + assert spool._rolled is True + assert spool.read(3) == b"zzz" + assert len(seen) == 1 + finally: + spool.close() + + +def test_body_deadline_fails_closed_before_reading(): + broker = _load("scm_broker") + with pytest.raises(broker.PolicyError, match="deadline"): + broker._spool_bounded( + io.BytesIO(b"body"), + broker.MAX_GIT_REQUEST, + 4, + token="runtime-sentinel", + context="Git request", + deadline=broker.time.monotonic() - 1, + ) + + +def test_upstream_stream_applies_absolute_socket_deadline(): + broker = _load("scm_broker") + + class DeadlineSocket: + def __init__(self): + self.values = [] + + def settimeout(self, value): + self.values.append(value) + + response = Response(b"safe", content_type="application/x-git-upload-pack-result") + sock = DeadlineSocket() + response.fp = type("FP", (), {"raw": type("Raw", (), {"_sock": sock})()})() + spool, length = broker._spool_response( + response, broker.MAX_GIT_RESPONSE, "sentinel", broker.INBOUND_BODY_TIMEOUT + ) + try: + assert length == 4 + assert sock.values + assert all(0 < value <= broker.INBOUND_BODY_TIMEOUT for value in sock.values) + finally: + spool.close() + + +def test_broker_rejects_work_when_concurrency_slots_are_exhausted(): + broker = _load("scm_broker") + + class FakeSocket: + def __init__(self): + self.value = b"" + + def sendall(self, value): + self.value += value + + def shutdown(self, _how): + return None + + def close(self): + return None + + server = broker.BoundedThreadingHTTPServer(("127.0.0.1", 0), broker.BrokerHandler) + try: + assert broker.BoundedThreadingHTTPServer.__module__ == "scm_broker_server" + assert server._slots.acquire(blocking=False) + assert server._slots.acquire(blocking=False) + assert server._slots.acquire(blocking=False) + request = FakeSocket() + server.process_request(request, ("127.0.0.1", 1)) + assert b"503 Service Unavailable" in request.value + finally: + server._slots.release() + server._slots.release() + server._slots.release() + server.server_close() + + +def test_three_simultaneous_broker_calls_have_bounded_capacity(): + server_module = _load("scm_broker_server") + semaphore = server_module.threading.BoundedSemaphore( + server_module.MAX_CONCURRENT_REQUESTS + ) + assert server_module.MAX_CONCURRENT_REQUESTS >= 3 + assert [semaphore.acquire(blocking=False) for _ in range(3)] == [True, True, True] + assert semaphore.acquire(blocking=False) is False + + +def test_header_deadline_is_absolute_against_slow_trickle(monkeypatch): + server_module = _load("scm_broker_server") + + class Stream: + def __init__(self): + self.value = io.BytesIO(b"GET /healthz HTTP/1.1\r\n") + + def read(self, length): + return self.value.read(length) + + class Connection: + def __init__(self): + self.timeouts = [] + + def settimeout(self, value): + self.timeouts.append(value) + + ticks = iter((0.0, 0.02, 0.04, 0.06, 0.08, 0.11)) + monkeypatch.setattr(server_module.time, "monotonic", lambda: next(ticks)) + connection = Connection() + reader = server_module._AbsoluteDeadlineReader(Stream(), connection) + reader.begin(0.1) + + with pytest.raises(TimeoutError, match="absolute SCM header deadline"): + reader.readline(65537) + + assert connection.timeouts == pytest.approx([0.08, 0.06, 0.04, 0.02]) + + +def test_broker_bounds_header_count_size_and_duplicate_lengths(): + broker = _load("scm_broker") + + def validate(headers): + handler = object.__new__(broker.BrokerHandler) + handler.headers = headers + handler._validate_headers() + + too_many = Message() + for index in range(broker.MAX_HEADERS + 1): + too_many[f"X-Test-{index}"] = "safe" + with pytest.raises(broker.PolicyError, match="too many headers"): + validate(too_many) + + too_large = Message() + too_large["X-Test"] = "x" * broker.MAX_HEADER_BYTES + with pytest.raises(broker.PolicyError, match="safe limit"): + validate(too_large) + + duplicate = Message() + duplicate["Content-Length"] = "1" + duplicate["Content-Length"] = "1" + with pytest.raises(broker.PolicyError, match="duplicate Content-Length"): + validate(duplicate) diff --git a/testing/tests/test_hermes_scm_broker_support.py b/testing/tests/test_hermes_scm_broker_support.py new file mode 100644 index 00000000..e3682494 --- /dev/null +++ b/testing/tests/test_hermes_scm_broker_support.py @@ -0,0 +1,84 @@ +"""Shared fixtures for credential-isolated Hermes SCM broker tests.""" + +from __future__ import annotations + +import hashlib +import importlib.util +import io +import sys +import zlib +from email.message import Message +from pathlib import Path + +ROOT = Path(__file__).parents[2] +SCRIPTS = ROOT / "services/hermes/scm-common/scripts" +sys.path.insert(0, str(SCRIPTS)) + + +def _load(name: str): + path = SCRIPTS / f"{name}.py" + spec = importlib.util.spec_from_file_location(f"test_{name}", path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _load_path(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +class Response: + def __init__(self, body: bytes, *, status: int = 200, content_type: str): + self.body = body + self.stream = io.BytesIO(body) + self.status = status + self.headers = Message() + self.headers["Content-Type"] = content_type + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def read(self, limit=-1): + return self.stream.read(limit) + + +def _object_entry(type_code: int, payload: bytes) -> bytes: + """Encode one non-delta pack entry with a real header and zlib body.""" + size = len(payload) + byte = (type_code << 4) | (size & 0x0F) + size >>= 4 + header = bytearray() + while size: + header.append(byte | 0x80) + byte = size & 0x7F + size >>= 7 + header.append(byte) + return bytes(header) + zlib.compress(payload) + + +def _pack_of(entries: list[bytes]) -> bytes: + """Assemble encoded entries into a checksummed version-2 pack.""" + body = b"PACK" + (2).to_bytes(4, "big") + len(entries).to_bytes(4, "big") + for entry in entries: + body += entry + return body + hashlib.sha1(body).digest() + + +def _blob_pack(*payloads: bytes) -> bytes: + return _pack_of([_object_entry(3, payload) for payload in payloads]) + + +def _receive_command(old: bytes, new: bytes, ref: bytes, pack: bytes = b"") -> bytes: + command = old + b" " + new + b" " + ref + b"\x00report-status\n" + framed = f"{len(command) + 4:04x}".encode() + command + b"0000" + return framed + (pack or _pack_of([])) diff --git a/testing/tests/test_hermes_scm_server_coverage.py b/testing/tests/test_hermes_scm_server_coverage.py new file mode 100644 index 00000000..7a358640 --- /dev/null +++ b/testing/tests/test_hermes_scm_server_coverage.py @@ -0,0 +1,180 @@ +"""Behavioral branch coverage for the SCM broker HTTP server primitives.""" + +from __future__ import annotations + +import io + +import pytest + +from testing.tests.test_hermes_scm_broker_support import _load + + +class _Connection: + def __init__(self): + self.timeouts = [] + + def settimeout(self, value): + self.timeouts.append(value) + + +class _FlushBuffer(io.BytesIO): + def __init__(self): + super().__init__() + self.flushes = 0 + + def flush(self): + self.flushes += 1 + + +def test_absolute_reader_supports_idle_limit_newline_eof_and_attribute_delegation(): + module = _load("scm_broker_server") + stream = io.BytesIO(b"first\nsecond") + reader = module._AbsoluteDeadlineReader(stream, _Connection()) + assert reader.readline(3) == b"fir" + reader.begin(5) + assert reader.readline() == b"st\n" + assert reader.readline() == b"second" + assert reader.readline() == b"" + assert reader.closed is False + reader.end() + assert reader.readline() == b"" + + +def _handler_type(module): + class StubHandler: + request_version = "HTTP/1.1" + command = "GET" + + def setup(self): + self.connection = _Connection() + self.rfile = io.BytesIO(b"") + self.wfile = _FlushBuffer() + + def parse_request(self): + return self.parse_result + + def send_error(self, code, *_args): + self.errors.append(code) + + def do_GET(self): + self.calls.append("GET") + + return type( + "DeadlineHandler", (module.AbsoluteHeaderDeadlineMixin, StubHandler), {} + ) + + +def _handler(module, raw: bytes, *, parse=True): + handler_type = _handler_type(module) + handler = object.__new__(handler_type) + handler.setup() + handler._header_reader._stream = io.BytesIO(raw) + handler.parse_result = parse + handler.errors = [] + handler.calls = [] + return handler + + +def test_header_mixin_setup_and_known_request_dispatch(): + module = _load("scm_broker_server") + handler = _handler(module, b"GET /healthz HTTP/1.1\r\n") + handler.handle_one_request() + assert handler.calls == ["GET"] + assert handler.wfile.flushes == 1 + assert handler._header_reader._deadline is None + + +@pytest.mark.parametrize( + ("raw", "parse", "command", "expected_error", "closed"), + [ + (b"x" * 65537, True, "GET", 414, False), + (b"", True, "GET", None, True), + (b"GET / HTTP/1.1\r\n", False, "GET", None, False), + (b"TRACE / HTTP/1.1\r\n", True, "TRACE", 501, False), + ], +) +def test_header_mixin_rejects_long_empty_unparsed_and_unknown_requests( + raw, parse, command, expected_error, closed +): + module = _load("scm_broker_server") + handler = _handler(module, raw, parse=parse) + handler.command = command + handler.close_connection = False + handler.handle_one_request() + assert handler.errors == ([] if expected_error is None else [expected_error]) + assert handler.close_connection is closed + + +def test_header_mixin_closes_on_absolute_timeout(monkeypatch): + module = _load("scm_broker_server") + handler = _handler(module, b"GET / HTTP/1.1\r\n") + handler.close_connection = False + monkeypatch.setattr( + handler._header_reader, + "readline", + lambda *_a, **_k: (_ for _ in ()).throw(TimeoutError()), + ) + handler.handle_one_request() + assert handler.close_connection is True + + +class _Slots: + def __init__(self, available=True): + self.available = available + self.releases = 0 + + def acquire(self, *, blocking): + assert blocking is False + return self.available + + def release(self): + self.releases += 1 + + +def _server_shell(module, available=True): + server = object.__new__(module.BoundedThreadingHTTPServer) + server._slots = _Slots(available) + server.shutdowns = [] + server.shutdown_request = server.shutdowns.append + return server + + +def test_bounded_server_delegates_accepted_request(monkeypatch): + module = _load("scm_broker_server") + server = _server_shell(module) + calls = [] + monkeypatch.setattr( + module.ThreadingHTTPServer, + "process_request", + lambda self, request, address: calls.append((request, address)), + ) + server.process_request("socket", ("127.0.0.1", 1)) + assert calls == [("socket", ("127.0.0.1", 1))] + assert server._slots.releases == 0 + + +def test_bounded_server_releases_on_dispatch_failure(monkeypatch): + module = _load("scm_broker_server") + server = _server_shell(module) + monkeypatch.setattr( + module.ThreadingHTTPServer, + "process_request", + lambda *_a, **_k: (_ for _ in ()).throw(RuntimeError("dispatch")), + ) + with pytest.raises(RuntimeError, match="dispatch"): + server.process_request("socket", ("127.0.0.1", 1)) + assert server._slots.releases == 1 + assert server.shutdowns == ["socket"] + + +def test_bounded_server_thread_always_releases(monkeypatch): + module = _load("scm_broker_server") + server = _server_shell(module) + monkeypatch.setattr( + module.ThreadingHTTPServer, + "process_request_thread", + lambda *_a, **_k: (_ for _ in ()).throw(RuntimeError("handler")), + ) + with pytest.raises(RuntimeError, match="handler"): + server.process_request_thread("socket", ("127.0.0.1", 1)) + assert server._slots.releases == 1